Verify fresh IndeeHub backup restores and record remaining release tasks

tests/regression/test_indeehub_maintenance_controller.py

@@ -56,10 +56,31 @@ class MaintenanceTests(unittest.TestCase):
   for name in ['database.dump',*(v+'.tar' for v in module.VOLUMES)]:
    path=c.root/'backup'/name;path.write_bytes(b'original')
    c.record['artifacts'][name]={'bytes':path.stat().st_size,'sha256':module.sha(path)}
+  c.record['original_members']=members()
+  c.record['database_before']={'operation_id':self.operation,'tables':{},'migrations':[]}
+  c.record['backup_restore_verified']=c.backup_restore_terms()
   c.save()
   return c
  def test_complete_backup_checksums_allow_verification(self):
   self.assertEqual(self.completed_backup().verify()['state'],'held')
+ def test_restore_proof_must_match_fresh_dump_baseline_image_and_operation(self):
+  c=self.completed_backup();proof=dict(c.record['backup_restore_verified'])
+  for field in proof:
+   c.record['backup_restore_verified']={**proof,field:'changed'}
+   with self.assertRaisesRegex(RuntimeError,'restore is not verified'):c.verify()
+   self.assertEqual(c.fence.read_text(),self.operation)
+  c.record.pop('backup_restore_verified')
+  with self.assertRaisesRegex(RuntimeError,'restore is not verified'):c.verify()
+ def test_foreign_restore_fixture_is_never_removed(self):
+  c=self.completed_backup();name='archy-backup-restore-'+'a'*32
+  c.record['restore_fixture']={'name':name,'image_id':'a'*64}
+  def command(argv,timeout,output):
+   if argv[:2]==['podman','ps']:return b'container-id'
+   if argv[:2]==['podman','inspect']:return json.dumps([{'Name':name,'Image':'a'*64,'Config':{'Labels':{'io.archipelago.backup.operation':str(uuid.uuid4())}}}]).encode()
+   raise AssertionError('Unexpected mutation '+str(argv))
+  c.runner=command
+  with self.assertRaisesRegex(RuntimeError,'ownership changed'):c.cleanup_restore_fixture()
+  self.assertIn('restore_fixture',c.record)
  def test_same_size_corruption_keeps_admission_closed(self):
   c=self.completed_backup();(c.root/'backup'/'database.dump').write_bytes(b'corrupt!')
   with self.assertRaisesRegex(RuntimeError,'checksum changed'):c.verify()

tests/regression/test_indeehub_maintenance_postgres.py

@@ -5,6 +5,7 @@ Requires an already imported image: --image IMAGE. Never mounts node volumes,
 publishes ports, or invokes the maintenance entrypoint against installed apps.
 """
 import argparse
+import copy
 import importlib.util
 import json
 from pathlib import Path
@@ -36,7 +37,7 @@ def main():
             '-e', 'POSTGRES_DB=indeedhub', image,
         ], text=True).strip()
         deadline = time.monotonic() + 60
-        while subprocess.run(['podman', 'exec', container, 'pg_isready', '-U', 'indeedhub'],
+        while subprocess.run(['podman', 'exec', container, 'pg_isready', '-h', '127.0.0.1', '-U', 'indeedhub'],
                              stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode:
             if time.monotonic() > deadline:
                 raise RuntimeError('Disposable PostgreSQL did not become ready')
@@ -68,6 +69,59 @@ def main():
                 'podman', 'exec', container, 'pg_dump', '-U', 'indeedhub',
                 '-d', 'indeedhub', '--format=custom', '--no-owner', '--no-acl',
             ], timeout=60)
+            # Exercise the production fresh-backup restore barrier, not just
+            # hand-written pg_restore commands. All containers are owned fixtures.
+            fresh = maintenance.Controller(root, str(uuid.uuid4()), 0)
+            fresh.record = {'operation_id': fresh.operation,
+                'original_members': [{'name': member, 'container_id': 'a'*64,
+                    'image_id': image.removeprefix('sha256:'), 'unit_sha256': 'b'*64,
+                    'config_sha256': 'c'*64, 'running': True} for member in maintenance.NAMES],
+                'database_before': {**copy.deepcopy(before), 'operation_id': fresh.operation}, 'artifacts': {}}
+            holds = fresh.data/'update-transactions'/'holds'
+            holds.mkdir(parents=True)
+            for member in maintenance.NAMES: (holds/member).write_text(fresh.operation)
+            fresh.fence.parent.mkdir(parents=True)
+            fresh.fence.write_text(fresh.operation)
+            backup = fresh.root/'backup'
+            backup.mkdir(parents=True)
+            for artifact in ['database.dump', *(v+'.tar' for v in maintenance.VOLUMES)]:
+                path = backup/artifact
+                path.write_bytes(dump if artifact == 'database.dump' else b'volume-fixture')
+                fresh.record['artifacts'][artifact] = {'bytes': path.stat().st_size, 'sha256': maintenance.sha(path)}
+            fresh.save()
+            try:
+                fresh.verify_database_backup()
+            except Exception:
+                # Fixture-only SQL diagnostics; this test never opens live data.
+                diagnostic = fresh.root/'commands.private.log'
+                if diagnostic.exists():
+                    Path('/tmp/archy-backup-fixture-failure.log').write_bytes(diagnostic.read_bytes())
+                raise
+            assert fresh.record['backup_restore_verified'] == fresh.backup_restore_terms()
+            assert 'restore_fixture' not in fresh.record
+            # A readable dump from the wrong database must also fail the barrier.
+            fresh.record.pop('backup_restore_verified')
+            fresh.record['database_before']['tables']['contents']['rows_sha256'] = '0'*64
+            try:
+                fresh.verify_database_backup()
+            except RuntimeError as error:
+                assert 'differs' in str(error)
+            else:
+                raise AssertionError('Wrong database backup incorrectly accepted')
+            assert 'restore_fixture' not in fresh.record
+            assert 'backup_restore_verified' not in fresh.record
+            path = backup/'database.dump'
+            path.write_bytes(dump[:32])
+            fresh.record['artifacts']['database.dump'] = {'bytes': path.stat().st_size, 'sha256': maintenance.sha(path)}
+            try:
+                fresh.verify_database_backup()
+            except subprocess.CalledProcessError:
+                pass
+            else:
+                raise AssertionError('Truncated fresh backup incorrectly accepted')
+            assert 'restore_fixture' not in fresh.record
+            assert 'backup_restore_verified' not in fresh.record
+            assert fresh.fence.read_text() == fresh.operation
             sql('CREATE DATABASE restore_check')
             restore_command = ['podman', 'exec', '-i', container, 'pg_restore',
                                '-U', 'indeedhub', '-d', 'restore_check',
@@ -104,7 +158,8 @@ def main():
             maintenance.verify_database_compatibility(before, controller.database_commitments())
             print(json.dumps({'postgres_commitments': 'passed', 'rejected_mutations': rejected,
                               'network': 'none', 'live_volumes_mounted': False,
-                              'custom_dump_restored': True, 'truncated_dump_rejected': True}))
+                              'custom_dump_restored': True, 'truncated_dump_rejected': True,
+                              'production_restore_barrier': 'passed', 'wrong_backup_rejected': True}))
     finally:
         if container:
             subprocess.run(['podman', 'rm', '-f', container], check=True, stdout=subprocess.DEVNULL)