Verify fresh IndeeHub backup restores and record remaining release tasks

tests/regression/test_indeehub_maintenance_postgres.py

@@ -5,6 +5,7 @@ Requires an already imported image: --image IMAGE. Never mounts node volumes,
 publishes ports, or invokes the maintenance entrypoint against installed apps.
 """
 import argparse
+import copy
 import importlib.util
 import json
 from pathlib import Path
@@ -36,7 +37,7 @@ def main():
             '-e', 'POSTGRES_DB=indeedhub', image,
         ], text=True).strip()
         deadline = time.monotonic() + 60
-        while subprocess.run(['podman', 'exec', container, 'pg_isready', '-U', 'indeedhub'],
+        while subprocess.run(['podman', 'exec', container, 'pg_isready', '-h', '127.0.0.1', '-U', 'indeedhub'],
                              stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode:
             if time.monotonic() > deadline:
                 raise RuntimeError('Disposable PostgreSQL did not become ready')
@@ -68,6 +69,59 @@ def main():
                 'podman', 'exec', container, 'pg_dump', '-U', 'indeedhub',
                 '-d', 'indeedhub', '--format=custom', '--no-owner', '--no-acl',
             ], timeout=60)
+            # Exercise the production fresh-backup restore barrier, not just
+            # hand-written pg_restore commands. All containers are owned fixtures.
+            fresh = maintenance.Controller(root, str(uuid.uuid4()), 0)
+            fresh.record = {'operation_id': fresh.operation,
+                'original_members': [{'name': member, 'container_id': 'a'*64,
+                    'image_id': image.removeprefix('sha256:'), 'unit_sha256': 'b'*64,
+                    'config_sha256': 'c'*64, 'running': True} for member in maintenance.NAMES],
+                'database_before': {**copy.deepcopy(before), 'operation_id': fresh.operation}, 'artifacts': {}}
+            holds = fresh.data/'update-transactions'/'holds'
+            holds.mkdir(parents=True)
+            for member in maintenance.NAMES: (holds/member).write_text(fresh.operation)
+            fresh.fence.parent.mkdir(parents=True)
+            fresh.fence.write_text(fresh.operation)
+            backup = fresh.root/'backup'
+            backup.mkdir(parents=True)
+            for artifact in ['database.dump', *(v+'.tar' for v in maintenance.VOLUMES)]:
+                path = backup/artifact
+                path.write_bytes(dump if artifact == 'database.dump' else b'volume-fixture')
+                fresh.record['artifacts'][artifact] = {'bytes': path.stat().st_size, 'sha256': maintenance.sha(path)}
+            fresh.save()
+            try:
+                fresh.verify_database_backup()
+            except Exception:
+                # Fixture-only SQL diagnostics; this test never opens live data.
+                diagnostic = fresh.root/'commands.private.log'
+                if diagnostic.exists():
+                    Path('/tmp/archy-backup-fixture-failure.log').write_bytes(diagnostic.read_bytes())
+                raise
+            assert fresh.record['backup_restore_verified'] == fresh.backup_restore_terms()
+            assert 'restore_fixture' not in fresh.record
+            # A readable dump from the wrong database must also fail the barrier.
+            fresh.record.pop('backup_restore_verified')
+            fresh.record['database_before']['tables']['contents']['rows_sha256'] = '0'*64
+            try:
+                fresh.verify_database_backup()
+            except RuntimeError as error:
+                assert 'differs' in str(error)
+            else:
+                raise AssertionError('Wrong database backup incorrectly accepted')
+            assert 'restore_fixture' not in fresh.record
+            assert 'backup_restore_verified' not in fresh.record
+            path = backup/'database.dump'
+            path.write_bytes(dump[:32])
+            fresh.record['artifacts']['database.dump'] = {'bytes': path.stat().st_size, 'sha256': maintenance.sha(path)}
+            try:
+                fresh.verify_database_backup()
+            except subprocess.CalledProcessError:
+                pass
+            else:
+                raise AssertionError('Truncated fresh backup incorrectly accepted')
+            assert 'restore_fixture' not in fresh.record
+            assert 'backup_restore_verified' not in fresh.record
+            assert fresh.fence.read_text() == fresh.operation
             sql('CREATE DATABASE restore_check')
             restore_command = ['podman', 'exec', '-i', container, 'pg_restore',
                                '-U', 'indeedhub', '-d', 'restore_check',
@@ -104,7 +158,8 @@ def main():
             maintenance.verify_database_compatibility(before, controller.database_commitments())
             print(json.dumps({'postgres_commitments': 'passed', 'rejected_mutations': rejected,
                               'network': 'none', 'live_volumes_mounted': False,
-                              'custom_dump_restored': True, 'truncated_dump_rejected': True}))
+                              'custom_dump_restored': True, 'truncated_dump_rejected': True,
+                              'production_restore_barrier': 'passed', 'wrong_backup_rejected': True}))
     finally:
         if container:
             subprocess.run(['podman', 'rm', '-f', container], check=True, stdout=subprocess.DEVNULL)