Verify fresh IndeeHub backup restores and record remaining release tasks

tests/lifecycle/supervised-runtime-primitives.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+"""Qualify real AutoRemove/Quadlet recovery primitives using owned fixtures only.
+
+This does not replace full app-specific drain or production updater acceptance.
+No app volume, port, wallet, catalog, or installed unit is used.
+"""
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import subprocess
+import tempfile
+import time
+import uuid
+
+
+def run(*args, check=True, timeout=90):
+    result = subprocess.run(args, check=False, timeout=timeout, capture_output=True, text=True)
+    if check and result.returncode:
+        raise RuntimeError(f'{args[0]} failed ({result.returncode}): {result.stderr.strip()}')
+    return result
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--image', required=True, help='Already imported local image containing /bin/sh')
+    args = parser.parse_args()
+    image = run('podman', 'image', 'inspect', '--format', '{{.Id}}', args.image).stdout.strip()
+    operation = str(uuid.uuid4())
+    name = 'archy-recovery-fixture-' + operation
+    tag = 'localhost/archy-update-recovery:' + operation + '-0'
+    root = Path(tempfile.mkdtemp(prefix=name + '-'))
+    data = root / 'data'
+    data.mkdir()
+    units = Path.home() / '.config/containers/systemd'
+    units.mkdir(parents=True, exist_ok=True)
+    unit = units / (name + '.container')
+    assert not unit.exists()
+    service = name + '.service'
+    snapshot = None
+    original = f'''[Container]
+Image=sha256:{image.removeprefix('sha256:')}
+ContainerName={name}
+Network=none
+Pull=never
+Volume={data}:/state
+Environment=MODE=original
+Entrypoint=/bin/sh
+Exec=-c "trap 'exit 0' TERM; while sleep 1; do :; done"
+
+[Service]
+Restart=no
+TimeoutStartSec=60
+TimeoutStopSec=15
+'''
+    def write(body):
+        temporary = unit.with_suffix('.next')
+        temporary.write_text(body)
+        temporary.chmod(0o600)
+        temporary.replace(unit)
+        run('systemctl', '--user', 'daemon-reload')
+    def inspect():
+        rows = json.loads(run('podman', 'inspect', name).stdout)
+        assert len(rows) == 1 and rows[0]['Name'] == name
+        return rows[0]
+    try:
+        write(original)
+        run('systemctl', '--user', 'start', service)
+        old = inspect()
+        assert old['State']['Running'] and old['HostConfig']['AutoRemove']
+        run('podman', 'exec', name, '/bin/sh', '-c',
+            'printf original-layer > /original-layer; printf persistent-bytes > /state/value')
+        volume_hash = hashlib.sha256((data / 'value').read_bytes()).hexdigest()
+        run('podman', 'commit', '--pause=true', '--include-volumes=false',
+            '--change', 'LABEL io.archipelago.recovery.operation=' + operation,
+            '--change', 'LABEL io.archipelago.recovery.container=' + old['Id'], old['Id'], tag)
+        captured = json.loads(run('podman', 'image', 'inspect', tag).stdout)[0]
+        snapshot = captured['Id']
+        assert captured['Config']['Labels']['io.archipelago.recovery.operation'] == operation
+        assert captured['Config']['Labels']['io.archipelago.recovery.container'] == old['Id']
+        run('podman', 'run', '--rm', '--network=none', '--pull=never', '--entrypoint=/bin/sh', snapshot,
+            '-c', 'test "$(cat /original-layer)" = original-layer; test ! -f /state/value')
+        run('systemctl', '--user', 'stop', service)
+        assert run('podman', 'container', 'exists', old['Id'], check=False).returncode == 1
+        failed = original.replace('Environment=MODE=original', 'Environment=MODE=candidate').replace(
+            'Exec=-c "trap \'exit 0\' TERM; while sleep 1; do :; done"', 'Exec=-c "exit 77"')
+        assert failed != original
+        write(failed)
+        run('systemctl', '--user', 'start', service, check=False)
+        deadline = time.monotonic() + 15
+        while run('systemctl', '--user', 'is-active', service, check=False).returncode == 0:
+            assert time.monotonic() < deadline, 'Fault injection unexpectedly remained active'
+            time.sleep(0.2)
+        assert run('systemctl', '--user', 'show', service, '--property=Result', '--value').stdout.strip() != 'success'
+        run('systemctl', '--user', 'stop', service, check=False)
+        restored = original.replace('Image=sha256:' + image.removeprefix('sha256:'),
+                                    'Image=sha256:' + snapshot.removeprefix('sha256:'))
+        write(restored)
+        run('systemctl', '--user', 'reset-failed', service, check=False)
+        run('systemctl', '--user', 'start', service)
+        current = inspect()
+        assert current['State']['Running'] and current['Id'] != old['Id']
+        assert current['Image'].removeprefix('sha256:') == snapshot.removeprefix('sha256:')
+        assert 'MODE=original' in current['Config']['Env']
+        assert unit.read_text() == restored and unit.stat().st_mode & 0o777 == 0o600
+        assert run('podman', 'exec', name, 'cat', '/original-layer').stdout == 'original-layer'
+        assert hashlib.sha256((data / 'value').read_bytes()).hexdigest() == volume_hash
+        print(json.dumps({'auto_remove_recovery': 'passed', 'writable_layer_preserved': True,
+                          'volume_bytes_preserved': True, 'original_configuration_restored': True,
+                          'injected_target_failure': True, 'network': 'none',
+                          'full_supervised_application_cutover': 'not tested'}))
+    finally:
+        run('systemctl', '--user', 'stop', service, check=False)
+        unit.unlink(missing_ok=True)
+        unit.with_suffix('.next').unlink(missing_ok=True)
+        run('systemctl', '--user', 'daemon-reload')
+        run('systemctl', '--user', 'reset-failed', service, check=False)
+        run('podman', 'rm', '-f', name, check=False)
+        if snapshot:
+            run('podman', 'rmi', tag)
+        assert root.parent == Path('/tmp') and root.name.startswith(name + '-')
+        run('podman', 'unshare', 'rm', '-rf', str(root))
+
+
+if __name__ == '__main__':
+    main()

tests/regression/test_indeehub_maintenance_controller.py

@@ -56,10 +56,31 @@ class MaintenanceTests(unittest.TestCase):
   for name in ['database.dump',*(v+'.tar' for v in module.VOLUMES)]:
    path=c.root/'backup'/name;path.write_bytes(b'original')
    c.record['artifacts'][name]={'bytes':path.stat().st_size,'sha256':module.sha(path)}
+  c.record['original_members']=members()
+  c.record['database_before']={'operation_id':self.operation,'tables':{},'migrations':[]}
+  c.record['backup_restore_verified']=c.backup_restore_terms()
   c.save()
   return c
  def test_complete_backup_checksums_allow_verification(self):
   self.assertEqual(self.completed_backup().verify()['state'],'held')
+ def test_restore_proof_must_match_fresh_dump_baseline_image_and_operation(self):
+  c=self.completed_backup();proof=dict(c.record['backup_restore_verified'])
+  for field in proof:
+   c.record['backup_restore_verified']={**proof,field:'changed'}
+   with self.assertRaisesRegex(RuntimeError,'restore is not verified'):c.verify()
+   self.assertEqual(c.fence.read_text(),self.operation)
+  c.record.pop('backup_restore_verified')
+  with self.assertRaisesRegex(RuntimeError,'restore is not verified'):c.verify()
+ def test_foreign_restore_fixture_is_never_removed(self):
+  c=self.completed_backup();name='archy-backup-restore-'+'a'*32
+  c.record['restore_fixture']={'name':name,'image_id':'a'*64}
+  def command(argv,timeout,output):
+   if argv[:2]==['podman','ps']:return b'container-id'
+   if argv[:2]==['podman','inspect']:return json.dumps([{'Name':name,'Image':'a'*64,'Config':{'Labels':{'io.archipelago.backup.operation':str(uuid.uuid4())}}}]).encode()
+   raise AssertionError('Unexpected mutation '+str(argv))
+  c.runner=command
+  with self.assertRaisesRegex(RuntimeError,'ownership changed'):c.cleanup_restore_fixture()
+  self.assertIn('restore_fixture',c.record)
  def test_same_size_corruption_keeps_admission_closed(self):
   c=self.completed_backup();(c.root/'backup'/'database.dump').write_bytes(b'corrupt!')
   with self.assertRaisesRegex(RuntimeError,'checksum changed'):c.verify()

tests/regression/test_indeehub_maintenance_postgres.py

@@ -5,6 +5,7 @@ Requires an already imported image: --image IMAGE. Never mounts node volumes,
 publishes ports, or invokes the maintenance entrypoint against installed apps.
 """
 import argparse
+import copy
 import importlib.util
 import json
 from pathlib import Path
@@ -36,7 +37,7 @@ def main():
             '-e', 'POSTGRES_DB=indeedhub', image,
         ], text=True).strip()
         deadline = time.monotonic() + 60
-        while subprocess.run(['podman', 'exec', container, 'pg_isready', '-U', 'indeedhub'],
+        while subprocess.run(['podman', 'exec', container, 'pg_isready', '-h', '127.0.0.1', '-U', 'indeedhub'],
                              stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode:
             if time.monotonic() > deadline:
                 raise RuntimeError('Disposable PostgreSQL did not become ready')
@@ -68,6 +69,59 @@ def main():
                 'podman', 'exec', container, 'pg_dump', '-U', 'indeedhub',
                 '-d', 'indeedhub', '--format=custom', '--no-owner', '--no-acl',
             ], timeout=60)
+            # Exercise the production fresh-backup restore barrier, not just
+            # hand-written pg_restore commands. All containers are owned fixtures.
+            fresh = maintenance.Controller(root, str(uuid.uuid4()), 0)
+            fresh.record = {'operation_id': fresh.operation,
+                'original_members': [{'name': member, 'container_id': 'a'*64,
+                    'image_id': image.removeprefix('sha256:'), 'unit_sha256': 'b'*64,
+                    'config_sha256': 'c'*64, 'running': True} for member in maintenance.NAMES],
+                'database_before': {**copy.deepcopy(before), 'operation_id': fresh.operation}, 'artifacts': {}}
+            holds = fresh.data/'update-transactions'/'holds'
+            holds.mkdir(parents=True)
+            for member in maintenance.NAMES: (holds/member).write_text(fresh.operation)
+            fresh.fence.parent.mkdir(parents=True)
+            fresh.fence.write_text(fresh.operation)
+            backup = fresh.root/'backup'
+            backup.mkdir(parents=True)
+            for artifact in ['database.dump', *(v+'.tar' for v in maintenance.VOLUMES)]:
+                path = backup/artifact
+                path.write_bytes(dump if artifact == 'database.dump' else b'volume-fixture')
+                fresh.record['artifacts'][artifact] = {'bytes': path.stat().st_size, 'sha256': maintenance.sha(path)}
+            fresh.save()
+            try:
+                fresh.verify_database_backup()
+            except Exception:
+                # Fixture-only SQL diagnostics; this test never opens live data.
+                diagnostic = fresh.root/'commands.private.log'
+                if diagnostic.exists():
+                    Path('/tmp/archy-backup-fixture-failure.log').write_bytes(diagnostic.read_bytes())
+                raise
+            assert fresh.record['backup_restore_verified'] == fresh.backup_restore_terms()
+            assert 'restore_fixture' not in fresh.record
+            # A readable dump from the wrong database must also fail the barrier.
+            fresh.record.pop('backup_restore_verified')
+            fresh.record['database_before']['tables']['contents']['rows_sha256'] = '0'*64
+            try:
+                fresh.verify_database_backup()
+            except RuntimeError as error:
+                assert 'differs' in str(error)
+            else:
+                raise AssertionError('Wrong database backup incorrectly accepted')
+            assert 'restore_fixture' not in fresh.record
+            assert 'backup_restore_verified' not in fresh.record
+            path = backup/'database.dump'
+            path.write_bytes(dump[:32])
+            fresh.record['artifacts']['database.dump'] = {'bytes': path.stat().st_size, 'sha256': maintenance.sha(path)}
+            try:
+                fresh.verify_database_backup()
+            except subprocess.CalledProcessError:
+                pass
+            else:
+                raise AssertionError('Truncated fresh backup incorrectly accepted')
+            assert 'restore_fixture' not in fresh.record
+            assert 'backup_restore_verified' not in fresh.record
+            assert fresh.fence.read_text() == fresh.operation
             sql('CREATE DATABASE restore_check')
             restore_command = ['podman', 'exec', '-i', container, 'pg_restore',
                                '-U', 'indeedhub', '-d', 'restore_check',
@@ -104,7 +158,8 @@ def main():
             maintenance.verify_database_compatibility(before, controller.database_commitments())
             print(json.dumps({'postgres_commitments': 'passed', 'rejected_mutations': rejected,
                               'network': 'none', 'live_volumes_mounted': False,
-                              'custom_dump_restored': True, 'truncated_dump_rejected': True}))
+                              'custom_dump_restored': True, 'truncated_dump_rejected': True,
+                              'production_restore_barrier': 'passed', 'wrong_backup_rejected': True}))
     finally:
         if container:
             subprocess.run(['podman', 'rm', '-f', container], check=True, stdout=subprocess.DEVNULL)