From d4f3cceb52f4ae7d3de72ff0868aca0914748cad Mon Sep 17 00:00:00 2001 From: archipelago Date: Wed, 7 Oct 2026 14:43:22 -0400 Subject: [PATCH] Verify fresh IndeeHub backup restores and record remaining release tasks --- docs/SESSION-HANDOVER-20261007.md | 89 +++++++++++++ .../managed-update-recovery-implementation.md | 39 +++++- docs/post-1.9.0-work-backlog.md | 29 ++++ docs/release-uat-checklist-20261007.md | 8 +- scripts/indeehub-maintenance-controller.py | 86 ++++++++++-- .../supervised-runtime-primitives.py | 126 ++++++++++++++++++ .../test_indeehub_maintenance_controller.py | 21 +++ .../test_indeehub_maintenance_postgres.py | 59 +++++++- 8 files changed, 440 insertions(+), 17 deletions(-) create mode 100755 tests/lifecycle/supervised-runtime-primitives.py diff --git a/docs/SESSION-HANDOVER-20261007.md b/docs/SESSION-HANDOVER-20261007.md index 7fc0df61..1458e9df 100644 --- a/docs/SESSION-HANDOVER-20261007.md +++ b/docs/SESSION-HANDOVER-20261007.md @@ -126,6 +126,11 @@ Private backup and artifacts are under should continue on the phone after app close, with native controls, queue, artwork, authorization and video PiP. Physical V4V acceptance is open. +21. **Public files/folders — queued, perform last:** Make public in both More + menus, discovery by non-peered/non-federated nodes, the same peer-sharing + pricing interface with blue tints instead of orange, optional charging and + a tiny blue marker at the icon's top-left. Full scope is backlog task21. + ## Additional release-regression items - Paid-file recovery must never issue a second payment; source protections and @@ -223,3 +228,87 @@ release receipt or preservation check as a stop-and-investigate condition. passes unchanged/additive schema and rejects four data/schema/history mutations. Full Rust and real supervised Podman/systemd qualification remain required before integration or IndeeHub activation. + + +## Operator PiP acceptance and continued release work — 2026-10-07 + +- Operator confirmed the delivered companion PiP works (“works great”). Record + this as operator acceptance of the reported PiP flow, not independent proof + of every rotation/network/expiry case or native V4V background-audio support. +- Retained updater is now integrated locally at `7fb7ee80`. The combined isolated + backend suite passed **2,000 tests, zero failures, five ignored**. Prior notes + saying integration/Rust qualification are pending are historical. +- Real disposable Quadlet/Podman AutoRemove primitive qualification passed: + injected target failure, restoration from the original writable-layer image, + preserved volume bytes and original configuration. The full seven-member + application drain/cutover adapter is still not qualified by this primitive test. +- Evidence: `/tmp/archy-resumed-20261007-integrated-full-backend.log` and + `/tmp/archy-resumed-20261007-runtime-primitives-recheck.log`. +- Active work: require restoration of each fresh database backup in an owned, + network-isolated PostgreSQL before IndeeHub target startup. Hash verification + alone is insufficient. All 21 controller tests pass; real restore/fault checks + passed, including the actual production restore method, rejection of truncated + and wrong-database dumps, owned fixture cleanup and retained admission fences. + Evidence: `/tmp/archy-20261007-fresh-backup-restore.log`. + This script change needs a fresh embedded-controller backend + build; do not call the older 2,000-test result evidence for this new change. +- Task21 is appended at the end, including the operator's explicit requirement + to reuse peer-sharing pricing with blue tints. It is not implemented yet. + + +### Current status overview requested by the operator + +“Nearly finished” means implemented with a bounded acceptance/review step left; +“In progress” still includes substantive implementation or distributed testing. +No percentage is inferred from test counts. + +| Task | Status | Remaining work | +| --- | --- | --- | +| 1 IndeeHub publishing/paid viewing | In progress | Full supervised cutover, distributed publication/payment/timed playback | +| 2 Native signer/companion grey screen | Nearly finished | Physical login/resume/session recovery acceptance | +| 3 Peering/discovery | In progress | Reciprocal offline/reconnect and expanded connection UX | +| 4 Framework Monitoring | Nearly finished | Owner-browser Monitoring check | +| 5 Immich/Nextcloud | Slightly started | Design assessment exists; connectors unimplemented | +| 6 Web5 connection journey | In progress | Deferred final flow review and expanded UX | +| 7 Companion launch speed | In progress | Actual device measurements and remaining latency work | +| 8 Fleet acceptance | In progress | Full supported-function/fault matrix | +| 9 AIUI/provider/funding | In progress | Provider/funding and companion acceptance | +| 10 Offline/network map | In progress | Real outage and recovery qualification | +| 11 Web5/Cloud/tab speed | In progress | Complete performance evidence | +| 12 Fleet metrics/FIPS | In progress | Fleet failure and transport qualification | +| 13 V4V Yaya demo | In progress | Browser demo live; native phone background playback remains | +| 14 Peer files/Indee streaming | In progress | Distributed timed playback | +| 15 MeshCore | Queued | Two-radio work and acceptance | +| 16 Web5 cards/footer | Nearly finished | Final visual/functional review | +| 17 HTTPS apps | In progress | Exact hostname/trust and companion acceptance | +| 18 Firewall/tunnel | In progress | Read-only UI done; settings persistence/reboot/rollback qualification | +| 19 Media guide/Cloud PiP | Nearly finished | PiP operator-accepted; finish reusable contract and remaining edge cases | +| 20 Native background media | In progress | Cloud PiP done; native audio service/controls and device checks remain | +| 21 Public files/folders | Queued, final task | Shared pricing interface with blue tints, unrelated-node discovery and tiny marker | + +Completed subitems: Framework LND startup incident, companion download/viewer +acceptance, physical upload acceptance, APK55 delivery and reported Cloud PiP +flow. Earlier complete UI/Android suites pass; a fresh backend rebuild is required +for the final restore barrier. OTA/ISO is not ready: payment/recovery, IndeeHub, +fleet/device qualification, mirror review/parity and final artifact gates remain. + + +### Fresh backup barrier qualification result + +- Final controller:21 pure tests pass. Real production restore-barrier tests + pass on PostgreSQL15.17 and16.13, including valid restoration, wrong-database + and truncated-dump refusal, retained admission, owned fixture cleanup, and + four original data/schema/history mutation rejections. +- The first PG15 attempt failed during pg_restore. That attempt's command error + log was in an automatically removed fixture directory, so its exact cause is + not proven. Readiness inspection found the bootstrap Unix-socket server could + be mistaken for the final server. The controller/fixture now wait for TCP + readiness; both final version runs pass. Do not erase the failed attempt. +- The backend rebuild begun before the readiness edit was interrupted deliberately + because its embedded source was stale and it competed with restore tests. + **Final embedded-controller backend rebuild/suite and deployment remain pending.** + The prior 2,000-pass receipt applies to `7fb7ee80`, not this changed controller. +- Durable logs: + `~/.local/state/archipelago/release-qualification/indeehub-backup-restore-20261007/`. +- No live app/container, database, wallet, catalog, payment or public file was + changed. Disposable fixtures used network-none and no live volumes. diff --git a/docs/managed-update-recovery-implementation.md b/docs/managed-update-recovery-implementation.md index 9f59b0c0..dd29f09e 100644 --- a/docs/managed-update-recovery-implementation.md +++ b/docs/managed-update-recovery-implementation.md @@ -1,7 +1,7 @@ # Managed update runtime recovery -Status: isolated source implementation; Rust and real Podman/systemd qualification -pending. No live update, snapshot, stop, backup or rollback has been performed by +Status: integrated candidate; combined Rust suite and real Podman/systemd +recovery primitives pass. Full application cutover qualification is pending. No live update, snapshot, stop, backup or rollback has been performed by this work. Active deployed source is unchanged. The managed path captures original source Quadlet bytes, mode, immutable image, @@ -75,3 +75,38 @@ and PostgreSQL timeout remain failed/incomplete attempts, not acceptance. Evidence: `/tmp/archy-resumed-20261007-updater-full-backend.log`, `/tmp/archy-resumed-20261007-indeehub-controller-tests-final.log`, and `/tmp/archy-resumed-20261007-indeehub-postgres-restore.log`. + + +## Integrated candidate checkpoint — 2026-10-07 + +Local integration at `7fb7ee80` passes the complete isolated backend suite: +2,000 passed, zero failed, five ignored. The stale receipt fixture failure above +is resolved by the already-integrated correction. Disposable real Quadlet +AutoRemove recovery primitives pass with injected target failure, original +writable-layer/configuration restoration and unchanged persistent fixture bytes. +Repeatable fixture: `tests/lifecycle/supervised-runtime-primitives.py`. +This does not qualify the complete seven-member app drain/cutover adapter. + +The fresh-backup restore barrier is being added after this checkpoint; its +qualification and new embedded-controller build remain separate from these +previously passing results. No live IndeeHub deployment has been changed. + + +Fresh database backup restoration now runs through the controller's production +method in a disposable network-none PostgreSQL with no external mounts or +published ports. The original local image is pinned, dump restore must exactly +match captured database commitments, and durable proof binds the operation, +image, dump hash and baseline. Ownership-checked cleanup survives retry and +refuses foreign fixtures. Verification rejects a missing/stale restore proof. + +All21 pure controller tests pass. The real PostgreSQL fixture passes valid +restoration, rejects truncated and wrong-database dumps, checks cleanup and +retained admission on failure, and retains the four prior mutation rejection +checks. Evidence: `/tmp/archy-20261007-fresh-backup-restore.log`. +Final real restore-barrier checks pass on PostgreSQL15.17 and16.13 after waiting +for the final TCP server instead of the temporary Unix-socket bootstrap server. +The initial PG15 restore failure is retained as failed evidence; its private +command stderr was removed by fixture cleanup, so no exact cause is claimed. +The stale backend compile was interrupted after the readiness edit; a fresh +backend build/suite remains required for the final embedded controller. +Volume-archive restore and the full supervised app cutover remain open gates. diff --git a/docs/post-1.9.0-work-backlog.md b/docs/post-1.9.0-work-backlog.md index 40b61446..24e2c109 100644 --- a/docs/post-1.9.0-work-backlog.md +++ b/docs/post-1.9.0-work-backlog.md @@ -549,3 +549,32 @@ and video, with video using picture-in-picture where the device supports it. reconnect, completion and reopen states, then publish the reusable contract for other audio/video apps. No live app deployment or payment is accepted from browser-only tests. + + +## 21. Public files and folders — final task + +Added by the operator on 2026-10-07. Perform after the existing tasks, including +previously deferred work. Status: queued; no public visibility has been changed. + +- Add **Make public** to the More menus for both files and folders. Public means + discoverable by any node, including nodes with no peering or federation + relationship to the owner. Relationship approval must not be required merely + to discover a public item. +- Reuse the existing **Share with peers pricing interface**, including optional + charging and its supported pricing/payment choices. Use the existing brand + **blue tints instead of orange** for this public-sharing variant; do not create + a separate pricing interaction or change the peer-sharing colour treatment. +- Place a very small blue public-state marker at the **top-left of the file or + folder icon**. Keep the marker readable and accessible without covering the + thumbnail or replacing the existing icon. +- Separate public discoverability from paid access: listed paid content must + still enforce its price/entitlement. Reuse settlement and recovery protections + so retries do not charge twice. +- Make folder scope explicit in the reused flow, including existing/new children, + nested items and individual visibility/pricing overrides. Preserve private + items until the owner actually applies the action; provide a way to withdraw + public sharing and reflect that state in menus, listings and markers. +- Qualify file/folder menus, shared pricing UI with blue styling, tiny marker, + unrelated-node discovery, paid access, withdrawal and mobile/desktop layouts. + No live file publication or real payment is authorized merely by adding this + task to the backlog. diff --git a/docs/release-uat-checklist-20261007.md b/docs/release-uat-checklist-20261007.md index 6f9d4781..985213db 100644 --- a/docs/release-uat-checklist-20261007.md +++ b/docs/release-uat-checklist-20261007.md @@ -128,9 +128,11 @@ does not close a release gate. ## Known release blockers 1. IndeeHub distributed paid playback and supervised cutover rollback. -2. Physical companion background media and video PiP. -3. Framework owner-browser Receive/balance confirmation still documented as - pending in the incident record. +2. Companion native background audio/media; operator accepted delivered Cloud + PiP on 2026-10-07. Detailed interruption/expiry cases remain distinct. +3. Framework Monitoring owner-browser acceptance. The earlier LND startup, + Receive and false-zero incident is closed with operator acceptance; it is + not a release blocker. Paid-file settlement/recovery is a separate open gate. 4. Fleet-wide offline/reconnect and FIPS acceptance. 5. Firewall/tunnel persistence and reboot qualification. 6. HTTPS hostname/trust and companion acceptance. diff --git a/scripts/indeehub-maintenance-controller.py b/scripts/indeehub-maintenance-controller.py index daee9d4d..e7961b59 100644 --- a/scripts/indeehub-maintenance-controller.py +++ b/scripts/indeehub-maintenance-controller.py @@ -108,11 +108,11 @@ class Controller: self.record=json.loads(self.path.read_text()) if self.path.exists() else None if self.record:require(self.record['operation_id']==operation,'Maintenance journal changed') def save(self): atomic(self.path,self.record) - def run(self, argv, timeout=30, output=None, input_bytes=None): + def run(self, argv, timeout=30, output=None, input_bytes=None, input_file=None): if self.runner:return self.runner(argv,timeout,output) self.root.mkdir(mode=0o700,parents=True,exist_ok=True) with (self.root/'commands.private.log').open('ab') as errors: - result=subprocess.run(argv,stdout=output or subprocess.PIPE,stderr=errors,timeout=timeout,check=True,pass_fds=(self.lock_fd,),input=input_bytes) + result=subprocess.run(argv,stdout=output or subprocess.PIPE,stderr=errors,timeout=timeout,check=True,pass_fds=(self.lock_fd,),input=input_bytes,stdin=input_file) if output:return b'' require(len(result.stdout)<=2*1024*1024,'Command response exceeds bound') return result.stdout @@ -197,8 +197,8 @@ class Controller: rows=json.loads(self.run(['podman','volume','inspect',*expected])) require({row['Name'] for row in rows}==set(expected),'Persistent volume scope changed') return {row['Name']:row['Mountpoint'] for row in rows} - def database_commitments(self): - raw=self.run(['podman','exec','-i','indeedhub-postgres','psql','-XqAt','--set=ON_ERROR_STOP=1','-U','indeedhub','-d','indeedhub'],timeout=300,input_bytes=DB_COMMITMENTS_SQL.encode()) + def database_commitments(self, container="indeedhub-postgres"): + raw=self.run(['podman','exec','-i',container,'psql','-XqAt','--set=ON_ERROR_STOP=1','-U','indeedhub','-d','indeedhub'],timeout=300,input_bytes=DB_COMMITMENTS_SQL.encode()) rows=[json.loads(line) for line in raw.decode().splitlines() if line.strip()] tables={};migrations=None for row in rows: @@ -271,7 +271,76 @@ class Controller: require(time.monotonic() /original-layer; printf persistent-bytes > /state/value') + volume_hash = hashlib.sha256((data / 'value').read_bytes()).hexdigest() + run('podman', 'commit', '--pause=true', '--include-volumes=false', + '--change', 'LABEL io.archipelago.recovery.operation=' + operation, + '--change', 'LABEL io.archipelago.recovery.container=' + old['Id'], old['Id'], tag) + captured = json.loads(run('podman', 'image', 'inspect', tag).stdout)[0] + snapshot = captured['Id'] + assert captured['Config']['Labels']['io.archipelago.recovery.operation'] == operation + assert captured['Config']['Labels']['io.archipelago.recovery.container'] == old['Id'] + run('podman', 'run', '--rm', '--network=none', '--pull=never', '--entrypoint=/bin/sh', snapshot, + '-c', 'test "$(cat /original-layer)" = original-layer; test ! -f /state/value') + run('systemctl', '--user', 'stop', service) + assert run('podman', 'container', 'exists', old['Id'], check=False).returncode == 1 + failed = original.replace('Environment=MODE=original', 'Environment=MODE=candidate').replace( + 'Exec=-c "trap \'exit 0\' TERM; while sleep 1; do :; done"', 'Exec=-c "exit 77"') + assert failed != original + write(failed) + run('systemctl', '--user', 'start', service, check=False) + deadline = time.monotonic() + 15 + while run('systemctl', '--user', 'is-active', service, check=False).returncode == 0: + assert time.monotonic() < deadline, 'Fault injection unexpectedly remained active' + time.sleep(0.2) + assert run('systemctl', '--user', 'show', service, '--property=Result', '--value').stdout.strip() != 'success' + run('systemctl', '--user', 'stop', service, check=False) + restored = original.replace('Image=sha256:' + image.removeprefix('sha256:'), + 'Image=sha256:' + snapshot.removeprefix('sha256:')) + write(restored) + run('systemctl', '--user', 'reset-failed', service, check=False) + run('systemctl', '--user', 'start', service) + current = inspect() + assert current['State']['Running'] and current['Id'] != old['Id'] + assert current['Image'].removeprefix('sha256:') == snapshot.removeprefix('sha256:') + assert 'MODE=original' in current['Config']['Env'] + assert unit.read_text() == restored and unit.stat().st_mode & 0o777 == 0o600 + assert run('podman', 'exec', name, 'cat', '/original-layer').stdout == 'original-layer' + assert hashlib.sha256((data / 'value').read_bytes()).hexdigest() == volume_hash + print(json.dumps({'auto_remove_recovery': 'passed', 'writable_layer_preserved': True, + 'volume_bytes_preserved': True, 'original_configuration_restored': True, + 'injected_target_failure': True, 'network': 'none', + 'full_supervised_application_cutover': 'not tested'})) + finally: + run('systemctl', '--user', 'stop', service, check=False) + unit.unlink(missing_ok=True) + unit.with_suffix('.next').unlink(missing_ok=True) + run('systemctl', '--user', 'daemon-reload') + run('systemctl', '--user', 'reset-failed', service, check=False) + run('podman', 'rm', '-f', name, check=False) + if snapshot: + run('podman', 'rmi', tag) + assert root.parent == Path('/tmp') and root.name.startswith(name + '-') + run('podman', 'unshare', 'rm', '-rf', str(root)) + + +if __name__ == '__main__': + main() diff --git a/tests/regression/test_indeehub_maintenance_controller.py b/tests/regression/test_indeehub_maintenance_controller.py index 96ef50bb..65ed2ac4 100644 --- a/tests/regression/test_indeehub_maintenance_controller.py +++ b/tests/regression/test_indeehub_maintenance_controller.py @@ -56,10 +56,31 @@ class MaintenanceTests(unittest.TestCase): for name in ['database.dump',*(v+'.tar' for v in module.VOLUMES)]: path=c.root/'backup'/name;path.write_bytes(b'original') c.record['artifacts'][name]={'bytes':path.stat().st_size,'sha256':module.sha(path)} + c.record['original_members']=members() + c.record['database_before']={'operation_id':self.operation,'tables':{},'migrations':[]} + c.record['backup_restore_verified']=c.backup_restore_terms() c.save() return c def test_complete_backup_checksums_allow_verification(self): self.assertEqual(self.completed_backup().verify()['state'],'held') + def test_restore_proof_must_match_fresh_dump_baseline_image_and_operation(self): + c=self.completed_backup();proof=dict(c.record['backup_restore_verified']) + for field in proof: + c.record['backup_restore_verified']={**proof,field:'changed'} + with self.assertRaisesRegex(RuntimeError,'restore is not verified'):c.verify() + self.assertEqual(c.fence.read_text(),self.operation) + c.record.pop('backup_restore_verified') + with self.assertRaisesRegex(RuntimeError,'restore is not verified'):c.verify() + def test_foreign_restore_fixture_is_never_removed(self): + c=self.completed_backup();name='archy-backup-restore-'+'a'*32 + c.record['restore_fixture']={'name':name,'image_id':'a'*64} + def command(argv,timeout,output): + if argv[:2]==['podman','ps']:return b'container-id' + if argv[:2]==['podman','inspect']:return json.dumps([{'Name':name,'Image':'a'*64,'Config':{'Labels':{'io.archipelago.backup.operation':str(uuid.uuid4())}}}]).encode() + raise AssertionError('Unexpected mutation '+str(argv)) + c.runner=command + with self.assertRaisesRegex(RuntimeError,'ownership changed'):c.cleanup_restore_fixture() + self.assertIn('restore_fixture',c.record) def test_same_size_corruption_keeps_admission_closed(self): c=self.completed_backup();(c.root/'backup'/'database.dump').write_bytes(b'corrupt!') with self.assertRaisesRegex(RuntimeError,'checksum changed'):c.verify() diff --git a/tests/regression/test_indeehub_maintenance_postgres.py b/tests/regression/test_indeehub_maintenance_postgres.py index 8410c35c..4ea8d6fd 100644 --- a/tests/regression/test_indeehub_maintenance_postgres.py +++ b/tests/regression/test_indeehub_maintenance_postgres.py @@ -5,6 +5,7 @@ Requires an already imported image: --image IMAGE. Never mounts node volumes, publishes ports, or invokes the maintenance entrypoint against installed apps. """ import argparse +import copy import importlib.util import json from pathlib import Path @@ -36,7 +37,7 @@ def main(): '-e', 'POSTGRES_DB=indeedhub', image, ], text=True).strip() deadline = time.monotonic() + 60 - while subprocess.run(['podman', 'exec', container, 'pg_isready', '-U', 'indeedhub'], + while subprocess.run(['podman', 'exec', container, 'pg_isready', '-h', '127.0.0.1', '-U', 'indeedhub'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode: if time.monotonic() > deadline: raise RuntimeError('Disposable PostgreSQL did not become ready') @@ -68,6 +69,59 @@ def main(): 'podman', 'exec', container, 'pg_dump', '-U', 'indeedhub', '-d', 'indeedhub', '--format=custom', '--no-owner', '--no-acl', ], timeout=60) + # Exercise the production fresh-backup restore barrier, not just + # hand-written pg_restore commands. All containers are owned fixtures. + fresh = maintenance.Controller(root, str(uuid.uuid4()), 0) + fresh.record = {'operation_id': fresh.operation, + 'original_members': [{'name': member, 'container_id': 'a'*64, + 'image_id': image.removeprefix('sha256:'), 'unit_sha256': 'b'*64, + 'config_sha256': 'c'*64, 'running': True} for member in maintenance.NAMES], + 'database_before': {**copy.deepcopy(before), 'operation_id': fresh.operation}, 'artifacts': {}} + holds = fresh.data/'update-transactions'/'holds' + holds.mkdir(parents=True) + for member in maintenance.NAMES: (holds/member).write_text(fresh.operation) + fresh.fence.parent.mkdir(parents=True) + fresh.fence.write_text(fresh.operation) + backup = fresh.root/'backup' + backup.mkdir(parents=True) + for artifact in ['database.dump', *(v+'.tar' for v in maintenance.VOLUMES)]: + path = backup/artifact + path.write_bytes(dump if artifact == 'database.dump' else b'volume-fixture') + fresh.record['artifacts'][artifact] = {'bytes': path.stat().st_size, 'sha256': maintenance.sha(path)} + fresh.save() + try: + fresh.verify_database_backup() + except Exception: + # Fixture-only SQL diagnostics; this test never opens live data. + diagnostic = fresh.root/'commands.private.log' + if diagnostic.exists(): + Path('/tmp/archy-backup-fixture-failure.log').write_bytes(diagnostic.read_bytes()) + raise + assert fresh.record['backup_restore_verified'] == fresh.backup_restore_terms() + assert 'restore_fixture' not in fresh.record + # A readable dump from the wrong database must also fail the barrier. + fresh.record.pop('backup_restore_verified') + fresh.record['database_before']['tables']['contents']['rows_sha256'] = '0'*64 + try: + fresh.verify_database_backup() + except RuntimeError as error: + assert 'differs' in str(error) + else: + raise AssertionError('Wrong database backup incorrectly accepted') + assert 'restore_fixture' not in fresh.record + assert 'backup_restore_verified' not in fresh.record + path = backup/'database.dump' + path.write_bytes(dump[:32]) + fresh.record['artifacts']['database.dump'] = {'bytes': path.stat().st_size, 'sha256': maintenance.sha(path)} + try: + fresh.verify_database_backup() + except subprocess.CalledProcessError: + pass + else: + raise AssertionError('Truncated fresh backup incorrectly accepted') + assert 'restore_fixture' not in fresh.record + assert 'backup_restore_verified' not in fresh.record + assert fresh.fence.read_text() == fresh.operation sql('CREATE DATABASE restore_check') restore_command = ['podman', 'exec', '-i', container, 'pg_restore', '-U', 'indeedhub', '-d', 'restore_check', @@ -104,7 +158,8 @@ def main(): maintenance.verify_database_compatibility(before, controller.database_commitments()) print(json.dumps({'postgres_commitments': 'passed', 'rejected_mutations': rejected, 'network': 'none', 'live_volumes_mounted': False, - 'custom_dump_restored': True, 'truncated_dump_rejected': True})) + 'custom_dump_restored': True, 'truncated_dump_rejected': True, + 'production_restore_barrier': 'passed', 'wrong_backup_rejected': True})) finally: if container: subprocess.run(['podman', 'rm', '-f', container], check=True, stdout=subprocess.DEVNULL)