diff --git a/docs/managed-update-recovery-implementation.md b/docs/managed-update-recovery-implementation.md index f0c80bff..a7209e27 100644 --- a/docs/managed-update-recovery-implementation.md +++ b/docs/managed-update-recovery-implementation.md @@ -354,3 +354,29 @@ VM fixture directory. This qualifies the primitive, not a full backend update. The earlier held API diagnostic lacks the new durable proof and remains unaccepted; it must be restored by the matching rebuilt manager before a fresh full update/rollback rehearsal. No live Yaya mutation or activation occurred. + +### Restart policy and recovery qualification checkpoint + +The actual retained Yaya units use `Restart=always`, `StopTimeout=30` and +`TimeoutStopSec=45`. The earlier synthetic `Restart=no` baseline was not +sufficient acceptance. The controller now holds automatic API restart through +an operation-owned runtime drop-in, without rewriting its retained unit body. +It records creation intent, requires exact owned bytes/mode and safe ancestry, +recreates a missing runtime override after reboot, verifies effective policy, +and removes only its own file at terminal release. The reviewed target must +retain the original Restart policy; a differing terminal policy stays held. + +All **45 pure controller tests pass**. The actual hardened VM primitive passed +`always -> no -> always`, lost-runtime-file recreation, unchanged unit body and +no API startup (`restart-policy-probe.receipt.json`). This is primitive evidence, +not full transaction acceptance. + +The rebuilt 9964b5c1 fixture executable reached native restoration but refused +with `Original launch configuration did not recover`. Its raw fingerprint +includes runtime-generated environment values/order. The manager was stopped; +all recovery images and the held transaction remain preserved. No old journal +hash was rewritten and no recovery was declared successful. Stable, versioned +fingerprints for fresh transactions are being qualified separately; legacy +records must retain strict comparison. The pre-target writer-preservation fix +in 07c7eb0f also awaits combined compilation/tests. No Yaya migration or catalog +activation has occurred. diff --git a/scripts/indeehub-maintenance-controller.py b/scripts/indeehub-maintenance-controller.py index 1e5b0347..13a745c5 100644 --- a/scripts/indeehub-maintenance-controller.py +++ b/scripts/indeehub-maintenance-controller.py @@ -177,6 +177,7 @@ def volume_archive_metadata(path): class Controller: def __init__(self, data, operation, lock_fd, runner=None): self.data=pathlib.Path(data);self.operation=operation;self.lock_fd=lock_fd;self.runner=runner + self.runtime_root=pathlib.Path('/run/user')/str(os.getuid()) require(str(uuid.UUID(operation))==operation,'Invalid operation UUID') self.root=self.data/'update-transactions'/'indeehub-maintenance'/operation self.path=self.root/'journal.json';self.fence=self.data/'app-maintenance'/'indeedhub' @@ -303,6 +304,48 @@ class Controller: require(observed.get('paused') is True and valid_queue_counts(observed.get('counts')) and all(v==0 for v in observed['counts'].values()),'Legacy API queue not paused and empty') extra=observed.get('extra');require(isinstance(extra,dict) and set(extra)=={'prioritized','waiting_children'} and all(type(v) is int and v==0 for v in extra.values()),'Legacy API has additional queued work') return observed + def api_restart_override_path(self): + require(self.runtime_root.is_dir() and not self.runtime_root.is_symlink() and self.runtime_root.stat().st_uid==os.getuid() and self.runtime_root.stat().st_mode & 0o022==0,'Unsafe user runtime directory') + parent=self.runtime_root + for name in ('systemd','user','indeedhub-api.service.d'): + parent=parent/name + parent.mkdir(mode=0o700,exist_ok=True) + require(parent.is_dir() and not parent.is_symlink() and parent.stat().st_uid==os.getuid() and parent.stat().st_mode & 0o022==0,'Unsafe API restart override directory') + return parent/('zz-archipelago-maintenance-'+self.operation+'.conf') + def api_restart_override_bytes(self): + return ('# Archipelago maintenance operation '+self.operation+'\n[Service]\nRestart=no\n').encode() + def verify_api_restart_override(self, path): + require(path.is_file() and not path.is_symlink() and path.stat().st_uid==os.getuid() and path.stat().st_mode & 0o777==0o600 and path.read_bytes()==self.api_restart_override_bytes(),'API restart override changed; hold retained') + def ensure_api_restart_override(self): + self.holds();self.fence_matches();path=self.api_restart_override_path();saved=self.record.get('api_restart_override') + if saved is None: + require(not path.exists() and not path.is_symlink(),'Unowned API restart override exists') + policy=self.run(['systemctl','--user','show','indeedhub-api.service','--property=Restart','--value']).decode().strip() + require(policy in ('no','always','on-success','on-failure','on-abnormal','on-watchdog','on-abort'),'Unrecognized original restart policy') + saved={'operation_id':self.operation,'original_policy':policy,'sha256':hashlib.sha256(self.api_restart_override_bytes()).hexdigest(),'released':False} + self.record['api_restart_override']=saved;self.save() + require(saved.get('operation_id')==self.operation and saved.get('sha256')==hashlib.sha256(self.api_restart_override_bytes()).hexdigest() and saved.get('released') is False,'API restart override obligation changed') + if path.exists() or path.is_symlink():self.verify_api_restart_override(path) + else: + descriptor=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600) + with os.fdopen(descriptor,'wb') as stream:stream.write(self.api_restart_override_bytes());stream.flush();os.fsync(stream.fileno()) + directory=os.open(path.parent,os.O_RDONLY);os.fsync(directory);os.close(directory) + self.run(['systemctl','--user','daemon-reload']) + require(self.run(['systemctl','--user','show','indeedhub-api.service','--property=Restart','--value']).decode().strip()=='no','API automatic restart did not close') + saved['installed']=True;self.save() + def release_api_restart_override(self): + saved=self.record.get('api_restart_override') + if not saved or saved.get('released') is True:return + require(saved.get('operation_id')==self.operation and saved.get('sha256')==hashlib.sha256(self.api_restart_override_bytes()).hexdigest(),'API restart override ownership changed') + path=self.api_restart_override_path() + if path.exists() or path.is_symlink(): + self.verify_api_restart_override(path);path.unlink() + directory=os.open(path.parent,os.O_RDONLY);os.fsync(directory);os.close(directory) + # /run may have been cleared by a reboot, or unlink may have completed + # before an interrupted reply. Absence still requires effective-policy verification. + self.run(['systemctl','--user','daemon-reload']) + require(self.run(['systemctl','--user','show','indeedhub-api.service','--property=Restart','--value']).decode().strip()==saved['original_policy'],'Original API restart policy was not restored; hold retained') + saved['released']=True;self.save() def signal_legacy_api(self, member): stopped=self.record['stopped'][member['name']] if stopped.get('api_signal'): @@ -311,8 +354,7 @@ class Controller: image=self.api_recovery_identity(member) require(datetime.datetime.fromisoformat(image['Created'].replace('Z','+00:00')).timestamp()<=stopped['intent_at'],'API recovery image was not captured before stop') actual=self.inspect(member['name']);require(actual['Id']==member['container_id'] and actual['Image']==member['image_id'] and actual['State']['Running'],'Original API changed before signal') - restart=self.run(['systemctl','--user','show',member['name']+'.service','--property=Restart','--value']).decode().strip() - require(restart=='no','Legacy API signal requires no automatic restart') + self.ensure_api_restart_override() binding=self.api_queue_binding(member,actual,image) queue=self.queue('status');prefix=queue.get('prefix');before=self.observe_empty_redis_queue(member,prefix,binding) self.legacy_api_idle() @@ -645,6 +687,7 @@ class Controller: else: self.record['rollback_data_claim']='No target startup/migration began; only original runtime restored.' + self.release_api_restart_override() if not self.record.get('queue_was_paused',True): state=self.queue('resume');require(not state['paused'],'Could not restore queue admission') self.record['phase']='Released';self.record['outcome']=outcome;self.save() diff --git a/tests/regression/test_indeehub_maintenance_controller.py b/tests/regression/test_indeehub_maintenance_controller.py index 544bc111..4b110334 100644 --- a/tests/regression/test_indeehub_maintenance_controller.py +++ b/tests/regression/test_indeehub_maintenance_controller.py @@ -149,6 +149,23 @@ class MaintenanceTests(unittest.TestCase): module.atomic(runtime,{'phase':'Restored','target_startup_began':False}) self.assertEqual(c.release('aborted')['state'],'released') self.assertIn('No target startup',c.record['rollback_data_claim']) + def test_partial_frontend_recovery_preserves_active_work_without_claiming_a_drain(self): + c=self.controller;c.record={'operation_id':self.operation,'original_members':module.validate_members(members()),'phase':'Prepared','queue_was_paused':False,'queue_pause_confirmed':True,'last_queue_counts':{name:(1 if name=='active' else 0) for name in module.QUEUE_COUNTS},'stopped':{'indeedhub':{'confirmed':True}}};c.save() + runtime=c.data/'update-transactions'/'supervised'/(self.operation+'.json');module.atomic(runtime,{'phase':'Restoring','target_startup_began':False}) + c.fence.parent.mkdir(parents=True);c.fence.write_text(self.operation);c.close_ingress=lambda:c.fence_matches() + self.assertEqual(c.acquire(members(),recovery=True)['state'],'recovering') + self.assertEqual(c.verify()['state'],'held');self.assertFalse(c.record.get('backup_complete',False)) + self.assertNotIn('indeedhub-ffmpeg',c.record['stopped']);self.assertEqual(self.calls,[]) + # Native restoration must first verify preserved originals and the restarted + # frontend. The helper never fabricates that terminal native decision. + with self.assertRaises(RuntimeError):c.release('restored') + module.atomic(runtime,{'phase':'Restored','target_startup_began':False}) + actions=[] + def queue(action):actions.append(action);return {'paused':False,'counts':dict(c.record['last_queue_counts'])} + c.queue=queue + self.assertEqual(c.release('restored')['state'],'released');self.assertEqual(actions,['resume']) + self.assertEqual(c.record['last_queue_counts']['active'],1) + self.assertFalse(c.record.get('backup_complete',False));self.assertEqual(self.calls,[]) def test_target_started_rollback_requires_data_compatibility(self): c=self.controller;c.record={'operation_id':self.operation,'phase':'Recovering'};c.save() c.fence.parent.mkdir(parents=True);c.fence.write_text(self.operation) @@ -329,6 +346,54 @@ class MaintenanceTests(unittest.TestCase): with self.assertRaisesRegex(RuntimeError,'OOM'):c.legacy_api_wrapper_termination(m,p) c.runner=lambda argv,timeout,output: b'replacement-id' if argv[:2]==['podman','ps'] and any('name=^' in x for x in argv) else runner(argv,timeout,output) with self.assertRaisesRegex(RuntimeError,'writer is still running'):c.legacy_api_wrapper_termination(m,p) + def restart_policy_fixture(self): + c=self.controller;c.record={'operation_id':self.operation};c.runtime_root=self.root/'runtime';c.runtime_root.mkdir(mode=0o700) + c.fence.parent.mkdir(parents=True);c.fence.write_text(self.operation) + path=c.api_restart_override_path();state={'original':'always'} + def runner(argv,timeout,output): + self.calls.append(argv) + if argv==['systemctl','--user','daemon-reload']:return b'' + if argv==['systemctl','--user','show','indeedhub-api.service','--property=Restart','--value']:return ('no' if path.exists() else state['original']).encode() + raise AssertionError(argv) + c.runner=runner + return c,path,state + def test_owned_restart_override_restores_always_without_rewriting_unit(self): + c,path,state=self.restart_policy_fixture();c.ensure_api_restart_override() + self.assertEqual(path.read_bytes(),c.api_restart_override_bytes());self.assertEqual(c.record['api_restart_override']['original_policy'],'always') + self.assertEqual(path.stat().st_mode&0o777,0o600) + c.ensure_api_restart_override();c.release_api_restart_override();self.assertFalse(path.exists()) + self.assertTrue(c.record['api_restart_override']['released']);calls=len(self.calls);c.release_api_restart_override();self.assertEqual(len(self.calls),calls) + self.assertTrue(all('stop' not in argv and 'revert' not in argv for argv in self.calls)) + def test_restart_override_conflict_or_tamper_never_overwrites_or_unlinks(self): + c,path,state=self.restart_policy_fixture();path.write_text('foreign') + with self.assertRaises(RuntimeError):c.ensure_api_restart_override() + self.assertEqual(path.read_text(),'foreign');self.assertEqual(self.calls,[]) + path.unlink();c.ensure_api_restart_override();path.write_text('changed') + for action in (c.ensure_api_restart_override,c.release_api_restart_override): + with self.assertRaises(RuntimeError):action() + self.assertEqual(path.read_text(),'changed') + def test_restart_override_lost_create_reply_and_reboot_missing_file_are_reverified(self): + c,path,state=self.restart_policy_fixture();c.ensure_api_restart_override();c.record['api_restart_override'].pop('installed');c.save() + resumed=module.Controller(c.data,self.operation,0,c.runner);resumed.runtime_root=c.runtime_root;resumed.ensure_api_restart_override() + self.assertEqual(resumed.record['api_restart_override']['original_policy'],'always') + path.unlink();resumed.ensure_api_restart_override();self.assertTrue(path.exists()) + self.assertEqual(resumed.record['api_restart_override']['original_policy'],'always') + def test_restart_policy_mismatch_retains_obligation_after_override_removal(self): + c,path,state=self.restart_policy_fixture();c.ensure_api_restart_override();state['original']='on-failure' + with self.assertRaises(RuntimeError):c.release_api_restart_override() + self.assertFalse(path.exists());self.assertFalse(c.record['api_restart_override']['released']);self.assertTrue(c.fence.exists()) + state['original']='always';c.release_api_restart_override();self.assertTrue(c.record['api_restart_override']['released']) + def test_restart_override_symlink_or_wrong_mode_is_refused(self): + c,path,state=self.restart_policy_fixture();c.ensure_api_restart_override();path.chmod(0o644) + with self.assertRaises(RuntimeError):c.release_api_restart_override() + path.unlink();other=self.root/'foreign';other.write_text('retain');path.symlink_to(other) + with self.assertRaises(RuntimeError):c.ensure_api_restart_override() + with self.assertRaises(RuntimeError):c.release_api_restart_override() + self.assertEqual(other.read_text(),'retain') + def test_restart_override_writable_ancestor_is_refused(self): + c,path,state=self.restart_policy_fixture();c.runtime_root.chmod(0o770) + with self.assertRaises(RuntimeError):c.ensure_api_restart_override() + self.assertFalse(path.exists());self.assertEqual(self.calls,[]) def test_unacknowledged_api_signal_is_never_retried(self): c,m,p,*_=self.legacy_api_fixture();c.record['stopped'][m['name']]['api_signal']['acknowledged']=False with self.assertRaisesRegex(RuntimeError,'Unacknowledged'):c.signal_legacy_api(m)