fix(indeehub): retain API restart policy across maintenance

docs/managed-update-recovery-implementation.md

@@ -354,3 +354,29 @@ VM fixture directory. This qualifies the primitive, not a full backend update.
 The earlier held API diagnostic lacks the new durable proof and remains
 unaccepted; it must be restored by the matching rebuilt manager before a fresh
 full update/rollback rehearsal. No live Yaya mutation or activation occurred.
+
+### Restart policy and recovery qualification checkpoint
+
+The actual retained Yaya units use `Restart=always`, `StopTimeout=30` and
+`TimeoutStopSec=45`. The earlier synthetic `Restart=no` baseline was not
+sufficient acceptance. The controller now holds automatic API restart through
+an operation-owned runtime drop-in, without rewriting its retained unit body.
+It records creation intent, requires exact owned bytes/mode and safe ancestry,
+recreates a missing runtime override after reboot, verifies effective policy,
+and removes only its own file at terminal release. The reviewed target must
+retain the original Restart policy; a differing terminal policy stays held.
+
+All **45 pure controller tests pass**. The actual hardened VM primitive passed
+`always -> no -> always`, lost-runtime-file recreation, unchanged unit body and
+no API startup (`restart-policy-probe.receipt.json`). This is primitive evidence,
+not full transaction acceptance.
+
+The rebuilt 9964b5c1 fixture executable reached native restoration but refused
+with `Original launch configuration did not recover`. Its raw fingerprint
+includes runtime-generated environment values/order. The manager was stopped;
+all recovery images and the held transaction remain preserved. No old journal
+hash was rewritten and no recovery was declared successful. Stable, versioned
+fingerprints for fresh transactions are being qualified separately; legacy
+records must retain strict comparison. The pre-target writer-preservation fix
+in 07c7eb0f also awaits combined compilation/tests. No Yaya migration or catalog
+activation has occurred.

scripts/indeehub-maintenance-controller.py

@@ -177,6 +177,7 @@ def volume_archive_metadata(path):
 class Controller:
     def __init__(self, data, operation, lock_fd, runner=None):
         self.data=pathlib.Path(data);self.operation=operation;self.lock_fd=lock_fd;self.runner=runner
+        self.runtime_root=pathlib.Path('/run/user')/str(os.getuid())
         require(str(uuid.UUID(operation))==operation,'Invalid operation UUID')
         self.root=self.data/'update-transactions'/'indeehub-maintenance'/operation
         self.path=self.root/'journal.json';self.fence=self.data/'app-maintenance'/'indeedhub'
@@ -303,6 +304,48 @@ class Controller:
         require(observed.get('paused') is True and valid_queue_counts(observed.get('counts')) and all(v==0 for v in observed['counts'].values()),'Legacy API queue not paused and empty')
         extra=observed.get('extra');require(isinstance(extra,dict) and set(extra)=={'prioritized','waiting_children'} and all(type(v) is int and v==0 for v in extra.values()),'Legacy API has additional queued work')
         return observed
+    def api_restart_override_path(self):
+        require(self.runtime_root.is_dir() and not self.runtime_root.is_symlink() and self.runtime_root.stat().st_uid==os.getuid() and self.runtime_root.stat().st_mode & 0o022==0,'Unsafe user runtime directory')
+        parent=self.runtime_root
+        for name in ('systemd','user','indeedhub-api.service.d'):
+            parent=parent/name
+            parent.mkdir(mode=0o700,exist_ok=True)
+            require(parent.is_dir() and not parent.is_symlink() and parent.stat().st_uid==os.getuid() and parent.stat().st_mode & 0o022==0,'Unsafe API restart override directory')
+        return parent/('zz-archipelago-maintenance-'+self.operation+'.conf')
+    def api_restart_override_bytes(self):
+        return ('# Archipelago maintenance operation '+self.operation+'\n[Service]\nRestart=no\n').encode()
+    def verify_api_restart_override(self, path):
+        require(path.is_file() and not path.is_symlink() and path.stat().st_uid==os.getuid() and path.stat().st_mode & 0o777==0o600 and path.read_bytes()==self.api_restart_override_bytes(),'API restart override changed; hold retained')
+    def ensure_api_restart_override(self):
+        self.holds();self.fence_matches();path=self.api_restart_override_path();saved=self.record.get('api_restart_override')
+        if saved is None:
+            require(not path.exists() and not path.is_symlink(),'Unowned API restart override exists')
+            policy=self.run(['systemctl','--user','show','indeedhub-api.service','--property=Restart','--value']).decode().strip()
+            require(policy in ('no','always','on-success','on-failure','on-abnormal','on-watchdog','on-abort'),'Unrecognized original restart policy')
+            saved={'operation_id':self.operation,'original_policy':policy,'sha256':hashlib.sha256(self.api_restart_override_bytes()).hexdigest(),'released':False}
+            self.record['api_restart_override']=saved;self.save()
+        require(saved.get('operation_id')==self.operation and saved.get('sha256')==hashlib.sha256(self.api_restart_override_bytes()).hexdigest() and saved.get('released') is False,'API restart override obligation changed')
+        if path.exists() or path.is_symlink():self.verify_api_restart_override(path)
+        else:
+            descriptor=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
+            with os.fdopen(descriptor,'wb') as stream:stream.write(self.api_restart_override_bytes());stream.flush();os.fsync(stream.fileno())
+            directory=os.open(path.parent,os.O_RDONLY);os.fsync(directory);os.close(directory)
+        self.run(['systemctl','--user','daemon-reload'])
+        require(self.run(['systemctl','--user','show','indeedhub-api.service','--property=Restart','--value']).decode().strip()=='no','API automatic restart did not close')
+        saved['installed']=True;self.save()
+    def release_api_restart_override(self):
+        saved=self.record.get('api_restart_override')
+        if not saved or saved.get('released') is True:return
+        require(saved.get('operation_id')==self.operation and saved.get('sha256')==hashlib.sha256(self.api_restart_override_bytes()).hexdigest(),'API restart override ownership changed')
+        path=self.api_restart_override_path()
+        if path.exists() or path.is_symlink():
+            self.verify_api_restart_override(path);path.unlink()
+            directory=os.open(path.parent,os.O_RDONLY);os.fsync(directory);os.close(directory)
+        # /run may have been cleared by a reboot, or unlink may have completed
+        # before an interrupted reply. Absence still requires effective-policy verification.
+        self.run(['systemctl','--user','daemon-reload'])
+        require(self.run(['systemctl','--user','show','indeedhub-api.service','--property=Restart','--value']).decode().strip()==saved['original_policy'],'Original API restart policy was not restored; hold retained')
+        saved['released']=True;self.save()
     def signal_legacy_api(self, member):
         stopped=self.record['stopped'][member['name']]
         if stopped.get('api_signal'):
@@ -311,8 +354,7 @@ class Controller:
         image=self.api_recovery_identity(member)
         require(datetime.datetime.fromisoformat(image['Created'].replace('Z','+00:00')).timestamp()<=stopped['intent_at'],'API recovery image was not captured before stop')
         actual=self.inspect(member['name']);require(actual['Id']==member['container_id'] and actual['Image']==member['image_id'] and actual['State']['Running'],'Original API changed before signal')
-        restart=self.run(['systemctl','--user','show',member['name']+'.service','--property=Restart','--value']).decode().strip()
-        require(restart=='no','Legacy API signal requires no automatic restart')
+        self.ensure_api_restart_override()
         binding=self.api_queue_binding(member,actual,image)
         queue=self.queue('status');prefix=queue.get('prefix');before=self.observe_empty_redis_queue(member,prefix,binding)
         self.legacy_api_idle()
@@ -645,6 +687,7 @@ class Controller:
             else:
                 self.record['rollback_data_claim']='No target startup/migration began; only original runtime restored.'
 
+        self.release_api_restart_override()
         if not self.record.get('queue_was_paused',True):
             state=self.queue('resume');require(not state['paused'],'Could not restore queue admission')
         self.record['phase']='Released';self.record['outcome']=outcome;self.save()

tests/regression/test_indeehub_maintenance_controller.py

@@ -149,6 +149,23 @@ class MaintenanceTests(unittest.TestCase):
   module.atomic(runtime,{'phase':'Restored','target_startup_began':False})
   self.assertEqual(c.release('aborted')['state'],'released')
   self.assertIn('No target startup',c.record['rollback_data_claim'])
+ def test_partial_frontend_recovery_preserves_active_work_without_claiming_a_drain(self):
+  c=self.controller;c.record={'operation_id':self.operation,'original_members':module.validate_members(members()),'phase':'Prepared','queue_was_paused':False,'queue_pause_confirmed':True,'last_queue_counts':{name:(1 if name=='active' else 0) for name in module.QUEUE_COUNTS},'stopped':{'indeedhub':{'confirmed':True}}};c.save()
+  runtime=c.data/'update-transactions'/'supervised'/(self.operation+'.json');module.atomic(runtime,{'phase':'Restoring','target_startup_began':False})
+  c.fence.parent.mkdir(parents=True);c.fence.write_text(self.operation);c.close_ingress=lambda:c.fence_matches()
+  self.assertEqual(c.acquire(members(),recovery=True)['state'],'recovering')
+  self.assertEqual(c.verify()['state'],'held');self.assertFalse(c.record.get('backup_complete',False))
+  self.assertNotIn('indeedhub-ffmpeg',c.record['stopped']);self.assertEqual(self.calls,[])
+  # Native restoration must first verify preserved originals and the restarted
+  # frontend. The helper never fabricates that terminal native decision.
+  with self.assertRaises(RuntimeError):c.release('restored')
+  module.atomic(runtime,{'phase':'Restored','target_startup_began':False})
+  actions=[]
+  def queue(action):actions.append(action);return {'paused':False,'counts':dict(c.record['last_queue_counts'])}
+  c.queue=queue
+  self.assertEqual(c.release('restored')['state'],'released');self.assertEqual(actions,['resume'])
+  self.assertEqual(c.record['last_queue_counts']['active'],1)
+  self.assertFalse(c.record.get('backup_complete',False));self.assertEqual(self.calls,[])
  def test_target_started_rollback_requires_data_compatibility(self):
   c=self.controller;c.record={'operation_id':self.operation,'phase':'Recovering'};c.save()
   c.fence.parent.mkdir(parents=True);c.fence.write_text(self.operation)
@@ -329,6 +346,54 @@ class MaintenanceTests(unittest.TestCase):
   with self.assertRaisesRegex(RuntimeError,'OOM'):c.legacy_api_wrapper_termination(m,p)
   c.runner=lambda argv,timeout,output: b'replacement-id' if argv[:2]==['podman','ps'] and any('name=^' in x for x in argv) else runner(argv,timeout,output)
   with self.assertRaisesRegex(RuntimeError,'writer is still running'):c.legacy_api_wrapper_termination(m,p)
+ def restart_policy_fixture(self):
+  c=self.controller;c.record={'operation_id':self.operation};c.runtime_root=self.root/'runtime';c.runtime_root.mkdir(mode=0o700)
+  c.fence.parent.mkdir(parents=True);c.fence.write_text(self.operation)
+  path=c.api_restart_override_path();state={'original':'always'}
+  def runner(argv,timeout,output):
+   self.calls.append(argv)
+   if argv==['systemctl','--user','daemon-reload']:return b''
+   if argv==['systemctl','--user','show','indeedhub-api.service','--property=Restart','--value']:return ('no' if path.exists() else state['original']).encode()
+   raise AssertionError(argv)
+  c.runner=runner
+  return c,path,state
+ def test_owned_restart_override_restores_always_without_rewriting_unit(self):
+  c,path,state=self.restart_policy_fixture();c.ensure_api_restart_override()
+  self.assertEqual(path.read_bytes(),c.api_restart_override_bytes());self.assertEqual(c.record['api_restart_override']['original_policy'],'always')
+  self.assertEqual(path.stat().st_mode&0o777,0o600)
+  c.ensure_api_restart_override();c.release_api_restart_override();self.assertFalse(path.exists())
+  self.assertTrue(c.record['api_restart_override']['released']);calls=len(self.calls);c.release_api_restart_override();self.assertEqual(len(self.calls),calls)
+  self.assertTrue(all('stop' not in argv and 'revert' not in argv for argv in self.calls))
+ def test_restart_override_conflict_or_tamper_never_overwrites_or_unlinks(self):
+  c,path,state=self.restart_policy_fixture();path.write_text('foreign')
+  with self.assertRaises(RuntimeError):c.ensure_api_restart_override()
+  self.assertEqual(path.read_text(),'foreign');self.assertEqual(self.calls,[])
+  path.unlink();c.ensure_api_restart_override();path.write_text('changed')
+  for action in (c.ensure_api_restart_override,c.release_api_restart_override):
+   with self.assertRaises(RuntimeError):action()
+   self.assertEqual(path.read_text(),'changed')
+ def test_restart_override_lost_create_reply_and_reboot_missing_file_are_reverified(self):
+  c,path,state=self.restart_policy_fixture();c.ensure_api_restart_override();c.record['api_restart_override'].pop('installed');c.save()
+  resumed=module.Controller(c.data,self.operation,0,c.runner);resumed.runtime_root=c.runtime_root;resumed.ensure_api_restart_override()
+  self.assertEqual(resumed.record['api_restart_override']['original_policy'],'always')
+  path.unlink();resumed.ensure_api_restart_override();self.assertTrue(path.exists())
+  self.assertEqual(resumed.record['api_restart_override']['original_policy'],'always')
+ def test_restart_policy_mismatch_retains_obligation_after_override_removal(self):
+  c,path,state=self.restart_policy_fixture();c.ensure_api_restart_override();state['original']='on-failure'
+  with self.assertRaises(RuntimeError):c.release_api_restart_override()
+  self.assertFalse(path.exists());self.assertFalse(c.record['api_restart_override']['released']);self.assertTrue(c.fence.exists())
+  state['original']='always';c.release_api_restart_override();self.assertTrue(c.record['api_restart_override']['released'])
+ def test_restart_override_symlink_or_wrong_mode_is_refused(self):
+  c,path,state=self.restart_policy_fixture();c.ensure_api_restart_override();path.chmod(0o644)
+  with self.assertRaises(RuntimeError):c.release_api_restart_override()
+  path.unlink();other=self.root/'foreign';other.write_text('retain');path.symlink_to(other)
+  with self.assertRaises(RuntimeError):c.ensure_api_restart_override()
+  with self.assertRaises(RuntimeError):c.release_api_restart_override()
+  self.assertEqual(other.read_text(),'retain')
+ def test_restart_override_writable_ancestor_is_refused(self):
+  c,path,state=self.restart_policy_fixture();c.runtime_root.chmod(0o770)
+  with self.assertRaises(RuntimeError):c.ensure_api_restart_override()
+  self.assertFalse(path.exists());self.assertEqual(self.calls,[])
  def test_unacknowledged_api_signal_is_never_retried(self):
   c,m,p,*_=self.legacy_api_fixture();c.record['stopped'][m['name']]['api_signal']['acknowledged']=False
   with self.assertRaisesRegex(RuntimeError,'Unacknowledged'):c.signal_legacy_api(m)