Qualify idle legacy process termination without inventing completed work

docs/indeehub-legacy-maintenance-controller.md

@@ -1,6 +1,6 @@
 # Legacy IndeeHub maintenance controller
 
-Status: isolated source implementation. Twelve pure Python fake-runtime regressions
+Status: isolated source implementation. Fourteen pure Python fake-runtime regressions
 pass; no live invocation or production qualification. The controller is not part
 of the already signed private app candidate and needs no new app image/API.
 
@@ -55,7 +55,7 @@ prove compatibility with newly changed data. No automatic DB/media restore exist
 
 ## Qualification and remaining integration
 
-`python3 tests/regression/test_indeehub_maintenance_controller.py` passes twelve
+`python3 tests/regression/test_indeehub_maintenance_controller.py` passes fourteen
 fake-runtime cases in temporary directories, without services/network/containers.
 Source nginx template guard coverage also passes its parser check. Production
 adapter compilation, actual Podman event format/systemd clean-exit behavior,
@@ -76,3 +76,11 @@ A pre-acquire snapshot/preflight failure may leave no controller journal. An
 Aborted node journal with target_startup_began=false then permits idempotent
 no-op acknowledgement, without touching any other operation’s admission fence.
 A matching fence without its controller journal requires recovery investigation.
+
+Read-only source evidence from actual old API/ffmpeg shows neither has SIGTERM
+shutdown hooks. The controller permits worker143 only after a paused queue has
+zero active jobs. Legacy API143 additionally requires closed/stopped frontend,
+stopped worker, and a fresh empty projects/contents/payments/shareholders/
+subscriptions/library_items store with no other active DB transaction. This is
+a narrow first-upgrade compatibility path, not evidence populated work completed.
+Populated or ambiguous legacy state remains a refused forward cutover.

scripts/indeehub-maintenance-controller.py

@@ -107,6 +107,20 @@ class Controller:
             original=self.queue('status');self.record['queue_was_paused']=original['paused'];self.record['queue_original_counts']=original['counts'];self.save()
         state=self.queue('pause');require(state['paused'],'Worker admission did not close')
         self.record['queue_pause_confirmed']=True;self.save()
+    def legacy_api_idle(self):
+        # Narrow first-upgrade compatibility for the observed legacy API which
+        # has no SIGTERM hooks. Existing customer/business work is never inferred
+        # completed: this path requires a fresh empty store behind closed ingress.
+        require(self.record.get('stopped',{}).get('indeedhub',{}).get('confirmed'),'Frontend ingress must already be stopped')
+        require(self.record.get('stopped',{}).get('indeedhub-ffmpeg',{}).get('confirmed'),'Transcode worker must already be stopped')
+        require(self.record.get('queue_pause_confirmed') is True and self.record.get('last_queue_counts',{}).get('active')==0,'Worker queue is not proven idle')
+        tables=('projects','contents','payments','shareholders','subscriptions','library_items')
+        fields=','.join("'%s',(SELECT count(*) FROM public.%s)"%(name,name) for name in tables)
+        sql="SELECT json_build_object("+fields+",'other_active_transactions',(SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND pid<>pg_backend_pid() AND state<>'idle'))"
+        counts=json.loads(self.run(['podman','exec','indeedhub-postgres','psql','-XAt','-U','indeedhub','-d','indeedhub','-c',sql]))
+        require(set(counts)==set(tables)|{'other_active_transactions'},'Legacy API business-state observation incomplete')
+        require(all(type(value) is int and value==0 for value in counts.values()),'Legacy API has business work or active transactions; completion cannot be inferred')
+        self.record['legacy_api_empty_state']=counts;self.save()
     def graceful_stop(self, name):
         # Save the obligation before systemd can remove an AutoRemove container.
         stopped=self.record.setdefault('stopped',{})
@@ -125,8 +139,11 @@ class Controller:
         matching=[event for event in matching if event.get('ID',event.get('id'))==member['container_id']]
         require(matching,'Original process exit evidence unavailable; hold retained')
         code=matching[-1].get('ContainerExitCode',matching[-1].get('containerExitCode'))
-        require(str(code)=='0','Original process did not exit cleanly; active work is not claimed completed')
+        idle_worker = name=='indeedhub-ffmpeg' and self.record.get('queue_pause_confirmed') is True and self.record.get('last_queue_counts',{}).get('active')==0
-        stopped[name].update(confirmed=True,exit_code=0,confirmed_at=time.time());self.save()
+        empty_api = name=='indeedhub-api' and self.record.get('legacy_api_empty_state') is not None
+        require(str(code)=='0' or (str(code)=='143' and (idle_worker or empty_api)),'Original process did not exit cleanly; active work is not claimed completed')
+        classification=('idle-worker-terminated-after-queue-drain' if idle_worker else 'empty-business-store-legacy-api-terminated') if str(code)=='143' else 'clean-process-exit'
+        stopped[name].update(confirmed=True,exit_code=int(code),classification=classification,confirmed_at=time.time());self.save()
     def volume_sources(self):
         expected=['indeedhub-minio-data','indeedhub-postgres-data','indeedhub-redis-data','indeedhub-relay-data']
         rows=json.loads(self.run(['podman','volume','inspect',*expected]))
@@ -186,7 +203,7 @@ class Controller:
                 if state['counts'].get('active',0)==0:break
                 require(time.monotonic()<deadline,'Transcodes still active; retained job state, no forced completion')
                 time.sleep(1)
-            self.graceful_stop('indeedhub-ffmpeg');self.graceful_stop('indeedhub-api')
+            self.graceful_stop('indeedhub-ffmpeg');self.legacy_api_idle();self.graceful_stop('indeedhub-api')
         self.backup();self.verify();return {'operation_id':self.operation,'state':'drained'}
     def verify(self):
         self.holds();self.fence_matches()

tests/regression/test_indeehub_maintenance_controller.py

@@ -95,4 +95,30 @@ class MaintenanceTests(unittest.TestCase):
   c.fence.parent.mkdir(parents=True);c.fence.write_text(self.operation)
   with self.assertRaisesRegex(RuntimeError,'without journal'):c.release('aborted')
   self.assertTrue(c.fence.exists())
+ def test_legacy_worker_sigterm_requires_proven_paused_idle_queue(self):
+  c=self.controller;c.record={'operation_id':self.operation,'phase':'Prepared','original_members':module.validate_members(members())};c.save()
+  worker=next(m for m in members() if m['name']=='indeedhub-ffmpeg')
+  def command(argv,timeout,output):
+   if argv[:2]==['podman','inspect']:return self.command_runner(argv,timeout,output)
+   if argv[:3]==['systemctl','--user','stop']:return b''
+   if argv[:3]==['systemctl','--user','show']:return b'ActiveState=inactive\nResult=success\n'
+   if argv[:2]==['podman','events']:return json.dumps({'ID':worker['container_id'],'ContainerExitCode':143}).encode()
+   raise AssertionError(argv)
+  c.runner=command
+  with self.assertRaises(RuntimeError):c.graceful_stop(worker['name'])
+  c.record['queue_pause_confirmed']=True;c.record['last_queue_counts']={'active':1}
+  with self.assertRaises(RuntimeError):c.graceful_stop(worker['name'])
+  c.record['last_queue_counts']['active']=0;c.graceful_stop(worker['name'])
+  self.assertEqual(c.record['stopped'][worker['name']]['classification'],'idle-worker-terminated-after-queue-drain')
+ def test_legacy_api_compatibility_requires_fresh_empty_business_state(self):
+  c=self.controller;c.record={'operation_id':self.operation,'phase':'Prepared','stopped':{'indeedhub':{'confirmed':True},'indeedhub-ffmpeg':{'confirmed':True}},'queue_pause_confirmed':True,'last_queue_counts':{'active':0}};c.save()
+  counts={name:0 for name in ('projects','contents','payments','shareholders','subscriptions','library_items','other_active_transactions')}
+  c.runner=lambda argv,timeout,output:json.dumps(counts).encode()
+  counts['payments']=1
+  with self.assertRaisesRegex(RuntimeError,'business work'):c.legacy_api_idle()
+  self.assertNotIn('legacy_api_empty_state',c.record)
+  counts['payments']=0;counts['other_active_transactions']=1
+  with self.assertRaisesRegex(RuntimeError,'business work'):c.legacy_api_idle()
+  counts['other_active_transactions']=0;c.legacy_api_idle()
+  self.assertEqual(c.record['legacy_api_empty_state'],counts)
 if __name__=='__main__':unittest.main()