fix(indeehub): qualify exact legacy relay native shutdown
This commit is contained in:
archipelago
@@ -418,3 +418,34 @@ remains under investigation. The held operation is
|
||||
images and failure evidence are retained. The manager is stopped and startup
|
||||
barred. The candidate helper was separate from the installed pinned helper.
|
||||
Neither full target rollback nor successful update has passed. Yaya is unchanged.
|
||||
|
||||
### Relay native shutdown qualification (2026-10-08)
|
||||
|
||||
The earlier exit 130 probe signalled before the relay completed initialization.
|
||||
A new disposable probe waited for the real listener, then signalled the exact
|
||||
shell wrapper's sole relay child. Original `nostr-rs-relay 0.10.0` exited **0**
|
||||
without OOM. The final controller script independently binds parent/child PID,
|
||||
PPID, start time and command bytes, the child's listening socket inode, and the
|
||||
executing binary SHA256
|
||||
`e4d5d1ceb80150dd8bf4dd55b4f937a9d260cad0c19d616a974dcfaa6e82eb3c`.
|
||||
A changed proof was refused while the probe stayed running; the matching proof
|
||||
then received SIGINT and exited 0. These probes used the exact original image,
|
||||
network none and tmpfs only. The final probe's receipt formatter had a variable
|
||||
collision after shutdown; independent exact-container inspection confirmed exit
|
||||
0/no OOM and retained that limitation in `relay-ready6-probe.receipt.json`.
|
||||
|
||||
The helper now has separate API/relay operation-owned runtime restart overrides.
|
||||
It still rejects relay exit 137/130. Both API and relay acknowledged-signal retries
|
||||
must prove no live replacement before any systemd stop. Relay admission retains
|
||||
exact original-image provenance: a different image requires a unique completed
|
||||
owned recovery chain plus the current installed recipe's operation/body binding;
|
||||
the executing binary hash is an additional check. Native finite-role override
|
||||
validation is checkpointed in `f78ee252`. **53 pure Python tests pass**; combined
|
||||
Rust tests and a matching executable remain pending.
|
||||
|
||||
The actual held operation `3b3c564b-cce6-4727-8be8-369a95c479e4` still has its
|
||||
unaccepted earlier relay-137 evidence. It is not retroactively reclassified.
|
||||
The isolated manager remains stopped, PostgreSQL's original live identity and
|
||||
all recovery evidence remain intact. Keep the guest idle during serialized
|
||||
builds rather than rebooting and invalidating that identity. No Yaya application
|
||||
or catalog mutation has occurred, and full rollback/success remains pending.
|
||||
|
||||
@@ -48,6 +48,82 @@ if(action==='signal'){
|
||||
}else if(action==='probe')fs.writeSync(1,JSON.stringify(proof)+'\n');else throw Error('Unsupported action');'''
|
||||
LEGACY_API_CMD = ['sh','-c',"echo 'Running database migrations...' && npx typeorm migration:run -d dist/database/ormconfig.js && echo 'Migrations complete.' && npm run start:prod"]
|
||||
|
||||
# Qualified in original image 061516573b143b44e331f960036a6a3dc43c9b256ef8ca71afedbeb2cf797a4b.
|
||||
# Bind executing bytes so a legitimate writable-layer recovery image remains usable.
|
||||
LEGACY_RELAY_BINARY_SHA256 = 'e4d5d1ceb80150dd8bf4dd55b4f937a9d260cad0c19d616a974dcfaa6e82eb3c'
|
||||
LEGACY_RELAY_CMD = ['/bin/sh','-c','./nostr-rs-relay --db ${APP_DATA}']
|
||||
RELAY_PROCESS_SCRIPT = r'''set -eu
|
||||
identity() {
|
||||
pid="$1"; stat=$(cat "/proc/$pid/stat"); rest=${stat##*) }; set -- $rest
|
||||
ppid="$2"; shift 19; start="$1"
|
||||
command=$(od -An -v -tx1 "/proc/$pid/cmdline" | tr -d ' \n')
|
||||
printf '%s %s %s %s\n' "$pid" "$ppid" "$start" "$command"
|
||||
}
|
||||
observe() {
|
||||
identity 1
|
||||
count=0; child=''
|
||||
for status in /proc/[0-9]*/status; do
|
||||
ppid=$(sed -n 's/^PPid:[[:space:]]*//p' "$status" 2>/dev/null) || continue
|
||||
if [ "$ppid" = 1 ]; then child=${status%/status}; child=${child##*/}; count=$((count+1)); fi
|
||||
done
|
||||
[ "$count" = 1 ]; identity "$child"
|
||||
listeners=$(awk '$4 == "0A" {print $10}' /proc/net/tcp /proc/net/tcp6)
|
||||
ready=0
|
||||
for descriptor in /proc/"$child"/fd/*; do
|
||||
target=$(readlink "$descriptor") || continue
|
||||
for inode in $listeners; do [ "$target" != "socket:[$inode]" ] || ready=1; done
|
||||
done
|
||||
[ "$ready" = 1 ]
|
||||
sha256sum "/proc/$child/exe" | cut -d " " -f1
|
||||
}
|
||||
proof=$(observe)
|
||||
if [ "$1" = probe ]; then printf '%s\n' "$proof"
|
||||
elif [ "$1" = signal ]; then
|
||||
[ "$proof" = "$2" ]; [ "$(observe)" = "$2" ]
|
||||
child=$(printf '%s\n' "$proof" | sed -n '2p'); child=${child%% *}
|
||||
kill -INT "$child"
|
||||
printf 'acknowledged SIGINT\n%s\n' "$proof"
|
||||
else exit 1; fi
|
||||
'''
|
||||
LEGACY_RELAY_IMAGE = '061516573b143b44e331f960036a6a3dc43c9b256ef8ca71afedbeb2cf797a4b'
|
||||
def verify_relay_image_lineage(image, unit_sha256, records, installed=None):
|
||||
seen=set()
|
||||
for _ in range(16):
|
||||
image=image.removeprefix('sha256:')
|
||||
require(re.fullmatch('[0-9a-f]{64}',image) is not None,'Invalid relay image lineage hash')
|
||||
if image==LEGACY_RELAY_IMAGE:return
|
||||
require(re.fullmatch('[0-9a-f]{64}',unit_sha256) is not None,'Invalid relay unit lineage hash')
|
||||
require(image not in seen,'Cyclic relay recovery image lineage');seen.add(image)
|
||||
matches=[]
|
||||
for record in records:
|
||||
if record.get('schema') not in (1,2) or record.get('package')!='indeedhub' or record.get('phase')!='Restored' or record.get('cleanup_done') is not True:continue
|
||||
try:require(str(uuid.UUID(record['id']))==record['id'],'Invalid relay lineage operation')
|
||||
except (KeyError,ValueError,AttributeError):continue
|
||||
relay=[m for m in record.get('members',[]) if m.get('original',{}).get('name')=='indeedhub-relay']
|
||||
if len(relay)!=1:continue
|
||||
for member in relay:
|
||||
original=member.get('original',{});recovery=member.get('recovery_image') or {}
|
||||
if original.get('name')!='indeedhub-relay' or recovery.get('image','').removeprefix('sha256:')!=image:continue
|
||||
require((record['schema']==1 and (member.get('preserve_original') is None or member.get('preserve_original') is False) or record['schema']==2 and member.get('preserve_original') is False) and recovery.get('operation_id')==record['id'] and recovery.get('source_container_id')==original.get('container_id'),'Relay recovery ownership changed')
|
||||
require(hashlib.sha256(member['pinned_original_body'].encode()).hexdigest()==unit_sha256,'Relay recovery unit lineage changed')
|
||||
require(re.fullmatch('[0-9a-f]{64}',original.get('container_id','')) is not None,'Invalid relay source identity')
|
||||
if len(seen)==1:require(isinstance(installed,dict) and installed.get('schema')==1 and installed.get('name')=='indeedhub-relay' and installed.get('operation')==record['id'] and installed.get('body')==member['pinned_original_body'],'Relay installed recipe does not own recovery lineage')
|
||||
matches.append((original['image'],hashlib.sha256(original['body'].encode()).hexdigest()))
|
||||
require(len(matches)==1,'Relay image lacks unique completed owned recovery lineage')
|
||||
image,unit_sha256=matches[0]
|
||||
raise RuntimeError('Relay recovery image lineage is too deep')
|
||||
def relay_process_proof(raw, data_path):
|
||||
require(isinstance(data_path,str) and data_path.startswith('/') and '\0' not in data_path,'Invalid relay data path')
|
||||
lines=raw.strip().splitlines();require(len(lines)==3 and lines[2]==LEGACY_RELAY_BINARY_SHA256,'Incomplete or unqualified relay executable proof')
|
||||
proof=[]
|
||||
for line in lines[:2]:
|
||||
fields=line.split();require(len(fields)==4 and all(re.fullmatch('[0-9]+',v) for v in fields[:3]) and re.fullmatch('[0-9a-f]+',fields[3]),'Malformed relay process proof')
|
||||
proof.append({'pid':int(fields[0]),'ppid':int(fields[1]),'starttime':fields[2],'command':bytes.fromhex(fields[3]).decode()})
|
||||
parent,child=proof
|
||||
require(parent['pid']==1 and parent['ppid']==0 and parent['command']=='\0'.join(LEGACY_RELAY_CMD)+'\0','Unrecognized relay wrapper')
|
||||
require(child['pid']>1 and child['ppid']==1 and child['command']=='./nostr-rs-relay\0--db\0'+data_path+'\0','Unrecognized relay child')
|
||||
return {'parent':parent,'child':child,'executable_sha256':lines[2]}
|
||||
|
||||
# The exact three migrations in the privately qualified API candidate. This is
|
||||
# an allowlist of additive schema history, never permission to discard app data.
|
||||
ADDITIVE_MIGRATIONS = {
|
||||
@@ -251,9 +327,9 @@ class Controller:
|
||||
require(set(counts)==set(tables)|{'other_active_transactions'},'Legacy API business-state observation incomplete')
|
||||
require(all(type(value) is int and value==0 for value in counts.values()),'Legacy API has business work or active transactions; completion cannot be inferred')
|
||||
self.record['legacy_api_empty_state']=counts;self.save()
|
||||
def api_recovery_identity(self, member):
|
||||
def api_recovery_identity(self, member, role="api"):
|
||||
self.holds();self.fence_matches()
|
||||
require(member['name']=='indeedhub-api','Legacy API member required')
|
||||
require(role in ('api','relay') and member['name']=='indeedhub-'+role,'Legacy signal member required')
|
||||
runtime=json.loads((self.data/'update-transactions'/'supervised'/(self.operation+'.json')).read_text())
|
||||
require(runtime.get('id')==self.operation and runtime.get('phase') in ('Editing','Restoring') and runtime.get('target_startup_began') is False,'Legacy API operation changed')
|
||||
rows=[m for m in runtime['members'] if m['original']['name']==member['name']]
|
||||
@@ -262,7 +338,20 @@ class Controller:
|
||||
require(recovery['source_container_id']==member['container_id'] and recovery['operation_id']==self.operation,'Legacy API recovery identity changed')
|
||||
images=json.loads(self.run(['podman','image','inspect',recovery['image']]))
|
||||
require(len(images)==1 and images[0]['Id'].removeprefix('sha256:')==recovery['image'].removeprefix('sha256:'),'Legacy API recovery image changed')
|
||||
image=images[0];require(image['Config'].get('Cmd')==LEGACY_API_CMD and image['Config'].get('Entrypoint')==['docker-entrypoint.sh'],'Unrecognized legacy API command')
|
||||
if role=='relay':
|
||||
records=[];installed=None
|
||||
if member['image_id'].removeprefix('sha256:')!=LEGACY_RELAY_IMAGE:
|
||||
directory=self.data/'update-transactions'/'supervised'
|
||||
require(directory.is_dir() and not directory.is_symlink() and directory.stat().st_uid==os.getuid() and directory.stat().st_mode & 0o022==0,'Unsafe relay recovery lineage directory')
|
||||
for path in directory.glob('*.json'):
|
||||
require(path.is_file() and not path.is_symlink() and path.stat().st_uid==os.getuid() and path.stat().st_mode & 0o077==0 and path.stat().st_size<=4*1024*1024,'Unsafe relay recovery lineage record')
|
||||
record=json.loads(path.read_text());require(path.stem==record.get('id'),'Relay recovery journal filename mismatch');records.append(record)
|
||||
directory=self.data/'update-transactions'/'installed-units';path=directory/'indeedhub-relay.json'
|
||||
require(directory.is_dir() and not directory.is_symlink() and directory.stat().st_uid==os.getuid() and directory.stat().st_mode & 0o077==0 and path.is_file() and not path.is_symlink() and path.stat().st_uid==os.getuid() and path.stat().st_mode & 0o077==0 and path.stat().st_size<=1024*1024,'Unsafe relay installed recipe')
|
||||
installed=json.loads(path.read_text())
|
||||
verify_relay_image_lineage(member['image_id'],member['unit_sha256'],records,installed)
|
||||
image=images[0];expected=(LEGACY_API_CMD,['docker-entrypoint.sh']) if role=='api' else (LEGACY_RELAY_CMD,None)
|
||||
require((image['Config'].get('Cmd'),image['Config'].get('Entrypoint'))==expected,'Unrecognized legacy signal command')
|
||||
source=pathlib.Path(self.run(['systemctl','--user','show',member['name']+'.service','--property=SourcePath','--value']).decode().strip())
|
||||
require(source.is_file() and not source.is_symlink() and source.suffix=='.container' and source.stat().st_uid==os.getuid() and sha(source)==member['unit_sha256'],'Legacy API saved unit changed')
|
||||
return image
|
||||
@@ -305,10 +394,11 @@ class Controller:
|
||||
require(observed.get('paused') is True and valid_queue_counts(observed.get('counts')) and all(v==0 for v in observed['counts'].values()),'Legacy API queue not paused and empty')
|
||||
extra=observed.get('extra');require(isinstance(extra,dict) and set(extra)=={'prioritized','waiting_children'} and all(type(v) is int and v==0 for v in extra.values()),'Legacy API has additional queued work')
|
||||
return observed
|
||||
def api_restart_override_path(self):
|
||||
def api_restart_override_path(self, role="api"):
|
||||
require(role in ("api","relay"),"Unsupported restart override role")
|
||||
require(self.runtime_root.is_dir() and not self.runtime_root.is_symlink() and self.runtime_root.stat().st_uid==os.getuid() and self.runtime_root.stat().st_mode & 0o022==0,'Unsafe user runtime directory')
|
||||
parent=self.runtime_root
|
||||
for name in ('systemd','user','indeedhub-api.service.d'):
|
||||
for name in ('systemd','user','indeedhub-'+role+'.service.d'):
|
||||
parent=parent/name
|
||||
parent.mkdir(mode=0o700,exist_ok=True)
|
||||
require(parent.is_dir() and not parent.is_symlink() and parent.stat().st_uid==os.getuid() and parent.stat().st_mode & 0o022==0,'Unsafe API restart override directory')
|
||||
@@ -317,14 +407,14 @@ class Controller:
|
||||
return ('# Archipelago maintenance operation '+self.operation+'\n[Service]\nRestart=no\n').encode()
|
||||
def verify_api_restart_override(self, path):
|
||||
require(path.is_file() and not path.is_symlink() and path.stat().st_uid==os.getuid() and path.stat().st_mode & 0o777==0o600 and path.read_bytes()==self.api_restart_override_bytes(),'API restart override changed; hold retained')
|
||||
def ensure_api_restart_override(self):
|
||||
self.holds();self.fence_matches();path=self.api_restart_override_path();saved=self.record.get('api_restart_override')
|
||||
def ensure_api_restart_override(self, role="api"):
|
||||
self.holds();self.fence_matches();path=self.api_restart_override_path(role);saved=self.record.get(role+'_restart_override')
|
||||
if saved is None:
|
||||
require(not path.exists() and not path.is_symlink(),'Unowned API restart override exists')
|
||||
policy=self.run(['systemctl','--user','show','indeedhub-api.service','--property=Restart','--value']).decode().strip()
|
||||
policy=self.run(['systemctl','--user','show','indeedhub-'+role+'.service','--property=Restart','--value']).decode().strip()
|
||||
require(policy in ('no','always','on-success','on-failure','on-abnormal','on-watchdog','on-abort'),'Unrecognized original restart policy')
|
||||
saved={'operation_id':self.operation,'original_policy':policy,'sha256':hashlib.sha256(self.api_restart_override_bytes()).hexdigest(),'released':False}
|
||||
self.record['api_restart_override']=saved;self.save()
|
||||
self.record[role+'_restart_override']=saved;self.save()
|
||||
require(saved.get('operation_id')==self.operation and saved.get('sha256')==hashlib.sha256(self.api_restart_override_bytes()).hexdigest() and saved.get('released') is False,'API restart override obligation changed')
|
||||
if path.exists() or path.is_symlink():self.verify_api_restart_override(path)
|
||||
else:
|
||||
@@ -332,20 +422,20 @@ class Controller:
|
||||
with os.fdopen(descriptor,'wb') as stream:stream.write(self.api_restart_override_bytes());stream.flush();os.fsync(stream.fileno())
|
||||
directory=os.open(path.parent,os.O_RDONLY);os.fsync(directory);os.close(directory)
|
||||
self.run(['systemctl','--user','daemon-reload'])
|
||||
require(self.run(['systemctl','--user','show','indeedhub-api.service','--property=Restart','--value']).decode().strip()=='no','API automatic restart did not close')
|
||||
require(self.run(['systemctl','--user','show','indeedhub-'+role+'.service','--property=Restart','--value']).decode().strip()=='no','API automatic restart did not close')
|
||||
saved['installed']=True;self.save()
|
||||
def release_api_restart_override(self):
|
||||
saved=self.record.get('api_restart_override')
|
||||
def release_api_restart_override(self, role="api"):
|
||||
saved=self.record.get(role+'_restart_override')
|
||||
if not saved or saved.get('released') is True:return
|
||||
require(saved.get('operation_id')==self.operation and saved.get('sha256')==hashlib.sha256(self.api_restart_override_bytes()).hexdigest(),'API restart override ownership changed')
|
||||
path=self.api_restart_override_path()
|
||||
path=self.api_restart_override_path(role)
|
||||
if path.exists() or path.is_symlink():
|
||||
self.verify_api_restart_override(path);path.unlink()
|
||||
directory=os.open(path.parent,os.O_RDONLY);os.fsync(directory);os.close(directory)
|
||||
# /run may have been cleared by a reboot, or unlink may have completed
|
||||
# before an interrupted reply. Absence still requires effective-policy verification.
|
||||
self.run(['systemctl','--user','daemon-reload'])
|
||||
require(self.run(['systemctl','--user','show','indeedhub-api.service','--property=Restart','--value']).decode().strip()==saved['original_policy'],'Original API restart policy was not restored; hold retained')
|
||||
require(self.run(['systemctl','--user','show','indeedhub-'+role+'.service','--property=Restart','--value']).decode().strip()==saved['original_policy'],'Original API restart policy was not restored; hold retained')
|
||||
saved['released']=True;self.save()
|
||||
def signal_legacy_api(self, member):
|
||||
stopped=self.record['stopped'][member['name']]
|
||||
@@ -418,6 +508,34 @@ class Controller:
|
||||
'original_container_id':member['container_id'],'original_image_id':member['image_id'],
|
||||
'recovery_image_id':recovery['image'],'unit_sha256':member['unit_sha256'],
|
||||
'before_counts':before,'after_counts':after['counts'],'process_dead':True}
|
||||
def signal_legacy_relay(self, member):
|
||||
stopped=self.record['stopped'][member['name']]
|
||||
image=self.api_recovery_identity(member,'relay')
|
||||
require(datetime.datetime.fromisoformat(image['Created'].replace('Z','+00:00')).timestamp()<=stopped['intent_at'],'Relay recovery image was not captured before stop')
|
||||
paths=[value.split('=',1)[1] for value in image['Config'].get('Env',[]) if value.startswith('APP_DATA=')]
|
||||
require(len(paths)==1,'Ambiguous relay data path');data_path=paths[0]
|
||||
saved=stopped.get('relay_signal')
|
||||
if saved:
|
||||
require(saved.get('operation_id')==self.operation and saved.get('container_id')==member['container_id'] and saved.get('acknowledged') is True and saved.get('signal')=='SIGINT' and saved.get('proof')==relay_process_proof(saved.get('raw',''),data_path) and type(saved.get('intent_at')) in (int,float) and type(saved.get('acknowledged_at')) in (int,float) and saved['acknowledged_at']>=saved['intent_at']>=stopped['intent_at'],'Incomplete durable relay signal proof; hold retained')
|
||||
return
|
||||
actual=self.inspect(member['name']);require(actual['Id']==member['container_id'] and actual['Image']==member['image_id'] and actual['State']['Running'],'Original relay changed before signal')
|
||||
self.ensure_api_restart_override('relay')
|
||||
deadline=time.monotonic()+60
|
||||
while True:
|
||||
sockets=self.run(['podman','exec',member['container_id'],'sh','-c','cat /proc/net/tcp /proc/net/tcp6']).decode().splitlines()
|
||||
if any(len(line.split())>3 and line.split()[3]=='0A' for line in sockets):break
|
||||
require(time.monotonic()<deadline,'Relay listener not ready for native shutdown');time.sleep(0.2)
|
||||
raw=self.run(['podman','exec',member['container_id'],'sh','-c',RELAY_PROCESS_SCRIPT,'relay-process','probe']).decode().strip()
|
||||
proof=relay_process_proof(raw,data_path)
|
||||
saved={'operation_id':self.operation,'container_id':member['container_id'],'signal':'SIGINT','proof':proof,'raw':raw,'intent_at':time.time(),'acknowledged':False}
|
||||
stopped['relay_signal']=saved;self.save()
|
||||
ack=self.run(['podman','exec',member['container_id'],'sh','-c',RELAY_PROCESS_SCRIPT,'relay-process','signal',raw]).decode().strip()
|
||||
require(ack=='acknowledged SIGINT\n'+raw,'Relay signal acknowledgement changed; hold retained')
|
||||
saved['acknowledged']=True;saved['acknowledged_at']=time.time();self.save()
|
||||
deadline=time.monotonic()+60
|
||||
while self.run(['podman','ps','--no-trunc','--filter','id='+member['container_id'],'--format','{{.ID}}']).strip():
|
||||
require(time.monotonic()<deadline,'Relay did not terminate after native shutdown signal');time.sleep(0.2)
|
||||
self.require_api_stopped(member)
|
||||
def graceful_stop(self, name):
|
||||
# Save the obligation before systemd can remove an AutoRemove container.
|
||||
stopped=self.record.setdefault('stopped',{})
|
||||
@@ -427,6 +545,8 @@ class Controller:
|
||||
actual=self.inspect(name);require(actual['Id']==member['container_id'] and actual['Image']==member['image_id'],'Original container changed before stop')
|
||||
stopped[name]={'intent_at':time.time(),'container_id':actual['Id']};self.save()
|
||||
if name=='indeedhub-api':self.signal_legacy_api(member)
|
||||
if name=='indeedhub-relay':self.signal_legacy_relay(member)
|
||||
if name in ('indeedhub-api','indeedhub-relay'):self.require_api_stopped(member)
|
||||
self.run(['systemctl','--user','stop',name+'.service'],timeout=180)
|
||||
properties=self.run(['systemctl','--user','show',name+'.service','--property=ActiveState,SubState,Result,ExecMainStatus']).decode()
|
||||
# --rm removes inspect state. Require a persisted Podman died event for
|
||||
@@ -442,6 +562,9 @@ class Controller:
|
||||
if name=='indeedhub-api' and str(code)=='1':forced=self.legacy_api_wrapper_termination(member,properties)
|
||||
require(forced is not None or ('ActiveState=inactive' in properties and 'Result=success' in properties),'Service did not stop successfully')
|
||||
require(forced is not None or str(code)=='0' or (str(code)=='143' and (idle_worker or empty_api)),'Original process did not exit cleanly; active work is not claimed completed')
|
||||
if name=='indeedhub-relay':
|
||||
require(str(code)=='0','Relay native shutdown did not exit cleanly')
|
||||
self.require_api_stopped(member)
|
||||
classification=forced['classification'] if forced else (('idle-worker-terminated-after-queue-drain' if idle_worker else 'empty-business-store-legacy-api-terminated') if str(code)=='143' else 'clean-process-exit')
|
||||
if forced:stopped[name]['legacy_idle_termination']=forced
|
||||
stopped[name].update(confirmed=True,exit_code=int(code),classification=classification,confirmed_at=time.time());self.save()
|
||||
@@ -689,6 +812,7 @@ class Controller:
|
||||
self.record['rollback_data_claim']='No target startup/migration began; only original runtime restored.'
|
||||
|
||||
self.release_api_restart_override()
|
||||
self.release_api_restart_override("relay")
|
||||
if not self.record.get('queue_was_paused',True):
|
||||
state=self.queue('resume');require(not state['paused'],'Could not restore queue admission')
|
||||
self.record['phase']='Released';self.record['outcome']=outcome;self.save()
|
||||
|
||||
@@ -486,4 +486,73 @@ console.log('process identity cases passed');'''
|
||||
self.assertEqual(c.release('restored')['state'],'released')
|
||||
self.assertEqual(c.record['recovery_data_verification']['operation_id'],self.operation)
|
||||
self.assertEqual(c.record['recovery_data_verification']['before_sha256'],c.record['recovery_data_verification']['after_sha256'])
|
||||
def relay_raw(self, child_pid=35, parent_pid=1, ppid=1, command=None):
|
||||
parent='\0'.join(module.LEGACY_RELAY_CMD)+'\0'
|
||||
child=command or './nostr-rs-relay\0--db\0/usr/src/app/db\0'
|
||||
return f'{parent_pid} 0 100 {parent.encode().hex()}\n{child_pid} {ppid} 200 {child.encode().hex()}\n{module.LEGACY_RELAY_BINARY_SHA256}'
|
||||
def test_relay_process_proof_requires_exact_wrapper_child_and_complete_identity(self):
|
||||
raw=self.relay_raw();proof=module.relay_process_proof(raw,'/usr/src/app/db')
|
||||
self.assertEqual(proof['child']['pid'],35)
|
||||
for bad in [raw+'\n'+raw.splitlines()[1],self.relay_raw(child_pid=1),self.relay_raw(parent_pid=2),self.relay_raw(ppid=0),self.relay_raw(command='./other\0'),raw.replace(' 200 ',' invalid '),raw.replace(module.LEGACY_RELAY_BINARY_SHA256,'a'*64)]:
|
||||
with self.assertRaises((RuntimeError,ValueError)):module.relay_process_proof(bad,'/usr/src/app/db')
|
||||
with self.assertRaises(RuntimeError):module.relay_process_proof(raw,'/other')
|
||||
def test_relay_override_has_separate_owned_path_and_journal_key(self):
|
||||
c,path,state=self.restart_policy_fixture();relay=c.api_restart_override_path('relay');old=c.runner
|
||||
def runner(argv,timeout,output):
|
||||
if argv==['systemctl','--user','show','indeedhub-relay.service','--property=Restart','--value']:return ('no' if relay.exists() else 'always').encode()
|
||||
return old(argv,timeout,output)
|
||||
c.runner=runner;c.ensure_api_restart_override('relay')
|
||||
self.assertTrue(relay.exists());self.assertFalse(path.exists());self.assertNotIn('api_restart_override',c.record)
|
||||
self.assertEqual(c.record['relay_restart_override']['original_policy'],'always')
|
||||
c.ensure_api_restart_override();c.release_api_restart_override('relay')
|
||||
self.assertTrue(path.exists());self.assertFalse(relay.exists());self.assertFalse(c.record['api_restart_override']['released'])
|
||||
with self.assertRaises(RuntimeError):c.api_restart_override_path('postgres')
|
||||
def test_relay_unacknowledged_or_changed_durable_signal_never_retries(self):
|
||||
c=self.controller;m=next(x for x in members() if x['name']=='indeedhub-relay')
|
||||
raw=self.relay_raw();saved={'operation_id':self.operation,'container_id':m['container_id'],'signal':'SIGINT','proof':module.relay_process_proof(raw,'/usr/src/app/db'),'raw':raw,'intent_at':1700000001,'acknowledged_at':1700000002,'acknowledged':True}
|
||||
c.record={'stopped':{m['name']:{'intent_at':1700000000,'relay_signal':saved}}}
|
||||
c.api_recovery_identity=lambda member,role:{'Created':'2020-01-01T00:00:00Z','Config':{'Env':['APP_DATA=/usr/src/app/db']}}
|
||||
c.signal_legacy_relay(m);self.assertEqual(self.calls,[])
|
||||
for key,bad in [('acknowledged',False),('operation_id',str(uuid.uuid4())),('container_id','wrong'),('proof',{}),('signal','SIGTERM'),('acknowledged_at',1)]:
|
||||
old=saved[key];saved[key]=bad
|
||||
with self.assertRaises(RuntimeError):c.signal_legacy_relay(m)
|
||||
saved[key]=old
|
||||
self.assertEqual(self.calls,[])
|
||||
def test_acknowledged_api_and_relay_retry_refuses_replacement_before_stop(self):
|
||||
for name in ('indeedhub-api','indeedhub-relay'):
|
||||
c=self.controller;m=next(x for x in members() if x['name']==name)
|
||||
c.record={'original_members':members(),'stopped':{name:{'container_id':m['container_id'],'intent_at':1700000000}}}
|
||||
c.signal_legacy_api=lambda member:None;c.signal_legacy_relay=lambda member:None
|
||||
calls=[]
|
||||
def runner(argv,timeout,output):
|
||||
calls.append(argv)
|
||||
if argv[:2]==['podman','ps']:return b'unexpected-replacement'
|
||||
raise AssertionError('Replacement must be refused before any stop '+str(argv))
|
||||
c.runner=runner
|
||||
with self.assertRaisesRegex(RuntimeError,'writer is still running'):c.graceful_stop(name)
|
||||
self.assertEqual(len(calls),1);self.assertEqual(calls[0][:2],['podman','ps'])
|
||||
def test_relay_image_requires_qualified_base_or_completed_owned_recovery_lineage(self):
|
||||
import copy,hashlib
|
||||
image='d'*64;body='[Container]\nImage='+image+'\n';before='[Container]\nImage='+module.LEGACY_RELAY_IMAGE+'\n'
|
||||
record={'schema':2,'id':self.operation,'package':'indeedhub','phase':'Restored','cleanup_done':True,'members':[{'original':{'name':'indeedhub-relay','image':module.LEGACY_RELAY_IMAGE,'body':before,'container_id':'e'*64},'pinned_original_body':body,'preserve_original':False,'recovery_image':{'image':image,'operation_id':self.operation,'source_container_id':'e'*64}}]}
|
||||
digest=hashlib.sha256(body.encode()).hexdigest()
|
||||
module.verify_relay_image_lineage(module.LEGACY_RELAY_IMAGE,'unused',[])
|
||||
installed={'schema':1,'name':'indeedhub-relay','operation':self.operation,'body':body}
|
||||
module.verify_relay_image_lineage(image,digest,[record],installed)
|
||||
for missing in (None,dict(installed,operation=str(uuid.uuid4())),dict(installed,body='changed')):
|
||||
with self.assertRaises(RuntimeError):module.verify_relay_image_lineage(image,digest,[record],missing)
|
||||
for bad in [[],[record,record]]:
|
||||
with self.assertRaises(RuntimeError):module.verify_relay_image_lineage(image,digest,bad,installed)
|
||||
legacy=copy.deepcopy(record);legacy['schema']=1
|
||||
for malformed in (0,'false',[],{}):
|
||||
legacy['members'][0]['preserve_original']=malformed
|
||||
with self.assertRaises(RuntimeError):module.verify_relay_image_lineage(image,digest,[legacy],installed)
|
||||
for change in ('unfinished','foreign','preserved','wrong-body','cycle'):
|
||||
bad=copy.deepcopy(record)
|
||||
if change=='unfinished':bad['cleanup_done']=False
|
||||
if change=='foreign':bad['members'][0]['recovery_image']['operation_id']=str(uuid.uuid4())
|
||||
if change=='preserved':bad['members'][0]['preserve_original']=True
|
||||
if change=='wrong-body':bad['members'][0]['pinned_original_body']='changed'
|
||||
if change=='cycle':bad['members'][0]['original']['image']=image
|
||||
with self.assertRaises(RuntimeError):module.verify_relay_image_lineage(image,digest,[bad],installed)
|
||||
if __name__=='__main__':unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user