233 lines
12 KiB
Python
233 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""Real apt/dpkg/filesystem faults on an independent prepared disposable disk."""
|
|
import fcntl
|
|
import errno
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
|
|
assert 'QEMU' in Path('/sys/class/dmi/id/sys_vendor').read_text()
|
|
assert subprocess.check_output(['findmnt', '-n', '-o', 'SOURCE', '/'], text=True).strip() == '/dev/vda'
|
|
sys.path.insert(0, '/opt/ota-source/scripts')
|
|
import os_update_executor as executor
|
|
from os_update_plan import canonical, digest
|
|
|
|
evidence = Path('/var/log/os-ota-stage')
|
|
mode = Path('/opt/os-ota-scenario').read_text()
|
|
binding = json.loads((evidence/'binding.json').read_text())
|
|
plan = json.loads((evidence/'plan.json').read_text())
|
|
canaries = Path('/var/lib/archipelago/ota-synthetic-canaries')
|
|
hashes = lambda: {p.name: executor.file_hash(p) for p in canaries.iterdir()}
|
|
before = hashes()
|
|
inventory = executor.inventory()
|
|
result = dict(scenario=mode, production_approved=False, passed=False,
|
|
recovery_policy='no automatic dpkg replay, no financial-data rollback', events=[])
|
|
|
|
|
|
def refused(label, phase='apply', expected=None):
|
|
observed = None
|
|
try:
|
|
executor.stage(binding, 'APPLY', phase)
|
|
except Exception as error:
|
|
observed = error
|
|
if expected is not None:
|
|
assert expected in str(error), (label,type(error).__name__,str(error))
|
|
result['events'].append(dict(label=label, exception=type(error).__name__, error=str(error)))
|
|
else:
|
|
raise AssertionError(label+' unexpectedly succeeded')
|
|
assert hashes() == before
|
|
return observed
|
|
|
|
|
|
def fresh_appointment():
|
|
"""A distinct root fixture candidate; never reset an existing consumed status."""
|
|
global binding
|
|
old = executor.Executor(binding['os_update'])
|
|
preservation = json.loads((old.source/'preservation.json').read_text())
|
|
authority = digest(('independent-VM-fault-'+mode).encode())
|
|
preservation['authority_digest'] = authority
|
|
plan['preservation_sha256'] = digest(canonical(preservation))
|
|
plan_sha = digest(canonical(plan))
|
|
manifest = dict(os_update=dict(version=1, plan_sha256=plan_sha),
|
|
qualification='ROOT-APPOINTED-DISPOSABLE-FAULT-NOT-PRODUCTION', scenario=mode)
|
|
source = executor.ROOT/'plans'/plan_sha
|
|
source.mkdir(mode=0o700)
|
|
for name, value in (('plan.json',plan),('manifest.json',manifest),('preservation.json',preservation)):
|
|
(source/name).write_bytes(canonical(value))
|
|
# Retain the initial appointed verifier as evidence before appointing the
|
|
# separate fault candidate on this independent VM only.
|
|
executor.VERIFIER.rename(executor.VERIFIER.with_name(executor.VERIFIER.name+'.initial-fixture'))
|
|
verifier = '''#!/usr/bin/python3
|
|
import hashlib,pathlib,sys
|
|
assert 'QEMU' in pathlib.Path('/sys/class/dmi/id/sys_vendor').read_text()
|
|
assert sys.argv[1:3] == ['ceremony','verify'] and len(sys.argv)==4
|
|
assert hashlib.sha256(pathlib.Path(sys.argv[3]).read_bytes()).hexdigest()==%r
|
|
''' % digest(canonical(manifest))
|
|
executor.VERIFIER.write_text(verifier)
|
|
executor.VERIFIER.chmod(0o755)
|
|
binding = dict(os_update=dict(version=1, plan_sha256=plan_sha,
|
|
manifest_sha256=digest(canonical(manifest)), authority_digest=authority,
|
|
verifier_sha256=executor.file_hash(executor.VERIFIER)))
|
|
(evidence/('fault-'+mode+'-binding.json')).write_bytes(canonical(binding))
|
|
|
|
|
|
def fill_private_filesystem(filler):
|
|
"""Reserve evidence space first; ext4 f_bfree is not all fallocatable."""
|
|
reserve=filler.with_suffix('.reserve')
|
|
with reserve.open('xb') as stream:
|
|
os.posix_fallocate(stream.fileno(),0,8*1024**2)
|
|
try:
|
|
with filler.open('xb') as stream:
|
|
offset=0
|
|
for chunk in (16*1024**2,64*1024):
|
|
while offset < 8*1024**3:
|
|
try:
|
|
os.posix_fallocate(stream.fileno(),offset,chunk)
|
|
except OSError as error:
|
|
if error.errno != errno.ENOSPC:
|
|
raise
|
|
break
|
|
offset+=chunk
|
|
result['space_filler_bytes']=offset
|
|
finally:
|
|
reserve.unlink()
|
|
|
|
|
|
try:
|
|
if mode == 'locked-apt':
|
|
holder=subprocess.Popen([sys.executable,'-c',
|
|
'import fcntl,time;f=open("/var/lib/dpkg/lock-frontend","a");'
|
|
'fcntl.lockf(f,fcntl.LOCK_EX|fcntl.LOCK_NB);print("LOCKED",flush=True);time.sleep(600)'],
|
|
stdout=subprocess.PIPE,text=True)
|
|
try:
|
|
assert holder.stdout.readline().strip()=='LOCKED'
|
|
result['competing_package_writer_pid']=holder.pid
|
|
refused('actual-dpkg-frontend-lock',expected='lock')
|
|
finally:
|
|
holder.terminate()
|
|
holder.wait()
|
|
holder.stdout.close()
|
|
assert executor.inventory() == inventory
|
|
if executor.Executor(binding['os_update']).status['phase']=='VERIFIED':
|
|
status=executor.stage(binding,'APPLY','apply')
|
|
assert status['phase']=='REBOOT_REQUIRED'
|
|
assert executor.stage(binding,'APPLY','apply')==status
|
|
result['recovery']='pre-effect real lock refusal; released lock; actual apply completed once under retained libapt locks'
|
|
else:
|
|
refused('lock-released-consumed-authority-not-replayed',expected='never retry dpkg')
|
|
result['reconciliation'] = executor.stage(binding,'APPLY','reconcile')
|
|
result['recovery'] = 'lock released; dpkg unchanged; externally reconciled authority still required'
|
|
elif mode == 'space':
|
|
filler = Path('/var/tmp/os-ota-private-space-fault')
|
|
try:
|
|
fill_private_filesystem(filler)
|
|
refused('actual-filesystem-space-exhaustion')
|
|
finally:
|
|
filler.unlink(missing_ok=True)
|
|
assert executor.inventory() == inventory
|
|
if executor.Executor(binding['os_update']).status['phase'] == 'VERIFIED':
|
|
recovered = executor.stage(binding,'APPLY','apply')
|
|
assert recovered['phase']=='REBOOT_REQUIRED'
|
|
result['recovery'] = 'pre-effect space refusal; owned filler removed; original prepared authority applied successfully'
|
|
else:
|
|
refused('space-recovered-authority-not-replayed')
|
|
result['reconciliation'] = executor.stage(binding,'APPLY','reconcile')
|
|
result['recovery'] = 'only owned filler removed; dpkg unchanged; evidence-only reconciliation; external authority required'
|
|
elif mode == 'partial-dpkg':
|
|
preparation = executor.Executor(binding['os_update']).status['phase'] != 'VERIFIED'
|
|
if preparation:
|
|
fresh_appointment()
|
|
download = subprocess.run(['apt-get','download','curl=8.14.1-2+deb13u5'], cwd=evidence,
|
|
text=True, capture_output=True)
|
|
print(download.stdout, download.stderr)
|
|
download.check_returncode()
|
|
archive = next(evidence.glob('curl_*deb13u5_amd64.deb'))
|
|
subprocess.run(['dpkg','--unpack',str(archive)],check=True)
|
|
audit = subprocess.check_output(['dpkg','--audit'],text=True)
|
|
assert audit.strip()
|
|
result['partial_dpkg_audit'] = audit
|
|
refused('actual-unpacked-unconfigured-package',phase='prepare' if preparation else 'apply',
|
|
expected='dpkg inventory changed')
|
|
subprocess.run(['dpkg','--configure','curl'],check=True)
|
|
assert executor.inventory() == inventory
|
|
assert not subprocess.check_output(['dpkg','--audit'],text=True).strip()
|
|
executor.Executor(binding['os_update']).preflight()
|
|
if preparation:
|
|
refused('dpkg-repaired-preparation-authority-not-replayed','prepare',expected='external reconciliation')
|
|
result['recovery'] = 'explicit configure of same-version curl; clean dpkg and original inventory; no snapshot restore'
|
|
elif mode in ('offline','missing-dependency','interrupted-before','bad-hash','space-prepare'):
|
|
if mode == 'missing-dependency':
|
|
removed = next(p for p in plan['packages'] if p['name']=='libcurl4t64')
|
|
plan['packages'].remove(removed)
|
|
result['missing_dependency'] = removed
|
|
if mode == 'bad-hash':
|
|
next(p for p in plan['packages'] if p['name']=='curl')['sha256']='0'*64
|
|
fresh_appointment()
|
|
if mode == 'bad-hash':
|
|
refused('package-hash-disagrees-with-authenticated-index','prepare',expected='Packages/archive chain')
|
|
refused('bad-hash-candidate-not-replayed','prepare',expected='external reconciliation')
|
|
result['recovery']='bad candidate retained; no package effect; corrected new authority required'
|
|
elif mode == 'space-prepare':
|
|
filler=Path('/var/tmp/os-ota-private-space-fault')
|
|
try:
|
|
fill_private_filesystem(filler)
|
|
error=refused('real-ENOSPC-during-preparation','prepare')
|
|
assert isinstance(error,OSError) and error.errno==errno.ENOSPC, str(error)
|
|
finally:
|
|
filler.unlink(missing_ok=True)
|
|
refused('space-restored-preparation-not-replayed','prepare',expected='external reconciliation')
|
|
result['recovery']='only owned filler removed; disk space restored; partial preparation retained; dpkg unchanged'
|
|
elif mode == 'offline':
|
|
# Actual private guest network is removed; no transport monkeypatch.
|
|
subprocess.run(['ip','link','set','eth0','down'],check=True)
|
|
error = refused('actual-offline-preparation', 'prepare')
|
|
assert isinstance(error,urllib.error.URLError), 'must be real transport failure, not another preparation refusal'
|
|
subprocess.run(['ip','link','set','eth0','up'],check=True)
|
|
subprocess.run(['ip','route','replace','default','via','10.0.2.2'],check=True)
|
|
refused('network-restored-preparation-not-replayed', 'prepare',expected='external reconciliation')
|
|
result['recovery'] = 'guest route restored; partial preparation retained for external reconciliation'
|
|
elif mode == 'missing-dependency':
|
|
refused('incomplete-authenticated-dependency-closure', 'prepare',expected='incomplete archive transaction')
|
|
result['recovery'] = 'incomplete candidate refused without dpkg effects; corrected new authority required'
|
|
else:
|
|
path = evidence/('fault-'+mode+'-binding.json')
|
|
child = subprocess.Popen([sys.executable,'-B','-c',
|
|
'import sys,json;sys.path.insert(0,"/opt/ota-source/scripts");'
|
|
'import os_update_executor as e;e.stage(json.load(open(sys.argv[1])),"APPLY","prepare")',str(path)],
|
|
start_new_session=True)
|
|
try:
|
|
deadline = time.monotonic()+60
|
|
while time.monotonic()<deadline:
|
|
status = executor.Executor(binding['os_update']).status
|
|
if status['phase']=='DOWNLOAD':
|
|
break
|
|
if child.poll() is not None:
|
|
raise AssertionError('preparation exited before interruption')
|
|
time.sleep(.01)
|
|
else:
|
|
raise TimeoutError('DOWNLOAD boundary not reached')
|
|
finally:
|
|
if child.poll() is None:
|
|
os.killpg(child.pid,signal.SIGKILL)
|
|
child.wait()
|
|
assert child.returncode==-9
|
|
result['interrupted_exit'] = child.returncode
|
|
refused('interrupted-before-effects-not-replayed','prepare',expected='external reconciliation')
|
|
result['recovery'] = 'interrupted preparation retained; no package effect or financial rollback'
|
|
assert executor.inventory() == inventory
|
|
else:
|
|
raise ValueError('unknown fault scenario')
|
|
assert hashes() == before
|
|
result.update(passed=True, canaries_preserved=True,
|
|
status=executor.Executor(binding['os_update']).status)
|
|
finally:
|
|
(evidence/('fault-'+mode+'-result.json')).write_text(json.dumps(result,indent=2)+'\n')
|
|
summary={k:v for k,v in result.items() if k!='status'}
|
|
summary['status_phase']=result.get('status',{}).get('phase')
|
|
print(json.dumps(summary,indent=2),flush=True)
|