feat(bootstrap): two OTA heals for network moves + registry renames

Both failure modes are from framework-pt relocating (2026-08-15):

- archy-ha-btc-rpc-proxy bound socat to the LAN IP baked in at unit
  generation; after a move the address no longer exists and the unit
  restart-looped forever (counter 2446). run_ha_rpc_proxy_bind_repair
  rewrites ExecStart to compute the bind address at each start, so
  Restart=always itself heals any future move.
- homeassistant's quadlet pointed at the domain image ref with --pull
  never while local storage held the same name:tag under the bare-IP
  registry ref (catalog signing rename) — 761 restarts on 'image not
  known'. run_pull_never_image_repair retags a matching local image;
  it deliberately never pulls.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
archipelago
2026-08-15 09:23:57 -04:00
co-authored by Claude Fable 5
parent 1587853ce2
commit ec2e6375ed
+206
View File
@@ -178,6 +178,18 @@ pub async fn ensure_doctor_installed() {
Ok(false) => debug!("Console welcome banner already current (or not an ISO node)"), Ok(false) => debug!("Console welcome banner already current (or not an ISO node)"),
Err(e) => warn!("Welcome banner sync failed (non-fatal): {:#}", e), Err(e) => warn!("Welcome banner sync failed (non-fatal): {:#}", e),
} }
match run_ha_rpc_proxy_bind_repair().await {
Ok(true) => info!(
"HA bitcoind RPC forwarder rebound dynamically — survives network moves now"
),
Ok(false) => debug!("HA bitcoind RPC forwarder absent or already dynamic"),
Err(e) => warn!("HA RPC forwarder bind repair failed (non-fatal): {:#}", e),
}
match run_pull_never_image_repair().await {
Ok(n) if n > 0 => info!(retagged = n, "Healed quadlet image refs orphaned by registry rename"),
Ok(_) => debug!("All quadlet image refs resolve locally"),
Err(e) => warn!("Quadlet image ref repair failed (non-fatal): {:#}", e),
}
match run_tor_torrc_repair().await { match run_tor_torrc_repair().await {
Ok(true) => info!("Tor healed at boot (torrc rebuilt and/or daemon restarted)"), Ok(true) => info!("Tor healed at boot (torrc rebuilt and/or daemon restarted)"),
Ok(false) => debug!("Tor healthy and torrc in sync — no heal needed"), Ok(false) => debug!("Tor healthy and torrc in sync — no heal needed"),
@@ -661,6 +673,167 @@ exit 2
const TOR_HELPER_SH: &str = include_str!("../../../scripts/tor-helper.sh"); const TOR_HELPER_SH: &str = include_str!("../../../scripts/tor-helper.sh");
const TOR_HELPER_PATH: &str = "/opt/archipelago/scripts/tor-helper.sh"; const TOR_HELPER_PATH: &str = "/opt/archipelago/scripts/tor-helper.sh";
/// Heal socat forwarder units that were generated with the node's LAN IP
/// baked into `bind=`.
///
/// `archy-ha-btc-rpc-proxy.service` (written on-node during the Pine/HA
/// integration) bound socat to the box's DHCP address at generation time —
/// pasta containers reach the host via its LAN address, so that was the
/// address that worked. Move the box to a new network and the address no
/// longer exists: `bind()` fails and the unit restart-loops forever
/// (framework-pt after relocating, 2026-08-15: restart counter 2446, and
/// Home Assistant's bitcoind sensor dead with it).
///
/// The rewrite computes the bind address at every service start instead, so
/// `Restart=always` itself becomes the heal: plug the box into any network
/// and the next restart binds to the new address.
const HA_RPC_PROXY_UNIT_PATH: &str = "/etc/systemd/system/archy-ha-btc-rpc-proxy.service";
/// Parse `TCP-LISTEN:<port>,bind=<ipv4>` + trailing `TCP:<target>` out of a
/// socat ExecStart line. Returns (listen_port, target).
fn parse_socat_static_bind(exec_line: &str) -> Option<(String, String)> {
let after_listen = exec_line.split("TCP-LISTEN:").nth(1)?;
let port = after_listen.split(',').next()?.trim();
if port.is_empty() || !port.chars().all(|c| c.is_ascii_digit()) {
return None;
}
// Only rewrite units pinned to a concrete address; a unit already using
// a computed bind (or none) needs no heal.
let bind = after_listen.split("bind=").nth(1)?.split(',').next()?.trim();
if !bind.chars().all(|c| c.is_ascii_digit() || c == '.') || bind.starts_with("127.") {
return None;
}
let target = exec_line.rsplit(" TCP:").next()?.trim();
if target.is_empty() || target == exec_line {
return None;
}
Some((port.to_string(), target.to_string()))
}
fn dynamic_bind_execstart(listen_port: &str, target: &str) -> String {
// `$$` survives systemd's own expansion as a literal `$`, so the command
// substitution runs in the shell at ExecStart time. If the box has no
// default route yet, exit non-zero and let Restart=always retry.
format!(
"ExecStart=/bin/sh -c 'IP=$$(ip -4 route get 1.1.1.1 | sed -n \"s/.*src \\([0-9.]*\\).*/\\1/p\"); \
[ -n \"$$IP\" ] || exit 1; \
exec /usr/bin/socat TCP-LISTEN:{listen_port},bind=$$IP,fork,reuseaddr TCP:{target}'"
)
}
async fn run_ha_rpc_proxy_bind_repair() -> Result<bool> {
let unit = match tokio::fs::read_to_string(HA_RPC_PROXY_UNIT_PATH).await {
Ok(s) => s,
Err(_) => return Ok(false), // node never grew the forwarder
};
let Some(exec_line) = unit.lines().find(|l| l.trim_start().starts_with("ExecStart=")) else {
return Ok(false);
};
let Some((port, target)) = parse_socat_static_bind(exec_line) else {
return Ok(false); // already dynamic (or not the shape we heal)
};
let healed = unit.replace(exec_line, &dynamic_bind_execstart(&port, &target));
let staged = "/var/lib/archipelago/ha-rpc-proxy.staged";
if let Some(dir) = Path::new(staged).parent() {
tokio::fs::create_dir_all(dir).await.ok();
}
tokio::fs::write(staged, &healed)
.await
.context("stage ha-rpc-proxy unit")?;
let script = format!(
"set -eu\ninstall -m 0644 {staged} {dest}\nsystemctl daemon-reload\nsystemctl restart archy-ha-btc-rpc-proxy 2>/dev/null || true\nexit 0\n",
staged = staged,
dest = HA_RPC_PROXY_UNIT_PATH
);
host_sudo(&["sh", "-lc", &script])
.await
.context("install ha-rpc-proxy unit")?;
Ok(true)
}
/// Re-point `--pull never` quadlets whose image ref no longer matches local
/// storage.
///
/// The catalog signing pass rewrites image refs (bare-IP registry → domain),
/// so a quadlet regenerated with the new ref points at an image the local
/// store only holds under the old name. With `--pull never` the app can
/// never start again on its own — Home Assistant looped 761 restarts on
/// "image not known" (framework-pt, 2026-08-15) while an identical
/// `name:tag` sat in storage under the bare-IP ref. If any local image
/// shares the wanted `name:tag`, retag it; pulling is deliberately NOT
/// attempted here (offline nodes, metered links — the doctor handles pulls).
async fn run_pull_never_image_repair() -> Result<usize> {
let home = std::env::var("HOME").unwrap_or_else(|_| "/home/archipelago".to_string());
let quadlet_dir = format!("{home}/.config/containers/systemd");
let mut wanted: Vec<String> = Vec::new();
let mut entries = match tokio::fs::read_dir(&quadlet_dir).await {
Ok(e) => e,
Err(_) => return Ok(0),
};
while let Ok(Some(entry)) = entries.next_entry().await {
let path = entry.path();
if path.extension().and_then(|e| e.to_str()) != Some("container") {
continue;
}
let Ok(text) = tokio::fs::read_to_string(&path).await else {
continue;
};
for line in text.lines() {
if let Some(image) = line.trim().strip_prefix("Image=") {
let image = image.trim();
if !image.is_empty() {
wanted.push(image.to_string());
}
}
}
}
if wanted.is_empty() {
return Ok(0);
}
let local = podman_stdout(&["images", "--format", "{{.Repository}}:{{.Tag}}"]).await;
let local: Vec<&str> = local
.lines()
.map(str::trim)
.filter(|l| !l.is_empty() && !l.contains("<none>"))
.collect();
let mut retagged = 0usize;
for want in wanted {
if local.iter().any(|l| *l == want) {
continue;
}
// Same `name:tag`, any registry prefix, is the rename we heal.
let Some(name_tag) = want.rsplit('/').next() else {
continue;
};
if !name_tag.contains(':') {
continue;
}
let suffix = format!("/{name_tag}");
let Some(src) = local.iter().find(|l| l.ends_with(&suffix)) else {
continue;
};
let status = tokio::process::Command::new("podman")
.args(["tag", src, &want])
.status()
.await;
match status {
Ok(s) if s.success() => {
info!(from = %src, to = %want, "Retagged image for a --pull never quadlet");
retagged += 1;
}
_ => warn!(from = %src, to = %want, "Image retag failed (non-fatal)"),
}
}
Ok(retagged)
}
async fn podman_stdout(args: &[&str]) -> String {
match tokio::process::Command::new("podman").args(args).output().await {
Ok(out) if out.status.success() => String::from_utf8_lossy(&out.stdout).into_owned(),
_ => String::new(),
}
}
/// The console welcome banner, embedded so the OTA can fix it on deployed /// The console welcome banner, embedded so the OTA can fix it on deployed
/// nodes. `/etc/profile.d/archipelago.sh` is baked by the ISO installer and /// nodes. `/etc/profile.d/archipelago.sh` is baked by the ISO installer and
/// no OTA path touched it, so every node kept whatever its ISO generation /// no OTA path touched it, so every node kept whatever its ISO generation
@@ -1472,6 +1645,39 @@ mod tests {
heal_stale_web_search_block("location / { try_files $uri /index.html; }").is_none() heal_stale_web_search_block("location / { try_files $uri /index.html; }").is_none()
); );
} }
/// The exact ExecStart framework-pt shipped with must parse, and the
/// rewrite must preserve its listen port and forward target.
#[test]
fn static_socat_bind_is_parsed_and_rewritten_dynamically() {
let line = "ExecStart=/usr/bin/socat TCP-LISTEN:18332,bind=192.168.1.249,fork,reuseaddr TCP:127.0.0.1:8332";
let (port, target) = parse_socat_static_bind(line).expect("must parse");
assert_eq!(port, "18332");
assert_eq!(target, "127.0.0.1:8332");
let dynamic = dynamic_bind_execstart(&port, &target);
assert!(dynamic.contains("TCP-LISTEN:18332,bind=$$IP"));
assert!(dynamic.contains("TCP:127.0.0.1:8332"));
assert!(dynamic.contains("route get 1.1.1.1"));
// The heal is idempotent: its own output no longer parses as a
// static bind (bind=$$IP is not a concrete address).
assert!(parse_socat_static_bind(&dynamic).is_none());
}
#[test]
fn socat_units_that_need_no_heal_are_left_alone() {
// Loopback bind is intentional (Tor bootstrap forwarder) — not ours.
assert!(parse_socat_static_bind(
"ExecStart=/usr/bin/socat TCP-LISTEN:18332,bind=127.0.0.1,reuseaddr,fork SOCKS4A:127.0.0.1:x.onion:8332,socksport=9050"
)
.is_none());
// No bind at all.
assert!(parse_socat_static_bind(
"ExecStart=/usr/bin/socat TCP-LISTEN:18332,fork,reuseaddr TCP:127.0.0.1:8332"
)
.is_none());
// Not a socat line.
assert!(parse_socat_static_bind("ExecStart=/usr/bin/true").is_none());
}
} }
/// Repair this node's own systemd restart policy. /// Repair this node's own systemd restart policy.