fix(orchestrator): reconcile recreates of bitcoin cascade-restart lnd

24fd97ed cascades on package.start/restart, but a reconcile pass that
recreates or starts a bitcoin backend (desired-state recovery, repair
recreate, boot InstallMissing) also moves the RPC address behind a
running lnd's back — §C 'restart lnd after ANY bitcoin recreate'.
After the pass, dependents from the shared address_caching_dependents
table (moved to app_ops as the single source of truth) that sat
untouched (NoOp) and aren't user-stopped are restarted under their op
lock; a dependent the pass itself (re)created already resolved the
fresh address and is left alone.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
archipelago 2026-07-10 10:27:36 -04:00
parent 854925c598
commit 755cd4f235
3 changed files with 127 additions and 10 deletions

View File

@ -1131,17 +1131,10 @@ fn orchestrator_stack_members(has_orchestrator: bool, package_id: &str) -> Optio
}
}
/// Dependents that resolve a backend's container address once at startup and
/// hold it: restarting the backend moves its IP and strands them until they
/// restart too. lnd dials the bitcoin RPC address it resolved at boot and
/// never re-resolves — after a bitcoin restart it spins on "dial tcp
/// <old-ip>:8332: no route to host" until lnd itself restarts (gate lnd
/// getinfo test, .228 2026-07-09; hardening plan §C cascade item).
/// See crate::app_ops::address_caching_dependents — shared with the
/// reconciler, which cascades after its own recreates.
fn address_caching_dependents(package_id: &str) -> &'static [&'static str] {
match package_id {
"bitcoin" | "bitcoin-core" | "bitcoin-knots" => &["lnd"],
_ => &[],
}
crate::app_ops::address_caching_dependents(package_id)
}
/// After a backend start/restart lands, bounce its address-caching dependents

View File

@ -60,6 +60,19 @@ pub fn stack_member_app_ids(package_id: &str) -> &'static [&'static str] {
}
}
/// Dependents that resolve a backend's container address once at startup and
/// hold it: moving the backend's IP (restart OR recreate) strands them until
/// they restart too. lnd dials the bitcoin RPC address it resolved at boot
/// and never re-resolves (gate lnd getinfo test, .228 2026-07-09; hardening
/// plan §C). The RPC start/restart workers and the reconciler both consult
/// this — single source of truth, like the stack table above.
pub fn address_caching_dependents(package_id: &str) -> &'static [&'static str] {
match package_id {
"bitcoin" | "bitcoin-core" | "bitcoin-knots" => &["lnd"],
_ => &[],
}
}
/// The package whose lifecycle lock covers `app_id`: the stack package when
/// `app_id` is a member (RPC ops on "mempool" hold the "mempool" lock while
/// they drive archy-mempool-web), otherwise the app itself.

View File

@ -951,6 +951,37 @@ impl ReconcileReport {
}
}
/// (backend, dependent) pairs a reconcile pass must cascade-restart: the pass
/// (re)created/started a backend whose address-caching dependents (see
/// crate::app_ops) hold its old resolved IP. Only dependents the pass left
/// untouched (NoOp = was already running) qualify — one the pass itself
/// (re)created resolved the fresh address on its own start — and a
/// user-stopped dependent holds no live connection to strand.
fn cascade_pairs_for_report<'r>(
report: &'r ReconcileReport,
user_stopped: &std::collections::HashSet<String>,
) -> Vec<(&'r str, &'static str)> {
let mut pairs = Vec::new();
for (backend, action) in &report.actions {
if !matches!(
action,
ReconcileAction::Installed | ReconcileAction::Started
) {
continue;
}
for dep in crate::app_ops::address_caching_dependents(backend) {
let dep_untouched = report
.actions
.iter()
.any(|(id, action)| id == dep && matches!(action, ReconcileAction::NoOp));
if dep_untouched && !user_stopped.contains(*dep) {
pairs.push((backend.as_str(), *dep));
}
}
}
pairs
}
#[derive(Debug, Default)]
pub struct AdoptionReport {
pub adopted: Vec<String>,
@ -1626,6 +1657,29 @@ impl ProdContainerOrchestrator {
}
}
// lnd resolves the bitcoin RPC address once at startup and caches it.
// The RPC start/restart workers already cascade-restart it (24fd97ed),
// but a reconcile pass that recreates or starts a backend (desired-
// state recovery, repair recreate, boot InstallMissing) moves the
// address behind a running dependent's back — §C "restart lnd after
// ANY bitcoin recreate".
for (backend, dep) in cascade_pairs_for_report(&report, &user_stopped) {
// Same rule as the RPC cascade: hold the dependent's op lock
// across the restart; skip when a worker is mid-sequence.
let lock = crate::app_ops::op_lock(dep);
let Ok(_guard) = lock.try_lock() else {
continue;
};
tracing::info!(
dependent = %dep,
backend = %backend,
"cascade restart: reconcile (re)created the backend and the dependent caches its resolved address"
);
if let Err(err) = self.restart(dep).await {
tracing::error!(dependent = %dep, backend = %backend, error = %err, "cascade restart failed");
}
}
report
}
@ -5215,6 +5269,63 @@ app:
assert!(!calls.iter().any(|c| c.starts_with("start_container:")));
}
#[test]
fn cascade_pairs_cover_backend_recreate_with_running_dependent() {
use std::collections::HashSet;
let report = |actions: Vec<(&str, ReconcileAction)>| ReconcileReport {
actions: actions
.into_iter()
.map(|(id, a)| (id.to_string(), a))
.collect(),
failures: vec![],
};
let none = HashSet::new();
// Backend recreated while lnd sat running (NoOp) → cascade.
let r = report(vec![
("bitcoin-knots", ReconcileAction::Installed),
("lnd", ReconcileAction::NoOp),
]);
assert_eq!(cascade_pairs_for_report(&r, &none), vec![("bitcoin-knots", "lnd")]);
// Backend merely started from stopped also moves the IP → cascade.
let r = report(vec![
("bitcoin-core", ReconcileAction::Started),
("lnd", ReconcileAction::NoOp),
]);
assert_eq!(cascade_pairs_for_report(&r, &none), vec![("bitcoin-core", "lnd")]);
// Backend untouched → no cascade.
let r = report(vec![
("bitcoin-knots", ReconcileAction::NoOp),
("lnd", ReconcileAction::NoOp),
]);
assert!(cascade_pairs_for_report(&r, &none).is_empty());
// Dependent itself (re)started this pass → it already resolved the
// fresh address; no cascade.
let r = report(vec![
("bitcoin-knots", ReconcileAction::Installed),
("lnd", ReconcileAction::Started),
]);
assert!(cascade_pairs_for_report(&r, &none).is_empty());
// User-stopped dependent is never bounced.
let r = report(vec![
("bitcoin-knots", ReconcileAction::Installed),
("lnd", ReconcileAction::NoOp),
]);
let stopped: HashSet<String> = ["lnd".to_string()].into();
assert!(cascade_pairs_for_report(&r, &stopped).is_empty());
// Non-backend recreates don't cascade anything.
let r = report(vec![
("grafana", ReconcileAction::Installed),
("lnd", ReconcileAction::NoOp),
]);
assert!(cascade_pairs_for_report(&r, &none).is_empty());
}
#[tokio::test]
async fn reconcile_starts_exited_container() {
let rt = Arc::new(MockRuntime::default());