fix(orchestrator): reconcile recreates of bitcoin cascade-restart lnd
24fd97ed cascades on package.start/restart, but a reconcile pass that recreates or starts a bitcoin backend (desired-state recovery, repair recreate, boot InstallMissing) also moves the RPC address behind a running lnd's back — §C 'restart lnd after ANY bitcoin recreate'. After the pass, dependents from the shared address_caching_dependents table (moved to app_ops as the single source of truth) that sat untouched (NoOp) and aren't user-stopped are restarted under their op lock; a dependent the pass itself (re)created already resolved the fresh address and is left alone. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
854925c598
commit
755cd4f235
@ -1131,17 +1131,10 @@ fn orchestrator_stack_members(has_orchestrator: bool, package_id: &str) -> Optio
|
||||
}
|
||||
}
|
||||
|
||||
/// Dependents that resolve a backend's container address once at startup and
|
||||
/// hold it: restarting the backend moves its IP and strands them until they
|
||||
/// restart too. lnd dials the bitcoin RPC address it resolved at boot and
|
||||
/// never re-resolves — after a bitcoin restart it spins on "dial tcp
|
||||
/// <old-ip>:8332: no route to host" until lnd itself restarts (gate lnd
|
||||
/// getinfo test, .228 2026-07-09; hardening plan §C cascade item).
|
||||
/// See crate::app_ops::address_caching_dependents — shared with the
|
||||
/// reconciler, which cascades after its own recreates.
|
||||
fn address_caching_dependents(package_id: &str) -> &'static [&'static str] {
|
||||
match package_id {
|
||||
"bitcoin" | "bitcoin-core" | "bitcoin-knots" => &["lnd"],
|
||||
_ => &[],
|
||||
}
|
||||
crate::app_ops::address_caching_dependents(package_id)
|
||||
}
|
||||
|
||||
/// After a backend start/restart lands, bounce its address-caching dependents
|
||||
|
||||
@ -60,6 +60,19 @@ pub fn stack_member_app_ids(package_id: &str) -> &'static [&'static str] {
|
||||
}
|
||||
}
|
||||
|
||||
/// Dependents that resolve a backend's container address once at startup and
|
||||
/// hold it: moving the backend's IP (restart OR recreate) strands them until
|
||||
/// they restart too. lnd dials the bitcoin RPC address it resolved at boot
|
||||
/// and never re-resolves (gate lnd getinfo test, .228 2026-07-09; hardening
|
||||
/// plan §C). The RPC start/restart workers and the reconciler both consult
|
||||
/// this — single source of truth, like the stack table above.
|
||||
pub fn address_caching_dependents(package_id: &str) -> &'static [&'static str] {
|
||||
match package_id {
|
||||
"bitcoin" | "bitcoin-core" | "bitcoin-knots" => &["lnd"],
|
||||
_ => &[],
|
||||
}
|
||||
}
|
||||
|
||||
/// The package whose lifecycle lock covers `app_id`: the stack package when
|
||||
/// `app_id` is a member (RPC ops on "mempool" hold the "mempool" lock while
|
||||
/// they drive archy-mempool-web), otherwise the app itself.
|
||||
|
||||
@ -951,6 +951,37 @@ impl ReconcileReport {
|
||||
}
|
||||
}
|
||||
|
||||
/// (backend, dependent) pairs a reconcile pass must cascade-restart: the pass
|
||||
/// (re)created/started a backend whose address-caching dependents (see
|
||||
/// crate::app_ops) hold its old resolved IP. Only dependents the pass left
|
||||
/// untouched (NoOp = was already running) qualify — one the pass itself
|
||||
/// (re)created resolved the fresh address on its own start — and a
|
||||
/// user-stopped dependent holds no live connection to strand.
|
||||
fn cascade_pairs_for_report<'r>(
|
||||
report: &'r ReconcileReport,
|
||||
user_stopped: &std::collections::HashSet<String>,
|
||||
) -> Vec<(&'r str, &'static str)> {
|
||||
let mut pairs = Vec::new();
|
||||
for (backend, action) in &report.actions {
|
||||
if !matches!(
|
||||
action,
|
||||
ReconcileAction::Installed | ReconcileAction::Started
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
for dep in crate::app_ops::address_caching_dependents(backend) {
|
||||
let dep_untouched = report
|
||||
.actions
|
||||
.iter()
|
||||
.any(|(id, action)| id == dep && matches!(action, ReconcileAction::NoOp));
|
||||
if dep_untouched && !user_stopped.contains(*dep) {
|
||||
pairs.push((backend.as_str(), *dep));
|
||||
}
|
||||
}
|
||||
}
|
||||
pairs
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct AdoptionReport {
|
||||
pub adopted: Vec<String>,
|
||||
@ -1626,6 +1657,29 @@ impl ProdContainerOrchestrator {
|
||||
}
|
||||
}
|
||||
|
||||
// lnd resolves the bitcoin RPC address once at startup and caches it.
|
||||
// The RPC start/restart workers already cascade-restart it (24fd97ed),
|
||||
// but a reconcile pass that recreates or starts a backend (desired-
|
||||
// state recovery, repair recreate, boot InstallMissing) moves the
|
||||
// address behind a running dependent's back — §C "restart lnd after
|
||||
// ANY bitcoin recreate".
|
||||
for (backend, dep) in cascade_pairs_for_report(&report, &user_stopped) {
|
||||
// Same rule as the RPC cascade: hold the dependent's op lock
|
||||
// across the restart; skip when a worker is mid-sequence.
|
||||
let lock = crate::app_ops::op_lock(dep);
|
||||
let Ok(_guard) = lock.try_lock() else {
|
||||
continue;
|
||||
};
|
||||
tracing::info!(
|
||||
dependent = %dep,
|
||||
backend = %backend,
|
||||
"cascade restart: reconcile (re)created the backend and the dependent caches its resolved address"
|
||||
);
|
||||
if let Err(err) = self.restart(dep).await {
|
||||
tracing::error!(dependent = %dep, backend = %backend, error = %err, "cascade restart failed");
|
||||
}
|
||||
}
|
||||
|
||||
report
|
||||
}
|
||||
|
||||
@ -5215,6 +5269,63 @@ app:
|
||||
assert!(!calls.iter().any(|c| c.starts_with("start_container:")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cascade_pairs_cover_backend_recreate_with_running_dependent() {
|
||||
use std::collections::HashSet;
|
||||
let report = |actions: Vec<(&str, ReconcileAction)>| ReconcileReport {
|
||||
actions: actions
|
||||
.into_iter()
|
||||
.map(|(id, a)| (id.to_string(), a))
|
||||
.collect(),
|
||||
failures: vec![],
|
||||
};
|
||||
let none = HashSet::new();
|
||||
|
||||
// Backend recreated while lnd sat running (NoOp) → cascade.
|
||||
let r = report(vec![
|
||||
("bitcoin-knots", ReconcileAction::Installed),
|
||||
("lnd", ReconcileAction::NoOp),
|
||||
]);
|
||||
assert_eq!(cascade_pairs_for_report(&r, &none), vec![("bitcoin-knots", "lnd")]);
|
||||
|
||||
// Backend merely started from stopped also moves the IP → cascade.
|
||||
let r = report(vec![
|
||||
("bitcoin-core", ReconcileAction::Started),
|
||||
("lnd", ReconcileAction::NoOp),
|
||||
]);
|
||||
assert_eq!(cascade_pairs_for_report(&r, &none), vec![("bitcoin-core", "lnd")]);
|
||||
|
||||
// Backend untouched → no cascade.
|
||||
let r = report(vec![
|
||||
("bitcoin-knots", ReconcileAction::NoOp),
|
||||
("lnd", ReconcileAction::NoOp),
|
||||
]);
|
||||
assert!(cascade_pairs_for_report(&r, &none).is_empty());
|
||||
|
||||
// Dependent itself (re)started this pass → it already resolved the
|
||||
// fresh address; no cascade.
|
||||
let r = report(vec![
|
||||
("bitcoin-knots", ReconcileAction::Installed),
|
||||
("lnd", ReconcileAction::Started),
|
||||
]);
|
||||
assert!(cascade_pairs_for_report(&r, &none).is_empty());
|
||||
|
||||
// User-stopped dependent is never bounced.
|
||||
let r = report(vec![
|
||||
("bitcoin-knots", ReconcileAction::Installed),
|
||||
("lnd", ReconcileAction::NoOp),
|
||||
]);
|
||||
let stopped: HashSet<String> = ["lnd".to_string()].into();
|
||||
assert!(cascade_pairs_for_report(&r, &stopped).is_empty());
|
||||
|
||||
// Non-backend recreates don't cascade anything.
|
||||
let r = report(vec![
|
||||
("grafana", ReconcileAction::Installed),
|
||||
("lnd", ReconcileAction::NoOp),
|
||||
]);
|
||||
assert!(cascade_pairs_for_report(&r, &none).is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reconcile_starts_exited_container() {
|
||||
let rt = Arc::new(MockRuntime::default());
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user