fix(lifecycle): cascade-restart lnd after bitcoin backend start/restart
lnd resolves the bitcoin RPC address once at startup and never re-resolves; when bitcoin-core/knots restarts (new container IP) lnd spins on "dial tcp <old-ip>:8332: no route to host" and its own RPC wedges until lnd restarts (gate lnd getinfo test 75 failed every iteration on .228, 2026-07-09; hardening plan §C cascade item). electrumx is already restarted by the version-switch path — lnd was the only dependent left stranded. After a successful package.start/restart of a bitcoin backend, restart running, not-user-stopped address-caching dependents under their own app-op lock (lock order backend→dependent only, so no deadlock). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
4fbe3d4ba0
commit
24fd97ed98
@ -94,6 +94,7 @@ impl RpcHandler {
|
|||||||
.await;
|
.await;
|
||||||
|
|
||||||
let op_lock = app_op_lock(package_id);
|
let op_lock = app_op_lock(package_id);
|
||||||
|
let data_dir = self.config.data_dir.clone();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let _op_guard = op_lock.lock().await;
|
let _op_guard = op_lock.lock().await;
|
||||||
let result = if let Some(orchestrator) = orchestrator.as_ref() {
|
let result = if let Some(orchestrator) = orchestrator.as_ref() {
|
||||||
@ -106,6 +107,13 @@ impl RpcHandler {
|
|||||||
reconcile_companions_for(&companion_app_id).await;
|
reconcile_companions_for(&companion_app_id).await;
|
||||||
set_package_state(&state_manager, &package_id_owned, PackageState::Running)
|
set_package_state(&state_manager, &package_id_owned, PackageState::Running)
|
||||||
.await;
|
.await;
|
||||||
|
cascade_restart_address_caching_dependents(
|
||||||
|
orchestrator.as_ref(),
|
||||||
|
&state_manager,
|
||||||
|
&data_dir,
|
||||||
|
&package_id_owned,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::error!("package.start {} failed: {:#}", package_id_owned, e);
|
tracing::error!("package.start {} failed: {:#}", package_id_owned, e);
|
||||||
@ -293,6 +301,7 @@ impl RpcHandler {
|
|||||||
.await;
|
.await;
|
||||||
|
|
||||||
let op_lock = app_op_lock(package_id);
|
let op_lock = app_op_lock(package_id);
|
||||||
|
let data_dir = self.config.data_dir.clone();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let _op_guard = op_lock.lock().await;
|
let _op_guard = op_lock.lock().await;
|
||||||
let result = if let Some(orchestrator) = orchestrator.as_ref() {
|
let result = if let Some(orchestrator) = orchestrator.as_ref() {
|
||||||
@ -305,6 +314,13 @@ impl RpcHandler {
|
|||||||
reconcile_companions_for(&companion_app_id).await;
|
reconcile_companions_for(&companion_app_id).await;
|
||||||
set_package_state(&state_manager, &package_id_owned, PackageState::Running)
|
set_package_state(&state_manager, &package_id_owned, PackageState::Running)
|
||||||
.await;
|
.await;
|
||||||
|
cascade_restart_address_caching_dependents(
|
||||||
|
orchestrator.as_ref(),
|
||||||
|
&state_manager,
|
||||||
|
&data_dir,
|
||||||
|
&package_id_owned,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::error!("package.restart {} failed: {:#}", package_id_owned, e);
|
tracing::error!("package.restart {} failed: {:#}", package_id_owned, e);
|
||||||
@ -1115,6 +1131,76 @@ fn orchestrator_stack_members(has_orchestrator: bool, package_id: &str) -> Optio
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Dependents that resolve a backend's container address once at startup and
|
||||||
|
/// hold it: restarting the backend moves its IP and strands them until they
|
||||||
|
/// restart too. lnd dials the bitcoin RPC address it resolved at boot and
|
||||||
|
/// never re-resolves — after a bitcoin restart it spins on "dial tcp
|
||||||
|
/// <old-ip>:8332: no route to host" until lnd itself restarts (gate lnd
|
||||||
|
/// getinfo test, .228 2026-07-09; hardening plan §C cascade item).
|
||||||
|
fn address_caching_dependents(package_id: &str) -> &'static [&'static str] {
|
||||||
|
match package_id {
|
||||||
|
"bitcoin" | "bitcoin-core" | "bitcoin-knots" => &["lnd"],
|
||||||
|
_ => &[],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// After a backend start/restart lands, bounce its address-caching dependents
|
||||||
|
/// (see above). Runs inside the backend worker's op-lock scope and takes each
|
||||||
|
/// dependent's own op lock — lock order is always backend → dependent, and no
|
||||||
|
/// dependent op takes a backend lock, so the nesting cannot deadlock.
|
||||||
|
async fn cascade_restart_address_caching_dependents(
|
||||||
|
orchestrator: Option<&Arc<dyn crate::container::traits::ContainerOrchestrator>>,
|
||||||
|
state_manager: &crate::state::StateManager,
|
||||||
|
data_dir: &std::path::Path,
|
||||||
|
backend_id: &str,
|
||||||
|
) {
|
||||||
|
for dep in address_caching_dependents(backend_id) {
|
||||||
|
// A user-stopped or not-running dependent holds no live connection —
|
||||||
|
// starting it here would override an explicit operator decision.
|
||||||
|
if crate::crash_recovery::load_user_stopped(data_dir)
|
||||||
|
.await
|
||||||
|
.contains(*dep)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let running = match podman_control(&["ps", "--format", "{{.Names}}"]).await {
|
||||||
|
Ok(o) if o.status.success() => String::from_utf8_lossy(&o.stdout)
|
||||||
|
.lines()
|
||||||
|
.any(|l| l.trim() == *dep),
|
||||||
|
_ => false,
|
||||||
|
};
|
||||||
|
if !running {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
install_log(&format!(
|
||||||
|
"CASCADE RESTART: {dep} (backend {backend_id} restarted; {dep} caches its resolved address)"
|
||||||
|
))
|
||||||
|
.await;
|
||||||
|
let op_lock = app_op_lock(dep);
|
||||||
|
let _op_guard = op_lock.lock().await;
|
||||||
|
let prev = flip_package_state(state_manager, dep, PackageState::Restarting).await;
|
||||||
|
let target = vec![dep.to_string()];
|
||||||
|
let result = if let Some(orch) = orchestrator {
|
||||||
|
do_orchestrator_package_restart(orch.as_ref(), &target).await
|
||||||
|
} else {
|
||||||
|
do_package_restart(&target).await
|
||||||
|
};
|
||||||
|
match result {
|
||||||
|
Ok(()) => {
|
||||||
|
reconcile_companions_for(dep).await;
|
||||||
|
set_package_state(state_manager, dep, PackageState::Running).await;
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
tracing::error!(dependent = %dep, backend = %backend_id, error = %e, "cascade restart failed");
|
||||||
|
install_log(&format!("CASCADE RESTART FAIL: {dep} — {e:#}")).await;
|
||||||
|
if let Some(prev) = prev {
|
||||||
|
set_package_state(state_manager, dep, prev).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Per-app lifecycle-operation locks. package.start/stop/restart reply
|
/// Per-app lifecycle-operation locks. package.start/stop/restart reply
|
||||||
/// immediately and run their multi-container sequences in spawned tasks;
|
/// immediately and run their multi-container sequences in spawned tasks;
|
||||||
/// unserialized, back-to-back RPCs interleave those sequences (gate run D,
|
/// unserialized, back-to-back RPCs interleave those sequences (gate run D,
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user