lfg2025/archy
1
0
Fork 0
Code Issues 16 Pull Requests Actions Packages Projects Releases 44 Wiki Activity

fix(lifecycle): cascade-restart lnd after bitcoin backend start/restart

Browse Source 
lnd resolves the bitcoin RPC address once at startup and never re-resolves;
when bitcoin-core/knots restarts (new container IP) lnd spins on "dial tcp
<old-ip>:8332: no route to host" and its own RPC wedges until lnd restarts
(gate lnd getinfo test 75 failed every iteration on .228, 2026-07-09;
hardening plan §C cascade item). electrumx is already restarted by the
version-switch path — lnd was the only dependent left stranded.

After a successful package.start/restart of a bitcoin backend, restart
running, not-user-stopped address-caching dependents under their own app-op
lock (lock order backend→dependent only, so no deadlock).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
archipelago 2026-07-09 05:53:53 -04:00
parent 4fbe3d4ba0
commit 24fd97ed98
1 changed files with 86 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
core/archipelago/src/api/rpc/package/runtime.rs
View File

@ -94,6 +94,7 @@ impl RpcHandler {
.await;
let op_lock = app_op_lock(package_id);
let data_dir = self.config.data_dir.clone();
tokio::spawn(async move {
let _op_guard = op_lock.lock().await;
let result = if let Some(orchestrator) = orchestrator.as_ref() {
@ -106,6 +107,13 @@ impl RpcHandler {
reconcile_companions_for(&companion_app_id).await;
set_package_state(&state_manager, &package_id_owned, PackageState::Running)
.await;
cascade_restart_address_caching_dependents(
orchestrator.as_ref(),
&state_manager,
&data_dir,
&package_id_owned,
)
.await;
}
Err(e) => {
tracing::error!("package.start {} failed: {:#}", package_id_owned, e);
@ -293,6 +301,7 @@ impl RpcHandler {
.await;
let op_lock = app_op_lock(package_id);
let data_dir = self.config.data_dir.clone();
tokio::spawn(async move {
let _op_guard = op_lock.lock().await;
let result = if let Some(orchestrator) = orchestrator.as_ref() {
@ -305,6 +314,13 @@ impl RpcHandler {
reconcile_companions_for(&companion_app_id).await;
set_package_state(&state_manager, &package_id_owned, PackageState::Running)
.await;
cascade_restart_address_caching_dependents(
orchestrator.as_ref(),
&state_manager,
&data_dir,
&package_id_owned,
)
.await;
}
Err(e) => {
tracing::error!("package.restart {} failed: {:#}", package_id_owned, e);
@ -1115,6 +1131,76 @@ fn orchestrator_stack_members(has_orchestrator: bool, package_id: &str) -> Optio
}
}
/// Dependents that resolve a backend's container address once at startup and
/// hold it: restarting the backend moves its IP and strands them until they
/// restart too. lnd dials the bitcoin RPC address it resolved at boot and
/// never re-resolves — after a bitcoin restart it spins on "dial tcp
/// <old-ip>:8332: no route to host" until lnd itself restarts (gate lnd
/// getinfo test, .228 2026-07-09; hardening plan §C cascade item).
fn address_caching_dependents(package_id: &str) -> &'static [&'static str] {
match package_id {
"bitcoin" | "bitcoin-core" | "bitcoin-knots" => &["lnd"],
_ => &[],
}
}
/// After a backend start/restart lands, bounce its address-caching dependents
/// (see above). Runs inside the backend worker's op-lock scope and takes each
/// dependent's own op lock — lock order is always backend → dependent, and no
/// dependent op takes a backend lock, so the nesting cannot deadlock.
async fn cascade_restart_address_caching_dependents(
orchestrator: Option<&Arc<dyn crate::container::traits::ContainerOrchestrator>>,
state_manager: &crate::state::StateManager,
data_dir: &std::path::Path,
backend_id: &str,
) {
for dep in address_caching_dependents(backend_id) {
// A user-stopped or not-running dependent holds no live connection —
// starting it here would override an explicit operator decision.
if crate::crash_recovery::load_user_stopped(data_dir)
.await
.contains(*dep)
{
continue;
}
let running = match podman_control(&["ps", "--format", "{{.Names}}"]).await {
Ok(o) if o.status.success() => String::from_utf8_lossy(&o.stdout)
.lines()
.any(|l| l.trim() == *dep),
_ => false,
};
if !running {
continue;
}
install_log(&format!(
"CASCADE RESTART: {dep} (backend {backend_id} restarted; {dep} caches its resolved address)"
))
.await;
let op_lock = app_op_lock(dep);
let _op_guard = op_lock.lock().await;
let prev = flip_package_state(state_manager, dep, PackageState::Restarting).await;
let target = vec![dep.to_string()];
let result = if let Some(orch) = orchestrator {
do_orchestrator_package_restart(orch.as_ref(), &target).await
} else {
do_package_restart(&target).await
};
match result {
Ok(()) => {
reconcile_companions_for(dep).await;
set_package_state(state_manager, dep, PackageState::Running).await;
}
Err(e) => {
tracing::error!(dependent = %dep, backend = %backend_id, error = %e, "cascade restart failed");
install_log(&format!("CASCADE RESTART FAIL: {dep} — {e:#}")).await;
if let Some(prev) = prev {
set_package_state(state_manager, dep, prev).await;
}
}
}
}
}
/// Per-app lifecycle-operation locks. package.start/stop/restart reply
/// immediately and run their multi-container sequences in spawned tasks;
/// unserialized, back-to-back RPCs interleave those sequences (gate run D,