Integrate retained updater candidate for release qualification
This commit is contained in:
@@ -362,7 +362,11 @@ impl RpcHandler {
|
||||
set_package_state(
|
||||
&handler.state_manager,
|
||||
&package_id_spawn,
|
||||
if result.get("status").and_then(|v| v.as_str()) == Some("up-to-date") {
|
||||
if result.get("status").and_then(|v| v.as_str()) == Some("staged") {
|
||||
PackageState::Stopped
|
||||
} else if result.get("status").and_then(|v| v.as_str())
|
||||
== Some("up-to-date")
|
||||
{
|
||||
pre_state.clone().unwrap_or(PackageState::Running)
|
||||
} else {
|
||||
PackageState::Running
|
||||
@@ -373,12 +377,34 @@ impl RpcHandler {
|
||||
Err(e) => {
|
||||
error!("package.update {} failed: {:#}", package_id_spawn, e);
|
||||
install_log(&format!("UPDATE FAIL: {} — {:#}", package_id_spawn, e)).await;
|
||||
// Inner handler already ran rollback_update + cleared
|
||||
// update state, but be defensive: revert to pre-state
|
||||
// in case the inner flow died before its cleanup.
|
||||
if let Some(prev) = pre_state {
|
||||
set_package_state(&handler.state_manager, &package_id_spawn, prev).await;
|
||||
}
|
||||
// Release the transitional overlay before asking the scanner
|
||||
// for real state. Prior Running is not proof of successful
|
||||
// rollback, and a failed preflight is not proof of Stopped.
|
||||
handler
|
||||
.state_manager
|
||||
.mutate_data(|data| {
|
||||
if let Some(entry) = data.package_data.get_mut(&package_id_spawn) {
|
||||
finish_failed_update(entry);
|
||||
}
|
||||
data.notifications.retain(|item| {
|
||||
item.id != format!("update-failed-{package_id_spawn}")
|
||||
});
|
||||
data.notifications.push(crate::data_model::Notification {
|
||||
id: format!("update-failed-{package_id_spawn}"),
|
||||
level: crate::data_model::NotificationLevel::Error,
|
||||
title: format!("Could not update {package_id_spawn}"),
|
||||
message: format!(
|
||||
"{e}. Runtime recovery does not roll back database changes."
|
||||
),
|
||||
timestamp: chrono::Utc::now().to_rfc3339(),
|
||||
app_id: Some(package_id_spawn.clone()),
|
||||
});
|
||||
while data.notifications.len() > 20 {
|
||||
data.notifications.remove(0);
|
||||
}
|
||||
})
|
||||
.await;
|
||||
kick_scanner_and_wait(&handler).await;
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -581,3 +607,34 @@ async fn kick_scanner_and_wait(handler: &RpcHandler) {
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
fn finish_failed_update(entry: &mut crate::data_model::PackageDataEntry) {
|
||||
if entry.state == PackageState::Updating {
|
||||
entry.state = PackageState::Installed;
|
||||
}
|
||||
entry.install_progress = None;
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod update_completion_tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn failure_releases_spinner_without_inventing_stopped_or_restored_runtime() {
|
||||
let mut entry = super::super::progress::create_installing_entry("movie");
|
||||
entry.state = PackageState::Updating;
|
||||
finish_failed_update(&mut entry);
|
||||
assert_eq!(entry.state, PackageState::Installed);
|
||||
assert!(entry.install_progress.is_none());
|
||||
for actual in [
|
||||
PackageState::Running,
|
||||
PackageState::Stopped,
|
||||
PackageState::Exited,
|
||||
] {
|
||||
entry.state = actual.clone();
|
||||
finish_failed_update(&mut entry);
|
||||
assert_eq!(
|
||||
entry.state, actual,
|
||||
"Fresh scanner evidence must win over old pre-update intent"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -526,7 +526,18 @@ pub(in crate::api::rpc) async fn get_containers_for_app(package_id: &str) -> Res
|
||||
.await
|
||||
.context("podman ps timed out while listing containers")?
|
||||
.context("Failed to list containers")?;
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
containers_from_list_output(package_id, &output)
|
||||
}
|
||||
|
||||
fn containers_from_list_output(
|
||||
package_id: &str,
|
||||
output: &std::process::Output,
|
||||
) -> Result<Vec<String>> {
|
||||
anyhow::ensure!(
|
||||
output.status.success(),
|
||||
"podman ps failed while listing containers"
|
||||
);
|
||||
let stdout = std::str::from_utf8(&output.stdout).context("Invalid container list response")?;
|
||||
let all: Vec<&str> = stdout.lines().filter(|s| !s.is_empty()).collect();
|
||||
|
||||
let patterns = all_container_names(package_id);
|
||||
@@ -543,6 +554,32 @@ pub(in crate::api::rpc) async fn get_containers_for_app(package_id: &str) -> Res
|
||||
mod tests {
|
||||
use super::{all_container_names, get_data_dirs_for_app, get_health_check_args};
|
||||
|
||||
#[test]
|
||||
fn failed_container_listing_is_not_an_absent_app() {
|
||||
use std::os::unix::process::ExitStatusExt;
|
||||
let mut output = std::process::Output {
|
||||
status: std::process::ExitStatus::from_raw(1 << 8),
|
||||
stdout: vec![],
|
||||
stderr: b"store unavailable".to_vec(),
|
||||
};
|
||||
assert!(super::containers_from_list_output("node-demo-music", &output).is_err());
|
||||
output.stdout = b"node-demo-music\n".to_vec();
|
||||
assert!(super::containers_from_list_output("node-demo-music", &output).is_err());
|
||||
output.status = std::process::ExitStatus::from_raw(0);
|
||||
assert_eq!(
|
||||
super::containers_from_list_output("node-demo-music", &output).unwrap(),
|
||||
vec!["node-demo-music"]
|
||||
);
|
||||
output.stdout.clear();
|
||||
assert!(
|
||||
super::containers_from_list_output("node-demo-music", &output)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
output.stdout = vec![0xff];
|
||||
assert!(super::containers_from_list_output("node-demo-music", &output).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bitcoin_variant_container_names_are_precise() {
|
||||
let core = all_container_names("bitcoin-core");
|
||||
|
||||
@@ -280,6 +280,10 @@ impl RpcHandler {
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or_else(|| anyhow::anyhow!("Missing package id"))?;
|
||||
validate_app_id(package_id)?;
|
||||
let lifecycle_guard =
|
||||
crate::container::update_transaction::Guard::acquire(&self.config.data_dir)?;
|
||||
lifecycle_guard.require_clear()?;
|
||||
lifecycle_guard.require_unheld(&super::config::all_container_names(package_id))?;
|
||||
|
||||
let docker_image = params
|
||||
.get("dockerImage")
|
||||
@@ -1901,7 +1905,10 @@ autopilot.active=false\n",
|
||||
.await
|
||||
.context("DATUM credentials are not available yet; wait for installation to finish")?;
|
||||
let password = password.trim();
|
||||
anyhow::ensure!(!password.is_empty(), "DATUM administrator password is empty");
|
||||
anyhow::ensure!(
|
||||
!password.is_empty(),
|
||||
"DATUM administrator password is empty"
|
||||
);
|
||||
return Ok(serde_json::json!({
|
||||
"title": "DATUM Gateway login",
|
||||
"description": "Use this password when DATUM asks you to unlock configuration. In Config, set your Bitcoin payout address. Point miners at this node's IP address on Stratum port 23334 (stratum+tcp://NODE-IP:23334). Gashboard connects automatically.",
|
||||
|
||||
@@ -60,6 +60,10 @@ impl RpcHandler {
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or_else(|| anyhow::anyhow!("Missing package id"))?;
|
||||
validate_app_id(package_id)?;
|
||||
let lifecycle_guard =
|
||||
crate::container::update_transaction::Guard::acquire(&self.config.data_dir)?;
|
||||
lifecycle_guard.require_clear()?;
|
||||
lifecycle_guard.require_unheld(&super::config::all_container_names(package_id))?;
|
||||
// A cuprate node that starts on a too-small disk fills it and takes
|
||||
// Archipelago down with it (no upstream pruning — see
|
||||
// dependencies::check_cuprate_disk_compatibility). Fail the start
|
||||
@@ -104,6 +108,7 @@ impl RpcHandler {
|
||||
let op_lock = app_op_lock(package_id);
|
||||
let data_dir = self.config.data_dir.clone();
|
||||
tokio::spawn(async move {
|
||||
let _lifecycle_guard = lifecycle_guard;
|
||||
let _op_guard = op_lock.lock().await;
|
||||
let result = if let Some(orchestrator) = orchestrator.as_ref() {
|
||||
do_orchestrator_package_start(orchestrator.as_ref(), &to_start).await
|
||||
@@ -126,7 +131,19 @@ impl RpcHandler {
|
||||
Err(e) => {
|
||||
tracing::error!("package.start {} failed: {:#}", package_id_owned, e);
|
||||
install_log(&format!("START FAIL: {} — {:#}", package_id_owned, e)).await;
|
||||
if let Some(prev) = pre_state {
|
||||
if e.downcast_ref::<crate::container::prod_orchestrator::StagedCleanupFailure>()
|
||||
.is_some()
|
||||
{
|
||||
// Installed is neutral and scanner-owned. Restoring the
|
||||
// prior Stopped state would conceal a failed cleanup;
|
||||
// keeping Starting would prevent scanner convergence.
|
||||
set_package_state(
|
||||
&state_manager,
|
||||
&package_id_owned,
|
||||
PackageState::Installed,
|
||||
)
|
||||
.await;
|
||||
} else if let Some(prev) = pre_state {
|
||||
set_package_state(&state_manager, &package_id_owned, prev).await;
|
||||
} else {
|
||||
warn!(
|
||||
@@ -155,6 +172,10 @@ impl RpcHandler {
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or_else(|| anyhow::anyhow!("Missing package id"))?;
|
||||
validate_app_id(package_id)?;
|
||||
let lifecycle_guard =
|
||||
crate::container::update_transaction::Guard::acquire(&self.config.data_dir)?;
|
||||
lifecycle_guard.require_clear()?;
|
||||
lifecycle_guard.require_unheld(&super::config::all_container_names(package_id))?;
|
||||
|
||||
let single_orchestrator_app =
|
||||
self.orchestrator.is_some() && uses_single_orchestrator_app(package_id);
|
||||
@@ -219,6 +240,7 @@ impl RpcHandler {
|
||||
|
||||
let op_lock = app_op_lock(package_id);
|
||||
tokio::spawn(async move {
|
||||
let _lifecycle_guard = lifecycle_guard;
|
||||
let _op_guard = op_lock.lock().await;
|
||||
let result = if let Some(orchestrator) = orchestrator.as_ref() {
|
||||
do_orchestrator_package_stop(orchestrator.as_ref(), &to_stop).await
|
||||
@@ -257,6 +279,10 @@ impl RpcHandler {
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or_else(|| anyhow::anyhow!("Missing package id"))?;
|
||||
validate_app_id(package_id)?;
|
||||
let lifecycle_guard =
|
||||
crate::container::update_transaction::Guard::acquire(&self.config.data_dir)?;
|
||||
lifecycle_guard.require_clear()?;
|
||||
lifecycle_guard.require_unheld(&super::config::all_container_names(package_id))?;
|
||||
// Restart is stop + recreate, so on a disk that shrank below the cuprate
|
||||
// minimum after install it resumes the doomed unprunable sync just like
|
||||
// start would — same gate, same "fail before clearing user-stopped /
|
||||
@@ -319,6 +345,7 @@ impl RpcHandler {
|
||||
let op_lock = app_op_lock(package_id);
|
||||
let data_dir = self.config.data_dir.clone();
|
||||
tokio::spawn(async move {
|
||||
let _lifecycle_guard = lifecycle_guard;
|
||||
let _op_guard = op_lock.lock().await;
|
||||
let result = if let Some(orchestrator) = orchestrator.as_ref() {
|
||||
do_orchestrator_package_restart(orchestrator.as_ref(), &to_restart).await
|
||||
@@ -362,6 +389,10 @@ impl RpcHandler {
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or_else(|| anyhow::anyhow!("Missing package id"))?;
|
||||
validate_app_id(package_id)?;
|
||||
let lifecycle_guard =
|
||||
crate::container::update_transaction::Guard::acquire(&self.config.data_dir)?;
|
||||
lifecycle_guard.require_clear()?;
|
||||
lifecycle_guard.require_unheld(&super::config::all_container_names(package_id))?;
|
||||
let preserve_data = params
|
||||
.get("preserve_data")
|
||||
.and_then(|v| v.as_bool())
|
||||
|
||||
@@ -4,15 +4,15 @@
|
||||
//! remove old container(s) → recreate (orchestrator-first, legacy fallback) → verify running.
|
||||
//! Data volumes are preserved (bind mounts, not stored in container).
|
||||
|
||||
use super::config::get_containers_for_app;
|
||||
use super::config::{all_container_names, get_containers_for_app};
|
||||
use super::install::install_log;
|
||||
use super::progress::parse_pull_progress;
|
||||
use super::runtime::stop_timeout_secs;
|
||||
use super::validation::validate_app_id;
|
||||
use crate::api::rpc::RpcHandler;
|
||||
use crate::container::image_versions;
|
||||
use crate::data_model::{InstallPhase, PackageState};
|
||||
use anyhow::{Context, Result};
|
||||
use std::{collections::HashSet, path::Path};
|
||||
use tokio::io::{AsyncBufReadExt, BufReader};
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
@@ -31,6 +31,9 @@ impl RpcHandler {
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or_else(|| anyhow::anyhow!("Missing package id"))?;
|
||||
validate_app_id(package_id)?;
|
||||
let lifecycle_guard =
|
||||
crate::container::update_transaction::Guard::acquire(&self.config.data_dir)?;
|
||||
lifecycle_guard.require_clear()?;
|
||||
|
||||
// An Update click must not act on an hourly cache that predates the
|
||||
// button. Fetch and verify first; failure leaves running containers alone.
|
||||
@@ -60,9 +63,24 @@ impl RpcHandler {
|
||||
let targets = pinned
|
||||
.as_ref()
|
||||
.map(|target| self.resolve_images_to_pull(package_id, target));
|
||||
// A stopped Quadlet normally removes its --rm container. Absence is
|
||||
// not an install decision: only a known single managed app with durable
|
||||
// installed AND stopped evidence may enter the recreate path.
|
||||
let known_managed =
|
||||
if should_try_orchestrator_update(package_id, self.orchestrator.is_some()) {
|
||||
self.orchestrator
|
||||
.as_ref()
|
||||
.expect("orchestrator presence checked")
|
||||
.knows_app(orchestrator_update_app_id(package_id))
|
||||
.await
|
||||
} else {
|
||||
false
|
||||
};
|
||||
let markers = UpdateMarkers::load(&self.config.data_dir, package_id).await?;
|
||||
let installed = inspect_update_images(package_id).await?;
|
||||
validate_update_presence(package_id, known_managed, !installed.is_empty(), &markers)?;
|
||||
if let Some(targets) = &targets {
|
||||
let installed = inspect_update_images(package_id).await?;
|
||||
if !update_targets_need_change(targets, &installed)? {
|
||||
if !update_targets_need_change(targets, &installed)? && !markers.stopped {
|
||||
install_log(&format!(
|
||||
"UPDATE SKIP: {} — target versions already installed",
|
||||
package_id
|
||||
@@ -111,6 +129,19 @@ impl RpcHandler {
|
||||
if let Some(orchestrator) = self.orchestrator.as_ref() {
|
||||
match orchestrator.upgrade(orchestrator_app_id).await {
|
||||
Ok(()) => {
|
||||
if let Some(image) = orchestrator
|
||||
.staged_upgrade_image(orchestrator_app_id)
|
||||
.await?
|
||||
{
|
||||
// The orchestrator proved the pinned image exists and
|
||||
// persisted its reviewed manifest. No running container
|
||||
// or healthy service is claimed for a stopped update.
|
||||
self.clear_install_progress(package_id).await;
|
||||
return Ok(serde_json::json!({
|
||||
"status": "staged", "state": "stopped",
|
||||
"package_id": package_id, "image": image,
|
||||
}));
|
||||
}
|
||||
if let Some(targets) = &targets {
|
||||
verify_update_targets(
|
||||
targets,
|
||||
@@ -188,7 +219,7 @@ impl RpcHandler {
|
||||
match preflighted_stack_update(
|
||||
&images_to_pull,
|
||||
|image| async move { self.pull_update_image(package_id, &image).await },
|
||||
|| self.execute_update(package_id, &containers, &images_to_pull),
|
||||
|| self.execute_update(package_id, &containers, &images_to_pull, &lifecycle_guard),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -212,10 +243,14 @@ impl RpcHandler {
|
||||
package_id, e
|
||||
))
|
||||
.await;
|
||||
self.rollback_update(package_id, &containers).await;
|
||||
// Transaction executor already recovered original identities or
|
||||
// returned an explicit unresolved state. Never guess/reinstall.
|
||||
self.clear_install_progress(package_id).await;
|
||||
self.clear_update_state(package_id).await;
|
||||
Err(e.context(format!("Update {} failed, rolled back", package_id)))
|
||||
Err(e.context(format!(
|
||||
"Update {} failed; see retained-runtime recovery result",
|
||||
package_id
|
||||
)))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -260,114 +295,59 @@ impl RpcHandler {
|
||||
}
|
||||
}
|
||||
|
||||
/// Images are prepared first; then stop → remove → recreate → verify.
|
||||
/// Images are prepared first. The transaction preserves original identities,
|
||||
/// creates stopped replacements, and restores the exact old states on failure.
|
||||
async fn execute_update(
|
||||
&self,
|
||||
package_id: &str,
|
||||
containers: &[String],
|
||||
images_to_pull: &[(String, String)],
|
||||
guard: &crate::container::update_transaction::Guard,
|
||||
) -> Result<()> {
|
||||
// Phase: Preparing — about to stop the running container(s) so
|
||||
// we can swap images. Fast.
|
||||
use crate::container::update_transaction::{self, Podman};
|
||||
let mut supervised = 0;
|
||||
for name in containers {
|
||||
if crate::container::quadlet::unit_exists(name).await {
|
||||
supervised += 1;
|
||||
}
|
||||
}
|
||||
anyhow::ensure!(
|
||||
supervised == 0 || supervised == containers.len(),
|
||||
"Mixed managed/unmanaged stack requires an explicit recovery plan; originals unchanged"
|
||||
);
|
||||
let managed = supervised != 0;
|
||||
anyhow::ensure!(
|
||||
!managed || package_id == "indeedhub",
|
||||
"This managed app has no qualified maintenance controller; originals unchanged"
|
||||
);
|
||||
let targets = Podman::targets(images_to_pull, !managed).await?;
|
||||
let names: HashSet<_> = containers.iter().map(String::as_str).collect();
|
||||
anyhow::ensure!(
|
||||
targets.len() == names.len()
|
||||
&& targets
|
||||
.iter()
|
||||
.all(|target| names.contains(target.name.as_str())),
|
||||
"Update target membership differs from installed stack; no containers changed"
|
||||
);
|
||||
self.set_install_phase(package_id, InstallPhase::Preparing)
|
||||
.await;
|
||||
|
||||
// 1. Graceful stop all containers (reverse order for dependencies)
|
||||
info!(
|
||||
"Update {}: stopping {} containers",
|
||||
package_id,
|
||||
containers.len()
|
||||
);
|
||||
for name in containers.iter().rev() {
|
||||
let timeout = stop_timeout_secs(name);
|
||||
info!(
|
||||
"Update {}: stopping {} (timeout: {}s)",
|
||||
package_id, name, timeout
|
||||
);
|
||||
let out = tokio::process::Command::new("podman")
|
||||
.args(["stop", "-t", timeout, name])
|
||||
.output()
|
||||
if managed {
|
||||
use crate::container::supervised_runtime::{
|
||||
load_reviewed_plans, LegacyIndeeMaintenance, SystemdSupervisor,
|
||||
};
|
||||
let plans = load_reviewed_plans(&self.config.data_dir, package_id)?;
|
||||
let adapter = SystemdSupervisor::new(
|
||||
self.config.data_dir.clone(),
|
||||
plans,
|
||||
LegacyIndeeMaintenance::new(guard)?,
|
||||
)
|
||||
.await?;
|
||||
let targets = adapter.reviewed_targets(&targets).await?;
|
||||
crate::container::supervised_update::execute(guard, package_id, &targets, &adapter)
|
||||
.await
|
||||
.context(format!("Failed to stop {}", name))?;
|
||||
if !out.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||
warn!(
|
||||
"Update {}: stop {} failed: {}",
|
||||
package_id,
|
||||
name,
|
||||
stderr.trim()
|
||||
);
|
||||
// Continue — container might already be stopped
|
||||
}
|
||||
} else {
|
||||
update_transaction::execute(guard, package_id, &targets, &Podman).await
|
||||
}
|
||||
|
||||
// 3. Remove old containers
|
||||
info!("Update {}: removing old containers", package_id);
|
||||
for name in containers {
|
||||
let out = tokio::process::Command::new("podman")
|
||||
.args(["rm", name])
|
||||
.output()
|
||||
.await
|
||||
.context(format!("Failed to remove {}", name))?;
|
||||
if !out.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||
// Force remove as fallback
|
||||
warn!(
|
||||
"Update {}: rm {} failed ({}), forcing",
|
||||
package_id,
|
||||
name,
|
||||
stderr.trim()
|
||||
);
|
||||
let _ = tokio::process::Command::new("podman")
|
||||
.args(["rm", "-f", name])
|
||||
.output()
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
// Phase: CreatingContainer — about to recreate each container.
|
||||
self.set_install_phase(package_id, InstallPhase::CreatingContainer)
|
||||
.await;
|
||||
|
||||
// 4. Recreate containers (orchestrator-first, reconcile fallback)
|
||||
info!("Update {}: recreating containers", package_id);
|
||||
for name in containers {
|
||||
self.recreate_container_for_update(package_id, name).await?;
|
||||
// Brief delay between containers for dependency initialization
|
||||
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
|
||||
}
|
||||
|
||||
// Phase: WaitingHealthy — reconcile has started every container,
|
||||
// now verifying each reached running state.
|
||||
self.set_install_phase(package_id, InstallPhase::WaitingHealthy)
|
||||
.await;
|
||||
|
||||
// 5. Verify containers reached running state
|
||||
tokio::time::sleep(std::time::Duration::from_secs(5)).await;
|
||||
for name in containers {
|
||||
let status = tokio::process::Command::new("podman")
|
||||
.args(["inspect", name, "--format", "{{.State.Status}}"])
|
||||
.output()
|
||||
.await;
|
||||
if let Ok(o) = status {
|
||||
let state = String::from_utf8_lossy(&o.stdout).trim().to_string();
|
||||
anyhow::ensure!(
|
||||
o.status.success() && state == "running",
|
||||
"Update {}: container {} is not running after recreate",
|
||||
package_id,
|
||||
name
|
||||
);
|
||||
} else {
|
||||
anyhow::bail!(
|
||||
"Update {}: cannot inspect recreated container {}",
|
||||
package_id,
|
||||
name
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
verify_update_targets(images_to_pull, &inspect_update_images(package_id).await?)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn recreate_container_for_update(
|
||||
@@ -526,61 +506,84 @@ impl RpcHandler {
|
||||
stack_images
|
||||
}
|
||||
|
||||
/// Rollback: restart old containers if they still exist.
|
||||
/// Called when update fails partway through.
|
||||
async fn rollback_update(&self, package_id: &str, containers: &[String]) {
|
||||
warn!("Rolling back update for {}", package_id);
|
||||
for name in containers {
|
||||
// Try to start — works if container still exists (wasn't removed yet)
|
||||
let out = tokio::process::Command::new("podman")
|
||||
.args(["start", name])
|
||||
.output()
|
||||
.await;
|
||||
match out {
|
||||
Ok(o) if o.status.success() => {
|
||||
info!("Rollback: restarted {}", name);
|
||||
}
|
||||
Ok(o) => {
|
||||
let stderr = String::from_utf8_lossy(&o.stderr);
|
||||
warn!("Rollback: could not restart {}: {}", name, stderr.trim());
|
||||
// Container was already removed (forward path ran `podman rm`).
|
||||
// Recreate via orchestrator-first path with legacy fallback.
|
||||
if let Err(recreate_err) =
|
||||
self.recreate_container_for_update(package_id, name).await
|
||||
{
|
||||
error!(
|
||||
"Rollback: failed to recreate {} during rollback of {}: {}",
|
||||
name, package_id, recreate_err
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Rollback: failed to restart {}: {}", name, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Clear the Updating state (used on failure/rollback).
|
||||
async fn clear_update_state(&self, package_id: &str) {
|
||||
let (mut data, _) = self.state_manager.get_snapshot().await;
|
||||
if let Some(entry) = data.package_data.get_mut(package_id) {
|
||||
// Don't overwrite state from scanner — just clear if still Updating
|
||||
if entry.state == PackageState::Updating {
|
||||
entry.state = PackageState::Stopped;
|
||||
// Unknown is not stopped: the authoritative scanner will
|
||||
// refresh actual retained runtime immediately in the wrapper.
|
||||
entry.state = PackageState::Installed;
|
||||
}
|
||||
}
|
||||
self.state_manager.update_data(data).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn inspect_update_images(package_id: &str) -> Result<Vec<(String, String)>> {
|
||||
let containers = get_containers_for_app(package_id).await?;
|
||||
#[derive(Default)]
|
||||
struct UpdateMarkers {
|
||||
installed: bool,
|
||||
stopped: bool,
|
||||
uninstalled: bool,
|
||||
}
|
||||
|
||||
impl UpdateMarkers {
|
||||
async fn load(data_dir: &Path, package_id: &str) -> Result<Self> {
|
||||
let mut names = all_container_names(package_id);
|
||||
names.push(package_id.to_string());
|
||||
names.push(orchestrator_update_app_id(package_id).to_string());
|
||||
let installed = read_update_markers(data_dir, "installed-apps.json").await?;
|
||||
let stopped = read_update_markers(data_dir, "user-stopped.json").await?;
|
||||
let uninstalled = read_update_markers(data_dir, "user-uninstalled.json").await?;
|
||||
Ok(Self {
|
||||
installed: names.iter().any(|name| installed.contains(name)),
|
||||
stopped: names.iter().any(|name| stopped.contains(name)),
|
||||
uninstalled: names.iter().any(|name| uninstalled.contains(name)),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The recovery loaders intentionally return empty on malformed state. An update
|
||||
// must instead distinguish unavailable evidence from a proven absence of an
|
||||
// uninstall decision before it can recreate a removed Quadlet container.
|
||||
async fn read_update_markers(data_dir: &Path, filename: &str) -> Result<HashSet<String>> {
|
||||
match tokio::fs::read(data_dir.join(filename)).await {
|
||||
Ok(bytes) => serde_json::from_slice(&bytes)
|
||||
.with_context(|| format!("Cannot read {filename}; update cancelled")),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(HashSet::new()),
|
||||
Err(error) => {
|
||||
Err(error).with_context(|| format!("Cannot read {filename}; update cancelled"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_update_presence(
|
||||
package_id: &str,
|
||||
known_managed: bool,
|
||||
has_containers: bool,
|
||||
markers: &UpdateMarkers,
|
||||
) -> Result<()> {
|
||||
anyhow::ensure!(
|
||||
!containers.is_empty(),
|
||||
"No containers found for {}",
|
||||
!markers.uninstalled,
|
||||
"{} was uninstalled; use an explicit install instead of update",
|
||||
package_id
|
||||
);
|
||||
anyhow::ensure!(
|
||||
has_containers || (known_managed && markers.installed && markers.stopped),
|
||||
"No containers or durable stopped installation found for {}",
|
||||
package_id
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn inspect_update_images(package_id: &str) -> Result<Vec<(String, String)>> {
|
||||
let containers = get_containers_for_app(package_id).await?;
|
||||
if containers.is_empty() {
|
||||
// The caller decides whether durable installation evidence authorizes
|
||||
// a missing runtime. Never invoke bare `podman inspect` here.
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let mut command = tokio::process::Command::new("podman");
|
||||
command.arg("inspect").args(&containers).kill_on_drop(true);
|
||||
let output = tokio::time::timeout(std::time::Duration::from_secs(30), command.output())
|
||||
@@ -626,6 +629,7 @@ fn verify_update_targets(
|
||||
targets: &[(String, String)],
|
||||
installed: &[(String, String)],
|
||||
) -> Result<()> {
|
||||
anyhow::ensure!(!targets.is_empty(), "No update targets resolved");
|
||||
for (app_id, target) in targets {
|
||||
let running = installed_image_for_target(app_id, installed).ok_or_else(|| {
|
||||
anyhow::anyhow!("Update {}: target container missing after recreate", app_id)
|
||||
@@ -651,6 +655,7 @@ fn update_targets_need_change(
|
||||
installed: &[(String, String)],
|
||||
) -> Result<bool> {
|
||||
use std::cmp::Ordering;
|
||||
anyhow::ensure!(!targets.is_empty(), "No update targets resolved");
|
||||
let mut changed = false;
|
||||
for (app_id, target) in targets {
|
||||
let running = installed_image_for_target(app_id, installed);
|
||||
@@ -771,9 +776,83 @@ mod tests {
|
||||
use super::{
|
||||
candidate_app_ids_for_container, immutable_update_image, orchestrator_update_app_id,
|
||||
should_try_orchestrator_update, update_targets_need_change, uses_legacy_update_flow,
|
||||
verify_update_targets,
|
||||
validate_update_presence, verify_update_targets, UpdateMarkers,
|
||||
};
|
||||
|
||||
#[tokio::test]
|
||||
async fn stopped_installed_managed_app_updates_after_quadlet_container_disappears() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
crate::crash_recovery::mark_installed(root.path(), "node-demo-music").await;
|
||||
crate::crash_recovery::mark_user_stopped(root.path(), "node-demo-music").await;
|
||||
let markers = UpdateMarkers::load(root.path(), "node-demo-music")
|
||||
.await
|
||||
.unwrap();
|
||||
validate_update_presence("node-demo-music", true, false, &markers).unwrap();
|
||||
let target = vec![("node-demo-music".into(), "localhost/music:2".into())];
|
||||
assert!(super::update_targets_need_change(&target, &[]).unwrap());
|
||||
// Success must still prove that upgrade actually created the target.
|
||||
assert!(verify_update_targets(&target, &[]).is_err());
|
||||
assert!(verify_update_targets(&target, &target).is_ok());
|
||||
assert!(crate::crash_recovery::load_user_stopped(root.path())
|
||||
.await
|
||||
.contains("node-demo-music"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absent_catalog_app_or_unmanaged_runtime_is_not_an_update_installation() {
|
||||
for (managed, installed, stopped) in [
|
||||
(true, false, false),
|
||||
(true, false, true),
|
||||
(true, true, false),
|
||||
(false, true, true),
|
||||
] {
|
||||
let markers = UpdateMarkers {
|
||||
installed,
|
||||
stopped,
|
||||
uninstalled: false,
|
||||
};
|
||||
assert!(validate_update_presence("optional", managed, false, &markers).is_err());
|
||||
}
|
||||
// Existing legacy containers remain updateable without modern markers.
|
||||
assert!(validate_update_presence("legacy", false, true, &UpdateMarkers::default()).is_ok());
|
||||
assert!(super::update_targets_need_change(&[], &[]).is_err());
|
||||
assert!(verify_update_targets(&[], &[]).is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn uninstall_tombstone_beats_stale_installed_and_stopped_markers() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
crate::crash_recovery::mark_installed(root.path(), "node-demo-music").await;
|
||||
crate::crash_recovery::mark_user_stopped(root.path(), "node-demo-music").await;
|
||||
crate::crash_recovery::mark_user_uninstalled(root.path(), "archy-node-demo-music").await;
|
||||
let markers = UpdateMarkers::load(root.path(), "node-demo-music")
|
||||
.await
|
||||
.unwrap();
|
||||
for present in [false, true] {
|
||||
assert!(validate_update_presence("node-demo-music", true, present, &markers).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn damaged_lifecycle_markers_cannot_authorize_recreation() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let absent = UpdateMarkers::load(root.path(), "optional").await.unwrap();
|
||||
assert!(validate_update_presence("optional", true, false, &absent).is_err());
|
||||
for name in [
|
||||
"installed-apps.json",
|
||||
"user-stopped.json",
|
||||
"user-uninstalled.json",
|
||||
] {
|
||||
tokio::fs::write(root.path().join(name), b"not-json")
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(UpdateMarkers::load(root.path(), "optional").await.is_err());
|
||||
tokio::fs::remove_file(root.path().join(name))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stack_image_failure_precedes_every_lifecycle_action() {
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
Reference in New Issue
Block a user