Add managed runtime adapter and require operation-owned write drain through update
This commit is contained in:
@@ -34,4 +34,5 @@ mod staged_update;
|
|||||||
|
|
||||||
pub(crate) mod update_transaction;
|
pub(crate) mod update_transaction;
|
||||||
|
|
||||||
|
pub(crate) mod supervised_runtime;
|
||||||
pub(crate) mod supervised_update;
|
pub(crate) mod supervised_update;
|
||||||
|
|||||||
@@ -0,0 +1,427 @@
|
|||||||
|
//! Production systemd/Podman adapter. Application write admission/drain is an
|
||||||
|
//! explicit dependency: neither process pause nor a filesystem receipt is drain.
|
||||||
|
use super::{
|
||||||
|
supervised_update::{self, PreparedTarget, RecoveryImage, Supervisor, Unit},
|
||||||
|
update_transaction::{Observed, Podman, Runtime, Target},
|
||||||
|
};
|
||||||
|
use anyhow::{Context, Result};
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
use std::{
|
||||||
|
collections::HashMap,
|
||||||
|
future::Future,
|
||||||
|
io::Write,
|
||||||
|
os::unix::fs::{MetadataExt, OpenOptionsExt},
|
||||||
|
path::{Path, PathBuf},
|
||||||
|
time::Duration,
|
||||||
|
};
|
||||||
|
|
||||||
|
pub(crate) trait DrainBarrier: Sync {
|
||||||
|
/// Persist ownership before blocking admissions. Return only after API,
|
||||||
|
/// direct uploads and workers have drained and the coherent backup finished.
|
||||||
|
fn acquire(
|
||||||
|
&self,
|
||||||
|
operation: &str,
|
||||||
|
originals: &[Unit],
|
||||||
|
) -> impl Future<Output = Result<()>> + Send;
|
||||||
|
/// Must inspect the live admission fence and same-operation ownership, not
|
||||||
|
/// merely trust an old 'backup complete' file. Keep it through snapshot/stop.
|
||||||
|
fn verify(&self, operation: &str) -> impl Future<Output = Result<()>> + Send;
|
||||||
|
/// Idempotent; an old recovery must never clear a newer operation's fence.
|
||||||
|
fn release(&self, operation: &str) -> impl Future<Output = Result<()>> + Send;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, serde::Serialize, serde::Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub(crate) struct MigrationPlan {
|
||||||
|
original_sha256: String,
|
||||||
|
prepared: PreparedTarget,
|
||||||
|
}
|
||||||
|
impl MigrationPlan {
|
||||||
|
/// The old render must be reproducible from an image-bound retained original
|
||||||
|
/// manifest. A current target catalog is not an installed-version receipt.
|
||||||
|
pub(crate) fn from_installed_render(
|
||||||
|
original: &str,
|
||||||
|
old_render: &str,
|
||||||
|
target_render: &str,
|
||||||
|
manifest: archipelago_container::AppManifest,
|
||||||
|
) -> Result<Self> {
|
||||||
|
manifest.validate()?;
|
||||||
|
let body = supervised_update::merge_reviewed_unit(original, old_render, target_render)?;
|
||||||
|
Ok(Self {
|
||||||
|
original_sha256: hex::encode(Sha256::digest(original.as_bytes())),
|
||||||
|
prepared: PreparedTarget { body, manifest },
|
||||||
|
})
|
||||||
|
}
|
||||||
|
/// Legacy overrides whose provenance cannot be reconstructed need a reviewed
|
||||||
|
/// exact-original-hash-bound migration, prepared by node administration code.
|
||||||
|
/// No browser RPC accepts unit bodies or this type.
|
||||||
|
pub(crate) fn from_reviewed_legacy(
|
||||||
|
original_sha256: &str,
|
||||||
|
body: String,
|
||||||
|
manifest: archipelago_container::AppManifest,
|
||||||
|
) -> Result<Self> {
|
||||||
|
anyhow::ensure!(
|
||||||
|
original_sha256.len() == 64 && original_sha256.bytes().all(|v| v.is_ascii_hexdigit()),
|
||||||
|
"Invalid original unit commitment"
|
||||||
|
);
|
||||||
|
manifest.validate()?;
|
||||||
|
Ok(Self {
|
||||||
|
original_sha256: original_sha256.into(),
|
||||||
|
prepared: PreparedTarget { body, manifest },
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(serde::Serialize, serde::Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
struct PlanFile {
|
||||||
|
schema: u8,
|
||||||
|
package: String,
|
||||||
|
plans: HashMap<String, MigrationPlan>,
|
||||||
|
}
|
||||||
|
/// Administrative preparation is separate from the Update RPC. The RPC never
|
||||||
|
/// accepts unit bodies, hook commands, or a browser-chosen preparation path.
|
||||||
|
pub(crate) fn load_reviewed_plans(
|
||||||
|
data_dir: &Path,
|
||||||
|
package: &str,
|
||||||
|
) -> Result<HashMap<String, MigrationPlan>> {
|
||||||
|
anyhow::ensure!(
|
||||||
|
!package.is_empty()
|
||||||
|
&& package.len() <= 128
|
||||||
|
&& package
|
||||||
|
.bytes()
|
||||||
|
.all(|v| v.is_ascii_alphanumeric() || matches!(v, b'-' | b'_')),
|
||||||
|
"Invalid managed package"
|
||||||
|
);
|
||||||
|
let dir = data_dir.join("managed-update-plans");
|
||||||
|
let directory = std::fs::symlink_metadata(&dir)
|
||||||
|
.context("No original-bound managed update plan; existing app remains unchanged")?;
|
||||||
|
anyhow::ensure!(
|
||||||
|
directory.is_dir()
|
||||||
|
&& !directory.file_type().is_symlink()
|
||||||
|
&& directory.uid() == unsafe { libc::geteuid() }
|
||||||
|
&& directory.mode() & 0o077 == 0,
|
||||||
|
"Managed update plan directory must be private and node-owned"
|
||||||
|
);
|
||||||
|
let path = dir.join(format!("{package}.json"));
|
||||||
|
let metadata = std::fs::symlink_metadata(&path)?;
|
||||||
|
anyhow::ensure!(
|
||||||
|
metadata.is_file()
|
||||||
|
&& !metadata.file_type().is_symlink()
|
||||||
|
&& metadata.uid() == unsafe { libc::geteuid() }
|
||||||
|
&& metadata.mode() & 0o077 == 0
|
||||||
|
&& metadata.len() <= 4 * 1024 * 1024,
|
||||||
|
"Managed update plan must be private and node-owned"
|
||||||
|
);
|
||||||
|
let record: PlanFile = serde_json::from_slice(&std::fs::read(path)?)?;
|
||||||
|
anyhow::ensure!(
|
||||||
|
record.schema == 1
|
||||||
|
&& record.package == package
|
||||||
|
&& !record.plans.is_empty()
|
||||||
|
&& record.plans.len() <= 32,
|
||||||
|
"Managed update plan does not match this package"
|
||||||
|
);
|
||||||
|
for plan in record.plans.values() {
|
||||||
|
anyhow::ensure!(
|
||||||
|
plan.original_sha256.len() == 64
|
||||||
|
&& plan.original_sha256.bytes().all(|v| v.is_ascii_hexdigit()),
|
||||||
|
"Invalid original unit commitment"
|
||||||
|
);
|
||||||
|
plan.prepared.manifest.validate()?;
|
||||||
|
}
|
||||||
|
Ok(record.plans)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) struct SystemdSupervisor<B> {
|
||||||
|
data_dir: PathBuf,
|
||||||
|
unit_dir: PathBuf,
|
||||||
|
plans: HashMap<String, MigrationPlan>,
|
||||||
|
barrier: B,
|
||||||
|
}
|
||||||
|
impl<B: DrainBarrier> SystemdSupervisor<B> {
|
||||||
|
pub(crate) async fn new(
|
||||||
|
data_dir: PathBuf,
|
||||||
|
plans: HashMap<String, MigrationPlan>,
|
||||||
|
barrier: B,
|
||||||
|
) -> Result<Self> {
|
||||||
|
let unit_dir = super::quadlet::unit_dir().await?;
|
||||||
|
let meta = std::fs::symlink_metadata(&unit_dir)?;
|
||||||
|
anyhow::ensure!(
|
||||||
|
meta.is_dir()
|
||||||
|
&& !meta.file_type().is_symlink()
|
||||||
|
&& meta.uid() == unsafe { libc::geteuid() }
|
||||||
|
&& meta.mode() & 0o022 == 0,
|
||||||
|
"Quadlet directory ownership changed"
|
||||||
|
);
|
||||||
|
Ok(Self {
|
||||||
|
data_dir,
|
||||||
|
unit_dir,
|
||||||
|
plans,
|
||||||
|
barrier,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
fn path(&self, name: &str) -> Result<PathBuf> {
|
||||||
|
anyhow::ensure!(
|
||||||
|
!name.is_empty()
|
||||||
|
&& name.len() <= 128
|
||||||
|
&& name
|
||||||
|
.bytes()
|
||||||
|
.all(|v| v.is_ascii_alphanumeric() || matches!(v, b'-' | b'_')),
|
||||||
|
"Invalid supervised unit name"
|
||||||
|
);
|
||||||
|
Ok(self.unit_dir.join(format!("{name}.container")))
|
||||||
|
}
|
||||||
|
async fn manager(args: &[&str]) -> Result<String> {
|
||||||
|
let output = tokio::time::timeout(
|
||||||
|
Duration::from_secs(120),
|
||||||
|
tokio::process::Command::new("systemctl")
|
||||||
|
.arg("--user")
|
||||||
|
.args(args)
|
||||||
|
.kill_on_drop(true)
|
||||||
|
.output(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.context("User service manager timed out")??;
|
||||||
|
anyhow::ensure!(
|
||||||
|
output.status.success(),
|
||||||
|
"User service manager rejected operation"
|
||||||
|
);
|
||||||
|
Ok(String::from_utf8(output.stdout)?)
|
||||||
|
}
|
||||||
|
async fn owned_file(&self, name: &str) -> Result<(PathBuf, std::fs::Metadata)> {
|
||||||
|
let expected = self.path(name)?;
|
||||||
|
let service = format!("{name}.service");
|
||||||
|
let output = Self::manager(&[
|
||||||
|
"show",
|
||||||
|
&service,
|
||||||
|
"--property=SourcePath",
|
||||||
|
"--property=DropInPaths",
|
||||||
|
"--property=LoadState",
|
||||||
|
])
|
||||||
|
.await?;
|
||||||
|
let values: HashMap<_, _> = output
|
||||||
|
.lines()
|
||||||
|
.filter_map(|line| line.split_once('='))
|
||||||
|
.collect();
|
||||||
|
anyhow::ensure!(
|
||||||
|
values.get("LoadState") == Some(&"loaded")
|
||||||
|
&& values.get("DropInPaths") == Some(&"")
|
||||||
|
&& values
|
||||||
|
.get("SourcePath")
|
||||||
|
.is_some_and(|path| Path::new(path) == expected),
|
||||||
|
"Service is not owned by the exact original source Quadlet or has external overrides"
|
||||||
|
);
|
||||||
|
let meta = std::fs::symlink_metadata(&expected)?;
|
||||||
|
anyhow::ensure!(
|
||||||
|
meta.is_file()
|
||||||
|
&& !meta.file_type().is_symlink()
|
||||||
|
&& meta.uid() == unsafe { libc::geteuid() }
|
||||||
|
&& meta.mode() & 0o022 == 0
|
||||||
|
&& meta.len() <= 1024 * 1024,
|
||||||
|
"Original Quadlet ownership changed"
|
||||||
|
);
|
||||||
|
Ok((expected, meta))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
impl<B: DrainBarrier> Supervisor for SystemdSupervisor<B> {
|
||||||
|
async fn begin_barrier(&self, operation: &str, originals: &[Unit]) -> Result<()> {
|
||||||
|
self.barrier.acquire(operation, originals).await
|
||||||
|
}
|
||||||
|
async fn verify_barrier(&self, operation: &str) -> Result<()> {
|
||||||
|
self.barrier.verify(operation).await
|
||||||
|
}
|
||||||
|
async fn release_barrier(&self, operation: &str) -> Result<()> {
|
||||||
|
self.barrier.release(operation).await
|
||||||
|
}
|
||||||
|
async fn prepare_target(&self, target: &Target, original: &Unit) -> Result<PreparedTarget> {
|
||||||
|
let plan = self
|
||||||
|
.plans
|
||||||
|
.get(&target.name)
|
||||||
|
.context("No reviewed original-bound managed migration; app unchanged")?;
|
||||||
|
anyhow::ensure!(
|
||||||
|
hex::encode(Sha256::digest(original.body.as_bytes())) == plan.original_sha256
|
||||||
|
&& plan.prepared.manifest.app.container.image.as_deref()
|
||||||
|
== Some(target.reference.as_str())
|
||||||
|
&& super::prod_orchestrator::compute_container_name(&plan.prepared.manifest)
|
||||||
|
== target.name,
|
||||||
|
"Original unit or reviewed immutable target changed before update"
|
||||||
|
);
|
||||||
|
Ok(plan.prepared.clone())
|
||||||
|
}
|
||||||
|
async fn target_hooks(
|
||||||
|
&self,
|
||||||
|
name: &str,
|
||||||
|
manifest: &archipelago_container::AppManifest,
|
||||||
|
) -> Result<()> {
|
||||||
|
super::hooks::run_post_install_strict(manifest, name, &self.data_dir).await
|
||||||
|
}
|
||||||
|
async fn capture(&self, name: &str) -> Result<Unit> {
|
||||||
|
let (path, meta) = self.owned_file(name).await?;
|
||||||
|
let body = std::fs::read_to_string(path)?;
|
||||||
|
let observed = Podman
|
||||||
|
.inspect(name)
|
||||||
|
.await?
|
||||||
|
.context("Original supervised runtime missing")?;
|
||||||
|
Ok(Unit {
|
||||||
|
name: name.into(),
|
||||||
|
body,
|
||||||
|
image: observed.image,
|
||||||
|
container_id: observed.id,
|
||||||
|
file_mode: meta.mode() & 0o777,
|
||||||
|
running: observed.running,
|
||||||
|
config_sha256: observed.config_sha256,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
async fn validate_original_file(&self, original: &Unit) -> Result<()> {
|
||||||
|
let (_, meta) = self.owned_file(&original.name).await?;
|
||||||
|
anyhow::ensure!(
|
||||||
|
meta.mode() & 0o777 == original.file_mode,
|
||||||
|
"Original Quadlet mode changed"
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
async fn read(&self, name: &str) -> Result<String> {
|
||||||
|
let (path, _) = self.owned_file(name).await?;
|
||||||
|
Ok(std::fs::read_to_string(path)?)
|
||||||
|
}
|
||||||
|
async fn write(&self, original: &Unit, expected: &[String], body: &str) -> Result<()> {
|
||||||
|
let (path, meta) = self.owned_file(&original.name).await?;
|
||||||
|
anyhow::ensure!(
|
||||||
|
meta.mode() & 0o777 == original.file_mode
|
||||||
|
&& expected.contains(&std::fs::read_to_string(&path)?),
|
||||||
|
"Original Quadlet changed; refusing to replace a foreign edit"
|
||||||
|
);
|
||||||
|
let temporary = self
|
||||||
|
.unit_dir
|
||||||
|
.join(format!(".archy-update-{}.tmp", uuid::Uuid::new_v4()));
|
||||||
|
// Commit point is synchronous while the transaction guard is held;
|
||||||
|
// cancelled futures cannot rename over a later recovery after unlock.
|
||||||
|
let result = (|| -> Result<()> {
|
||||||
|
let mut file = std::fs::OpenOptions::new()
|
||||||
|
.create_new(true)
|
||||||
|
.write(true)
|
||||||
|
.mode(original.file_mode)
|
||||||
|
.open(&temporary)?;
|
||||||
|
file.write_all(body.as_bytes())?;
|
||||||
|
file.sync_all()?;
|
||||||
|
std::fs::rename(&temporary, &path)?;
|
||||||
|
std::fs::File::open(&self.unit_dir)?.sync_all()?;
|
||||||
|
Ok(())
|
||||||
|
})();
|
||||||
|
if result.is_err() {
|
||||||
|
let _ = std::fs::remove_file(&temporary);
|
||||||
|
}
|
||||||
|
result
|
||||||
|
}
|
||||||
|
async fn snapshot(&self, original: &Unit, operation: &str, tag: &str) -> Result<RecoveryImage> {
|
||||||
|
self.barrier.verify(operation).await?;
|
||||||
|
supervised_update::capture_local_recovery_image(original, operation, tag).await
|
||||||
|
}
|
||||||
|
async fn pin(&self, image: &str, tag: &str) -> Result<()> {
|
||||||
|
anyhow::ensure!(
|
||||||
|
image.len() == 64
|
||||||
|
&& image.bytes().all(|v| v.is_ascii_hexdigit())
|
||||||
|
&& tag.starts_with("localhost/archy-update-recovery:"),
|
||||||
|
"Invalid recovery image pin"
|
||||||
|
);
|
||||||
|
let output = tokio::time::timeout(
|
||||||
|
Duration::from_secs(30),
|
||||||
|
tokio::process::Command::new("podman")
|
||||||
|
.args(["tag", &format!("sha256:{image}"), tag])
|
||||||
|
.kill_on_drop(true)
|
||||||
|
.output(),
|
||||||
|
)
|
||||||
|
.await??;
|
||||||
|
anyhow::ensure!(
|
||||||
|
output.status.success(),
|
||||||
|
"Original recovery image unavailable"
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
async fn stop(&self, name: &str) -> Result<()> {
|
||||||
|
self.owned_file(name).await?;
|
||||||
|
super::quadlet::stop_service_with_timeout(
|
||||||
|
&format!("{name}.service"),
|
||||||
|
Duration::from_secs(archipelago_container::runtime::stop_grace_secs_for(name) + 30),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
async fn reload(&self) -> Result<()> {
|
||||||
|
super::quadlet::daemon_reload_user().await
|
||||||
|
}
|
||||||
|
async fn start(&self, name: &str) -> Result<()> {
|
||||||
|
self.owned_file(name).await?;
|
||||||
|
super::quadlet::enable_now(&format!("{name}.service")).await
|
||||||
|
}
|
||||||
|
async fn observed(&self, name: &str) -> Result<Option<Observed>> {
|
||||||
|
Podman.inspect(name).await
|
||||||
|
}
|
||||||
|
async fn healthy(&self, name: &str) -> Result<bool> {
|
||||||
|
Podman.healthy(name).await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
struct UnusedBarrier;
|
||||||
|
impl DrainBarrier for UnusedBarrier {
|
||||||
|
async fn acquire(&self, _: &str, _: &[Unit]) -> Result<()> {
|
||||||
|
anyhow::bail!("No application barrier installed")
|
||||||
|
}
|
||||||
|
async fn verify(&self, _: &str) -> Result<()> {
|
||||||
|
anyhow::bail!("No application barrier installed")
|
||||||
|
}
|
||||||
|
async fn release(&self, _: &str) -> Result<()> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#[tokio::test]
|
||||||
|
async fn original_bound_plan_rejects_changed_unit_or_image_without_service_calls() {
|
||||||
|
let image = format!("localhost/movie@sha256:{}", "b".repeat(64));
|
||||||
|
let manifest = archipelago_container::AppManifest::parse(&format!(
|
||||||
|
"app:\n id: movie\n name: Movie\n version: 2.0.0\n container:\n image: {image}\n")).unwrap();
|
||||||
|
let original_body = "[Container]\nContainerName=movie\nImage=old\n";
|
||||||
|
let next_body = format!("[Container]\nContainerName=movie\nImage={image}\n");
|
||||||
|
let plan = MigrationPlan::from_reviewed_legacy(
|
||||||
|
&hex::encode(Sha256::digest(original_body)),
|
||||||
|
next_body.clone(),
|
||||||
|
manifest,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let adapter = SystemdSupervisor {
|
||||||
|
data_dir: PathBuf::from("/unused"),
|
||||||
|
unit_dir: PathBuf::from("/unused"),
|
||||||
|
plans: HashMap::from([("movie".into(), plan)]),
|
||||||
|
barrier: UnusedBarrier,
|
||||||
|
};
|
||||||
|
let mut original = Unit {
|
||||||
|
name: "movie".into(),
|
||||||
|
body: original_body.into(),
|
||||||
|
image: "a".repeat(64),
|
||||||
|
container_id: "c".repeat(64),
|
||||||
|
file_mode: 0o600,
|
||||||
|
running: true,
|
||||||
|
config_sha256: "d".repeat(64),
|
||||||
|
};
|
||||||
|
let mut target = Target {
|
||||||
|
name: "movie".into(),
|
||||||
|
reference: image,
|
||||||
|
image: "b".repeat(64),
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
adapter
|
||||||
|
.prepare_target(&target, &original)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.body,
|
||||||
|
next_body
|
||||||
|
);
|
||||||
|
original.body.push_str("Environment=OPERATOR=changed\n");
|
||||||
|
assert!(adapter.prepare_target(&target, &original).await.is_err());
|
||||||
|
original.body = original_body.into();
|
||||||
|
target.reference = format!("localhost/movie@sha256:{}", "e".repeat(64));
|
||||||
|
assert!(adapter.prepare_target(&target, &original).await.is_err());
|
||||||
|
assert!(adapter.begin_barrier("unused", &[original]).await.is_err());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -15,6 +15,7 @@ pub(crate) struct Unit {
|
|||||||
pub body: String,
|
pub body: String,
|
||||||
pub image: String,
|
pub image: String,
|
||||||
pub container_id: String,
|
pub container_id: String,
|
||||||
|
pub file_mode: u32,
|
||||||
pub running: bool,
|
pub running: bool,
|
||||||
pub config_sha256: String,
|
pub config_sha256: String,
|
||||||
}
|
}
|
||||||
@@ -59,6 +60,16 @@ struct Journal {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) trait Supervisor: Sync {
|
pub(crate) trait Supervisor: Sync {
|
||||||
|
/// Admission must be fenced and in-flight application work drained under
|
||||||
|
/// this durable operation before snapshots. A paused process is not proof.
|
||||||
|
/// Acquisition/release are idempotent and operation-owned across restart.
|
||||||
|
fn begin_barrier(
|
||||||
|
&self,
|
||||||
|
operation: &str,
|
||||||
|
originals: &[Unit],
|
||||||
|
) -> impl Future<Output = Result<()>> + Send;
|
||||||
|
fn verify_barrier(&self, operation: &str) -> impl Future<Output = Result<()>> + Send;
|
||||||
|
fn release_barrier(&self, operation: &str) -> impl Future<Output = Result<()>> + Send;
|
||||||
/// Only an internal reviewed signed-manifest planner may supply this value;
|
/// Only an internal reviewed signed-manifest planner may supply this value;
|
||||||
/// browser parameters must never become a unit body or hook recipe.
|
/// browser parameters must never become a unit body or hook recipe.
|
||||||
fn prepare_target(
|
fn prepare_target(
|
||||||
@@ -83,10 +94,11 @@ pub(crate) trait Supervisor: Sync {
|
|||||||
tag: &str,
|
tag: &str,
|
||||||
) -> impl Future<Output = Result<RecoveryImage>> + Send;
|
) -> impl Future<Output = Result<RecoveryImage>> + Send;
|
||||||
fn capture(&self, name: &str) -> impl Future<Output = Result<Unit>> + Send;
|
fn capture(&self, name: &str) -> impl Future<Output = Result<Unit>> + Send;
|
||||||
|
fn validate_original_file(&self, original: &Unit) -> impl Future<Output = Result<()>> + Send;
|
||||||
fn read(&self, name: &str) -> impl Future<Output = Result<String>> + Send;
|
fn read(&self, name: &str) -> impl Future<Output = Result<String>> + Send;
|
||||||
fn write(
|
fn write(
|
||||||
&self,
|
&self,
|
||||||
name: &str,
|
original: &Unit,
|
||||||
expected: &[String],
|
expected: &[String],
|
||||||
body: &str,
|
body: &str,
|
||||||
) -> impl Future<Output = Result<()>> + Send;
|
) -> impl Future<Output = Result<()>> + Send;
|
||||||
@@ -406,6 +418,8 @@ fn validate(record: &Journal) -> Result<()> {
|
|||||||
&& digest(&member.target.image)
|
&& digest(&member.target.image)
|
||||||
&& digest(&member.original.image)
|
&& digest(&member.original.image)
|
||||||
&& digest(&member.original.container_id)
|
&& digest(&member.original.container_id)
|
||||||
|
&& member.original.file_mode & !0o777 == 0
|
||||||
|
&& member.original.file_mode & 0o022 == 0
|
||||||
&& digest(&member.original.config_sha256),
|
&& digest(&member.original.config_sha256),
|
||||||
"Invalid original supervised identity"
|
"Invalid original supervised identity"
|
||||||
);
|
);
|
||||||
@@ -567,6 +581,10 @@ pub(crate) async fn execute(
|
|||||||
}
|
}
|
||||||
let result = apply(guard, &mut record, supervisor).await;
|
let result = apply(guard, &mut record, supervisor).await;
|
||||||
if let Err(error) = result {
|
if let Err(error) = result {
|
||||||
|
if record.phase == Phase::Committed {
|
||||||
|
return Err(error
|
||||||
|
.context("Update committed; completion cleanup must resume without rolling back"));
|
||||||
|
}
|
||||||
return match restore(guard,&mut record,supervisor).await {
|
return match restore(guard,&mut record,supervisor).await {
|
||||||
Ok(())=>Err(error.context("Original supervised image/configuration and running intent restored; container IDs may change and data was not rolled back")),
|
Ok(())=>Err(error.context("Original supervised image/configuration and running intent restored; container IDs may change and data was not rolled back")),
|
||||||
Err(recovery)=>Err(error.context(format!("Supervised runtime recovery remains unresolved: {recovery:#}"))),
|
Err(recovery)=>Err(error.context(format!("Supervised runtime recovery remains unresolved: {recovery:#}"))),
|
||||||
@@ -575,12 +593,20 @@ pub(crate) async fn execute(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
async fn apply(guard: &Guard, record: &mut Journal, supervisor: &impl Supervisor) -> Result<()> {
|
async fn apply(guard: &Guard, record: &mut Journal, supervisor: &impl Supervisor) -> Result<()> {
|
||||||
|
let originals: Vec<_> = record
|
||||||
|
.members
|
||||||
|
.iter()
|
||||||
|
.map(|member| member.original.clone())
|
||||||
|
.collect();
|
||||||
|
supervisor.begin_barrier(&record.id, &originals).await?;
|
||||||
for index in 0..record.members.len() {
|
for index in 0..record.members.len() {
|
||||||
let member = &record.members[index];
|
let member = &record.members[index];
|
||||||
|
supervisor.validate_original_file(&member.original).await?;
|
||||||
anyhow::ensure!(
|
anyhow::ensure!(
|
||||||
supervisor.read(&member.original.name).await? == member.original.body,
|
supervisor.read(&member.original.name).await? == member.original.body,
|
||||||
"Unit edited before update; originals retained"
|
"Unit edited before update; originals retained"
|
||||||
);
|
);
|
||||||
|
supervisor.verify_barrier(&record.id).await?;
|
||||||
let image = supervisor
|
let image = supervisor
|
||||||
.snapshot(&member.original, &record.id, &member.original_tag)
|
.snapshot(&member.original, &record.id, &member.original_tag)
|
||||||
.await?;
|
.await?;
|
||||||
@@ -600,15 +626,17 @@ async fn apply(guard: &Guard, record: &mut Journal, supervisor: &impl Supervisor
|
|||||||
// The image acknowledgement becomes durable before any original stop.
|
// The image acknowledgement becomes durable before any original stop.
|
||||||
save(guard, record)?;
|
save(guard, record)?;
|
||||||
}
|
}
|
||||||
|
supervisor.verify_barrier(&record.id).await?;
|
||||||
record.phase = Phase::Editing;
|
record.phase = Phase::Editing;
|
||||||
save(guard, record)?;
|
save(guard, record)?;
|
||||||
for member in record.members.iter().rev() {
|
for member in record.members.iter().rev() {
|
||||||
|
supervisor.verify_barrier(&record.id).await?;
|
||||||
supervisor.stop(&member.original.name).await?;
|
supervisor.stop(&member.original.name).await?;
|
||||||
}
|
}
|
||||||
for member in &record.members {
|
for member in &record.members {
|
||||||
supervisor
|
supervisor
|
||||||
.write(
|
.write(
|
||||||
&member.original.name,
|
&member.original,
|
||||||
&[member.original.body.clone()],
|
&[member.original.body.clone()],
|
||||||
&member.target_body,
|
&member.target_body,
|
||||||
)
|
)
|
||||||
@@ -635,6 +663,7 @@ async fn apply(guard: &Guard, record: &mut Journal, supervisor: &impl Supervisor
|
|||||||
}
|
}
|
||||||
record.phase = Phase::Committed;
|
record.phase = Phase::Committed;
|
||||||
save(guard, record)?;
|
save(guard, record)?;
|
||||||
|
supervisor.release_barrier(&record.id).await?;
|
||||||
for member in &record.members {
|
for member in &record.members {
|
||||||
guard.release_hold(&member.original.name, &record.id)?;
|
guard.release_hold(&member.original.name, &record.id)?;
|
||||||
}
|
}
|
||||||
@@ -660,13 +689,16 @@ async fn restore(guard: &Guard, record: &mut Journal, supervisor: &impl Supervis
|
|||||||
}
|
}
|
||||||
record.phase = Phase::Aborted;
|
record.phase = Phase::Aborted;
|
||||||
save(guard, record)?;
|
save(guard, record)?;
|
||||||
|
supervisor.release_barrier(&record.id).await?;
|
||||||
for member in &record.members {
|
for member in &record.members {
|
||||||
guard.release_hold(&member.original.name, &record.id)?;
|
guard.release_hold(&member.original.name, &record.id)?;
|
||||||
}
|
}
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
supervisor.verify_barrier(&record.id).await?;
|
||||||
// Refuse to overwrite a foreign edit before stopping any surviving member.
|
// Refuse to overwrite a foreign edit before stopping any surviving member.
|
||||||
for member in &record.members {
|
for member in &record.members {
|
||||||
|
supervisor.validate_original_file(&member.original).await?;
|
||||||
let body = supervisor.read(&member.original.name).await?;
|
let body = supervisor.read(&member.original.name).await?;
|
||||||
anyhow::ensure!(
|
anyhow::ensure!(
|
||||||
[
|
[
|
||||||
@@ -694,7 +726,7 @@ async fn restore(guard: &Guard, record: &mut Journal, supervisor: &impl Supervis
|
|||||||
for member in &record.members {
|
for member in &record.members {
|
||||||
supervisor
|
supervisor
|
||||||
.write(
|
.write(
|
||||||
&member.original.name,
|
&member.original,
|
||||||
&[
|
&[
|
||||||
member.original.body.clone(),
|
member.original.body.clone(),
|
||||||
member.target_body.clone(),
|
member.target_body.clone(),
|
||||||
@@ -734,17 +766,21 @@ async fn restore(guard: &Guard, record: &mut Journal, supervisor: &impl Supervis
|
|||||||
guard.hold(&member.original.name, &record.id)?;
|
guard.hold(&member.original.name, &record.id)?;
|
||||||
}
|
}
|
||||||
record.phase = Phase::Restored;
|
record.phase = Phase::Restored;
|
||||||
save(guard, record)
|
save(guard, record)?;
|
||||||
|
supervisor.release_barrier(&record.id).await
|
||||||
}
|
}
|
||||||
pub(crate) async fn recover(guard: &Guard, supervisor: &impl Supervisor) -> Result<()> {
|
pub(crate) async fn recover(guard: &Guard, supervisor: &impl Supervisor) -> Result<()> {
|
||||||
for mut record in records(guard)? {
|
for mut record in records(guard)? {
|
||||||
match record.phase {
|
match record.phase {
|
||||||
Phase::Committed | Phase::Aborted => {
|
Phase::Committed | Phase::Aborted => {
|
||||||
|
supervisor.release_barrier(&record.id).await?;
|
||||||
for member in &record.members {
|
for member in &record.members {
|
||||||
guard.release_hold(&member.original.name, &record.id)?;
|
guard.release_hold(&member.original.name, &record.id)?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Phase::Restored => {} // A later retry may own the current hold.
|
Phase::Restored => {
|
||||||
|
supervisor.release_barrier(&record.id).await?;
|
||||||
|
} // Never release a newer owner.
|
||||||
_ => restore(guard, &mut record, supervisor).await?,
|
_ => restore(guard, &mut record, supervisor).await?,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -765,6 +801,8 @@ mod tests {
|
|||||||
calls: Mutex<Vec<String>>,
|
calls: Mutex<Vec<String>>,
|
||||||
fail_new_hooks: AtomicBool,
|
fail_new_hooks: AtomicBool,
|
||||||
fail_snapshot: AtomicBool,
|
fail_snapshot: AtomicBool,
|
||||||
|
fail_barrier_release: AtomicBool,
|
||||||
|
barrier: Mutex<Option<String>>,
|
||||||
generation: AtomicUsize,
|
generation: AtomicUsize,
|
||||||
}
|
}
|
||||||
impl Mock {
|
impl Mock {
|
||||||
@@ -776,6 +814,7 @@ mod tests {
|
|||||||
body: body.clone(),
|
body: body.clone(),
|
||||||
image: "a".repeat(64),
|
image: "a".repeat(64),
|
||||||
container_id: format!("{:064x}", 1),
|
container_id: format!("{:064x}", 1),
|
||||||
|
file_mode: 0o600,
|
||||||
running: true,
|
running: true,
|
||||||
config_sha256: "c".repeat(64),
|
config_sha256: "c".repeat(64),
|
||||||
},
|
},
|
||||||
@@ -784,6 +823,8 @@ mod tests {
|
|||||||
calls: Default::default(),
|
calls: Default::default(),
|
||||||
fail_new_hooks: AtomicBool::new(false),
|
fail_new_hooks: AtomicBool::new(false),
|
||||||
fail_snapshot: AtomicBool::new(false),
|
fail_snapshot: AtomicBool::new(false),
|
||||||
|
fail_barrier_release: AtomicBool::new(false),
|
||||||
|
barrier: Mutex::new(None),
|
||||||
generation: AtomicUsize::new(1),
|
generation: AtomicUsize::new(1),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -796,6 +837,33 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
impl Supervisor for Mock {
|
impl Supervisor for Mock {
|
||||||
|
async fn begin_barrier(&self, operation: &str, _originals: &[Unit]) -> Result<()> {
|
||||||
|
let mut held = self.barrier.lock().unwrap();
|
||||||
|
anyhow::ensure!(
|
||||||
|
held.as_deref().is_none_or(|id| id == operation),
|
||||||
|
"Another drain barrier owns admission"
|
||||||
|
);
|
||||||
|
*held = Some(operation.into());
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
async fn verify_barrier(&self, operation: &str) -> Result<()> {
|
||||||
|
anyhow::ensure!(
|
||||||
|
self.barrier.lock().unwrap().as_deref() == Some(operation),
|
||||||
|
"Write barrier not held"
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
async fn release_barrier(&self, operation: &str) -> Result<()> {
|
||||||
|
anyhow::ensure!(
|
||||||
|
!self.fail_barrier_release.load(Ordering::SeqCst),
|
||||||
|
"Barrier release unavailable"
|
||||||
|
);
|
||||||
|
let mut held = self.barrier.lock().unwrap();
|
||||||
|
if held.as_deref() == Some(operation) {
|
||||||
|
*held = None;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
async fn prepare_target(
|
async fn prepare_target(
|
||||||
&self,
|
&self,
|
||||||
target: &Target,
|
target: &Target,
|
||||||
@@ -843,10 +911,13 @@ mod tests {
|
|||||||
async fn capture(&self, _name: &str) -> Result<Unit> {
|
async fn capture(&self, _name: &str) -> Result<Unit> {
|
||||||
Ok(self.original.clone())
|
Ok(self.original.clone())
|
||||||
}
|
}
|
||||||
|
async fn validate_original_file(&self, _original: &Unit) -> Result<()> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
async fn read(&self, _name: &str) -> Result<String> {
|
async fn read(&self, _name: &str) -> Result<String> {
|
||||||
Ok(self.body.lock().unwrap().clone())
|
Ok(self.body.lock().unwrap().clone())
|
||||||
}
|
}
|
||||||
async fn write(&self, _name: &str, expected: &[String], body: &str) -> Result<()> {
|
async fn write(&self, _original: &Unit, expected: &[String], body: &str) -> Result<()> {
|
||||||
let mut current = self.body.lock().unwrap();
|
let mut current = self.body.lock().unwrap();
|
||||||
anyhow::ensure!(expected.contains(&*current), "Foreign unit edit");
|
anyhow::ensure!(expected.contains(&*current), "Foreign unit edit");
|
||||||
*current = body.into();
|
*current = body.into();
|
||||||
@@ -933,6 +1004,38 @@ mod tests {
|
|||||||
assert!(merge_reviewed_unit("[Container]\nEnvironment=A=1 B=2\n", old, old).is_err());
|
assert!(merge_reviewed_unit("[Container]\nEnvironment=A=1 B=2\n", old, old).is_err());
|
||||||
}
|
}
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
async fn lost_barrier_release_after_commit_never_rolls_back_a_verified_update() {
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let guard = Guard::acquire(root.path()).unwrap();
|
||||||
|
let runtime = Mock::new();
|
||||||
|
runtime.fail_barrier_release.store(true, Ordering::SeqCst);
|
||||||
|
let error = execute(&guard, "movie", &[Mock::target()], &runtime)
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(error.to_string().contains("Update committed"));
|
||||||
|
assert_eq!(records(&guard).unwrap()[0].phase, Phase::Committed);
|
||||||
|
assert_eq!(
|
||||||
|
runtime.observed("movie").await.unwrap().unwrap().image,
|
||||||
|
"b".repeat(64)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.calls
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.filter(|call| *call == "stop-unit")
|
||||||
|
.count(),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
runtime.calls.lock().unwrap().clear();
|
||||||
|
runtime.fail_barrier_release.store(false, Ordering::SeqCst);
|
||||||
|
recover(&guard, &runtime).await.unwrap();
|
||||||
|
assert!(runtime.calls.lock().unwrap().is_empty());
|
||||||
|
assert!(runtime.barrier.lock().unwrap().is_none());
|
||||||
|
assert!(!super::super::update_transaction::is_held(root.path(), "movie").unwrap());
|
||||||
|
}
|
||||||
|
#[tokio::test]
|
||||||
async fn snapshot_failure_never_stops_or_recreates_original_runtime() {
|
async fn snapshot_failure_never_stops_or_recreates_original_runtime() {
|
||||||
let root = tempfile::tempdir().unwrap();
|
let root = tempfile::tempdir().unwrap();
|
||||||
let guard = Guard::acquire(root.path()).unwrap();
|
let guard = Guard::acquire(root.path()).unwrap();
|
||||||
@@ -1022,6 +1125,7 @@ mod tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
let mut record = records(&guard).unwrap().pop().unwrap();
|
let mut record = records(&guard).unwrap().pop().unwrap();
|
||||||
record.phase = Phase::Starting;
|
record.phase = Phase::Starting;
|
||||||
|
*runtime.barrier.lock().unwrap() = Some(record.id.clone());
|
||||||
save(&guard, &record).unwrap();
|
save(&guard, &record).unwrap();
|
||||||
runtime.calls.lock().unwrap().clear();
|
runtime.calls.lock().unwrap().clear();
|
||||||
recover(&guard, &runtime).await.unwrap();
|
recover(&guard, &runtime).await.unwrap();
|
||||||
@@ -1046,6 +1150,7 @@ mod tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
let mut record = records(&guard).unwrap().pop().unwrap();
|
let mut record = records(&guard).unwrap().pop().unwrap();
|
||||||
record.phase = Phase::Starting;
|
record.phase = Phase::Starting;
|
||||||
|
*runtime.barrier.lock().unwrap() = Some(record.id.clone());
|
||||||
save(&guard, &record).unwrap();
|
save(&guard, &record).unwrap();
|
||||||
*runtime.body.lock().unwrap() = "operator replaced unit".into();
|
*runtime.body.lock().unwrap() = "operator replaced unit".into();
|
||||||
runtime.calls.lock().unwrap().clear();
|
runtime.calls.lock().unwrap().clear();
|
||||||
|
|||||||
Reference in New Issue
Block a user