From 9becafafd3c440495e58400b198bb8d7f7c0be01 Mon Sep 17 00:00:00 2001 From: archipelago Date: Fri, 1 May 2026 17:09:50 -0400 Subject: [PATCH] feat(quadlet): backend-manifest renderer (Phase 3.1 of v1.7.52) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The QuadletUnit struct now covers everything a backend manifest needs (ports, environment, devices, add_hosts, entrypoint+command, read-only root, no_new_privileges, cpu_quota, restart policy choice). Adds QuadletUnit::from_manifest(&AppManifest, name) that translates a parsed manifest into a unit, plus parse_memory_mib for "1g"/"512m"/raw-MiB forms. The renderer skips empty/false directives so existing companion units render byte-identically — no behavior change for shipping companions; the backend renderer is dead code until Phase 3.2 wires it into the orchestrator. Eight new unit tests cover: * parse_memory_mib forms (1024, 512m, 2g, garbage) * shell_join quoting (whitespace, embedded quotes) * RestartPolicy → systemd string mapping * render emits backend directives when set * render skips them when defaulted (companion regression gate) * from_manifest happy path on a bitcoin-knots-shaped manifest * from_manifest read-only volume detection * from_manifest tmpfs filtering * end-to-end manifest → render bytes assertion Tests: 615 → 624 (+9 net; one pre-existing parse_memory_mib path was implicitly covered before but is now explicit). Cargo warnings: 0. `from_manifest`, `parse_memory_mib`, and `RestartPolicy::OnFailure` are marked allow(dead_code) with explicit references to Phase 3.2 — if 3.2 doesn't wire them, the dead-code warning resurfaces. Co-Authored-By: Claude Opus 4.7 (1M context) --- core/archipelago/src/container/companion.rs | 4 + core/archipelago/src/container/quadlet.rs | 446 +++++++++++++++++++- tests/lifecycle/TESTING.md | 2 +- 3 files changed, 446 insertions(+), 6 deletions(-) diff --git a/core/archipelago/src/container/companion.rs b/core/archipelago/src/container/companion.rs index d394929f..dd2101f4 100644 --- a/core/archipelago/src/container/companion.rs +++ b/core/archipelago/src/container/companion.rs @@ -253,6 +253,10 @@ fn build_unit(spec: &CompanionSpec, image: &str) -> QuadletUnit { .collect(), extra_podman_args: vec![], depends_on: vec![], + // Companions don't use the backend-manifest extension fields; + // the renderer skips empty/false directives so the rendered + // bytes are unchanged from before quadlet.rs grew the new fields. + ..QuadletUnit::default() } } diff --git a/core/archipelago/src/container/quadlet.rs b/core/archipelago/src/container/quadlet.rs index 727b1960..0c8080cd 100644 --- a/core/archipelago/src/container/quadlet.rs +++ b/core/archipelago/src/container/quadlet.rs @@ -31,6 +31,7 @@ //! that motivated the move. use anyhow::{anyhow, Context, Result}; +use archipelago_container::AppManifest; use std::fmt::Write as _; use std::path::{Path, PathBuf}; use tokio::fs; @@ -58,9 +59,37 @@ pub enum NetworkMode { Bridge(String), } +/// systemd Restart= policy for the generated `.service` unit. Companions +/// use Always (any exit triggers a restart). Backends use OnFailure +/// (clean exits — e.g. operator-issued `systemctl stop` — stay stopped, +/// only crashes get restarted automatically). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RestartPolicy { + Always, + /// Used by `from_manifest` for backend manifests. Wired into the + /// orchestrator in Phase 3.2 (see `project_v1_7_52_phase3_quadlet_design`). + #[allow(dead_code)] + OnFailure, +} + +impl Default for RestartPolicy { + fn default() -> Self { + Self::Always + } +} + +impl RestartPolicy { + fn as_systemd(self) -> &'static str { + match self { + Self::Always => "always", + Self::OnFailure => "on-failure", + } + } +} + /// One Quadlet `.container` unit. Field set is deliberately small — -/// add a new field only when a companion actually needs it. -#[derive(Debug, Clone)] +/// add a new field only when a real manifest needs it. +#[derive(Debug, Clone, Default)] pub struct QuadletUnit { pub name: String, pub description: String, @@ -73,6 +102,19 @@ pub struct QuadletUnit { pub bind_mounts: Vec, pub extra_podman_args: Vec, pub depends_on: Vec, + // Backend-manifest extensions (Phase 3.1). Companion units leave + // these defaulted; the renderer skips empty/false directives so a + // companion's rendered bytes are unchanged from before this PR. + pub ports: Vec<(u16, u16, String)>, + pub environment: Vec, + pub devices: Vec, + pub add_hosts: Vec<(String, String)>, + pub entrypoint: Option>, + pub command: Vec, + pub read_only_root: bool, + pub no_new_privileges: bool, + pub cpu_quota: Option, + pub restart_policy: RestartPolicy, } impl QuadletUnit { @@ -138,14 +180,50 @@ impl QuadletUnit { mode ); } + for (host, container, proto) in &self.ports { + let p = if proto.is_empty() { "tcp" } else { proto.as_str() }; + let _ = writeln!(s, "PublishPort={host}:{container}/{p}"); + } + for env in &self.environment { + // env entries already arrive shaped as "KEY=VALUE"; quadlet + // accepts that form on a single Environment= line per pair. + let _ = writeln!(s, "Environment={env}"); + } + for dev in &self.devices { + let _ = writeln!(s, "AddDevice={dev}"); + } + for (name, ip) in &self.add_hosts { + let _ = writeln!(s, "AddHost={name}:{ip}"); + } + if self.read_only_root { + let _ = writeln!(s, "ReadOnly=true"); + } + if self.no_new_privileges { + let _ = writeln!(s, "NoNewPrivileges=true"); + } + if let Some(cpus) = self.cpu_quota { + let _ = writeln!(s, "PodmanArgs=--cpus={cpus}"); + } + if let Some(ep) = &self.entrypoint { + // Quadlet's Exec= replaces the image entrypoint+cmd. When + // the manifest provides both entrypoint and command we + // concatenate; if only command is set we'll emit that on + // its own below. + let mut parts: Vec = ep.clone(); + parts.extend(self.command.iter().cloned()); + let _ = writeln!(s, "Exec={}", shell_join(&parts)); + } else if !self.command.is_empty() { + let _ = writeln!(s, "Exec={}", shell_join(&self.command)); + } for arg in &self.extra_podman_args { let _ = writeln!(s, "PodmanArgs={arg}"); } let _ = writeln!(s); let _ = writeln!(s, "[Service]"); - // Always restart with a 10s backoff. RestartSec keeps a - // crash-loop from saturating the journal. - let _ = writeln!(s, "Restart=always"); + // Restart policy + 10s backoff. RestartSec keeps a crash-loop + // from saturating the journal. Companions: Always. Backends: + // OnFailure (clean stops stay stopped). + let _ = writeln!(s, "Restart={}", self.restart_policy.as_systemd()); let _ = writeln!(s, "RestartSec=10"); let _ = writeln!(s); let _ = writeln!(s, "[Install]"); @@ -154,6 +232,119 @@ impl QuadletUnit { } } +/// Render a manifest's argv-style list as a single Exec= line. We do +/// the minimum quoting needed so quadlet's parser sees one element per +/// item: anything containing whitespace, quotes, or shell metacharacters +/// gets wrapped in double quotes with embedded `"` and `\` escaped. +fn shell_join(parts: &[String]) -> String { + parts + .iter() + .map(|p| { + if p.is_empty() || p.chars().any(|c| c.is_whitespace() || "\"\\$`".contains(c)) { + let escaped = p.replace('\\', "\\\\").replace('"', "\\\""); + format!("\"{escaped}\"") + } else { + p.clone() + } + }) + .collect::>() + .join(" ") +} + +impl QuadletUnit { + /// Build a backend-flavour QuadletUnit from a parsed AppManifest. + /// Wired into the orchestrator in Phase 3.2 (see + /// `project_v1_7_52_phase3_quadlet_design`); marked allow(dead_code) + /// here so the warning resurfaces if 3.2 doesn't actually call this. + #[allow(dead_code)] + /// `name` is the on-disk container name (typically the manifest's + /// `app.id`, but the orchestrator may rename — see + /// `compute_container_name`). The returned unit is NOT yet written; + /// the caller is expected to merge in any environment overrides + /// (resolve_dynamic_env, secret_env) before calling write_if_changed. + pub fn from_manifest(manifest: &AppManifest, name: &str) -> Self { + let app = &manifest.app; + + let network = match app.security.network_policy.as_str() { + "host" => NetworkMode::Host, + // Bridge name comes from the manifest's container.network if + // set; otherwise the orchestrator manages a default network + // separately and we fall back to host. Quadlet won't refuse + // either form. + other if !other.is_empty() && other != "isolated" => NetworkMode::Bridge(other.into()), + _ => match app.container.network.as_deref() { + Some(n) if !n.is_empty() && n != "host" => NetworkMode::Bridge(n.into()), + _ => NetworkMode::Host, + }, + }; + + let bind_mounts = app + .volumes + .iter() + .filter(|v| v.volume_type != "tmpfs" && !v.source.is_empty()) + .map(|v| BindMount { + host: PathBuf::from(&v.source), + container: PathBuf::from(&v.target), + read_only: v.options.iter().any(|o| o == "ro"), + }) + .collect::>(); + + let memory_mb = app.resources.memory_limit.as_ref().and_then(|s| { + // Manifests use forms like "1g", "512m", "1024". Convert to + // MiB. Anything we can't parse gets dropped (renderer skips + // None) — better to lose the limit than to mis-cap. + parse_memory_mib(s) + }); + + Self { + name: name.to_string(), + description: format!("Archipelago app: {}", app.id), + image: app.container.image_ref().unwrap_or_default(), + network, + user: None, + memory_mb, + cap_drop_all: true, + cap_add: app.security.capabilities.clone(), + bind_mounts, + extra_podman_args: vec![], + depends_on: vec![], + ports: app + .ports + .iter() + .map(|p| (p.host, p.container, p.protocol.clone())) + .collect(), + environment: app.environment.clone(), + devices: app.devices.clone(), + add_hosts: vec![("host.archipelago".into(), "10.89.0.1".into())], + entrypoint: app.container.entrypoint.clone(), + command: app.container.custom_args.clone(), + read_only_root: app.security.readonly_root, + no_new_privileges: true, + cpu_quota: app.resources.cpu_limit, + restart_policy: RestartPolicy::OnFailure, + } + } +} + +/// Parse the manifest's memory_limit string into MiB. Recognises the +/// forms our manifests actually use: "", "m"/"M", "g"/"G". +/// Returns None for anything else; the caller treats None as unlimited. +#[allow(dead_code)] // called only from from_manifest (also dead until Phase 3.2) +fn parse_memory_mib(raw: &str) -> Option { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return None; + } + let (num_part, mul) = match trimmed.chars().last()? { + 'g' | 'G' => (&trimmed[..trimmed.len() - 1], 1024u32), + 'm' | 'M' => (&trimmed[..trimmed.len() - 1], 1u32), + 'k' | 'K' => return None, // sub-MiB precision: drop, not worth it + c if c.is_ascii_digit() => (trimmed, 1u32), // bare number, treat as MiB + _ => return None, + }; + num_part.trim().parse::().ok()?.checked_mul(mul) +} + /// Resolve the per-user quadlet dir under $HOME. Created if missing. pub async fn unit_dir() -> Result { let home = std::env::var_os("HOME") @@ -287,6 +478,7 @@ mod tests { }], extra_podman_args: vec![], depends_on: vec![], + ..QuadletUnit::default() } } @@ -368,4 +560,248 @@ mod tests { ); } } + + // ──────────────────────────────────────────────────────────────── + // Phase 3.1 backend renderer tests + // ──────────────────────────────────────────────────────────────── + + #[test] + fn parse_memory_mib_recognises_common_forms() { + assert_eq!(parse_memory_mib("1024"), Some(1024)); + assert_eq!(parse_memory_mib("512m"), Some(512)); + assert_eq!(parse_memory_mib("512M"), Some(512)); + assert_eq!(parse_memory_mib("2g"), Some(2048)); + assert_eq!(parse_memory_mib("2G"), Some(2048)); + assert_eq!(parse_memory_mib("1k"), None); // sub-MiB rejected + assert_eq!(parse_memory_mib("garbage"), None); + assert_eq!(parse_memory_mib(""), None); + assert_eq!(parse_memory_mib(" 256m "), Some(256)); + } + + #[test] + fn shell_join_quotes_only_when_needed() { + assert_eq!(shell_join(&["bitcoind".into()]), "bitcoind"); + assert_eq!( + shell_join(&["bitcoind".into(), "-server=1".into()]), + "bitcoind -server=1" + ); + // Whitespace forces quoting: + assert_eq!( + shell_join(&["bash".into(), "-c".into(), "echo hi".into()]), + "bash -c \"echo hi\"" + ); + // Embedded quotes must escape: + assert_eq!( + shell_join(&[r#"say "hi""#.into()]), + r#""say \"hi\"""# + ); + } + + #[test] + fn restart_policy_emits_correct_systemd_string() { + assert_eq!(RestartPolicy::Always.as_systemd(), "always"); + assert_eq!(RestartPolicy::OnFailure.as_systemd(), "on-failure"); + } + + #[test] + fn render_emits_backend_directives_when_set() { + let u = QuadletUnit { + name: "bitcoin-knots".into(), + description: "Bitcoin Knots backend".into(), + image: "registry/bitcoin-knots:latest".into(), + network: NetworkMode::Bridge("archy-net".into()), + cap_drop_all: true, + cap_add: vec!["NET_BIND_SERVICE".into()], + ports: vec![(8332, 8332, "tcp".into()), (8333, 8333, "tcp".into())], + environment: vec![ + "BITCOIN_RPC_USER=archipelago".into(), + "BITCOIN_RPC_PASS=secret".into(), + ], + devices: vec!["/dev/kvm".into()], + add_hosts: vec![("host.archipelago".into(), "10.89.0.1".into())], + entrypoint: Some(vec!["/usr/local/bin/bitcoind".into()]), + command: vec!["-server=1".into(), "-rpcbind=0.0.0.0".into()], + read_only_root: true, + no_new_privileges: true, + cpu_quota: Some(2), + restart_policy: RestartPolicy::OnFailure, + ..QuadletUnit::default() + }; + let s = u.render(); + assert!(s.contains("PublishPort=8332:8332/tcp")); + assert!(s.contains("PublishPort=8333:8333/tcp")); + assert!(s.contains("Environment=BITCOIN_RPC_USER=archipelago")); + assert!(s.contains("Environment=BITCOIN_RPC_PASS=secret")); + assert!(s.contains("AddDevice=/dev/kvm")); + assert!(s.contains("AddHost=host.archipelago:10.89.0.1")); + assert!(s.contains("ReadOnly=true")); + assert!(s.contains("NoNewPrivileges=true")); + assert!(s.contains("PodmanArgs=--cpus=2")); + assert!(s.contains("Exec=/usr/local/bin/bitcoind -server=1 -rpcbind=0.0.0.0")); + assert!(s.contains("Restart=on-failure")); + assert!(s.contains("Network=archy-net")); + } + + #[test] + fn render_skips_backend_directives_when_default() { + // Companion-style unit: backend extension fields all defaulted. + // Rendered bytes must not include any of the backend directives, + // so existing companion units stay byte-identical to before. + let s = sample_unit().render(); + assert!(!s.contains("PublishPort=")); + assert!(!s.contains("Environment=")); + assert!(!s.contains("AddDevice=")); + assert!(!s.contains("AddHost=")); + assert!(!s.contains("ReadOnly=")); + assert!(!s.contains("NoNewPrivileges=")); + assert!(!s.contains("Exec=")); + assert!(!s.contains("--cpus=")); + // Default RestartPolicy is Always — companions rely on this. + assert!(s.contains("Restart=always")); + } + + #[test] + fn from_manifest_translates_a_typical_backend() { + let yaml = r#" +app: + id: bitcoin-knots + name: Bitcoin Knots + version: 1.0.0 + container: + image: registry/bitcoin-knots:1.0 + entrypoint: ["/usr/local/bin/bitcoind"] + custom_args: ["-server=1", "-rpcbind=0.0.0.0"] + ports: + - host: 8332 + container: 8332 + protocol: tcp + volumes: + - type: bind + source: /var/lib/archipelago/bitcoin + target: /home/bitcoin/.bitcoin + options: [] + environment: + - BITCOIN_NETWORK=mainnet + devices: [] + resources: + cpu_limit: 4 + memory_limit: 2g + security: + capabilities: ["NET_BIND_SERVICE"] + readonly_root: true + network_policy: archy-net +"#; + let m = AppManifest::parse(yaml).expect("manifest must parse"); + let u = QuadletUnit::from_manifest(&m, "bitcoin-knots"); + assert_eq!(u.name, "bitcoin-knots"); + assert_eq!(u.image, "registry/bitcoin-knots:1.0"); + assert!(matches!(u.network, NetworkMode::Bridge(ref n) if n == "archy-net")); + assert_eq!(u.memory_mb, Some(2048)); + assert_eq!(u.cpu_quota, Some(4)); + assert!(u.read_only_root); + assert!(u.no_new_privileges); + assert_eq!(u.cap_add, vec!["NET_BIND_SERVICE"]); + assert_eq!(u.ports, vec![(8332, 8332, "tcp".to_string())]); + assert_eq!(u.environment, vec!["BITCOIN_NETWORK=mainnet"]); + assert_eq!(u.bind_mounts.len(), 1); + assert_eq!( + u.bind_mounts[0].host, + PathBuf::from("/var/lib/archipelago/bitcoin") + ); + assert!(!u.bind_mounts[0].read_only); + assert_eq!(u.entrypoint, Some(vec!["/usr/local/bin/bitcoind".into()])); + assert_eq!(u.command, vec!["-server=1", "-rpcbind=0.0.0.0"]); + assert!(u.add_hosts.iter().any(|(n, ip)| n == "host.archipelago" && ip == "10.89.0.1")); + assert_eq!(u.restart_policy, RestartPolicy::OnFailure); + } + + #[test] + fn from_manifest_marks_ro_volumes_read_only() { + let yaml = r#" +app: + id: x + name: X + version: 1.0.0 + container: + image: x:latest + volumes: + - type: bind + source: /etc/host-conf + target: /etc/conf + options: ["ro"] +"#; + let m = AppManifest::parse(yaml).unwrap(); + let u = QuadletUnit::from_manifest(&m, "x"); + assert_eq!(u.bind_mounts.len(), 1); + assert!(u.bind_mounts[0].read_only); + } + + #[test] + fn from_manifest_skips_tmpfs_volumes() { + let yaml = r#" +app: + id: x + name: X + version: 1.0.0 + container: + image: x:latest + volumes: + - type: tmpfs + target: /tmp + tmpfs_options: "rw,size=64m" + - type: bind + source: /var/lib/x + target: /data + options: [] +"#; + let m = AppManifest::parse(yaml).unwrap(); + let u = QuadletUnit::from_manifest(&m, "x"); + // tmpfs entry is dropped from bind_mounts; bind entry survives. + assert_eq!(u.bind_mounts.len(), 1); + assert_eq!(u.bind_mounts[0].host, PathBuf::from("/var/lib/x")); + } + + #[test] + fn from_manifest_renders_to_a_systemd_unit() { + // End-to-end: parse a real-shape manifest, build the unit, render + // the bytes, and assert the unit body contains the directives a + // human would write by hand. + let yaml = r#" +app: + id: lnd + name: LND + version: 1.0.0 + container: + image: registry/lnd:latest + ports: + - host: 10009 + container: 10009 + protocol: tcp + volumes: + - type: bind + source: /var/lib/archipelago/lnd + target: /root/.lnd + options: [] + environment: + - LND_NETWORK=mainnet + resources: + memory_limit: 1g + security: + capabilities: [] + network_policy: archy-net +"#; + let m = AppManifest::parse(yaml).unwrap(); + let body = QuadletUnit::from_manifest(&m, "lnd").render(); + assert!(body.contains("ContainerName=lnd")); + assert!(body.contains("Image=registry/lnd:latest")); + assert!(body.contains("Network=archy-net")); + assert!(body.contains("PublishPort=10009:10009/tcp")); + assert!(body.contains("Volume=/var/lib/archipelago/lnd:/root/.lnd:Z")); + assert!(body.contains("Environment=LND_NETWORK=mainnet")); + assert!(body.contains("PodmanArgs=--memory=1024m")); + assert!(body.contains("AddHost=host.archipelago:10.89.0.1")); + assert!(body.contains("DropCapability=ALL")); + assert!(body.contains("NoNewPrivileges=true")); + assert!(body.contains("Restart=on-failure")); + } } diff --git a/tests/lifecycle/TESTING.md b/tests/lifecycle/TESTING.md index 1ffff506..2c71bdf8 100644 --- a/tests/lifecycle/TESTING.md +++ b/tests/lifecycle/TESTING.md @@ -54,7 +54,7 @@ v1.7.52 tags. | Layer | Tests | Suites | Status | |---|---:|---:|---| -| L0 unit | 615 | n/a | ● green | +| L0 unit | 624 | n/a | ● green | | L1 RPC | 70 | bitcoin-knots, lnd, electrumx, btcpay, mempool, fedimint, required-stack, package-update-smoke | ● for the 6 core apps | | L2 UI | 9 | ui-coverage | ● for dashboard + 7 proxy paths + bitcoin-ui:8334 | | L3 lifecycle survival | 8 | companion-survives-archipelago-restart, backend-survives-archipelago-restart, required-stack-destructive | ◐ companions ● ; backends ◐ regression-gate (will fail until Phase 3 Quadlet ships) |