lfg2025/archy
1
0
Fork 0
Code Issues 11 Pull Requests Actions Packages Projects Releases 11 Wiki Activity

fix(portainer): repair same-node Git routing with recoverable network migration

Browse Source
This commit is contained in:
archipelago
2026-09-30 09:57:25 -04:00
parent 6ac26f637c
commit eda28c4cd6
16 changed files with 856 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/archipelago/src/api/rpc/package/install.rs
+4 -26
View File
@@ -1699,32 +1699,10 @@ autopilot.active=false\n",
patch_indeedhub_nostr_provider().await;
}
// Gitea: keep it on its native host port (3001). The UI opens Gitea
// in a new tab on that direct port so absolute asset URLs must be
// rooted at the host port rather than Archipelago's /app/gitea/ path.
if package_id == "gitea" {
let _ = tokio::fs::remove_file("/etc/nginx/conf.d/gitea-iframe.conf").await;
// Set ROOT_URL to the direct launch route so links/assets stay
// anchored under the same origin Gitea is launched from.
let host_ip = &self.config.host_ip;
let _ = tokio::process::Command::new("podman")
.args(["exec", "gitea", "sh", "-c",
&format!("grep -q ROOT_URL /data/gitea/conf/app.ini && sed -i 's|ROOT_URL.*|ROOT_URL = http://{}:3001/|' /data/gitea/conf/app.ini || true", host_ip)])
.output()
.await;
// Also ensure X_FRAME_OPTIONS is empty so Gitea doesn't send the header
let _ = tokio::process::Command::new("podman")
.args(["exec", "gitea", "sh", "-c",
"grep -q X_FRAME_OPTIONS /data/gitea/conf/app.ini && sed -i 's|X_FRAME_OPTIONS.*|X_FRAME_OPTIONS =|' /data/gitea/conf/app.ini || sed -i '/^\\[security\\]/a X_FRAME_OPTIONS =' /data/gitea/conf/app.ini"])
.output()
.await;
info!(
"Gitea: ROOT_URL set to http://{}:3001/, X_FRAME_OPTIONS cleared",
host_ip
);
}
// Gitea owns its public URL and security settings in app.ini, including
// values chosen in its first-run setup. Do not rewrite operator values
// or claim success from best-effort grep/sed commands. The app gate
// fronts its declared HTTP port and handles frame headers separately.
if package_id == "nextcloud" {
let host_ip = &self.config.host_ip;
core/archipelago/src/container/migration_backup.rs
+251
View File
@@ -0,0 +1,251 @@
//! Consistent, private snapshots for declaratively opted-in network migrations.
use anyhow::{bail, Context, Result};
use archipelago_container::AppManifest;
use std::os::unix::fs::PermissionsExt;
use std::path::{Path, PathBuf};
pub fn enabled(manifest: &AppManifest) -> Result<bool> {
match manifest.app.extensions.get("backup_on_network_change") {
None => Ok(false),
Some(value) => value
.as_bool()
.context("backup_on_network_change must be boolean"),
}
}
fn relative_sources(manifest: &AppManifest, data_dir: &Path) -> Result<Vec<PathBuf>> {
let mut sources = Vec::new();
for volume in &manifest.app.volumes {
if volume.options.iter().any(|v| v == "ro") || volume.volume_type == "tmpfs" {
continue;
}
// A runtime socket is a connection, not application state.
if volume.source == "/run/user/1000/podman/podman.sock" {
continue;
}
if volume.volume_type != "bind" {
bail!("network migration backup requires bind-mounted persistent state");
}
let path = Path::new(&volume.source);
let relative = path
.strip_prefix(data_dir)
.context("network migration state must be inside the node data directory")?;
if relative.as_os_str().is_empty()
|| relative
.components()
.any(|c| !matches!(c, std::path::Component::Normal(_)))
{
bail!("invalid network migration state path");
}
sources.push(relative.to_path_buf());
}
sources.sort();
sources.dedup();
let mut roots: Vec<PathBuf> = Vec::new();
for source in sources {
if !roots.iter().any(|root| source.starts_with(root)) {
roots.push(source);
}
}
if roots.is_empty() {
bail!("network migration backup has no persistent state mounts");
}
Ok(roots)
}
/// Caller must gracefully stop the app before this function, and resume the old
/// service if it fails. No source files are changed or deleted by this operation.
pub async fn snapshot(
manifest: &AppManifest,
data_dir: &Path,
previous_unit: Option<&[u8]>,
) -> Result<PathBuf> {
let mut command = tokio::process::Command::new("podman");
command.args(["unshare", "tar"]);
snapshot_with_command(manifest, data_dir, previous_unit, command).await
}
async fn snapshot_with_command(
manifest: &AppManifest,
data_dir: &Path,
previous_unit: Option<&[u8]>,
mut command: tokio::process::Command,
) -> Result<PathBuf> {
let sources = relative_sources(manifest, data_dir)?;
let canonical_root = tokio::fs::canonicalize(data_dir).await?;
for source in &sources {
let path = data_dir.join(source);
if tokio::fs::symlink_metadata(&path)
.await?
.file_type()
.is_symlink()
{
bail!("network migration state mount is a symlink; explicit backup required");
}
let canonical = tokio::fs::canonicalize(&path).await?;
if !canonical.starts_with(&canonical_root) {
bail!("network migration state path resolves outside node data directory");
}
}
let root = data_dir.join("migration-backups");
tokio::fs::create_dir_all(&root).await?;
tokio::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)).await?;
let dir = root.join(uuid::Uuid::new_v4().to_string());
tokio::fs::create_dir(&dir).await?;
tokio::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).await?;
if let Some(unit) = previous_unit {
let path = dir.join("previous.container");
tokio::fs::write(&path, unit).await?;
tokio::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).await?;
tokio::fs::File::open(&path).await?.sync_all().await?;
}
let partial = dir.join("state.tar.partial");
let archive = dir.join("state.tar");
let output = command
.args([
"--create",
"--numeric-owner",
"--acls",
"--xattrs",
"--file",
])
.arg(&partial)
.arg("--directory")
.arg(data_dir)
.arg("--")
.args(&sources)
.output()
.await
.context("start rootless migration snapshot")?;
if !output.status.success() {
// No tar stderr in public logs: it can contain private filenames.
let _ = tokio::fs::remove_file(&partial).await;
bail!("persistent-state snapshot failed; original state was left intact");
}
tokio::fs::set_permissions(&partial, std::fs::Permissions::from_mode(0o600)).await?;
tokio::fs::File::open(&partial).await?.sync_all().await?;
tokio::fs::rename(&partial, &archive).await?;
let metadata = serde_json::json!({"app": manifest.app.id, "version": manifest.app.version,
"network": manifest.app.container.network, "sources": sources});
tokio::fs::write(
dir.join("metadata.json"),
serde_json::to_vec_pretty(&metadata)?,
)
.await?;
tokio::fs::File::open(&dir).await?.sync_all().await?;
Ok(archive)
}
#[cfg(test)]
mod tests {
use super::*;
fn portainer() -> AppManifest {
AppManifest::parse(include_str!("../../../../apps/portainer/manifest.yml")).unwrap()
}
#[tokio::test]
async fn stopped_state_archive_round_trips_database_compose_and_old_unit() {
let dir = tempfile::tempdir().unwrap();
let state = dir.path().join("portainer");
tokio::fs::create_dir_all(state.join("compose"))
.await
.unwrap();
tokio::fs::write(state.join("portainer.db"), b"fixture database")
.await
.unwrap();
tokio::fs::write(state.join("compose/stack.yml"), b"services: {}\n")
.await
.unwrap();
let mut m = portainer();
m.app.volumes[0].source = state.display().to_string();
m.app.volumes[1].source = state.join("compose").display().to_string();
let archive = snapshot_with_command(
&m,
dir.path(),
Some(b"old unit"),
tokio::process::Command::new("tar"),
)
.await
.unwrap();
assert_eq!(
std::fs::metadata(&archive).unwrap().permissions().mode() & 0o777,
0o600
);
assert_eq!(
tokio::fs::read(archive.parent().unwrap().join("previous.container"))
.await
.unwrap(),
b"old unit"
);
let restored = tempfile::tempdir().unwrap();
assert!(tokio::process::Command::new("tar")
.arg("-xf")
.arg(archive)
.arg("-C")
.arg(restored.path())
.status()
.await
.unwrap()
.success());
assert_eq!(
tokio::fs::read(restored.path().join("portainer/portainer.db"))
.await
.unwrap(),
b"fixture database"
);
assert_eq!(
tokio::fs::read(restored.path().join("portainer/compose/stack.yml"))
.await
.unwrap(),
b"services: {}\n"
);
assert_eq!(
tokio::fs::read(state.join("portainer.db")).await.unwrap(),
b"fixture database"
);
}
#[tokio::test]
async fn failed_snapshot_never_publishes_archive_or_changes_original_state() {
let dir = tempfile::tempdir().unwrap();
let state = dir.path().join("portainer");
tokio::fs::create_dir_all(state.join("compose"))
.await
.unwrap();
tokio::fs::write(state.join("portainer.db"), b"unchanged")
.await
.unwrap();
let mut m = portainer();
m.app.volumes[0].source = state.display().to_string();
m.app.volumes[1].source = state.join("compose").display().to_string();
assert!(
snapshot_with_command(&m, dir.path(), None, tokio::process::Command::new("false"))
.await
.is_err()
);
assert_eq!(
tokio::fs::read(state.join("portainer.db")).await.unwrap(),
b"unchanged"
);
for entry in std::fs::read_dir(dir.path().join("migration-backups")).unwrap() {
assert!(!entry.unwrap().path().join("state.tar").exists());
}
}
#[test]
fn backup_covers_all_portainer_state_once_and_excludes_runtime_socket() {
let m = portainer();
assert!(enabled(&m).unwrap());
assert_eq!(
relative_sources(&m, Path::new("/var/lib/archipelago")).unwrap(),
vec![PathBuf::from("portainer")]
);
}
#[test]
fn backup_refuses_unknown_state_locations_instead_of_silently_omitting_them() {
let mut m = portainer();
m.app.volumes[0].source = "/other/operator/state".into();
assert!(relative_sources(&m, Path::new("/var/lib/archipelago")).is_err());
m.app.volumes[0].source = "/var/lib/archipelago/../secret".into();
assert!(relative_sources(&m, Path::new("/var/lib/archipelago")).is_err());
}
}
core/archipelago/src/container/mod.rs
+1
View File
@@ -12,6 +12,7 @@ pub mod hooks;
pub mod image_policy;
pub mod image_versions;
pub mod lnd;
pub mod migration_backup;
pub mod prod_orchestrator;
pub mod quadlet;
pub mod registry;
core/archipelago/src/container/prod_orchestrator.rs
+122 -35
View File
@@ -91,6 +91,14 @@ fn is_builtin_network_mode(network: &str) -> bool {
)
}
// Only an explicitly selected rootless mode establishes drift. An omitted
// network delegates to Podman and must not recreate unrelated installed apps.
fn rootless_network_mode_drifted(expected: Option<&str>, actual: &str) -> bool {
matches!(expected, Some("slirp4netns" | "pasta"))
&& !actual.trim().is_empty()
&& actual.trim().split(':').next() != expected
}
fn uses_pasta_network(manifest: &AppManifest) -> bool {
manifest.app.container.network.as_deref() == Some("pasta")
}
@@ -2499,6 +2507,7 @@ impl ProdContainerOrchestrator {
return Ok(ReconcileAction::NoOp);
}
tracing::info!(app_id = %app_id, container = %name, "container env drift detected — recreating");
self.backup_network_change(&name, &resolved_manifest).await?;
let _ = self.runtime.stop_container(&name).await;
let _ = self.runtime.remove_container(&name).await;
self.install_fresh(lm).await?;
@@ -2555,6 +2564,7 @@ impl ProdContainerOrchestrator {
.await
{
tracing::info!(app_id = %app_id, container = %name, "stopped container env/port drift detected — recreating");
self.backup_network_change(&name, &resolved_manifest).await?;
let _ = self.runtime.remove_container(&name).await;
self.install_fresh(lm).await?;
return Ok(ReconcileAction::Installed);
@@ -3080,13 +3090,9 @@ impl ProdContainerOrchestrator {
/// app is a companion (companion.rs owns those units), or when no
/// unit file exists yet (install_via_quadlet handles first-write).
///
/// We DON'T restart the .service when content changes — running
/// containers keep their current config until an operator-initiated
/// restart picks up the new file. That's the right tradeoff: file
/// updates are cheap and non-destructive; service restarts are
/// destructive (the SIGKILL cascade we're trying to eliminate).
/// systemctl --user daemon-reload runs only when content actually
/// changed, so steady-state reconcile ticks pay just one fs read.
/// Ordinary metadata changes wait for an operator restart. Runtime-affecting
/// changes restart the service and retain a durable pending marker until
/// that succeeds, including across daemon restarts and failed reloads.
async fn sync_quadlet_unit(&self, lm: &LoadedManifest, name: &str) -> Result<()> {
// Companions: same reasoning as migrate_to_quadlet_if_needed —
// companion.rs renders these units with a different shape, syncing
@@ -3106,7 +3112,7 @@ impl ProdContainerOrchestrator {
}
let old_body = tokio::fs::read_to_string(&unit_path)
.await
.unwrap_or_default();
.with_context(|| format!("read existing quadlet for {name}"))?;
let restart_required = quadlet::contains_stale_health_gate(&old_body);
let mut resolved = lm.manifest.clone();
@@ -3122,49 +3128,47 @@ impl ProdContainerOrchestrator {
quadlet::network_aliases_changed(&old_body, &new_body);
let restart_for_exec_change = quadlet::exec_changed(&old_body, &new_body);
let restart_for_health_change = quadlet::health_cmd_changed(&old_body, &new_body);
let needs_restart = restart_required
|| restart_for_port_change
|| restart_for_network_alias_change
|| restart_for_exec_change
|| restart_for_health_change;
// Record the obligation BEFORE replacing the unit. A failed reload or
// restart must not become a no-op on the next tick just because the
// generated file already matches the manifest.
let pending = quadlet::RestartObligation::prepare(&unit_path, needs_restart).await?;
if pending.is_pending() {
self.ensure_resolved_source_available(lm).await?;
}
if restart_for_network_alias_change {
self.backup_network_change(name, &resolved).await?;
}
let changed = quadlet::write_if_changed(&unit, &unit_dir)
.await
.with_context(|| format!("drift-sync quadlet unit for {name}"))?;
if changed {
if changed || pending.is_pending() {
quadlet::daemon_reload_user()
.await
.context("systemctl --user daemon-reload after drift-syncing quadlet unit")?;
tracing::info!(
app_id = %lm.manifest.app.id,
container = %name,
"Quadlet unit drift-synced — file rewritten, .service NOT restarted (operator restart picks up new config)"
);
}
if changed
&& (restart_required
|| restart_for_port_change
|| restart_for_network_alias_change
|| restart_for_exec_change
|| restart_for_health_change)
{
self.ensure_resolved_source_available(lm).await?;
if pending.is_pending() {
let service = unit.service_name();
let reason = if restart_required {
"stale health gate"
} else if restart_for_port_change {
"port binding drift"
} else if restart_for_network_alias_change {
"network alias drift"
} else if restart_for_health_change {
"health command drift"
} else {
"exec drift"
};
tracing::info!(
app_id = %lm.manifest.app.id,
container = %name,
service = %service,
reason = reason,
"Quadlet unit rewrite requires service restart"
"Applying pending Quadlet runtime change"
);
quadlet::restart_service(&service)
.await
.with_context(|| format!("restart drifted quadlet service {service}"))?;
pending.complete().await?;
} else if changed {
tracing::info!(
app_id = %lm.manifest.app.id,
container = %name,
"Quadlet metadata updated; operator restart will apply it"
);
}
Ok(())
}
@@ -3866,6 +3870,61 @@ impl ProdContainerOrchestrator {
Ok(())
}
async fn backup_network_change(&self, name: &str, manifest: &AppManifest) -> Result<()> {
if !crate::container::migration_backup::enabled(manifest)? {
return Ok(());
}
// Only back up an actual network migration, not ordinary env drift.
let output = tokio::process::Command::new("podman")
.args(["inspect", name, "--format", "{{.HostConfig.NetworkMode}}"])
.output().await.context("inspect network before migration backup")?;
let present = if output.status.success() {
if !rootless_network_mode_drifted(manifest.app.container.network.as_deref(), &String::from_utf8_lossy(&output.stdout)) {
return Ok(());
}
true
} else {
// A crash after gracefully stopping a --rm Quadlet container can
// leave only its data and old unit. Prove absence before snapshotting
// stopped state; an inspect/Podman failure is not proof of absence.
let exists = tokio::process::Command::new("podman")
.args(["container", "exists", name]).status().await?;
if exists.code() != Some(1) {
anyhow::bail!("cannot verify existing container before network migration backup");
}
false
};
let service = format!("{name}.service");
let managed = quadlet::unit_exists(name).await;
let previous_unit = if managed {
Some(tokio::fs::read(quadlet::unit_dir().await?.join(format!("{name}.container"))).await?)
} else {
None
};
if managed {
quadlet::stop_service(&service).await?;
} else if present {
self.runtime.stop_container(name).await?;
}
match crate::container::migration_backup::snapshot(manifest, &self.data_dir, previous_unit.as_deref()).await {
Ok(archive) => {
tracing::info!(container = %name, backup = %archive.display(), "Persistent state saved before network migration");
Ok(())
}
Err(error) => {
// The unit has not been rewritten yet. Restore its previous
// service on backup failure and report the migration failure.
let restored = if managed {
quadlet::enable_now(&service).await
} else {
self.runtime.start_container(name).await
};
restored.context("restore original app after failed migration snapshot")?;
Err(error)
}
}
}
async fn container_env_drifted(&self, name: &str, manifest: &AppManifest) -> bool {
if cfg!(test) {
return false;
@@ -3875,6 +3934,23 @@ impl ProdContainerOrchestrator {
return true;
}
// Quadlet handles declarative Network= drift above. Legacy rootless
// Podman containers need the same convergence when no unit owns them.
if matches!(manifest.app.container.network.as_deref(), Some("slirp4netns" | "pasta")) {
if let Ok(output) = tokio::process::Command::new("podman")
.args(["inspect", name, "--format", "{{.HostConfig.NetworkMode}}"])
.output()
.await
{
if output.status.success() && rootless_network_mode_drifted(
manifest.app.container.network.as_deref(),
&String::from_utf8_lossy(&output.stdout),
) {
return true;
}
}
}
let inspect = tokio::process::Command::new("podman")
.args([
"inspect",
@@ -4917,6 +4993,17 @@ mod tests {
/// recovered when its siblings have live containers (the stack is
/// installed), and left alone when the whole stack is gone or the app
/// is not a stack member at all.
#[test]
fn explicit_rootless_network_change_converges_without_guessing_defaults() {
assert!(rootless_network_mode_drifted(Some("slirp4netns"), "pasta"));
assert!(rootless_network_mode_drifted(Some("slirp4netns"), "bridge"));
assert!(!rootless_network_mode_drifted(Some("slirp4netns"), "slirp4netns"));
assert!(!rootless_network_mode_drifted(Some("slirp4netns"), "slirp4netns:allow_host_loopback=true"));
assert!(!rootless_network_mode_drifted(None, "pasta"));
assert!(!rootless_network_mode_drifted(Some("slirp4netns"), ""));
assert!(!rootless_network_mode_drifted(Some("archy-net"), "bridge"));
}
#[test]
fn absent_stack_member_recovery_requires_a_live_sibling() {
let present: HashSet<String> = ["indeedhub-redis", "indeedhub-relay", "indeedhub"]
core/archipelago/src/container/quadlet.rs
+98
View File
@@ -938,6 +938,53 @@ pub fn health_cmd_changed(old_body: &str, new_body: &str) -> bool {
!= directive_values(new_body, "HealthRetries=")
}
/// A unit rewrite and a successful systemd restart are separate operations.
/// Keep the restart obligation across errors or a management-daemon restart.
pub struct RestartObligation {
marker: PathBuf,
pending: bool,
}
impl RestartObligation {
pub async fn prepare(unit_path: &Path, newly_required: bool) -> Result<Self> {
let marker = unit_path.with_extension("restart-pending");
if newly_required {
// Contents contain no manifest environment or credentials. sync_all
// makes the obligation durable before the subsequent unit rename.
let file = tokio::fs::OpenOptions::new()
.write(true)
.create(true)
.truncate(false)
.open(&marker)
.await
.context("record pending Quadlet restart")?;
file.sync_all().await?;
if let Some(parent) = marker.parent() {
tokio::fs::File::open(parent).await?.sync_all().await?;
}
}
let pending = tokio::fs::try_exists(&marker).await?;
Ok(Self { marker, pending })
}
pub fn is_pending(&self) -> bool {
self.pending
}
/// Call only after systemd accepted the replacement service successfully.
pub async fn complete(self) -> Result<()> {
if self.pending {
tokio::fs::remove_file(&self.marker)
.await
.context("clear completed Quadlet restart")?;
if let Some(parent) = self.marker.parent() {
tokio::fs::File::open(parent).await?.sync_all().await?;
}
}
Ok(())
}
}
pub fn publish_ports_changed(old_body: &str, new_body: &str) -> bool {
let old_ports = directive_values(old_body, "PublishPort=");
let new_ports = directive_values(new_body, "PublishPort=");
@@ -1541,6 +1588,28 @@ app:
assert!(!s.contains("Network=host"));
}
#[test]
fn portainer_catalog_network_repairs_same_node_routing_without_exposing_backend() {
let manifest = AppManifest::parse(include_str!(
"../../../../apps/portainer/manifest.yml"
))
.expect("shipped Portainer manifest must parse");
let new = QuadletUnit::from_manifest(&manifest, "portainer").render();
assert!(new.contains("Network=slirp4netns\n"));
assert!(!new.contains("NetworkAlias="));
assert!(new.contains("PublishPort=127.0.0.1:9000:9000/tcp"));
assert!(!new.contains("PublishPort=0.0.0.0"));
// The upgrade changes networking only: retain both state mounts and the
// existing rootless socket, without an app.ini or repository rewrite.
assert!(new.contains("Volume=/var/lib/archipelago/portainer:/data"));
assert!(new.contains("Volume=/var/lib/archipelago/portainer/compose:/data/compose"));
assert!(new.contains("Volume=/run/user/1000/podman/podman.sock:/var/run/docker.sock"));
let old = new.replace("Network=slirp4netns\n", "");
assert!(network_aliases_changed(&old, &new));
assert!(!network_aliases_changed(&new, &new));
assert!(!publish_ports_changed(&old, &new));
}
#[test]
fn from_manifest_slirp4netns_omits_network_alias() {
let yaml = r#"
@@ -1891,6 +1960,35 @@ app:
assert!(!network_aliases_changed(new, new));
}
#[tokio::test]
async fn failed_runtime_change_remains_pending_when_unit_already_matches() {
let dir = tempfile::tempdir().unwrap();
let unit = dir.path().join("portainer.container");
tokio::fs::write(&unit, "[Container]\n").await.unwrap();
let pending = RestartObligation::prepare(&unit, true).await.unwrap();
assert!(pending.is_pending());
tokio::fs::write(&unit, "[Container]\nNetwork=slirp4netns\n")
.await
.unwrap();
// Simulate systemctl failure or daemon interruption after unit rewrite.
drop(pending);
let retry = RestartObligation::prepare(&unit, false).await.unwrap();
assert!(retry.is_pending(), "matching unit must not discard failed restart");
retry.complete().await.unwrap();
assert!(!RestartObligation::prepare(&unit, false).await.unwrap().is_pending());
}
#[tokio::test]
async fn pending_runtime_change_errors_are_not_reported_as_success() {
let dir = tempfile::tempdir().unwrap();
let missing = dir.path().join("missing/app.container");
assert!(RestartObligation::prepare(&missing, true).await.is_err());
let unit = dir.path().join("app.container");
let pending = RestartObligation::prepare(&unit, true).await.unwrap();
tokio::fs::remove_file(unit.with_extension("restart-pending")).await.unwrap();
assert!(pending.complete().await.is_err());
}
#[test]
fn network_aliases_changed_detects_network_mode_drift() {
let old = "[Container]\nNetwork=slirp4netns\n";