diff --git a/core/archipelago/src/container/app_catalog.rs b/core/archipelago/src/container/app_catalog.rs index b32980d0..3c47ed71 100644 --- a/core/archipelago/src/container/app_catalog.rs +++ b/core/archipelago/src/container/app_catalog.rs @@ -86,6 +86,15 @@ pub struct AppCatalogEntry { /// Optional human-readable changelog lines for this version. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub changelog: Vec, + /// Full app manifest, embedded so the app installs from the registry alone — + /// no OTA-shipped `apps//manifest.yml`. Carried as the raw value the + /// publisher signed (so it stays part of the verified preimage) and + /// deserialized into an `AppManifest` by the orchestrator at load time, where + /// it overrides the disk manifest (origin-wins). Absent during the migration + /// window => the node falls back to the disk manifest. See + /// `docs/registry-manifest-design.md`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub manifest: Option, } /// Read-side cache file search order. Mirrors `image_versions.rs`: the running @@ -166,6 +175,18 @@ pub fn catalog_stack_images(app_id: &str) -> HashMap { entry_for(app_id).and_then(|e| e.images).unwrap_or_default() } +/// All `(app_id, manifest-value)` pairs the registry catalog carries. The +/// orchestrator deserializes + validates each into an `AppManifest` and prefers +/// it over the disk manifest (origin-wins); disk remains the migration fallback. +/// Empty when the catalog is absent or no entry embeds a manifest. +pub fn catalog_manifest_values() -> Vec<(String, serde_json::Value)> { + load_catalog() + .apps + .into_iter() + .filter_map(|(id, e)| e.manifest.map(|m| (id, m))) + .collect() +} + /// Image override for the orchestrator's install/upgrade path. Returns the /// catalog's primary image for `app_id` ONLY when it refers to the same /// repository as the manifest's current image — a guard so a catalog typo can @@ -346,6 +367,30 @@ mod tests { assert_eq!(e.digest.as_deref(), Some("blake3:deadbeef")); } + #[test] + fn entry_carries_embedded_manifest() { + let json = r#"{ + "schema": 1, + "apps": { + "demo": { + "version": "1.0.0", + "manifest": { + "app": { + "id": "demo", + "name": "Demo", + "version": "1.0.0", + "container": { "image": "registry/demo:1.0.0" } + } + } + } + } + }"#; + let cat: AppCatalog = serde_json::from_str(json).unwrap(); + let e = cat.apps.get("demo").unwrap(); + let m = e.manifest.as_ref().expect("manifest present"); + assert_eq!(m["app"]["id"], "demo"); + } + #[test] fn empty_catalog_when_absent_is_default() { let cat = AppCatalog::default(); diff --git a/core/archipelago/src/container/prod_orchestrator.rs b/core/archipelago/src/container/prod_orchestrator.rs index 2c50d6d3..080b20b5 100644 --- a/core/archipelago/src/container/prod_orchestrator.rs +++ b/core/archipelago/src/container/prod_orchestrator.rs @@ -909,6 +909,39 @@ struct LoadedManifest { manifest_dir: PathBuf, } +/// Validate a catalog-carried manifest value for `app_id`, returning the +/// `AppManifest` to overlay over the disk manifest, or `None` to keep the disk +/// fallback. Returns `None` on: an unparseable value, an embedded app id that +/// mismatches the catalog key, a manifest that fails `validate()`, or a build +/// source (build contexts aren't registry-distributed yet — phase 1 is +/// image-only). See `docs/registry-manifest-design.md`. +fn catalog_manifest_to_overlay(app_id: &str, value: serde_json::Value) -> Option { + let m: AppManifest = match serde_json::from_value(value) { + Ok(m) => m, + Err(e) => { + tracing::warn!(app = %app_id, error = %e, + "skipping unparseable catalog manifest; using disk fallback"); + return None; + } + }; + if m.app.id != app_id { + tracing::warn!(catalog_id = %app_id, manifest_id = %m.app.id, + "skipping catalog manifest: embedded app id mismatches catalog key"); + return None; + } + if let Err(e) = m.validate() { + tracing::warn!(app = %app_id, error = %e, + "skipping invalid catalog manifest; using disk fallback"); + return None; + } + if m.app.container.build.is_some() { + tracing::debug!(app = %app_id, + "catalog manifest has a build source; deferring to disk (phase 1 = image-only)"); + return None; + } + Some(m) +} + struct OrchestratorState { /// app_id → loaded manifest manifests: HashMap, @@ -1139,7 +1172,35 @@ impl ProdContainerOrchestrator { } state.manifests.insert(lm.manifest.app.id.clone(), lm); } - Ok(count) + + // Registry-distributed manifests (workstream B): the signed catalog may + // carry full manifests. Overlay them over disk — the registry is the + // authoritative origin; disk is the migration fallback. Image-only apps + // (phase 1); build-source catalog manifests defer to disk. + // See docs/registry-manifest-design.md. + let _ = count; // disk count subsumed by the merged total below + let mut overlaid = 0usize; + for (app_id, value) in crate::container::app_catalog::catalog_manifest_values() { + if let Some(m) = catalog_manifest_to_overlay(&app_id, value) { + // Reuse the disk dir when the app also exists on disk (so a future + // build-source catalog manifest can still resolve its context); + // otherwise a sentinel under the manifests dir. Image-only apps + // never read manifest_dir. + let manifest_dir = state + .manifests + .get(&app_id) + .map(|lm| lm.manifest_dir.clone()) + .unwrap_or_else(|| root.join(&app_id)); + state + .manifests + .insert(app_id.clone(), LoadedManifest { manifest: m, manifest_dir }); + overlaid += 1; + } + } + if overlaid > 0 { + tracing::info!("registry catalog overlaid {overlaid} manifest(s) over disk"); + } + Ok(state.manifests.len()) } /// Test helper: inject a manifest directly without touching the filesystem. @@ -3665,6 +3726,41 @@ app: assert!(required.contains("archy-nbxplorer")); } + #[test] + fn catalog_overlay_accepts_valid_image_manifest() { + let v = serde_json::to_value(pull_manifest("demo", "registry/demo:1.0.0")).unwrap(); + let m = catalog_manifest_to_overlay("demo", v).expect("valid image manifest accepted"); + assert_eq!(m.app.id, "demo"); + } + + #[test] + fn catalog_overlay_rejects_app_id_mismatch() { + let v = serde_json::to_value(pull_manifest("demo", "registry/demo:1.0.0")).unwrap(); + assert!(catalog_manifest_to_overlay("other", v).is_none()); + } + + #[test] + fn catalog_overlay_defers_build_source_to_disk() { + // Build contexts aren't registry-distributed yet (phase 1 = image-only). + let v = serde_json::to_value(build_manifest("demo", "ctx", "demo:1.0.0")).unwrap(); + assert!(catalog_manifest_to_overlay("demo", v).is_none()); + } + + #[test] + fn catalog_overlay_rejects_invalid_manifest() { + // Deserializes (image and build both absent) but fails validate(). + let v = serde_json::json!({ + "app": { "id": "demo", "name": "demo", "version": "1.0.0", "container": {} } + }); + assert!(catalog_manifest_to_overlay("demo", v).is_none()); + } + + #[test] + fn catalog_overlay_rejects_unparseable_value() { + let v = serde_json::json!({ "not": "a manifest" }); + assert!(catalog_manifest_to_overlay("demo", v).is_none()); + } + fn manifest_with_container_name(id: &str, image: &str, name: &str) -> AppManifest { let yaml = format!( "app:\n id: {id}\n name: {id}\n version: 1.0.0\n container_name: {name}\n container:\n image: {image}\n" diff --git a/core/archipelago/src/main.rs b/core/archipelago/src/main.rs index 3ca0aba4..c49e72eb 100644 --- a/core/archipelago/src/main.rs +++ b/core/archipelago/src/main.rs @@ -201,7 +201,7 @@ async fn main() -> Result<()> { // Best-effort manifest load; a missing /opt/archipelago/apps is // logged inside load_manifests and not fatal. match prod.load_manifests().await { - Ok(n) => info!("📦 Loaded {n} app manifest(s) from disk"), + Ok(n) => info!("📦 Loaded {n} app manifest(s) (disk + registry catalog)"), Err(e) => { tracing::error!(error = %e, "prod orchestrator: load_manifests failed at startup"); } diff --git a/docs/registry-manifest-design.md b/docs/registry-manifest-design.md index 3e8f6adb..324dad19 100644 --- a/docs/registry-manifest-design.md +++ b/docs/registry-manifest-design.md @@ -83,17 +83,22 @@ Extend (not replace) the disk walk: 4. A catalog manifest that fails parse/validate is logged and skipped → disk fallback used (one bad entry never blocks the fleet, same as the disk walk). -### `manifest_dir` for registry manifests +### `manifest_dir` for registry manifests — IMPLEMENTED -`LoadedManifest.manifest_dir` is used for **build contexts** and **generated files** -(`GeneratedFile`) that live next to a disk manifest. Registry manifests have no dir. +`LoadedManifest.manifest_dir` is used **only** in the `ResolvedSource::Build` branch +(relative `container.build.context` resolution — two call sites). Image-only apps +(`ResolvedSource::Pull`) never read it. -- **Phase 1 scope = image-only apps** (no `build:`, no `generated_files:` needing a - source dir). immich, grafana, the fedimint apps, postgres/redis all qualify. -- Represent the absent dir explicitly: `manifest_dir: Option` (or a - sentinel under `/registry-apps//` materialized on demand). - Companion build apps (bitcoin-ui, …) keep their disk path until a later phase - teaches the catalog to carry build contexts (content-addressed, per the DHT plan). +**Decision (phase 1, shipped):** keep `manifest_dir: PathBuf` (no `Option` ripple +through the codebase). A catalog manifest with a **build source is skipped** so its +disk manifest stays in effect — build contexts aren't registry-distributed until a +later phase (content-addressed, per the DHT plan). For an accepted (image-only) +catalog manifest, `manifest_dir` = the disk app dir if the app also exists on disk, +else a sentinel `/` (never read for image-only apps). + +This is enforced by `catalog_manifest_to_overlay(app_id, value) -> Option` +in `prod_orchestrator.rs`, which returns `None` (→ disk fallback) for: unparseable +value, embedded-id ≠ catalog-key, failed `validate()`, or a build source. ## 5. Publishing (publish-side generator)