From 7bfbe8fe4084828c6f13cd27c6cd04797428dc59 Mon Sep 17 00:00:00 2001 From: archipelago Date: Sun, 21 Jun 2026 05:46:17 -0400 Subject: [PATCH] =?UTF-8?q?feat(registry-manifest):=20phase=202=20?= =?UTF-8?q?=E2=80=94=20publisher=20embeds=20manifests=20into=20signed=20ca?= =?UTF-8?q?talog?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit generate-app-catalog.sh gains opt-in EMBED_MANIFESTS=1: embeds each apps//manifest.yml into its catalog entry's `manifest` field (whole document, top-level app: preserved — exactly what the Rust side deserializes). Default off so routine catalog regen is unchanged during the migration window; turn on deliberately, then sign via the existing release-root ceremony. Verified: default embeds 0; EMBED_MANIFESTS=1 embeds 40 manifests (generated_secrets preserved). Adds a round-trip guard test: every shipped apps/*/manifest.yml must deserialize + validate through catalog_manifest_to_overlay (image apps accepted, build apps defer to disk) — catches schema drift between disk manifests and the catalog path. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/container/prod_orchestrator.rs | 38 +++++++++++++++ scripts/generate-app-catalog.sh | 46 ++++++++++++++++++- 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/core/archipelago/src/container/prod_orchestrator.rs b/core/archipelago/src/container/prod_orchestrator.rs index 080b20b5..48609098 100644 --- a/core/archipelago/src/container/prod_orchestrator.rs +++ b/core/archipelago/src/container/prod_orchestrator.rs @@ -3761,6 +3761,44 @@ app: assert!(catalog_manifest_to_overlay("demo", v).is_none()); } + #[test] + fn catalog_overlay_accepts_all_real_image_manifests() { + // Guard the registry-distribution round-trip for the WHOLE shipped app + // set: every apps/*/manifest.yml must deserialize + validate when carried + // through the catalog as a value. Image-only apps must be accepted; + // build-source apps must defer to disk (phase 1 = image-only). Catches + // schema drift between disk manifests and the catalog path. + let apps_dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../apps"); + if !apps_dir.exists() { + return; // packaged/CI layout without the repo apps/ tree — skip + } + let mut image_apps = 0; + for entry in std::fs::read_dir(&apps_dir).unwrap().flatten() { + let mf = entry.path().join("manifest.yml"); + if !mf.exists() { + continue; + } + let m = match AppManifest::from_file(&mf) { + Ok(m) => m, + Err(_) => continue, // a malformed disk manifest is a separate concern + }; + let id = m.app.id.clone(); + let is_build = m.app.container.build.is_some(); + let value = serde_json::to_value(&m).expect("manifest serializes to JSON"); + let overlay = catalog_manifest_to_overlay(&id, value); + if is_build { + assert!(overlay.is_none(), "{id}: build-source app must defer to disk"); + } else { + assert!( + overlay.is_some(), + "{id}: image-only app must round-trip through the catalog" + ); + image_apps += 1; + } + } + assert!(image_apps > 0, "expected at least one image-only manifest"); + } + fn manifest_with_container_name(id: &str, image: &str, name: &str) -> AppManifest { let yaml = format!( "app:\n id: {id}\n name: {id}\n version: 1.0.0\n container_name: {name}\n container:\n image: {image}\n" diff --git a/scripts/generate-app-catalog.sh b/scripts/generate-app-catalog.sh index f3648c54..61ac1796 100755 --- a/scripts/generate-app-catalog.sh +++ b/scripts/generate-app-catalog.sh @@ -14,7 +14,16 @@ # # Usage: # scripts/generate-app-catalog.sh [output-path] +# EMBED_MANIFESTS=1 scripts/generate-app-catalog.sh # also embed full manifests # # then publish: push releases/app-catalog.json to the OVH gitea (raw URL). +# +# EMBED_MANIFESTS (opt-in, default off): also embed each app's full +# apps//manifest.yml into its catalog entry's `manifest` field, so nodes can +# install from the signed registry alone (no OTA-shipped disk manifest). Consumed +# by container::app_catalog + the orchestrator's load_manifests overlay +# (origin-wins, disk = fallback). See docs/registry-manifest-design.md. Kept +# opt-in during the migration window so a routine catalog regen never changes +# what phase-1 nodes install until we deliberately turn it on. set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" @@ -26,9 +35,16 @@ set -a source "$ROOT/scripts/image-versions.sh" set +a -UPDATED="$(date -u +%Y-%m-%d)" OUT="$OUT" python3 - <<'PY' +UPDATED="$(date -u +%Y-%m-%d)" OUT="$OUT" APPS_DIR="$ROOT/apps" \ +EMBED_MANIFESTS="${EMBED_MANIFESTS:-}" python3 - <<'PY' +import glob import json, os +try: + import yaml +except ImportError: + yaml = None + def img(var): v = os.environ.get(var) return v if v else None @@ -121,6 +137,31 @@ for app_id, comps in STACK.items(): entry["images"] = images apps[app_id] = entry +# Opt-in (EMBED_MANIFESTS): embed each app's full manifest so nodes install from +# the registry alone. The whole manifest document is embedded under `manifest` +# (top-level `app:` preserved) — that is exactly what the Rust side deserializes +# into an AppManifest. Apps not already in SINGLE/STACK get a new entry whose +# version comes from the manifest. A bad embed is harmless: the node validates and +# falls back to its disk manifest. +embedded = 0 +apps_dir = os.environ.get("APPS_DIR") +if os.environ.get("EMBED_MANIFESTS") and apps_dir: + if yaml is None: + raise SystemExit("EMBED_MANIFESTS set but PyYAML is not available") + for path in sorted(glob.glob(os.path.join(apps_dir, "*", "manifest.yml"))): + with open(path) as fh: + data = yaml.safe_load(fh) + if not isinstance(data, dict) or not isinstance(data.get("app"), dict): + continue + app = data["app"] + app_id = app.get("id") + if not app_id: + continue + entry = apps.setdefault(str(app_id), {}) + entry.setdefault("version", str(app.get("version", "")) or "0") + entry["manifest"] = data + embedded += 1 + catalog = { "schema": 1, "updated": os.environ["UPDATED"], @@ -130,5 +171,6 @@ catalog = { with open(os.environ["OUT"], "w") as f: json.dump(catalog, f, indent=2) f.write("\n") -print(f"Wrote {os.environ['OUT']} with {len(apps)} apps") +suffix = f" (embedded {embedded} manifests)" if embedded else "" +print(f"Wrote {os.environ['OUT']} with {len(apps)} apps{suffix}") PY