From ceb319c2923ad5d9fd969b20ceb14dfd7a70cb48 Mon Sep 17 00:00:00 2001 From: archipelago Date: Thu, 9 Jul 2026 18:54:00 -0400 Subject: [PATCH] feat(catalog): reload manifest overlay when a refreshed catalog changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit load_manifests() only ran at startup, so a manifest published via the signed catalog sat dormant until the next service restart (found while shipping the strfry manifest fixes: package.check-updates refreshed the cache but the orchestrator kept rendering the old overlay). - refresh_catalog now reports whether the cached bytes actually changed (write_cache skips identical rewrites), so unchanged hourly polls don't churn the manifest map. - New ContainerOrchestrator::reload_manifests (default no-op); prod delegates to load_manifests — the rebuild is atomic under the state write lock, and the reconciler's durable user-stopped/uninstalled marker filters make a mid-session reload equivalent to the restart path that already runs on every boot. - package.check-updates reloads on change and reports catalog_changed + manifests_reloaded; the hourly update-scheduler tick (and its startup refresh) do the same. Tests: app_catalog 5/5 (new write_cache changed-detection test), reconcile 16/16, knows_app 1/1. Co-Authored-By: Claude Fable 5 --- .../archipelago/src/api/rpc/package/update.rs | 32 ++++++-- core/archipelago/src/container/app_catalog.rs | 81 ++++++++++++++++--- .../src/container/prod_orchestrator.rs | 8 ++ core/archipelago/src/container/traits.rs | 10 +++ core/archipelago/src/main.rs | 5 +- core/archipelago/src/update.rs | 31 +++++-- 6 files changed, 142 insertions(+), 25 deletions(-) diff --git a/core/archipelago/src/api/rpc/package/update.rs b/core/archipelago/src/api/rpc/package/update.rs index a90bd5f3..2df467b6 100644 --- a/core/archipelago/src/api/rpc/package/update.rs +++ b/core/archipelago/src/api/rpc/package/update.rs @@ -172,18 +172,36 @@ impl RpcHandler { /// Manual "check for updates": refresh the remote app catalog now. The /// package scanner recomputes each app's `available-update` from the fresh - /// catalog on its next cycle and pushes it to the UI. Best-effort — a fetch - /// failure leaves the cached catalog in place and reports `refreshed: false`. + /// catalog on its next cycle and pushes it to the UI. When the catalog + /// bytes changed, the orchestrator's manifest overlay is reloaded in the + /// same call so catalog-shipped manifest fixes apply without a service + /// restart. Best-effort — a fetch failure leaves the cached catalog in + /// place and reports `refreshed: false`. pub(in crate::api::rpc) async fn handle_package_check_updates( &self, _params: Option, ) -> Result { match crate::container::app_catalog::refresh_catalog(&self.config.data_dir).await { - Ok(count) => Ok(serde_json::json!({ - "status": "ok", - "refreshed": true, - "catalog_apps": count, - })), + Ok(refresh) => { + let mut manifests_reloaded = serde_json::Value::Null; + if refresh.changed { + if let Some(orch) = &self.orchestrator { + match orch.reload_manifests().await { + Ok(n) => manifests_reloaded = serde_json::json!(n), + Err(e) => tracing::warn!( + "check-updates: manifest reload after catalog change failed: {e}" + ), + } + } + } + Ok(serde_json::json!({ + "status": "ok", + "refreshed": true, + "catalog_apps": refresh.apps, + "catalog_changed": refresh.changed, + "manifests_reloaded": manifests_reloaded, + })) + } Err(e) => Ok(serde_json::json!({ "status": "ok", "refreshed": false, diff --git a/core/archipelago/src/container/app_catalog.rs b/core/archipelago/src/container/app_catalog.rs index b5eddc9c..8fb6803e 100644 --- a/core/archipelago/src/container/app_catalog.rs +++ b/core/archipelago/src/container/app_catalog.rs @@ -339,18 +339,31 @@ fn catalog_urls_from_mirrors(mirrors: &[crate::update::UpdateMirror]) -> Vec/app-catalog.json`. Returns the number of apps in the catalog on -/// success. Best-effort: a fetch failure leaves the existing cache untouched +/// `/app-catalog.json`. Returns the app count and whether the cache +/// changed. Best-effort: a fetch failure leaves the existing cache untouched /// (origin-always-wins; updates simply aren't refreshed this cycle). -pub async fn refresh_catalog(data_dir: &Path) -> anyhow::Result { +pub async fn refresh_catalog(data_dir: &Path) -> anyhow::Result { let mirrors = crate::update::load_mirrors(data_dir) .await .unwrap_or_default(); let urls = catalog_urls_from_mirrors(&mirrors); if urls.is_empty() { debug!("app-catalog: no mirror-derived URLs to fetch from"); - return Ok(0); + return Ok(CatalogRefresh { + apps: 0, + changed: false, + }); } let client = reqwest::Client::builder() @@ -362,11 +375,21 @@ pub async fn refresh_catalog(data_dir: &Path) -> anyhow::Result { match fetch_one(&client, url).await { Ok(catalog) => { let count = catalog.apps.len(); - write_cache(data_dir, &catalog)?; - // Invalidate the in-process cache so the next read re-parses. - *CACHE.lock().unwrap() = None; - info!("app-catalog: refreshed from {} ({} apps)", url, count); - return Ok(count); + let changed = write_cache(data_dir, &catalog)?; + if changed { + // Invalidate the in-process cache so the next read re-parses. + *CACHE.lock().unwrap() = None; + } + info!( + "app-catalog: refreshed from {} ({} apps{})", + url, + count, + if changed { ", changed" } else { ", unchanged" } + ); + return Ok(CatalogRefresh { + apps: count, + changed, + }); } Err(e) => { debug!("app-catalog: fetch {} failed: {}", url, e); @@ -419,13 +442,22 @@ async fn fetch_one(client: &reqwest::Client, url: &str) -> anyhow::Result anyhow::Result<()> { +/// Atomically write the catalog cache. Returns `false` (skipping the write) +/// when the serialized bytes are identical to what's already cached, so +/// callers can tell a genuine catalog change from an unchanged poll. +fn write_cache(data_dir: &Path, catalog: &AppCatalog) -> anyhow::Result { let dest = data_dir.join(APP_CATALOG_FILE); - let tmp = data_dir.join(format!("{}.tmp", APP_CATALOG_FILE)); let json = serde_json::to_string_pretty(catalog)?; + if std::fs::read_to_string(&dest) + .map(|current| current == json) + .unwrap_or(false) + { + return Ok(false); + } + let tmp = data_dir.join(format!("{}.tmp", APP_CATALOG_FILE)); std::fs::write(&tmp, json)?; std::fs::rename(&tmp, &dest)?; - Ok(()) + Ok(true) } #[cfg(test)] @@ -462,6 +494,31 @@ mod tests { assert_eq!(e.digest.as_deref(), Some("blake3:deadbeef")); } + #[test] + fn write_cache_reports_changed_only_on_new_bytes() { + // The changed flag gates the runtime manifest-overlay reload: an + // unchanged hourly poll must NOT report changed (or every tick would + // rebuild the manifest map), while a genuinely new catalog must. + let dir = tempfile::tempdir().unwrap(); + let cat: AppCatalog = serde_json::from_str( + r#"{"schema":1,"apps":{"demo":{"version":"1.0.0"}}}"#, + ) + .unwrap(); + assert!(write_cache(dir.path(), &cat).unwrap(), "first write is a change"); + assert!( + !write_cache(dir.path(), &cat).unwrap(), + "identical rewrite is not a change" + ); + let cat2: AppCatalog = serde_json::from_str( + r#"{"schema":1,"apps":{"demo":{"version":"1.0.1"}}}"#, + ) + .unwrap(); + assert!( + write_cache(dir.path(), &cat2).unwrap(), + "new catalog bytes are a change" + ); + } + #[test] fn entry_carries_embedded_manifest() { let json = r#"{ diff --git a/core/archipelago/src/container/prod_orchestrator.rs b/core/archipelago/src/container/prod_orchestrator.rs index 255d3b81..cf5ff8af 100644 --- a/core/archipelago/src/container/prod_orchestrator.rs +++ b/core/archipelago/src/container/prod_orchestrator.rs @@ -3614,6 +3614,14 @@ impl ContainerOrchestrator for ProdContainerOrchestrator { self.state.read().await.manifests.contains_key(app_id) } + async fn reload_manifests(&self) -> Result { + // Safe mid-session: the rebuild is atomic under the state write lock, + // and although it wipes the in-memory `disabled` set, reconcile also + // filters on the durable user-stopped/user-uninstalled markers — the + // same contract every service restart already relies on. + self.load_manifests().await + } + async fn install(&self, app_id: &str) -> Result { { let mut state = self.state.write().await; diff --git a/core/archipelago/src/container/traits.rs b/core/archipelago/src/container/traits.rs index b69ea915..0bc0319c 100644 --- a/core/archipelago/src/container/traits.rs +++ b/core/archipelago/src/container/traits.rs @@ -35,6 +35,16 @@ pub trait ContainerOrchestrator: Send + Sync { false } + /// Rebuild the in-memory manifest map (disk + signed-catalog overlay). + /// Called after a runtime catalog refresh detects changed bytes so catalog + /// manifest changes take effect without a service restart — without this, + /// `load_manifests` only runs at startup and a freshly published manifest + /// sits dormant until the next restart. Returns the merged manifest count. + /// Defaults to a no-op for orchestrators without a manifest registry. + async fn reload_manifests(&self) -> Result { + Ok(0) + } + /// Start an already-created container. async fn start(&self, app_id: &str) -> Result<()>; diff --git a/core/archipelago/src/main.rs b/core/archipelago/src/main.rs index 0db5dca4..5a80230e 100644 --- a/core/archipelago/src/main.rs +++ b/core/archipelago/src/main.rs @@ -222,7 +222,10 @@ async fn main() -> Result<()> { ) .await { - Ok(Ok(n)) => info!("🛰️ app-catalog refreshed before manifest load ({n} apps)"), + Ok(Ok(r)) => info!( + "🛰️ app-catalog refreshed before manifest load ({} apps)", + r.apps + ), Ok(Err(e)) => tracing::debug!("app-catalog pre-load refresh failed (using cache): {e}"), Err(_) => tracing::debug!("app-catalog pre-load refresh timed out (using cache)"), } diff --git a/core/archipelago/src/update.rs b/core/archipelago/src/update.rs index 848e464a..10fe1246 100644 --- a/core/archipelago/src/update.rs +++ b/core/archipelago/src/update.rs @@ -1897,6 +1897,23 @@ async fn apply_per_app_auto_updates( } } +/// After a catalog refresh that changed the cached bytes, rebuild the +/// orchestrator's manifest map so registry-shipped manifest changes take +/// effect now instead of at the next service restart. +async fn reload_manifests_if_changed( + refresh: crate::container::app_catalog::CatalogRefresh, + orchestrator: &Option>, +) { + if !refresh.changed { + return; + } + let Some(orch) = orchestrator else { return }; + match orch.reload_manifests().await { + Ok(n) => info!("Update scheduler: catalog changed, reloaded {n} manifest(s)"), + Err(e) => warn!("Update scheduler: manifest reload after catalog change failed: {e}"), + } +} + pub async fn run_update_scheduler( data_dir: std::path::PathBuf, orchestrator: Option>, @@ -1908,11 +1925,12 @@ pub async fn run_update_scheduler( // Refresh the app catalog once at startup so per-app "update available" // badges appear without waiting for the first hourly tick. - if let Err(e) = crate::container::app_catalog::refresh_catalog(&data_dir).await { - debug!( + match crate::container::app_catalog::refresh_catalog(&data_dir).await { + Ok(refresh) => reload_manifests_if_changed(refresh, &orchestrator).await, + Err(e) => debug!( "Update scheduler: initial app-catalog refresh failed: {}", e - ); + ), } loop { @@ -1922,8 +1940,11 @@ pub async fn run_update_scheduler( // populates per-app update availability (the "Update" button still has // to be clicked — nothing auto-applies). Best-effort; on failure the // previously cached catalog stays in place (origin-always-wins). - if let Err(e) = crate::container::app_catalog::refresh_catalog(&data_dir).await { - debug!("Update scheduler: app-catalog refresh failed: {}", e); + // A changed catalog also reloads the orchestrator's manifest overlay so + // catalog-shipped manifest fixes apply without a service restart. + match crate::container::app_catalog::refresh_catalog(&data_dir).await { + Ok(refresh) => reload_manifests_if_changed(refresh, &orchestrator).await, + Err(e) => debug!("Update scheduler: app-catalog refresh failed: {}", e), } // Per-app auto-update-to-latest (multi-version support). Runs every tick