feat(catalog): reload manifest overlay when a refreshed catalog changes

load_manifests() only ran at startup, so a manifest published via the
signed catalog sat dormant until the next service restart (found while
shipping the strfry manifest fixes: package.check-updates refreshed the
cache but the orchestrator kept rendering the old overlay).

- refresh_catalog now reports whether the cached bytes actually changed
  (write_cache skips identical rewrites), so unchanged hourly polls
  don't churn the manifest map.
- New ContainerOrchestrator::reload_manifests (default no-op); prod
  delegates to load_manifests — the rebuild is atomic under the state
  write lock, and the reconciler's durable user-stopped/uninstalled
  marker filters make a mid-session reload equivalent to the restart
  path that already runs on every boot.
- package.check-updates reloads on change and reports catalog_changed +
  manifests_reloaded; the hourly update-scheduler tick (and its startup
  refresh) do the same.

Tests: app_catalog 5/5 (new write_cache changed-detection test),
reconcile 16/16, knows_app 1/1.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
archipelago 2026-07-09 18:54:00 -04:00
parent 2c46da387c
commit ceb319c292
6 changed files with 142 additions and 25 deletions

View File

@ -172,18 +172,36 @@ impl RpcHandler {
/// Manual "check for updates": refresh the remote app catalog now. The /// Manual "check for updates": refresh the remote app catalog now. The
/// package scanner recomputes each app's `available-update` from the fresh /// package scanner recomputes each app's `available-update` from the fresh
/// catalog on its next cycle and pushes it to the UI. Best-effort — a fetch /// catalog on its next cycle and pushes it to the UI. When the catalog
/// failure leaves the cached catalog in place and reports `refreshed: false`. /// bytes changed, the orchestrator's manifest overlay is reloaded in the
/// same call so catalog-shipped manifest fixes apply without a service
/// restart. Best-effort — a fetch failure leaves the cached catalog in
/// place and reports `refreshed: false`.
pub(in crate::api::rpc) async fn handle_package_check_updates( pub(in crate::api::rpc) async fn handle_package_check_updates(
&self, &self,
_params: Option<serde_json::Value>, _params: Option<serde_json::Value>,
) -> Result<serde_json::Value> { ) -> Result<serde_json::Value> {
match crate::container::app_catalog::refresh_catalog(&self.config.data_dir).await { match crate::container::app_catalog::refresh_catalog(&self.config.data_dir).await {
Ok(count) => Ok(serde_json::json!({ Ok(refresh) => {
"status": "ok", let mut manifests_reloaded = serde_json::Value::Null;
"refreshed": true, if refresh.changed {
"catalog_apps": count, if let Some(orch) = &self.orchestrator {
})), match orch.reload_manifests().await {
Ok(n) => manifests_reloaded = serde_json::json!(n),
Err(e) => tracing::warn!(
"check-updates: manifest reload after catalog change failed: {e}"
),
}
}
}
Ok(serde_json::json!({
"status": "ok",
"refreshed": true,
"catalog_apps": refresh.apps,
"catalog_changed": refresh.changed,
"manifests_reloaded": manifests_reloaded,
}))
}
Err(e) => Ok(serde_json::json!({ Err(e) => Ok(serde_json::json!({
"status": "ok", "status": "ok",
"refreshed": false, "refreshed": false,

View File

@ -339,18 +339,31 @@ fn catalog_urls_from_mirrors(mirrors: &[crate::update::UpdateMirror]) -> Vec<Str
urls urls
} }
/// Outcome of [`refresh_catalog`]: the app count of the fetched catalog and
/// whether the cached bytes actually changed. `changed` drives the manifest-
/// overlay reload — catalog manifests only take effect once `load_manifests`
/// re-runs, and reloading on every unchanged hourly poll would be pure churn.
#[derive(Debug, Clone, Copy)]
pub struct CatalogRefresh {
pub apps: usize,
pub changed: bool,
}
/// Fetch the catalog from the first reachable mirror and atomically write it to /// Fetch the catalog from the first reachable mirror and atomically write it to
/// `<data_dir>/app-catalog.json`. Returns the number of apps in the catalog on /// `<data_dir>/app-catalog.json`. Returns the app count and whether the cache
/// success. Best-effort: a fetch failure leaves the existing cache untouched /// changed. Best-effort: a fetch failure leaves the existing cache untouched
/// (origin-always-wins; updates simply aren't refreshed this cycle). /// (origin-always-wins; updates simply aren't refreshed this cycle).
pub async fn refresh_catalog(data_dir: &Path) -> anyhow::Result<usize> { pub async fn refresh_catalog(data_dir: &Path) -> anyhow::Result<CatalogRefresh> {
let mirrors = crate::update::load_mirrors(data_dir) let mirrors = crate::update::load_mirrors(data_dir)
.await .await
.unwrap_or_default(); .unwrap_or_default();
let urls = catalog_urls_from_mirrors(&mirrors); let urls = catalog_urls_from_mirrors(&mirrors);
if urls.is_empty() { if urls.is_empty() {
debug!("app-catalog: no mirror-derived URLs to fetch from"); debug!("app-catalog: no mirror-derived URLs to fetch from");
return Ok(0); return Ok(CatalogRefresh {
apps: 0,
changed: false,
});
} }
let client = reqwest::Client::builder() let client = reqwest::Client::builder()
@ -362,11 +375,21 @@ pub async fn refresh_catalog(data_dir: &Path) -> anyhow::Result<usize> {
match fetch_one(&client, url).await { match fetch_one(&client, url).await {
Ok(catalog) => { Ok(catalog) => {
let count = catalog.apps.len(); let count = catalog.apps.len();
write_cache(data_dir, &catalog)?; let changed = write_cache(data_dir, &catalog)?;
// Invalidate the in-process cache so the next read re-parses. if changed {
*CACHE.lock().unwrap() = None; // Invalidate the in-process cache so the next read re-parses.
info!("app-catalog: refreshed from {} ({} apps)", url, count); *CACHE.lock().unwrap() = None;
return Ok(count); }
info!(
"app-catalog: refreshed from {} ({} apps{})",
url,
count,
if changed { ", changed" } else { ", unchanged" }
);
return Ok(CatalogRefresh {
apps: count,
changed,
});
} }
Err(e) => { Err(e) => {
debug!("app-catalog: fetch {} failed: {}", url, e); debug!("app-catalog: fetch {} failed: {}", url, e);
@ -419,13 +442,22 @@ async fn fetch_one(client: &reqwest::Client, url: &str) -> anyhow::Result<AppCat
Ok(catalog) Ok(catalog)
} }
fn write_cache(data_dir: &Path, catalog: &AppCatalog) -> anyhow::Result<()> { /// Atomically write the catalog cache. Returns `false` (skipping the write)
/// when the serialized bytes are identical to what's already cached, so
/// callers can tell a genuine catalog change from an unchanged poll.
fn write_cache(data_dir: &Path, catalog: &AppCatalog) -> anyhow::Result<bool> {
let dest = data_dir.join(APP_CATALOG_FILE); let dest = data_dir.join(APP_CATALOG_FILE);
let tmp = data_dir.join(format!("{}.tmp", APP_CATALOG_FILE));
let json = serde_json::to_string_pretty(catalog)?; let json = serde_json::to_string_pretty(catalog)?;
if std::fs::read_to_string(&dest)
.map(|current| current == json)
.unwrap_or(false)
{
return Ok(false);
}
let tmp = data_dir.join(format!("{}.tmp", APP_CATALOG_FILE));
std::fs::write(&tmp, json)?; std::fs::write(&tmp, json)?;
std::fs::rename(&tmp, &dest)?; std::fs::rename(&tmp, &dest)?;
Ok(()) Ok(true)
} }
#[cfg(test)] #[cfg(test)]
@ -462,6 +494,31 @@ mod tests {
assert_eq!(e.digest.as_deref(), Some("blake3:deadbeef")); assert_eq!(e.digest.as_deref(), Some("blake3:deadbeef"));
} }
#[test]
fn write_cache_reports_changed_only_on_new_bytes() {
// The changed flag gates the runtime manifest-overlay reload: an
// unchanged hourly poll must NOT report changed (or every tick would
// rebuild the manifest map), while a genuinely new catalog must.
let dir = tempfile::tempdir().unwrap();
let cat: AppCatalog = serde_json::from_str(
r#"{"schema":1,"apps":{"demo":{"version":"1.0.0"}}}"#,
)
.unwrap();
assert!(write_cache(dir.path(), &cat).unwrap(), "first write is a change");
assert!(
!write_cache(dir.path(), &cat).unwrap(),
"identical rewrite is not a change"
);
let cat2: AppCatalog = serde_json::from_str(
r#"{"schema":1,"apps":{"demo":{"version":"1.0.1"}}}"#,
)
.unwrap();
assert!(
write_cache(dir.path(), &cat2).unwrap(),
"new catalog bytes are a change"
);
}
#[test] #[test]
fn entry_carries_embedded_manifest() { fn entry_carries_embedded_manifest() {
let json = r#"{ let json = r#"{

View File

@ -3614,6 +3614,14 @@ impl ContainerOrchestrator for ProdContainerOrchestrator {
self.state.read().await.manifests.contains_key(app_id) self.state.read().await.manifests.contains_key(app_id)
} }
async fn reload_manifests(&self) -> Result<usize> {
// Safe mid-session: the rebuild is atomic under the state write lock,
// and although it wipes the in-memory `disabled` set, reconcile also
// filters on the durable user-stopped/user-uninstalled markers — the
// same contract every service restart already relies on.
self.load_manifests().await
}
async fn install(&self, app_id: &str) -> Result<String> { async fn install(&self, app_id: &str) -> Result<String> {
{ {
let mut state = self.state.write().await; let mut state = self.state.write().await;

View File

@ -35,6 +35,16 @@ pub trait ContainerOrchestrator: Send + Sync {
false false
} }
/// Rebuild the in-memory manifest map (disk + signed-catalog overlay).
/// Called after a runtime catalog refresh detects changed bytes so catalog
/// manifest changes take effect without a service restart — without this,
/// `load_manifests` only runs at startup and a freshly published manifest
/// sits dormant until the next restart. Returns the merged manifest count.
/// Defaults to a no-op for orchestrators without a manifest registry.
async fn reload_manifests(&self) -> Result<usize> {
Ok(0)
}
/// Start an already-created container. /// Start an already-created container.
async fn start(&self, app_id: &str) -> Result<()>; async fn start(&self, app_id: &str) -> Result<()>;

View File

@ -222,7 +222,10 @@ async fn main() -> Result<()> {
) )
.await .await
{ {
Ok(Ok(n)) => info!("🛰️ app-catalog refreshed before manifest load ({n} apps)"), Ok(Ok(r)) => info!(
"🛰️ app-catalog refreshed before manifest load ({} apps)",
r.apps
),
Ok(Err(e)) => tracing::debug!("app-catalog pre-load refresh failed (using cache): {e}"), Ok(Err(e)) => tracing::debug!("app-catalog pre-load refresh failed (using cache): {e}"),
Err(_) => tracing::debug!("app-catalog pre-load refresh timed out (using cache)"), Err(_) => tracing::debug!("app-catalog pre-load refresh timed out (using cache)"),
} }

View File

@ -1897,6 +1897,23 @@ async fn apply_per_app_auto_updates(
} }
} }
/// After a catalog refresh that changed the cached bytes, rebuild the
/// orchestrator's manifest map so registry-shipped manifest changes take
/// effect now instead of at the next service restart.
async fn reload_manifests_if_changed(
refresh: crate::container::app_catalog::CatalogRefresh,
orchestrator: &Option<std::sync::Arc<dyn crate::container::traits::ContainerOrchestrator>>,
) {
if !refresh.changed {
return;
}
let Some(orch) = orchestrator else { return };
match orch.reload_manifests().await {
Ok(n) => info!("Update scheduler: catalog changed, reloaded {n} manifest(s)"),
Err(e) => warn!("Update scheduler: manifest reload after catalog change failed: {e}"),
}
}
pub async fn run_update_scheduler( pub async fn run_update_scheduler(
data_dir: std::path::PathBuf, data_dir: std::path::PathBuf,
orchestrator: Option<std::sync::Arc<dyn crate::container::traits::ContainerOrchestrator>>, orchestrator: Option<std::sync::Arc<dyn crate::container::traits::ContainerOrchestrator>>,
@ -1908,11 +1925,12 @@ pub async fn run_update_scheduler(
// Refresh the app catalog once at startup so per-app "update available" // Refresh the app catalog once at startup so per-app "update available"
// badges appear without waiting for the first hourly tick. // badges appear without waiting for the first hourly tick.
if let Err(e) = crate::container::app_catalog::refresh_catalog(&data_dir).await { match crate::container::app_catalog::refresh_catalog(&data_dir).await {
debug!( Ok(refresh) => reload_manifests_if_changed(refresh, &orchestrator).await,
Err(e) => debug!(
"Update scheduler: initial app-catalog refresh failed: {}", "Update scheduler: initial app-catalog refresh failed: {}",
e e
); ),
} }
loop { loop {
@ -1922,8 +1940,11 @@ pub async fn run_update_scheduler(
// populates per-app update availability (the "Update" button still has // populates per-app update availability (the "Update" button still has
// to be clicked — nothing auto-applies). Best-effort; on failure the // to be clicked — nothing auto-applies). Best-effort; on failure the
// previously cached catalog stays in place (origin-always-wins). // previously cached catalog stays in place (origin-always-wins).
if let Err(e) = crate::container::app_catalog::refresh_catalog(&data_dir).await { // A changed catalog also reloads the orchestrator's manifest overlay so
debug!("Update scheduler: app-catalog refresh failed: {}", e); // catalog-shipped manifest fixes apply without a service restart.
match crate::container::app_catalog::refresh_catalog(&data_dir).await {
Ok(refresh) => reload_manifests_if_changed(refresh, &orchestrator).await,
Err(e) => debug!("Update scheduler: app-catalog refresh failed: {}", e),
} }
// Per-app auto-update-to-latest (multi-version support). Runs every tick // Per-app auto-update-to-latest (multi-version support). Runs every tick