feat(catalog): reload manifest overlay when a refreshed catalog changes
load_manifests() only ran at startup, so a manifest published via the signed catalog sat dormant until the next service restart (found while shipping the strfry manifest fixes: package.check-updates refreshed the cache but the orchestrator kept rendering the old overlay). - refresh_catalog now reports whether the cached bytes actually changed (write_cache skips identical rewrites), so unchanged hourly polls don't churn the manifest map. - New ContainerOrchestrator::reload_manifests (default no-op); prod delegates to load_manifests — the rebuild is atomic under the state write lock, and the reconciler's durable user-stopped/uninstalled marker filters make a mid-session reload equivalent to the restart path that already runs on every boot. - package.check-updates reloads on change and reports catalog_changed + manifests_reloaded; the hourly update-scheduler tick (and its startup refresh) do the same. Tests: app_catalog 5/5 (new write_cache changed-detection test), reconcile 16/16, knows_app 1/1. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
2c46da387c
commit
ceb319c292
@ -172,18 +172,36 @@ impl RpcHandler {
|
|||||||
|
|
||||||
/// Manual "check for updates": refresh the remote app catalog now. The
|
/// Manual "check for updates": refresh the remote app catalog now. The
|
||||||
/// package scanner recomputes each app's `available-update` from the fresh
|
/// package scanner recomputes each app's `available-update` from the fresh
|
||||||
/// catalog on its next cycle and pushes it to the UI. Best-effort — a fetch
|
/// catalog on its next cycle and pushes it to the UI. When the catalog
|
||||||
/// failure leaves the cached catalog in place and reports `refreshed: false`.
|
/// bytes changed, the orchestrator's manifest overlay is reloaded in the
|
||||||
|
/// same call so catalog-shipped manifest fixes apply without a service
|
||||||
|
/// restart. Best-effort — a fetch failure leaves the cached catalog in
|
||||||
|
/// place and reports `refreshed: false`.
|
||||||
pub(in crate::api::rpc) async fn handle_package_check_updates(
|
pub(in crate::api::rpc) async fn handle_package_check_updates(
|
||||||
&self,
|
&self,
|
||||||
_params: Option<serde_json::Value>,
|
_params: Option<serde_json::Value>,
|
||||||
) -> Result<serde_json::Value> {
|
) -> Result<serde_json::Value> {
|
||||||
match crate::container::app_catalog::refresh_catalog(&self.config.data_dir).await {
|
match crate::container::app_catalog::refresh_catalog(&self.config.data_dir).await {
|
||||||
Ok(count) => Ok(serde_json::json!({
|
Ok(refresh) => {
|
||||||
"status": "ok",
|
let mut manifests_reloaded = serde_json::Value::Null;
|
||||||
"refreshed": true,
|
if refresh.changed {
|
||||||
"catalog_apps": count,
|
if let Some(orch) = &self.orchestrator {
|
||||||
})),
|
match orch.reload_manifests().await {
|
||||||
|
Ok(n) => manifests_reloaded = serde_json::json!(n),
|
||||||
|
Err(e) => tracing::warn!(
|
||||||
|
"check-updates: manifest reload after catalog change failed: {e}"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(serde_json::json!({
|
||||||
|
"status": "ok",
|
||||||
|
"refreshed": true,
|
||||||
|
"catalog_apps": refresh.apps,
|
||||||
|
"catalog_changed": refresh.changed,
|
||||||
|
"manifests_reloaded": manifests_reloaded,
|
||||||
|
}))
|
||||||
|
}
|
||||||
Err(e) => Ok(serde_json::json!({
|
Err(e) => Ok(serde_json::json!({
|
||||||
"status": "ok",
|
"status": "ok",
|
||||||
"refreshed": false,
|
"refreshed": false,
|
||||||
|
|||||||
@ -339,18 +339,31 @@ fn catalog_urls_from_mirrors(mirrors: &[crate::update::UpdateMirror]) -> Vec<Str
|
|||||||
urls
|
urls
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Outcome of [`refresh_catalog`]: the app count of the fetched catalog and
|
||||||
|
/// whether the cached bytes actually changed. `changed` drives the manifest-
|
||||||
|
/// overlay reload — catalog manifests only take effect once `load_manifests`
|
||||||
|
/// re-runs, and reloading on every unchanged hourly poll would be pure churn.
|
||||||
|
#[derive(Debug, Clone, Copy)]
|
||||||
|
pub struct CatalogRefresh {
|
||||||
|
pub apps: usize,
|
||||||
|
pub changed: bool,
|
||||||
|
}
|
||||||
|
|
||||||
/// Fetch the catalog from the first reachable mirror and atomically write it to
|
/// Fetch the catalog from the first reachable mirror and atomically write it to
|
||||||
/// `<data_dir>/app-catalog.json`. Returns the number of apps in the catalog on
|
/// `<data_dir>/app-catalog.json`. Returns the app count and whether the cache
|
||||||
/// success. Best-effort: a fetch failure leaves the existing cache untouched
|
/// changed. Best-effort: a fetch failure leaves the existing cache untouched
|
||||||
/// (origin-always-wins; updates simply aren't refreshed this cycle).
|
/// (origin-always-wins; updates simply aren't refreshed this cycle).
|
||||||
pub async fn refresh_catalog(data_dir: &Path) -> anyhow::Result<usize> {
|
pub async fn refresh_catalog(data_dir: &Path) -> anyhow::Result<CatalogRefresh> {
|
||||||
let mirrors = crate::update::load_mirrors(data_dir)
|
let mirrors = crate::update::load_mirrors(data_dir)
|
||||||
.await
|
.await
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
let urls = catalog_urls_from_mirrors(&mirrors);
|
let urls = catalog_urls_from_mirrors(&mirrors);
|
||||||
if urls.is_empty() {
|
if urls.is_empty() {
|
||||||
debug!("app-catalog: no mirror-derived URLs to fetch from");
|
debug!("app-catalog: no mirror-derived URLs to fetch from");
|
||||||
return Ok(0);
|
return Ok(CatalogRefresh {
|
||||||
|
apps: 0,
|
||||||
|
changed: false,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
let client = reqwest::Client::builder()
|
let client = reqwest::Client::builder()
|
||||||
@ -362,11 +375,21 @@ pub async fn refresh_catalog(data_dir: &Path) -> anyhow::Result<usize> {
|
|||||||
match fetch_one(&client, url).await {
|
match fetch_one(&client, url).await {
|
||||||
Ok(catalog) => {
|
Ok(catalog) => {
|
||||||
let count = catalog.apps.len();
|
let count = catalog.apps.len();
|
||||||
write_cache(data_dir, &catalog)?;
|
let changed = write_cache(data_dir, &catalog)?;
|
||||||
// Invalidate the in-process cache so the next read re-parses.
|
if changed {
|
||||||
*CACHE.lock().unwrap() = None;
|
// Invalidate the in-process cache so the next read re-parses.
|
||||||
info!("app-catalog: refreshed from {} ({} apps)", url, count);
|
*CACHE.lock().unwrap() = None;
|
||||||
return Ok(count);
|
}
|
||||||
|
info!(
|
||||||
|
"app-catalog: refreshed from {} ({} apps{})",
|
||||||
|
url,
|
||||||
|
count,
|
||||||
|
if changed { ", changed" } else { ", unchanged" }
|
||||||
|
);
|
||||||
|
return Ok(CatalogRefresh {
|
||||||
|
apps: count,
|
||||||
|
changed,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
debug!("app-catalog: fetch {} failed: {}", url, e);
|
debug!("app-catalog: fetch {} failed: {}", url, e);
|
||||||
@ -419,13 +442,22 @@ async fn fetch_one(client: &reqwest::Client, url: &str) -> anyhow::Result<AppCat
|
|||||||
Ok(catalog)
|
Ok(catalog)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn write_cache(data_dir: &Path, catalog: &AppCatalog) -> anyhow::Result<()> {
|
/// Atomically write the catalog cache. Returns `false` (skipping the write)
|
||||||
|
/// when the serialized bytes are identical to what's already cached, so
|
||||||
|
/// callers can tell a genuine catalog change from an unchanged poll.
|
||||||
|
fn write_cache(data_dir: &Path, catalog: &AppCatalog) -> anyhow::Result<bool> {
|
||||||
let dest = data_dir.join(APP_CATALOG_FILE);
|
let dest = data_dir.join(APP_CATALOG_FILE);
|
||||||
let tmp = data_dir.join(format!("{}.tmp", APP_CATALOG_FILE));
|
|
||||||
let json = serde_json::to_string_pretty(catalog)?;
|
let json = serde_json::to_string_pretty(catalog)?;
|
||||||
|
if std::fs::read_to_string(&dest)
|
||||||
|
.map(|current| current == json)
|
||||||
|
.unwrap_or(false)
|
||||||
|
{
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
let tmp = data_dir.join(format!("{}.tmp", APP_CATALOG_FILE));
|
||||||
std::fs::write(&tmp, json)?;
|
std::fs::write(&tmp, json)?;
|
||||||
std::fs::rename(&tmp, &dest)?;
|
std::fs::rename(&tmp, &dest)?;
|
||||||
Ok(())
|
Ok(true)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@ -462,6 +494,31 @@ mod tests {
|
|||||||
assert_eq!(e.digest.as_deref(), Some("blake3:deadbeef"));
|
assert_eq!(e.digest.as_deref(), Some("blake3:deadbeef"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn write_cache_reports_changed_only_on_new_bytes() {
|
||||||
|
// The changed flag gates the runtime manifest-overlay reload: an
|
||||||
|
// unchanged hourly poll must NOT report changed (or every tick would
|
||||||
|
// rebuild the manifest map), while a genuinely new catalog must.
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let cat: AppCatalog = serde_json::from_str(
|
||||||
|
r#"{"schema":1,"apps":{"demo":{"version":"1.0.0"}}}"#,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(write_cache(dir.path(), &cat).unwrap(), "first write is a change");
|
||||||
|
assert!(
|
||||||
|
!write_cache(dir.path(), &cat).unwrap(),
|
||||||
|
"identical rewrite is not a change"
|
||||||
|
);
|
||||||
|
let cat2: AppCatalog = serde_json::from_str(
|
||||||
|
r#"{"schema":1,"apps":{"demo":{"version":"1.0.1"}}}"#,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
write_cache(dir.path(), &cat2).unwrap(),
|
||||||
|
"new catalog bytes are a change"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn entry_carries_embedded_manifest() {
|
fn entry_carries_embedded_manifest() {
|
||||||
let json = r#"{
|
let json = r#"{
|
||||||
|
|||||||
@ -3614,6 +3614,14 @@ impl ContainerOrchestrator for ProdContainerOrchestrator {
|
|||||||
self.state.read().await.manifests.contains_key(app_id)
|
self.state.read().await.manifests.contains_key(app_id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn reload_manifests(&self) -> Result<usize> {
|
||||||
|
// Safe mid-session: the rebuild is atomic under the state write lock,
|
||||||
|
// and although it wipes the in-memory `disabled` set, reconcile also
|
||||||
|
// filters on the durable user-stopped/user-uninstalled markers — the
|
||||||
|
// same contract every service restart already relies on.
|
||||||
|
self.load_manifests().await
|
||||||
|
}
|
||||||
|
|
||||||
async fn install(&self, app_id: &str) -> Result<String> {
|
async fn install(&self, app_id: &str) -> Result<String> {
|
||||||
{
|
{
|
||||||
let mut state = self.state.write().await;
|
let mut state = self.state.write().await;
|
||||||
|
|||||||
@ -35,6 +35,16 @@ pub trait ContainerOrchestrator: Send + Sync {
|
|||||||
false
|
false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Rebuild the in-memory manifest map (disk + signed-catalog overlay).
|
||||||
|
/// Called after a runtime catalog refresh detects changed bytes so catalog
|
||||||
|
/// manifest changes take effect without a service restart — without this,
|
||||||
|
/// `load_manifests` only runs at startup and a freshly published manifest
|
||||||
|
/// sits dormant until the next restart. Returns the merged manifest count.
|
||||||
|
/// Defaults to a no-op for orchestrators without a manifest registry.
|
||||||
|
async fn reload_manifests(&self) -> Result<usize> {
|
||||||
|
Ok(0)
|
||||||
|
}
|
||||||
|
|
||||||
/// Start an already-created container.
|
/// Start an already-created container.
|
||||||
async fn start(&self, app_id: &str) -> Result<()>;
|
async fn start(&self, app_id: &str) -> Result<()>;
|
||||||
|
|
||||||
|
|||||||
@ -222,7 +222,10 @@ async fn main() -> Result<()> {
|
|||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(Ok(n)) => info!("🛰️ app-catalog refreshed before manifest load ({n} apps)"),
|
Ok(Ok(r)) => info!(
|
||||||
|
"🛰️ app-catalog refreshed before manifest load ({} apps)",
|
||||||
|
r.apps
|
||||||
|
),
|
||||||
Ok(Err(e)) => tracing::debug!("app-catalog pre-load refresh failed (using cache): {e}"),
|
Ok(Err(e)) => tracing::debug!("app-catalog pre-load refresh failed (using cache): {e}"),
|
||||||
Err(_) => tracing::debug!("app-catalog pre-load refresh timed out (using cache)"),
|
Err(_) => tracing::debug!("app-catalog pre-load refresh timed out (using cache)"),
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1897,6 +1897,23 @@ async fn apply_per_app_auto_updates(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// After a catalog refresh that changed the cached bytes, rebuild the
|
||||||
|
/// orchestrator's manifest map so registry-shipped manifest changes take
|
||||||
|
/// effect now instead of at the next service restart.
|
||||||
|
async fn reload_manifests_if_changed(
|
||||||
|
refresh: crate::container::app_catalog::CatalogRefresh,
|
||||||
|
orchestrator: &Option<std::sync::Arc<dyn crate::container::traits::ContainerOrchestrator>>,
|
||||||
|
) {
|
||||||
|
if !refresh.changed {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let Some(orch) = orchestrator else { return };
|
||||||
|
match orch.reload_manifests().await {
|
||||||
|
Ok(n) => info!("Update scheduler: catalog changed, reloaded {n} manifest(s)"),
|
||||||
|
Err(e) => warn!("Update scheduler: manifest reload after catalog change failed: {e}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn run_update_scheduler(
|
pub async fn run_update_scheduler(
|
||||||
data_dir: std::path::PathBuf,
|
data_dir: std::path::PathBuf,
|
||||||
orchestrator: Option<std::sync::Arc<dyn crate::container::traits::ContainerOrchestrator>>,
|
orchestrator: Option<std::sync::Arc<dyn crate::container::traits::ContainerOrchestrator>>,
|
||||||
@ -1908,11 +1925,12 @@ pub async fn run_update_scheduler(
|
|||||||
|
|
||||||
// Refresh the app catalog once at startup so per-app "update available"
|
// Refresh the app catalog once at startup so per-app "update available"
|
||||||
// badges appear without waiting for the first hourly tick.
|
// badges appear without waiting for the first hourly tick.
|
||||||
if let Err(e) = crate::container::app_catalog::refresh_catalog(&data_dir).await {
|
match crate::container::app_catalog::refresh_catalog(&data_dir).await {
|
||||||
debug!(
|
Ok(refresh) => reload_manifests_if_changed(refresh, &orchestrator).await,
|
||||||
|
Err(e) => debug!(
|
||||||
"Update scheduler: initial app-catalog refresh failed: {}",
|
"Update scheduler: initial app-catalog refresh failed: {}",
|
||||||
e
|
e
|
||||||
);
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
@ -1922,8 +1940,11 @@ pub async fn run_update_scheduler(
|
|||||||
// populates per-app update availability (the "Update" button still has
|
// populates per-app update availability (the "Update" button still has
|
||||||
// to be clicked — nothing auto-applies). Best-effort; on failure the
|
// to be clicked — nothing auto-applies). Best-effort; on failure the
|
||||||
// previously cached catalog stays in place (origin-always-wins).
|
// previously cached catalog stays in place (origin-always-wins).
|
||||||
if let Err(e) = crate::container::app_catalog::refresh_catalog(&data_dir).await {
|
// A changed catalog also reloads the orchestrator's manifest overlay so
|
||||||
debug!("Update scheduler: app-catalog refresh failed: {}", e);
|
// catalog-shipped manifest fixes apply without a service restart.
|
||||||
|
match crate::container::app_catalog::refresh_catalog(&data_dir).await {
|
||||||
|
Ok(refresh) => reload_manifests_if_changed(refresh, &orchestrator).await,
|
||||||
|
Err(e) => debug!("Update scheduler: app-catalog refresh failed: {}", e),
|
||||||
}
|
}
|
||||||
|
|
||||||
// Per-app auto-update-to-latest (multi-version support). Runs every tick
|
// Per-app auto-update-to-latest (multi-version support). Runs every tick
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user