lfg2025/archy
1
0
Fork 0
Code Issues 9 Pull Requests Actions Packages Projects Releases 44 Wiki Activity

refactor(indeedhub): delete orchestrator special-cases; use generic path (#20 phase 3)

Browse Source 
The fresh-create path was blocked by hardcoded indeedhub orchestrator logic
that predated and conflicted with the manifest migration:
- ensure_running routed app_id=="indeedhub" → reconcile_indeedhub_stack, which
  REFUSED to create the frontend from its manifest (returned Left("stack-managed")).
- run_pre_start_hooks("indeedhub") → start_indeedhub_backends →
  wait_for_indeedhub_dependencies_ready(120) — a DNS gate with a chicken-and-egg
  bug (required the frontend's own alias present before the frontend could be
  created), which failed install_fresh with "dependencies were not ready within
  120s" and left the frontend down (caught live on .228).

Delete all of it (−382 lines): reconcile_indeedhub_stack, start_indeedhub_backends,
wait_for_indeedhub_dependencies_ready, indeedhub_api_dependency_dns_ready,
indeedhub_required_aliases_present, repair_indeedhub_network_aliases,
indeedhub_alias_present, patch_indeedhub_nostr_provider, and the INDEEDHUB_*
consts. The manifests now carry everything these did: network_aliases (short
hostnames), generated_secrets, dependencies, and the post_install nginx hook. So
"indeedhub" + every member flows through the generic install_fresh/reconcile path
— the frontend fresh-creates normally and runs its hook.

(crash_recovery.rs's frontend-after-deps ordering guard is kept — it's beneficial
startup ordering, not a blocker.) cargo check + release build green.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
archipelago 2026-06-21 17:11:33 -04:00
parent 84031e6209
commit b73084dbb0
1 changed files with 6 additions and 382 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

388
core/archipelago/src/container/prod_orchestrator.rs
View File

@ -50,15 +50,6 @@ use crate::update::host_sudo;
/// so the rule is visible in one place and unit-testable.
const UI_APP_IDS: &[&str] = &["bitcoin-ui", "electrs-ui", "lnd-ui"];
const ARCHIVAL_BITCOIN_DISK_GB: u64 = 1000;
const INDEEDHUB_BACKEND_CONTAINERS: &[&str] = &[
"indeedhub-postgres",
"indeedhub-redis",
"indeedhub-minio",
"indeedhub-relay",
"indeedhub-api",
"indeedhub-ffmpeg",
];
const INDEEDHUB_FRONTEND_READY_TIMEOUT_SECS: u64 = 90;
fn is_required_baseline_app(app_id: &str) -> bool {
matches!(
@ -765,102 +756,6 @@ async fn restart_container_scoped_if_pasta(
}
}
async fn patch_indeedhub_nostr_provider() {
let _ = tokio::process::Command::new("podman")
.args([
"exec",
"indeedhub",
"sed",
"-i",
"/X-Frame-Options/d",
"/etc/nginx/conf.d/default.conf",
])
.output()
.await;
let provider_src = "/opt/archipelago/web-ui/nostr-provider.js";
if tokio::fs::metadata(provider_src).await.is_ok() {
let _ = tokio::process::Command::new("podman")
.args([
"cp",
provider_src,
"indeedhub:/usr/share/nginx/html/nostr-provider.js",
])
.output()
.await;
}
let check = tokio::process::Command::new("podman")
.args([
"exec",
"indeedhub",
"grep",
"-q",
"nostr-provider",
"/etc/nginx/conf.d/default.conf",
])
.output()
.await;
let already_patched = check.map(|o| o.status.success()).unwrap_or(false);
if !already_patched {
let cat_out = tokio::process::Command::new("podman")
.args(["exec", "indeedhub", "cat", "/etc/nginx/conf.d/default.conf"])
.output()
.await;
if let Ok(out) = cat_out {
if out.status.success() {
let conf = String::from_utf8_lossy(&out.stdout).to_string();
let conf = conf.replace(
"location = /sw.js {",
"location = /nostr-provider.js {\n\
add_header Cache-Control \"no-cache, no-store, must-revalidate\";\n\
expires off;\n\
}\n\n\
location = /sw.js {",
);
let conf = if conf.contains("try_files") && !conf.contains("sub_filter") {
conf.replacen(
"try_files $uri $uri/ /index.html;",
"try_files $uri $uri/ /index.html;\n\
sub_filter_once on;\n\
sub_filter '</head>' '<script src=\"/nostr-provider.js\"></script></head>';",
1,
)
} else {
conf
};
let tmp_path = "/tmp/indeedhub-nginx-patch.conf";
if tokio::fs::write(tmp_path, &conf).await.is_ok() {
let _ = tokio::process::Command::new("podman")
.args(["cp", tmp_path, "indeedhub:/etc/nginx/conf.d/default.conf"])
.output()
.await;
let _ = tokio::fs::remove_file(tmp_path).await;
}
}
}
}
let _ = tokio::process::Command::new("podman")
.args([
"exec",
"indeedhub",
"sed",
"-i",
"s|proxy_set_header X-Forwarded-Prefix /api;|proxy_set_header X-Forwarded-Prefix $http_x_forwarded_prefix/api;|",
"/etc/nginx/conf.d/default.conf",
])
.output()
.await;
let _ = tokio::process::Command::new("podman")
.args(["exec", "indeedhub", "nginx", "-s", "reload"])
.output()
.await;
}
/// Outcome of `reconcile_all` for a single app.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ReconcileAction {
@ -1374,13 +1269,12 @@ impl ProdContainerOrchestrator {
mode: ReconcileMode,
) -> Result<ReconcileAction> {
let app_id = lm.manifest.app.id.clone();
if app_id == "indeedhub" {
// IndeedHub is a multi-container stack installed by the package
// stack path. Boot reconcile must not fresh-install the catalog
// manifest, but it does need to start/repair an already-installed
// stack and reapply the frontend's Nostr provider patch after boot.
return self.reconcile_indeedhub_stack(mode).await;
}
// IndeedHub used to be a hardcoded orchestrator special-case
// (reconcile_indeedhub_stack + a dependency-DNS gate) that refused to
// create the frontend from its manifest. It is now fully manifest-driven
// (apps/indeedhub-* + apps/indeedhub): network_aliases, generated_secrets,
// dependencies, and the post_install nginx hook live in the manifests, so
// every member — frontend included — flows through the generic path here.
let lock = self.app_lock(&app_id).await;
let _guard = lock.lock().await;
@ -2321,10 +2215,6 @@ impl ProdContainerOrchestrator {
self.ensure_btcpay_stack_dirs().await?;
Ok(Some(HookOutcome::Unchanged))
}
"indeedhub" => {
self.start_indeedhub_backends().await?;
Ok(Some(HookOutcome::Unchanged))
}
"grafana" => {
self.cleanup_stale_grafana_port().await;
Ok(Some(HookOutcome::Unchanged))
@ -2398,272 +2288,6 @@ impl ProdContainerOrchestrator {
Ok(())
}
async fn start_indeedhub_backends(&self) -> Result<()> {
let _ = tokio::process::Command::new("podman")
.args(["network", "create", "indeedhub-net"])
.output()
.await;
for name in INDEEDHUB_BACKEND_CONTAINERS {
let status = match self.runtime.get_container_status(name).await {
Ok(status) => status,
Err(_) => continue,
};
if !matches!(status.state, ContainerState::Running) {
if let Err(err) = podman_user_scope(&["start", name]).await {
tracing::warn!(
container = %name,
error = %err,
"IndeedHub scoped backend start failed; falling back to runtime start"
);
self.runtime
.start_container(name)
.await
.with_context(|| format!("start IndeedHub backend {name}"))?;
}
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
}
}
self.repair_indeedhub_network_aliases().await;
self.wait_for_indeedhub_dependencies_ready(120).await?;
Ok(())
}
async fn reconcile_indeedhub_stack(&self, mode: ReconcileMode) -> Result<ReconcileAction> {
let frontend_status = match self.runtime.get_container_status("indeedhub").await {
Ok(status) => status,
Err(_) => {
if mode == ReconcileMode::ExistingOnly {
return Ok(ReconcileAction::Left("absent".to_string()));
}
// Fresh stack creation is owned by package::stacks so we do not
// create a single broken frontend container from the manifest.
return Ok(ReconcileAction::Left("stack-managed".to_string()));
}
};
self.start_indeedhub_backends().await?;
let mut started = false;
match frontend_status.state {
ContainerState::Running => {}
ContainerState::Stopped
| ContainerState::Exited
| ContainerState::Created
| ContainerState::Stopping => {
if let Err(err) = podman_user_scope(&["start", "indeedhub"]).await {
tracing::warn!(
error = %err,
"IndeedHub scoped frontend start failed; falling back to runtime start"
);
self.runtime
.start_container("indeedhub")
.await
.context("start IndeedHub frontend during reconcile")?;
}
started = true;
}
ContainerState::Paused => return Ok(ReconcileAction::Left("paused".to_string())),
ContainerState::Unknown(s) => return Ok(ReconcileAction::Left(s)),
}
tokio::time::sleep(std::time::Duration::from_secs(5)).await;
self.repair_indeedhub_network_aliases().await;
patch_indeedhub_nostr_provider().await;
let frontend_stable = wait_for_container_stable_running(
self.runtime.as_ref(),
"indeedhub",
5,
INDEEDHUB_FRONTEND_READY_TIMEOUT_SECS,
)
.await;
if frontend_stable.is_err() || !wait_for_host_port(7778, 10).await {
tracing::warn!(
error = ?frontend_stable.err(),
"IndeedHub frontend did not stay reachable after reconcile; restarting"
);
let _ = self.runtime.stop_container("indeedhub").await;
if let Err(err) = podman_user_scope(&["start", "indeedhub"]).await {
tracing::warn!(
error = %err,
"IndeedHub scoped frontend restart failed; falling back to runtime start"
);
self.runtime
.start_container("indeedhub")
.await
.context("restart IndeedHub frontend after failed readiness")?;
}
started = true;
tokio::time::sleep(std::time::Duration::from_secs(5)).await;
patch_indeedhub_nostr_provider().await;
wait_for_container_stable_running(
self.runtime.as_ref(),
"indeedhub",
5,
INDEEDHUB_FRONTEND_READY_TIMEOUT_SECS,
)
.await
.context("IndeedHub frontend did not remain running after restart")?;
if !wait_for_host_port(7778, 30).await {
return Err(anyhow::anyhow!(
"IndeedHub frontend did not expose host port 7778 after restart"
));
}
}
if started {
Ok(ReconcileAction::Started)
} else {
Ok(ReconcileAction::NoOp)
}
}
async fn wait_for_indeedhub_dependencies_ready(&self, timeout_secs: u64) -> Result<()> {
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(timeout_secs);
let mut last = String::from("not checked");
loop {
let mut all_running = true;
for name in INDEEDHUB_BACKEND_CONTAINERS {
match self.runtime.get_container_status(name).await {
Ok(status) if matches!(status.state, ContainerState::Running) => {}
Ok(status) => {
all_running = false;
last = format!("{name} state {:?}", status.state);
break;
}
Err(err) => {
all_running = false;
last = format!("{name} status error: {err}");
break;
}
}
}
if all_running && self.indeedhub_api_dependency_dns_ready().await {
return Ok(());
}
if all_running {
last = "indeedhub-api dependency DNS not ready".to_string();
}
if std::time::Instant::now() >= deadline {
return Err(anyhow::anyhow!(
"IndeedHub dependencies were not ready within {}s ({})",
timeout_secs,
last
));
}
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
}
}
async fn indeedhub_api_dependency_dns_ready(&self) -> bool {
let aliases_ready = self.indeedhub_required_aliases_present().await;
if cfg!(test) {
return true;
}
for host in ["postgres", "redis", "minio", "relay"] {
let Ok(Ok(output)) = tokio::time::timeout(
std::time::Duration::from_secs(5),
tokio::process::Command::new("podman")
.args(["exec", "indeedhub-api", "getent", "hosts", host])
.output(),
)
.await
else {
return aliases_ready;
};
if !output.status.success() {
return aliases_ready;
}
}
true
}
async fn indeedhub_required_aliases_present(&self) -> bool {
for (container, alias) in [
("indeedhub-postgres", "postgres"),
("indeedhub-redis", "redis"),
("indeedhub-minio", "minio"),
("indeedhub-relay", "relay"),
("indeedhub-api", "api"),
("indeedhub", "indeedhub"),
] {
if !self.indeedhub_alias_present(container, alias).await {
return false;
}
}
true
}
async fn repair_indeedhub_network_aliases(&self) {
for (container, alias) in [
("indeedhub-postgres", "postgres"),
("indeedhub-redis", "redis"),
("indeedhub-minio", "minio"),
("indeedhub-relay", "relay"),
("indeedhub-api", "api"),
("indeedhub", "indeedhub"),
] {
let exists = tokio::process::Command::new("podman")
.args(["container", "exists", container])
.status()
.await
.map(|s| s.success())
.unwrap_or(false);
if !exists {
continue;
}
if self.indeedhub_alias_present(container, alias).await {
continue;
}
let _ = tokio::process::Command::new("podman")
.args(["network", "disconnect", "-f", "indeedhub-net", container])
.output()
.await;
let _ = tokio::process::Command::new("podman")
.args([
"network",
"connect",
"--alias",
alias,
"indeedhub-net",
container,
])
.output()
.await;
}
}
async fn indeedhub_alias_present(&self, container: &str, alias: &str) -> bool {
let output = match tokio::process::Command::new("podman")
.args([
"inspect",
container,
"--format",
"{{json .NetworkSettings.Networks}}",
])
.output()
.await
{
Ok(output) if output.status.success() => output,
_ => return false,
};
let Ok(networks) = serde_json::from_slice::<serde_json::Value>(&output.stdout) else {
return false;
};
networks
.get("indeedhub-net")
.and_then(|network| network.get("Aliases"))
.and_then(|aliases| aliases.as_array())
.map(|aliases| aliases.iter().any(|value| value.as_str() == Some(alias)))
.unwrap_or(false)
}
async fn cleanup_stale_grafana_port(&self) {
let _ = tokio::process::Command::new("pkill")
.args(["-f", "pasta.*3001"])