Merge branch 'bitcoin-version-bulletproof' into mesh-multiversion-integration

2026-06-29 06:45:50 -04:00 · 2026-06-29 06:45:50 -04:00 · 067002b04b
commit 067002b04b
parent 20f762cb2c f4f45c1a09
13 changed files with 478 additions and 76 deletions
--- a/apps/bitcoin-core/Dockerfile
+++ b/apps/bitcoin-core/Dockerfile
@ -22,7 +22,12 @@ RUN set -eux; \
 COPY bin/bitcoind /usr/local/bin/bitcoind
 COPY bin/bitcoin-cli /usr/local/bin/bitcoin-cli
 RUN chmod 0755 /usr/local/bin/bitcoind /usr/local/bin/bitcoin-cli
-USER bitcoin
+# Run as (container) root, like the legacy hand-built :latest image. Rootless
+# Podman maps container-root to the unprivileged host service user; the manifest
+# grants CAP_DAC_OVERRIDE so bitcoind can read its data dir, which the
+# orchestrator chowns to the data_uid (host 100101 / container uid 102), not to
+# this image's `bitcoin` user. A non-root USER can't read existing chain data and
+# bitcoind crash-loops with "Error initializing block database".
 WORKDIR /home/bitcoin
 VOLUME ["/home/bitcoin/.bitcoin"]
 EXPOSE 8332 8333
--- a/apps/bitcoin-knots/Dockerfile
+++ b/apps/bitcoin-knots/Dockerfile
@ -23,7 +23,12 @@ RUN set -eux; \
 COPY bin/bitcoind /usr/local/bin/bitcoind
 COPY bin/bitcoin-cli /usr/local/bin/bitcoin-cli
 RUN chmod 0755 /usr/local/bin/bitcoind /usr/local/bin/bitcoin-cli
-USER bitcoin
+# Run as (container) root, like the legacy hand-built :latest image. Rootless
+# Podman maps container-root to the unprivileged host service user; the manifest
+# grants CAP_DAC_OVERRIDE so bitcoind can read its data dir, which the
+# orchestrator chowns to the data_uid (host 100101 / container uid 102), not to
+# this image's `bitcoin` user. A non-root USER can't read existing chain data and
+# bitcoind crash-loops with "Error initializing block database".
 WORKDIR /home/bitcoin
 VOLUME ["/home/bitcoin/.bitcoin"]
 EXPOSE 8332 8333
--- a/core/archipelago/src/container/prod_orchestrator.rs
+++ b/core/archipelago/src/container/prod_orchestrator.rs
@ -929,6 +929,51 @@ pub struct AdoptionReport {
    pub adopted: Vec<String>,
 }

+/// Decouple the app image from the shipped manifest: prefer the remote app
+/// catalog when it covers this app with a same-repo image, and let a
+/// runner-pinned version win over the catalog default so "install/switch to
+/// the version I chose" is honored across every install + recreate. Falls
+/// through to the catalog default when unpinned or the pin can't be resolved
+/// (unknown/repo-mismatch), and to the manifest image when the catalog is
+/// absent/uncovered.
+///
+/// Applied in-place to a cloned manifest. MUST run anywhere a quadlet is
+/// rendered — both install_fresh (pull + create) and sync_quadlet_unit (the
+/// reconciler's drift re-render); otherwise the reconciler rewrites the unit
+/// back to the manifest's shipped `:latest` tag and silently reverts a pinned
+/// version on the next tick.
+fn resolve_catalog_image(resolved_manifest: &mut AppManifest) {
+    let Some(current) = resolved_manifest.app.container.image.clone() else {
+        return;
+    };
+    let app_id = resolved_manifest.app.id.clone();
+    let resolved_image = crate::container::version_config::pinned_version(&app_id)
+        .and_then(|v| {
+            crate::container::app_catalog::catalog_image_for_version(&app_id, &v, &current).or_else(
+                || {
+                    tracing::warn!(
+                        app_id = %app_id,
+                        pinned = %v,
+                        "version_config: pinned version not resolvable in catalog — using default"
+                    );
+                    None
+                },
+            )
+        })
+        .or_else(|| crate::container::app_catalog::catalog_image_override(&app_id, &current));
+    if let Some(catalog_image) = resolved_image {
+        if catalog_image != current {
+            tracing::info!(
+                app_id = %app_id,
+                from = %current,
+                to = %catalog_image,
+                "app-catalog: overriding manifest image"
+            );
+            resolved_manifest.app.container.image = Some(catalog_image);
+        }
+    }
+}
+
 /// Internal: track a manifest together with the absolute directory it was loaded
 /// from, so Build sources can resolve relative `context:` paths.
 #[derive(Debug, Clone)]
@ -1809,45 +1854,7 @@ impl ProdContainerOrchestrator {
        self.ensure_app_secrets(&lm.manifest.app.id).await?;
        let mut resolved_manifest = lm.manifest.clone();
        self.resolve_dynamic_env(&mut resolved_manifest)?;
-
-        // Decouple the app image from the shipped manifest: prefer the remote
-        // app catalog when it covers this app with a same-repo image. This makes
-        // both the pull below and create_container() below use the catalog tag,
-        // so an app update no longer requires a binary/runtime release. Falls
-        // back to the manifest image when the catalog is absent/uncovered.
-        if let Some(current) = resolved_manifest.app.container.image.clone() {
-            let app_id = resolved_manifest.app.id.clone();
-            // Multi-version support: a runner-pinned version wins over the catalog
-            // default so "install/switch to the version I chose" is honored across
-            // every install + recreate. Falls through to the catalog default when
-            // unpinned or the pin can't be resolved (unknown/repo-mismatch).
-            let resolved_image = crate::container::version_config::pinned_version(&app_id)
-                .and_then(|v| {
-                    crate::container::app_catalog::catalog_image_for_version(&app_id, &v, &current)
-                        .or_else(|| {
-                            tracing::warn!(
-                                app_id = %app_id,
-                                pinned = %v,
-                                "version_config: pinned version not resolvable in catalog — using default"
-                            );
-                            None
-                        })
-                })
-                .or_else(|| {
-                    crate::container::app_catalog::catalog_image_override(&app_id, &current)
-                });
-            if let Some(catalog_image) = resolved_image {
-                if catalog_image != current {
-                    tracing::info!(
-                        app_id = %app_id,
-                        from = %current,
-                        to = %catalog_image,
-                        "app-catalog: overriding manifest image"
-                    );
-                    resolved_manifest.app.container.image = Some(catalog_image);
-                }
-            }
-        }
+        resolve_catalog_image(&mut resolved_manifest);

        let resolved = resolved_manifest.app.container.resolve().ok_or_else(|| {
            anyhow::anyhow!(
@ -2206,6 +2213,10 @@ impl ProdContainerOrchestrator {

        let mut resolved = lm.manifest.clone();
        self.resolve_dynamic_env(&mut resolved)?;
+        // Same catalog/pinned-version image resolution the installer applies, so
+        // the drift re-render doesn't revert a pinned version back to the
+        // manifest's shipped `:latest` tag on the next reconcile tick.
+        resolve_catalog_image(&mut resolved);
        let unit = quadlet::QuadletUnit::from_manifest(&resolved, name);
        let new_body = unit.render();
        let restart_for_port_change = quadlet::publish_ports_changed(&old_body, &new_body);
--- a/core/archipelago/src/container/quadlet.rs
+++ b/core/archipelago/src/container/quadlet.rs
@ -268,14 +268,21 @@ impl QuadletUnit {
            let _ = writeln!(s, "HealthTimeout={}", h.timeout);
            let _ = writeln!(s, "HealthRetries={}", h.retries);
        }
-        if let Some(ep) = &self.entrypoint {
-            // Quadlet's Exec= replaces the image entrypoint+cmd. When
-            // the manifest provides both entrypoint and command we
-            // concatenate; if only command is set we'll emit that on
-            // its own below.
-            let mut parts: Vec<String> = ep.clone();
+        if let Some((first, rest)) = self.entrypoint.as_deref().and_then(<[String]>::split_first) {
+            // Quadlet's Exec= sets only the command (the args passed to the
+            // image's ENTRYPOINT) — it does NOT replace the entrypoint. So a
+            // manifest entrypoint like `sh -lc` must be emitted as a real
+            // Entrypoint= override; otherwise it gets appended to whatever
+            // ENTRYPOINT the image baked in (e.g. the versioned bitcoind
+            // images use `ENTRYPOINT ["bitcoind"]`, which turned the wrapper
+            // into `bitcoind sh -lc ...` and crash-looped). Emitting
+            // Entrypoint= makes the unit independent of the image's entrypoint.
+            let _ = writeln!(s, "Entrypoint={first}");
+            let mut parts: Vec<String> = rest.to_vec();
            parts.extend(self.command.iter().cloned());
-            let _ = writeln!(s, "Exec={}", shell_join(&parts));
+            if !parts.is_empty() {
+                let _ = writeln!(s, "Exec={}", shell_join(&parts));
+            }
        } else if !self.command.is_empty() {
            let _ = writeln!(s, "Exec={}", shell_join(&self.command));
        }
@ -769,9 +776,11 @@ pub fn network_aliases_changed(old_body: &str, new_body: &str) -> bool {
 }

 pub fn exec_changed(old_body: &str, new_body: &str) -> bool {
-    let old_exec = directive_values(old_body, "Exec=");
-    let new_exec = directive_values(new_body, "Exec=");
-    old_exec != new_exec
+    // Entrypoint= and Exec= together define what the container runs, so a drift
+    // in either must recreate the container (e.g. when this renderer first
+    // splits a folded `Exec=sh -lc ...` into `Entrypoint=sh` + `Exec=-lc ...`).
+    directive_values(old_body, "Exec=") != directive_values(new_body, "Exec=")
+        || directive_values(old_body, "Entrypoint=") != directive_values(new_body, "Entrypoint=")
 }

 fn directive_values(unit_body: &str, prefix: &str) -> Vec<String> {
@ -1063,7 +1072,10 @@ mod tests {
        assert!(s.contains("ReadOnly=true"));
        assert!(s.contains("NoNewPrivileges=true"));
        assert!(s.contains("PodmanArgs=--cpus=2"));
-        assert!(s.contains("Exec=/usr/local/bin/bitcoind -server=1 -rpcbind=0.0.0.0"));
+        // Manifest entrypoint becomes a real Entrypoint= override (not folded
+        // into Exec=), so the unit doesn't depend on the image's own ENTRYPOINT.
+        assert!(s.contains("Entrypoint=/usr/local/bin/bitcoind"));
+        assert!(s.contains("Exec=-server=1 -rpcbind=0.0.0.0"));
        assert!(s.contains("Restart=on-failure"));
        assert!(s.contains("Network=archy-net"));
    }
--- a/docs/PRODUCTION-MASTER-PLAN.md
+++ b/docs/PRODUCTION-MASTER-PLAN.md
@ -135,6 +135,16 @@ After the 2026-06-23 multinode test deploy (latest backend + UX frontend to .116
 3. **§6c Lifecycle perfection** (workstream F) — the comprehensive uninstall/reinstall +
   progress-UI + all-apps gate expansion below.

+## 6b-bis. Bitcoin multi-version bulletproofing (2026-06-29) — READY TO MERGE + DEPLOY
+
+Branch `bitcoin-version-bulletproof` (base `095a76cd`). Fixes the "switch version silently
+fails / crash-loops" class + a data-access mismatch that can corrupt a node's index. All
+code + images + catalog + frontend DONE; **.228** carries it (Knots chainstate mid-reindex
+recovery). The **coordinated fleet rollout** (OTA binary+frontend, mirror catalog publish,
+`:latest` repoint sequencing, full switch-matrix test) is the remaining work — fold it into
+the next release. **Authoritative detail + exact remaining steps + test matrix →
+`docs/bitcoin-version-bulletproof-rollout.md`.** Pairs with `docs/bitcoin-multi-version-design.md`.
+
 ## 6c. Lifecycle perfection — what "green" MISSED (workstream F, the perfection bar)

 **Why this exists:** the 2026-06-23 single-node gate went 5×-green but is **NOT** the
--- a/docs/bitcoin-version-bulletproof-rollout.md
+++ b/docs/bitcoin-version-bulletproof-rollout.md
@ -0,0 +1,131 @@
+# Bitcoin Multi-Version — Bulletproofing & Rollout (handoff)
+
+> **Status 2026-06-29:** code + images + catalog + frontend DONE on branch
+> `bitcoin-version-bulletproof` (base commit `095a76cd`, plus the catalog-generator
+> + handoff follow-ups). **.228 is the test node**: binary + frontend + catalog are
+> live there; its Knots chainstate is mid-**reindex recovery** (see §5). The fleet
+> rollout (OTA binary+frontend, mirror catalog publish, `:latest` repoint) is the
+> **coordinated step the other agent owns** — see §4. Pairs with
+> `docs/bitcoin-multi-version-design.md` (the original design).
+
+## 1. What was broken (root causes)
+
+User report: "switched Knots to `v29.3.knots20260508`, version didn't update in the UI."
+Three **stacked** bugs, plus a data-corruption hazard:
+
+1. **Reconciler reverted the pin.** `prod_orchestrator::sync_quadlet_unit` re-rendered the
+   quadlet every reconcile tick using the manifest's `:latest`, ignoring the per-app
+   pinned version → any switch silently reverted within one tick.
+2. **Entrypoint render bug.** The renderer folded the manifest `entrypoint: ["sh","-lc"]`
+   into `Exec=`. That only works when the image ENTRYPOINT is a passthrough shell wrapper.
+   The versioned images use `ENTRYPOINT ["bitcoind"]`, so `Exec=sh -lc …` became
+   `bitcoind sh -lc …` → `unexpected token 'sh'` → crash loop.
+3. **Image USER divergence.** The versioned images were built `USER bitcoin` (uid 1000);
+   the legacy `:latest` ran as **root**. Chain data is owned by the `data_uid`
+   (host 100101 / container uid 102). Root reads it via `CAP_DAC_OVERRIDE` (granted in the
+   manifest); uid-1000 cannot → `Error initializing block database`.
+4. **Data hazard (already hit on .228).** Repeated failed starts under mixed UIDs left
+   bitcoind's two LevelDBs (`blocks/index/` + `chainstate/`) truncated to KB stubs while
+   the raw `blocks/blk*.dat` (797 GB) stayed intact. Recovery = `bitcoind -reindex` from
+   local blocks (no re-download). The uniform-root image fix (below) removes the mixed-UID
+   cause going forward; the proper switch flow was already data-safe (600s stop grace,
+   clean stop→rm→recreate, conflict-stops the other impl — they share port 8332 + datadir
+   `/var/lib/archipelago/bitcoin`).
+
+## 2. What was fixed (all on the branch)
+
+- **Renderer** (`core/archipelago/src/container/`):
+  - `prod_orchestrator.rs`: factored `resolve_catalog_image()` (catalog/pinned-version →
+    image) and call it in BOTH `install_fresh` and `sync_quadlet_unit` — the pin now
+    survives reconcile.
+  - `quadlet.rs`: emit a real `Entrypoint=<first>` + `Exec=<rest+cmd>` instead of folding;
+    `exec_changed` now also diffs `Entrypoint=` so the recreate fires. Validated against
+    the live podman 5.4.2 quadlet generator.
+- **Images** (`scripts/build-bitcoin-image.sh`, `apps/bitcoin-{knots,core}/Dockerfile`):
+  removed `USER bitcoin` → run as **container-root** like legacy (still 100% rootless:
+  container-root maps to the unprivileged host service user; `CAP_DAC_OVERRIDE` from the
+  manifest lets bitcoind read the `data_uid`-owned datadir). **All** images rebuilt root +
+  pushed to the mirror (`146.59.87.168:3000/lfg2025`):
+  - Knots: `29.3.knots20260508`, `29.3.knots20260507`, `29.3.knots20260210`, `29.2.knots20251110`
+  - Core: `25.2 26.2 27.2 28.4 29.2 29.3 30.2 31.0` + `latest` (→31.0)
+- **Catalog** (`scripts/generate-app-catalog.sh` VERSIONS map + regenerated
+  `releases/app-catalog.json`): Knots & Core `versions[]` populated; the generator now
+  forces top-level `version` == the `default` entry's version (the `169ff2e2` invariant)
+  regardless of the manifest version. Knots `latest` entry points at the newest **dated**
+  image (`29.3.knots20260508`) so "Always use latest" = newest on fixed-binary nodes.
+- **Frontend** (`neode-ui/`):
+  - `AppSidebar.vue`: rename the latest option to **"Always use the latest version"**
+    (no `v` prefix), fix right padding, and `pickSelection()` guarantees the bound value is
+    a real option (fixes the blank dropdown).
+  - New `components/InstallVersionModal.vue`: full-screen version chooser shown from the
+    App Store / Discover **card** install button for multi-version apps — app icon +
+    "Install <name>", latest pre-selected. Wired in `Discover.vue handleInstall`.
+  - i18n keys: `appDetails.alwaysUseLatestVersion`, `marketplace.installModalTitle/Hint`.
+
+## 3. Current live state on .228 (test node)
+
+- Binary with both renderer fixes: **deployed** (`/usr/local/bin/archipelago`).
+- New frontend bundle: **deployed** to `/opt/archipelago/web-ui` (hard-refresh to see it).
+- Updated catalog: placed at `/var/lib/archipelago/app-catalog.json` (local override —
+  will refresh from the mirror's OLDER copy at the next hourly fetch until §4 publishes it).
+- Knots: `bitcoin-knots` service held **stopped** (`package.stop`, user_stopped);
+  a detached `bitcoin-knots-reindex` container is rebuilding the index+UTXO (§5).
+
+## 4. Remaining — coordinated fleet rollout (OTHER AGENT)
+
+Do this together with the other workstream's release, AFTER both are ready:
+
+1. **Merge** branch `bitcoin-version-bulletproof` into the release line.
+2. **Build + OTA** the binary + frontend (these carry the renderer fix + UI). The renderer
+   fix is a **hard prerequisite** for the new images everywhere — see fleet-safety below.
+3. **Publish the catalog** to the mirror (push `releases/app-catalog.json` to gitea-vps2
+   `main`, the raw URL nodes fetch hourly). The current catalog is **fleet-safe even before
+   the binary lands**: unpinned/auto-update nodes resolve via the manifest's floating
+   `:latest` (still the legacy image); only explicit version selection (needs the new UI)
+   uses the new root images.
+4. **Only AFTER the binary is fleet-wide:** optionally repoint the `bitcoin-knots:latest`
+   tag → `29.3.knots20260508` (root) and simplify the catalog `latest` entry back to the
+   `:latest` tag. **Do NOT repoint `:latest` before then** — old-binary nodes fold
+   `Exec=sh -lc …` and would crash on an `ENTRYPOINT ["bitcoind"]` image. (Core never
+   worked on old binaries — it always shipped `ENTRYPOINT ["bitcoind"]` — so Core has no
+   such constraint.)
+5. **Verify the full switch matrix** on a healthy node (§6).
+
+## 5. Finishing .228's reindex (OTHER AGENT owns this — not babysat by the original author)
+
+The detached `bitcoin-knots-reindex` container runs the new **root** `29.3.knots20260508`
+image with `-reindex -server=0` against `/var/lib/archipelago/bitcoin`. It holds the datadir
+lock, so the managed service (held stopped) can't collide. When it has connected blocks up
+to ~the prior tip (height ≥ ~955800) it's done; then:
+
+```sh
+# on .228 (SSH/sudo/UI pw all: ThisIsWeb54321@)
+podman stop -t 600 bitcoin-knots-reindex && podman rm bitcoin-knots-reindex
+# start the managed service via RPC (sets desired=running, clears user_stopped):
+#   package.start {id: bitcoin-knots}  (POST https://127.0.0.1/rpc/v1, CSRF: echo csrf_token cookie as X-CSRF-Token)
+# verify:
+podman exec bitcoin-knots sh -lc '$(command -v bitcoind) --version | head -1'   # → v29.3.knots20260508
+# RPC up → the Bitcoin UI populates; it syncs the gap to tip.
+```
+The "Bitcoin RPC connection refused (127.0.0.1:8332)" the UI shows is EXPECTED until this
+swap (reindex runs with RPC off).
+
+## 6. Switch-matrix test plan (what "bulletproof" must prove)
+
+On a healthy node, each step must end with bitcoind running + RPC answering + syncing, with
+NO `Error initializing block database` and NO data loss:
+- Knots: switch `latest` → `29.3.knots20260507` → `29.3.knots20260210` → back to `latest`.
+- Core: install `latest`; switch `31.0` → `28.4.0`.
+- **Knots ↔ Core** (shared datadir/port): Knots→Core upgrade path (Core ≥ data version) and
+  the reverse. **Cross-major DOWNGRADES** (e.g. 29.x data → Core 28.4) legitimately need a
+  reindex — the UI already surfaces a downgrade warning; confirm it does and that confirming
+  reindexes cleanly rather than crash-looping.
+- Reboot survival after each switch.
+
+## 7. Notes / assumptions
+
+- **"29.2"** in the request doesn't exist as a Knots build (404 upstream); added as **Bitcoin
+  Core 29.2** (exists). Revisit if a Knots 29.2 was meant.
+- Reindex is unavoidable ONLY because .228's index was already corrupted by the pre-fix
+  crash loop; a normal switch on the fixed binary does NOT reindex.
+- Creds for .228: SSH/sudo + UI/RPC all `ThisIsWeb54321@`.
--- a/neode-ui/src/components/InstallVersionModal.vue
+++ b/neode-ui/src/components/InstallVersionModal.vue
@ -0,0 +1,120 @@
+<template>
+  <BaseModal :show="show" title="" max-width="max-w-lg" @close="emit('close')">
+    <!-- Header: app icon + "Install Bitcoin Knots/Core" -->
+    <div class="flex items-center gap-4 mb-5 -mt-2">
+      <img
+        v-if="app?.icon"
+        :src="app.icon"
+        :alt="app?.title || ''"
+        class="w-14 h-14 rounded-xl shadow-lg shrink-0"
+      />
+      <div v-else class="w-14 h-14 rounded-xl bg-white/10 flex items-center justify-center shrink-0">
+        <svg class="w-7 h-7 text-white/40" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+          <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M20 7l-8-4-8 4m16 0l-8 4m8-4v10l-8 4m0-10L4 7m8 4v10M4 7v10l8 4" />
+        </svg>
+      </div>
+      <h3 class="text-xl font-semibold text-white leading-snug">
+        {{ t('marketplace.installModalTitle', { name: app?.title || appId }) }}
+      </h3>
+    </div>
+
+    <div v-if="loading" class="py-6 text-center text-white/60 text-sm">{{ t('common.loading') }}</div>
+
+    <div v-else class="space-y-2">
+      <label class="block text-white/60 text-sm">{{ t('appDetails.selectVersion') }}</label>
+      <select
+        v-model="selected"
+        class="w-full rounded-lg bg-white/[0.06] border border-white/10 text-white pl-3 pr-9 py-2 text-sm focus:outline-none focus:border-blue-400/60"
+      >
+        <option v-for="v in versions" :key="v.version" :value="v.version">{{ optionLabel(v) }}</option>
+      </select>
+      <p class="text-white/40 text-xs">{{ t('marketplace.installModalHint') }}</p>
+    </div>
+
+    <template #footer>
+      <div class="flex gap-2 mt-6">
+        <button
+          type="button"
+          class="flex-1 rounded-lg bg-blue-600 hover:bg-blue-500 disabled:opacity-50 text-white text-sm font-semibold py-2.5 transition-colors"
+          :disabled="loading || !selected"
+          @click="confirm"
+        >
+          {{ t('common.install') }}
+        </button>
+        <button
+          type="button"
+          class="rounded-lg bg-white/10 hover:bg-white/20 text-white text-sm font-medium px-5 py-2.5 transition-colors"
+          @click="emit('close')"
+        >
+          {{ t('common.cancel') }}
+        </button>
+      </div>
+    </template>
+  </BaseModal>
+</template>
+
+<script setup lang="ts">
+import { ref, watch } from 'vue'
+import { useI18n } from 'vue-i18n'
+import BaseModal from './BaseModal.vue'
+import { rpcClient, type CatalogVersionInfo } from '../api/rpc-client'
+import { displayVersion } from '@/utils/version'
+
+const props = defineProps<{
+  show: boolean
+  appId: string
+  app: { id: string; title?: string; icon?: string } | null
+}>()
+
+const emit = defineEmits<{
+  close: []
+  // Emits the version string the runner chose (e.g. "latest" or "29.3.knots20260508").
+  confirm: [version: string]
+}>()
+
+const { t } = useI18n()
+const loading = ref(false)
+const versions = ref<CatalogVersionInfo[]>([])
+const selected = ref('')
+
+// Latest reads as a sentence (no "v" prefix); concrete versions are normalized.
+function optionLabel(v: CatalogVersionInfo): string {
+  if (v.version === 'latest') return t('appDetails.alwaysUseLatestVersion')
+  let label = displayVersion(v.version)
+  if (v.deprecated) label += ' (deprecated)'
+  if (v.eol) label += ` · EOL ${v.eol}`
+  return label
+}
+
+async function load() {
+  loading.value = true
+  versions.value = []
+  selected.value = ''
+  try {
+    const info = await rpcClient.getPackageVersions(props.appId)
+    // catalog_versions() returns the list default(=latest)-first, so versions[0]
+    // is the latest — pre-select it.
+    versions.value = info.versions || []
+    selected.value = info.default || versions.value.find((v) => v.default)?.version || versions.value[0]?.version || 'latest'
+  } catch (err) {
+    if (import.meta.env.DEV) console.warn('[InstallVersionModal] getPackageVersions failed:', err)
+    // Fall back to the floating "latest" so the install can still proceed.
+    selected.value = 'latest'
+  } finally {
+    loading.value = false
+  }
+}
+
+function confirm() {
+  if (!selected.value) return
+  emit('confirm', selected.value)
+}
+
+watch(
+  () => props.show,
+  (open) => {
+    if (open) void load()
+  },
+  { immediate: true },
+)
+</script>
--- a/neode-ui/src/locales/en.json
+++ b/neode-ui/src/locales/en.json
@ -330,6 +330,8 @@
  "marketplace": {
    "title": "App Store",
    "subtitle": "Discover and install apps for your new sovereign life",
+    "installModalTitle": "Install {name}",
+    "installModalHint": "Choose a version to install. The latest is recommended.",
    "curatedTab": "Curated",
    "communityTab": "Community",
    "nostrCommunityTab": "Nostr Community",
@ -572,6 +574,7 @@
    "versionUpdates": "Version & Updates",
    "runningVersion": "Running version",
    "selectVersion": "Version",
+    "alwaysUseLatestVersion": "Always use the latest version",
    "autoUpdateLatest": "Auto-update to latest",
    "autoUpdatePinnedNote": "Auto-update is disabled while a version is pinned.",
    "confirmDowngrade": "Downgrade anyway",
--- a/neode-ui/src/views/Discover.vue
+++ b/neode-ui/src/views/Discover.vue
@ -207,6 +207,15 @@
      <p class="text-white/60 text-xl mt-4 font-mono">// Cypherpunks write code. We run nodes.</p>
    </div>

+    <!-- First-install version chooser (Bitcoin Knots / Core) -->
+    <InstallVersionModal
+      :show="showInstallModal"
+      :app-id="installModalApp?.id || ''"
+      :app="installModalApp"
+      @close="showInstallModal = false; installModalApp = null"
+      @confirm="onInstallModalConfirm"
+    />
+
  </div>
 </template>

@ -228,6 +237,7 @@ import { APP_STORE_SECTIONS } from './appStoreCategories'
 import DiscoverHero from './discover/DiscoverHero.vue'
 import FeaturedApps from './discover/FeaturedApps.vue'
 import AppGrid from './discover/AppGrid.vue'
+import InstallVersionModal from '@/components/InstallVersionModal.vue'
 import type { MarketplaceApp, FeaturedApp } from './discover/types'
 import { getCuratedAppList, INSTALLED_ALIASES, FEATURED_DEFINITIONS, categorizeCommunityApp, fetchAppCatalog, type CatalogFeatured } from './discover/curatedApps'

@ -262,6 +272,10 @@ const appStoreSections = computed(() => APP_STORE_SECTIONS)
 // been removed in favour of the store's phase-aware mapping.
 const installingApps = serverStore.installingApps

+// First-install version-choice modal (multi-version apps: Bitcoin Knots / Core)
+const showInstallModal = ref(false)
+const installModalApp = ref<MarketplaceApp | null>(null)
+
 function navigateToMarketplace(categoryId: string) {
  router.push({ name: 'marketplace', query: { category: categoryId } })
 }
@ -427,19 +441,42 @@ function launchInstalledApp(app: MarketplaceApp) {
  appLauncher.openSession(app.id)
 }

-function handleInstall(app: MarketplaceApp) {
+async function handleInstall(app: MarketplaceApp) {
  const blocked = installBlockedReason(app.id)
  if (blocked) {
    toast.error(blocked)
    return
  }
+  if (installingApps.has(app.id) || isInstalled(app.id)) return
+  // Multi-version apps (Bitcoin Knots / Core): let the runner pick a version up
+  // front via a full-screen modal (latest pre-selected) instead of silently
+  // installing the default. Best-effort — if the lookup fails we install directly.
+  try {
+    const info = await rpcClient.getPackageVersions(app.id)
+    if (info.supportsVersions && info.versions.length > 1) {
+      installModalApp.value = app
+      showInstallModal.value = true
+      return
+    }
+  } catch { /* no catalog versions — fall through to direct install */ }
+  startInstall(app)
+}
+
+function startInstall(app: MarketplaceApp, versionOverride?: string) {
  if (app.source === 'local') {
-    installApp(app)
+    installApp(app, versionOverride)
  } else {
-    installCommunityApp(app)
+    installCommunityApp(app, versionOverride)
  }
 }

+function onInstallModalConfirm(version: string) {
+  const app = installModalApp.value
+  showInstallModal.value = false
+  installModalApp.value = null
+  if (app) startInstall(app, version)
+}
+
 function viewAppDetails(app: MarketplaceApp) {
  try {
    if (isInstalled(app.id)) {
@ -514,27 +551,27 @@ function failInstall(app: MarketplaceApp, err: unknown) {
  trackTimeout(() => { serverStore.clearInstallProgress(app.id) }, 5000)
 }

-async function installApp(app: MarketplaceApp) {
+async function installApp(app: MarketplaceApp, versionOverride?: string) {
  if (installingApps.has(app.id) || isInstalled(app.id)) return
  queueInstall(app)
  toast.info("Installing " + (app.title ?? app.id) + " - check My Apps")
  router.push('/dashboard/apps').catch(() => {})
  try {
    const installUrl = app.url || app.manifestUrl || app.s9pkUrl
-    await rpcClient.call({ method: 'package.install', params: { id: app.id, url: installUrl, version: app.version }, timeout: 600000 })
+    await rpcClient.call({ method: 'package.install', params: { id: app.id, url: installUrl, version: versionOverride || app.version }, timeout: 600000 })
  } catch (err) {
    if (import.meta.env.DEV) console.error('Installation failed:', err)
    failInstall(app, err)
  }
 }

-async function installCommunityApp(app: MarketplaceApp) {
+async function installCommunityApp(app: MarketplaceApp, versionOverride?: string) {
  if (installingApps.has(app.id) || isInstalled(app.id) || !app.dockerImage) return
  queueInstall(app)
  toast.info("Installing " + (app.title ?? app.id) + " - check My Apps")
  router.push('/dashboard/apps').catch(() => {})
  try {
-    const installParams: Record<string, unknown> = { id: app.id, dockerImage: app.dockerImage, version: app.version }
+    const installParams: Record<string, unknown> = { id: app.id, dockerImage: app.dockerImage, version: versionOverride || app.version }
    if ((app as Record<string, unknown>).containerConfig) {
      installParams.containerConfig = (app as Record<string, unknown>).containerConfig
    }
--- a/neode-ui/src/views/appDetails/AppSidebar.vue
+++ b/neode-ui/src/views/appDetails/AppSidebar.vue
@ -48,11 +48,9 @@
          <select
            v-model="selectedVersion"
            :disabled="versionBusy"
-            class="w-full rounded-lg bg-white/[0.06] border border-white/10 text-white px-3 py-2 text-sm focus:outline-none focus:border-blue-400/60"
+            class="w-full rounded-lg bg-white/[0.06] border border-white/10 text-white pl-3 pr-9 py-2 text-sm focus:outline-none focus:border-blue-400/60"
          >
-            <option v-for="v in versionInfo.versions" :key="v.version" :value="v.version">
-              {{ $ver(v.version) }}{{ v.default ? ' — latest' : '' }}{{ v.deprecated ? ' (deprecated)' : '' }}{{ v.eol ? ` · EOL ${v.eol}` : '' }}
-            </option>
+            <option v-for="v in versionInfo.versions" :key="v.version" :value="v.version">{{ versionOptionLabel(v) }}</option>
          </select>
        </div>

@ -252,7 +250,8 @@
 import { computed, ref, watch } from 'vue'
 import { useI18n } from 'vue-i18n'
 import type { AppCredentialsResponse } from '@/types/api'
-import { rpcClient, type PackageVersionsResponse } from '../../api/rpc-client'
+import { rpcClient, type PackageVersionsResponse, type CatalogVersionInfo } from '../../api/rpc-client'
+import { displayVersion } from '@/utils/version'

 const { t } = useI18n()
 const copiedCredential = ref('')
@ -334,10 +333,33 @@ const isPinned = computed(() => !!versionInfo.value?.pinnedVersion)
 const versionDirty = computed(() => {
  const info = versionInfo.value
  if (!info) return false
-  const currentSelection = info.pinnedVersion || info.default || info.installedVersion || ''
-  return selectedVersion.value !== currentSelection || autoUpdate.value !== info.autoUpdate
+  return selectedVersion.value !== pickSelection(info) || autoUpdate.value !== info.autoUpdate
 })

+// Option label: the floating "latest" entry reads as a sentence (no "v"
+// prefix); every concrete version is normalized via $ver + status suffixes.
+function versionOptionLabel(v: CatalogVersionInfo): string {
+  if (v.version === 'latest') return t('appDetails.alwaysUseLatestVersion')
+  let label = displayVersion(v.version)
+  if (v.deprecated) label += ' (deprecated)'
+  if (v.eol) label += ` · EOL ${v.eol}`
+  return label
+}
+
+// Resolve the dropdown's current value and GUARANTEE it matches a real option,
+// otherwise the native <select> renders blank. Prefer the pin, then the catalog
+// default, then the running version — but fall back to the default/first option
+// when none of those is actually in the list (e.g. a stale installedVersion the
+// catalog no longer carries, which is what left the control blank).
+function pickSelection(info: PackageVersionsResponse): string {
+  const options = info.versions.map((v) => v.version)
+  const preferred = info.pinnedVersion || info.default || info.installedVersion || ''
+  if (preferred && options.includes(preferred)) return preferred
+  return info.default && options.includes(info.default)
+    ? info.default
+    : info.versions[0]?.version || ''
+}
+
 async function loadVersions(appId: string) {
  versionInfo.value = null
  versionError.value = ''
@ -346,7 +368,7 @@ async function loadVersions(appId: string) {
    const info = await rpcClient.getPackageVersions(appId)
    if (!info.supportsVersions || !info.versions.length) return
    versionInfo.value = info
-    selectedVersion.value = info.pinnedVersion || info.default || info.installedVersion || info.versions[0]?.version || ''
+    selectedVersion.value = pickSelection(info)
    autoUpdate.value = info.autoUpdate
  } catch (err) {
    if (import.meta.env.DEV) console.warn('[AppSidebar] getPackageVersions failed:', err)
@ -381,7 +403,7 @@ function cancelDowngrade() {
  downgradeWarning.value = ''
  // Reset the dropdown to the current selection.
  const info = versionInfo.value
-  if (info) selectedVersion.value = info.pinnedVersion || info.default || info.installedVersion || ''
+  if (info) selectedVersion.value = pickSelection(info)
 }

 watch(
--- a/releases/app-catalog.json
+++ b/releases/app-catalog.json
@ -1,6 +1,6 @@
 {
  "schema": 1,
-  "updated": "2026-06-28",
+  "updated": "2026-06-29",
  "apps": {
    "adguardhome": {
      "version": "v0.107.55",
@ -334,7 +334,7 @@
      }
    },
    "bitcoin-core": {
-      "version": "28.4.0",
+      "version": "latest",
      "manifest": {
        "app": {
          "id": "bitcoin-core",
@ -433,6 +433,11 @@
        }
      },
      "versions": [
+        {
+          "version": "latest",
+          "image": "146.59.87.168:3000/lfg2025/bitcoin:latest",
+          "default": true
+        },
        {
          "version": "31.0",
          "image": "146.59.87.168:3000/lfg2025/bitcoin:31.0"
@ -445,10 +450,13 @@
          "version": "29.3",
          "image": "146.59.87.168:3000/lfg2025/bitcoin:29.3"
        },
+        {
+          "version": "29.2",
+          "image": "146.59.87.168:3000/lfg2025/bitcoin:29.2"
+        },
        {
          "version": "28.4.0",
-          "image": "146.59.87.168:3000/lfg2025/bitcoin:28.4",
-          "default": true
+          "image": "146.59.87.168:3000/lfg2025/bitcoin:28.4"
        },
        {
          "version": "27.2",
@ -569,12 +577,24 @@
      "versions": [
        {
          "version": "latest",
-          "image": "146.59.87.168:3000/lfg2025/bitcoin-knots:latest",
+          "image": "146.59.87.168:3000/lfg2025/bitcoin-knots:29.3.knots20260508",
          "default": true
        },
        {
          "version": "29.3.knots20260508",
          "image": "146.59.87.168:3000/lfg2025/bitcoin-knots:29.3.knots20260508"
+        },
+        {
+          "version": "29.3.knots20260507",
+          "image": "146.59.87.168:3000/lfg2025/bitcoin-knots:29.3.knots20260507"
+        },
+        {
+          "version": "29.3.knots20260210",
+          "image": "146.59.87.168:3000/lfg2025/bitcoin-knots:29.3.knots20260210"
+        },
+        {
+          "version": "29.2.knots20251110",
+          "image": "146.59.87.168:3000/lfg2025/bitcoin-knots:29.2.knots20251110"
        }
      ]
    },
--- a/scripts/build-bitcoin-image.sh
+++ b/scripts/build-bitcoin-image.sh
@ -144,7 +144,12 @@ RUN set -eux; \
 COPY bin/bitcoind /usr/local/bin/bitcoind
 COPY bin/bitcoin-cli /usr/local/bin/bitcoin-cli
 RUN chmod 0755 /usr/local/bin/bitcoind /usr/local/bin/bitcoin-cli
-USER bitcoin
+# Run as (container) root, exactly like the legacy hand-built :latest image.
+# Rootless Podman maps container-root to the unprivileged host service user, and
+# the manifest grants CAP_DAC_OVERRIDE so bitcoind can read its data dir — which
+# the orchestrator chowns to the data_uid (host 100101 / container uid 102), NOT
+# to this image's `bitcoin` user. A non-root USER here can't read existing chain
+# data and bitcoind crash-loops with "Error initializing block database".
 WORKDIR /home/bitcoin
 VOLUME ["/home/bitcoin/.bitcoin"]
 EXPOSE 8332 8333
--- a/scripts/generate-app-catalog.sh
+++ b/scripts/generate-app-catalog.sh
@ -182,10 +182,12 @@ VERSIONS = {
    # "Version & Updates" card. Add the next release by building its image then
    # prepending it here.
    "bitcoin-core": [
+        {"version": "latest", "image": f"{REGISTRY}/bitcoin:latest", "default": True},
        {"version": "31.0", "image": f"{REGISTRY}/bitcoin:31.0"},
        {"version": "30.2", "image": f"{REGISTRY}/bitcoin:30.2"},
        {"version": "29.3", "image": f"{REGISTRY}/bitcoin:29.3"},
-        {"version": "28.4.0", "image": f"{REGISTRY}/bitcoin:28.4", "default": True},
+        {"version": "29.2", "image": f"{REGISTRY}/bitcoin:29.2"},
+        {"version": "28.4.0", "image": f"{REGISTRY}/bitcoin:28.4"},
        {"version": "27.2", "image": f"{REGISTRY}/bitcoin:27.2"},
        {"version": "26.2", "image": f"{REGISTRY}/bitcoin:26.2", "deprecated": True},
        {"version": "25.2", "image": f"{REGISTRY}/bitcoin:25.2", "deprecated": True},
@ -196,16 +198,35 @@ VERSIONS = {
    # top-level catalog version (L167-168) or the card can't reach "latest" and
    # selecting the highlighted default would instead pin+recreate. Pinning
    # 29.3.knots20260508 moves a runner off the floating tag.
+    # `latest` is the default and points at the NEWEST published dated image
+    # (not the bare :latest tag) so "Always use the latest version" installs the
+    # newest build on fixed-binary nodes, while UNPINNED nodes still resolve via
+    # the manifest's floating :latest tag (kept on the legacy image until the
+    # entrypoint-render fix is fleet-deployed — see
+    # docs/bitcoin-version-bulletproof-rollout.md).
    "bitcoin-knots": [
        {"version": "latest",
-         "image": f"{REGISTRY}/bitcoin-knots:latest", "default": True},
+         "image": f"{REGISTRY}/bitcoin-knots:29.3.knots20260508", "default": True},
        {"version": "29.3.knots20260508",
         "image": f"{REGISTRY}/bitcoin-knots:29.3.knots20260508"},
+        {"version": "29.3.knots20260507",
+         "image": f"{REGISTRY}/bitcoin-knots:29.3.knots20260507"},
+        {"version": "29.3.knots20260210",
+         "image": f"{REGISTRY}/bitcoin-knots:29.3.knots20260210"},
+        {"version": "29.2.knots20251110",
+         "image": f"{REGISTRY}/bitcoin-knots:29.2.knots20251110"},
    ],
 }
 for app_id, versions in VERSIONS.items():
    if app_id in apps and versions:
        apps[app_id]["versions"] = versions
+        # The default/latest entry MUST equal the app's top-level catalog
+        # `version` (commit 169ff2e2) so selecting the highlighted default
+        # un-pins / tracks latest instead of pinning+recreating. Enforce it here
+        # rather than relying on the manifest version matching.
+        default_entry = next((v for v in versions if v.get("default")), None)
+        if default_entry:
+            apps[app_id]["version"] = default_entry["version"]

 catalog = {
    "schema": 1,