diff --git a/image-recipe/build-auto-installer-iso.sh b/image-recipe/build-auto-installer-iso.sh index 3bbf9ade..8aa7e5bf 100755 --- a/image-recipe/build-auto-installer-iso.sh +++ b/image-recipe/build-auto-installer-iso.sh @@ -21,6 +21,26 @@ # - Automatic installation with progress display # - Boots directly to web UI after install # +# Image versions: sourced from scripts/image-versions.sh (single source of truth). +# All container image references MUST use the $*_IMAGE variables defined there. +# +# --- PLANNED REFACTOR (post-beta) --- +# This script is ~1870 lines and should be split into a modular library. +# Proposed structure: +# image-recipe/ +# build-auto-installer-iso.sh — Main orchestrator (config, CLI args, step sequencing) +# lib/ +# rootfs.sh — Step 1: Build root filesystem via Docker (~185 lines) +# installer-env.sh — Step 2: Download/extract Debian Live base ISO (~80 lines) +# components.sh — Step 3: Add Archipelago components (binary, configs, web UI) (~120 lines) +# container-images.sh — Step 3b: Bundle container images for offline install (~330 lines) +# auto-install-script.sh — Step 4: Generate the embedded auto-install.sh (~615 lines) +# boot-config.sh — Step 5: Configure live boot auto-start + overlay squashfs (~215 lines) +# create-iso.sh — Step 6: Build final bootable ISO with xorriso/grub (~140 lines) +# Each lib/ script exports functions; main script sources them and calls in sequence. +# DO NOT split until tested on the build server — this is critical infrastructure. +# --- +# set -e @@ -616,7 +636,7 @@ ${FEDIMINT_GATEWAY_IMAGE:-docker.io/fedimint/gatewayd:v0.5.1} fedimint-gateway.t ${FILEBROWSER_IMAGE:-docker.io/filebrowser/filebrowser:v2} filebrowser.tar ${ALPINE_TOR_IMAGE:-docker.io/andrius/alpine-tor:0.4.8.13} alpine-tor.tar ${NGINX_ALPINE_IMAGE:-docker.io/library/nginx:alpine} nginx-alpine.tar -ghcr.io/tbd54566975/dwn-server:main dwn-server.tar +${DWN_SERVER_IMAGE:-ghcr.io/tbd54566975/dwn-server:main} dwn-server.tar ${GRAFANA_IMAGE:-docker.io/grafana/grafana:11.4.0} grafana.tar ${UPTIME_KUMA_IMAGE:-docker.io/louislam/uptime-kuma:1} uptime-kuma.tar ${VAULTWARDEN_IMAGE:-docker.io/vaultwarden/server:1.32.5} vaultwarden.tar @@ -628,7 +648,7 @@ ${PHOTOPRISM_IMAGE:-docker.io/photoprism/photoprism:240915} photoprism.tar ${NEXTCLOUD_IMAGE:-docker.io/library/nextcloud:30} nextcloud.tar ${NPM_IMAGE:-docker.io/jc21/nginx-proxy-manager:2} nginx-proxy-manager.tar ${IMMICH_IMAGE:-ghcr.io/immich-app/immich-server:v1.123.0} immich-server.tar -docker.io/library/postgres:14-alpine postgres-immich.tar +${IMMICH_POSTGRES_IMAGE:-ghcr.io/immich-app/postgres:14-vectorchord0.4.3-pgvectors0.2.0} postgres-immich.tar ${INDEEDHUB_REDIS_IMAGE:-docker.io/library/redis:7-alpine} redis-immich.tar ${ONLYOFFICE_IMAGE:-docker.io/onlyoffice/documentserver:8.2} onlyoffice.tar ${ADGUARDHOME_IMAGE:-docker.io/adguard/adguardhome:v0.107.55} adguardhome.tar @@ -840,6 +860,11 @@ if [ "$UNBUNDLED" = "1" ]; then echo " Skipping first-boot containers (UNBUNDLED: apps installed from Marketplace)" else echo " Creating first-boot container creation service..." + # Copy shared script library + if [ -d "$SCRIPT_DIR/../scripts/lib" ]; then + mkdir -p "$ARCH_DIR/scripts/lib" + cp "$SCRIPT_DIR/../scripts/lib/"*.sh "$ARCH_DIR/scripts/lib/" 2>/dev/null || true + fi if [ -f "$SCRIPT_DIR/../scripts/first-boot-containers.sh" ]; then cp "$SCRIPT_DIR/../scripts/first-boot-containers.sh" "$ARCH_DIR/scripts/" chmod +x "$ARCH_DIR/scripts/first-boot-containers.sh" @@ -1152,6 +1177,11 @@ if [ -d "$BOOT_MEDIA/archipelago/container-images" ]; then if [ -f "$BOOT_MEDIA/archipelago/scripts/archipelago-setup-tor.service" ]; then cp "$BOOT_MEDIA/archipelago/scripts/archipelago-setup-tor.service" /mnt/target/etc/systemd/system/ fi + # Copy shared script library + if [ -d "$BOOT_MEDIA/archipelago/scripts/lib" ]; then + mkdir -p /mnt/target/opt/archipelago/scripts/lib + cp -r "$BOOT_MEDIA/archipelago/scripts/lib/"* /mnt/target/opt/archipelago/scripts/lib/ 2>/dev/null || true + fi if [ -f "$BOOT_MEDIA/archipelago/scripts/first-boot-containers.sh" ]; then cp "$BOOT_MEDIA/archipelago/scripts/first-boot-containers.sh" /mnt/target/opt/archipelago/scripts/ chmod +x /mnt/target/opt/archipelago/scripts/first-boot-containers.sh diff --git a/scripts/deploy-tailscale.sh b/scripts/deploy-tailscale.sh index 719f07e7..fd5ba795 100755 --- a/scripts/deploy-tailscale.sh +++ b/scripts/deploy-tailscale.sh @@ -27,6 +27,9 @@ TARGET_DIR="/home/archipelago/archy" # Source pinned image versions (single source of truth) [ -f "$SCRIPT_DIR/image-versions.sh" ] && . "$SCRIPT_DIR/image-versions.sh" +# Source shared utility library +[ -f "$SCRIPT_DIR/lib/common.sh" ] && . "$SCRIPT_DIR/lib/common.sh" + SSH_KEY="${ARCHIPELAGO_SSH_KEY:-$HOME/.ssh/archipelago-deploy}" SSH_OPTS="-o StrictHostKeyChecking=no -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -o ConnectTimeout=10 -i $SSH_KEY" BUILD_SOURCE="archipelago@${DEFAULT_PRIMARY:-192.168.1.228}" diff --git a/scripts/deploy-to-target.sh b/scripts/deploy-to-target.sh index f2b0a80e..f3866492 100755 --- a/scripts/deploy-to-target.sh +++ b/scripts/deploy-to-target.sh @@ -25,6 +25,9 @@ PROJECT_DIR="$(dirname "$SCRIPT_DIR")" # Source pinned image versions (single source of truth) [ -f "$SCRIPT_DIR/image-versions.sh" ] && . "$SCRIPT_DIR/image-versions.sh" +# Source shared utility library +[ -f "$SCRIPT_DIR/lib/common.sh" ] && . "$SCRIPT_DIR/lib/common.sh" + # Configuration TARGET_HOST="${ARCHIPELAGO_TARGET:-archipelago@192.168.1.228}" TARGET_DIR="/home/archipelago/archy" diff --git a/scripts/first-boot-containers.sh b/scripts/first-boot-containers.sh index e0a88791..d150b171 100644 --- a/scripts/first-boot-containers.sh +++ b/scripts/first-boot-containers.sh @@ -7,6 +7,51 @@ # Based on scripts/deploy-to-target.sh (--live) container logic - do not diverge. # No set -e: each section continues even if one fails (idempotent, best-effort). # +# Image versions: sourced from /opt/archipelago/image-versions.sh (single source of truth). +# All container image references MUST use the $*_IMAGE variables defined there. +# NOTE: Many container creation lines below still use hardcoded versions instead of +# the $*_IMAGE variables. These must be migrated to use the variables for consistency. +# See the version mismatch list in the planned refactor below. +# +# --- PLANNED REFACTOR (post-beta) --- +# This script is ~995 lines and should be split into a modular library. +# Proposed structure: +# scripts/ +# first-boot-containers.sh — Main orchestrator (prereqs, sequencing, summary) +# lib/ +# container-prereqs.sh — Swap setup, rootless podman config, UID mapping (~120 lines) +# container-secrets.sh — RPC auth, DB passwords, bitcoin.conf generation (~80 lines) +# container-helpers.sh — mem_limit(), wait_for_container(), track_container() (~60 lines) +# tier1-databases.sh — Tier 1: Bitcoin Knots, MariaDB, Postgres, ElectrumX (~200 lines) +# tier2-services.sh — Tier 2: LND, Mempool, BTCPay, Fedimint (~200 lines) +# tier3-apps.sh — Tier 3: Home Assistant, Grafana, Jellyfin, etc. (~250 lines) +# tier3-stacks.sh — Tier 3: Multi-container stacks (Immich, Penpot, Nostr) (~100 lines) +# custom-ui.sh — Custom UI containers (bitcoin-ui, lnd-ui, electrs-ui) (~60 lines) +# Each lib/ script exports functions; main script sources them and calls in sequence. +# DO NOT split until tested on the build server — this is critical infrastructure. +# +# KNOWN VERSION MISMATCHES (hardcoded vs image-versions.sh): +# - MariaDB: hardcoded 10.11, pinned 11.4 +# - ElectrumX: hardcoded v1.18.0, pinned v1.16.0 +# - Mempool backend/frontend: hardcoded v2.5.0, pinned v3.0.0 +# - Postgres (BTCPay): hardcoded 15-alpine, pinned 16 +# - NBXplorer: hardcoded 2.6.0, pinned 2.5.13 +# - BTCPay: hardcoded 1.13.5, pinned 1.14.5 +# - LND: hardcoded v0.18.4-beta, pinned v0.18.5-beta +# - Fedimint: hardcoded v0.10.0, pinned v0.5.1 +# - Home Assistant: hardcoded 2024.1, pinned 2024.12 +# - Grafana: hardcoded 10.2.0, pinned 11.4.0 +# - Jellyfin: hardcoded 10.8.13, pinned 10.10.3 +# - Vaultwarden: hardcoded 1.30.0-alpine, pinned 1.32.5 +# - Nextcloud: hardcoded 28, pinned 30 +# - OnlyOffice: hardcoded 7.5.1, pinned 8.2 +# - FileBrowser: hardcoded v2.27.0, pinned v2 +# - Portainer: hardcoded 2.19.4, pinned 2.21.5 +# - Tailscale: hardcoded :stable, pinned v1.78.3 +# - Immich: hardcoded :release, pinned v1.123.0 +# Fix these by replacing hardcoded values with ${VAR:-fallback} pattern. +# --- +# LOG="/var/log/archipelago-first-boot.log" DOCKER=podman command -v podman >/dev/null 2>&1 || DOCKER=docker @@ -14,6 +59,10 @@ command -v podman >/dev/null 2>&1 || DOCKER=docker # Source pinned image versions (single source of truth) source /opt/archipelago/image-versions.sh 2>/dev/null || true +# Source shared utility library +SCRIPT_DIR_FBC="$(cd "$(dirname "$0")" && pwd)" +[ -f "$SCRIPT_DIR_FBC/lib/common.sh" ] && source "$SCRIPT_DIR_FBC/lib/common.sh" || true + # Must run as root for podman [ "$(id -u)" -eq 0 ] || { echo "Must run as root" >&2; exit 1; } diff --git a/scripts/image-versions.sh b/scripts/image-versions.sh index 090f3af5..d51d5f01 100644 --- a/scripts/image-versions.sh +++ b/scripts/image-versions.sh @@ -46,6 +46,8 @@ FEDIMINT_GATEWAY_IMAGE="docker.io/fedimint/gatewayd:v0.5.1" # Media IMMICH_IMAGE="ghcr.io/immich-app/immich-server:v1.123.0" +IMMICH_POSTGRES_IMAGE="ghcr.io/immich-app/postgres:14-vectorchord0.4.3-pgvectors0.2.0" +IMMICH_VALKEY_IMAGE="docker.io/valkey/valkey:7-alpine" REDIS_IMAGE="docker.io/library/redis:7" # Penpot @@ -63,5 +65,11 @@ MINIO_IMAGE="docker.io/minio/minio:RELEASE.2024-11-07T00-52-20Z" INDEEDHUB_POSTGRES_IMAGE="docker.io/library/postgres:16-alpine" INDEEDHUB_REDIS_IMAGE="docker.io/library/redis:7-alpine" +# DWN (Decentralized Web Node) +DWN_SERVER_IMAGE="ghcr.io/tbd54566975/dwn-server:main" + +# Penpot postgres (separate from BTCPay postgres — different version) +PENPOT_POSTGRES_IMAGE="docker.io/library/postgres:15" + # Base images NGINX_ALPINE_IMAGE="docker.io/library/nginx:alpine" diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh new file mode 100755 index 00000000..9bf7136f --- /dev/null +++ b/scripts/lib/common.sh @@ -0,0 +1,184 @@ +#!/bin/bash +# Shared utility functions for Archipelago scripts +# +# Source this from any script: +# source "$(dirname "$0")/lib/common.sh" +# +# Provides: logging, SSH helpers, health checks, disk checks, memory limits + +# Guard against double-sourcing +[ -n "$_ARCHY_COMMON_LOADED" ] && return 0 +_ARCHY_COMMON_LOADED=1 + +# ── Colored logging ───────────────────────────────────────────────────── + +log_info() { echo -e "\033[0;32m[INFO]\033[0m $(date '+%H:%M:%S') $*"; } +log_warn() { echo -e "\033[0;33m[WARN]\033[0m $(date '+%H:%M:%S') $*"; } +log_error() { echo -e "\033[0;31m[ERROR]\033[0m $(date '+%H:%M:%S') $*"; } + +# ── SSH wrapper with deploy key ───────────────────────────────────────── + +# Usage: ssh_cmd +# Uses the standard deploy key and safe defaults. +ssh_cmd() { + local host="$1"; shift + local key="${ARCHIPELAGO_SSH_KEY:-$HOME/.ssh/archipelago-deploy}" + ssh -i "$key" \ + -o StrictHostKeyChecking=no \ + -o ConnectTimeout=10 \ + -o ServerAliveInterval=15 \ + -o ServerAliveCountMax=4 \ + "archipelago@${host}" "$@" +} + +# Usage: scp_cmd +# Wraps scp with the same deploy key and options. +scp_cmd() { + local key="${ARCHIPELAGO_SSH_KEY:-$HOME/.ssh/archipelago-deploy}" + scp -i "$key" \ + -o StrictHostKeyChecking=no \ + -o ConnectTimeout=10 \ + "$@" +} + +# ── Health check ──────────────────────────────────────────────────────── + +# Wait for an HTTP health endpoint to respond successfully. +# Usage: wait_for_health [max_wait_seconds] [path] +wait_for_health() { + local host="$1" max_wait="${2:-60}" path="${3:-/health}" + local waited=0 + while [ $waited -lt $max_wait ]; do + if curl -sf "http://${host}${path}" >/dev/null 2>&1; then + log_info "Health check passed for ${host}" + return 0 + fi + sleep 2 + waited=$((waited + 2)) + done + log_error "Health check failed for ${host} after ${max_wait}s" + return 1 +} + +# ── Disk space check ─────────────────────────────────────────────────── + +# Check that disk usage on a remote host is below a threshold. +# Usage: check_disk_space [max_percent] +check_disk_space() { + local host="$1" max_pct="${2:-85}" + local pct + pct=$(ssh_cmd "$host" "df / | tail -1 | awk '{print \$(NF-1)}' | tr -d '%'" 2>/dev/null) + if [ -n "$pct" ] && [ "$pct" -gt "$max_pct" ] 2>/dev/null; then + log_error "Disk at ${pct}% on ${host} (max ${max_pct}%)" + return 1 + fi + return 0 +} + +# ── Memory limit calculator ──────────────────────────────────────────── + +# Returns the memory limit for a container by name. +# Checks /etc/archipelago/memory-limits.conf first (override), then falls +# back to built-in defaults. Mirrors the pattern in first-boot-containers.sh. +# +# Low-memory mode: set LOW_MEM=true before calling to get reduced limits +# on certain heavy containers. +# +# Usage: mem_limit +mem_limit() { + local name="$1" + + # Allow per-host overrides via config file + local limit + limit=$(grep "^${name}=" /etc/archipelago/memory-limits.conf 2>/dev/null | cut -d= -f2) + if [ -n "$limit" ]; then + echo "$limit" + return + fi + + # Built-in defaults (keep in sync with first-boot-containers.sh) + local low="${LOW_MEM:-false}" + case "$name" in + bitcoin-knots) $low && echo "1g" || echo "2g" ;; + onlyoffice) $low && echo "1g" || echo "2g" ;; + ollama) $low && echo "1g" || echo "4g" ;; + lnd) echo "512m" ;; + electrumx) echo "1g" ;; + nextcloud) echo "1g" ;; + immich_server) echo "1g" ;; + btcpay-server) echo "1g" ;; + homeassistant) echo "512m" ;; + fedimint) echo "512m" ;; + fedimint-gateway) echo "512m" ;; + photoprism) $low && echo "512m" || echo "1g" ;; + mempool-api) echo "512m" ;; + jellyfin) echo "1g" ;; + searxng) echo "512m" ;; + archy-btcpay-db) echo "512m" ;; + archy-nbxplorer) echo "512m" ;; + archy-mempool-db) echo "512m" ;; + archy-mempool-web) echo "256m" ;; + grafana) echo "256m" ;; + vaultwarden) echo "256m" ;; + uptime-kuma) echo "256m" ;; + filebrowser) echo "256m" ;; + portainer) echo "256m" ;; + nginx-proxy-manager) echo "256m" ;; + immich_postgres) echo "256m" ;; + immich_redis) echo "128m" ;; + tailscale) echo "256m" ;; + penpot-postgres) echo "256m" ;; + penpot-valkey) echo "128m" ;; + penpot-backend) echo "512m" ;; + penpot-exporter) echo "256m" ;; + penpot-frontend) echo "256m" ;; + nostr-rs-relay) echo "256m" ;; + strfry) echo "256m" ;; + indeedhub|archy-bitcoin-ui|archy-lnd-ui|archy-electrs-ui) echo "128m" ;; + *) echo "512m" ;; + esac +} + +# ── Wait for container readiness ─────────────────────────────────────── + +# Wait for a container health check command to succeed. +# Usage: wait_for_container [max_wait_seconds] +wait_for_container() { + local name="$1" check_cmd="$2" max_wait="${3:-30}" + local waited=0 + while [ $waited -lt $max_wait ]; do + if eval "$check_cmd" 2>/dev/null; then + log_info "$name is ready (${waited}s)" + return 0 + fi + sleep 2 + waited=$((waited + 2)) + done + log_warn "$name not ready after ${max_wait}s" + return 1 +} + +# ── Section timing ───────────────────────────────────────────────────── + +# Track elapsed time for deploy sections. +# Usage: +# section_start "Building frontend" +# ... do work ... +# section_end +_SECTION_START=0 +_SECTION_NAME="" + +section_start() { + _SECTION_NAME="${1:-}" + _SECTION_START=$(date +%s) + [ -n "$_SECTION_NAME" ] && log_info "$_SECTION_NAME" +} + +section_end() { + local elapsed=$(( $(date +%s) - _SECTION_START )) + if [ -n "$_SECTION_NAME" ]; then + log_info "$_SECTION_NAME done (${elapsed}s)" + else + echo " (${elapsed}s)" + fi +}