archy/core/archipelago/src/api/rpc/package.rs

use super::RpcHandler;
use crate::data_model::{
    Description, InstallProgress, Manifest, PackageDataEntry, PackageState, StaticFiles,
};
use crate::port_allocator::PortAllocator;
use anyhow::{Context, Result};
use tokio::io::{AsyncBufReadExt, BufReader};
use tracing::{debug, info};

impl RpcHandler {
    /// Install a package from a Docker image
    /// Security: Image verification, resource limits, network isolation
    pub(super) async fn handle_package_install(
        &self,
        params: Option<serde_json::Value>,
    ) -> Result<serde_json::Value> {
        let params = params.ok_or_else(|| anyhow::anyhow!("Missing params"))?;

        let package_id = params
            .get("id")
            .and_then(|v| v.as_str())
            .ok_or_else(|| anyhow::anyhow!("Missing package id"))?;
        validate_app_id(package_id)?;

        let docker_image = params
            .get("dockerImage")
            .and_then(|v| v.as_str())
            .ok_or_else(|| anyhow::anyhow!("Missing dockerImage"))?;

        debug!("Installing package {} from image {}", package_id, docker_image);

        // Security: Validate image name format (prevent injection)
        if !is_valid_docker_image(docker_image) {
            return Err(anyhow::anyhow!("Invalid Docker image format"));
        }

        // Multi-container apps: create full stack
        if package_id == "immich" {
            return self.install_immich_stack().await;
        }
        if package_id == "penpot" || package_id == "penpot-frontend" {
            return self.install_penpot_stack().await;
        }

        // Dependency checks: verify required services are running before install
        let has_lnd;
        {
            let dep_check = tokio::process::Command::new("podman")
                .args(["ps", "--format", "{{.Names}}"])
                .output()
                .await
                .context("Failed to check running containers")?;
            let running = String::from_utf8_lossy(&dep_check.stdout);
            let is_running = |names: &[&str]| {
                running.lines().any(|l| {
                    let name = l.trim();
                    names.iter().any(|n| name == *n)
                })
            };
            let has_bitcoin = is_running(&["bitcoin-knots", "bitcoin-core", "bitcoin"]);
            let has_electrumx = is_running(&["electrumx", "mempool-electrs", "electrs"]);
            has_lnd = is_running(&["lnd"]);

            match package_id {
                "electrumx" | "mempool-electrs" | "electrs" if !has_bitcoin => {
                    return Err(anyhow::anyhow!(
                        "ElectrumX requires a running Bitcoin node (Bitcoin Knots). Please install and start Bitcoin Knots first."
                    ));
                }
                "lnd" if !has_bitcoin => {
                    return Err(anyhow::anyhow!(
                        "LND requires a running Bitcoin node (Bitcoin Knots). Please install and start Bitcoin Knots first."
                    ));
                }
                "btcpay-server" | "btcpayserver" if !has_bitcoin => {
                    return Err(anyhow::anyhow!(
                        "BTCPay Server requires a running Bitcoin node (Bitcoin Knots). Please install and start Bitcoin Knots first."
                    ));
                }
                "mempool" | "mempool-web" if !has_bitcoin || !has_electrumx => {
                    let mut missing = vec![];
                    if !has_bitcoin { missing.push("Bitcoin Knots"); }
                    if !has_electrumx { missing.push("ElectrumX"); }
                    return Err(anyhow::anyhow!(
                        "Mempool requires {} to be running. Please install and start {} first.",
                        missing.join(" and "),
                        missing.join(" and ")
                    ));
                }
                "fedimint" if !has_bitcoin => {
                    return Err(anyhow::anyhow!(
                        "Fedimint requires a running Bitcoin node (Bitcoin Knots). Please install and start Bitcoin Knots first."
                    ));
                }
                _ => {}
            }
            // Log dependency info for apps that have optional deps
            if matches!(package_id, "btcpay-server" | "btcpayserver") && !has_lnd {
                info!("BTCPay Server installing without LND — Lightning payments won't be available until LND is installed");
            }
        }

        // Check if container already exists
        let check_output = tokio::process::Command::new("podman")
            .args(["ps", "-a", "--format", "{{.Names}}", "--filter", &format!("name=^{}$", package_id)])
            .output()
            .await
            .context("Failed to check existing containers")?;

        if !String::from_utf8_lossy(&check_output.stdout).trim().is_empty() {
            return Err(anyhow::anyhow!("Container {} already exists. Stop and remove it first.", package_id));
        }

        // Pull the image (skip for local images - must be built locally first)
        // For registry images, also check if a local build exists first (avoids
        // pull failures when the registry image hasn't been pushed yet).
        let is_local_image = docker_image.starts_with("localhost/");
        let has_local_fallback = if !is_local_image {
            let local_tag = format!("localhost/{}:latest", package_id);
            let check = tokio::process::Command::new("podman")
                .args(["images", "-q", &local_tag])
                .output().await.ok();
            check.map_or(false, |o| !String::from_utf8_lossy(&o.stdout).trim().is_empty())
        } else { false };
        if !is_local_image && !has_local_fallback {
            debug!("Pulling image: {}", docker_image);

            // Set package state to Installing with progress
            self.set_install_progress(package_id, 0, 0).await;

            // Stream pull progress via piped stderr
            let mut child = tokio::process::Command::new("podman")
                .args(["pull", docker_image])
                .stdout(std::process::Stdio::piped())
                .stderr(std::process::Stdio::piped())
                .spawn()
                .context("Failed to start image pull")?;

            // Parse stderr for progress updates
            if let Some(stderr) = child.stderr.take() {
                let reader = BufReader::new(stderr);
                let mut lines = reader.lines();
                let pkg_id = package_id.to_string();
                let state_mgr = self.state_manager.clone();

                while let Ok(Some(line)) = lines.next_line().await {
                    // Podman outputs lines like: "Copying blob sha256:abc123 [=====>  ] 50.0MiB / 100.0MiB"
                    // or "Getting image source signatures" etc.
                    if let Some((downloaded, total)) = parse_pull_progress(&line) {
                        Self::update_install_progress(&state_mgr, &pkg_id, downloaded, total).await;
                    }
                }
            }

            let status = child.wait().await.context("Failed to wait for image pull")?;
            if !status.success() {
                self.clear_install_progress(package_id).await;
                return Err(anyhow::anyhow!("Failed to pull image"));
            }

            // Mark pull as complete (100%)
            self.set_install_progress(package_id, 100, 100).await;
        } else if has_local_fallback {
            // Registry image exists locally — use the local build
            debug!("Using local build for {} (skipping registry pull)", package_id);
        } else {
            // Verify local image exists
            let images_output = tokio::process::Command::new("podman")
                .args(["images", "-q", docker_image])
                .output()
                .await
                .context("Failed to check local image")?;
            if String::from_utf8_lossy(&images_output.stdout).trim().is_empty() {
                return Err(anyhow::anyhow!(
                    "Local image {} not found. Build the image first or ensure the registry is reachable.",
                    docker_image
                ));
            }
            debug!("Using local image: {}", docker_image);
        }

        // Normalize container name: legacy "electrs"/"mempool-electrs" aliases -> "electrumx"
        let container_name = match package_id {
            "electrs" | "mempool-electrs" => "electrumx",
            _ => package_id,
        };

        // Create and start container with security constraints
        let mut run_args = vec![
            "run",
            "-d",  // Detached
            "--name", container_name,
            "--restart=unless-stopped",  // Auto-restart policy
        ];

        // Read Bitcoin RPC password from secrets for container configs
        let rpc_pass = crate::bitcoin_rpc::bitcoin_rpc_password().await;

        // App-specific configuration (should come from manifest)
        let (mut ports, mut volumes, env_vars, custom_command, mut custom_args) = {
            let mut allocator = self.port_allocator.lock().map_err(|e| {
                anyhow::anyhow!("Port allocator lock poisoned: {}", e)
            })?;
            get_app_config(package_id, &self.config.host_ip, &mut allocator, &rpc_pass)
        };

        // Fedimint Gateway: auto-detect LND and switch to lnd mode
        if package_id == "fedimint-gateway" && has_lnd {
            let lnd_cert = "/var/lib/archipelago/lnd/tls.cert";
            let lnd_macaroon = "/var/lib/archipelago/lnd/data/chain/bitcoin/mainnet/admin.macaroon";
            if std::path::Path::new(lnd_cert).exists() && std::path::Path::new(lnd_macaroon).exists() {
                info!("LND detected with credentials — configuring gateway in lnd mode");
                // Remove LDK port (9737) since we'll use LND
                ports.retain(|p| p != "9737:9737");
                // Mount LND credentials read-only
                volumes.push(format!("{}:/lnd/tls.cert:ro", lnd_cert));
                volumes.push(format!("{}:/lnd/admin.macaroon:ro", lnd_macaroon));
                // Switch args from ldk to lnd
                custom_args = Some(vec![
                    "gatewayd".to_string(),
                    "--data-dir".to_string(), "/data".to_string(),
                    "--listen".to_string(), "0.0.0.0:8176".to_string(),
                    "--bcrypt-password-hash".to_string(),
                    "$2y$10$t9YjjxkiktrlYvjajB/zgOMDnSNVg4HqrbDqh47u7Jf42whNdxNqC".to_string(),
                    "--network".to_string(), "bitcoin".to_string(),
                    "--bitcoind-url".to_string(), format!("http://{}:8332", self.config.host_ip),
                    "--bitcoind-username".to_string(), "archipelago".to_string(),
                    "--bitcoind-password".to_string(), rpc_pass.clone(),
                    "lnd".to_string(),
                    "--lnd-rpc-host".to_string(), format!("{}:10009", self.config.host_ip),
                    "--lnd-tls-cert".to_string(), "/lnd/tls.cert".to_string(),
                    "--lnd-macaroon".to_string(), "/lnd/admin.macaroon".to_string(),
                ]);
            }
        }

        // Special handling: Tailscale needs host network; mempool stack needs archy-net
        let is_tailscale = package_id == "tailscale";
        let needs_archy_net = matches!(
            package_id,
            "bitcoin-knots" | "bitcoin" | "bitcoin-core"
                | "lnd"
                | "mempool" | "mempool-web" | "mempool-api" | "electrumx" | "mempool-electrs" | "electrs" | "mysql-mempool" | "archy-mempool-db" | "archy-mempool-web"
                | "btcpay-server" | "btcpayserver" | "archy-btcpay-db" | "archy-nbxplorer" | "nbxplorer"
                | "fedimint" | "fedimint-gateway"
        );

        if is_tailscale {
            run_args.push("--network=host");
            run_args.push("--privileged");
            run_args.push("--cap-add=NET_ADMIN");
            run_args.push("--cap-add=NET_RAW");
            run_args.push("--device=/dev/net/tun");
        } else if needs_archy_net {
            let _ = tokio::process::Command::new("podman")
                .args(["network", "create", "archy-net"])
                .output()
                .await;
            run_args.push("--network=archy-net");
        }

        // Security hardening (skip for privileged containers like Tailscale)
        let security_caps: Vec<String> = if !is_tailscale {
            get_app_capabilities(package_id)
        } else {
            vec![]
        };
        let readonly_compatible = !is_tailscale && is_readonly_compatible(package_id);

        if !is_tailscale {
            run_args.push("--cap-drop=ALL");
            run_args.push("--security-opt=no-new-privileges:true");
            for cap in &security_caps {
                run_args.push(cap);
            }
            if readonly_compatible {
                run_args.push("--read-only");
                run_args.push("--tmpfs=/tmp:rw,noexec,nosuid,size=256m");
                run_args.push("--tmpfs=/run:rw,noexec,nosuid,size=64m");
            }
        }

        // Create data directories if they don't exist
        for volume in &volumes {
            if let Some(host_path) = volume.split(':').next() {
                if host_path.starts_with("/var/lib/archipelago/") {
                    debug!("Creating directory: {}", host_path);
                    let create_dir = tokio::process::Command::new("sudo")
                        .args(["mkdir", "-p", host_path])
                        .output()
                        .await;

                    if let Err(e) = create_dir {
                        debug!("Failed to create directory {}: {}", host_path, e);
                    }
                    // Grafana runs as UID 472 - fix permissions so it can write
                    if package_id == "grafana" && host_path.contains("grafana") {
                        let _ = tokio::process::Command::new("sudo")
                            .args(["chown", "-R", "472:472", host_path])
                            .output()
                            .await;
                    }
                }
            }
        }

        // Pre-install: Create bitcoin.conf for Bitcoin nodes with RPC + txindex
        if matches!(package_id, "bitcoin" | "bitcoin-core" | "bitcoin-knots") {
            let bitcoin_dir = "/var/lib/archipelago/bitcoin";
            let conf_path = format!("{}/bitcoin.conf", bitcoin_dir);
            let bitcoin_conf = format!("\
server=1\n\
prune=550\n\
rpcuser=archipelago\n\
rpcpassword={}\n\
rpcbind=0.0.0.0\n\
rpcallowip=127.0.0.1/32\n\
rpcallowip=10.88.0.0/16\n\
rpcport=8332\n\
listen=1\n\
printtoconsole=1\n", rpc_pass);
            let _ = tokio::fs::create_dir_all(bitcoin_dir).await;
            let _ = tokio::fs::write(&conf_path, bitcoin_conf).await;
            info!("Created bitcoin.conf at {} with RPC + txindex enabled", conf_path);
        }

        // Add port mappings (skip if host network mode like Tailscale)
        if !is_tailscale {
            for port in &ports {
                run_args.push("-p");
                run_args.push(port);
            }
        }

        // Add volume mounts
        for volume in &volumes {
            run_args.push("-v");
            run_args.push(volume);
        }

        // Add environment variables
        for env in &env_vars {
            run_args.push("-e");
            run_args.push(env);
        }

        // Resource limits: per-app memory and CPU
        let memory_limit = get_memory_limit(package_id);
        let mem_arg = format!("--memory={}", memory_limit);
        run_args.push(&mem_arg);
        run_args.push("--cpus=2");

        // Health check definitions
        let health_args = get_health_check_args(package_id, &rpc_pass);
        for arg in &health_args {
            run_args.push(arg);
        }

        // Finally, the image — use local build if available, otherwise registry image
        let effective_image = if has_local_fallback {
            format!("localhost/{}:latest", package_id)
        } else {
            docker_image.to_string()
        };
        run_args.push(&effective_image);

        debug!("Running container with args: {:?}", run_args);

        // Build command with optional custom command
        let mut cmd = tokio::process::Command::new("podman");
        cmd.args(&run_args);

        // Add custom command/args if specified
        if let Some(custom_cmd) = custom_command {
            cmd.arg(custom_cmd);
        } else if let Some(args) = custom_args {
            cmd.args(args);
        }

        let run_output = cmd
            .output()
            .await
            .context("Failed to run container")?;

        if !run_output.status.success() {
            let stderr = String::from_utf8_lossy(&run_output.stderr);
            return Err(anyhow::anyhow!("Failed to start container: {}", stderr));
        }

        let container_id = String::from_utf8_lossy(&run_output.stdout).trim().to_string();

        // Post-install: Nextcloud needs trusted domains configured for iframe embedding
        if package_id == "nextcloud" {
            let host_ip = self.config.host_ip.clone();
            tokio::spawn(async move {
                // Wait for Nextcloud to finish first-run initialization
                tokio::time::sleep(std::time::Duration::from_secs(30)).await;
                for domain_idx in 1..=2u8 {
                    let value = if domain_idx == 1 { host_ip.as_str() } else { "localhost" };
                    let _ = tokio::process::Command::new("podman")
                        .args([
                            "exec", "-u", "33", "nextcloud",
                            "php", "occ", "config:system:set",
                            "trusted_domains", &domain_idx.to_string(),
                            "--value", value,
                        ])
                        .output()
                        .await;
                }
                info!("Nextcloud trusted domains configured for {}", host_ip);
            });
        }

        // Post-install: Build and start bitcoin-ui container for Bitcoin Knots
        if matches!(package_id, "bitcoin" | "bitcoin-core" | "bitcoin-knots") {
            tokio::spawn(async move {
                let ui_dir = "/opt/archipelago/docker/bitcoin-ui";
                let _ = tokio::process::Command::new("podman")
                    .args(["build", "-t", "localhost/bitcoin-ui", ui_dir])
                    .output()
                    .await;
                let _ = tokio::process::Command::new("podman")
                    .args(["rm", "-f", "bitcoin-ui"])
                    .output()
                    .await;
                let _ = tokio::process::Command::new("podman")
                    .args([
                        "run", "-d", "--name", "bitcoin-ui",
                        "--restart=unless-stopped",
                        "-p", "8334:80",
                        "localhost/bitcoin-ui:latest",
                    ])
                    .output()
                    .await;
                info!("Bitcoin UI container started on port 8334");
            });
        }

        Ok(serde_json::json!({
            "success": true,
            "package_id": package_id,
            "container_id": container_id,
            "message": format!("Package {} installed and started", package_id)
        }))
    }

    /// Install Immich stack (postgres + redis + server)
    async fn install_immich_stack(&self) -> Result<serde_json::Value> {
        let check = tokio::process::Command::new("podman")
            .args(["ps", "-a", "--format", "{{.Names}}"])
            .output()
            .await
            .context("Failed to list containers")?;
        let stdout = String::from_utf8_lossy(&check.stdout);
        if stdout.contains("immich_server") {
            return Err(anyhow::anyhow!("Immich already installed. Stop and remove it first."));
        }
        if stdout.contains("immich\n") || stdout.lines().any(|l| l.trim() == "immich") {
            let _ = tokio::process::Command::new("podman")
                .args(["stop", "immich"])
                .output()
                .await;
            let _ = tokio::process::Command::new("podman")
                .args(["rm", "-f", "immich"])
                .output()
                .await;
        }

        let images = [
            "ghcr.io/immich-app/postgres:14-vectorchord0.4.3-pgvectors0.2.0",
            "docker.io/valkey/valkey:7-alpine",
            "ghcr.io/immich-app/immich-server:release",
        ];
        for img in &images {
            let _ = tokio::process::Command::new("podman")
                .args(["pull", img])
                .output()
                .await;
        }

        let _ = tokio::process::Command::new("sudo")
            .args(["mkdir", "-p", "/var/lib/archipelago/immich", "/var/lib/archipelago/immich-db"])
            .output()
            .await;
        let _ = tokio::process::Command::new("podman")
            .args(["network", "create", "immich-net"])
            .output()
            .await;

        let _ = tokio::process::Command::new("podman")
            .args([
                "run", "-d", "--name", "immich_postgres", "--restart", "unless-stopped",
                "--network", "immich-net",
                "-v", "/var/lib/archipelago/immich-db:/var/lib/postgresql/data",
                "-e", "POSTGRES_PASSWORD=immichpass", "-e", "POSTGRES_USER=postgres", "-e", "POSTGRES_DB=immich",
                "ghcr.io/immich-app/postgres:14-vectorchord0.4.3-pgvectors0.2.0",
            ])
            .output()
            .await;
        tokio::time::sleep(std::time::Duration::from_secs(5)).await;

        let _ = tokio::process::Command::new("podman")
            .args([
                "run", "-d", "--name", "immich_redis", "--restart", "unless-stopped",
                "--network", "immich-net",
                "docker.io/valkey/valkey:7-alpine",
            ])
            .output()
            .await;
        tokio::time::sleep(std::time::Duration::from_secs(2)).await;

        let run = tokio::process::Command::new("podman")
            .args([
                "run", "-d", "--name", "immich_server", "--restart", "unless-stopped",
                "--network", "immich-net", "-p", "2283:2283",
                "-v", "/var/lib/archipelago/immich:/usr/src/app/upload",
                "-e", "DB_HOSTNAME=immich_postgres", "-e", "DB_USERNAME=postgres",
                "-e", "DB_PASSWORD=immichpass", "-e", "DB_DATABASE_NAME=immich",
                "-e", "REDIS_HOSTNAME=immich_redis", "-e", "UPLOAD_LOCATION=/usr/src/app/upload",
                "ghcr.io/immich-app/immich-server:release",
            ])
            .output()
            .await
            .context("Failed to start immich_server")?;

        if !run.status.success() {
            let stderr = String::from_utf8_lossy(&run.stderr);
            return Err(anyhow::anyhow!("Failed to start Immich server: {}", stderr));
        }

        Ok(serde_json::json!({
            "success": true,
            "package_id": "immich",
            "message": "Immich stack installed and started"
        }))
    }

    /// Install Penpot stack (postgres + valkey + backend + exporter + frontend)
    async fn install_penpot_stack(&self) -> Result<serde_json::Value> {
        let check = tokio::process::Command::new("podman")
            .args(["ps", "-a", "--format", "{{.Names}}"])
            .output()
            .await
            .context("Failed to list containers")?;
        let stdout = String::from_utf8_lossy(&check.stdout);
        if stdout.contains("penpot-frontend") {
            return Err(anyhow::anyhow!("Penpot already installed. Stop and remove it first."));
        }

        let images = [
            "docker.io/postgres:15",
            "docker.io/valkey/valkey:8.1",
            "docker.io/penpotapp/backend:2.4",
            "docker.io/penpotapp/exporter:2.4",
            "docker.io/penpotapp/frontend:2.4",
        ];
        for img in &images {
            let _ = tokio::process::Command::new("podman")
                .args(["pull", img])
                .output()
                .await;
        }

        let _ = tokio::process::Command::new("sudo")
            .args(["mkdir", "-p", "/var/lib/archipelago/penpot-assets"])
            .output()
            .await;
        let _ = tokio::process::Command::new("podman")
            .args(["network", "create", "penpot-net"])
            .output()
            .await;

        // Generate a stable secret key derived from the data directory
        let secret = {
            use sha2::{Digest, Sha256};
            let mut hasher = Sha256::new();
            hasher.update(b"penpot-secret-");
            hasher.update(self.config.data_dir.to_string_lossy().as_bytes());
            hex::encode(hasher.finalize())
        };
        let host_ip = &self.config.host_ip;

        let _ = tokio::process::Command::new("podman")
            .args([
                "run", "-d", "--name", "penpot-postgres", "--restart", "unless-stopped",
                "--network", "penpot-net",
                "-v", "/var/lib/archipelago/penpot-postgres:/var/lib/postgresql/data",
                "-e", "POSTGRES_DB=penpot", "-e", "POSTGRES_USER=penpot", "-e", "POSTGRES_PASSWORD=penpot",
                "docker.io/postgres:15",
            ])
            .output()
            .await;
        tokio::time::sleep(std::time::Duration::from_secs(5)).await;

        let _ = tokio::process::Command::new("podman")
            .args([
                "run", "-d", "--name", "penpot-valkey", "--restart", "unless-stopped",
                "--network", "penpot-net",
                "-e", "VALKEY_EXTRA_FLAGS=--maxmemory 128mb --maxmemory-policy volatile-lfu",
                "docker.io/valkey/valkey:8.1",
            ])
            .output()
            .await;
        tokio::time::sleep(std::time::Duration::from_secs(3)).await;

        let _ = tokio::process::Command::new("podman")
            .args([
                "run", "-d", "--name", "penpot-backend", "--restart", "unless-stopped",
                "--network", "penpot-net",
                "-v", "/var/lib/archipelago/penpot-assets:/opt/data/assets",
                "-e", &format!("PENPOT_PUBLIC_URI=http://{}:9001", host_ip),
                "-e", &format!("PENPOT_SECRET_KEY={}", secret),
                "-e", "PENPOT_DATABASE_URI=postgresql://penpot-postgres/penpot",
                "-e", "PENPOT_DATABASE_USERNAME=penpot", "-e", "PENPOT_DATABASE_PASSWORD=penpot",
                "-e", "PENPOT_REDIS_URI=redis://penpot-valkey/0",
                "-e", "PENPOT_OBJECTS_STORAGE_BACKEND=fs",
                "-e", "PENPOT_OBJECTS_STORAGE_FS_DIRECTORY=/opt/data/assets",
                "-e", "PENPOT_FLAGS=disable-email-verification enable-smtp enable-prepl-server disable-secure-session-cookies",
                "docker.io/penpotapp/backend:2.4",
            ])
            .output()
            .await;
        tokio::time::sleep(std::time::Duration::from_secs(5)).await;

        let _ = tokio::process::Command::new("podman")
            .args([
                "run", "-d", "--name", "penpot-exporter", "--restart", "unless-stopped",
                "--network", "penpot-net",
                "-e", &format!("PENPOT_SECRET_KEY={}", secret),
                "-e", "PENPOT_PUBLIC_URI=http://penpot-frontend:8080",
                "-e", "PENPOT_REDIS_URI=redis://penpot-valkey/0",
                "docker.io/penpotapp/exporter:2.4",
            ])
            .output()
            .await;
        tokio::time::sleep(std::time::Duration::from_secs(2)).await;

        let run = tokio::process::Command::new("podman")
            .args([
                "run", "-d", "--name", "penpot-frontend", "--restart", "unless-stopped",
                "--network", "penpot-net", "-p", "9001:8080",
                "-v", "/var/lib/archipelago/penpot-assets:/opt/data/assets",
                "-e", &format!("PENPOT_PUBLIC_URI=http://{}:9001", host_ip),
                "-e", "PENPOT_FLAGS=disable-email-verification enable-smtp enable-prepl-server disable-secure-session-cookies",
                "docker.io/penpotapp/frontend:2.4",
            ])
            .output()
            .await
            .context("Failed to start penpot-frontend")?;

        if !run.status.success() {
            let stderr = String::from_utf8_lossy(&run.stderr);
            return Err(anyhow::anyhow!("Failed to start Penpot frontend: {}", stderr));
        }

        Ok(serde_json::json!({
            "success": true,
            "package_id": "penpot",
            "message": "Penpot stack installed and started"
        }))
    }

    pub(super) async fn handle_package_start(
        &self,
        params: Option<serde_json::Value>,
    ) -> Result<serde_json::Value> {
        let params = params.ok_or_else(|| anyhow::anyhow!("Missing params"))?;
        let package_id = params
            .get("id")
            .and_then(|v| v.as_str())
            .ok_or_else(|| anyhow::anyhow!("Missing package id"))?;
        validate_app_id(package_id)?;

        let containers = get_containers_for_app(package_id).await?;
        let to_start: Vec<String> = if containers.is_empty() {
            vec![format!("archy-{}", package_id)]
        } else {
            let order: &[&str] = match package_id {
                "mempool" | "mempool-web" => &["archy-mempool-db", "mysql-mempool", "electrumx", "mempool-electrs", "mempool-api", "archy-mempool-api", "archy-mempool-web", "mempool"],
                "immich" => &["immich_postgres", "immich_redis", "immich_server"],
                "penpot" | "penpot-frontend" => &["penpot-postgres", "penpot-valkey", "penpot-backend", "penpot-exporter", "penpot-frontend"],
                _ => &["archy-mempool-db", "mysql-mempool", "electrumx", "mempool-electrs", "mempool-api", "archy-mempool-api", "archy-mempool-web", "mempool"],
            };
            let mut sorted = containers;
            sorted.sort_by_key(|c| order.iter().position(|o| *o == c).unwrap_or(99));
            sorted
        };

        // Clear user-stopped flag — user explicitly started this app
        crate::crash_recovery::clear_user_stopped(&self.config.data_dir, package_id).await;
        for name in &to_start {
            crate::crash_recovery::clear_user_stopped(&self.config.data_dir, name).await;
        }

        for name in to_start {
            let _ = tokio::process::Command::new("podman")
                .args(["start", &name])
                .output()
                .await;
        }

        Ok(serde_json::Value::Null)
    }

    pub(super) async fn handle_package_stop(
        &self,
        params: Option<serde_json::Value>,
    ) -> Result<serde_json::Value> {
        let params = params.ok_or_else(|| anyhow::anyhow!("Missing params"))?;
        let package_id = params
            .get("id")
            .and_then(|v| v.as_str())
            .ok_or_else(|| anyhow::anyhow!("Missing package id"))?;
        validate_app_id(package_id)?;

        // Mark as user-stopped so health monitor and crash recovery don't auto-restart
        crate::crash_recovery::mark_user_stopped(&self.config.data_dir, package_id).await;

        let containers = get_containers_for_app(package_id).await?;
        if containers.is_empty() {
            let container_name = format!("archy-{}", package_id);
            crate::crash_recovery::mark_user_stopped(&self.config.data_dir, &container_name).await;
            let _ = tokio::process::Command::new("podman")
                .args(["stop", &container_name])
                .output()
                .await;
            return Ok(serde_json::Value::Null);
        }

        for name in &containers {
            crate::crash_recovery::mark_user_stopped(&self.config.data_dir, name).await;
        }
        for name in containers {
            let _ = tokio::process::Command::new("podman")
                .args(["stop", &name])
                .output()
                .await;
        }

        Ok(serde_json::Value::Null)
    }

    pub(super) async fn handle_package_restart(
        &self,
        params: Option<serde_json::Value>,
    ) -> Result<serde_json::Value> {
        let params = params.ok_or_else(|| anyhow::anyhow!("Missing params"))?;
        let package_id = params
            .get("id")
            .and_then(|v| v.as_str())
            .ok_or_else(|| anyhow::anyhow!("Missing package id"))?;
        validate_app_id(package_id)?;

        let containers = get_containers_for_app(package_id).await?;
        if containers.is_empty() {
            let container_name = format!("archy-{}", package_id);
            let _ = tokio::process::Command::new("podman")
                .args(["restart", &container_name])
                .output()
                .await;
            return Ok(serde_json::Value::Null);
        }

        for name in containers {
            let _ = tokio::process::Command::new("podman")
                .args(["restart", &name])
                .output()
                .await;
        }

        Ok(serde_json::Value::Null)
    }

    /// Uninstall a package: stop and remove all related containers, clean data.
    pub(super) async fn handle_package_uninstall(
        &self,
        params: Option<serde_json::Value>,
    ) -> Result<serde_json::Value> {
        let params = params.ok_or_else(|| anyhow::anyhow!("Missing params"))?;
        let package_id = params
            .get("id")
            .and_then(|v| v.as_str())
            .ok_or_else(|| anyhow::anyhow!("Missing package id"))?;
        validate_app_id(package_id)?;
        let preserve_data = params
            .get("preserve_data")
            .and_then(|v| v.as_bool())
            .unwrap_or(false);

        let containers_to_remove = get_containers_for_app(package_id).await?;

        if containers_to_remove.is_empty() {
            tracing::warn!("Uninstall {}: no containers found", package_id);
        }

        let mut stopped = 0u32;
        let mut removed = 0u32;
        let mut errors = Vec::new();

        for name in &containers_to_remove {
            tracing::info!("Uninstall {}: stopping container {}", package_id, name);
            let stop_out = tokio::process::Command::new("podman")
                .args(["stop", "-t", "10", name])
                .output()
                .await;
            match stop_out {
                Ok(o) if o.status.success() => stopped += 1,
                Ok(o) => {
                    let stderr = String::from_utf8_lossy(&o.stderr);
                    tracing::warn!("Uninstall {}: stop {} failed: {}", package_id, name, stderr.trim());
                }
                Err(e) => {
                    tracing::warn!("Uninstall {}: stop {} error: {}", package_id, name, e);
                }
            }

            tracing::info!("Uninstall {}: removing container {}", package_id, name);
            let rm_out = tokio::process::Command::new("podman")
                .args(["rm", "-f", name])
                .output()
                .await;
            match rm_out {
                Ok(o) if o.status.success() => removed += 1,
                Ok(o) => {
                    let stderr = String::from_utf8_lossy(&o.stderr);
                    let msg = format!("Failed to remove {}: {}", name, stderr.trim());
                    tracing::error!("Uninstall {}: {}", package_id, msg);
                    errors.push(msg);
                }
                Err(e) => {
                    let msg = format!("Failed to remove {}: {}", name, e);
                    tracing::error!("Uninstall {}: {}", package_id, msg);
                    errors.push(msg);
                }
            }
        }

        // Release port allocation
        if let Ok(mut allocator) = self.port_allocator.lock() {
            let _ = allocator.release(package_id);
        }

        // Clean data directories unless preserve_data
        if !preserve_data {
            let data_dirs = get_data_dirs_for_app(package_id);
            for dir in &data_dirs {
                tracing::info!("Uninstall {}: removing data {}", package_id, dir);
                let rm_out = tokio::process::Command::new("sudo")
                    .args(["rm", "-rf", dir])
                    .output()
                    .await;
                if let Ok(o) = rm_out {
                    if !o.status.success() {
                        tracing::warn!("Uninstall {}: rm {} failed", package_id, dir);
                    }
                }
            }
        }

        if !errors.is_empty() {
            tracing::error!("Uninstall {} completed with errors: {:?}", package_id, errors);
        } else {
            tracing::info!("Uninstall {} complete: stopped={}, removed={}", package_id, stopped, removed);
        }

        Ok(serde_json::json!({
            "status": if errors.is_empty() { "uninstalled" } else { "partial" },
            "stopped": stopped,
            "removed": removed,
            "errors": errors,
        }))
    }

    /// Start a bundled app (create container from pre-loaded image if needed, then start)
    pub(super) async fn handle_bundled_app_start(
        &self,
        params: Option<serde_json::Value>,
    ) -> Result<serde_json::Value> {
        let params = params.ok_or_else(|| anyhow::anyhow!("Missing params"))?;
        let app_id = params
            .get("app_id")
            .and_then(|v| v.as_str())
            .ok_or_else(|| anyhow::anyhow!("Missing app_id"))?;
        validate_app_id(app_id)?;
        let image = params
            .get("image")
            .and_then(|v| v.as_str())
            .ok_or_else(|| anyhow::anyhow!("Missing image"))?;
        if !is_valid_docker_image(image) {
            return Err(anyhow::anyhow!("Invalid Docker image format"));
        }
        let ports = params
            .get("ports")
            .and_then(|v| v.as_array())
            .ok_or_else(|| anyhow::anyhow!("Missing ports"))?;
        let volumes = params
            .get("volumes")
            .and_then(|v| v.as_array())
            .ok_or_else(|| anyhow::anyhow!("Missing volumes"))?;

        let check_output = tokio::process::Command::new("podman")
            .args(["ps", "-a", "--format", "{{.Names}}", "--filter", &format!("name={}", app_id)])
            .output()
            .await
            .context("Failed to check container")?;

        let existing = String::from_utf8_lossy(&check_output.stdout);

        if existing.trim().is_empty() {
            let mut cmd = tokio::process::Command::new("podman");
            cmd.args(["run", "-d", "--name", app_id]);

            for port in ports {
                if let (Some(host), Some(container)) = (
                    port.get("host").and_then(|v| v.as_u64()),
                    port.get("container").and_then(|v| v.as_u64()),
                ) {
                    cmd.arg("-p").arg(format!("{}:{}", host, container));
                }
            }

            for volume in volumes {
                if let (Some(host), Some(container)) = (
                    volume.get("host").and_then(|v| v.as_str()),
                    volume.get("container").and_then(|v| v.as_str()),
                ) {
                    // Validate host path: must be under /var/lib/archipelago/ and no traversal
                    if !host.starts_with("/var/lib/archipelago/") || host.contains("..") || host.contains('\0') {
                        return Err(anyhow::anyhow!(
                            "Volume host path must be under /var/lib/archipelago/ and cannot contain path traversal"
                        ));
                    }
                    // Validate container path
                    if container.contains("..") || container.contains('\0') {
                        return Err(anyhow::anyhow!("Invalid container mount path"));
                    }
                    let _ = tokio::process::Command::new("sudo")
                        .args(["mkdir", "-p", host])
                        .output()
                        .await;
                    cmd.arg("-v").arg(format!("{}:{}", host, container));
                }
            }

            cmd.arg(image);

            let output = cmd.output().await.context("Failed to create container")?;

            if !output.status.success() {
                let stderr = String::from_utf8_lossy(&output.stderr);
                return Err(anyhow::anyhow!("Failed to create container: {}", stderr));
            }
        } else {
            let output = tokio::process::Command::new("podman")
                .args(["start", app_id])
                .output()
                .await
                .context("Failed to start container")?;

            if !output.status.success() {
                let stderr = String::from_utf8_lossy(&output.stderr);
                return Err(anyhow::anyhow!("Failed to start container: {}", stderr));
            }
        }

        Ok(serde_json::json!({ "status": "started", "app_id": app_id }))
    }

    /// Stop a bundled app
    pub(super) async fn handle_bundled_app_stop(
        &self,
        params: Option<serde_json::Value>,
    ) -> Result<serde_json::Value> {
        let params = params.ok_or_else(|| anyhow::anyhow!("Missing params"))?;
        let app_id = params
            .get("app_id")
            .and_then(|v| v.as_str())
            .ok_or_else(|| anyhow::anyhow!("Missing app_id"))?;
        validate_app_id(app_id)?;

        let output = tokio::process::Command::new("podman")
            .args(["stop", app_id])
            .output()
            .await
            .context("Failed to stop container")?;

        if !output.status.success() {
            let stderr = String::from_utf8_lossy(&output.stderr);
            return Err(anyhow::anyhow!("Failed to stop container: {}", stderr));
        }

        Ok(serde_json::json!({ "status": "stopped", "app_id": app_id }))
    }

    /// Set install progress for a package and broadcast the update.
    /// Creates a minimal package entry if one doesn't exist yet.
    async fn set_install_progress(&self, package_id: &str, downloaded: u64, size: u64) {
        let (mut data, _rev) = self.state_manager.get_snapshot().await;
        let entry = data
            .package_data
            .entry(package_id.to_string())
            .or_insert_with(|| create_installing_entry(package_id));
        entry.state = PackageState::Installing;
        entry.install_progress = Some(InstallProgress { size, downloaded });
        self.state_manager.update_data(data).await;
    }

    /// Clear install progress after pull completes or fails
    async fn clear_install_progress(&self, package_id: &str) {
        let (mut data, _rev) = self.state_manager.get_snapshot().await;
        if let Some(entry) = data.package_data.get_mut(package_id) {
            entry.install_progress = None;
        }
        self.state_manager.update_data(data).await;
    }

    /// Update install progress (static method for use in async closures)
    async fn update_install_progress(
        state_manager: &crate::state::StateManager,
        package_id: &str,
        downloaded: u64,
        total: u64,
    ) {
        let (mut data, _rev) = state_manager.get_snapshot().await;
        let entry = data
            .package_data
            .entry(package_id.to_string())
            .or_insert_with(|| create_installing_entry(package_id));
        entry.install_progress = Some(InstallProgress {
            size: total,
            downloaded,
        });
        state_manager.update_data(data).await;
    }
}

/// Create a minimal PackageDataEntry for a package being installed
fn create_installing_entry(package_id: &str) -> PackageDataEntry {
    PackageDataEntry {
        state: PackageState::Installing,
        health: None,
        static_files: StaticFiles {
            license: String::new(),
            instructions: String::new(),
            icon: format!("/assets/img/app-icons/{}.png", package_id),
        },
        manifest: Manifest {
            id: package_id.to_string(),
            title: package_id.to_string(),
            version: String::new(),
            description: Description {
                short: "Installing...".to_string(),
                long: String::new(),
            },
            release_notes: String::new(),
            license: String::new(),
            wrapper_repo: String::new(),
            upstream_repo: String::new(),
            support_site: String::new(),
            marketing_site: String::new(),
            donation_url: None,
            author: None,
            website: None,
            interfaces: None,
            tier: None,
        },
        installed: None,
        install_progress: None,
    }
}

/// Parse podman pull progress output.
/// Podman outputs lines like: "Copying blob sha256:abc done  |  50.0MiB / 100.0MiB"
/// Returns (downloaded_bytes, total_bytes) if parseable.
fn parse_pull_progress(line: &str) -> Option<(u64, u64)> {
    // Look for "X.YMiB / Z.WMiB" or "X.YGiB / Z.WGiB" patterns
    let line = line.trim();

    // Find the pattern "NUMBER UNIT / NUMBER UNIT"
    let parts: Vec<&str> = line.split('/').collect();
    if parts.len() != 2 {
        return None;
    }

    let downloaded = parse_size_value(parts[0].trim())?;
    let total = parse_size_value(parts[1].trim())?;

    if total > 0 {
        Some((downloaded, total))
    } else {
        None
    }
}

/// Parse a size value like "50.0MiB", "1.2GiB", "500KiB" into bytes
fn parse_size_value(s: &str) -> Option<u64> {
    // Extract the last token which should be "NUMBER UNIT" or "NUMBERUnit"
    let s = s.trim();

    // Try to find the numeric part at the end of the string
    // Podman formats: "50.0MiB", "1.2 GiB", etc.
    let (num_str, multiplier) = if let Some(pos) = s.rfind("GiB") {
        (s[..pos].trim().split_whitespace().last()?, 1024 * 1024 * 1024)
    } else if let Some(pos) = s.rfind("MiB") {
        (s[..pos].trim().split_whitespace().last()?, 1024 * 1024)
    } else if let Some(pos) = s.rfind("KiB") {
        (s[..pos].trim().split_whitespace().last()?, 1024)
    } else if let Some(pos) = s.rfind("GB") {
        (s[..pos].trim().split_whitespace().last()?, 1_000_000_000)
    } else if let Some(pos) = s.rfind("MB") {
        (s[..pos].trim().split_whitespace().last()?, 1_000_000)
    } else if let Some(pos) = s.rfind("KB") {
        (s[..pos].trim().split_whitespace().last()?, 1_000)
    } else if let Some(pos) = s.rfind('B') {
        (s[..pos].trim().split_whitespace().last()?, 1)
    } else {
        return None;
    };

    let num: f64 = num_str.parse().ok()?;
    Some((num * multiplier as f64) as u64)
}

/// Get all container names for an app (handles multi-container apps like mempool)
async fn get_containers_for_app(package_id: &str) -> Result<Vec<String>> {
    validate_app_id(package_id)?;
    let output = tokio::process::Command::new("podman")
        .args(["ps", "-a", "--format", "{{.Names}}"])
        .output()
        .await
        .context("Failed to list containers")?;
    let stdout = String::from_utf8_lossy(&output.stdout);
    let all: Vec<&str> = stdout.lines().filter(|s| !s.is_empty()).collect();

    let patterns: Vec<String> = match package_id {
        "mempool" | "mempool-web" => {
            vec![
                "electrumx".into(),
                "mempool-electrs".into(),
                "mempool-api".into(),
                "archy-mempool-api".into(),
                "archy-mempool-web".into(),
                "mempool".into(),
                "archy-mempool-db".into(),
                "mysql-mempool".into(),
            ]
        }
        "fedimint" => vec!["fedimint".into(), "fedimint-ui".into(), "archy-fedimint".into(), "fedimint-gateway".into()],
        "fedimint-gateway" => vec!["fedimint-gateway".into()],
        "immich" => vec![
            "immich_postgres".into(),
            "immich_redis".into(),
            "immich_server".into(),
        ],
        "penpot" | "penpot-frontend" => vec![
            "penpot-postgres".into(),
            "penpot-valkey".into(),
            "penpot-backend".into(),
            "penpot-exporter".into(),
            "penpot-frontend".into(),
        ],
        _ => vec![package_id.to_string(), format!("archy-{}", package_id)],
    };

    let mut result = Vec::new();
    for name in all {
        for pat in &patterns {
            if name == pat {
                result.push(name.to_string());
                break;
            }
        }
    }
    Ok(result)
}

/// Get data directories to clean for an app.
/// Caller must validate package_id before calling.
fn get_data_dirs_for_app(package_id: &str) -> Vec<String> {
    let base = "/var/lib/archipelago";
    match package_id {
        "mempool" | "mempool-web" => vec![
            format!("{}/mempool", base),
            format!("{}/mysql-mempool", base),
            format!("{}/electrumx", base),
            format!("{}/mempool-electrs", base),
        ],
        "fedimint" => vec![format!("{}/fedimint", base), format!("{}/fedimint-gateway", base)],
        "fedimint-gateway" => vec![format!("{}/fedimint-gateway", base)],
        "immich" => vec![
            format!("{}/immich", base),
            format!("{}/immich-db", base),
        ],
        "penpot" | "penpot-frontend" => vec![
            format!("{}/penpot-assets", base),
            format!("{}/penpot-postgres", base),
        ],
        _ => vec![format!("{}/{}", base, package_id)],
    }
}

/// Trusted Docker registries. Only images from these sources are allowed.
const TRUSTED_REGISTRIES: &[&str] = &[
    "docker.io/",
    "ghcr.io/",
    "localhost/",
];

/// Validate Docker image against trusted registry allowlist.
/// Detect which Bitcoin container is running on archy-net for DNS resolution.
/// Returns the container name to use as the RPC host (e.g., "bitcoin-knots").
fn detect_bitcoin_container_name() -> String {
    // Synchronous check — called from get_app_config which is sync
    let output = std::process::Command::new("podman")
        .args(["ps", "--format", "{{.Names}}"])
        .output();
    if let Ok(out) = output {
        let names = String::from_utf8_lossy(&out.stdout);
        for candidate in &["bitcoin-knots", "bitcoin-core", "bitcoin"] {
            if names.lines().any(|l| l.trim() == *candidate) {
                return candidate.to_string();
            }
        }
    }
    // Default to bitcoin-knots (most common)
    "bitcoin-knots".to_string()
}

fn is_valid_docker_image(image: &str) -> bool {
    if image.is_empty() || image.len() > 256 {
        return false;
    }
    // Reject shell metacharacters
    let dangerous_chars = ['&', '|', ';', '`', '$', '(', ')', '<', '>', '\n', '\r'];
    if image.chars().any(|c| dangerous_chars.contains(&c)) {
        return false;
    }
    // Must come from a trusted registry — match the exact domain, not just prefix
    let registry = match image.split('/').next() {
        Some(r) => r,
        None => return false,
    };
    matches!(registry, "docker.io" | "ghcr.io" | "localhost")
}

/// Validate that a package/app ID is safe (lowercase alphanumeric + hyphens, 1-64 chars).
pub(super) fn validate_app_id(id: &str) -> Result<()> {
    if id.is_empty() || id.len() > 64 {
        anyhow::bail!("Invalid app id: must be 1-64 characters");
    }
    if !id.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') {
        anyhow::bail!("Invalid app id: only lowercase letters, digits, and hyphens allowed");
    }
    if id.starts_with('-') {
        anyhow::bail!("Invalid app id: must not start with a hyphen");
    }
    Ok(())
}

/// Per-app Linux capabilities needed beyond the default cap-drop=ALL.
/// Most apps need CHOWN/SETUID/SETGID for internal user switching.
fn get_app_capabilities(app_id: &str) -> Vec<String> {
    match app_id {
        // Apps that need user switching and file ownership changes
        "nextcloud" | "homeassistant" | "home-assistant" | "btcpay-server" | "btcpayserver"
        | "jellyfin" | "onlyoffice" | "onlyoffice-documentserver" | "portainer" => vec![
            "--cap-add=CHOWN".to_string(),
            "--cap-add=SETUID".to_string(),
            "--cap-add=SETGID".to_string(),
            "--cap-add=DAC_OVERRIDE".to_string(),
        ],
        // Nginx Proxy Manager needs to bind low ports
        "nginx-proxy-manager" => vec![
            "--cap-add=CHOWN".to_string(),
            "--cap-add=SETUID".to_string(),
            "--cap-add=SETGID".to_string(),
            "--cap-add=NET_BIND_SERVICE".to_string(),
        ],
        // Bitcoin and Lightning need file ownership ops + DAC_OVERRIDE for data dir access
        "bitcoin" | "bitcoin-core" | "bitcoin-knots" | "lnd" | "fedimint" | "fedimint-gateway" => vec![
            "--cap-add=CHOWN".to_string(),
            "--cap-add=FOWNER".to_string(),
            "--cap-add=SETUID".to_string(),
            "--cap-add=SETGID".to_string(),
            "--cap-add=DAC_OVERRIDE".to_string(),
        ],
        // Vaultwarden needs file ownership + NET_BIND_SERVICE (binds port 80 internally)
        "vaultwarden" => vec![
            "--cap-add=CHOWN".to_string(),
            "--cap-add=SETUID".to_string(),
            "--cap-add=SETGID".to_string(),
            "--cap-add=NET_BIND_SERVICE".to_string(),
        ],
        // PhotoPrism uses s6-overlay which needs privilege ops
        "photoprism" => vec![
            "--cap-add=CHOWN".to_string(),
            "--cap-add=SETUID".to_string(),
            "--cap-add=SETGID".to_string(),
        ],
        // Grafana runs as specific UID (472)
        "grafana" => vec![
            "--cap-add=CHOWN".to_string(),
            "--cap-add=SETUID".to_string(),
            "--cap-add=SETGID".to_string(),
        ],
        // Uptime-kuma startup script needs chown/fowner for /app/data ownership
        "uptime-kuma" => vec![
            "--cap-add=CHOWN".to_string(),
            "--cap-add=FOWNER".to_string(),
            "--cap-add=SETUID".to_string(),
            "--cap-add=SETGID".to_string(),
        ],
        // Minimal apps (searxng, filebrowser, etc.) need no extra caps
        _ => vec![],
    }
}

/// Apps safe to run with --read-only root filesystem.
/// These work correctly with volume mounts + tmpfs for /tmp and /run.
fn is_readonly_compatible(app_id: &str) -> bool {
    matches!(
        app_id,
        "searxng"
            | "grafana"
            | "filebrowser"
            | "electrumx"
            | "mempool-electrs"
            | "electrs"
            | "nostr-rs-relay"
            | "ollama"
            | "indeedhub"
    )
}

/// Get container health check arguments for podman run.
/// Returns (health-cmd, interval, retries) args to append to run_args.
fn get_health_check_args(app_id: &str, rpc_pass: &str) -> Vec<String> {
    let btc_health = format!("bitcoin-cli -rpcuser=archipelago -rpcpassword={} getblockchaininfo || exit 1", rpc_pass);
    let (cmd, interval, retries) = match app_id {
        "bitcoin" | "bitcoin-core" | "bitcoin-knots" => (
            btc_health.as_str(),
            "30s", "3",
        ),
        "lnd" => (
            "lncli getinfo || exit 1",
            "30s", "3",
        ),
        "btcpay-server" | "btcpayserver" => (
            "curl -sf http://localhost:49392/ || exit 1",
            "30s", "3",
        ),
        "mempool-api" => (
            "curl -sf http://localhost:8999/api/v1/backend-info || exit 1",
            "30s", "3",
        ),
        "mempool" | "mempool-web" | "archy-mempool-web" => (
            "curl -sf http://localhost:8080/ || exit 1",
            "30s", "3",
        ),
        "electrumx" | "mempool-electrs" | "electrs" => (
            "curl -sf http://localhost:8000/ || exit 1",
            "60s", "3",
        ),
        "nextcloud" => (
            "curl -sf http://localhost:80/status.php || exit 1",
            "30s", "3",
        ),
        "homeassistant" | "home-assistant" => (
            "curl -sf http://localhost:8123/api/ || exit 1",
            "30s", "3",
        ),
        "grafana" => (
            "curl -sf http://localhost:3000/api/health || exit 1",
            "30s", "3",
        ),
        "jellyfin" => (
            "curl -sf http://localhost:8096/health || exit 1",
            "30s", "3",
        ),
        "vaultwarden" => (
            "curl -sf http://localhost:80/alive || exit 1",
            "30s", "3",
        ),
        "uptime-kuma" => (
            "curl -sf http://localhost:3001/ || exit 1",
            "30s", "3",
        ),
        "filebrowser" => (
            "curl -sf http://localhost:80/health || exit 1",
            "30s", "3",
        ),
        "searxng" => (
            "curl -sf http://localhost:8080/ || exit 1",
            "30s", "3",
        ),
        "photoprism" => (
            "curl -sf http://localhost:2342/api/v1/status || exit 1",
            "60s", "3",
        ),
        "immich_server" | "immich" => (
            "curl -sf http://localhost:2283/api/server/ping || exit 1",
            "30s", "3",
        ),
        "dwn" => (
            "curl -sf http://localhost:3000/health || exit 1",
            "30s", "3",
        ),
        "portainer" => (
            "curl -sf http://localhost:9000/api/status || exit 1",
            "30s", "3",
        ),
        "ollama" => (
            "curl -sf http://localhost:11434/ || exit 1",
            "30s", "3",
        ),
        "fedimint" => (
            "curl -sf http://localhost:8174/health || exit 1",
            "60s", "3",
        ),
        "nostr-rs-relay" | "nostr-relay" => (
            "curl -sf http://localhost:8080/ || exit 1",
            "30s", "3",
        ),
        "nginx-proxy-manager" => (
            "curl -sf http://localhost:81/api/ || exit 1",
            "30s", "3",
        ),
        _ => return vec![],
    };

    vec![
        format!("--health-cmd={}", cmd),
        format!("--health-interval={}", interval),
        format!("--health-retries={}", retries),
        "--health-start-period=60s".to_string(),
    ]
}

/// Get per-app memory limit.
fn get_memory_limit(app_id: &str) -> &'static str {
    match app_id {
        // Heavy apps
        "bitcoin" | "bitcoin-core" | "bitcoin-knots" => "2g",
        "onlyoffice" | "onlyoffice-documentserver" => "2g",
        "ollama" => "4g",
        // Medium apps
        "lnd" => "512m",
        "electrumx" | "mempool-electrs" | "electrs" => "1g",
        "nextcloud" => "1g",
        "immich_server" | "immich" => "1g",
        "btcpay-server" | "btcpayserver" => "1g",
        "homeassistant" | "home-assistant" => "512m",
        "fedimint" => "512m",
        "fedimint-gateway" => "512m",
        "photoprism" => "1g",
        // Light apps
        "mempool-api" => "512m",
        "mempool" | "mempool-web" | "archy-mempool-web" => "256m",
        "grafana" => "256m",
        "jellyfin" => "1g",
        "vaultwarden" => "256m",
        "uptime-kuma" => "256m",
        "filebrowser" => "256m",
        "searxng" => "512m",
        "dwn" => "256m",
        "portainer" => "256m",
        "nostr-rs-relay" | "nostr-relay" => "256m",
        "nginx-proxy-manager" => "256m",
        // Databases
        "archy-btcpay-db" | "archy-mempool-db" | "mysql-mempool" => "512m",
        "immich_postgres" | "penpot-postgres" => "256m",
        "immich_redis" | "penpot-valkey" => "128m",
        // Default
        _ => "512m",
    }
}

/// Get app-specific configuration
/// Returns: (ports, volumes, env_vars, custom_command, custom_args)
fn get_app_config(
    app_id: &str,
    host_ip: &str,
    allocator: &mut PortAllocator,
    rpc_pass: &str,
) -> (Vec<String>, Vec<String>, Vec<String>, Option<String>, Option<Vec<String>>) {
    match app_id {
        "homeassistant" | "home-assistant" => (
            vec!["8123:8123".to_string()],
            vec!["/var/lib/archipelago/home-assistant:/config".to_string()],
            vec!["TZ=UTC".to_string()],
            None,
            None,
        ),
        "bitcoin" | "bitcoin-core" | "bitcoin-knots" => (
            vec!["8332:8332".to_string(), "8333:8333".to_string()],
            vec!["/var/lib/archipelago/bitcoin:/home/bitcoin/.bitcoin".to_string()],
            vec![],
            None,
            None,
        ),
        "lnd" => (
            vec!["9735:9735".to_string(), "10009:10009".to_string(), "8080:8080".to_string()],
            vec!["/var/lib/archipelago/lnd:/root/.lnd".to_string()],
            vec!["BITCOIN_ACTIVE=1".to_string()],
            None,
            None,
        ),
        "btcpay-server" | "btcpayserver" => (
            vec!["23000:49392".to_string()],
            vec!["/var/lib/archipelago/btcpay:/datadir".to_string()],
            vec![
                "ASPNETCORE_URLS=http://0.0.0.0:49392".to_string(),
                "BTCPAY_PROTOCOL=http".to_string(),
                format!("BTCPAY_HOST={}:23000", host_ip),
                "BTCPAY_CHAINS=btc".to_string(),
                format!("BTCPAY_BTCRPCURL=http://{}:8332", host_ip),
                "BTCPAY_BTCRPCUSER=archipelago".to_string(),
                format!("BTCPAY_BTCRPCPASSWORD={}", rpc_pass),
                "BTCPAY_POSTGRES=User ID=btcpay;Password=btcpaypass;Host=archy-btcpay-db;Port=5432;Database=btcpay;Include Error Detail=true".to_string(),
            ],
            None,
            None,
        ),
        "mempool" | "mempool-web" => (
            vec!["4080:8080".to_string()],
            vec![],
            vec![format!("BACKEND_MAINNET_HTTP_HOST={}", host_ip)],
            None,
            None,
        ),
        "mempool-api" => (
            vec!["8999:8999".to_string()],
            vec!["/var/lib/archipelago/mempool:/data".to_string()],
            vec![
                "MEMPOOL_BACKEND=electrum".to_string(),
                "ELECTRUM_HOST=electrumx".to_string(),
                "ELECTRUM_PORT=50001".to_string(),
                "ELECTRUM_TLS_ENABLED=false".to_string(),
                format!("CORE_RPC_HOST={}", host_ip),
                "CORE_RPC_PORT=8332".to_string(),
                "CORE_RPC_USERNAME=archipelago".to_string(),
                format!("CORE_RPC_PASSWORD={}", rpc_pass),
                "DATABASE_ENABLED=true".to_string(),
                "DATABASE_HOST=archy-mempool-db".to_string(),
                "DATABASE_DATABASE=mempool".to_string(),
                "DATABASE_USERNAME=mempool".to_string(),
                "DATABASE_PASSWORD=mempoolpass".to_string(),
            ],
            None,
            None,
        ),
        "electrumx" | "mempool-electrs" | "electrs" => {
            // Detect which bitcoin container is running for archy-net DNS resolution
            let bitcoin_host = detect_bitcoin_container_name();
            (
                vec!["50001:50001".to_string()],
                vec!["/var/lib/archipelago/electrumx:/data".to_string()],
                vec![
                    format!("DAEMON_URL=http://archipelago:{}@{}:8332/", rpc_pass, bitcoin_host),
                    "COIN=Bitcoin".to_string(),
                    "DB_DIRECTORY=/data".to_string(),
                    "SERVICES=tcp://:50001,rpc://0.0.0.0:8000".to_string(),
                ],
                None,
                None,
            )
        },
        "mysql-mempool" => (
            vec![],
            vec!["/var/lib/archipelago/mysql-mempool:/var/lib/mysql".to_string()],
            vec![
                "MYSQL_DATABASE=mempool".to_string(),
                "MYSQL_USER=mempool".to_string(),
                "MYSQL_PASSWORD=mempoolpass".to_string(),
                "MYSQL_ROOT_PASSWORD=rootpass".to_string(),
            ],
            None,
            None,
        ),
        "grafana" => (
            vec!["3000:3000".to_string()],
            vec!["/var/lib/archipelago/grafana:/var/lib/grafana".to_string()],
            vec!["GF_PATHS_DATA=/var/lib/grafana".to_string(), "GF_USERS_ALLOW_SIGN_UP=false".to_string()],
            None,
            None,
        ),
        "searxng" => (
            vec!["8888:8080".to_string()],
            vec![],
            vec![],
            None,
            None,
        ),
        "ollama" => (
            vec!["11434:11434".to_string()],
            vec!["/var/lib/archipelago/ollama:/root/.ollama".to_string()],
            vec![],
            None,
            None,
        ),
        "onlyoffice" | "onlyoffice-documentserver" => (
            vec!["9980:80".to_string()],
            vec![],
            vec![],
            None,
            None,
        ),
        "penpot" | "penpot-frontend" => (
            vec!["9001:80".to_string()],
            vec![],
            vec![],
            None,
            None,
        ),
        "nextcloud" => {
            let host_port = allocator
                .allocate_or_get(app_id, 8085, 80)
                .unwrap_or(8085);
            (
                vec![format!("{}:80", host_port)],
                vec!["/var/lib/archipelago/nextcloud:/var/www/html".to_string()],
                vec![],
                None,
                None,
            )
        }
        "vaultwarden" => {
            let host_port = allocator
                .allocate_or_get(app_id, 8082, 80)
                .unwrap_or(8082);
            (
                vec![format!("{}:80", host_port)],
                vec!["/var/lib/archipelago/vaultwarden:/data".to_string()],
                vec![],
                None,
                None,
            )
        }
        "jellyfin" => (
            vec!["8096:8096".to_string()],
            vec!["/var/lib/archipelago/jellyfin/config:/config".to_string(), "/var/lib/archipelago/jellyfin/cache:/cache".to_string()],
            vec![],
            None,
            None,
        ),
        "photoprism" => (
            vec!["2342:2342".to_string()],
            vec!["/var/lib/archipelago/photoprism:/photoprism/storage".to_string()],
            vec!["PHOTOPRISM_ADMIN_PASSWORD=archipelago".to_string(), "PHOTOPRISM_DEFAULT_LOCALE=en".to_string()],
            None,
            None,
        ),
        "immich" => (
            vec!["2283:2283".to_string()],
            vec!["/var/lib/archipelago/immich:/usr/src/app/upload".to_string()],
            vec![
                "DB_HOSTNAME=immich_postgres".to_string(),
                "DB_USERNAME=postgres".to_string(),
                "DB_PASSWORD=immichpass".to_string(),
                "DB_DATABASE_NAME=immich".to_string(),
                "REDIS_HOSTNAME=immich_redis".to_string(),
                "UPLOAD_LOCATION=/usr/src/app/upload".to_string(),
            ],
            None,
            None,
        ),
        "filebrowser" => {
            let host_port = allocator
                .allocate_or_get(app_id, 8083, 80)
                .unwrap_or(8083);
            (
                vec![format!("{}:80", host_port)],
                vec!["/var/lib/archipelago/filebrowser:/srv".to_string()],
                vec![],
                None,
                None,
            )
        }
        "nginx-proxy-manager" => (
            vec!["81:81".to_string(), "8084:80".to_string(), "8443:443".to_string()],
            vec![
                "/var/lib/archipelago/nginx-proxy-manager/data:/data".to_string(),
                "/var/lib/archipelago/nginx-proxy-manager/letsencrypt:/etc/letsencrypt".to_string(),
            ],
            vec![],
            None,
            None,
        ),
        "portainer" => (
            vec!["9000:9000".to_string()],
            vec!["/var/lib/archipelago/portainer:/data".to_string(), "/var/run/podman/podman.sock:/var/run/docker.sock".to_string()],
            vec![],
            None,
            None,
        ),
        "uptime-kuma" => (
            vec!["3001:3001".to_string()],
            vec!["/var/lib/archipelago/uptime-kuma:/app/data".to_string()],
            vec!["TZ=UTC".to_string()],
            None,
            None,
        ),
        "tailscale" => (
            vec!["8240:8240".to_string()],
            vec![
                "/var/lib/archipelago/tailscale:/var/lib/tailscale".to_string(),
            ],
            vec![
                "TS_STATE_DIR=/var/lib/tailscale".to_string(),
            ],
            Some("sh -c 'tailscale web --listen 0.0.0.0:8240 & exec tailscaled'".to_string()),
            None,
        ),
        "fedimint" => (
            vec![
                "8173:8173".to_string(),
                "8174:8174".to_string(),
                "8175:8175".to_string(),
            ],
            vec!["/var/lib/archipelago/fedimint:/data".to_string()],
            vec![
                "FM_DATA_DIR=/data".to_string(),
                "FM_BITCOIND_USERNAME=archipelago".to_string(),
                format!("FM_BITCOIND_PASSWORD={}", rpc_pass),
                "FM_BITCOIN_NETWORK=bitcoin".to_string(),
                "FM_BIND_P2P=0.0.0.0:8173".to_string(),
                "FM_BIND_API=0.0.0.0:8174".to_string(),
                "FM_BIND_UI=0.0.0.0:8175".to_string(),
                format!("FM_P2P_URL=fedimint://{}:8173", host_ip),
                format!("FM_API_URL=ws://{}:8174", host_ip),
                format!("FM_BITCOIND_URL=http://{}:8332", host_ip),
            ],
            None,
            None,
        ),
        "fedimint-gateway" => (
            vec!["8176:8176".to_string(), "9737:9737".to_string()],
            vec!["/var/lib/archipelago/fedimint-gateway:/data".to_string()],
            vec![],
            None,
            Some(vec![
                "gatewayd".to_string(),
                "--data-dir".to_string(), "/data".to_string(),
                "--listen".to_string(), "0.0.0.0:8176".to_string(),
                "--bcrypt-password-hash".to_string(),
                "$2y$10$t9YjjxkiktrlYvjajB/zgOMDnSNVg4HqrbDqh47u7Jf42whNdxNqC".to_string(),
                "--network".to_string(), "bitcoin".to_string(),
                "--bitcoind-url".to_string(), format!("http://{}:8332", host_ip),
                "--bitcoind-username".to_string(), "archipelago".to_string(),
                "--bitcoind-password".to_string(), rpc_pass.to_string(),
                "ldk".to_string(),
                "--ldk-lightning-port".to_string(), "9737".to_string(),
                "--ldk-alias".to_string(), "archipelago-gateway".to_string(),
            ]),
        ),
        "indeedhub" => (
            vec!["8190:3000".to_string()],
            vec![],
            vec!["NODE_ENV=production".to_string(), "NEXT_TELEMETRY_DISABLED=1".to_string()],
            None,
            None,
        ),
        "nostr-rs-relay" => (
            vec!["18081:8080".to_string()],
            vec!["/var/lib/archipelago/nostr-rs-relay:/usr/src/app/db".to_string()],
            vec![],
            None,
            None,
        ),
        "dwn" => (
            vec!["3100:3000".to_string()],
            vec!["/var/lib/archipelago/dwn:/dwn/data".to_string()],
            vec![
                "DS_PORT=3000".to_string(),
                "DS_MESSAGES_STORE_URI=level://data/messages".to_string(),
                "DS_DATA_STORE_URI=level://data/data".to_string(),
                "DS_EVENT_LOG_URI=level://data/events".to_string(),
            ],
            None,
            None,
        ),
        _ => (vec![], vec![], vec![], None, None),
    }
}