Enhance Immich deployment process by removing old single-container instances and updating service configurations

- Added logic to remove any existing single-container 'immich' instances to prevent conflicts with the new multi-container 'immich_server' stack.
- Updated `build-auto-installer-iso.sh` to utilize configuration files from the `configs/` directory for Nginx and systemd service, ensuring proper setup.
- Modified deployment scripts to ensure the removal of old containers and improved handling of Immich stack creation.
- Updated documentation to reflect changes in service configurations and critical build checklist items.

core/archipelago/src/api/rpc.rs

@@ -791,6 +791,17 @@ impl RpcHandler {
         if stdout.contains("immich_server") {
             return Err(anyhow::anyhow!("Immich already installed. Stop and remove it first."));
         }
+        // Remove old single-container 'immich' if present (wrong port mapping, blocks immich_server)
+        if stdout.contains("immich\n") || stdout.lines().any(|l| l.trim() == "immich") {
+            let _ = tokio::process::Command::new("sudo")
+                .args(["podman", "stop", "immich"])
+                .output()
+                .await;
+            let _ = tokio::process::Command::new("sudo")
+                .args(["podman", "rm", "-f", "immich"])
+                .output()
+                .await;
+        }
 
         let images = [
             "ghcr.io/immich-app/postgres:14-vectorchord0.4.3-pgvectors0.2.0",

image-recipe/ISO-BUILD-CHECKLIST.md

@@ -12,11 +12,13 @@ cd image-recipe
 ```
 
 This captures:
-- [ ] Systemd service configuration (`archipelago.service`)
-- [ ] Nginx configuration (`nginx-archipelago.conf`)
+- [ ] Systemd service configuration (`archipelago.service`) - **User=root** required for Podman
+- [ ] Nginx configuration (`nginx-archipelago.conf`) - includes app proxies (Nextcloud, Vaultwarden, Immich, Penpot)
 - [ ] Logrotate configuration (if exists)
 - [ ] Any custom scripts in `/opt/archipelago/scripts/`
 
+**Critical**: `build-auto-installer-iso.sh` uses `configs/` for nginx and archipelago.service. Ensure these are synced before building.
+
 ### 2. Verify Code Changes
 
 Ensure all code changes are committed:
@@ -86,6 +88,14 @@ Test checklist:
 - [ ] Can access UI at `http://localhost:8080` (or mapped port)
 - [ ] Container detection works: Check logs for "Detected container"
 
+## App Stack Hardening (Immich, Penpot, etc.)
+
+The first-boot script and deploy script ensure:
+- [ ] **Immich**: Old single-container `immich` (wrong port) is removed before creating `immich_server` stack
+- [ ] **First-boot**: Waits for postgres (pg_isready) before starting Immich server
+- [ ] **Backend**: `package.install` for Immich removes old container before creating stack
+- [ ] **Deploy**: Ensures Immich stack on every deploy, cleans up conflicts
+
 ## Post-Build
 
 ### 8. Write to USB (Optional)

image-recipe/build-auto-installer-iso.sh

@@ -200,54 +200,32 @@ RUN apt-get clean && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 DOCKERFILE
 
-    # Create nginx config for the Docker build
-    cat > "$WORK_DIR/nginx-archipelago.conf" <<'NGINXCONF'
+    # Use nginx config from configs/ (includes app proxies for Nextcloud, Vaultwarden, Immich, Penpot)
+    if [ -f "$SCRIPT_DIR/configs/nginx-archipelago.conf" ]; then
+        cp "$SCRIPT_DIR/configs/nginx-archipelago.conf" "$WORK_DIR/nginx-archipelago.conf"
+        echo "   Using nginx config from configs/nginx-archipelago.conf"
+    else
+        echo "   ⚠ configs/nginx-archipelago.conf not found, using minimal config"
+        cat > "$WORK_DIR/nginx-archipelago.conf" <<'NGINXCONF'
 server {
     listen 80;
     server_name _;
-    
     root /opt/archipelago/web-ui;
     index index.html;
-    
-    # Serve static files (Vue.js SPA)
-    location / {
-        try_files $uri $uri/ /index.html;
-    }
-    
-    # Peer-to-peer node messaging (receives from other nodes over Tor)
-    location /archipelago/ {
-        proxy_pass http://127.0.0.1:5678;
-        proxy_http_version 1.1;
-        proxy_set_header Host $host;
-        proxy_set_header X-Real-IP $remote_addr;
-    }
-    
-    # Proxy API requests to backend
-    location /rpc/ {
-        proxy_pass http://127.0.0.1:5678;
-        proxy_http_version 1.1;
-        proxy_set_header Host $host;
-        proxy_set_header X-Real-IP $remote_addr;
-        # Increase timeout for long-running operations (e.g., Docker image pulls)
-        proxy_connect_timeout 300s;
-        proxy_send_timeout 300s;
-        proxy_read_timeout 300s;
-    }
-    
-    # Proxy WebSocket
-    location /ws {
-        proxy_pass http://127.0.0.1:5678;
-        proxy_http_version 1.1;
-        proxy_set_header Upgrade $http_upgrade;
-        proxy_set_header Connection "upgrade";
-        proxy_set_header Host $host;
-        proxy_read_timeout 86400s;
-    }
+    location / { try_files $uri $uri/ /index.html; }
+    location /archipelago/ { proxy_pass http://127.0.0.1:5678; proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; }
+    location /rpc/ { proxy_pass http://127.0.0.1:5678; proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_connect_timeout 300s; proxy_send_timeout 300s; proxy_read_timeout 300s; }
+    location /ws { proxy_pass http://127.0.0.1:5678; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; proxy_set_header Host $host; proxy_read_timeout 86400s; }
 }
 NGINXCONF
+    fi
 
-    # Create systemd service for the Docker build
-    cat > "$WORK_DIR/archipelago.service" <<'SYSTEMDSERVICE'
+    # Use archipelago.service from configs/ (User=root for Podman container access)
+    if [ -f "$SCRIPT_DIR/configs/archipelago.service" ]; then
+        cp "$SCRIPT_DIR/configs/archipelago.service" "$WORK_DIR/archipelago.service"
+        echo "   Using archipelago.service from configs/"
+    else
+        cat > "$WORK_DIR/archipelago.service" <<'SYSTEMDSERVICE'
 [Unit]
 Description=Archipelago Backend
 After=network-online.target
@@ -255,7 +233,7 @@ Wants=network-online.target
 
 [Service]
 Type=simple
-User=archipelago
+User=root
 Environment="ARCHIPELAGO_BIND=0.0.0.0:5678"
 Environment="ARCHIPELAGO_DEV_MODE=true"
 ExecStart=/usr/local/bin/archipelago
@@ -265,6 +243,7 @@ RestartSec=5
 [Install]
 WantedBy=multi-user.target
 SYSTEMDSERVICE
+    fi
 
     echo "   Building $CONTAINER_CMD image (this may take a few minutes)..."
     $CONTAINER_CMD build --platform linux/amd64 -t archipelago-rootfs -f "$WORK_DIR/Dockerfile.rootfs" "$WORK_DIR"
@@ -1017,7 +996,7 @@ fi
 PROFILE
 chmod +x /mnt/target/etc/profile.d/archipelago.sh
 
-# Systemd service is already in rootfs, but ensure it has correct config
+# Systemd service: User=root required for Podman container access
 cat > /mnt/target/etc/systemd/system/archipelago.service <<'SERVICE'
 [Unit]
 Description=Archipelago Backend
@@ -1026,9 +1005,10 @@ Wants=network-online.target
 
 [Service]
 Type=simple
-User=archipelago
+User=root
 Environment="ARCHIPELAGO_BIND=0.0.0.0:5678"
 Environment="ARCHIPELAGO_DEV_MODE=true"
+ExecStartPre=/bin/bash -c 'mkdir -p /etc/archipelago && echo "ARCHIPELAGO_HOST_IP=$(hostname -I 2>/dev/null | awk \"{print \$1}\")" > /etc/archipelago/host-ip.env'
 ExecStart=/usr/local/bin/archipelago
 Restart=on-failure
 RestartSec=5

scripts/deploy-to-target.sh

@@ -401,6 +401,13 @@ if [ "$LIVE" = true ]; then
     sshpass -p "$ARCHIPELAGO_PASSWORD" ssh $SSH_OPTS "$TARGET_HOST" "
       DOCKER=podman
       command -v podman >/dev/null 2>&1 || DOCKER=docker
+      # Remove old single-container 'immich' if present (wrong port mapping, conflicts with immich_server)
+      if sudo \$DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx immich; then
+        echo '   Removing old immich container (use immich_server)...'
+        sudo \$DOCKER stop immich 2>/dev/null
+        sudo \$DOCKER rm -f immich 2>/dev/null
+        sudo \$DOCKER start immich_server 2>/dev/null || true
+      fi
       if ! sudo \$DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q immich_server; then
         echo '   Creating Immich stack...'
         sudo mkdir -p /var/lib/archipelago/immich /var/lib/archipelago/immich-db

scripts/first-boot-containers.sh

@@ -234,6 +234,13 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q nextcloud; then
 fi
 
 # Immich stack (postgres + redis + server - ML optional)
+# Remove old single-container 'immich' if present (wrong port 2283:3001, conflicts with immich_server)
+if $DOCKER ps -a --format '{{.Names}}' 2>/dev/null | grep -qx immich; then
+  log "Removing old immich container (use immich_server stack)..."
+  $DOCKER stop immich 2>/dev/null || true
+  $DOCKER rm -f immich 2>/dev/null || true
+  $DOCKER start immich_server 2>/dev/null || true
+fi
 if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q immich_server; then
   log "Creating Immich stack..."
   mkdir -p /var/lib/archipelago/immich /var/lib/archipelago/immich-db
@@ -243,7 +250,11 @@ if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q immich_server; then
       -v /var/lib/archipelago/immich-db:/var/lib/postgresql/data \
       -e POSTGRES_PASSWORD=immichpass -e POSTGRES_USER=postgres -e POSTGRES_DB=immich \
       ghcr.io/immich-app/postgres:14-vectorchord0.4.3-pgvectors0.2.0 2>>"$LOG" || true
-    sleep 5
+    sleep 3
+    for i in 1 2 3 4 5 6 7 8 9 10; do
+      $DOCKER exec immich_postgres pg_isready -U postgres 2>/dev/null && break
+      sleep 2
+    done
   fi
   if ! $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -q immich_redis; then
     $DOCKER run -d --name immich_redis --restart unless-stopped --network immich-net \