Enhance development workflow and deployment practices for Archipelago

- Updated the Development-Workflow documentation to clarify deployment strategy, emphasizing direct deployment to the live system for testing.
- Added detailed instructions for the deployment command, including syncing code, building frontend and backend, and restarting services.
- Improved SSH key management section to assist with authentication issues.
- Expanded the testing workflow to include steps for checking logs and syncing changes back to the ISO build.
- Updated the ISO build integration section to ensure system-level changes are captured for future builds.
- Refactored various sections for clarity and completeness, including deployment paths and system configuration files.
This commit is contained in:
Dorian
2026-02-01 13:24:03 +00:00
parent 00d1af12f0
commit 34fc06726e
28 changed files with 1248 additions and 285 deletions
+27 -1
View File
@@ -6,6 +6,7 @@ use futures_util::{SinkExt, StreamExt};
use hyper::{Method, Request, Response, StatusCode};
use hyper_ws_listener::WsStream;
use std::sync::Arc;
use tokio::sync::broadcast;
use tokio_tungstenite::tungstenite::Message;
use tracing::{debug, info};
@@ -94,11 +95,14 @@ impl ApiHandler {
debug!("Sent initial data dump at revision {}", initial_msg.rev);
}
// Subscribe to state updates
let mut state_rx = state_manager.subscribe();
// Send periodic pings to keep connection alive
let ping_interval = tokio::time::interval(tokio::time::Duration::from_secs(30));
tokio::pin!(ping_interval);
// Keep connection open; UI may send/receive JSON patches. For now just accept and ignore.
// Keep connection open and forward state updates to client
loop {
tokio::select! {
_ = ping_interval.tick() => {
@@ -107,6 +111,28 @@ impl ApiHandler {
break;
}
}
// Forward state updates from broadcast channel to WebSocket
update = state_rx.recv() => {
match update {
Ok(msg) => {
if let Ok(json_msg) = serde_json::to_string(&msg) {
if let Err(e) = tx.send(Message::Text(json_msg)).await {
debug!("Failed to send state update: {}", e);
break;
}
debug!("Sent state update at revision {}", msg.rev);
}
}
Err(broadcast::error::RecvError::Lagged(skipped)) => {
debug!("Client lagged behind, skipped {} messages", skipped);
// Continue receiving - the client will get the next update
}
Err(broadcast::error::RecvError::Closed) => {
debug!("Broadcast channel closed");
break;
}
}
}
msg = rx.next() => {
match msg {
Some(Ok(Message::Close(_))) => break,
+64 -21
View File
@@ -445,15 +445,30 @@ impl RpcHandler {
.ok_or_else(|| anyhow::anyhow!("Missing package id"))?;
// Convert package ID to container name (e.g., "bitcoin" -> "archy-bitcoin")
let container_name = format!("archy-{}", package_id);
// Use docker CLI to start the container
let output = tokio::process::Command::new("docker")
.arg("start")
.arg(&container_name)
// But also check if container exists without the prefix
let container_name = if let Ok(output) = tokio::process::Command::new("sudo")
.args(["podman", "ps", "-a", "--format", "{{.Names}}", "--filter", &format!("name=^{}$", package_id)])
.output()
.await
.context("Failed to execute docker start")?;
{
let stdout = String::from_utf8_lossy(&output.stdout);
if !stdout.trim().is_empty() {
debug!("Found container without prefix: {}", package_id);
package_id.to_string()
} else {
debug!("Using archy- prefix: archy-{}", package_id);
format!("archy-{}", package_id)
}
} else {
format!("archy-{}", package_id)
};
// Use podman CLI to start the container
let output = tokio::process::Command::new("sudo")
.args(["podman", "start", &container_name])
.output()
.await
.context("Failed to execute podman start")?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
@@ -474,15 +489,29 @@ impl RpcHandler {
.ok_or_else(|| anyhow::anyhow!("Missing package id"))?;
// Convert package ID to container name
let container_name = format!("archy-{}", package_id);
// Use docker CLI to stop the container
let output = tokio::process::Command::new("docker")
.arg("stop")
.arg(&container_name)
let container_name = if let Ok(output) = tokio::process::Command::new("sudo")
.args(["podman", "ps", "-a", "--format", "{{.Names}}", "--filter", &format!("name=^{}$", package_id)])
.output()
.await
.context("Failed to execute docker stop")?;
{
let stdout = String::from_utf8_lossy(&output.stdout);
if !stdout.trim().is_empty() {
debug!("Found container without prefix: {}", package_id);
package_id.to_string()
} else {
debug!("Using archy- prefix: archy-{}", package_id);
format!("archy-{}", package_id)
}
} else {
format!("archy-{}", package_id)
};
// Use podman CLI to stop the container
let output = tokio::process::Command::new("sudo")
.args(["podman", "stop", &container_name])
.output()
.await
.context("Failed to execute podman stop")?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
@@ -503,15 +532,29 @@ impl RpcHandler {
.ok_or_else(|| anyhow::anyhow!("Missing package id"))?;
// Convert package ID to container name
let container_name = format!("archy-{}", package_id);
// Use docker CLI to restart the container
let output = tokio::process::Command::new("docker")
.arg("restart")
.arg(&container_name)
let container_name = if let Ok(output) = tokio::process::Command::new("sudo")
.args(["podman", "ps", "-a", "--format", "{{.Names}}", "--filter", &format!("name=^{}$", package_id)])
.output()
.await
.context("Failed to execute docker restart")?;
{
let stdout = String::from_utf8_lossy(&output.stdout);
if !stdout.trim().is_empty() {
debug!("Found container without prefix: {}", package_id);
package_id.to_string()
} else {
debug!("Using archy- prefix: archy-{}", package_id);
format!("archy-{}", package_id)
}
} else {
format!("archy-{}", package_id)
};
// Use podman CLI to restart the container
let output = tokio::process::Command::new("sudo")
.args(["podman", "restart", &container_name])
.output()
.await
.context("Failed to execute podman restart")?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
@@ -23,7 +23,13 @@ impl DockerPackageScanner {
/// Scan Docker containers and convert to package data
pub async fn scan_containers(&self) -> Result<HashMap<String, PackageDataEntry>> {
let containers = self.runtime.list_containers().await?;
let containers = match self.runtime.list_containers().await {
Ok(c) => c,
Err(e) => {
debug!("Failed to list containers: {}", e);
return Ok(HashMap::new());
}
};
debug!("Found {} containers", containers.len());
@@ -39,22 +45,37 @@ impl DockerPackageScanner {
"penpot-exporter",
"penpot-valkey",
"penpot-mailcatch",
"bitcoin-ui",
"lnd-ui",
"endurain-db",
"nextcloud-db",
];
for container in containers {
// Only process archy-* containers from docker-compose
if !container.name.starts_with("archy-") {
continue;
// First pass: collect UI containers
let mut ui_containers: HashMap<String, String> = HashMap::new();
for container in &containers {
if container.name.ends_with("-ui") {
// Map bitcoin-ui -> bitcoin, lnd-ui -> lnd
let parent_app = container.name.strip_suffix("-ui").unwrap_or(&container.name);
if !container.ports.is_empty() {
if let Some(ui_address) = extract_lan_address(&container.ports) {
ui_containers.insert(parent_app.to_string(), ui_address);
}
}
}
// Extract app ID from container name (archy-bitcoin -> bitcoin)
let app_id = container.name.strip_prefix("archy-")
.unwrap_or(&container.name)
.to_string();
}
debug!("Found {} UI containers", ui_containers.len());
for container in containers {
// Extract app ID from container name
// Support both archy-* containers (docker-compose) and plain names (manual)
let app_id = if container.name.starts_with("archy-") {
container.name.strip_prefix("archy-")
.unwrap_or(&container.name)
.to_string()
} else {
// Use the container name as-is for manually started containers
container.name.clone()
};
// Skip backend services (databases, APIs, etc.)
if excluded_services.contains(&app_id.as_str()) {
@@ -62,11 +83,25 @@ impl DockerPackageScanner {
continue;
}
// Skip UI containers (they're merged with their parent apps)
if app_id.ends_with("-ui") {
debug!("Skipping UI container: {}", app_id);
continue;
}
// Get metadata for this app
let metadata = get_app_metadata(&app_id);
// Extract port from container
let lan_address = extract_lan_address(&container.ports);
// Check if this app has a separate UI container
let lan_address = if let Some(ui_address) = ui_containers.get(&app_id) {
debug!("Using UI container address for {}: {}", app_id, ui_address);
Some(ui_address.clone())
} else {
// Extract port from the main container
extract_lan_address(&container.ports)
};
debug!("Container {}: ports={:?}, lan_address={:?}", app_id, container.ports, lan_address);
// Convert container state to package/service state
let (package_state, service_status) = convert_state(&container.state);
@@ -146,11 +181,11 @@ struct AppMetadata {
fn get_app_metadata(app_id: &str) -> AppMetadata {
match app_id {
"bitcoin" => AppMetadata {
title: "Bitcoin Core".to_string(),
"bitcoin" | "bitcoin-core" | "bitcoin-knots" => AppMetadata {
title: "Bitcoin Knots".to_string(),
description: "Full Bitcoin node implementation".to_string(),
icon: "/assets/img/app-icons/bitcoin-core.png".to_string(),
repo: "https://github.com/bitcoin/bitcoin".to_string(),
icon: "/assets/img/app-icons/bitcoin-knots.webp".to_string(),
repo: "https://github.com/bitcoinknots/bitcoin".to_string(),
},
"btcpay" | "btcpay-server" => AppMetadata {
title: "BTCPay Server".to_string(),
+18 -18
View File
@@ -3,7 +3,7 @@ use std::collections::HashMap;
/// The main data model that mirrors the frontend's DataModel type.
/// This is sent via WebSocket as the initial state and updated via patches.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct DataModel {
#[serde(rename = "server-info")]
pub server_info: ServerInfo,
@@ -12,7 +12,7 @@ pub struct DataModel {
pub ui: UIData,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct ServerInfo {
pub id: String,
pub version: String,
@@ -29,7 +29,7 @@ pub struct ServerInfo {
pub zram_enabled: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct StatusInfo {
pub restarting: bool,
#[serde(rename = "shutting-down")]
@@ -41,7 +41,7 @@ pub struct StatusInfo {
pub update_progress: Option<f32>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct UIData {
pub name: Option<String>,
#[serde(rename = "ack-welcome")]
@@ -50,7 +50,7 @@ pub struct UIData {
pub theme: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct UIMarketplaceData {
#[serde(rename = "selected-hosts")]
pub selected_hosts: Vec<String>,
@@ -58,13 +58,13 @@ pub struct UIMarketplaceData {
pub known_hosts: HashMap<String, MarketplaceHost>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct MarketplaceHost {
pub name: String,
pub url: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "kebab-case")]
pub enum PackageState {
Installing,
@@ -83,7 +83,7 @@ pub enum PackageState {
BackingUp,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct PackageDataEntry {
pub state: PackageState,
#[serde(rename = "static-files")]
@@ -94,14 +94,14 @@ pub struct PackageDataEntry {
pub install_progress: Option<InstallProgress>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct StaticFiles {
pub license: String,
pub instructions: String,
pub icon: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct Manifest {
pub id: String,
pub title: String,
@@ -125,18 +125,18 @@ pub struct Manifest {
pub interfaces: Option<Interfaces>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct Description {
pub short: String,
pub long: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct Interfaces {
pub main: Option<MainInterface>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct MainInterface {
pub ui: Option<String>,
#[serde(rename = "tor-config")]
@@ -145,7 +145,7 @@ pub struct MainInterface {
pub lan_config: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct InstalledPackageDataEntry {
#[serde(rename = "current-dependents")]
pub current_dependents: HashMap<String, CurrentDependencyInfo>,
@@ -158,13 +158,13 @@ pub struct InstalledPackageDataEntry {
pub status: ServiceStatus,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct CurrentDependencyInfo {
#[serde(rename = "health-checks")]
pub health_checks: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct InterfaceAddress {
#[serde(rename = "tor-address")]
pub tor_address: String,
@@ -172,7 +172,7 @@ pub struct InterfaceAddress {
pub lan_address: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "lowercase")]
pub enum ServiceStatus {
Stopped,
@@ -182,7 +182,7 @@ pub enum ServiceStatus {
Restarting,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct InstallProgress {
pub size: u64,
pub downloaded: u64,
+15 -6
View File
@@ -9,7 +9,7 @@ use std::net::SocketAddr;
use std::sync::Arc;
use std::time::Duration;
use tokio::net::TcpListener;
use tracing::{error, info};
use tracing::{debug, error, info};
pub struct Server {
_config: Config,
@@ -34,8 +34,8 @@ impl Server {
error!("Failed to scan Docker containers: {}", e);
}
// Periodic scan every 5 seconds
let mut interval = tokio::time::interval(Duration::from_secs(5));
// Periodic scan every 10 seconds (only broadcasts if state changed)
let mut interval = tokio::time::interval(Duration::from_secs(10));
loop {
interval.tick().await;
if let Err(e) = scan_and_update_packages(&scanner, &state).await {
@@ -114,10 +114,19 @@ async fn scan_and_update_packages(
) -> Result<()> {
let packages = scanner.scan_containers().await?;
// Only update if we have packages AND they're different from current state
if !packages.is_empty() {
let (mut data, _) = state.get_snapshot().await;
data.package_data = packages;
state.update_data(data).await;
let (current_data, _) = state.get_snapshot().await;
// Check if packages actually changed to avoid unnecessary broadcasts
let packages_changed = current_data.package_data != packages;
if packages_changed {
let mut data = current_data;
data.package_data = packages;
state.update_data(data).await;
debug!("📦 Container state changed, broadcasting update");
}
}
Ok(())
+21 -5
View File
@@ -1,19 +1,22 @@
use crate::data_model::{DataModel, WebSocketMessage};
use std::sync::Arc;
use tokio::sync::RwLock;
use tokio::sync::{broadcast, RwLock};
use tracing::debug;
/// Manages the application state and broadcasts updates to WebSocket clients
pub struct StateManager {
data: Arc<RwLock<DataModel>>,
revision: Arc<RwLock<u32>>,
broadcast_tx: broadcast::Sender<WebSocketMessage>,
}
impl StateManager {
pub fn new() -> Self {
let (broadcast_tx, _) = broadcast::channel(100);
Self {
data: Arc::new(RwLock::new(DataModel::new())),
revision: Arc::new(RwLock::new(0)),
broadcast_tx,
}
}
@@ -24,18 +27,31 @@ impl StateManager {
(data, rev)
}
/// Update the data model (will broadcast patches in the future)
/// Subscribe to state updates
pub fn subscribe(&self) -> broadcast::Receiver<WebSocketMessage> {
self.broadcast_tx.subscribe()
}
/// Update the data model and broadcast to all connected clients
pub async fn update_data(&self, new_data: DataModel) {
let mut data = self.data.write().await;
let mut rev = self.revision.write().await;
*data = new_data;
*data = new_data.clone();
*rev += 1;
debug!("Data model updated to revision {}", *rev);
// TODO: In the future, compute JSON patches and broadcast to all connected clients
// For now, clients will need to reconnect to get updates
// Broadcast full data dump to all connected clients
// In the future, we can optimize this by computing and sending JSON patches
let message = WebSocketMessage {
rev: *rev,
data: Some(new_data),
patch: None,
};
// Ignore errors if no receivers are connected
let _ = self.broadcast_tx.send(message);
}
/// Get a WebSocket message with the current state
+92 -12
View File
@@ -55,9 +55,13 @@ pub struct PodmanClient {
impl PodmanClient {
pub fn new(user: String) -> Self {
// If running as root, use root podman context
let is_root = std::env::var("USER").unwrap_or_default() == "root" ||
std::env::var("HOME").unwrap_or_default() == "/root";
Self {
_user: user,
rootless: true,
rootless: !is_root,
}
}
@@ -315,25 +319,101 @@ impl PodmanClient {
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
log::error!("Podman list failed: {}", stderr);
return Err(anyhow::anyhow!("Failed to list containers: {}", stderr));
}
let json = String::from_utf8_lossy(&output.stdout);
let containers: Vec<serde_json::Value> = serde_json::from_str(&json)
.context("Failed to parse container list")?;
log::debug!("Podman JSON output ({} bytes): {}", json.len(),
if json.len() > 200 { &json[..200] } else { &json });
// Podman can return either a JSON array or NDJSON (newline-delimited JSON)
let mut result = Vec::new();
for container in containers {
result.push(ContainerStatus {
id: container["Id"].as_str().unwrap_or("").to_string(),
name: container["Names"][0].as_str().unwrap_or("").to_string(),
state: ContainerState::from(container["State"].as_str().unwrap_or("unknown")),
image: container["Image"].as_str().unwrap_or("").to_string(),
created: container["Created"].as_str().unwrap_or("").to_string(),
ports: vec![],
});
// Try parsing as a JSON array first
if let Ok(containers) = serde_json::from_str::<Vec<serde_json::Value>>(&json) {
log::debug!("Parsed as JSON array with {} items", containers.len());
for container in containers {
// Handle both Names as array and Names as string
let name = if let Some(names_array) = container["Names"].as_array() {
names_array.get(0).and_then(|v| v.as_str()).unwrap_or("").to_string()
} else {
container["Names"].as_str().unwrap_or("").to_string()
};
// Parse ports from the Ports array
let ports = if let Some(ports_array) = container["Ports"].as_array() {
ports_array.iter().filter_map(|port| {
// Podman format: {"host_ip":"","container_port":8123,"host_port":8123,"range":1,"protocol":"tcp"}
if let (Some(host_port), Some(container_port), Some(protocol)) = (
port["host_port"].as_u64(),
port["container_port"].as_u64(),
port["protocol"].as_str()
) {
Some(format!("0.0.0.0:{}->{}/{}", host_port, container_port, protocol))
} else {
None
}
}).collect()
} else {
vec![]
};
result.push(ContainerStatus {
id: container["Id"].as_str().unwrap_or("").to_string(),
name,
state: ContainerState::from(container["State"].as_str().unwrap_or("unknown")),
image: container["Image"].as_str().unwrap_or("").to_string(),
created: container["Created"].as_str().unwrap_or("").to_string(),
ports,
});
}
} else {
log::debug!("Failed to parse as JSON array, trying NDJSON");
// Try parsing as NDJSON (newline-delimited JSON)
for line in json.lines() {
if line.trim().is_empty() {
continue;
}
if let Ok(container) = serde_json::from_str::<serde_json::Value>(line) {
// Handle both Names as array and Names as string
let name = if let Some(names_array) = container["Names"].as_array() {
names_array.get(0).and_then(|v| v.as_str()).unwrap_or("").to_string()
} else {
container["Names"].as_str().unwrap_or("").to_string()
};
// Parse ports from the Ports array
let ports = if let Some(ports_array) = container["Ports"].as_array() {
ports_array.iter().filter_map(|port| {
if let (Some(host_port), Some(container_port), Some(protocol)) = (
port["host_port"].as_u64(),
port["container_port"].as_u64(),
port["protocol"].as_str()
) {
Some(format!("0.0.0.0:{}->{}/{}", host_port, container_port, protocol))
} else {
None
}
}).collect()
} else {
vec![]
};
result.push(ContainerStatus {
id: container["Id"].as_str().unwrap_or("").to_string(),
name,
state: ContainerState::from(container["State"].as_str().unwrap_or("unknown")),
image: container["Image"].as_str().unwrap_or("").to_string(),
created: container["Created"].as_str().unwrap_or("").to_string(),
ports,
});
}
}
}
log::debug!("Returning {} containers", result.len());
Ok(result)
}
}