345 lines
12 KiB
Rust
345 lines
12 KiB
Rust
|
//! Host-level fixups: OS packages, kernel parameters and system services the
|
|||
|
|
//! node needs, delivered by the same signed-binary OTA that ships everything
|
|||
|
|
//! else (docs/system-level-ota-design.md).
|
|||
|
|
//!
|
|||
|
|
//! Scope and posture — read before adding anything here:
|
|||
|
|
//!
|
|||
|
|
//! * **Idempotent + non-fatal.** Every step is a no-op when the host already
|
|||
|
|
//! has the desired state, and a failure (offline box, locked dpkg, missing
|
|||
|
|
//! package in the release's Debian suite) logs a warning and moves on. A
|
|||
|
|
//! host fixup must never be able to stop the node from starting.
|
|||
|
|
//! * **Curated, pinned intent — not dist-upgrade automation.** We deliver the
|
|||
|
|
//! specific packages and settings a release deliberately adds (crash
|
|||
|
|
//! capture, hardware-error logging, later: unattended-upgrades posture, host
|
|||
|
|
//! firewall). Regular Debian upgrades stay with the operator; this channel
|
|||
|
|
//! never silently swaps a kernel or a libc.
|
|||
|
|
//! * **Fresh installs converge too.** The ISO bakes the same end state in
|
|||
|
|
//! (Dockerfile.rootfs, auto-install.sh cmdline), so the fixup is a no-op on
|
|||
|
|
//! new machines and only does real work on already-deployed nodes.
|
|||
|
|
//! * **Kernel cmdline can't move at runtime.** `crashkernel=` reserves memory
|
|||
|
|
//! at boot; the fixup writes GRUB and update-grub so the change lands on the
|
|||
|
|
//! next reboot, and says so in the log. Everything else (packages, sysctls,
|
|||
|
|
//! services) applies immediately.
|
|||
|
|
//!
|
|||
|
|
//! First payload (#144, docs/kdump-rasdaemon-design.md): kdump + rasdaemon —
|
|||
|
|
//! post-mortem and hardware-error capture:
|
|||
|
|
//! * kdump-tools/kexec-tools/rasdaemon installed
|
|||
|
|
//! * /etc/default/kdump-tools: USE_KDUMP=1, dumps to /var/crash, compressed
|
|||
|
|
//! core collector
|
|||
|
|
//! * /etc/sysctl.d/99-archipelago-kdump.conf: a wedged node dumps and
|
|||
|
|
//! reboots rather than sitting dead until power-cycled
|
|||
|
|
//! * crashkernel=256M appended to the installed GRUB cmdline (next reboot)
|
|||
|
|
//! * /var/crash pruned to the two newest dumps
|
|||
|
|
//!
|
|||
|
|
//! The module is skipped on dev boxes (same guard bootstrap::run uses) and on
|
|||
|
|
//! hosts without dpkg.
|
|||
|
|
|
|||
|
|
use anyhow::{Context, Result};
|
|||
|
|
use tracing::{debug, info, warn};
|
|||
|
|
|
|||
|
|
use crate::update::host_sudo;
|
|||
|
|
|
|||
|
|
/// Packages the node's host must have. Keep this list short and justified —
|
|||
|
|
/// every entry is state we now own on the fleet's OS images.
|
|||
|
|
const HOST_PACKAGES: &[&str] = &["kdump-tools", "kexec-tools", "rasdaemon"];
|
|||
|
|
|
|||
|
|
/// Crash-kernel reservation. 256M covers the capture kernel plus makedumpfile
|
|||
|
|
/// on the fleet's 16–64GB amd64 machines (~1–2% of RAM, permanently reserved).
|
|||
|
|
/// The arm image (RPi) is out of scope for phase 1 — see the design doc.
|
|||
|
|
const CRASHKERNEL_PARAM: &str = "crashkernel=256M";
|
|||
|
|
|
|||
|
|
const KDUMP_SYSDROPIN_PATH: &str = "/etc/sysctl.d/99-archipelago-kdump.conf";
|
|||
|
|
const KDUMP_SYSDROPIN: &str = "\
|
|||
|
|
# Archipelago kdump policy (#144). A wedged kiosk is useless until someone
|
|||
|
|
# power-cycles it — capture the evidence, then reboot by itself. Dumps land in
|
|||
|
|
# /var/crash (see docs/kdump-rasdaemon-design.md); keep-2 pruning is done by
|
|||
|
|
# the host fixup pass, not a timer.
|
|||
|
|
kernel.panic = 10
|
|||
|
|
kernel.panic_on_oops = 1
|
|||
|
|
kernel.hung_task_panic = 1
|
|||
|
|
kernel.hardlockup_panic = 1
|
|||
|
|
";
|
|||
|
|
|
|||
|
|
/// How many dumps to keep in /var/crash. Two ≈ 4 GiB worst case on the 30 GiB
|
|||
|
|
/// unencrypted root — the partition usage itself is tracked by disk_monitor.
|
|||
|
|
const KEEP_DUMPS: usize = 2;
|
|||
|
|
|
|||
|
|
/// Entry point, spawned from main.rs at startup like the other ensure_* heals.
|
|||
|
|
pub async fn ensure_host_fixups() {
|
|||
|
|
// Dev-box guard (same rationale as bootstrap::run): on contributor
|
|||
|
|
// machines /home/archipelago/archy is a symlink into a git checkout and
|
|||
|
|
// the host is the contributor's own OS — never touch it.
|
|||
|
|
let home_archy = std::path::Path::new("/home/archipelago/archy");
|
|||
|
|
if tokio::fs::symlink_metadata(home_archy)
|
|||
|
|
.await
|
|||
|
|
.map(|m| m.file_type().is_symlink())
|
|||
|
|
.unwrap_or(false)
|
|||
|
|
{
|
|||
|
|
debug!("/home/archipelago/archy is a symlink — skipping host fixups (dev box)");
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
// Non-Debian hosts: nothing we manage here applies.
|
|||
|
|
if tokio::fs::symlink_metadata("/usr/bin/dpkg").await.is_err() {
|
|||
|
|
debug!("no dpkg on this host — skipping host fixups");
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if let Err(e) = run_host_fixups().await {
|
|||
|
|
warn!("host fixups failed (non-fatal): {:#}", e);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async fn run_host_fixups() -> Result<()> {
|
|||
|
|
// 1. Packages — install only what's missing; a locked/offline apt must
|
|||
|
|
// never block anything downstream (steps below degrade to no-ops).
|
|||
|
|
match ensure_packages().await {
|
|||
|
|
Ok(true) => info!("host fixups: installed missing packages"),
|
|||
|
|
Ok(false) => debug!("host fixups: all packages present"),
|
|||
|
|
Err(e) => warn!("host fixups: package install failed (non-fatal): {:#}", e),
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 2. kdump config + sysctl drop-in + GRUB cmdline + services. One helper
|
|||
|
|
// per concern so a failure in one logs and leaves the others running.
|
|||
|
|
if let Err(e) = ensure_kdump_sysdropin().await {
|
|||
|
|
warn!(
|
|||
|
|
"host fixups: kdump sysctl drop-in failed (non-fatal): {:#}",
|
|||
|
|
e
|
|||
|
|
);
|
|||
|
|
}
|
|||
|
|
if let Err(e) = ensure_kdump_defaults().await {
|
|||
|
|
warn!(
|
|||
|
|
"host fixups: kdump-tools config failed (non-fatal): {:#}",
|
|||
|
|
e
|
|||
|
|
);
|
|||
|
|
}
|
|||
|
|
match ensure_crashkernel_cmdline().await? {
|
|||
|
|
true => {
|
|||
|
|
warn!("host fixups: crashkernel= written to GRUB — takes effect on the NEXT reboot")
|
|||
|
|
}
|
|||
|
|
false => debug!("host fixups: crashkernel already in GRUB cmdline"),
|
|||
|
|
}
|
|||
|
|
if let Err(e) = ensure_rasdaemon_enabled().await {
|
|||
|
|
warn!("host fixups: rasdaemon enable failed (non-fatal): {:#}", e);
|
|||
|
|
}
|
|||
|
|
if let Err(e) = prune_crash_dumps().await {
|
|||
|
|
debug!("host fixups: /var/crash prune skipped: {:#}", e);
|
|||
|
|
}
|
|||
|
|
Ok(())
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/// True if any package was installed. Mirrors the polkit repair's apt posture:
|
|||
|
|
/// install without `apt-get update` first; only if that fails (fresh suite,
|
|||
|
|
/// stale index), update once and retry. Both under timeout, both non-fatal.
|
|||
|
|
async fn ensure_packages() -> Result<bool> {
|
|||
|
|
let wanted = HOST_PACKAGES
|
|||
|
|
.iter()
|
|||
|
|
.map(|p| format!("'{p}'"))
|
|||
|
|
.collect::<Vec<_>>()
|
|||
|
|
.join(" ");
|
|||
|
|
let script = format!(
|
|||
|
|
r#"
|
|||
|
|
set -u
|
|||
|
|
WANTED="{wanted}"
|
|||
|
|
MISSING=""
|
|||
|
|
for p in $WANTED; do
|
|||
|
|
dpkg-query -W -f='${{Status}}' "$p" 2>/dev/null | grep -q 'install ok installed' || MISSING="$MISSING $p"
|
|||
|
|
done
|
|||
|
|
[ -z "$MISSING" ] && exit 0
|
|||
|
|
timeout 240 apt-get install -y --no-install-recommends $MISSING >/dev/null 2>&1 \
|
|||
|
|
|| timeout 240 sh -c 'apt-get update >/dev/null 2>&1 && apt-get install -y --no-install-recommends $MISSING >/dev/null 2>&1' \
|
|||
|
|
|| exit 3
|
|||
|
|
exit 2
|
|||
|
|
"#
|
|||
|
|
);
|
|||
|
|
let status = host_sudo(&["sh", "-lc", &script])
|
|||
|
|
.await
|
|||
|
|
.context("install host packages")?;
|
|||
|
|
match status.code() {
|
|||
|
|
Some(0) => Ok(false),
|
|||
|
|
Some(2) => Ok(true),
|
|||
|
|
code => anyhow::bail!("host package install exited with {code:?}"),
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/// Write the sysctl drop-in and apply it live (these four keys are all
|
|||
|
|
/// runtime-settable, so the hang/panic policy takes effect without a reboot).
|
|||
|
|
async fn ensure_kdump_sysdropin() -> Result<()> {
|
|||
|
|
let script = format!(
|
|||
|
|
r#"
|
|||
|
|
set -u
|
|||
|
|
PATH_FILE='{KDUMP_SYSDROPIN_PATH}'
|
|||
|
|
CONTENT_FILE=/tmp/archy-kdump-sysctl.$$.tmp
|
|||
|
|
cat > "$CONTENT_FILE" <<'SYSEOF'
|
|||
|
|
{KDUMP_SYSDROPIN}SYSEOF
|
|||
|
|
if [ -f "$PATH_FILE" ] && cmp -s "$CONTENT_FILE" "$PATH_FILE"; then
|
|||
|
|
rm -f "$CONTENT_FILE"
|
|||
|
|
exit 0
|
|||
|
|
fi
|
|||
|
|
mv "$CONTENT_FILE" "$PATH_FILE"
|
|||
|
|
chmod 644 "$PATH_FILE"
|
|||
|
|
sysctl --system >/dev/null 2>&1 || true
|
|||
|
|
exit 2
|
|||
|
|
"#
|
|||
|
|
);
|
|||
|
|
let status = host_sudo(&["sh", "-lc", &script])
|
|||
|
|
.await
|
|||
|
|
.context("write kdump sysctl drop-in")?;
|
|||
|
|
match status.code() {
|
|||
|
|
Some(0) => Ok(()),
|
|||
|
|
Some(2) => {
|
|||
|
|
info!("host fixups: installed {KDUMP_SYSDROPIN_PATH} (hang/panic policy)");
|
|||
|
|
Ok(())
|
|||
|
|
}
|
|||
|
|
code => anyhow::bail!("kdump sysctl drop-in exited with {code:?}"),
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/// Point kdump-tools at /var/crash with a compressed core collector. Works on
|
|||
|
|
/// the package's shipped defaults file (USE_KDUMP=0, commented KDUMP_COREDIR)
|
|||
|
|
/// and on any state we already wrote — pure line surgery, idempotent.
|
|||
|
|
async fn ensure_kdump_defaults() -> Result<()> {
|
|||
|
|
let script = r#"
|
|||
|
|
set -u
|
|||
|
|
CONF=/etc/default/kdump-tools
|
|||
|
|
[ -f "$CONF" ] || exit 3
|
|||
|
|
CHANGED=0
|
|||
|
|
set_kv() {
|
|||
|
|
# set_kv KEY VALUE — replace any (possibly commented) KEY= line with
|
|||
|
|
# KEY='VALUE', appending at the end when absent.
|
|||
|
|
KEY="$1"; VAL="$2"
|
|||
|
|
if grep -qE "^${KEY}=" "$CONF" 2>/dev/null; then
|
|||
|
|
if ! grep -qE "^${KEY}='?${VAL}'?$" "$CONF"; then
|
|||
|
|
sed -i "s|^${KEY}=.*|${KEY}=\"${VAL}\"|" "$CONF"
|
|||
|
|
CHANGED=1
|
|||
|
|
fi
|
|||
|
|
else
|
|||
|
|
printf '\n%s="%s"\n' "$KEY" "$VAL" >> "$CONF"
|
|||
|
|
CHANGED=1
|
|||
|
|
fi
|
|||
|
|
}
|
|||
|
|
set_kv USE_KDUMP 1
|
|||
|
|
set_kv KDUMP_COREDIR /var/crash
|
|||
|
|
set_kv CORE_COLLECTOR 'makedumpfile -l --message-level 1 -d 31'
|
|||
|
|
[ "$CHANGED" -eq 1 ] || exit 0
|
|||
|
|
systemctl enable kdump-tools >/dev/null 2>&1 || true
|
|||
|
|
exit 2
|
|||
|
|
"#;
|
|||
|
|
let status = host_sudo(&["sh", "-lc", script])
|
|||
|
|
.await
|
|||
|
|
.context("configure kdump-tools")?;
|
|||
|
|
match status.code() {
|
|||
|
|
Some(0) => Ok(()),
|
|||
|
|
Some(2) => {
|
|||
|
|
info!("host fixups: kdump-tools configured (USE_KDUMP=1, /var/crash)");
|
|||
|
|
Ok(())
|
|||
|
|
}
|
|||
|
|
code => anyhow::bail!("kdump-tools config exited with {code:?}"),
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/// Append `crashkernel=` to the installed GRUB cmdline and run update-grub.
|
|||
|
|
/// The reservation itself only exists after the next reboot — memory cannot
|
|||
|
|
/// be set aside at runtime — so the caller must log the reboot caveat.
|
|||
|
|
/// Returns true if the cmdline changed.
|
|||
|
|
async fn ensure_crashkernel_cmdline() -> Result<bool> {
|
|||
|
|
let script = format!(
|
|||
|
|
r#"
|
|||
|
|
set -u
|
|||
|
|
GRUB=/etc/default/grub
|
|||
|
|
PARAM='{CRASHKERNEL_PARAM}'
|
|||
|
|
[ -f "$GRUB" ] || exit 3
|
|||
|
|
LINE=$(grep -E '^GRUB_CMDLINE_LINUX_DEFAULT=' "$GRUB" | head -1)
|
|||
|
|
[ -n "$LINE" ] || exit 3
|
|||
|
|
case "$LINE" in
|
|||
|
|
*"$PARAM"*) exit 0 ;;
|
|||
|
|
esac
|
|||
|
|
NEWLINE=$(printf '%s' "$LINE" | sed "s/\"$/ $PARAM\"/")
|
|||
|
|
sed -i "s|^GRUB_CMDLINE_LINUX_DEFAULT=.*|$NEWLINE|" "$GRUB"
|
|||
|
|
timeout 120 update-grub >/dev/null 2>&1 || true
|
|||
|
|
exit 2
|
|||
|
|
"#
|
|||
|
|
);
|
|||
|
|
let status = host_sudo(&["sh", "-lc", &script])
|
|||
|
|
.await
|
|||
|
|
.context("set crashkernel= in GRUB")?;
|
|||
|
|
match status.code() {
|
|||
|
|
Some(0) => Ok(false),
|
|||
|
|
Some(2) => Ok(true),
|
|||
|
|
code => anyhow::bail!("crashkernel cmdline fixup exited with {code:?}"),
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async fn ensure_rasdaemon_enabled() -> Result<()> {
|
|||
|
|
let status = host_sudo(&["systemctl", "enable", "--now", "rasdaemon"])
|
|||
|
|
.await
|
|||
|
|
.context("enable rasdaemon")?;
|
|||
|
|
if status.success() {
|
|||
|
|
Ok(())
|
|||
|
|
} else {
|
|||
|
|
anyhow::bail!("systemctl enable --now rasdaemon exited with {status}")
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/// Keep only the newest [`KEEP_DUMPS`] dumps in /var/crash. Called on every
|
|||
|
|
/// fixup pass rather than by a timer: the pass runs at every startup, which is
|
|||
|
|
/// exactly the cadence at which new dumps appear (a dump ends in a reboot).
|
|||
|
|
async fn prune_crash_dumps() -> Result<()> {
|
|||
|
|
let script = format!(
|
|||
|
|
r#"
|
|||
|
|
set -u
|
|||
|
|
DIR=/var/crash
|
|||
|
|
[ -d "$DIR" ] || exit 0
|
|||
|
|
KEEP={KEEP_DUMPS}
|
|||
|
|
COUNT=$(ls -1 "$DIR" 2>/dev/null | wc -l)
|
|||
|
|
[ "$COUNT" -gt "$KEEP" ] || exit 0
|
|||
|
|
ls -1dt "$DIR"/* 2>/dev/null | tail -n +"$((KEEP + 1))" | while IFS= read -r victim; do
|
|||
|
|
rm -rf -- "$victim"
|
|||
|
|
done
|
|||
|
|
exit 2
|
|||
|
|
"#
|
|||
|
|
);
|
|||
|
|
let status = host_sudo(&["sh", "-lc", &script])
|
|||
|
|
.await
|
|||
|
|
.context("prune /var/crash")?;
|
|||
|
|
match status.code() {
|
|||
|
|
Some(0) => Ok(()),
|
|||
|
|
Some(2) => {
|
|||
|
|
info!("host fixups: pruned old dumps in /var/crash (keep {KEEP_DUMPS})");
|
|||
|
|
Ok(())
|
|||
|
|
}
|
|||
|
|
code => anyhow::bail!("/var/crash prune exited with {code:?}"),
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
#[cfg(test)]
|
|||
|
|
mod tests {
|
|||
|
|
use super::*;
|
|||
|
|
|
|||
|
|
#[test]
|
|||
|
|
fn sysctl_dropin_carries_the_full_hang_capture_policy() {
|
|||
|
|
for key in [
|
|||
|
|
"kernel.panic = 10",
|
|||
|
|
"kernel.panic_on_oops = 1",
|
|||
|
|
"kernel.hung_task_panic = 1",
|
|||
|
|
"kernel.hardlockup_panic = 1",
|
|||
|
|
] {
|
|||
|
|
assert!(KDUMP_SYSDROPIN.contains(key), "drop-in missing {key}");
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
#[test]
|
|||
|
|
fn package_list_is_exactly_the_kdump_rasdaemon_set() {
|
|||
|
|
assert_eq!(HOST_PACKAGES, &["kdump-tools", "kexec-tools", "rasdaemon"]);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
#[test]
|
|||
|
|
fn crashkernel_param_is_sized_and_unprefixed() {
|
|||
|
|
assert_eq!(CRASHKERNEL_PARAM, "crashkernel=256M");
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
#[test]
|
|||
|
|
fn keep_dumps_is_two() {
|
|||
|
|
assert_eq!(KEEP_DUMPS, 2);
|
|||
|
|
}
|
|||
|
|
}
|