feat: automated backup and production monitoring

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

server/src/app.ts

@@ -20,6 +20,8 @@ import { join, dirname } from 'path'
 import { fileURLToPath } from 'url'
 import { cleanupOrphanedFights } from './engine/orchestrator.js'
 import { recoverOrphanedPayments } from './engine/payments.js'
+import { startDailyBackups } from './engine/backup.js'
+import { startMemoryTracking } from './engine/analytics.js'
 
 export const app = new Hono()
 
@@ -170,3 +172,11 @@ cleanupOrphanedFights().then(() => {
 recoverOrphanedPayments().catch(err => {
   console.error('[botfights] payment recovery error:', err)
 })
+
+// Start daily database backups (production only)
+if (process.env.NODE_ENV === 'production') {
+  startDailyBackups()
+}
+
+// Start memory tracking
+startMemoryTracking()

server/src/engine/analytics.ts

@@ -65,3 +65,45 @@ export function getPublicStats(days: number = 30): Array<{ date: string; metric:
   )
   return stmt.all(cutoffStr) as Array<{ date: string; metric: string; value: number }>
 }
+
+// --- Production monitoring ---
+
+/** Track webhook response latency */
+export function trackWebhookLatency(botId: string, latencyMs: number): void {
+  trackMetricBy('webhook_latency_total', latencyMs)
+  trackMetric('webhook_calls')
+  if (latencyMs > 5000) trackMetric('webhook_slow')
+}
+
+/** Track payment settlement time */
+export function trackPaymentSettlement(latencyMs: number, success: boolean): void {
+  trackMetricBy('payment_settlement_total', latencyMs)
+  trackMetric(success ? 'payment_success' : 'payment_failure')
+}
+
+/** Track errors */
+export function trackError(category: string): void {
+  trackMetric(`error_${category}`)
+}
+
+/** Memory metrics snapshot (call periodically) */
+export function snapshotMemory(): void {
+  const mem = process.memoryUsage()
+  const rssMB = Math.round(mem.rss / 1024 / 1024)
+  trackMetricBy('rss_peak_mb', 0) // Initialize if not exists
+  // Track peak RSS for today
+  const row = sqlite.prepare(
+    "SELECT value FROM analytics WHERE date = ? AND metric = 'rss_peak_mb'"
+  ).get(today()) as { value: number } | undefined
+  if (!row || rssMB > row.value) {
+    sqlite.prepare(
+      `INSERT INTO analytics (date, metric, value) VALUES (?, 'rss_peak_mb', ?)
+       ON CONFLICT(date, metric) DO UPDATE SET value = ?`
+    ).run(today(), rssMB, rssMB)
+  }
+}
+
+/** Start periodic memory snapshots */
+export function startMemoryTracking(): void {
+  setInterval(snapshotMemory, 60_000)
+}

server/src/engine/backup.ts

@@ -0,0 +1,53 @@
+import { copyFileSync, readdirSync, unlinkSync, mkdirSync, existsSync } from 'fs'
+import { join, dirname } from 'path'
+import { fileURLToPath } from 'url'
+import { logger } from '../lib/logger.js'
+
+const __dirname = dirname(fileURLToPath(import.meta.url))
+const DB_PATH = process.env.DB_PATH || join(__dirname, '..', '..', 'data', 'botfights.db')
+const BACKUP_DIR = join(dirname(DB_PATH), 'backups')
+const MAX_BACKUPS = 7
+
+/** Create a timestamped backup of the database */
+export function createBackup(): string {
+  mkdirSync(BACKUP_DIR, { recursive: true })
+
+  const timestamp = new Date().toISOString().slice(0, 10)
+  const backupPath = join(BACKUP_DIR, `botfights-${timestamp}.db`)
+
+  copyFileSync(DB_PATH, backupPath)
+  logger.info('backup', `created backup: ${backupPath}`)
+
+  // Rotate: keep only the latest MAX_BACKUPS
+  rotateBackups()
+
+  return backupPath
+}
+
+/** Remove old backups beyond MAX_BACKUPS */
+function rotateBackups(): void {
+  if (!existsSync(BACKUP_DIR)) return
+
+  const files = readdirSync(BACKUP_DIR)
+    .filter(f => f.startsWith('botfights-') && f.endsWith('.db'))
+    .sort()
+    .reverse()
+
+  for (let i = MAX_BACKUPS; i < files.length; i++) {
+    const path = join(BACKUP_DIR, files[i])
+    try {
+      unlinkSync(path)
+      logger.info('backup', `rotated out: ${files[i]}`)
+    } catch { /* ok */ }
+  }
+}
+
+/** Start daily backup timer */
+export function startDailyBackups(): void {
+  // Run first backup after 1 minute, then every 24 hours
+  setTimeout(() => {
+    createBackup()
+    setInterval(() => createBackup(), 24 * 60 * 60 * 1000)
+  }, 60_000)
+  logger.info('backup', 'daily backups scheduled')
+}

server/src/routes/admin.ts

@@ -2,6 +2,7 @@ import { Hono } from 'hono'
 import { db, schema, sqlite } from '../db/index.js'
 import { eq, desc, sql, count } from 'drizzle-orm'
 import { getActiveSSECount } from './fights.js'
+import { createBackup } from '../engine/backup.js'
 
 const CREATOR_PUBKEY = 'da5e0c1b646bdb13c2300f805b0ca3e5afe5b052c594ce78bac8978d21c3fa39'
 const startTime = Date.now()
@@ -30,6 +31,8 @@ adminRouter.get('/stats', async (c) => {
   const totalFights = await db.select({ count: count() }).from(schema.fights)
   const totalSats = await db.select({ total: sql<number>`coalesce(sum(${schema.fights.potSats}), 0)` }).from(schema.fights)
 
+  const rssMB = Math.round(mem.rss / 1024 / 1024)
+
   return c.json({
     uptime: Math.round((Date.now() - startTime) / 1000),
     rssBytes: mem.rss,
@@ -41,6 +44,10 @@ adminRouter.get('/stats', async (c) => {
     totalBots: totalBots[0]?.count || 0,
     totalFights: totalFights[0]?.count || 0,
     totalSatsMoved: totalSats[0]?.total || 0,
+    alerts: {
+      memoryWarning: rssMB > 200,
+      memoryCritical: rssMB > 300,
+    },
   })
 })
 
@@ -105,3 +112,14 @@ adminRouter.get('/fights', async (c) => {
 
   return c.json({ fights })
 })
+
+// GET /backup — trigger manual backup
+adminRouter.get('/backup', (c) => {
+  try {
+    const path = createBackup()
+    return c.json({ ok: true, path })
+  } catch (err: unknown) {
+    const msg = err instanceof Error ? err.message : 'Backup failed'
+    return c.json({ error: msg }, 500)
+  }
+})