Go global: worldwide BTC Map sync, marker clustering, response compression

- Sync now pulls every BTC Map place worldwide (~29k live), not just
  Madeira — dropped BTCMAP_CENTER_LAT/LON/RADIUS_KM entirely.
- Incremental via a stored watermark (settings.btcmap_synced_since):
  only fetches what changed since the last sync, not the whole dataset
  every time. Handles deletions too (BTC Map delistings correctly evict
  the local place + cascade its claim/links, not just go unclaimed).
- Real bug caught and fixed before it shipped: tried BTC Map's own
  recommended updated_since+limit pagination first, but their timestamps
  mix millisecond and whole-second precision, and naive string comparison
  across that isn't chronologically safe — silently truncated a real
  sync to ~4,000 of ~42,000 records with no error. Measured the
  alternative (single unpaginated request) instead: 2-3s for the full
  dataset, simpler and actually correct. Full writeup in
  services/btcmap.ts, regression test for the specific bug in
  services/btcmap.test.ts.
- @fastify/compress added — the places list is now tens of thousands of
  rows, gzip/br/zstd auto-negotiated.
- Also fixed while touching dependencies: @fastify/static had a real
  high-severity path-traversal/auth-bypass advisory (GHSA-pr96-94w5-mx2h
  et al) affecting the version we were pinned to — bumped to the patched
  10.1.2.
- Frontend: leaflet.markercluster (raw per-marker rendering doesn't scale
  to tens of thousands of points), cyberpunk-themed cluster icons to
  match the existing HUD styling, batch marker insertion (addLayers, not
  a per-marker addLayer loop) since that's dramatically faster at this
  scale. Default map view is now world-scale, recentering on the
  player's location if geolocation is available.
- 60 backend tests passing throughout (7 new for the sync rewrite).
This commit is contained in:
2026-08-05 17:32:54 +00:00
parent 4fef930029
commit bb03bfc365
14 changed files with 464 additions and 200 deletions
+46 -21
View File
@@ -1,9 +1,17 @@
import type { FastifyInstance } from 'fastify';
import { fetchAreaPlaces } from '../services/btcmap.js';
import { fetchAllPlaces, latestUpdatedAt } from '../services/btcmap.js';
const WATERMARK_KEY = 'btcmap_synced_since';
const EPOCH = '1970-01-01T00:00:00Z';
export default async function syncRoutes(app: FastifyInstance) {
const { db, config } = app.ctx;
const getWatermark = db.prepare('SELECT value FROM settings WHERE key = ?');
const setWatermark = db.prepare(`
INSERT INTO settings (key, value) VALUES (?, ?)
ON CONFLICT(key) DO UPDATE SET value = excluded.value
`);
const upsertPlace = db.prepare(`
INSERT INTO places (id, lat, lon, name, icon, address, osm_id, btcmap_updated_at, synced_at)
VALUES (@id, @lat, @lon, @name, @icon, @address, @osm_id, @btcmap_updated_at, @synced_at)
@@ -17,30 +25,47 @@ export default async function syncRoutes(app: FastifyInstance) {
btcmap_updated_at = excluded.btcmap_updated_at,
synced_at = excluded.synced_at
`);
// ON DELETE CASCADE on claims/links (see db/migrations.ts) means a place
// that BTC Map delists correctly reverts to fully gone here too, not just
// unclaimed — its claim and any links through it disappear with it.
const deletePlace = db.prepare('DELETE FROM places WHERE id = ?');
// Public on purpose for now (single-region MVP, no write access to real money);
// revisit if/when this needs to be admin-gated for a larger, costlier sync area.
// Public on purpose (read-only pull from a public API, no write access to
// real money) — revisit if this ever needs admin-gating, e.g. to bound
// how often the full worldwide dataset gets re-pulled.
app.post('/api/sync', async () => {
const places = await fetchAreaPlaces(config.BTCMAP_CENTER_LAT, config.BTCMAP_CENTER_LON, config.BTCMAP_RADIUS_KM);
const since = (getWatermark.get(WATERMARK_KEY) as { value: string } | undefined)?.value ?? EPOCH;
const syncedAt = Math.floor(Date.now() / 1000);
const insertAll = db.transaction((rows: typeof places) => {
for (const p of rows) {
upsertPlace.run({
id: p.id,
lat: p.lat,
lon: p.lon,
name: p.name ?? '',
icon: p.icon ?? null,
address: p.address ?? null,
osm_id: p.osm_id ?? null,
btcmap_updated_at: p.updated_at ?? null,
synced_at: syncedAt,
});
}
});
insertAll(places);
const places = await fetchAllPlaces(config.BTCMAP_API_BASE_URL, since);
let upserted = 0;
let deleted = 0;
return { synced: places.length, center: { lat: config.BTCMAP_CENTER_LAT, lon: config.BTCMAP_CENTER_LON }, radiusKm: config.BTCMAP_RADIUS_KM };
db.transaction(() => {
for (const p of places) {
if (p.deleted_at) {
deletePlace.run(p.id);
deleted += 1;
} else {
upsertPlace.run({
id: p.id,
lat: p.lat,
lon: p.lon,
name: p.name ?? '',
icon: p.icon ?? null,
address: p.address ?? null,
osm_id: p.osm_id ?? null,
btcmap_updated_at: p.updated_at ?? null,
synced_at: syncedAt,
});
upserted += 1;
}
}
})();
const newWatermark = latestUpdatedAt(places, since);
setWatermark.run(WATERMARK_KEY, newWatermark);
return { fetched: places.length, upserted, deleted, since, watermark: newWatermark };
});
}