Go global: worldwide BTC Map sync, marker clustering, response compression
- Sync now pulls every BTC Map place worldwide (~29k live), not just Madeira — dropped BTCMAP_CENTER_LAT/LON/RADIUS_KM entirely. - Incremental via a stored watermark (settings.btcmap_synced_since): only fetches what changed since the last sync, not the whole dataset every time. Handles deletions too (BTC Map delistings correctly evict the local place + cascade its claim/links, not just go unclaimed). - Real bug caught and fixed before it shipped: tried BTC Map's own recommended updated_since+limit pagination first, but their timestamps mix millisecond and whole-second precision, and naive string comparison across that isn't chronologically safe — silently truncated a real sync to ~4,000 of ~42,000 records with no error. Measured the alternative (single unpaginated request) instead: 2-3s for the full dataset, simpler and actually correct. Full writeup in services/btcmap.ts, regression test for the specific bug in services/btcmap.test.ts. - @fastify/compress added — the places list is now tens of thousands of rows, gzip/br/zstd auto-negotiated. - Also fixed while touching dependencies: @fastify/static had a real high-severity path-traversal/auth-bypass advisory (GHSA-pr96-94w5-mx2h et al) affecting the version we were pinned to — bumped to the patched 10.1.2. - Frontend: leaflet.markercluster (raw per-marker rendering doesn't scale to tens of thousands of points), cyberpunk-themed cluster icons to match the existing HUD styling, batch marker insertion (addLayers, not a per-marker addLayer loop) since that's dramatically faster at this scale. Default map view is now world-scale, recentering on the player's location if geolocation is available. - 60 backend tests passing throughout (7 new for the sync rewrite).
This commit is contained in:
+46
-21
@@ -1,9 +1,17 @@
|
||||
import type { FastifyInstance } from 'fastify';
|
||||
import { fetchAreaPlaces } from '../services/btcmap.js';
|
||||
import { fetchAllPlaces, latestUpdatedAt } from '../services/btcmap.js';
|
||||
|
||||
const WATERMARK_KEY = 'btcmap_synced_since';
|
||||
const EPOCH = '1970-01-01T00:00:00Z';
|
||||
|
||||
export default async function syncRoutes(app: FastifyInstance) {
|
||||
const { db, config } = app.ctx;
|
||||
|
||||
const getWatermark = db.prepare('SELECT value FROM settings WHERE key = ?');
|
||||
const setWatermark = db.prepare(`
|
||||
INSERT INTO settings (key, value) VALUES (?, ?)
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value
|
||||
`);
|
||||
const upsertPlace = db.prepare(`
|
||||
INSERT INTO places (id, lat, lon, name, icon, address, osm_id, btcmap_updated_at, synced_at)
|
||||
VALUES (@id, @lat, @lon, @name, @icon, @address, @osm_id, @btcmap_updated_at, @synced_at)
|
||||
@@ -17,30 +25,47 @@ export default async function syncRoutes(app: FastifyInstance) {
|
||||
btcmap_updated_at = excluded.btcmap_updated_at,
|
||||
synced_at = excluded.synced_at
|
||||
`);
|
||||
// ON DELETE CASCADE on claims/links (see db/migrations.ts) means a place
|
||||
// that BTC Map delists correctly reverts to fully gone here too, not just
|
||||
// unclaimed — its claim and any links through it disappear with it.
|
||||
const deletePlace = db.prepare('DELETE FROM places WHERE id = ?');
|
||||
|
||||
// Public on purpose for now (single-region MVP, no write access to real money);
|
||||
// revisit if/when this needs to be admin-gated for a larger, costlier sync area.
|
||||
// Public on purpose (read-only pull from a public API, no write access to
|
||||
// real money) — revisit if this ever needs admin-gating, e.g. to bound
|
||||
// how often the full worldwide dataset gets re-pulled.
|
||||
app.post('/api/sync', async () => {
|
||||
const places = await fetchAreaPlaces(config.BTCMAP_CENTER_LAT, config.BTCMAP_CENTER_LON, config.BTCMAP_RADIUS_KM);
|
||||
const since = (getWatermark.get(WATERMARK_KEY) as { value: string } | undefined)?.value ?? EPOCH;
|
||||
const syncedAt = Math.floor(Date.now() / 1000);
|
||||
|
||||
const insertAll = db.transaction((rows: typeof places) => {
|
||||
for (const p of rows) {
|
||||
upsertPlace.run({
|
||||
id: p.id,
|
||||
lat: p.lat,
|
||||
lon: p.lon,
|
||||
name: p.name ?? '',
|
||||
icon: p.icon ?? null,
|
||||
address: p.address ?? null,
|
||||
osm_id: p.osm_id ?? null,
|
||||
btcmap_updated_at: p.updated_at ?? null,
|
||||
synced_at: syncedAt,
|
||||
});
|
||||
}
|
||||
});
|
||||
insertAll(places);
|
||||
const places = await fetchAllPlaces(config.BTCMAP_API_BASE_URL, since);
|
||||
let upserted = 0;
|
||||
let deleted = 0;
|
||||
|
||||
return { synced: places.length, center: { lat: config.BTCMAP_CENTER_LAT, lon: config.BTCMAP_CENTER_LON }, radiusKm: config.BTCMAP_RADIUS_KM };
|
||||
db.transaction(() => {
|
||||
for (const p of places) {
|
||||
if (p.deleted_at) {
|
||||
deletePlace.run(p.id);
|
||||
deleted += 1;
|
||||
} else {
|
||||
upsertPlace.run({
|
||||
id: p.id,
|
||||
lat: p.lat,
|
||||
lon: p.lon,
|
||||
name: p.name ?? '',
|
||||
icon: p.icon ?? null,
|
||||
address: p.address ?? null,
|
||||
osm_id: p.osm_id ?? null,
|
||||
btcmap_updated_at: p.updated_at ?? null,
|
||||
synced_at: syncedAt,
|
||||
});
|
||||
upserted += 1;
|
||||
}
|
||||
}
|
||||
})();
|
||||
|
||||
const newWatermark = latestUpdatedAt(places, since);
|
||||
setWatermark.run(WATERMARK_KEY, newWatermark);
|
||||
|
||||
return { fetched: places.length, upserted, deleted, since, watermark: newWatermark };
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user