Go global: worldwide BTC Map sync, marker clustering, response compression
- Sync now pulls every BTC Map place worldwide (~29k live), not just Madeira — dropped BTCMAP_CENTER_LAT/LON/RADIUS_KM entirely. - Incremental via a stored watermark (settings.btcmap_synced_since): only fetches what changed since the last sync, not the whole dataset every time. Handles deletions too (BTC Map delistings correctly evict the local place + cascade its claim/links, not just go unclaimed). - Real bug caught and fixed before it shipped: tried BTC Map's own recommended updated_since+limit pagination first, but their timestamps mix millisecond and whole-second precision, and naive string comparison across that isn't chronologically safe — silently truncated a real sync to ~4,000 of ~42,000 records with no error. Measured the alternative (single unpaginated request) instead: 2-3s for the full dataset, simpler and actually correct. Full writeup in services/btcmap.ts, regression test for the specific bug in services/btcmap.test.ts. - @fastify/compress added — the places list is now tens of thousands of rows, gzip/br/zstd auto-negotiated. - Also fixed while touching dependencies: @fastify/static had a real high-severity path-traversal/auth-bypass advisory (GHSA-pr96-94w5-mx2h et al) affecting the version we were pinned to — bumped to the patched 10.1.2. - Frontend: leaflet.markercluster (raw per-marker rendering doesn't scale to tens of thousands of points), cyberpunk-themed cluster icons to match the existing HUD styling, batch marker insertion (addLayers, not a per-marker addLayer loop) since that's dramatically faster at this scale. Default map view is now world-scale, recentering on the player's location if geolocation is available. - 60 backend tests passing throughout (7 new for the sync rewrite).
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
import Fastify, { type FastifyInstance } from 'fastify';
|
||||
import compress from '@fastify/compress';
|
||||
import cookie from '@fastify/cookie';
|
||||
import cors from '@fastify/cors';
|
||||
import fastifyStatic from '@fastify/static';
|
||||
@@ -49,6 +50,9 @@ export async function buildApp(opts: BuildAppOptions): Promise<FastifyInstance>
|
||||
|
||||
await app.register(cookie);
|
||||
await app.register(cors, { origin: true, credentials: true, methods: ['GET', 'POST', 'DELETE'] });
|
||||
// Global place list is tens of thousands of rows — gzip is the single
|
||||
// biggest win for that payload size, worth having by default everywhere.
|
||||
await app.register(compress, { global: true });
|
||||
await app.register(nostrAuth, { db, config });
|
||||
|
||||
// Everything below is mounted under ROUTE_PREFIX (empty by default, i.e.
|
||||
|
||||
@@ -15,10 +15,11 @@ const envSchema = z.object({
|
||||
ROUTE_PREFIX: z.string().default(''),
|
||||
NIP98_MAX_SKEW_SECS: z.coerce.number().default(60),
|
||||
SESSION_TTL_DAYS: z.coerce.number().default(30),
|
||||
// Default sync area: Madeira, Portugal (Funchal-centered radius covering the whole island).
|
||||
BTCMAP_CENTER_LAT: z.coerce.number().default(32.7607),
|
||||
BTCMAP_CENTER_LON: z.coerce.number().default(-16.9595),
|
||||
BTCMAP_RADIUS_KM: z.coerce.number().default(45),
|
||||
// Base URL for BTC Map's API — global sync pulls every place worldwide via
|
||||
// the chronological /v4/places endpoint (paginated, incremental via a
|
||||
// stored watermark — see routes/sync.ts), not scoped to any region.
|
||||
// Overridable for tests to point at a mock server instead of the real API.
|
||||
BTCMAP_API_BASE_URL: z.string().url().default('https://api.btcmap.org'),
|
||||
// How close (in meters) a player must be to a place to claim it or link from it.
|
||||
CLAIM_RADIUS_METERS: z.coerce.number().default(40),
|
||||
// Maximum link length, in kilometers — mirrors Ingress's level-scaled link
|
||||
|
||||
+46
-21
@@ -1,9 +1,17 @@
|
||||
import type { FastifyInstance } from 'fastify';
|
||||
import { fetchAreaPlaces } from '../services/btcmap.js';
|
||||
import { fetchAllPlaces, latestUpdatedAt } from '../services/btcmap.js';
|
||||
|
||||
const WATERMARK_KEY = 'btcmap_synced_since';
|
||||
const EPOCH = '1970-01-01T00:00:00Z';
|
||||
|
||||
export default async function syncRoutes(app: FastifyInstance) {
|
||||
const { db, config } = app.ctx;
|
||||
|
||||
const getWatermark = db.prepare('SELECT value FROM settings WHERE key = ?');
|
||||
const setWatermark = db.prepare(`
|
||||
INSERT INTO settings (key, value) VALUES (?, ?)
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value
|
||||
`);
|
||||
const upsertPlace = db.prepare(`
|
||||
INSERT INTO places (id, lat, lon, name, icon, address, osm_id, btcmap_updated_at, synced_at)
|
||||
VALUES (@id, @lat, @lon, @name, @icon, @address, @osm_id, @btcmap_updated_at, @synced_at)
|
||||
@@ -17,30 +25,47 @@ export default async function syncRoutes(app: FastifyInstance) {
|
||||
btcmap_updated_at = excluded.btcmap_updated_at,
|
||||
synced_at = excluded.synced_at
|
||||
`);
|
||||
// ON DELETE CASCADE on claims/links (see db/migrations.ts) means a place
|
||||
// that BTC Map delists correctly reverts to fully gone here too, not just
|
||||
// unclaimed — its claim and any links through it disappear with it.
|
||||
const deletePlace = db.prepare('DELETE FROM places WHERE id = ?');
|
||||
|
||||
// Public on purpose for now (single-region MVP, no write access to real money);
|
||||
// revisit if/when this needs to be admin-gated for a larger, costlier sync area.
|
||||
// Public on purpose (read-only pull from a public API, no write access to
|
||||
// real money) — revisit if this ever needs admin-gating, e.g. to bound
|
||||
// how often the full worldwide dataset gets re-pulled.
|
||||
app.post('/api/sync', async () => {
|
||||
const places = await fetchAreaPlaces(config.BTCMAP_CENTER_LAT, config.BTCMAP_CENTER_LON, config.BTCMAP_RADIUS_KM);
|
||||
const since = (getWatermark.get(WATERMARK_KEY) as { value: string } | undefined)?.value ?? EPOCH;
|
||||
const syncedAt = Math.floor(Date.now() / 1000);
|
||||
|
||||
const insertAll = db.transaction((rows: typeof places) => {
|
||||
for (const p of rows) {
|
||||
upsertPlace.run({
|
||||
id: p.id,
|
||||
lat: p.lat,
|
||||
lon: p.lon,
|
||||
name: p.name ?? '',
|
||||
icon: p.icon ?? null,
|
||||
address: p.address ?? null,
|
||||
osm_id: p.osm_id ?? null,
|
||||
btcmap_updated_at: p.updated_at ?? null,
|
||||
synced_at: syncedAt,
|
||||
});
|
||||
}
|
||||
});
|
||||
insertAll(places);
|
||||
const places = await fetchAllPlaces(config.BTCMAP_API_BASE_URL, since);
|
||||
let upserted = 0;
|
||||
let deleted = 0;
|
||||
|
||||
return { synced: places.length, center: { lat: config.BTCMAP_CENTER_LAT, lon: config.BTCMAP_CENTER_LON }, radiusKm: config.BTCMAP_RADIUS_KM };
|
||||
db.transaction(() => {
|
||||
for (const p of places) {
|
||||
if (p.deleted_at) {
|
||||
deletePlace.run(p.id);
|
||||
deleted += 1;
|
||||
} else {
|
||||
upsertPlace.run({
|
||||
id: p.id,
|
||||
lat: p.lat,
|
||||
lon: p.lon,
|
||||
name: p.name ?? '',
|
||||
icon: p.icon ?? null,
|
||||
address: p.address ?? null,
|
||||
osm_id: p.osm_id ?? null,
|
||||
btcmap_updated_at: p.updated_at ?? null,
|
||||
synced_at: syncedAt,
|
||||
});
|
||||
upserted += 1;
|
||||
}
|
||||
}
|
||||
})();
|
||||
|
||||
const newWatermark = latestUpdatedAt(places, since);
|
||||
setWatermark.run(WATERMARK_KEY, newWatermark);
|
||||
|
||||
return { fetched: places.length, upserted, deleted, since, watermark: newWatermark };
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { fetchAllPlaces, latestUpdatedAt, type BtcMapPlace } from './btcmap.js';
|
||||
|
||||
const BASE = 'https://fake-btcmap.example';
|
||||
|
||||
function place(id: number, updatedAt: string, deleted = false): BtcMapPlace {
|
||||
return {
|
||||
id,
|
||||
lat: id,
|
||||
lon: id,
|
||||
name: `place-${id}`,
|
||||
updated_at: updatedAt,
|
||||
...(deleted ? { deleted_at: updatedAt } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
describe('fetchAllPlaces', () => {
|
||||
it('requests include_deleted and the given since, with no pagination params', async () => {
|
||||
let capturedUrl: URL | undefined;
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn(async (input: string | URL) => {
|
||||
capturedUrl = input instanceof URL ? input : new URL(String(input));
|
||||
return new Response(JSON.stringify([]), { status: 200 });
|
||||
}),
|
||||
);
|
||||
|
||||
await fetchAllPlaces(BASE, '2026-01-01T00:00:00Z');
|
||||
expect(capturedUrl?.searchParams.get('include_deleted')).toBe('true');
|
||||
expect(capturedUrl?.searchParams.get('updated_since')).toBe('2026-01-01T00:00:00Z');
|
||||
expect(capturedUrl?.searchParams.has('limit')).toBe(false);
|
||||
});
|
||||
|
||||
it('returns whatever the API responds with, deleted entries included', async () => {
|
||||
const places = [place(1, '2026-01-01T00:00:00Z'), place(2, '2026-01-02T00:00:00Z', true)];
|
||||
vi.stubGlobal('fetch', vi.fn(async () => new Response(JSON.stringify(places), { status: 200 })));
|
||||
|
||||
const result = await fetchAllPlaces(BASE, '1970-01-01T00:00:00Z');
|
||||
expect(result).toEqual(places);
|
||||
});
|
||||
|
||||
it('throws on a non-ok response', async () => {
|
||||
vi.stubGlobal('fetch', vi.fn(async () => new Response('nope', { status: 500, statusText: 'Internal Error' })));
|
||||
await expect(fetchAllPlaces(BASE, '1970-01-01T00:00:00Z')).rejects.toThrow(/500/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('latestUpdatedAt', () => {
|
||||
it('returns the fallback for an empty list', () => {
|
||||
expect(latestUpdatedAt([], '2026-01-01T00:00:00Z')).toBe('2026-01-01T00:00:00Z');
|
||||
});
|
||||
|
||||
it('picks the chronologically latest timestamp, not the lexicographically largest', () => {
|
||||
// The whole reason this exists: naive string comparison gets this backwards —
|
||||
// "...27Z" (whole-second) sorts AFTER "...27.500Z" (mid-second) as a string,
|
||||
// even though .500Z happened later within that same second.
|
||||
const places = [place(1, '2026-01-01T00:00:27Z'), place(2, '2026-01-01T00:00:27.500Z')];
|
||||
expect(latestUpdatedAt(places, '1970-01-01T00:00:00Z')).toBe('2026-01-01T00:00:27.500Z');
|
||||
});
|
||||
|
||||
it('ignores entries with no updated_at', () => {
|
||||
const places: BtcMapPlace[] = [{ id: 1, lat: 0, lon: 0 }, place(2, '2026-01-01T00:00:00Z')];
|
||||
expect(latestUpdatedAt(places, '1970-01-01T00:00:00Z')).toBe('2026-01-01T00:00:00Z');
|
||||
});
|
||||
|
||||
it('keeps the fallback if nothing in the batch is newer', () => {
|
||||
const places = [place(1, '2020-01-01T00:00:00Z')];
|
||||
expect(latestUpdatedAt(places, '2026-01-01T00:00:00Z')).toBe('2026-01-01T00:00:00Z');
|
||||
});
|
||||
});
|
||||
@@ -7,22 +7,52 @@ export interface BtcMapPlace {
|
||||
address?: string;
|
||||
osm_id?: string;
|
||||
updated_at?: string;
|
||||
deleted_at?: string;
|
||||
}
|
||||
|
||||
const BTCMAP_FIELDS = 'id,lat,lon,name,icon,address,osm_id,updated_at';
|
||||
const BTCMAP_FIELDS = 'id,lat,lon,name,icon,address,osm_id,updated_at,deleted_at';
|
||||
|
||||
/**
|
||||
* Pull every BTC Map place within a radius of a center point. Used for the
|
||||
* initial/periodic sync into our local `places` cache — see routes/sync.ts.
|
||||
* Every BTC Map place worldwide updated since `since` (ISO 8601), deleted
|
||||
* ones included so the caller can evict them locally too — one unpaginated
|
||||
* request, not the chronological-cursor pagination BTC Map's own docs
|
||||
* describe as the "recommended" pattern.
|
||||
*
|
||||
* That paginated pattern was tried first and abandoned: with updated_since
|
||||
* as a plain ISO string, BTC Map's timestamps mix millisecond-precision and
|
||||
* whole-second values, and naive string comparison of ISO-8601 strings
|
||||
* across mixed precision is NOT chronologically safe ("...27Z" sorts after
|
||||
* "...27.137Z" lexicographically, even though .137 happened later within
|
||||
* that same second) — this silently truncated a real sync to the first
|
||||
* ~4,000 of ~42,000 records with no error. Measured the alternative
|
||||
* instead: omitting `limit` entirely returns the full dataset — currently
|
||||
* ~42k records (including historical deletions), ~9MB uncompressed, in
|
||||
* ~2-3s — comfortably fine as a single request. Revisit with real
|
||||
* pagination (and a numeric/parsed-Date cursor, not string comparison) if
|
||||
* the dataset ever grows enough to make that request unwieldy.
|
||||
*/
|
||||
export async function fetchAreaPlaces(lat: number, lon: number, radiusKm: number): Promise<BtcMapPlace[]> {
|
||||
const url = new URL('https://api.btcmap.org/v4/places/search/');
|
||||
url.searchParams.set('lat', String(lat));
|
||||
url.searchParams.set('lon', String(lon));
|
||||
url.searchParams.set('radius_km', String(radiusKm));
|
||||
export async function fetchAllPlaces(apiBaseUrl: string, since: string): Promise<BtcMapPlace[]> {
|
||||
const url = new URL('/v4/places', apiBaseUrl);
|
||||
url.searchParams.set('fields', BTCMAP_FIELDS);
|
||||
url.searchParams.set('updated_since', since);
|
||||
url.searchParams.set('include_deleted', 'true');
|
||||
|
||||
const res = await fetch(url);
|
||||
if (!res.ok) throw new Error(`btcmap search failed: ${res.status} ${res.statusText}`);
|
||||
if (!res.ok) throw new Error(`btcmap places fetch failed: ${res.status} ${res.statusText}`);
|
||||
return (await res.json()) as BtcMapPlace[];
|
||||
}
|
||||
|
||||
/** Latest updated_at among a batch, compared as real timestamps (not strings — see fetchAllPlaces). */
|
||||
export function latestUpdatedAt(places: BtcMapPlace[], fallback: string): string {
|
||||
let best = fallback;
|
||||
let bestMs = Date.parse(fallback);
|
||||
for (const p of places) {
|
||||
if (!p.updated_at) continue;
|
||||
const ms = Date.parse(p.updated_at);
|
||||
if (!Number.isNaN(ms) && ms > bestMs) {
|
||||
best = p.updated_at;
|
||||
bestMs = ms;
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user