ssmithx/regress
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Go global: worldwide BTC Map sync, marker clustering, response compression

Browse Source 
- Sync now pulls every BTC Map place worldwide (~29k live), not just
  Madeira — dropped BTCMAP_CENTER_LAT/LON/RADIUS_KM entirely.
- Incremental via a stored watermark (settings.btcmap_synced_since):
  only fetches what changed since the last sync, not the whole dataset
  every time. Handles deletions too (BTC Map delistings correctly evict
  the local place + cascade its claim/links, not just go unclaimed).
- Real bug caught and fixed before it shipped: tried BTC Map's own
  recommended updated_since+limit pagination first, but their timestamps
  mix millisecond and whole-second precision, and naive string comparison
  across that isn't chronologically safe — silently truncated a real
  sync to ~4,000 of ~42,000 records with no error. Measured the
  alternative (single unpaginated request) instead: 2-3s for the full
  dataset, simpler and actually correct. Full writeup in
  services/btcmap.ts, regression test for the specific bug in
  services/btcmap.test.ts.
- @fastify/compress added — the places list is now tens of thousands of
  rows, gzip/br/zstd auto-negotiated.
- Also fixed while touching dependencies: @fastify/static had a real
  high-severity path-traversal/auth-bypass advisory (GHSA-pr96-94w5-mx2h
  et al) affecting the version we were pinned to — bumped to the patched
  10.1.2.
- Frontend: leaflet.markercluster (raw per-marker rendering doesn't scale
  to tens of thousands of points), cyberpunk-themed cluster icons to
  match the existing HUD styling, batch marker insertion (addLayers, not
  a per-marker addLayer loop) since that's dramatically faster at this
  scale. Default map view is now world-scale, recentering on the
  player's location if geolocation is available.
- 60 backend tests passing throughout (7 new for the sync rewrite).
This commit is contained in:
ssmithx
2026-08-05 17:32:54 +00:00
parent 4fef930029
commit bb03bfc365
14 changed files with 464 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/package-lock.json
Generated
+166 -150
View File
@@ -8,9 +8,10 @@
"name": "regress-server",
"version": "0.1.0",
"dependencies": {
"@fastify/compress": "^9.1.1",
"@fastify/cookie": "^11.0.2",
"@fastify/cors": "^11.3.0",
"@fastify/static": "^8.1.1",
"@fastify/static": "^10.1.2",
"better-sqlite3": "^12.2.0",
"fastify": "^5.4.0",
"fastify-plugin": "^5.1.0",
@@ -504,6 +505,85 @@
"fast-uri": "^3.0.0"
}
},
"node_modules/@fastify/compress": {
"version": "9.1.1",
"resolved": "https://registry.npmjs.org/@fastify/compress/-/compress-9.1.1.tgz",
"integrity": "sha512-YRv6HsKhI8TjdW/Etwo2GskrN8x5YaacSQhmeyc1dIgptvKPCFGRkXVciNvnCZULM1Np90b0TUM3D90l3VF1Zw==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/fastify"
},
{
"type": "opencollective",
"url": "https://opencollective.com/fastify"
}
],
"license": "MIT",
"dependencies": {
"@fastify/accept-negotiator": "^2.0.0",
"fastify-plugin": "^6.0.0",
"mime-db": "^1.52.0",
"minipass": "^7.0.4",
"readable-stream": "^4.5.2"
}
},
"node_modules/@fastify/compress/node_modules/buffer": {
"version": "6.0.3",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz",
"integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/feross"
},
{
"type": "patreon",
"url": "https://www.patreon.com/feross"
},
{
"type": "consulting",
"url": "https://feross.org/support"
}
],
"license": "MIT",
"dependencies": {
"base64-js": "^1.3.1",
"ieee754": "^1.2.1"
}
},
"node_modules/@fastify/compress/node_modules/fastify-plugin": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/fastify-plugin/-/fastify-plugin-6.0.0.tgz",
"integrity": "sha512-fZOty7z3O7vOliF6d8bHE3wiEh1KcNnKEQensSgTk9C1DvN6nRLS++XVd86v33Hw/8u9Un8A1zDrQ8ujcQDHEg==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/fastify"
},
{
"type": "opencollective",
"url": "https://opencollective.com/fastify"
}
],
"license": "MIT"
},
"node_modules/@fastify/compress/node_modules/readable-stream": {
"version": "4.7.0",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz",
"integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==",
"license": "MIT",
"dependencies": {
"abort-controller": "^3.0.0",
"buffer": "^6.0.3",
"events": "^3.3.0",
"process": "^0.11.10",
"string_decoder": "^1.3.0"
},
"engines": {
"node": "^12.22.0 || ^14.17.0 || >=16.0.0"
}
},
"node_modules/@fastify/cookie": {
"version": "11.1.2",
"resolved": "https://registry.npmjs.org/@fastify/cookie/-/cookie-11.1.2.tgz",
@@ -690,9 +770,9 @@
}
},
"node_modules/@fastify/static": {
"version": "8.3.0",
"resolved": "https://registry.npmjs.org/@fastify/static/-/static-8.3.0.tgz",
"integrity": "sha512-yKxviR5PH1OKNnisIzZKmgZSus0r2OZb8qCSbqmw34aolT4g3UlzYfeBRym+HJ1J471CR8e2ldNub4PubD1coA==",
"version": "10.1.2",
"resolved": "https://registry.npmjs.org/@fastify/static/-/static-10.1.2.tgz",
"integrity": "sha512-G/g18cG9tLutT/OVyN1AIsHIl9L1UwmJ+S3dkyhVpplIx0nEMicd7RGQ+uJLyhKKF4a3tTcQydccn3Mop1fX+Q==",
"funding": [
{
"type": "github",
@@ -706,21 +786,29 @@
"license": "MIT",
"dependencies": {
"@fastify/accept-negotiator": "^2.0.0",
"@fastify/error": "^4.0.0",
"@fastify/send": "^4.0.0",
"content-disposition": "^0.5.4",
"fastify-plugin": "^5.0.0",
"content-disposition": "^2.0.1",
"fastify-plugin": "^6.0.0",
"fastq": "^1.17.1",
"glob": "^11.0.0"
"glob": "^13.0.0"
}
},
"node_modules/@isaacs/cliui": {
"version": "9.0.0",
"resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-9.0.0.tgz",
"integrity": "sha512-AokJm4tuBHillT+FpMtxQ60n8ObyXBatq7jD2/JA9dxbDDokKQm8KMht5ibGzLVU9IJDIKK4TPKgMHEYMn3lMg==",
"license": "BlueOak-1.0.0",
"engines": {
"node": ">=18"
}
"node_modules/@fastify/static/node_modules/fastify-plugin": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/fastify-plugin/-/fastify-plugin-6.0.0.tgz",
"integrity": "sha512-fZOty7z3O7vOliF6d8bHE3wiEh1KcNnKEQensSgTk9C1DvN6nRLS++XVd86v33Hw/8u9Un8A1zDrQ8ujcQDHEg==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/fastify"
},
{
"type": "opencollective",
"url": "https://opencollective.com/fastify"
}
],
"license": "MIT"
},
"node_modules/@jridgewell/sourcemap-codec": {
"version": "1.5.5",
@@ -1346,6 +1434,18 @@
"url": "https://opencollective.com/vitest"
}
},
"node_modules/abort-controller": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
"integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==",
"license": "MIT",
"dependencies": {
"event-target-shim": "^5.0.0"
},
"engines": {
"node": ">=6.5"
}
},
"node_modules/abstract-logging": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/abstract-logging/-/abstract-logging-2.0.1.tgz",
@@ -1567,15 +1667,16 @@
"license": "ISC"
},
"node_modules/content-disposition": {
"version": "0.5.4",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
"integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==",
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-2.0.1.tgz",
"integrity": "sha512-e+H0ZXHSWYrENhQzw1LPuP4oF5MzVKmDU6d3hxlvaPEYLLg62MxtQNPRx4SYSuYJSBUgnQIG4HIN2tEtNv7Dog==",
"license": "MIT",
"dependencies": {
"safe-buffer": "5.2.1"
},
"engines": {
"node": ">= 0.6"
"node": ">=18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/cookie": {
@@ -1591,20 +1692,6 @@
"url": "https://opencollective.com/express"
}
},
"node_modules/cross-spawn": {
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"license": "MIT",
"dependencies": {
"path-key": "^3.1.0",
"shebang-command": "^2.0.0",
"which": "^2.0.1"
},
"engines": {
"node": ">= 8"
}
},
"node_modules/debug": {
"version": "4.4.3",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
@@ -1758,6 +1845,24 @@
"@types/estree": "^1.0.0"
}
},
"node_modules/event-target-shim": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
"integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==",
"license": "MIT",
"engines": {
"node": ">=6"
}
},
"node_modules/events": {
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
"integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==",
"license": "MIT",
"engines": {
"node": ">=0.8.x"
}
},
"node_modules/expand-template": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
@@ -1950,22 +2055,6 @@
"node": ">=20"
}
},
"node_modules/foreground-child": {
"version": "3.3.1",
"resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz",
"integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==",
"license": "ISC",
"dependencies": {
"cross-spawn": "^7.0.6",
"signal-exit": "^4.0.1"
},
"engines": {
"node": ">=14"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/fs-constants": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
@@ -1994,24 +2083,17 @@
"license": "MIT"
},
"node_modules/glob": {
"version": "11.1.0",
"resolved": "https://registry.npmjs.org/glob/-/glob-11.1.0.tgz",
"integrity": "sha512-vuNwKSaKiqm7g0THUBu2x7ckSs3XJLXE+2ssL7/MfTGPLLcrJQ/4Uq1CjPTtO5cCIiRxqvN6Twy1qOwhL0Xjcw==",
"deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me",
"version": "13.0.6",
"resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz",
"integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==",
"license": "BlueOak-1.0.0",
"dependencies": {
"foreground-child": "^3.3.1",
"jackspeak": "^4.1.1",
"minimatch": "^10.1.1",
"minipass": "^7.1.2",
"package-json-from-dist": "^1.0.0",
"path-scurry": "^2.0.0"
},
"bin": {
"glob": "dist/esm/bin.mjs"
"minimatch": "^10.2.2",
"minipass": "^7.1.3",
"path-scurry": "^2.0.2"
},
"engines": {
"node": "20 || >=22"
"node": "18 || 20 || >=22"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
@@ -2078,27 +2160,6 @@
"node": ">= 10"
}
},
"node_modules/isexe": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
"integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==",
"license": "ISC"
},
"node_modules/jackspeak": {
"version": "4.2.3",
"resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.2.3.tgz",
"integrity": "sha512-ykkVRwrYvFm1nb2AJfKKYPr0emF6IiXDYUaFx4Zn9ZuIH7MrzEZ3sD5RlqGXNRpHtvUHJyOnCEFxOlNDtGo7wg==",
"license": "BlueOak-1.0.0",
"dependencies": {
"@isaacs/cliui": "^9.0.0"
},
"engines": {
"node": "20 || >=22"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/js-tokens": {
"version": "9.0.1",
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-9.0.1.tgz",
@@ -2219,6 +2280,15 @@
"node": ">=10.0.0"
}
},
"node_modules/mime-db": {
"version": "1.54.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz",
"integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==",
"license": "MIT",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/mimic-response": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",
@@ -2361,21 +2431,6 @@
"wrappy": "1"
}
},
"node_modules/package-json-from-dist": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz",
"integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==",
"license": "BlueOak-1.0.0"
},
"node_modules/path-key": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
"integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==",
"license": "MIT",
"engines": {
"node": ">=8"
}
},
"node_modules/path-scurry": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz",
@@ -2522,6 +2577,15 @@
"node": ">=10"
}
},
"node_modules/process": {
"version": "0.11.10",
"resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz",
"integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==",
"license": "MIT",
"engines": {
"node": ">= 0.6.0"
}
},
"node_modules/process-warning": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.1.0.tgz",
@@ -2763,27 +2827,6 @@
"integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==",
"license": "ISC"
},
"node_modules/shebang-command": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
"integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
"license": "MIT",
"dependencies": {
"shebang-regex": "^3.0.0"
},
"engines": {
"node": ">=8"
}
},
"node_modules/shebang-regex": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
"integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
"license": "MIT",
"engines": {
"node": ">=8"
}
},
"node_modules/siginfo": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz",
@@ -2791,18 +2834,6 @@
"dev": true,
"license": "ISC"
},
"node_modules/signal-exit": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz",
"integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==",
"license": "ISC",
"engines": {
"node": ">=14"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/simple-concat": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz",
@@ -3284,21 +3315,6 @@
}
}
},
"node_modules/which": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
"integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
"license": "ISC",
"dependencies": {
"isexe": "^2.0.0"
},
"bin": {
"node-which": "bin/node-which"
},
"engines": {
"node": ">= 8"
}
},
"node_modules/why-is-node-running": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz",
server/package.json
+2 -1
View File
@@ -11,9 +11,10 @@
"typecheck": "tsc --noEmit"
},
"dependencies": {
"@fastify/compress": "^9.1.1",
"@fastify/cookie": "^11.0.2",
"@fastify/cors": "^11.3.0",
"@fastify/static": "^8.1.1",
"@fastify/static": "^10.1.2",
"better-sqlite3": "^12.2.0",
"fastify": "^5.4.0",
"fastify-plugin": "^5.1.0",
server/src/app.ts
+4
View File
@@ -1,4 +1,5 @@
import Fastify, { type FastifyInstance } from 'fastify';
import compress from '@fastify/compress';
import cookie from '@fastify/cookie';
import cors from '@fastify/cors';
import fastifyStatic from '@fastify/static';
@@ -49,6 +50,9 @@ export async function buildApp(opts: BuildAppOptions): Promise<FastifyInstance>
await app.register(cookie);
await app.register(cors, { origin: true, credentials: true, methods: ['GET', 'POST', 'DELETE'] });
// Global place list is tens of thousands of rows — gzip is the single
// biggest win for that payload size, worth having by default everywhere.
await app.register(compress, { global: true });
await app.register(nostrAuth, { db, config });
// Everything below is mounted under ROUTE_PREFIX (empty by default, i.e.
server/src/config.ts
+5 -4
View File
@@ -15,10 +15,11 @@ const envSchema = z.object({
ROUTE_PREFIX: z.string().default(''),
NIP98_MAX_SKEW_SECS: z.coerce.number().default(60),
SESSION_TTL_DAYS: z.coerce.number().default(30),
// Default sync area: Madeira, Portugal (Funchal-centered radius covering the whole island).
BTCMAP_CENTER_LAT: z.coerce.number().default(32.7607),
BTCMAP_CENTER_LON: z.coerce.number().default(-16.9595),
BTCMAP_RADIUS_KM: z.coerce.number().default(45),
// Base URL for BTC Map's API — global sync pulls every place worldwide via
// the chronological /v4/places endpoint (paginated, incremental via a
// stored watermark — see routes/sync.ts), not scoped to any region.
// Overridable for tests to point at a mock server instead of the real API.
BTCMAP_API_BASE_URL: z.string().url().default('https://api.btcmap.org'),
// How close (in meters) a player must be to a place to claim it or link from it.
CLAIM_RADIUS_METERS: z.coerce.number().default(40),
// Maximum link length, in kilometers — mirrors Ingress's level-scaled link
server/src/routes/sync.ts
+46 -21
View File
@@ -1,9 +1,17 @@
import type { FastifyInstance } from 'fastify';
import { fetchAreaPlaces } from '../services/btcmap.js';
import { fetchAllPlaces, latestUpdatedAt } from '../services/btcmap.js';
const WATERMARK_KEY = 'btcmap_synced_since';
const EPOCH = '1970-01-01T00:00:00Z';
export default async function syncRoutes(app: FastifyInstance) {
const { db, config } = app.ctx;
const getWatermark = db.prepare('SELECT value FROM settings WHERE key = ?');
const setWatermark = db.prepare(`
INSERT INTO settings (key, value) VALUES (?, ?)
ON CONFLICT(key) DO UPDATE SET value = excluded.value
`);
const upsertPlace = db.prepare(`
INSERT INTO places (id, lat, lon, name, icon, address, osm_id, btcmap_updated_at, synced_at)
VALUES (@id, @lat, @lon, @name, @icon, @address, @osm_id, @btcmap_updated_at, @synced_at)
@@ -17,30 +25,47 @@ export default async function syncRoutes(app: FastifyInstance) {
btcmap_updated_at = excluded.btcmap_updated_at,
synced_at = excluded.synced_at
`);
// ON DELETE CASCADE on claims/links (see db/migrations.ts) means a place
// that BTC Map delists correctly reverts to fully gone here too, not just
// unclaimed — its claim and any links through it disappear with it.
const deletePlace = db.prepare('DELETE FROM places WHERE id = ?');
// Public on purpose for now (single-region MVP, no write access to real money);
// revisit if/when this needs to be admin-gated for a larger, costlier sync area.
// Public on purpose (read-only pull from a public API, no write access to
// real money) — revisit if this ever needs admin-gating, e.g. to bound
// how often the full worldwide dataset gets re-pulled.
app.post('/api/sync', async () => {
const places = await fetchAreaPlaces(config.BTCMAP_CENTER_LAT, config.BTCMAP_CENTER_LON, config.BTCMAP_RADIUS_KM);
const since = (getWatermark.get(WATERMARK_KEY) as { value: string } | undefined)?.value ?? EPOCH;
const syncedAt = Math.floor(Date.now() / 1000);
const insertAll = db.transaction((rows: typeof places) => {
for (const p of rows) {
upsertPlace.run({
id: p.id,
lat: p.lat,
lon: p.lon,
name: p.name ?? '',
icon: p.icon ?? null,
address: p.address ?? null,
osm_id: p.osm_id ?? null,
btcmap_updated_at: p.updated_at ?? null,
synced_at: syncedAt,
});
}
});
insertAll(places);
const places = await fetchAllPlaces(config.BTCMAP_API_BASE_URL, since);
let upserted = 0;
let deleted = 0;
return { synced: places.length, center: { lat: config.BTCMAP_CENTER_LAT, lon: config.BTCMAP_CENTER_LON }, radiusKm: config.BTCMAP_RADIUS_KM };
db.transaction(() => {
for (const p of places) {
if (p.deleted_at) {
deletePlace.run(p.id);
deleted += 1;
} else {
upsertPlace.run({
id: p.id,
lat: p.lat,
lon: p.lon,
name: p.name ?? '',
icon: p.icon ?? null,
address: p.address ?? null,
osm_id: p.osm_id ?? null,
btcmap_updated_at: p.updated_at ?? null,
synced_at: syncedAt,
});
upserted += 1;
}
}
})();
const newWatermark = latestUpdatedAt(places, since);
setWatermark.run(WATERMARK_KEY, newWatermark);
return { fetched: places.length, upserted, deleted, since, watermark: newWatermark };
});
}
server/src/services/btcmap.test.ts
+74
View File
@@ -0,0 +1,74 @@
import { afterEach, describe, expect, it, vi } from 'vitest';
import { fetchAllPlaces, latestUpdatedAt, type BtcMapPlace } from './btcmap.js';
const BASE = 'https://fake-btcmap.example';
function place(id: number, updatedAt: string, deleted = false): BtcMapPlace {
return {
id,
lat: id,
lon: id,
name: `place-${id}`,
updated_at: updatedAt,
...(deleted ? { deleted_at: updatedAt } : {}),
};
}
afterEach(() => {
vi.unstubAllGlobals();
});
describe('fetchAllPlaces', () => {
it('requests include_deleted and the given since, with no pagination params', async () => {
let capturedUrl: URL | undefined;
vi.stubGlobal(
'fetch',
vi.fn(async (input: string | URL) => {
capturedUrl = input instanceof URL ? input : new URL(String(input));
return new Response(JSON.stringify([]), { status: 200 });
}),
);
await fetchAllPlaces(BASE, '2026-01-01T00:00:00Z');
expect(capturedUrl?.searchParams.get('include_deleted')).toBe('true');
expect(capturedUrl?.searchParams.get('updated_since')).toBe('2026-01-01T00:00:00Z');
expect(capturedUrl?.searchParams.has('limit')).toBe(false);
});
it('returns whatever the API responds with, deleted entries included', async () => {
const places = [place(1, '2026-01-01T00:00:00Z'), place(2, '2026-01-02T00:00:00Z', true)];
vi.stubGlobal('fetch', vi.fn(async () => new Response(JSON.stringify(places), { status: 200 })));
const result = await fetchAllPlaces(BASE, '1970-01-01T00:00:00Z');
expect(result).toEqual(places);
});
it('throws on a non-ok response', async () => {
vi.stubGlobal('fetch', vi.fn(async () => new Response('nope', { status: 500, statusText: 'Internal Error' })));
await expect(fetchAllPlaces(BASE, '1970-01-01T00:00:00Z')).rejects.toThrow(/500/);
});
});
describe('latestUpdatedAt', () => {
it('returns the fallback for an empty list', () => {
expect(latestUpdatedAt([], '2026-01-01T00:00:00Z')).toBe('2026-01-01T00:00:00Z');
});
it('picks the chronologically latest timestamp, not the lexicographically largest', () => {
// The whole reason this exists: naive string comparison gets this backwards —
// "...27Z" (whole-second) sorts AFTER "...27.500Z" (mid-second) as a string,
// even though .500Z happened later within that same second.
const places = [place(1, '2026-01-01T00:00:27Z'), place(2, '2026-01-01T00:00:27.500Z')];
expect(latestUpdatedAt(places, '1970-01-01T00:00:00Z')).toBe('2026-01-01T00:00:27.500Z');
});
it('ignores entries with no updated_at', () => {
const places: BtcMapPlace[] = [{ id: 1, lat: 0, lon: 0 }, place(2, '2026-01-01T00:00:00Z')];
expect(latestUpdatedAt(places, '1970-01-01T00:00:00Z')).toBe('2026-01-01T00:00:00Z');
});
it('keeps the fallback if nothing in the batch is newer', () => {
const places = [place(1, '2020-01-01T00:00:00Z')];
expect(latestUpdatedAt(places, '2026-01-01T00:00:00Z')).toBe('2026-01-01T00:00:00Z');
});
});
server/src/services/btcmap.ts
+39 -9
View File
@@ -7,22 +7,52 @@ export interface BtcMapPlace {
address?: string;
osm_id?: string;
updated_at?: string;
deleted_at?: string;
}
const BTCMAP_FIELDS = 'id,lat,lon,name,icon,address,osm_id,updated_at';
const BTCMAP_FIELDS = 'id,lat,lon,name,icon,address,osm_id,updated_at,deleted_at';
/**
* Pull every BTC Map place within a radius of a center point. Used for the
* initial/periodic sync into our local `places` cache — see routes/sync.ts.
* Every BTC Map place worldwide updated since `since` (ISO 8601), deleted
* ones included so the caller can evict them locally too — one unpaginated
* request, not the chronological-cursor pagination BTC Map's own docs
* describe as the "recommended" pattern.
*
* That paginated pattern was tried first and abandoned: with updated_since
* as a plain ISO string, BTC Map's timestamps mix millisecond-precision and
* whole-second values, and naive string comparison of ISO-8601 strings
* across mixed precision is NOT chronologically safe ("...27Z" sorts after
* "...27.137Z" lexicographically, even though .137 happened later within
* that same second) — this silently truncated a real sync to the first
* ~4,000 of ~42,000 records with no error. Measured the alternative
* instead: omitting `limit` entirely returns the full dataset — currently
* ~42k records (including historical deletions), ~9MB uncompressed, in
* ~2-3s — comfortably fine as a single request. Revisit with real
* pagination (and a numeric/parsed-Date cursor, not string comparison) if
* the dataset ever grows enough to make that request unwieldy.
*/
export async function fetchAreaPlaces(lat: number, lon: number, radiusKm: number): Promise<BtcMapPlace[]> {
const url = new URL('https://api.btcmap.org/v4/places/search/');
url.searchParams.set('lat', String(lat));
url.searchParams.set('lon', String(lon));
url.searchParams.set('radius_km', String(radiusKm));
export async function fetchAllPlaces(apiBaseUrl: string, since: string): Promise<BtcMapPlace[]> {
const url = new URL('/v4/places', apiBaseUrl);
url.searchParams.set('fields', BTCMAP_FIELDS);
url.searchParams.set('updated_since', since);
url.searchParams.set('include_deleted', 'true');
const res = await fetch(url);
if (!res.ok) throw new Error(`btcmap search failed: ${res.status} ${res.statusText}`);
if (!res.ok) throw new Error(`btcmap places fetch failed: ${res.status} ${res.statusText}`);
return (await res.json()) as BtcMapPlace[];
}
/** Latest updated_at among a batch, compared as real timestamps (not strings — see fetchAllPlaces). */
export function latestUpdatedAt(places: BtcMapPlace[], fallback: string): string {
let best = fallback;
let bestMs = Date.parse(fallback);
for (const p of places) {
if (!p.updated_at) continue;
const ms = Date.parse(p.updated_at);
if (!Number.isNaN(ms) && ms > bestMs) {
best = p.updated_at;
bestMs = ms;
}
}
return best;
}