feat(dojobay): add DojoBay app (manifest, image, catalog, ports)
This commit is contained in:
@@ -0,0 +1,317 @@
|
||||
#!/usr/bin/env node
|
||||
// Bootstrap a new Dojo Bay from a TRUSTED existing instance, so a fresh
|
||||
// directory is mature the moment it starts: its nodes become approved store
|
||||
// records here and their reliability histories carry over.
|
||||
//
|
||||
// node scripts/bootstrap-import.mjs --onion <56-char>.onion \
|
||||
// --code PM8T... [--dry-run]
|
||||
//
|
||||
// Trust is verified before anything is imported: the remote instance's
|
||||
// data/operator.json must bind that onion to exactly the payment code YOU
|
||||
// typed in, under a valid wallet signature (server/crypto.ts). If the
|
||||
// signature does not verify, or binds a different onion or code, nothing is
|
||||
// fetched further. After that: dojos.json supplies the nodes, both history
|
||||
// files supply the record, and each PayNym is resolved against paynym.rs
|
||||
// (over Tor) for its full BIP47 code-variant set so imported operators can
|
||||
// sign in here with either variant. Existing ids are never touched; history
|
||||
// is only written for ids that have none.
|
||||
import { readFile, writeFile, rename, mkdir } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { httpOverTor } from "./update.mjs";
|
||||
import { store, hasSignedBlock } from "../server/store.ts";
|
||||
import { verifySignedPayload, canonicalPairing } from "../server/crypto.ts";
|
||||
|
||||
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const DATA_DIR = process.env.PUBLIC_DATA_DIR || path.join(ROOT, "data");
|
||||
|
||||
const defaultCfg = () => ({
|
||||
proxyHost: process.env.TOR_SOCKS_HOST || "127.0.0.1",
|
||||
proxyPort: +(process.env.TOR_SOCKS_PORT || 9050),
|
||||
});
|
||||
|
||||
// GET a JSON document from the remote instance over Tor.
|
||||
async function torFetchJSON(onionHost, urlPath, cfg, timeoutMs = 30000) {
|
||||
const req = `GET ${urlPath} HTTP/1.0\r\nHost: ${onionHost}\r\nUser-Agent: dojobay-bootstrap\r\nConnection: close\r\n\r\n`;
|
||||
const res = await httpOverTor(cfg, onionHost, 80, req, timeoutMs);
|
||||
if (res.status !== 200) throw new Error(`${urlPath}: HTTP ${res.status || "no response"}`);
|
||||
return JSON.parse(res.body);
|
||||
}
|
||||
|
||||
// A temporary name no other writer can take; see server/build-public.ts. The
|
||||
// counter matters as well as the pid: one import writes the seed, both history
|
||||
// files and the avatars in quick succession.
|
||||
let tmpSeq = 0;
|
||||
async function writeJSONAtomic(p, obj) {
|
||||
await mkdir(path.dirname(p), { recursive: true });
|
||||
const tmp = `${p}.${process.pid}.${(tmpSeq = (tmpSeq + 1) % 1e6)}.tmp`;
|
||||
await writeFile(tmp, JSON.stringify(obj, null, 2) + "\n");
|
||||
await rename(tmp, p);
|
||||
}
|
||||
|
||||
// fetchers are injectable for the self-test: fetchDoc(urlPath) -> object,
|
||||
// fetchCodes(paynymOrCode) -> [{code, segwit}, ...]
|
||||
/**
|
||||
* @param {{ onionHost?: string, trustedCode?: string, dryRun?: boolean, dataDir?: string,
|
||||
* log?: (...a: any[]) => void, fetchDoc?: any, fetchCodes?: any,
|
||||
* status?: "approved" | "pending" }} [opts]
|
||||
*/
|
||||
export async function bootstrapImport({
|
||||
onionHost, trustedCode, dryRun = false, dataDir = DATA_DIR, log = console.error,
|
||||
fetchDoc, fetchCodes, status = "approved",
|
||||
} = {}) {
|
||||
const cfg = defaultCfg();
|
||||
fetchDoc = fetchDoc || ((p) => torFetchJSON(onionHost, p, cfg));
|
||||
if (!fetchCodes) {
|
||||
const { fetchNymCodes } = await import("../server/paynym.mjs");
|
||||
fetchCodes = (nym) => fetchNymCodes(nym);
|
||||
}
|
||||
|
||||
// 1) trust gate: the remote operator binding must verify for THIS onion and
|
||||
// exactly the payment code the operator typed in.
|
||||
const { verifyOperatorDoc } = await import("../server/crypto.ts");
|
||||
const opDoc = await fetchDoc("/data/operator.json");
|
||||
const v = verifyOperatorDoc(opDoc, { expectedOnion: `http://${onionHost}` });
|
||||
if (!v.ok) throw new Error(`refusing to import: remote operator binding does not verify (${v.error})`);
|
||||
if (opDoc.paymentCode !== trustedCode) {
|
||||
throw new Error("refusing to import: the remote instance is operated by a DIFFERENT payment code than the one you trusted");
|
||||
}
|
||||
log(`trusted: ${onionHost} is signed by ${trustedCode.slice(0, 12)}… ✓`);
|
||||
|
||||
// 2) data
|
||||
const dojos = await fetchDoc("/data/dojos.json");
|
||||
const hist = await fetchDoc("/data/history.json").catch(() => ({ nodes: {} }));
|
||||
const daily = await fetchDoc("/data/history-daily.json").catch(() => ({ nodes: {} }));
|
||||
const nodes = (dojos.nodes || []).filter((n) => n.payload?.pairing?.url);
|
||||
|
||||
// The pairing URL identifies a physical Dojo; an id does not.
|
||||
//
|
||||
// An operator installing a new instance names their own node in the anchor,
|
||||
// then bootstraps from a directory that already lists it. The two ids differ,
|
||||
// because each instance derives one from the name it was given, so the same
|
||||
// machine arrived twice: once as the anchor and once as an import, with its
|
||||
// reliability history split between them. What is actually the same thing is
|
||||
// the onion address in the signed pairing payload, which is why matching on
|
||||
// it is not a heuristic. Two listings cannot share one, and an operator
|
||||
// cannot claim somebody else's without the signature failing.
|
||||
//
|
||||
// Compared as a whole URL rather than by host alone, because one machine may
|
||||
// legitimately serve mainnet at /v2 and testnet at /test/v2, and those are
|
||||
// two listings. Lower-cased and stripped of a trailing slash, since neither
|
||||
// changes which endpoint is meant.
|
||||
const pairingKey = (n) => {
|
||||
const u = n?.payload?.pairing?.url;
|
||||
if (typeof u !== "string" || !u) return null;
|
||||
return u.trim().toLowerCase().replace(/\/+$/, "");
|
||||
};
|
||||
|
||||
// Everything this instance already lists, from the store AND from the seed
|
||||
// anchor. The anchor is not a store record, which is exactly why it was
|
||||
// invisible to this check and why the operator's own node was the one node
|
||||
// guaranteed to duplicate.
|
||||
const localByUrl = new Map();
|
||||
for (const r of await store.listSubmissions()) {
|
||||
const k = pairingKey(r);
|
||||
if (k) localByUrl.set(k, r.id);
|
||||
}
|
||||
try {
|
||||
const seed = JSON.parse(await readFile(path.join(dataDir, "seed.json"), "utf8"));
|
||||
for (const n of seed.nodes || []) {
|
||||
const k = pairingKey(n);
|
||||
if (k && !localByUrl.has(k)) localByUrl.set(k, n.id);
|
||||
}
|
||||
} catch { /* no anchor yet, which is normal on a bare install */ }
|
||||
|
||||
// 3) plan records: skip existing ids; resolve full code sets per PayNym
|
||||
const existingIds = new Set((await store.listSubmissions()).map((r) => r.id));
|
||||
const plan = [];
|
||||
const codeCache = new Map();
|
||||
for (const n of nodes) {
|
||||
if (existingIds.has(n.id)) { plan.push({ action: "skip", n }); continue; }
|
||||
// Same machine under a different id. The record is not created, because a
|
||||
// second listing for one Dojo is worse than a missing one, but the history
|
||||
// is worth having: it is the same node's record of itself, and dropping it
|
||||
// would restart an operator's reliability figures from nothing on a machine
|
||||
// that has been up for months. Carried onto the id this instance uses.
|
||||
const dupOf = localByUrl.get(pairingKey(n));
|
||||
if (dupOf) { plan.push({ action: "merge", n, dupOf }); continue; }
|
||||
// A published node from another instance carries its signed block in
|
||||
// dojos.json, so an unsigned one either predates the rule there or was
|
||||
// published by an instance that does not enforce it. Either way it cannot
|
||||
// enter this store, and saying so in the plan is better than a throw from
|
||||
// putSubmission half way through the import.
|
||||
if (!hasSignedBlock(n)) { plan.push({ action: "refuse", n, why: "no signed pairing block" }); continue; }
|
||||
// And the block must actually verify, here, against the payload it claims
|
||||
// to cover.
|
||||
//
|
||||
// hasSignedBlock only looks for the two header lines, and putSubmission
|
||||
// enforces nothing more, so until this check an imported listing's
|
||||
// signature was taken on the source instance's word: a directory that was
|
||||
// careless or compromised could publish a well-formed block that verifies
|
||||
// against nothing, and every instance bootstrapping from it would list the
|
||||
// node. This is the same standard the domain badges above are already held
|
||||
// to, and for the same reason: one compromised directory must not be able
|
||||
// to place listings across a federation.
|
||||
//
|
||||
// Offline and self-contained. canonicalPairing derives the message from the
|
||||
// payload being imported, so a payload altered in transit no longer matches
|
||||
// what was signed, and the addresses come from the payment code named
|
||||
// inside the block itself rather than from anything the source asserts.
|
||||
const sig = verifySignedPayload({
|
||||
signedText: n.signed,
|
||||
expectedMessage: canonicalPairing(n.payload),
|
||||
network: n.network === "testnet" ? "testnet" : "bitcoin",
|
||||
});
|
||||
if (!sig.ok) { plan.push({ action: "refuse", n, why: `signature does not verify (${sig.error})` }); continue; }
|
||||
let codes = n.paymentCode ? [n.paymentCode] : [];
|
||||
if (n.paynym) {
|
||||
if (!codeCache.has(n.paynym)) codeCache.set(n.paynym, await fetchCodes(n.paynym).catch(() => []));
|
||||
const all = codeCache.get(n.paynym).map((c) => c.code);
|
||||
if (all.length) codes = [...new Set([...all, ...codes])];
|
||||
}
|
||||
if (!codes.length) { plan.push({ action: "refuse", n, why: "no BIP47 payment code" }); continue; }
|
||||
plan.push({ action: "import", n, codes });
|
||||
}
|
||||
|
||||
const now = new Date().toISOString();
|
||||
for (const { action, n, codes, why } of plan) {
|
||||
log(` ${action.padEnd(6)} ${n.id.padEnd(28)} ${n.paynym || "(no PayNym)"} (${(codes || []).length} codes)${why ? " — " + why : ""}`);
|
||||
}
|
||||
const imports = plan.filter((p) => p.action === "import");
|
||||
const merges = plan.filter((p) => p.action === "merge");
|
||||
const refused = plan.filter((p) => p.action === "refuse");
|
||||
for (const m of merges) {
|
||||
log(` merge ${m.n.id.padEnd(28)} same Dojo as ${m.dupOf}: history only, no second listing`);
|
||||
}
|
||||
if (refused.length) log(`refused ${refused.length} node(s) that cannot be listed here: ${refused.map((p) => p.n.id).join(", ")}`);
|
||||
// The plan as data, not as log lines. The command line reads the log; the
|
||||
// admin console has to render this and let an operator decide, and parsing
|
||||
// the log back out would be inventing a format nobody agreed on.
|
||||
const rows = plan.map(({ action, n, codes, dupOf, why }) => ({
|
||||
action, id: n.id, name: n.name || n.id, network: n.network || null,
|
||||
paynym: n.paynym || null, url: n?.payload?.pairing?.url || null,
|
||||
codes: (codes || []).length, dupOf: dupOf || null, why: why || null,
|
||||
}));
|
||||
if (dryRun) {
|
||||
log(`dry run: ${imports.length} node(s) would be imported`
|
||||
+ (merges.length ? `, ${merges.length} recognised as already listed here` : "")
|
||||
+ ", nothing written.");
|
||||
return { imported: 0, planned: imports.length, merged: merges.length,
|
||||
refused: refused.length, plan: rows, status };
|
||||
}
|
||||
|
||||
for (const { n, codes } of imports) {
|
||||
await store.putSubmission({
|
||||
id: n.id, network: n.network, name: n.name || n.id,
|
||||
paymentCodes: codes, paynym: n.paynym || null,
|
||||
jurisdiction: n.jurisdiction || null, country: n.country || null,
|
||||
hardware: n.hardware || null, payload: n.payload,
|
||||
signed: n.signed || null,
|
||||
// approved at install, because choosing to bootstrap from a directory IS
|
||||
// the decision to trust its list. An import into a running instance
|
||||
// arrives pending instead, so it lands in the moderation queue the
|
||||
// operator already uses and nothing is published until they say so.
|
||||
status, source: `bootstrap-import:${onionHost}`,
|
||||
created_at: now, updated_at: now,
|
||||
});
|
||||
}
|
||||
|
||||
// 3b) verified operator domains.
|
||||
//
|
||||
// dojos.json publishes each badge's proof, and the signed statement is
|
||||
// deliberately portable: it names the domain and the payment code, never the
|
||||
// instance that verified it. So a claim travels intact — but it is NOT taken
|
||||
// on the source's word. We re-verify the signature here, locally and offline,
|
||||
// and store the claim UNVERIFIED so this instance's own sweep must see the TXT
|
||||
// record with its own eyes before any badge appears. Importing a badge because
|
||||
// another instance said so would make one compromised directory able to mint
|
||||
// verified domains across a federation.
|
||||
const claims = new Map();
|
||||
for (const n of dojos.nodes || []) {
|
||||
const pf = n.operator_domain_proof;
|
||||
if (!pf || !pf.domain || !pf.paymentCode || !pf.signed) continue;
|
||||
if (claims.has(pf.paymentCode)) continue;
|
||||
claims.set(pf.paymentCode, pf);
|
||||
}
|
||||
let domainsImported = 0, domainsRefused = 0;
|
||||
if (claims.size) {
|
||||
const { verifySignedUrlClaim } = await import("../server/crypto.ts");
|
||||
for (const [code, pf] of claims) {
|
||||
if (await store.getDomain(code)) continue; // never overwrite a local claim
|
||||
const v = verifySignedUrlClaim({ signed: pf.signed, expectedUrl: `https://${pf.domain}`, paymentCode: code });
|
||||
if (!v.ok) {
|
||||
log(` domain ${pf.domain}: refused (${v.error})`);
|
||||
domainsRefused++;
|
||||
continue;
|
||||
}
|
||||
await store.putDomain({
|
||||
paymentCode: code, domain: pf.domain, signed: pf.signed,
|
||||
verified: false, // this instance has not seen the DNS yet
|
||||
verified_at: null,
|
||||
last_check: null, // so the sweep picks it up immediately
|
||||
last_result: `imported from ${onionHost}; awaiting our own DNS check`,
|
||||
fail_since: null, created_at: now,
|
||||
});
|
||||
log(` domain ${pf.domain}: signature verified, awaiting our own TXT lookup`);
|
||||
domainsImported++;
|
||||
}
|
||||
}
|
||||
|
||||
// 4) histories: only for ids we have no history for
|
||||
for (const [file, remote] of [["history.json", hist], ["history-daily.json", daily]]) {
|
||||
const p = path.join(dataDir, file);
|
||||
let local; try { local = JSON.parse(await readFile(p, "utf8")); } catch { local = { nodes: {} } }
|
||||
local.nodes = local.nodes || {};
|
||||
let added = 0;
|
||||
for (const [id, entry] of Object.entries(remote.nodes || {})) {
|
||||
if (!local.nodes[id] && imports.some((x) => x.n.id === id)) { local.nodes[id] = entry; added++; continue; }
|
||||
// A duplicate contributes its history under the id this instance uses.
|
||||
//
|
||||
// The two series are combined rather than one replacing the other. An
|
||||
// anchor installed an hour ago has a handful of checks of its own and the
|
||||
// remote has months: overwriting throws away the local ones, skipping
|
||||
// throws away the months, and neither is what an operator means by
|
||||
// importing history. Combined, de-duplicated on the timestamp, sorted,
|
||||
// and trimmed to the same window the updater keeps.
|
||||
const merged = merges.find((x) => x.n.id === id);
|
||||
if (!merged) continue;
|
||||
const key = entry.checks ? "checks" : "days";
|
||||
const stamp = key === "checks" ? "t" : "d";
|
||||
const mine = (local.nodes[merged.dupOf] || {})[key] || [];
|
||||
const theirs = entry[key] || [];
|
||||
if (!theirs.length) continue;
|
||||
const byStamp = new Map();
|
||||
// Local last, so a period this instance measured itself wins over the
|
||||
// remote's account of the same period.
|
||||
for (const row of [...theirs, ...mine]) if (row && row[stamp]) byStamp.set(row[stamp], row);
|
||||
const all = [...byStamp.values()].sort((x, y) => String(x[stamp]).localeCompare(String(y[stamp])));
|
||||
const cap = key === "checks" ? (remote.window_checks || local.window_checks || 144) : 90;
|
||||
local.nodes[merged.dupOf] = { [key]: all.slice(-cap) };
|
||||
added++;
|
||||
}
|
||||
if (added) {
|
||||
if (remote.interval_minutes && !local.interval_minutes) local.interval_minutes = remote.interval_minutes;
|
||||
if (remote.window_checks && !local.window_checks) local.window_checks = remote.window_checks;
|
||||
await writeJSONAtomic(p, local);
|
||||
log(` history: ${added} node(s) carried into ${file}`);
|
||||
}
|
||||
}
|
||||
log(`imported ${imports.length} node(s) from ${onionHost}`
|
||||
+ (merges.length ? `, and recognised ${merges.length} as node(s) this instance already lists` : "")
|
||||
+ ". Now run: node server/build-public.mjs");
|
||||
return { imported: imports.length, planned: imports.length, merged: merges.length,
|
||||
refused: refused.length, plan: rows, status,
|
||||
domains_imported: domainsImported, domains_refused: domainsRefused };
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1] || "").href) {
|
||||
const arg = (k) => { const i = process.argv.indexOf(k); return i > 0 ? process.argv[i + 1] : null; };
|
||||
const onionHost = String(arg("--onion") || "").replace(/^https?:\/\//, "").replace(/\/.*$/, "");
|
||||
const trustedCode = arg("--code");
|
||||
if (!/^[a-z2-7]{56}\.onion$/.test(onionHost) || !trustedCode) {
|
||||
console.error("usage: node scripts/bootstrap-import.mjs --onion <56-char>.onion --code PM8T... [--dry-run]");
|
||||
process.exit(1);
|
||||
}
|
||||
bootstrapImport({ onionHost, trustedCode, dryRun: process.argv.includes("--dry-run") })
|
||||
.catch((e) => { console.error("fatal:", e.message); process.exit(1); });
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
#!/usr/bin/env node
|
||||
// Move seed nodes into the operator-managed store, idempotently.
|
||||
//
|
||||
// node scripts/migrate-seed-to-store.mjs --dry-run print the plan, write nothing
|
||||
// node scripts/migrate-seed-to-store.mjs apply it
|
||||
//
|
||||
// The seed's role is the instance ANCHOR: exactly one node, the instance
|
||||
// operator's own Dojo (mainnet or testnet), carrying their PayNym and BIP47
|
||||
// payment code. Everything else belongs in the store, where operators manage
|
||||
// their listings over Auth47. This script is the transition tool for an
|
||||
// instance whose seed still carries an old-style curated list:
|
||||
//
|
||||
// - a seed node with a PayNym present in data/paynym-codes.json becomes an
|
||||
// APPROVED store record owned by every BIP47 code variant of that PayNym
|
||||
// - a seed node WITHOUT a PayNym is REFUSED. Every listing must carry a BIP47
|
||||
// payment code: it is the identity a listing is owned, edited, verified and
|
||||
// recognised by. Code-less records were once adopted as admin-managed
|
||||
// exceptions; that door is closed, and the store refuses to write one.
|
||||
// - a seed node whose id already exists in the store is SKIPPED untouched,
|
||||
// which is what makes re-runs no-ops and lets the anchor node coexist as
|
||||
// both seed entry (bootstrap guarantee) and store record (Auth47-managed:
|
||||
// the store record shadows the seed copy in the public list)
|
||||
//
|
||||
// The script never rewrites data/seed.json: slimming the seed down to the
|
||||
// anchor is a deliberate, separate commit made AFTER the store records exist,
|
||||
// because a deploy that removes a node's seed entry before its store record
|
||||
// exists delists it (the history survives under the fourteen-day grace stamp,
|
||||
// but there is no reason to invite the gap).
|
||||
//
|
||||
// Record ids are the original seed ids, so reliability history (keyed by id)
|
||||
// carries over untouched. Afterwards run `node server/build-public.mjs`.
|
||||
import { readFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { store, hasSignedBlock } from "../server/store.ts";
|
||||
|
||||
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const DATA_DIR = process.env.PUBLIC_DATA_DIR || path.join(ROOT, "data");
|
||||
const SEED_PATH = path.join(DATA_DIR, "seed.json");
|
||||
const CODES_PATH = path.join(DATA_DIR, "paynym-codes.json");
|
||||
const DRY = process.argv.includes("--dry-run");
|
||||
|
||||
async function readJSON(p, fallback) {
|
||||
try { return JSON.parse(await readFile(p, "utf8")); }
|
||||
catch (e) { if (fallback !== undefined) return fallback; throw e; }
|
||||
}
|
||||
|
||||
const slugOf = (v) => String(v || "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
|
||||
|
||||
// Name derivation for owned groups. Remainder = seed id minus `${network}-`.
|
||||
// When one owner's several nodes share a first hyphen-token and stripping it
|
||||
// leaves something for each, drop the shared token; and prefer the seed's
|
||||
// display name whenever it slugs to the derived value, so capitalisation like
|
||||
// "wanderinKing072" survives.
|
||||
function deriveNames(nodes) {
|
||||
const rem = nodes.map((n) => n.id.replace(new RegExp(`^${n.network}-`), ""));
|
||||
let names = rem;
|
||||
if (nodes.length > 1) {
|
||||
const first = rem.map((r) => r.split("-")[0]);
|
||||
if (first.every((t) => t === first[0]) && rem.every((r) => r.includes("-"))) {
|
||||
names = rem.map((r) => r.split("-").slice(1).join("-"));
|
||||
}
|
||||
}
|
||||
return nodes.map((n, i) => (n.name && slugOf(n.name) === names[i]) ? n.name : names[i]);
|
||||
}
|
||||
|
||||
function toRecord(n, name, codes, now) {
|
||||
return {
|
||||
id: n.id, network: n.network, name,
|
||||
paymentCodes: codes,
|
||||
paynym: n.paynym || null,
|
||||
jurisdiction: n.jurisdiction || null,
|
||||
country: n.country || null,
|
||||
hardware: n.hardware || null,
|
||||
payload: n.payload,
|
||||
signed: n.signed || null,
|
||||
status: "approved",
|
||||
source: "seed-migration",
|
||||
created_at: now, updated_at: now,
|
||||
};
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const seed = await readJSON(SEED_PATH);
|
||||
const mapping = (await readJSON(CODES_PATH, { mapping: {} })).mapping || {};
|
||||
const existing = await store.listSubmissions();
|
||||
const nodes = seed.nodes || [];
|
||||
|
||||
const owned = nodes.filter((n) => n.paynym);
|
||||
const missing = owned.filter((n) => !mapping[n.paynym]);
|
||||
if (missing.length) {
|
||||
console.error("aborting: no payment codes in", path.relative(ROOT, CODES_PATH), "for:");
|
||||
for (const n of missing) console.error(" ", n.id, n.paynym);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Derive names per owner; code-less nodes keep their seed name (or the id
|
||||
// remainder). Then refuse any per-network name collision against the plan
|
||||
// itself or records already in the store under a DIFFERENT id.
|
||||
const byOwner = new Map();
|
||||
for (const n of owned) (byOwner.get(n.paynym) || byOwner.set(n.paynym, []).get(n.paynym)).push(n);
|
||||
const nameOf = new Map();
|
||||
for (const group of byOwner.values()) deriveNames(group).forEach((nm, i) => nameOf.set(group[i].id, nm));
|
||||
for (const n of nodes.filter((x) => !x.paynym)) {
|
||||
const rem = n.id.replace(new RegExp(`^${n.network}-`), "");
|
||||
nameOf.set(n.id, (n.name && slugOf(n.name) === rem) ? n.name : (n.name || rem));
|
||||
}
|
||||
const seen = new Set();
|
||||
for (const n of nodes) {
|
||||
const key = `${n.network}:${slugOf(nameOf.get(n.id))}`;
|
||||
if (seen.has(key)) { console.error("aborting: duplicate node name per network:", key); process.exit(1); }
|
||||
seen.add(key);
|
||||
}
|
||||
for (const r of existing) {
|
||||
for (const n of nodes) {
|
||||
if (r.id !== n.id && r.network === n.network && slugOf(r.name) === slugOf(nameOf.get(n.id))) {
|
||||
console.error(`aborting: seed node ${n.id} clashes with store record ${r.id} on name "${r.name}"`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const now = new Date().toISOString();
|
||||
const byId = new Map(existing.map((r) => [r.id, r]));
|
||||
const plan = nodes.map((n) => {
|
||||
if (byId.has(n.id)) return { action: "skip", why: "already in store (left untouched)", node: byId.get(n.id) };
|
||||
const codes = n.paynym ? mapping[n.paynym].codes.map((c) => c.code) : [];
|
||||
// Two things make a node unmigratable, and both are the store's rules
|
||||
// rather than this script's: no payment code means no owner, and no signed
|
||||
// pairing block means nothing a visitor can check. Refusing here rather
|
||||
// than letting putSubmission throw is what turns a stack trace part-way
|
||||
// through a migration into a plan you can read before anything is written.
|
||||
const node = toRecord(n, nameOf.get(n.id), codes, now);
|
||||
if (!codes.length) return { action: "refuse", why: "no BIP47 payment code", node };
|
||||
if (!hasSignedBlock(node)) return { action: "refuse", why: "no signed pairing block", node };
|
||||
return { action: "create", node };
|
||||
});
|
||||
|
||||
console.log(`${DRY ? "DRY RUN — " : ""}migration plan (${nodes.length} seed nodes):`);
|
||||
for (const { action, why, node } of plan) {
|
||||
const owner = node.paynym || "(no PayNym)";
|
||||
console.log(` ${action.padEnd(6)} ${node.id.padEnd(26)} name=${String(node.name).padEnd(18)} ${owner} (${(node.paymentCodes || []).length} codes)${why ? " — " + why : ""}`);
|
||||
if (action === "refuse") {
|
||||
console.log(` REFUSED: ${node.id} ${why}, so it cannot be migrated.`);
|
||||
console.log(` Give it a PayNym in data/paynym-codes.json and a signed pairing block, or drop it from the seed.`);
|
||||
}
|
||||
}
|
||||
|
||||
const changes = plan.filter((p) => p.action === "create");
|
||||
const refused = plan.filter((p) => p.action === "refuse");
|
||||
const tail = refused.length ? ` ${refused.length} refused: ${refused.map((p) => p.node.id).join(", ")}.` : "";
|
||||
if (DRY) { console.log(`\ndry run: ${changes.length} change(s) would be made, nothing written.${tail}`); return; }
|
||||
if (!changes.length) { console.log(`\nnothing to do: every seed node already has a store record.${tail}`); return; }
|
||||
for (const { node } of changes) await store.putSubmission(node);
|
||||
console.log(`\napplied ${changes.length} change(s).${tail} Now run: node server/build-public.mjs`);
|
||||
console.log("Once the store records exist, slim data/seed.json to the anchor (your own node) in a separate commit.");
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1] || "").href) {
|
||||
main().catch((e) => { console.error("fatal:", e.message); process.exit(1); });
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
#!/usr/bin/env node
|
||||
// Pack this instance's own codebase into data/dojobay-src.zip, so the running
|
||||
// site is its own distribution point: visitors download exactly the code the
|
||||
// instance runs (the footer's source icon), with no reliance on GitHub being
|
||||
// reachable. Node builtins only -- the ZIP container is written by hand
|
||||
// (deflate entries via zlib + a central directory), because a bare box has no
|
||||
// `zip` binary and scripts/ must run everywhere.
|
||||
//
|
||||
// node scripts/pack-source.mjs write data/dojobay-src.zip
|
||||
//
|
||||
// What goes in is manifest-driven, and what stays out matters more than what
|
||||
// goes in: NEVER the submission store (Dojo API keys, sessions), never the
|
||||
// instance's generated data (dojos.json, history, avatars), and never its
|
||||
// identity (seed.json anchor, operator.json binding, paynym-codes.json), so
|
||||
// extracting the zip over an existing web root upgrades the CODE and touches
|
||||
// nothing the instance owns. data/version.json IS included: it states which
|
||||
// commit the code is, which is exactly what a downloader wants to know.
|
||||
import { readFile, writeFile, rename, readdir, stat, mkdir } from "node:fs/promises";
|
||||
import { deflateRawSync } from "node:zlib";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
|
||||
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const PREFIX = "dojobay/"; // extraction lands in one folder
|
||||
|
||||
const INCLUDE_FILES = [
|
||||
"index.html", "manifest.json", "sw.js", "favicon.svg", "og-image.png",
|
||||
// LICENSE travels with THIRD-PARTY-NOTICES.md: the archive is a distributed
|
||||
// copy of the source, and the README it contains links to the notices.
|
||||
// SECURITY.md travels for the same reason: a recipient who finds a
|
||||
// vulnerability in this copy needs to be told where to send it.
|
||||
"LICENSE", "THIRD-PARTY-NOTICES.md", "README.md", "CONTRIBUTING.md", "SECURITY.md", "package.json",
|
||||
"tsconfig.json", "types.d.ts",
|
||||
"install.sh", "uninstall.sh",
|
||||
"data/version.json",
|
||||
];
|
||||
// docs/ holds the reasoning: why things are shaped as they are and what was
|
||||
// tried and rejected. It is the most useful thing in the tree to anyone
|
||||
// changing the code, and this archive is how a peer instance receives the code.
|
||||
const INCLUDE_DIRS = ["assets", "content", "deploy", "docs", "scripts", "server", ".github"];
|
||||
const DENY = [
|
||||
"server/data", "server/node_modules", "node_modules", ".git",
|
||||
"data/dojos.json", "data/history.json", "data/history-daily.json",
|
||||
"data/avatars", "data/seed.json", "data/operator.json", "data/paynym-codes.json",
|
||||
"data/updates", "data/backups",
|
||||
];
|
||||
const denied = (rel) => DENY.some((d) => rel === d || rel.startsWith(d + "/"))
|
||||
|| rel.endsWith(".zip") || path.basename(rel) === ".DS_Store";
|
||||
|
||||
async function collect(root) {
|
||||
const out = [];
|
||||
for (const f of INCLUDE_FILES) {
|
||||
try { await stat(path.join(root, f)); out.push(f); } catch { /* absent on this instance */ }
|
||||
}
|
||||
async function walk(rel) {
|
||||
for (const e of await readdir(path.join(root, rel), { withFileTypes: true })) {
|
||||
const r = rel + "/" + e.name;
|
||||
if (denied(r)) continue;
|
||||
if (e.isDirectory()) await walk(r);
|
||||
else if (e.isFile()) out.push(r);
|
||||
}
|
||||
}
|
||||
for (const d of INCLUDE_DIRS) {
|
||||
try { await stat(path.join(root, d)); await walk(d); } catch { /* absent */ }
|
||||
}
|
||||
return out.sort();
|
||||
}
|
||||
|
||||
// ---- minimal ZIP writer (PKZIP appnote: local headers + central directory) --
|
||||
const CRC_TABLE = (() => {
|
||||
const t = new Uint32Array(256);
|
||||
for (let n = 0; n < 256; n++) {
|
||||
let c = n;
|
||||
for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
|
||||
t[n] = c >>> 0;
|
||||
}
|
||||
return t;
|
||||
})();
|
||||
const crc32 = (buf) => {
|
||||
let c = 0xffffffff;
|
||||
for (let i = 0; i < buf.length; i++) c = CRC_TABLE[(c ^ buf[i]) & 0xff] ^ (c >>> 8);
|
||||
return (c ^ 0xffffffff) >>> 0;
|
||||
};
|
||||
const dosTime = (d) => (((d.getHours() << 11) | (d.getMinutes() << 5) | (d.getSeconds() >> 1)) & 0xffff);
|
||||
const dosDate = (d) => ((((d.getFullYear() - 1980) << 9) | ((d.getMonth() + 1) << 5) | d.getDate()) & 0xffff);
|
||||
const u16 = (n) => { const b = Buffer.alloc(2); b.writeUInt16LE(n & 0xffff); return b; };
|
||||
const u32 = (n) => { const b = Buffer.alloc(4); b.writeUInt32LE(n >>> 0); return b; };
|
||||
|
||||
function buildZip(entries) { // entries: [{name, data, mtime, mode}]
|
||||
const locals = [], centrals = [];
|
||||
let offset = 0;
|
||||
for (const { name, data, mtime, mode = 0o644 } of entries) {
|
||||
const nameBuf = Buffer.from(name, "utf8");
|
||||
const deflated = deflateRawSync(data, { level: 9 });
|
||||
const stored = deflated.length < data.length;
|
||||
const body = stored ? deflated : data;
|
||||
const method = stored ? 8 : 0;
|
||||
const crc = crc32(data);
|
||||
const t = u16(dosTime(mtime)), dt = u16(dosDate(mtime));
|
||||
const common = Buffer.concat([
|
||||
u16(20), u16(0x0800 /* UTF-8 names */), u16(method), t, dt,
|
||||
u32(crc), u32(body.length), u32(data.length), u16(nameBuf.length), u16(0),
|
||||
]);
|
||||
locals.push(Buffer.concat([u32(0x04034b50), common, nameBuf, body]));
|
||||
centrals.push(Buffer.concat([
|
||||
u32(0x02014b50), u16((3 << 8) | 20 /* unix */), common, u16(0), u16(0), u16(0),
|
||||
u32(((0o100000 | mode) >>> 0) * 0x10000) /* unix mode in high word */, u32(offset), nameBuf,
|
||||
]));
|
||||
offset += locals[locals.length - 1].length;
|
||||
}
|
||||
const cd = Buffer.concat(centrals);
|
||||
const end = Buffer.concat([
|
||||
u32(0x06054b50), u16(0), u16(0), u16(entries.length), u16(entries.length),
|
||||
u32(cd.length), u32(offset), u16(0),
|
||||
]);
|
||||
return Buffer.concat([...locals, cd, end]);
|
||||
}
|
||||
|
||||
export async function packSource({ root = ROOT, outDir = path.join(ROOT, "data") } = {}) {
|
||||
const files = await collect(root);
|
||||
const entries = [];
|
||||
for (const rel of files) {
|
||||
const p = path.join(root, rel);
|
||||
const [data, st] = [await readFile(p), await stat(p)];
|
||||
entries.push({ name: PREFIX + rel, data, mtime: st.mtime, mode: st.mode & 0o777 });
|
||||
}
|
||||
const zip = buildZip(entries);
|
||||
await mkdir(outDir, { recursive: true });
|
||||
const out = path.join(outDir, "dojobay-src.zip");
|
||||
await writeFile(out + ".tmp", zip);
|
||||
await rename(out + ".tmp", out);
|
||||
return { out, files: files.length, bytes: zip.length };
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1] || "").href) {
|
||||
packSource().then((r) => console.log(`wrote ${r.out}: ${r.files} files, ${(r.bytes / 1024).toFixed(0)} KiB`))
|
||||
.catch((e) => { console.error("fatal:", e.message); process.exit(1); });
|
||||
}
|
||||
@@ -0,0 +1,806 @@
|
||||
#!/usr/bin/env node
|
||||
// =============================================================================
|
||||
// The Dojo Bay — directory updater
|
||||
//
|
||||
// Probes every node's .onion pairing endpoint over Tor and rewrites the two
|
||||
// JSON databases the website reads:
|
||||
//
|
||||
// data/dojos.json current snapshot -> node.status + node.checked_at
|
||||
// data/history.json rolling history -> one {t, up} per node, per run
|
||||
//
|
||||
// dojos.json is also the source of truth for the node LIST. To add or remove a
|
||||
// node, edit dojos.json (name, paynym, payload, etc.); this script only fills
|
||||
// in status/checked_at and appends to the history. New nodes get a fresh
|
||||
// history series automatically; removed nodes are retired under a grace stamp
|
||||
// and only pruned HISTORY_GRACE_DAYS (default 14) after leaving the list.
|
||||
//
|
||||
// Health is checked through Tor's SOCKS5 proxy (no external npm deps). For a
|
||||
// node whose pairing payload carries an apikey, the check logs in to the Dojo
|
||||
// API and reads info.latest_block.height from GET /v2/wallet: the node is
|
||||
// "active" only if it returns a chain tip, which proves the whole stack (Tor,
|
||||
// nginx, Dojo API, bitcoind) is serving block data, and the height is recorded
|
||||
// on the node. Nodes without an apikey fall back to a plain HTTP reachability
|
||||
// probe (active if the onion returns an HTTP response line).
|
||||
//
|
||||
// Every Dojo response carries its running version in the X-Dojo-Version header;
|
||||
// the probe reads it and records node.detected_version, so a card can show the
|
||||
// live version rather than the one frozen into the pairing payload at signing
|
||||
// time. build-public.mjs decides the effective version an operator override
|
||||
// still wins over it.
|
||||
//
|
||||
// Run once (intended to be driven by cron/systemd every 10 minutes):
|
||||
// node scripts/update.mjs
|
||||
//
|
||||
// Config via environment variables (all optional):
|
||||
// TOR_SOCKS_HOST default 127.0.0.1
|
||||
// TOR_SOCKS_PORT default 9050
|
||||
// DATA_DIR default <repo>/data
|
||||
// TIMEOUT_MS default 45000 per-node Tor timeout
|
||||
// CONCURRENCY default 3 simultaneous Tor circuits
|
||||
// WINDOW_CHECKS default 144 history length kept per node (24h @ 10min)
|
||||
// RETENTION_DAYS default 90 daily-rollup days kept per node (~3 months)
|
||||
// CONNECT_ONLY default 0 "1" = treat a successful Tor connect as up
|
||||
// without waiting for an HTTP response line
|
||||
// DOJO_VERSION_HEADER default X-Dojo-Version response header carrying the
|
||||
// node's running Dojo version
|
||||
// =============================================================================
|
||||
|
||||
import net from "node:net";
|
||||
import { retireUnlisted } from "../server/build-public.ts";
|
||||
import { readFile, writeFile, rename, stat as fsStat, mkdir as fsMkdir } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
// Chosen for a home connection as much as a VPS, because the unit that would
|
||||
// override them lives in /etc and no update can reach it. A node answering at
|
||||
// 23 seconds was being recorded as down against a 30 second ceiling, and six
|
||||
// circuits at once through one Tor client on a domestic line makes every probe
|
||||
// slow together, which reads as every node being down.
|
||||
export const DEFAULT_TIMEOUT_MS = 45000;
|
||||
export const DEFAULT_CONCURRENCY = 3;
|
||||
|
||||
const CFG = {
|
||||
proxyHost: process.env.TOR_SOCKS_HOST || "127.0.0.1",
|
||||
proxyPort: +(process.env.TOR_SOCKS_PORT || 9050),
|
||||
dataDir: process.env.DATA_DIR || path.resolve(__dirname, "..", "data"),
|
||||
timeoutMs: +(process.env.TIMEOUT_MS || DEFAULT_TIMEOUT_MS),
|
||||
concurrency: +(process.env.CONCURRENCY || DEFAULT_CONCURRENCY),
|
||||
windowChecks: +(process.env.WINDOW_CHECKS || 144),
|
||||
retentionDays: +(process.env.RETENTION_DAYS || 90),
|
||||
connectOnly: process.env.CONNECT_ONLY === "1",
|
||||
// The Dojo API stamps its running version on every response via this header
|
||||
// (Dojo's http-server appends X-Dojo-Version: <DOJO_VERSION_TAG> as global
|
||||
// middleware). Read it during the probe so a node's displayed version tracks
|
||||
// what it is actually running, instead of the value frozen into its pairing
|
||||
// payload at submission time. Overridable in case a fork renames the header.
|
||||
dojoVersionHeader: (process.env.DOJO_VERSION_HEADER || "X-Dojo-Version").toLowerCase(),
|
||||
};
|
||||
|
||||
// ---- SOCKS5 reply codes (RFC 1928 §6) ---------------------------------------
|
||||
const SOCKS_ERR = {
|
||||
0x01: "general failure",
|
||||
0x02: "connection not allowed",
|
||||
0x03: "network unreachable",
|
||||
0x04: "host unreachable", // Tor: onion descriptor not found / service down
|
||||
0x05: "connection refused",
|
||||
0x06: "TTL expired",
|
||||
0x07: "command not supported",
|
||||
0x08: "address type not supported",
|
||||
};
|
||||
|
||||
class SocksError extends Error {
|
||||
constructor(code) {
|
||||
super("SOCKS " + (SOCKS_ERR[code] || "error 0x" + code.toString(16)));
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// Open a TCP stream to host:port THROUGH a SOCKS5 proxy (Tor), using a remote
|
||||
// hostname so the .onion is resolved by Tor, not locally. Resolves with a
|
||||
// connected socket on success; rejects on any handshake/connect failure.
|
||||
// -----------------------------------------------------------------------------
|
||||
export function socks5Connect(proxyHost, proxyPort, host, port, timeoutMs) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const socket = net.connect(proxyPort, proxyHost);
|
||||
let stage = "greet";
|
||||
let buf = Buffer.alloc(0);
|
||||
let settled = false;
|
||||
|
||||
const fail = (e) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
socket.destroy();
|
||||
reject(e instanceof Error ? e : new Error(String(e)));
|
||||
};
|
||||
const timer = setTimeout(() => fail(new Error("timeout")), timeoutMs);
|
||||
|
||||
socket.once("connect", () => {
|
||||
// greeting: VER=5, NMETHODS=1, METHOD=0 (no auth)
|
||||
socket.write(Buffer.from([0x05, 0x01, 0x00]));
|
||||
});
|
||||
socket.on("error", fail);
|
||||
socket.on("close", () => fail(new Error("proxy closed")));
|
||||
|
||||
socket.on("data", (d) => {
|
||||
buf = Buffer.concat([buf, d]);
|
||||
|
||||
if (stage === "greet") {
|
||||
if (buf.length < 2) return;
|
||||
if (buf[0] !== 0x05 || buf[1] !== 0x00) return fail(new Error("proxy refused no-auth handshake"));
|
||||
buf = buf.subarray(2);
|
||||
stage = "reply";
|
||||
// CONNECT request with ATYP=3 (domain name), so Tor resolves the onion
|
||||
const hb = Buffer.from(host, "utf8");
|
||||
socket.write(Buffer.concat([
|
||||
Buffer.from([0x05, 0x01, 0x00, 0x03, hb.length]),
|
||||
hb,
|
||||
Buffer.from([(port >> 8) & 0xff, port & 0xff]),
|
||||
]));
|
||||
}
|
||||
|
||||
if (stage === "reply") {
|
||||
if (buf.length < 4) return;
|
||||
if (buf[1] !== 0x00) return fail(new SocksError(buf[1]));
|
||||
const atyp = buf[3];
|
||||
const addrLen =
|
||||
atyp === 0x01 ? 4 :
|
||||
atyp === 0x04 ? 16 :
|
||||
atyp === 0x03 ? (buf.length >= 5 ? 1 + buf[4] : Infinity) : 0;
|
||||
if (buf.length < 4 + addrLen + 2) return; // wait for the full bound-addr
|
||||
// success: hand the live stream back to the caller
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
socket.removeAllListeners("data");
|
||||
socket.removeAllListeners("error");
|
||||
socket.removeAllListeners("close");
|
||||
resolve(socket);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// Well-formed dummy extended keys, used only to elicit info.latest_block from
|
||||
// the Dojo /wallet endpoint. They are passed as `new` so the node performs no
|
||||
// rescan or historical import; they derive from a throwaway seed and can never
|
||||
// receive funds. One per network so the Dojo never rejects them on format.
|
||||
const DUMMY_XPUB = "xpub661MyMwAqRbcFhv1kNXxwyGrJUVPrmiBNTVDYAtpzF5zu9ceuhn5yV6oaSdveis14LSeBLzpWb58pDNN6hC59TTDyiN7iJR7kUQgXNMfZCL";
|
||||
const DUMMY_TPUB = "tpubD6NzVbkrYhZ4XW6sCZX49tcDdbb3rADEv65WtiwyL9qteSHMyvdB7vmdpUiiBDpErEyYnvWh3guBWPryVZ3K2tuX3K7RPq5MLS16HN9awey";
|
||||
|
||||
// The most bytes a response may accumulate before the read is abandoned.
|
||||
//
|
||||
// Every caller of httpOverTor is talking to a machine somebody else controls:
|
||||
// that is the point of the probe. Without a ceiling the reader accumulates
|
||||
// whatever arrives until the socket closes or the timeout fires, so a listed
|
||||
// node that simply never stops sending can push thirty seconds of Tor
|
||||
// throughput into the heap, times CONCURRENCY parallel probes, on a VPS whose
|
||||
// documented minimum is 1 GB. Nothing about that requires malice: a Dojo
|
||||
// misconfigured to return a file rather than JSON does it by accident.
|
||||
//
|
||||
// 2 MiB is chosen against the largest legitimate response any probe path sees,
|
||||
// which is a Dojo /wallet reply for two dummy xpubs, single-digit kilobytes.
|
||||
// A PayNym avatar is a small PNG and sits under the same ceiling comfortably;
|
||||
// it does not get a tighter limit of its own, because a second constant would
|
||||
// have to be kept in a sensible relationship with this one, and 2 MiB already
|
||||
// bounds the disk that syncAvatars can consume to a few tens of megabytes
|
||||
// across every listed code. The one caller that legitimately needs more is
|
||||
// self-update fetching a peer's source zip, and it passes its own value.
|
||||
export const MAX_RESPONSE_BYTES = 2 * 1024 * 1024;
|
||||
|
||||
// The unauthenticated probe reads only until it recognises an HTTP status line,
|
||||
// so it needs a far smaller ceiling than a full response: this bounds how long
|
||||
// it will listen to something that is not speaking HTTP at all.
|
||||
export const MAX_STATUS_LINE_BYTES = 64 * 1024;
|
||||
|
||||
// Send one HTTP/1.0 request over a fresh Tor stream and read the whole reply
|
||||
// (Connection: close means the server ends the body by closing). Resolves with
|
||||
// { status, body } or rejects on connect failure, read timeout, or a reply that
|
||||
// runs past maxBytes.
|
||||
export function httpOverTor(cfg, host, port, rawRequest, timeoutMs, maxBytes = MAX_RESPONSE_BYTES) {
|
||||
return new Promise(async (resolve, reject) => {
|
||||
let socket;
|
||||
try {
|
||||
socket = await socks5Connect(cfg.proxyHost, cfg.proxyPort, host, port, timeoutMs);
|
||||
} catch (e) { return reject(e); }
|
||||
let buf = Buffer.alloc(0);
|
||||
let settled = false;
|
||||
const done = (fn, v) => { if (settled) return; settled = true; clearTimeout(timer); try { socket.destroy(); } catch {} fn(v); };
|
||||
const timer = setTimeout(() => done(reject, new Error("read-timeout")), timeoutMs);
|
||||
socket.on("data", (d) => {
|
||||
buf = Buffer.concat([buf, d]);
|
||||
// Rejected the moment the ceiling is crossed rather than at close, so the
|
||||
// socket is destroyed and the memory released now. Waiting would mean a
|
||||
// node that never closes still occupies the full timeout while holding
|
||||
// everything it has sent. done() destroys the socket, so no further data
|
||||
// events arrive and the partial buffer goes out of scope with this call.
|
||||
if (buf.length > maxBytes) {
|
||||
done(reject, new Error(`response exceeded ${maxBytes} bytes`));
|
||||
}
|
||||
});
|
||||
socket.on("error", (e) => done(reject, e));
|
||||
socket.on("close", () => {
|
||||
const s = buf.toString("latin1");
|
||||
const m = s.match(/^HTTP\/1\.[01] (\d{3})/);
|
||||
const i = s.indexOf("\r\n\r\n");
|
||||
done(resolve, {
|
||||
status: m ? +m[1] : 0,
|
||||
body: i >= 0 ? s.slice(i + 4) : "",
|
||||
rawHead: i >= 0 ? s.slice(0, i + 2) : s, // headers incl. trailing CRLF
|
||||
bodyBuf: i >= 0 ? buf.subarray(i + 4) : Buffer.alloc(0), // exact bytes for binary payloads
|
||||
});
|
||||
});
|
||||
socket.write(rawRequest);
|
||||
});
|
||||
}
|
||||
|
||||
// ---- Dojo version from response headers -------------------------------------
|
||||
// The Dojo API sets its running version on every response (X-Dojo-Version). We
|
||||
// read it opportunistically while probing so the card can show the live value.
|
||||
// A node is only semi-trusted, so the value is validated and length-capped
|
||||
// before it can reach a data file: a version looks like 1, 1.28, 1.28.0 or
|
||||
// 1.28.0-rc1, with an optional leading v that we strip. Anything else -> null.
|
||||
export function normaliseVersion(raw) {
|
||||
if (typeof raw !== "string") return null;
|
||||
const v = raw.trim().replace(/^v/i, "").trim();
|
||||
if (!v || v.length > 32) return null;
|
||||
return /^\d+(\.\d+){0,3}([-+][0-9A-Za-z.]+)?$/.test(v) ? v : null;
|
||||
}
|
||||
|
||||
// Pull the version out of a raw header block (the CRLF-joined header lines from
|
||||
// httpOverTor's rawHead, or the accumulated first bytes of a plain probe).
|
||||
// Header names are case-insensitive; the first occurrence wins.
|
||||
export function parseDojoVersion(rawHead, headerName = CFG.dojoVersionHeader) {
|
||||
if (typeof rawHead !== "string" || !rawHead) return null;
|
||||
const name = String(headerName).toLowerCase();
|
||||
for (const line of rawHead.split(/\r?\n/)) {
|
||||
const idx = line.indexOf(":");
|
||||
if (idx < 0) continue;
|
||||
if (line.slice(0, idx).trim().toLowerCase() !== name) continue;
|
||||
return normaliseVersion(line.slice(idx + 1));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ---- Electrum (indexer) endpoint from /support/services ---------------------
|
||||
// Dojo v1.27.0 added GET /support/services (ordinary apikey auth, not admin),
|
||||
// which returns { services: [ { type, kind, url }, … ] }. The "indexer" entry
|
||||
// is the node's Electrum server, published by the Dojo as
|
||||
// "<tcp|ssl>://<onion>:<port>" and present only when the operator exposes a
|
||||
// local indexer. Older Dojos have no such route, so absence is normal and is
|
||||
// reported as "not found" rather than an error.
|
||||
export function parseIndexerUrl(body) {
|
||||
let doc;
|
||||
try { doc = JSON.parse(body); } catch { return null; }
|
||||
const list = Array.isArray(doc?.services) ? doc.services : null;
|
||||
if (!list) return null;
|
||||
const hit = list.find((s) => s && s.type === "indexer" && typeof s.url === "string");
|
||||
return hit ? normaliseIndexerUrl(hit.url) : null;
|
||||
}
|
||||
|
||||
// A listed node is only semi-trusted, so the URL is validated and length-capped
|
||||
// before it can reach a data file or be rendered as a copyable string. Same
|
||||
// shape the card already accepts: tcp/ssl, v3 onion, explicit port.
|
||||
export function normaliseIndexerUrl(raw) {
|
||||
if (typeof raw !== "string") return null;
|
||||
const u = raw.trim();
|
||||
if (!u || u.length > 120) return null;
|
||||
return /^(tcp|ssl):\/\/[a-z2-7]{56}\.onion:\d{2,5}$/i.test(u) ? u : null;
|
||||
}
|
||||
|
||||
// ---- PayNym avatars ---------------------------------------------------------
|
||||
// Cards embed each node's PayNym avatar in the centre of its pairing QR. The
|
||||
// front end never fetches from third parties, so the avatar is mirrored here:
|
||||
// downloaded over Tor from the paynym.rs onion and served locally from
|
||||
// data/avatars/<paymentCode>.png. Missing files are fetched every cycle (which
|
||||
// also covers newly approved nodes within ten minutes) and existing ones are
|
||||
// refreshed weekly. Only verified PNG bytes are written; anything else -- an
|
||||
// error page, a redirect chain, an empty body -- is skipped without touching
|
||||
// the file, and failures are logged, never fatal.
|
||||
const PAYNYM_ONION = process.env.PAYNYM_ONION_HOST || "paynym25chftmsywv4v2r67agbrr62lcxagsf4tymbzpeeucucy2ivad.onion";
|
||||
const AVATAR_MAX_AGE_MS = 7 * 86400000;
|
||||
const PNG_MAGIC = Buffer.from([0x89, 0x50, 0x4e, 0x47]);
|
||||
|
||||
/**
|
||||
* @param {string} paymentCode
|
||||
* @param {{ proxyHost?: string, proxyPort?: number, destDir?: string,
|
||||
* timeoutMs?: number, host?: string, port?: number }} [opts]
|
||||
*/
|
||||
export async function fetchAvatar(paymentCode, { proxyHost, proxyPort, destDir, timeoutMs = 25000, host = PAYNYM_ONION, port = 80 } = {}) {
|
||||
const cfg = { proxyHost, proxyPort };
|
||||
let pathPart = `/${encodeURIComponent(paymentCode)}/avatar`;
|
||||
for (let hop = 0; hop < 2; hop++) { // follow at most one same-host redirect
|
||||
const req = `GET ${pathPart} HTTP/1.0\r\nHost: ${host}\r\nUser-Agent: dojobay-checker\r\nConnection: close\r\n\r\n`;
|
||||
const res = await httpOverTor(cfg, host, port, req, timeoutMs);
|
||||
if ([301, 302, 307, 308].includes(res.status)) {
|
||||
const m = res.rawHead && res.rawHead.match(/\r\nlocation:\s*([^\r\n]+)/i);
|
||||
if (!m) throw new Error("redirect without location");
|
||||
const loc = m[1].trim();
|
||||
if (/^https?:\/\//i.test(loc)) {
|
||||
const u = new URL(loc);
|
||||
if (u.hostname !== host) throw new Error("cross-host redirect");
|
||||
pathPart = u.pathname + u.search;
|
||||
} else pathPart = loc;
|
||||
continue;
|
||||
}
|
||||
if (res.status !== 200) throw new Error(`HTTP ${res.status || "no-response"}`);
|
||||
const bytes = res.bodyBuf || Buffer.from(res.body, "latin1");
|
||||
if (bytes.length < 8 || !bytes.subarray(0, 4).equals(PNG_MAGIC)) throw new Error("not a PNG");
|
||||
await fsMkdir(destDir, { recursive: true });
|
||||
const dest = path.join(destDir, `${paymentCode}.png`);
|
||||
const atmp = tmpName(dest);
|
||||
await writeFile(atmp, bytes);
|
||||
await rename(atmp, dest);
|
||||
return dest;
|
||||
}
|
||||
throw new Error("too many redirects");
|
||||
}
|
||||
|
||||
// Ensure a local avatar exists (and is reasonably fresh) for every listed
|
||||
// payment code. Small concurrency; per-code failures are logged and skipped.
|
||||
async function syncAvatars(nodes, destDir) {
|
||||
const codes = [...new Set(nodes.map((n) => n.paymentCode).filter(Boolean))];
|
||||
const wanted = [];
|
||||
for (const code of codes) {
|
||||
try {
|
||||
const st = await fsStat(path.join(destDir, `${code}.png`));
|
||||
if (Date.now() - st.mtimeMs < AVATAR_MAX_AGE_MS) continue;
|
||||
} catch { /* missing -> fetch */ }
|
||||
wanted.push(code);
|
||||
}
|
||||
let i = 0;
|
||||
const worker = async () => {
|
||||
for (;;) {
|
||||
const code = wanted[i++];
|
||||
if (!code) return;
|
||||
try {
|
||||
await fetchAvatar(code, { proxyHost: CFG.proxyHost, proxyPort: CFG.proxyPort, destDir });
|
||||
console.error(`[avatar] fetched ${code.slice(0, 12)}…`);
|
||||
} catch (e) {
|
||||
console.error(`[avatar] ${code.slice(0, 12)}…: ${e.message}`);
|
||||
}
|
||||
}
|
||||
};
|
||||
await Promise.all(Array.from({ length: Math.min(3, wanted.length) }, worker));
|
||||
}
|
||||
|
||||
// Authenticated health check: log in with the node's apikey, then read the
|
||||
// chain tip from GET /v2/wallet. The Dojo stamps X-Dojo-Version on every
|
||||
// response, so we harvest it from the first response that carries it (the login
|
||||
// reply always does) even on an otherwise-down cycle. Returns
|
||||
// { up, reason, ms, height?, blockTime?, detectedVersion? }.
|
||||
async function probeHeight(url, cfg) {
|
||||
const t0 = Date.now();
|
||||
const u = new URL(url);
|
||||
const host = u.hostname;
|
||||
const port = u.port ? +u.port : 80;
|
||||
const base = (u.pathname || "/v2").replace(/\/+$/, "") || "/v2"; // e.g. /v2
|
||||
const dummy = cfg.network === "testnet" ? DUMMY_TPUB : DUMMY_XPUB;
|
||||
let detectedVersion = null;
|
||||
|
||||
// 1) login -> access token
|
||||
let token;
|
||||
try {
|
||||
const body = `apikey=${encodeURIComponent(cfg.apikey)}`;
|
||||
const req =
|
||||
`POST ${base}/auth/login HTTP/1.0\r\nHost: ${host}\r\n` +
|
||||
`Content-Type: application/x-www-form-urlencoded\r\nContent-Length: ${Buffer.byteLength(body)}\r\n` +
|
||||
`User-Agent: dojobay-checker\r\nConnection: close\r\n\r\n${body}`;
|
||||
const res = await httpOverTor(cfg, host, port, req, cfg.timeoutMs);
|
||||
detectedVersion = parseDojoVersion(res.rawHead, cfg.dojoVersionHeader) || detectedVersion;
|
||||
if (res.status !== 200) return { up: false, reason: `login HTTP ${res.status || "no-response"}`, ms: Date.now() - t0, detectedVersion };
|
||||
token = JSON.parse(res.body)?.authorizations?.access_token;
|
||||
if (!token) return { up: false, reason: "login: no token", ms: Date.now() - t0, detectedVersion };
|
||||
} catch (e) {
|
||||
return { up: false, reason: "login: " + e.message, ms: Date.now() - t0, detectedVersion };
|
||||
}
|
||||
|
||||
// 2) wallet -> info.latest_block.height
|
||||
try {
|
||||
const q = `active=${dummy}&new=${dummy}`;
|
||||
const req =
|
||||
`GET ${base}/wallet?${q} HTTP/1.0\r\nHost: ${host}\r\n` +
|
||||
`Authorization: Bearer ${token}\r\nUser-Agent: dojobay-checker\r\nConnection: close\r\n\r\n`;
|
||||
const res = await httpOverTor(cfg, host, port, req, cfg.timeoutMs);
|
||||
detectedVersion = detectedVersion || parseDojoVersion(res.rawHead, cfg.dojoVersionHeader);
|
||||
if (res.status !== 200) return { up: false, reason: `wallet HTTP ${res.status || "no-response"}`, ms: Date.now() - t0, detectedVersion };
|
||||
const info = JSON.parse(res.body)?.info?.latest_block;
|
||||
const height = info?.height;
|
||||
if (typeof height !== "number") return { up: false, reason: "wallet: no block height", ms: Date.now() - t0, detectedVersion };
|
||||
|
||||
// 3) services -> Electrum (indexer) endpoint. Best-effort and strictly
|
||||
// additive: the node is already known up, so a missing route (pre-1.27.0),
|
||||
// a node that exposes no indexer, or any error here must never downgrade
|
||||
// the result. Absence simply means the card shows N/A.
|
||||
let detectedIndexer = null;
|
||||
try {
|
||||
const sreq =
|
||||
`GET ${base}/support/services HTTP/1.0\r\nHost: ${host}\r\n` +
|
||||
`Authorization: Bearer ${token}\r\nUser-Agent: dojobay-checker\r\nConnection: close\r\n\r\n`;
|
||||
const sres = await httpOverTor(cfg, host, port, sreq, cfg.timeoutMs);
|
||||
detectedVersion = detectedVersion || parseDojoVersion(sres.rawHead, cfg.dojoVersionHeader);
|
||||
if (sres.status === 200) detectedIndexer = parseIndexerUrl(sres.body);
|
||||
} catch { /* leave null */ }
|
||||
|
||||
return { up: true, reason: "height", height, blockTime: info.time ?? null, ms: Date.now() - t0, detectedVersion, detectedIndexer };
|
||||
} catch (e) {
|
||||
return { up: false, reason: "wallet: " + e.message, ms: Date.now() - t0, detectedVersion };
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// Probe a single onion URL. Returns { up, reason, ms }.
|
||||
// up = Tor connected AND (CONNECT_ONLY, or an HTTP status line came back)
|
||||
// -----------------------------------------------------------------------------
|
||||
// Fill in the transport settings a probe cannot work without. Callers pass a
|
||||
// partial config (an apikey and a network, say) and it is easy to forget to
|
||||
// spread PROBE_CFG or CFG alongside it; without these, net.connect is handed an
|
||||
// undefined port and Node reports 'The "options" or "port" or "path" argument
|
||||
// must be specified', which says nothing about the real mistake. The defaults
|
||||
// are the same ones PROBE_CFG uses, so a partial config now behaves rather than
|
||||
// failing obscurely. Explicitly supplied values always win.
|
||||
/**
|
||||
* @param {Partial<import("../types.js").ProbeCfg>} [cfg]
|
||||
* @returns {import("../types.js").ProbeCfg}
|
||||
*/
|
||||
export function probeCfg(cfg = {}) {
|
||||
return {
|
||||
...cfg,
|
||||
proxyHost: cfg.proxyHost ?? (process.env.TOR_SOCKS_HOST || "127.0.0.1"),
|
||||
proxyPort: cfg.proxyPort ?? +(process.env.TOR_SOCKS_PORT || 9050),
|
||||
// Same default as CFG below, from one place. These were separate literals
|
||||
// and had already diverged: the cron path waited 45 seconds while anything
|
||||
// going through this helper waited 30, so the same node could be up for one
|
||||
// caller and down for the other.
|
||||
timeoutMs: cfg.timeoutMs ?? +(process.env.TIMEOUT_MS || DEFAULT_TIMEOUT_MS),
|
||||
concurrency: cfg.concurrency ?? +(process.env.CONCURRENCY || DEFAULT_CONCURRENCY),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} url
|
||||
* @param {Partial<import("../types.js").ProbeCfg>} [cfgIn]
|
||||
* @returns {Promise<import("../types.js").ProbeResult>}
|
||||
*/
|
||||
export async function probe(url, cfgIn = CFG) {
|
||||
const cfg = probeCfg(cfgIn);
|
||||
// Preferred path: authenticated chain-tip check when an apikey is available.
|
||||
if (cfg.apikey) return probeHeight(url, cfg);
|
||||
const u = new URL(url);
|
||||
const host = u.hostname;
|
||||
const port = u.port ? +u.port : (u.protocol === "https:" ? 443 : 80);
|
||||
const reqPath = (u.pathname || "/") + (u.search || "");
|
||||
const t0 = Date.now();
|
||||
|
||||
let socket;
|
||||
try {
|
||||
socket = await socks5Connect(cfg.proxyHost, cfg.proxyPort, host, port, cfg.timeoutMs);
|
||||
} catch (e) {
|
||||
return { up: false, reason: e.message, ms: Date.now() - t0 };
|
||||
}
|
||||
|
||||
// TLS onions or connect-only mode: a successful Tor stream is the signal.
|
||||
if (cfg.connectOnly || u.protocol === "https:") {
|
||||
socket.destroy();
|
||||
return { up: true, reason: u.protocol === "https:" ? "tls-connect" : "connect", ms: Date.now() - t0 };
|
||||
}
|
||||
|
||||
// Otherwise confirm the Dojo HTTP server actually answers.
|
||||
return await new Promise((resolve) => {
|
||||
let got = "";
|
||||
let settled = false;
|
||||
const finish = (up, reason) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
socket.destroy();
|
||||
// A code-less node has no apikey, so this is the only chance to read its
|
||||
// version; the header rides in the same first packet as the status line
|
||||
// often enough to be worth a look. Absent -> null, harmless.
|
||||
resolve({ up, reason, ms: Date.now() - t0, detectedVersion: parseDojoVersion(got, cfg.dojoVersionHeader) });
|
||||
};
|
||||
const timer = setTimeout(() => finish(got.length > 0, got ? "partial" : "read-timeout"), cfg.timeoutMs);
|
||||
|
||||
socket.on("data", (d) => {
|
||||
got += d.toString("latin1");
|
||||
if (/^HTTP\//i.test(got)) finish(true, "http");
|
||||
// The same unbounded accumulation httpOverTor had, reached by a different
|
||||
// door. A well-behaved server puts its status line in the first packet
|
||||
// and the test above ends the read immediately, but a node that sends
|
||||
// anything NOT starting with "HTTP/" is never matched, so before this
|
||||
// guard `got` grew until the timeout with no ceiling at all. A status
|
||||
// line is a few dozen bytes; 64 KiB without one means this is not an HTTP
|
||||
// server, which is the answer the probe wanted anyway.
|
||||
else if (got.length > MAX_STATUS_LINE_BYTES) finish(false, "no-http-response");
|
||||
});
|
||||
socket.on("error", () => finish(got.length > 0, "socket-error"));
|
||||
socket.on("close", () => finish(got.length > 0, "closed"));
|
||||
|
||||
socket.write(
|
||||
`HEAD ${reqPath} HTTP/1.0\r\nHost: ${host}\r\nUser-Agent: dojobay-checker\r\nConnection: close\r\n\r\n`
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
// ---- date helpers (UTC, matching the formats already in the JSON) -----------
|
||||
const p2 = (n) => String(n).padStart(2, "0");
|
||||
function stamps(d = new Date()) {
|
||||
const Y = d.getUTCFullYear(), M = p2(d.getUTCMonth() + 1), D = p2(d.getUTCDate());
|
||||
const h = p2(d.getUTCHours()), m = p2(d.getUTCMinutes()), s = p2(d.getUTCSeconds());
|
||||
return {
|
||||
isoSec: `${Y}-${M}-${D}T${h}:${m}:${s}Z`, // generated_at
|
||||
isoMin: `${Y}-${M}-${D}T${h}:${m}Z`, // history check timestamp
|
||||
dateTime: `${Y}-${M}-${D} ${h}:${m}:${s}`, // node.checked_at
|
||||
};
|
||||
}
|
||||
|
||||
// ---- small concurrency pool -------------------------------------------------
|
||||
async function pool(items, limit, fn) {
|
||||
const out = new Array(items.length);
|
||||
let i = 0;
|
||||
const worker = async () => {
|
||||
while (i < items.length) {
|
||||
const idx = i++;
|
||||
out[idx] = await fn(items[idx], idx);
|
||||
}
|
||||
};
|
||||
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
|
||||
return out;
|
||||
}
|
||||
|
||||
async function readJSON(file, fallback) {
|
||||
try { return JSON.parse(await readFile(file, "utf8")); }
|
||||
catch (e) { if (e.code === "ENOENT" && fallback !== undefined) return fallback; throw e; }
|
||||
}
|
||||
|
||||
// A temporary name no other writer can take.
|
||||
//
|
||||
// Every atomic write here was `<file>.tmp`, which is not atomic between
|
||||
// processes: two writers produce the same path, the first rename consumes it,
|
||||
// and the second fails with ENOENT on a file it had just written. That is not
|
||||
// hypothetical. The installer enables the update timer and then runs its own
|
||||
// first probe cycle, and once the timer gained a calendar schedule with
|
||||
// Persistent=true, enabling it fired a catch-up run immediately rather than
|
||||
// after two minutes. Two updaters wrote data/dojos.json.tmp at once and the
|
||||
// install ended by announcing failures on a directory that was already
|
||||
// updating.
|
||||
//
|
||||
// The pid and a counter are enough: the collision is between processes on one
|
||||
// machine, and the rename is what makes the swap atomic for readers.
|
||||
function tmpName(file) {
|
||||
return `${file}.${process.pid}.${(tmpSeq = (tmpSeq + 1) % 1e6)}.tmp`;
|
||||
}
|
||||
let tmpSeq = 0;
|
||||
|
||||
// Write atomically: a reader (the website) never sees a half-written file.
|
||||
async function writeJSONAtomic(file, obj) {
|
||||
const tmp = tmpName(file);
|
||||
await writeFile(tmp, JSON.stringify(obj, null, 2) + "\n");
|
||||
await rename(tmp, file);
|
||||
}
|
||||
|
||||
// Merge seed + approved submissions into the public list (delegates to
|
||||
// server/build-public.mjs, which preserves live statuses and histories).
|
||||
// Exported so the self-test can drive it against isolated data directories.
|
||||
export async function reconcilePublicList() {
|
||||
if (!process.env.PUBLIC_DATA_DIR) process.env.PUBLIC_DATA_DIR = CFG.dataDir;
|
||||
const { rebuild } = await import("../server/build-public.ts");
|
||||
return rebuild();
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
async function main() {
|
||||
const dojosPath = path.join(CFG.dataDir, "dojos.json");
|
||||
// Reconcile FIRST: fold the curated seed and every APPROVED submission into
|
||||
// dojos.json before this cycle reads it. The admin approve does its own
|
||||
// rebuild, but that write is lost if it lands while a probe cycle (minutes
|
||||
// long over Tor) is in flight, because the cycle writes back the node list
|
||||
// it read at the start. Rebuilding here means an approved node can be absent
|
||||
// for at most one cycle, never indefinitely.
|
||||
try {
|
||||
const r = await reconcilePublicList();
|
||||
console.error(`[reconcile] ${r.msg}`);
|
||||
} catch (e) {
|
||||
console.error(`[reconcile] skipped: ${e.message}`);
|
||||
}
|
||||
const historyPath = path.join(CFG.dataDir, "history.json");
|
||||
|
||||
const dojos = await readJSON(dojosPath);
|
||||
if (!dojos || !Array.isArray(dojos.nodes)) throw new Error(`bad or missing ${dojosPath}`);
|
||||
// Keep the self-hosted source download current: regenerate the zip when it
|
||||
// is missing or older than data/version.json (i.e. after any code deploy).
|
||||
try {
|
||||
const zipPath = path.join(CFG.dataDir, "dojobay-src.zip");
|
||||
const verPath = path.join(CFG.dataDir, "version.json");
|
||||
const zipSt = await fsStat(zipPath).catch(() => null);
|
||||
const verSt = await fsStat(verPath).catch(() => null);
|
||||
if (!zipSt || (verSt && verSt.mtimeMs > zipSt.mtimeMs)) {
|
||||
const { packSource } = await import("./pack-source.mjs");
|
||||
const r = await packSource({ outDir: CFG.dataDir });
|
||||
console.error(`[src-zip] repacked: ${r.files} files, ${(r.bytes / 1024).toFixed(0)} KiB`);
|
||||
}
|
||||
} catch (e) { console.error(`[src-zip] skipped: ${e.message}`); }
|
||||
|
||||
// Mirror PayNym avatars for every listed code (non-blocking for the probes).
|
||||
const operatorDoc = await readJSON(path.join(CFG.dataDir, "operator.json")).catch(() => null) ?? {};
|
||||
const avatarSubjects = dojos.nodes.concat(operatorDoc.paymentCode ? [{ paymentCode: operatorDoc.paymentCode }] : []);
|
||||
const avatarsDone = syncAvatars(avatarSubjects, path.join(CFG.dataDir, "avatars")).catch((e) => console.error("[avatar]", e.message));
|
||||
const history = await readJSON(historyPath, { interval_minutes: 10, window_checks: CFG.windowChecks, nodes: {} });
|
||||
const window = history.window_checks || CFG.windowChecks;
|
||||
|
||||
const now = new Date();
|
||||
const ts = stamps(now);
|
||||
console.error(`[${ts.isoSec}] probing ${dojos.nodes.length} nodes via socks5h://${CFG.proxyHost}:${CFG.proxyPort} (timeout ${CFG.timeoutMs}ms, concurrency ${CFG.concurrency})`);
|
||||
|
||||
const results = await pool(dojos.nodes, CFG.concurrency, async (n) => {
|
||||
const url = n?.payload?.pairing?.url;
|
||||
if (!url) return { up: false, reason: "no pairing url", ms: 0 };
|
||||
return probe(url, { ...CFG, apikey: n?.payload?.pairing?.apikey, network: n.network });
|
||||
});
|
||||
|
||||
// ---- did this cycle learn anything? ----
|
||||
//
|
||||
// Fifteen independently operated nodes on different continents do not fail in
|
||||
// the same ten-minute window. When every one of them fails, the cause is here:
|
||||
// Tor rebuilding circuits after a suspend, a home connection renegotiating,
|
||||
// the daemon restarted underneath us. Recording that would write a DOWN check
|
||||
// against every operator in the directory and pull down reliability figures
|
||||
// this instance publishes about other people's machines, for a fault of its
|
||||
// own. So it is not recorded.
|
||||
//
|
||||
// The threshold is zero rather than a proportion. A cycle where some nodes
|
||||
// answer proves the local path works, and the ones that did not answer really
|
||||
// did not; only a clean sweep is evidence about this machine instead of about
|
||||
// them. A directory with one listing would trip this on a genuine outage, and
|
||||
// that is the right trade: withholding one node's bad cycle costs far less
|
||||
// than publishing a false one against everybody.
|
||||
const allFailed = dojos.nodes.length > 0 && results.every((r) => !r.up);
|
||||
|
||||
// ---- update current snapshot ----
|
||||
let up = 0;
|
||||
dojos.nodes.forEach((n, i) => {
|
||||
const r = results[i];
|
||||
if (r.up) up++;
|
||||
n.status = r.up ? "active" : "inactive";
|
||||
n.checked_at = ts.dateTime;
|
||||
// Record the tip height when we read one; keep the last known height on a
|
||||
// down cycle so the card can still show where the node last was.
|
||||
if (typeof r.height === "number") n.block_height = r.height;
|
||||
else if (!("block_height" in n)) n.block_height = null;
|
||||
// Same sticky rule for the version read from X-Dojo-Version: update it when
|
||||
// this cycle saw one, otherwise leave the last known value in place. The
|
||||
// effective card version (operator override > detected > pairing default)
|
||||
// is computed by build-public.mjs, which carries this field across the
|
||||
// reconcile rebuild that opens every cycle.
|
||||
if (r.detectedVersion) n.detected_version = r.detectedVersion;
|
||||
else if (!("detected_version" in n)) n.detected_version = null;
|
||||
// Same sticky rule for the Electrum endpoint read from /support/services:
|
||||
// keep the last known value when a cycle didn't read one, so a node that is
|
||||
// merely down for a cycle doesn't flip its card to N/A. build-public.mjs
|
||||
// computes the published value and carries this field across the rebuild.
|
||||
if (r.detectedIndexer) n.detected_indexer = r.detectedIndexer;
|
||||
else if (!("detected_indexer" in n)) n.detected_indexer = null;
|
||||
});
|
||||
dojos.interval_minutes = dojos.interval_minutes || 10;
|
||||
|
||||
if (allFailed) {
|
||||
// Publish the fault and nothing else. Statuses, heights and checked_at stay
|
||||
// as the last cycle that actually reached something left them, and
|
||||
// generated_at is deliberately not advanced, so the staleness banner keeps
|
||||
// measuring the age of real data rather than the age of a failure.
|
||||
const fresh = await readJSON(dojosPath, null);
|
||||
if (fresh) {
|
||||
fresh.probe_fault = { at: ts.isoSec, nodes: dojos.nodes.length };
|
||||
await writeJSONAtomic(dojosPath, fresh);
|
||||
}
|
||||
console.error(`[${ts.isoSec}] every one of ${dojos.nodes.length} nodes failed, which is`
|
||||
+ " almost certainly a fault here rather than all of them at once.");
|
||||
console.error(" Nothing was recorded: no statuses changed and no history written.");
|
||||
console.error(" Check Tor on this machine (systemctl status tor@default), and the clock.");
|
||||
return;
|
||||
}
|
||||
dojos.generated_at = ts.isoSec;
|
||||
delete dojos.probe_fault;
|
||||
|
||||
// ---- update rolling history (append + trim, retire stale ids) ----
|
||||
const listed = new Set(dojos.nodes.map((n) => n.id));
|
||||
const histNodes = {};
|
||||
dojos.nodes.forEach((n, i) => {
|
||||
const prev = (history.nodes?.[n.id]?.checks) || [];
|
||||
const checks = prev.concat([{ t: ts.isoMin, up: results[i].up }]);
|
||||
if (checks.length > window) checks.splice(0, checks.length - window);
|
||||
histNodes[n.id] = { checks };
|
||||
});
|
||||
// Unlisted ids are kept under a `retired` stamp for HISTORY_GRACE_DAYS (same
|
||||
// rule as build-public.mjs), so a bad or transient node list cannot destroy
|
||||
// accumulated history; a resurrected id resumes where it left off.
|
||||
for (const id of Object.keys(history.nodes || {})) if (!histNodes[id]) histNodes[id] = history.nodes[id];
|
||||
retireUnlisted(histNodes, (id) => listed.has(id), ts.isoSec);
|
||||
|
||||
await writeJSONAtomic(dojosPath, dojos);
|
||||
await writeJSONAtomic(historyPath, {
|
||||
generated_at: ts.isoSec,
|
||||
interval_minutes: history.interval_minutes || 10,
|
||||
window_checks: window,
|
||||
nodes: histNodes,
|
||||
});
|
||||
|
||||
// ---- update 90-day daily rollup (per-day uptime + closing block height) ----
|
||||
// One record per node per UTC day; `close` is the last height read that day,
|
||||
// so at day's end it holds the closing height. Retained RETENTION_DAYS days.
|
||||
const dailyPath = path.join(CFG.dataDir, "history-daily.json");
|
||||
const daily = await readJSON(dailyPath, { retention_days: CFG.retentionDays, nodes: {} });
|
||||
const today = ts.dateTime.slice(0, 10); // YYYY-MM-DD (UTC)
|
||||
const dailyNodes = {};
|
||||
dojos.nodes.forEach((n, i) => {
|
||||
const r = results[i];
|
||||
const days = ((daily.nodes?.[n.id]?.days) || []).map((d) => ({ ...d }));
|
||||
let rec = days.length && days[days.length - 1].d === today ? days[days.length - 1] : null;
|
||||
if (!rec) { rec = { d: today, up: 0, total: 0, pct: 0, close: null }; days.push(rec); }
|
||||
rec.total += 1;
|
||||
if (r.up) rec.up += 1;
|
||||
rec.pct = Math.round((rec.up / rec.total) * 1000) / 10;
|
||||
if (typeof r.height === "number") rec.close = r.height;
|
||||
if (days.length > CFG.retentionDays) days.splice(0, days.length - CFG.retentionDays);
|
||||
dailyNodes[n.id] = { days };
|
||||
});
|
||||
for (const id of Object.keys(daily.nodes || {})) if (!dailyNodes[id]) dailyNodes[id] = daily.nodes[id];
|
||||
retireUnlisted(dailyNodes, (id) => listed.has(id), ts.isoSec);
|
||||
await writeJSONAtomic(dailyPath, {
|
||||
generated_at: ts.isoSec,
|
||||
retention_days: CFG.retentionDays,
|
||||
nodes: dailyNodes,
|
||||
});
|
||||
|
||||
// ---- probe PENDING submissions so the operator sees uptime before approving
|
||||
// Results are written server-side only (server/data/pending-probe.json), never
|
||||
// to the public data/, so an unapproved submission is not exposed over Tor.
|
||||
try {
|
||||
const { store } = await import("../server/store.ts");
|
||||
const serverDataDir = process.env.SERVER_DATA_DIR
|
||||
|| path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "server", "data");
|
||||
const pendingPath = path.join(serverDataDir, "pending-probe.json");
|
||||
const subs = (await store.listSubmissions()).filter((s) => s.status === "pending");
|
||||
if (subs.length) {
|
||||
const prevDoc = await readJSON(pendingPath, { window_checks: window, nodes: {} });
|
||||
const presults = await pool(subs, CFG.concurrency, async (s) => {
|
||||
const url = s?.payload?.pairing?.url;
|
||||
if (!url) return { up: false, reason: "no pairing url", ms: 0 };
|
||||
return probe(url, { ...CFG, apikey: s?.payload?.pairing?.apikey, network: s.network });
|
||||
});
|
||||
const pnodes = {};
|
||||
subs.forEach((s, i) => {
|
||||
const r = presults[i];
|
||||
const prev = (prevDoc.nodes?.[s.id]?.checks) || [];
|
||||
const checks = prev.concat([{ t: ts.isoMin, up: r.up }]);
|
||||
if (checks.length > window) checks.splice(0, checks.length - window);
|
||||
pnodes[s.id] = {
|
||||
status: r.up ? "active" : "inactive",
|
||||
checked_at: ts.dateTime,
|
||||
block_height: typeof r.height === "number" ? r.height
|
||||
: (prevDoc.nodes?.[s.id]?.block_height ?? null),
|
||||
detected_version: r.detectedVersion || (prevDoc.nodes?.[s.id]?.detected_version ?? null),
|
||||
detected_indexer: r.detectedIndexer || (prevDoc.nodes?.[s.id]?.detected_indexer ?? null),
|
||||
checks,
|
||||
};
|
||||
});
|
||||
await writeJSONAtomic(pendingPath, { generated_at: ts.isoSec, window_checks: window, nodes: pnodes });
|
||||
console.error(`[${ts.isoSec}] probed ${subs.length} pending submission(s)`);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(`[${ts.isoSec}] pending probe skipped: ${e.message}`);
|
||||
}
|
||||
|
||||
console.error(`[${ts.isoSec}] done: ${up}/${dojos.nodes.length} active`);
|
||||
for (const [i, n] of dojos.nodes.entries()) {
|
||||
const r = results[i];
|
||||
console.error(` ${r.up ? "UP " : "DOWN"} ${n.id.padEnd(28)} ${String(r.ms).padStart(6)}ms ${r.reason || ""}`);
|
||||
}
|
||||
await avatarsDone; // let in-flight avatar mirrors finish before the timer unit exits
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1] || "").href) {
|
||||
main().catch((e) => { console.error("fatal:", e.message); process.exit(1); });
|
||||
}
|
||||
Reference in New Issue
Block a user