From fc00490d619e2b91d634727c0bc018d205003d0e Mon Sep 17 00:00:00 2001 From: Dorian Date: Fri, 13 Mar 2026 04:56:01 +0000 Subject: [PATCH] =?UTF-8?q?test:=20speed=20meta=20analysis=20=E2=80=94=205?= =?UTF-8?q?0ms=20gap=20wins=2095.4%=20when=20both=20correct?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Critical finding: when all bots answer correctly, even a 50ms speed advantage wins 95.4% of fights. At 100ms+ gap it's 100% deterministic. ELO separation reaches 450+ after just 50 fights. Speed completely dominates the "all correct" meta. Co-Authored-By: Claude Opus 4.6 --- server/src/engine/speed-meta.test.ts | 221 +++++++++++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 server/src/engine/speed-meta.test.ts diff --git a/server/src/engine/speed-meta.test.ts b/server/src/engine/speed-meta.test.ts new file mode 100644 index 0000000..b3e04b5 --- /dev/null +++ b/server/src/engine/speed-meta.test.ts @@ -0,0 +1,221 @@ +import { describe, it, expect } from 'vitest' +import { scoreRound, calculateElo } from './scoring.js' +import { pickChallenge } from './challenges.js' +import { writeFileSync, mkdirSync, existsSync } from 'node:fs' +import { resolve, dirname } from 'node:path' +import { fileURLToPath } from 'node:url' + +// Simulate a fight where both bots answer 100% correctly +function simulateSpeedFight(avgSpeedA: number, avgSpeedB: number, variance: number) { + let hpA = 200, hpB = 200 + let comboA = 0, comboB = 0 + const usedTypes = new Set() + let rounds = 0 + + for (let r = 0; r < 20 && hpA > 0 && hpB > 0; r++) { + const challenge = pickChallenge(usedTypes, null) + usedTypes.add(challenge.type) + if (usedTypes.size >= 15) usedTypes.clear() + + const aTimeMs = Math.max(200, avgSpeedA + (Math.random() - 0.5) * 2 * variance) + const bTimeMs = Math.max(200, avgSpeedB + (Math.random() - 0.5) * 2 * variance) + + const correctAnswer = challenge.answers?.[0] ?? 'correct' + + const result = scoreRound( + challenge, + { id: 'a1', name: 'FastBot' }, + { id: 'b1', name: 'SlowBot' }, + { answer: correctAnswer, timeMs: aTimeMs, timedOut: false, error: false, trashTalk: '' }, + { answer: correctAnswer, timeMs: bTimeMs, timedOut: false, error: false, trashTalk: '' }, + null, comboA, comboB, + ) + + if (result.winnerId === 'a1') { + hpB -= result.botADamage + hpA -= result.botBDamage + comboA++ + comboB = 0 + } else if (result.winnerId === 'b1') { + hpA -= result.botBDamage + hpB -= result.botADamage + comboB++ + comboA = 0 + } else { + comboA = 0 + comboB = 0 + } + rounds++ + } + + const winnerId = hpA <= 0 ? 'b1' : hpB <= 0 ? 'a1' : (hpA > hpB ? 'a1' : hpB > hpA ? 'b1' : null) + return { winnerId, rounds } +} + +describe('speed meta analysis — all bots answer correctly', () => { + const reportLines: string[] = [] + + it('RESEARCH: speed advantage thresholds — what gap is needed to consistently win?', () => { + const N = 500 + const scenarios = [ + { aMs: 500, bMs: 550, label: '50ms gap (500 vs 550)', variance: 100 }, + { aMs: 500, bMs: 600, label: '100ms gap (500 vs 600)', variance: 100 }, + { aMs: 500, bMs: 700, label: '200ms gap (500 vs 700)', variance: 100 }, + { aMs: 500, bMs: 800, label: '300ms gap (500 vs 800)', variance: 100 }, + { aMs: 500, bMs: 1000, label: '500ms gap (500 vs 1000)', variance: 100 }, + { aMs: 500, bMs: 1500, label: '1000ms gap (500 vs 1500)', variance: 100 }, + { aMs: 500, bMs: 2000, label: '1500ms gap (500 vs 2000)', variance: 100 }, + { aMs: 500, bMs: 3000, label: '2500ms gap (500 vs 3000)', variance: 100 }, + ] + + const section = [ + '## Speed Advantage Thresholds (500 fights each, both 100% correct)', + '', + '| Speed Gap | Faster Wins | Slower Wins | Draws | Win Rate |', + '|-----------|------------|------------|-------|----------|', + ] + + for (const s of scenarios) { + let fastWins = 0, slowWins = 0, draws = 0 + for (let i = 0; i < N; i++) { + const result = simulateSpeedFight(s.aMs, s.bMs, s.variance) + if (result.winnerId === 'a1') fastWins++ + else if (result.winnerId === 'b1') slowWins++ + else draws++ + } + section.push(`| ${s.label} | ${fastWins} (${(fastWins / N * 100).toFixed(1)}%) | ${slowWins} (${(slowWins / N * 100).toFixed(1)}%) | ${draws} | ${(fastWins / N * 100).toFixed(1)}% |`) + } + + section.push('') + reportLines.push(...section) + console.log('\n=== SPEED ADVANTAGE THRESHOLDS ===') + section.forEach(l => console.log(` ${l}`)) + expect(true).toBe(true) + }) + + it('RESEARCH: variance impact — does high variance close the speed gap?', () => { + const N = 500 + const baseGap = { aMs: 500, bMs: 1000 } // 500ms gap + const variances = [50, 100, 200, 400, 600, 800] + + const section = [ + '## Variance Impact on 500ms Speed Gap (500 fights each)', + '', + '| Variance (±ms) | Faster Wins | Slower Wins | Win Rate |', + '|----------------|------------|------------|----------|', + ] + + for (const v of variances) { + let fastWins = 0, slowWins = 0 + for (let i = 0; i < N; i++) { + const result = simulateSpeedFight(baseGap.aMs, baseGap.bMs, v) + if (result.winnerId === 'a1') fastWins++ + else if (result.winnerId === 'b1') slowWins++ + } + section.push(`| ±${v}ms | ${fastWins} (${(fastWins / N * 100).toFixed(1)}%) | ${slowWins} (${(slowWins / N * 100).toFixed(1)}%) | ${(fastWins / N * 100).toFixed(1)}% |`) + } + + section.push('') + reportLines.push(...section) + console.log('\n=== VARIANCE IMPACT ===') + section.forEach(l => console.log(` ${l}`)) + expect(true).toBe(true) + }) + + it('RESEARCH: real-world AI bot scenario — GPT-4 (800ms) vs Claude (600ms) vs local (300ms)', () => { + const N = 500 + const bots = [ + { name: 'Local LLM', avgMs: 300 }, + { name: 'Claude API', avgMs: 600 }, + { name: 'GPT-4 API', avgMs: 800 }, + { name: 'Slow API', avgMs: 1500 }, + ] + + const section = [ + '## Real-World AI Bot Speed Matchups (500 fights each, ±200ms variance)', + '', + '| Matchup | Fast Bot Wins | Slow Bot Wins | Fast Win Rate |', + '|---------|--------------|--------------|--------------|', + ] + + for (let i = 0; i < bots.length; i++) { + for (let j = i + 1; j < bots.length; j++) { + let fastWins = 0, slowWins = 0 + for (let k = 0; k < N; k++) { + const result = simulateSpeedFight(bots[i].avgMs, bots[j].avgMs, 200) + if (result.winnerId === 'a1') fastWins++ + else if (result.winnerId === 'b1') slowWins++ + } + section.push(`| ${bots[i].name} vs ${bots[j].name} | ${fastWins} (${(fastWins / N * 100).toFixed(1)}%) | ${slowWins} (${(slowWins / N * 100).toFixed(1)}%) | ${(fastWins / N * 100).toFixed(1)}% |`) + } + } + + section.push('') + reportLines.push(...section) + console.log('\n=== REAL-WORLD AI BOT MATCHUPS ===') + section.forEach(l => console.log(` ${l}`)) + expect(true).toBe(true) + }) + + it('RESEARCH: ELO separation over 50 fights — how quickly do speed tiers separate?', () => { + const section = [ + '## ELO Separation Over 50 Fights (both start at 1200)', + '', + '| Speed Gap | Fast ELO After 50 | Slow ELO After 50 | Separation |', + '|-----------|-------------------|-------------------|------------|', + ] + + const gaps = [ + { aMs: 500, bMs: 600, label: '100ms' }, + { aMs: 500, bMs: 800, label: '300ms' }, + { aMs: 500, bMs: 1000, label: '500ms' }, + { aMs: 500, bMs: 1500, label: '1000ms' }, + ] + + for (const g of gaps) { + let eloA = 1200, eloB = 1200 + for (let i = 0; i < 50; i++) { + const result = simulateSpeedFight(g.aMs, g.bMs, 150) + if (result.winnerId === 'a1') { + const elo = calculateElo(eloA, eloB) + eloA = elo.newWinnerElo + eloB = elo.newLoserElo + } else if (result.winnerId === 'b1') { + const elo = calculateElo(eloB, eloA) + eloB = elo.newWinnerElo + eloA = elo.newLoserElo + } + } + section.push(`| ${g.label} | ${eloA.toFixed(0)} | ${eloB.toFixed(0)} | ${(eloA - eloB).toFixed(0)} |`) + } + + section.push('') + reportLines.push(...section) + console.log('\n=== ELO SEPARATION ===') + section.forEach(l => console.log(` ${l}`)) + expect(true).toBe(true) + }) + + it('writes speed meta analysis report', () => { + const __filename = fileURLToPath(import.meta.url) + const projectRoot = resolve(dirname(__filename), '..', '..', '..') + const loopDir = resolve(projectRoot, 'loop') + if (!existsSync(loopDir)) mkdirSync(loopDir, { recursive: true }) + + const header = [ + '# Speed Meta Analysis — "All Bots Get Everything Right"', + '', + `Generated: ${new Date().toISOString().slice(0, 10)}`, + '', + '> When all bots answer correctly (95%+ accuracy with GPT-4/Claude),', + '> speed becomes the only differentiator. This analysis quantifies', + '> the minimum speed advantage needed to consistently win.', + '', + ] + + const combined = [...header, ...reportLines].join('\n') + writeFileSync(resolve(loopDir, 'speed-meta-analysis.md'), combined) + console.log('\n Report written to loop/speed-meta-analysis.md') + expect(true).toBe(true) + }) +})