test: speed meta analysis — 50ms gap wins 95.4% when both correct
Critical finding: when all bots answer correctly, even a 50ms speed advantage wins 95.4% of fights. At 100ms+ gap it's 100% deterministic. ELO separation reaches 450+ after just 50 fights. Speed completely dominates the "all correct" meta. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Dorian
co-authored by
Claude Opus 4.6
Claude Opus 4.6
parent
9c55850b70
commit
fc00490d61
@@ -0,0 +1,221 @@
|
|||||||
|
import { describe, it, expect } from 'vitest'
|
||||||
|
import { scoreRound, calculateElo } from './scoring.js'
|
||||||
|
import { pickChallenge } from './challenges.js'
|
||||||
|
import { writeFileSync, mkdirSync, existsSync } from 'node:fs'
|
||||||
|
import { resolve, dirname } from 'node:path'
|
||||||
|
import { fileURLToPath } from 'node:url'
|
||||||
|
|
||||||
|
// Simulate a fight where both bots answer 100% correctly
|
||||||
|
function simulateSpeedFight(avgSpeedA: number, avgSpeedB: number, variance: number) {
|
||||||
|
let hpA = 200, hpB = 200
|
||||||
|
let comboA = 0, comboB = 0
|
||||||
|
const usedTypes = new Set<string>()
|
||||||
|
let rounds = 0
|
||||||
|
|
||||||
|
for (let r = 0; r < 20 && hpA > 0 && hpB > 0; r++) {
|
||||||
|
const challenge = pickChallenge(usedTypes, null)
|
||||||
|
usedTypes.add(challenge.type)
|
||||||
|
if (usedTypes.size >= 15) usedTypes.clear()
|
||||||
|
|
||||||
|
const aTimeMs = Math.max(200, avgSpeedA + (Math.random() - 0.5) * 2 * variance)
|
||||||
|
const bTimeMs = Math.max(200, avgSpeedB + (Math.random() - 0.5) * 2 * variance)
|
||||||
|
|
||||||
|
const correctAnswer = challenge.answers?.[0] ?? 'correct'
|
||||||
|
|
||||||
|
const result = scoreRound(
|
||||||
|
challenge,
|
||||||
|
{ id: 'a1', name: 'FastBot' },
|
||||||
|
{ id: 'b1', name: 'SlowBot' },
|
||||||
|
{ answer: correctAnswer, timeMs: aTimeMs, timedOut: false, error: false, trashTalk: '' },
|
||||||
|
{ answer: correctAnswer, timeMs: bTimeMs, timedOut: false, error: false, trashTalk: '' },
|
||||||
|
null, comboA, comboB,
|
||||||
|
)
|
||||||
|
|
||||||
|
if (result.winnerId === 'a1') {
|
||||||
|
hpB -= result.botADamage
|
||||||
|
hpA -= result.botBDamage
|
||||||
|
comboA++
|
||||||
|
comboB = 0
|
||||||
|
} else if (result.winnerId === 'b1') {
|
||||||
|
hpA -= result.botBDamage
|
||||||
|
hpB -= result.botADamage
|
||||||
|
comboB++
|
||||||
|
comboA = 0
|
||||||
|
} else {
|
||||||
|
comboA = 0
|
||||||
|
comboB = 0
|
||||||
|
}
|
||||||
|
rounds++
|
||||||
|
}
|
||||||
|
|
||||||
|
const winnerId = hpA <= 0 ? 'b1' : hpB <= 0 ? 'a1' : (hpA > hpB ? 'a1' : hpB > hpA ? 'b1' : null)
|
||||||
|
return { winnerId, rounds }
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('speed meta analysis — all bots answer correctly', () => {
|
||||||
|
const reportLines: string[] = []
|
||||||
|
|
||||||
|
it('RESEARCH: speed advantage thresholds — what gap is needed to consistently win?', () => {
|
||||||
|
const N = 500
|
||||||
|
const scenarios = [
|
||||||
|
{ aMs: 500, bMs: 550, label: '50ms gap (500 vs 550)', variance: 100 },
|
||||||
|
{ aMs: 500, bMs: 600, label: '100ms gap (500 vs 600)', variance: 100 },
|
||||||
|
{ aMs: 500, bMs: 700, label: '200ms gap (500 vs 700)', variance: 100 },
|
||||||
|
{ aMs: 500, bMs: 800, label: '300ms gap (500 vs 800)', variance: 100 },
|
||||||
|
{ aMs: 500, bMs: 1000, label: '500ms gap (500 vs 1000)', variance: 100 },
|
||||||
|
{ aMs: 500, bMs: 1500, label: '1000ms gap (500 vs 1500)', variance: 100 },
|
||||||
|
{ aMs: 500, bMs: 2000, label: '1500ms gap (500 vs 2000)', variance: 100 },
|
||||||
|
{ aMs: 500, bMs: 3000, label: '2500ms gap (500 vs 3000)', variance: 100 },
|
||||||
|
]
|
||||||
|
|
||||||
|
const section = [
|
||||||
|
'## Speed Advantage Thresholds (500 fights each, both 100% correct)',
|
||||||
|
'',
|
||||||
|
'| Speed Gap | Faster Wins | Slower Wins | Draws | Win Rate |',
|
||||||
|
'|-----------|------------|------------|-------|----------|',
|
||||||
|
]
|
||||||
|
|
||||||
|
for (const s of scenarios) {
|
||||||
|
let fastWins = 0, slowWins = 0, draws = 0
|
||||||
|
for (let i = 0; i < N; i++) {
|
||||||
|
const result = simulateSpeedFight(s.aMs, s.bMs, s.variance)
|
||||||
|
if (result.winnerId === 'a1') fastWins++
|
||||||
|
else if (result.winnerId === 'b1') slowWins++
|
||||||
|
else draws++
|
||||||
|
}
|
||||||
|
section.push(`| ${s.label} | ${fastWins} (${(fastWins / N * 100).toFixed(1)}%) | ${slowWins} (${(slowWins / N * 100).toFixed(1)}%) | ${draws} | ${(fastWins / N * 100).toFixed(1)}% |`)
|
||||||
|
}
|
||||||
|
|
||||||
|
section.push('')
|
||||||
|
reportLines.push(...section)
|
||||||
|
console.log('\n=== SPEED ADVANTAGE THRESHOLDS ===')
|
||||||
|
section.forEach(l => console.log(` ${l}`))
|
||||||
|
expect(true).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('RESEARCH: variance impact — does high variance close the speed gap?', () => {
|
||||||
|
const N = 500
|
||||||
|
const baseGap = { aMs: 500, bMs: 1000 } // 500ms gap
|
||||||
|
const variances = [50, 100, 200, 400, 600, 800]
|
||||||
|
|
||||||
|
const section = [
|
||||||
|
'## Variance Impact on 500ms Speed Gap (500 fights each)',
|
||||||
|
'',
|
||||||
|
'| Variance (±ms) | Faster Wins | Slower Wins | Win Rate |',
|
||||||
|
'|----------------|------------|------------|----------|',
|
||||||
|
]
|
||||||
|
|
||||||
|
for (const v of variances) {
|
||||||
|
let fastWins = 0, slowWins = 0
|
||||||
|
for (let i = 0; i < N; i++) {
|
||||||
|
const result = simulateSpeedFight(baseGap.aMs, baseGap.bMs, v)
|
||||||
|
if (result.winnerId === 'a1') fastWins++
|
||||||
|
else if (result.winnerId === 'b1') slowWins++
|
||||||
|
}
|
||||||
|
section.push(`| ±${v}ms | ${fastWins} (${(fastWins / N * 100).toFixed(1)}%) | ${slowWins} (${(slowWins / N * 100).toFixed(1)}%) | ${(fastWins / N * 100).toFixed(1)}% |`)
|
||||||
|
}
|
||||||
|
|
||||||
|
section.push('')
|
||||||
|
reportLines.push(...section)
|
||||||
|
console.log('\n=== VARIANCE IMPACT ===')
|
||||||
|
section.forEach(l => console.log(` ${l}`))
|
||||||
|
expect(true).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('RESEARCH: real-world AI bot scenario — GPT-4 (800ms) vs Claude (600ms) vs local (300ms)', () => {
|
||||||
|
const N = 500
|
||||||
|
const bots = [
|
||||||
|
{ name: 'Local LLM', avgMs: 300 },
|
||||||
|
{ name: 'Claude API', avgMs: 600 },
|
||||||
|
{ name: 'GPT-4 API', avgMs: 800 },
|
||||||
|
{ name: 'Slow API', avgMs: 1500 },
|
||||||
|
]
|
||||||
|
|
||||||
|
const section = [
|
||||||
|
'## Real-World AI Bot Speed Matchups (500 fights each, ±200ms variance)',
|
||||||
|
'',
|
||||||
|
'| Matchup | Fast Bot Wins | Slow Bot Wins | Fast Win Rate |',
|
||||||
|
'|---------|--------------|--------------|--------------|',
|
||||||
|
]
|
||||||
|
|
||||||
|
for (let i = 0; i < bots.length; i++) {
|
||||||
|
for (let j = i + 1; j < bots.length; j++) {
|
||||||
|
let fastWins = 0, slowWins = 0
|
||||||
|
for (let k = 0; k < N; k++) {
|
||||||
|
const result = simulateSpeedFight(bots[i].avgMs, bots[j].avgMs, 200)
|
||||||
|
if (result.winnerId === 'a1') fastWins++
|
||||||
|
else if (result.winnerId === 'b1') slowWins++
|
||||||
|
}
|
||||||
|
section.push(`| ${bots[i].name} vs ${bots[j].name} | ${fastWins} (${(fastWins / N * 100).toFixed(1)}%) | ${slowWins} (${(slowWins / N * 100).toFixed(1)}%) | ${(fastWins / N * 100).toFixed(1)}% |`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
section.push('')
|
||||||
|
reportLines.push(...section)
|
||||||
|
console.log('\n=== REAL-WORLD AI BOT MATCHUPS ===')
|
||||||
|
section.forEach(l => console.log(` ${l}`))
|
||||||
|
expect(true).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('RESEARCH: ELO separation over 50 fights — how quickly do speed tiers separate?', () => {
|
||||||
|
const section = [
|
||||||
|
'## ELO Separation Over 50 Fights (both start at 1200)',
|
||||||
|
'',
|
||||||
|
'| Speed Gap | Fast ELO After 50 | Slow ELO After 50 | Separation |',
|
||||||
|
'|-----------|-------------------|-------------------|------------|',
|
||||||
|
]
|
||||||
|
|
||||||
|
const gaps = [
|
||||||
|
{ aMs: 500, bMs: 600, label: '100ms' },
|
||||||
|
{ aMs: 500, bMs: 800, label: '300ms' },
|
||||||
|
{ aMs: 500, bMs: 1000, label: '500ms' },
|
||||||
|
{ aMs: 500, bMs: 1500, label: '1000ms' },
|
||||||
|
]
|
||||||
|
|
||||||
|
for (const g of gaps) {
|
||||||
|
let eloA = 1200, eloB = 1200
|
||||||
|
for (let i = 0; i < 50; i++) {
|
||||||
|
const result = simulateSpeedFight(g.aMs, g.bMs, 150)
|
||||||
|
if (result.winnerId === 'a1') {
|
||||||
|
const elo = calculateElo(eloA, eloB)
|
||||||
|
eloA = elo.newWinnerElo
|
||||||
|
eloB = elo.newLoserElo
|
||||||
|
} else if (result.winnerId === 'b1') {
|
||||||
|
const elo = calculateElo(eloB, eloA)
|
||||||
|
eloB = elo.newWinnerElo
|
||||||
|
eloA = elo.newLoserElo
|
||||||
|
}
|
||||||
|
}
|
||||||
|
section.push(`| ${g.label} | ${eloA.toFixed(0)} | ${eloB.toFixed(0)} | ${(eloA - eloB).toFixed(0)} |`)
|
||||||
|
}
|
||||||
|
|
||||||
|
section.push('')
|
||||||
|
reportLines.push(...section)
|
||||||
|
console.log('\n=== ELO SEPARATION ===')
|
||||||
|
section.forEach(l => console.log(` ${l}`))
|
||||||
|
expect(true).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('writes speed meta analysis report', () => {
|
||||||
|
const __filename = fileURLToPath(import.meta.url)
|
||||||
|
const projectRoot = resolve(dirname(__filename), '..', '..', '..')
|
||||||
|
const loopDir = resolve(projectRoot, 'loop')
|
||||||
|
if (!existsSync(loopDir)) mkdirSync(loopDir, { recursive: true })
|
||||||
|
|
||||||
|
const header = [
|
||||||
|
'# Speed Meta Analysis — "All Bots Get Everything Right"',
|
||||||
|
'',
|
||||||
|
`Generated: ${new Date().toISOString().slice(0, 10)}`,
|
||||||
|
'',
|
||||||
|
'> When all bots answer correctly (95%+ accuracy with GPT-4/Claude),',
|
||||||
|
'> speed becomes the only differentiator. This analysis quantifies',
|
||||||
|
'> the minimum speed advantage needed to consistently win.',
|
||||||
|
'',
|
||||||
|
]
|
||||||
|
|
||||||
|
const combined = [...header, ...reportLines].join('\n')
|
||||||
|
writeFileSync(resolve(loopDir, 'speed-meta-analysis.md'), combined)
|
||||||
|
console.log('\n Report written to loop/speed-meta-analysis.md')
|
||||||
|
expect(true).toBe(true)
|
||||||
|
})
|
||||||
|
})
|
||||||
Reference in New Issue
Block a user