diff --git a/server/src/engine/tier-balance.test.ts b/server/src/engine/tier-balance.test.ts new file mode 100644 index 0000000..82c202b --- /dev/null +++ b/server/src/engine/tier-balance.test.ts @@ -0,0 +1,236 @@ +import { describe, it, expect } from 'vitest' +import { scoreRound, calculateElo, calculateTier } from './scoring.js' +import { pickChallenge } from './challenges.js' +import { mockResponse } from './mock.js' +import { writeFileSync, mkdirSync, existsSync } from 'node:fs' +import { resolve, dirname } from 'node:path' +import { fileURLToPath } from 'node:url' + +// Simulate a full fight between two bots with given ELOs +function simulateTierFight(eloA: number, eloB: number) { + let hpA = 200, hpB = 200 + let comboA = 0, comboB = 0 + const usedTypes = new Set() + let rounds = 0 + + for (let r = 0; r < 20 && hpA > 0 && hpB > 0; r++) { + const challenge = pickChallenge(usedTypes, null) + usedTypes.add(challenge.type) + if (usedTypes.size >= 15) usedTypes.clear() + + const respA = mockResponse(challenge, 'default', eloA) + const respB = mockResponse(challenge, 'default', eloB) + + const result = scoreRound( + challenge, + { id: 'a1', name: 'BotA' }, + { id: 'b1', name: 'BotB' }, + { answer: respA.answer, timeMs: respA.timeMs, timedOut: respA.timedOut, error: respA.error, trashTalk: '' }, + { answer: respB.answer, timeMs: respB.timeMs, timedOut: respB.timedOut, error: respB.error, trashTalk: '' }, + null, comboA, comboB, + ) + + if (result.winnerId === 'a1') { + hpB -= result.botADamage + hpA -= result.botBDamage + comboA++ + comboB = 0 + } else if (result.winnerId === 'b1') { + hpA -= result.botBDamage + hpB -= result.botADamage + comboB++ + comboA = 0 + } else { + comboA = 0 + comboB = 0 + } + rounds++ + } + + const winnerId = hpA <= 0 ? 'b1' : hpB <= 0 ? 'a1' : (hpA > hpB ? 'a1' : hpB > hpA ? 'b1' : null) + return { winnerId, rounds } +} + +// Tier representative ELOs (midpoint of each tier range) +const TIER_REPS: { tier: number; name: string; elo: number; wins: number }[] = [ + { tier: 0, name: 'Baby', elo: 1000, wins: 0 }, + { tier: 1, name: 'Bronze', elo: 1100, wins: 2 }, + { tier: 2, name: 'Silver', elo: 1275, wins: 5 }, + { tier: 3, name: 'Gold', elo: 1425, wins: 10 }, + { tier: 4, name: 'Platinum', elo: 1600, wins: 20 }, + { tier: 5, name: 'Diamond', elo: 1800, wins: 30 }, + { tier: 6, name: 'Legend', elo: 1950, wins: 45 }, +] + +describe('tier balance analysis', () => { + const reportLines: string[] = [] + + it('RESEARCH: same-tier matchups — win rates should be ~50/50', () => { + const N = 100 + const section = [ + '## Same-Tier Matchups (100 fights each)', + '', + '| Tier | Win A | Win B | Draws | Avg Rounds |', + '|------|-------|-------|-------|------------|', + ] + + for (const t of TIER_REPS) { + let aWins = 0, bWins = 0, draws = 0, totalRounds = 0 + for (let i = 0; i < N; i++) { + const result = simulateTierFight(t.elo, t.elo) + if (result.winnerId === 'a1') aWins++ + else if (result.winnerId === 'b1') bWins++ + else draws++ + totalRounds += result.rounds + } + section.push(`| ${t.name} (${t.elo}) | ${aWins}% | ${bWins}% | ${draws}% | ${(totalRounds / N).toFixed(1)} |`) + } + + section.push('') + reportLines.push(...section) + console.log('\n=== SAME-TIER MATCHUPS ===') + section.forEach(l => console.log(` ${l}`)) + expect(true).toBe(true) + }) + + it('RESEARCH: cross-tier matchups — higher tier should win consistently', () => { + const N = 100 + const section = [ + '## Cross-Tier Matchups (100 fights each)', + '', + '| Matchup | Higher Wins | Lower Wins | Upset Rate | Avg Rounds |', + '|---------|------------|------------|------------|------------|', + ] + + for (let i = 0; i < TIER_REPS.length - 1; i++) { + const higher = TIER_REPS[i + 1] + const lower = TIER_REPS[i] + + let higherWins = 0, lowerWins = 0, draws = 0, totalRounds = 0 + for (let j = 0; j < N; j++) { + const result = simulateTierFight(higher.elo, lower.elo) + if (result.winnerId === 'a1') higherWins++ + else if (result.winnerId === 'b1') lowerWins++ + else draws++ + totalRounds += result.rounds + } + + const upsetRate = (lowerWins / N * 100).toFixed(1) + section.push(`| ${higher.name} vs ${lower.name} | ${higherWins}% | ${lowerWins}% | ${upsetRate}% | ${(totalRounds / N).toFixed(1)} |`) + } + + section.push('') + reportLines.push(...section) + console.log('\n=== CROSS-TIER MATCHUPS ===') + section.forEach(l => console.log(` ${l}`)) + expect(true).toBe(true) + }) + + it('RESEARCH: 2-tier gap matchups — should be nearly unwinnable for lower tier', () => { + const N = 100 + const section = [ + '## 2-Tier Gap Matchups (100 fights each)', + '', + '| Matchup | Higher Wins | Lower Wins | Upset Rate |', + '|---------|------------|------------|------------|', + ] + + const gaps = [ + [TIER_REPS[6], TIER_REPS[4]], // Legend vs Platinum + [TIER_REPS[5], TIER_REPS[3]], // Diamond vs Gold + [TIER_REPS[4], TIER_REPS[2]], // Platinum vs Silver + [TIER_REPS[3], TIER_REPS[1]], // Gold vs Bronze + [TIER_REPS[2], TIER_REPS[0]], // Silver vs Baby + ] + + for (const [higher, lower] of gaps) { + let higherWins = 0, lowerWins = 0 + for (let j = 0; j < N; j++) { + const result = simulateTierFight(higher.elo, lower.elo) + if (result.winnerId === 'a1') higherWins++ + else if (result.winnerId === 'b1') lowerWins++ + } + section.push(`| ${higher.name} vs ${lower.name} | ${higherWins}% | ${lowerWins}% | ${(lowerWins / N * 100).toFixed(1)}% |`) + } + + section.push('') + reportLines.push(...section) + console.log('\n=== 2-TIER GAP MATCHUPS ===') + section.forEach(l => console.log(` ${l}`)) + expect(true).toBe(true) + }) + + it('RESEARCH: ELO delta per tier matchup — K-factor appropriateness', () => { + const section = [ + '## ELO Delta Analysis (K=32)', + '', + '| Matchup | ELO Gap | Winner Gain | Loser Loss | Expected Win% |', + '|---------|---------|------------|------------|---------------|', + ] + + for (let i = 0; i < TIER_REPS.length - 1; i++) { + const higher = TIER_REPS[i + 1] + const lower = TIER_REPS[i] + const gap = higher.elo - lower.elo + + // Higher tier wins + const eloHighWins = calculateElo(higher.elo, lower.elo) + // Lower tier upsets + const eloLowWins = calculateElo(lower.elo, higher.elo) + + const expectedWin = 1 / (1 + Math.pow(10, (lower.elo - higher.elo) / 400)) + + section.push(`| ${higher.name} beats ${lower.name} | ${gap} | +${(eloHighWins.newWinnerElo - higher.elo).toFixed(1)} | ${(eloHighWins.newLoserElo - lower.elo).toFixed(1)} | ${(expectedWin * 100).toFixed(1)}% |`) + section.push(`| ${lower.name} upsets ${higher.name} | ${gap} | +${(eloLowWins.newWinnerElo - lower.elo).toFixed(1)} | ${(eloLowWins.newLoserElo - higher.elo).toFixed(1)} | ${((1 - expectedWin) * 100).toFixed(1)}% |`) + } + + section.push('') + reportLines.push(...section) + console.log('\n=== ELO DELTA ANALYSIS ===') + section.forEach(l => console.log(` ${l}`)) + expect(true).toBe(true) + }) + + it('RESEARCH: tier calculation correctness', () => { + // Verify tier boundaries + const tests = [ + { elo: 1000, wins: 0, expected: 0 }, // Baby + { elo: 1200, wins: 1, expected: 1 }, // Bronze + { elo: 1200, wins: 3, expected: 2 }, // Silver + { elo: 1350, wins: 7, expected: 3 }, // Gold + { elo: 1500, wins: 15, expected: 4 }, // Platinum + { elo: 1700, wins: 25, expected: 5 }, // Diamond + { elo: 1900, wins: 40, expected: 6 }, // Legend + // Edge cases + { elo: 1899, wins: 40, expected: 5 }, // Just below Legend ELO → Diamond + { elo: 1900, wins: 39, expected: 5 }, // Enough ELO but not enough wins → Diamond + { elo: 1100, wins: 0, expected: 0 }, // High ELO but no wins → Baby + ] + + for (const t of tests) { + expect(calculateTier(t.elo, t.wins)).toBe(t.expected) + } + }) + + it('writes tier balance report', () => { + const __filename = fileURLToPath(import.meta.url) + const projectRoot = resolve(dirname(__filename), '..', '..', '..') + const loopDir = resolve(projectRoot, 'loop') + if (!existsSync(loopDir)) mkdirSync(loopDir, { recursive: true }) + + const header = [ + '# Tier Balance Analysis', + '', + `Generated: ${new Date().toISOString().slice(0, 10)}`, + '', + '> Simulated fights between tier-representative bots to analyze', + '> skill gaps, upset rates, and ELO K-factor appropriateness.', + '', + ] + + const combined = [...header, ...reportLines].join('\n') + writeFileSync(resolve(loopDir, 'tier-balance-analysis.md'), combined) + console.log('\n Report written to loop/tier-balance-analysis.md') + expect(true).toBe(true) + }) +})