test: tier balance analysis — system well-balanced across all tiers
Same-tier: ~50/50 win rates. Adjacent tiers: 70-93% higher-tier wins. 2-tier gap: 87-99% higher wins. K=32 ELO factor appropriate. Legend vs Platinum: 99% win rate confirms clear skill separation. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Dorian
co-authored by
Claude Opus 4.6
Claude Opus 4.6
parent
929758ed1b
commit
9c55850b70
@@ -0,0 +1,236 @@
|
|||||||
|
import { describe, it, expect } from 'vitest'
|
||||||
|
import { scoreRound, calculateElo, calculateTier } from './scoring.js'
|
||||||
|
import { pickChallenge } from './challenges.js'
|
||||||
|
import { mockResponse } from './mock.js'
|
||||||
|
import { writeFileSync, mkdirSync, existsSync } from 'node:fs'
|
||||||
|
import { resolve, dirname } from 'node:path'
|
||||||
|
import { fileURLToPath } from 'node:url'
|
||||||
|
|
||||||
|
// Simulate a full fight between two bots with given ELOs
|
||||||
|
function simulateTierFight(eloA: number, eloB: number) {
|
||||||
|
let hpA = 200, hpB = 200
|
||||||
|
let comboA = 0, comboB = 0
|
||||||
|
const usedTypes = new Set<string>()
|
||||||
|
let rounds = 0
|
||||||
|
|
||||||
|
for (let r = 0; r < 20 && hpA > 0 && hpB > 0; r++) {
|
||||||
|
const challenge = pickChallenge(usedTypes, null)
|
||||||
|
usedTypes.add(challenge.type)
|
||||||
|
if (usedTypes.size >= 15) usedTypes.clear()
|
||||||
|
|
||||||
|
const respA = mockResponse(challenge, 'default', eloA)
|
||||||
|
const respB = mockResponse(challenge, 'default', eloB)
|
||||||
|
|
||||||
|
const result = scoreRound(
|
||||||
|
challenge,
|
||||||
|
{ id: 'a1', name: 'BotA' },
|
||||||
|
{ id: 'b1', name: 'BotB' },
|
||||||
|
{ answer: respA.answer, timeMs: respA.timeMs, timedOut: respA.timedOut, error: respA.error, trashTalk: '' },
|
||||||
|
{ answer: respB.answer, timeMs: respB.timeMs, timedOut: respB.timedOut, error: respB.error, trashTalk: '' },
|
||||||
|
null, comboA, comboB,
|
||||||
|
)
|
||||||
|
|
||||||
|
if (result.winnerId === 'a1') {
|
||||||
|
hpB -= result.botADamage
|
||||||
|
hpA -= result.botBDamage
|
||||||
|
comboA++
|
||||||
|
comboB = 0
|
||||||
|
} else if (result.winnerId === 'b1') {
|
||||||
|
hpA -= result.botBDamage
|
||||||
|
hpB -= result.botADamage
|
||||||
|
comboB++
|
||||||
|
comboA = 0
|
||||||
|
} else {
|
||||||
|
comboA = 0
|
||||||
|
comboB = 0
|
||||||
|
}
|
||||||
|
rounds++
|
||||||
|
}
|
||||||
|
|
||||||
|
const winnerId = hpA <= 0 ? 'b1' : hpB <= 0 ? 'a1' : (hpA > hpB ? 'a1' : hpB > hpA ? 'b1' : null)
|
||||||
|
return { winnerId, rounds }
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tier representative ELOs (midpoint of each tier range)
|
||||||
|
const TIER_REPS: { tier: number; name: string; elo: number; wins: number }[] = [
|
||||||
|
{ tier: 0, name: 'Baby', elo: 1000, wins: 0 },
|
||||||
|
{ tier: 1, name: 'Bronze', elo: 1100, wins: 2 },
|
||||||
|
{ tier: 2, name: 'Silver', elo: 1275, wins: 5 },
|
||||||
|
{ tier: 3, name: 'Gold', elo: 1425, wins: 10 },
|
||||||
|
{ tier: 4, name: 'Platinum', elo: 1600, wins: 20 },
|
||||||
|
{ tier: 5, name: 'Diamond', elo: 1800, wins: 30 },
|
||||||
|
{ tier: 6, name: 'Legend', elo: 1950, wins: 45 },
|
||||||
|
]
|
||||||
|
|
||||||
|
describe('tier balance analysis', () => {
|
||||||
|
const reportLines: string[] = []
|
||||||
|
|
||||||
|
it('RESEARCH: same-tier matchups — win rates should be ~50/50', () => {
|
||||||
|
const N = 100
|
||||||
|
const section = [
|
||||||
|
'## Same-Tier Matchups (100 fights each)',
|
||||||
|
'',
|
||||||
|
'| Tier | Win A | Win B | Draws | Avg Rounds |',
|
||||||
|
'|------|-------|-------|-------|------------|',
|
||||||
|
]
|
||||||
|
|
||||||
|
for (const t of TIER_REPS) {
|
||||||
|
let aWins = 0, bWins = 0, draws = 0, totalRounds = 0
|
||||||
|
for (let i = 0; i < N; i++) {
|
||||||
|
const result = simulateTierFight(t.elo, t.elo)
|
||||||
|
if (result.winnerId === 'a1') aWins++
|
||||||
|
else if (result.winnerId === 'b1') bWins++
|
||||||
|
else draws++
|
||||||
|
totalRounds += result.rounds
|
||||||
|
}
|
||||||
|
section.push(`| ${t.name} (${t.elo}) | ${aWins}% | ${bWins}% | ${draws}% | ${(totalRounds / N).toFixed(1)} |`)
|
||||||
|
}
|
||||||
|
|
||||||
|
section.push('')
|
||||||
|
reportLines.push(...section)
|
||||||
|
console.log('\n=== SAME-TIER MATCHUPS ===')
|
||||||
|
section.forEach(l => console.log(` ${l}`))
|
||||||
|
expect(true).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('RESEARCH: cross-tier matchups — higher tier should win consistently', () => {
|
||||||
|
const N = 100
|
||||||
|
const section = [
|
||||||
|
'## Cross-Tier Matchups (100 fights each)',
|
||||||
|
'',
|
||||||
|
'| Matchup | Higher Wins | Lower Wins | Upset Rate | Avg Rounds |',
|
||||||
|
'|---------|------------|------------|------------|------------|',
|
||||||
|
]
|
||||||
|
|
||||||
|
for (let i = 0; i < TIER_REPS.length - 1; i++) {
|
||||||
|
const higher = TIER_REPS[i + 1]
|
||||||
|
const lower = TIER_REPS[i]
|
||||||
|
|
||||||
|
let higherWins = 0, lowerWins = 0, draws = 0, totalRounds = 0
|
||||||
|
for (let j = 0; j < N; j++) {
|
||||||
|
const result = simulateTierFight(higher.elo, lower.elo)
|
||||||
|
if (result.winnerId === 'a1') higherWins++
|
||||||
|
else if (result.winnerId === 'b1') lowerWins++
|
||||||
|
else draws++
|
||||||
|
totalRounds += result.rounds
|
||||||
|
}
|
||||||
|
|
||||||
|
const upsetRate = (lowerWins / N * 100).toFixed(1)
|
||||||
|
section.push(`| ${higher.name} vs ${lower.name} | ${higherWins}% | ${lowerWins}% | ${upsetRate}% | ${(totalRounds / N).toFixed(1)} |`)
|
||||||
|
}
|
||||||
|
|
||||||
|
section.push('')
|
||||||
|
reportLines.push(...section)
|
||||||
|
console.log('\n=== CROSS-TIER MATCHUPS ===')
|
||||||
|
section.forEach(l => console.log(` ${l}`))
|
||||||
|
expect(true).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('RESEARCH: 2-tier gap matchups — should be nearly unwinnable for lower tier', () => {
|
||||||
|
const N = 100
|
||||||
|
const section = [
|
||||||
|
'## 2-Tier Gap Matchups (100 fights each)',
|
||||||
|
'',
|
||||||
|
'| Matchup | Higher Wins | Lower Wins | Upset Rate |',
|
||||||
|
'|---------|------------|------------|------------|',
|
||||||
|
]
|
||||||
|
|
||||||
|
const gaps = [
|
||||||
|
[TIER_REPS[6], TIER_REPS[4]], // Legend vs Platinum
|
||||||
|
[TIER_REPS[5], TIER_REPS[3]], // Diamond vs Gold
|
||||||
|
[TIER_REPS[4], TIER_REPS[2]], // Platinum vs Silver
|
||||||
|
[TIER_REPS[3], TIER_REPS[1]], // Gold vs Bronze
|
||||||
|
[TIER_REPS[2], TIER_REPS[0]], // Silver vs Baby
|
||||||
|
]
|
||||||
|
|
||||||
|
for (const [higher, lower] of gaps) {
|
||||||
|
let higherWins = 0, lowerWins = 0
|
||||||
|
for (let j = 0; j < N; j++) {
|
||||||
|
const result = simulateTierFight(higher.elo, lower.elo)
|
||||||
|
if (result.winnerId === 'a1') higherWins++
|
||||||
|
else if (result.winnerId === 'b1') lowerWins++
|
||||||
|
}
|
||||||
|
section.push(`| ${higher.name} vs ${lower.name} | ${higherWins}% | ${lowerWins}% | ${(lowerWins / N * 100).toFixed(1)}% |`)
|
||||||
|
}
|
||||||
|
|
||||||
|
section.push('')
|
||||||
|
reportLines.push(...section)
|
||||||
|
console.log('\n=== 2-TIER GAP MATCHUPS ===')
|
||||||
|
section.forEach(l => console.log(` ${l}`))
|
||||||
|
expect(true).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('RESEARCH: ELO delta per tier matchup — K-factor appropriateness', () => {
|
||||||
|
const section = [
|
||||||
|
'## ELO Delta Analysis (K=32)',
|
||||||
|
'',
|
||||||
|
'| Matchup | ELO Gap | Winner Gain | Loser Loss | Expected Win% |',
|
||||||
|
'|---------|---------|------------|------------|---------------|',
|
||||||
|
]
|
||||||
|
|
||||||
|
for (let i = 0; i < TIER_REPS.length - 1; i++) {
|
||||||
|
const higher = TIER_REPS[i + 1]
|
||||||
|
const lower = TIER_REPS[i]
|
||||||
|
const gap = higher.elo - lower.elo
|
||||||
|
|
||||||
|
// Higher tier wins
|
||||||
|
const eloHighWins = calculateElo(higher.elo, lower.elo)
|
||||||
|
// Lower tier upsets
|
||||||
|
const eloLowWins = calculateElo(lower.elo, higher.elo)
|
||||||
|
|
||||||
|
const expectedWin = 1 / (1 + Math.pow(10, (lower.elo - higher.elo) / 400))
|
||||||
|
|
||||||
|
section.push(`| ${higher.name} beats ${lower.name} | ${gap} | +${(eloHighWins.newWinnerElo - higher.elo).toFixed(1)} | ${(eloHighWins.newLoserElo - lower.elo).toFixed(1)} | ${(expectedWin * 100).toFixed(1)}% |`)
|
||||||
|
section.push(`| ${lower.name} upsets ${higher.name} | ${gap} | +${(eloLowWins.newWinnerElo - lower.elo).toFixed(1)} | ${(eloLowWins.newLoserElo - higher.elo).toFixed(1)} | ${((1 - expectedWin) * 100).toFixed(1)}% |`)
|
||||||
|
}
|
||||||
|
|
||||||
|
section.push('')
|
||||||
|
reportLines.push(...section)
|
||||||
|
console.log('\n=== ELO DELTA ANALYSIS ===')
|
||||||
|
section.forEach(l => console.log(` ${l}`))
|
||||||
|
expect(true).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('RESEARCH: tier calculation correctness', () => {
|
||||||
|
// Verify tier boundaries
|
||||||
|
const tests = [
|
||||||
|
{ elo: 1000, wins: 0, expected: 0 }, // Baby
|
||||||
|
{ elo: 1200, wins: 1, expected: 1 }, // Bronze
|
||||||
|
{ elo: 1200, wins: 3, expected: 2 }, // Silver
|
||||||
|
{ elo: 1350, wins: 7, expected: 3 }, // Gold
|
||||||
|
{ elo: 1500, wins: 15, expected: 4 }, // Platinum
|
||||||
|
{ elo: 1700, wins: 25, expected: 5 }, // Diamond
|
||||||
|
{ elo: 1900, wins: 40, expected: 6 }, // Legend
|
||||||
|
// Edge cases
|
||||||
|
{ elo: 1899, wins: 40, expected: 5 }, // Just below Legend ELO → Diamond
|
||||||
|
{ elo: 1900, wins: 39, expected: 5 }, // Enough ELO but not enough wins → Diamond
|
||||||
|
{ elo: 1100, wins: 0, expected: 0 }, // High ELO but no wins → Baby
|
||||||
|
]
|
||||||
|
|
||||||
|
for (const t of tests) {
|
||||||
|
expect(calculateTier(t.elo, t.wins)).toBe(t.expected)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('writes tier balance report', () => {
|
||||||
|
const __filename = fileURLToPath(import.meta.url)
|
||||||
|
const projectRoot = resolve(dirname(__filename), '..', '..', '..')
|
||||||
|
const loopDir = resolve(projectRoot, 'loop')
|
||||||
|
if (!existsSync(loopDir)) mkdirSync(loopDir, { recursive: true })
|
||||||
|
|
||||||
|
const header = [
|
||||||
|
'# Tier Balance Analysis',
|
||||||
|
'',
|
||||||
|
`Generated: ${new Date().toISOString().slice(0, 10)}`,
|
||||||
|
'',
|
||||||
|
'> Simulated fights between tier-representative bots to analyze',
|
||||||
|
'> skill gaps, upset rates, and ELO K-factor appropriateness.',
|
||||||
|
'',
|
||||||
|
]
|
||||||
|
|
||||||
|
const combined = [...header, ...reportLines].join('\n')
|
||||||
|
writeFileSync(resolve(loopDir, 'tier-balance-analysis.md'), combined)
|
||||||
|
console.log('\n Report written to loop/tier-balance-analysis.md')
|
||||||
|
expect(true).toBe(true)
|
||||||
|
})
|
||||||
|
})
|
||||||
Reference in New Issue
Block a user