diff --git a/server/src/engine/difficulty.test.ts b/server/src/engine/difficulty.test.ts new file mode 100644 index 0000000..d282807 --- /dev/null +++ b/server/src/engine/difficulty.test.ts @@ -0,0 +1,160 @@ +import { describe, it, expect } from 'vitest' +import { roundToDifficulty, pickChallenge, getAllChallengeTypes } from './challenges.js' +import { TEMPLATES, type PromptDifficulty } from './challenge-data.js' +import { EXTRA_PROMPTS } from './challenges-extra.js' +import { BITCOIN_PROMPTS } from './challenges-bitcoin.js' +import { CONSPIRACY_PROMPTS } from './challenges-conspiracy.js' +import { PC_PROMPTS } from './challenges-pc.js' +import { VIBE_PROMPTS } from './challenges-vibe.js' +import { writeFileSync, mkdirSync, existsSync } from 'node:fs' +import { resolve, dirname } from 'node:path' +import { fileURLToPath } from 'node:url' + +// Build merged templates (non-mutating copy for counting) +function buildMergedTemplates() { + return TEMPLATES.map(t => { + const allPrompts = [...t.prompts] + for (const source of [EXTRA_PROMPTS, BITCOIN_PROMPTS, CONSPIRACY_PROMPTS, PC_PROMPTS, VIBE_PROMPTS]) { + const extras = source[t.type] + if (extras) allPrompts.push(...extras) + } + return { ...t, prompts: allPrompts } + }) +} + +describe('roundToDifficulty', () => { + it('rounds 1-2 return easy', () => { + expect(roundToDifficulty(1)).toBe('easy') + expect(roundToDifficulty(2)).toBe('easy') + }) + + it('rounds 3-4 return medium', () => { + expect(roundToDifficulty(3)).toBe('medium') + expect(roundToDifficulty(4)).toBe('medium') + }) + + it('rounds 5+ return hard', () => { + expect(roundToDifficulty(5)).toBe('hard') + expect(roundToDifficulty(6)).toBe('hard') + expect(roundToDifficulty(10)).toBe('hard') + expect(roundToDifficulty(20)).toBe('hard') + }) +}) + +describe('pickChallenge difficulty filtering', () => { + it('pickChallenge with roundNumber filters by difficulty when tagged prompts exist', () => { + // Pick many challenges at round 5 (hard) and check that the prompt pool was filtered + // We can't verify the internal filtering directly, but we can verify the function works + const usedTypes = new Set() + for (let i = 0; i < 50; i++) { + const challenge = pickChallenge(usedTypes, null, undefined, 5) + expect(challenge).toBeDefined() + expect(challenge.prompt).toBeTruthy() + } + }) + + it('pickChallenge without roundNumber does not filter by difficulty', () => { + const usedTypes = new Set() + const challenge = pickChallenge(usedTypes, null) + expect(challenge).toBeDefined() + expect(challenge.prompt).toBeTruthy() + }) + + it('pickChallenge at round 1 prefers easy prompts', () => { + const usedTypes = new Set() + const challenge = pickChallenge(usedTypes, null, undefined, 1) + expect(challenge).toBeDefined() + }) +}) + +describe('difficulty distribution audit', () => { + it('counts prompts per difficulty level per challenge type', () => { + const templates = buildMergedTemplates() + const difficulties: PromptDifficulty[] = ['easy', 'medium', 'hard'] + const matrix: Record> = {} + const typeTotals: Record = {} + + for (const t of templates) { + matrix[t.type] = { easy: 0, medium: 0, hard: 0, untagged: 0 } + typeTotals[t.type] = t.prompts.length + + for (const p of t.prompts) { + if (p.difficulty && difficulties.includes(p.difficulty)) { + matrix[t.type][p.difficulty]++ + } else { + matrix[t.type].untagged++ + } + } + } + + // Console output + console.log('\n══════════════════════════════════════════════════════') + console.log(' DIFFICULTY DISTRIBUTION MATRIX') + console.log('══════════════════════════════════════════════════════') + const header = ' Type'.padEnd(24) + 'Easy'.padStart(8) + 'Medium'.padStart(8) + 'Hard'.padStart(8) + 'Untagged'.padStart(10) + 'Total'.padStart(8) + console.log(header) + console.log(' ' + '─'.repeat(62)) + + const sortedTypes = Object.keys(matrix).sort() + const globalCounts = { easy: 0, medium: 0, hard: 0, untagged: 0, total: 0 } + const typesLackingHard: string[] = [] + const typesLackingMedium: string[] = [] + + for (const type of sortedTypes) { + const row = matrix[type] + const total = typeTotals[type] + const line = ` ${type}`.padEnd(24) + + String(row.easy).padStart(8) + + String(row.medium).padStart(8) + + String(row.hard).padStart(8) + + String(row.untagged).padStart(10) + + String(total).padStart(8) + console.log(line) + + globalCounts.easy += row.easy + globalCounts.medium += row.medium + globalCounts.hard += row.hard + globalCounts.untagged += row.untagged + globalCounts.total += total + + if (row.hard === 0) typesLackingHard.push(type) + if (row.medium === 0) typesLackingMedium.push(type) + } + + console.log(' ' + '─'.repeat(62)) + console.log(` ${'TOTAL'.padEnd(22)}${String(globalCounts.easy).padStart(8)}${String(globalCounts.medium).padStart(8)}${String(globalCounts.hard).padStart(8)}${String(globalCounts.untagged).padStart(10)}${String(globalCounts.total).padStart(8)}`) + console.log('') + console.log(` Types lacking hard prompts: ${typesLackingHard.length > 0 ? typesLackingHard.join(', ') : 'none'}`) + console.log(` Types lacking medium prompts: ${typesLackingMedium.length > 0 ? typesLackingMedium.join(', ') : 'none'}`) + console.log('══════════════════════════════════════════════════════\n') + + // Write markdown report + const __filename = fileURLToPath(import.meta.url) + const projectRoot = resolve(dirname(__filename), '..', '..', '..') + const loopDir = resolve(projectRoot, 'loop') + if (!existsSync(loopDir)) mkdirSync(loopDir, { recursive: true }) + + let md = `# Difficulty Distribution\n\nGenerated: ${new Date().toISOString().slice(0, 10)}\n\n` + md += `## Summary\n\n` + md += `- **Total prompts**: ${globalCounts.total}\n` + md += `- **Tagged**: ${globalCounts.easy + globalCounts.medium + globalCounts.hard} (${((globalCounts.easy + globalCounts.medium + globalCounts.hard) / globalCounts.total * 100).toFixed(1)}%)\n` + md += `- **Untagged**: ${globalCounts.untagged} (${(globalCounts.untagged / globalCounts.total * 100).toFixed(1)}%)\n\n` + md += `## Distribution\n\n` + md += `| Type | Easy | Medium | Hard | Untagged | Total |\n` + md += `|------|-----:|-------:|-----:|---------:|------:|\n` + for (const type of sortedTypes) { + const row = matrix[type] + md += `| ${type} | ${row.easy} | ${row.medium} | ${row.hard} | ${row.untagged} | ${typeTotals[type]} |\n` + } + md += `| **TOTAL** | **${globalCounts.easy}** | **${globalCounts.medium}** | **${globalCounts.hard}** | **${globalCounts.untagged}** | **${globalCounts.total}** |\n\n` + md += `## Gaps\n\n` + md += `- Types lacking hard prompts: ${typesLackingHard.length > 0 ? typesLackingHard.join(', ') : 'none'}\n` + md += `- Types lacking medium prompts: ${typesLackingMedium.length > 0 ? typesLackingMedium.join(', ') : 'none'}\n` + + writeFileSync(resolve(loopDir, 'difficulty-distribution.md'), md) + console.log(` Report written to loop/difficulty-distribution.md`) + + // Assertions + expect(globalCounts.total).toBeGreaterThan(800) + }) +})