test: verify all factual answers work with checkAnswer, fix 2 data bugs
- Hash rate question: added ×10^20 notation to answers array - Phishing question: added correct MC choice text to answers array - New tests: self-match, case-insensitive, MC correctness validation Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
a4b4963c32
commit
070e41f97b
@@ -1,5 +1,7 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { checkAnswer } from './answers.js'
|
||||
import { TEMPLATES } from './challenge-data.js'
|
||||
import { EXTRA_PROMPTS } from './challenges-extra.js'
|
||||
|
||||
describe('checkAnswer', () => {
|
||||
// --- Tier 1: exact match after normalization ---
|
||||
@@ -146,6 +148,59 @@ describe('checkAnswer', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('all factual prompt answers work with checkAnswer', () => {
|
||||
// Merge extras into templates (same as challenges.ts does)
|
||||
const templates = TEMPLATES.map(t => {
|
||||
const extras = EXTRA_PROMPTS[t.type]
|
||||
return extras ? { ...t, prompts: [...t.prompts, ...extras] } : t
|
||||
})
|
||||
|
||||
const factualTemplates = templates.filter(t => t.scoring === 'factual')
|
||||
|
||||
it('every factual prompt has at least one answer', () => {
|
||||
for (const t of factualTemplates) {
|
||||
for (const p of t.prompts) {
|
||||
expect(p.answers, `${t.type}: "${p.prompt}" has no answers`).toBeTruthy()
|
||||
expect(p.answers!.length, `${t.type}: "${p.prompt}" has empty answers array`).toBeGreaterThan(0)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
it('every answer self-matches with checkAnswer (case-insensitive)', () => {
|
||||
let checked = 0
|
||||
for (const t of factualTemplates) {
|
||||
for (const p of t.prompts) {
|
||||
if (!p.answers) continue
|
||||
for (const answer of p.answers) {
|
||||
const score = checkAnswer(answer, p.answers)
|
||||
expect(score, `${t.type}: answer "${answer}" for "${p.prompt}" should self-match`).toBeGreaterThanOrEqual(0.75)
|
||||
// Also test uppercase variant
|
||||
const upper = checkAnswer(answer.toUpperCase(), p.answers)
|
||||
expect(upper, `${t.type}: UPPER "${answer.toUpperCase()}" should match`).toBeGreaterThanOrEqual(0.75)
|
||||
checked++
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(checked).toBeGreaterThan(100)
|
||||
})
|
||||
|
||||
it('MC choices always include the correct answer', () => {
|
||||
let checked = 0
|
||||
for (const t of factualTemplates) {
|
||||
for (const p of t.prompts) {
|
||||
if (!p.choices || !p.answers) continue
|
||||
// At least one choice should match at least one answer
|
||||
const hasCorrect = p.choices.some(choice =>
|
||||
checkAnswer(choice, p.answers!) > 0
|
||||
)
|
||||
expect(hasCorrect, `${t.type}: "${p.prompt}" choices ${JSON.stringify(p.choices)} don't include correct answer ${JSON.stringify(p.answers)}`).toBe(true)
|
||||
checked++
|
||||
}
|
||||
}
|
||||
expect(checked).toBeGreaterThan(50)
|
||||
})
|
||||
})
|
||||
|
||||
describe('checkAnswer performance', () => {
|
||||
it('completes 1000 checks in under 50ms (<0.05ms each)', () => {
|
||||
const answers = ['Paris', 'London', 'Tokyo']
|
||||
|
||||
Reference in New Issue
Block a user