import { describe, it, expect } from 'vitest' import { checkAnswer } from './answers.js' import { TEMPLATES } from './challenge-data.js' import { EXTRA_PROMPTS } from './challenges-extra.js' describe('checkAnswer', () => { // --- Tier 1: exact match after normalization --- it('exact match (case insensitive)', () => { expect(checkAnswer('Paris', ['paris'])).toBe(1.0) expect(checkAnswer('PARIS', ['paris'])).toBe(1.0) expect(checkAnswer('paris', ['Paris'])).toBe(1.0) }) it('exact match strips punctuation', () => { expect(checkAnswer('Paris!', ['paris'])).toBe(1.0) expect(checkAnswer(' Paris ', ['paris'])).toBe(1.0) }) it('strips leading articles', () => { expect(checkAnswer('the Eiffel Tower', ['Eiffel Tower'])).toBe(1.0) expect(checkAnswer('an apple', ['apple'])).toBe(1.0) expect(checkAnswer('a dog', ['dog'])).toBe(1.0) }) // --- Tier 2: contraction expansion --- it('contraction expansion', () => { expect(checkAnswer("can't", ['cannot'])).toBe(1.0) expect(checkAnswer("don't", ['do not'])).toBe(1.0) expect(checkAnswer("it's", ['it is'])).toBe(1.0) expect(checkAnswer("they're", ['they are'])).toBe(1.0) }) // --- Tier 3: stemmed match --- it('stemmed match (plural/singular)', () => { expect(checkAnswer('dogs', ['dog'])).toBe(1.0) expect(checkAnswer('buses', ['bus'])).toBe(1.0) expect(checkAnswer('foxes', ['fox'])).toBe(1.0) }) // --- Tier 4: numeric equivalence --- it('numeric equivalence', () => { expect(checkAnswer('42', ['42'])).toBe(1.0) expect(checkAnswer('42.0', ['42'])).toBe(1.0) expect(checkAnswer('forty', ['40'])).toBe(1.0) expect(checkAnswer('twenty', ['20'])).toBe(1.0) expect(checkAnswer('one hundred', ['100'])).toBe(1.0) expect(checkAnswer('three hundred', ['300'])).toBe(1.0) }) it('compound number words', () => { expect(checkAnswer('twenty one', ['21'])).toBe(1.0) expect(checkAnswer('three hundred', ['300'])).toBe(1.0) }) // --- Tier 5: containment --- it('response contains accepted answer', () => { expect(checkAnswer('The answer is Paris of course', ['paris'])).toBe(1.0) expect(checkAnswer('I believe the answer is 42', ['42'])).toBe(1.0) }) it('stemmed containment', () => { const score = checkAnswer('There are many dogs in the park', ['dog']) expect(score).toBeGreaterThanOrEqual(0.95) }) // --- Tier 6: accepted contains response --- it('accepted answer contains the response (short answer)', () => { const score = checkAnswer('Paris', ['The city of Paris']) expect(score).toBeGreaterThanOrEqual(0.75) }) // --- Tier 7: number in longer response --- it('number embedded in response', () => { expect(checkAnswer('I think it is about 42 meters', ['42'])).toBe(1.0) }) it('number word in response matches', () => { const score = checkAnswer('I think forty is the answer', ['40']) expect(score).toBeGreaterThanOrEqual(0.9) }) // --- Tier 8: word-level containment --- it('all words of answer appear in response', () => { const score = checkAnswer('The Great Wall is in China and is very long', ['Great Wall']) expect(score).toBeGreaterThanOrEqual(0.9) }) // --- Tier 10: true/false --- it('true/false match', () => { expect(checkAnswer('True', ['true'])).toBe(1.0) expect(checkAnswer('true, definitely', ['true'])).toBeGreaterThanOrEqual(0.9) expect(checkAnswer('yes', ['true'])).toBeGreaterThanOrEqual(0.9) expect(checkAnswer('correct', ['true'])).toBeGreaterThanOrEqual(0.9) expect(checkAnswer('no', ['false'])).toBeGreaterThanOrEqual(0.9) expect(checkAnswer('wrong', ['false'])).toBeGreaterThanOrEqual(0.9) expect(checkAnswer('incorrect', ['false'])).toBeGreaterThanOrEqual(0.9) }) // --- Edge cases --- it('null response returns 0', () => { expect(checkAnswer(null, ['Paris'])).toBe(0) }) it('empty response returns 0', () => { expect(checkAnswer('', ['Paris'])).toBe(0) expect(checkAnswer(' ', ['Paris'])).toBe(0) }) it('completely wrong answer returns 0', () => { expect(checkAnswer('banana', ['Paris'])).toBe(0) }) it('multiple accepted answers', () => { expect(checkAnswer('NYC', ['New York City', 'NYC', 'New York'])).toBe(1.0) // "New York" contained in "New York City" (tier 6: accepted contains response) scores 0.8 expect(checkAnswer('New York', ['New York City', 'NYC', 'New York'])).toBeGreaterThanOrEqual(0.8) }) it('very short accepted answer needs >= 2 chars for containment', () => { // Single char answers shouldn't trigger containment on random text const score = checkAnswer('absolutely nothing relevant', ['a']) // 'a' length is 1, so containment shouldn't trigger (requires >= 2) expect(score).toBeLessThanOrEqual(0.8) }) it('special characters in answer', () => { expect(checkAnswer('C++', ['C++'])).toBe(1.0) expect(checkAnswer('c++', ['C++'])).toBe(1.0) expect(checkAnswer('$100', ['$100'])).toBe(1.0) expect(checkAnswer('42%', ['42'])).toBe(1.0) }) it('very long answer still matches if correct keyword present', () => { const longAnswer = 'Well, after much deliberation and careful consideration of all the facts, ' + 'weighing the evidence both for and against, consulting multiple sources, and thinking deeply ' + 'about the philosophical implications, I believe the answer you are looking for is Paris, ' + 'which is of course the beautiful capital of France.' expect(checkAnswer(longAnswer, ['Paris'])).toBe(1.0) }) it('very long answer with no match returns 0', () => { const longWrong = 'A'.repeat(2000) + ' banana ' + 'B'.repeat(2000) expect(checkAnswer(longWrong, ['Paris'])).toBe(0) }) it('whitespace-only answer returns 0', () => { expect(checkAnswer('\t\n \r', ['Paris'])).toBe(0) }) }) describe('all factual prompt answers work with checkAnswer', () => { // Merge extras into templates (same as challenges.ts does) const templates = TEMPLATES.map(t => { const extras = EXTRA_PROMPTS[t.type] return extras ? { ...t, prompts: [...t.prompts, ...extras] } : t }) const factualTemplates = templates.filter(t => t.scoring === 'factual') it('every factual prompt has at least one answer', () => { for (const t of factualTemplates) { for (const p of t.prompts) { expect(p.answers, `${t.type}: "${p.prompt}" has no answers`).toBeTruthy() expect(p.answers!.length, `${t.type}: "${p.prompt}" has empty answers array`).toBeGreaterThan(0) } } }) it('every answer self-matches with checkAnswer (case-insensitive)', () => { let checked = 0 for (const t of factualTemplates) { for (const p of t.prompts) { if (!p.answers) continue for (const answer of p.answers) { const score = checkAnswer(answer, p.answers) expect(score, `${t.type}: answer "${answer}" for "${p.prompt}" should self-match`).toBeGreaterThanOrEqual(0.75) // Also test uppercase variant const upper = checkAnswer(answer.toUpperCase(), p.answers) expect(upper, `${t.type}: UPPER "${answer.toUpperCase()}" should match`).toBeGreaterThanOrEqual(0.75) checked++ } } } expect(checked).toBeGreaterThan(100) }) it('MC choices always include the correct answer', () => { let checked = 0 for (const t of factualTemplates) { for (const p of t.prompts) { if (!p.choices || !p.answers) continue // At least one choice should match at least one answer const hasCorrect = p.choices.some(choice => checkAnswer(choice, p.answers!) > 0 ) expect(hasCorrect, `${t.type}: "${p.prompt}" choices ${JSON.stringify(p.choices)} don't include correct answer ${JSON.stringify(p.answers)}`).toBe(true) checked++ } } expect(checked).toBeGreaterThan(50) }) }) describe('checkAnswer performance', () => { it('completes 1000 checks in under 50ms (<0.05ms each)', () => { const answers = ['Paris', 'London', 'Tokyo'] const start = performance.now() for (let i = 0; i < 1000; i++) { checkAnswer('I think the answer is probably Paris', answers) } const elapsed = performance.now() - start expect(elapsed).toBeLessThan(50) }) it('no regex backtracking on adversarial input', () => { // ReDoS-style strings that could cause catastrophic backtracking const adversarial = [ 'a'.repeat(10000), 'a'.repeat(5000) + '!' + 'a'.repeat(5000), 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaab', '(((((((((((((((((((((((((((((((', 'x'.repeat(2000) + 'y'.repeat(2000), ] const start = performance.now() for (const input of adversarial) { checkAnswer(input, ['correct answer', '42', 'true']) } const elapsed = performance.now() - start // Must complete in <100ms total for all adversarial inputs expect(elapsed).toBeLessThan(100) }) })