botfights/server/src/engine/answers.test.ts

import { describe, it, expect } from 'vitest'
import { checkAnswer } from './answers.js'
import { TEMPLATES } from './challenge-data.js'
import { EXTRA_PROMPTS } from './challenges-extra.js'

describe('checkAnswer', () => {
  // --- Tier 1: exact match after normalization ---
  it('exact match (case insensitive)', () => {
    expect(checkAnswer('Paris', ['paris'])).toBe(1.0)
    expect(checkAnswer('PARIS', ['paris'])).toBe(1.0)
    expect(checkAnswer('paris', ['Paris'])).toBe(1.0)
  })

  it('exact match strips punctuation', () => {
    expect(checkAnswer('Paris!', ['paris'])).toBe(1.0)
    expect(checkAnswer('  Paris  ', ['paris'])).toBe(1.0)
  })

  it('strips leading articles', () => {
    expect(checkAnswer('the Eiffel Tower', ['Eiffel Tower'])).toBe(1.0)
    expect(checkAnswer('an apple', ['apple'])).toBe(1.0)
    expect(checkAnswer('a dog', ['dog'])).toBe(1.0)
  })

  // --- Tier 2: contraction expansion ---
  it('contraction expansion', () => {
    expect(checkAnswer("can't", ['cannot'])).toBe(1.0)
    expect(checkAnswer("don't", ['do not'])).toBe(1.0)
    expect(checkAnswer("it's", ['it is'])).toBe(1.0)
    expect(checkAnswer("they're", ['they are'])).toBe(1.0)
  })

  // --- Tier 3: stemmed match ---
  it('stemmed match (plural/singular)', () => {
    expect(checkAnswer('dogs', ['dog'])).toBe(1.0)
    expect(checkAnswer('buses', ['bus'])).toBe(1.0)
    expect(checkAnswer('foxes', ['fox'])).toBe(1.0)
  })

  // --- Tier 4: numeric equivalence ---
  it('numeric equivalence', () => {
    expect(checkAnswer('42', ['42'])).toBe(1.0)
    expect(checkAnswer('42.0', ['42'])).toBe(1.0)
    expect(checkAnswer('forty', ['40'])).toBe(1.0)
    expect(checkAnswer('twenty', ['20'])).toBe(1.0)
    expect(checkAnswer('one hundred', ['100'])).toBe(1.0)
    expect(checkAnswer('three hundred', ['300'])).toBe(1.0)
  })

  it('compound number words', () => {
    expect(checkAnswer('twenty one', ['21'])).toBe(1.0)
    expect(checkAnswer('three hundred', ['300'])).toBe(1.0)
  })

  // --- Tier 5: containment ---
  it('response contains accepted answer', () => {
    expect(checkAnswer('The answer is Paris of course', ['paris'])).toBe(1.0)
    expect(checkAnswer('I believe the answer is 42', ['42'])).toBe(1.0)
  })

  it('stemmed containment', () => {
    const score = checkAnswer('There are many dogs in the park', ['dog'])
    expect(score).toBeGreaterThanOrEqual(0.95)
  })

  // --- Tier 6: accepted contains response ---
  it('accepted answer contains the response (short answer)', () => {
    const score = checkAnswer('Paris', ['The city of Paris'])
    expect(score).toBeGreaterThanOrEqual(0.75)
  })

  // --- Tier 7: number in longer response ---
  it('number embedded in response', () => {
    expect(checkAnswer('I think it is about 42 meters', ['42'])).toBe(1.0)
  })

  it('number word in response matches', () => {
    const score = checkAnswer('I think forty is the answer', ['40'])
    expect(score).toBeGreaterThanOrEqual(0.9)
  })

  // --- Tier 8: word-level containment ---
  it('all words of answer appear in response', () => {
    const score = checkAnswer('The Great Wall is in China and is very long', ['Great Wall'])
    expect(score).toBeGreaterThanOrEqual(0.9)
  })

  // --- Tier 10: true/false ---
  it('true/false match', () => {
    expect(checkAnswer('True', ['true'])).toBe(1.0)
    expect(checkAnswer('true, definitely', ['true'])).toBeGreaterThanOrEqual(0.9)
    expect(checkAnswer('yes', ['true'])).toBeGreaterThanOrEqual(0.9)
    expect(checkAnswer('correct', ['true'])).toBeGreaterThanOrEqual(0.9)
    expect(checkAnswer('no', ['false'])).toBeGreaterThanOrEqual(0.9)
    expect(checkAnswer('wrong', ['false'])).toBeGreaterThanOrEqual(0.9)
    expect(checkAnswer('incorrect', ['false'])).toBeGreaterThanOrEqual(0.9)
  })

  // --- Edge cases ---
  it('null response returns 0', () => {
    expect(checkAnswer(null, ['Paris'])).toBe(0)
  })

  it('empty response returns 0', () => {
    expect(checkAnswer('', ['Paris'])).toBe(0)
    expect(checkAnswer('   ', ['Paris'])).toBe(0)
  })

  it('completely wrong answer returns 0', () => {
    expect(checkAnswer('banana', ['Paris'])).toBe(0)
  })

  it('multiple accepted answers', () => {
    expect(checkAnswer('NYC', ['New York City', 'NYC', 'New York'])).toBe(1.0)
    // "New York" contained in "New York City" (tier 6: accepted contains response) scores 0.8
    expect(checkAnswer('New York', ['New York City', 'NYC', 'New York'])).toBeGreaterThanOrEqual(0.8)
  })

  it('very short accepted answer needs >= 2 chars for containment', () => {
    // Single char answers shouldn't trigger containment on random text
    const score = checkAnswer('absolutely nothing relevant', ['a'])
    // 'a' length is 1, so containment shouldn't trigger (requires >= 2)
    expect(score).toBeLessThanOrEqual(0.8)
  })

  it('special characters in answer', () => {
    expect(checkAnswer('C++', ['C++'])).toBe(1.0)
    expect(checkAnswer('c++', ['C++'])).toBe(1.0)
    expect(checkAnswer('$100', ['$100'])).toBe(1.0)
    expect(checkAnswer('42%', ['42'])).toBe(1.0)
  })

  it('very long answer still matches if correct keyword present', () => {
    const longAnswer = 'Well, after much deliberation and careful consideration of all the facts, ' +
      'weighing the evidence both for and against, consulting multiple sources, and thinking deeply ' +
      'about the philosophical implications, I believe the answer you are looking for is Paris, ' +
      'which is of course the beautiful capital of France.'
    expect(checkAnswer(longAnswer, ['Paris'])).toBe(1.0)
  })

  it('very long answer with no match returns 0', () => {
    const longWrong = 'A'.repeat(2000) + ' banana ' + 'B'.repeat(2000)
    expect(checkAnswer(longWrong, ['Paris'])).toBe(0)
  })

  it('whitespace-only answer returns 0', () => {
    expect(checkAnswer('\t\n  \r', ['Paris'])).toBe(0)
  })
})

describe('all factual prompt answers work with checkAnswer', () => {
  // Merge extras into templates (same as challenges.ts does)
  const templates = TEMPLATES.map(t => {
    const extras = EXTRA_PROMPTS[t.type]
    return extras ? { ...t, prompts: [...t.prompts, ...extras] } : t
  })

  const factualTemplates = templates.filter(t => t.scoring === 'factual')

  it('every factual prompt has at least one answer', () => {
    for (const t of factualTemplates) {
      for (const p of t.prompts) {
        expect(p.answers, `${t.type}: "${p.prompt}" has no answers`).toBeTruthy()
        expect(p.answers!.length, `${t.type}: "${p.prompt}" has empty answers array`).toBeGreaterThan(0)
      }
    }
  })

  it('every answer self-matches with checkAnswer (case-insensitive)', () => {
    let checked = 0
    for (const t of factualTemplates) {
      for (const p of t.prompts) {
        if (!p.answers) continue
        for (const answer of p.answers) {
          const score = checkAnswer(answer, p.answers)
          expect(score, `${t.type}: answer "${answer}" for "${p.prompt}" should self-match`).toBeGreaterThanOrEqual(0.75)
          // Also test uppercase variant
          const upper = checkAnswer(answer.toUpperCase(), p.answers)
          expect(upper, `${t.type}: UPPER "${answer.toUpperCase()}" should match`).toBeGreaterThanOrEqual(0.75)
          checked++
        }
      }
    }
    expect(checked).toBeGreaterThan(100)
  })

  it('MC choices always include the correct answer', () => {
    let checked = 0
    for (const t of factualTemplates) {
      for (const p of t.prompts) {
        if (!p.choices || !p.answers) continue
        // At least one choice should match at least one answer
        const hasCorrect = p.choices.some(choice =>
          checkAnswer(choice, p.answers!) > 0
        )
        expect(hasCorrect, `${t.type}: "${p.prompt}" choices ${JSON.stringify(p.choices)} don't include correct answer ${JSON.stringify(p.answers)}`).toBe(true)
        checked++
      }
    }
    expect(checked).toBeGreaterThan(50)
  })
})

describe('checkAnswer performance', () => {
  it('completes 1000 checks in under 50ms (<0.05ms each)', () => {
    const answers = ['Paris', 'London', 'Tokyo']
    const start = performance.now()
    for (let i = 0; i < 1000; i++) {
      checkAnswer('I think the answer is probably Paris', answers)
    }
    const elapsed = performance.now() - start
    expect(elapsed).toBeLessThan(50)
  })

  it('no regex backtracking on adversarial input', () => {
    // ReDoS-style strings that could cause catastrophic backtracking
    const adversarial = [
      'a'.repeat(10000),
      'a'.repeat(5000) + '!' + 'a'.repeat(5000),
      'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaab',
      '(((((((((((((((((((((((((((((((',
      'x'.repeat(2000) + 'y'.repeat(2000),
    ]
    const start = performance.now()
    for (const input of adversarial) {
      checkAnswer(input, ['correct answer', '42', 'true'])
    }
    const elapsed = performance.now() - start
    // Must complete in <100ms total for all adversarial inputs
    expect(elapsed).toBeLessThan(100)
  })
})