Files
botfights/server/src/engine/answers.ts
T
DorianandClaude Opus 4.6 4d6e50d988 fix: checkAnswer best-match + decimal preservation + prompt data fixes
- checkAnswer now returns highest score across all accepted answers
  instead of first match, fixing 95 false-low-confidence results
- Skip string containment for purely numeric strings to prevent
  false positives like "1000" matching inside "10000"
- Preserve decimal points in normalize() (42.0 no longer becomes 420)
- Use word-boundary regex for number matching in responses
- Fix 47 wrong choices scoring too high (comma-formatted numbers,
  verbose choices matching terse answers)
- Fix 17 prompts where no choice matched any accepted answer
- Challenge audit now reports zero failures across all 1472 prompts

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-13 04:35:54 +00:00

202 lines
7.9 KiB
TypeScript

/**
* Answer checking for factual challenges.
* Handles: case insensitivity, numeric equivalence, containment matching,
* number words (forty = 40), basic stemming, contraction normalization.
*/
const NUMBER_WORDS: Record<string, number> = {
zero: 0, one: 1, two: 2, three: 3, four: 4, five: 5,
six: 6, seven: 7, eight: 8, nine: 9, ten: 10,
eleven: 11, twelve: 12, thirteen: 13, fourteen: 14, fifteen: 15,
sixteen: 16, seventeen: 17, eighteen: 18, nineteen: 19, twenty: 20,
thirty: 30, forty: 40, fifty: 50, sixty: 60, seventy: 70,
eighty: 80, ninety: 90, hundred: 100, thousand: 1000, million: 1000000,
}
function normalize(s: string): string {
return s
.toLowerCase()
.trim()
.replace(/[,!?;:'"()\[\]{}<>]/g, '') // strip punctuation (NOT periods)
.replace(/\.(?!\d)/g, '') // strip periods NOT followed by a digit (preserve decimals)
.replace(/\s+/g, ' ') // collapse whitespace
.replace(/^(the|a|an|its|it is|it's) /i, '') // strip leading articles
.trim()
}
/** Expand contractions so "can't" matches "cannot", "don't" matches "do not" etc. */
function expandContractions(s: string): string {
return s
.replace(/\bcan'?t\b/gi, 'cannot')
.replace(/\bdon'?t\b/gi, 'do not')
.replace(/\bwon'?t\b/gi, 'will not')
.replace(/\bdoesn'?t\b/gi, 'does not')
.replace(/\bisn'?t\b/gi, 'is not')
.replace(/\baren'?t\b/gi, 'are not')
.replace(/\bwasn'?t\b/gi, 'was not')
.replace(/\bweren'?t\b/gi, 'were not')
.replace(/\bhasn'?t\b/gi, 'has not')
.replace(/\bhaven'?t\b/gi, 'have not')
.replace(/\bhadn'?t\b/gi, 'had not')
.replace(/\bcouldn'?t\b/gi, 'could not')
.replace(/\bshouldn'?t\b/gi, 'should not')
.replace(/\bwouldn'?t\b/gi, 'would not')
.replace(/\bit'?s\b/gi, 'it is')
.replace(/\bthat'?s\b/gi, 'that is')
.replace(/\bthey'?re\b/gi, 'they are')
.replace(/\bwe'?re\b/gi, 'we are')
.replace(/\byou'?re\b/gi, 'you are')
}
/** Basic English stemming: strip common suffixes for loose comparison. */
function stem(word: string): string {
if (word.length <= 3) return word
// Plural: buses → bus, foxes → fox, tries → tri (imperfect but ok)
if (word.endsWith('ies') && word.length > 4) return word.slice(0, -3) + 'y'
if (word.endsWith('ses') || word.endsWith('xes') || word.endsWith('zes') || word.endsWith('ches') || word.endsWith('shes')) {
return word.slice(0, -2)
}
if (word.endsWith('s') && !word.endsWith('ss')) return word.slice(0, -1)
return word
}
/** Stem all words in a string. */
function stemAll(s: string): string {
return s.split(/\s+/).map(stem).join(' ')
}
function tryParseNumber(s: string): number | null {
const cleaned = normalize(s)
// Direct numeric parse
const direct = Number(cleaned)
if (!isNaN(direct) && cleaned !== '') return direct
// Number word lookup
if (NUMBER_WORDS[cleaned] !== undefined) return NUMBER_WORDS[cleaned]
// Compound number words: "twenty one" → 21, "three hundred" → 300
const words = cleaned.split(/[\s-]+/)
if (words.length <= 4 && words.every(w => NUMBER_WORDS[w] !== undefined)) {
let total = 0
let current = 0
for (const w of words) {
const val = NUMBER_WORDS[w]
if (val >= 100) {
current = (current || 1) * val
} else {
current += val
}
}
total += current
return total
}
return null
}
/** Check if a string is purely digits (after normalization). */
function isPurelyNumeric(s: string): boolean {
return /^\d+$/.test(s)
}
/**
* Check if a bot's response matches any of the accepted answers.
* Returns a confidence score: 1.0 = definite match, 0.5-0.9 = partial, 0 = no match.
* Uses best-match: tries all accepted answers and returns the highest score.
*/
export function checkAnswer(response: string | null, acceptedAnswers: string[]): number {
if (!response || response.trim() === '') return 0
const normResponse = normalize(response)
if (normResponse === '') return 0
// Pre-compute expanded/stemmed versions once
const expandedResponse = normalize(expandContractions(response))
const stemmedResponse = stemAll(normResponse)
let maxScore = 0
for (const accepted of acceptedAnswers) {
const normAccepted = normalize(accepted)
const expandedAccepted = normalize(expandContractions(accepted))
const stemmedAccepted = stemAll(normAccepted)
// 1. Exact match after normalization
if (normResponse === normAccepted) return 1.0
// 2. Exact match after contraction expansion
if (expandedResponse === expandedAccepted) return 1.0
// 3. Exact match after stemming (catches plural/singular)
if (stemmedResponse === stemmedAccepted) return 1.0
// 4. Numeric equivalence
const respNum = tryParseNumber(normResponse)
const accNum = tryParseNumber(normAccepted)
if (respNum !== null && accNum !== null && respNum === accNum) return 1.0
// Guard: skip string containment for purely numeric strings — rule 4 handles those
const bothNumeric = isPurelyNumeric(normResponse) && isPurelyNumeric(normAccepted)
// 5. Response contains the accepted answer (or stemmed version)
if (!bothNumeric && normResponse.includes(normAccepted) && normAccepted.length >= 2)
return 1.0
if (!bothNumeric && stemmedResponse.includes(stemmedAccepted) && stemmedAccepted.length >= 2)
maxScore = Math.max(maxScore, 0.95)
// 6. Accepted answer contains the response (for short definitive answers)
if (!bothNumeric && normAccepted.includes(normResponse) && normResponse.length >= 3)
maxScore = Math.max(maxScore, 0.8)
if (!bothNumeric && stemmedAccepted.includes(stemmedResponse) && stemmedResponse.length >= 3)
maxScore = Math.max(maxScore, 0.75)
// 7. Check if number appears anywhere in a longer response
if (accNum !== null) {
const numStr = String(accNum)
// Only match if the number appears as a whole token, not as a substring of a larger number
const numRegex = new RegExp(`(?<![\\d])${numStr.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?![\\d])`)
if (numRegex.test(normResponse)) return 1.0
// Check number words in response
const responseWords = normResponse.split(/\s+/)
for (const w of responseWords) {
const parsed = tryParseNumber(w)
if (parsed !== null && parsed === accNum) maxScore = Math.max(maxScore, 0.9)
}
}
// 8. Word-level containment — all words of the answer appear in the response
const acceptedWords = normAccepted.split(/\s+/)
if (acceptedWords.length >= 2) {
const allFound = acceptedWords.every(w => normResponse.includes(w))
if (allFound) maxScore = Math.max(maxScore, 0.9)
// Try with stemming
const stemmedAccWords = stemmedAccepted.split(/\s+/)
const allStemFound = stemmedAccWords.every(w => stemmedResponse.includes(w))
if (allStemFound) maxScore = Math.max(maxScore, 0.85)
}
// 9. Contraction-expanded word containment
if (expandedAccepted.split(/\s+/).length >= 2) {
const allFound = expandedAccepted.split(/\s+/).every(w => expandedResponse.includes(w))
if (allFound) maxScore = Math.max(maxScore, 0.85)
}
}
if (maxScore > 0) return maxScore
// 10. For true/false questions, check if the response starts with the right keyword
const tfAnswer = acceptedAnswers.find(a => a.toLowerCase() === 'true' || a.toLowerCase() === 'false')
if (tfAnswer) {
const firstWord = normResponse.split(/\s+/)[0]
if (firstWord === tfAnswer.toLowerCase()) return 1.0
// "that's true" / "this is false" / "yes" for true / "no" for false
if (normResponse.includes(tfAnswer.toLowerCase())) return 0.9
// "yes" ≈ true, "no" ≈ false
if (tfAnswer.toLowerCase() === 'true' && (firstWord === 'yes' || firstWord === 'correct' || firstWord === 'right')) return 0.9
if (tfAnswer.toLowerCase() === 'false' && (firstWord === 'no' || firstWord === 'incorrect' || firstWord === 'wrong')) return 0.9
}
return 0
}