- checkAnswer now returns highest score across all accepted answers instead of first match, fixing 95 false-low-confidence results - Skip string containment for purely numeric strings to prevent false positives like "1000" matching inside "10000" - Preserve decimal points in normalize() (42.0 no longer becomes 420) - Use word-boundary regex for number matching in responses - Fix 47 wrong choices scoring too high (comma-formatted numbers, verbose choices matching terse answers) - Fix 17 prompts where no choice matched any accepted answer - Challenge audit now reports zero failures across all 1472 prompts Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
202 lines
7.9 KiB
TypeScript
202 lines
7.9 KiB
TypeScript
/**
|
|
* Answer checking for factual challenges.
|
|
* Handles: case insensitivity, numeric equivalence, containment matching,
|
|
* number words (forty = 40), basic stemming, contraction normalization.
|
|
*/
|
|
|
|
const NUMBER_WORDS: Record<string, number> = {
|
|
zero: 0, one: 1, two: 2, three: 3, four: 4, five: 5,
|
|
six: 6, seven: 7, eight: 8, nine: 9, ten: 10,
|
|
eleven: 11, twelve: 12, thirteen: 13, fourteen: 14, fifteen: 15,
|
|
sixteen: 16, seventeen: 17, eighteen: 18, nineteen: 19, twenty: 20,
|
|
thirty: 30, forty: 40, fifty: 50, sixty: 60, seventy: 70,
|
|
eighty: 80, ninety: 90, hundred: 100, thousand: 1000, million: 1000000,
|
|
}
|
|
|
|
function normalize(s: string): string {
|
|
return s
|
|
.toLowerCase()
|
|
.trim()
|
|
.replace(/[,!?;:'"()\[\]{}<>]/g, '') // strip punctuation (NOT periods)
|
|
.replace(/\.(?!\d)/g, '') // strip periods NOT followed by a digit (preserve decimals)
|
|
.replace(/\s+/g, ' ') // collapse whitespace
|
|
.replace(/^(the|a|an|its|it is|it's) /i, '') // strip leading articles
|
|
.trim()
|
|
}
|
|
|
|
/** Expand contractions so "can't" matches "cannot", "don't" matches "do not" etc. */
|
|
function expandContractions(s: string): string {
|
|
return s
|
|
.replace(/\bcan'?t\b/gi, 'cannot')
|
|
.replace(/\bdon'?t\b/gi, 'do not')
|
|
.replace(/\bwon'?t\b/gi, 'will not')
|
|
.replace(/\bdoesn'?t\b/gi, 'does not')
|
|
.replace(/\bisn'?t\b/gi, 'is not')
|
|
.replace(/\baren'?t\b/gi, 'are not')
|
|
.replace(/\bwasn'?t\b/gi, 'was not')
|
|
.replace(/\bweren'?t\b/gi, 'were not')
|
|
.replace(/\bhasn'?t\b/gi, 'has not')
|
|
.replace(/\bhaven'?t\b/gi, 'have not')
|
|
.replace(/\bhadn'?t\b/gi, 'had not')
|
|
.replace(/\bcouldn'?t\b/gi, 'could not')
|
|
.replace(/\bshouldn'?t\b/gi, 'should not')
|
|
.replace(/\bwouldn'?t\b/gi, 'would not')
|
|
.replace(/\bit'?s\b/gi, 'it is')
|
|
.replace(/\bthat'?s\b/gi, 'that is')
|
|
.replace(/\bthey'?re\b/gi, 'they are')
|
|
.replace(/\bwe'?re\b/gi, 'we are')
|
|
.replace(/\byou'?re\b/gi, 'you are')
|
|
}
|
|
|
|
/** Basic English stemming: strip common suffixes for loose comparison. */
|
|
function stem(word: string): string {
|
|
if (word.length <= 3) return word
|
|
// Plural: buses → bus, foxes → fox, tries → tri (imperfect but ok)
|
|
if (word.endsWith('ies') && word.length > 4) return word.slice(0, -3) + 'y'
|
|
if (word.endsWith('ses') || word.endsWith('xes') || word.endsWith('zes') || word.endsWith('ches') || word.endsWith('shes')) {
|
|
return word.slice(0, -2)
|
|
}
|
|
if (word.endsWith('s') && !word.endsWith('ss')) return word.slice(0, -1)
|
|
return word
|
|
}
|
|
|
|
/** Stem all words in a string. */
|
|
function stemAll(s: string): string {
|
|
return s.split(/\s+/).map(stem).join(' ')
|
|
}
|
|
|
|
function tryParseNumber(s: string): number | null {
|
|
const cleaned = normalize(s)
|
|
|
|
// Direct numeric parse
|
|
const direct = Number(cleaned)
|
|
if (!isNaN(direct) && cleaned !== '') return direct
|
|
|
|
// Number word lookup
|
|
if (NUMBER_WORDS[cleaned] !== undefined) return NUMBER_WORDS[cleaned]
|
|
|
|
// Compound number words: "twenty one" → 21, "three hundred" → 300
|
|
const words = cleaned.split(/[\s-]+/)
|
|
if (words.length <= 4 && words.every(w => NUMBER_WORDS[w] !== undefined)) {
|
|
let total = 0
|
|
let current = 0
|
|
for (const w of words) {
|
|
const val = NUMBER_WORDS[w]
|
|
if (val >= 100) {
|
|
current = (current || 1) * val
|
|
} else {
|
|
current += val
|
|
}
|
|
}
|
|
total += current
|
|
return total
|
|
}
|
|
|
|
return null
|
|
}
|
|
|
|
/** Check if a string is purely digits (after normalization). */
|
|
function isPurelyNumeric(s: string): boolean {
|
|
return /^\d+$/.test(s)
|
|
}
|
|
|
|
/**
|
|
* Check if a bot's response matches any of the accepted answers.
|
|
* Returns a confidence score: 1.0 = definite match, 0.5-0.9 = partial, 0 = no match.
|
|
* Uses best-match: tries all accepted answers and returns the highest score.
|
|
*/
|
|
export function checkAnswer(response: string | null, acceptedAnswers: string[]): number {
|
|
if (!response || response.trim() === '') return 0
|
|
|
|
const normResponse = normalize(response)
|
|
if (normResponse === '') return 0
|
|
|
|
// Pre-compute expanded/stemmed versions once
|
|
const expandedResponse = normalize(expandContractions(response))
|
|
const stemmedResponse = stemAll(normResponse)
|
|
|
|
let maxScore = 0
|
|
|
|
for (const accepted of acceptedAnswers) {
|
|
const normAccepted = normalize(accepted)
|
|
const expandedAccepted = normalize(expandContractions(accepted))
|
|
const stemmedAccepted = stemAll(normAccepted)
|
|
|
|
// 1. Exact match after normalization
|
|
if (normResponse === normAccepted) return 1.0
|
|
|
|
// 2. Exact match after contraction expansion
|
|
if (expandedResponse === expandedAccepted) return 1.0
|
|
|
|
// 3. Exact match after stemming (catches plural/singular)
|
|
if (stemmedResponse === stemmedAccepted) return 1.0
|
|
|
|
// 4. Numeric equivalence
|
|
const respNum = tryParseNumber(normResponse)
|
|
const accNum = tryParseNumber(normAccepted)
|
|
if (respNum !== null && accNum !== null && respNum === accNum) return 1.0
|
|
|
|
// Guard: skip string containment for purely numeric strings — rule 4 handles those
|
|
const bothNumeric = isPurelyNumeric(normResponse) && isPurelyNumeric(normAccepted)
|
|
|
|
// 5. Response contains the accepted answer (or stemmed version)
|
|
if (!bothNumeric && normResponse.includes(normAccepted) && normAccepted.length >= 2)
|
|
return 1.0
|
|
if (!bothNumeric && stemmedResponse.includes(stemmedAccepted) && stemmedAccepted.length >= 2)
|
|
maxScore = Math.max(maxScore, 0.95)
|
|
|
|
// 6. Accepted answer contains the response (for short definitive answers)
|
|
if (!bothNumeric && normAccepted.includes(normResponse) && normResponse.length >= 3)
|
|
maxScore = Math.max(maxScore, 0.8)
|
|
if (!bothNumeric && stemmedAccepted.includes(stemmedResponse) && stemmedResponse.length >= 3)
|
|
maxScore = Math.max(maxScore, 0.75)
|
|
|
|
// 7. Check if number appears anywhere in a longer response
|
|
if (accNum !== null) {
|
|
const numStr = String(accNum)
|
|
// Only match if the number appears as a whole token, not as a substring of a larger number
|
|
const numRegex = new RegExp(`(?<![\\d])${numStr.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?![\\d])`)
|
|
if (numRegex.test(normResponse)) return 1.0
|
|
// Check number words in response
|
|
const responseWords = normResponse.split(/\s+/)
|
|
for (const w of responseWords) {
|
|
const parsed = tryParseNumber(w)
|
|
if (parsed !== null && parsed === accNum) maxScore = Math.max(maxScore, 0.9)
|
|
}
|
|
}
|
|
|
|
// 8. Word-level containment — all words of the answer appear in the response
|
|
const acceptedWords = normAccepted.split(/\s+/)
|
|
if (acceptedWords.length >= 2) {
|
|
const allFound = acceptedWords.every(w => normResponse.includes(w))
|
|
if (allFound) maxScore = Math.max(maxScore, 0.9)
|
|
// Try with stemming
|
|
const stemmedAccWords = stemmedAccepted.split(/\s+/)
|
|
const allStemFound = stemmedAccWords.every(w => stemmedResponse.includes(w))
|
|
if (allStemFound) maxScore = Math.max(maxScore, 0.85)
|
|
}
|
|
|
|
// 9. Contraction-expanded word containment
|
|
if (expandedAccepted.split(/\s+/).length >= 2) {
|
|
const allFound = expandedAccepted.split(/\s+/).every(w => expandedResponse.includes(w))
|
|
if (allFound) maxScore = Math.max(maxScore, 0.85)
|
|
}
|
|
}
|
|
|
|
if (maxScore > 0) return maxScore
|
|
|
|
// 10. For true/false questions, check if the response starts with the right keyword
|
|
const tfAnswer = acceptedAnswers.find(a => a.toLowerCase() === 'true' || a.toLowerCase() === 'false')
|
|
if (tfAnswer) {
|
|
const firstWord = normResponse.split(/\s+/)[0]
|
|
if (firstWord === tfAnswer.toLowerCase()) return 1.0
|
|
// "that's true" / "this is false" / "yes" for true / "no" for false
|
|
if (normResponse.includes(tfAnswer.toLowerCase())) return 0.9
|
|
// "yes" ≈ true, "no" ≈ false
|
|
if (tfAnswer.toLowerCase() === 'true' && (firstWord === 'yes' || firstWord === 'correct' || firstWord === 'right')) return 0.9
|
|
if (tfAnswer.toLowerCase() === 'false' && (firstWord === 'no' || firstWord === 'incorrect' || firstWord === 'wrong')) return 0.9
|
|
}
|
|
|
|
return 0
|
|
}
|