fix: checkAnswer best-match + decimal preservation + prompt data fixes

- checkAnswer now returns highest score across all accepted answers
  instead of first match, fixing 95 false-low-confidence results
- Skip string containment for purely numeric strings to prevent
  false positives like "1000" matching inside "10000"
- Preserve decimal points in normalize() (42.0 no longer becomes 420)
- Use word-boundary regex for number matching in responses
- Fix 47 wrong choices scoring too high (comma-formatted numbers,
  verbose choices matching terse answers)
- Fix 17 prompts where no choice matched any accepted answer
- Challenge audit now reports zero failures across all 1472 prompts

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Dorian
2026-03-13 04:35:54 +00:00
co-authored by Claude Opus 4.6
parent 6314861513
commit 4d6e50d988
7 changed files with 90 additions and 70 deletions
+30 -10
View File
@@ -17,7 +17,8 @@ function normalize(s: string): string {
return s
.toLowerCase()
.trim()
.replace(/[.,!?;:'"()\[\]{}<>]/g, '') // strip punctuation
.replace(/[,!?;:'"()\[\]{}<>]/g, '') // strip punctuation (NOT periods)
.replace(/\.(?!\d)/g, '') // strip periods NOT followed by a digit (preserve decimals)
.replace(/\s+/g, ' ') // collapse whitespace
.replace(/^(the|a|an|its|it is|it's) /i, '') // strip leading articles
.trim()
@@ -94,9 +95,15 @@ function tryParseNumber(s: string): number | null {
return null
}
/** Check if a string is purely digits (after normalization). */
function isPurelyNumeric(s: string): boolean {
return /^\d+$/.test(s)
}
/**
* Check if a bot's response matches any of the accepted answers.
* Returns a confidence score: 1.0 = definite match, 0.5-0.9 = partial, 0 = no match.
* Uses best-match: tries all accepted answers and returns the highest score.
*/
export function checkAnswer(response: string | null, acceptedAnswers: string[]): number {
if (!response || response.trim() === '') return 0
@@ -108,6 +115,8 @@ export function checkAnswer(response: string | null, acceptedAnswers: string[]):
const expandedResponse = normalize(expandContractions(response))
const stemmedResponse = stemAll(normResponse)
let maxScore = 0
for (const accepted of acceptedAnswers) {
const normAccepted = normalize(accepted)
const expandedAccepted = normalize(expandContractions(accepted))
@@ -127,23 +136,32 @@ export function checkAnswer(response: string | null, acceptedAnswers: string[]):
const accNum = tryParseNumber(normAccepted)
if (respNum !== null && accNum !== null && respNum === accNum) return 1.0
// Guard: skip string containment for purely numeric strings — rule 4 handles those
const bothNumeric = isPurelyNumeric(normResponse) && isPurelyNumeric(normAccepted)
// 5. Response contains the accepted answer (or stemmed version)
if (normResponse.includes(normAccepted) && normAccepted.length >= 2) return 1.0
if (stemmedResponse.includes(stemmedAccepted) && stemmedAccepted.length >= 2) return 0.95
if (!bothNumeric && normResponse.includes(normAccepted) && normAccepted.length >= 2)
return 1.0
if (!bothNumeric && stemmedResponse.includes(stemmedAccepted) && stemmedAccepted.length >= 2)
maxScore = Math.max(maxScore, 0.95)
// 6. Accepted answer contains the response (for short definitive answers)
if (normAccepted.includes(normResponse) && normResponse.length >= 3) return 0.8
if (stemmedAccepted.includes(stemmedResponse) && stemmedResponse.length >= 3) return 0.75
if (!bothNumeric && normAccepted.includes(normResponse) && normResponse.length >= 3)
maxScore = Math.max(maxScore, 0.8)
if (!bothNumeric && stemmedAccepted.includes(stemmedResponse) && stemmedResponse.length >= 3)
maxScore = Math.max(maxScore, 0.75)
// 7. Check if number appears anywhere in a longer response
if (accNum !== null) {
const numStr = String(accNum)
if (normResponse.includes(numStr)) return 1.0
// Only match if the number appears as a whole token, not as a substring of a larger number
const numRegex = new RegExp(`(?<![\\d])${numStr.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?![\\d])`)
if (numRegex.test(normResponse)) return 1.0
// Check number words in response
const responseWords = normResponse.split(/\s+/)
for (const w of responseWords) {
const parsed = tryParseNumber(w)
if (parsed !== null && parsed === accNum) return 0.9
if (parsed !== null && parsed === accNum) maxScore = Math.max(maxScore, 0.9)
}
}
@@ -151,20 +169,22 @@ export function checkAnswer(response: string | null, acceptedAnswers: string[]):
const acceptedWords = normAccepted.split(/\s+/)
if (acceptedWords.length >= 2) {
const allFound = acceptedWords.every(w => normResponse.includes(w))
if (allFound) return 0.9
if (allFound) maxScore = Math.max(maxScore, 0.9)
// Try with stemming
const stemmedAccWords = stemmedAccepted.split(/\s+/)
const allStemFound = stemmedAccWords.every(w => stemmedResponse.includes(w))
if (allStemFound) return 0.85
if (allStemFound) maxScore = Math.max(maxScore, 0.85)
}
// 9. Contraction-expanded word containment
if (expandedAccepted.split(/\s+/).length >= 2) {
const allFound = expandedAccepted.split(/\s+/).every(w => expandedResponse.includes(w))
if (allFound) return 0.85
if (allFound) maxScore = Math.max(maxScore, 0.85)
}
}
if (maxScore > 0) return maxScore
// 10. For true/false questions, check if the response starts with the right keyword
const tfAnswer = acceptedAnswers.find(a => a.toLowerCase() === 'true' || a.toLowerCase() === 'false')
if (tfAnswer) {