fix: checkAnswer best-match + decimal preservation + prompt data fixes
- checkAnswer now returns highest score across all accepted answers instead of first match, fixing 95 false-low-confidence results - Skip string containment for purely numeric strings to prevent false positives like "1000" matching inside "10000" - Preserve decimal points in normalize() (42.0 no longer becomes 420) - Use word-boundary regex for number matching in responses - Fix 47 wrong choices scoring too high (comma-formatted numbers, verbose choices matching terse answers) - Fix 17 prompts where no choice matched any accepted answer - Challenge audit now reports zero failures across all 1472 prompts Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
6314861513
commit
4d6e50d988
@@ -17,7 +17,8 @@ function normalize(s: string): string {
|
||||
return s
|
||||
.toLowerCase()
|
||||
.trim()
|
||||
.replace(/[.,!?;:'"()\[\]{}<>]/g, '') // strip punctuation
|
||||
.replace(/[,!?;:'"()\[\]{}<>]/g, '') // strip punctuation (NOT periods)
|
||||
.replace(/\.(?!\d)/g, '') // strip periods NOT followed by a digit (preserve decimals)
|
||||
.replace(/\s+/g, ' ') // collapse whitespace
|
||||
.replace(/^(the|a|an|its|it is|it's) /i, '') // strip leading articles
|
||||
.trim()
|
||||
@@ -94,9 +95,15 @@ function tryParseNumber(s: string): number | null {
|
||||
return null
|
||||
}
|
||||
|
||||
/** Check if a string is purely digits (after normalization). */
|
||||
function isPurelyNumeric(s: string): boolean {
|
||||
return /^\d+$/.test(s)
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a bot's response matches any of the accepted answers.
|
||||
* Returns a confidence score: 1.0 = definite match, 0.5-0.9 = partial, 0 = no match.
|
||||
* Uses best-match: tries all accepted answers and returns the highest score.
|
||||
*/
|
||||
export function checkAnswer(response: string | null, acceptedAnswers: string[]): number {
|
||||
if (!response || response.trim() === '') return 0
|
||||
@@ -108,6 +115,8 @@ export function checkAnswer(response: string | null, acceptedAnswers: string[]):
|
||||
const expandedResponse = normalize(expandContractions(response))
|
||||
const stemmedResponse = stemAll(normResponse)
|
||||
|
||||
let maxScore = 0
|
||||
|
||||
for (const accepted of acceptedAnswers) {
|
||||
const normAccepted = normalize(accepted)
|
||||
const expandedAccepted = normalize(expandContractions(accepted))
|
||||
@@ -127,23 +136,32 @@ export function checkAnswer(response: string | null, acceptedAnswers: string[]):
|
||||
const accNum = tryParseNumber(normAccepted)
|
||||
if (respNum !== null && accNum !== null && respNum === accNum) return 1.0
|
||||
|
||||
// Guard: skip string containment for purely numeric strings — rule 4 handles those
|
||||
const bothNumeric = isPurelyNumeric(normResponse) && isPurelyNumeric(normAccepted)
|
||||
|
||||
// 5. Response contains the accepted answer (or stemmed version)
|
||||
if (normResponse.includes(normAccepted) && normAccepted.length >= 2) return 1.0
|
||||
if (stemmedResponse.includes(stemmedAccepted) && stemmedAccepted.length >= 2) return 0.95
|
||||
if (!bothNumeric && normResponse.includes(normAccepted) && normAccepted.length >= 2)
|
||||
return 1.0
|
||||
if (!bothNumeric && stemmedResponse.includes(stemmedAccepted) && stemmedAccepted.length >= 2)
|
||||
maxScore = Math.max(maxScore, 0.95)
|
||||
|
||||
// 6. Accepted answer contains the response (for short definitive answers)
|
||||
if (normAccepted.includes(normResponse) && normResponse.length >= 3) return 0.8
|
||||
if (stemmedAccepted.includes(stemmedResponse) && stemmedResponse.length >= 3) return 0.75
|
||||
if (!bothNumeric && normAccepted.includes(normResponse) && normResponse.length >= 3)
|
||||
maxScore = Math.max(maxScore, 0.8)
|
||||
if (!bothNumeric && stemmedAccepted.includes(stemmedResponse) && stemmedResponse.length >= 3)
|
||||
maxScore = Math.max(maxScore, 0.75)
|
||||
|
||||
// 7. Check if number appears anywhere in a longer response
|
||||
if (accNum !== null) {
|
||||
const numStr = String(accNum)
|
||||
if (normResponse.includes(numStr)) return 1.0
|
||||
// Only match if the number appears as a whole token, not as a substring of a larger number
|
||||
const numRegex = new RegExp(`(?<![\\d])${numStr.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?![\\d])`)
|
||||
if (numRegex.test(normResponse)) return 1.0
|
||||
// Check number words in response
|
||||
const responseWords = normResponse.split(/\s+/)
|
||||
for (const w of responseWords) {
|
||||
const parsed = tryParseNumber(w)
|
||||
if (parsed !== null && parsed === accNum) return 0.9
|
||||
if (parsed !== null && parsed === accNum) maxScore = Math.max(maxScore, 0.9)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -151,20 +169,22 @@ export function checkAnswer(response: string | null, acceptedAnswers: string[]):
|
||||
const acceptedWords = normAccepted.split(/\s+/)
|
||||
if (acceptedWords.length >= 2) {
|
||||
const allFound = acceptedWords.every(w => normResponse.includes(w))
|
||||
if (allFound) return 0.9
|
||||
if (allFound) maxScore = Math.max(maxScore, 0.9)
|
||||
// Try with stemming
|
||||
const stemmedAccWords = stemmedAccepted.split(/\s+/)
|
||||
const allStemFound = stemmedAccWords.every(w => stemmedResponse.includes(w))
|
||||
if (allStemFound) return 0.85
|
||||
if (allStemFound) maxScore = Math.max(maxScore, 0.85)
|
||||
}
|
||||
|
||||
// 9. Contraction-expanded word containment
|
||||
if (expandedAccepted.split(/\s+/).length >= 2) {
|
||||
const allFound = expandedAccepted.split(/\s+/).every(w => expandedResponse.includes(w))
|
||||
if (allFound) return 0.85
|
||||
if (allFound) maxScore = Math.max(maxScore, 0.85)
|
||||
}
|
||||
}
|
||||
|
||||
if (maxScore > 0) return maxScore
|
||||
|
||||
// 10. For true/false questions, check if the response starts with the right keyword
|
||||
const tfAnswer = acceptedAnswers.find(a => a.toLowerCase() === 'true' || a.toLowerCase() === 'false')
|
||||
if (tfAnswer) {
|
||||
|
||||
Reference in New Issue
Block a user