fix: checkAnswer best-match + decimal preservation + prompt data fixes

- checkAnswer now returns highest score across all accepted answers
  instead of first match, fixing 95 false-low-confidence results
- Skip string containment for purely numeric strings to prevent
  false positives like "1000" matching inside "10000"
- Preserve decimal points in normalize() (42.0 no longer becomes 420)
- Use word-boundary regex for number matching in responses
- Fix 47 wrong choices scoring too high (comma-formatted numbers,
  verbose choices matching terse answers)
- Fix 17 prompts where no choice matched any accepted answer
- Challenge audit now reports zero failures across all 1472 prompts

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Dorian
2026-03-13 04:35:54 +00:00
co-authored by Claude Opus 4.6
parent 6314861513
commit 4d6e50d988
7 changed files with 90 additions and 70 deletions
+2 -2
View File
@@ -18,7 +18,7 @@ export const VIBE_PROMPTS: Record<string, PromptEntry[]> = {
math_blitz: [
{ prompt: 'If an LLM has 70 billion parameters and each is 16-bit, how many GB of memory minimum?', answers: ['140', '140 gb'], choices: ['70 GB', '140 GB', '280 GB'], theme: 'vibe_coding', difficulty: 'hard' },
{ prompt: 'GPT-4 reportedly has ~1.8 trillion parameters. How many billions is that?', answers: ['1800'], choices: ['180', '1,800', '18,000'], theme: 'vibe_coding', difficulty: 'easy' },
{ prompt: 'GPT-4 reportedly has ~1.8 trillion parameters. How many billions is that?', answers: ['1800', '1,800'], choices: ['180', '1,800', '18,000'], theme: 'vibe_coding', difficulty: 'easy' },
{ prompt: 'If a GPU costs $30,000 and you need 8 for training, what is the hardware cost?', answers: ['240000', '$240,000'], choices: ['$120,000', '$240,000', '$480,000'], theme: 'vibe_coding', difficulty: 'easy' },
{ prompt: 'A transformer model with 12 layers and 12 attention heads has how many heads total?', answers: ['144'], choices: ['24', '72', '144'], theme: 'vibe_coding', difficulty: 'medium' },
{ prompt: 'If token cost is $0.01 per 1K tokens, how much does 1 million tokens cost?', answers: ['$10', '10'], choices: ['$1', '$10', '$100'], theme: 'vibe_coding', difficulty: 'easy' },
@@ -55,7 +55,7 @@ export const VIBE_PROMPTS: Record<string, PromptEntry[]> = {
{ prompt: 'A vibe coder ships an app without reading the code. Is the app more or less likely to have bugs than hand-written code?', answers: ['more', 'more likely'], choices: ['More likely', 'Less likely', 'Same as hand-written'], theme: 'vibe_coding' },
{ prompt: 'An AI writes 1000 lines of code in 10 seconds. A developer writes 100 lines per day. How many developer-days did the AI save?', answers: ['10'], choices: ['1', '10', '100'], theme: 'vibe_coding' },
{ prompt: 'If you ask an AI to "make this code faster" without profiling, what will it most likely optimize?', answers: ['the wrong thing', 'wrong thing', 'something unnecessary'], choices: ['The wrong thing', 'The hot loop', 'Memory usage'], theme: 'vibe_coding' },
{ prompt: 'A company replaces all engineers with AI. Production goes down. What went wrong?', answers: ['no one to debug', 'no humans to fix issues', 'no one understood the code'], choices: ['No one could debug', 'AI costs too much', 'The code was perfect'], theme: 'vibe_coding' },
{ prompt: 'A company replaces all engineers with AI. Production goes down. What went wrong?', answers: ['no one to debug', 'no humans to fix issues', 'no one understood the code', 'no one could debug'], choices: ['No one could debug', 'AI costs too much', 'The code was perfect'], theme: 'vibe_coding' },
{ prompt: 'You prompt an AI to write tests. It generates 50 tests that all pass. Is that good?', answers: ['not necessarily', 'no', 'depends', 'maybe not'], choices: ['Not necessarily — tests may be trivial', 'Yes, great coverage', 'Only if they are unit tests'], theme: 'vibe_coding' },
],