test: comprehensive challenge audit — 1472 prompts, 176 issues found
Audit covers all factual prompts through checkAnswer. Findings: - 95 low-confidence correct answers (substring collision at 0.8) - 64 false-positive wrong choices (normalization strips commas) - 17 choices missing correct answer (paraphrasing mismatch) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Dorian
co-authored by
Claude Opus 4.6
Claude Opus 4.6
parent
39b8504157
commit
b8acc1c95b
@@ -0,0 +1,289 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { TEMPLATES, type ChallengeTemplate, type PromptEntry } from './challenge-data.js'
|
||||
import { EXTRA_PROMPTS } from './challenges-extra.js'
|
||||
import { BITCOIN_PROMPTS } from './challenges-bitcoin.js'
|
||||
import { CONSPIRACY_PROMPTS } from './challenges-conspiracy.js'
|
||||
import { PC_PROMPTS } from './challenges-pc.js'
|
||||
import { VIBE_PROMPTS } from './challenges-vibe.js'
|
||||
import { checkAnswer } from './answers.js'
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════
|
||||
// CHALLENGE AUDIT — exhaustive sweep of every prompt through checkAnswer
|
||||
// ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
// Build merged templates the same way challenges.ts does (non-mutating copy)
|
||||
function buildMergedTemplates(): ChallengeTemplate[] {
|
||||
return TEMPLATES.map(t => {
|
||||
const allPrompts = [...t.prompts]
|
||||
const extras = EXTRA_PROMPTS[t.type]
|
||||
if (extras) allPrompts.push(...extras)
|
||||
const btc = BITCOIN_PROMPTS[t.type]
|
||||
if (btc) allPrompts.push(...btc)
|
||||
const conspiracy = CONSPIRACY_PROMPTS[t.type]
|
||||
if (conspiracy) allPrompts.push(...conspiracy)
|
||||
const pc = PC_PROMPTS[t.type]
|
||||
if (pc) allPrompts.push(...pc)
|
||||
const vibe = VIBE_PROMPTS[t.type]
|
||||
if (vibe) allPrompts.push(...vibe)
|
||||
return { ...t, prompts: allPrompts }
|
||||
})
|
||||
}
|
||||
|
||||
interface AuditFailure {
|
||||
type: string
|
||||
prompt: string
|
||||
category: string
|
||||
detail: string
|
||||
}
|
||||
|
||||
describe('Challenge Prompt Audit', () => {
|
||||
const templates = buildMergedTemplates()
|
||||
const factualTemplates = templates.filter(t => t.scoring === 'factual')
|
||||
const creativeTemplates = templates.filter(t => t.scoring === 'creative')
|
||||
|
||||
// Collect all failures for the final report
|
||||
const failures: AuditFailure[] = []
|
||||
const stats = {
|
||||
totalTemplates: templates.length,
|
||||
factualTemplates: 0,
|
||||
creativeTemplates: 0,
|
||||
totalPrompts: 0,
|
||||
factualPrompts: 0,
|
||||
creativePrompts: 0,
|
||||
answersChecked: 0,
|
||||
wrongChoicesChecked: 0,
|
||||
missingAnswers: 0,
|
||||
lowConfidenceCorrect: 0,
|
||||
highConfidenceWrong: 0,
|
||||
choicesMissingCorrect: 0,
|
||||
}
|
||||
|
||||
// ─── 1. Count everything ───────────────────────────────────────────
|
||||
it('counts all templates and prompts', () => {
|
||||
stats.factualTemplates = factualTemplates.length
|
||||
stats.creativeTemplates = creativeTemplates.length
|
||||
stats.totalPrompts = templates.reduce((sum, t) => sum + t.prompts.length, 0)
|
||||
stats.factualPrompts = factualTemplates.reduce((sum, t) => sum + t.prompts.length, 0)
|
||||
stats.creativePrompts = creativeTemplates.reduce((sum, t) => sum + t.prompts.length, 0)
|
||||
|
||||
console.log(`\n══════════════════════════════════════════`)
|
||||
console.log(` CHALLENGE AUDIT — PROMPT COUNTS`)
|
||||
console.log(`══════════════════════════════════════════`)
|
||||
console.log(` Total templates: ${stats.totalTemplates}`)
|
||||
console.log(` Factual templates: ${stats.factualTemplates}`)
|
||||
console.log(` Creative templates: ${stats.creativeTemplates}`)
|
||||
console.log(` Total prompts: ${stats.totalPrompts}`)
|
||||
console.log(` Factual prompts: ${stats.factualPrompts}`)
|
||||
console.log(` Creative prompts: ${stats.creativePrompts}`)
|
||||
console.log(`══════════════════════════════════════════\n`)
|
||||
|
||||
// Soft assertion: we expect 800+ prompts
|
||||
expect(stats.totalPrompts).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
// ─── 2. Every factual prompt must have answers ─────────────────────
|
||||
it('every factual prompt has answers array', () => {
|
||||
for (const t of factualTemplates) {
|
||||
for (const p of t.prompts) {
|
||||
if (!p.answers || p.answers.length === 0) {
|
||||
stats.missingAnswers++
|
||||
failures.push({
|
||||
type: t.type,
|
||||
prompt: p.prompt,
|
||||
category: 'MISSING_ANSWERS',
|
||||
detail: 'No answers array or empty answers',
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log(` Missing answers: ${stats.missingAnswers}`)
|
||||
if (stats.missingAnswers > 0) {
|
||||
console.warn(` WARNING: ${stats.missingAnswers} factual prompts have no answers`)
|
||||
}
|
||||
// Soft check — document but do not hard-fail
|
||||
expect(true).toBe(true)
|
||||
})
|
||||
|
||||
// ─── 3. checkAnswer(answer, answers) >= 0.9 for every correct answer
|
||||
it('every correct answer scores >= 0.9 via checkAnswer', () => {
|
||||
for (const t of factualTemplates) {
|
||||
for (const p of t.prompts) {
|
||||
if (!p.answers) continue
|
||||
for (const answer of p.answers) {
|
||||
stats.answersChecked++
|
||||
const score = checkAnswer(answer, p.answers)
|
||||
if (score < 0.9) {
|
||||
stats.lowConfidenceCorrect++
|
||||
failures.push({
|
||||
type: t.type,
|
||||
prompt: p.prompt,
|
||||
category: 'LOW_CONFIDENCE_CORRECT',
|
||||
detail: `answer="${answer}" scored ${score} (expected >= 0.9)`,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log(` Answers checked: ${stats.answersChecked}`)
|
||||
console.log(` Low confidence correct: ${stats.lowConfidenceCorrect}`)
|
||||
|
||||
if (stats.lowConfidenceCorrect > 0) {
|
||||
console.warn(` WARNING: ${stats.lowConfidenceCorrect} correct answers scored below 0.9`)
|
||||
}
|
||||
// Soft check — document findings, do not hard-fail
|
||||
expect(true).toBe(true)
|
||||
})
|
||||
|
||||
// ─── 4. Wrong choices must score < 0.5 ─────────────────────────────
|
||||
it('wrong choices score < 0.5 via checkAnswer', () => {
|
||||
for (const t of factualTemplates) {
|
||||
for (const p of t.prompts) {
|
||||
if (!p.choices || !p.answers) continue
|
||||
|
||||
// Identify wrong choices: those that do NOT match any accepted answer
|
||||
const wrongChoices = p.choices.filter(choice => {
|
||||
const score = checkAnswer(choice, p.answers!)
|
||||
return score === 0 || score < 0.5
|
||||
})
|
||||
|
||||
// The "correct" choices are those that DO match
|
||||
const correctChoices = p.choices.filter(choice =>
|
||||
checkAnswer(choice, p.answers!) >= 0.5
|
||||
)
|
||||
|
||||
// Every prompt with choices should have at least one correct choice
|
||||
if (correctChoices.length === 0) {
|
||||
stats.choicesMissingCorrect++
|
||||
failures.push({
|
||||
type: t.type,
|
||||
prompt: p.prompt,
|
||||
category: 'CHOICES_MISSING_CORRECT',
|
||||
detail: `No choice matches answers. choices=${JSON.stringify(p.choices)} answers=${JSON.stringify(p.answers)}`,
|
||||
})
|
||||
}
|
||||
|
||||
// Now check that wrong choices truly score low
|
||||
for (const choice of p.choices) {
|
||||
const score = checkAnswer(choice, p.answers)
|
||||
if (score >= 0.5) {
|
||||
// This is a "correct" choice — skip
|
||||
continue
|
||||
}
|
||||
// This is a wrong choice — it should be < 0.5 (already is, by filter)
|
||||
stats.wrongChoicesChecked++
|
||||
}
|
||||
|
||||
// Also: explicitly test that each non-matching choice is < 0.5
|
||||
for (const choice of p.choices) {
|
||||
const score = checkAnswer(choice, p.answers)
|
||||
// If a choice scores >= 0.5 it should be a valid answer
|
||||
// If it scores >= 0.5 but ISN'T in the answers, flag it
|
||||
if (score >= 0.5) {
|
||||
// Check: is this choice genuinely correct?
|
||||
const isGenuineAnswer = p.answers.some(a => {
|
||||
const n1 = a.toLowerCase().trim()
|
||||
const n2 = choice.toLowerCase().trim()
|
||||
return n1 === n2 || n1.includes(n2) || n2.includes(n1)
|
||||
})
|
||||
if (!isGenuineAnswer) {
|
||||
stats.highConfidenceWrong++
|
||||
failures.push({
|
||||
type: t.type,
|
||||
prompt: p.prompt,
|
||||
category: 'HIGH_CONFIDENCE_WRONG',
|
||||
detail: `wrong choice="${choice}" scored ${score} (expected < 0.5). answers=${JSON.stringify(p.answers)}`,
|
||||
})
|
||||
}
|
||||
} else {
|
||||
stats.wrongChoicesChecked++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(` Wrong choices checked: ${stats.wrongChoicesChecked}`)
|
||||
console.log(` High confidence wrong: ${stats.highConfidenceWrong}`)
|
||||
console.log(` Choices missing correct answer: ${stats.choicesMissingCorrect}`)
|
||||
|
||||
if (stats.highConfidenceWrong > 0) {
|
||||
console.warn(` WARNING: ${stats.highConfidenceWrong} wrong choices scored >= 0.5`)
|
||||
}
|
||||
// Soft: allow test to pass even with some cross-match issues
|
||||
})
|
||||
|
||||
// ─── 5. Also test uppercase variants of correct answers ────────────
|
||||
it('correct answers match when uppercased', () => {
|
||||
let uppercaseFailures = 0
|
||||
for (const t of factualTemplates) {
|
||||
for (const p of t.prompts) {
|
||||
if (!p.answers) continue
|
||||
for (const answer of p.answers) {
|
||||
const score = checkAnswer(answer.toUpperCase(), p.answers)
|
||||
if (score < 0.75) {
|
||||
uppercaseFailures++
|
||||
failures.push({
|
||||
type: t.type,
|
||||
prompt: p.prompt,
|
||||
category: 'UPPERCASE_MISMATCH',
|
||||
detail: `UPPER "${answer.toUpperCase()}" scored ${score}`,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log(` Uppercase mismatches: ${uppercaseFailures}`)
|
||||
})
|
||||
|
||||
// ─── 6. Final audit report ─────────────────────────────────────────
|
||||
it('prints the full audit report', () => {
|
||||
console.log(`\n══════════════════════════════════════════`)
|
||||
console.log(` CHALLENGE AUDIT — FINAL REPORT`)
|
||||
console.log(`══════════════════════════════════════════`)
|
||||
console.log(` Total templates: ${stats.totalTemplates}`)
|
||||
console.log(` Factual templates: ${stats.factualTemplates}`)
|
||||
console.log(` Creative templates: ${stats.creativeTemplates}`)
|
||||
console.log(` Total prompts: ${stats.totalPrompts}`)
|
||||
console.log(` Factual prompts: ${stats.factualPrompts}`)
|
||||
console.log(` Creative prompts: ${stats.creativePrompts}`)
|
||||
console.log(` ────────────────────────────────────────`)
|
||||
console.log(` Answers checked: ${stats.answersChecked}`)
|
||||
console.log(` Wrong choices checked: ${stats.wrongChoicesChecked}`)
|
||||
console.log(` ────────────────────────────────────────`)
|
||||
console.log(` FAILURES:`)
|
||||
console.log(` Missing answers: ${stats.missingAnswers}`)
|
||||
console.log(` Low confidence correct: ${stats.lowConfidenceCorrect}`)
|
||||
console.log(` High confidence wrong: ${stats.highConfidenceWrong}`)
|
||||
console.log(` Choices missing correct: ${stats.choicesMissingCorrect}`)
|
||||
console.log(` ────────────────────────────────────────`)
|
||||
console.log(` Total failures: ${failures.length}`)
|
||||
console.log(`══════════════════════════════════════════`)
|
||||
|
||||
if (failures.length > 0) {
|
||||
console.log(`\n FAILURE DETAILS:`)
|
||||
console.log(` ────────────────────────────────────────`)
|
||||
|
||||
// Group by category
|
||||
const byCategory = new Map<string, AuditFailure[]>()
|
||||
for (const f of failures) {
|
||||
const list = byCategory.get(f.category) || []
|
||||
list.push(f)
|
||||
byCategory.set(f.category, list)
|
||||
}
|
||||
|
||||
for (const [category, items] of byCategory) {
|
||||
console.log(`\n [${category}] (${items.length} failures)`)
|
||||
for (const item of items) {
|
||||
console.log(` ${item.type}: "${item.prompt.slice(0, 60)}..."`)
|
||||
console.log(` ${item.detail}`)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
console.log(`\n ALL PROMPTS PASSED AUDIT`)
|
||||
}
|
||||
|
||||
console.log(`\n══════════════════════════════════════════\n`)
|
||||
|
||||
// This test always passes — it's just the report printer
|
||||
expect(true).toBe(true)
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user