DorianandClaude Opus 4.6
6314861513
Run heuristic LLM difficulty classification on all 1472 prompts: - 43.6% TRIVIAL, 27.9% MODERATE, 21.4% HARD, 7.1% TRICK - Hypothesis "80%+ TRIVIAL" rejected — distribution more varied Ambiguous prompts audit found 517 issues: - 22 rejected alternatives (single-char answers fail with prefixes) - 368 substring conflicts between accepted answers - 127 first-match-not-best scoring issues Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
318 lines
14 KiB
TypeScript
318 lines
14 KiB
TypeScript
import { describe, it, expect } from 'vitest'
|
|
import { TEMPLATES, type ChallengeTemplate, type PromptEntry } from './challenge-data.js'
|
|
import { EXTRA_PROMPTS } from './challenges-extra.js'
|
|
import { BITCOIN_PROMPTS } from './challenges-bitcoin.js'
|
|
import { CONSPIRACY_PROMPTS } from './challenges-conspiracy.js'
|
|
import { PC_PROMPTS } from './challenges-pc.js'
|
|
import { VIBE_PROMPTS } from './challenges-vibe.js'
|
|
import { checkAnswer } from './answers.js'
|
|
import { writeFileSync, mkdirSync } from 'node:fs'
|
|
import { resolve, dirname } from 'node:path'
|
|
import { fileURLToPath } from 'node:url'
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════
|
|
// AMBIGUOUS PROMPTS AUDIT
|
|
// Identifies prompts with ambiguous or multiple valid answers that the
|
|
// checkAnswer function may not handle correctly.
|
|
// ═══════════════════════════════════════════════════════════════════════
|
|
|
|
// Build merged templates (same logic as challenge-audit)
|
|
function buildMergedTemplates(): ChallengeTemplate[] {
|
|
return TEMPLATES.map(t => {
|
|
const allPrompts = [...t.prompts]
|
|
const extras = EXTRA_PROMPTS[t.type]
|
|
if (extras) allPrompts.push(...extras)
|
|
const btc = BITCOIN_PROMPTS[t.type]
|
|
if (btc) allPrompts.push(...btc)
|
|
const conspiracy = CONSPIRACY_PROMPTS[t.type]
|
|
if (conspiracy) allPrompts.push(...conspiracy)
|
|
const pc = PC_PROMPTS[t.type]
|
|
if (pc) allPrompts.push(...pc)
|
|
const vibe = VIBE_PROMPTS[t.type]
|
|
if (vibe) allPrompts.push(...vibe)
|
|
return { ...t, prompts: allPrompts }
|
|
})
|
|
}
|
|
|
|
interface AmbiguityFinding {
|
|
type: string
|
|
prompt: string
|
|
category: string
|
|
detail: string
|
|
}
|
|
|
|
/** Generate reasonable alternative phrasings for an answer */
|
|
function generateAlternatives(answer: string, allAnswers: string[]): string[] {
|
|
const alts: string[] = []
|
|
|
|
// 1. Prefix with "It's ..."
|
|
alts.push(`It's ${answer}`)
|
|
|
|
// 2. Prefix with "The ..."
|
|
if (!answer.toLowerCase().startsWith('the ')) {
|
|
alts.push(`The ${answer}`)
|
|
}
|
|
|
|
// 3. Prefix with "A ..."
|
|
if (!answer.toLowerCase().startsWith('a ')) {
|
|
alts.push(`A ${answer}`)
|
|
}
|
|
|
|
// 4. Trailing period
|
|
alts.push(`${answer}.`)
|
|
|
|
// 5. Trailing question mark (bots sometimes answer with question mark)
|
|
alts.push(`${answer}?`)
|
|
|
|
// 6. Add "I think" prefix
|
|
alts.push(`I think ${answer}`)
|
|
|
|
// 7. Extended answer: combine first two accepted answers with "and"
|
|
if (allAnswers.length >= 2) {
|
|
const other = allAnswers.find(a => a.toLowerCase() !== answer.toLowerCase())
|
|
if (other) {
|
|
alts.push(`${answer} and ${other}`)
|
|
}
|
|
}
|
|
|
|
// 8. Add context words — "The answer is X"
|
|
alts.push(`The answer is ${answer}`)
|
|
|
|
// 9. Leading "It is"
|
|
alts.push(`It is ${answer}`)
|
|
|
|
return alts
|
|
}
|
|
|
|
describe('Ambiguous Prompts Audit', () => {
|
|
const templates = buildMergedTemplates()
|
|
const factualTemplates = templates.filter(t => t.scoring === 'factual')
|
|
|
|
const findings: AmbiguityFinding[] = []
|
|
const stats = {
|
|
promptsScanned: 0,
|
|
alternativesTested: 0,
|
|
rejectedAlternatives: 0,
|
|
substringConflicts: 0,
|
|
firstMatchNotBest: 0,
|
|
}
|
|
|
|
// ─── 1. Alternative phrasings rejected ────────────────────────────
|
|
it('identifies reasonable alternatives rejected by checkAnswer', () => {
|
|
for (const t of factualTemplates) {
|
|
for (const p of t.prompts) {
|
|
if (!p.answers || p.answers.length === 0) continue
|
|
stats.promptsScanned++
|
|
|
|
for (const answer of p.answers) {
|
|
const alternatives = generateAlternatives(answer, p.answers)
|
|
|
|
for (const alt of alternatives) {
|
|
stats.alternativesTested++
|
|
const score = checkAnswer(alt, p.answers)
|
|
if (score === 0) {
|
|
stats.rejectedAlternatives++
|
|
findings.push({
|
|
type: t.type,
|
|
prompt: p.prompt,
|
|
category: 'REJECTED_ALTERNATIVE',
|
|
detail: `"${alt}" scored 0 (base answer: "${answer}")`,
|
|
})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
console.log(`\n══════════════════════════════════════════`)
|
|
console.log(` AMBIGUITY AUDIT — ALTERNATIVE PHRASINGS`)
|
|
console.log(`══════════════════════════════════════════`)
|
|
console.log(` Prompts scanned: ${stats.promptsScanned}`)
|
|
console.log(` Alternatives tested: ${stats.alternativesTested}`)
|
|
console.log(` Rejected (score = 0): ${stats.rejectedAlternatives}`)
|
|
console.log(`══════════════════════════════════════════\n`)
|
|
|
|
// Soft assertion — this is an audit, not a hard gate
|
|
expect(stats.promptsScanned).toBeGreaterThan(0)
|
|
})
|
|
|
|
// ─── 2. Substring conflicts between accepted answers ──────────────
|
|
it('identifies answers that are substrings of each other', () => {
|
|
for (const t of factualTemplates) {
|
|
for (const p of t.prompts) {
|
|
if (!p.answers || p.answers.length < 2) continue
|
|
|
|
const normalized = p.answers.map(a => a.toLowerCase().trim())
|
|
|
|
for (let i = 0; i < normalized.length; i++) {
|
|
for (let j = 0; j < normalized.length; j++) {
|
|
if (i === j) continue
|
|
// Check if answer[i] is a strict substring of answer[j]
|
|
if (normalized[j].includes(normalized[i]) && normalized[i] !== normalized[j]) {
|
|
stats.substringConflicts++
|
|
findings.push({
|
|
type: t.type,
|
|
prompt: p.prompt,
|
|
category: 'SUBSTRING_CONFLICT',
|
|
detail: `"${p.answers[i]}" is a substring of "${p.answers[j]}" — first-match ordering may return suboptimal score`,
|
|
})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
console.log(`══════════════════════════════════════════`)
|
|
console.log(` AMBIGUITY AUDIT — SUBSTRING CONFLICTS`)
|
|
console.log(`══════════════════════════════════════════`)
|
|
console.log(` Substring conflicts: ${stats.substringConflicts}`)
|
|
console.log(`══════════════════════════════════════════\n`)
|
|
|
|
// Soft check
|
|
expect(true).toBe(true)
|
|
})
|
|
|
|
// ─── 3. First-match-not-best scoring issue ────────────────────────
|
|
it('identifies prompts where answer ordering causes suboptimal scores', () => {
|
|
for (const t of factualTemplates) {
|
|
for (const p of t.prompts) {
|
|
if (!p.answers || p.answers.length < 2) continue
|
|
|
|
// For each accepted answer, check if answering with it gets full marks
|
|
for (const answer of p.answers) {
|
|
const score = checkAnswer(answer, p.answers)
|
|
if (score > 0 && score < 1.0) {
|
|
// This accepted answer does not get a perfect score — ordering issue
|
|
stats.firstMatchNotBest++
|
|
findings.push({
|
|
type: t.type,
|
|
prompt: p.prompt,
|
|
category: 'FIRST_MATCH_NOT_BEST',
|
|
detail: `answering "${answer}" scores ${score} instead of 1.0 (answers: [${p.answers.map(a => `"${a}"`).join(', ')}])`,
|
|
})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
console.log(`══════════════════════════════════════════`)
|
|
console.log(` AMBIGUITY AUDIT — FIRST-MATCH-NOT-BEST`)
|
|
console.log(`══════════════════════════════════════════`)
|
|
console.log(` Suboptimal scores: ${stats.firstMatchNotBest}`)
|
|
console.log(`══════════════════════════════════════════\n`)
|
|
|
|
// Soft check
|
|
expect(true).toBe(true)
|
|
})
|
|
|
|
// ─── 4. Final report and markdown output ──────────────────────────
|
|
it('prints full ambiguity report and writes markdown', () => {
|
|
console.log(`\n══════════════════════════════════════════`)
|
|
console.log(` AMBIGUOUS PROMPTS AUDIT — FINAL REPORT`)
|
|
console.log(`══════════════════════════════════════════`)
|
|
console.log(` Prompts scanned: ${stats.promptsScanned}`)
|
|
console.log(` Alternatives tested: ${stats.alternativesTested}`)
|
|
console.log(` ────────────────────────────────────────`)
|
|
console.log(` FINDINGS:`)
|
|
console.log(` Rejected alternatives: ${stats.rejectedAlternatives}`)
|
|
console.log(` Substring conflicts: ${stats.substringConflicts}`)
|
|
console.log(` First-match-not-best: ${stats.firstMatchNotBest}`)
|
|
console.log(` Total findings: ${findings.length}`)
|
|
console.log(`══════════════════════════════════════════`)
|
|
|
|
if (findings.length > 0) {
|
|
// Group by category
|
|
const byCategory = new Map<string, AmbiguityFinding[]>()
|
|
for (const f of findings) {
|
|
const list = byCategory.get(f.category) || []
|
|
list.push(f)
|
|
byCategory.set(f.category, list)
|
|
}
|
|
|
|
for (const [category, items] of byCategory) {
|
|
console.log(`\n [${category}] (${items.length} findings)`)
|
|
// Print first 10 per category to avoid overwhelming output
|
|
const shown = items.slice(0, 10)
|
|
for (const item of shown) {
|
|
console.log(` ${item.type}: "${item.prompt.slice(0, 60)}..."`)
|
|
console.log(` ${item.detail}`)
|
|
}
|
|
if (items.length > 10) {
|
|
console.log(` ... and ${items.length - 10} more`)
|
|
}
|
|
}
|
|
}
|
|
|
|
// ─── Write markdown report ────────────────────────────────────
|
|
const lines: string[] = []
|
|
lines.push('# Ambiguous Prompts Audit')
|
|
lines.push('')
|
|
lines.push(`Generated: ${new Date().toISOString()}`)
|
|
lines.push('')
|
|
lines.push('## Summary')
|
|
lines.push('')
|
|
lines.push(`| Metric | Count |`)
|
|
lines.push(`|--------|-------|`)
|
|
lines.push(`| Prompts scanned | ${stats.promptsScanned} |`)
|
|
lines.push(`| Alternatives tested | ${stats.alternativesTested} |`)
|
|
lines.push(`| Rejected alternatives | ${stats.rejectedAlternatives} |`)
|
|
lines.push(`| Substring conflicts | ${stats.substringConflicts} |`)
|
|
lines.push(`| First-match-not-best | ${stats.firstMatchNotBest} |`)
|
|
lines.push(`| **Total findings** | **${findings.length}** |`)
|
|
lines.push('')
|
|
|
|
// Group by category for markdown
|
|
const byCategory = new Map<string, AmbiguityFinding[]>()
|
|
for (const f of findings) {
|
|
const list = byCategory.get(f.category) || []
|
|
list.push(f)
|
|
byCategory.set(f.category, list)
|
|
}
|
|
|
|
for (const [category, items] of byCategory) {
|
|
lines.push(`## ${category} (${items.length})`)
|
|
lines.push('')
|
|
|
|
if (category === 'REJECTED_ALTERNATIVE') {
|
|
lines.push('Reasonable answer phrasings that checkAnswer rejects (score = 0).')
|
|
lines.push('These represent cases where a bot giving a correct but differently-phrased answer would get zero credit.')
|
|
lines.push('')
|
|
} else if (category === 'SUBSTRING_CONFLICT') {
|
|
lines.push('Answer arrays where one answer is a substring of another.')
|
|
lines.push('Because checkAnswer returns the first match (not the best), answer ordering matters.')
|
|
lines.push('')
|
|
} else if (category === 'FIRST_MATCH_NOT_BEST') {
|
|
lines.push('Accepted answers that score below 1.0 when checked against the full answers array.')
|
|
lines.push('This happens when an earlier answer in the array partially matches via containment before the exact match is reached.')
|
|
lines.push('')
|
|
}
|
|
|
|
lines.push('| Type | Prompt | Detail |')
|
|
lines.push('|------|--------|--------|')
|
|
for (const item of items) {
|
|
const escapedPrompt = item.prompt.replace(/\|/g, '\\|').slice(0, 80)
|
|
const escapedDetail = item.detail.replace(/\|/g, '\\|')
|
|
lines.push(`| ${item.type} | ${escapedPrompt} | ${escapedDetail} |`)
|
|
}
|
|
lines.push('')
|
|
}
|
|
|
|
// Write to loop/ambiguous-prompts.md
|
|
const thisDir = dirname(fileURLToPath(import.meta.url))
|
|
const loopDir = resolve(thisDir, '..', '..', '..', 'loop')
|
|
try {
|
|
mkdirSync(loopDir, { recursive: true })
|
|
} catch {
|
|
// already exists
|
|
}
|
|
const outPath = resolve(loopDir, 'ambiguous-prompts.md')
|
|
writeFileSync(outPath, lines.join('\n'), 'utf-8')
|
|
console.log(`\n Report written to: ${outPath}`)
|
|
|
|
console.log(`\n══════════════════════════════════════════\n`)
|
|
|
|
// Soft assertion — audit always passes
|
|
expect(true).toBe(true)
|
|
})
|
|
})
|