test: verify all factual answers work with checkAnswer, fix 2 data bugs
- Hash rate question: added ×10^20 notation to answers array - Phishing question: added correct MC choice text to answers array - New tests: self-match, case-insensitive, MC correctness validation Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
a4b4963c32
commit
070e41f97b
@@ -1,5 +1,7 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { checkAnswer } from './answers.js'
|
||||
import { TEMPLATES } from './challenge-data.js'
|
||||
import { EXTRA_PROMPTS } from './challenges-extra.js'
|
||||
|
||||
describe('checkAnswer', () => {
|
||||
// --- Tier 1: exact match after normalization ---
|
||||
@@ -146,6 +148,59 @@ describe('checkAnswer', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('all factual prompt answers work with checkAnswer', () => {
|
||||
// Merge extras into templates (same as challenges.ts does)
|
||||
const templates = TEMPLATES.map(t => {
|
||||
const extras = EXTRA_PROMPTS[t.type]
|
||||
return extras ? { ...t, prompts: [...t.prompts, ...extras] } : t
|
||||
})
|
||||
|
||||
const factualTemplates = templates.filter(t => t.scoring === 'factual')
|
||||
|
||||
it('every factual prompt has at least one answer', () => {
|
||||
for (const t of factualTemplates) {
|
||||
for (const p of t.prompts) {
|
||||
expect(p.answers, `${t.type}: "${p.prompt}" has no answers`).toBeTruthy()
|
||||
expect(p.answers!.length, `${t.type}: "${p.prompt}" has empty answers array`).toBeGreaterThan(0)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
it('every answer self-matches with checkAnswer (case-insensitive)', () => {
|
||||
let checked = 0
|
||||
for (const t of factualTemplates) {
|
||||
for (const p of t.prompts) {
|
||||
if (!p.answers) continue
|
||||
for (const answer of p.answers) {
|
||||
const score = checkAnswer(answer, p.answers)
|
||||
expect(score, `${t.type}: answer "${answer}" for "${p.prompt}" should self-match`).toBeGreaterThanOrEqual(0.75)
|
||||
// Also test uppercase variant
|
||||
const upper = checkAnswer(answer.toUpperCase(), p.answers)
|
||||
expect(upper, `${t.type}: UPPER "${answer.toUpperCase()}" should match`).toBeGreaterThanOrEqual(0.75)
|
||||
checked++
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(checked).toBeGreaterThan(100)
|
||||
})
|
||||
|
||||
it('MC choices always include the correct answer', () => {
|
||||
let checked = 0
|
||||
for (const t of factualTemplates) {
|
||||
for (const p of t.prompts) {
|
||||
if (!p.choices || !p.answers) continue
|
||||
// At least one choice should match at least one answer
|
||||
const hasCorrect = p.choices.some(choice =>
|
||||
checkAnswer(choice, p.answers!) > 0
|
||||
)
|
||||
expect(hasCorrect, `${t.type}: "${p.prompt}" choices ${JSON.stringify(p.choices)} don't include correct answer ${JSON.stringify(p.answers)}`).toBe(true)
|
||||
checked++
|
||||
}
|
||||
}
|
||||
expect(checked).toBeGreaterThan(50)
|
||||
})
|
||||
})
|
||||
|
||||
describe('checkAnswer performance', () => {
|
||||
it('completes 1000 checks in under 50ms (<0.05ms each)', () => {
|
||||
const answers = ['Paris', 'London', 'Tokyo']
|
||||
|
||||
@@ -139,7 +139,7 @@ export const EXTRA_PROMPTS: Record<string, PromptEntry[]> = {
|
||||
{ prompt: 'How many zeros trailing 100! (100 factorial)?', answers: ['24'], choices: ['20', '22', '24', '25'] },
|
||||
{ prompt: 'What is the volume of a sphere with radius 3? (use π≈3.14)', answers: ['113.04', '113.1'], choices: ['84.78', '113.04', '150.72', '36'] },
|
||||
{ prompt: 'What is 0o777 in decimal?', answers: ['511'], choices: ['255', '511', '777', '999'] },
|
||||
{ prompt: 'If hash rate is 400 EH/s, how many hashes per second is that?', answers: ['4e20', '400000000000000000000'], choices: ['4×10^17', '4×10^18', '4×10^20', '4×10^22'] },
|
||||
{ prompt: 'If hash rate is 400 EH/s, how many hashes per second is that?', answers: ['4e20', '400000000000000000000', '4×10^20', '4*10^20'], choices: ['4×10^17', '4×10^18', '4×10^20', '4×10^22'] },
|
||||
{ prompt: 'What is the derivative of x³?', answers: ['3x²', '3x^2'], choices: ['x²', '2x³', '3x²', '3x'] },
|
||||
{ prompt: 'What is sin(90°)?', answers: ['1'], choices: ['0', '0.5', '1', '-1'] },
|
||||
{ prompt: 'How many unique secp256k1 private keys exist (approximately)?', answers: ['2^256', '1.16e77'], choices: ['2^128', '2^256', '2^512', '2^64'] },
|
||||
@@ -428,7 +428,7 @@ export const EXTRA_PROMPTS: Record<string, PromptEntry[]> = {
|
||||
{ prompt: 'What is a CVE?', answers: ['common vulnerabilities and exposures'], choices: ['Common Vulnerabilities and Exposures', 'Computer Virus Encyclopedia', 'Certified Vulnerability Expert', 'Core Validation Engine'] },
|
||||
{ prompt: 'What is the purpose of a firewall?', answers: ['filter network traffic', 'block unauthorized access'], choices: ['Filter network traffic', 'Speed up internet', 'Store passwords', 'Encrypt files'] },
|
||||
{ prompt: 'What tool is commonly used for penetration testing?', answers: ['metasploit', 'nmap', 'burp suite', 'kali'], choices: ['Metasploit', 'Microsoft Word', 'Photoshop', 'Notepad'] },
|
||||
{ prompt: 'What is phishing?', answers: ['fake emails to steal information', 'fraudulent communication'], choices: ['Fake communications to steal data', 'Fishing for data on the dark web', 'A type of malware', 'Hacking via phone'] },
|
||||
{ prompt: 'What is phishing?', answers: ['fake communications to steal data', 'fake emails to steal information', 'fraudulent communication'], choices: ['Fake communications to steal data', 'Fishing for data on the dark web', 'A type of malware', 'Hacking via phone'] },
|
||||
{ prompt: 'What does HTTPS use for encryption?', answers: ['tls', 'ssl', 'ssl/tls'], choices: ['TLS/SSL', 'AES only', 'RSA only', 'PGP'] },
|
||||
{ prompt: 'What is a rootkit?', answers: ['malware that hides in the os', 'hidden malware', 'stealthy malware'], choices: ['Malware that hides deep in the OS', 'A Linux admin tool', 'A password manager', 'A type of firewall'] },
|
||||
{ prompt: 'What hash algorithm is considered broken for security?', answers: ['md5', 'sha1', 'sha-1'], choices: ['MD5/SHA-1', 'SHA-256', 'bcrypt', 'Argon2'] },
|
||||
|
||||
Reference in New Issue
Block a user