feat: add theme system, improve creative scoring spam detection

Theme system (Section 5.1):
- Add PromptTheme type ('bitcoin' | 'conspiracy' | 'pc_culture' | 'bot_coding')
- Add optional theme field to PromptEntry interface
- pickChallenge now accepts optional theme bias, picks target theme with
  30/20/20/30 distribution, prefers themed prompts when available
- challenges-extra.ts now imports PromptEntry type from challenge-data.ts

Creative scoring improvements (Section 7.2):
- Detect repeated phrases via trigram analysis (>25% duplicate = spam)
- Detect question echo (answer copies prompt back)
- Detect all-caps spam (>80% uppercase letters)
- Detect punctuation-only spam (<30% letter content)
- Minimum word count threshold (< 3 words = low score)
- 5 new tests: repeated phrases, question echo, all-caps, short answer, legit short

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Dorian
2026-03-09 07:56:51 +00:00
co-authored by Claude Opus 4.6
parent 2ceef08f55
commit 3b863d1f5f
5 changed files with 134 additions and 14 deletions
+3
View File
@@ -1,9 +1,12 @@
// Challenge prompt data — separated from challenge logic
export type PromptTheme = 'bitcoin' | 'conspiracy' | 'pc_culture' | 'bot_coding'
export interface PromptEntry {
prompt: string
answers?: string[]
choices?: string[]
theme?: PromptTheme
}
export interface ChallengeTemplate {
+1 -5
View File
@@ -1,11 +1,7 @@
// Extra challenge prompts — Bitcoin/cypherpunk themed + general expansion
// Adds ~75 prompts per type to reach 2,000+ total
interface PromptEntry {
prompt: string
answers?: string[]
choices?: string[]
}
import type { PromptEntry } from './challenge-data.js'
export const EXTRA_PROMPTS: Record<string, PromptEntry[]> = {
speed_blitz: [
+31 -6
View File
@@ -1,4 +1,4 @@
import { TEMPLATES, type ChallengeTemplate, type PromptEntry } from './challenge-data.js'
import { TEMPLATES, type ChallengeTemplate, type PromptEntry, type PromptTheme } from './challenge-data.js'
import { EXTRA_PROMPTS } from './challenges-extra.js'
import { pick } from '../lib/utils.js'
@@ -14,7 +14,7 @@ export interface Challenge {
displayPrompt?: string
}
export type { ChallengeTemplate, PromptEntry }
export type { ChallengeTemplate, PromptEntry, PromptTheme }
// Merge extra prompts into templates
@@ -36,7 +36,25 @@ function shuffleArray<T>(arr: T[]): T[] {
return s
}
export function pickChallenge(usedTypes: Set<string>, _arenaModifier: string | null): Challenge {
// Target theme distribution: 30% bitcoin, 20% conspiracy, 20% pc_culture, 30% bot_coding
const THEME_WEIGHTS: Record<PromptTheme, number> = {
bitcoin: 0.3,
conspiracy: 0.2,
pc_culture: 0.2,
bot_coding: 0.3,
}
function pickTheme(): PromptTheme | undefined {
const roll = Math.random()
let cumulative = 0
for (const [theme, weight] of Object.entries(THEME_WEIGHTS)) {
cumulative += weight
if (roll < cumulative) return theme as PromptTheme
}
return undefined
}
export function pickChallenge(usedTypes: Set<string>, _arenaModifier: string | null, themeBias?: PromptTheme): Challenge {
let available = TEMPLATES.filter(t => !usedTypes.has(t.type))
if (available.length === 0) available = TEMPLATES
@@ -52,7 +70,8 @@ export function pickChallenge(usedTypes: Set<string>, _arenaModifier: string | n
}
const template = pick(pool)
return templateToChallenge(template)
const targetTheme = themeBias || pickTheme()
return templateToChallenge(template, targetTheme)
}
/** Ranked challenge: no multiple choice, only harder creative/open-ended prompts */
@@ -83,8 +102,14 @@ export function pickRankedChallenge(usedTypes: Set<string>): Challenge {
}
}
function templateToChallenge(template: ChallengeTemplate): Challenge {
const entry = pick(template.prompts)
function templateToChallenge(template: ChallengeTemplate, targetTheme?: PromptTheme): Challenge {
// Prefer prompts matching target theme if any are tagged
let prompts = template.prompts
if (targetTheme) {
const themed = template.prompts.filter(p => p.theme === targetTheme)
if (themed.length > 0) prompts = themed
}
const entry = pick(prompts)
// Determine choices
let choices: string[] | undefined
+63
View File
@@ -346,3 +346,66 @@ describe('scoreRound performance', () => {
expect(elapsed).toBeLessThan(100) // <0.1ms per call
})
})
describe('creative scoring spam detection', () => {
const botA = { id: 'a1', name: 'AlphaBot' }
const botB = { id: 'b1', name: 'BetaBot' }
const creative = makeChallenge({
answers: undefined,
scoring: 'creative',
type: 'roast_battle',
prompt: 'Write a two-sentence roast of JavaScript',
})
it('repeated phrase answer loses to quality answer', () => {
const result = scoreRound(
creative, botA, botB,
makeResponse('lol lol lol lol lol lol lol lol lol lol', 300),
makeResponse('Your code is so bad even ChatGPT refuses to debug it. Every function you write is a monument to incompetence.', 300),
null, 0, 0,
)
expect(result.winnerId).toBe('b1')
})
it('question echo answer scores low', () => {
const result = scoreRound(
creative, botA, botB,
makeResponse('Write a two-sentence roast of JavaScript', 300),
makeResponse('JavaScript has more callbacks than a desperate ex. Even its creators apologize for it.', 300),
null, 0, 0,
)
expect(result.winnerId).toBe('b1')
})
it('all-caps spam scores lower than normal text', () => {
const result = scoreRound(
creative, botA, botB,
makeResponse('THIS IS ALL CAPS AND IT IS VERY ANNOYING AND NOT CREATIVE AT ALL', 300),
makeResponse('Your framework choices make me question if you have taste or just throw darts at a list.', 300),
null, 0, 0,
)
expect(result.winnerId).toBe('b1')
})
it('very short creative answer loses to longer quality answer', () => {
const result = scoreRound(
creative, botA, botB,
makeResponse('ok', 300),
makeResponse('Your code is so bad the compiler files a restraining order every time you open an IDE.', 300),
null, 0, 0,
)
expect(result.winnerId).toBe('b1')
})
it('legitimate short creative answer still gets reasonable score', () => {
const result = scoreRound(
creative, botA, botB,
makeResponse('Your code has more bugs than a rainforest. Even Stack Overflow gave up on you.', 200),
makeResponse('You write code like a poet writes math: beautifully wrong in every conceivable way.', 400),
null, 0, 0,
)
// Both should get reasonable scores (not zeroed)
expect(result.botAScore).toBeGreaterThan(2)
expect(result.botBScore).toBeGreaterThan(2)
})
})
+36 -3
View File
@@ -154,8 +154,8 @@ export function scoreRound(
}
} else {
// === CREATIVE SCORING ===
const qualA = estimateQuality(responseA)
const qualB = estimateQuality(responseB)
const qualA = estimateQuality(responseA, challenge.prompt)
const qualB = estimateQuality(responseB, challenge.prompt)
const total = qualA + qualB || 1
scoreA = (qualA / total) * CREATIVE_TOTAL_SCORE
scoreB = (qualB / total) * CREATIVE_TOTAL_SCORE
@@ -232,7 +232,7 @@ function applyModifiers(
return d
}
function estimateQuality(response: BotResponse): number {
function estimateQuality(response: BotResponse, challengePrompt?: string): number {
if (!response.answer) return EMPTY_RESPONSE_QUALITY
const text = response.answer.trim()
const len = text.length
@@ -249,6 +249,39 @@ function estimateQuality(response: BotResponse): number {
const uniqueWords = new Set(words.map(w => w.toLowerCase()))
const wordDiversity = uniqueWords.size / Math.max(words.length, 1)
// Minimum word count — single-word or two-word answers score low for creative
if (words.length < 3) return 1.5
// Detect repeated phrases (same 3+ word sequence appears twice)
if (words.length >= 6) {
const trigrams = new Set<string>()
let dupeTrigramCount = 0
for (let i = 0; i <= words.length - 3; i++) {
const tri = words.slice(i, i + 3).join(' ').toLowerCase()
if (trigrams.has(tri)) dupeTrigramCount++
else trigrams.add(tri)
}
if (dupeTrigramCount > words.length / 4) return 1 // >25% duplicate trigrams
}
// Detect question echo (response copies the prompt back)
if (challengePrompt) {
const normPrompt = challengePrompt.toLowerCase().replace(/[^\w\s]/g, '').trim()
const normAnswer = text.toLowerCase().replace(/[^\w\s]/g, '').trim()
if (normPrompt.length > 10 && normAnswer.includes(normPrompt)) return 1
}
// Detect all-caps spam
const upperCount = text.replace(/[^A-Z]/g, '').length
const letterCount = text.replace(/[^a-zA-Z]/g, '').length
if (letterCount > 20 && upperCount / letterCount > 0.8) {
// Heavy penalty for all-caps but don't zero it
return 1.5
}
// Detect punctuation-only or near-punctuation spam
if (letterCount < len * 0.3 && len > 10) return EMPTY_RESPONSE_QUALITY
// Ideal length window
let lengthScore: number
if (len >= QUALITY_LENGTH_IDEAL_MIN && len <= QUALITY_LENGTH_IDEAL_MAX) lengthScore = QUALITY_SCORE_IDEAL