feat: add theme system, improve creative scoring spam detection

Theme system (Section 5.1):
- Add PromptTheme type ('bitcoin' | 'conspiracy' | 'pc_culture' | 'bot_coding')
- Add optional theme field to PromptEntry interface
- pickChallenge now accepts optional theme bias, picks target theme with
  30/20/20/30 distribution, prefers themed prompts when available
- challenges-extra.ts now imports PromptEntry type from challenge-data.ts

Creative scoring improvements (Section 7.2):
- Detect repeated phrases via trigram analysis (>25% duplicate = spam)
- Detect question echo (answer copies prompt back)
- Detect all-caps spam (>80% uppercase letters)
- Detect punctuation-only spam (<30% letter content)
- Minimum word count threshold (< 3 words = low score)
- 5 new tests: repeated phrases, question echo, all-caps, short answer, legit short

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Dorian
2026-03-09 07:56:51 +00:00
co-authored by Claude Opus 4.6
parent 2ceef08f55
commit 3b863d1f5f
5 changed files with 134 additions and 14 deletions
+31 -6
View File
@@ -1,4 +1,4 @@
import { TEMPLATES, type ChallengeTemplate, type PromptEntry } from './challenge-data.js'
import { TEMPLATES, type ChallengeTemplate, type PromptEntry, type PromptTheme } from './challenge-data.js'
import { EXTRA_PROMPTS } from './challenges-extra.js'
import { pick } from '../lib/utils.js'
@@ -14,7 +14,7 @@ export interface Challenge {
displayPrompt?: string
}
export type { ChallengeTemplate, PromptEntry }
export type { ChallengeTemplate, PromptEntry, PromptTheme }
// Merge extra prompts into templates
@@ -36,7 +36,25 @@ function shuffleArray<T>(arr: T[]): T[] {
return s
}
export function pickChallenge(usedTypes: Set<string>, _arenaModifier: string | null): Challenge {
// Target theme distribution: 30% bitcoin, 20% conspiracy, 20% pc_culture, 30% bot_coding
const THEME_WEIGHTS: Record<PromptTheme, number> = {
bitcoin: 0.3,
conspiracy: 0.2,
pc_culture: 0.2,
bot_coding: 0.3,
}
function pickTheme(): PromptTheme | undefined {
const roll = Math.random()
let cumulative = 0
for (const [theme, weight] of Object.entries(THEME_WEIGHTS)) {
cumulative += weight
if (roll < cumulative) return theme as PromptTheme
}
return undefined
}
export function pickChallenge(usedTypes: Set<string>, _arenaModifier: string | null, themeBias?: PromptTheme): Challenge {
let available = TEMPLATES.filter(t => !usedTypes.has(t.type))
if (available.length === 0) available = TEMPLATES
@@ -52,7 +70,8 @@ export function pickChallenge(usedTypes: Set<string>, _arenaModifier: string | n
}
const template = pick(pool)
return templateToChallenge(template)
const targetTheme = themeBias || pickTheme()
return templateToChallenge(template, targetTheme)
}
/** Ranked challenge: no multiple choice, only harder creative/open-ended prompts */
@@ -83,8 +102,14 @@ export function pickRankedChallenge(usedTypes: Set<string>): Challenge {
}
}
function templateToChallenge(template: ChallengeTemplate): Challenge {
const entry = pick(template.prompts)
function templateToChallenge(template: ChallengeTemplate, targetTheme?: PromptTheme): Challenge {
// Prefer prompts matching target theme if any are tagged
let prompts = template.prompts
if (targetTheme) {
const themed = template.prompts.filter(p => p.theme === targetTheme)
if (themed.length > 0) prompts = themed
}
const entry = pick(prompts)
// Determine choices
let choices: string[] | undefined