feat: add theme system, improve creative scoring spam detection
Theme system (Section 5.1):
- Add PromptTheme type ('bitcoin' | 'conspiracy' | 'pc_culture' | 'bot_coding')
- Add optional theme field to PromptEntry interface
- pickChallenge now accepts optional theme bias, picks target theme with
30/20/20/30 distribution, prefers themed prompts when available
- challenges-extra.ts now imports PromptEntry type from challenge-data.ts
Creative scoring improvements (Section 7.2):
- Detect repeated phrases via trigram analysis (>25% duplicate = spam)
- Detect question echo (answer copies prompt back)
- Detect all-caps spam (>80% uppercase letters)
- Detect punctuation-only spam (<30% letter content)
- Minimum word count threshold (< 3 words = low score)
- 5 new tests: repeated phrases, question echo, all-caps, short answer, legit short
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
2ceef08f55
commit
3b863d1f5f
@@ -1,4 +1,4 @@
|
||||
import { TEMPLATES, type ChallengeTemplate, type PromptEntry } from './challenge-data.js'
|
||||
import { TEMPLATES, type ChallengeTemplate, type PromptEntry, type PromptTheme } from './challenge-data.js'
|
||||
import { EXTRA_PROMPTS } from './challenges-extra.js'
|
||||
import { pick } from '../lib/utils.js'
|
||||
|
||||
@@ -14,7 +14,7 @@ export interface Challenge {
|
||||
displayPrompt?: string
|
||||
}
|
||||
|
||||
export type { ChallengeTemplate, PromptEntry }
|
||||
export type { ChallengeTemplate, PromptEntry, PromptTheme }
|
||||
|
||||
|
||||
// Merge extra prompts into templates
|
||||
@@ -36,7 +36,25 @@ function shuffleArray<T>(arr: T[]): T[] {
|
||||
return s
|
||||
}
|
||||
|
||||
export function pickChallenge(usedTypes: Set<string>, _arenaModifier: string | null): Challenge {
|
||||
// Target theme distribution: 30% bitcoin, 20% conspiracy, 20% pc_culture, 30% bot_coding
|
||||
const THEME_WEIGHTS: Record<PromptTheme, number> = {
|
||||
bitcoin: 0.3,
|
||||
conspiracy: 0.2,
|
||||
pc_culture: 0.2,
|
||||
bot_coding: 0.3,
|
||||
}
|
||||
|
||||
function pickTheme(): PromptTheme | undefined {
|
||||
const roll = Math.random()
|
||||
let cumulative = 0
|
||||
for (const [theme, weight] of Object.entries(THEME_WEIGHTS)) {
|
||||
cumulative += weight
|
||||
if (roll < cumulative) return theme as PromptTheme
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
export function pickChallenge(usedTypes: Set<string>, _arenaModifier: string | null, themeBias?: PromptTheme): Challenge {
|
||||
let available = TEMPLATES.filter(t => !usedTypes.has(t.type))
|
||||
if (available.length === 0) available = TEMPLATES
|
||||
|
||||
@@ -52,7 +70,8 @@ export function pickChallenge(usedTypes: Set<string>, _arenaModifier: string | n
|
||||
}
|
||||
|
||||
const template = pick(pool)
|
||||
return templateToChallenge(template)
|
||||
const targetTheme = themeBias || pickTheme()
|
||||
return templateToChallenge(template, targetTheme)
|
||||
}
|
||||
|
||||
/** Ranked challenge: no multiple choice, only harder creative/open-ended prompts */
|
||||
@@ -83,8 +102,14 @@ export function pickRankedChallenge(usedTypes: Set<string>): Challenge {
|
||||
}
|
||||
}
|
||||
|
||||
function templateToChallenge(template: ChallengeTemplate): Challenge {
|
||||
const entry = pick(template.prompts)
|
||||
function templateToChallenge(template: ChallengeTemplate, targetTheme?: PromptTheme): Challenge {
|
||||
// Prefer prompts matching target theme if any are tagged
|
||||
let prompts = template.prompts
|
||||
if (targetTheme) {
|
||||
const themed = template.prompts.filter(p => p.theme === targetTheme)
|
||||
if (themed.length > 0) prompts = themed
|
||||
}
|
||||
const entry = pick(prompts)
|
||||
|
||||
// Determine choices
|
||||
let choices: string[] | undefined
|
||||
|
||||
Reference in New Issue
Block a user