feat: add theme system, improve creative scoring spam detection
Theme system (Section 5.1):
- Add PromptTheme type ('bitcoin' | 'conspiracy' | 'pc_culture' | 'bot_coding')
- Add optional theme field to PromptEntry interface
- pickChallenge now accepts optional theme bias, picks target theme with
30/20/20/30 distribution, prefers themed prompts when available
- challenges-extra.ts now imports PromptEntry type from challenge-data.ts
Creative scoring improvements (Section 7.2):
- Detect repeated phrases via trigram analysis (>25% duplicate = spam)
- Detect question echo (answer copies prompt back)
- Detect all-caps spam (>80% uppercase letters)
- Detect punctuation-only spam (<30% letter content)
- Minimum word count threshold (< 3 words = low score)
- 5 new tests: repeated phrases, question echo, all-caps, short answer, legit short
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
2ceef08f55
commit
3b863d1f5f
@@ -1,9 +1,12 @@
|
||||
// Challenge prompt data — separated from challenge logic
|
||||
|
||||
export type PromptTheme = 'bitcoin' | 'conspiracy' | 'pc_culture' | 'bot_coding'
|
||||
|
||||
export interface PromptEntry {
|
||||
prompt: string
|
||||
answers?: string[]
|
||||
choices?: string[]
|
||||
theme?: PromptTheme
|
||||
}
|
||||
|
||||
export interface ChallengeTemplate {
|
||||
|
||||
@@ -1,11 +1,7 @@
|
||||
// Extra challenge prompts — Bitcoin/cypherpunk themed + general expansion
|
||||
// Adds ~75 prompts per type to reach 2,000+ total
|
||||
|
||||
interface PromptEntry {
|
||||
prompt: string
|
||||
answers?: string[]
|
||||
choices?: string[]
|
||||
}
|
||||
import type { PromptEntry } from './challenge-data.js'
|
||||
|
||||
export const EXTRA_PROMPTS: Record<string, PromptEntry[]> = {
|
||||
speed_blitz: [
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { TEMPLATES, type ChallengeTemplate, type PromptEntry } from './challenge-data.js'
|
||||
import { TEMPLATES, type ChallengeTemplate, type PromptEntry, type PromptTheme } from './challenge-data.js'
|
||||
import { EXTRA_PROMPTS } from './challenges-extra.js'
|
||||
import { pick } from '../lib/utils.js'
|
||||
|
||||
@@ -14,7 +14,7 @@ export interface Challenge {
|
||||
displayPrompt?: string
|
||||
}
|
||||
|
||||
export type { ChallengeTemplate, PromptEntry }
|
||||
export type { ChallengeTemplate, PromptEntry, PromptTheme }
|
||||
|
||||
|
||||
// Merge extra prompts into templates
|
||||
@@ -36,7 +36,25 @@ function shuffleArray<T>(arr: T[]): T[] {
|
||||
return s
|
||||
}
|
||||
|
||||
export function pickChallenge(usedTypes: Set<string>, _arenaModifier: string | null): Challenge {
|
||||
// Target theme distribution: 30% bitcoin, 20% conspiracy, 20% pc_culture, 30% bot_coding
|
||||
const THEME_WEIGHTS: Record<PromptTheme, number> = {
|
||||
bitcoin: 0.3,
|
||||
conspiracy: 0.2,
|
||||
pc_culture: 0.2,
|
||||
bot_coding: 0.3,
|
||||
}
|
||||
|
||||
function pickTheme(): PromptTheme | undefined {
|
||||
const roll = Math.random()
|
||||
let cumulative = 0
|
||||
for (const [theme, weight] of Object.entries(THEME_WEIGHTS)) {
|
||||
cumulative += weight
|
||||
if (roll < cumulative) return theme as PromptTheme
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
export function pickChallenge(usedTypes: Set<string>, _arenaModifier: string | null, themeBias?: PromptTheme): Challenge {
|
||||
let available = TEMPLATES.filter(t => !usedTypes.has(t.type))
|
||||
if (available.length === 0) available = TEMPLATES
|
||||
|
||||
@@ -52,7 +70,8 @@ export function pickChallenge(usedTypes: Set<string>, _arenaModifier: string | n
|
||||
}
|
||||
|
||||
const template = pick(pool)
|
||||
return templateToChallenge(template)
|
||||
const targetTheme = themeBias || pickTheme()
|
||||
return templateToChallenge(template, targetTheme)
|
||||
}
|
||||
|
||||
/** Ranked challenge: no multiple choice, only harder creative/open-ended prompts */
|
||||
@@ -83,8 +102,14 @@ export function pickRankedChallenge(usedTypes: Set<string>): Challenge {
|
||||
}
|
||||
}
|
||||
|
||||
function templateToChallenge(template: ChallengeTemplate): Challenge {
|
||||
const entry = pick(template.prompts)
|
||||
function templateToChallenge(template: ChallengeTemplate, targetTheme?: PromptTheme): Challenge {
|
||||
// Prefer prompts matching target theme if any are tagged
|
||||
let prompts = template.prompts
|
||||
if (targetTheme) {
|
||||
const themed = template.prompts.filter(p => p.theme === targetTheme)
|
||||
if (themed.length > 0) prompts = themed
|
||||
}
|
||||
const entry = pick(prompts)
|
||||
|
||||
// Determine choices
|
||||
let choices: string[] | undefined
|
||||
|
||||
@@ -346,3 +346,66 @@ describe('scoreRound performance', () => {
|
||||
expect(elapsed).toBeLessThan(100) // <0.1ms per call
|
||||
})
|
||||
})
|
||||
|
||||
describe('creative scoring spam detection', () => {
|
||||
const botA = { id: 'a1', name: 'AlphaBot' }
|
||||
const botB = { id: 'b1', name: 'BetaBot' }
|
||||
const creative = makeChallenge({
|
||||
answers: undefined,
|
||||
scoring: 'creative',
|
||||
type: 'roast_battle',
|
||||
prompt: 'Write a two-sentence roast of JavaScript',
|
||||
})
|
||||
|
||||
it('repeated phrase answer loses to quality answer', () => {
|
||||
const result = scoreRound(
|
||||
creative, botA, botB,
|
||||
makeResponse('lol lol lol lol lol lol lol lol lol lol', 300),
|
||||
makeResponse('Your code is so bad even ChatGPT refuses to debug it. Every function you write is a monument to incompetence.', 300),
|
||||
null, 0, 0,
|
||||
)
|
||||
expect(result.winnerId).toBe('b1')
|
||||
})
|
||||
|
||||
it('question echo answer scores low', () => {
|
||||
const result = scoreRound(
|
||||
creative, botA, botB,
|
||||
makeResponse('Write a two-sentence roast of JavaScript', 300),
|
||||
makeResponse('JavaScript has more callbacks than a desperate ex. Even its creators apologize for it.', 300),
|
||||
null, 0, 0,
|
||||
)
|
||||
expect(result.winnerId).toBe('b1')
|
||||
})
|
||||
|
||||
it('all-caps spam scores lower than normal text', () => {
|
||||
const result = scoreRound(
|
||||
creative, botA, botB,
|
||||
makeResponse('THIS IS ALL CAPS AND IT IS VERY ANNOYING AND NOT CREATIVE AT ALL', 300),
|
||||
makeResponse('Your framework choices make me question if you have taste or just throw darts at a list.', 300),
|
||||
null, 0, 0,
|
||||
)
|
||||
expect(result.winnerId).toBe('b1')
|
||||
})
|
||||
|
||||
it('very short creative answer loses to longer quality answer', () => {
|
||||
const result = scoreRound(
|
||||
creative, botA, botB,
|
||||
makeResponse('ok', 300),
|
||||
makeResponse('Your code is so bad the compiler files a restraining order every time you open an IDE.', 300),
|
||||
null, 0, 0,
|
||||
)
|
||||
expect(result.winnerId).toBe('b1')
|
||||
})
|
||||
|
||||
it('legitimate short creative answer still gets reasonable score', () => {
|
||||
const result = scoreRound(
|
||||
creative, botA, botB,
|
||||
makeResponse('Your code has more bugs than a rainforest. Even Stack Overflow gave up on you.', 200),
|
||||
makeResponse('You write code like a poet writes math: beautifully wrong in every conceivable way.', 400),
|
||||
null, 0, 0,
|
||||
)
|
||||
// Both should get reasonable scores (not zeroed)
|
||||
expect(result.botAScore).toBeGreaterThan(2)
|
||||
expect(result.botBScore).toBeGreaterThan(2)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -154,8 +154,8 @@ export function scoreRound(
|
||||
}
|
||||
} else {
|
||||
// === CREATIVE SCORING ===
|
||||
const qualA = estimateQuality(responseA)
|
||||
const qualB = estimateQuality(responseB)
|
||||
const qualA = estimateQuality(responseA, challenge.prompt)
|
||||
const qualB = estimateQuality(responseB, challenge.prompt)
|
||||
const total = qualA + qualB || 1
|
||||
scoreA = (qualA / total) * CREATIVE_TOTAL_SCORE
|
||||
scoreB = (qualB / total) * CREATIVE_TOTAL_SCORE
|
||||
@@ -232,7 +232,7 @@ function applyModifiers(
|
||||
return d
|
||||
}
|
||||
|
||||
function estimateQuality(response: BotResponse): number {
|
||||
function estimateQuality(response: BotResponse, challengePrompt?: string): number {
|
||||
if (!response.answer) return EMPTY_RESPONSE_QUALITY
|
||||
const text = response.answer.trim()
|
||||
const len = text.length
|
||||
@@ -249,6 +249,39 @@ function estimateQuality(response: BotResponse): number {
|
||||
const uniqueWords = new Set(words.map(w => w.toLowerCase()))
|
||||
const wordDiversity = uniqueWords.size / Math.max(words.length, 1)
|
||||
|
||||
// Minimum word count — single-word or two-word answers score low for creative
|
||||
if (words.length < 3) return 1.5
|
||||
|
||||
// Detect repeated phrases (same 3+ word sequence appears twice)
|
||||
if (words.length >= 6) {
|
||||
const trigrams = new Set<string>()
|
||||
let dupeTrigramCount = 0
|
||||
for (let i = 0; i <= words.length - 3; i++) {
|
||||
const tri = words.slice(i, i + 3).join(' ').toLowerCase()
|
||||
if (trigrams.has(tri)) dupeTrigramCount++
|
||||
else trigrams.add(tri)
|
||||
}
|
||||
if (dupeTrigramCount > words.length / 4) return 1 // >25% duplicate trigrams
|
||||
}
|
||||
|
||||
// Detect question echo (response copies the prompt back)
|
||||
if (challengePrompt) {
|
||||
const normPrompt = challengePrompt.toLowerCase().replace(/[^\w\s]/g, '').trim()
|
||||
const normAnswer = text.toLowerCase().replace(/[^\w\s]/g, '').trim()
|
||||
if (normPrompt.length > 10 && normAnswer.includes(normPrompt)) return 1
|
||||
}
|
||||
|
||||
// Detect all-caps spam
|
||||
const upperCount = text.replace(/[^A-Z]/g, '').length
|
||||
const letterCount = text.replace(/[^a-zA-Z]/g, '').length
|
||||
if (letterCount > 20 && upperCount / letterCount > 0.8) {
|
||||
// Heavy penalty for all-caps but don't zero it
|
||||
return 1.5
|
||||
}
|
||||
|
||||
// Detect punctuation-only or near-punctuation spam
|
||||
if (letterCount < len * 0.3 && len > 10) return EMPTY_RESPONSE_QUALITY
|
||||
|
||||
// Ideal length window
|
||||
let lengthScore: number
|
||||
if (len >= QUALITY_LENGTH_IDEAL_MIN && len <= QUALITY_LENGTH_IDEAL_MAX) lengthScore = QUALITY_SCORE_IDEAL
|
||||
|
||||
Reference in New Issue
Block a user