diff --git a/server/src/engine/challenge-data.ts b/server/src/engine/challenge-data.ts index c09081b..447dbd4 100644 --- a/server/src/engine/challenge-data.ts +++ b/server/src/engine/challenge-data.ts @@ -1,9 +1,12 @@ // Challenge prompt data — separated from challenge logic +export type PromptTheme = 'bitcoin' | 'conspiracy' | 'pc_culture' | 'bot_coding' + export interface PromptEntry { prompt: string answers?: string[] choices?: string[] + theme?: PromptTheme } export interface ChallengeTemplate { diff --git a/server/src/engine/challenges-extra.ts b/server/src/engine/challenges-extra.ts index cc2590b..f650801 100644 --- a/server/src/engine/challenges-extra.ts +++ b/server/src/engine/challenges-extra.ts @@ -1,11 +1,7 @@ // Extra challenge prompts — Bitcoin/cypherpunk themed + general expansion // Adds ~75 prompts per type to reach 2,000+ total -interface PromptEntry { - prompt: string - answers?: string[] - choices?: string[] -} +import type { PromptEntry } from './challenge-data.js' export const EXTRA_PROMPTS: Record = { speed_blitz: [ diff --git a/server/src/engine/challenges.ts b/server/src/engine/challenges.ts index 9b183fe..d00490e 100644 --- a/server/src/engine/challenges.ts +++ b/server/src/engine/challenges.ts @@ -1,4 +1,4 @@ -import { TEMPLATES, type ChallengeTemplate, type PromptEntry } from './challenge-data.js' +import { TEMPLATES, type ChallengeTemplate, type PromptEntry, type PromptTheme } from './challenge-data.js' import { EXTRA_PROMPTS } from './challenges-extra.js' import { pick } from '../lib/utils.js' @@ -14,7 +14,7 @@ export interface Challenge { displayPrompt?: string } -export type { ChallengeTemplate, PromptEntry } +export type { ChallengeTemplate, PromptEntry, PromptTheme } // Merge extra prompts into templates @@ -36,7 +36,25 @@ function shuffleArray(arr: T[]): T[] { return s } -export function pickChallenge(usedTypes: Set, _arenaModifier: string | null): Challenge { +// Target theme distribution: 30% bitcoin, 20% conspiracy, 20% pc_culture, 30% bot_coding +const THEME_WEIGHTS: Record = { + bitcoin: 0.3, + conspiracy: 0.2, + pc_culture: 0.2, + bot_coding: 0.3, +} + +function pickTheme(): PromptTheme | undefined { + const roll = Math.random() + let cumulative = 0 + for (const [theme, weight] of Object.entries(THEME_WEIGHTS)) { + cumulative += weight + if (roll < cumulative) return theme as PromptTheme + } + return undefined +} + +export function pickChallenge(usedTypes: Set, _arenaModifier: string | null, themeBias?: PromptTheme): Challenge { let available = TEMPLATES.filter(t => !usedTypes.has(t.type)) if (available.length === 0) available = TEMPLATES @@ -52,7 +70,8 @@ export function pickChallenge(usedTypes: Set, _arenaModifier: string | n } const template = pick(pool) - return templateToChallenge(template) + const targetTheme = themeBias || pickTheme() + return templateToChallenge(template, targetTheme) } /** Ranked challenge: no multiple choice, only harder creative/open-ended prompts */ @@ -83,8 +102,14 @@ export function pickRankedChallenge(usedTypes: Set): Challenge { } } -function templateToChallenge(template: ChallengeTemplate): Challenge { - const entry = pick(template.prompts) +function templateToChallenge(template: ChallengeTemplate, targetTheme?: PromptTheme): Challenge { + // Prefer prompts matching target theme if any are tagged + let prompts = template.prompts + if (targetTheme) { + const themed = template.prompts.filter(p => p.theme === targetTheme) + if (themed.length > 0) prompts = themed + } + const entry = pick(prompts) // Determine choices let choices: string[] | undefined diff --git a/server/src/engine/scoring.test.ts b/server/src/engine/scoring.test.ts index a5034fd..11836f6 100644 --- a/server/src/engine/scoring.test.ts +++ b/server/src/engine/scoring.test.ts @@ -346,3 +346,66 @@ describe('scoreRound performance', () => { expect(elapsed).toBeLessThan(100) // <0.1ms per call }) }) + +describe('creative scoring spam detection', () => { + const botA = { id: 'a1', name: 'AlphaBot' } + const botB = { id: 'b1', name: 'BetaBot' } + const creative = makeChallenge({ + answers: undefined, + scoring: 'creative', + type: 'roast_battle', + prompt: 'Write a two-sentence roast of JavaScript', + }) + + it('repeated phrase answer loses to quality answer', () => { + const result = scoreRound( + creative, botA, botB, + makeResponse('lol lol lol lol lol lol lol lol lol lol', 300), + makeResponse('Your code is so bad even ChatGPT refuses to debug it. Every function you write is a monument to incompetence.', 300), + null, 0, 0, + ) + expect(result.winnerId).toBe('b1') + }) + + it('question echo answer scores low', () => { + const result = scoreRound( + creative, botA, botB, + makeResponse('Write a two-sentence roast of JavaScript', 300), + makeResponse('JavaScript has more callbacks than a desperate ex. Even its creators apologize for it.', 300), + null, 0, 0, + ) + expect(result.winnerId).toBe('b1') + }) + + it('all-caps spam scores lower than normal text', () => { + const result = scoreRound( + creative, botA, botB, + makeResponse('THIS IS ALL CAPS AND IT IS VERY ANNOYING AND NOT CREATIVE AT ALL', 300), + makeResponse('Your framework choices make me question if you have taste or just throw darts at a list.', 300), + null, 0, 0, + ) + expect(result.winnerId).toBe('b1') + }) + + it('very short creative answer loses to longer quality answer', () => { + const result = scoreRound( + creative, botA, botB, + makeResponse('ok', 300), + makeResponse('Your code is so bad the compiler files a restraining order every time you open an IDE.', 300), + null, 0, 0, + ) + expect(result.winnerId).toBe('b1') + }) + + it('legitimate short creative answer still gets reasonable score', () => { + const result = scoreRound( + creative, botA, botB, + makeResponse('Your code has more bugs than a rainforest. Even Stack Overflow gave up on you.', 200), + makeResponse('You write code like a poet writes math: beautifully wrong in every conceivable way.', 400), + null, 0, 0, + ) + // Both should get reasonable scores (not zeroed) + expect(result.botAScore).toBeGreaterThan(2) + expect(result.botBScore).toBeGreaterThan(2) + }) +}) diff --git a/server/src/engine/scoring.ts b/server/src/engine/scoring.ts index 20cf101..ed4a2cc 100644 --- a/server/src/engine/scoring.ts +++ b/server/src/engine/scoring.ts @@ -154,8 +154,8 @@ export function scoreRound( } } else { // === CREATIVE SCORING === - const qualA = estimateQuality(responseA) - const qualB = estimateQuality(responseB) + const qualA = estimateQuality(responseA, challenge.prompt) + const qualB = estimateQuality(responseB, challenge.prompt) const total = qualA + qualB || 1 scoreA = (qualA / total) * CREATIVE_TOTAL_SCORE scoreB = (qualB / total) * CREATIVE_TOTAL_SCORE @@ -232,7 +232,7 @@ function applyModifiers( return d } -function estimateQuality(response: BotResponse): number { +function estimateQuality(response: BotResponse, challengePrompt?: string): number { if (!response.answer) return EMPTY_RESPONSE_QUALITY const text = response.answer.trim() const len = text.length @@ -249,6 +249,39 @@ function estimateQuality(response: BotResponse): number { const uniqueWords = new Set(words.map(w => w.toLowerCase())) const wordDiversity = uniqueWords.size / Math.max(words.length, 1) + // Minimum word count — single-word or two-word answers score low for creative + if (words.length < 3) return 1.5 + + // Detect repeated phrases (same 3+ word sequence appears twice) + if (words.length >= 6) { + const trigrams = new Set() + let dupeTrigramCount = 0 + for (let i = 0; i <= words.length - 3; i++) { + const tri = words.slice(i, i + 3).join(' ').toLowerCase() + if (trigrams.has(tri)) dupeTrigramCount++ + else trigrams.add(tri) + } + if (dupeTrigramCount > words.length / 4) return 1 // >25% duplicate trigrams + } + + // Detect question echo (response copies the prompt back) + if (challengePrompt) { + const normPrompt = challengePrompt.toLowerCase().replace(/[^\w\s]/g, '').trim() + const normAnswer = text.toLowerCase().replace(/[^\w\s]/g, '').trim() + if (normPrompt.length > 10 && normAnswer.includes(normPrompt)) return 1 + } + + // Detect all-caps spam + const upperCount = text.replace(/[^A-Z]/g, '').length + const letterCount = text.replace(/[^a-zA-Z]/g, '').length + if (letterCount > 20 && upperCount / letterCount > 0.8) { + // Heavy penalty for all-caps but don't zero it + return 1.5 + } + + // Detect punctuation-only or near-punctuation spam + if (letterCount < len * 0.3 && len > 10) return EMPTY_RESPONSE_QUALITY + // Ideal length window let lengthScore: number if (len >= QUALITY_LENGTH_IDEAL_MIN && len <= QUALITY_LENGTH_IDEAL_MAX) lengthScore = QUALITY_SCORE_IDEAL