feat: add theme system, improve creative scoring spam detection
Theme system (Section 5.1):
- Add PromptTheme type ('bitcoin' | 'conspiracy' | 'pc_culture' | 'bot_coding')
- Add optional theme field to PromptEntry interface
- pickChallenge now accepts optional theme bias, picks target theme with
30/20/20/30 distribution, prefers themed prompts when available
- challenges-extra.ts now imports PromptEntry type from challenge-data.ts
Creative scoring improvements (Section 7.2):
- Detect repeated phrases via trigram analysis (>25% duplicate = spam)
- Detect question echo (answer copies prompt back)
- Detect all-caps spam (>80% uppercase letters)
- Detect punctuation-only spam (<30% letter content)
- Minimum word count threshold (< 3 words = low score)
- 5 new tests: repeated phrases, question echo, all-caps, short answer, legit short
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
2ceef08f55
commit
3b863d1f5f
@@ -346,3 +346,66 @@ describe('scoreRound performance', () => {
|
||||
expect(elapsed).toBeLessThan(100) // <0.1ms per call
|
||||
})
|
||||
})
|
||||
|
||||
describe('creative scoring spam detection', () => {
|
||||
const botA = { id: 'a1', name: 'AlphaBot' }
|
||||
const botB = { id: 'b1', name: 'BetaBot' }
|
||||
const creative = makeChallenge({
|
||||
answers: undefined,
|
||||
scoring: 'creative',
|
||||
type: 'roast_battle',
|
||||
prompt: 'Write a two-sentence roast of JavaScript',
|
||||
})
|
||||
|
||||
it('repeated phrase answer loses to quality answer', () => {
|
||||
const result = scoreRound(
|
||||
creative, botA, botB,
|
||||
makeResponse('lol lol lol lol lol lol lol lol lol lol', 300),
|
||||
makeResponse('Your code is so bad even ChatGPT refuses to debug it. Every function you write is a monument to incompetence.', 300),
|
||||
null, 0, 0,
|
||||
)
|
||||
expect(result.winnerId).toBe('b1')
|
||||
})
|
||||
|
||||
it('question echo answer scores low', () => {
|
||||
const result = scoreRound(
|
||||
creative, botA, botB,
|
||||
makeResponse('Write a two-sentence roast of JavaScript', 300),
|
||||
makeResponse('JavaScript has more callbacks than a desperate ex. Even its creators apologize for it.', 300),
|
||||
null, 0, 0,
|
||||
)
|
||||
expect(result.winnerId).toBe('b1')
|
||||
})
|
||||
|
||||
it('all-caps spam scores lower than normal text', () => {
|
||||
const result = scoreRound(
|
||||
creative, botA, botB,
|
||||
makeResponse('THIS IS ALL CAPS AND IT IS VERY ANNOYING AND NOT CREATIVE AT ALL', 300),
|
||||
makeResponse('Your framework choices make me question if you have taste or just throw darts at a list.', 300),
|
||||
null, 0, 0,
|
||||
)
|
||||
expect(result.winnerId).toBe('b1')
|
||||
})
|
||||
|
||||
it('very short creative answer loses to longer quality answer', () => {
|
||||
const result = scoreRound(
|
||||
creative, botA, botB,
|
||||
makeResponse('ok', 300),
|
||||
makeResponse('Your code is so bad the compiler files a restraining order every time you open an IDE.', 300),
|
||||
null, 0, 0,
|
||||
)
|
||||
expect(result.winnerId).toBe('b1')
|
||||
})
|
||||
|
||||
it('legitimate short creative answer still gets reasonable score', () => {
|
||||
const result = scoreRound(
|
||||
creative, botA, botB,
|
||||
makeResponse('Your code has more bugs than a rainforest. Even Stack Overflow gave up on you.', 200),
|
||||
makeResponse('You write code like a poet writes math: beautifully wrong in every conceivable way.', 400),
|
||||
null, 0, 0,
|
||||
)
|
||||
// Both should get reasonable scores (not zeroed)
|
||||
expect(result.botAScore).toBeGreaterThan(2)
|
||||
expect(result.botBScore).toBeGreaterThan(2)
|
||||
})
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user