fix: only generate MC choices for human fights, not bot-vs-bot
Bots answer via webhook and don't need multiple choice options. Add forHuman parameter to pickChallenge, update tests to cover both modes. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
dcd8570ed8
commit
592841d7b1
@@ -48,9 +48,9 @@ describe('pickChallenge', () => {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
it('creative challenges have MC choices and a correct answer', () => {
|
it('creative challenges have MC choices and a correct answer (human mode)', () => {
|
||||||
for (let i = 0; i < 100; i++) {
|
for (let i = 0; i < 100; i++) {
|
||||||
const c = pickChallenge(new Set(), null)
|
const c = pickChallenge(new Set(), null, undefined, undefined, true)
|
||||||
if (c.scoring === 'creative') {
|
if (c.scoring === 'creative') {
|
||||||
// Creative challenges now have auto-generated MC choices with a correct answer
|
// Creative challenges now have auto-generated MC choices with a correct answer
|
||||||
expect(c.choices).toBeDefined()
|
expect(c.choices).toBeDefined()
|
||||||
@@ -63,10 +63,17 @@ describe('pickChallenge', () => {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
it('factual challenges with choices have shuffled choices', () => {
|
it('bot challenges have no MC choices', () => {
|
||||||
|
for (let i = 0; i < 100; i++) {
|
||||||
|
const c = pickChallenge(new Set(), null)
|
||||||
|
expect(c.choices).toBeUndefined()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('factual challenges with choices have shuffled choices (human mode)', () => {
|
||||||
const orders = new Set<string>()
|
const orders = new Set<string>()
|
||||||
for (let i = 0; i < 50; i++) {
|
for (let i = 0; i < 50; i++) {
|
||||||
const c = pickChallenge(new Set(), null)
|
const c = pickChallenge(new Set(), null, undefined, undefined, true)
|
||||||
if (c.scoring === 'factual' && c.choices) {
|
if (c.scoring === 'factual' && c.choices) {
|
||||||
orders.add(c.choices.join(','))
|
orders.add(c.choices.join(','))
|
||||||
}
|
}
|
||||||
@@ -90,11 +97,11 @@ describe('pickChallenge', () => {
|
|||||||
expect(factualPct).toBeLessThan(0.85)
|
expect(factualPct).toBeLessThan(0.85)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('True/False auto-generation for boolean answers', () => {
|
it('True/False auto-generation for boolean answers (human mode)', () => {
|
||||||
// Pick many challenges, find ones with answers = ['true'] or ['false']
|
// Pick many challenges, find ones with answers = ['true'] or ['false']
|
||||||
let foundTFWithChoices = false
|
let foundTFWithChoices = false
|
||||||
for (let i = 0; i < 500; i++) {
|
for (let i = 0; i < 500; i++) {
|
||||||
const c = pickChallenge(new Set(), null)
|
const c = pickChallenge(new Set(), null, undefined, undefined, true)
|
||||||
if (c.answers?.length === 1 && ['true', 'false'].includes(c.answers[0].toLowerCase())) {
|
if (c.answers?.length === 1 && ['true', 'false'].includes(c.answers[0].toLowerCase())) {
|
||||||
expect(c.choices).toBeTruthy()
|
expect(c.choices).toBeTruthy()
|
||||||
expect(c.choices!.length).toBe(2)
|
expect(c.choices!.length).toBe(2)
|
||||||
|
|||||||
@@ -70,7 +70,7 @@ export function roundToDifficulty(round: number): PromptDifficulty {
|
|||||||
return 'hard'
|
return 'hard'
|
||||||
}
|
}
|
||||||
|
|
||||||
export function pickChallenge(usedTypes: Set<string>, _arenaModifier: string | null, themeBias?: PromptTheme, roundNumber?: number): Challenge {
|
export function pickChallenge(usedTypes: Set<string>, _arenaModifier: string | null, themeBias?: PromptTheme, roundNumber?: number, forHuman = false): Challenge {
|
||||||
let available = TEMPLATES.filter(t => !usedTypes.has(t.type))
|
let available = TEMPLATES.filter(t => !usedTypes.has(t.type))
|
||||||
if (available.length === 0) available = TEMPLATES
|
if (available.length === 0) available = TEMPLATES
|
||||||
|
|
||||||
@@ -88,7 +88,7 @@ export function pickChallenge(usedTypes: Set<string>, _arenaModifier: string | n
|
|||||||
const template = pick(pool)
|
const template = pick(pool)
|
||||||
const targetTheme = themeBias || pickTheme()
|
const targetTheme = themeBias || pickTheme()
|
||||||
const targetDifficulty = roundNumber ? roundToDifficulty(roundNumber) : undefined
|
const targetDifficulty = roundNumber ? roundToDifficulty(roundNumber) : undefined
|
||||||
return templateToChallenge(template, targetTheme, targetDifficulty)
|
return templateToChallenge(template, targetTheme, targetDifficulty, forHuman)
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Ranked challenge: no multiple choice, only harder creative/open-ended prompts */
|
/** Ranked challenge: no multiple choice, only harder creative/open-ended prompts */
|
||||||
@@ -262,7 +262,7 @@ function generateCreativeChoices(type: string): { choices: string[]; answer: str
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function templateToChallenge(template: ChallengeTemplate, targetTheme?: PromptTheme, targetDifficulty?: PromptDifficulty): Challenge {
|
function templateToChallenge(template: ChallengeTemplate, targetTheme?: PromptTheme, targetDifficulty?: PromptDifficulty, forHuman = false): Challenge {
|
||||||
// Prefer prompts matching target theme if any are tagged
|
// Prefer prompts matching target theme if any are tagged
|
||||||
let prompts = template.prompts
|
let prompts = template.prompts
|
||||||
if (targetTheme) {
|
if (targetTheme) {
|
||||||
@@ -276,19 +276,19 @@ function templateToChallenge(template: ChallengeTemplate, targetTheme?: PromptTh
|
|||||||
}
|
}
|
||||||
const entry = pick(prompts)
|
const entry = pick(prompts)
|
||||||
|
|
||||||
// Determine choices
|
// Determine choices — only for human fights (bots answer via webhook)
|
||||||
let choices: string[] | undefined
|
let choices: string[] | undefined
|
||||||
let answers = entry.answers
|
let answers = entry.answers
|
||||||
if (entry.choices) {
|
if (forHuman) {
|
||||||
choices = shuffleArray(entry.choices)
|
if (entry.choices) {
|
||||||
} else if (entry.answers?.length === 1 && ['true', 'false'].includes(entry.answers[0].toLowerCase())) {
|
choices = shuffleArray(entry.choices)
|
||||||
// Auto-generate True/False choices for boolean questions
|
} else if (entry.answers?.length === 1 && ['true', 'false'].includes(entry.answers[0].toLowerCase())) {
|
||||||
choices = shuffleArray(['True', 'False'])
|
choices = shuffleArray(['True', 'False'])
|
||||||
} else if (template.scoring === 'creative' && CREATIVE_CHOICES[template.type]) {
|
} else if (template.scoring === 'creative' && CREATIVE_CHOICES[template.type]) {
|
||||||
// Auto-generate multiple choice for creative challenges
|
const generated = generateCreativeChoices(template.type)
|
||||||
const generated = generateCreativeChoices(template.type)
|
choices = generated.choices
|
||||||
choices = generated.choices
|
answers = [generated.answer]
|
||||||
answers = [generated.answer]
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// For creative challenges with choices, add a "Pick the best response:" prefix
|
// For creative challenges with choices, add a "Pick the best response:" prefix
|
||||||
|
|||||||
Reference in New Issue
Block a user