fix: only generate MC choices for human fights, not bot-vs-bot
Bots answer via webhook and don't need multiple choice options. Add forHuman parameter to pickChallenge, update tests to cover both modes. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
dcd8570ed8
commit
592841d7b1
@@ -48,9 +48,9 @@ describe('pickChallenge', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('creative challenges have MC choices and a correct answer', () => {
|
||||
it('creative challenges have MC choices and a correct answer (human mode)', () => {
|
||||
for (let i = 0; i < 100; i++) {
|
||||
const c = pickChallenge(new Set(), null)
|
||||
const c = pickChallenge(new Set(), null, undefined, undefined, true)
|
||||
if (c.scoring === 'creative') {
|
||||
// Creative challenges now have auto-generated MC choices with a correct answer
|
||||
expect(c.choices).toBeDefined()
|
||||
@@ -63,10 +63,17 @@ describe('pickChallenge', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('factual challenges with choices have shuffled choices', () => {
|
||||
it('bot challenges have no MC choices', () => {
|
||||
for (let i = 0; i < 100; i++) {
|
||||
const c = pickChallenge(new Set(), null)
|
||||
expect(c.choices).toBeUndefined()
|
||||
}
|
||||
})
|
||||
|
||||
it('factual challenges with choices have shuffled choices (human mode)', () => {
|
||||
const orders = new Set<string>()
|
||||
for (let i = 0; i < 50; i++) {
|
||||
const c = pickChallenge(new Set(), null)
|
||||
const c = pickChallenge(new Set(), null, undefined, undefined, true)
|
||||
if (c.scoring === 'factual' && c.choices) {
|
||||
orders.add(c.choices.join(','))
|
||||
}
|
||||
@@ -90,11 +97,11 @@ describe('pickChallenge', () => {
|
||||
expect(factualPct).toBeLessThan(0.85)
|
||||
})
|
||||
|
||||
it('True/False auto-generation for boolean answers', () => {
|
||||
it('True/False auto-generation for boolean answers (human mode)', () => {
|
||||
// Pick many challenges, find ones with answers = ['true'] or ['false']
|
||||
let foundTFWithChoices = false
|
||||
for (let i = 0; i < 500; i++) {
|
||||
const c = pickChallenge(new Set(), null)
|
||||
const c = pickChallenge(new Set(), null, undefined, undefined, true)
|
||||
if (c.answers?.length === 1 && ['true', 'false'].includes(c.answers[0].toLowerCase())) {
|
||||
expect(c.choices).toBeTruthy()
|
||||
expect(c.choices!.length).toBe(2)
|
||||
|
||||
@@ -70,7 +70,7 @@ export function roundToDifficulty(round: number): PromptDifficulty {
|
||||
return 'hard'
|
||||
}
|
||||
|
||||
export function pickChallenge(usedTypes: Set<string>, _arenaModifier: string | null, themeBias?: PromptTheme, roundNumber?: number): Challenge {
|
||||
export function pickChallenge(usedTypes: Set<string>, _arenaModifier: string | null, themeBias?: PromptTheme, roundNumber?: number, forHuman = false): Challenge {
|
||||
let available = TEMPLATES.filter(t => !usedTypes.has(t.type))
|
||||
if (available.length === 0) available = TEMPLATES
|
||||
|
||||
@@ -88,7 +88,7 @@ export function pickChallenge(usedTypes: Set<string>, _arenaModifier: string | n
|
||||
const template = pick(pool)
|
||||
const targetTheme = themeBias || pickTheme()
|
||||
const targetDifficulty = roundNumber ? roundToDifficulty(roundNumber) : undefined
|
||||
return templateToChallenge(template, targetTheme, targetDifficulty)
|
||||
return templateToChallenge(template, targetTheme, targetDifficulty, forHuman)
|
||||
}
|
||||
|
||||
/** Ranked challenge: no multiple choice, only harder creative/open-ended prompts */
|
||||
@@ -262,7 +262,7 @@ function generateCreativeChoices(type: string): { choices: string[]; answer: str
|
||||
}
|
||||
}
|
||||
|
||||
function templateToChallenge(template: ChallengeTemplate, targetTheme?: PromptTheme, targetDifficulty?: PromptDifficulty): Challenge {
|
||||
function templateToChallenge(template: ChallengeTemplate, targetTheme?: PromptTheme, targetDifficulty?: PromptDifficulty, forHuman = false): Challenge {
|
||||
// Prefer prompts matching target theme if any are tagged
|
||||
let prompts = template.prompts
|
||||
if (targetTheme) {
|
||||
@@ -276,19 +276,19 @@ function templateToChallenge(template: ChallengeTemplate, targetTheme?: PromptTh
|
||||
}
|
||||
const entry = pick(prompts)
|
||||
|
||||
// Determine choices
|
||||
// Determine choices — only for human fights (bots answer via webhook)
|
||||
let choices: string[] | undefined
|
||||
let answers = entry.answers
|
||||
if (entry.choices) {
|
||||
choices = shuffleArray(entry.choices)
|
||||
} else if (entry.answers?.length === 1 && ['true', 'false'].includes(entry.answers[0].toLowerCase())) {
|
||||
// Auto-generate True/False choices for boolean questions
|
||||
choices = shuffleArray(['True', 'False'])
|
||||
} else if (template.scoring === 'creative' && CREATIVE_CHOICES[template.type]) {
|
||||
// Auto-generate multiple choice for creative challenges
|
||||
const generated = generateCreativeChoices(template.type)
|
||||
choices = generated.choices
|
||||
answers = [generated.answer]
|
||||
if (forHuman) {
|
||||
if (entry.choices) {
|
||||
choices = shuffleArray(entry.choices)
|
||||
} else if (entry.answers?.length === 1 && ['true', 'false'].includes(entry.answers[0].toLowerCase())) {
|
||||
choices = shuffleArray(['True', 'False'])
|
||||
} else if (template.scoring === 'creative' && CREATIVE_CHOICES[template.type]) {
|
||||
const generated = generateCreativeChoices(template.type)
|
||||
choices = generated.choices
|
||||
answers = [generated.answer]
|
||||
}
|
||||
}
|
||||
|
||||
// For creative challenges with choices, add a "Pick the best response:" prefix
|
||||
|
||||
Reference in New Issue
Block a user