feat: sync TTS voice with chat bubbles and battle log for zero-gap playback
Adds await-then-play pattern: audio is pre-generated and cached before visuals appear, so log text + speech bubble + mouth animation + voice all fire in the same frame. Prefetches both answers during question playback for instant transitions. Adds hideSpeechBubble() to dismiss bubbles when voice ends instead of fixed 5s timer. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
e7d1f9b97b
commit
cb8a1d50fe
@@ -10,6 +10,8 @@ import {
|
|||||||
setMasterMute, isMasterMuted, ensureAudioContext,
|
setMasterMute, isMasterMuted, ensureAudioContext,
|
||||||
speakQuestion, speakAnswer, speakNarration,
|
speakQuestion, speakAnswer, speakNarration,
|
||||||
prefetchQuestion, prefetchAnswer, prefetchNarration,
|
prefetchQuestion, prefetchAnswer, prefetchNarration,
|
||||||
|
awaitQuestionReady, awaitAnswerReady,
|
||||||
|
playQuestionNow, playAnswerNow, playNarrationNow,
|
||||||
sfxRandomComedy, sfxRandomFail, sfxVineBoom, sfxEmotionalDamage,
|
sfxRandomComedy, sfxRandomFail, sfxVineBoom, sfxEmotionalDamage,
|
||||||
} from '../game/audio'
|
} from '../game/audio'
|
||||||
import { isKokoroLoading, getKokoroProgress } from '../game/tts'
|
import { isKokoroLoading, getKokoroProgress } from '../game/tts'
|
||||||
@@ -400,60 +402,61 @@ async function _doReplay() {
|
|||||||
scene.startShowboating('b')
|
scene.startShowboating('b')
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1. Show question in log AND speak it (audio should be pre-generated by now)
|
// 1. Ensure question audio is ready BEFORE showing anything (zero-gap)
|
||||||
|
if (doTTS && questionText) {
|
||||||
|
// Kick off answer prefetches early so they generate in parallel
|
||||||
|
if (round.botAResponse) prefetchAnswer(props.fight.botA!.name, round.botAResponse)
|
||||||
|
if (round.botBResponse) prefetchAnswer(props.fight.botB!.name, round.botBResponse)
|
||||||
|
await awaitQuestionReady(questionText)
|
||||||
|
}
|
||||||
|
// Show question text + play voice at the exact same moment
|
||||||
logItems.value.push(
|
logItems.value.push(
|
||||||
{ type: 'header', round: round.roundNumber, text: `ROUND ${round.roundNumber}: ${challengeLabel(round.challengeType)}`, color: 'neon-purple' },
|
{ type: 'header', round: round.roundNumber, text: `ROUND ${round.roundNumber}: ${challengeLabel(round.challengeType)}`, color: 'neon-purple' },
|
||||||
)
|
)
|
||||||
scrollLog(); await sleep(insanityMode.value ? 30 : 100)
|
|
||||||
logItems.value.push(
|
logItems.value.push(
|
||||||
{ type: 'prompt', round: round.roundNumber, text: questionText, color: 'text-muted' },
|
{ type: 'prompt', round: round.roundNumber, text: questionText, color: 'text-muted' },
|
||||||
)
|
)
|
||||||
scrollLog()
|
scrollLog()
|
||||||
if (doTTS && questionText) {
|
if (doTTS && questionText) {
|
||||||
// Prefetch bot A answer while question plays
|
await playQuestionNow(questionText)
|
||||||
if (round.botAResponse) prefetchAnswer(props.fight.botA!.name, round.botAResponse)
|
|
||||||
await speakQuestion(questionText)
|
|
||||||
await sleep(150)
|
|
||||||
} else {
|
} else {
|
||||||
await sleep(insanityMode.value ? 50 : 600)
|
await sleep(insanityMode.value ? 50 : 300)
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. Bot A: log + bubble + mouth + TTS (all synced)
|
// 2. Bot A: ensure audio ready, then show log + bubble + mouth + voice all at once
|
||||||
// Stop A's showboat when it's their turn to talk, B keeps showboating
|
|
||||||
if (round.botAResponse) {
|
if (round.botAResponse) {
|
||||||
|
if (doTTS) await awaitAnswerReady(props.fight.botA!.name, round.botAResponse)
|
||||||
scene?.stopShowboating('a')
|
scene?.stopShowboating('a')
|
||||||
logItems.value.push({ type: 'responseA', round: round.roundNumber, text: `${props.fight.botA?.name}: ${round.botAResponse.slice(0, 120) || '[NO RESPONSE]'}`, color: 'neon-cyan' })
|
logItems.value.push({ type: 'responseA', round: round.roundNumber, text: `${props.fight.botA?.name}: ${round.botAResponse.slice(0, 120) || '[NO RESPONSE]'}`, color: 'neon-cyan' })
|
||||||
logItems.value.push({ type: 'time', round: round.roundNumber, text: ` ${round.botATimeMs}ms | Score: ${round.botAScore}`, color: 'text-muted' })
|
logItems.value.push({ type: 'time', round: round.roundNumber, text: ` ${round.botATimeMs}ms | Score: ${round.botAScore}`, color: 'text-muted' })
|
||||||
scrollLog()
|
scrollLog()
|
||||||
if (scene && !insanityMode.value) {
|
if (scene && !insanityMode.value) {
|
||||||
scene.showSpeechBubble('a', round.botAResponse.slice(0, 60), 5)
|
scene.showSpeechBubble('a', round.botAResponse.slice(0, 60), 8)
|
||||||
scene.startTalking('a')
|
scene.startTalking('a')
|
||||||
}
|
}
|
||||||
// Prefetch bot B answer while bot A talks
|
if (doTTS) await playAnswerNow(props.fight.botA!.name, round.botAResponse)
|
||||||
if (doTTS && round.botBResponse) prefetchAnswer(props.fight.botB!.name, round.botBResponse)
|
else await sleep(insanityMode.value ? 30 : 400)
|
||||||
if (doTTS) await speakAnswer(props.fight.botA!.name, round.botAResponse)
|
|
||||||
else await sleep(insanityMode.value ? 30 : 800)
|
|
||||||
scene?.stopTalking('a')
|
scene?.stopTalking('a')
|
||||||
if (!insanityMode.value) await sleep(150)
|
scene?.hideSpeechBubble('a')
|
||||||
}
|
}
|
||||||
|
|
||||||
// 3. Bot B: log + bubble + mouth + TTS
|
// 3. Bot B: ensure audio ready, then show log + bubble + mouth + voice all at once
|
||||||
// Stop B's showboat when it's their turn to talk
|
|
||||||
if (round.botBResponse) {
|
if (round.botBResponse) {
|
||||||
|
if (doTTS) await awaitAnswerReady(props.fight.botB!.name, round.botBResponse)
|
||||||
scene?.stopShowboating('b')
|
scene?.stopShowboating('b')
|
||||||
logItems.value.push({ type: 'responseB', round: round.roundNumber, text: `${props.fight.botB?.name}: ${round.botBResponse.slice(0, 120) || '[NO RESPONSE]'}`, color: 'neon-pink' })
|
logItems.value.push({ type: 'responseB', round: round.roundNumber, text: `${props.fight.botB?.name}: ${round.botBResponse.slice(0, 120) || '[NO RESPONSE]'}`, color: 'neon-pink' })
|
||||||
logItems.value.push({ type: 'time', round: round.roundNumber, text: ` ${round.botBTimeMs}ms | Score: ${round.botBScore}`, color: 'text-muted' })
|
logItems.value.push({ type: 'time', round: round.roundNumber, text: ` ${round.botBTimeMs}ms | Score: ${round.botBScore}`, color: 'text-muted' })
|
||||||
scrollLog()
|
scrollLog()
|
||||||
if (scene && !insanityMode.value) {
|
|
||||||
scene.showSpeechBubble('b', round.botBResponse.slice(0, 60), 5)
|
|
||||||
scene.startTalking('b')
|
|
||||||
}
|
|
||||||
// Prefetch narration while bot B talks
|
// Prefetch narration while bot B talks
|
||||||
if (doTTS && round.narration) prefetchNarration(round.narration)
|
if (doTTS && round.narration) prefetchNarration(round.narration)
|
||||||
if (doTTS) await speakAnswer(props.fight.botB!.name, round.botBResponse)
|
if (scene && !insanityMode.value) {
|
||||||
else await sleep(insanityMode.value ? 30 : 800)
|
scene.showSpeechBubble('b', round.botBResponse.slice(0, 60), 8)
|
||||||
|
scene.startTalking('b')
|
||||||
|
}
|
||||||
|
if (doTTS) await playAnswerNow(props.fight.botB!.name, round.botBResponse)
|
||||||
|
else await sleep(insanityMode.value ? 30 : 400)
|
||||||
scene?.stopTalking('b')
|
scene?.stopTalking('b')
|
||||||
if (!insanityMode.value) await sleep(150)
|
scene?.hideSpeechBubble('b')
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop all showboating before fight animation
|
// Stop all showboating before fight animation
|
||||||
@@ -815,14 +818,11 @@ async function _doReplay() {
|
|||||||
</svg>
|
</svg>
|
||||||
</button>
|
</button>
|
||||||
<button
|
<button
|
||||||
class="w-8 h-8 flex items-center justify-center border text-[9px] font-pixel tracking-wider transition-all"
|
class="h-8 px-2 flex items-center justify-center border text-[9px] font-pixel tracking-wider transition-all border-neon-green/70 text-neon-green bg-neon-green/10"
|
||||||
:class="perfMode
|
:title="perfMode ? 'Performance mode ON (fewer particles)' : 'Performance mode OFF (full effects)'"
|
||||||
? 'border-neon-green/70 text-neon-green bg-neon-green/10'
|
|
||||||
: 'border-border/50 text-text-muted hover:text-neon-green hover:border-neon-green/50'"
|
|
||||||
:title="perfMode ? 'Performance mode ON (fewer particles)' : 'Performance mode OFF'"
|
|
||||||
@click="togglePerfMode"
|
@click="togglePerfMode"
|
||||||
>
|
>
|
||||||
{{ perfMode ? 'LO' : 'HI' }}
|
{{ perfMode ? 'Lo GFX' : 'Hi GFX' }}
|
||||||
</button>
|
</button>
|
||||||
<button
|
<button
|
||||||
v-if="isReplaying"
|
v-if="isReplaying"
|
||||||
@@ -833,7 +833,7 @@ async function _doReplay() {
|
|||||||
title="INSANITY MODE: Skip TTS, max speed"
|
title="INSANITY MODE: Skip TTS, max speed"
|
||||||
@click="toggleInsanity"
|
@click="toggleInsanity"
|
||||||
>
|
>
|
||||||
{{ insanityMode ? 'INSANITY' : 'SKIP' }}
|
{{ insanityMode ? 'INSANITY' : 'REGULAR' }}
|
||||||
</button>
|
</button>
|
||||||
<!-- Zap winner button -->
|
<!-- Zap winner button -->
|
||||||
<button
|
<button
|
||||||
|
|||||||
@@ -734,6 +734,19 @@ export async function createFightScene(config: FightSceneConfig) {
|
|||||||
}, duration * 1000)
|
}, duration * 1000)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function hideSpeechBubble(side: 'a' | 'b') {
|
||||||
|
const els = side === 'a' ? activeBubbleA : activeBubbleB
|
||||||
|
if (!els.length) return
|
||||||
|
els.forEach(el => {
|
||||||
|
if (!el.exists()) return
|
||||||
|
k.tween(el.opacity, 0, 0.25, (v) => { el.opacity = v }).then(() => {
|
||||||
|
if (el.exists()) el.destroy()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
if (side === 'a') activeBubbleA = []
|
||||||
|
else activeBubbleB = []
|
||||||
|
}
|
||||||
|
|
||||||
function wrapBubbleText(text: string, maxChars: number): string[] {
|
function wrapBubbleText(text: string, maxChars: number): string[] {
|
||||||
const words = text.split(' ')
|
const words = text.split(' ')
|
||||||
const lines: string[] = []
|
const lines: string[] = []
|
||||||
@@ -1195,6 +1208,7 @@ export async function createFightScene(config: FightSceneConfig) {
|
|||||||
showSpeechBubble(side: 'a' | 'b', text: string, duration?: number) {
|
showSpeechBubble(side: 'a' | 'b', text: string, duration?: number) {
|
||||||
showSpeechBubble(side, text, duration)
|
showSpeechBubble(side, text, duration)
|
||||||
},
|
},
|
||||||
|
hideSpeechBubble(side: 'a' | 'b') { hideSpeechBubble(side) },
|
||||||
|
|
||||||
startTalking(side: 'a' | 'b') { startTalking(side) },
|
startTalking(side: 'a' | 'b') { startTalking(side) },
|
||||||
stopTalking(side: 'a' | 'b') { stopTalking(side) },
|
stopTalking(side: 'a' | 'b') { stopTalking(side) },
|
||||||
|
|||||||
@@ -205,6 +205,11 @@ export {
|
|||||||
prefetchQuestion,
|
prefetchQuestion,
|
||||||
prefetchAnswer,
|
prefetchAnswer,
|
||||||
prefetchNarration,
|
prefetchNarration,
|
||||||
|
awaitQuestionReady,
|
||||||
|
awaitAnswerReady,
|
||||||
|
playQuestionNow,
|
||||||
|
playAnswerNow,
|
||||||
|
playNarrationNow,
|
||||||
} from './voice'
|
} from './voice'
|
||||||
|
|
||||||
// music
|
// music
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
import { getCtx, getSfxDest, getMasterMuted, getSfxGain } from './context'
|
import { getCtx, getSfxDest, getMasterMuted, getSfxGain } from './context'
|
||||||
import { tone, noise } from './primitives'
|
import { tone, noise } from './primitives'
|
||||||
import { initKokoro, isKokoroReady, kokoroSpeak, kokoroSpeakAsync, kokoroStop, kokoroPrefetch, setAudioContext } from '../tts'
|
import { initKokoro, isKokoroReady, kokoroSpeak, kokoroSpeakAsync, kokoroStop, kokoroPrefetch, kokoroAwaitReady, kokoroPlayCached, setAudioContext } from '../tts'
|
||||||
|
|
||||||
// === VOICE PROFILES ===
|
// === VOICE PROFILES ===
|
||||||
|
|
||||||
@@ -756,6 +756,48 @@ export function prefetchNarration(text: string) {
|
|||||||
kokoroPrefetch(smartTruncate(text, 200), 'sportscaster')
|
kokoroPrefetch(smartTruncate(text, 200), 'sportscaster')
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// === AWAIT + PLAY-NOW pattern for tight sync ===
|
||||||
|
// Call awaitReady to block until audio is generated, then playNow fires instantly
|
||||||
|
|
||||||
|
export async function awaitQuestionReady(text: string): Promise<void> {
|
||||||
|
if (getMasterMuted()) return
|
||||||
|
const trimmed = smartTruncate(text, 200)
|
||||||
|
if (isKokoroReady()) await kokoroAwaitReady(trimmed, 'question_reader')
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function awaitAnswerReady(botName: string, answer: string): Promise<void> {
|
||||||
|
if (getMasterMuted()) return
|
||||||
|
const trimmed = smartTruncate(answer, 200)
|
||||||
|
if (isKokoroReady()) await kokoroAwaitReady(trimmed, botVoiceKey(botName))
|
||||||
|
}
|
||||||
|
|
||||||
|
export function playQuestionNow(text: string): Promise<void> {
|
||||||
|
const trimmed = smartTruncate(text, 200)
|
||||||
|
return _playNowCore(trimmed, 'question_reader')
|
||||||
|
}
|
||||||
|
|
||||||
|
export function playAnswerNow(botName: string, answer: string): Promise<void> {
|
||||||
|
const trimmed = smartTruncate(answer, 200)
|
||||||
|
return _playNowCore(trimmed, botVoiceKey(botName), 1.15)
|
||||||
|
}
|
||||||
|
|
||||||
|
export function playNarrationNow(text: string): Promise<void> {
|
||||||
|
const trimmed = smartTruncate(text, 200)
|
||||||
|
return _playNowCore(trimmed, 'sportscaster', 1.2)
|
||||||
|
}
|
||||||
|
|
||||||
|
async function _playNowCore(text: string, profileName: string, rateOverride?: number): Promise<void> {
|
||||||
|
if (getMasterMuted()) return
|
||||||
|
const sfxGain = getSfxGain()
|
||||||
|
if (isKokoroReady() && sfxGain) {
|
||||||
|
const profile = voiceProfiles[profileName] || voiceProfiles.announcer
|
||||||
|
await kokoroPlayCached(text, profileName, sfxGain, profile.volume * VOICE_VOLUME_SCALE)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Fallback to Web Speech
|
||||||
|
return _speakAsyncCore(text, profileName, rateOverride)
|
||||||
|
}
|
||||||
|
|
||||||
// Re-export TTS utilities needed by ensureAudioContext (in index.ts)
|
// Re-export TTS utilities needed by ensureAudioContext (in index.ts)
|
||||||
export { initKokoro, setAudioContext, loadVoices as _loadVoices }
|
export { initKokoro, setAudioContext, loadVoices as _loadVoices }
|
||||||
|
|
||||||
|
|||||||
@@ -376,6 +376,33 @@ export function kokoroPrefetch(text: string, profileName: string): void {
|
|||||||
_generateAndCache(text, profileName).catch(() => {})
|
_generateAndCache(text, profileName).catch(() => {})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Await until prefetched audio is cached and ready. Returns true if cached, false on failure. */
|
||||||
|
export async function kokoroAwaitReady(text: string, profileName: string): Promise<boolean> {
|
||||||
|
if (!_workerReady) return false
|
||||||
|
try {
|
||||||
|
const buf = await _generateAndCache(text, profileName)
|
||||||
|
return !!buf
|
||||||
|
} catch { return false }
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Play already-cached audio immediately. Returns playback promise. If not cached, returns resolved. */
|
||||||
|
export function kokoroPlayCached(
|
||||||
|
text: string,
|
||||||
|
profileName: string,
|
||||||
|
dest: AudioNode,
|
||||||
|
volume: number = 0.7,
|
||||||
|
): Promise<void> {
|
||||||
|
if (!_workerReady) return Promise.resolve()
|
||||||
|
const key = _cacheKey(text, profileName)
|
||||||
|
const cached = audioCache.get(key)
|
||||||
|
if (cached) {
|
||||||
|
cached.lastAccess = Date.now()
|
||||||
|
return _playBuffer(cached.buf, dest, volume)
|
||||||
|
}
|
||||||
|
// Not cached — fall through to generate+play
|
||||||
|
return kokoroSpeakAsync(text, profileName, dest, volume).then(() => {})
|
||||||
|
}
|
||||||
|
|
||||||
/** Stop all currently playing kokoro audio */
|
/** Stop all currently playing kokoro audio */
|
||||||
export function kokoroStop() {
|
export function kokoroStop() {
|
||||||
for (const src of activeSources) {
|
for (const src of activeSources) {
|
||||||
|
|||||||
Reference in New Issue
Block a user