feat(stage): group chat controls and track speech mute (#2139)

This commit is contained in:
RainbowBird
2026-07-28 19:34:14 +08:00
committed by GitHub
parent 899131b0a1
commit ae35b580d8
16 changed files with 299 additions and 107 deletions
@@ -44,7 +44,6 @@ const { activeCard, activeCardId } = storeToRefs(airiCardStore)
const { t } = useI18n()
const { openImagePreview } = journalPreviewStore
const isComposing = ref(false)
const sessionsDrawerOpen = defineModel<boolean>('sessionsDrawerOpen', { default: false })
const DOUBLE_ENTER_INTERVAL_MS = 300
const TRAILING_NEWLINES_REGEX = /[\r\n]+$/
const SEND_MODES = ['enter', 'ctrl-enter', 'double-enter'] as const
@@ -329,17 +328,6 @@ async function handleCleanupMessages() {
</div>
</div>
<div :class="['flex items-center justify-end gap-2 py-1']">
<button
:class="[
'max-h-[10lh] min-h-[1lh] flex items-center justify-center rounded-md p-2 outline-none',
'bg-neutral-100 text-lg text-neutral-500 transition-colors transition-transform active:scale-95',
'dark:bg-neutral-800 dark:text-neutral-400 hover:text-primary-500 dark:hover:text-primary-400',
]"
title="Conversations"
@click="sessionsDrawerOpen = true"
>
<div class="i-solar:chat-line-bold-duotone" />
</button>
<DropdownMenuRoot>
<DropdownMenuTrigger as-child>
<button
@@ -179,15 +179,6 @@ function resetMainWindowPosition() {
</template>
</ControlButtonTooltip>
<ControlButtonTooltip disable-hoverable-content>
<ControlButton :button-style="adjustStyleClasses.button" @click="openChat">
<div i-solar:chat-line-line-duotone :class="adjustStyleClasses.icon" text="neutral-800 dark:neutral-300" />
</ControlButton>
<template #tooltip>
{{ t('tamagotchi.stage.controls-island.open-chat') }}
</template>
</ControlButtonTooltip>
<ControlButtonTooltip disable-hoverable-content>
<ControlButton :button-style="adjustStyleClasses.button" @click="refreshWindow">
<div i-solar:refresh-linear :class="adjustStyleClasses.icon" text="neutral-800 dark:neutral-300" />
@@ -257,6 +248,15 @@ function resetMainWindowPosition() {
</template>
</ControlButtonTooltip>
<ControlButtonTooltip side="left">
<ControlButton :button-style="adjustStyleClasses.button" @click="openChat">
<div i-solar:chat-line-line-duotone :class="adjustStyleClasses.icon" text="neutral-800 dark:neutral-300" />
</ControlButton>
<template #tooltip>
{{ t('tamagotchi.stage.controls-island.open-chat') }}
</template>
</ControlButtonTooltip>
<ControlButtonTooltip side="left">
<ControlsIslandHearingConfig :show="blockingOverlays.has('hearing')" @update:show="setOverlay('hearing', $event)">
<div class="relative">
@@ -1,6 +1,8 @@
<script setup lang="ts">
import { defineInvoke } from '@moeru/eventa'
import { useStopSpeakingButton } from '@proj-airi/stage-layouts/composables/useStopSpeakingButton'
import { ChatSessionsDrawer } from '@proj-airi/stage-ui/components'
import { getSpeechBusContext, speechOutputGetPlaybackState } from '@proj-airi/stage-ui/services/speech/bus'
import { shallowRef } from 'vue'
import { useI18n } from 'vue-i18n'
@@ -8,7 +10,17 @@ import InteractiveArea from '../components/InteractiveArea.vue'
import WindowTitleBar from '../components/Window/TitleBar.vue'
const sessionsDrawerOpen = shallowRef(false)
const { speechMuted, toggleSpeechMuted } = useStopSpeakingButton()
const getOutputPlaybackState = defineInvoke(getSpeechBusContext(), speechOutputGetPlaybackState)
const { speechMuted, toggleSpeechMuted } = useStopSpeakingButton({
resolveSpeakingState: async () => {
// A BroadcastChannel round trip is normally immediate. Bound the
// analytics-only lookup so a reloading output renderer cannot stall mute.
const state = await getOutputPlaybackState(undefined, {
signal: AbortSignal.timeout(1000),
})
return state.speaking
},
})
const { t } = useI18n()
</script>
@@ -20,6 +32,19 @@ const { t } = useI18n()
@title-click="sessionsDrawerOpen = true"
>
<template #actions>
<button
data-testid="conversation-selector-button"
:class="[
'h-7 w-7 flex items-center justify-center rounded-md outline-none',
'text-base text-neutral-400 transition-colors transition-transform active:scale-95',
'hover:bg-neutral-200 hover:text-primary-500 dark:text-neutral-500 dark:hover:bg-neutral-800 dark:hover:text-primary-400',
]"
:title="t('stage.chat.sessions.title')"
:aria-label="t('stage.chat.sessions.title')"
@click="sessionsDrawerOpen = true"
>
<div class="i-solar:chat-line-bold-duotone" />
</button>
<button
data-testid="speech-mute-button"
:class="[
@@ -40,7 +65,6 @@ const { t } = useI18n()
</template>
</WindowTitleBar>
<InteractiveArea
v-model:sessions-drawer-open="sessionsDrawerOpen"
class="interaction-area block"
h-full w-full p-4 transition="opacity duration-250"
/>
@@ -236,6 +236,34 @@ describe('createChatOrchestratorRuntime', () => {
expect(harness.promptProjections).toHaveLength(1)
})
// ROOT CAUSE:
//
// Speech-muted consumers dispatch plugin CALL markers without a TTS
// session. If the hook context has no turn id, a locally unhandled call
// cannot be correlated and relayed to another Electron renderer.
it('preserves the round turn id on special-token hooks', async () => {
const harness = createHarness()
let specialTurnId = ''
harness.runtime.hooks.onTokenSpecial(async (_special, context) => {
specialTurnId = context.turnId
})
harness.stream.mockImplementationOnce(async (_model, _chatProvider, _messages, options) => {
await options?.onStreamEvent?.({ type: 'text-delta', text: '<|CALL ["plugin.action"]|>' })
await options?.onStreamEvent?.({ type: 'finish', finishReason: 'stop' })
})
await harness.runtime.ingest('trigger special', {
model: 'gpt-test',
chatProvider: provider,
})
expect(specialTurnId).toBe('user-id')
expect(harness.telemetry.messageSendStarted).toEqual([
expect.objectContaining({ roundId: specialTurnId }),
])
})
it('keeps timestamp prefixes stable for legacy user messages without createdAt', async () => {
const harness = createHarness()
const legacyUserMessage: ChatHistoryItem = {
@@ -442,8 +442,14 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps)
const sendingCreatedAt = now()
// TODO: Expire or prune stale runtime contexts from disconnected services before composing.
// Allocate the three per-round ids in their historical order so callers
// with deterministic id factories keep the same durable message ids.
const streamContextMessageId = createId()
const assistantMessageId = createId()
const roundId = createId()
const streamingMessageContext: ChatStreamEventContext = {
message: { role: 'user', content: sendingMessage, createdAt: sendingCreatedAt, id: createId() },
turnId: roundId,
message: { role: 'user', content: sendingMessage, createdAt: sendingCreatedAt, id: streamContextMessageId },
contexts: deps.context.snapshot(),
composedMessage: [],
input: options.input,
@@ -471,14 +477,13 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps)
slices: [],
tool_results: [],
createdAt: now(),
id: createId(),
id: assistantMessageId,
}
patchForegroundStream(sessionId, buildingMessage)
const sendSource = options.input ? 'voice' : 'text'
const activeProvider = deps.getActiveProvider?.() ?? ''
// The user message is the durable start of a round, so its ID also serves
// as the correlation key for every telemetry milestone emitted by it.
const roundId = createId()
const correlation: ChatRoundCorrelation = {
conversationId: sessionId,
roundId,
+2
View File
@@ -50,6 +50,8 @@ export interface ContextMessage extends ContextUpdate<Record<string, unknown>, u
export type ChatHistoryItem = (ChatMessage | ErrorMessage) & { context?: ContextMessage } & { createdAt?: number, id?: string }
export interface ChatStreamEventContext {
/** Stable correlation id shared by every hook emitted for one user turn. */
turnId: string
message: ChatHistoryItem
contexts: Record<string, ContextMessage[]>
composedMessage: Array<Message>
@@ -199,15 +199,36 @@ onMounted(() => {
<div translate-y="[-100%]" absolute right-0 px-3 pb-3 font-sans>
<div flex="~ col" gap-1>
<ActionAbout />
<button
border="2 solid neutral-100/60 dark:neutral-800/30"
bg="neutral-50/70 dark:neutral-800/70"
w-fit flex items-center self-end justify-center rounded-xl p-2 backdrop-blur-md
title="Conversations"
@click="sessionsDrawerOpen = true"
>
<div i-solar:chat-line-bold-duotone size-5 text="neutral-500 dark:neutral-400" />
</button>
<div flex="~ col" items-end gap-1>
<button
data-testid="conversation-selector-button"
border="2 solid neutral-100/60 dark:neutral-800/30"
bg="neutral-50/70 dark:neutral-800/70"
w-fit flex items-center self-end justify-center rounded-xl p-2 backdrop-blur-md
:title="t('stage.chat.sessions.title')"
:aria-label="t('stage.chat.sessions.title')"
@click="sessionsDrawerOpen = true"
>
<div i-solar:chat-line-bold-duotone size-5 text="neutral-500 dark:neutral-400" />
</button>
<button
data-testid="speech-mute-button"
:class="[
'w-fit flex items-center self-end justify-center rounded-xl border-2 border-solid p-2 backdrop-blur-md',
'border-neutral-100/60 text-neutral-500 transition-colors active:scale-95 dark:border-neutral-800/30 dark:text-neutral-400',
speechMuted
? 'bg-primary-100/80 text-primary-600 dark:bg-primary-900/60 dark:text-primary-300'
: 'bg-neutral-50/70 hover:text-primary-500 dark:bg-neutral-800/70 dark:hover:text-primary-400',
]"
:title="speechMuted ? t('stage.speech-output.unmute') : t('stage.speech-output.mute')"
:aria-label="speechMuted ? t('stage.speech-output.unmute') : t('stage.speech-output.mute')"
:aria-pressed="speechMuted"
@click="toggleSpeechMuted"
>
<div v-if="speechMuted" class="i-solar:volume-cross-bold-duotone size-5" />
<div v-else class="i-solar:volume-loud-bold-duotone size-5" />
</button>
</div>
<ChatSessionsDrawer v-model="sessionsDrawerOpen" />
<HearingConfigDialog
v-model:enabled="enabled"
@@ -227,23 +248,6 @@ onMounted(() => {
</Transition>
</button>
</HearingConfigDialog>
<button
data-testid="speech-mute-button"
:class="[
'w-fit flex items-center self-end justify-center rounded-xl border-2 border-solid p-2 backdrop-blur-md',
'border-neutral-100/60 text-neutral-500 transition-colors active:scale-95 dark:border-neutral-800/30 dark:text-neutral-400',
speechMuted
? 'bg-primary-100/80 text-primary-600 dark:bg-primary-900/60 dark:text-primary-300'
: 'bg-neutral-50/70 hover:text-primary-500 dark:bg-neutral-800/70 dark:hover:text-primary-400',
]"
:title="speechMuted ? t('stage.speech-output.unmute') : t('stage.speech-output.mute')"
:aria-label="speechMuted ? t('stage.speech-output.unmute') : t('stage.speech-output.mute')"
:aria-pressed="speechMuted"
@click="toggleSpeechMuted"
>
<div v-if="speechMuted" class="i-solar:volume-cross-bold-duotone size-5" />
<div v-else class="i-solar:volume-loud-bold-duotone size-5" />
</button>
<button border="2 solid neutral-100/60 dark:neutral-800/30" bg="neutral-50/70 dark:neutral-800/70" w-fit flex items-center self-end justify-center rounded-xl p-2 backdrop-blur-md title="Theme" @click="toggleDark()">
<Transition name="fade" mode="out-in">
<div v-if="isDark" i-solar:moon-outline size-5 text="neutral-500 dark:neutral-400" />
@@ -1,4 +1,5 @@
<script setup lang="ts">
import { ChatSessionsDrawer } from '@proj-airi/stage-ui/components/scenarios/chat'
import { useAnalytics } from '@proj-airi/stage-ui/composables/use-analytics'
import { useChatMaintenanceStore } from '@proj-airi/stage-ui/stores/chat/maintenance'
import { useChatSessionStore } from '@proj-airi/stage-ui/stores/chat/session-store'
@@ -20,6 +21,7 @@ const { speechMuted, toggleSpeechMuted } = useStopSpeakingButton()
const { t } = useI18n()
const backgroundDialogOpen = ref(false)
const sessionsDrawerOpen = ref(false)
function handleCleanupMessages() {
const messageCount = messages.value.filter(message => message.role !== 'system').length
@@ -33,24 +35,40 @@ function handleCleanupMessages() {
<template>
<BackgroundDialogPicker v-model="backgroundDialogOpen" />
<ChatSessionsDrawer v-model="sessionsDrawerOpen" />
<div absolute bottom--8 right-0 flex gap-2>
<button
data-testid="speech-mute-button"
:class="[
'max-h-[10lh] min-h-[1lh] flex items-center justify-center rounded-md p-2 outline-none',
'text-lg transition-colors transition-transform active:scale-95',
speechMuted
? 'bg-primary-100 text-primary-600 dark:bg-primary-900/40 dark:text-primary-300'
: 'bg-neutral-100 text-neutral-500 hover:text-primary-500 dark:bg-neutral-800 dark:text-neutral-400 dark:hover:text-primary-400',
]"
:title="speechMuted ? t('stage.speech-output.unmute') : t('stage.speech-output.mute')"
:aria-label="speechMuted ? t('stage.speech-output.unmute') : t('stage.speech-output.mute')"
:aria-pressed="speechMuted"
@click="toggleSpeechMuted"
>
<div v-if="speechMuted" class="i-solar:volume-cross-bold-duotone" />
<div v-else class="i-solar:volume-loud-bold-duotone" />
</button>
<div flex gap-1>
<button
data-testid="conversation-selector-button"
:class="[
'max-h-[10lh] min-h-[1lh] flex items-center justify-center rounded-md p-2 outline-none',
'bg-neutral-100 text-lg text-neutral-500 transition-colors transition-transform active:scale-95',
'hover:text-primary-500 dark:bg-neutral-800 dark:text-neutral-400 dark:hover:text-primary-400',
]"
:title="t('stage.chat.sessions.title')"
:aria-label="t('stage.chat.sessions.title')"
@click="sessionsDrawerOpen = true"
>
<div class="i-solar:chat-line-bold-duotone" />
</button>
<button
data-testid="speech-mute-button"
:class="[
'max-h-[10lh] min-h-[1lh] flex items-center justify-center rounded-md p-2 outline-none',
'text-lg transition-colors transition-transform active:scale-95',
speechMuted
? 'bg-primary-100 text-primary-600 dark:bg-primary-900/40 dark:text-primary-300'
: 'bg-neutral-100 text-neutral-500 hover:text-primary-500 dark:bg-neutral-800 dark:text-neutral-400 dark:hover:text-primary-400',
]"
:title="speechMuted ? t('stage.speech-output.unmute') : t('stage.speech-output.mute')"
:aria-label="speechMuted ? t('stage.speech-output.unmute') : t('stage.speech-output.mute')"
:aria-pressed="speechMuted"
@click="toggleSpeechMuted"
>
<div v-if="speechMuted" class="i-solar:volume-cross-bold-duotone" />
<div v-else class="i-solar:volume-loud-bold-duotone" />
</button>
</div>
<ViewControls />
<button
class="max-h-[10lh] min-h-[1lh]"
@@ -3,7 +3,6 @@ import type { ChatProvider } from '@xsai-ext/providers/utils'
import { errorMessageFrom } from '@moeru/std'
import { isStageTamagotchi } from '@proj-airi/stage-shared'
import { ChatSessionsDrawer } from '@proj-airi/stage-ui/components/scenarios/chat'
import { HearingConfig } from '@proj-airi/stage-ui/components/scenarios/dialogs/audio-input/index'
import { useAudioAnalyzer } from '@proj-airi/stage-ui/composables'
import { useAudioContext } from '@proj-airi/stage-ui/stores/audio'
@@ -26,7 +25,6 @@ import { useStopSpeakingButton } from '../../composables/useStopSpeakingButton'
const messageInput = ref<string>('')
const hearingPopoverOpen = ref(false)
const sessionsDrawerOpen = ref(false)
const isComposing = ref(false)
const DOUBLE_ENTER_INTERVAL_MS = 300
const TRAILING_NEWLINES_REGEX = /[\r\n]+$/
@@ -205,24 +203,10 @@ watch(sendMode, () => {
@compositionend="isComposing = false"
/>
<!-- Bottom-left action button: Microphone -->
<!-- Input configuration controls -->
<div
absolute bottom-2 left-2 z-10 flex items-center gap-2
>
<!-- Conversations drawer trigger -->
<button
:class="[
'h-8 w-8 flex items-center justify-center rounded-md outline-none transition-all duration-200 active:scale-95',
'text-lg text-neutral-500 dark:text-neutral-400',
]"
title="Conversations"
@click="sessionsDrawerOpen = true"
>
<div class="i-solar:chat-line-bold-duotone h-5 w-5" />
</button>
<ChatSessionsDrawer v-model="sessionsDrawerOpen" />
<DropdownMenuRoot>
<DropdownMenuTrigger as-child>
<button
@@ -6,7 +6,8 @@ import { useStopSpeakingButton } from './useStopSpeakingButton'
const nowSpeaking = ref(false)
const speechMuted = ref(false)
const requestStopSpeakingMock = vi.fn()
const toggleSpeechMutedMock = vi.fn()
const setSpeechMutedMock = vi.fn()
const trackSpeechMuteToggledMock = vi.fn()
const trackTtsStopClickedMock = vi.fn()
vi.mock('@proj-airi/stage-ui/stores/audio', () => ({
@@ -18,13 +19,14 @@ vi.mock('@proj-airi/stage-ui/stores/audio', () => ({
vi.mock('@proj-airi/stage-ui/stores/speech-output-control', () => ({
useSpeechOutputControlStore: () => ({
requestStopSpeaking: requestStopSpeakingMock,
setSpeechMuted: setSpeechMutedMock,
speechMuted,
toggleSpeechMuted: toggleSpeechMutedMock,
}),
}))
vi.mock('@proj-airi/stage-ui/composables/use-analytics', () => ({
useAnalytics: () => ({
trackSpeechMuteToggled: trackSpeechMuteToggledMock,
trackTtsStopClicked: trackTtsStopClickedMock,
}),
}))
@@ -74,16 +76,72 @@ describe('useStopSpeakingButton', () => {
})
})
it('exposes persisted mute state and toggles it through the shared output store', () => {
speechMuted.value = true
toggleSpeechMutedMock.mockClear()
it('tracks mute and unmute with the active playback state', async () => {
speechMuted.value = false
nowSpeaking.value = true
setSpeechMutedMock.mockClear()
trackSpeechMuteToggledMock.mockClear()
const controls = useStopSpeakingButton()
expect(controls.speechMuted.value).toBe(true)
await controls.toggleSpeechMuted()
controls.toggleSpeechMuted()
expect(setSpeechMutedMock).toHaveBeenCalledWith(true)
expect(trackSpeechMuteToggledMock).toHaveBeenCalledWith({
muted: true,
was_speaking: true,
})
expect(toggleSpeechMutedMock).toHaveBeenCalledOnce()
speechMuted.value = true
nowSpeaking.value = false
await controls.toggleSpeechMuted()
expect(setSpeechMutedMock).toHaveBeenLastCalledWith(false)
expect(trackSpeechMuteToggledMock).toHaveBeenLastCalledWith({
muted: false,
was_speaking: false,
})
})
// ROOT CAUSE:
//
// Electron's auxiliary /chat renderer has its own Pinia instance, so its
// local nowSpeaking value stays false while the main Stage renderer speaks.
//
// The title-bar mute control now resolves state from the output host before
// capturing speech_mute_toggled.
it('tracks the speaking state resolved from a remote output host', async () => {
speechMuted.value = false
nowSpeaking.value = false
setSpeechMutedMock.mockClear()
trackSpeechMuteToggledMock.mockClear()
const resolveSpeakingState = vi.fn().mockResolvedValue(true)
const controls = useStopSpeakingButton({ resolveSpeakingState })
await controls.toggleSpeechMuted()
expect(resolveSpeakingState).toHaveBeenCalledTimes(1)
expect(setSpeechMutedMock).toHaveBeenCalledWith(true)
expect(trackSpeechMuteToggledMock).toHaveBeenCalledWith({
muted: true,
was_speaking: true,
})
})
it('still toggles mute without capturing a false metric when the output host is unavailable', async () => {
speechMuted.value = false
setSpeechMutedMock.mockClear()
trackSpeechMuteToggledMock.mockClear()
const controls = useStopSpeakingButton({
resolveSpeakingState: () => Promise.reject(new Error('output host reloading')),
})
await controls.toggleSpeechMuted()
expect(setSpeechMutedMock).toHaveBeenCalledWith(true)
expect(trackSpeechMuteToggledMock).not.toHaveBeenCalled()
})
})
@@ -10,11 +10,18 @@ import { computed } from 'vue'
* Manual stops affect current playback without cancelling text generation.
* Mute is persisted by the shared store and also blocks future TTS sessions.
*/
export function useStopSpeakingButton() {
export function useStopSpeakingButton(options: {
/**
* Reads speaking state from the renderer that owns playback.
*
* @default The current renderer's speaking store.
*/
resolveSpeakingState?: () => boolean | Promise<boolean>
} = {}) {
const { nowSpeaking } = storeToRefs(useSpeakingStore())
const speechOutputControlStore = useSpeechOutputControlStore()
const { speechMuted } = storeToRefs(speechOutputControlStore)
const { trackTtsStopClicked } = useAnalytics()
const { trackSpeechMuteToggled, trackTtsStopClicked } = useAnalytics()
const showStopSpeakingButton = computed(() => nowSpeaking.value)
@@ -28,11 +35,32 @@ export function useStopSpeakingButton() {
speechOutputControlStore.requestStopSpeaking('manual-all')
}
async function toggleSpeechMuted() {
let wasSpeaking: boolean
try {
wasSpeaking = await (options.resolveSpeakingState?.() ?? nowSpeaking.value)
}
catch {
// Muting is the user action; analytics must not make it fail when an
// auxiliary renderer cannot reach the output host during a reload.
speechOutputControlStore.setSpeechMuted(!speechMuted.value)
return
}
const muted = !speechMuted.value
speechOutputControlStore.setSpeechMuted(muted)
trackSpeechMuteToggled({
muted,
was_speaking: wasSpeaking,
})
}
return {
showStopSpeakingButton,
speechMuted,
stopSpeakingFromChat,
stopAllSpeaking,
toggleSpeechMuted: speechOutputControlStore.toggleSpeechMuted,
toggleSpeechMuted,
}
}
@@ -8,6 +8,7 @@ import type { UnElevenLabsOptions } from 'unspeech'
import type { EmotionPayload } from '../../constants/emotions'
import type { SpeechTransport, StageTtsSession, StreamingSessionSnapshot } from '../../libs/speech/tts-session'
import { defineInvokeHandler } from '@moeru/eventa'
import { sleep } from '@moeru/std'
import { createLive2DLipSync } from '@proj-airi/model-driver-lipsync'
import { wlipsyncProfile } from '@proj-airi/model-driver-lipsync/shared/wlipsync'
@@ -38,6 +39,7 @@ import { getDefaultStreamingModel, getDefinedProvider } from '../../libs/provide
import { OFFICIAL_SPEECH_PROVIDER_ID, OFFICIAL_SPEECH_STREAMING_PROVIDER_ID } from '../../libs/providers/providers/official'
import { bindSpeakingStateToPlaybackManager } from '../../libs/speech/playback-speaking-state'
import { createStageTtsSession } from '../../libs/speech/tts-session'
import { getSpeechBusContext, speechOutputGetPlaybackState } from '../../services/speech/bus'
import { useAudioContext, useSpeakingStore } from '../../stores/audio'
import { useBackgroundStore } from '../../stores/background'
import { useChatOrchestratorStore } from '../../stores/chat'
@@ -91,6 +93,11 @@ const {
spineRenderScale,
} = storeToRefs(settingsStore)
const { mouthOpenSize, nowSpeaking } = storeToRefs(useSpeakingStore())
const disposePlaybackStateHandler = defineInvokeHandler(
getSpeechBusContext(),
speechOutputGetPlaybackState,
() => ({ speaking: nowSpeaking.value }),
)
const { audioContext } = useAudioContext()
const currentAudioSource = ref<AudioBufferSourceNode>()
const speechOutputControlStore = useSpeechOutputControlStore()
@@ -696,7 +703,7 @@ function resolveStreamingSessionModel(): string | null {
return sessionModel
}
function buildStreamingSnapshot(): StreamingSessionSnapshot | null {
function buildStreamingSnapshot(turnId: string): StreamingSessionSnapshot | null {
if (speechMuted.value)
return null
@@ -733,7 +740,7 @@ function buildStreamingSnapshot(): StreamingSessionSnapshot | null {
audio: { sample_rate: 24000, bit_rate: 64000 },
},
ownerId: activeCardId.value,
onImmediateSpecial: playSpecialToken,
onImmediateSpecial: special => playSpecialToken(special, { turnId }),
}
}
@@ -747,7 +754,7 @@ function resolveSpeechTransport(providerId: string | null | undefined): SpeechTr
return getDefinedProvider(providerId)?.capabilities?.speech?.transport
}
function openTtsSession(): StageTtsSession {
function openTtsSession(turnId: string): StageTtsSession {
// A session must only clear the module-level `currentSession` if it IS that session. The previous
// code cleared it whenever any `stream-` session completed, which is unsafe once sessions exist that
// are not assigned to `currentSession` (e.g. one-off read-aloud sessions): one of those finishing
@@ -760,11 +767,12 @@ function openTtsSession(): StageTtsSession {
}
session = createStageTtsSession<AudioBuffer>({
transport: resolveSpeechTransport(activeSpeechProvider.value),
streaming: buildStreamingSnapshot,
streaming: () => buildStreamingSnapshot(turnId),
audioContext,
playbackManager,
openIntent: opts => speechRuntimeStore.openIntent(opts),
intentOptions: () => ({
turnId,
ownerId: activeCardId.value,
priority: 'normal',
behavior: 'queue',
@@ -802,7 +810,7 @@ watch(speechMuted, (muted) => {
stopSpeechOutput('muted')
}, { immediate: true })
chatHookCleanups.push(onBeforeMessageComposed(async () => {
chatHookCleanups.push(onBeforeMessageComposed(async (_message, context) => {
officialAutoTtsTrackedForTurn = false
playbackManager.stopAll('new-message')
resetAssistantSpeechSurface('new-message')
@@ -815,7 +823,7 @@ chatHookCleanups.push(onBeforeMessageComposed(async () => {
setupAnalyser()
await setupLipSync()
currentSession = openTtsSession()
currentSession = openTtsSession(context.turnId)
}))
chatHookCleanups.push(onBeforeSend(async () => {
@@ -826,11 +834,11 @@ chatHookCleanups.push(onTokenLiteral(async (literal) => {
currentSession?.appendText(literal)
}))
chatHookCleanups.push(onTokenSpecial(async (special) => {
chatHookCleanups.push(onTokenSpecial(async (special, context) => {
// Muting speech must not suppress non-audio signals such as emotion, motion,
// delay, or plugin calls that normally travel through the TTS session.
if (speechMuted.value) {
await playSpecialToken(special)
await playSpecialToken(special, { turnId: context.turnId })
return
}
@@ -996,6 +1004,7 @@ async function captureFrame() {
}
onUnmounted(() => {
disposePlaybackStateHandler()
resetLive2dLipSync()
chatHookCleanups.forEach(dispose => dispose?.())
viewUpdateCleanups.forEach(dispose => dispose?.())
@@ -83,6 +83,21 @@ describe('useAnalytics conversation product events', () => {
})
})
it('captures speech mute state changes without conversation or audio content', () => {
const analytics = useAnalytics()
analytics.trackSpeechMuteToggled({
muted: true,
was_speaking: true,
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenCalledWith('speech_mute_toggled', {
app_surface: 'web',
muted: true,
was_speaking: true,
})
})
it('captures custom-provider token usage without prompt or response content', () => {
const analytics = useAnalytics()
@@ -654,6 +654,18 @@ export function useAnalytics() {
})
}
function trackSpeechMuteToggled(properties: {
muted: boolean
was_speaking: boolean
}) {
if (!canCapture())
return
posthog.capture('speech_mute_toggled', {
...properties,
app_surface: getConversationAnalyticsSurface(),
})
}
function trackChatSessionSelected(properties: { source: 'sessions_drawer', message_count: number, cloud_synced: boolean }) {
if (!canCapture())
return
@@ -1305,6 +1317,7 @@ export function useAnalytics() {
trackProviderConfigCompleted,
trackOfficialProviderEnabled,
trackTtsStopClicked,
trackSpeechMuteToggled,
trackChatSessionSelected,
trackChatMessageDeleted,
trackChatMessagesCleared,
+10 -1
View File
@@ -1,4 +1,4 @@
import { defineEventa } from '@moeru/eventa'
import { defineEventa, defineInvokeEventa } from '@moeru/eventa'
import { createContext as createBroadcastChannelContext } from '@moeru/eventa/adapters/broadcast-channel'
export interface SpeechIntentStartPayload {
@@ -42,6 +42,15 @@ export const speechIntentFlushEvent = defineEventa<SpeechIntentTokenPayload>('ev
export const speechIntentEndEvent = defineEventa<SpeechIntentEndPayload>('eventa:audio:speech:intent:end')
export const speechIntentCancelEvent = defineEventa<SpeechIntentCancelPayload>('eventa:audio:speech:intent:cancel')
/** Snapshot served by the renderer that owns active speech playback. */
export interface SpeechOutputPlaybackState {
/** Whether the output host is currently playing assistant speech. */
speaking: boolean
}
/** Cross-renderer request for the active speech output host's playback state. */
export const speechOutputGetPlaybackState = defineInvokeEventa<SpeechOutputPlaybackState>('eventa:audio:speech:output:get-playback-state')
const BUS_CHANNEL_NAME = 'proj-airi:pipelines:outputs:speech'
let context: ReturnType<typeof createBroadcastChannelContext>['context'] | undefined
@@ -501,7 +501,12 @@ describe('chat orchestrator contract', () => {
expect(syntheticContextText).toContain('- system:weather: sunny')
})
it('emits special tokens for speech timeline handling during chat streaming', async () => {
// ROOT CAUSE:
//
// Muted speech dispatches special tokens without opening a TTS session.
// Without a turn id in the hook context, an unhandled plugin CALL cannot be
// correlated and relayed to a handler in another Electron renderer.
it('emits special tokens with a stable turn id for cross-renderer routing', async () => {
getContextsSnapshotMock.mockReturnValue({})
llmStreamMock.mockImplementationOnce(async (_model, _provider, _messages, options) => {
await options.onStreamEvent({ type: 'text-delta', text: '<|CALL ["plugin.action"]|>' })
@@ -518,7 +523,9 @@ describe('chat orchestrator contract', () => {
expect(specialHook).toHaveBeenCalledWith('<|CALL ["plugin.action"]|>', expect.objectContaining({
contexts: {},
turnId: expect.any(String),
}))
expect(specialHook.mock.calls[0]?.[1].turnId.length).toBeGreaterThan(0)
})
/**