mirror of
https://github.com/moeru-ai/airi.git
synced 2026-08-14 08:52:42 +00:00
feat(stage): group chat controls and track speech mute (#2139)
This commit is contained in:
@@ -44,7 +44,6 @@ const { activeCard, activeCardId } = storeToRefs(airiCardStore)
|
||||
const { t } = useI18n()
|
||||
const { openImagePreview } = journalPreviewStore
|
||||
const isComposing = ref(false)
|
||||
const sessionsDrawerOpen = defineModel<boolean>('sessionsDrawerOpen', { default: false })
|
||||
const DOUBLE_ENTER_INTERVAL_MS = 300
|
||||
const TRAILING_NEWLINES_REGEX = /[\r\n]+$/
|
||||
const SEND_MODES = ['enter', 'ctrl-enter', 'double-enter'] as const
|
||||
@@ -329,17 +328,6 @@ async function handleCleanupMessages() {
|
||||
</div>
|
||||
</div>
|
||||
<div :class="['flex items-center justify-end gap-2 py-1']">
|
||||
<button
|
||||
:class="[
|
||||
'max-h-[10lh] min-h-[1lh] flex items-center justify-center rounded-md p-2 outline-none',
|
||||
'bg-neutral-100 text-lg text-neutral-500 transition-colors transition-transform active:scale-95',
|
||||
'dark:bg-neutral-800 dark:text-neutral-400 hover:text-primary-500 dark:hover:text-primary-400',
|
||||
]"
|
||||
title="Conversations"
|
||||
@click="sessionsDrawerOpen = true"
|
||||
>
|
||||
<div class="i-solar:chat-line-bold-duotone" />
|
||||
</button>
|
||||
<DropdownMenuRoot>
|
||||
<DropdownMenuTrigger as-child>
|
||||
<button
|
||||
|
||||
+9
-9
@@ -179,15 +179,6 @@ function resetMainWindowPosition() {
|
||||
</template>
|
||||
</ControlButtonTooltip>
|
||||
|
||||
<ControlButtonTooltip disable-hoverable-content>
|
||||
<ControlButton :button-style="adjustStyleClasses.button" @click="openChat">
|
||||
<div i-solar:chat-line-line-duotone :class="adjustStyleClasses.icon" text="neutral-800 dark:neutral-300" />
|
||||
</ControlButton>
|
||||
<template #tooltip>
|
||||
{{ t('tamagotchi.stage.controls-island.open-chat') }}
|
||||
</template>
|
||||
</ControlButtonTooltip>
|
||||
|
||||
<ControlButtonTooltip disable-hoverable-content>
|
||||
<ControlButton :button-style="adjustStyleClasses.button" @click="refreshWindow">
|
||||
<div i-solar:refresh-linear :class="adjustStyleClasses.icon" text="neutral-800 dark:neutral-300" />
|
||||
@@ -257,6 +248,15 @@ function resetMainWindowPosition() {
|
||||
</template>
|
||||
</ControlButtonTooltip>
|
||||
|
||||
<ControlButtonTooltip side="left">
|
||||
<ControlButton :button-style="adjustStyleClasses.button" @click="openChat">
|
||||
<div i-solar:chat-line-line-duotone :class="adjustStyleClasses.icon" text="neutral-800 dark:neutral-300" />
|
||||
</ControlButton>
|
||||
<template #tooltip>
|
||||
{{ t('tamagotchi.stage.controls-island.open-chat') }}
|
||||
</template>
|
||||
</ControlButtonTooltip>
|
||||
|
||||
<ControlButtonTooltip side="left">
|
||||
<ControlsIslandHearingConfig :show="blockingOverlays.has('hearing')" @update:show="setOverlay('hearing', $event)">
|
||||
<div class="relative">
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
<script setup lang="ts">
|
||||
import { defineInvoke } from '@moeru/eventa'
|
||||
import { useStopSpeakingButton } from '@proj-airi/stage-layouts/composables/useStopSpeakingButton'
|
||||
import { ChatSessionsDrawer } from '@proj-airi/stage-ui/components'
|
||||
import { getSpeechBusContext, speechOutputGetPlaybackState } from '@proj-airi/stage-ui/services/speech/bus'
|
||||
import { shallowRef } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
|
||||
@@ -8,7 +10,17 @@ import InteractiveArea from '../components/InteractiveArea.vue'
|
||||
import WindowTitleBar from '../components/Window/TitleBar.vue'
|
||||
|
||||
const sessionsDrawerOpen = shallowRef(false)
|
||||
const { speechMuted, toggleSpeechMuted } = useStopSpeakingButton()
|
||||
const getOutputPlaybackState = defineInvoke(getSpeechBusContext(), speechOutputGetPlaybackState)
|
||||
const { speechMuted, toggleSpeechMuted } = useStopSpeakingButton({
|
||||
resolveSpeakingState: async () => {
|
||||
// A BroadcastChannel round trip is normally immediate. Bound the
|
||||
// analytics-only lookup so a reloading output renderer cannot stall mute.
|
||||
const state = await getOutputPlaybackState(undefined, {
|
||||
signal: AbortSignal.timeout(1000),
|
||||
})
|
||||
return state.speaking
|
||||
},
|
||||
})
|
||||
const { t } = useI18n()
|
||||
</script>
|
||||
|
||||
@@ -20,6 +32,19 @@ const { t } = useI18n()
|
||||
@title-click="sessionsDrawerOpen = true"
|
||||
>
|
||||
<template #actions>
|
||||
<button
|
||||
data-testid="conversation-selector-button"
|
||||
:class="[
|
||||
'h-7 w-7 flex items-center justify-center rounded-md outline-none',
|
||||
'text-base text-neutral-400 transition-colors transition-transform active:scale-95',
|
||||
'hover:bg-neutral-200 hover:text-primary-500 dark:text-neutral-500 dark:hover:bg-neutral-800 dark:hover:text-primary-400',
|
||||
]"
|
||||
:title="t('stage.chat.sessions.title')"
|
||||
:aria-label="t('stage.chat.sessions.title')"
|
||||
@click="sessionsDrawerOpen = true"
|
||||
>
|
||||
<div class="i-solar:chat-line-bold-duotone" />
|
||||
</button>
|
||||
<button
|
||||
data-testid="speech-mute-button"
|
||||
:class="[
|
||||
@@ -40,7 +65,6 @@ const { t } = useI18n()
|
||||
</template>
|
||||
</WindowTitleBar>
|
||||
<InteractiveArea
|
||||
v-model:sessions-drawer-open="sessionsDrawerOpen"
|
||||
class="interaction-area block"
|
||||
h-full w-full p-4 transition="opacity duration-250"
|
||||
/>
|
||||
|
||||
@@ -236,6 +236,34 @@ describe('createChatOrchestratorRuntime', () => {
|
||||
expect(harness.promptProjections).toHaveLength(1)
|
||||
})
|
||||
|
||||
// ROOT CAUSE:
|
||||
//
|
||||
// Speech-muted consumers dispatch plugin CALL markers without a TTS
|
||||
// session. If the hook context has no turn id, a locally unhandled call
|
||||
// cannot be correlated and relayed to another Electron renderer.
|
||||
it('preserves the round turn id on special-token hooks', async () => {
|
||||
const harness = createHarness()
|
||||
let specialTurnId = ''
|
||||
|
||||
harness.runtime.hooks.onTokenSpecial(async (_special, context) => {
|
||||
specialTurnId = context.turnId
|
||||
})
|
||||
harness.stream.mockImplementationOnce(async (_model, _chatProvider, _messages, options) => {
|
||||
await options?.onStreamEvent?.({ type: 'text-delta', text: '<|CALL ["plugin.action"]|>' })
|
||||
await options?.onStreamEvent?.({ type: 'finish', finishReason: 'stop' })
|
||||
})
|
||||
|
||||
await harness.runtime.ingest('trigger special', {
|
||||
model: 'gpt-test',
|
||||
chatProvider: provider,
|
||||
})
|
||||
|
||||
expect(specialTurnId).toBe('user-id')
|
||||
expect(harness.telemetry.messageSendStarted).toEqual([
|
||||
expect.objectContaining({ roundId: specialTurnId }),
|
||||
])
|
||||
})
|
||||
|
||||
it('keeps timestamp prefixes stable for legacy user messages without createdAt', async () => {
|
||||
const harness = createHarness()
|
||||
const legacyUserMessage: ChatHistoryItem = {
|
||||
|
||||
@@ -442,8 +442,14 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps)
|
||||
const sendingCreatedAt = now()
|
||||
|
||||
// TODO: Expire or prune stale runtime contexts from disconnected services before composing.
|
||||
// Allocate the three per-round ids in their historical order so callers
|
||||
// with deterministic id factories keep the same durable message ids.
|
||||
const streamContextMessageId = createId()
|
||||
const assistantMessageId = createId()
|
||||
const roundId = createId()
|
||||
const streamingMessageContext: ChatStreamEventContext = {
|
||||
message: { role: 'user', content: sendingMessage, createdAt: sendingCreatedAt, id: createId() },
|
||||
turnId: roundId,
|
||||
message: { role: 'user', content: sendingMessage, createdAt: sendingCreatedAt, id: streamContextMessageId },
|
||||
contexts: deps.context.snapshot(),
|
||||
composedMessage: [],
|
||||
input: options.input,
|
||||
@@ -471,14 +477,13 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps)
|
||||
slices: [],
|
||||
tool_results: [],
|
||||
createdAt: now(),
|
||||
id: createId(),
|
||||
id: assistantMessageId,
|
||||
}
|
||||
patchForegroundStream(sessionId, buildingMessage)
|
||||
const sendSource = options.input ? 'voice' : 'text'
|
||||
const activeProvider = deps.getActiveProvider?.() ?? ''
|
||||
// The user message is the durable start of a round, so its ID also serves
|
||||
// as the correlation key for every telemetry milestone emitted by it.
|
||||
const roundId = createId()
|
||||
const correlation: ChatRoundCorrelation = {
|
||||
conversationId: sessionId,
|
||||
roundId,
|
||||
|
||||
@@ -50,6 +50,8 @@ export interface ContextMessage extends ContextUpdate<Record<string, unknown>, u
|
||||
export type ChatHistoryItem = (ChatMessage | ErrorMessage) & { context?: ContextMessage } & { createdAt?: number, id?: string }
|
||||
|
||||
export interface ChatStreamEventContext {
|
||||
/** Stable correlation id shared by every hook emitted for one user turn. */
|
||||
turnId: string
|
||||
message: ChatHistoryItem
|
||||
contexts: Record<string, ContextMessage[]>
|
||||
composedMessage: Array<Message>
|
||||
|
||||
@@ -199,15 +199,36 @@ onMounted(() => {
|
||||
<div translate-y="[-100%]" absolute right-0 px-3 pb-3 font-sans>
|
||||
<div flex="~ col" gap-1>
|
||||
<ActionAbout />
|
||||
<button
|
||||
border="2 solid neutral-100/60 dark:neutral-800/30"
|
||||
bg="neutral-50/70 dark:neutral-800/70"
|
||||
w-fit flex items-center self-end justify-center rounded-xl p-2 backdrop-blur-md
|
||||
title="Conversations"
|
||||
@click="sessionsDrawerOpen = true"
|
||||
>
|
||||
<div i-solar:chat-line-bold-duotone size-5 text="neutral-500 dark:neutral-400" />
|
||||
</button>
|
||||
<div flex="~ col" items-end gap-1>
|
||||
<button
|
||||
data-testid="conversation-selector-button"
|
||||
border="2 solid neutral-100/60 dark:neutral-800/30"
|
||||
bg="neutral-50/70 dark:neutral-800/70"
|
||||
w-fit flex items-center self-end justify-center rounded-xl p-2 backdrop-blur-md
|
||||
:title="t('stage.chat.sessions.title')"
|
||||
:aria-label="t('stage.chat.sessions.title')"
|
||||
@click="sessionsDrawerOpen = true"
|
||||
>
|
||||
<div i-solar:chat-line-bold-duotone size-5 text="neutral-500 dark:neutral-400" />
|
||||
</button>
|
||||
<button
|
||||
data-testid="speech-mute-button"
|
||||
:class="[
|
||||
'w-fit flex items-center self-end justify-center rounded-xl border-2 border-solid p-2 backdrop-blur-md',
|
||||
'border-neutral-100/60 text-neutral-500 transition-colors active:scale-95 dark:border-neutral-800/30 dark:text-neutral-400',
|
||||
speechMuted
|
||||
? 'bg-primary-100/80 text-primary-600 dark:bg-primary-900/60 dark:text-primary-300'
|
||||
: 'bg-neutral-50/70 hover:text-primary-500 dark:bg-neutral-800/70 dark:hover:text-primary-400',
|
||||
]"
|
||||
:title="speechMuted ? t('stage.speech-output.unmute') : t('stage.speech-output.mute')"
|
||||
:aria-label="speechMuted ? t('stage.speech-output.unmute') : t('stage.speech-output.mute')"
|
||||
:aria-pressed="speechMuted"
|
||||
@click="toggleSpeechMuted"
|
||||
>
|
||||
<div v-if="speechMuted" class="i-solar:volume-cross-bold-duotone size-5" />
|
||||
<div v-else class="i-solar:volume-loud-bold-duotone size-5" />
|
||||
</button>
|
||||
</div>
|
||||
<ChatSessionsDrawer v-model="sessionsDrawerOpen" />
|
||||
<HearingConfigDialog
|
||||
v-model:enabled="enabled"
|
||||
@@ -227,23 +248,6 @@ onMounted(() => {
|
||||
</Transition>
|
||||
</button>
|
||||
</HearingConfigDialog>
|
||||
<button
|
||||
data-testid="speech-mute-button"
|
||||
:class="[
|
||||
'w-fit flex items-center self-end justify-center rounded-xl border-2 border-solid p-2 backdrop-blur-md',
|
||||
'border-neutral-100/60 text-neutral-500 transition-colors active:scale-95 dark:border-neutral-800/30 dark:text-neutral-400',
|
||||
speechMuted
|
||||
? 'bg-primary-100/80 text-primary-600 dark:bg-primary-900/60 dark:text-primary-300'
|
||||
: 'bg-neutral-50/70 hover:text-primary-500 dark:bg-neutral-800/70 dark:hover:text-primary-400',
|
||||
]"
|
||||
:title="speechMuted ? t('stage.speech-output.unmute') : t('stage.speech-output.mute')"
|
||||
:aria-label="speechMuted ? t('stage.speech-output.unmute') : t('stage.speech-output.mute')"
|
||||
:aria-pressed="speechMuted"
|
||||
@click="toggleSpeechMuted"
|
||||
>
|
||||
<div v-if="speechMuted" class="i-solar:volume-cross-bold-duotone size-5" />
|
||||
<div v-else class="i-solar:volume-loud-bold-duotone size-5" />
|
||||
</button>
|
||||
<button border="2 solid neutral-100/60 dark:neutral-800/30" bg="neutral-50/70 dark:neutral-800/70" w-fit flex items-center self-end justify-center rounded-xl p-2 backdrop-blur-md title="Theme" @click="toggleDark()">
|
||||
<Transition name="fade" mode="out-in">
|
||||
<div v-if="isDark" i-solar:moon-outline size-5 text="neutral-500 dark:neutral-400" />
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
<script setup lang="ts">
|
||||
import { ChatSessionsDrawer } from '@proj-airi/stage-ui/components/scenarios/chat'
|
||||
import { useAnalytics } from '@proj-airi/stage-ui/composables/use-analytics'
|
||||
import { useChatMaintenanceStore } from '@proj-airi/stage-ui/stores/chat/maintenance'
|
||||
import { useChatSessionStore } from '@proj-airi/stage-ui/stores/chat/session-store'
|
||||
@@ -20,6 +21,7 @@ const { speechMuted, toggleSpeechMuted } = useStopSpeakingButton()
|
||||
const { t } = useI18n()
|
||||
|
||||
const backgroundDialogOpen = ref(false)
|
||||
const sessionsDrawerOpen = ref(false)
|
||||
|
||||
function handleCleanupMessages() {
|
||||
const messageCount = messages.value.filter(message => message.role !== 'system').length
|
||||
@@ -33,24 +35,40 @@ function handleCleanupMessages() {
|
||||
|
||||
<template>
|
||||
<BackgroundDialogPicker v-model="backgroundDialogOpen" />
|
||||
<ChatSessionsDrawer v-model="sessionsDrawerOpen" />
|
||||
<div absolute bottom--8 right-0 flex gap-2>
|
||||
<button
|
||||
data-testid="speech-mute-button"
|
||||
:class="[
|
||||
'max-h-[10lh] min-h-[1lh] flex items-center justify-center rounded-md p-2 outline-none',
|
||||
'text-lg transition-colors transition-transform active:scale-95',
|
||||
speechMuted
|
||||
? 'bg-primary-100 text-primary-600 dark:bg-primary-900/40 dark:text-primary-300'
|
||||
: 'bg-neutral-100 text-neutral-500 hover:text-primary-500 dark:bg-neutral-800 dark:text-neutral-400 dark:hover:text-primary-400',
|
||||
]"
|
||||
:title="speechMuted ? t('stage.speech-output.unmute') : t('stage.speech-output.mute')"
|
||||
:aria-label="speechMuted ? t('stage.speech-output.unmute') : t('stage.speech-output.mute')"
|
||||
:aria-pressed="speechMuted"
|
||||
@click="toggleSpeechMuted"
|
||||
>
|
||||
<div v-if="speechMuted" class="i-solar:volume-cross-bold-duotone" />
|
||||
<div v-else class="i-solar:volume-loud-bold-duotone" />
|
||||
</button>
|
||||
<div flex gap-1>
|
||||
<button
|
||||
data-testid="conversation-selector-button"
|
||||
:class="[
|
||||
'max-h-[10lh] min-h-[1lh] flex items-center justify-center rounded-md p-2 outline-none',
|
||||
'bg-neutral-100 text-lg text-neutral-500 transition-colors transition-transform active:scale-95',
|
||||
'hover:text-primary-500 dark:bg-neutral-800 dark:text-neutral-400 dark:hover:text-primary-400',
|
||||
]"
|
||||
:title="t('stage.chat.sessions.title')"
|
||||
:aria-label="t('stage.chat.sessions.title')"
|
||||
@click="sessionsDrawerOpen = true"
|
||||
>
|
||||
<div class="i-solar:chat-line-bold-duotone" />
|
||||
</button>
|
||||
<button
|
||||
data-testid="speech-mute-button"
|
||||
:class="[
|
||||
'max-h-[10lh] min-h-[1lh] flex items-center justify-center rounded-md p-2 outline-none',
|
||||
'text-lg transition-colors transition-transform active:scale-95',
|
||||
speechMuted
|
||||
? 'bg-primary-100 text-primary-600 dark:bg-primary-900/40 dark:text-primary-300'
|
||||
: 'bg-neutral-100 text-neutral-500 hover:text-primary-500 dark:bg-neutral-800 dark:text-neutral-400 dark:hover:text-primary-400',
|
||||
]"
|
||||
:title="speechMuted ? t('stage.speech-output.unmute') : t('stage.speech-output.mute')"
|
||||
:aria-label="speechMuted ? t('stage.speech-output.unmute') : t('stage.speech-output.mute')"
|
||||
:aria-pressed="speechMuted"
|
||||
@click="toggleSpeechMuted"
|
||||
>
|
||||
<div v-if="speechMuted" class="i-solar:volume-cross-bold-duotone" />
|
||||
<div v-else class="i-solar:volume-loud-bold-duotone" />
|
||||
</button>
|
||||
</div>
|
||||
<ViewControls />
|
||||
<button
|
||||
class="max-h-[10lh] min-h-[1lh]"
|
||||
|
||||
@@ -3,7 +3,6 @@ import type { ChatProvider } from '@xsai-ext/providers/utils'
|
||||
|
||||
import { errorMessageFrom } from '@moeru/std'
|
||||
import { isStageTamagotchi } from '@proj-airi/stage-shared'
|
||||
import { ChatSessionsDrawer } from '@proj-airi/stage-ui/components/scenarios/chat'
|
||||
import { HearingConfig } from '@proj-airi/stage-ui/components/scenarios/dialogs/audio-input/index'
|
||||
import { useAudioAnalyzer } from '@proj-airi/stage-ui/composables'
|
||||
import { useAudioContext } from '@proj-airi/stage-ui/stores/audio'
|
||||
@@ -26,7 +25,6 @@ import { useStopSpeakingButton } from '../../composables/useStopSpeakingButton'
|
||||
|
||||
const messageInput = ref<string>('')
|
||||
const hearingPopoverOpen = ref(false)
|
||||
const sessionsDrawerOpen = ref(false)
|
||||
const isComposing = ref(false)
|
||||
const DOUBLE_ENTER_INTERVAL_MS = 300
|
||||
const TRAILING_NEWLINES_REGEX = /[\r\n]+$/
|
||||
@@ -205,24 +203,10 @@ watch(sendMode, () => {
|
||||
@compositionend="isComposing = false"
|
||||
/>
|
||||
|
||||
<!-- Bottom-left action button: Microphone -->
|
||||
<!-- Input configuration controls -->
|
||||
<div
|
||||
absolute bottom-2 left-2 z-10 flex items-center gap-2
|
||||
>
|
||||
<!-- Conversations drawer trigger -->
|
||||
<button
|
||||
:class="[
|
||||
'h-8 w-8 flex items-center justify-center rounded-md outline-none transition-all duration-200 active:scale-95',
|
||||
'text-lg text-neutral-500 dark:text-neutral-400',
|
||||
]"
|
||||
title="Conversations"
|
||||
@click="sessionsDrawerOpen = true"
|
||||
>
|
||||
<div class="i-solar:chat-line-bold-duotone h-5 w-5" />
|
||||
</button>
|
||||
|
||||
<ChatSessionsDrawer v-model="sessionsDrawerOpen" />
|
||||
|
||||
<DropdownMenuRoot>
|
||||
<DropdownMenuTrigger as-child>
|
||||
<button
|
||||
|
||||
@@ -6,7 +6,8 @@ import { useStopSpeakingButton } from './useStopSpeakingButton'
|
||||
const nowSpeaking = ref(false)
|
||||
const speechMuted = ref(false)
|
||||
const requestStopSpeakingMock = vi.fn()
|
||||
const toggleSpeechMutedMock = vi.fn()
|
||||
const setSpeechMutedMock = vi.fn()
|
||||
const trackSpeechMuteToggledMock = vi.fn()
|
||||
const trackTtsStopClickedMock = vi.fn()
|
||||
|
||||
vi.mock('@proj-airi/stage-ui/stores/audio', () => ({
|
||||
@@ -18,13 +19,14 @@ vi.mock('@proj-airi/stage-ui/stores/audio', () => ({
|
||||
vi.mock('@proj-airi/stage-ui/stores/speech-output-control', () => ({
|
||||
useSpeechOutputControlStore: () => ({
|
||||
requestStopSpeaking: requestStopSpeakingMock,
|
||||
setSpeechMuted: setSpeechMutedMock,
|
||||
speechMuted,
|
||||
toggleSpeechMuted: toggleSpeechMutedMock,
|
||||
}),
|
||||
}))
|
||||
|
||||
vi.mock('@proj-airi/stage-ui/composables/use-analytics', () => ({
|
||||
useAnalytics: () => ({
|
||||
trackSpeechMuteToggled: trackSpeechMuteToggledMock,
|
||||
trackTtsStopClicked: trackTtsStopClickedMock,
|
||||
}),
|
||||
}))
|
||||
@@ -74,16 +76,72 @@ describe('useStopSpeakingButton', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('exposes persisted mute state and toggles it through the shared output store', () => {
|
||||
speechMuted.value = true
|
||||
toggleSpeechMutedMock.mockClear()
|
||||
it('tracks mute and unmute with the active playback state', async () => {
|
||||
speechMuted.value = false
|
||||
nowSpeaking.value = true
|
||||
setSpeechMutedMock.mockClear()
|
||||
trackSpeechMuteToggledMock.mockClear()
|
||||
|
||||
const controls = useStopSpeakingButton()
|
||||
|
||||
expect(controls.speechMuted.value).toBe(true)
|
||||
await controls.toggleSpeechMuted()
|
||||
|
||||
controls.toggleSpeechMuted()
|
||||
expect(setSpeechMutedMock).toHaveBeenCalledWith(true)
|
||||
expect(trackSpeechMuteToggledMock).toHaveBeenCalledWith({
|
||||
muted: true,
|
||||
was_speaking: true,
|
||||
})
|
||||
|
||||
expect(toggleSpeechMutedMock).toHaveBeenCalledOnce()
|
||||
speechMuted.value = true
|
||||
nowSpeaking.value = false
|
||||
|
||||
await controls.toggleSpeechMuted()
|
||||
|
||||
expect(setSpeechMutedMock).toHaveBeenLastCalledWith(false)
|
||||
expect(trackSpeechMuteToggledMock).toHaveBeenLastCalledWith({
|
||||
muted: false,
|
||||
was_speaking: false,
|
||||
})
|
||||
})
|
||||
|
||||
// ROOT CAUSE:
|
||||
//
|
||||
// Electron's auxiliary /chat renderer has its own Pinia instance, so its
|
||||
// local nowSpeaking value stays false while the main Stage renderer speaks.
|
||||
//
|
||||
// The title-bar mute control now resolves state from the output host before
|
||||
// capturing speech_mute_toggled.
|
||||
it('tracks the speaking state resolved from a remote output host', async () => {
|
||||
speechMuted.value = false
|
||||
nowSpeaking.value = false
|
||||
setSpeechMutedMock.mockClear()
|
||||
trackSpeechMuteToggledMock.mockClear()
|
||||
|
||||
const resolveSpeakingState = vi.fn().mockResolvedValue(true)
|
||||
const controls = useStopSpeakingButton({ resolveSpeakingState })
|
||||
|
||||
await controls.toggleSpeechMuted()
|
||||
|
||||
expect(resolveSpeakingState).toHaveBeenCalledTimes(1)
|
||||
expect(setSpeechMutedMock).toHaveBeenCalledWith(true)
|
||||
expect(trackSpeechMuteToggledMock).toHaveBeenCalledWith({
|
||||
muted: true,
|
||||
was_speaking: true,
|
||||
})
|
||||
})
|
||||
|
||||
it('still toggles mute without capturing a false metric when the output host is unavailable', async () => {
|
||||
speechMuted.value = false
|
||||
setSpeechMutedMock.mockClear()
|
||||
trackSpeechMuteToggledMock.mockClear()
|
||||
|
||||
const controls = useStopSpeakingButton({
|
||||
resolveSpeakingState: () => Promise.reject(new Error('output host reloading')),
|
||||
})
|
||||
|
||||
await controls.toggleSpeechMuted()
|
||||
|
||||
expect(setSpeechMutedMock).toHaveBeenCalledWith(true)
|
||||
expect(trackSpeechMuteToggledMock).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -10,11 +10,18 @@ import { computed } from 'vue'
|
||||
* Manual stops affect current playback without cancelling text generation.
|
||||
* Mute is persisted by the shared store and also blocks future TTS sessions.
|
||||
*/
|
||||
export function useStopSpeakingButton() {
|
||||
export function useStopSpeakingButton(options: {
|
||||
/**
|
||||
* Reads speaking state from the renderer that owns playback.
|
||||
*
|
||||
* @default The current renderer's speaking store.
|
||||
*/
|
||||
resolveSpeakingState?: () => boolean | Promise<boolean>
|
||||
} = {}) {
|
||||
const { nowSpeaking } = storeToRefs(useSpeakingStore())
|
||||
const speechOutputControlStore = useSpeechOutputControlStore()
|
||||
const { speechMuted } = storeToRefs(speechOutputControlStore)
|
||||
const { trackTtsStopClicked } = useAnalytics()
|
||||
const { trackSpeechMuteToggled, trackTtsStopClicked } = useAnalytics()
|
||||
|
||||
const showStopSpeakingButton = computed(() => nowSpeaking.value)
|
||||
|
||||
@@ -28,11 +35,32 @@ export function useStopSpeakingButton() {
|
||||
speechOutputControlStore.requestStopSpeaking('manual-all')
|
||||
}
|
||||
|
||||
async function toggleSpeechMuted() {
|
||||
let wasSpeaking: boolean
|
||||
try {
|
||||
wasSpeaking = await (options.resolveSpeakingState?.() ?? nowSpeaking.value)
|
||||
}
|
||||
catch {
|
||||
// Muting is the user action; analytics must not make it fail when an
|
||||
// auxiliary renderer cannot reach the output host during a reload.
|
||||
speechOutputControlStore.setSpeechMuted(!speechMuted.value)
|
||||
return
|
||||
}
|
||||
|
||||
const muted = !speechMuted.value
|
||||
|
||||
speechOutputControlStore.setSpeechMuted(muted)
|
||||
trackSpeechMuteToggled({
|
||||
muted,
|
||||
was_speaking: wasSpeaking,
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
showStopSpeakingButton,
|
||||
speechMuted,
|
||||
stopSpeakingFromChat,
|
||||
stopAllSpeaking,
|
||||
toggleSpeechMuted: speechOutputControlStore.toggleSpeechMuted,
|
||||
toggleSpeechMuted,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import type { UnElevenLabsOptions } from 'unspeech'
|
||||
import type { EmotionPayload } from '../../constants/emotions'
|
||||
import type { SpeechTransport, StageTtsSession, StreamingSessionSnapshot } from '../../libs/speech/tts-session'
|
||||
|
||||
import { defineInvokeHandler } from '@moeru/eventa'
|
||||
import { sleep } from '@moeru/std'
|
||||
import { createLive2DLipSync } from '@proj-airi/model-driver-lipsync'
|
||||
import { wlipsyncProfile } from '@proj-airi/model-driver-lipsync/shared/wlipsync'
|
||||
@@ -38,6 +39,7 @@ import { getDefaultStreamingModel, getDefinedProvider } from '../../libs/provide
|
||||
import { OFFICIAL_SPEECH_PROVIDER_ID, OFFICIAL_SPEECH_STREAMING_PROVIDER_ID } from '../../libs/providers/providers/official'
|
||||
import { bindSpeakingStateToPlaybackManager } from '../../libs/speech/playback-speaking-state'
|
||||
import { createStageTtsSession } from '../../libs/speech/tts-session'
|
||||
import { getSpeechBusContext, speechOutputGetPlaybackState } from '../../services/speech/bus'
|
||||
import { useAudioContext, useSpeakingStore } from '../../stores/audio'
|
||||
import { useBackgroundStore } from '../../stores/background'
|
||||
import { useChatOrchestratorStore } from '../../stores/chat'
|
||||
@@ -91,6 +93,11 @@ const {
|
||||
spineRenderScale,
|
||||
} = storeToRefs(settingsStore)
|
||||
const { mouthOpenSize, nowSpeaking } = storeToRefs(useSpeakingStore())
|
||||
const disposePlaybackStateHandler = defineInvokeHandler(
|
||||
getSpeechBusContext(),
|
||||
speechOutputGetPlaybackState,
|
||||
() => ({ speaking: nowSpeaking.value }),
|
||||
)
|
||||
const { audioContext } = useAudioContext()
|
||||
const currentAudioSource = ref<AudioBufferSourceNode>()
|
||||
const speechOutputControlStore = useSpeechOutputControlStore()
|
||||
@@ -696,7 +703,7 @@ function resolveStreamingSessionModel(): string | null {
|
||||
return sessionModel
|
||||
}
|
||||
|
||||
function buildStreamingSnapshot(): StreamingSessionSnapshot | null {
|
||||
function buildStreamingSnapshot(turnId: string): StreamingSessionSnapshot | null {
|
||||
if (speechMuted.value)
|
||||
return null
|
||||
|
||||
@@ -733,7 +740,7 @@ function buildStreamingSnapshot(): StreamingSessionSnapshot | null {
|
||||
audio: { sample_rate: 24000, bit_rate: 64000 },
|
||||
},
|
||||
ownerId: activeCardId.value,
|
||||
onImmediateSpecial: playSpecialToken,
|
||||
onImmediateSpecial: special => playSpecialToken(special, { turnId }),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -747,7 +754,7 @@ function resolveSpeechTransport(providerId: string | null | undefined): SpeechTr
|
||||
return getDefinedProvider(providerId)?.capabilities?.speech?.transport
|
||||
}
|
||||
|
||||
function openTtsSession(): StageTtsSession {
|
||||
function openTtsSession(turnId: string): StageTtsSession {
|
||||
// A session must only clear the module-level `currentSession` if it IS that session. The previous
|
||||
// code cleared it whenever any `stream-` session completed, which is unsafe once sessions exist that
|
||||
// are not assigned to `currentSession` (e.g. one-off read-aloud sessions): one of those finishing
|
||||
@@ -760,11 +767,12 @@ function openTtsSession(): StageTtsSession {
|
||||
}
|
||||
session = createStageTtsSession<AudioBuffer>({
|
||||
transport: resolveSpeechTransport(activeSpeechProvider.value),
|
||||
streaming: buildStreamingSnapshot,
|
||||
streaming: () => buildStreamingSnapshot(turnId),
|
||||
audioContext,
|
||||
playbackManager,
|
||||
openIntent: opts => speechRuntimeStore.openIntent(opts),
|
||||
intentOptions: () => ({
|
||||
turnId,
|
||||
ownerId: activeCardId.value,
|
||||
priority: 'normal',
|
||||
behavior: 'queue',
|
||||
@@ -802,7 +810,7 @@ watch(speechMuted, (muted) => {
|
||||
stopSpeechOutput('muted')
|
||||
}, { immediate: true })
|
||||
|
||||
chatHookCleanups.push(onBeforeMessageComposed(async () => {
|
||||
chatHookCleanups.push(onBeforeMessageComposed(async (_message, context) => {
|
||||
officialAutoTtsTrackedForTurn = false
|
||||
playbackManager.stopAll('new-message')
|
||||
resetAssistantSpeechSurface('new-message')
|
||||
@@ -815,7 +823,7 @@ chatHookCleanups.push(onBeforeMessageComposed(async () => {
|
||||
|
||||
setupAnalyser()
|
||||
await setupLipSync()
|
||||
currentSession = openTtsSession()
|
||||
currentSession = openTtsSession(context.turnId)
|
||||
}))
|
||||
|
||||
chatHookCleanups.push(onBeforeSend(async () => {
|
||||
@@ -826,11 +834,11 @@ chatHookCleanups.push(onTokenLiteral(async (literal) => {
|
||||
currentSession?.appendText(literal)
|
||||
}))
|
||||
|
||||
chatHookCleanups.push(onTokenSpecial(async (special) => {
|
||||
chatHookCleanups.push(onTokenSpecial(async (special, context) => {
|
||||
// Muting speech must not suppress non-audio signals such as emotion, motion,
|
||||
// delay, or plugin calls that normally travel through the TTS session.
|
||||
if (speechMuted.value) {
|
||||
await playSpecialToken(special)
|
||||
await playSpecialToken(special, { turnId: context.turnId })
|
||||
return
|
||||
}
|
||||
|
||||
@@ -996,6 +1004,7 @@ async function captureFrame() {
|
||||
}
|
||||
|
||||
onUnmounted(() => {
|
||||
disposePlaybackStateHandler()
|
||||
resetLive2dLipSync()
|
||||
chatHookCleanups.forEach(dispose => dispose?.())
|
||||
viewUpdateCleanups.forEach(dispose => dispose?.())
|
||||
|
||||
@@ -83,6 +83,21 @@ describe('useAnalytics conversation product events', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('captures speech mute state changes without conversation or audio content', () => {
|
||||
const analytics = useAnalytics()
|
||||
|
||||
analytics.trackSpeechMuteToggled({
|
||||
muted: true,
|
||||
was_speaking: true,
|
||||
})
|
||||
|
||||
expect(analyticsMocks.posthogCaptureMock).toHaveBeenCalledWith('speech_mute_toggled', {
|
||||
app_surface: 'web',
|
||||
muted: true,
|
||||
was_speaking: true,
|
||||
})
|
||||
})
|
||||
|
||||
it('captures custom-provider token usage without prompt or response content', () => {
|
||||
const analytics = useAnalytics()
|
||||
|
||||
|
||||
@@ -654,6 +654,18 @@ export function useAnalytics() {
|
||||
})
|
||||
}
|
||||
|
||||
function trackSpeechMuteToggled(properties: {
|
||||
muted: boolean
|
||||
was_speaking: boolean
|
||||
}) {
|
||||
if (!canCapture())
|
||||
return
|
||||
posthog.capture('speech_mute_toggled', {
|
||||
...properties,
|
||||
app_surface: getConversationAnalyticsSurface(),
|
||||
})
|
||||
}
|
||||
|
||||
function trackChatSessionSelected(properties: { source: 'sessions_drawer', message_count: number, cloud_synced: boolean }) {
|
||||
if (!canCapture())
|
||||
return
|
||||
@@ -1305,6 +1317,7 @@ export function useAnalytics() {
|
||||
trackProviderConfigCompleted,
|
||||
trackOfficialProviderEnabled,
|
||||
trackTtsStopClicked,
|
||||
trackSpeechMuteToggled,
|
||||
trackChatSessionSelected,
|
||||
trackChatMessageDeleted,
|
||||
trackChatMessagesCleared,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { defineEventa } from '@moeru/eventa'
|
||||
import { defineEventa, defineInvokeEventa } from '@moeru/eventa'
|
||||
import { createContext as createBroadcastChannelContext } from '@moeru/eventa/adapters/broadcast-channel'
|
||||
|
||||
export interface SpeechIntentStartPayload {
|
||||
@@ -42,6 +42,15 @@ export const speechIntentFlushEvent = defineEventa<SpeechIntentTokenPayload>('ev
|
||||
export const speechIntentEndEvent = defineEventa<SpeechIntentEndPayload>('eventa:audio:speech:intent:end')
|
||||
export const speechIntentCancelEvent = defineEventa<SpeechIntentCancelPayload>('eventa:audio:speech:intent:cancel')
|
||||
|
||||
/** Snapshot served by the renderer that owns active speech playback. */
|
||||
export interface SpeechOutputPlaybackState {
|
||||
/** Whether the output host is currently playing assistant speech. */
|
||||
speaking: boolean
|
||||
}
|
||||
|
||||
/** Cross-renderer request for the active speech output host's playback state. */
|
||||
export const speechOutputGetPlaybackState = defineInvokeEventa<SpeechOutputPlaybackState>('eventa:audio:speech:output:get-playback-state')
|
||||
|
||||
const BUS_CHANNEL_NAME = 'proj-airi:pipelines:outputs:speech'
|
||||
|
||||
let context: ReturnType<typeof createBroadcastChannelContext>['context'] | undefined
|
||||
|
||||
@@ -501,7 +501,12 @@ describe('chat orchestrator contract', () => {
|
||||
expect(syntheticContextText).toContain('- system:weather: sunny')
|
||||
})
|
||||
|
||||
it('emits special tokens for speech timeline handling during chat streaming', async () => {
|
||||
// ROOT CAUSE:
|
||||
//
|
||||
// Muted speech dispatches special tokens without opening a TTS session.
|
||||
// Without a turn id in the hook context, an unhandled plugin CALL cannot be
|
||||
// correlated and relayed to a handler in another Electron renderer.
|
||||
it('emits special tokens with a stable turn id for cross-renderer routing', async () => {
|
||||
getContextsSnapshotMock.mockReturnValue({})
|
||||
llmStreamMock.mockImplementationOnce(async (_model, _provider, _messages, options) => {
|
||||
await options.onStreamEvent({ type: 'text-delta', text: '<|CALL ["plugin.action"]|>' })
|
||||
@@ -518,7 +523,9 @@ describe('chat orchestrator contract', () => {
|
||||
|
||||
expect(specialHook).toHaveBeenCalledWith('<|CALL ["plugin.action"]|>', expect.objectContaining({
|
||||
contexts: {},
|
||||
turnId: expect.any(String),
|
||||
}))
|
||||
expect(specialHook.mock.calls[0]?.[1].turnId.length).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user