diff --git a/apps/stage-tamagotchi/src/renderer/main.ts b/apps/stage-tamagotchi/src/renderer/main.ts index 220b2fda0..a42a0f198 100644 --- a/apps/stage-tamagotchi/src/renderer/main.ts +++ b/apps/stage-tamagotchi/src/renderer/main.ts @@ -7,7 +7,7 @@ import { autoAnimatePlugin } from '@formkit/auto-animate/vue' import { PiniaColada } from '@pinia/colada' import { trackButtonPlugin } from '@proj-airi/stage-ui/directives/track-button' import { configureAnalyticsAdapter } from '@proj-airi/stage-ui/libs/analytics' -import { setupSynced } from '@proj-airi/stage-ui/libs/pinia' +import { piniaPluginTracing, setupSynced } from '@proj-airi/stage-ui/libs/pinia' import { MotionPlugin } from '@vueuse/motion' import { createPinia } from 'pinia' import { setupLayouts } from 'virtual:generated-layouts' @@ -47,6 +47,7 @@ configureAnalyticsAdapter(async (options) => { const pinia = createPinia() const synced = setupSynced() pinia.use(synced.pinia) +pinia.use(piniaPluginTracing) const router = createRouter({ history: createWebHashHistory(), diff --git a/apps/stage-web/src/main.ts b/apps/stage-web/src/main.ts index 48b57fa0d..62ed9164d 100644 --- a/apps/stage-web/src/main.ts +++ b/apps/stage-web/src/main.ts @@ -9,7 +9,7 @@ import { PiniaColada } from '@pinia/colada' import { isEnvTruthy } from '@proj-airi/stage-shared' import { trackButtonPlugin } from '@proj-airi/stage-ui/directives/track-button' import { configureAnalyticsAdapter } from '@proj-airi/stage-ui/libs/analytics' -import { setupSynced } from '@proj-airi/stage-ui/libs/pinia' +import { piniaPluginTracing, setupSynced } from '@proj-airi/stage-ui/libs/pinia' import { MotionPlugin } from '@vueuse/motion' import { createPinia } from 'pinia' import { setupLayouts } from 'virtual:generated-layouts' @@ -37,6 +37,7 @@ configureAnalyticsAdapter(async (options) => { const pinia = createPinia() const synced = setupSynced() pinia.use(synced.pinia) +pinia.use(piniaPluginTracing) // TODO: vite-plugin-vue-layouts is long deprecated, replace with another layout solution const routeRecords = setupLayouts(routes as RouteRecordRaw[]) diff --git a/packages/stage-shared/package.json b/packages/stage-shared/package.json index b3a428c9b..ec56472ce 100644 --- a/packages/stage-shared/package.json +++ b/packages/stage-shared/package.json @@ -20,8 +20,10 @@ "./beat-sync": "./src/beat-sync/index.ts", "./global-shortcut": "./src/global-shortcut/index.ts", "./godot-stage": "./src/godot-stage/index.ts", + "./perf/io-trace": "./src/perf/io-trace.ts", "./server-channel-qr": "./src/server-channel-qr.ts", "./types/io-trace": "./src/types/io-trace.ts", + "./types/pinia-action-event": "./src/types/pinia-action-event.ts", "./electron-renderer": "./src/electron-renderer.d.ts", "./composables": "./src/composables/index.ts", "./webgpu": "./src/webgpu/index.ts" diff --git a/packages/stage-shared/src/perf/io-trace.ts b/packages/stage-shared/src/perf/io-trace.ts index d14bf4efe..b7cd28d39 100644 --- a/packages/stage-shared/src/perf/io-trace.ts +++ b/packages/stage-shared/src/perf/io-trace.ts @@ -10,6 +10,7 @@ export type IOSubsystem = (typeof IOSubsystems)[keyof typeof IOSubsystems] export const IOSpanNames = { InteractionTurn: 'Interaction turn', + SpeechTurn: 'Speech turn', VoiceActivityDetection: 'Voice activity detection', SpeechRecognition: 'Speech recognition', LLMInference: 'LLM inference', @@ -32,6 +33,11 @@ export const IOAttributes = { VADAborted: `${customPrefix}.vad.aborted`, ASRText: `${customPrefix}.asr.text`, ASRAbort: `${customPrefix}.asr.abort`, + LLMInputMessageCount: `${customPrefix}.llm.input_message_count`, + LLMInputMessageRoles: `${customPrefix}.llm.input_message_roles`, + LLMInputUserMessageCount: `${customPrefix}.llm.input_user_message_count`, + LLMOutputChunkCount: `${customPrefix}.llm.output_chunk_count`, + LLMOutputChunkLengths: `${customPrefix}.llm.output_chunk_lengths`, LLMTextLength: `${customPrefix}.llm.text_length`, StreamingControlCallName: `${customPrefix}.streaming_control.call_name`, StreamingControlHandlerCount: `${customPrefix}.streaming_control.handler_count`, @@ -47,9 +53,11 @@ export const IOAttributes = { TTSSegmentId: `${customPrefix}.tts.segment_id`, TTSText: `${customPrefix}.tts.text`, TTSChunkReason: `${customPrefix}.tts.chunk_reason`, + TTSAudioDurationMs: `${customPrefix}.tts.audio_duration_ms`, TTSInterrupted: `${customPrefix}.tts.interrupted`, TTSInterruptReason: `${customPrefix}.tts.interrupt_reason`, TTSCanceled: `${customPrefix}.tts.canceled`, + TurnId: `${customPrefix}.turn_id`, } as const export const IOEvents = { diff --git a/packages/stage-shared/src/types/pinia-action-event.ts b/packages/stage-shared/src/types/pinia-action-event.ts new file mode 100644 index 000000000..c2d82f462 --- /dev/null +++ b/packages/stage-shared/src/types/pinia-action-event.ts @@ -0,0 +1,21 @@ +/** Lifecycle phase observed for one Pinia action invocation. */ +export type PiniaActionEventStatus = 'started' | 'completed' | 'failed' + +/** Broadcast channel used by Pinia action tracing producers and consumers. */ +export const piniaActionTracingChannelName = 'airi-pinia-action-tracing' + +/** Test-safe metadata emitted for one Pinia action lifecycle transition. */ +export interface PiniaActionEvent { + /** Correlates lifecycle events from the same action invocation. */ + invocationId: string + /** Name passed to `defineStore`. */ + storeId: string + /** Action property name reported by Pinia. */ + actionName: string + status: PiniaActionEventStatus + timestamp: number + /** Renderer URL that invoked the action, when available. */ + sourceUrl?: string + /** Safe error text included only for failed actions. */ + errorMessage?: string +} diff --git a/packages/stage-ui/src/components/scenes/Stage.vue b/packages/stage-ui/src/components/scenes/Stage.vue index 8b88dca6b..5a16961cb 100644 --- a/packages/stage-ui/src/components/scenes/Stage.vue +++ b/packages/stage-ui/src/components/scenes/Stage.vue @@ -26,7 +26,6 @@ import { useBroadcastChannel } from '@vueuse/core' // import { createTransformers } from '@xsai-transformers/embed' // import embedWorkerURL from '@xsai-transformers/embed/worker?worker&url' // import { embed } from '@xsai/embed' -import { generateSpeech } from '@xsai/generate-speech' import { storeToRefs } from 'pinia' import { computed, nextTick, onMounted, onUnmounted, ref, shallowRef, watch } from 'vue' @@ -520,24 +519,18 @@ const speechPipeline = createSpeechPipeline({ // Non-streaming providers only: synth via REST. Streaming provider // was already early-returned above; it owns its own ws path opened // in `onBeforeMessageComposed`. - const providerConfigWithAnalytics = activeSpeechProvider.value === OFFICIAL_SPEECH_PROVIDER_ID - ? { - ...speechRequest.providerConfig, - extraBody: { - ...(speechRequest.providerConfig.extraBody as Record | undefined), - airi_analytics: { - trigger: 'auto', - source: 'chat_auto_tts', - voice_type: resolveStageVoiceType(), - }, - }, - } - : speechRequest.providerConfig - const res = await generateSpeech({ - ...provider.speech(model, providerConfigWithAnalytics), - input: speechRequest.input, - voice: voice.id, - }) + const res = await speechStore.speech( + provider, + model, + speechRequest.input, + voice.id, + speechRequest.providerConfig, + { + trigger: 'auto', + source: 'chat_auto_tts', + voice_type: resolveStageVoiceType(), + }, + ) if (signal.aborted || !res || res.byteLength === 0) return null diff --git a/packages/stage-ui/src/composables/use-io-trace-bridge.ts b/packages/stage-ui/src/composables/use-io-trace-bridge.ts index 69c36fcc1..23f5076db 100644 --- a/packages/stage-ui/src/composables/use-io-trace-bridge.ts +++ b/packages/stage-ui/src/composables/use-io-trace-bridge.ts @@ -11,8 +11,25 @@ export function useIOTraceBridge(pipeline: ReturnType() const playbackSpans = new Map() + const speechTurnSpans = new Map() const segmentReasons = new Map() + cleanupFns.push(pipeline.on('onTurnStart', (turnId) => { + speechTurnSpans.set(turnId, startSpan(IOSpanNames.SpeechTurn, activeTurnSpan.value, { + [IOAttributes.TurnId]: turnId, + })) + })) + + cleanupFns.push(pipeline.on('onTurnEnd', (turnId) => { + speechTurnSpans.get(turnId)?.end() + speechTurnSpans.delete(turnId) + })) + + cleanupFns.push(pipeline.on('onTurnCancel', ({ turnId }) => { + speechTurnSpans.get(turnId)?.end() + speechTurnSpans.delete(turnId) + })) + cleanupFns.push(pipeline.on('onSegment', (segment) => { segmentReasons.set(segment.segmentId, segment.reason) })) @@ -23,6 +40,7 @@ export function useIOTraceBridge(pipeline: ReturnType { const span = synthesisSpans.get(result.segmentId) if (span) { + if (result.audio instanceof AudioBuffer) + span.setAttribute(IOAttributes.TTSAudioDurationMs, result.audio.duration * 1000) span.end() synthesisSpans.delete(result.segmentId) } @@ -77,6 +97,8 @@ export function useIOTraceBridge(pipeline: ReturnType { + for (const span of speechTurnSpans.values()) + span.end() for (const cleanup of cleanupFns) cleanup() }) diff --git a/packages/stage-ui/src/libs/pinia/index.ts b/packages/stage-ui/src/libs/pinia/index.ts index 4287e3c7e..f0a60a4c9 100644 --- a/packages/stage-ui/src/libs/pinia/index.ts +++ b/packages/stage-ui/src/libs/pinia/index.ts @@ -1 +1,2 @@ +export * from './pinia-plugin-tracing' export * from './setup-synced' diff --git a/packages/stage-ui/src/libs/pinia/pinia-plugin-tracing.test.ts b/packages/stage-ui/src/libs/pinia/pinia-plugin-tracing.test.ts new file mode 100644 index 000000000..12f4c29ad --- /dev/null +++ b/packages/stage-ui/src/libs/pinia/pinia-plugin-tracing.test.ts @@ -0,0 +1,83 @@ +import type { PiniaActionEvent } from '@proj-airi/stage-shared/types/pinia-action-event' + +import { piniaActionTracingChannelName } from '@proj-airi/stage-shared/types/pinia-action-event' +import { createPinia, defineStore } from 'pinia' +import { describe, expect, it } from 'vitest' +import { createApp } from 'vue' + +import { piniaPluginTracing } from './pinia-plugin-tracing' + +describe('piniaPluginTracing over BroadcastChannel', () => { + it('emits correlated start and completion events without action values', async () => { + const observer = observeActionEvents(2) + + const pinia = createPinia() + pinia.use(piniaPluginTracing) + createApp({}).use(pinia) + + const useStore = defineStore('action-event-test', { + actions: { + async complete(secret: string) { + return `returned:${secret}` + }, + }, + }) + + await useStore(pinia).complete('must-not-be-recorded') + const events = await observer.events + + observer.close() + expect(events).toHaveLength(2) + expect(events.map(event => event.status)).toEqual(['started', 'completed']) + expect(events[0]?.invocationId).toBe(events[1]?.invocationId) + for (const event of events) { + expect(event).toEqual(expect.objectContaining({ + actionName: 'complete', + storeId: 'action-event-test', + })) + } + expect(JSON.stringify(events)).not.toContain('must-not-be-recorded') + expect(JSON.stringify(events)).not.toContain('returned:') + }) + + it('emits a failed event without changing the action error', async () => { + const observer = observeActionEvents(2) + + const pinia = createPinia() + pinia.use(piniaPluginTracing) + createApp({}).use(pinia) + + const failure = new Error('provider unavailable') + const useStore = defineStore('failed-action-test', { + actions: { + fail() { + throw failure + }, + }, + }) + + expect(() => useStore(pinia).fail()).toThrow(failure) + const events = await observer.events + + observer.close() + expect(events.map(event => event.status)).toEqual(['started', 'failed']) + expect(events[1]?.errorMessage).toBe('provider unavailable') + }) +}) + +function observeActionEvents(count: number) { + const channel = new BroadcastChannel(piniaActionTracingChannelName) + const captured: PiniaActionEvent[] = [] + const events = new Promise((resolve) => { + channel.addEventListener('message', (message: MessageEvent) => { + captured.push(message.data) + if (captured.length === count) + resolve(captured) + }) + }) + + return { + close: () => channel.close(), + events, + } +} diff --git a/packages/stage-ui/src/libs/pinia/pinia-plugin-tracing.ts b/packages/stage-ui/src/libs/pinia/pinia-plugin-tracing.ts new file mode 100644 index 000000000..193604732 --- /dev/null +++ b/packages/stage-ui/src/libs/pinia/pinia-plugin-tracing.ts @@ -0,0 +1,41 @@ +import type { PiniaActionEvent, PiniaActionEventStatus } from '@proj-airi/stage-shared/types/pinia-action-event' +import type { PiniaPlugin } from 'pinia' + +import { errorMessageFrom } from '@moeru/std' +import { piniaActionTracingChannelName } from '@proj-airi/stage-shared/types/pinia-action-event' +import { nanoid } from 'nanoid/non-secure' + +const piniaActionChannel = new BroadcastChannel(piniaActionTracingChannelName) + +function emitActionEvent( + event: Omit, + status: PiniaActionEventStatus, + error?: unknown, +): void { + piniaActionChannel.postMessage({ + ...event, + status, + timestamp: Date.now(), + ...(status === 'failed' ? { errorMessage: errorMessageFrom(error) ?? 'Unknown action failure' } : {}), + }) +} + +/** + * Traces Pinia action lifecycle events through a broadcast channel. + * + * The plugin never retains action arguments, results, or state snapshots. + */ +export const piniaPluginTracing: PiniaPlugin = ({ store }) => { + store.$onAction(({ name, after, onError }) => { + const event = { + invocationId: nanoid(), + storeId: store.$id, + actionName: name, + ...(typeof location === 'undefined' ? {} : { sourceUrl: location.href }), + } + + emitActionEvent(event, 'started') + after(() => emitActionEvent(event, 'completed')) + onError(error => emitActionEvent(event, 'failed', error)) + }) +} diff --git a/packages/stage-ui/src/stores/chat.contract.test.ts b/packages/stage-ui/src/stores/chat.contract.test.ts index baddfa8b3..772c20109 100644 --- a/packages/stage-ui/src/stores/chat.contract.test.ts +++ b/packages/stage-ui/src/stores/chat.contract.test.ts @@ -1,7 +1,7 @@ import type { ChatProvider } from '@xsai-ext/providers/utils' import type { Message, Tool } from '@xsai/shared-chat' -import { IOSpanNames } from '@proj-airi/stage-shared' +import { IOAttributes, IOSpanNames } from '@proj-airi/stage-shared' import { createPinia, setActivePinia } from 'pinia' import { beforeEach, describe, expect, it, vi } from 'vitest' import { nextTick, ref } from 'vue' @@ -584,6 +584,20 @@ describe('chat store contract', () => { expect(composedMessages).toHaveLength(2) expect(composedMessages[0]).toMatchObject({ role: 'system' }) expect(composedMessages[1]).toMatchObject({ role: 'user' }) + expect(ioTracerMocks.startSpanMock).toHaveBeenCalledWith( + IOSpanNames.LLMInference, + expect.anything(), + expect.objectContaining({ + [IOAttributes.LLMInputMessageCount]: 2, + [IOAttributes.LLMInputUserMessageCount]: 1, + [IOAttributes.TurnId]: expect.any(String), + }), + ) + const llmSpan = ioTracerMocks.spans.find(span => span.name === IOSpanNames.LLMInference) + expect(llmSpan.setAttribute).toHaveBeenCalledWith(IOAttributes.LLMInputMessageRoles, ['system', 'user']) + expect(llmSpan.setAttribute).toHaveBeenCalledWith(IOAttributes.LLMOutputChunkCount, 1) + expect(llmSpan.setAttribute).toHaveBeenCalledWith(IOAttributes.LLMOutputChunkLengths, [5]) + expect(llmSpan.setAttribute).toHaveBeenCalledWith(IOAttributes.LLMTextLength, 5) // System message stays untouched: keeping it 100% static is what makes // the prefix permanently KV-cache friendly across turns and across day diff --git a/packages/stage-ui/src/stores/chat.ts b/packages/stage-ui/src/stores/chat.ts index 6e855db60..2dc8a9b52 100644 --- a/packages/stage-ui/src/stores/chat.ts +++ b/packages/stage-ui/src/stores/chat.ts @@ -171,6 +171,8 @@ export const useChatStore = defineStore('chat', () => { options?: StreamOptions, ) { let llmTextLength = 0 + let llmOutputChunkCount = 0 + const llmOutputChunkLengths: number[] = [] const headers = { ...options?.headers } if (getProviderMode(activeProvider.value) === 'official' && options?.requestCorrelation) { headers[AIRI_CHAT_SESSION_ID_HEADER] = options.requestCorrelation.conversationId @@ -188,7 +190,11 @@ export const useChatStore = defineStore('chat', () => { const llmSpan = startSpan(IOSpanNames.LLMInference, activeTurnSpan.value, { [IOAttributes.Subsystem]: IOSubsystems.LLM, [IOAttributes.GenAIRequestModel]: model, + [IOAttributes.LLMInputMessageCount]: messages.length, + [IOAttributes.LLMInputUserMessageCount]: messages.filter(message => message.role === 'user').length, + [IOAttributes.TurnId]: options?.requestCorrelation?.roundId ?? '', }) + llmSpan.setAttribute(IOAttributes.LLMInputMessageRoles, messages.map(message => message.role)) const llmRequestTs = performance.now() let llmFirstTokenEmitted = false @@ -198,6 +204,8 @@ export const useChatStore = defineStore('chat', () => { headers, onStreamEvent: async (event: StreamEvent) => { if (isTextDelta(event)) { + llmOutputChunkCount += 1 + llmOutputChunkLengths.push(event.text.length) if (!llmFirstTokenEmitted) { llmFirstTokenEmitted = true llmSpan.addEvent(IOEvents.LLMFirstToken, { @@ -210,10 +218,11 @@ export const useChatStore = defineStore('chat', () => { await options?.onStreamEvent?.(event) }, }) - - llmSpan.setAttribute(IOAttributes.LLMTextLength, llmTextLength) } finally { + llmSpan.setAttribute(IOAttributes.LLMOutputChunkCount, llmOutputChunkCount) + llmSpan.setAttribute(IOAttributes.LLMOutputChunkLengths, llmOutputChunkLengths) + llmSpan.setAttribute(IOAttributes.LLMTextLength, llmTextLength) llmSpan.end() } } diff --git a/packages/stage-ui/src/stores/modules/speech.ts b/packages/stage-ui/src/stores/modules/speech.ts index 95336ad34..c91b337eb 100644 --- a/packages/stage-ui/src/stores/modules/speech.ts +++ b/packages/stage-ui/src/stores/modules/speech.ts @@ -37,6 +37,12 @@ interface SpeechInput { providerConfig: Record } +interface SpeechAnalytics { + trigger: 'auto' | 'manual' + source: 'chat_auto_tts' | 'manual_preview' | 'settings_test' + voice_type?: 'official_default' | 'official_selected' | 'custom_configured' | 'voice_pack' +} + export const useSpeechStore = defineStore('speech', () => { const providersStore = useProviderStore() const providerStore = useProviderConfigStore() @@ -301,14 +307,15 @@ export const useSpeechStore = defineStore('speech', () => { input: string, voice: string, providerConfig: Record = {}, + analytics: SpeechAnalytics = { + trigger: 'manual', + source: 'manual_preview', + voice_type: resolveVoiceType(voice), + }, ): Promise { const requestProviderConfig = activeSpeechProvider.value === OFFICIAL_SPEECH_PROVIDER_ID || activeSpeechProvider.value === OFFICIAL_SPEECH_STREAMING_PROVIDER_ID - ? withAiriTtsAnalytics(providerConfig, { - trigger: 'manual', - source: 'manual_preview', - voice_type: resolveVoiceType(voice), - }) + ? withAiriTtsAnalytics(providerConfig, analytics) : providerConfig const response = await generateSpeech({ ...provider.speech(model, requestProviderConfig), @@ -321,11 +328,7 @@ export const useSpeechStore = defineStore('speech', () => { function withAiriTtsAnalytics( providerConfig: Record, - analytics: { - trigger: 'auto' | 'manual' - source: 'chat_auto_tts' | 'manual_preview' | 'settings_test' - voice_type?: 'official_default' | 'official_selected' | 'custom_configured' | 'voice_pack' - }, + analytics: SpeechAnalytics, ): Record { return { ...providerConfig, diff --git a/packages/testing-audio/cases/long-leading-silence/case.audio.test.ts b/packages/testing-audio/cases/long-leading-silence/case.audio.test.ts index 2e74415ed..77bf3a93a 100644 --- a/packages/testing-audio/cases/long-leading-silence/case.audio.test.ts +++ b/packages/testing-audio/cases/long-leading-silence/case.audio.test.ts @@ -43,6 +43,7 @@ describe('audio input pipeline', () => { await expect(audio).not.toHaveTranscriptions([ ['There is no meaning to your existence, just let go.'], ]) + await expect(audio).toHaveCompletedTranscription() }) it('does not preserve the complete phrase with Aliyun NLS', { @@ -78,5 +79,6 @@ describe('audio input pipeline', () => { await expect(audio).not.toHaveTranscriptions([ ['There is no meaning to your existence, just let go.'], ]) + await expect(audio).toHaveCompletedTranscription() }) }) diff --git a/packages/testing-audio/cases/single-utterance-pipeline/case.audio.test.ts b/packages/testing-audio/cases/single-utterance-pipeline/case.audio.test.ts index 839aff10d..98c25a732 100644 --- a/packages/testing-audio/cases/single-utterance-pipeline/case.audio.test.ts +++ b/packages/testing-audio/cases/single-utterance-pipeline/case.audio.test.ts @@ -1,12 +1,12 @@ import { describe, expect, it } from '../../src' import { configureModuleConsciousness, configureModuleHearing, configureModuleSpeech, configureOnboarding, loadCaseEnvironment } from '../shared/configurations' -import { assistantMessages, enableChatMicrophone, openChat } from '../shared/interactions' +import { enableChatMicrophone } from '../shared/interactions' import { aliyunNlsAsr, openaiAsr, openaiLlm, openaiTts } from '../shared/providers' describe('audio input pipeline', () => { // An OpenAI-compatible TTS Provider generated the fixture in mono 16 kHz PCM WAV format. // The fixture contains 14 seconds of leading silence for VAD initialization and 3 seconds of trailing silence. - // Its warm-up phrase gives the VAD time to start. Only "Please say hello." is required in the transcript. + // Its warm-up phrase gives the VAD time to start. The final "say hello" command is required in the transcript. it('runs an OpenAI-compatible request through the complete pipeline', { input: new URL('./input.test.wav', import.meta.url), // This case keeps AIRI's default VAD and selects every remote Provider explicitly. @@ -72,16 +72,22 @@ describe('audio input pipeline', () => { } await expect(audio).toHaveTranscriptions([ - ['Please say hello.'], + ['Please say hello.', 'Say hello.'], ], { match: 'contains' }) - await expect.poll(async () => (await audio.completedSpans('LLM inference')).length, { timeout: 60_000 }).toBeGreaterThanOrEqual(1) - await expect.poll(async () => (await audio.completedSpans('TTS synthesis')).length, { timeout: 60_000 }).toBeGreaterThanOrEqual(1) - await expect.poll(async () => (await audio.completedSpans('Audio playback')).length, { timeout: 60_000 }).toBeGreaterThanOrEqual(1) + const turn = await audio.waitForTurn() - await openChat(audio) - const messages = await assistantMessages(audio).allTextContents() - expect(messages.at(-1)).toMatch(/.+/s) + expect(turn.chat.messages).toHaveLength(2) + expect(turn.chat.messages.map(message => message.role)).toEqual(['user', 'assistant']) + expect(turn.llm.inputMessages).toHaveLength(2) + expect(turn.llm.inputMessages.filter(message => message.role === 'user')).toHaveLength(1) + expect(turn.llm.outputChunks.length).toBeGreaterThan(0) + expect(turn.llm.outputCharacters).toBeGreaterThan(0) + expect(turn.tts.audioSegments.length).toBeGreaterThan(0) + for (const segment of turn.tts.audioSegments) { + expect(segment.text.length).toBeGreaterThan(0) + expect(segment.durationMs).toBeGreaterThan(100) + } }) it('transcribes the greeting with Aliyun NLS', { @@ -117,5 +123,6 @@ describe('audio input pipeline', () => { await expect(audio).toHaveTranscriptions([ ['Please say hello.'], ], { match: 'contains' }) + await expect(audio).toHaveCompletedTranscription() }) }) diff --git a/packages/testing-audio/cases/two-utterance-streaming/case.audio.test.ts b/packages/testing-audio/cases/two-utterance-streaming/case.audio.test.ts index b4639e51d..6770f20be 100644 --- a/packages/testing-audio/cases/two-utterance-streaming/case.audio.test.ts +++ b/packages/testing-audio/cases/two-utterance-streaming/case.audio.test.ts @@ -54,6 +54,7 @@ describe('audio input pipeline', () => { const finalTranscriptions = await audio.transcriptionResults(expectedTranscriptions.length) const streamingUpdates = await audio.streamingTranscriptionUpdates() expect(streamingUpdates.some(update => !finalTranscriptions.includes(update))).toBe(true) + await expect(audio).toHaveCompletedTranscription() }) it('keeps two Aliyun NLS utterances in streaming transcription', { @@ -101,5 +102,6 @@ describe('audio input pipeline', () => { const finalTranscriptions = await audio.transcriptionResults(expectedTranscriptions.length) const streamingUpdates = await audio.streamingTranscriptionUpdates() expect(streamingUpdates.some(update => !finalTranscriptions.includes(update))).toBe(true) + await expect(audio).toHaveCompletedTranscription() }) }) diff --git a/packages/testing-audio/src/expect-extend.test.ts b/packages/testing-audio/src/expect-extend.test.ts index 38e72d812..ba67c1034 100644 --- a/packages/testing-audio/src/expect-extend.test.ts +++ b/packages/testing-audio/src/expect-extend.test.ts @@ -1,3 +1,5 @@ +import type { PiniaActionEvent } from '@proj-airi/stage-shared/types/pinia-action-event' + import type { AudioInputObservations } from './types' import { describe, it } from 'vitest' @@ -14,15 +16,45 @@ describe('audio input matchers', () => { ['please say hello'], ]) }) + + it('reports a failed transcription action', async () => { + const session = createAudioInputSession([], [{ + ...actionEvent('modules:hearing:speech:audio-input-pipeline', 'transcribeForRecording'), + errorMessage: 'Request failed', + status: 'failed', + }]) + + await expect(expect(session).toHaveCompletedTranscription()).rejects.toThrow('ASR failed: Request failed') + }) }) -function createAudioInputSession(transcriptions: string[]): AudioInputObservations { +function createAudioInputSession( + transcriptions: string[], + actions: PiniaActionEvent[] = [], +): AudioInputObservations { return { capturedTranscriptionAudio: async () => [], streamingTranscriptionUpdates: async () => [], transcriptionResults: async () => transcriptions, completedSpans: async () => [], + piniaActionEvents: async () => actions, + waitForPiniaAction: async () => { + throw new Error('This matcher fixture does not observe Pinia actions.') + }, waitForStreamingTranscriptionReady: async () => {}, + waitForTurn: async () => { + throw new Error('This matcher fixture does not observe completed turns.') + }, waitForVadReady: async () => {}, } } + +function actionEvent(storeId: string, actionName: string): PiniaActionEvent { + return { + actionName, + invocationId: `${storeId}:${actionName}`, + status: 'completed', + storeId, + timestamp: 0, + } +} diff --git a/packages/testing-audio/src/expect-extend.ts b/packages/testing-audio/src/expect-extend.ts index 9814f6391..6a1c1fdc1 100644 --- a/packages/testing-audio/src/expect-extend.ts +++ b/packages/testing-audio/src/expect-extend.ts @@ -12,6 +12,12 @@ export interface TranscriptionExpectationOptions { match?: 'exact' | 'contains' } +/** Sets the wait limit for a transcription action assertion. */ +export interface TranscriptionActionExpectationOptions { + /** Maximum time to wait for a completed ASR action. @default 60000 */ + timeout?: number +} + declare module 'vitest' { interface Assertion { toHaveCapturedTranscriptionAudio: T extends AudioInputObservations @@ -23,6 +29,9 @@ declare module 'vitest' { options?: TranscriptionExpectationOptions, ) => Promise : never + toHaveCompletedTranscription: T extends AudioInputObservations + ? (options?: TranscriptionActionExpectationOptions) => Promise + : never } } @@ -30,13 +39,13 @@ declare module 'vitest' { export const expect = vitestExpect /** - * Normalizes a transcript for speech-recognition comparison. + * Normalizes speech text for transcript comparison. * * @example - * normalizeTranscript(' Hello, AIRI! ') + * normalizeSpeechText(' Hello, AIRI! ') * // => 'helloairi' */ -function normalizeTranscript(value: string): string { +function normalizeSpeechText(value: string): string { return value .normalize('NFKC') .toLocaleLowerCase() @@ -79,8 +88,8 @@ export function installAudioInputMatchers(): void { options: TranscriptionExpectationOptions = {}, ) { const actual = await session.transcriptionResults(expected.length) - const normalizedActual = actual.map(normalizeTranscript) - const normalizedExpected = expected.map(alternatives => alternatives.map(normalizeTranscript)) + const normalizedActual = actual.map(normalizeSpeechText) + const normalizedExpected = expected.map(alternatives => alternatives.map(normalizeSpeechText)) const match = options.match ?? 'exact' const pass = normalizedActual.length === normalizedExpected.length && normalizedActual.every((transcript, index) => ( @@ -96,5 +105,57 @@ export function installAudioInputMatchers(): void { : `Expected transcriptions ${JSON.stringify(expected)}, but received ${JSON.stringify(actual)}.`, } }, + async toHaveCompletedTranscription( + session: AudioInputObservations, + options: TranscriptionActionExpectationOptions = {}, + ) { + const result = await waitForTranscription(session, options.timeout ?? 60_000) + return { + pass: result.complete, + message: () => result.complete + ? 'Expected the transcription action not to complete.' + : `Expected the transcription action to complete, but ${result.summary}.`, + } + }, }) } + +const transcriptionActions = [ + { storeId: 'modules:hearing:speech:audio-input-pipeline', actionName: 'transcribeForRecording' }, + { storeId: 'modules:hearing:speech:audio-input-pipeline', actionName: 'transcribeForMediaStream' }, +] + +async function waitForTranscription( + session: AudioInputObservations, + timeout: number, +): Promise<{ complete: boolean, failed: boolean, summary: string }> { + const deadline = Date.now() + timeout + let result = transcriptionResult(await session.piniaActionEvents()) + + while (!result.complete && !result.failed && Date.now() < deadline) { + await new Promise(resolve => setTimeout(resolve, 100)) + result = transcriptionResult(await session.piniaActionEvents()) + } + return result +} + +function transcriptionResult( + events: Awaited>, +): { complete: boolean, failed: boolean, summary: string } { + const matchingEvents = events.filter(event => transcriptionActions.some(action => ( + event.storeId === action.storeId && event.actionName === action.actionName + ))) + const latestTerminalEvent = matchingEvents.findLast(event => event.status !== 'started') + if (latestTerminalEvent?.status === 'failed') { + return { + complete: false, + failed: true, + summary: `ASR failed${latestTerminalEvent.errorMessage ? `: ${latestTerminalEvent.errorMessage}` : ''}`, + } + } + return { + complete: latestTerminalEvent?.status === 'completed', + failed: false, + summary: 'ASR did not complete', + } +} diff --git a/packages/testing-audio/src/index.ts b/packages/testing-audio/src/index.ts index b2619c5eb..6cd8d01d2 100644 --- a/packages/testing-audio/src/index.ts +++ b/packages/testing-audio/src/index.ts @@ -1,10 +1,18 @@ export { describe, expect, it } from './describe' +export type { TranscriptionActionExpectationOptions } from './expect-extend' export type { + AudioInputChatMessage, + AudioInputLLMMessage, + AudioInputLLMOutputChunk, AudioInputObservations, AudioInputPreflightCallback, AudioInputPreflightContext, AudioInputSession, AudioInputTarget, AudioInputTestCase, + AudioInputTTSSegment, + AudioInputTurn, } from './types' + +export type { PiniaActionEvent, PiniaActionEventStatus } from '@proj-airi/stage-shared/types/pinia-action-event' diff --git a/packages/testing-audio/src/runtimes/prepare-electron.ts b/packages/testing-audio/src/runtimes/prepare-electron.ts index 9195d65b5..a7bea5286 100644 --- a/packages/testing-audio/src/runtimes/prepare-electron.ts +++ b/packages/testing-audio/src/runtimes/prepare-electron.ts @@ -3,18 +3,20 @@ import type { Page } from 'playwright' import type { AudioInputSession } from '../types' +import { piniaActionTracingChannelName } from '@proj-airi/stage-shared/types/pinia-action-event' + import { stubForBrowser } from '../setup/browser-probe' import { createSession } from '../setup/session' /** Adapts a Fakemic Electron process into an AIRI desktop audio session. */ export default async function prepareElectronRuntime(context: FakemicElectronPrepareContext): Promise { - await context.app.context().addInitScript(stubForBrowser) + await context.app.context().addInitScript(stubForBrowser, piniaActionTracingChannelName) const page = await waitForPage(context, (page) => { const url = new URL(page.url()) return url.pathname.endsWith('/index.html') && url.hash === '#/' }) await page.locator('[i-solar\\:alt-arrow-up-line-duotone]').first().waitFor({ state: 'visible', timeout: 30_000 }) - await page.evaluate(stubForBrowser) + await page.evaluate(stubForBrowser, piniaActionTracingChannelName) return createSession({ electronApp: context.app, diff --git a/packages/testing-audio/src/runtimes/prepare-web.ts b/packages/testing-audio/src/runtimes/prepare-web.ts index b8dea08b0..4298832c3 100644 --- a/packages/testing-audio/src/runtimes/prepare-web.ts +++ b/packages/testing-audio/src/runtimes/prepare-web.ts @@ -2,12 +2,14 @@ import type { FakemicWebPrepareContext } from '@proj-airi/vitest-plugin-fakemic' import type { AudioInputSession } from '../types' +import { piniaActionTracingChannelName } from '@proj-airi/stage-shared/types/pinia-action-event' + import { stubForBrowser } from '../setup/browser-probe' import { createSession } from '../setup/session' /** Adapts a Fakemic Chromium process into an AIRI Web audio session. */ export default async function prepareWebRuntime(context: FakemicWebPrepareContext): Promise { - await context.context.addInitScript(stubForBrowser) + await context.context.addInitScript(stubForBrowser, piniaActionTracingChannelName) const page = await context.context.newPage() await page.goto(context.runtime.url) diff --git a/packages/testing-audio/src/setup/browser-probe.ts b/packages/testing-audio/src/setup/browser-probe.ts index 4dee09346..e473bb21b 100644 --- a/packages/testing-audio/src/setup/browser-probe.ts +++ b/packages/testing-audio/src/setup/browser-probe.ts @@ -1,4 +1,5 @@ import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace' +import type { PiniaActionEvent } from '@proj-airi/stage-shared/types/pinia-action-event' /** * Installs passive browser probes before the application starts. @@ -8,7 +9,8 @@ import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace' * `BrowserContext.addInitScript` * -> {@link stubForBrowser} * -> `window.fetch` - * -> `BroadcastChannel('io-tracer-channel')` + * -> Pinia action tracing channel + * -> I/O tracing channel * * Upstream: * - `BrowserContext.addInitScript` @@ -16,10 +18,15 @@ import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace' * Downstream: * - `window.__airiAudioInputE2E` */ -export function stubForBrowser() { - const state: BrowserAudioInputState = { spans: [], streamingTranscriptionReady: false, streamingTranscriptionUpdates: [], transcriptionAudio: [], transcriptionResults: [], vadReady: false } +export function stubForBrowser(piniaActionChannelName: string) { + const state: BrowserAudioInputState = { piniaActionEvents: [], spans: [], streamingTranscriptionReady: false, streamingTranscriptionUpdates: [], transcriptionAudio: [], transcriptionResults: [], vadReady: false } window.__airiAudioInputE2E = state + const piniaActionChannel = new BroadcastChannel(piniaActionChannelName) + piniaActionChannel.addEventListener('message', (message: MessageEvent) => { + state.piniaActionEvents.push(message.data) + }) + const originalConsoleInfo = console.info.bind(console) console.info = (...values: unknown[]) => { if (typeof values[0] === 'string' && values[0].startsWith('[Voice Input] vad-ready:')) diff --git a/packages/testing-audio/src/setup/session.ts b/packages/testing-audio/src/setup/session.ts index 90481953e..d0c98a82c 100644 --- a/packages/testing-audio/src/setup/session.ts +++ b/packages/testing-audio/src/setup/session.ts @@ -1,13 +1,17 @@ +import type { PiniaActionEvent } from '@proj-airi/stage-shared/types/pinia-action-event' import type { AudioCapture } from '@proj-airi/vitest-plugin-fakemic' import type { ElectronApplication, Page } from 'playwright' -import type { AudioInputSession, AudioInputTarget } from '../types' +import type { AudioInputChatMessage, AudioInputSession, AudioInputTarget, AudioInputTurn } from '../types' import { Buffer } from 'node:buffer' +import { IOAttributes, IOSpanNames } from '@proj-airi/stage-shared/perf/io-trace' +import { piniaActionTracingChannelName } from '@proj-airi/stage-shared/types/pinia-action-event' + import { readCompletedSpans } from './browser-probe' -/** Creates the runtime session and records its page diagnostics. */ +/** Creates the runtime session for one audio test. */ export function createSession(options: { electronApp?: ElectronApplication page: Page @@ -15,26 +19,6 @@ export function createSession(options: { close: () => Promise transcriptionCaptureFormat?: AudioCapture['format'] }): AudioInputSession { - const diagnostics: string[] = [] - const observedPages = new WeakSet() - - function observePage(page: Page) { - if (observedPages.has(page)) - return - - observedPages.add(page) - page.on('console', (message) => { - const text = message.text() - const isAudioPipelineInfo = message.type() === 'info' - && (text.includes('[Hearing Pipeline]') || text.includes('[Voice Input]') || text.includes('transcription')) - if (['error', 'warning'].includes(message.type()) || isAudioPipelineInfo) - diagnostics.push(`[console:${message.type()}] ${text}`) - }) - page.on('pageerror', error => diagnostics.push(`[pageerror] ${error.message}`)) - } - - observePage(options.page) - const session: AudioInputSession = { electronApp: options.electronApp, page: options.page, @@ -42,7 +26,6 @@ export function createSession(options: { target: options.target, transcriptionCaptureFormat: options.transcriptionCaptureFormat, activatePage(page) { - observePage(page) session.page = page }, async capturedTranscriptionAudio(count) { @@ -63,12 +46,13 @@ export function createSession(options: { microphoneEnabled: localStorage.getItem('settings/audio/input/enabled'), microphoneInput: localStorage.getItem('settings/audio/input'), microphoneOffIconVisible: Boolean(document.querySelector('[i-ph\\:microphone-slash]')), + piniaActionEvents: window.__airiAudioInputE2E?.piniaActionEvents ?? [], probeInstalled: Boolean(window.__airiAudioInputE2E), streamingTranscriptionReady: window.__airiAudioInputE2E?.streamingTranscriptionReady ?? false, url: window.location.href, vadReady: window.__airiAudioInputE2E?.vadReady ?? false, })) - throw new Error(`Timed out waiting for captured transcription audio: ${JSON.stringify({ diagnostics, runtimeState })}`, { cause: error }) + throw new Error(`Timed out waiting for captured transcription audio: ${JSON.stringify(runtimeState)}`, { cause: error }) } const capturedAudio = await options.page.evaluate(() => window.__airiAudioInputE2E?.transcriptionAudio ?? []) return capturedAudio.map(audio => ({ @@ -91,22 +75,64 @@ export function createSession(options: { const interactionSpans = await session.page.evaluate(readCompletedSpans, name) return [...runtimeSpans, ...interactionSpans] }, + async waitForTurn(waitOptions = {}) { + const timeout = waitOptions.timeout ?? 60_000 + const deadline = Date.now() + timeout + let turn = createTurnObservation(await session.completedSpans()) + + while (!turn && Date.now() < deadline) { + await new Promise(resolve => setTimeout(resolve, 100)) + turn = createTurnObservation(await session.completedSpans()) + } + + if (!turn) + throw new Error('Timed out waiting for a completed LLM and speech turn.') + + turn.chat.messages = await readChatMessages(session.page) + return turn + }, + piniaActionEvents: () => options.page.evaluate(() => window.__airiAudioInputE2E?.piniaActionEvents ?? []), async waitForVadReady() { await options.page.waitForFunction(() => window.__airiAudioInputE2E?.vadReady === true, undefined, { timeout: 30_000 }) }, async waitForStreamingTranscriptionReady() { await options.page.waitForFunction(() => window.__airiAudioInputE2E?.streamingTranscriptionReady === true, undefined, { timeout: 30_000 }) }, + async waitForPiniaAction(waitOptions) { + return options.page.evaluate(({ actionName, channelName, status, storeId, timeout }) => new Promise((resolve, reject) => { + const channel = new BroadcastChannel(channelName) + const timeoutId = window.setTimeout(() => { + channel.close() + reject(new Error(`Timed out waiting for Pinia action ${storeId}.${actionName} (${status})`)) + }, timeout) + + channel.addEventListener('message', (message: MessageEvent) => { + const action = message.data + if (action.storeId !== storeId || action.actionName !== actionName || action.status !== status) + return + + window.clearTimeout(timeoutId) + channel.close() + resolve(action) + }) + }), { + actionName: waitOptions.actionName, + channelName: piniaActionTracingChannelName, + status: waitOptions.status ?? 'completed', + storeId: waitOptions.storeId, + timeout: waitOptions.timeout ?? 60_000, + }) + }, async snapshot() { const runtimeState = await options.page.evaluate(() => window.__airiAudioInputE2E) const interactionState = session.page === options.page ? runtimeState : await session.page.evaluate(() => window.__airiAudioInputE2E) return { + piniaActionEvents: runtimeState?.piniaActionEvents ?? [], spans: runtimeState?.spans ?? [], streamingTranscriptionUpdates: interactionState?.streamingTranscriptionUpdates ?? [], transcriptionResults: runtimeState?.transcriptionResults ?? [], - diagnostics: [...diagnostics], } }, close: options.close, @@ -114,3 +140,71 @@ export function createSession(options: { return session } + +function createTurnObservation(spans: Awaited>): AudioInputTurn | undefined { + const speechTurnSpan = spans.findLast(span => span.name === IOSpanNames.SpeechTurn) + if (!speechTurnSpan) + return undefined + + const turnId = stringAttribute(speechTurnSpan, IOAttributes.TurnId) + const llmSpan = spans.findLast(span => ( + span.name === IOSpanNames.LLMInference + && stringAttribute(span, IOAttributes.TurnId) === turnId + )) + if (!turnId || !llmSpan) + return undefined + + const inputMessageRoles = stringArrayAttribute(llmSpan, IOAttributes.LLMInputMessageRoles) + const outputChunkLengths = numberArrayAttribute(llmSpan, IOAttributes.LLMOutputChunkLengths) + const audioSegments = spans + .filter(span => ( + span.name === IOSpanNames.TTSSynthesis + && stringAttribute(span, IOAttributes.TurnId) === turnId + )) + .toSorted((left, right) => Number(left.startTimeNano) - Number(right.startTimeNano)) + .map(span => ({ + durationMs: numberAttribute(span, IOAttributes.TTSAudioDurationMs), + text: stringAttribute(span, IOAttributes.TTSText), + })) + + return { + id: turnId, + chat: { messages: [] }, + llm: { + inputMessages: inputMessageRoles.map(role => ({ role })), + outputCharacters: numberAttribute(llmSpan, IOAttributes.LLMTextLength), + outputChunks: outputChunkLengths.map(characters => ({ characters })), + }, + tts: { audioSegments }, + } +} + +async function readChatMessages(page: Page): Promise { + return page.locator('[data-chat-message-role]').evaluateAll(elements => elements.flatMap((element) => { + const role = element.getAttribute('data-chat-message-role') + const text = element.textContent?.trim() ?? '' + if ((role !== 'assistant' && role !== 'user') || !text) + return [] + return [{ role, text }] + })) +} + +function numberAttribute(span: Awaited>[number], name: string): number { + const value = span.attributes[name] + return typeof value === 'number' ? value : 0 +} + +function numberArrayAttribute(span: Awaited>[number], name: string): number[] { + const value = span.attributes[name] + return Array.isArray(value) ? value.filter(item => typeof item === 'number') : [] +} + +function stringAttribute(span: Awaited>[number], name: string): string { + const value = span.attributes[name] + return typeof value === 'string' ? value : '' +} + +function stringArrayAttribute(span: Awaited>[number], name: string): string[] { + const value = span.attributes[name] + return Array.isArray(value) ? value.filter(item => typeof item === 'string') : [] +} diff --git a/packages/testing-audio/src/types.ts b/packages/testing-audio/src/types.ts index 44ae685ba..247b20c08 100644 --- a/packages/testing-audio/src/types.ts +++ b/packages/testing-audio/src/types.ts @@ -1,4 +1,5 @@ import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace' +import type { PiniaActionEvent, PiniaActionEventStatus } from '@proj-airi/stage-shared/types/pinia-action-event' import type { AudioCapture, AudioCaptureFormat, AudioTestCase, AudioTestPreflightCallback, AudioTestSession } from '@proj-airi/vitest-plugin-fakemic' import type { ElectronApplication, Page } from 'playwright' @@ -24,10 +25,48 @@ export type AudioInputTestCase = AudioTestCase /** Snapshot of the observable AIRI audio pipeline state. */ export interface AudioInputSnapshot { + piniaActionEvents: PiniaActionEvent[] spans: SerializedIOSpan[] streamingTranscriptionUpdates: string[] transcriptionResults: string[] - diagnostics: string[] +} + +/** One rendered chat message from the completed turn. */ +export interface AudioInputChatMessage { + role: 'assistant' | 'user' + text: string +} + +/** One LLM input message represented without its potentially sensitive content. */ +export interface AudioInputLLMMessage { + role: string +} + +/** One text chunk emitted by the LLM stream. */ +export interface AudioInputLLMOutputChunk { + characters: number +} + +/** One synthesized audio segment routed from the LLM turn. */ +export interface AudioInputTTSSegment { + durationMs: number + text: string +} + +/** Structured observations for one completed voice interaction turn. */ +export interface AudioInputTurn { + id: string + chat: { + messages: AudioInputChatMessage[] + } + llm: { + inputMessages: AudioInputLLMMessage[] + outputCharacters: number + outputChunks: AudioInputLLMOutputChunk[] + } + tts: { + audioSegments: AudioInputTTSSegment[] + } } /** Observable audio values used by AIRI matchers. */ @@ -38,10 +77,36 @@ export interface AudioInputObservations { streamingTranscriptionUpdates: () => Promise transcriptionResults: (count: number) => Promise completedSpans: (name?: string) => Promise + /** Waits until all speech work for the latest LLM turn is complete. */ + waitForTurn: (options?: { + /** @default 60000 */ + timeout?: number + }) => Promise + /** Returns the Pinia action events collected by the runtime probe. */ + piniaActionEvents: () => Promise /** Waits until the VAD audio graph is connected to the microphone stream. */ waitForVadReady: () => Promise /** Waits until a streaming transcription transport accepts microphone audio. */ waitForStreamingTranscriptionReady: () => Promise + /** + * Waits for the next matching Pinia action event after this method is called. + * + * @example + * const completed = audio.waitForPiniaAction({ + * storeId: 'modules:hearing:speech:audio-input-pipeline', + * actionName: 'transcribeForMediaStream', + * }) + * await enableButton.click() + * await completed + */ + waitForPiniaAction: (options: { + storeId: string + actionName: string + /** @default 'completed' */ + status?: PiniaActionEventStatus + /** @default 60000 */ + timeout?: number + }) => Promise } /** Runtime handle for one AIRI audio-input test. */ diff --git a/packages/testing-audio/src/types/browser.d.ts b/packages/testing-audio/src/types/browser.d.ts index 4111bcdaa..712a1658b 100644 --- a/packages/testing-audio/src/types/browser.d.ts +++ b/packages/testing-audio/src/types/browser.d.ts @@ -1,7 +1,9 @@ import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace' +import type { PiniaActionEvent } from '@proj-airi/stage-shared/types/pinia-action-event' declare global { interface BrowserAudioInputState { + piniaActionEvents: PiniaActionEvent[] spans: SerializedIOSpan[] streamingTranscriptionReady: boolean streamingTranscriptionUpdates: string[]