refactor(testing-audio,stage-ui): added pinia-plugin-tracing, directly assert from history of pinia

This commit is contained in:
Neko Ayaka
2026-08-14 03:22:32 +08:00
parent d4fe2bef59
commit addfec1981
25 changed files with 564 additions and 81 deletions
+2 -1
View File
@@ -7,7 +7,7 @@ import { autoAnimatePlugin } from '@formkit/auto-animate/vue'
import { PiniaColada } from '@pinia/colada'
import { trackButtonPlugin } from '@proj-airi/stage-ui/directives/track-button'
import { configureAnalyticsAdapter } from '@proj-airi/stage-ui/libs/analytics'
import { setupSynced } from '@proj-airi/stage-ui/libs/pinia'
import { piniaPluginTracing, setupSynced } from '@proj-airi/stage-ui/libs/pinia'
import { MotionPlugin } from '@vueuse/motion'
import { createPinia } from 'pinia'
import { setupLayouts } from 'virtual:generated-layouts'
@@ -47,6 +47,7 @@ configureAnalyticsAdapter(async (options) => {
const pinia = createPinia()
const synced = setupSynced()
pinia.use(synced.pinia)
pinia.use(piniaPluginTracing)
const router = createRouter({
history: createWebHashHistory(),
+2 -1
View File
@@ -9,7 +9,7 @@ import { PiniaColada } from '@pinia/colada'
import { isEnvTruthy } from '@proj-airi/stage-shared'
import { trackButtonPlugin } from '@proj-airi/stage-ui/directives/track-button'
import { configureAnalyticsAdapter } from '@proj-airi/stage-ui/libs/analytics'
import { setupSynced } from '@proj-airi/stage-ui/libs/pinia'
import { piniaPluginTracing, setupSynced } from '@proj-airi/stage-ui/libs/pinia'
import { MotionPlugin } from '@vueuse/motion'
import { createPinia } from 'pinia'
import { setupLayouts } from 'virtual:generated-layouts'
@@ -37,6 +37,7 @@ configureAnalyticsAdapter(async (options) => {
const pinia = createPinia()
const synced = setupSynced()
pinia.use(synced.pinia)
pinia.use(piniaPluginTracing)
// TODO: vite-plugin-vue-layouts is long deprecated, replace with another layout solution
const routeRecords = setupLayouts(routes as RouteRecordRaw[])
+2
View File
@@ -20,8 +20,10 @@
"./beat-sync": "./src/beat-sync/index.ts",
"./global-shortcut": "./src/global-shortcut/index.ts",
"./godot-stage": "./src/godot-stage/index.ts",
"./perf/io-trace": "./src/perf/io-trace.ts",
"./server-channel-qr": "./src/server-channel-qr.ts",
"./types/io-trace": "./src/types/io-trace.ts",
"./types/pinia-action-event": "./src/types/pinia-action-event.ts",
"./electron-renderer": "./src/electron-renderer.d.ts",
"./composables": "./src/composables/index.ts",
"./webgpu": "./src/webgpu/index.ts"
@@ -10,6 +10,7 @@ export type IOSubsystem = (typeof IOSubsystems)[keyof typeof IOSubsystems]
export const IOSpanNames = {
InteractionTurn: 'Interaction turn',
SpeechTurn: 'Speech turn',
VoiceActivityDetection: 'Voice activity detection',
SpeechRecognition: 'Speech recognition',
LLMInference: 'LLM inference',
@@ -32,6 +33,11 @@ export const IOAttributes = {
VADAborted: `${customPrefix}.vad.aborted`,
ASRText: `${customPrefix}.asr.text`,
ASRAbort: `${customPrefix}.asr.abort`,
LLMInputMessageCount: `${customPrefix}.llm.input_message_count`,
LLMInputMessageRoles: `${customPrefix}.llm.input_message_roles`,
LLMInputUserMessageCount: `${customPrefix}.llm.input_user_message_count`,
LLMOutputChunkCount: `${customPrefix}.llm.output_chunk_count`,
LLMOutputChunkLengths: `${customPrefix}.llm.output_chunk_lengths`,
LLMTextLength: `${customPrefix}.llm.text_length`,
StreamingControlCallName: `${customPrefix}.streaming_control.call_name`,
StreamingControlHandlerCount: `${customPrefix}.streaming_control.handler_count`,
@@ -47,9 +53,11 @@ export const IOAttributes = {
TTSSegmentId: `${customPrefix}.tts.segment_id`,
TTSText: `${customPrefix}.tts.text`,
TTSChunkReason: `${customPrefix}.tts.chunk_reason`,
TTSAudioDurationMs: `${customPrefix}.tts.audio_duration_ms`,
TTSInterrupted: `${customPrefix}.tts.interrupted`,
TTSInterruptReason: `${customPrefix}.tts.interrupt_reason`,
TTSCanceled: `${customPrefix}.tts.canceled`,
TurnId: `${customPrefix}.turn_id`,
} as const
export const IOEvents = {
@@ -0,0 +1,21 @@
/** Lifecycle phase observed for one Pinia action invocation. */
export type PiniaActionEventStatus = 'started' | 'completed' | 'failed'
/** Broadcast channel used by Pinia action tracing producers and consumers. */
export const piniaActionTracingChannelName = 'airi-pinia-action-tracing'
/** Test-safe metadata emitted for one Pinia action lifecycle transition. */
export interface PiniaActionEvent {
/** Correlates lifecycle events from the same action invocation. */
invocationId: string
/** Name passed to `defineStore`. */
storeId: string
/** Action property name reported by Pinia. */
actionName: string
status: PiniaActionEventStatus
timestamp: number
/** Renderer URL that invoked the action, when available. */
sourceUrl?: string
/** Safe error text included only for failed actions. */
errorMessage?: string
}
@@ -26,7 +26,6 @@ import { useBroadcastChannel } from '@vueuse/core'
// import { createTransformers } from '@xsai-transformers/embed'
// import embedWorkerURL from '@xsai-transformers/embed/worker?worker&url'
// import { embed } from '@xsai/embed'
import { generateSpeech } from '@xsai/generate-speech'
import { storeToRefs } from 'pinia'
import { computed, nextTick, onMounted, onUnmounted, ref, shallowRef, watch } from 'vue'
@@ -520,24 +519,18 @@ const speechPipeline = createSpeechPipeline<AudioBuffer>({
// Non-streaming providers only: synth via REST. Streaming provider
// was already early-returned above; it owns its own ws path opened
// in `onBeforeMessageComposed`.
const providerConfigWithAnalytics = activeSpeechProvider.value === OFFICIAL_SPEECH_PROVIDER_ID
? {
...speechRequest.providerConfig,
extraBody: {
...(speechRequest.providerConfig.extraBody as Record<string, unknown> | undefined),
airi_analytics: {
trigger: 'auto',
source: 'chat_auto_tts',
voice_type: resolveStageVoiceType(),
},
},
}
: speechRequest.providerConfig
const res = await generateSpeech({
...provider.speech(model, providerConfigWithAnalytics),
input: speechRequest.input,
voice: voice.id,
})
const res = await speechStore.speech(
provider,
model,
speechRequest.input,
voice.id,
speechRequest.providerConfig,
{
trigger: 'auto',
source: 'chat_auto_tts',
voice_type: resolveStageVoiceType(),
},
)
if (signal.aborted || !res || res.byteLength === 0)
return null
@@ -11,8 +11,25 @@ export function useIOTraceBridge(pipeline: ReturnType<typeof createSpeechPipelin
const synthesisSpans = new Map<string, Span>()
const playbackSpans = new Map<string, Span>()
const speechTurnSpans = new Map<string, Span>()
const segmentReasons = new Map<string, string>()
cleanupFns.push(pipeline.on('onTurnStart', (turnId) => {
speechTurnSpans.set(turnId, startSpan(IOSpanNames.SpeechTurn, activeTurnSpan.value, {
[IOAttributes.TurnId]: turnId,
}))
}))
cleanupFns.push(pipeline.on('onTurnEnd', (turnId) => {
speechTurnSpans.get(turnId)?.end()
speechTurnSpans.delete(turnId)
}))
cleanupFns.push(pipeline.on('onTurnCancel', ({ turnId }) => {
speechTurnSpans.get(turnId)?.end()
speechTurnSpans.delete(turnId)
}))
cleanupFns.push(pipeline.on('onSegment', (segment) => {
segmentReasons.set(segment.segmentId, segment.reason)
}))
@@ -23,6 +40,7 @@ export function useIOTraceBridge(pipeline: ReturnType<typeof createSpeechPipelin
[IOAttributes.TTSSegmentId]: request.segmentId,
[IOAttributes.TTSText]: request.text,
[IOAttributes.TTSChunkReason]: segmentReasons.get(request.segmentId) ?? '',
[IOAttributes.TurnId]: request.turnId ?? '',
})
segmentReasons.delete(request.segmentId)
synthesisSpans.set(request.segmentId, ttsSynthesisSpan)
@@ -31,6 +49,8 @@ export function useIOTraceBridge(pipeline: ReturnType<typeof createSpeechPipelin
cleanupFns.push(pipeline.on('onTtsResult', (result) => {
const span = synthesisSpans.get(result.segmentId)
if (span) {
if (result.audio instanceof AudioBuffer)
span.setAttribute(IOAttributes.TTSAudioDurationMs, result.audio.duration * 1000)
span.end()
synthesisSpans.delete(result.segmentId)
}
@@ -77,6 +97,8 @@ export function useIOTraceBridge(pipeline: ReturnType<typeof createSpeechPipelin
}))
onScopeDispose(() => {
for (const span of speechTurnSpans.values())
span.end()
for (const cleanup of cleanupFns)
cleanup()
})
@@ -1 +1,2 @@
export * from './pinia-plugin-tracing'
export * from './setup-synced'
@@ -0,0 +1,83 @@
import type { PiniaActionEvent } from '@proj-airi/stage-shared/types/pinia-action-event'
import { piniaActionTracingChannelName } from '@proj-airi/stage-shared/types/pinia-action-event'
import { createPinia, defineStore } from 'pinia'
import { describe, expect, it } from 'vitest'
import { createApp } from 'vue'
import { piniaPluginTracing } from './pinia-plugin-tracing'
describe('piniaPluginTracing over BroadcastChannel', () => {
it('emits correlated start and completion events without action values', async () => {
const observer = observeActionEvents(2)
const pinia = createPinia()
pinia.use(piniaPluginTracing)
createApp({}).use(pinia)
const useStore = defineStore('action-event-test', {
actions: {
async complete(secret: string) {
return `returned:${secret}`
},
},
})
await useStore(pinia).complete('must-not-be-recorded')
const events = await observer.events
observer.close()
expect(events).toHaveLength(2)
expect(events.map(event => event.status)).toEqual(['started', 'completed'])
expect(events[0]?.invocationId).toBe(events[1]?.invocationId)
for (const event of events) {
expect(event).toEqual(expect.objectContaining({
actionName: 'complete',
storeId: 'action-event-test',
}))
}
expect(JSON.stringify(events)).not.toContain('must-not-be-recorded')
expect(JSON.stringify(events)).not.toContain('returned:')
})
it('emits a failed event without changing the action error', async () => {
const observer = observeActionEvents(2)
const pinia = createPinia()
pinia.use(piniaPluginTracing)
createApp({}).use(pinia)
const failure = new Error('provider unavailable')
const useStore = defineStore('failed-action-test', {
actions: {
fail() {
throw failure
},
},
})
expect(() => useStore(pinia).fail()).toThrow(failure)
const events = await observer.events
observer.close()
expect(events.map(event => event.status)).toEqual(['started', 'failed'])
expect(events[1]?.errorMessage).toBe('provider unavailable')
})
})
function observeActionEvents(count: number) {
const channel = new BroadcastChannel(piniaActionTracingChannelName)
const captured: PiniaActionEvent[] = []
const events = new Promise<PiniaActionEvent[]>((resolve) => {
channel.addEventListener('message', (message: MessageEvent<PiniaActionEvent>) => {
captured.push(message.data)
if (captured.length === count)
resolve(captured)
})
})
return {
close: () => channel.close(),
events,
}
}
@@ -0,0 +1,41 @@
import type { PiniaActionEvent, PiniaActionEventStatus } from '@proj-airi/stage-shared/types/pinia-action-event'
import type { PiniaPlugin } from 'pinia'
import { errorMessageFrom } from '@moeru/std'
import { piniaActionTracingChannelName } from '@proj-airi/stage-shared/types/pinia-action-event'
import { nanoid } from 'nanoid/non-secure'
const piniaActionChannel = new BroadcastChannel(piniaActionTracingChannelName)
function emitActionEvent(
event: Omit<PiniaActionEvent, 'status' | 'timestamp'>,
status: PiniaActionEventStatus,
error?: unknown,
): void {
piniaActionChannel.postMessage({
...event,
status,
timestamp: Date.now(),
...(status === 'failed' ? { errorMessage: errorMessageFrom(error) ?? 'Unknown action failure' } : {}),
})
}
/**
* Traces Pinia action lifecycle events through a broadcast channel.
*
* The plugin never retains action arguments, results, or state snapshots.
*/
export const piniaPluginTracing: PiniaPlugin = ({ store }) => {
store.$onAction(({ name, after, onError }) => {
const event = {
invocationId: nanoid(),
storeId: store.$id,
actionName: name,
...(typeof location === 'undefined' ? {} : { sourceUrl: location.href }),
}
emitActionEvent(event, 'started')
after(() => emitActionEvent(event, 'completed'))
onError(error => emitActionEvent(event, 'failed', error))
})
}
@@ -1,7 +1,7 @@
import type { ChatProvider } from '@xsai-ext/providers/utils'
import type { Message, Tool } from '@xsai/shared-chat'
import { IOSpanNames } from '@proj-airi/stage-shared'
import { IOAttributes, IOSpanNames } from '@proj-airi/stage-shared'
import { createPinia, setActivePinia } from 'pinia'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { nextTick, ref } from 'vue'
@@ -584,6 +584,20 @@ describe('chat store contract', () => {
expect(composedMessages).toHaveLength(2)
expect(composedMessages[0]).toMatchObject({ role: 'system' })
expect(composedMessages[1]).toMatchObject({ role: 'user' })
expect(ioTracerMocks.startSpanMock).toHaveBeenCalledWith(
IOSpanNames.LLMInference,
expect.anything(),
expect.objectContaining({
[IOAttributes.LLMInputMessageCount]: 2,
[IOAttributes.LLMInputUserMessageCount]: 1,
[IOAttributes.TurnId]: expect.any(String),
}),
)
const llmSpan = ioTracerMocks.spans.find(span => span.name === IOSpanNames.LLMInference)
expect(llmSpan.setAttribute).toHaveBeenCalledWith(IOAttributes.LLMInputMessageRoles, ['system', 'user'])
expect(llmSpan.setAttribute).toHaveBeenCalledWith(IOAttributes.LLMOutputChunkCount, 1)
expect(llmSpan.setAttribute).toHaveBeenCalledWith(IOAttributes.LLMOutputChunkLengths, [5])
expect(llmSpan.setAttribute).toHaveBeenCalledWith(IOAttributes.LLMTextLength, 5)
// System message stays untouched: keeping it 100% static is what makes
// the prefix permanently KV-cache friendly across turns and across day
+11 -2
View File
@@ -171,6 +171,8 @@ export const useChatStore = defineStore('chat', () => {
options?: StreamOptions,
) {
let llmTextLength = 0
let llmOutputChunkCount = 0
const llmOutputChunkLengths: number[] = []
const headers = { ...options?.headers }
if (getProviderMode(activeProvider.value) === 'official' && options?.requestCorrelation) {
headers[AIRI_CHAT_SESSION_ID_HEADER] = options.requestCorrelation.conversationId
@@ -188,7 +190,11 @@ export const useChatStore = defineStore('chat', () => {
const llmSpan = startSpan(IOSpanNames.LLMInference, activeTurnSpan.value, {
[IOAttributes.Subsystem]: IOSubsystems.LLM,
[IOAttributes.GenAIRequestModel]: model,
[IOAttributes.LLMInputMessageCount]: messages.length,
[IOAttributes.LLMInputUserMessageCount]: messages.filter(message => message.role === 'user').length,
[IOAttributes.TurnId]: options?.requestCorrelation?.roundId ?? '',
})
llmSpan.setAttribute(IOAttributes.LLMInputMessageRoles, messages.map(message => message.role))
const llmRequestTs = performance.now()
let llmFirstTokenEmitted = false
@@ -198,6 +204,8 @@ export const useChatStore = defineStore('chat', () => {
headers,
onStreamEvent: async (event: StreamEvent) => {
if (isTextDelta(event)) {
llmOutputChunkCount += 1
llmOutputChunkLengths.push(event.text.length)
if (!llmFirstTokenEmitted) {
llmFirstTokenEmitted = true
llmSpan.addEvent(IOEvents.LLMFirstToken, {
@@ -210,10 +218,11 @@ export const useChatStore = defineStore('chat', () => {
await options?.onStreamEvent?.(event)
},
})
llmSpan.setAttribute(IOAttributes.LLMTextLength, llmTextLength)
}
finally {
llmSpan.setAttribute(IOAttributes.LLMOutputChunkCount, llmOutputChunkCount)
llmSpan.setAttribute(IOAttributes.LLMOutputChunkLengths, llmOutputChunkLengths)
llmSpan.setAttribute(IOAttributes.LLMTextLength, llmTextLength)
llmSpan.end()
}
}
+13 -10
View File
@@ -37,6 +37,12 @@ interface SpeechInput {
providerConfig: Record<string, unknown>
}
interface SpeechAnalytics {
trigger: 'auto' | 'manual'
source: 'chat_auto_tts' | 'manual_preview' | 'settings_test'
voice_type?: 'official_default' | 'official_selected' | 'custom_configured' | 'voice_pack'
}
export const useSpeechStore = defineStore('speech', () => {
const providersStore = useProviderStore()
const providerStore = useProviderConfigStore()
@@ -301,14 +307,15 @@ export const useSpeechStore = defineStore('speech', () => {
input: string,
voice: string,
providerConfig: Record<string, any> = {},
analytics: SpeechAnalytics = {
trigger: 'manual',
source: 'manual_preview',
voice_type: resolveVoiceType(voice),
},
): Promise<ArrayBuffer> {
const requestProviderConfig = activeSpeechProvider.value === OFFICIAL_SPEECH_PROVIDER_ID
|| activeSpeechProvider.value === OFFICIAL_SPEECH_STREAMING_PROVIDER_ID
? withAiriTtsAnalytics(providerConfig, {
trigger: 'manual',
source: 'manual_preview',
voice_type: resolveVoiceType(voice),
})
? withAiriTtsAnalytics(providerConfig, analytics)
: providerConfig
const response = await generateSpeech({
...provider.speech(model, requestProviderConfig),
@@ -321,11 +328,7 @@ export const useSpeechStore = defineStore('speech', () => {
function withAiriTtsAnalytics(
providerConfig: Record<string, any>,
analytics: {
trigger: 'auto' | 'manual'
source: 'chat_auto_tts' | 'manual_preview' | 'settings_test'
voice_type?: 'official_default' | 'official_selected' | 'custom_configured' | 'voice_pack'
},
analytics: SpeechAnalytics,
): Record<string, any> {
return {
...providerConfig,
@@ -43,6 +43,7 @@ describe('audio input pipeline', () => {
await expect(audio).not.toHaveTranscriptions([
['There is no meaning to your existence, just let go.'],
])
await expect(audio).toHaveCompletedTranscription()
})
it('does not preserve the complete phrase with Aliyun NLS', {
@@ -78,5 +79,6 @@ describe('audio input pipeline', () => {
await expect(audio).not.toHaveTranscriptions([
['There is no meaning to your existence, just let go.'],
])
await expect(audio).toHaveCompletedTranscription()
})
})
@@ -1,12 +1,12 @@
import { describe, expect, it } from '../../src'
import { configureModuleConsciousness, configureModuleHearing, configureModuleSpeech, configureOnboarding, loadCaseEnvironment } from '../shared/configurations'
import { assistantMessages, enableChatMicrophone, openChat } from '../shared/interactions'
import { enableChatMicrophone } from '../shared/interactions'
import { aliyunNlsAsr, openaiAsr, openaiLlm, openaiTts } from '../shared/providers'
describe('audio input pipeline', () => {
// An OpenAI-compatible TTS Provider generated the fixture in mono 16 kHz PCM WAV format.
// The fixture contains 14 seconds of leading silence for VAD initialization and 3 seconds of trailing silence.
// Its warm-up phrase gives the VAD time to start. Only "Please say hello." is required in the transcript.
// Its warm-up phrase gives the VAD time to start. The final "say hello" command is required in the transcript.
it('runs an OpenAI-compatible request through the complete pipeline', {
input: new URL('./input.test.wav', import.meta.url),
// This case keeps AIRI's default VAD and selects every remote Provider explicitly.
@@ -72,16 +72,22 @@ describe('audio input pipeline', () => {
}
await expect(audio).toHaveTranscriptions([
['Please say hello.'],
['Please say hello.', 'Say hello.'],
], { match: 'contains' })
await expect.poll(async () => (await audio.completedSpans('LLM inference')).length, { timeout: 60_000 }).toBeGreaterThanOrEqual(1)
await expect.poll(async () => (await audio.completedSpans('TTS synthesis')).length, { timeout: 60_000 }).toBeGreaterThanOrEqual(1)
await expect.poll(async () => (await audio.completedSpans('Audio playback')).length, { timeout: 60_000 }).toBeGreaterThanOrEqual(1)
const turn = await audio.waitForTurn()
await openChat(audio)
const messages = await assistantMessages(audio).allTextContents()
expect(messages.at(-1)).toMatch(/.+/s)
expect(turn.chat.messages).toHaveLength(2)
expect(turn.chat.messages.map(message => message.role)).toEqual(['user', 'assistant'])
expect(turn.llm.inputMessages).toHaveLength(2)
expect(turn.llm.inputMessages.filter(message => message.role === 'user')).toHaveLength(1)
expect(turn.llm.outputChunks.length).toBeGreaterThan(0)
expect(turn.llm.outputCharacters).toBeGreaterThan(0)
expect(turn.tts.audioSegments.length).toBeGreaterThan(0)
for (const segment of turn.tts.audioSegments) {
expect(segment.text.length).toBeGreaterThan(0)
expect(segment.durationMs).toBeGreaterThan(100)
}
})
it('transcribes the greeting with Aliyun NLS', {
@@ -117,5 +123,6 @@ describe('audio input pipeline', () => {
await expect(audio).toHaveTranscriptions([
['Please say hello.'],
], { match: 'contains' })
await expect(audio).toHaveCompletedTranscription()
})
})
@@ -54,6 +54,7 @@ describe('audio input pipeline', () => {
const finalTranscriptions = await audio.transcriptionResults(expectedTranscriptions.length)
const streamingUpdates = await audio.streamingTranscriptionUpdates()
expect(streamingUpdates.some(update => !finalTranscriptions.includes(update))).toBe(true)
await expect(audio).toHaveCompletedTranscription()
})
it('keeps two Aliyun NLS utterances in streaming transcription', {
@@ -101,5 +102,6 @@ describe('audio input pipeline', () => {
const finalTranscriptions = await audio.transcriptionResults(expectedTranscriptions.length)
const streamingUpdates = await audio.streamingTranscriptionUpdates()
expect(streamingUpdates.some(update => !finalTranscriptions.includes(update))).toBe(true)
await expect(audio).toHaveCompletedTranscription()
})
})
@@ -1,3 +1,5 @@
import type { PiniaActionEvent } from '@proj-airi/stage-shared/types/pinia-action-event'
import type { AudioInputObservations } from './types'
import { describe, it } from 'vitest'
@@ -14,15 +16,45 @@ describe('audio input matchers', () => {
['please say hello'],
])
})
it('reports a failed transcription action', async () => {
const session = createAudioInputSession([], [{
...actionEvent('modules:hearing:speech:audio-input-pipeline', 'transcribeForRecording'),
errorMessage: 'Request failed',
status: 'failed',
}])
await expect(expect(session).toHaveCompletedTranscription()).rejects.toThrow('ASR failed: Request failed')
})
})
function createAudioInputSession(transcriptions: string[]): AudioInputObservations {
function createAudioInputSession(
transcriptions: string[],
actions: PiniaActionEvent[] = [],
): AudioInputObservations {
return {
capturedTranscriptionAudio: async () => [],
streamingTranscriptionUpdates: async () => [],
transcriptionResults: async () => transcriptions,
completedSpans: async () => [],
piniaActionEvents: async () => actions,
waitForPiniaAction: async () => {
throw new Error('This matcher fixture does not observe Pinia actions.')
},
waitForStreamingTranscriptionReady: async () => {},
waitForTurn: async () => {
throw new Error('This matcher fixture does not observe completed turns.')
},
waitForVadReady: async () => {},
}
}
function actionEvent(storeId: string, actionName: string): PiniaActionEvent {
return {
actionName,
invocationId: `${storeId}:${actionName}`,
status: 'completed',
storeId,
timestamp: 0,
}
}
+66 -5
View File
@@ -12,6 +12,12 @@ export interface TranscriptionExpectationOptions {
match?: 'exact' | 'contains'
}
/** Sets the wait limit for a transcription action assertion. */
export interface TranscriptionActionExpectationOptions {
/** Maximum time to wait for a completed ASR action. @default 60000 */
timeout?: number
}
declare module 'vitest' {
interface Assertion<T> {
toHaveCapturedTranscriptionAudio: T extends AudioInputObservations
@@ -23,6 +29,9 @@ declare module 'vitest' {
options?: TranscriptionExpectationOptions,
) => Promise<void>
: never
toHaveCompletedTranscription: T extends AudioInputObservations
? (options?: TranscriptionActionExpectationOptions) => Promise<void>
: never
}
}
@@ -30,13 +39,13 @@ declare module 'vitest' {
export const expect = vitestExpect
/**
* Normalizes a transcript for speech-recognition comparison.
* Normalizes speech text for transcript comparison.
*
* @example
* normalizeTranscript(' Hello, AIRI! ')
* normalizeSpeechText(' Hello, AIRI! ')
* // => 'helloairi'
*/
function normalizeTranscript(value: string): string {
function normalizeSpeechText(value: string): string {
return value
.normalize('NFKC')
.toLocaleLowerCase()
@@ -79,8 +88,8 @@ export function installAudioInputMatchers(): void {
options: TranscriptionExpectationOptions = {},
) {
const actual = await session.transcriptionResults(expected.length)
const normalizedActual = actual.map(normalizeTranscript)
const normalizedExpected = expected.map(alternatives => alternatives.map(normalizeTranscript))
const normalizedActual = actual.map(normalizeSpeechText)
const normalizedExpected = expected.map(alternatives => alternatives.map(normalizeSpeechText))
const match = options.match ?? 'exact'
const pass = normalizedActual.length === normalizedExpected.length
&& normalizedActual.every((transcript, index) => (
@@ -96,5 +105,57 @@ export function installAudioInputMatchers(): void {
: `Expected transcriptions ${JSON.stringify(expected)}, but received ${JSON.stringify(actual)}.`,
}
},
async toHaveCompletedTranscription(
session: AudioInputObservations,
options: TranscriptionActionExpectationOptions = {},
) {
const result = await waitForTranscription(session, options.timeout ?? 60_000)
return {
pass: result.complete,
message: () => result.complete
? 'Expected the transcription action not to complete.'
: `Expected the transcription action to complete, but ${result.summary}.`,
}
},
})
}
const transcriptionActions = [
{ storeId: 'modules:hearing:speech:audio-input-pipeline', actionName: 'transcribeForRecording' },
{ storeId: 'modules:hearing:speech:audio-input-pipeline', actionName: 'transcribeForMediaStream' },
]
async function waitForTranscription(
session: AudioInputObservations,
timeout: number,
): Promise<{ complete: boolean, failed: boolean, summary: string }> {
const deadline = Date.now() + timeout
let result = transcriptionResult(await session.piniaActionEvents())
while (!result.complete && !result.failed && Date.now() < deadline) {
await new Promise(resolve => setTimeout(resolve, 100))
result = transcriptionResult(await session.piniaActionEvents())
}
return result
}
function transcriptionResult(
events: Awaited<ReturnType<AudioInputObservations['piniaActionEvents']>>,
): { complete: boolean, failed: boolean, summary: string } {
const matchingEvents = events.filter(event => transcriptionActions.some(action => (
event.storeId === action.storeId && event.actionName === action.actionName
)))
const latestTerminalEvent = matchingEvents.findLast(event => event.status !== 'started')
if (latestTerminalEvent?.status === 'failed') {
return {
complete: false,
failed: true,
summary: `ASR failed${latestTerminalEvent.errorMessage ? `: ${latestTerminalEvent.errorMessage}` : ''}`,
}
}
return {
complete: latestTerminalEvent?.status === 'completed',
failed: false,
summary: 'ASR did not complete',
}
}
+8
View File
@@ -1,10 +1,18 @@
export { describe, expect, it } from './describe'
export type { TranscriptionActionExpectationOptions } from './expect-extend'
export type {
AudioInputChatMessage,
AudioInputLLMMessage,
AudioInputLLMOutputChunk,
AudioInputObservations,
AudioInputPreflightCallback,
AudioInputPreflightContext,
AudioInputSession,
AudioInputTarget,
AudioInputTestCase,
AudioInputTTSSegment,
AudioInputTurn,
} from './types'
export type { PiniaActionEvent, PiniaActionEventStatus } from '@proj-airi/stage-shared/types/pinia-action-event'
@@ -3,18 +3,20 @@ import type { Page } from 'playwright'
import type { AudioInputSession } from '../types'
import { piniaActionTracingChannelName } from '@proj-airi/stage-shared/types/pinia-action-event'
import { stubForBrowser } from '../setup/browser-probe'
import { createSession } from '../setup/session'
/** Adapts a Fakemic Electron process into an AIRI desktop audio session. */
export default async function prepareElectronRuntime(context: FakemicElectronPrepareContext): Promise<AudioInputSession> {
await context.app.context().addInitScript(stubForBrowser)
await context.app.context().addInitScript(stubForBrowser, piniaActionTracingChannelName)
const page = await waitForPage(context, (page) => {
const url = new URL(page.url())
return url.pathname.endsWith('/index.html') && url.hash === '#/'
})
await page.locator('[i-solar\\:alt-arrow-up-line-duotone]').first().waitFor({ state: 'visible', timeout: 30_000 })
await page.evaluate(stubForBrowser)
await page.evaluate(stubForBrowser, piniaActionTracingChannelName)
return createSession({
electronApp: context.app,
@@ -2,12 +2,14 @@ import type { FakemicWebPrepareContext } from '@proj-airi/vitest-plugin-fakemic'
import type { AudioInputSession } from '../types'
import { piniaActionTracingChannelName } from '@proj-airi/stage-shared/types/pinia-action-event'
import { stubForBrowser } from '../setup/browser-probe'
import { createSession } from '../setup/session'
/** Adapts a Fakemic Chromium process into an AIRI Web audio session. */
export default async function prepareWebRuntime(context: FakemicWebPrepareContext): Promise<AudioInputSession> {
await context.context.addInitScript(stubForBrowser)
await context.context.addInitScript(stubForBrowser, piniaActionTracingChannelName)
const page = await context.context.newPage()
await page.goto(context.runtime.url)
@@ -1,4 +1,5 @@
import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
import type { PiniaActionEvent } from '@proj-airi/stage-shared/types/pinia-action-event'
/**
* Installs passive browser probes before the application starts.
@@ -8,7 +9,8 @@ import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
* `BrowserContext.addInitScript`
* -> {@link stubForBrowser}
* -> `window.fetch`
* -> `BroadcastChannel('io-tracer-channel')`
* -> Pinia action tracing channel
* -> I/O tracing channel
*
* Upstream:
* - `BrowserContext.addInitScript`
@@ -16,10 +18,15 @@ import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
* Downstream:
* - `window.__airiAudioInputE2E`
*/
export function stubForBrowser() {
const state: BrowserAudioInputState = { spans: [], streamingTranscriptionReady: false, streamingTranscriptionUpdates: [], transcriptionAudio: [], transcriptionResults: [], vadReady: false }
export function stubForBrowser(piniaActionChannelName: string) {
const state: BrowserAudioInputState = { piniaActionEvents: [], spans: [], streamingTranscriptionReady: false, streamingTranscriptionUpdates: [], transcriptionAudio: [], transcriptionResults: [], vadReady: false }
window.__airiAudioInputE2E = state
const piniaActionChannel = new BroadcastChannel(piniaActionChannelName)
piniaActionChannel.addEventListener('message', (message: MessageEvent<PiniaActionEvent>) => {
state.piniaActionEvents.push(message.data)
})
const originalConsoleInfo = console.info.bind(console)
console.info = (...values: unknown[]) => {
if (typeof values[0] === 'string' && values[0].startsWith('[Voice Input] vad-ready:'))
+119 -25
View File
@@ -1,13 +1,17 @@
import type { PiniaActionEvent } from '@proj-airi/stage-shared/types/pinia-action-event'
import type { AudioCapture } from '@proj-airi/vitest-plugin-fakemic'
import type { ElectronApplication, Page } from 'playwright'
import type { AudioInputSession, AudioInputTarget } from '../types'
import type { AudioInputChatMessage, AudioInputSession, AudioInputTarget, AudioInputTurn } from '../types'
import { Buffer } from 'node:buffer'
import { IOAttributes, IOSpanNames } from '@proj-airi/stage-shared/perf/io-trace'
import { piniaActionTracingChannelName } from '@proj-airi/stage-shared/types/pinia-action-event'
import { readCompletedSpans } from './browser-probe'
/** Creates the runtime session and records its page diagnostics. */
/** Creates the runtime session for one audio test. */
export function createSession(options: {
electronApp?: ElectronApplication
page: Page
@@ -15,26 +19,6 @@ export function createSession(options: {
close: () => Promise<void>
transcriptionCaptureFormat?: AudioCapture['format']
}): AudioInputSession {
const diagnostics: string[] = []
const observedPages = new WeakSet<Page>()
function observePage(page: Page) {
if (observedPages.has(page))
return
observedPages.add(page)
page.on('console', (message) => {
const text = message.text()
const isAudioPipelineInfo = message.type() === 'info'
&& (text.includes('[Hearing Pipeline]') || text.includes('[Voice Input]') || text.includes('transcription'))
if (['error', 'warning'].includes(message.type()) || isAudioPipelineInfo)
diagnostics.push(`[console:${message.type()}] ${text}`)
})
page.on('pageerror', error => diagnostics.push(`[pageerror] ${error.message}`))
}
observePage(options.page)
const session: AudioInputSession = {
electronApp: options.electronApp,
page: options.page,
@@ -42,7 +26,6 @@ export function createSession(options: {
target: options.target,
transcriptionCaptureFormat: options.transcriptionCaptureFormat,
activatePage(page) {
observePage(page)
session.page = page
},
async capturedTranscriptionAudio(count) {
@@ -63,12 +46,13 @@ export function createSession(options: {
microphoneEnabled: localStorage.getItem('settings/audio/input/enabled'),
microphoneInput: localStorage.getItem('settings/audio/input'),
microphoneOffIconVisible: Boolean(document.querySelector('[i-ph\\:microphone-slash]')),
piniaActionEvents: window.__airiAudioInputE2E?.piniaActionEvents ?? [],
probeInstalled: Boolean(window.__airiAudioInputE2E),
streamingTranscriptionReady: window.__airiAudioInputE2E?.streamingTranscriptionReady ?? false,
url: window.location.href,
vadReady: window.__airiAudioInputE2E?.vadReady ?? false,
}))
throw new Error(`Timed out waiting for captured transcription audio: ${JSON.stringify({ diagnostics, runtimeState })}`, { cause: error })
throw new Error(`Timed out waiting for captured transcription audio: ${JSON.stringify(runtimeState)}`, { cause: error })
}
const capturedAudio = await options.page.evaluate(() => window.__airiAudioInputE2E?.transcriptionAudio ?? [])
return capturedAudio.map(audio => ({
@@ -91,22 +75,64 @@ export function createSession(options: {
const interactionSpans = await session.page.evaluate(readCompletedSpans, name)
return [...runtimeSpans, ...interactionSpans]
},
async waitForTurn(waitOptions = {}) {
const timeout = waitOptions.timeout ?? 60_000
const deadline = Date.now() + timeout
let turn = createTurnObservation(await session.completedSpans())
while (!turn && Date.now() < deadline) {
await new Promise(resolve => setTimeout(resolve, 100))
turn = createTurnObservation(await session.completedSpans())
}
if (!turn)
throw new Error('Timed out waiting for a completed LLM and speech turn.')
turn.chat.messages = await readChatMessages(session.page)
return turn
},
piniaActionEvents: () => options.page.evaluate(() => window.__airiAudioInputE2E?.piniaActionEvents ?? []),
async waitForVadReady() {
await options.page.waitForFunction(() => window.__airiAudioInputE2E?.vadReady === true, undefined, { timeout: 30_000 })
},
async waitForStreamingTranscriptionReady() {
await options.page.waitForFunction(() => window.__airiAudioInputE2E?.streamingTranscriptionReady === true, undefined, { timeout: 30_000 })
},
async waitForPiniaAction(waitOptions) {
return options.page.evaluate(({ actionName, channelName, status, storeId, timeout }) => new Promise<PiniaActionEvent>((resolve, reject) => {
const channel = new BroadcastChannel(channelName)
const timeoutId = window.setTimeout(() => {
channel.close()
reject(new Error(`Timed out waiting for Pinia action ${storeId}.${actionName} (${status})`))
}, timeout)
channel.addEventListener('message', (message: MessageEvent<PiniaActionEvent>) => {
const action = message.data
if (action.storeId !== storeId || action.actionName !== actionName || action.status !== status)
return
window.clearTimeout(timeoutId)
channel.close()
resolve(action)
})
}), {
actionName: waitOptions.actionName,
channelName: piniaActionTracingChannelName,
status: waitOptions.status ?? 'completed',
storeId: waitOptions.storeId,
timeout: waitOptions.timeout ?? 60_000,
})
},
async snapshot() {
const runtimeState = await options.page.evaluate(() => window.__airiAudioInputE2E)
const interactionState = session.page === options.page
? runtimeState
: await session.page.evaluate(() => window.__airiAudioInputE2E)
return {
piniaActionEvents: runtimeState?.piniaActionEvents ?? [],
spans: runtimeState?.spans ?? [],
streamingTranscriptionUpdates: interactionState?.streamingTranscriptionUpdates ?? [],
transcriptionResults: runtimeState?.transcriptionResults ?? [],
diagnostics: [...diagnostics],
}
},
close: options.close,
@@ -114,3 +140,71 @@ export function createSession(options: {
return session
}
function createTurnObservation(spans: Awaited<ReturnType<AudioInputSession['completedSpans']>>): AudioInputTurn | undefined {
const speechTurnSpan = spans.findLast(span => span.name === IOSpanNames.SpeechTurn)
if (!speechTurnSpan)
return undefined
const turnId = stringAttribute(speechTurnSpan, IOAttributes.TurnId)
const llmSpan = spans.findLast(span => (
span.name === IOSpanNames.LLMInference
&& stringAttribute(span, IOAttributes.TurnId) === turnId
))
if (!turnId || !llmSpan)
return undefined
const inputMessageRoles = stringArrayAttribute(llmSpan, IOAttributes.LLMInputMessageRoles)
const outputChunkLengths = numberArrayAttribute(llmSpan, IOAttributes.LLMOutputChunkLengths)
const audioSegments = spans
.filter(span => (
span.name === IOSpanNames.TTSSynthesis
&& stringAttribute(span, IOAttributes.TurnId) === turnId
))
.toSorted((left, right) => Number(left.startTimeNano) - Number(right.startTimeNano))
.map(span => ({
durationMs: numberAttribute(span, IOAttributes.TTSAudioDurationMs),
text: stringAttribute(span, IOAttributes.TTSText),
}))
return {
id: turnId,
chat: { messages: [] },
llm: {
inputMessages: inputMessageRoles.map(role => ({ role })),
outputCharacters: numberAttribute(llmSpan, IOAttributes.LLMTextLength),
outputChunks: outputChunkLengths.map(characters => ({ characters })),
},
tts: { audioSegments },
}
}
async function readChatMessages(page: Page): Promise<AudioInputChatMessage[]> {
return page.locator('[data-chat-message-role]').evaluateAll(elements => elements.flatMap((element) => {
const role = element.getAttribute('data-chat-message-role')
const text = element.textContent?.trim() ?? ''
if ((role !== 'assistant' && role !== 'user') || !text)
return []
return [{ role, text }]
}))
}
function numberAttribute(span: Awaited<ReturnType<AudioInputSession['completedSpans']>>[number], name: string): number {
const value = span.attributes[name]
return typeof value === 'number' ? value : 0
}
function numberArrayAttribute(span: Awaited<ReturnType<AudioInputSession['completedSpans']>>[number], name: string): number[] {
const value = span.attributes[name]
return Array.isArray(value) ? value.filter(item => typeof item === 'number') : []
}
function stringAttribute(span: Awaited<ReturnType<AudioInputSession['completedSpans']>>[number], name: string): string {
const value = span.attributes[name]
return typeof value === 'string' ? value : ''
}
function stringArrayAttribute(span: Awaited<ReturnType<AudioInputSession['completedSpans']>>[number], name: string): string[] {
const value = span.attributes[name]
return Array.isArray(value) ? value.filter(item => typeof item === 'string') : []
}
+66 -1
View File
@@ -1,4 +1,5 @@
import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
import type { PiniaActionEvent, PiniaActionEventStatus } from '@proj-airi/stage-shared/types/pinia-action-event'
import type { AudioCapture, AudioCaptureFormat, AudioTestCase, AudioTestPreflightCallback, AudioTestSession } from '@proj-airi/vitest-plugin-fakemic'
import type { ElectronApplication, Page } from 'playwright'
@@ -24,10 +25,48 @@ export type AudioInputTestCase = AudioTestCase<AudioInputPreflightContext>
/** Snapshot of the observable AIRI audio pipeline state. */
export interface AudioInputSnapshot {
piniaActionEvents: PiniaActionEvent[]
spans: SerializedIOSpan[]
streamingTranscriptionUpdates: string[]
transcriptionResults: string[]
diagnostics: string[]
}
/** One rendered chat message from the completed turn. */
export interface AudioInputChatMessage {
role: 'assistant' | 'user'
text: string
}
/** One LLM input message represented without its potentially sensitive content. */
export interface AudioInputLLMMessage {
role: string
}
/** One text chunk emitted by the LLM stream. */
export interface AudioInputLLMOutputChunk {
characters: number
}
/** One synthesized audio segment routed from the LLM turn. */
export interface AudioInputTTSSegment {
durationMs: number
text: string
}
/** Structured observations for one completed voice interaction turn. */
export interface AudioInputTurn {
id: string
chat: {
messages: AudioInputChatMessage[]
}
llm: {
inputMessages: AudioInputLLMMessage[]
outputCharacters: number
outputChunks: AudioInputLLMOutputChunk[]
}
tts: {
audioSegments: AudioInputTTSSegment[]
}
}
/** Observable audio values used by AIRI matchers. */
@@ -38,10 +77,36 @@ export interface AudioInputObservations {
streamingTranscriptionUpdates: () => Promise<string[]>
transcriptionResults: (count: number) => Promise<string[]>
completedSpans: (name?: string) => Promise<SerializedIOSpan[]>
/** Waits until all speech work for the latest LLM turn is complete. */
waitForTurn: (options?: {
/** @default 60000 */
timeout?: number
}) => Promise<AudioInputTurn>
/** Returns the Pinia action events collected by the runtime probe. */
piniaActionEvents: () => Promise<PiniaActionEvent[]>
/** Waits until the VAD audio graph is connected to the microphone stream. */
waitForVadReady: () => Promise<void>
/** Waits until a streaming transcription transport accepts microphone audio. */
waitForStreamingTranscriptionReady: () => Promise<void>
/**
* Waits for the next matching Pinia action event after this method is called.
*
* @example
* const completed = audio.waitForPiniaAction({
* storeId: 'modules:hearing:speech:audio-input-pipeline',
* actionName: 'transcribeForMediaStream',
* })
* await enableButton.click()
* await completed
*/
waitForPiniaAction: (options: {
storeId: string
actionName: string
/** @default 'completed' */
status?: PiniaActionEventStatus
/** @default 60000 */
timeout?: number
}) => Promise<PiniaActionEvent>
}
/** Runtime handle for one AIRI audio-input test. */
+2
View File
@@ -1,7 +1,9 @@
import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
import type { PiniaActionEvent } from '@proj-airi/stage-shared/types/pinia-action-event'
declare global {
interface BrowserAudioInputState {
piniaActionEvents: PiniaActionEvent[]
spans: SerializedIOSpan[]
streamingTranscriptionReady: boolean
streamingTranscriptionUpdates: string[]