mirror of
https://github.com/moeru-ai/airi.git
synced 2026-08-14 00:48:06 +00:00
refactor(testing-audio,stage-ui): added pinia-plugin-tracing, directly assert from history of pinia
This commit is contained in:
@@ -7,7 +7,7 @@ import { autoAnimatePlugin } from '@formkit/auto-animate/vue'
|
||||
import { PiniaColada } from '@pinia/colada'
|
||||
import { trackButtonPlugin } from '@proj-airi/stage-ui/directives/track-button'
|
||||
import { configureAnalyticsAdapter } from '@proj-airi/stage-ui/libs/analytics'
|
||||
import { setupSynced } from '@proj-airi/stage-ui/libs/pinia'
|
||||
import { piniaPluginTracing, setupSynced } from '@proj-airi/stage-ui/libs/pinia'
|
||||
import { MotionPlugin } from '@vueuse/motion'
|
||||
import { createPinia } from 'pinia'
|
||||
import { setupLayouts } from 'virtual:generated-layouts'
|
||||
@@ -47,6 +47,7 @@ configureAnalyticsAdapter(async (options) => {
|
||||
const pinia = createPinia()
|
||||
const synced = setupSynced()
|
||||
pinia.use(synced.pinia)
|
||||
pinia.use(piniaPluginTracing)
|
||||
|
||||
const router = createRouter({
|
||||
history: createWebHashHistory(),
|
||||
|
||||
@@ -9,7 +9,7 @@ import { PiniaColada } from '@pinia/colada'
|
||||
import { isEnvTruthy } from '@proj-airi/stage-shared'
|
||||
import { trackButtonPlugin } from '@proj-airi/stage-ui/directives/track-button'
|
||||
import { configureAnalyticsAdapter } from '@proj-airi/stage-ui/libs/analytics'
|
||||
import { setupSynced } from '@proj-airi/stage-ui/libs/pinia'
|
||||
import { piniaPluginTracing, setupSynced } from '@proj-airi/stage-ui/libs/pinia'
|
||||
import { MotionPlugin } from '@vueuse/motion'
|
||||
import { createPinia } from 'pinia'
|
||||
import { setupLayouts } from 'virtual:generated-layouts'
|
||||
@@ -37,6 +37,7 @@ configureAnalyticsAdapter(async (options) => {
|
||||
const pinia = createPinia()
|
||||
const synced = setupSynced()
|
||||
pinia.use(synced.pinia)
|
||||
pinia.use(piniaPluginTracing)
|
||||
|
||||
// TODO: vite-plugin-vue-layouts is long deprecated, replace with another layout solution
|
||||
const routeRecords = setupLayouts(routes as RouteRecordRaw[])
|
||||
|
||||
@@ -20,8 +20,10 @@
|
||||
"./beat-sync": "./src/beat-sync/index.ts",
|
||||
"./global-shortcut": "./src/global-shortcut/index.ts",
|
||||
"./godot-stage": "./src/godot-stage/index.ts",
|
||||
"./perf/io-trace": "./src/perf/io-trace.ts",
|
||||
"./server-channel-qr": "./src/server-channel-qr.ts",
|
||||
"./types/io-trace": "./src/types/io-trace.ts",
|
||||
"./types/pinia-action-event": "./src/types/pinia-action-event.ts",
|
||||
"./electron-renderer": "./src/electron-renderer.d.ts",
|
||||
"./composables": "./src/composables/index.ts",
|
||||
"./webgpu": "./src/webgpu/index.ts"
|
||||
|
||||
@@ -10,6 +10,7 @@ export type IOSubsystem = (typeof IOSubsystems)[keyof typeof IOSubsystems]
|
||||
|
||||
export const IOSpanNames = {
|
||||
InteractionTurn: 'Interaction turn',
|
||||
SpeechTurn: 'Speech turn',
|
||||
VoiceActivityDetection: 'Voice activity detection',
|
||||
SpeechRecognition: 'Speech recognition',
|
||||
LLMInference: 'LLM inference',
|
||||
@@ -32,6 +33,11 @@ export const IOAttributes = {
|
||||
VADAborted: `${customPrefix}.vad.aborted`,
|
||||
ASRText: `${customPrefix}.asr.text`,
|
||||
ASRAbort: `${customPrefix}.asr.abort`,
|
||||
LLMInputMessageCount: `${customPrefix}.llm.input_message_count`,
|
||||
LLMInputMessageRoles: `${customPrefix}.llm.input_message_roles`,
|
||||
LLMInputUserMessageCount: `${customPrefix}.llm.input_user_message_count`,
|
||||
LLMOutputChunkCount: `${customPrefix}.llm.output_chunk_count`,
|
||||
LLMOutputChunkLengths: `${customPrefix}.llm.output_chunk_lengths`,
|
||||
LLMTextLength: `${customPrefix}.llm.text_length`,
|
||||
StreamingControlCallName: `${customPrefix}.streaming_control.call_name`,
|
||||
StreamingControlHandlerCount: `${customPrefix}.streaming_control.handler_count`,
|
||||
@@ -47,9 +53,11 @@ export const IOAttributes = {
|
||||
TTSSegmentId: `${customPrefix}.tts.segment_id`,
|
||||
TTSText: `${customPrefix}.tts.text`,
|
||||
TTSChunkReason: `${customPrefix}.tts.chunk_reason`,
|
||||
TTSAudioDurationMs: `${customPrefix}.tts.audio_duration_ms`,
|
||||
TTSInterrupted: `${customPrefix}.tts.interrupted`,
|
||||
TTSInterruptReason: `${customPrefix}.tts.interrupt_reason`,
|
||||
TTSCanceled: `${customPrefix}.tts.canceled`,
|
||||
TurnId: `${customPrefix}.turn_id`,
|
||||
} as const
|
||||
|
||||
export const IOEvents = {
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
/** Lifecycle phase observed for one Pinia action invocation. */
|
||||
export type PiniaActionEventStatus = 'started' | 'completed' | 'failed'
|
||||
|
||||
/** Broadcast channel used by Pinia action tracing producers and consumers. */
|
||||
export const piniaActionTracingChannelName = 'airi-pinia-action-tracing'
|
||||
|
||||
/** Test-safe metadata emitted for one Pinia action lifecycle transition. */
|
||||
export interface PiniaActionEvent {
|
||||
/** Correlates lifecycle events from the same action invocation. */
|
||||
invocationId: string
|
||||
/** Name passed to `defineStore`. */
|
||||
storeId: string
|
||||
/** Action property name reported by Pinia. */
|
||||
actionName: string
|
||||
status: PiniaActionEventStatus
|
||||
timestamp: number
|
||||
/** Renderer URL that invoked the action, when available. */
|
||||
sourceUrl?: string
|
||||
/** Safe error text included only for failed actions. */
|
||||
errorMessage?: string
|
||||
}
|
||||
@@ -26,7 +26,6 @@ import { useBroadcastChannel } from '@vueuse/core'
|
||||
// import { createTransformers } from '@xsai-transformers/embed'
|
||||
// import embedWorkerURL from '@xsai-transformers/embed/worker?worker&url'
|
||||
// import { embed } from '@xsai/embed'
|
||||
import { generateSpeech } from '@xsai/generate-speech'
|
||||
import { storeToRefs } from 'pinia'
|
||||
import { computed, nextTick, onMounted, onUnmounted, ref, shallowRef, watch } from 'vue'
|
||||
|
||||
@@ -520,24 +519,18 @@ const speechPipeline = createSpeechPipeline<AudioBuffer>({
|
||||
// Non-streaming providers only: synth via REST. Streaming provider
|
||||
// was already early-returned above; it owns its own ws path opened
|
||||
// in `onBeforeMessageComposed`.
|
||||
const providerConfigWithAnalytics = activeSpeechProvider.value === OFFICIAL_SPEECH_PROVIDER_ID
|
||||
? {
|
||||
...speechRequest.providerConfig,
|
||||
extraBody: {
|
||||
...(speechRequest.providerConfig.extraBody as Record<string, unknown> | undefined),
|
||||
airi_analytics: {
|
||||
const res = await speechStore.speech(
|
||||
provider,
|
||||
model,
|
||||
speechRequest.input,
|
||||
voice.id,
|
||||
speechRequest.providerConfig,
|
||||
{
|
||||
trigger: 'auto',
|
||||
source: 'chat_auto_tts',
|
||||
voice_type: resolveStageVoiceType(),
|
||||
},
|
||||
},
|
||||
}
|
||||
: speechRequest.providerConfig
|
||||
const res = await generateSpeech({
|
||||
...provider.speech(model, providerConfigWithAnalytics),
|
||||
input: speechRequest.input,
|
||||
voice: voice.id,
|
||||
})
|
||||
)
|
||||
|
||||
if (signal.aborted || !res || res.byteLength === 0)
|
||||
return null
|
||||
|
||||
@@ -11,8 +11,25 @@ export function useIOTraceBridge(pipeline: ReturnType<typeof createSpeechPipelin
|
||||
|
||||
const synthesisSpans = new Map<string, Span>()
|
||||
const playbackSpans = new Map<string, Span>()
|
||||
const speechTurnSpans = new Map<string, Span>()
|
||||
const segmentReasons = new Map<string, string>()
|
||||
|
||||
cleanupFns.push(pipeline.on('onTurnStart', (turnId) => {
|
||||
speechTurnSpans.set(turnId, startSpan(IOSpanNames.SpeechTurn, activeTurnSpan.value, {
|
||||
[IOAttributes.TurnId]: turnId,
|
||||
}))
|
||||
}))
|
||||
|
||||
cleanupFns.push(pipeline.on('onTurnEnd', (turnId) => {
|
||||
speechTurnSpans.get(turnId)?.end()
|
||||
speechTurnSpans.delete(turnId)
|
||||
}))
|
||||
|
||||
cleanupFns.push(pipeline.on('onTurnCancel', ({ turnId }) => {
|
||||
speechTurnSpans.get(turnId)?.end()
|
||||
speechTurnSpans.delete(turnId)
|
||||
}))
|
||||
|
||||
cleanupFns.push(pipeline.on('onSegment', (segment) => {
|
||||
segmentReasons.set(segment.segmentId, segment.reason)
|
||||
}))
|
||||
@@ -23,6 +40,7 @@ export function useIOTraceBridge(pipeline: ReturnType<typeof createSpeechPipelin
|
||||
[IOAttributes.TTSSegmentId]: request.segmentId,
|
||||
[IOAttributes.TTSText]: request.text,
|
||||
[IOAttributes.TTSChunkReason]: segmentReasons.get(request.segmentId) ?? '',
|
||||
[IOAttributes.TurnId]: request.turnId ?? '',
|
||||
})
|
||||
segmentReasons.delete(request.segmentId)
|
||||
synthesisSpans.set(request.segmentId, ttsSynthesisSpan)
|
||||
@@ -31,6 +49,8 @@ export function useIOTraceBridge(pipeline: ReturnType<typeof createSpeechPipelin
|
||||
cleanupFns.push(pipeline.on('onTtsResult', (result) => {
|
||||
const span = synthesisSpans.get(result.segmentId)
|
||||
if (span) {
|
||||
if (result.audio instanceof AudioBuffer)
|
||||
span.setAttribute(IOAttributes.TTSAudioDurationMs, result.audio.duration * 1000)
|
||||
span.end()
|
||||
synthesisSpans.delete(result.segmentId)
|
||||
}
|
||||
@@ -77,6 +97,8 @@ export function useIOTraceBridge(pipeline: ReturnType<typeof createSpeechPipelin
|
||||
}))
|
||||
|
||||
onScopeDispose(() => {
|
||||
for (const span of speechTurnSpans.values())
|
||||
span.end()
|
||||
for (const cleanup of cleanupFns)
|
||||
cleanup()
|
||||
})
|
||||
|
||||
@@ -1 +1,2 @@
|
||||
export * from './pinia-plugin-tracing'
|
||||
export * from './setup-synced'
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
import type { PiniaActionEvent } from '@proj-airi/stage-shared/types/pinia-action-event'
|
||||
|
||||
import { piniaActionTracingChannelName } from '@proj-airi/stage-shared/types/pinia-action-event'
|
||||
import { createPinia, defineStore } from 'pinia'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { createApp } from 'vue'
|
||||
|
||||
import { piniaPluginTracing } from './pinia-plugin-tracing'
|
||||
|
||||
describe('piniaPluginTracing over BroadcastChannel', () => {
|
||||
it('emits correlated start and completion events without action values', async () => {
|
||||
const observer = observeActionEvents(2)
|
||||
|
||||
const pinia = createPinia()
|
||||
pinia.use(piniaPluginTracing)
|
||||
createApp({}).use(pinia)
|
||||
|
||||
const useStore = defineStore('action-event-test', {
|
||||
actions: {
|
||||
async complete(secret: string) {
|
||||
return `returned:${secret}`
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
await useStore(pinia).complete('must-not-be-recorded')
|
||||
const events = await observer.events
|
||||
|
||||
observer.close()
|
||||
expect(events).toHaveLength(2)
|
||||
expect(events.map(event => event.status)).toEqual(['started', 'completed'])
|
||||
expect(events[0]?.invocationId).toBe(events[1]?.invocationId)
|
||||
for (const event of events) {
|
||||
expect(event).toEqual(expect.objectContaining({
|
||||
actionName: 'complete',
|
||||
storeId: 'action-event-test',
|
||||
}))
|
||||
}
|
||||
expect(JSON.stringify(events)).not.toContain('must-not-be-recorded')
|
||||
expect(JSON.stringify(events)).not.toContain('returned:')
|
||||
})
|
||||
|
||||
it('emits a failed event without changing the action error', async () => {
|
||||
const observer = observeActionEvents(2)
|
||||
|
||||
const pinia = createPinia()
|
||||
pinia.use(piniaPluginTracing)
|
||||
createApp({}).use(pinia)
|
||||
|
||||
const failure = new Error('provider unavailable')
|
||||
const useStore = defineStore('failed-action-test', {
|
||||
actions: {
|
||||
fail() {
|
||||
throw failure
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
expect(() => useStore(pinia).fail()).toThrow(failure)
|
||||
const events = await observer.events
|
||||
|
||||
observer.close()
|
||||
expect(events.map(event => event.status)).toEqual(['started', 'failed'])
|
||||
expect(events[1]?.errorMessage).toBe('provider unavailable')
|
||||
})
|
||||
})
|
||||
|
||||
function observeActionEvents(count: number) {
|
||||
const channel = new BroadcastChannel(piniaActionTracingChannelName)
|
||||
const captured: PiniaActionEvent[] = []
|
||||
const events = new Promise<PiniaActionEvent[]>((resolve) => {
|
||||
channel.addEventListener('message', (message: MessageEvent<PiniaActionEvent>) => {
|
||||
captured.push(message.data)
|
||||
if (captured.length === count)
|
||||
resolve(captured)
|
||||
})
|
||||
})
|
||||
|
||||
return {
|
||||
close: () => channel.close(),
|
||||
events,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
import type { PiniaActionEvent, PiniaActionEventStatus } from '@proj-airi/stage-shared/types/pinia-action-event'
|
||||
import type { PiniaPlugin } from 'pinia'
|
||||
|
||||
import { errorMessageFrom } from '@moeru/std'
|
||||
import { piniaActionTracingChannelName } from '@proj-airi/stage-shared/types/pinia-action-event'
|
||||
import { nanoid } from 'nanoid/non-secure'
|
||||
|
||||
const piniaActionChannel = new BroadcastChannel(piniaActionTracingChannelName)
|
||||
|
||||
function emitActionEvent(
|
||||
event: Omit<PiniaActionEvent, 'status' | 'timestamp'>,
|
||||
status: PiniaActionEventStatus,
|
||||
error?: unknown,
|
||||
): void {
|
||||
piniaActionChannel.postMessage({
|
||||
...event,
|
||||
status,
|
||||
timestamp: Date.now(),
|
||||
...(status === 'failed' ? { errorMessage: errorMessageFrom(error) ?? 'Unknown action failure' } : {}),
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Traces Pinia action lifecycle events through a broadcast channel.
|
||||
*
|
||||
* The plugin never retains action arguments, results, or state snapshots.
|
||||
*/
|
||||
export const piniaPluginTracing: PiniaPlugin = ({ store }) => {
|
||||
store.$onAction(({ name, after, onError }) => {
|
||||
const event = {
|
||||
invocationId: nanoid(),
|
||||
storeId: store.$id,
|
||||
actionName: name,
|
||||
...(typeof location === 'undefined' ? {} : { sourceUrl: location.href }),
|
||||
}
|
||||
|
||||
emitActionEvent(event, 'started')
|
||||
after(() => emitActionEvent(event, 'completed'))
|
||||
onError(error => emitActionEvent(event, 'failed', error))
|
||||
})
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
import type { ChatProvider } from '@xsai-ext/providers/utils'
|
||||
import type { Message, Tool } from '@xsai/shared-chat'
|
||||
|
||||
import { IOSpanNames } from '@proj-airi/stage-shared'
|
||||
import { IOAttributes, IOSpanNames } from '@proj-airi/stage-shared'
|
||||
import { createPinia, setActivePinia } from 'pinia'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { nextTick, ref } from 'vue'
|
||||
@@ -584,6 +584,20 @@ describe('chat store contract', () => {
|
||||
expect(composedMessages).toHaveLength(2)
|
||||
expect(composedMessages[0]).toMatchObject({ role: 'system' })
|
||||
expect(composedMessages[1]).toMatchObject({ role: 'user' })
|
||||
expect(ioTracerMocks.startSpanMock).toHaveBeenCalledWith(
|
||||
IOSpanNames.LLMInference,
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
[IOAttributes.LLMInputMessageCount]: 2,
|
||||
[IOAttributes.LLMInputUserMessageCount]: 1,
|
||||
[IOAttributes.TurnId]: expect.any(String),
|
||||
}),
|
||||
)
|
||||
const llmSpan = ioTracerMocks.spans.find(span => span.name === IOSpanNames.LLMInference)
|
||||
expect(llmSpan.setAttribute).toHaveBeenCalledWith(IOAttributes.LLMInputMessageRoles, ['system', 'user'])
|
||||
expect(llmSpan.setAttribute).toHaveBeenCalledWith(IOAttributes.LLMOutputChunkCount, 1)
|
||||
expect(llmSpan.setAttribute).toHaveBeenCalledWith(IOAttributes.LLMOutputChunkLengths, [5])
|
||||
expect(llmSpan.setAttribute).toHaveBeenCalledWith(IOAttributes.LLMTextLength, 5)
|
||||
|
||||
// System message stays untouched: keeping it 100% static is what makes
|
||||
// the prefix permanently KV-cache friendly across turns and across day
|
||||
|
||||
@@ -171,6 +171,8 @@ export const useChatStore = defineStore('chat', () => {
|
||||
options?: StreamOptions,
|
||||
) {
|
||||
let llmTextLength = 0
|
||||
let llmOutputChunkCount = 0
|
||||
const llmOutputChunkLengths: number[] = []
|
||||
const headers = { ...options?.headers }
|
||||
if (getProviderMode(activeProvider.value) === 'official' && options?.requestCorrelation) {
|
||||
headers[AIRI_CHAT_SESSION_ID_HEADER] = options.requestCorrelation.conversationId
|
||||
@@ -188,7 +190,11 @@ export const useChatStore = defineStore('chat', () => {
|
||||
const llmSpan = startSpan(IOSpanNames.LLMInference, activeTurnSpan.value, {
|
||||
[IOAttributes.Subsystem]: IOSubsystems.LLM,
|
||||
[IOAttributes.GenAIRequestModel]: model,
|
||||
[IOAttributes.LLMInputMessageCount]: messages.length,
|
||||
[IOAttributes.LLMInputUserMessageCount]: messages.filter(message => message.role === 'user').length,
|
||||
[IOAttributes.TurnId]: options?.requestCorrelation?.roundId ?? '',
|
||||
})
|
||||
llmSpan.setAttribute(IOAttributes.LLMInputMessageRoles, messages.map(message => message.role))
|
||||
const llmRequestTs = performance.now()
|
||||
let llmFirstTokenEmitted = false
|
||||
|
||||
@@ -198,6 +204,8 @@ export const useChatStore = defineStore('chat', () => {
|
||||
headers,
|
||||
onStreamEvent: async (event: StreamEvent) => {
|
||||
if (isTextDelta(event)) {
|
||||
llmOutputChunkCount += 1
|
||||
llmOutputChunkLengths.push(event.text.length)
|
||||
if (!llmFirstTokenEmitted) {
|
||||
llmFirstTokenEmitted = true
|
||||
llmSpan.addEvent(IOEvents.LLMFirstToken, {
|
||||
@@ -210,10 +218,11 @@ export const useChatStore = defineStore('chat', () => {
|
||||
await options?.onStreamEvent?.(event)
|
||||
},
|
||||
})
|
||||
|
||||
llmSpan.setAttribute(IOAttributes.LLMTextLength, llmTextLength)
|
||||
}
|
||||
finally {
|
||||
llmSpan.setAttribute(IOAttributes.LLMOutputChunkCount, llmOutputChunkCount)
|
||||
llmSpan.setAttribute(IOAttributes.LLMOutputChunkLengths, llmOutputChunkLengths)
|
||||
llmSpan.setAttribute(IOAttributes.LLMTextLength, llmTextLength)
|
||||
llmSpan.end()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,6 +37,12 @@ interface SpeechInput {
|
||||
providerConfig: Record<string, unknown>
|
||||
}
|
||||
|
||||
interface SpeechAnalytics {
|
||||
trigger: 'auto' | 'manual'
|
||||
source: 'chat_auto_tts' | 'manual_preview' | 'settings_test'
|
||||
voice_type?: 'official_default' | 'official_selected' | 'custom_configured' | 'voice_pack'
|
||||
}
|
||||
|
||||
export const useSpeechStore = defineStore('speech', () => {
|
||||
const providersStore = useProviderStore()
|
||||
const providerStore = useProviderConfigStore()
|
||||
@@ -301,14 +307,15 @@ export const useSpeechStore = defineStore('speech', () => {
|
||||
input: string,
|
||||
voice: string,
|
||||
providerConfig: Record<string, any> = {},
|
||||
): Promise<ArrayBuffer> {
|
||||
const requestProviderConfig = activeSpeechProvider.value === OFFICIAL_SPEECH_PROVIDER_ID
|
||||
|| activeSpeechProvider.value === OFFICIAL_SPEECH_STREAMING_PROVIDER_ID
|
||||
? withAiriTtsAnalytics(providerConfig, {
|
||||
analytics: SpeechAnalytics = {
|
||||
trigger: 'manual',
|
||||
source: 'manual_preview',
|
||||
voice_type: resolveVoiceType(voice),
|
||||
})
|
||||
},
|
||||
): Promise<ArrayBuffer> {
|
||||
const requestProviderConfig = activeSpeechProvider.value === OFFICIAL_SPEECH_PROVIDER_ID
|
||||
|| activeSpeechProvider.value === OFFICIAL_SPEECH_STREAMING_PROVIDER_ID
|
||||
? withAiriTtsAnalytics(providerConfig, analytics)
|
||||
: providerConfig
|
||||
const response = await generateSpeech({
|
||||
...provider.speech(model, requestProviderConfig),
|
||||
@@ -321,11 +328,7 @@ export const useSpeechStore = defineStore('speech', () => {
|
||||
|
||||
function withAiriTtsAnalytics(
|
||||
providerConfig: Record<string, any>,
|
||||
analytics: {
|
||||
trigger: 'auto' | 'manual'
|
||||
source: 'chat_auto_tts' | 'manual_preview' | 'settings_test'
|
||||
voice_type?: 'official_default' | 'official_selected' | 'custom_configured' | 'voice_pack'
|
||||
},
|
||||
analytics: SpeechAnalytics,
|
||||
): Record<string, any> {
|
||||
return {
|
||||
...providerConfig,
|
||||
|
||||
@@ -43,6 +43,7 @@ describe('audio input pipeline', () => {
|
||||
await expect(audio).not.toHaveTranscriptions([
|
||||
['There is no meaning to your existence, just let go.'],
|
||||
])
|
||||
await expect(audio).toHaveCompletedTranscription()
|
||||
})
|
||||
|
||||
it('does not preserve the complete phrase with Aliyun NLS', {
|
||||
@@ -78,5 +79,6 @@ describe('audio input pipeline', () => {
|
||||
await expect(audio).not.toHaveTranscriptions([
|
||||
['There is no meaning to your existence, just let go.'],
|
||||
])
|
||||
await expect(audio).toHaveCompletedTranscription()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
import { describe, expect, it } from '../../src'
|
||||
import { configureModuleConsciousness, configureModuleHearing, configureModuleSpeech, configureOnboarding, loadCaseEnvironment } from '../shared/configurations'
|
||||
import { assistantMessages, enableChatMicrophone, openChat } from '../shared/interactions'
|
||||
import { enableChatMicrophone } from '../shared/interactions'
|
||||
import { aliyunNlsAsr, openaiAsr, openaiLlm, openaiTts } from '../shared/providers'
|
||||
|
||||
describe('audio input pipeline', () => {
|
||||
// An OpenAI-compatible TTS Provider generated the fixture in mono 16 kHz PCM WAV format.
|
||||
// The fixture contains 14 seconds of leading silence for VAD initialization and 3 seconds of trailing silence.
|
||||
// Its warm-up phrase gives the VAD time to start. Only "Please say hello." is required in the transcript.
|
||||
// Its warm-up phrase gives the VAD time to start. The final "say hello" command is required in the transcript.
|
||||
it('runs an OpenAI-compatible request through the complete pipeline', {
|
||||
input: new URL('./input.test.wav', import.meta.url),
|
||||
// This case keeps AIRI's default VAD and selects every remote Provider explicitly.
|
||||
@@ -72,16 +72,22 @@ describe('audio input pipeline', () => {
|
||||
}
|
||||
|
||||
await expect(audio).toHaveTranscriptions([
|
||||
['Please say hello.'],
|
||||
['Please say hello.', 'Say hello.'],
|
||||
], { match: 'contains' })
|
||||
|
||||
await expect.poll(async () => (await audio.completedSpans('LLM inference')).length, { timeout: 60_000 }).toBeGreaterThanOrEqual(1)
|
||||
await expect.poll(async () => (await audio.completedSpans('TTS synthesis')).length, { timeout: 60_000 }).toBeGreaterThanOrEqual(1)
|
||||
await expect.poll(async () => (await audio.completedSpans('Audio playback')).length, { timeout: 60_000 }).toBeGreaterThanOrEqual(1)
|
||||
const turn = await audio.waitForTurn()
|
||||
|
||||
await openChat(audio)
|
||||
const messages = await assistantMessages(audio).allTextContents()
|
||||
expect(messages.at(-1)).toMatch(/.+/s)
|
||||
expect(turn.chat.messages).toHaveLength(2)
|
||||
expect(turn.chat.messages.map(message => message.role)).toEqual(['user', 'assistant'])
|
||||
expect(turn.llm.inputMessages).toHaveLength(2)
|
||||
expect(turn.llm.inputMessages.filter(message => message.role === 'user')).toHaveLength(1)
|
||||
expect(turn.llm.outputChunks.length).toBeGreaterThan(0)
|
||||
expect(turn.llm.outputCharacters).toBeGreaterThan(0)
|
||||
expect(turn.tts.audioSegments.length).toBeGreaterThan(0)
|
||||
for (const segment of turn.tts.audioSegments) {
|
||||
expect(segment.text.length).toBeGreaterThan(0)
|
||||
expect(segment.durationMs).toBeGreaterThan(100)
|
||||
}
|
||||
})
|
||||
|
||||
it('transcribes the greeting with Aliyun NLS', {
|
||||
@@ -117,5 +123,6 @@ describe('audio input pipeline', () => {
|
||||
await expect(audio).toHaveTranscriptions([
|
||||
['Please say hello.'],
|
||||
], { match: 'contains' })
|
||||
await expect(audio).toHaveCompletedTranscription()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -54,6 +54,7 @@ describe('audio input pipeline', () => {
|
||||
const finalTranscriptions = await audio.transcriptionResults(expectedTranscriptions.length)
|
||||
const streamingUpdates = await audio.streamingTranscriptionUpdates()
|
||||
expect(streamingUpdates.some(update => !finalTranscriptions.includes(update))).toBe(true)
|
||||
await expect(audio).toHaveCompletedTranscription()
|
||||
})
|
||||
|
||||
it('keeps two Aliyun NLS utterances in streaming transcription', {
|
||||
@@ -101,5 +102,6 @@ describe('audio input pipeline', () => {
|
||||
const finalTranscriptions = await audio.transcriptionResults(expectedTranscriptions.length)
|
||||
const streamingUpdates = await audio.streamingTranscriptionUpdates()
|
||||
expect(streamingUpdates.some(update => !finalTranscriptions.includes(update))).toBe(true)
|
||||
await expect(audio).toHaveCompletedTranscription()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import type { PiniaActionEvent } from '@proj-airi/stage-shared/types/pinia-action-event'
|
||||
|
||||
import type { AudioInputObservations } from './types'
|
||||
|
||||
import { describe, it } from 'vitest'
|
||||
@@ -14,15 +16,45 @@ describe('audio input matchers', () => {
|
||||
['please say hello'],
|
||||
])
|
||||
})
|
||||
|
||||
it('reports a failed transcription action', async () => {
|
||||
const session = createAudioInputSession([], [{
|
||||
...actionEvent('modules:hearing:speech:audio-input-pipeline', 'transcribeForRecording'),
|
||||
errorMessage: 'Request failed',
|
||||
status: 'failed',
|
||||
}])
|
||||
|
||||
await expect(expect(session).toHaveCompletedTranscription()).rejects.toThrow('ASR failed: Request failed')
|
||||
})
|
||||
})
|
||||
|
||||
function createAudioInputSession(transcriptions: string[]): AudioInputObservations {
|
||||
function createAudioInputSession(
|
||||
transcriptions: string[],
|
||||
actions: PiniaActionEvent[] = [],
|
||||
): AudioInputObservations {
|
||||
return {
|
||||
capturedTranscriptionAudio: async () => [],
|
||||
streamingTranscriptionUpdates: async () => [],
|
||||
transcriptionResults: async () => transcriptions,
|
||||
completedSpans: async () => [],
|
||||
piniaActionEvents: async () => actions,
|
||||
waitForPiniaAction: async () => {
|
||||
throw new Error('This matcher fixture does not observe Pinia actions.')
|
||||
},
|
||||
waitForStreamingTranscriptionReady: async () => {},
|
||||
waitForTurn: async () => {
|
||||
throw new Error('This matcher fixture does not observe completed turns.')
|
||||
},
|
||||
waitForVadReady: async () => {},
|
||||
}
|
||||
}
|
||||
|
||||
function actionEvent(storeId: string, actionName: string): PiniaActionEvent {
|
||||
return {
|
||||
actionName,
|
||||
invocationId: `${storeId}:${actionName}`,
|
||||
status: 'completed',
|
||||
storeId,
|
||||
timestamp: 0,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,12 @@ export interface TranscriptionExpectationOptions {
|
||||
match?: 'exact' | 'contains'
|
||||
}
|
||||
|
||||
/** Sets the wait limit for a transcription action assertion. */
|
||||
export interface TranscriptionActionExpectationOptions {
|
||||
/** Maximum time to wait for a completed ASR action. @default 60000 */
|
||||
timeout?: number
|
||||
}
|
||||
|
||||
declare module 'vitest' {
|
||||
interface Assertion<T> {
|
||||
toHaveCapturedTranscriptionAudio: T extends AudioInputObservations
|
||||
@@ -23,6 +29,9 @@ declare module 'vitest' {
|
||||
options?: TranscriptionExpectationOptions,
|
||||
) => Promise<void>
|
||||
: never
|
||||
toHaveCompletedTranscription: T extends AudioInputObservations
|
||||
? (options?: TranscriptionActionExpectationOptions) => Promise<void>
|
||||
: never
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,13 +39,13 @@ declare module 'vitest' {
|
||||
export const expect = vitestExpect
|
||||
|
||||
/**
|
||||
* Normalizes a transcript for speech-recognition comparison.
|
||||
* Normalizes speech text for transcript comparison.
|
||||
*
|
||||
* @example
|
||||
* normalizeTranscript(' Hello, AIRI! ')
|
||||
* normalizeSpeechText(' Hello, AIRI! ')
|
||||
* // => 'helloairi'
|
||||
*/
|
||||
function normalizeTranscript(value: string): string {
|
||||
function normalizeSpeechText(value: string): string {
|
||||
return value
|
||||
.normalize('NFKC')
|
||||
.toLocaleLowerCase()
|
||||
@@ -79,8 +88,8 @@ export function installAudioInputMatchers(): void {
|
||||
options: TranscriptionExpectationOptions = {},
|
||||
) {
|
||||
const actual = await session.transcriptionResults(expected.length)
|
||||
const normalizedActual = actual.map(normalizeTranscript)
|
||||
const normalizedExpected = expected.map(alternatives => alternatives.map(normalizeTranscript))
|
||||
const normalizedActual = actual.map(normalizeSpeechText)
|
||||
const normalizedExpected = expected.map(alternatives => alternatives.map(normalizeSpeechText))
|
||||
const match = options.match ?? 'exact'
|
||||
const pass = normalizedActual.length === normalizedExpected.length
|
||||
&& normalizedActual.every((transcript, index) => (
|
||||
@@ -96,5 +105,57 @@ export function installAudioInputMatchers(): void {
|
||||
: `Expected transcriptions ${JSON.stringify(expected)}, but received ${JSON.stringify(actual)}.`,
|
||||
}
|
||||
},
|
||||
async toHaveCompletedTranscription(
|
||||
session: AudioInputObservations,
|
||||
options: TranscriptionActionExpectationOptions = {},
|
||||
) {
|
||||
const result = await waitForTranscription(session, options.timeout ?? 60_000)
|
||||
return {
|
||||
pass: result.complete,
|
||||
message: () => result.complete
|
||||
? 'Expected the transcription action not to complete.'
|
||||
: `Expected the transcription action to complete, but ${result.summary}.`,
|
||||
}
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
const transcriptionActions = [
|
||||
{ storeId: 'modules:hearing:speech:audio-input-pipeline', actionName: 'transcribeForRecording' },
|
||||
{ storeId: 'modules:hearing:speech:audio-input-pipeline', actionName: 'transcribeForMediaStream' },
|
||||
]
|
||||
|
||||
async function waitForTranscription(
|
||||
session: AudioInputObservations,
|
||||
timeout: number,
|
||||
): Promise<{ complete: boolean, failed: boolean, summary: string }> {
|
||||
const deadline = Date.now() + timeout
|
||||
let result = transcriptionResult(await session.piniaActionEvents())
|
||||
|
||||
while (!result.complete && !result.failed && Date.now() < deadline) {
|
||||
await new Promise(resolve => setTimeout(resolve, 100))
|
||||
result = transcriptionResult(await session.piniaActionEvents())
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
function transcriptionResult(
|
||||
events: Awaited<ReturnType<AudioInputObservations['piniaActionEvents']>>,
|
||||
): { complete: boolean, failed: boolean, summary: string } {
|
||||
const matchingEvents = events.filter(event => transcriptionActions.some(action => (
|
||||
event.storeId === action.storeId && event.actionName === action.actionName
|
||||
)))
|
||||
const latestTerminalEvent = matchingEvents.findLast(event => event.status !== 'started')
|
||||
if (latestTerminalEvent?.status === 'failed') {
|
||||
return {
|
||||
complete: false,
|
||||
failed: true,
|
||||
summary: `ASR failed${latestTerminalEvent.errorMessage ? `: ${latestTerminalEvent.errorMessage}` : ''}`,
|
||||
}
|
||||
}
|
||||
return {
|
||||
complete: latestTerminalEvent?.status === 'completed',
|
||||
failed: false,
|
||||
summary: 'ASR did not complete',
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,10 +1,18 @@
|
||||
export { describe, expect, it } from './describe'
|
||||
|
||||
export type { TranscriptionActionExpectationOptions } from './expect-extend'
|
||||
export type {
|
||||
AudioInputChatMessage,
|
||||
AudioInputLLMMessage,
|
||||
AudioInputLLMOutputChunk,
|
||||
AudioInputObservations,
|
||||
AudioInputPreflightCallback,
|
||||
AudioInputPreflightContext,
|
||||
AudioInputSession,
|
||||
AudioInputTarget,
|
||||
AudioInputTestCase,
|
||||
AudioInputTTSSegment,
|
||||
AudioInputTurn,
|
||||
} from './types'
|
||||
|
||||
export type { PiniaActionEvent, PiniaActionEventStatus } from '@proj-airi/stage-shared/types/pinia-action-event'
|
||||
|
||||
@@ -3,18 +3,20 @@ import type { Page } from 'playwright'
|
||||
|
||||
import type { AudioInputSession } from '../types'
|
||||
|
||||
import { piniaActionTracingChannelName } from '@proj-airi/stage-shared/types/pinia-action-event'
|
||||
|
||||
import { stubForBrowser } from '../setup/browser-probe'
|
||||
import { createSession } from '../setup/session'
|
||||
|
||||
/** Adapts a Fakemic Electron process into an AIRI desktop audio session. */
|
||||
export default async function prepareElectronRuntime(context: FakemicElectronPrepareContext): Promise<AudioInputSession> {
|
||||
await context.app.context().addInitScript(stubForBrowser)
|
||||
await context.app.context().addInitScript(stubForBrowser, piniaActionTracingChannelName)
|
||||
const page = await waitForPage(context, (page) => {
|
||||
const url = new URL(page.url())
|
||||
return url.pathname.endsWith('/index.html') && url.hash === '#/'
|
||||
})
|
||||
await page.locator('[i-solar\\:alt-arrow-up-line-duotone]').first().waitFor({ state: 'visible', timeout: 30_000 })
|
||||
await page.evaluate(stubForBrowser)
|
||||
await page.evaluate(stubForBrowser, piniaActionTracingChannelName)
|
||||
|
||||
return createSession({
|
||||
electronApp: context.app,
|
||||
|
||||
@@ -2,12 +2,14 @@ import type { FakemicWebPrepareContext } from '@proj-airi/vitest-plugin-fakemic'
|
||||
|
||||
import type { AudioInputSession } from '../types'
|
||||
|
||||
import { piniaActionTracingChannelName } from '@proj-airi/stage-shared/types/pinia-action-event'
|
||||
|
||||
import { stubForBrowser } from '../setup/browser-probe'
|
||||
import { createSession } from '../setup/session'
|
||||
|
||||
/** Adapts a Fakemic Chromium process into an AIRI Web audio session. */
|
||||
export default async function prepareWebRuntime(context: FakemicWebPrepareContext): Promise<AudioInputSession> {
|
||||
await context.context.addInitScript(stubForBrowser)
|
||||
await context.context.addInitScript(stubForBrowser, piniaActionTracingChannelName)
|
||||
|
||||
const page = await context.context.newPage()
|
||||
await page.goto(context.runtime.url)
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
|
||||
import type { PiniaActionEvent } from '@proj-airi/stage-shared/types/pinia-action-event'
|
||||
|
||||
/**
|
||||
* Installs passive browser probes before the application starts.
|
||||
@@ -8,7 +9,8 @@ import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
|
||||
* `BrowserContext.addInitScript`
|
||||
* -> {@link stubForBrowser}
|
||||
* -> `window.fetch`
|
||||
* -> `BroadcastChannel('io-tracer-channel')`
|
||||
* -> Pinia action tracing channel
|
||||
* -> I/O tracing channel
|
||||
*
|
||||
* Upstream:
|
||||
* - `BrowserContext.addInitScript`
|
||||
@@ -16,10 +18,15 @@ import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
|
||||
* Downstream:
|
||||
* - `window.__airiAudioInputE2E`
|
||||
*/
|
||||
export function stubForBrowser() {
|
||||
const state: BrowserAudioInputState = { spans: [], streamingTranscriptionReady: false, streamingTranscriptionUpdates: [], transcriptionAudio: [], transcriptionResults: [], vadReady: false }
|
||||
export function stubForBrowser(piniaActionChannelName: string) {
|
||||
const state: BrowserAudioInputState = { piniaActionEvents: [], spans: [], streamingTranscriptionReady: false, streamingTranscriptionUpdates: [], transcriptionAudio: [], transcriptionResults: [], vadReady: false }
|
||||
window.__airiAudioInputE2E = state
|
||||
|
||||
const piniaActionChannel = new BroadcastChannel(piniaActionChannelName)
|
||||
piniaActionChannel.addEventListener('message', (message: MessageEvent<PiniaActionEvent>) => {
|
||||
state.piniaActionEvents.push(message.data)
|
||||
})
|
||||
|
||||
const originalConsoleInfo = console.info.bind(console)
|
||||
console.info = (...values: unknown[]) => {
|
||||
if (typeof values[0] === 'string' && values[0].startsWith('[Voice Input] vad-ready:'))
|
||||
|
||||
@@ -1,13 +1,17 @@
|
||||
import type { PiniaActionEvent } from '@proj-airi/stage-shared/types/pinia-action-event'
|
||||
import type { AudioCapture } from '@proj-airi/vitest-plugin-fakemic'
|
||||
import type { ElectronApplication, Page } from 'playwright'
|
||||
|
||||
import type { AudioInputSession, AudioInputTarget } from '../types'
|
||||
import type { AudioInputChatMessage, AudioInputSession, AudioInputTarget, AudioInputTurn } from '../types'
|
||||
|
||||
import { Buffer } from 'node:buffer'
|
||||
|
||||
import { IOAttributes, IOSpanNames } from '@proj-airi/stage-shared/perf/io-trace'
|
||||
import { piniaActionTracingChannelName } from '@proj-airi/stage-shared/types/pinia-action-event'
|
||||
|
||||
import { readCompletedSpans } from './browser-probe'
|
||||
|
||||
/** Creates the runtime session and records its page diagnostics. */
|
||||
/** Creates the runtime session for one audio test. */
|
||||
export function createSession(options: {
|
||||
electronApp?: ElectronApplication
|
||||
page: Page
|
||||
@@ -15,26 +19,6 @@ export function createSession(options: {
|
||||
close: () => Promise<void>
|
||||
transcriptionCaptureFormat?: AudioCapture['format']
|
||||
}): AudioInputSession {
|
||||
const diagnostics: string[] = []
|
||||
const observedPages = new WeakSet<Page>()
|
||||
|
||||
function observePage(page: Page) {
|
||||
if (observedPages.has(page))
|
||||
return
|
||||
|
||||
observedPages.add(page)
|
||||
page.on('console', (message) => {
|
||||
const text = message.text()
|
||||
const isAudioPipelineInfo = message.type() === 'info'
|
||||
&& (text.includes('[Hearing Pipeline]') || text.includes('[Voice Input]') || text.includes('transcription'))
|
||||
if (['error', 'warning'].includes(message.type()) || isAudioPipelineInfo)
|
||||
diagnostics.push(`[console:${message.type()}] ${text}`)
|
||||
})
|
||||
page.on('pageerror', error => diagnostics.push(`[pageerror] ${error.message}`))
|
||||
}
|
||||
|
||||
observePage(options.page)
|
||||
|
||||
const session: AudioInputSession = {
|
||||
electronApp: options.electronApp,
|
||||
page: options.page,
|
||||
@@ -42,7 +26,6 @@ export function createSession(options: {
|
||||
target: options.target,
|
||||
transcriptionCaptureFormat: options.transcriptionCaptureFormat,
|
||||
activatePage(page) {
|
||||
observePage(page)
|
||||
session.page = page
|
||||
},
|
||||
async capturedTranscriptionAudio(count) {
|
||||
@@ -63,12 +46,13 @@ export function createSession(options: {
|
||||
microphoneEnabled: localStorage.getItem('settings/audio/input/enabled'),
|
||||
microphoneInput: localStorage.getItem('settings/audio/input'),
|
||||
microphoneOffIconVisible: Boolean(document.querySelector('[i-ph\\:microphone-slash]')),
|
||||
piniaActionEvents: window.__airiAudioInputE2E?.piniaActionEvents ?? [],
|
||||
probeInstalled: Boolean(window.__airiAudioInputE2E),
|
||||
streamingTranscriptionReady: window.__airiAudioInputE2E?.streamingTranscriptionReady ?? false,
|
||||
url: window.location.href,
|
||||
vadReady: window.__airiAudioInputE2E?.vadReady ?? false,
|
||||
}))
|
||||
throw new Error(`Timed out waiting for captured transcription audio: ${JSON.stringify({ diagnostics, runtimeState })}`, { cause: error })
|
||||
throw new Error(`Timed out waiting for captured transcription audio: ${JSON.stringify(runtimeState)}`, { cause: error })
|
||||
}
|
||||
const capturedAudio = await options.page.evaluate(() => window.__airiAudioInputE2E?.transcriptionAudio ?? [])
|
||||
return capturedAudio.map(audio => ({
|
||||
@@ -91,22 +75,64 @@ export function createSession(options: {
|
||||
const interactionSpans = await session.page.evaluate(readCompletedSpans, name)
|
||||
return [...runtimeSpans, ...interactionSpans]
|
||||
},
|
||||
async waitForTurn(waitOptions = {}) {
|
||||
const timeout = waitOptions.timeout ?? 60_000
|
||||
const deadline = Date.now() + timeout
|
||||
let turn = createTurnObservation(await session.completedSpans())
|
||||
|
||||
while (!turn && Date.now() < deadline) {
|
||||
await new Promise(resolve => setTimeout(resolve, 100))
|
||||
turn = createTurnObservation(await session.completedSpans())
|
||||
}
|
||||
|
||||
if (!turn)
|
||||
throw new Error('Timed out waiting for a completed LLM and speech turn.')
|
||||
|
||||
turn.chat.messages = await readChatMessages(session.page)
|
||||
return turn
|
||||
},
|
||||
piniaActionEvents: () => options.page.evaluate(() => window.__airiAudioInputE2E?.piniaActionEvents ?? []),
|
||||
async waitForVadReady() {
|
||||
await options.page.waitForFunction(() => window.__airiAudioInputE2E?.vadReady === true, undefined, { timeout: 30_000 })
|
||||
},
|
||||
async waitForStreamingTranscriptionReady() {
|
||||
await options.page.waitForFunction(() => window.__airiAudioInputE2E?.streamingTranscriptionReady === true, undefined, { timeout: 30_000 })
|
||||
},
|
||||
async waitForPiniaAction(waitOptions) {
|
||||
return options.page.evaluate(({ actionName, channelName, status, storeId, timeout }) => new Promise<PiniaActionEvent>((resolve, reject) => {
|
||||
const channel = new BroadcastChannel(channelName)
|
||||
const timeoutId = window.setTimeout(() => {
|
||||
channel.close()
|
||||
reject(new Error(`Timed out waiting for Pinia action ${storeId}.${actionName} (${status})`))
|
||||
}, timeout)
|
||||
|
||||
channel.addEventListener('message', (message: MessageEvent<PiniaActionEvent>) => {
|
||||
const action = message.data
|
||||
if (action.storeId !== storeId || action.actionName !== actionName || action.status !== status)
|
||||
return
|
||||
|
||||
window.clearTimeout(timeoutId)
|
||||
channel.close()
|
||||
resolve(action)
|
||||
})
|
||||
}), {
|
||||
actionName: waitOptions.actionName,
|
||||
channelName: piniaActionTracingChannelName,
|
||||
status: waitOptions.status ?? 'completed',
|
||||
storeId: waitOptions.storeId,
|
||||
timeout: waitOptions.timeout ?? 60_000,
|
||||
})
|
||||
},
|
||||
async snapshot() {
|
||||
const runtimeState = await options.page.evaluate(() => window.__airiAudioInputE2E)
|
||||
const interactionState = session.page === options.page
|
||||
? runtimeState
|
||||
: await session.page.evaluate(() => window.__airiAudioInputE2E)
|
||||
return {
|
||||
piniaActionEvents: runtimeState?.piniaActionEvents ?? [],
|
||||
spans: runtimeState?.spans ?? [],
|
||||
streamingTranscriptionUpdates: interactionState?.streamingTranscriptionUpdates ?? [],
|
||||
transcriptionResults: runtimeState?.transcriptionResults ?? [],
|
||||
diagnostics: [...diagnostics],
|
||||
}
|
||||
},
|
||||
close: options.close,
|
||||
@@ -114,3 +140,71 @@ export function createSession(options: {
|
||||
|
||||
return session
|
||||
}
|
||||
|
||||
function createTurnObservation(spans: Awaited<ReturnType<AudioInputSession['completedSpans']>>): AudioInputTurn | undefined {
|
||||
const speechTurnSpan = spans.findLast(span => span.name === IOSpanNames.SpeechTurn)
|
||||
if (!speechTurnSpan)
|
||||
return undefined
|
||||
|
||||
const turnId = stringAttribute(speechTurnSpan, IOAttributes.TurnId)
|
||||
const llmSpan = spans.findLast(span => (
|
||||
span.name === IOSpanNames.LLMInference
|
||||
&& stringAttribute(span, IOAttributes.TurnId) === turnId
|
||||
))
|
||||
if (!turnId || !llmSpan)
|
||||
return undefined
|
||||
|
||||
const inputMessageRoles = stringArrayAttribute(llmSpan, IOAttributes.LLMInputMessageRoles)
|
||||
const outputChunkLengths = numberArrayAttribute(llmSpan, IOAttributes.LLMOutputChunkLengths)
|
||||
const audioSegments = spans
|
||||
.filter(span => (
|
||||
span.name === IOSpanNames.TTSSynthesis
|
||||
&& stringAttribute(span, IOAttributes.TurnId) === turnId
|
||||
))
|
||||
.toSorted((left, right) => Number(left.startTimeNano) - Number(right.startTimeNano))
|
||||
.map(span => ({
|
||||
durationMs: numberAttribute(span, IOAttributes.TTSAudioDurationMs),
|
||||
text: stringAttribute(span, IOAttributes.TTSText),
|
||||
}))
|
||||
|
||||
return {
|
||||
id: turnId,
|
||||
chat: { messages: [] },
|
||||
llm: {
|
||||
inputMessages: inputMessageRoles.map(role => ({ role })),
|
||||
outputCharacters: numberAttribute(llmSpan, IOAttributes.LLMTextLength),
|
||||
outputChunks: outputChunkLengths.map(characters => ({ characters })),
|
||||
},
|
||||
tts: { audioSegments },
|
||||
}
|
||||
}
|
||||
|
||||
async function readChatMessages(page: Page): Promise<AudioInputChatMessage[]> {
|
||||
return page.locator('[data-chat-message-role]').evaluateAll(elements => elements.flatMap((element) => {
|
||||
const role = element.getAttribute('data-chat-message-role')
|
||||
const text = element.textContent?.trim() ?? ''
|
||||
if ((role !== 'assistant' && role !== 'user') || !text)
|
||||
return []
|
||||
return [{ role, text }]
|
||||
}))
|
||||
}
|
||||
|
||||
function numberAttribute(span: Awaited<ReturnType<AudioInputSession['completedSpans']>>[number], name: string): number {
|
||||
const value = span.attributes[name]
|
||||
return typeof value === 'number' ? value : 0
|
||||
}
|
||||
|
||||
function numberArrayAttribute(span: Awaited<ReturnType<AudioInputSession['completedSpans']>>[number], name: string): number[] {
|
||||
const value = span.attributes[name]
|
||||
return Array.isArray(value) ? value.filter(item => typeof item === 'number') : []
|
||||
}
|
||||
|
||||
function stringAttribute(span: Awaited<ReturnType<AudioInputSession['completedSpans']>>[number], name: string): string {
|
||||
const value = span.attributes[name]
|
||||
return typeof value === 'string' ? value : ''
|
||||
}
|
||||
|
||||
function stringArrayAttribute(span: Awaited<ReturnType<AudioInputSession['completedSpans']>>[number], name: string): string[] {
|
||||
const value = span.attributes[name]
|
||||
return Array.isArray(value) ? value.filter(item => typeof item === 'string') : []
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
|
||||
import type { PiniaActionEvent, PiniaActionEventStatus } from '@proj-airi/stage-shared/types/pinia-action-event'
|
||||
import type { AudioCapture, AudioCaptureFormat, AudioTestCase, AudioTestPreflightCallback, AudioTestSession } from '@proj-airi/vitest-plugin-fakemic'
|
||||
import type { ElectronApplication, Page } from 'playwright'
|
||||
|
||||
@@ -24,10 +25,48 @@ export type AudioInputTestCase = AudioTestCase<AudioInputPreflightContext>
|
||||
|
||||
/** Snapshot of the observable AIRI audio pipeline state. */
|
||||
export interface AudioInputSnapshot {
|
||||
piniaActionEvents: PiniaActionEvent[]
|
||||
spans: SerializedIOSpan[]
|
||||
streamingTranscriptionUpdates: string[]
|
||||
transcriptionResults: string[]
|
||||
diagnostics: string[]
|
||||
}
|
||||
|
||||
/** One rendered chat message from the completed turn. */
|
||||
export interface AudioInputChatMessage {
|
||||
role: 'assistant' | 'user'
|
||||
text: string
|
||||
}
|
||||
|
||||
/** One LLM input message represented without its potentially sensitive content. */
|
||||
export interface AudioInputLLMMessage {
|
||||
role: string
|
||||
}
|
||||
|
||||
/** One text chunk emitted by the LLM stream. */
|
||||
export interface AudioInputLLMOutputChunk {
|
||||
characters: number
|
||||
}
|
||||
|
||||
/** One synthesized audio segment routed from the LLM turn. */
|
||||
export interface AudioInputTTSSegment {
|
||||
durationMs: number
|
||||
text: string
|
||||
}
|
||||
|
||||
/** Structured observations for one completed voice interaction turn. */
|
||||
export interface AudioInputTurn {
|
||||
id: string
|
||||
chat: {
|
||||
messages: AudioInputChatMessage[]
|
||||
}
|
||||
llm: {
|
||||
inputMessages: AudioInputLLMMessage[]
|
||||
outputCharacters: number
|
||||
outputChunks: AudioInputLLMOutputChunk[]
|
||||
}
|
||||
tts: {
|
||||
audioSegments: AudioInputTTSSegment[]
|
||||
}
|
||||
}
|
||||
|
||||
/** Observable audio values used by AIRI matchers. */
|
||||
@@ -38,10 +77,36 @@ export interface AudioInputObservations {
|
||||
streamingTranscriptionUpdates: () => Promise<string[]>
|
||||
transcriptionResults: (count: number) => Promise<string[]>
|
||||
completedSpans: (name?: string) => Promise<SerializedIOSpan[]>
|
||||
/** Waits until all speech work for the latest LLM turn is complete. */
|
||||
waitForTurn: (options?: {
|
||||
/** @default 60000 */
|
||||
timeout?: number
|
||||
}) => Promise<AudioInputTurn>
|
||||
/** Returns the Pinia action events collected by the runtime probe. */
|
||||
piniaActionEvents: () => Promise<PiniaActionEvent[]>
|
||||
/** Waits until the VAD audio graph is connected to the microphone stream. */
|
||||
waitForVadReady: () => Promise<void>
|
||||
/** Waits until a streaming transcription transport accepts microphone audio. */
|
||||
waitForStreamingTranscriptionReady: () => Promise<void>
|
||||
/**
|
||||
* Waits for the next matching Pinia action event after this method is called.
|
||||
*
|
||||
* @example
|
||||
* const completed = audio.waitForPiniaAction({
|
||||
* storeId: 'modules:hearing:speech:audio-input-pipeline',
|
||||
* actionName: 'transcribeForMediaStream',
|
||||
* })
|
||||
* await enableButton.click()
|
||||
* await completed
|
||||
*/
|
||||
waitForPiniaAction: (options: {
|
||||
storeId: string
|
||||
actionName: string
|
||||
/** @default 'completed' */
|
||||
status?: PiniaActionEventStatus
|
||||
/** @default 60000 */
|
||||
timeout?: number
|
||||
}) => Promise<PiniaActionEvent>
|
||||
}
|
||||
|
||||
/** Runtime handle for one AIRI audio-input test. */
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
|
||||
import type { PiniaActionEvent } from '@proj-airi/stage-shared/types/pinia-action-event'
|
||||
|
||||
declare global {
|
||||
interface BrowserAudioInputState {
|
||||
piniaActionEvents: PiniaActionEvent[]
|
||||
spans: SerializedIOSpan[]
|
||||
streamingTranscriptionReady: boolean
|
||||
streamingTranscriptionUpdates: string[]
|
||||
|
||||
Reference in New Issue
Block a user