mirror of
https://github.com/moeru-ai/airi.git
synced 2026-08-14 00:48:06 +00:00
refactor(testing-audio,stage-ui): added pinia-plugin-tracing, directly assert from history of pinia
This commit is contained in:
@@ -7,7 +7,7 @@ import { autoAnimatePlugin } from '@formkit/auto-animate/vue'
|
|||||||
import { PiniaColada } from '@pinia/colada'
|
import { PiniaColada } from '@pinia/colada'
|
||||||
import { trackButtonPlugin } from '@proj-airi/stage-ui/directives/track-button'
|
import { trackButtonPlugin } from '@proj-airi/stage-ui/directives/track-button'
|
||||||
import { configureAnalyticsAdapter } from '@proj-airi/stage-ui/libs/analytics'
|
import { configureAnalyticsAdapter } from '@proj-airi/stage-ui/libs/analytics'
|
||||||
import { setupSynced } from '@proj-airi/stage-ui/libs/pinia'
|
import { piniaPluginTracing, setupSynced } from '@proj-airi/stage-ui/libs/pinia'
|
||||||
import { MotionPlugin } from '@vueuse/motion'
|
import { MotionPlugin } from '@vueuse/motion'
|
||||||
import { createPinia } from 'pinia'
|
import { createPinia } from 'pinia'
|
||||||
import { setupLayouts } from 'virtual:generated-layouts'
|
import { setupLayouts } from 'virtual:generated-layouts'
|
||||||
@@ -47,6 +47,7 @@ configureAnalyticsAdapter(async (options) => {
|
|||||||
const pinia = createPinia()
|
const pinia = createPinia()
|
||||||
const synced = setupSynced()
|
const synced = setupSynced()
|
||||||
pinia.use(synced.pinia)
|
pinia.use(synced.pinia)
|
||||||
|
pinia.use(piniaPluginTracing)
|
||||||
|
|
||||||
const router = createRouter({
|
const router = createRouter({
|
||||||
history: createWebHashHistory(),
|
history: createWebHashHistory(),
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import { PiniaColada } from '@pinia/colada'
|
|||||||
import { isEnvTruthy } from '@proj-airi/stage-shared'
|
import { isEnvTruthy } from '@proj-airi/stage-shared'
|
||||||
import { trackButtonPlugin } from '@proj-airi/stage-ui/directives/track-button'
|
import { trackButtonPlugin } from '@proj-airi/stage-ui/directives/track-button'
|
||||||
import { configureAnalyticsAdapter } from '@proj-airi/stage-ui/libs/analytics'
|
import { configureAnalyticsAdapter } from '@proj-airi/stage-ui/libs/analytics'
|
||||||
import { setupSynced } from '@proj-airi/stage-ui/libs/pinia'
|
import { piniaPluginTracing, setupSynced } from '@proj-airi/stage-ui/libs/pinia'
|
||||||
import { MotionPlugin } from '@vueuse/motion'
|
import { MotionPlugin } from '@vueuse/motion'
|
||||||
import { createPinia } from 'pinia'
|
import { createPinia } from 'pinia'
|
||||||
import { setupLayouts } from 'virtual:generated-layouts'
|
import { setupLayouts } from 'virtual:generated-layouts'
|
||||||
@@ -37,6 +37,7 @@ configureAnalyticsAdapter(async (options) => {
|
|||||||
const pinia = createPinia()
|
const pinia = createPinia()
|
||||||
const synced = setupSynced()
|
const synced = setupSynced()
|
||||||
pinia.use(synced.pinia)
|
pinia.use(synced.pinia)
|
||||||
|
pinia.use(piniaPluginTracing)
|
||||||
|
|
||||||
// TODO: vite-plugin-vue-layouts is long deprecated, replace with another layout solution
|
// TODO: vite-plugin-vue-layouts is long deprecated, replace with another layout solution
|
||||||
const routeRecords = setupLayouts(routes as RouteRecordRaw[])
|
const routeRecords = setupLayouts(routes as RouteRecordRaw[])
|
||||||
|
|||||||
@@ -20,8 +20,10 @@
|
|||||||
"./beat-sync": "./src/beat-sync/index.ts",
|
"./beat-sync": "./src/beat-sync/index.ts",
|
||||||
"./global-shortcut": "./src/global-shortcut/index.ts",
|
"./global-shortcut": "./src/global-shortcut/index.ts",
|
||||||
"./godot-stage": "./src/godot-stage/index.ts",
|
"./godot-stage": "./src/godot-stage/index.ts",
|
||||||
|
"./perf/io-trace": "./src/perf/io-trace.ts",
|
||||||
"./server-channel-qr": "./src/server-channel-qr.ts",
|
"./server-channel-qr": "./src/server-channel-qr.ts",
|
||||||
"./types/io-trace": "./src/types/io-trace.ts",
|
"./types/io-trace": "./src/types/io-trace.ts",
|
||||||
|
"./types/pinia-action-event": "./src/types/pinia-action-event.ts",
|
||||||
"./electron-renderer": "./src/electron-renderer.d.ts",
|
"./electron-renderer": "./src/electron-renderer.d.ts",
|
||||||
"./composables": "./src/composables/index.ts",
|
"./composables": "./src/composables/index.ts",
|
||||||
"./webgpu": "./src/webgpu/index.ts"
|
"./webgpu": "./src/webgpu/index.ts"
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ export type IOSubsystem = (typeof IOSubsystems)[keyof typeof IOSubsystems]
|
|||||||
|
|
||||||
export const IOSpanNames = {
|
export const IOSpanNames = {
|
||||||
InteractionTurn: 'Interaction turn',
|
InteractionTurn: 'Interaction turn',
|
||||||
|
SpeechTurn: 'Speech turn',
|
||||||
VoiceActivityDetection: 'Voice activity detection',
|
VoiceActivityDetection: 'Voice activity detection',
|
||||||
SpeechRecognition: 'Speech recognition',
|
SpeechRecognition: 'Speech recognition',
|
||||||
LLMInference: 'LLM inference',
|
LLMInference: 'LLM inference',
|
||||||
@@ -32,6 +33,11 @@ export const IOAttributes = {
|
|||||||
VADAborted: `${customPrefix}.vad.aborted`,
|
VADAborted: `${customPrefix}.vad.aborted`,
|
||||||
ASRText: `${customPrefix}.asr.text`,
|
ASRText: `${customPrefix}.asr.text`,
|
||||||
ASRAbort: `${customPrefix}.asr.abort`,
|
ASRAbort: `${customPrefix}.asr.abort`,
|
||||||
|
LLMInputMessageCount: `${customPrefix}.llm.input_message_count`,
|
||||||
|
LLMInputMessageRoles: `${customPrefix}.llm.input_message_roles`,
|
||||||
|
LLMInputUserMessageCount: `${customPrefix}.llm.input_user_message_count`,
|
||||||
|
LLMOutputChunkCount: `${customPrefix}.llm.output_chunk_count`,
|
||||||
|
LLMOutputChunkLengths: `${customPrefix}.llm.output_chunk_lengths`,
|
||||||
LLMTextLength: `${customPrefix}.llm.text_length`,
|
LLMTextLength: `${customPrefix}.llm.text_length`,
|
||||||
StreamingControlCallName: `${customPrefix}.streaming_control.call_name`,
|
StreamingControlCallName: `${customPrefix}.streaming_control.call_name`,
|
||||||
StreamingControlHandlerCount: `${customPrefix}.streaming_control.handler_count`,
|
StreamingControlHandlerCount: `${customPrefix}.streaming_control.handler_count`,
|
||||||
@@ -47,9 +53,11 @@ export const IOAttributes = {
|
|||||||
TTSSegmentId: `${customPrefix}.tts.segment_id`,
|
TTSSegmentId: `${customPrefix}.tts.segment_id`,
|
||||||
TTSText: `${customPrefix}.tts.text`,
|
TTSText: `${customPrefix}.tts.text`,
|
||||||
TTSChunkReason: `${customPrefix}.tts.chunk_reason`,
|
TTSChunkReason: `${customPrefix}.tts.chunk_reason`,
|
||||||
|
TTSAudioDurationMs: `${customPrefix}.tts.audio_duration_ms`,
|
||||||
TTSInterrupted: `${customPrefix}.tts.interrupted`,
|
TTSInterrupted: `${customPrefix}.tts.interrupted`,
|
||||||
TTSInterruptReason: `${customPrefix}.tts.interrupt_reason`,
|
TTSInterruptReason: `${customPrefix}.tts.interrupt_reason`,
|
||||||
TTSCanceled: `${customPrefix}.tts.canceled`,
|
TTSCanceled: `${customPrefix}.tts.canceled`,
|
||||||
|
TurnId: `${customPrefix}.turn_id`,
|
||||||
} as const
|
} as const
|
||||||
|
|
||||||
export const IOEvents = {
|
export const IOEvents = {
|
||||||
|
|||||||
@@ -0,0 +1,21 @@
|
|||||||
|
/** Lifecycle phase observed for one Pinia action invocation. */
|
||||||
|
export type PiniaActionEventStatus = 'started' | 'completed' | 'failed'
|
||||||
|
|
||||||
|
/** Broadcast channel used by Pinia action tracing producers and consumers. */
|
||||||
|
export const piniaActionTracingChannelName = 'airi-pinia-action-tracing'
|
||||||
|
|
||||||
|
/** Test-safe metadata emitted for one Pinia action lifecycle transition. */
|
||||||
|
export interface PiniaActionEvent {
|
||||||
|
/** Correlates lifecycle events from the same action invocation. */
|
||||||
|
invocationId: string
|
||||||
|
/** Name passed to `defineStore`. */
|
||||||
|
storeId: string
|
||||||
|
/** Action property name reported by Pinia. */
|
||||||
|
actionName: string
|
||||||
|
status: PiniaActionEventStatus
|
||||||
|
timestamp: number
|
||||||
|
/** Renderer URL that invoked the action, when available. */
|
||||||
|
sourceUrl?: string
|
||||||
|
/** Safe error text included only for failed actions. */
|
||||||
|
errorMessage?: string
|
||||||
|
}
|
||||||
@@ -26,7 +26,6 @@ import { useBroadcastChannel } from '@vueuse/core'
|
|||||||
// import { createTransformers } from '@xsai-transformers/embed'
|
// import { createTransformers } from '@xsai-transformers/embed'
|
||||||
// import embedWorkerURL from '@xsai-transformers/embed/worker?worker&url'
|
// import embedWorkerURL from '@xsai-transformers/embed/worker?worker&url'
|
||||||
// import { embed } from '@xsai/embed'
|
// import { embed } from '@xsai/embed'
|
||||||
import { generateSpeech } from '@xsai/generate-speech'
|
|
||||||
import { storeToRefs } from 'pinia'
|
import { storeToRefs } from 'pinia'
|
||||||
import { computed, nextTick, onMounted, onUnmounted, ref, shallowRef, watch } from 'vue'
|
import { computed, nextTick, onMounted, onUnmounted, ref, shallowRef, watch } from 'vue'
|
||||||
|
|
||||||
@@ -520,24 +519,18 @@ const speechPipeline = createSpeechPipeline<AudioBuffer>({
|
|||||||
// Non-streaming providers only: synth via REST. Streaming provider
|
// Non-streaming providers only: synth via REST. Streaming provider
|
||||||
// was already early-returned above; it owns its own ws path opened
|
// was already early-returned above; it owns its own ws path opened
|
||||||
// in `onBeforeMessageComposed`.
|
// in `onBeforeMessageComposed`.
|
||||||
const providerConfigWithAnalytics = activeSpeechProvider.value === OFFICIAL_SPEECH_PROVIDER_ID
|
const res = await speechStore.speech(
|
||||||
? {
|
provider,
|
||||||
...speechRequest.providerConfig,
|
model,
|
||||||
extraBody: {
|
speechRequest.input,
|
||||||
...(speechRequest.providerConfig.extraBody as Record<string, unknown> | undefined),
|
voice.id,
|
||||||
airi_analytics: {
|
speechRequest.providerConfig,
|
||||||
trigger: 'auto',
|
{
|
||||||
source: 'chat_auto_tts',
|
trigger: 'auto',
|
||||||
voice_type: resolveStageVoiceType(),
|
source: 'chat_auto_tts',
|
||||||
},
|
voice_type: resolveStageVoiceType(),
|
||||||
},
|
},
|
||||||
}
|
)
|
||||||
: speechRequest.providerConfig
|
|
||||||
const res = await generateSpeech({
|
|
||||||
...provider.speech(model, providerConfigWithAnalytics),
|
|
||||||
input: speechRequest.input,
|
|
||||||
voice: voice.id,
|
|
||||||
})
|
|
||||||
|
|
||||||
if (signal.aborted || !res || res.byteLength === 0)
|
if (signal.aborted || !res || res.byteLength === 0)
|
||||||
return null
|
return null
|
||||||
|
|||||||
@@ -11,8 +11,25 @@ export function useIOTraceBridge(pipeline: ReturnType<typeof createSpeechPipelin
|
|||||||
|
|
||||||
const synthesisSpans = new Map<string, Span>()
|
const synthesisSpans = new Map<string, Span>()
|
||||||
const playbackSpans = new Map<string, Span>()
|
const playbackSpans = new Map<string, Span>()
|
||||||
|
const speechTurnSpans = new Map<string, Span>()
|
||||||
const segmentReasons = new Map<string, string>()
|
const segmentReasons = new Map<string, string>()
|
||||||
|
|
||||||
|
cleanupFns.push(pipeline.on('onTurnStart', (turnId) => {
|
||||||
|
speechTurnSpans.set(turnId, startSpan(IOSpanNames.SpeechTurn, activeTurnSpan.value, {
|
||||||
|
[IOAttributes.TurnId]: turnId,
|
||||||
|
}))
|
||||||
|
}))
|
||||||
|
|
||||||
|
cleanupFns.push(pipeline.on('onTurnEnd', (turnId) => {
|
||||||
|
speechTurnSpans.get(turnId)?.end()
|
||||||
|
speechTurnSpans.delete(turnId)
|
||||||
|
}))
|
||||||
|
|
||||||
|
cleanupFns.push(pipeline.on('onTurnCancel', ({ turnId }) => {
|
||||||
|
speechTurnSpans.get(turnId)?.end()
|
||||||
|
speechTurnSpans.delete(turnId)
|
||||||
|
}))
|
||||||
|
|
||||||
cleanupFns.push(pipeline.on('onSegment', (segment) => {
|
cleanupFns.push(pipeline.on('onSegment', (segment) => {
|
||||||
segmentReasons.set(segment.segmentId, segment.reason)
|
segmentReasons.set(segment.segmentId, segment.reason)
|
||||||
}))
|
}))
|
||||||
@@ -23,6 +40,7 @@ export function useIOTraceBridge(pipeline: ReturnType<typeof createSpeechPipelin
|
|||||||
[IOAttributes.TTSSegmentId]: request.segmentId,
|
[IOAttributes.TTSSegmentId]: request.segmentId,
|
||||||
[IOAttributes.TTSText]: request.text,
|
[IOAttributes.TTSText]: request.text,
|
||||||
[IOAttributes.TTSChunkReason]: segmentReasons.get(request.segmentId) ?? '',
|
[IOAttributes.TTSChunkReason]: segmentReasons.get(request.segmentId) ?? '',
|
||||||
|
[IOAttributes.TurnId]: request.turnId ?? '',
|
||||||
})
|
})
|
||||||
segmentReasons.delete(request.segmentId)
|
segmentReasons.delete(request.segmentId)
|
||||||
synthesisSpans.set(request.segmentId, ttsSynthesisSpan)
|
synthesisSpans.set(request.segmentId, ttsSynthesisSpan)
|
||||||
@@ -31,6 +49,8 @@ export function useIOTraceBridge(pipeline: ReturnType<typeof createSpeechPipelin
|
|||||||
cleanupFns.push(pipeline.on('onTtsResult', (result) => {
|
cleanupFns.push(pipeline.on('onTtsResult', (result) => {
|
||||||
const span = synthesisSpans.get(result.segmentId)
|
const span = synthesisSpans.get(result.segmentId)
|
||||||
if (span) {
|
if (span) {
|
||||||
|
if (result.audio instanceof AudioBuffer)
|
||||||
|
span.setAttribute(IOAttributes.TTSAudioDurationMs, result.audio.duration * 1000)
|
||||||
span.end()
|
span.end()
|
||||||
synthesisSpans.delete(result.segmentId)
|
synthesisSpans.delete(result.segmentId)
|
||||||
}
|
}
|
||||||
@@ -77,6 +97,8 @@ export function useIOTraceBridge(pipeline: ReturnType<typeof createSpeechPipelin
|
|||||||
}))
|
}))
|
||||||
|
|
||||||
onScopeDispose(() => {
|
onScopeDispose(() => {
|
||||||
|
for (const span of speechTurnSpans.values())
|
||||||
|
span.end()
|
||||||
for (const cleanup of cleanupFns)
|
for (const cleanup of cleanupFns)
|
||||||
cleanup()
|
cleanup()
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -1 +1,2 @@
|
|||||||
|
export * from './pinia-plugin-tracing'
|
||||||
export * from './setup-synced'
|
export * from './setup-synced'
|
||||||
|
|||||||
@@ -0,0 +1,83 @@
|
|||||||
|
import type { PiniaActionEvent } from '@proj-airi/stage-shared/types/pinia-action-event'
|
||||||
|
|
||||||
|
import { piniaActionTracingChannelName } from '@proj-airi/stage-shared/types/pinia-action-event'
|
||||||
|
import { createPinia, defineStore } from 'pinia'
|
||||||
|
import { describe, expect, it } from 'vitest'
|
||||||
|
import { createApp } from 'vue'
|
||||||
|
|
||||||
|
import { piniaPluginTracing } from './pinia-plugin-tracing'
|
||||||
|
|
||||||
|
describe('piniaPluginTracing over BroadcastChannel', () => {
|
||||||
|
it('emits correlated start and completion events without action values', async () => {
|
||||||
|
const observer = observeActionEvents(2)
|
||||||
|
|
||||||
|
const pinia = createPinia()
|
||||||
|
pinia.use(piniaPluginTracing)
|
||||||
|
createApp({}).use(pinia)
|
||||||
|
|
||||||
|
const useStore = defineStore('action-event-test', {
|
||||||
|
actions: {
|
||||||
|
async complete(secret: string) {
|
||||||
|
return `returned:${secret}`
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
await useStore(pinia).complete('must-not-be-recorded')
|
||||||
|
const events = await observer.events
|
||||||
|
|
||||||
|
observer.close()
|
||||||
|
expect(events).toHaveLength(2)
|
||||||
|
expect(events.map(event => event.status)).toEqual(['started', 'completed'])
|
||||||
|
expect(events[0]?.invocationId).toBe(events[1]?.invocationId)
|
||||||
|
for (const event of events) {
|
||||||
|
expect(event).toEqual(expect.objectContaining({
|
||||||
|
actionName: 'complete',
|
||||||
|
storeId: 'action-event-test',
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
expect(JSON.stringify(events)).not.toContain('must-not-be-recorded')
|
||||||
|
expect(JSON.stringify(events)).not.toContain('returned:')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('emits a failed event without changing the action error', async () => {
|
||||||
|
const observer = observeActionEvents(2)
|
||||||
|
|
||||||
|
const pinia = createPinia()
|
||||||
|
pinia.use(piniaPluginTracing)
|
||||||
|
createApp({}).use(pinia)
|
||||||
|
|
||||||
|
const failure = new Error('provider unavailable')
|
||||||
|
const useStore = defineStore('failed-action-test', {
|
||||||
|
actions: {
|
||||||
|
fail() {
|
||||||
|
throw failure
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
expect(() => useStore(pinia).fail()).toThrow(failure)
|
||||||
|
const events = await observer.events
|
||||||
|
|
||||||
|
observer.close()
|
||||||
|
expect(events.map(event => event.status)).toEqual(['started', 'failed'])
|
||||||
|
expect(events[1]?.errorMessage).toBe('provider unavailable')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
function observeActionEvents(count: number) {
|
||||||
|
const channel = new BroadcastChannel(piniaActionTracingChannelName)
|
||||||
|
const captured: PiniaActionEvent[] = []
|
||||||
|
const events = new Promise<PiniaActionEvent[]>((resolve) => {
|
||||||
|
channel.addEventListener('message', (message: MessageEvent<PiniaActionEvent>) => {
|
||||||
|
captured.push(message.data)
|
||||||
|
if (captured.length === count)
|
||||||
|
resolve(captured)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
return {
|
||||||
|
close: () => channel.close(),
|
||||||
|
events,
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
import type { PiniaActionEvent, PiniaActionEventStatus } from '@proj-airi/stage-shared/types/pinia-action-event'
|
||||||
|
import type { PiniaPlugin } from 'pinia'
|
||||||
|
|
||||||
|
import { errorMessageFrom } from '@moeru/std'
|
||||||
|
import { piniaActionTracingChannelName } from '@proj-airi/stage-shared/types/pinia-action-event'
|
||||||
|
import { nanoid } from 'nanoid/non-secure'
|
||||||
|
|
||||||
|
const piniaActionChannel = new BroadcastChannel(piniaActionTracingChannelName)
|
||||||
|
|
||||||
|
function emitActionEvent(
|
||||||
|
event: Omit<PiniaActionEvent, 'status' | 'timestamp'>,
|
||||||
|
status: PiniaActionEventStatus,
|
||||||
|
error?: unknown,
|
||||||
|
): void {
|
||||||
|
piniaActionChannel.postMessage({
|
||||||
|
...event,
|
||||||
|
status,
|
||||||
|
timestamp: Date.now(),
|
||||||
|
...(status === 'failed' ? { errorMessage: errorMessageFrom(error) ?? 'Unknown action failure' } : {}),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Traces Pinia action lifecycle events through a broadcast channel.
|
||||||
|
*
|
||||||
|
* The plugin never retains action arguments, results, or state snapshots.
|
||||||
|
*/
|
||||||
|
export const piniaPluginTracing: PiniaPlugin = ({ store }) => {
|
||||||
|
store.$onAction(({ name, after, onError }) => {
|
||||||
|
const event = {
|
||||||
|
invocationId: nanoid(),
|
||||||
|
storeId: store.$id,
|
||||||
|
actionName: name,
|
||||||
|
...(typeof location === 'undefined' ? {} : { sourceUrl: location.href }),
|
||||||
|
}
|
||||||
|
|
||||||
|
emitActionEvent(event, 'started')
|
||||||
|
after(() => emitActionEvent(event, 'completed'))
|
||||||
|
onError(error => emitActionEvent(event, 'failed', error))
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
import type { ChatProvider } from '@xsai-ext/providers/utils'
|
import type { ChatProvider } from '@xsai-ext/providers/utils'
|
||||||
import type { Message, Tool } from '@xsai/shared-chat'
|
import type { Message, Tool } from '@xsai/shared-chat'
|
||||||
|
|
||||||
import { IOSpanNames } from '@proj-airi/stage-shared'
|
import { IOAttributes, IOSpanNames } from '@proj-airi/stage-shared'
|
||||||
import { createPinia, setActivePinia } from 'pinia'
|
import { createPinia, setActivePinia } from 'pinia'
|
||||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||||
import { nextTick, ref } from 'vue'
|
import { nextTick, ref } from 'vue'
|
||||||
@@ -584,6 +584,20 @@ describe('chat store contract', () => {
|
|||||||
expect(composedMessages).toHaveLength(2)
|
expect(composedMessages).toHaveLength(2)
|
||||||
expect(composedMessages[0]).toMatchObject({ role: 'system' })
|
expect(composedMessages[0]).toMatchObject({ role: 'system' })
|
||||||
expect(composedMessages[1]).toMatchObject({ role: 'user' })
|
expect(composedMessages[1]).toMatchObject({ role: 'user' })
|
||||||
|
expect(ioTracerMocks.startSpanMock).toHaveBeenCalledWith(
|
||||||
|
IOSpanNames.LLMInference,
|
||||||
|
expect.anything(),
|
||||||
|
expect.objectContaining({
|
||||||
|
[IOAttributes.LLMInputMessageCount]: 2,
|
||||||
|
[IOAttributes.LLMInputUserMessageCount]: 1,
|
||||||
|
[IOAttributes.TurnId]: expect.any(String),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
const llmSpan = ioTracerMocks.spans.find(span => span.name === IOSpanNames.LLMInference)
|
||||||
|
expect(llmSpan.setAttribute).toHaveBeenCalledWith(IOAttributes.LLMInputMessageRoles, ['system', 'user'])
|
||||||
|
expect(llmSpan.setAttribute).toHaveBeenCalledWith(IOAttributes.LLMOutputChunkCount, 1)
|
||||||
|
expect(llmSpan.setAttribute).toHaveBeenCalledWith(IOAttributes.LLMOutputChunkLengths, [5])
|
||||||
|
expect(llmSpan.setAttribute).toHaveBeenCalledWith(IOAttributes.LLMTextLength, 5)
|
||||||
|
|
||||||
// System message stays untouched: keeping it 100% static is what makes
|
// System message stays untouched: keeping it 100% static is what makes
|
||||||
// the prefix permanently KV-cache friendly across turns and across day
|
// the prefix permanently KV-cache friendly across turns and across day
|
||||||
|
|||||||
@@ -171,6 +171,8 @@ export const useChatStore = defineStore('chat', () => {
|
|||||||
options?: StreamOptions,
|
options?: StreamOptions,
|
||||||
) {
|
) {
|
||||||
let llmTextLength = 0
|
let llmTextLength = 0
|
||||||
|
let llmOutputChunkCount = 0
|
||||||
|
const llmOutputChunkLengths: number[] = []
|
||||||
const headers = { ...options?.headers }
|
const headers = { ...options?.headers }
|
||||||
if (getProviderMode(activeProvider.value) === 'official' && options?.requestCorrelation) {
|
if (getProviderMode(activeProvider.value) === 'official' && options?.requestCorrelation) {
|
||||||
headers[AIRI_CHAT_SESSION_ID_HEADER] = options.requestCorrelation.conversationId
|
headers[AIRI_CHAT_SESSION_ID_HEADER] = options.requestCorrelation.conversationId
|
||||||
@@ -188,7 +190,11 @@ export const useChatStore = defineStore('chat', () => {
|
|||||||
const llmSpan = startSpan(IOSpanNames.LLMInference, activeTurnSpan.value, {
|
const llmSpan = startSpan(IOSpanNames.LLMInference, activeTurnSpan.value, {
|
||||||
[IOAttributes.Subsystem]: IOSubsystems.LLM,
|
[IOAttributes.Subsystem]: IOSubsystems.LLM,
|
||||||
[IOAttributes.GenAIRequestModel]: model,
|
[IOAttributes.GenAIRequestModel]: model,
|
||||||
|
[IOAttributes.LLMInputMessageCount]: messages.length,
|
||||||
|
[IOAttributes.LLMInputUserMessageCount]: messages.filter(message => message.role === 'user').length,
|
||||||
|
[IOAttributes.TurnId]: options?.requestCorrelation?.roundId ?? '',
|
||||||
})
|
})
|
||||||
|
llmSpan.setAttribute(IOAttributes.LLMInputMessageRoles, messages.map(message => message.role))
|
||||||
const llmRequestTs = performance.now()
|
const llmRequestTs = performance.now()
|
||||||
let llmFirstTokenEmitted = false
|
let llmFirstTokenEmitted = false
|
||||||
|
|
||||||
@@ -198,6 +204,8 @@ export const useChatStore = defineStore('chat', () => {
|
|||||||
headers,
|
headers,
|
||||||
onStreamEvent: async (event: StreamEvent) => {
|
onStreamEvent: async (event: StreamEvent) => {
|
||||||
if (isTextDelta(event)) {
|
if (isTextDelta(event)) {
|
||||||
|
llmOutputChunkCount += 1
|
||||||
|
llmOutputChunkLengths.push(event.text.length)
|
||||||
if (!llmFirstTokenEmitted) {
|
if (!llmFirstTokenEmitted) {
|
||||||
llmFirstTokenEmitted = true
|
llmFirstTokenEmitted = true
|
||||||
llmSpan.addEvent(IOEvents.LLMFirstToken, {
|
llmSpan.addEvent(IOEvents.LLMFirstToken, {
|
||||||
@@ -210,10 +218,11 @@ export const useChatStore = defineStore('chat', () => {
|
|||||||
await options?.onStreamEvent?.(event)
|
await options?.onStreamEvent?.(event)
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
llmSpan.setAttribute(IOAttributes.LLMTextLength, llmTextLength)
|
|
||||||
}
|
}
|
||||||
finally {
|
finally {
|
||||||
|
llmSpan.setAttribute(IOAttributes.LLMOutputChunkCount, llmOutputChunkCount)
|
||||||
|
llmSpan.setAttribute(IOAttributes.LLMOutputChunkLengths, llmOutputChunkLengths)
|
||||||
|
llmSpan.setAttribute(IOAttributes.LLMTextLength, llmTextLength)
|
||||||
llmSpan.end()
|
llmSpan.end()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -37,6 +37,12 @@ interface SpeechInput {
|
|||||||
providerConfig: Record<string, unknown>
|
providerConfig: Record<string, unknown>
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface SpeechAnalytics {
|
||||||
|
trigger: 'auto' | 'manual'
|
||||||
|
source: 'chat_auto_tts' | 'manual_preview' | 'settings_test'
|
||||||
|
voice_type?: 'official_default' | 'official_selected' | 'custom_configured' | 'voice_pack'
|
||||||
|
}
|
||||||
|
|
||||||
export const useSpeechStore = defineStore('speech', () => {
|
export const useSpeechStore = defineStore('speech', () => {
|
||||||
const providersStore = useProviderStore()
|
const providersStore = useProviderStore()
|
||||||
const providerStore = useProviderConfigStore()
|
const providerStore = useProviderConfigStore()
|
||||||
@@ -301,14 +307,15 @@ export const useSpeechStore = defineStore('speech', () => {
|
|||||||
input: string,
|
input: string,
|
||||||
voice: string,
|
voice: string,
|
||||||
providerConfig: Record<string, any> = {},
|
providerConfig: Record<string, any> = {},
|
||||||
|
analytics: SpeechAnalytics = {
|
||||||
|
trigger: 'manual',
|
||||||
|
source: 'manual_preview',
|
||||||
|
voice_type: resolveVoiceType(voice),
|
||||||
|
},
|
||||||
): Promise<ArrayBuffer> {
|
): Promise<ArrayBuffer> {
|
||||||
const requestProviderConfig = activeSpeechProvider.value === OFFICIAL_SPEECH_PROVIDER_ID
|
const requestProviderConfig = activeSpeechProvider.value === OFFICIAL_SPEECH_PROVIDER_ID
|
||||||
|| activeSpeechProvider.value === OFFICIAL_SPEECH_STREAMING_PROVIDER_ID
|
|| activeSpeechProvider.value === OFFICIAL_SPEECH_STREAMING_PROVIDER_ID
|
||||||
? withAiriTtsAnalytics(providerConfig, {
|
? withAiriTtsAnalytics(providerConfig, analytics)
|
||||||
trigger: 'manual',
|
|
||||||
source: 'manual_preview',
|
|
||||||
voice_type: resolveVoiceType(voice),
|
|
||||||
})
|
|
||||||
: providerConfig
|
: providerConfig
|
||||||
const response = await generateSpeech({
|
const response = await generateSpeech({
|
||||||
...provider.speech(model, requestProviderConfig),
|
...provider.speech(model, requestProviderConfig),
|
||||||
@@ -321,11 +328,7 @@ export const useSpeechStore = defineStore('speech', () => {
|
|||||||
|
|
||||||
function withAiriTtsAnalytics(
|
function withAiriTtsAnalytics(
|
||||||
providerConfig: Record<string, any>,
|
providerConfig: Record<string, any>,
|
||||||
analytics: {
|
analytics: SpeechAnalytics,
|
||||||
trigger: 'auto' | 'manual'
|
|
||||||
source: 'chat_auto_tts' | 'manual_preview' | 'settings_test'
|
|
||||||
voice_type?: 'official_default' | 'official_selected' | 'custom_configured' | 'voice_pack'
|
|
||||||
},
|
|
||||||
): Record<string, any> {
|
): Record<string, any> {
|
||||||
return {
|
return {
|
||||||
...providerConfig,
|
...providerConfig,
|
||||||
|
|||||||
@@ -43,6 +43,7 @@ describe('audio input pipeline', () => {
|
|||||||
await expect(audio).not.toHaveTranscriptions([
|
await expect(audio).not.toHaveTranscriptions([
|
||||||
['There is no meaning to your existence, just let go.'],
|
['There is no meaning to your existence, just let go.'],
|
||||||
])
|
])
|
||||||
|
await expect(audio).toHaveCompletedTranscription()
|
||||||
})
|
})
|
||||||
|
|
||||||
it('does not preserve the complete phrase with Aliyun NLS', {
|
it('does not preserve the complete phrase with Aliyun NLS', {
|
||||||
@@ -78,5 +79,6 @@ describe('audio input pipeline', () => {
|
|||||||
await expect(audio).not.toHaveTranscriptions([
|
await expect(audio).not.toHaveTranscriptions([
|
||||||
['There is no meaning to your existence, just let go.'],
|
['There is no meaning to your existence, just let go.'],
|
||||||
])
|
])
|
||||||
|
await expect(audio).toHaveCompletedTranscription()
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -1,12 +1,12 @@
|
|||||||
import { describe, expect, it } from '../../src'
|
import { describe, expect, it } from '../../src'
|
||||||
import { configureModuleConsciousness, configureModuleHearing, configureModuleSpeech, configureOnboarding, loadCaseEnvironment } from '../shared/configurations'
|
import { configureModuleConsciousness, configureModuleHearing, configureModuleSpeech, configureOnboarding, loadCaseEnvironment } from '../shared/configurations'
|
||||||
import { assistantMessages, enableChatMicrophone, openChat } from '../shared/interactions'
|
import { enableChatMicrophone } from '../shared/interactions'
|
||||||
import { aliyunNlsAsr, openaiAsr, openaiLlm, openaiTts } from '../shared/providers'
|
import { aliyunNlsAsr, openaiAsr, openaiLlm, openaiTts } from '../shared/providers'
|
||||||
|
|
||||||
describe('audio input pipeline', () => {
|
describe('audio input pipeline', () => {
|
||||||
// An OpenAI-compatible TTS Provider generated the fixture in mono 16 kHz PCM WAV format.
|
// An OpenAI-compatible TTS Provider generated the fixture in mono 16 kHz PCM WAV format.
|
||||||
// The fixture contains 14 seconds of leading silence for VAD initialization and 3 seconds of trailing silence.
|
// The fixture contains 14 seconds of leading silence for VAD initialization and 3 seconds of trailing silence.
|
||||||
// Its warm-up phrase gives the VAD time to start. Only "Please say hello." is required in the transcript.
|
// Its warm-up phrase gives the VAD time to start. The final "say hello" command is required in the transcript.
|
||||||
it('runs an OpenAI-compatible request through the complete pipeline', {
|
it('runs an OpenAI-compatible request through the complete pipeline', {
|
||||||
input: new URL('./input.test.wav', import.meta.url),
|
input: new URL('./input.test.wav', import.meta.url),
|
||||||
// This case keeps AIRI's default VAD and selects every remote Provider explicitly.
|
// This case keeps AIRI's default VAD and selects every remote Provider explicitly.
|
||||||
@@ -72,16 +72,22 @@ describe('audio input pipeline', () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
await expect(audio).toHaveTranscriptions([
|
await expect(audio).toHaveTranscriptions([
|
||||||
['Please say hello.'],
|
['Please say hello.', 'Say hello.'],
|
||||||
], { match: 'contains' })
|
], { match: 'contains' })
|
||||||
|
|
||||||
await expect.poll(async () => (await audio.completedSpans('LLM inference')).length, { timeout: 60_000 }).toBeGreaterThanOrEqual(1)
|
const turn = await audio.waitForTurn()
|
||||||
await expect.poll(async () => (await audio.completedSpans('TTS synthesis')).length, { timeout: 60_000 }).toBeGreaterThanOrEqual(1)
|
|
||||||
await expect.poll(async () => (await audio.completedSpans('Audio playback')).length, { timeout: 60_000 }).toBeGreaterThanOrEqual(1)
|
|
||||||
|
|
||||||
await openChat(audio)
|
expect(turn.chat.messages).toHaveLength(2)
|
||||||
const messages = await assistantMessages(audio).allTextContents()
|
expect(turn.chat.messages.map(message => message.role)).toEqual(['user', 'assistant'])
|
||||||
expect(messages.at(-1)).toMatch(/.+/s)
|
expect(turn.llm.inputMessages).toHaveLength(2)
|
||||||
|
expect(turn.llm.inputMessages.filter(message => message.role === 'user')).toHaveLength(1)
|
||||||
|
expect(turn.llm.outputChunks.length).toBeGreaterThan(0)
|
||||||
|
expect(turn.llm.outputCharacters).toBeGreaterThan(0)
|
||||||
|
expect(turn.tts.audioSegments.length).toBeGreaterThan(0)
|
||||||
|
for (const segment of turn.tts.audioSegments) {
|
||||||
|
expect(segment.text.length).toBeGreaterThan(0)
|
||||||
|
expect(segment.durationMs).toBeGreaterThan(100)
|
||||||
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
it('transcribes the greeting with Aliyun NLS', {
|
it('transcribes the greeting with Aliyun NLS', {
|
||||||
@@ -117,5 +123,6 @@ describe('audio input pipeline', () => {
|
|||||||
await expect(audio).toHaveTranscriptions([
|
await expect(audio).toHaveTranscriptions([
|
||||||
['Please say hello.'],
|
['Please say hello.'],
|
||||||
], { match: 'contains' })
|
], { match: 'contains' })
|
||||||
|
await expect(audio).toHaveCompletedTranscription()
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -54,6 +54,7 @@ describe('audio input pipeline', () => {
|
|||||||
const finalTranscriptions = await audio.transcriptionResults(expectedTranscriptions.length)
|
const finalTranscriptions = await audio.transcriptionResults(expectedTranscriptions.length)
|
||||||
const streamingUpdates = await audio.streamingTranscriptionUpdates()
|
const streamingUpdates = await audio.streamingTranscriptionUpdates()
|
||||||
expect(streamingUpdates.some(update => !finalTranscriptions.includes(update))).toBe(true)
|
expect(streamingUpdates.some(update => !finalTranscriptions.includes(update))).toBe(true)
|
||||||
|
await expect(audio).toHaveCompletedTranscription()
|
||||||
})
|
})
|
||||||
|
|
||||||
it('keeps two Aliyun NLS utterances in streaming transcription', {
|
it('keeps two Aliyun NLS utterances in streaming transcription', {
|
||||||
@@ -101,5 +102,6 @@ describe('audio input pipeline', () => {
|
|||||||
const finalTranscriptions = await audio.transcriptionResults(expectedTranscriptions.length)
|
const finalTranscriptions = await audio.transcriptionResults(expectedTranscriptions.length)
|
||||||
const streamingUpdates = await audio.streamingTranscriptionUpdates()
|
const streamingUpdates = await audio.streamingTranscriptionUpdates()
|
||||||
expect(streamingUpdates.some(update => !finalTranscriptions.includes(update))).toBe(true)
|
expect(streamingUpdates.some(update => !finalTranscriptions.includes(update))).toBe(true)
|
||||||
|
await expect(audio).toHaveCompletedTranscription()
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
import type { PiniaActionEvent } from '@proj-airi/stage-shared/types/pinia-action-event'
|
||||||
|
|
||||||
import type { AudioInputObservations } from './types'
|
import type { AudioInputObservations } from './types'
|
||||||
|
|
||||||
import { describe, it } from 'vitest'
|
import { describe, it } from 'vitest'
|
||||||
@@ -14,15 +16,45 @@ describe('audio input matchers', () => {
|
|||||||
['please say hello'],
|
['please say hello'],
|
||||||
])
|
])
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('reports a failed transcription action', async () => {
|
||||||
|
const session = createAudioInputSession([], [{
|
||||||
|
...actionEvent('modules:hearing:speech:audio-input-pipeline', 'transcribeForRecording'),
|
||||||
|
errorMessage: 'Request failed',
|
||||||
|
status: 'failed',
|
||||||
|
}])
|
||||||
|
|
||||||
|
await expect(expect(session).toHaveCompletedTranscription()).rejects.toThrow('ASR failed: Request failed')
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
function createAudioInputSession(transcriptions: string[]): AudioInputObservations {
|
function createAudioInputSession(
|
||||||
|
transcriptions: string[],
|
||||||
|
actions: PiniaActionEvent[] = [],
|
||||||
|
): AudioInputObservations {
|
||||||
return {
|
return {
|
||||||
capturedTranscriptionAudio: async () => [],
|
capturedTranscriptionAudio: async () => [],
|
||||||
streamingTranscriptionUpdates: async () => [],
|
streamingTranscriptionUpdates: async () => [],
|
||||||
transcriptionResults: async () => transcriptions,
|
transcriptionResults: async () => transcriptions,
|
||||||
completedSpans: async () => [],
|
completedSpans: async () => [],
|
||||||
|
piniaActionEvents: async () => actions,
|
||||||
|
waitForPiniaAction: async () => {
|
||||||
|
throw new Error('This matcher fixture does not observe Pinia actions.')
|
||||||
|
},
|
||||||
waitForStreamingTranscriptionReady: async () => {},
|
waitForStreamingTranscriptionReady: async () => {},
|
||||||
|
waitForTurn: async () => {
|
||||||
|
throw new Error('This matcher fixture does not observe completed turns.')
|
||||||
|
},
|
||||||
waitForVadReady: async () => {},
|
waitForVadReady: async () => {},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function actionEvent(storeId: string, actionName: string): PiniaActionEvent {
|
||||||
|
return {
|
||||||
|
actionName,
|
||||||
|
invocationId: `${storeId}:${actionName}`,
|
||||||
|
status: 'completed',
|
||||||
|
storeId,
|
||||||
|
timestamp: 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -12,6 +12,12 @@ export interface TranscriptionExpectationOptions {
|
|||||||
match?: 'exact' | 'contains'
|
match?: 'exact' | 'contains'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Sets the wait limit for a transcription action assertion. */
|
||||||
|
export interface TranscriptionActionExpectationOptions {
|
||||||
|
/** Maximum time to wait for a completed ASR action. @default 60000 */
|
||||||
|
timeout?: number
|
||||||
|
}
|
||||||
|
|
||||||
declare module 'vitest' {
|
declare module 'vitest' {
|
||||||
interface Assertion<T> {
|
interface Assertion<T> {
|
||||||
toHaveCapturedTranscriptionAudio: T extends AudioInputObservations
|
toHaveCapturedTranscriptionAudio: T extends AudioInputObservations
|
||||||
@@ -23,6 +29,9 @@ declare module 'vitest' {
|
|||||||
options?: TranscriptionExpectationOptions,
|
options?: TranscriptionExpectationOptions,
|
||||||
) => Promise<void>
|
) => Promise<void>
|
||||||
: never
|
: never
|
||||||
|
toHaveCompletedTranscription: T extends AudioInputObservations
|
||||||
|
? (options?: TranscriptionActionExpectationOptions) => Promise<void>
|
||||||
|
: never
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -30,13 +39,13 @@ declare module 'vitest' {
|
|||||||
export const expect = vitestExpect
|
export const expect = vitestExpect
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Normalizes a transcript for speech-recognition comparison.
|
* Normalizes speech text for transcript comparison.
|
||||||
*
|
*
|
||||||
* @example
|
* @example
|
||||||
* normalizeTranscript(' Hello, AIRI! ')
|
* normalizeSpeechText(' Hello, AIRI! ')
|
||||||
* // => 'helloairi'
|
* // => 'helloairi'
|
||||||
*/
|
*/
|
||||||
function normalizeTranscript(value: string): string {
|
function normalizeSpeechText(value: string): string {
|
||||||
return value
|
return value
|
||||||
.normalize('NFKC')
|
.normalize('NFKC')
|
||||||
.toLocaleLowerCase()
|
.toLocaleLowerCase()
|
||||||
@@ -79,8 +88,8 @@ export function installAudioInputMatchers(): void {
|
|||||||
options: TranscriptionExpectationOptions = {},
|
options: TranscriptionExpectationOptions = {},
|
||||||
) {
|
) {
|
||||||
const actual = await session.transcriptionResults(expected.length)
|
const actual = await session.transcriptionResults(expected.length)
|
||||||
const normalizedActual = actual.map(normalizeTranscript)
|
const normalizedActual = actual.map(normalizeSpeechText)
|
||||||
const normalizedExpected = expected.map(alternatives => alternatives.map(normalizeTranscript))
|
const normalizedExpected = expected.map(alternatives => alternatives.map(normalizeSpeechText))
|
||||||
const match = options.match ?? 'exact'
|
const match = options.match ?? 'exact'
|
||||||
const pass = normalizedActual.length === normalizedExpected.length
|
const pass = normalizedActual.length === normalizedExpected.length
|
||||||
&& normalizedActual.every((transcript, index) => (
|
&& normalizedActual.every((transcript, index) => (
|
||||||
@@ -96,5 +105,57 @@ export function installAudioInputMatchers(): void {
|
|||||||
: `Expected transcriptions ${JSON.stringify(expected)}, but received ${JSON.stringify(actual)}.`,
|
: `Expected transcriptions ${JSON.stringify(expected)}, but received ${JSON.stringify(actual)}.`,
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
async toHaveCompletedTranscription(
|
||||||
|
session: AudioInputObservations,
|
||||||
|
options: TranscriptionActionExpectationOptions = {},
|
||||||
|
) {
|
||||||
|
const result = await waitForTranscription(session, options.timeout ?? 60_000)
|
||||||
|
return {
|
||||||
|
pass: result.complete,
|
||||||
|
message: () => result.complete
|
||||||
|
? 'Expected the transcription action not to complete.'
|
||||||
|
: `Expected the transcription action to complete, but ${result.summary}.`,
|
||||||
|
}
|
||||||
|
},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const transcriptionActions = [
|
||||||
|
{ storeId: 'modules:hearing:speech:audio-input-pipeline', actionName: 'transcribeForRecording' },
|
||||||
|
{ storeId: 'modules:hearing:speech:audio-input-pipeline', actionName: 'transcribeForMediaStream' },
|
||||||
|
]
|
||||||
|
|
||||||
|
async function waitForTranscription(
|
||||||
|
session: AudioInputObservations,
|
||||||
|
timeout: number,
|
||||||
|
): Promise<{ complete: boolean, failed: boolean, summary: string }> {
|
||||||
|
const deadline = Date.now() + timeout
|
||||||
|
let result = transcriptionResult(await session.piniaActionEvents())
|
||||||
|
|
||||||
|
while (!result.complete && !result.failed && Date.now() < deadline) {
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 100))
|
||||||
|
result = transcriptionResult(await session.piniaActionEvents())
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
function transcriptionResult(
|
||||||
|
events: Awaited<ReturnType<AudioInputObservations['piniaActionEvents']>>,
|
||||||
|
): { complete: boolean, failed: boolean, summary: string } {
|
||||||
|
const matchingEvents = events.filter(event => transcriptionActions.some(action => (
|
||||||
|
event.storeId === action.storeId && event.actionName === action.actionName
|
||||||
|
)))
|
||||||
|
const latestTerminalEvent = matchingEvents.findLast(event => event.status !== 'started')
|
||||||
|
if (latestTerminalEvent?.status === 'failed') {
|
||||||
|
return {
|
||||||
|
complete: false,
|
||||||
|
failed: true,
|
||||||
|
summary: `ASR failed${latestTerminalEvent.errorMessage ? `: ${latestTerminalEvent.errorMessage}` : ''}`,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
complete: latestTerminalEvent?.status === 'completed',
|
||||||
|
failed: false,
|
||||||
|
summary: 'ASR did not complete',
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,10 +1,18 @@
|
|||||||
export { describe, expect, it } from './describe'
|
export { describe, expect, it } from './describe'
|
||||||
|
|
||||||
|
export type { TranscriptionActionExpectationOptions } from './expect-extend'
|
||||||
export type {
|
export type {
|
||||||
|
AudioInputChatMessage,
|
||||||
|
AudioInputLLMMessage,
|
||||||
|
AudioInputLLMOutputChunk,
|
||||||
AudioInputObservations,
|
AudioInputObservations,
|
||||||
AudioInputPreflightCallback,
|
AudioInputPreflightCallback,
|
||||||
AudioInputPreflightContext,
|
AudioInputPreflightContext,
|
||||||
AudioInputSession,
|
AudioInputSession,
|
||||||
AudioInputTarget,
|
AudioInputTarget,
|
||||||
AudioInputTestCase,
|
AudioInputTestCase,
|
||||||
|
AudioInputTTSSegment,
|
||||||
|
AudioInputTurn,
|
||||||
} from './types'
|
} from './types'
|
||||||
|
|
||||||
|
export type { PiniaActionEvent, PiniaActionEventStatus } from '@proj-airi/stage-shared/types/pinia-action-event'
|
||||||
|
|||||||
@@ -3,18 +3,20 @@ import type { Page } from 'playwright'
|
|||||||
|
|
||||||
import type { AudioInputSession } from '../types'
|
import type { AudioInputSession } from '../types'
|
||||||
|
|
||||||
|
import { piniaActionTracingChannelName } from '@proj-airi/stage-shared/types/pinia-action-event'
|
||||||
|
|
||||||
import { stubForBrowser } from '../setup/browser-probe'
|
import { stubForBrowser } from '../setup/browser-probe'
|
||||||
import { createSession } from '../setup/session'
|
import { createSession } from '../setup/session'
|
||||||
|
|
||||||
/** Adapts a Fakemic Electron process into an AIRI desktop audio session. */
|
/** Adapts a Fakemic Electron process into an AIRI desktop audio session. */
|
||||||
export default async function prepareElectronRuntime(context: FakemicElectronPrepareContext): Promise<AudioInputSession> {
|
export default async function prepareElectronRuntime(context: FakemicElectronPrepareContext): Promise<AudioInputSession> {
|
||||||
await context.app.context().addInitScript(stubForBrowser)
|
await context.app.context().addInitScript(stubForBrowser, piniaActionTracingChannelName)
|
||||||
const page = await waitForPage(context, (page) => {
|
const page = await waitForPage(context, (page) => {
|
||||||
const url = new URL(page.url())
|
const url = new URL(page.url())
|
||||||
return url.pathname.endsWith('/index.html') && url.hash === '#/'
|
return url.pathname.endsWith('/index.html') && url.hash === '#/'
|
||||||
})
|
})
|
||||||
await page.locator('[i-solar\\:alt-arrow-up-line-duotone]').first().waitFor({ state: 'visible', timeout: 30_000 })
|
await page.locator('[i-solar\\:alt-arrow-up-line-duotone]').first().waitFor({ state: 'visible', timeout: 30_000 })
|
||||||
await page.evaluate(stubForBrowser)
|
await page.evaluate(stubForBrowser, piniaActionTracingChannelName)
|
||||||
|
|
||||||
return createSession({
|
return createSession({
|
||||||
electronApp: context.app,
|
electronApp: context.app,
|
||||||
|
|||||||
@@ -2,12 +2,14 @@ import type { FakemicWebPrepareContext } from '@proj-airi/vitest-plugin-fakemic'
|
|||||||
|
|
||||||
import type { AudioInputSession } from '../types'
|
import type { AudioInputSession } from '../types'
|
||||||
|
|
||||||
|
import { piniaActionTracingChannelName } from '@proj-airi/stage-shared/types/pinia-action-event'
|
||||||
|
|
||||||
import { stubForBrowser } from '../setup/browser-probe'
|
import { stubForBrowser } from '../setup/browser-probe'
|
||||||
import { createSession } from '../setup/session'
|
import { createSession } from '../setup/session'
|
||||||
|
|
||||||
/** Adapts a Fakemic Chromium process into an AIRI Web audio session. */
|
/** Adapts a Fakemic Chromium process into an AIRI Web audio session. */
|
||||||
export default async function prepareWebRuntime(context: FakemicWebPrepareContext): Promise<AudioInputSession> {
|
export default async function prepareWebRuntime(context: FakemicWebPrepareContext): Promise<AudioInputSession> {
|
||||||
await context.context.addInitScript(stubForBrowser)
|
await context.context.addInitScript(stubForBrowser, piniaActionTracingChannelName)
|
||||||
|
|
||||||
const page = await context.context.newPage()
|
const page = await context.context.newPage()
|
||||||
await page.goto(context.runtime.url)
|
await page.goto(context.runtime.url)
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
|
import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
|
||||||
|
import type { PiniaActionEvent } from '@proj-airi/stage-shared/types/pinia-action-event'
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Installs passive browser probes before the application starts.
|
* Installs passive browser probes before the application starts.
|
||||||
@@ -8,7 +9,8 @@ import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
|
|||||||
* `BrowserContext.addInitScript`
|
* `BrowserContext.addInitScript`
|
||||||
* -> {@link stubForBrowser}
|
* -> {@link stubForBrowser}
|
||||||
* -> `window.fetch`
|
* -> `window.fetch`
|
||||||
* -> `BroadcastChannel('io-tracer-channel')`
|
* -> Pinia action tracing channel
|
||||||
|
* -> I/O tracing channel
|
||||||
*
|
*
|
||||||
* Upstream:
|
* Upstream:
|
||||||
* - `BrowserContext.addInitScript`
|
* - `BrowserContext.addInitScript`
|
||||||
@@ -16,10 +18,15 @@ import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
|
|||||||
* Downstream:
|
* Downstream:
|
||||||
* - `window.__airiAudioInputE2E`
|
* - `window.__airiAudioInputE2E`
|
||||||
*/
|
*/
|
||||||
export function stubForBrowser() {
|
export function stubForBrowser(piniaActionChannelName: string) {
|
||||||
const state: BrowserAudioInputState = { spans: [], streamingTranscriptionReady: false, streamingTranscriptionUpdates: [], transcriptionAudio: [], transcriptionResults: [], vadReady: false }
|
const state: BrowserAudioInputState = { piniaActionEvents: [], spans: [], streamingTranscriptionReady: false, streamingTranscriptionUpdates: [], transcriptionAudio: [], transcriptionResults: [], vadReady: false }
|
||||||
window.__airiAudioInputE2E = state
|
window.__airiAudioInputE2E = state
|
||||||
|
|
||||||
|
const piniaActionChannel = new BroadcastChannel(piniaActionChannelName)
|
||||||
|
piniaActionChannel.addEventListener('message', (message: MessageEvent<PiniaActionEvent>) => {
|
||||||
|
state.piniaActionEvents.push(message.data)
|
||||||
|
})
|
||||||
|
|
||||||
const originalConsoleInfo = console.info.bind(console)
|
const originalConsoleInfo = console.info.bind(console)
|
||||||
console.info = (...values: unknown[]) => {
|
console.info = (...values: unknown[]) => {
|
||||||
if (typeof values[0] === 'string' && values[0].startsWith('[Voice Input] vad-ready:'))
|
if (typeof values[0] === 'string' && values[0].startsWith('[Voice Input] vad-ready:'))
|
||||||
|
|||||||
@@ -1,13 +1,17 @@
|
|||||||
|
import type { PiniaActionEvent } from '@proj-airi/stage-shared/types/pinia-action-event'
|
||||||
import type { AudioCapture } from '@proj-airi/vitest-plugin-fakemic'
|
import type { AudioCapture } from '@proj-airi/vitest-plugin-fakemic'
|
||||||
import type { ElectronApplication, Page } from 'playwright'
|
import type { ElectronApplication, Page } from 'playwright'
|
||||||
|
|
||||||
import type { AudioInputSession, AudioInputTarget } from '../types'
|
import type { AudioInputChatMessage, AudioInputSession, AudioInputTarget, AudioInputTurn } from '../types'
|
||||||
|
|
||||||
import { Buffer } from 'node:buffer'
|
import { Buffer } from 'node:buffer'
|
||||||
|
|
||||||
|
import { IOAttributes, IOSpanNames } from '@proj-airi/stage-shared/perf/io-trace'
|
||||||
|
import { piniaActionTracingChannelName } from '@proj-airi/stage-shared/types/pinia-action-event'
|
||||||
|
|
||||||
import { readCompletedSpans } from './browser-probe'
|
import { readCompletedSpans } from './browser-probe'
|
||||||
|
|
||||||
/** Creates the runtime session and records its page diagnostics. */
|
/** Creates the runtime session for one audio test. */
|
||||||
export function createSession(options: {
|
export function createSession(options: {
|
||||||
electronApp?: ElectronApplication
|
electronApp?: ElectronApplication
|
||||||
page: Page
|
page: Page
|
||||||
@@ -15,26 +19,6 @@ export function createSession(options: {
|
|||||||
close: () => Promise<void>
|
close: () => Promise<void>
|
||||||
transcriptionCaptureFormat?: AudioCapture['format']
|
transcriptionCaptureFormat?: AudioCapture['format']
|
||||||
}): AudioInputSession {
|
}): AudioInputSession {
|
||||||
const diagnostics: string[] = []
|
|
||||||
const observedPages = new WeakSet<Page>()
|
|
||||||
|
|
||||||
function observePage(page: Page) {
|
|
||||||
if (observedPages.has(page))
|
|
||||||
return
|
|
||||||
|
|
||||||
observedPages.add(page)
|
|
||||||
page.on('console', (message) => {
|
|
||||||
const text = message.text()
|
|
||||||
const isAudioPipelineInfo = message.type() === 'info'
|
|
||||||
&& (text.includes('[Hearing Pipeline]') || text.includes('[Voice Input]') || text.includes('transcription'))
|
|
||||||
if (['error', 'warning'].includes(message.type()) || isAudioPipelineInfo)
|
|
||||||
diagnostics.push(`[console:${message.type()}] ${text}`)
|
|
||||||
})
|
|
||||||
page.on('pageerror', error => diagnostics.push(`[pageerror] ${error.message}`))
|
|
||||||
}
|
|
||||||
|
|
||||||
observePage(options.page)
|
|
||||||
|
|
||||||
const session: AudioInputSession = {
|
const session: AudioInputSession = {
|
||||||
electronApp: options.electronApp,
|
electronApp: options.electronApp,
|
||||||
page: options.page,
|
page: options.page,
|
||||||
@@ -42,7 +26,6 @@ export function createSession(options: {
|
|||||||
target: options.target,
|
target: options.target,
|
||||||
transcriptionCaptureFormat: options.transcriptionCaptureFormat,
|
transcriptionCaptureFormat: options.transcriptionCaptureFormat,
|
||||||
activatePage(page) {
|
activatePage(page) {
|
||||||
observePage(page)
|
|
||||||
session.page = page
|
session.page = page
|
||||||
},
|
},
|
||||||
async capturedTranscriptionAudio(count) {
|
async capturedTranscriptionAudio(count) {
|
||||||
@@ -63,12 +46,13 @@ export function createSession(options: {
|
|||||||
microphoneEnabled: localStorage.getItem('settings/audio/input/enabled'),
|
microphoneEnabled: localStorage.getItem('settings/audio/input/enabled'),
|
||||||
microphoneInput: localStorage.getItem('settings/audio/input'),
|
microphoneInput: localStorage.getItem('settings/audio/input'),
|
||||||
microphoneOffIconVisible: Boolean(document.querySelector('[i-ph\\:microphone-slash]')),
|
microphoneOffIconVisible: Boolean(document.querySelector('[i-ph\\:microphone-slash]')),
|
||||||
|
piniaActionEvents: window.__airiAudioInputE2E?.piniaActionEvents ?? [],
|
||||||
probeInstalled: Boolean(window.__airiAudioInputE2E),
|
probeInstalled: Boolean(window.__airiAudioInputE2E),
|
||||||
streamingTranscriptionReady: window.__airiAudioInputE2E?.streamingTranscriptionReady ?? false,
|
streamingTranscriptionReady: window.__airiAudioInputE2E?.streamingTranscriptionReady ?? false,
|
||||||
url: window.location.href,
|
url: window.location.href,
|
||||||
vadReady: window.__airiAudioInputE2E?.vadReady ?? false,
|
vadReady: window.__airiAudioInputE2E?.vadReady ?? false,
|
||||||
}))
|
}))
|
||||||
throw new Error(`Timed out waiting for captured transcription audio: ${JSON.stringify({ diagnostics, runtimeState })}`, { cause: error })
|
throw new Error(`Timed out waiting for captured transcription audio: ${JSON.stringify(runtimeState)}`, { cause: error })
|
||||||
}
|
}
|
||||||
const capturedAudio = await options.page.evaluate(() => window.__airiAudioInputE2E?.transcriptionAudio ?? [])
|
const capturedAudio = await options.page.evaluate(() => window.__airiAudioInputE2E?.transcriptionAudio ?? [])
|
||||||
return capturedAudio.map(audio => ({
|
return capturedAudio.map(audio => ({
|
||||||
@@ -91,22 +75,64 @@ export function createSession(options: {
|
|||||||
const interactionSpans = await session.page.evaluate(readCompletedSpans, name)
|
const interactionSpans = await session.page.evaluate(readCompletedSpans, name)
|
||||||
return [...runtimeSpans, ...interactionSpans]
|
return [...runtimeSpans, ...interactionSpans]
|
||||||
},
|
},
|
||||||
|
async waitForTurn(waitOptions = {}) {
|
||||||
|
const timeout = waitOptions.timeout ?? 60_000
|
||||||
|
const deadline = Date.now() + timeout
|
||||||
|
let turn = createTurnObservation(await session.completedSpans())
|
||||||
|
|
||||||
|
while (!turn && Date.now() < deadline) {
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 100))
|
||||||
|
turn = createTurnObservation(await session.completedSpans())
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!turn)
|
||||||
|
throw new Error('Timed out waiting for a completed LLM and speech turn.')
|
||||||
|
|
||||||
|
turn.chat.messages = await readChatMessages(session.page)
|
||||||
|
return turn
|
||||||
|
},
|
||||||
|
piniaActionEvents: () => options.page.evaluate(() => window.__airiAudioInputE2E?.piniaActionEvents ?? []),
|
||||||
async waitForVadReady() {
|
async waitForVadReady() {
|
||||||
await options.page.waitForFunction(() => window.__airiAudioInputE2E?.vadReady === true, undefined, { timeout: 30_000 })
|
await options.page.waitForFunction(() => window.__airiAudioInputE2E?.vadReady === true, undefined, { timeout: 30_000 })
|
||||||
},
|
},
|
||||||
async waitForStreamingTranscriptionReady() {
|
async waitForStreamingTranscriptionReady() {
|
||||||
await options.page.waitForFunction(() => window.__airiAudioInputE2E?.streamingTranscriptionReady === true, undefined, { timeout: 30_000 })
|
await options.page.waitForFunction(() => window.__airiAudioInputE2E?.streamingTranscriptionReady === true, undefined, { timeout: 30_000 })
|
||||||
},
|
},
|
||||||
|
async waitForPiniaAction(waitOptions) {
|
||||||
|
return options.page.evaluate(({ actionName, channelName, status, storeId, timeout }) => new Promise<PiniaActionEvent>((resolve, reject) => {
|
||||||
|
const channel = new BroadcastChannel(channelName)
|
||||||
|
const timeoutId = window.setTimeout(() => {
|
||||||
|
channel.close()
|
||||||
|
reject(new Error(`Timed out waiting for Pinia action ${storeId}.${actionName} (${status})`))
|
||||||
|
}, timeout)
|
||||||
|
|
||||||
|
channel.addEventListener('message', (message: MessageEvent<PiniaActionEvent>) => {
|
||||||
|
const action = message.data
|
||||||
|
if (action.storeId !== storeId || action.actionName !== actionName || action.status !== status)
|
||||||
|
return
|
||||||
|
|
||||||
|
window.clearTimeout(timeoutId)
|
||||||
|
channel.close()
|
||||||
|
resolve(action)
|
||||||
|
})
|
||||||
|
}), {
|
||||||
|
actionName: waitOptions.actionName,
|
||||||
|
channelName: piniaActionTracingChannelName,
|
||||||
|
status: waitOptions.status ?? 'completed',
|
||||||
|
storeId: waitOptions.storeId,
|
||||||
|
timeout: waitOptions.timeout ?? 60_000,
|
||||||
|
})
|
||||||
|
},
|
||||||
async snapshot() {
|
async snapshot() {
|
||||||
const runtimeState = await options.page.evaluate(() => window.__airiAudioInputE2E)
|
const runtimeState = await options.page.evaluate(() => window.__airiAudioInputE2E)
|
||||||
const interactionState = session.page === options.page
|
const interactionState = session.page === options.page
|
||||||
? runtimeState
|
? runtimeState
|
||||||
: await session.page.evaluate(() => window.__airiAudioInputE2E)
|
: await session.page.evaluate(() => window.__airiAudioInputE2E)
|
||||||
return {
|
return {
|
||||||
|
piniaActionEvents: runtimeState?.piniaActionEvents ?? [],
|
||||||
spans: runtimeState?.spans ?? [],
|
spans: runtimeState?.spans ?? [],
|
||||||
streamingTranscriptionUpdates: interactionState?.streamingTranscriptionUpdates ?? [],
|
streamingTranscriptionUpdates: interactionState?.streamingTranscriptionUpdates ?? [],
|
||||||
transcriptionResults: runtimeState?.transcriptionResults ?? [],
|
transcriptionResults: runtimeState?.transcriptionResults ?? [],
|
||||||
diagnostics: [...diagnostics],
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
close: options.close,
|
close: options.close,
|
||||||
@@ -114,3 +140,71 @@ export function createSession(options: {
|
|||||||
|
|
||||||
return session
|
return session
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function createTurnObservation(spans: Awaited<ReturnType<AudioInputSession['completedSpans']>>): AudioInputTurn | undefined {
|
||||||
|
const speechTurnSpan = spans.findLast(span => span.name === IOSpanNames.SpeechTurn)
|
||||||
|
if (!speechTurnSpan)
|
||||||
|
return undefined
|
||||||
|
|
||||||
|
const turnId = stringAttribute(speechTurnSpan, IOAttributes.TurnId)
|
||||||
|
const llmSpan = spans.findLast(span => (
|
||||||
|
span.name === IOSpanNames.LLMInference
|
||||||
|
&& stringAttribute(span, IOAttributes.TurnId) === turnId
|
||||||
|
))
|
||||||
|
if (!turnId || !llmSpan)
|
||||||
|
return undefined
|
||||||
|
|
||||||
|
const inputMessageRoles = stringArrayAttribute(llmSpan, IOAttributes.LLMInputMessageRoles)
|
||||||
|
const outputChunkLengths = numberArrayAttribute(llmSpan, IOAttributes.LLMOutputChunkLengths)
|
||||||
|
const audioSegments = spans
|
||||||
|
.filter(span => (
|
||||||
|
span.name === IOSpanNames.TTSSynthesis
|
||||||
|
&& stringAttribute(span, IOAttributes.TurnId) === turnId
|
||||||
|
))
|
||||||
|
.toSorted((left, right) => Number(left.startTimeNano) - Number(right.startTimeNano))
|
||||||
|
.map(span => ({
|
||||||
|
durationMs: numberAttribute(span, IOAttributes.TTSAudioDurationMs),
|
||||||
|
text: stringAttribute(span, IOAttributes.TTSText),
|
||||||
|
}))
|
||||||
|
|
||||||
|
return {
|
||||||
|
id: turnId,
|
||||||
|
chat: { messages: [] },
|
||||||
|
llm: {
|
||||||
|
inputMessages: inputMessageRoles.map(role => ({ role })),
|
||||||
|
outputCharacters: numberAttribute(llmSpan, IOAttributes.LLMTextLength),
|
||||||
|
outputChunks: outputChunkLengths.map(characters => ({ characters })),
|
||||||
|
},
|
||||||
|
tts: { audioSegments },
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readChatMessages(page: Page): Promise<AudioInputChatMessage[]> {
|
||||||
|
return page.locator('[data-chat-message-role]').evaluateAll(elements => elements.flatMap((element) => {
|
||||||
|
const role = element.getAttribute('data-chat-message-role')
|
||||||
|
const text = element.textContent?.trim() ?? ''
|
||||||
|
if ((role !== 'assistant' && role !== 'user') || !text)
|
||||||
|
return []
|
||||||
|
return [{ role, text }]
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
function numberAttribute(span: Awaited<ReturnType<AudioInputSession['completedSpans']>>[number], name: string): number {
|
||||||
|
const value = span.attributes[name]
|
||||||
|
return typeof value === 'number' ? value : 0
|
||||||
|
}
|
||||||
|
|
||||||
|
function numberArrayAttribute(span: Awaited<ReturnType<AudioInputSession['completedSpans']>>[number], name: string): number[] {
|
||||||
|
const value = span.attributes[name]
|
||||||
|
return Array.isArray(value) ? value.filter(item => typeof item === 'number') : []
|
||||||
|
}
|
||||||
|
|
||||||
|
function stringAttribute(span: Awaited<ReturnType<AudioInputSession['completedSpans']>>[number], name: string): string {
|
||||||
|
const value = span.attributes[name]
|
||||||
|
return typeof value === 'string' ? value : ''
|
||||||
|
}
|
||||||
|
|
||||||
|
function stringArrayAttribute(span: Awaited<ReturnType<AudioInputSession['completedSpans']>>[number], name: string): string[] {
|
||||||
|
const value = span.attributes[name]
|
||||||
|
return Array.isArray(value) ? value.filter(item => typeof item === 'string') : []
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
|
import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
|
||||||
|
import type { PiniaActionEvent, PiniaActionEventStatus } from '@proj-airi/stage-shared/types/pinia-action-event'
|
||||||
import type { AudioCapture, AudioCaptureFormat, AudioTestCase, AudioTestPreflightCallback, AudioTestSession } from '@proj-airi/vitest-plugin-fakemic'
|
import type { AudioCapture, AudioCaptureFormat, AudioTestCase, AudioTestPreflightCallback, AudioTestSession } from '@proj-airi/vitest-plugin-fakemic'
|
||||||
import type { ElectronApplication, Page } from 'playwright'
|
import type { ElectronApplication, Page } from 'playwright'
|
||||||
|
|
||||||
@@ -24,10 +25,48 @@ export type AudioInputTestCase = AudioTestCase<AudioInputPreflightContext>
|
|||||||
|
|
||||||
/** Snapshot of the observable AIRI audio pipeline state. */
|
/** Snapshot of the observable AIRI audio pipeline state. */
|
||||||
export interface AudioInputSnapshot {
|
export interface AudioInputSnapshot {
|
||||||
|
piniaActionEvents: PiniaActionEvent[]
|
||||||
spans: SerializedIOSpan[]
|
spans: SerializedIOSpan[]
|
||||||
streamingTranscriptionUpdates: string[]
|
streamingTranscriptionUpdates: string[]
|
||||||
transcriptionResults: string[]
|
transcriptionResults: string[]
|
||||||
diagnostics: string[]
|
}
|
||||||
|
|
||||||
|
/** One rendered chat message from the completed turn. */
|
||||||
|
export interface AudioInputChatMessage {
|
||||||
|
role: 'assistant' | 'user'
|
||||||
|
text: string
|
||||||
|
}
|
||||||
|
|
||||||
|
/** One LLM input message represented without its potentially sensitive content. */
|
||||||
|
export interface AudioInputLLMMessage {
|
||||||
|
role: string
|
||||||
|
}
|
||||||
|
|
||||||
|
/** One text chunk emitted by the LLM stream. */
|
||||||
|
export interface AudioInputLLMOutputChunk {
|
||||||
|
characters: number
|
||||||
|
}
|
||||||
|
|
||||||
|
/** One synthesized audio segment routed from the LLM turn. */
|
||||||
|
export interface AudioInputTTSSegment {
|
||||||
|
durationMs: number
|
||||||
|
text: string
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Structured observations for one completed voice interaction turn. */
|
||||||
|
export interface AudioInputTurn {
|
||||||
|
id: string
|
||||||
|
chat: {
|
||||||
|
messages: AudioInputChatMessage[]
|
||||||
|
}
|
||||||
|
llm: {
|
||||||
|
inputMessages: AudioInputLLMMessage[]
|
||||||
|
outputCharacters: number
|
||||||
|
outputChunks: AudioInputLLMOutputChunk[]
|
||||||
|
}
|
||||||
|
tts: {
|
||||||
|
audioSegments: AudioInputTTSSegment[]
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Observable audio values used by AIRI matchers. */
|
/** Observable audio values used by AIRI matchers. */
|
||||||
@@ -38,10 +77,36 @@ export interface AudioInputObservations {
|
|||||||
streamingTranscriptionUpdates: () => Promise<string[]>
|
streamingTranscriptionUpdates: () => Promise<string[]>
|
||||||
transcriptionResults: (count: number) => Promise<string[]>
|
transcriptionResults: (count: number) => Promise<string[]>
|
||||||
completedSpans: (name?: string) => Promise<SerializedIOSpan[]>
|
completedSpans: (name?: string) => Promise<SerializedIOSpan[]>
|
||||||
|
/** Waits until all speech work for the latest LLM turn is complete. */
|
||||||
|
waitForTurn: (options?: {
|
||||||
|
/** @default 60000 */
|
||||||
|
timeout?: number
|
||||||
|
}) => Promise<AudioInputTurn>
|
||||||
|
/** Returns the Pinia action events collected by the runtime probe. */
|
||||||
|
piniaActionEvents: () => Promise<PiniaActionEvent[]>
|
||||||
/** Waits until the VAD audio graph is connected to the microphone stream. */
|
/** Waits until the VAD audio graph is connected to the microphone stream. */
|
||||||
waitForVadReady: () => Promise<void>
|
waitForVadReady: () => Promise<void>
|
||||||
/** Waits until a streaming transcription transport accepts microphone audio. */
|
/** Waits until a streaming transcription transport accepts microphone audio. */
|
||||||
waitForStreamingTranscriptionReady: () => Promise<void>
|
waitForStreamingTranscriptionReady: () => Promise<void>
|
||||||
|
/**
|
||||||
|
* Waits for the next matching Pinia action event after this method is called.
|
||||||
|
*
|
||||||
|
* @example
|
||||||
|
* const completed = audio.waitForPiniaAction({
|
||||||
|
* storeId: 'modules:hearing:speech:audio-input-pipeline',
|
||||||
|
* actionName: 'transcribeForMediaStream',
|
||||||
|
* })
|
||||||
|
* await enableButton.click()
|
||||||
|
* await completed
|
||||||
|
*/
|
||||||
|
waitForPiniaAction: (options: {
|
||||||
|
storeId: string
|
||||||
|
actionName: string
|
||||||
|
/** @default 'completed' */
|
||||||
|
status?: PiniaActionEventStatus
|
||||||
|
/** @default 60000 */
|
||||||
|
timeout?: number
|
||||||
|
}) => Promise<PiniaActionEvent>
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Runtime handle for one AIRI audio-input test. */
|
/** Runtime handle for one AIRI audio-input test. */
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
|
import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
|
||||||
|
import type { PiniaActionEvent } from '@proj-airi/stage-shared/types/pinia-action-event'
|
||||||
|
|
||||||
declare global {
|
declare global {
|
||||||
interface BrowserAudioInputState {
|
interface BrowserAudioInputState {
|
||||||
|
piniaActionEvents: PiniaActionEvent[]
|
||||||
spans: SerializedIOSpan[]
|
spans: SerializedIOSpan[]
|
||||||
streamingTranscriptionReady: boolean
|
streamingTranscriptionReady: boolean
|
||||||
streamingTranscriptionUpdates: string[]
|
streamingTranscriptionUpdates: string[]
|
||||||
|
|||||||
Reference in New Issue
Block a user