mirror of
https://github.com/moeru-ai/airi.git
synced 2026-08-14 00:48:06 +00:00
feat(testing-audio): now entire audio input/output pipeline can be tested
This commit is contained in:
@@ -598,6 +598,18 @@ function getVoiceInputGeneration(metadata?: Record<string, unknown>) {
|
||||
|
||||
const voiceInputSession = useVoiceInputSession(stream, {
|
||||
shouldUseStreamInput,
|
||||
onLog(level, event, message, details) {
|
||||
const output = `[Voice Input] ${event}: ${message}`
|
||||
if (level === 'error') {
|
||||
console.error(output, details ?? {})
|
||||
return
|
||||
}
|
||||
if (level === 'warn') {
|
||||
console.warn(output, details ?? {})
|
||||
return
|
||||
}
|
||||
console.info(output, details ?? {})
|
||||
},
|
||||
canStartSegment: () => enabled.value && !isVoiceInputSuppressed(),
|
||||
inspectBeforeTranscription: ({ metadata }) => inspectVoiceInputProviderRequestGate(getVoiceInputGeneration(metadata)),
|
||||
inspectAfterTranscription: ({ metadata }) => inspectVoiceInputProviderRequestGate(getVoiceInputGeneration(metadata)),
|
||||
|
||||
@@ -111,6 +111,7 @@ words:
|
||||
- esaxx
|
||||
- eventa
|
||||
- Factorio
|
||||
- fakemic
|
||||
- feaxios
|
||||
- fflate
|
||||
- Flathub
|
||||
|
||||
@@ -51,7 +51,7 @@
|
||||
},
|
||||
"inlinedDependencies": {
|
||||
"@electron-toolkit/preload": "3.0.2",
|
||||
"@moeru/eventa": "1.0.0-beta.13",
|
||||
"@moeru/eventa": "1.0.0-beta.15",
|
||||
"async-mutex": "0.5.0",
|
||||
"nanoid": [
|
||||
"5.1.11",
|
||||
|
||||
@@ -9,6 +9,7 @@ const audioRecorderMock = vi.hoisted(() => ({
|
||||
}))
|
||||
|
||||
const vadMock = vi.hoisted(() => ({
|
||||
init: vi.fn<() => Promise<void>>(),
|
||||
options: undefined as {
|
||||
onSpeechStart?: () => void
|
||||
onSpeechEnd?: () => void
|
||||
@@ -16,6 +17,7 @@ const vadMock = vi.hoisted(() => ({
|
||||
onSpeechReady?: (event: { buffer: Float32Array, duration: number }) => void
|
||||
minSilenceDurationMs?: number
|
||||
} | undefined,
|
||||
start: vi.fn<() => Promise<void>>(),
|
||||
}))
|
||||
|
||||
const hearingPipelineMock = vi.hoisted(() => ({
|
||||
@@ -33,9 +35,9 @@ vi.mock('../../stores/ai/models/vad', async () => {
|
||||
useVAD: (_workerUrl: string, options: typeof vadMock.options) => {
|
||||
vadMock.options = options
|
||||
return {
|
||||
init: vi.fn(),
|
||||
init: vadMock.init,
|
||||
dispose: vi.fn(),
|
||||
start: vi.fn(),
|
||||
start: vadMock.start,
|
||||
loaded: vue.ref(true),
|
||||
isSpeech: vue.ref(false),
|
||||
isSpeechProb: vue.ref(0),
|
||||
@@ -82,6 +84,8 @@ describe('useVoiceInputSession', () => {
|
||||
audioRecorderMock.isRecording.value = false
|
||||
audioRecorderMock.onStopRecordHook = undefined
|
||||
vadMock.options = undefined
|
||||
vadMock.init.mockReset().mockResolvedValue(undefined)
|
||||
vadMock.start.mockReset().mockResolvedValue(undefined)
|
||||
vi.useRealTimers()
|
||||
vi.unstubAllGlobals()
|
||||
vi.clearAllMocks()
|
||||
|
||||
@@ -3,6 +3,26 @@ import { describe, expect, it, vi } from 'vitest'
|
||||
import { startVoiceInputVadDetectionSafely } from './voice-input-vad-startup'
|
||||
|
||||
describe('voice input VAD startup', () => {
|
||||
it('reports readiness after VAD connects to the microphone stream', async () => {
|
||||
const log = vi.fn()
|
||||
const start = vi.fn().mockResolvedValue(undefined)
|
||||
|
||||
await expect(startVoiceInputVadDetectionSafely({
|
||||
init: vi.fn().mockResolvedValue(undefined),
|
||||
loaded: () => true,
|
||||
start,
|
||||
stream: {} as MediaStream,
|
||||
log,
|
||||
})).resolves.toBe(true)
|
||||
|
||||
expect(start).toHaveBeenCalledOnce()
|
||||
expect(log).toHaveBeenLastCalledWith(
|
||||
'info',
|
||||
'vad-ready',
|
||||
'VAD is connected to the microphone stream.',
|
||||
)
|
||||
})
|
||||
|
||||
it('returns false and logs when VAD initialization throws', async () => {
|
||||
const init = vi.fn().mockRejectedValue(new Error('vad unavailable'))
|
||||
const start = vi.fn()
|
||||
|
||||
@@ -18,6 +18,7 @@ export async function startVoiceInputVadDetectionSafely(options: VoiceInputVadSt
|
||||
stream: options.stream,
|
||||
})
|
||||
await options.start(options.stream)
|
||||
options.log?.('info', 'vad-ready', 'VAD is connected to the microphone stream.')
|
||||
return true
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
test-results/
|
||||
@@ -0,0 +1,127 @@
|
||||
# Testing audio
|
||||
|
||||
This package runs recorded microphone tests through the real AIRI audio pipeline.
|
||||
|
||||
```text
|
||||
input.wav -> virtual microphone -> VAD -> ASR -> LLM -> TTS -> playback -> UI
|
||||
```
|
||||
|
||||
The package exports audio-aware `describe`, `it`, and `expect` APIs. Vitest owns the task tree and result protocol. Each project starts its configured Playwright runtime.
|
||||
|
||||
`src` only owns test scheduling, runtime startup, browser probes, and matchers. Case-specific environment, storage, Provider, route, and UI operations live under `cases/shared`:
|
||||
|
||||
```text
|
||||
cases/shared/
|
||||
configurations/ # preflight callbacks that configure one concern
|
||||
interactions/ # reusable UI operations selected by a case
|
||||
```
|
||||
|
||||
## Configure a case
|
||||
|
||||
The optional `preflight` field is an ordered callback array. The fake-microphone Vitest integration starts the runtime before it invokes these callbacks. Each callback receives:
|
||||
|
||||
- `env`: the process environment for this case
|
||||
- `runtime`: the clean Playwright runtime
|
||||
- `skip`: Vitest's case-level skip control
|
||||
|
||||
Every case explicitly selects the configuration it needs. Do not create one shared “complete pipeline” preflight that hides the Provider combination.
|
||||
|
||||
```ts
|
||||
import { describe, expect, it } from '../../src'
|
||||
import { configureModuleHearing, configureOnboarding, loadCaseEnvironment } from '../shared/configurations'
|
||||
import { enableChatMicrophone } from '../shared/interactions'
|
||||
import { openaiAsr } from '../shared/providers'
|
||||
|
||||
describe('audio input pipeline', () => {
|
||||
it('transcribes a greeting', {
|
||||
input: new URL('./input.test.wav', import.meta.url),
|
||||
preflight: [
|
||||
configureOnboarding(() => ({ completed: true })),
|
||||
configureModuleHearing(async (context) => {
|
||||
const environment = await loadCaseEnvironment(context.env)
|
||||
const apiKey = environment.TESTING_AUDIO_ASR_API_KEY
|
||||
context.skip(!apiKey, 'Set TESTING_AUDIO_ASR_API_KEY to run this ASR case.')
|
||||
if (!apiKey)
|
||||
return undefined
|
||||
|
||||
return {
|
||||
provider: openaiAsr({
|
||||
apiKey,
|
||||
baseUrl: environment.TESTING_AUDIO_ASR_API_BASE_URL ?? 'https://api.openai.com/v1/',
|
||||
model: environment.TESTING_AUDIO_ASR_MODEL ?? 'whisper-1',
|
||||
provider: environment.TESTING_AUDIO_ASR_PROVIDER ?? 'openai-compatible-audio-transcription',
|
||||
}),
|
||||
captureFormat: 'wav',
|
||||
}
|
||||
}),
|
||||
],
|
||||
}, async ({ audio }) => {
|
||||
await enableChatMicrophone(audio)
|
||||
await expect(audio).toHaveTranscriptions([
|
||||
['Please say hello.'],
|
||||
])
|
||||
})
|
||||
})
|
||||
```
|
||||
|
||||
A case can independently choose its VAD behavior, ASR, LLM, and TTS configuration. A configuration callback can read its own environment, write local storage, use another persistence mechanism, or deliberately leave onboarding incomplete.
|
||||
|
||||
## Provider environment
|
||||
|
||||
Each case selects its environment variables. `loadCaseEnvironment` uses Vite test mode to read repository, `packages/stage-ui`, and package environment files. Process variables have the highest priority.
|
||||
|
||||
Put local test credentials in `packages/testing-audio/.env.test.local`. Git ignores this file.
|
||||
|
||||
The OpenAI-compatible Provider helpers do not read the environment. Pass the endpoint, API key, model, and voice from the case callback.
|
||||
|
||||
The included cases use these explicit variables:
|
||||
|
||||
```dotenv
|
||||
TESTING_AUDIO_ASR_PROVIDER=openai-compatible-audio-transcription
|
||||
TESTING_AUDIO_ASR_MODEL=whisper-1
|
||||
TESTING_AUDIO_ASR_API_BASE_URL=https://api.openai.com/v1/
|
||||
TESTING_AUDIO_ASR_API_KEY=...
|
||||
|
||||
TESTING_AUDIO_ASR_ALIYUN_NLS_PROVIDER=aliyun-nls-transcription
|
||||
TESTING_AUDIO_ASR_ALIYUN_NLS_ALIYUN_AK_ID=...
|
||||
TESTING_AUDIO_ASR_ALIYUN_NLS_ALIYUN_AK_SECRET=...
|
||||
TESTING_AUDIO_ASR_ALIYUN_NLS_APPKEY=...
|
||||
|
||||
TESTING_AUDIO_LLM_PROVIDER=openai-compatible
|
||||
TESTING_AUDIO_LLM_MODEL=gpt-4o-mini
|
||||
TESTING_AUDIO_LLM_API_BASE_URL=https://api.openai.com/v1/
|
||||
TESTING_AUDIO_LLM_API_KEY=...
|
||||
|
||||
TESTING_AUDIO_TTS_PROVIDER=openai-compatible-audio-speech
|
||||
TESTING_AUDIO_TTS_MODEL=tts-1
|
||||
TESTING_AUDIO_TTS_VOICE=alloy
|
||||
TESTING_AUDIO_TTS_API_BASE_URL=https://api.openai.com/v1/
|
||||
TESTING_AUDIO_TTS_API_KEY=...
|
||||
```
|
||||
|
||||
These tests send audio and text to external Providers. Each run can incur Provider charges.
|
||||
|
||||
## Run the tests
|
||||
|
||||
Build both targets and run all runtime projects:
|
||||
|
||||
```bash
|
||||
pnpm -F @proj-airi/testing-audio test:run
|
||||
```
|
||||
|
||||
Use existing builds:
|
||||
|
||||
```bash
|
||||
pnpm -F @proj-airi/testing-audio test:existing-builds
|
||||
```
|
||||
|
||||
Run one runtime project:
|
||||
|
||||
```bash
|
||||
pnpm -F @proj-airi/testing-audio exec vitest run --project audio-web
|
||||
pnpm -F @proj-airi/testing-audio exec vitest run --project audio-electron
|
||||
```
|
||||
|
||||
Use `*.audio.web.test.ts` for Web-only cases. Use `*.audio.electron.test.ts` for Electron-only cases. The `*.audio.test.ts` pattern runs in both projects.
|
||||
|
||||
Do not use Vitest Browser Mode for these cases. Each task needs a case-specific Chromium fake-microphone process argument.
|
||||
@@ -0,0 +1,82 @@
|
||||
import { describe, expect, it } from '../../src'
|
||||
import { configureModuleHearing, configureOnboarding, loadCaseEnvironment } from '../shared/configurations'
|
||||
import { enableChatMicrophone } from '../shared/interactions'
|
||||
import { aliyunNlsAsr, openaiAsr } from '../shared/providers'
|
||||
|
||||
describe('audio input pipeline', () => {
|
||||
// The fixture uses the repository recording at docs/content/en/blog/DevLog-2025.03.20/assets/ashley-pitch-test.mp3.
|
||||
// It has 20 seconds of leading silence and 4 seconds of trailing silence in mono 16 kHz PCM WAV format.
|
||||
it('does not preserve the complete phrase after long leading silence', {
|
||||
input: new URL('./input.test.wav', import.meta.url),
|
||||
// This regression isolates AIRI's default VAD and one explicit ASR Provider.
|
||||
preflight: [
|
||||
configureOnboarding(() => ({ completed: true })),
|
||||
configureModuleHearing(async (context) => {
|
||||
const environment = await loadCaseEnvironment(context.env)
|
||||
const apiKey = environment.TESTING_AUDIO_ASR_API_KEY
|
||||
context.skip(!apiKey, 'Set TESTING_AUDIO_ASR_API_KEY to run this ASR case.')
|
||||
if (!apiKey)
|
||||
return undefined
|
||||
|
||||
return {
|
||||
provider: openaiAsr({
|
||||
apiKey,
|
||||
baseUrl: environment.TESTING_AUDIO_ASR_API_BASE_URL ?? 'https://api.openai.com/v1/',
|
||||
model: environment.TESTING_AUDIO_ASR_MODEL ?? 'whisper-1',
|
||||
provider: environment.TESTING_AUDIO_ASR_PROVIDER ?? 'openai-compatible-audio-transcription',
|
||||
}),
|
||||
captureFormat: 'wav',
|
||||
}
|
||||
}),
|
||||
],
|
||||
}, async ({ audio }) => {
|
||||
await enableChatMicrophone(audio)
|
||||
|
||||
if (audio.transcriptionCaptureFormat) {
|
||||
await expect(audio).toHaveCapturedTranscriptionAudio({
|
||||
count: 1,
|
||||
minimumBytes: 8000,
|
||||
})
|
||||
}
|
||||
|
||||
// The VAD upload currently drops part of the sentence before it sends the recording to ASR.
|
||||
await expect(audio).not.toHaveTranscriptions([
|
||||
['There is no meaning to your existence, just let go.'],
|
||||
])
|
||||
})
|
||||
|
||||
it('does not preserve the complete phrase with Aliyun NLS', {
|
||||
input: new URL('./input.test.wav', import.meta.url),
|
||||
preflight: [
|
||||
configureOnboarding(() => ({ completed: true })),
|
||||
configureModuleHearing(async (context) => {
|
||||
const environment = await loadCaseEnvironment(context.env)
|
||||
const provider = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_PROVIDER
|
||||
const accessKeyId = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_ALIYUN_AK_ID
|
||||
const accessKeySecret = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_ALIYUN_AK_SECRET
|
||||
const appKey = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_APPKEY
|
||||
context.skip(
|
||||
!provider || !accessKeyId || !accessKeySecret || !appKey,
|
||||
'Set all TESTING_AUDIO_ASR_ALIYUN_NLS_* variables to run this ASR case.',
|
||||
)
|
||||
if (!provider || !accessKeyId || !accessKeySecret || !appKey)
|
||||
return undefined
|
||||
|
||||
return {
|
||||
provider: aliyunNlsAsr({ provider, accessKeyId, accessKeySecret, appKey }),
|
||||
captureFormat: 'pcm',
|
||||
}
|
||||
}),
|
||||
],
|
||||
}, async ({ audio }) => {
|
||||
await enableChatMicrophone(audio, { readiness: 'streaming-transcription' })
|
||||
|
||||
await expect(audio).toHaveCapturedTranscriptionAudio({
|
||||
count: 1,
|
||||
minimumBytes: 8000,
|
||||
})
|
||||
await expect(audio).not.toHaveTranscriptions([
|
||||
['There is no meaning to your existence, just let go.'],
|
||||
])
|
||||
})
|
||||
})
|
||||
Binary file not shown.
@@ -0,0 +1,35 @@
|
||||
import type { AiriCard, AiriExtension } from '@proj-airi/stage-ui/types'
|
||||
|
||||
import type { AudioInputSession } from '../../../src/types'
|
||||
|
||||
type ActiveCardModules = Partial<Pick<AiriExtension['modules'], 'consciousness' | 'speech'>>
|
||||
|
||||
/** Updates Provider selections that the active AIRI Card reapplies during application startup. */
|
||||
export async function configureActiveCardModules(
|
||||
runtime: AudioInputSession,
|
||||
modules: ActiveCardModules,
|
||||
): Promise<void> {
|
||||
await runtime.runtimePage.evaluate(({ configuredModules }) => {
|
||||
const serializedCards = localStorage.getItem('airi-cards')
|
||||
if (!serializedCards)
|
||||
throw new Error('The AIRI Card store is not initialized.')
|
||||
|
||||
const activeCardId = localStorage.getItem('airi-card-active-id') ?? 'default'
|
||||
const cards = JSON.parse(serializedCards) as Array<[string, AiriCard]>
|
||||
const activeCard = cards.find(([cardId]) => cardId === activeCardId)?.[1]
|
||||
if (!activeCard)
|
||||
throw new Error(`The active AIRI Card "${activeCardId}" does not exist.`)
|
||||
|
||||
const currentModules = activeCard.extensions.airi.modules
|
||||
activeCard.extensions.airi.modules = {
|
||||
...currentModules,
|
||||
...(configuredModules.consciousness
|
||||
? { consciousness: configuredModules.consciousness }
|
||||
: {}),
|
||||
...(configuredModules.speech
|
||||
? { speech: { ...currentModules.speech, ...configuredModules.speech } }
|
||||
: {}),
|
||||
}
|
||||
localStorage.setItem('airi-cards', JSON.stringify(cards))
|
||||
}, { configuredModules: modules })
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import type { AudioInputPreflightContext } from '../../../src/types'
|
||||
|
||||
import { resolve } from 'node:path'
|
||||
|
||||
import { findWorkspaceDir } from '@pnpm/find-workspace-dir'
|
||||
import { loadEnv } from 'vite'
|
||||
|
||||
/** Loads Vite test-mode environment files and applies the environment of the current case. */
|
||||
export async function loadCaseEnvironment(
|
||||
environment: AudioInputPreflightContext['env'],
|
||||
): Promise<Record<string, string | undefined>> {
|
||||
const repositoryRoot = await findWorkspaceDir(import.meta.dirname)
|
||||
if (!repositoryRoot)
|
||||
throw new Error(`Unable to find the pnpm workspace from ${import.meta.dirname}`)
|
||||
|
||||
const repositoryEnvironment = loadEnv('test', repositoryRoot, '')
|
||||
// Shared Provider development variables live in stage-ui. These values override repository files.
|
||||
const stageUiEnvironment = loadEnv('test', resolve(repositoryRoot, 'packages/stage-ui'), '')
|
||||
// Audio case credentials belong to this package. These values override shared Provider variables.
|
||||
const testingAudioEnvironment = loadEnv('test', resolve(repositoryRoot, 'packages/testing-audio'), '')
|
||||
// The case process has the highest priority so that CI and shell values override local files.
|
||||
return { ...repositoryEnvironment, ...stageUiEnvironment, ...testingAudioEnvironment, ...environment }
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
export { loadCaseEnvironment } from './environment'
|
||||
export { configureModuleConsciousness } from './module-consciousness'
|
||||
export type { ConsciousnessModuleConfiguration } from './module-consciousness'
|
||||
export { configureModuleHearing } from './module-hearing'
|
||||
export type { HearingModuleConfiguration } from './module-hearing'
|
||||
export { configureModuleSpeech } from './module-speech'
|
||||
export type { SpeechModuleConfiguration } from './module-speech'
|
||||
export { configureOnboarding } from './onboarding'
|
||||
export type { OnboardingConfiguration } from './onboarding'
|
||||
@@ -0,0 +1,33 @@
|
||||
import type { AudioInputPreflightCallback, AudioInputPreflightContext } from '../../../src/types'
|
||||
import type { ProviderConfiguration } from './provider'
|
||||
|
||||
import { configureActiveCardModules } from './active-card'
|
||||
import { configureProvider } from './provider'
|
||||
import { configureStorage } from './storage'
|
||||
|
||||
export interface ConsciousnessModuleConfiguration {
|
||||
provider: ProviderConfiguration
|
||||
}
|
||||
|
||||
type ConsciousnessModuleResolver = (context: AudioInputPreflightContext) => ConsciousnessModuleConfiguration | undefined | Promise<ConsciousnessModuleConfiguration | undefined>
|
||||
|
||||
/** Configures the consciousness module with the LLM Provider selected by one case. */
|
||||
export function configureModuleConsciousness(resolve: ConsciousnessModuleResolver): AudioInputPreflightCallback {
|
||||
return async (context) => {
|
||||
const configuration = await resolve(context)
|
||||
if (!configuration)
|
||||
return
|
||||
|
||||
await configureProvider(context.runtime, configuration.provider)
|
||||
await configureActiveCardModules(context.runtime, {
|
||||
consciousness: {
|
||||
provider: configuration.provider.id,
|
||||
model: configuration.provider.model,
|
||||
},
|
||||
})
|
||||
await configureStorage(context.runtime, {
|
||||
'settings/consciousness/active-provider': configuration.provider.id,
|
||||
'settings/consciousness/active-model': configuration.provider.model,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import type { AudioCaptureFormat } from '@proj-airi/vitest-plugin-fakemic'
|
||||
|
||||
import type { AudioInputPreflightCallback, AudioInputPreflightContext } from '../../../src/types'
|
||||
import type { ProviderConfiguration } from './provider'
|
||||
|
||||
import { configureProvider } from './provider'
|
||||
import { configureStorage } from './storage'
|
||||
|
||||
export interface HearingModuleConfiguration {
|
||||
/** @default undefined */
|
||||
captureFormat?: AudioCaptureFormat
|
||||
/** @default false */
|
||||
microphoneEnabled?: boolean
|
||||
provider: ProviderConfiguration
|
||||
}
|
||||
|
||||
type HearingModuleResolver = (context: AudioInputPreflightContext) => HearingModuleConfiguration | undefined | Promise<HearingModuleConfiguration | undefined>
|
||||
|
||||
/** Configures the hearing module with the ASR Provider selected by one case. */
|
||||
export function configureModuleHearing(resolve: HearingModuleResolver): AudioInputPreflightCallback {
|
||||
return async (context) => {
|
||||
const configuration = await resolve(context)
|
||||
if (!configuration)
|
||||
return
|
||||
|
||||
await configureProvider(context.runtime, configuration.provider)
|
||||
const settings: Record<string, string> = {
|
||||
'settings/hearing/active-provider': configuration.provider.id,
|
||||
'settings/hearing/active-model': configuration.provider.model,
|
||||
'settings/audio/input/enabled': String(configuration.microphoneEnabled ?? false),
|
||||
}
|
||||
|
||||
if (context.runtime.target === 'electron') {
|
||||
const microphoneInput = await context.runtime.runtimePage.evaluate(async () => {
|
||||
const devices = await navigator.mediaDevices.enumerateDevices()
|
||||
return devices.find(device => device.kind === 'audioinput' && device.label.includes('Fake'))?.deviceId
|
||||
})
|
||||
if (!microphoneInput)
|
||||
throw new Error('Chromium did not expose the file-backed fake microphone.')
|
||||
settings['settings/audio/input'] = microphoneInput
|
||||
}
|
||||
|
||||
await configureStorage(context.runtime, settings)
|
||||
context.runtime.transcriptionCaptureFormat = configuration.captureFormat
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import type { AudioInputPreflightCallback, AudioInputPreflightContext } from '../../../src/types'
|
||||
import type { ProviderConfiguration } from './provider'
|
||||
|
||||
import { configureActiveCardModules } from './active-card'
|
||||
import { configureProvider } from './provider'
|
||||
import { configureStorage } from './storage'
|
||||
|
||||
export interface SpeechModuleConfiguration {
|
||||
/** @default false */
|
||||
muted?: boolean
|
||||
provider: ProviderConfiguration
|
||||
voice: string
|
||||
}
|
||||
|
||||
type SpeechModuleResolver = (context: AudioInputPreflightContext) => SpeechModuleConfiguration | undefined | Promise<SpeechModuleConfiguration | undefined>
|
||||
|
||||
/** Configures the speech module with the TTS Provider selected by one case. */
|
||||
export function configureModuleSpeech(resolve: SpeechModuleResolver): AudioInputPreflightCallback {
|
||||
return async (context) => {
|
||||
const configuration = await resolve(context)
|
||||
if (!configuration)
|
||||
return
|
||||
|
||||
await configureProvider(context.runtime, configuration.provider)
|
||||
await configureActiveCardModules(context.runtime, {
|
||||
speech: {
|
||||
provider: configuration.provider.id,
|
||||
model: configuration.provider.model,
|
||||
voice_id: configuration.voice,
|
||||
},
|
||||
})
|
||||
await configureStorage(context.runtime, {
|
||||
'settings/speech/active-provider': configuration.provider.id,
|
||||
'settings/speech/active-model': configuration.provider.model,
|
||||
'settings/speech/voice': configuration.voice,
|
||||
'settings/speech/output-muted': String(configuration.muted ?? false),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import type { AudioInputPreflightCallback, AudioInputPreflightContext } from '../../../src/types'
|
||||
|
||||
import { configureStorage } from './storage'
|
||||
|
||||
export interface OnboardingConfiguration {
|
||||
completed: boolean
|
||||
/** @default false */
|
||||
skipped?: boolean
|
||||
}
|
||||
|
||||
type OnboardingResolver = (context: AudioInputPreflightContext) => OnboardingConfiguration | undefined | Promise<OnboardingConfiguration | undefined>
|
||||
|
||||
/** Configures onboarding with values selected by one case. */
|
||||
export function configureOnboarding(resolve: OnboardingResolver): AudioInputPreflightCallback {
|
||||
return async (context) => {
|
||||
const configuration = await resolve(context)
|
||||
if (!configuration)
|
||||
return
|
||||
|
||||
await configureStorage(context.runtime, {
|
||||
'onboarding/completed': String(configuration.completed),
|
||||
'onboarding/skipped': String(configuration.skipped ?? false),
|
||||
})
|
||||
|
||||
if (configuration.completed || configuration.skipped) {
|
||||
await new Promise(resolveWait => setTimeout(resolveWait, 1_000))
|
||||
const onboardingPages = context.runtime.electronApp
|
||||
?.windows()
|
||||
.filter(page => new URL(page.url()).hash.startsWith('#/onboarding')) ?? []
|
||||
await Promise.all(onboardingPages.map(page => page.close().catch(() => undefined)))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import type { AudioInputSession } from '../../../src/types'
|
||||
|
||||
/** Provider values stored by one case preflight callback. */
|
||||
export interface ProviderConfiguration {
|
||||
config: Record<string, unknown>
|
||||
definitionId: string
|
||||
id: string
|
||||
model: string
|
||||
}
|
||||
|
||||
/** Adds one Provider without replacing Providers configured by earlier callbacks. */
|
||||
export async function configureProvider(runtime: AudioInputSession, provider: ProviderConfiguration): Promise<void> {
|
||||
await runtime.page.evaluate(({ configuredProvider }) => {
|
||||
const credentials = JSON.parse(localStorage.getItem('settings/credentials/providers') ?? '{}') as Record<string, unknown>
|
||||
const configured = JSON.parse(localStorage.getItem('settings/providers/configured') ?? '{}') as Record<string, unknown>
|
||||
const added = JSON.parse(localStorage.getItem('settings/providers/added') ?? '{}') as Record<string, boolean>
|
||||
|
||||
credentials[configuredProvider.id] = configuredProvider.config
|
||||
configured[configuredProvider.id] = {
|
||||
id: configuredProvider.id,
|
||||
definitionId: configuredProvider.definitionId,
|
||||
config: configuredProvider.config,
|
||||
status: 'configured',
|
||||
}
|
||||
added[configuredProvider.id] = true
|
||||
|
||||
localStorage.setItem('settings/credentials/providers', JSON.stringify(credentials))
|
||||
localStorage.setItem('settings/providers/configured', JSON.stringify(configured))
|
||||
localStorage.setItem('settings/providers/added', JSON.stringify(added))
|
||||
}, { configuredProvider: provider })
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import type { AudioInputSession } from '../../../src/types'
|
||||
|
||||
/** Writes AIRI settings through the storage owned by the current runtime page. */
|
||||
export async function configureStorage(
|
||||
runtime: AudioInputSession,
|
||||
settings: Record<string, string>,
|
||||
): Promise<void> {
|
||||
await runtime.runtimePage.evaluate(({ entries }) => {
|
||||
for (const [key, value] of Object.entries(entries))
|
||||
localStorage.setItem(key, value)
|
||||
}, { entries: settings })
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
import type { Locator } from 'playwright'
|
||||
|
||||
import type { AudioInputSession } from '../../../src/types'
|
||||
|
||||
import { captureStreamingTranscription } from './streaming-transcription'
|
||||
|
||||
export interface EnableChatMicrophoneOptions {
|
||||
/** @default 'vad' */
|
||||
readiness?: 'streaming-transcription' | 'vad'
|
||||
}
|
||||
|
||||
/** Enables the chat microphone through the UI owned by the selected runtime. */
|
||||
export async function enableChatMicrophone(
|
||||
runtime: AudioInputSession,
|
||||
options: EnableChatMicrophoneOptions = {},
|
||||
): Promise<void> {
|
||||
if (runtime.target === 'electron') {
|
||||
const app = runtime.electronApp
|
||||
if (!app)
|
||||
throw new Error('The Electron audio session does not expose its application.')
|
||||
|
||||
const existingChatPage = app.windows().find(page => page.url().includes('index.html#/chat'))
|
||||
if (existingChatPage) {
|
||||
runtime.activatePage(existingChatPage)
|
||||
}
|
||||
else {
|
||||
const chatButton = runtime.runtimePage.locator('button').filter({
|
||||
has: runtime.runtimePage.locator('[i-solar\\:chat-line-line-duotone]'),
|
||||
}).first()
|
||||
await chatButton.waitFor({ state: 'visible', timeout: 30_000 })
|
||||
runtime.activatePage(await openElectronChat(app, chatButton))
|
||||
}
|
||||
}
|
||||
|
||||
const { page } = runtime
|
||||
await page.locator('textarea').first().waitFor({ state: 'visible', timeout: 60_000 })
|
||||
await captureStreamingTranscription(page, 'textarea')
|
||||
|
||||
if (runtime.target === 'electron') {
|
||||
await runtime.runtimePage.waitForFunction(async () => {
|
||||
const devices = await navigator.mediaDevices.enumerateDevices()
|
||||
return devices.some(device => device.kind === 'audioinput' && device.label.includes('Fake'))
|
||||
})
|
||||
|
||||
const hearingTrigger = runtime.runtimePage.locator('div[aria-haspopup="dialog"] button').first()
|
||||
await hearingTrigger.waitFor({ state: 'visible', timeout: 15_000 })
|
||||
await hearingTrigger.hover()
|
||||
await hearingTrigger.click({ force: true })
|
||||
await runtime.runtimePage.waitForTimeout(500)
|
||||
|
||||
const enableButton = runtime.runtimePage.locator('button[aria-label="Enable microphone input"]')
|
||||
await enableButton.waitFor({ state: 'visible', timeout: 15_000 })
|
||||
const inputReady = options.readiness === 'streaming-transcription'
|
||||
? runtime.waitForStreamingTranscriptionReady()
|
||||
: runtime.waitForVadReady()
|
||||
await enableButton.click({ force: true })
|
||||
const disableButton = runtime.runtimePage.locator('button[aria-label="Disable microphone input"]')
|
||||
await disableButton.waitFor({ state: 'visible', timeout: 15_000 })
|
||||
await inputReady
|
||||
return
|
||||
}
|
||||
|
||||
const microphoneTrigger = page.locator('button').filter({ has: page.locator('.i-ph\\:microphone-slash') }).first()
|
||||
await microphoneTrigger.click({ force: true })
|
||||
|
||||
const enableButton = page.locator('button[aria-label="Enable microphone input"]')
|
||||
await enableButton.waitFor({ state: 'visible' })
|
||||
const inputReady = options.readiness === 'streaming-transcription'
|
||||
? runtime.waitForStreamingTranscriptionReady()
|
||||
: runtime.waitForVadReady()
|
||||
await enableButton.click()
|
||||
await page.locator('button[aria-label="Disable microphone input"]').waitFor({ state: 'visible' })
|
||||
await inputReady
|
||||
}
|
||||
|
||||
/** Opens the chat page and waits for its input. */
|
||||
export async function openChat(runtime: AudioInputSession): Promise<void> {
|
||||
await runtime.page.evaluate(() => {
|
||||
window.location.hash = '/chat'
|
||||
})
|
||||
await runtime.page.locator('textarea').first().waitFor({ state: 'visible', timeout: 60_000 })
|
||||
}
|
||||
|
||||
/** Returns the assistant messages on the current chat page. */
|
||||
export function assistantMessages(runtime: AudioInputSession): Locator {
|
||||
return runtime.page.locator('[data-chat-message-role="assistant"] .markdown-content')
|
||||
}
|
||||
|
||||
async function openElectronChat(
|
||||
app: NonNullable<AudioInputSession['electronApp']>,
|
||||
chatButton: Locator,
|
||||
): Promise<AudioInputSession['page']> {
|
||||
let lastError: unknown
|
||||
for (let attempt = 0; attempt < 10; attempt++) {
|
||||
await chatButton.click({ force: true })
|
||||
try {
|
||||
return await waitForElectronPage(app, page => page.url().includes('index.html#/chat'), 3_000)
|
||||
}
|
||||
catch (error) {
|
||||
lastError = error
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError
|
||||
}
|
||||
|
||||
async function waitForElectronPage(
|
||||
app: NonNullable<AudioInputSession['electronApp']>,
|
||||
predicate: (page: AudioInputSession['page']) => boolean,
|
||||
timeoutMs = 60_000,
|
||||
): Promise<AudioInputSession['page']> {
|
||||
const deadline = Date.now() + timeoutMs
|
||||
while (Date.now() < deadline) {
|
||||
const page = app.windows().find(predicate)
|
||||
if (page) {
|
||||
await page.waitForLoadState('domcontentloaded')
|
||||
await page.bringToFront()
|
||||
await page.waitForTimeout(750)
|
||||
return page
|
||||
}
|
||||
|
||||
await new Promise(resolveWait => setTimeout(resolveWait, 100))
|
||||
}
|
||||
|
||||
throw new Error(`Timed out while waiting for the Electron chat window. Open pages: ${app.windows().map(page => page.url()).join(', ')}`)
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
import type { ElectronApplication, Page } from 'playwright'
|
||||
|
||||
import type { AudioInputSession } from '../../../src/types'
|
||||
|
||||
import { captureStreamingTranscription } from './streaming-transcription'
|
||||
|
||||
/** Opens the Electron hearing playground for a case that selects this input UI. */
|
||||
export async function openHearingPlayground(runtime: AudioInputSession): Promise<Page> {
|
||||
const app = runtime.electronApp
|
||||
if (!app)
|
||||
throw new Error('The hearing playground interaction requires an Electron runtime')
|
||||
|
||||
const settingsButton = runtime.page.getByRole('button', { name: /Open settings|打开设置/ }).last()
|
||||
if (!await settingsButton.isVisible().catch(() => false)) {
|
||||
await runtime.page.getByRole('button', { name: /Expand|展开/ }).last().click({ force: true })
|
||||
await settingsButton.waitFor({ state: 'visible', timeout: 15_000 })
|
||||
}
|
||||
|
||||
await settingsButton.click({ force: true })
|
||||
const settingsPage = await waitForElectronPage(app, page => page.url().includes('index.html#/settings'))
|
||||
await settingsPage.evaluate(() => {
|
||||
window.location.hash = '/settings/modules/hearing'
|
||||
})
|
||||
await settingsPage.waitForURL(/#\/settings\/modules\/hearing/)
|
||||
await settingsPage.getByTestId('hearing-playground-monitor-toggle').waitFor({ state: 'visible', timeout: 60_000 })
|
||||
return settingsPage
|
||||
}
|
||||
|
||||
/** Enables microphone monitoring on an open hearing playground. */
|
||||
export async function enableHearingPlaygroundMicrophone(page: Page): Promise<void> {
|
||||
await captureStreamingTranscription(page, '[data-testid="hearing-playground-current"] p')
|
||||
|
||||
const modelBasedToggle = page.getByRole('switch').last()
|
||||
if (await modelBasedToggle.isChecked())
|
||||
await modelBasedToggle.click()
|
||||
|
||||
const monitorToggle = page.getByTestId('hearing-playground-monitor-toggle')
|
||||
await monitorToggle.click()
|
||||
}
|
||||
|
||||
/** Reads the requested number of final hearing playground transcripts. */
|
||||
export async function readHearingPlaygroundTranscriptions(page: Page, count: number): Promise<string[]> {
|
||||
const transcriptions = page.getByTestId('hearing-playground-transcript')
|
||||
await transcriptions.nth(count - 1).waitFor({ state: 'visible', timeout: 60_000 })
|
||||
const results = (await transcriptions.allTextContents()).toReversed()
|
||||
await page.evaluate((transcriptionResults) => {
|
||||
if (window.__airiAudioInputE2E)
|
||||
window.__airiAudioInputE2E.transcriptionResults = transcriptionResults
|
||||
}, results)
|
||||
return results
|
||||
}
|
||||
|
||||
async function waitForElectronPage(
|
||||
app: ElectronApplication,
|
||||
predicate: (page: Page) => boolean,
|
||||
timeoutMs = 60_000,
|
||||
): Promise<Page> {
|
||||
const deadline = Date.now() + timeoutMs
|
||||
while (Date.now() < deadline) {
|
||||
const page = app.windows().find(predicate)
|
||||
if (page) {
|
||||
await page.waitForLoadState('domcontentloaded')
|
||||
return page
|
||||
}
|
||||
await new Promise(resolveWait => setTimeout(resolveWait, 100))
|
||||
}
|
||||
throw new Error('Timed out while waiting for the Electron renderer')
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
export { assistantMessages, enableChatMicrophone, openChat } from './chat'
|
||||
export type { EnableChatMicrophoneOptions } from './chat'
|
||||
export {
|
||||
enableHearingPlaygroundMicrophone,
|
||||
openHearingPlayground,
|
||||
readHearingPlaygroundTranscriptions,
|
||||
} from './hearing-playground'
|
||||
@@ -0,0 +1,20 @@
|
||||
import type { Page } from 'playwright'
|
||||
|
||||
/** Records distinct visible transcription values until the page closes. */
|
||||
export async function captureStreamingTranscription(page: Page, selector: string): Promise<void> {
|
||||
await page.evaluate((captureSelector) => {
|
||||
let lastUpdate = ''
|
||||
setInterval(() => {
|
||||
const element = document.querySelector(captureSelector)
|
||||
const value = element instanceof HTMLTextAreaElement
|
||||
? element.value
|
||||
: element?.textContent
|
||||
const update = value?.trim() ?? ''
|
||||
const updates = window.__airiAudioInputE2E?.streamingTranscriptionUpdates
|
||||
if (update && update !== lastUpdate && updates) {
|
||||
lastUpdate = update
|
||||
updates.push(update)
|
||||
}
|
||||
}, 20)
|
||||
}, selector)
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import type { ProviderConfiguration } from '../configurations/provider'
|
||||
|
||||
/** Credentials and Provider selection for Aliyun NLS transcription. */
|
||||
export interface AliyunNlsAsrOptions {
|
||||
accessKeyId: string
|
||||
accessKeySecret: string
|
||||
appKey: string
|
||||
provider: string
|
||||
}
|
||||
|
||||
/** Creates an Aliyun NLS realtime ASR Provider configuration. */
|
||||
export function aliyunNlsAsr(options: AliyunNlsAsrOptions): ProviderConfiguration {
|
||||
if (options.provider !== 'aliyun-nls-transcription')
|
||||
throw new Error('The Aliyun NLS Provider must be "aliyun-nls-transcription".')
|
||||
|
||||
return {
|
||||
id: options.provider,
|
||||
definitionId: options.provider,
|
||||
model: 'aliyun-nls-v1',
|
||||
config: {
|
||||
accessKeyId: options.accessKeyId,
|
||||
accessKeySecret: options.accessKeySecret,
|
||||
appKey: options.appKey,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
export { aliyunNlsAsr } from './aliyun-nls'
|
||||
export type { AliyunNlsAsrOptions } from './aliyun-nls'
|
||||
export { openaiAsr, openaiLlm, openaiTts } from './openai'
|
||||
export type { OpenAIProviderOptions, OpenAISpeechProviderOptions } from './openai'
|
||||
@@ -0,0 +1,58 @@
|
||||
import type { ProviderConfiguration } from '../configurations/provider'
|
||||
|
||||
/** Configuration for an OpenAI-compatible Provider selected by one case. */
|
||||
export interface OpenAIProviderOptions {
|
||||
apiKey: string
|
||||
baseUrl: string
|
||||
model: string
|
||||
provider: string
|
||||
}
|
||||
|
||||
/** Configuration for an OpenAI-compatible speech Provider selected by one case. */
|
||||
export interface OpenAISpeechProviderOptions extends OpenAIProviderOptions {
|
||||
voice: string
|
||||
}
|
||||
|
||||
/** Creates an OpenAI-compatible ASR Provider configuration. */
|
||||
export function openaiAsr(options: OpenAIProviderOptions): ProviderConfiguration {
|
||||
return {
|
||||
id: options.provider,
|
||||
definitionId: options.provider,
|
||||
model: options.model,
|
||||
config: {
|
||||
apiKey: options.apiKey,
|
||||
baseUrl: options.baseUrl,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/** Creates an OpenAI-compatible LLM Provider configuration. */
|
||||
export function openaiLlm(options: OpenAIProviderOptions): ProviderConfiguration {
|
||||
return {
|
||||
id: options.provider,
|
||||
definitionId: options.provider,
|
||||
model: options.model,
|
||||
config: {
|
||||
apiKey: options.apiKey,
|
||||
baseUrl: options.baseUrl,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/** Creates an OpenAI-compatible TTS Provider configuration. */
|
||||
export function openaiTts(options: OpenAISpeechProviderOptions): { provider: ProviderConfiguration, voice: string } {
|
||||
return {
|
||||
voice: options.voice,
|
||||
provider: {
|
||||
id: options.provider,
|
||||
definitionId: options.provider,
|
||||
model: options.model,
|
||||
config: {
|
||||
apiKey: options.apiKey,
|
||||
baseUrl: options.baseUrl,
|
||||
model: options.model,
|
||||
voice: options.voice,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
import { describe, expect, it } from '../../src'
|
||||
import { configureModuleConsciousness, configureModuleHearing, configureModuleSpeech, configureOnboarding, loadCaseEnvironment } from '../shared/configurations'
|
||||
import { assistantMessages, enableChatMicrophone, openChat } from '../shared/interactions'
|
||||
import { aliyunNlsAsr, openaiAsr, openaiLlm, openaiTts } from '../shared/providers'
|
||||
|
||||
describe('audio input pipeline', () => {
|
||||
// An OpenAI-compatible TTS Provider generated the fixture in mono 16 kHz PCM WAV format.
|
||||
// The fixture contains 14 seconds of leading silence for VAD initialization and 3 seconds of trailing silence.
|
||||
// Its warm-up phrase gives the VAD time to start. Only "Please say hello." is required in the transcript.
|
||||
it('runs an OpenAI-compatible request through the complete pipeline', {
|
||||
input: new URL('./input.test.wav', import.meta.url),
|
||||
// This case keeps AIRI's default VAD and selects every remote Provider explicitly.
|
||||
preflight: [
|
||||
configureOnboarding(() => ({ completed: true })),
|
||||
configureModuleHearing(async (context) => {
|
||||
const environment = await loadCaseEnvironment(context.env)
|
||||
const apiKey = environment.TESTING_AUDIO_ASR_API_KEY
|
||||
context.skip(!apiKey, 'Set TESTING_AUDIO_ASR_API_KEY to run this ASR case.')
|
||||
if (!apiKey)
|
||||
return undefined
|
||||
|
||||
return {
|
||||
provider: openaiAsr({
|
||||
apiKey,
|
||||
baseUrl: environment.TESTING_AUDIO_ASR_API_BASE_URL ?? 'https://api.openai.com/v1/',
|
||||
model: environment.TESTING_AUDIO_ASR_MODEL ?? 'whisper-1',
|
||||
provider: environment.TESTING_AUDIO_ASR_PROVIDER ?? 'openai-compatible-audio-transcription',
|
||||
}),
|
||||
captureFormat: 'wav',
|
||||
}
|
||||
}),
|
||||
configureModuleConsciousness(async (context) => {
|
||||
const environment = await loadCaseEnvironment(context.env)
|
||||
const apiKey = environment.TESTING_AUDIO_LLM_API_KEY
|
||||
context.skip(!apiKey, 'Set TESTING_AUDIO_LLM_API_KEY to run this LLM case.')
|
||||
if (!apiKey)
|
||||
return undefined
|
||||
|
||||
return {
|
||||
provider: openaiLlm({
|
||||
apiKey,
|
||||
baseUrl: environment.TESTING_AUDIO_LLM_API_BASE_URL ?? 'https://api.openai.com/v1/',
|
||||
model: environment.TESTING_AUDIO_LLM_MODEL ?? 'gpt-4o-mini',
|
||||
provider: environment.TESTING_AUDIO_LLM_PROVIDER ?? 'openai-compatible',
|
||||
}),
|
||||
}
|
||||
}),
|
||||
configureModuleSpeech(async (context) => {
|
||||
const environment = await loadCaseEnvironment(context.env)
|
||||
const apiKey = environment.TESTING_AUDIO_TTS_API_KEY
|
||||
context.skip(!apiKey, 'Set TESTING_AUDIO_TTS_API_KEY to run this TTS case.')
|
||||
if (!apiKey)
|
||||
return undefined
|
||||
|
||||
return openaiTts({
|
||||
apiKey,
|
||||
baseUrl: environment.TESTING_AUDIO_TTS_API_BASE_URL ?? 'https://api.openai.com/v1/',
|
||||
model: environment.TESTING_AUDIO_TTS_MODEL ?? 'tts-1',
|
||||
provider: environment.TESTING_AUDIO_TTS_PROVIDER ?? 'openai-compatible-audio-speech',
|
||||
voice: environment.TESTING_AUDIO_TTS_VOICE ?? 'alloy',
|
||||
})
|
||||
}),
|
||||
],
|
||||
}, async ({ audio }) => {
|
||||
await enableChatMicrophone(audio)
|
||||
|
||||
if (audio.transcriptionCaptureFormat) {
|
||||
await expect(audio).toHaveCapturedTranscriptionAudio({
|
||||
count: 1,
|
||||
minimumBytes: 8000,
|
||||
})
|
||||
}
|
||||
|
||||
await expect(audio).toHaveTranscriptions([
|
||||
['Please say hello.'],
|
||||
], { match: 'contains' })
|
||||
|
||||
await expect.poll(async () => (await audio.completedSpans('LLM inference')).length, { timeout: 60_000 }).toBeGreaterThanOrEqual(1)
|
||||
await expect.poll(async () => (await audio.completedSpans('TTS synthesis')).length, { timeout: 60_000 }).toBeGreaterThanOrEqual(1)
|
||||
await expect.poll(async () => (await audio.completedSpans('Audio playback')).length, { timeout: 60_000 }).toBeGreaterThanOrEqual(1)
|
||||
|
||||
await openChat(audio)
|
||||
const messages = await assistantMessages(audio).allTextContents()
|
||||
expect(messages.at(-1)).toMatch(/.+/s)
|
||||
})
|
||||
|
||||
it('transcribes the greeting with Aliyun NLS', {
|
||||
input: new URL('./input.test.wav', import.meta.url),
|
||||
preflight: [
|
||||
configureOnboarding(() => ({ completed: true })),
|
||||
configureModuleHearing(async (context) => {
|
||||
const environment = await loadCaseEnvironment(context.env)
|
||||
const provider = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_PROVIDER
|
||||
const accessKeyId = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_ALIYUN_AK_ID
|
||||
const accessKeySecret = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_ALIYUN_AK_SECRET
|
||||
const appKey = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_APPKEY
|
||||
context.skip(
|
||||
!provider || !accessKeyId || !accessKeySecret || !appKey,
|
||||
'Set all TESTING_AUDIO_ASR_ALIYUN_NLS_* variables to run this ASR case.',
|
||||
)
|
||||
if (!provider || !accessKeyId || !accessKeySecret || !appKey)
|
||||
return undefined
|
||||
|
||||
return {
|
||||
provider: aliyunNlsAsr({ provider, accessKeyId, accessKeySecret, appKey }),
|
||||
captureFormat: 'pcm',
|
||||
}
|
||||
}),
|
||||
],
|
||||
}, async ({ audio }) => {
|
||||
await enableChatMicrophone(audio, { readiness: 'streaming-transcription' })
|
||||
|
||||
await expect(audio).toHaveCapturedTranscriptionAudio({
|
||||
count: 1,
|
||||
minimumBytes: 8000,
|
||||
})
|
||||
await expect(audio).toHaveTranscriptions([
|
||||
['Please say hello.'],
|
||||
], { match: 'contains' })
|
||||
})
|
||||
})
|
||||
Binary file not shown.
@@ -0,0 +1,105 @@
|
||||
import { describe, expect, it } from '../../src'
|
||||
import { configureModuleHearing, configureOnboarding, loadCaseEnvironment } from '../shared/configurations'
|
||||
import { enableChatMicrophone } from '../shared/interactions'
|
||||
import { aliyunNlsAsr, openaiAsr } from '../shared/providers'
|
||||
|
||||
describe('audio input pipeline', () => {
|
||||
// The fixture has 12 seconds of leading silence so that the browser VAD can load.
|
||||
// It repeats the greeting after a 2-second pause to catch regressions that drop the second utterance.
|
||||
it('keeps two utterances in streaming transcription', {
|
||||
input: new URL('./input.test.wav', import.meta.url),
|
||||
// This regression isolates AIRI's default VAD and one explicit ASR Provider.
|
||||
preflight: [
|
||||
configureOnboarding(() => ({ completed: true })),
|
||||
configureModuleHearing(async (context) => {
|
||||
const environment = await loadCaseEnvironment(context.env)
|
||||
const apiKey = environment.TESTING_AUDIO_ASR_API_KEY
|
||||
context.skip(!apiKey, 'Set TESTING_AUDIO_ASR_API_KEY to run this ASR case.')
|
||||
if (!apiKey)
|
||||
return undefined
|
||||
|
||||
return {
|
||||
provider: openaiAsr({
|
||||
apiKey,
|
||||
baseUrl: environment.TESTING_AUDIO_ASR_API_BASE_URL ?? 'https://api.openai.com/v1/',
|
||||
model: environment.TESTING_AUDIO_ASR_MODEL ?? 'whisper-1',
|
||||
provider: environment.TESTING_AUDIO_ASR_PROVIDER ?? 'openai-compatible-audio-transcription',
|
||||
}),
|
||||
captureFormat: 'wav',
|
||||
}
|
||||
}),
|
||||
],
|
||||
}, async ({ audio }) => {
|
||||
await enableChatMicrophone(audio)
|
||||
|
||||
if (audio.transcriptionCaptureFormat) {
|
||||
await expect(audio).toHaveCapturedTranscriptionAudio({
|
||||
count: 1,
|
||||
minimumBytes: 8000,
|
||||
})
|
||||
}
|
||||
|
||||
const expectedTranscriptions = [
|
||||
[
|
||||
'Microphone warm up, microphone warm up. Hello, AIRI, please say hello.',
|
||||
'Microphone warm up, microphone warm up. Hello, Eric, please say hello.',
|
||||
],
|
||||
[
|
||||
'Microphone warm up, microphone warm up. Hello, AIRI, please say hello.',
|
||||
'Microphone warm up, microphone warm up. Hello, Eric, please say hello.',
|
||||
],
|
||||
]
|
||||
await expect(audio).toHaveTranscriptions(expectedTranscriptions)
|
||||
|
||||
const finalTranscriptions = await audio.transcriptionResults(expectedTranscriptions.length)
|
||||
const streamingUpdates = await audio.streamingTranscriptionUpdates()
|
||||
expect(streamingUpdates.some(update => !finalTranscriptions.includes(update))).toBe(true)
|
||||
})
|
||||
|
||||
it('keeps two Aliyun NLS utterances in streaming transcription', {
|
||||
input: new URL('./input.test.wav', import.meta.url),
|
||||
preflight: [
|
||||
configureOnboarding(() => ({ completed: true })),
|
||||
configureModuleHearing(async (context) => {
|
||||
const environment = await loadCaseEnvironment(context.env)
|
||||
const provider = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_PROVIDER
|
||||
const accessKeyId = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_ALIYUN_AK_ID
|
||||
const accessKeySecret = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_ALIYUN_AK_SECRET
|
||||
const appKey = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_APPKEY
|
||||
context.skip(
|
||||
!provider || !accessKeyId || !accessKeySecret || !appKey,
|
||||
'Set all TESTING_AUDIO_ASR_ALIYUN_NLS_* variables to run this ASR case.',
|
||||
)
|
||||
if (!provider || !accessKeyId || !accessKeySecret || !appKey)
|
||||
return undefined
|
||||
|
||||
return {
|
||||
provider: aliyunNlsAsr({ provider, accessKeyId, accessKeySecret, appKey }),
|
||||
captureFormat: 'pcm',
|
||||
}
|
||||
}),
|
||||
],
|
||||
}, async ({ audio }) => {
|
||||
await enableChatMicrophone(audio, { readiness: 'streaming-transcription' })
|
||||
|
||||
await expect(audio).toHaveCapturedTranscriptionAudio({
|
||||
count: 1,
|
||||
minimumBytes: 8000,
|
||||
})
|
||||
const expectedTranscriptions = [
|
||||
[
|
||||
'Microphone warm up, microphone warm up. Hello, AIRI, please say hello.',
|
||||
'Microphone warm up, microphone warm up. Hello, Eric, please say hello.',
|
||||
],
|
||||
[
|
||||
'Microphone warm up, microphone warm up. Hello, AIRI, please say hello.',
|
||||
'Microphone warm up, microphone warm up. Hello, Eric, please say hello.',
|
||||
],
|
||||
]
|
||||
await expect(audio).toHaveTranscriptions(expectedTranscriptions)
|
||||
|
||||
const finalTranscriptions = await audio.transcriptionResults(expectedTranscriptions.length)
|
||||
const streamingUpdates = await audio.streamingTranscriptionUpdates()
|
||||
expect(streamingUpdates.some(update => !finalTranscriptions.includes(update))).toBe(true)
|
||||
})
|
||||
})
|
||||
Binary file not shown.
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"name": "@proj-airi/testing-audio",
|
||||
"type": "module",
|
||||
"version": "0.11.3",
|
||||
"private": true,
|
||||
"description": "AIRI Web and Electron audio pipeline tests",
|
||||
"author": {
|
||||
"name": "Moeru AI Project AIRI Team",
|
||||
"email": "airi@moeru.ai",
|
||||
"url": "https://github.com/moeru-ai"
|
||||
},
|
||||
"license": "MIT",
|
||||
"exports": "./src/index.ts",
|
||||
"scripts": {
|
||||
"build:targets": "pnpm -F @proj-airi/stage-web build && pnpm -F @proj-airi/stage-tamagotchi build",
|
||||
"test": "vitest",
|
||||
"test:run": "pnpm run build:targets && vitest run",
|
||||
"test:existing-builds": "vitest run",
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@moeru/std": "catalog:",
|
||||
"@pnpm/find-workspace-dir": "catalog:",
|
||||
"@proj-airi/stage-shared": "workspace:^",
|
||||
"@proj-airi/stage-ui": "workspace:^",
|
||||
"@proj-airi/vitest-plugin-fakemic": "workspace:^",
|
||||
"playwright": "catalog:",
|
||||
"vite": "catalog:",
|
||||
"vitest": "catalog:vitest"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "catalog:"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
import type { AudioTestTask } from '@proj-airi/vitest-plugin-fakemic'
|
||||
|
||||
import type { AudioInputPreflightContext, AudioInputSession, AudioInputTestCase } from './types'
|
||||
|
||||
import { env } from 'node:process'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
import { errorMessageFrom } from '@moeru/std'
|
||||
import { createAudioTestAPI, createAudioTestTask, runAudioTestSession, startFakemicRuntime } from '@proj-airi/vitest-plugin-fakemic'
|
||||
import { inject } from 'vitest'
|
||||
|
||||
import { expect, installAudioInputMatchers } from './expect-extend'
|
||||
|
||||
type RunnableAudioInputTest = AudioTestTask
|
||||
|
||||
installAudioInputMatchers()
|
||||
|
||||
const audioTestAPI = createAudioTestAPI<
|
||||
AudioInputTestCase,
|
||||
RunnableAudioInputTest,
|
||||
{ audio: AudioInputSession },
|
||||
AudioInputPreflightContext
|
||||
>({
|
||||
preflight: definition => definition.preflight,
|
||||
createPlans(name, testCase) {
|
||||
const task = createAudioTestTask(name, testCase)
|
||||
return [{
|
||||
name: task.name,
|
||||
definition: task,
|
||||
metadata: {
|
||||
input: fileURLToPath(task.input),
|
||||
runtime: inject('fakemicRuntime').name,
|
||||
},
|
||||
}]
|
||||
},
|
||||
async execute({ plan, task, invokeHandler, runPreflight }) {
|
||||
await runAudioTestSession({
|
||||
start() {
|
||||
const microphoneInput = fileURLToPath(plan.definition.input)
|
||||
return startFakemicRuntime<AudioInputSession>(microphoneInput)
|
||||
},
|
||||
async execute(session) {
|
||||
await runPreflight({
|
||||
env,
|
||||
runtime: session,
|
||||
skip: (condition, note) => task.context.skip(Boolean(condition), note),
|
||||
})
|
||||
await session.runtimePage.reload({ waitUntil: 'domcontentloaded' })
|
||||
await session.runtimePage.locator('[i-solar\\:alt-arrow-up-line-duotone]').first().waitFor({ state: 'visible', timeout: 30_000 })
|
||||
await session.runtimePage.bringToFront()
|
||||
await session.runtimePage.waitForTimeout(750)
|
||||
Object.assign(task.context, { audio: session })
|
||||
await invokeHandler()
|
||||
},
|
||||
async recordArtifacts(session) {
|
||||
const snapshot = await session.snapshot().catch(error => ({
|
||||
snapshotError: errorMessageFrom(error) ?? 'Unknown snapshot error',
|
||||
}))
|
||||
await task.context.annotate('pipeline.json', {
|
||||
body: `${JSON.stringify(snapshot, null, 2)}\n`,
|
||||
bodyEncoding: 'utf-8',
|
||||
contentType: 'application/json',
|
||||
})
|
||||
},
|
||||
})
|
||||
},
|
||||
})
|
||||
|
||||
/** Groups AIRI audio-input tests in the Vitest task tree. */
|
||||
export const describe = audioTestAPI.describe
|
||||
|
||||
/** Defines an AIRI audio-input test for each selected target. */
|
||||
export const it = audioTestAPI.it
|
||||
|
||||
/** Vitest expect with AIRI audio-input matchers installed. */
|
||||
export { expect }
|
||||
@@ -0,0 +1,28 @@
|
||||
import type { AudioInputObservations } from './types'
|
||||
|
||||
import { describe, it } from 'vitest'
|
||||
|
||||
import { expect, installAudioInputMatchers } from './expect-extend'
|
||||
|
||||
installAudioInputMatchers()
|
||||
|
||||
describe('audio input matchers', () => {
|
||||
it('normalizes transcription case, width, punctuation, and whitespace', async () => {
|
||||
const session = createAudioInputSession(['Please, SAY hello!'])
|
||||
|
||||
await expect(session).toHaveTranscriptions([
|
||||
['please say hello'],
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
function createAudioInputSession(transcriptions: string[]): AudioInputObservations {
|
||||
return {
|
||||
capturedTranscriptionAudio: async () => [],
|
||||
streamingTranscriptionUpdates: async () => [],
|
||||
transcriptionResults: async () => transcriptions,
|
||||
completedSpans: async () => [],
|
||||
waitForStreamingTranscriptionReady: async () => {},
|
||||
waitForVadReady: async () => {},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
import type { AudioInputObservations } from './types'
|
||||
|
||||
import { expect as vitestExpect } from 'vitest'
|
||||
|
||||
export interface CapturedTranscriptionAudioExpectation {
|
||||
count: number
|
||||
minimumBytes: number
|
||||
}
|
||||
|
||||
export interface TranscriptionExpectationOptions {
|
||||
/** @default 'exact' */
|
||||
match?: 'exact' | 'contains'
|
||||
}
|
||||
|
||||
declare module 'vitest' {
|
||||
interface Assertion<T> {
|
||||
toHaveCapturedTranscriptionAudio: T extends AudioInputObservations
|
||||
? (expected: CapturedTranscriptionAudioExpectation) => Promise<void>
|
||||
: never
|
||||
toHaveTranscriptions: T extends AudioInputObservations
|
||||
? (
|
||||
expected: ReadonlyArray<ReadonlyArray<string>>,
|
||||
options?: TranscriptionExpectationOptions,
|
||||
) => Promise<void>
|
||||
: never
|
||||
}
|
||||
}
|
||||
|
||||
/** Vitest expect with AIRI audio-input matcher types. */
|
||||
export const expect = vitestExpect
|
||||
|
||||
/**
|
||||
* Normalizes a transcript for speech-recognition comparison.
|
||||
*
|
||||
* @example
|
||||
* normalizeTranscript(' Hello, AIRI! ')
|
||||
* // => 'helloairi'
|
||||
*/
|
||||
function normalizeTranscript(value: string): string {
|
||||
return value
|
||||
.normalize('NFKC')
|
||||
.toLocaleLowerCase()
|
||||
.replace(/[^\p{L}\p{N}]+/gu, '')
|
||||
}
|
||||
|
||||
/** Installs asynchronous matchers for AIRI audio-input observations. */
|
||||
export function installAudioInputMatchers(): void {
|
||||
vitestExpect.extend({
|
||||
async toHaveCapturedTranscriptionAudio(
|
||||
session: AudioInputObservations,
|
||||
expected: CapturedTranscriptionAudioExpectation,
|
||||
) {
|
||||
const format = session.transcriptionCaptureFormat
|
||||
if (!format) {
|
||||
return {
|
||||
pass: false,
|
||||
message: () => 'The active transcription Provider does not expose uploaded audio.',
|
||||
}
|
||||
}
|
||||
|
||||
const captures = await session.capturedTranscriptionAudio(expected.count)
|
||||
const invalidCapture = captures.find((capture) => {
|
||||
if (capture.format !== format || capture.data.byteLength < expected.minimumBytes)
|
||||
return true
|
||||
return format === 'wav' && new TextDecoder().decode(capture.data.subarray(0, 4)) !== 'RIFF'
|
||||
})
|
||||
const pass = captures.length === expected.count && !invalidCapture
|
||||
|
||||
return {
|
||||
pass,
|
||||
message: () => pass
|
||||
? 'Expected the session not to contain valid transcription audio.'
|
||||
: `Expected ${expected.count} ${format} capture(s) with at least ${expected.minimumBytes} bytes.`,
|
||||
}
|
||||
},
|
||||
async toHaveTranscriptions(
|
||||
session: AudioInputObservations,
|
||||
expected: ReadonlyArray<ReadonlyArray<string>>,
|
||||
options: TranscriptionExpectationOptions = {},
|
||||
) {
|
||||
const actual = await session.transcriptionResults(expected.length)
|
||||
const normalizedActual = actual.map(normalizeTranscript)
|
||||
const normalizedExpected = expected.map(alternatives => alternatives.map(normalizeTranscript))
|
||||
const match = options.match ?? 'exact'
|
||||
const pass = normalizedActual.length === normalizedExpected.length
|
||||
&& normalizedActual.every((transcript, index) => (
|
||||
match === 'contains'
|
||||
? normalizedExpected[index].some(candidate => transcript.includes(candidate))
|
||||
: normalizedExpected[index].includes(transcript)
|
||||
))
|
||||
|
||||
return {
|
||||
pass,
|
||||
message: () => pass
|
||||
? 'Expected the session not to contain the specified transcriptions.'
|
||||
: `Expected transcriptions ${JSON.stringify(expected)}, but received ${JSON.stringify(actual)}.`,
|
||||
}
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
export { describe, expect, it } from './describe'
|
||||
|
||||
export type {
|
||||
AudioInputObservations,
|
||||
AudioInputPreflightCallback,
|
||||
AudioInputPreflightContext,
|
||||
AudioInputSession,
|
||||
AudioInputTarget,
|
||||
AudioInputTestCase,
|
||||
} from './types'
|
||||
@@ -0,0 +1,44 @@
|
||||
import type { FakemicElectronPrepareContext } from '@proj-airi/vitest-plugin-fakemic'
|
||||
import type { Page } from 'playwright'
|
||||
|
||||
import type { AudioInputSession } from '../types'
|
||||
|
||||
import { stubForBrowser } from '../setup/browser-probe'
|
||||
import { createSession } from '../setup/session'
|
||||
|
||||
/** Adapts a Fakemic Electron process into an AIRI desktop audio session. */
|
||||
export default async function prepareElectronRuntime(context: FakemicElectronPrepareContext): Promise<AudioInputSession> {
|
||||
await context.app.context().addInitScript(stubForBrowser)
|
||||
const page = await waitForPage(context, (page) => {
|
||||
const url = new URL(page.url())
|
||||
return url.pathname.endsWith('/index.html') && url.hash === '#/'
|
||||
})
|
||||
await page.locator('[i-solar\\:alt-arrow-up-line-duotone]').first().waitFor({ state: 'visible', timeout: 30_000 })
|
||||
await page.evaluate(stubForBrowser)
|
||||
|
||||
return createSession({
|
||||
electronApp: context.app,
|
||||
page,
|
||||
target: 'electron',
|
||||
close: context.close,
|
||||
})
|
||||
}
|
||||
|
||||
async function waitForPage(
|
||||
context: FakemicElectronPrepareContext,
|
||||
predicate: (page: Page) => boolean,
|
||||
timeoutMs = 60_000,
|
||||
): Promise<Page> {
|
||||
const deadline = Date.now() + timeoutMs
|
||||
while (Date.now() < deadline) {
|
||||
const page = context.app.windows().find(predicate)
|
||||
if (page) {
|
||||
await page.waitForLoadState('domcontentloaded')
|
||||
return page
|
||||
}
|
||||
|
||||
await new Promise(resolveWait => setTimeout(resolveWait, 100))
|
||||
}
|
||||
|
||||
throw new Error('Timed out while waiting for the Electron renderer')
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import type { FakemicWebPrepareContext } from '@proj-airi/vitest-plugin-fakemic'
|
||||
|
||||
import type { AudioInputSession } from '../types'
|
||||
|
||||
import { stubForBrowser } from '../setup/browser-probe'
|
||||
import { createSession } from '../setup/session'
|
||||
|
||||
/** Adapts a Fakemic Chromium process into an AIRI Web audio session. */
|
||||
export default async function prepareWebRuntime(context: FakemicWebPrepareContext): Promise<AudioInputSession> {
|
||||
await context.context.addInitScript(stubForBrowser)
|
||||
|
||||
const page = await context.context.newPage()
|
||||
await page.goto(context.runtime.url)
|
||||
|
||||
return createSession({
|
||||
page,
|
||||
target: 'web',
|
||||
close: context.close,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
|
||||
|
||||
/**
|
||||
* Installs passive browser probes before the application starts.
|
||||
*
|
||||
* Triggering workflow:
|
||||
*
|
||||
* `BrowserContext.addInitScript`
|
||||
* -> {@link stubForBrowser}
|
||||
* -> `window.fetch`
|
||||
* -> `BroadcastChannel('io-tracer-channel')`
|
||||
*
|
||||
* Upstream:
|
||||
* - `BrowserContext.addInitScript`
|
||||
*
|
||||
* Downstream:
|
||||
* - `window.__airiAudioInputE2E`
|
||||
*/
|
||||
export function stubForBrowser() {
|
||||
const state: BrowserAudioInputState = { spans: [], streamingTranscriptionReady: false, streamingTranscriptionUpdates: [], transcriptionAudio: [], transcriptionResults: [], vadReady: false }
|
||||
window.__airiAudioInputE2E = state
|
||||
|
||||
const originalConsoleInfo = console.info.bind(console)
|
||||
console.info = (...values: unknown[]) => {
|
||||
if (typeof values[0] === 'string' && values[0].startsWith('[Voice Input] vad-ready:'))
|
||||
state.vadReady = true
|
||||
originalConsoleInfo(...values)
|
||||
}
|
||||
|
||||
const originalFetch = window.fetch.bind(window)
|
||||
|
||||
/**
|
||||
* Copies each ASR upload and response while it forwards the request.
|
||||
*
|
||||
* Triggering workflow:
|
||||
*
|
||||
* `window.fetch`
|
||||
* -> `POST /audio/transcriptions`
|
||||
* -> captureFetch
|
||||
*
|
||||
* Upstream:
|
||||
* - The OpenAI-compatible transcription Provider.
|
||||
*
|
||||
* Downstream:
|
||||
* - `window.__airiAudioInputE2E`
|
||||
* - The original `window.fetch` function.
|
||||
*/
|
||||
const captureFetch: typeof window.fetch = async (input, init) => {
|
||||
const requestUrl = typeof input === 'string'
|
||||
? input
|
||||
: input instanceof URL
|
||||
? input.href
|
||||
: input.url
|
||||
|
||||
const capturesTranscription = new URL(requestUrl, window.location.href).pathname.endsWith('/audio/transcriptions')
|
||||
if (capturesTranscription && init?.body instanceof FormData) {
|
||||
const file = init.body.get('file')
|
||||
if (file instanceof Blob) {
|
||||
state.transcriptionAudio.push({
|
||||
base64: new Uint8Array(await file.arrayBuffer()).toBase64(),
|
||||
format: 'wav',
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const response = await originalFetch(input, init)
|
||||
if (capturesTranscription && response.ok) {
|
||||
const payload = await response.clone().json() as { text?: unknown }
|
||||
if (typeof payload.text === 'string')
|
||||
state.transcriptionResults.push(payload.text)
|
||||
}
|
||||
|
||||
return response
|
||||
}
|
||||
|
||||
window.fetch = captureFetch
|
||||
|
||||
const OriginalWebSocket = window.WebSocket
|
||||
const audioChunksBySocket = new WeakMap<WebSocket, Uint8Array[]>()
|
||||
const capturesAliyunNlsBySocket = new WeakMap<WebSocket, boolean>()
|
||||
const capturedSockets = new WeakSet<WebSocket>()
|
||||
|
||||
function capturePcmAudio(socket: WebSocket) {
|
||||
const audioChunks = audioChunksBySocket.get(socket)
|
||||
if (!audioChunks || capturedSockets.has(socket) || !audioChunks.length)
|
||||
return
|
||||
|
||||
const byteLength = audioChunks.reduce((total, chunk) => total + chunk.byteLength, 0)
|
||||
if (byteLength < 8192)
|
||||
return
|
||||
|
||||
capturedSockets.add(socket)
|
||||
const audio = new Uint8Array(byteLength)
|
||||
let offset = 0
|
||||
for (const chunk of audioChunks) {
|
||||
audio.set(chunk, offset)
|
||||
offset += chunk.byteLength
|
||||
}
|
||||
state.transcriptionAudio.push({ base64: audio.toBase64(), format: 'pcm' })
|
||||
}
|
||||
|
||||
class CaptureAliyunNlsWebSocket extends OriginalWebSocket {
|
||||
constructor(url: string | URL, protocols?: string | string[]) {
|
||||
super(url, protocols)
|
||||
const target = new URL(url.toString(), window.location.href)
|
||||
const capturesAliyunNls = target.hostname.startsWith('nls-gateway-') && target.hostname.endsWith('.aliyuncs.com')
|
||||
capturesAliyunNlsBySocket.set(this, capturesAliyunNls)
|
||||
|
||||
if (capturesAliyunNls) {
|
||||
audioChunksBySocket.set(this, [])
|
||||
this.addEventListener('open', () => {
|
||||
state.streamingTranscriptionReady = true
|
||||
})
|
||||
this.addEventListener('message', (event) => {
|
||||
if (typeof event.data !== 'string')
|
||||
return
|
||||
|
||||
try {
|
||||
const payload = JSON.parse(event.data) as { header?: { name?: string }, payload?: { result?: unknown } }
|
||||
if (payload.header?.name === 'SentenceEnd' && typeof payload.payload?.result === 'string')
|
||||
state.transcriptionResults.push(payload.payload.result)
|
||||
}
|
||||
catch {
|
||||
// NLS can send non-transcription frames. The Provider handles those frames.
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
send(data: string | ArrayBufferLike | Blob | ArrayBufferView): void {
|
||||
const audioChunks = audioChunksBySocket.get(this)
|
||||
if (capturesAliyunNlsBySocket.get(this) && audioChunks && typeof data !== 'string') {
|
||||
if (data instanceof Blob) {
|
||||
void data.arrayBuffer().then((buffer) => {
|
||||
audioChunks.push(new Uint8Array(buffer))
|
||||
})
|
||||
}
|
||||
else if (ArrayBuffer.isView(data)) {
|
||||
audioChunks.push(new Uint8Array(data.buffer, data.byteOffset, data.byteLength))
|
||||
}
|
||||
else {
|
||||
audioChunks.push(new Uint8Array(data))
|
||||
}
|
||||
capturePcmAudio(this)
|
||||
}
|
||||
|
||||
let outboundData: string | Blob | BufferSource
|
||||
if (typeof data === 'string' || data instanceof Blob || data instanceof ArrayBuffer) {
|
||||
outboundData = data
|
||||
}
|
||||
else if (ArrayBuffer.isView(data)) {
|
||||
// WebSocket does not accept views backed by SharedArrayBuffer. The copy uses a regular ArrayBuffer.
|
||||
outboundData = new Uint8Array(data.buffer, data.byteOffset, data.byteLength).slice()
|
||||
}
|
||||
else {
|
||||
// ArrayBufferLike also includes SharedArrayBuffer. The copy keeps the original bytes in a supported buffer.
|
||||
outboundData = new Uint8Array(data).slice()
|
||||
}
|
||||
|
||||
super.send(outboundData)
|
||||
}
|
||||
|
||||
close(code?: number, reason?: string): void {
|
||||
const audioChunks = audioChunksBySocket.get(this)
|
||||
if (capturesAliyunNlsBySocket.get(this) && audioChunks && !capturedSockets.has(this) && audioChunks.length) {
|
||||
capturePcmAudio(this)
|
||||
}
|
||||
|
||||
super.close(code, reason)
|
||||
}
|
||||
}
|
||||
|
||||
window.WebSocket = CaptureAliyunNlsWebSocket
|
||||
|
||||
const channel = new BroadcastChannel('io-tracer-channel')
|
||||
|
||||
/**
|
||||
* Stores completed I/O spans for the case artifact.
|
||||
*
|
||||
* Triggering workflow:
|
||||
*
|
||||
* `BroadcastChannel('io-tracer-channel')`
|
||||
* -> `message`
|
||||
* -> captureSpan
|
||||
*
|
||||
* Upstream:
|
||||
* - The AIRI I/O trace exporter.
|
||||
*
|
||||
* Downstream:
|
||||
* - `window.__airiAudioInputE2E.spans`
|
||||
*/
|
||||
const captureSpan = (event: MessageEvent) => {
|
||||
if (event.data?.type === 'span' && event.data.span?.ended)
|
||||
state.spans.push(event.data.span as SerializedIOSpan)
|
||||
}
|
||||
|
||||
channel.addEventListener('message', captureSpan)
|
||||
}
|
||||
|
||||
/** Returns the completed browser spans that match the optional span name. */
|
||||
export function readCompletedSpans(name?: string) {
|
||||
const spans = window.__airiAudioInputE2E?.spans ?? []
|
||||
return name ? spans.filter(span => span.name === name) : spans
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
import type { AudioCapture } from '@proj-airi/vitest-plugin-fakemic'
|
||||
import type { ElectronApplication, Page } from 'playwright'
|
||||
|
||||
import type { AudioInputSession, AudioInputTarget } from '../types'
|
||||
|
||||
import { Buffer } from 'node:buffer'
|
||||
|
||||
import { readCompletedSpans } from './browser-probe'
|
||||
|
||||
/** Creates the runtime session and records its page diagnostics. */
|
||||
export function createSession(options: {
|
||||
electronApp?: ElectronApplication
|
||||
page: Page
|
||||
target: AudioInputTarget
|
||||
close: () => Promise<void>
|
||||
transcriptionCaptureFormat?: AudioCapture['format']
|
||||
}): AudioInputSession {
|
||||
const diagnostics: string[] = []
|
||||
const observedPages = new WeakSet<Page>()
|
||||
|
||||
function observePage(page: Page) {
|
||||
if (observedPages.has(page))
|
||||
return
|
||||
|
||||
observedPages.add(page)
|
||||
page.on('console', (message) => {
|
||||
const text = message.text()
|
||||
const isAudioPipelineInfo = message.type() === 'info'
|
||||
&& (text.includes('[Hearing Pipeline]') || text.includes('[Voice Input]') || text.includes('transcription'))
|
||||
if (['error', 'warning'].includes(message.type()) || isAudioPipelineInfo)
|
||||
diagnostics.push(`[console:${message.type()}] ${text}`)
|
||||
})
|
||||
page.on('pageerror', error => diagnostics.push(`[pageerror] ${error.message}`))
|
||||
}
|
||||
|
||||
observePage(options.page)
|
||||
|
||||
const session: AudioInputSession = {
|
||||
electronApp: options.electronApp,
|
||||
page: options.page,
|
||||
runtimePage: options.page,
|
||||
target: options.target,
|
||||
transcriptionCaptureFormat: options.transcriptionCaptureFormat,
|
||||
activatePage(page) {
|
||||
observePage(page)
|
||||
session.page = page
|
||||
},
|
||||
async capturedTranscriptionAudio(count) {
|
||||
try {
|
||||
await options.page.waitForFunction(expectedCount => (
|
||||
(window.__airiAudioInputE2E?.transcriptionAudio.length ?? 0) >= expectedCount
|
||||
), count, { timeout: 60_000 })
|
||||
}
|
||||
catch (error) {
|
||||
const runtimeState = await options.page.evaluate(async () => ({
|
||||
activeModel: localStorage.getItem('settings/hearing/active-model'),
|
||||
activeProvider: localStorage.getItem('settings/hearing/active-provider'),
|
||||
devices: (await navigator.mediaDevices.enumerateDevices()).map(device => ({
|
||||
deviceId: device.deviceId,
|
||||
kind: device.kind,
|
||||
label: device.label,
|
||||
})),
|
||||
microphoneEnabled: localStorage.getItem('settings/audio/input/enabled'),
|
||||
microphoneInput: localStorage.getItem('settings/audio/input'),
|
||||
microphoneOffIconVisible: Boolean(document.querySelector('[i-ph\\:microphone-slash]')),
|
||||
probeInstalled: Boolean(window.__airiAudioInputE2E),
|
||||
streamingTranscriptionReady: window.__airiAudioInputE2E?.streamingTranscriptionReady ?? false,
|
||||
url: window.location.href,
|
||||
vadReady: window.__airiAudioInputE2E?.vadReady ?? false,
|
||||
}))
|
||||
throw new Error(`Timed out waiting for captured transcription audio: ${JSON.stringify({ diagnostics, runtimeState })}`, { cause: error })
|
||||
}
|
||||
const capturedAudio = await options.page.evaluate(() => window.__airiAudioInputE2E?.transcriptionAudio ?? [])
|
||||
return capturedAudio.map(audio => ({
|
||||
format: audio.format,
|
||||
data: Buffer.from(audio.base64, 'base64'),
|
||||
}))
|
||||
},
|
||||
streamingTranscriptionUpdates: () => session.page.evaluate(() => window.__airiAudioInputE2E?.streamingTranscriptionUpdates ?? []),
|
||||
async transcriptionResults(count) {
|
||||
await options.page.waitForFunction(expectedCount => (
|
||||
(window.__airiAudioInputE2E?.transcriptionResults.length ?? 0) >= expectedCount
|
||||
), count, { timeout: 60_000 })
|
||||
return options.page.evaluate(() => window.__airiAudioInputE2E?.transcriptionResults ?? [])
|
||||
},
|
||||
async completedSpans(name) {
|
||||
const runtimeSpans = await options.page.evaluate(readCompletedSpans, name)
|
||||
if (session.page === options.page)
|
||||
return runtimeSpans
|
||||
|
||||
const interactionSpans = await session.page.evaluate(readCompletedSpans, name)
|
||||
return [...runtimeSpans, ...interactionSpans]
|
||||
},
|
||||
async waitForVadReady() {
|
||||
await options.page.waitForFunction(() => window.__airiAudioInputE2E?.vadReady === true, undefined, { timeout: 30_000 })
|
||||
},
|
||||
async waitForStreamingTranscriptionReady() {
|
||||
await options.page.waitForFunction(() => window.__airiAudioInputE2E?.streamingTranscriptionReady === true, undefined, { timeout: 30_000 })
|
||||
},
|
||||
async snapshot() {
|
||||
const runtimeState = await options.page.evaluate(() => window.__airiAudioInputE2E)
|
||||
const interactionState = session.page === options.page
|
||||
? runtimeState
|
||||
: await session.page.evaluate(() => window.__airiAudioInputE2E)
|
||||
return {
|
||||
spans: runtimeState?.spans ?? [],
|
||||
streamingTranscriptionUpdates: interactionState?.streamingTranscriptionUpdates ?? [],
|
||||
transcriptionResults: runtimeState?.transcriptionResults ?? [],
|
||||
diagnostics: [...diagnostics],
|
||||
}
|
||||
},
|
||||
close: options.close,
|
||||
}
|
||||
|
||||
return session
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
|
||||
import type { AudioCapture, AudioCaptureFormat, AudioTestCase, AudioTestPreflightCallback, AudioTestSession } from '@proj-airi/vitest-plugin-fakemic'
|
||||
import type { ElectronApplication, Page } from 'playwright'
|
||||
|
||||
export const audioInputTargets = ['web', 'electron'] as const
|
||||
|
||||
export type AudioInputTarget = (typeof audioInputTargets)[number]
|
||||
|
||||
/** Values available when one case resolves its preflight callbacks. */
|
||||
export interface AudioInputPreflightContext {
|
||||
/** Environment variables loaded for this case. */
|
||||
env: Readonly<NodeJS.ProcessEnv>
|
||||
/** Clean runtime that the configuration callback can prepare. */
|
||||
runtime: AudioInputSession
|
||||
/** Skips this case when its environment does not satisfy a case constraint. */
|
||||
skip: (condition: unknown, note?: string) => void
|
||||
}
|
||||
|
||||
/** One optional case callback that resolves configuration from its environment. */
|
||||
export type AudioInputPreflightCallback = AudioTestPreflightCallback<AudioInputPreflightContext>
|
||||
|
||||
/** One AIRI audio-input test definition. */
|
||||
export type AudioInputTestCase = AudioTestCase<AudioInputPreflightContext>
|
||||
|
||||
/** Snapshot of the observable AIRI audio pipeline state. */
|
||||
export interface AudioInputSnapshot {
|
||||
spans: SerializedIOSpan[]
|
||||
streamingTranscriptionUpdates: string[]
|
||||
transcriptionResults: string[]
|
||||
diagnostics: string[]
|
||||
}
|
||||
|
||||
/** Observable audio values used by AIRI matchers. */
|
||||
export interface AudioInputObservations {
|
||||
/** Capture format used by the active transcription Provider. */
|
||||
transcriptionCaptureFormat?: AudioCaptureFormat
|
||||
capturedTranscriptionAudio: (count: number) => Promise<AudioCapture[]>
|
||||
streamingTranscriptionUpdates: () => Promise<string[]>
|
||||
transcriptionResults: (count: number) => Promise<string[]>
|
||||
completedSpans: (name?: string) => Promise<SerializedIOSpan[]>
|
||||
/** Waits until the VAD audio graph is connected to the microphone stream. */
|
||||
waitForVadReady: () => Promise<void>
|
||||
/** Waits until a streaming transcription transport accepts microphone audio. */
|
||||
waitForStreamingTranscriptionReady: () => Promise<void>
|
||||
}
|
||||
|
||||
/** Runtime handle for one AIRI audio-input test. */
|
||||
export interface AudioInputSession extends AudioInputObservations, AudioTestSession {
|
||||
/** Electron application for Electron tasks. */
|
||||
electronApp?: ElectronApplication
|
||||
/** Page used by case interactions. */
|
||||
page: Page
|
||||
/** Page that owns the audio-input pipeline. */
|
||||
runtimePage: Page
|
||||
/** Runtime selected for this concrete task. */
|
||||
target: AudioInputTarget
|
||||
/** Selects the page used by subsequent case interactions. */
|
||||
activatePage: (page: Page) => void
|
||||
snapshot: () => Promise<AudioInputSnapshot>
|
||||
}
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
|
||||
|
||||
declare global {
|
||||
interface BrowserAudioInputState {
|
||||
spans: SerializedIOSpan[]
|
||||
streamingTranscriptionReady: boolean
|
||||
streamingTranscriptionUpdates: string[]
|
||||
transcriptionAudio: Array<{ base64: string, format: 'pcm' | 'wav' }>
|
||||
transcriptionResults: string[]
|
||||
vadReady: boolean
|
||||
}
|
||||
|
||||
interface Window {
|
||||
__airiAudioInputE2E?: BrowserAudioInputState
|
||||
}
|
||||
|
||||
// NOTICE:
|
||||
// TypeScript 5.9 does not declare the Baseline 2025 Uint8Array Base64 methods.
|
||||
// The test runs in current Playwright and Electron Chromium runtimes that implement this API.
|
||||
// Source: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Uint8Array/toBase64
|
||||
// Remove this declaration when the TypeScript standard library includes the method.
|
||||
interface Uint8Array<TArrayBuffer extends ArrayBufferLike = ArrayBufferLike> {
|
||||
toBase64: (options?: {
|
||||
alphabet?: 'base64' | 'base64url'
|
||||
omitPadding?: boolean
|
||||
}) => string
|
||||
}
|
||||
}
|
||||
|
||||
export {}
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ESNext",
|
||||
"lib": [
|
||||
"ESNext",
|
||||
"DOM",
|
||||
"DOM.Iterable"
|
||||
],
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Bundler",
|
||||
"types": [
|
||||
"node"
|
||||
],
|
||||
"strict": true,
|
||||
"noEmit": true,
|
||||
"isolatedModules": true,
|
||||
"verbatimModuleSyntax": true,
|
||||
"skipLibCheck": true
|
||||
},
|
||||
"include": [
|
||||
"cases/**/*.ts",
|
||||
"src/**/*.ts",
|
||||
"vitest.config.ts"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import { join } from 'node:path'
|
||||
|
||||
import fakemic, { electron, web } from '@proj-airi/vitest-plugin-fakemic'
|
||||
|
||||
import { defineConfig } from 'vitest/config'
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
projects: [
|
||||
fakemic({
|
||||
name: 'audio-web',
|
||||
include: ['cases/**/*.audio.test.ts', 'cases/**/*.audio.web.test.ts'],
|
||||
runtime: web({
|
||||
name: 'web',
|
||||
prepare: new URL('./src/runtimes/prepare-web.ts', import.meta.url).href,
|
||||
url: 'http://127.0.0.1:4173/',
|
||||
context: { permissions: ['microphone'] },
|
||||
preview: {
|
||||
configFile: join(import.meta.dirname, '../../apps/stage-web/vite.config.ts'),
|
||||
root: join(import.meta.dirname, '../../apps/stage-web'),
|
||||
},
|
||||
}),
|
||||
}),
|
||||
fakemic({
|
||||
name: 'audio-electron',
|
||||
include: ['cases/**/*.audio.test.ts', 'cases/**/*.audio.electron.test.ts'],
|
||||
runtime: electron({
|
||||
name: 'electron',
|
||||
prepare: new URL('./src/runtimes/prepare-electron.ts', import.meta.url).href,
|
||||
entry: join(import.meta.dirname, '../../apps/stage-tamagotchi/out/main/index.js'),
|
||||
args: ['--no-sandbox'],
|
||||
cwd: join(import.meta.dirname, '../..'),
|
||||
temporaryUserData: {
|
||||
env: 'APP_USER_DATA_PATH',
|
||||
prefix: 'airi-testing-audio-',
|
||||
},
|
||||
}),
|
||||
}),
|
||||
{
|
||||
extends: true,
|
||||
test: {
|
||||
name: 'unit',
|
||||
include: ['src/**/*.test.ts'],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
})
|
||||
@@ -0,0 +1,32 @@
|
||||
# Vitest plugin for fake microphones
|
||||
|
||||
This package supplies fake-microphone Web and Electron runtimes with Vitest project integration.
|
||||
|
||||
Import the Vitest interface from the package root:
|
||||
|
||||
```ts
|
||||
import fakemic, {
|
||||
createAudioTestAPI,
|
||||
electron,
|
||||
runAudioTestSession,
|
||||
web,
|
||||
} from '@proj-airi/vitest-plugin-fakemic'
|
||||
```
|
||||
|
||||
`src/index.ts` owns project configuration, runtime launch, task collection, and preflight scheduling. `src/runner.ts` owns task execution.
|
||||
|
||||
Runtime configuration contains only serializable values. A `prepare` module adapts the launched Playwright runtime into an application session.
|
||||
|
||||
```ts
|
||||
fakemic({
|
||||
name: 'audio-web',
|
||||
include: ['cases/**/*.audio.test.ts', 'cases/**/*.audio.web.test.ts'],
|
||||
runtime: web({
|
||||
name: 'web',
|
||||
prepare: new URL('./prepare-web.ts', import.meta.url).href,
|
||||
url: 'http://127.0.0.1:4173/',
|
||||
}),
|
||||
})
|
||||
```
|
||||
|
||||
Application packages own selectors, routes, probes, Provider settings, and prepare modules.
|
||||
@@ -1,22 +1,24 @@
|
||||
{
|
||||
"name": "@proj-airi/audio-input-e2e",
|
||||
"name": "@proj-airi/vitest-plugin-fakemic",
|
||||
"type": "module",
|
||||
"version": "0.11.3",
|
||||
"private": true,
|
||||
"description": "End-to-end audio input contracts for AIRI Web and Electron",
|
||||
"description": "Fake microphone test runtime and Vitest integration",
|
||||
"author": {
|
||||
"name": "Moeru AI Project AIRI Team",
|
||||
"email": "airi@moeru.ai",
|
||||
"url": "https://github.com/moeru-ai"
|
||||
},
|
||||
"license": "MIT",
|
||||
"exports": "./src/index.ts",
|
||||
"scripts": {
|
||||
"test": "vitest run",
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@pnpm/find-workspace-dir": "catalog:",
|
||||
"@proj-airi/stage-shared": "workspace:^",
|
||||
"playwright": "catalog:",
|
||||
"valibot": "catalog:"
|
||||
"vite": "catalog:",
|
||||
"vitest": "catalog:vitest"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "catalog:"
|
||||
@@ -0,0 +1,91 @@
|
||||
import {
|
||||
createAudioTestAPI,
|
||||
createAudioTestTask,
|
||||
runAudioTestSession,
|
||||
} from '@proj-airi/vitest-plugin-fakemic'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const calls: string[] = []
|
||||
const audio = createAudioTestAPI<
|
||||
{ value: string, preflight?: readonly ((context: { value: string }) => void)[] },
|
||||
{ value: string },
|
||||
{ capturedValue: string },
|
||||
{ value: string }
|
||||
>({
|
||||
createPlans: (name, definition) => [{
|
||||
name: `mock: ${name}`,
|
||||
definition,
|
||||
metadata: {
|
||||
input: '/fixtures/input.wav',
|
||||
runtime: 'mock',
|
||||
},
|
||||
}],
|
||||
preflight: definition => definition.preflight,
|
||||
async execute({ plan, task, invokeHandler, runPreflight }) {
|
||||
await runPreflight({ value: 'preflight' })
|
||||
Object.assign(task.context, {
|
||||
capturedValue: plan.definition.value,
|
||||
})
|
||||
await invokeHandler()
|
||||
},
|
||||
})
|
||||
|
||||
audio.describe('createAudioTestAPI', () => {
|
||||
audio.it('runs preflight before a registered task', {
|
||||
value: 'captured',
|
||||
preflight: [({ value }) => calls.push(value)],
|
||||
}, ({ capturedValue }) => {
|
||||
expect(calls).toEqual(['preflight'])
|
||||
expect(capturedValue).toBe('captured')
|
||||
})
|
||||
})
|
||||
|
||||
describe('audio test tasks', () => {
|
||||
it('creates one task for the current runtime project', () => {
|
||||
const input = new URL('file:///audio/input.wav')
|
||||
const task = createAudioTestTask('captures speech', { input })
|
||||
|
||||
expect(task).toEqual({ name: 'captures speech', input })
|
||||
})
|
||||
|
||||
it('records artifacts before it closes the session', async () => {
|
||||
const calls: string[] = []
|
||||
const session = {
|
||||
close: vi.fn(async () => {
|
||||
calls.push('close')
|
||||
}),
|
||||
}
|
||||
|
||||
await runAudioTestSession({
|
||||
start: async () => session,
|
||||
execute: async () => {
|
||||
calls.push('execute')
|
||||
},
|
||||
recordArtifacts: async () => {
|
||||
calls.push('record')
|
||||
},
|
||||
})
|
||||
|
||||
expect(calls).toEqual(['execute', 'record', 'close'])
|
||||
})
|
||||
|
||||
it('closes the session and keeps execution and cleanup failures', async () => {
|
||||
const executionError = new Error('execution failed')
|
||||
const closeError = new Error('close failed')
|
||||
|
||||
const result = runAudioTestSession({
|
||||
start: async () => ({
|
||||
close: async () => {
|
||||
throw closeError
|
||||
},
|
||||
}),
|
||||
execute: async () => {
|
||||
throw executionError
|
||||
},
|
||||
})
|
||||
|
||||
await expect(result).rejects.toMatchObject({
|
||||
errors: [executionError, closeError],
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,420 @@
|
||||
import type { Browser, BrowserContext, ElectronApplication } from 'playwright'
|
||||
import type { RunnerTestCase, TestAPI, TestContext } from 'vitest'
|
||||
import type { UserWorkspaceConfig } from 'vitest/config'
|
||||
|
||||
import { mkdtemp, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { env } from 'node:process'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
import { chromium, _electron as playwrightElectron } from 'playwright'
|
||||
import { preview } from 'vite'
|
||||
import { describe, inject, TestRunner } from 'vitest'
|
||||
|
||||
/** Supported encodings for captured audio. */
|
||||
export type AudioCaptureFormat = 'pcm' | 'wav'
|
||||
|
||||
/** One audio payload captured from a tested runtime. */
|
||||
export interface AudioCapture {
|
||||
format: AudioCaptureFormat
|
||||
data: Uint8Array
|
||||
}
|
||||
|
||||
/** A session that owns one audio test runtime. */
|
||||
export interface AudioTestSession {
|
||||
/** Releases the runtime and all resources that belong to the session. */
|
||||
close: () => Promise<void>
|
||||
}
|
||||
|
||||
/** One callback that runs after a runtime starts and before its test handler. */
|
||||
export type AudioTestPreflightCallback<Context> = (context: Context) => void | Promise<void>
|
||||
|
||||
/** A runner-neutral audio test definition. */
|
||||
export interface AudioTestCase<PreflightContext = never> {
|
||||
/** File-backed microphone input for the test. */
|
||||
input: URL
|
||||
/** @default [] */
|
||||
preflight?: readonly AudioTestPreflightCallback<PreflightContext>[]
|
||||
}
|
||||
|
||||
/** One runnable task derived from an audio test case. */
|
||||
export interface AudioTestTask {
|
||||
name: string
|
||||
input: URL
|
||||
}
|
||||
|
||||
/** Lifecycle operations for one audio test session. */
|
||||
export interface RunAudioTestSessionOptions<Session extends AudioTestSession> {
|
||||
start: () => Promise<Session>
|
||||
execute: (session: Session) => Promise<void>
|
||||
recordArtifacts?: (session: Session) => Promise<void>
|
||||
}
|
||||
|
||||
/** Metadata that identifies an audio task in Vitest reports. */
|
||||
export interface AudioVitestTaskMetadata {
|
||||
input: string
|
||||
runtime: string
|
||||
}
|
||||
|
||||
declare module 'vitest' {
|
||||
interface TaskMeta {
|
||||
audioTest?: AudioVitestTaskMetadata
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* One concrete audio task that the custom runner can execute.
|
||||
*
|
||||
* @param Definition - The concrete definition stored for this task.
|
||||
*/
|
||||
export interface AudioVitestPlan<Definition> {
|
||||
name: string
|
||||
definition: Definition
|
||||
metadata: AudioVitestTaskMetadata
|
||||
}
|
||||
|
||||
/**
|
||||
* Test context exposed to an audio test callback.
|
||||
*
|
||||
* @param Context - Fields added by the concrete audio framework.
|
||||
*/
|
||||
export type AudioVitestTaskContext<Context extends object> = TestContext & Context
|
||||
|
||||
/**
|
||||
* Callback for an audio task.
|
||||
*
|
||||
* @param Context - Fields added by the concrete audio framework.
|
||||
*/
|
||||
export type AudioTestHandler<Context extends object> = (
|
||||
context: AudioVitestTaskContext<Context>,
|
||||
) => void | Promise<void>
|
||||
|
||||
/**
|
||||
* A Vitest-like test function that accepts an audio definition.
|
||||
*
|
||||
* @param Definition - The definition supplied by each test.
|
||||
* @param Context - Fields exposed to the test callback.
|
||||
*/
|
||||
export interface AudioTestAPI<Definition, Context extends object> {
|
||||
(name: string, definition: Definition, handler: AudioTestHandler<Context>): void
|
||||
only: AudioTestAPI<Definition, Context>
|
||||
skip: AudioTestAPI<Definition, Context>
|
||||
todo: AudioTestAPI<Definition, Context>
|
||||
fails: AudioTestAPI<Definition, Context>
|
||||
}
|
||||
|
||||
/**
|
||||
* Configuration for a package-owned audio test interface.
|
||||
*
|
||||
* @param Definition - The definition supplied by each test.
|
||||
* @param Plan - One concrete task produced from that definition.
|
||||
*/
|
||||
export interface CreateAudioTestAPIOptions<Definition, Plan, PreflightContext> {
|
||||
createPlans: (name: string, definition: Definition) => Array<AudioVitestPlan<Plan>>
|
||||
execute: (options: {
|
||||
plan: AudioVitestPlan<Plan>
|
||||
task: RunnerTestCase
|
||||
invokeHandler: () => Promise<void>
|
||||
runPreflight: (context: PreflightContext) => Promise<void>
|
||||
}) => Promise<void>
|
||||
preflight?: (definition: Definition) => readonly AudioTestPreflightCallback<PreflightContext>[] | undefined
|
||||
}
|
||||
|
||||
/** Serializable Web runtime configuration. */
|
||||
export interface FakemicWebRuntime {
|
||||
kind: 'web'
|
||||
name: string
|
||||
prepare: string
|
||||
url: string
|
||||
launch?: Parameters<typeof chromium.launch>[0]
|
||||
context?: Parameters<Browser['newContext']>[0]
|
||||
preview?: {
|
||||
configFile: string
|
||||
root: string
|
||||
host?: string
|
||||
port?: number
|
||||
}
|
||||
}
|
||||
|
||||
/** Serializable Electron runtime configuration. */
|
||||
export interface FakemicElectronRuntime {
|
||||
kind: 'electron'
|
||||
name: string
|
||||
prepare: string
|
||||
entry: string
|
||||
args?: string[]
|
||||
cwd?: string
|
||||
env?: Record<string, string>
|
||||
temporaryUserData?: {
|
||||
env: string
|
||||
prefix?: string
|
||||
}
|
||||
}
|
||||
|
||||
/** Runtime selected by one Fakemic Vitest project. */
|
||||
export type FakemicRuntime = FakemicElectronRuntime | FakemicWebRuntime
|
||||
|
||||
/** Context supplied to a Web prepare module. */
|
||||
export interface FakemicWebPrepareContext {
|
||||
browser: Browser
|
||||
context: BrowserContext
|
||||
close: () => Promise<void>
|
||||
runtime: FakemicWebRuntime
|
||||
}
|
||||
|
||||
/** Context supplied to an Electron prepare module. */
|
||||
export interface FakemicElectronPrepareContext {
|
||||
app: ElectronApplication
|
||||
close: () => Promise<void>
|
||||
runtime: FakemicElectronRuntime
|
||||
}
|
||||
|
||||
/** Module that adapts a launched runtime into the application session. */
|
||||
export interface FakemicPrepareModule<Context, Session extends AudioTestSession> {
|
||||
default: (context: Context) => Promise<Session>
|
||||
}
|
||||
|
||||
/** Configuration for one Fakemic Vitest project. */
|
||||
export interface FakemicPluginOptions {
|
||||
include: string[]
|
||||
name: string
|
||||
runtime: FakemicRuntime
|
||||
/** @default 180000 */
|
||||
testTimeout?: number
|
||||
/** @default 120000 */
|
||||
hookTimeout?: number
|
||||
}
|
||||
|
||||
declare module 'vitest' {
|
||||
interface ProvidedContext {
|
||||
fakemicRuntime: FakemicRuntime
|
||||
}
|
||||
}
|
||||
|
||||
interface FakemicTaskExecution {
|
||||
run: (task: RunnerTestCase, invokeHandler: () => Promise<void>) => Promise<void>
|
||||
}
|
||||
|
||||
const registryKey = Symbol.for('airi.vitest-plugin-fakemic.executions')
|
||||
const registryHost = globalThis as typeof globalThis & Record<typeof registryKey, WeakMap<RunnerTestCase, FakemicTaskExecution> | undefined>
|
||||
|
||||
// NOTICE:
|
||||
// Vitest can load the runner and collected test modules through different module IDs.
|
||||
// The global symbol gives both module instances access to the same task registry.
|
||||
// Source: the Vitest ModuleRunner seam between the runner and collected test files.
|
||||
// Remove this registry when Vitest provides a public task execution registry.
|
||||
const fakemicTaskExecutions = registryHost[registryKey] ??= new WeakMap()
|
||||
|
||||
/** Creates package-owned `describe` and `it` functions for an audio framework. */
|
||||
export function createAudioTestAPI<Definition, Plan, Context extends object, PreflightContext = never>(
|
||||
options: CreateAudioTestAPIOptions<Definition, Plan, PreflightContext>,
|
||||
): {
|
||||
describe: typeof describe
|
||||
it: AudioTestAPI<Definition, Context>
|
||||
} {
|
||||
const collector = TestRunner.createTaskCollector(function (
|
||||
this: object,
|
||||
name: string,
|
||||
definition: Definition,
|
||||
handler: AudioTestHandler<Context>,
|
||||
) {
|
||||
const plans = options.createPlans(name, definition)
|
||||
const preflight = options.preflight?.(definition) ?? []
|
||||
|
||||
for (const plan of plans) {
|
||||
const task = TestRunner.getCurrentSuite<Context>().task(plan.name, {
|
||||
...this,
|
||||
meta: {
|
||||
audioTest: plan.metadata,
|
||||
},
|
||||
handler: async (context) => {
|
||||
await handler(context as AudioVitestTaskContext<Context>)
|
||||
},
|
||||
})
|
||||
|
||||
fakemicTaskExecutions.set(task, {
|
||||
run: (runnerTask, invokeHandler) => options.execute({
|
||||
plan,
|
||||
task: runnerTask,
|
||||
invokeHandler,
|
||||
async runPreflight(context) {
|
||||
for (const callback of preflight)
|
||||
await callback(context)
|
||||
},
|
||||
}),
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
return {
|
||||
describe,
|
||||
it: collector as TestAPI as AudioTestAPI<Definition, Context>,
|
||||
}
|
||||
}
|
||||
|
||||
/** Creates a Web runtime descriptor for one Fakemic project. */
|
||||
export function web(options: Omit<FakemicWebRuntime, 'kind'>): FakemicWebRuntime {
|
||||
return { kind: 'web', ...options }
|
||||
}
|
||||
|
||||
/** Creates an Electron runtime descriptor for one Fakemic project. */
|
||||
export function electron(options: Omit<FakemicElectronRuntime, 'kind'>): FakemicElectronRuntime {
|
||||
return { kind: 'electron', ...options }
|
||||
}
|
||||
|
||||
/** Configures a serial Node Vitest project for package-owned audio tests. */
|
||||
export default function fakemic(options: FakemicPluginOptions): UserWorkspaceConfig {
|
||||
return {
|
||||
test: {
|
||||
name: options.name,
|
||||
include: options.include,
|
||||
environment: 'node',
|
||||
runner: fileURLToPath(new URL('./runner.ts', import.meta.url)),
|
||||
fileParallelism: false,
|
||||
maxWorkers: 1,
|
||||
testTimeout: options.testTimeout ?? 180_000,
|
||||
hookTimeout: options.hookTimeout ?? 120_000,
|
||||
provide: {
|
||||
fakemicRuntime: options.runtime,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/** Creates Chromium arguments for a non-looping file-backed microphone. */
|
||||
export function createChromiumFileMicrophoneArguments(microphoneInput: string): string[] {
|
||||
return [
|
||||
'--use-fake-ui-for-media-stream',
|
||||
'--use-fake-device-for-media-stream',
|
||||
`--use-file-for-fake-audio-capture=${microphoneInput}%noloop`,
|
||||
'--autoplay-policy=no-user-gesture-required',
|
||||
]
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates one runnable task from an audio case.
|
||||
*
|
||||
* @example
|
||||
* createAudioTestTask('greets the user', { input })
|
||||
* // => { name: 'greets the user', input }
|
||||
*/
|
||||
export function createAudioTestTask<PreflightContext = never>(
|
||||
name: string,
|
||||
testCase: AudioTestCase<PreflightContext>,
|
||||
): AudioTestTask {
|
||||
return {
|
||||
name,
|
||||
input: testCase.input,
|
||||
}
|
||||
}
|
||||
|
||||
/** Launches the runtime selected by the current Vitest project. */
|
||||
export async function startFakemicRuntime<Session extends AudioTestSession>(microphoneInput: string): Promise<Session> {
|
||||
const runtime = inject('fakemicRuntime')
|
||||
if (runtime.kind === 'electron')
|
||||
return startElectronFakemicRuntime<Session>(runtime, microphoneInput)
|
||||
return startWebFakemicRuntime<Session>(runtime, microphoneInput)
|
||||
}
|
||||
|
||||
async function startWebFakemicRuntime<Session extends AudioTestSession>(runtime: FakemicWebRuntime, microphoneInput: string): Promise<Session> {
|
||||
const server = runtime.preview
|
||||
? await preview({
|
||||
configFile: runtime.preview.configFile,
|
||||
root: runtime.preview.root,
|
||||
preview: {
|
||||
host: runtime.preview.host ?? '127.0.0.1',
|
||||
port: runtime.preview.port ?? 4173,
|
||||
strictPort: true,
|
||||
},
|
||||
})
|
||||
: undefined
|
||||
let browser: Browser | undefined
|
||||
const close = async () => {
|
||||
await browser?.close()
|
||||
await server?.close()
|
||||
}
|
||||
|
||||
try {
|
||||
browser = await chromium.launch({
|
||||
...runtime.launch,
|
||||
args: [...(runtime.launch?.args ?? []), ...createChromiumFileMicrophoneArguments(microphoneInput)],
|
||||
})
|
||||
const context = await browser.newContext(runtime.context)
|
||||
const module = await import(runtime.prepare) as FakemicPrepareModule<FakemicWebPrepareContext, Session>
|
||||
return await module.default({ browser, context, close, runtime })
|
||||
}
|
||||
catch (error) {
|
||||
await close()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function startElectronFakemicRuntime<Session extends AudioTestSession>(runtime: FakemicElectronRuntime, microphoneInput: string): Promise<Session> {
|
||||
const temporaryUserData = runtime.temporaryUserData
|
||||
const userDataPath = temporaryUserData
|
||||
? await mkdtemp(join(tmpdir(), temporaryUserData.prefix ?? 'fakemic-electron-'))
|
||||
: undefined
|
||||
let app: ElectronApplication | undefined
|
||||
const close = async () => {
|
||||
await app?.close()
|
||||
if (userDataPath)
|
||||
await rm(userDataPath, { recursive: true, force: true })
|
||||
}
|
||||
|
||||
try {
|
||||
const launchEnvironment = Object.fromEntries(
|
||||
Object.entries({ ...env, ...runtime.env })
|
||||
.filter((entry): entry is [string, string] => entry[1] !== undefined),
|
||||
)
|
||||
if (temporaryUserData && userDataPath)
|
||||
launchEnvironment[temporaryUserData.env] = userDataPath
|
||||
|
||||
app = await playwrightElectron.launch({
|
||||
args: [runtime.entry, ...(runtime.args ?? []), ...createChromiumFileMicrophoneArguments(microphoneInput)],
|
||||
cwd: runtime.cwd,
|
||||
env: launchEnvironment,
|
||||
})
|
||||
const launchedApp = app
|
||||
const module = await import(runtime.prepare) as FakemicPrepareModule<FakemicElectronPrepareContext, Session>
|
||||
return await module.default({ app: launchedApp, close, runtime })
|
||||
}
|
||||
catch (error) {
|
||||
await close()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/** Runs one audio session and preserves execution and cleanup failures. */
|
||||
export async function runAudioTestSession<Session extends AudioTestSession>(
|
||||
options: RunAudioTestSessionOptions<Session>,
|
||||
): Promise<void> {
|
||||
const session = await options.start()
|
||||
const errors: unknown[] = []
|
||||
|
||||
try {
|
||||
await options.execute(session)
|
||||
}
|
||||
catch (error) {
|
||||
errors.push(error)
|
||||
}
|
||||
|
||||
try {
|
||||
await options.recordArtifacts?.(session)
|
||||
}
|
||||
catch (error) {
|
||||
errors.push(error)
|
||||
}
|
||||
|
||||
try {
|
||||
await session.close()
|
||||
}
|
||||
catch (error) {
|
||||
errors.push(error)
|
||||
}
|
||||
|
||||
if (errors.length === 1)
|
||||
throw errors[0]
|
||||
if (errors.length > 1)
|
||||
throw new AggregateError(errors, 'The audio test and its cleanup produced multiple errors')
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import type { RunnerTestCase } from 'vitest'
|
||||
|
||||
import { TestRunner } from 'vitest'
|
||||
|
||||
interface FakemicTaskExecution {
|
||||
run: (task: RunnerTestCase, invokeHandler: () => Promise<void>) => Promise<void>
|
||||
}
|
||||
|
||||
const registryKey = Symbol.for('airi.vitest-plugin-fakemic.executions')
|
||||
const registryHost = globalThis as typeof globalThis & Record<typeof registryKey, WeakMap<RunnerTestCase, FakemicTaskExecution> | undefined>
|
||||
const fakemicTaskExecutions = registryHost[registryKey] ??= new WeakMap()
|
||||
|
||||
/**
|
||||
* Runs package-owned audio tasks and delegates normal tasks to Vitest.
|
||||
*
|
||||
* Call stack:
|
||||
*
|
||||
* FakemicVitestRunner.runTask
|
||||
* -> fakemic task execution
|
||||
* -> {@link TestRunner.getTestFn}
|
||||
*/
|
||||
export default class FakemicVitestRunner extends TestRunner {
|
||||
async runTask(task: RunnerTestCase): Promise<void> {
|
||||
const handler = TestRunner.getTestFn(task)
|
||||
if (!handler)
|
||||
throw new Error(`Vitest did not collect a handler for "${task.name}"`)
|
||||
|
||||
const execution = fakemicTaskExecutions.get(task)
|
||||
if (!execution) {
|
||||
await handler()
|
||||
return
|
||||
}
|
||||
|
||||
await execution.run(task, async () => {
|
||||
await handler()
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ESNext",
|
||||
"lib": [
|
||||
"ESNext"
|
||||
],
|
||||
"rootDir": ".",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Bundler",
|
||||
"types": [
|
||||
"node"
|
||||
],
|
||||
"strict": true,
|
||||
"noEmit": true,
|
||||
"isolatedModules": true,
|
||||
"verbatimModuleSyntax": true,
|
||||
"skipLibCheck": true
|
||||
},
|
||||
"include": [
|
||||
"src/**/*.ts",
|
||||
"vitest.config.ts"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
import { defineConfig } from 'vitest/config'
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
include: ['src/**/*.test.ts'],
|
||||
runner: './src/runner.ts',
|
||||
},
|
||||
})
|
||||
Generated
+1741
-282
File diff suppressed because it is too large
Load Diff
+1
-1
@@ -115,7 +115,7 @@ catalog:
|
||||
'@mediapipe/tasks-vision': ^0.10.34
|
||||
'@modelcontextprotocol/sdk': ^1.29.0
|
||||
'@moeru/eslint-config': 0.1.0-beta.19
|
||||
'@moeru/eventa': 1.0.0-beta.13
|
||||
'@moeru/eventa': 1.0.0-beta.15
|
||||
'@moeru/std': 0.1.0-beta.17
|
||||
'@moeru/three-mmd': 0.1.0-beta.7
|
||||
'@moeru/three-mmd-physics-ammo': 0.1.0-beta.7
|
||||
|
||||
@@ -19,6 +19,7 @@ export default defineConfig({
|
||||
'packages/server-runtime',
|
||||
'packages/server-sdk',
|
||||
'packages/stage-shared',
|
||||
'packages/vitest-plugin-fakemic',
|
||||
],
|
||||
},
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user