feat(testing-audio): now entire audio input/output pipeline can be tested

This commit is contained in:
Neko Ayaka
2026-08-13 22:12:24 +08:00
parent 8d241c3d71
commit 0ef3a26f7d
53 changed files with 4204 additions and 291 deletions
@@ -598,6 +598,18 @@ function getVoiceInputGeneration(metadata?: Record<string, unknown>) {
const voiceInputSession = useVoiceInputSession(stream, {
shouldUseStreamInput,
onLog(level, event, message, details) {
const output = `[Voice Input] ${event}: ${message}`
if (level === 'error') {
console.error(output, details ?? {})
return
}
if (level === 'warn') {
console.warn(output, details ?? {})
return
}
console.info(output, details ?? {})
},
canStartSegment: () => enabled.value && !isVoiceInputSuppressed(),
inspectBeforeTranscription: ({ metadata }) => inspectVoiceInputProviderRequestGate(getVoiceInputGeneration(metadata)),
inspectAfterTranscription: ({ metadata }) => inspectVoiceInputProviderRequestGate(getVoiceInputGeneration(metadata)),
+1
View File
@@ -111,6 +111,7 @@ words:
- esaxx
- eventa
- Factorio
- fakemic
- feaxios
- fflate
- Flathub
@@ -51,7 +51,7 @@
},
"inlinedDependencies": {
"@electron-toolkit/preload": "3.0.2",
"@moeru/eventa": "1.0.0-beta.13",
"@moeru/eventa": "1.0.0-beta.15",
"async-mutex": "0.5.0",
"nanoid": [
"5.1.11",
@@ -9,6 +9,7 @@ const audioRecorderMock = vi.hoisted(() => ({
}))
const vadMock = vi.hoisted(() => ({
init: vi.fn<() => Promise<void>>(),
options: undefined as {
onSpeechStart?: () => void
onSpeechEnd?: () => void
@@ -16,6 +17,7 @@ const vadMock = vi.hoisted(() => ({
onSpeechReady?: (event: { buffer: Float32Array, duration: number }) => void
minSilenceDurationMs?: number
} | undefined,
start: vi.fn<() => Promise<void>>(),
}))
const hearingPipelineMock = vi.hoisted(() => ({
@@ -33,9 +35,9 @@ vi.mock('../../stores/ai/models/vad', async () => {
useVAD: (_workerUrl: string, options: typeof vadMock.options) => {
vadMock.options = options
return {
init: vi.fn(),
init: vadMock.init,
dispose: vi.fn(),
start: vi.fn(),
start: vadMock.start,
loaded: vue.ref(true),
isSpeech: vue.ref(false),
isSpeechProb: vue.ref(0),
@@ -82,6 +84,8 @@ describe('useVoiceInputSession', () => {
audioRecorderMock.isRecording.value = false
audioRecorderMock.onStopRecordHook = undefined
vadMock.options = undefined
vadMock.init.mockReset().mockResolvedValue(undefined)
vadMock.start.mockReset().mockResolvedValue(undefined)
vi.useRealTimers()
vi.unstubAllGlobals()
vi.clearAllMocks()
@@ -3,6 +3,26 @@ import { describe, expect, it, vi } from 'vitest'
import { startVoiceInputVadDetectionSafely } from './voice-input-vad-startup'
describe('voice input VAD startup', () => {
it('reports readiness after VAD connects to the microphone stream', async () => {
const log = vi.fn()
const start = vi.fn().mockResolvedValue(undefined)
await expect(startVoiceInputVadDetectionSafely({
init: vi.fn().mockResolvedValue(undefined),
loaded: () => true,
start,
stream: {} as MediaStream,
log,
})).resolves.toBe(true)
expect(start).toHaveBeenCalledOnce()
expect(log).toHaveBeenLastCalledWith(
'info',
'vad-ready',
'VAD is connected to the microphone stream.',
)
})
it('returns false and logs when VAD initialization throws', async () => {
const init = vi.fn().mockRejectedValue(new Error('vad unavailable'))
const start = vi.fn()
@@ -18,6 +18,7 @@ export async function startVoiceInputVadDetectionSafely(options: VoiceInputVadSt
stream: options.stream,
})
await options.start(options.stream)
options.log?.('info', 'vad-ready', 'VAD is connected to the microphone stream.')
return true
}
+1
View File
@@ -0,0 +1 @@
test-results/
+127
View File
@@ -0,0 +1,127 @@
# Testing audio
This package runs recorded microphone tests through the real AIRI audio pipeline.
```text
input.wav -> virtual microphone -> VAD -> ASR -> LLM -> TTS -> playback -> UI
```
The package exports audio-aware `describe`, `it`, and `expect` APIs. Vitest owns the task tree and result protocol. Each project starts its configured Playwright runtime.
`src` only owns test scheduling, runtime startup, browser probes, and matchers. Case-specific environment, storage, Provider, route, and UI operations live under `cases/shared`:
```text
cases/shared/
configurations/ # preflight callbacks that configure one concern
interactions/ # reusable UI operations selected by a case
```
## Configure a case
The optional `preflight` field is an ordered callback array. The fake-microphone Vitest integration starts the runtime before it invokes these callbacks. Each callback receives:
- `env`: the process environment for this case
- `runtime`: the clean Playwright runtime
- `skip`: Vitest's case-level skip control
Every case explicitly selects the configuration it needs. Do not create one shared “complete pipeline” preflight that hides the Provider combination.
```ts
import { describe, expect, it } from '../../src'
import { configureModuleHearing, configureOnboarding, loadCaseEnvironment } from '../shared/configurations'
import { enableChatMicrophone } from '../shared/interactions'
import { openaiAsr } from '../shared/providers'
describe('audio input pipeline', () => {
it('transcribes a greeting', {
input: new URL('./input.test.wav', import.meta.url),
preflight: [
configureOnboarding(() => ({ completed: true })),
configureModuleHearing(async (context) => {
const environment = await loadCaseEnvironment(context.env)
const apiKey = environment.TESTING_AUDIO_ASR_API_KEY
context.skip(!apiKey, 'Set TESTING_AUDIO_ASR_API_KEY to run this ASR case.')
if (!apiKey)
return undefined
return {
provider: openaiAsr({
apiKey,
baseUrl: environment.TESTING_AUDIO_ASR_API_BASE_URL ?? 'https://api.openai.com/v1/',
model: environment.TESTING_AUDIO_ASR_MODEL ?? 'whisper-1',
provider: environment.TESTING_AUDIO_ASR_PROVIDER ?? 'openai-compatible-audio-transcription',
}),
captureFormat: 'wav',
}
}),
],
}, async ({ audio }) => {
await enableChatMicrophone(audio)
await expect(audio).toHaveTranscriptions([
['Please say hello.'],
])
})
})
```
A case can independently choose its VAD behavior, ASR, LLM, and TTS configuration. A configuration callback can read its own environment, write local storage, use another persistence mechanism, or deliberately leave onboarding incomplete.
## Provider environment
Each case selects its environment variables. `loadCaseEnvironment` uses Vite test mode to read repository, `packages/stage-ui`, and package environment files. Process variables have the highest priority.
Put local test credentials in `packages/testing-audio/.env.test.local`. Git ignores this file.
The OpenAI-compatible Provider helpers do not read the environment. Pass the endpoint, API key, model, and voice from the case callback.
The included cases use these explicit variables:
```dotenv
TESTING_AUDIO_ASR_PROVIDER=openai-compatible-audio-transcription
TESTING_AUDIO_ASR_MODEL=whisper-1
TESTING_AUDIO_ASR_API_BASE_URL=https://api.openai.com/v1/
TESTING_AUDIO_ASR_API_KEY=...
TESTING_AUDIO_ASR_ALIYUN_NLS_PROVIDER=aliyun-nls-transcription
TESTING_AUDIO_ASR_ALIYUN_NLS_ALIYUN_AK_ID=...
TESTING_AUDIO_ASR_ALIYUN_NLS_ALIYUN_AK_SECRET=...
TESTING_AUDIO_ASR_ALIYUN_NLS_APPKEY=...
TESTING_AUDIO_LLM_PROVIDER=openai-compatible
TESTING_AUDIO_LLM_MODEL=gpt-4o-mini
TESTING_AUDIO_LLM_API_BASE_URL=https://api.openai.com/v1/
TESTING_AUDIO_LLM_API_KEY=...
TESTING_AUDIO_TTS_PROVIDER=openai-compatible-audio-speech
TESTING_AUDIO_TTS_MODEL=tts-1
TESTING_AUDIO_TTS_VOICE=alloy
TESTING_AUDIO_TTS_API_BASE_URL=https://api.openai.com/v1/
TESTING_AUDIO_TTS_API_KEY=...
```
These tests send audio and text to external Providers. Each run can incur Provider charges.
## Run the tests
Build both targets and run all runtime projects:
```bash
pnpm -F @proj-airi/testing-audio test:run
```
Use existing builds:
```bash
pnpm -F @proj-airi/testing-audio test:existing-builds
```
Run one runtime project:
```bash
pnpm -F @proj-airi/testing-audio exec vitest run --project audio-web
pnpm -F @proj-airi/testing-audio exec vitest run --project audio-electron
```
Use `*.audio.web.test.ts` for Web-only cases. Use `*.audio.electron.test.ts` for Electron-only cases. The `*.audio.test.ts` pattern runs in both projects.
Do not use Vitest Browser Mode for these cases. Each task needs a case-specific Chromium fake-microphone process argument.
@@ -0,0 +1,82 @@
import { describe, expect, it } from '../../src'
import { configureModuleHearing, configureOnboarding, loadCaseEnvironment } from '../shared/configurations'
import { enableChatMicrophone } from '../shared/interactions'
import { aliyunNlsAsr, openaiAsr } from '../shared/providers'
describe('audio input pipeline', () => {
// The fixture uses the repository recording at docs/content/en/blog/DevLog-2025.03.20/assets/ashley-pitch-test.mp3.
// It has 20 seconds of leading silence and 4 seconds of trailing silence in mono 16 kHz PCM WAV format.
it('does not preserve the complete phrase after long leading silence', {
input: new URL('./input.test.wav', import.meta.url),
// This regression isolates AIRI's default VAD and one explicit ASR Provider.
preflight: [
configureOnboarding(() => ({ completed: true })),
configureModuleHearing(async (context) => {
const environment = await loadCaseEnvironment(context.env)
const apiKey = environment.TESTING_AUDIO_ASR_API_KEY
context.skip(!apiKey, 'Set TESTING_AUDIO_ASR_API_KEY to run this ASR case.')
if (!apiKey)
return undefined
return {
provider: openaiAsr({
apiKey,
baseUrl: environment.TESTING_AUDIO_ASR_API_BASE_URL ?? 'https://api.openai.com/v1/',
model: environment.TESTING_AUDIO_ASR_MODEL ?? 'whisper-1',
provider: environment.TESTING_AUDIO_ASR_PROVIDER ?? 'openai-compatible-audio-transcription',
}),
captureFormat: 'wav',
}
}),
],
}, async ({ audio }) => {
await enableChatMicrophone(audio)
if (audio.transcriptionCaptureFormat) {
await expect(audio).toHaveCapturedTranscriptionAudio({
count: 1,
minimumBytes: 8000,
})
}
// The VAD upload currently drops part of the sentence before it sends the recording to ASR.
await expect(audio).not.toHaveTranscriptions([
['There is no meaning to your existence, just let go.'],
])
})
it('does not preserve the complete phrase with Aliyun NLS', {
input: new URL('./input.test.wav', import.meta.url),
preflight: [
configureOnboarding(() => ({ completed: true })),
configureModuleHearing(async (context) => {
const environment = await loadCaseEnvironment(context.env)
const provider = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_PROVIDER
const accessKeyId = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_ALIYUN_AK_ID
const accessKeySecret = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_ALIYUN_AK_SECRET
const appKey = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_APPKEY
context.skip(
!provider || !accessKeyId || !accessKeySecret || !appKey,
'Set all TESTING_AUDIO_ASR_ALIYUN_NLS_* variables to run this ASR case.',
)
if (!provider || !accessKeyId || !accessKeySecret || !appKey)
return undefined
return {
provider: aliyunNlsAsr({ provider, accessKeyId, accessKeySecret, appKey }),
captureFormat: 'pcm',
}
}),
],
}, async ({ audio }) => {
await enableChatMicrophone(audio, { readiness: 'streaming-transcription' })
await expect(audio).toHaveCapturedTranscriptionAudio({
count: 1,
minimumBytes: 8000,
})
await expect(audio).not.toHaveTranscriptions([
['There is no meaning to your existence, just let go.'],
])
})
})
@@ -0,0 +1,35 @@
import type { AiriCard, AiriExtension } from '@proj-airi/stage-ui/types'
import type { AudioInputSession } from '../../../src/types'
type ActiveCardModules = Partial<Pick<AiriExtension['modules'], 'consciousness' | 'speech'>>
/** Updates Provider selections that the active AIRI Card reapplies during application startup. */
export async function configureActiveCardModules(
runtime: AudioInputSession,
modules: ActiveCardModules,
): Promise<void> {
await runtime.runtimePage.evaluate(({ configuredModules }) => {
const serializedCards = localStorage.getItem('airi-cards')
if (!serializedCards)
throw new Error('The AIRI Card store is not initialized.')
const activeCardId = localStorage.getItem('airi-card-active-id') ?? 'default'
const cards = JSON.parse(serializedCards) as Array<[string, AiriCard]>
const activeCard = cards.find(([cardId]) => cardId === activeCardId)?.[1]
if (!activeCard)
throw new Error(`The active AIRI Card "${activeCardId}" does not exist.`)
const currentModules = activeCard.extensions.airi.modules
activeCard.extensions.airi.modules = {
...currentModules,
...(configuredModules.consciousness
? { consciousness: configuredModules.consciousness }
: {}),
...(configuredModules.speech
? { speech: { ...currentModules.speech, ...configuredModules.speech } }
: {}),
}
localStorage.setItem('airi-cards', JSON.stringify(cards))
}, { configuredModules: modules })
}
@@ -0,0 +1,23 @@
import type { AudioInputPreflightContext } from '../../../src/types'
import { resolve } from 'node:path'
import { findWorkspaceDir } from '@pnpm/find-workspace-dir'
import { loadEnv } from 'vite'
/** Loads Vite test-mode environment files and applies the environment of the current case. */
export async function loadCaseEnvironment(
environment: AudioInputPreflightContext['env'],
): Promise<Record<string, string | undefined>> {
const repositoryRoot = await findWorkspaceDir(import.meta.dirname)
if (!repositoryRoot)
throw new Error(`Unable to find the pnpm workspace from ${import.meta.dirname}`)
const repositoryEnvironment = loadEnv('test', repositoryRoot, '')
// Shared Provider development variables live in stage-ui. These values override repository files.
const stageUiEnvironment = loadEnv('test', resolve(repositoryRoot, 'packages/stage-ui'), '')
// Audio case credentials belong to this package. These values override shared Provider variables.
const testingAudioEnvironment = loadEnv('test', resolve(repositoryRoot, 'packages/testing-audio'), '')
// The case process has the highest priority so that CI and shell values override local files.
return { ...repositoryEnvironment, ...stageUiEnvironment, ...testingAudioEnvironment, ...environment }
}
@@ -0,0 +1,9 @@
export { loadCaseEnvironment } from './environment'
export { configureModuleConsciousness } from './module-consciousness'
export type { ConsciousnessModuleConfiguration } from './module-consciousness'
export { configureModuleHearing } from './module-hearing'
export type { HearingModuleConfiguration } from './module-hearing'
export { configureModuleSpeech } from './module-speech'
export type { SpeechModuleConfiguration } from './module-speech'
export { configureOnboarding } from './onboarding'
export type { OnboardingConfiguration } from './onboarding'
@@ -0,0 +1,33 @@
import type { AudioInputPreflightCallback, AudioInputPreflightContext } from '../../../src/types'
import type { ProviderConfiguration } from './provider'
import { configureActiveCardModules } from './active-card'
import { configureProvider } from './provider'
import { configureStorage } from './storage'
export interface ConsciousnessModuleConfiguration {
provider: ProviderConfiguration
}
type ConsciousnessModuleResolver = (context: AudioInputPreflightContext) => ConsciousnessModuleConfiguration | undefined | Promise<ConsciousnessModuleConfiguration | undefined>
/** Configures the consciousness module with the LLM Provider selected by one case. */
export function configureModuleConsciousness(resolve: ConsciousnessModuleResolver): AudioInputPreflightCallback {
return async (context) => {
const configuration = await resolve(context)
if (!configuration)
return
await configureProvider(context.runtime, configuration.provider)
await configureActiveCardModules(context.runtime, {
consciousness: {
provider: configuration.provider.id,
model: configuration.provider.model,
},
})
await configureStorage(context.runtime, {
'settings/consciousness/active-provider': configuration.provider.id,
'settings/consciousness/active-model': configuration.provider.model,
})
}
}
@@ -0,0 +1,46 @@
import type { AudioCaptureFormat } from '@proj-airi/vitest-plugin-fakemic'
import type { AudioInputPreflightCallback, AudioInputPreflightContext } from '../../../src/types'
import type { ProviderConfiguration } from './provider'
import { configureProvider } from './provider'
import { configureStorage } from './storage'
export interface HearingModuleConfiguration {
/** @default undefined */
captureFormat?: AudioCaptureFormat
/** @default false */
microphoneEnabled?: boolean
provider: ProviderConfiguration
}
type HearingModuleResolver = (context: AudioInputPreflightContext) => HearingModuleConfiguration | undefined | Promise<HearingModuleConfiguration | undefined>
/** Configures the hearing module with the ASR Provider selected by one case. */
export function configureModuleHearing(resolve: HearingModuleResolver): AudioInputPreflightCallback {
return async (context) => {
const configuration = await resolve(context)
if (!configuration)
return
await configureProvider(context.runtime, configuration.provider)
const settings: Record<string, string> = {
'settings/hearing/active-provider': configuration.provider.id,
'settings/hearing/active-model': configuration.provider.model,
'settings/audio/input/enabled': String(configuration.microphoneEnabled ?? false),
}
if (context.runtime.target === 'electron') {
const microphoneInput = await context.runtime.runtimePage.evaluate(async () => {
const devices = await navigator.mediaDevices.enumerateDevices()
return devices.find(device => device.kind === 'audioinput' && device.label.includes('Fake'))?.deviceId
})
if (!microphoneInput)
throw new Error('Chromium did not expose the file-backed fake microphone.')
settings['settings/audio/input'] = microphoneInput
}
await configureStorage(context.runtime, settings)
context.runtime.transcriptionCaptureFormat = configuration.captureFormat
}
}
@@ -0,0 +1,39 @@
import type { AudioInputPreflightCallback, AudioInputPreflightContext } from '../../../src/types'
import type { ProviderConfiguration } from './provider'
import { configureActiveCardModules } from './active-card'
import { configureProvider } from './provider'
import { configureStorage } from './storage'
export interface SpeechModuleConfiguration {
/** @default false */
muted?: boolean
provider: ProviderConfiguration
voice: string
}
type SpeechModuleResolver = (context: AudioInputPreflightContext) => SpeechModuleConfiguration | undefined | Promise<SpeechModuleConfiguration | undefined>
/** Configures the speech module with the TTS Provider selected by one case. */
export function configureModuleSpeech(resolve: SpeechModuleResolver): AudioInputPreflightCallback {
return async (context) => {
const configuration = await resolve(context)
if (!configuration)
return
await configureProvider(context.runtime, configuration.provider)
await configureActiveCardModules(context.runtime, {
speech: {
provider: configuration.provider.id,
model: configuration.provider.model,
voice_id: configuration.voice,
},
})
await configureStorage(context.runtime, {
'settings/speech/active-provider': configuration.provider.id,
'settings/speech/active-model': configuration.provider.model,
'settings/speech/voice': configuration.voice,
'settings/speech/output-muted': String(configuration.muted ?? false),
})
}
}
@@ -0,0 +1,33 @@
import type { AudioInputPreflightCallback, AudioInputPreflightContext } from '../../../src/types'
import { configureStorage } from './storage'
export interface OnboardingConfiguration {
completed: boolean
/** @default false */
skipped?: boolean
}
type OnboardingResolver = (context: AudioInputPreflightContext) => OnboardingConfiguration | undefined | Promise<OnboardingConfiguration | undefined>
/** Configures onboarding with values selected by one case. */
export function configureOnboarding(resolve: OnboardingResolver): AudioInputPreflightCallback {
return async (context) => {
const configuration = await resolve(context)
if (!configuration)
return
await configureStorage(context.runtime, {
'onboarding/completed': String(configuration.completed),
'onboarding/skipped': String(configuration.skipped ?? false),
})
if (configuration.completed || configuration.skipped) {
await new Promise(resolveWait => setTimeout(resolveWait, 1_000))
const onboardingPages = context.runtime.electronApp
?.windows()
.filter(page => new URL(page.url()).hash.startsWith('#/onboarding')) ?? []
await Promise.all(onboardingPages.map(page => page.close().catch(() => undefined)))
}
}
}
@@ -0,0 +1,31 @@
import type { AudioInputSession } from '../../../src/types'
/** Provider values stored by one case preflight callback. */
export interface ProviderConfiguration {
config: Record<string, unknown>
definitionId: string
id: string
model: string
}
/** Adds one Provider without replacing Providers configured by earlier callbacks. */
export async function configureProvider(runtime: AudioInputSession, provider: ProviderConfiguration): Promise<void> {
await runtime.page.evaluate(({ configuredProvider }) => {
const credentials = JSON.parse(localStorage.getItem('settings/credentials/providers') ?? '{}') as Record<string, unknown>
const configured = JSON.parse(localStorage.getItem('settings/providers/configured') ?? '{}') as Record<string, unknown>
const added = JSON.parse(localStorage.getItem('settings/providers/added') ?? '{}') as Record<string, boolean>
credentials[configuredProvider.id] = configuredProvider.config
configured[configuredProvider.id] = {
id: configuredProvider.id,
definitionId: configuredProvider.definitionId,
config: configuredProvider.config,
status: 'configured',
}
added[configuredProvider.id] = true
localStorage.setItem('settings/credentials/providers', JSON.stringify(credentials))
localStorage.setItem('settings/providers/configured', JSON.stringify(configured))
localStorage.setItem('settings/providers/added', JSON.stringify(added))
}, { configuredProvider: provider })
}
@@ -0,0 +1,12 @@
import type { AudioInputSession } from '../../../src/types'
/** Writes AIRI settings through the storage owned by the current runtime page. */
export async function configureStorage(
runtime: AudioInputSession,
settings: Record<string, string>,
): Promise<void> {
await runtime.runtimePage.evaluate(({ entries }) => {
for (const [key, value] of Object.entries(entries))
localStorage.setItem(key, value)
}, { entries: settings })
}
@@ -0,0 +1,126 @@
import type { Locator } from 'playwright'
import type { AudioInputSession } from '../../../src/types'
import { captureStreamingTranscription } from './streaming-transcription'
export interface EnableChatMicrophoneOptions {
/** @default 'vad' */
readiness?: 'streaming-transcription' | 'vad'
}
/** Enables the chat microphone through the UI owned by the selected runtime. */
export async function enableChatMicrophone(
runtime: AudioInputSession,
options: EnableChatMicrophoneOptions = {},
): Promise<void> {
if (runtime.target === 'electron') {
const app = runtime.electronApp
if (!app)
throw new Error('The Electron audio session does not expose its application.')
const existingChatPage = app.windows().find(page => page.url().includes('index.html#/chat'))
if (existingChatPage) {
runtime.activatePage(existingChatPage)
}
else {
const chatButton = runtime.runtimePage.locator('button').filter({
has: runtime.runtimePage.locator('[i-solar\\:chat-line-line-duotone]'),
}).first()
await chatButton.waitFor({ state: 'visible', timeout: 30_000 })
runtime.activatePage(await openElectronChat(app, chatButton))
}
}
const { page } = runtime
await page.locator('textarea').first().waitFor({ state: 'visible', timeout: 60_000 })
await captureStreamingTranscription(page, 'textarea')
if (runtime.target === 'electron') {
await runtime.runtimePage.waitForFunction(async () => {
const devices = await navigator.mediaDevices.enumerateDevices()
return devices.some(device => device.kind === 'audioinput' && device.label.includes('Fake'))
})
const hearingTrigger = runtime.runtimePage.locator('div[aria-haspopup="dialog"] button').first()
await hearingTrigger.waitFor({ state: 'visible', timeout: 15_000 })
await hearingTrigger.hover()
await hearingTrigger.click({ force: true })
await runtime.runtimePage.waitForTimeout(500)
const enableButton = runtime.runtimePage.locator('button[aria-label="Enable microphone input"]')
await enableButton.waitFor({ state: 'visible', timeout: 15_000 })
const inputReady = options.readiness === 'streaming-transcription'
? runtime.waitForStreamingTranscriptionReady()
: runtime.waitForVadReady()
await enableButton.click({ force: true })
const disableButton = runtime.runtimePage.locator('button[aria-label="Disable microphone input"]')
await disableButton.waitFor({ state: 'visible', timeout: 15_000 })
await inputReady
return
}
const microphoneTrigger = page.locator('button').filter({ has: page.locator('.i-ph\\:microphone-slash') }).first()
await microphoneTrigger.click({ force: true })
const enableButton = page.locator('button[aria-label="Enable microphone input"]')
await enableButton.waitFor({ state: 'visible' })
const inputReady = options.readiness === 'streaming-transcription'
? runtime.waitForStreamingTranscriptionReady()
: runtime.waitForVadReady()
await enableButton.click()
await page.locator('button[aria-label="Disable microphone input"]').waitFor({ state: 'visible' })
await inputReady
}
/** Opens the chat page and waits for its input. */
export async function openChat(runtime: AudioInputSession): Promise<void> {
await runtime.page.evaluate(() => {
window.location.hash = '/chat'
})
await runtime.page.locator('textarea').first().waitFor({ state: 'visible', timeout: 60_000 })
}
/** Returns the assistant messages on the current chat page. */
export function assistantMessages(runtime: AudioInputSession): Locator {
return runtime.page.locator('[data-chat-message-role="assistant"] .markdown-content')
}
async function openElectronChat(
app: NonNullable<AudioInputSession['electronApp']>,
chatButton: Locator,
): Promise<AudioInputSession['page']> {
let lastError: unknown
for (let attempt = 0; attempt < 10; attempt++) {
await chatButton.click({ force: true })
try {
return await waitForElectronPage(app, page => page.url().includes('index.html#/chat'), 3_000)
}
catch (error) {
lastError = error
}
}
throw lastError
}
async function waitForElectronPage(
app: NonNullable<AudioInputSession['electronApp']>,
predicate: (page: AudioInputSession['page']) => boolean,
timeoutMs = 60_000,
): Promise<AudioInputSession['page']> {
const deadline = Date.now() + timeoutMs
while (Date.now() < deadline) {
const page = app.windows().find(predicate)
if (page) {
await page.waitForLoadState('domcontentloaded')
await page.bringToFront()
await page.waitForTimeout(750)
return page
}
await new Promise(resolveWait => setTimeout(resolveWait, 100))
}
throw new Error(`Timed out while waiting for the Electron chat window. Open pages: ${app.windows().map(page => page.url()).join(', ')}`)
}
@@ -0,0 +1,68 @@
import type { ElectronApplication, Page } from 'playwright'
import type { AudioInputSession } from '../../../src/types'
import { captureStreamingTranscription } from './streaming-transcription'
/** Opens the Electron hearing playground for a case that selects this input UI. */
export async function openHearingPlayground(runtime: AudioInputSession): Promise<Page> {
const app = runtime.electronApp
if (!app)
throw new Error('The hearing playground interaction requires an Electron runtime')
const settingsButton = runtime.page.getByRole('button', { name: /Open settings|打开设置/ }).last()
if (!await settingsButton.isVisible().catch(() => false)) {
await runtime.page.getByRole('button', { name: /Expand|展开/ }).last().click({ force: true })
await settingsButton.waitFor({ state: 'visible', timeout: 15_000 })
}
await settingsButton.click({ force: true })
const settingsPage = await waitForElectronPage(app, page => page.url().includes('index.html#/settings'))
await settingsPage.evaluate(() => {
window.location.hash = '/settings/modules/hearing'
})
await settingsPage.waitForURL(/#\/settings\/modules\/hearing/)
await settingsPage.getByTestId('hearing-playground-monitor-toggle').waitFor({ state: 'visible', timeout: 60_000 })
return settingsPage
}
/** Enables microphone monitoring on an open hearing playground. */
export async function enableHearingPlaygroundMicrophone(page: Page): Promise<void> {
await captureStreamingTranscription(page, '[data-testid="hearing-playground-current"] p')
const modelBasedToggle = page.getByRole('switch').last()
if (await modelBasedToggle.isChecked())
await modelBasedToggle.click()
const monitorToggle = page.getByTestId('hearing-playground-monitor-toggle')
await monitorToggle.click()
}
/** Reads the requested number of final hearing playground transcripts. */
export async function readHearingPlaygroundTranscriptions(page: Page, count: number): Promise<string[]> {
const transcriptions = page.getByTestId('hearing-playground-transcript')
await transcriptions.nth(count - 1).waitFor({ state: 'visible', timeout: 60_000 })
const results = (await transcriptions.allTextContents()).toReversed()
await page.evaluate((transcriptionResults) => {
if (window.__airiAudioInputE2E)
window.__airiAudioInputE2E.transcriptionResults = transcriptionResults
}, results)
return results
}
async function waitForElectronPage(
app: ElectronApplication,
predicate: (page: Page) => boolean,
timeoutMs = 60_000,
): Promise<Page> {
const deadline = Date.now() + timeoutMs
while (Date.now() < deadline) {
const page = app.windows().find(predicate)
if (page) {
await page.waitForLoadState('domcontentloaded')
return page
}
await new Promise(resolveWait => setTimeout(resolveWait, 100))
}
throw new Error('Timed out while waiting for the Electron renderer')
}
@@ -0,0 +1,7 @@
export { assistantMessages, enableChatMicrophone, openChat } from './chat'
export type { EnableChatMicrophoneOptions } from './chat'
export {
enableHearingPlaygroundMicrophone,
openHearingPlayground,
readHearingPlaygroundTranscriptions,
} from './hearing-playground'
@@ -0,0 +1,20 @@
import type { Page } from 'playwright'
/** Records distinct visible transcription values until the page closes. */
export async function captureStreamingTranscription(page: Page, selector: string): Promise<void> {
await page.evaluate((captureSelector) => {
let lastUpdate = ''
setInterval(() => {
const element = document.querySelector(captureSelector)
const value = element instanceof HTMLTextAreaElement
? element.value
: element?.textContent
const update = value?.trim() ?? ''
const updates = window.__airiAudioInputE2E?.streamingTranscriptionUpdates
if (update && update !== lastUpdate && updates) {
lastUpdate = update
updates.push(update)
}
}, 20)
}, selector)
}
@@ -0,0 +1,26 @@
import type { ProviderConfiguration } from '../configurations/provider'
/** Credentials and Provider selection for Aliyun NLS transcription. */
export interface AliyunNlsAsrOptions {
accessKeyId: string
accessKeySecret: string
appKey: string
provider: string
}
/** Creates an Aliyun NLS realtime ASR Provider configuration. */
export function aliyunNlsAsr(options: AliyunNlsAsrOptions): ProviderConfiguration {
if (options.provider !== 'aliyun-nls-transcription')
throw new Error('The Aliyun NLS Provider must be "aliyun-nls-transcription".')
return {
id: options.provider,
definitionId: options.provider,
model: 'aliyun-nls-v1',
config: {
accessKeyId: options.accessKeyId,
accessKeySecret: options.accessKeySecret,
appKey: options.appKey,
},
}
}
@@ -0,0 +1,4 @@
export { aliyunNlsAsr } from './aliyun-nls'
export type { AliyunNlsAsrOptions } from './aliyun-nls'
export { openaiAsr, openaiLlm, openaiTts } from './openai'
export type { OpenAIProviderOptions, OpenAISpeechProviderOptions } from './openai'
@@ -0,0 +1,58 @@
import type { ProviderConfiguration } from '../configurations/provider'
/** Configuration for an OpenAI-compatible Provider selected by one case. */
export interface OpenAIProviderOptions {
apiKey: string
baseUrl: string
model: string
provider: string
}
/** Configuration for an OpenAI-compatible speech Provider selected by one case. */
export interface OpenAISpeechProviderOptions extends OpenAIProviderOptions {
voice: string
}
/** Creates an OpenAI-compatible ASR Provider configuration. */
export function openaiAsr(options: OpenAIProviderOptions): ProviderConfiguration {
return {
id: options.provider,
definitionId: options.provider,
model: options.model,
config: {
apiKey: options.apiKey,
baseUrl: options.baseUrl,
},
}
}
/** Creates an OpenAI-compatible LLM Provider configuration. */
export function openaiLlm(options: OpenAIProviderOptions): ProviderConfiguration {
return {
id: options.provider,
definitionId: options.provider,
model: options.model,
config: {
apiKey: options.apiKey,
baseUrl: options.baseUrl,
},
}
}
/** Creates an OpenAI-compatible TTS Provider configuration. */
export function openaiTts(options: OpenAISpeechProviderOptions): { provider: ProviderConfiguration, voice: string } {
return {
voice: options.voice,
provider: {
id: options.provider,
definitionId: options.provider,
model: options.model,
config: {
apiKey: options.apiKey,
baseUrl: options.baseUrl,
model: options.model,
voice: options.voice,
},
},
}
}
@@ -0,0 +1,121 @@
import { describe, expect, it } from '../../src'
import { configureModuleConsciousness, configureModuleHearing, configureModuleSpeech, configureOnboarding, loadCaseEnvironment } from '../shared/configurations'
import { assistantMessages, enableChatMicrophone, openChat } from '../shared/interactions'
import { aliyunNlsAsr, openaiAsr, openaiLlm, openaiTts } from '../shared/providers'
describe('audio input pipeline', () => {
// An OpenAI-compatible TTS Provider generated the fixture in mono 16 kHz PCM WAV format.
// The fixture contains 14 seconds of leading silence for VAD initialization and 3 seconds of trailing silence.
// Its warm-up phrase gives the VAD time to start. Only "Please say hello." is required in the transcript.
it('runs an OpenAI-compatible request through the complete pipeline', {
input: new URL('./input.test.wav', import.meta.url),
// This case keeps AIRI's default VAD and selects every remote Provider explicitly.
preflight: [
configureOnboarding(() => ({ completed: true })),
configureModuleHearing(async (context) => {
const environment = await loadCaseEnvironment(context.env)
const apiKey = environment.TESTING_AUDIO_ASR_API_KEY
context.skip(!apiKey, 'Set TESTING_AUDIO_ASR_API_KEY to run this ASR case.')
if (!apiKey)
return undefined
return {
provider: openaiAsr({
apiKey,
baseUrl: environment.TESTING_AUDIO_ASR_API_BASE_URL ?? 'https://api.openai.com/v1/',
model: environment.TESTING_AUDIO_ASR_MODEL ?? 'whisper-1',
provider: environment.TESTING_AUDIO_ASR_PROVIDER ?? 'openai-compatible-audio-transcription',
}),
captureFormat: 'wav',
}
}),
configureModuleConsciousness(async (context) => {
const environment = await loadCaseEnvironment(context.env)
const apiKey = environment.TESTING_AUDIO_LLM_API_KEY
context.skip(!apiKey, 'Set TESTING_AUDIO_LLM_API_KEY to run this LLM case.')
if (!apiKey)
return undefined
return {
provider: openaiLlm({
apiKey,
baseUrl: environment.TESTING_AUDIO_LLM_API_BASE_URL ?? 'https://api.openai.com/v1/',
model: environment.TESTING_AUDIO_LLM_MODEL ?? 'gpt-4o-mini',
provider: environment.TESTING_AUDIO_LLM_PROVIDER ?? 'openai-compatible',
}),
}
}),
configureModuleSpeech(async (context) => {
const environment = await loadCaseEnvironment(context.env)
const apiKey = environment.TESTING_AUDIO_TTS_API_KEY
context.skip(!apiKey, 'Set TESTING_AUDIO_TTS_API_KEY to run this TTS case.')
if (!apiKey)
return undefined
return openaiTts({
apiKey,
baseUrl: environment.TESTING_AUDIO_TTS_API_BASE_URL ?? 'https://api.openai.com/v1/',
model: environment.TESTING_AUDIO_TTS_MODEL ?? 'tts-1',
provider: environment.TESTING_AUDIO_TTS_PROVIDER ?? 'openai-compatible-audio-speech',
voice: environment.TESTING_AUDIO_TTS_VOICE ?? 'alloy',
})
}),
],
}, async ({ audio }) => {
await enableChatMicrophone(audio)
if (audio.transcriptionCaptureFormat) {
await expect(audio).toHaveCapturedTranscriptionAudio({
count: 1,
minimumBytes: 8000,
})
}
await expect(audio).toHaveTranscriptions([
['Please say hello.'],
], { match: 'contains' })
await expect.poll(async () => (await audio.completedSpans('LLM inference')).length, { timeout: 60_000 }).toBeGreaterThanOrEqual(1)
await expect.poll(async () => (await audio.completedSpans('TTS synthesis')).length, { timeout: 60_000 }).toBeGreaterThanOrEqual(1)
await expect.poll(async () => (await audio.completedSpans('Audio playback')).length, { timeout: 60_000 }).toBeGreaterThanOrEqual(1)
await openChat(audio)
const messages = await assistantMessages(audio).allTextContents()
expect(messages.at(-1)).toMatch(/.+/s)
})
it('transcribes the greeting with Aliyun NLS', {
input: new URL('./input.test.wav', import.meta.url),
preflight: [
configureOnboarding(() => ({ completed: true })),
configureModuleHearing(async (context) => {
const environment = await loadCaseEnvironment(context.env)
const provider = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_PROVIDER
const accessKeyId = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_ALIYUN_AK_ID
const accessKeySecret = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_ALIYUN_AK_SECRET
const appKey = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_APPKEY
context.skip(
!provider || !accessKeyId || !accessKeySecret || !appKey,
'Set all TESTING_AUDIO_ASR_ALIYUN_NLS_* variables to run this ASR case.',
)
if (!provider || !accessKeyId || !accessKeySecret || !appKey)
return undefined
return {
provider: aliyunNlsAsr({ provider, accessKeyId, accessKeySecret, appKey }),
captureFormat: 'pcm',
}
}),
],
}, async ({ audio }) => {
await enableChatMicrophone(audio, { readiness: 'streaming-transcription' })
await expect(audio).toHaveCapturedTranscriptionAudio({
count: 1,
minimumBytes: 8000,
})
await expect(audio).toHaveTranscriptions([
['Please say hello.'],
], { match: 'contains' })
})
})
@@ -0,0 +1,105 @@
import { describe, expect, it } from '../../src'
import { configureModuleHearing, configureOnboarding, loadCaseEnvironment } from '../shared/configurations'
import { enableChatMicrophone } from '../shared/interactions'
import { aliyunNlsAsr, openaiAsr } from '../shared/providers'
describe('audio input pipeline', () => {
// The fixture has 12 seconds of leading silence so that the browser VAD can load.
// It repeats the greeting after a 2-second pause to catch regressions that drop the second utterance.
it('keeps two utterances in streaming transcription', {
input: new URL('./input.test.wav', import.meta.url),
// This regression isolates AIRI's default VAD and one explicit ASR Provider.
preflight: [
configureOnboarding(() => ({ completed: true })),
configureModuleHearing(async (context) => {
const environment = await loadCaseEnvironment(context.env)
const apiKey = environment.TESTING_AUDIO_ASR_API_KEY
context.skip(!apiKey, 'Set TESTING_AUDIO_ASR_API_KEY to run this ASR case.')
if (!apiKey)
return undefined
return {
provider: openaiAsr({
apiKey,
baseUrl: environment.TESTING_AUDIO_ASR_API_BASE_URL ?? 'https://api.openai.com/v1/',
model: environment.TESTING_AUDIO_ASR_MODEL ?? 'whisper-1',
provider: environment.TESTING_AUDIO_ASR_PROVIDER ?? 'openai-compatible-audio-transcription',
}),
captureFormat: 'wav',
}
}),
],
}, async ({ audio }) => {
await enableChatMicrophone(audio)
if (audio.transcriptionCaptureFormat) {
await expect(audio).toHaveCapturedTranscriptionAudio({
count: 1,
minimumBytes: 8000,
})
}
const expectedTranscriptions = [
[
'Microphone warm up, microphone warm up. Hello, AIRI, please say hello.',
'Microphone warm up, microphone warm up. Hello, Eric, please say hello.',
],
[
'Microphone warm up, microphone warm up. Hello, AIRI, please say hello.',
'Microphone warm up, microphone warm up. Hello, Eric, please say hello.',
],
]
await expect(audio).toHaveTranscriptions(expectedTranscriptions)
const finalTranscriptions = await audio.transcriptionResults(expectedTranscriptions.length)
const streamingUpdates = await audio.streamingTranscriptionUpdates()
expect(streamingUpdates.some(update => !finalTranscriptions.includes(update))).toBe(true)
})
it('keeps two Aliyun NLS utterances in streaming transcription', {
input: new URL('./input.test.wav', import.meta.url),
preflight: [
configureOnboarding(() => ({ completed: true })),
configureModuleHearing(async (context) => {
const environment = await loadCaseEnvironment(context.env)
const provider = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_PROVIDER
const accessKeyId = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_ALIYUN_AK_ID
const accessKeySecret = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_ALIYUN_AK_SECRET
const appKey = environment.TESTING_AUDIO_ASR_ALIYUN_NLS_APPKEY
context.skip(
!provider || !accessKeyId || !accessKeySecret || !appKey,
'Set all TESTING_AUDIO_ASR_ALIYUN_NLS_* variables to run this ASR case.',
)
if (!provider || !accessKeyId || !accessKeySecret || !appKey)
return undefined
return {
provider: aliyunNlsAsr({ provider, accessKeyId, accessKeySecret, appKey }),
captureFormat: 'pcm',
}
}),
],
}, async ({ audio }) => {
await enableChatMicrophone(audio, { readiness: 'streaming-transcription' })
await expect(audio).toHaveCapturedTranscriptionAudio({
count: 1,
minimumBytes: 8000,
})
const expectedTranscriptions = [
[
'Microphone warm up, microphone warm up. Hello, AIRI, please say hello.',
'Microphone warm up, microphone warm up. Hello, Eric, please say hello.',
],
[
'Microphone warm up, microphone warm up. Hello, AIRI, please say hello.',
'Microphone warm up, microphone warm up. Hello, Eric, please say hello.',
],
]
await expect(audio).toHaveTranscriptions(expectedTranscriptions)
const finalTranscriptions = await audio.transcriptionResults(expectedTranscriptions.length)
const streamingUpdates = await audio.streamingTranscriptionUpdates()
expect(streamingUpdates.some(update => !finalTranscriptions.includes(update))).toBe(true)
})
})
+34
View File
@@ -0,0 +1,34 @@
{
"name": "@proj-airi/testing-audio",
"type": "module",
"version": "0.11.3",
"private": true,
"description": "AIRI Web and Electron audio pipeline tests",
"author": {
"name": "Moeru AI Project AIRI Team",
"email": "airi@moeru.ai",
"url": "https://github.com/moeru-ai"
},
"license": "MIT",
"exports": "./src/index.ts",
"scripts": {
"build:targets": "pnpm -F @proj-airi/stage-web build && pnpm -F @proj-airi/stage-tamagotchi build",
"test": "vitest",
"test:run": "pnpm run build:targets && vitest run",
"test:existing-builds": "vitest run",
"typecheck": "tsc --noEmit"
},
"dependencies": {
"@moeru/std": "catalog:",
"@pnpm/find-workspace-dir": "catalog:",
"@proj-airi/stage-shared": "workspace:^",
"@proj-airi/stage-ui": "workspace:^",
"@proj-airi/vitest-plugin-fakemic": "workspace:^",
"playwright": "catalog:",
"vite": "catalog:",
"vitest": "catalog:vitest"
},
"devDependencies": {
"@types/node": "catalog:"
}
}
+76
View File
@@ -0,0 +1,76 @@
import type { AudioTestTask } from '@proj-airi/vitest-plugin-fakemic'
import type { AudioInputPreflightContext, AudioInputSession, AudioInputTestCase } from './types'
import { env } from 'node:process'
import { fileURLToPath } from 'node:url'
import { errorMessageFrom } from '@moeru/std'
import { createAudioTestAPI, createAudioTestTask, runAudioTestSession, startFakemicRuntime } from '@proj-airi/vitest-plugin-fakemic'
import { inject } from 'vitest'
import { expect, installAudioInputMatchers } from './expect-extend'
type RunnableAudioInputTest = AudioTestTask
installAudioInputMatchers()
const audioTestAPI = createAudioTestAPI<
AudioInputTestCase,
RunnableAudioInputTest,
{ audio: AudioInputSession },
AudioInputPreflightContext
>({
preflight: definition => definition.preflight,
createPlans(name, testCase) {
const task = createAudioTestTask(name, testCase)
return [{
name: task.name,
definition: task,
metadata: {
input: fileURLToPath(task.input),
runtime: inject('fakemicRuntime').name,
},
}]
},
async execute({ plan, task, invokeHandler, runPreflight }) {
await runAudioTestSession({
start() {
const microphoneInput = fileURLToPath(plan.definition.input)
return startFakemicRuntime<AudioInputSession>(microphoneInput)
},
async execute(session) {
await runPreflight({
env,
runtime: session,
skip: (condition, note) => task.context.skip(Boolean(condition), note),
})
await session.runtimePage.reload({ waitUntil: 'domcontentloaded' })
await session.runtimePage.locator('[i-solar\\:alt-arrow-up-line-duotone]').first().waitFor({ state: 'visible', timeout: 30_000 })
await session.runtimePage.bringToFront()
await session.runtimePage.waitForTimeout(750)
Object.assign(task.context, { audio: session })
await invokeHandler()
},
async recordArtifacts(session) {
const snapshot = await session.snapshot().catch(error => ({
snapshotError: errorMessageFrom(error) ?? 'Unknown snapshot error',
}))
await task.context.annotate('pipeline.json', {
body: `${JSON.stringify(snapshot, null, 2)}\n`,
bodyEncoding: 'utf-8',
contentType: 'application/json',
})
},
})
},
})
/** Groups AIRI audio-input tests in the Vitest task tree. */
export const describe = audioTestAPI.describe
/** Defines an AIRI audio-input test for each selected target. */
export const it = audioTestAPI.it
/** Vitest expect with AIRI audio-input matchers installed. */
export { expect }
@@ -0,0 +1,28 @@
import type { AudioInputObservations } from './types'
import { describe, it } from 'vitest'
import { expect, installAudioInputMatchers } from './expect-extend'
installAudioInputMatchers()
describe('audio input matchers', () => {
it('normalizes transcription case, width, punctuation, and whitespace', async () => {
const session = createAudioInputSession(['lease, SAY hello!'])
await expect(session).toHaveTranscriptions([
['please say hello'],
])
})
})
function createAudioInputSession(transcriptions: string[]): AudioInputObservations {
return {
capturedTranscriptionAudio: async () => [],
streamingTranscriptionUpdates: async () => [],
transcriptionResults: async () => transcriptions,
completedSpans: async () => [],
waitForStreamingTranscriptionReady: async () => {},
waitForVadReady: async () => {},
}
}
+100
View File
@@ -0,0 +1,100 @@
import type { AudioInputObservations } from './types'
import { expect as vitestExpect } from 'vitest'
export interface CapturedTranscriptionAudioExpectation {
count: number
minimumBytes: number
}
export interface TranscriptionExpectationOptions {
/** @default 'exact' */
match?: 'exact' | 'contains'
}
declare module 'vitest' {
interface Assertion<T> {
toHaveCapturedTranscriptionAudio: T extends AudioInputObservations
? (expected: CapturedTranscriptionAudioExpectation) => Promise<void>
: never
toHaveTranscriptions: T extends AudioInputObservations
? (
expected: ReadonlyArray<ReadonlyArray<string>>,
options?: TranscriptionExpectationOptions,
) => Promise<void>
: never
}
}
/** Vitest expect with AIRI audio-input matcher types. */
export const expect = vitestExpect
/**
* Normalizes a transcript for speech-recognition comparison.
*
* @example
* normalizeTranscript(' Hello, AIRI! ')
* // => 'helloairi'
*/
function normalizeTranscript(value: string): string {
return value
.normalize('NFKC')
.toLocaleLowerCase()
.replace(/[^\p{L}\p{N}]+/gu, '')
}
/** Installs asynchronous matchers for AIRI audio-input observations. */
export function installAudioInputMatchers(): void {
vitestExpect.extend({
async toHaveCapturedTranscriptionAudio(
session: AudioInputObservations,
expected: CapturedTranscriptionAudioExpectation,
) {
const format = session.transcriptionCaptureFormat
if (!format) {
return {
pass: false,
message: () => 'The active transcription Provider does not expose uploaded audio.',
}
}
const captures = await session.capturedTranscriptionAudio(expected.count)
const invalidCapture = captures.find((capture) => {
if (capture.format !== format || capture.data.byteLength < expected.minimumBytes)
return true
return format === 'wav' && new TextDecoder().decode(capture.data.subarray(0, 4)) !== 'RIFF'
})
const pass = captures.length === expected.count && !invalidCapture
return {
pass,
message: () => pass
? 'Expected the session not to contain valid transcription audio.'
: `Expected ${expected.count} ${format} capture(s) with at least ${expected.minimumBytes} bytes.`,
}
},
async toHaveTranscriptions(
session: AudioInputObservations,
expected: ReadonlyArray<ReadonlyArray<string>>,
options: TranscriptionExpectationOptions = {},
) {
const actual = await session.transcriptionResults(expected.length)
const normalizedActual = actual.map(normalizeTranscript)
const normalizedExpected = expected.map(alternatives => alternatives.map(normalizeTranscript))
const match = options.match ?? 'exact'
const pass = normalizedActual.length === normalizedExpected.length
&& normalizedActual.every((transcript, index) => (
match === 'contains'
? normalizedExpected[index].some(candidate => transcript.includes(candidate))
: normalizedExpected[index].includes(transcript)
))
return {
pass,
message: () => pass
? 'Expected the session not to contain the specified transcriptions.'
: `Expected transcriptions ${JSON.stringify(expected)}, but received ${JSON.stringify(actual)}.`,
}
},
})
}
+10
View File
@@ -0,0 +1,10 @@
export { describe, expect, it } from './describe'
export type {
AudioInputObservations,
AudioInputPreflightCallback,
AudioInputPreflightContext,
AudioInputSession,
AudioInputTarget,
AudioInputTestCase,
} from './types'
@@ -0,0 +1,44 @@
import type { FakemicElectronPrepareContext } from '@proj-airi/vitest-plugin-fakemic'
import type { Page } from 'playwright'
import type { AudioInputSession } from '../types'
import { stubForBrowser } from '../setup/browser-probe'
import { createSession } from '../setup/session'
/** Adapts a Fakemic Electron process into an AIRI desktop audio session. */
export default async function prepareElectronRuntime(context: FakemicElectronPrepareContext): Promise<AudioInputSession> {
await context.app.context().addInitScript(stubForBrowser)
const page = await waitForPage(context, (page) => {
const url = new URL(page.url())
return url.pathname.endsWith('/index.html') && url.hash === '#/'
})
await page.locator('[i-solar\\:alt-arrow-up-line-duotone]').first().waitFor({ state: 'visible', timeout: 30_000 })
await page.evaluate(stubForBrowser)
return createSession({
electronApp: context.app,
page,
target: 'electron',
close: context.close,
})
}
async function waitForPage(
context: FakemicElectronPrepareContext,
predicate: (page: Page) => boolean,
timeoutMs = 60_000,
): Promise<Page> {
const deadline = Date.now() + timeoutMs
while (Date.now() < deadline) {
const page = context.app.windows().find(predicate)
if (page) {
await page.waitForLoadState('domcontentloaded')
return page
}
await new Promise(resolveWait => setTimeout(resolveWait, 100))
}
throw new Error('Timed out while waiting for the Electron renderer')
}
@@ -0,0 +1,20 @@
import type { FakemicWebPrepareContext } from '@proj-airi/vitest-plugin-fakemic'
import type { AudioInputSession } from '../types'
import { stubForBrowser } from '../setup/browser-probe'
import { createSession } from '../setup/session'
/** Adapts a Fakemic Chromium process into an AIRI Web audio session. */
export default async function prepareWebRuntime(context: FakemicWebPrepareContext): Promise<AudioInputSession> {
await context.context.addInitScript(stubForBrowser)
const page = await context.context.newPage()
await page.goto(context.runtime.url)
return createSession({
page,
target: 'web',
close: context.close,
})
}
@@ -0,0 +1,204 @@
import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
/**
* Installs passive browser probes before the application starts.
*
* Triggering workflow:
*
* `BrowserContext.addInitScript`
* -> {@link stubForBrowser}
* -> `window.fetch`
* -> `BroadcastChannel('io-tracer-channel')`
*
* Upstream:
* - `BrowserContext.addInitScript`
*
* Downstream:
* - `window.__airiAudioInputE2E`
*/
export function stubForBrowser() {
const state: BrowserAudioInputState = { spans: [], streamingTranscriptionReady: false, streamingTranscriptionUpdates: [], transcriptionAudio: [], transcriptionResults: [], vadReady: false }
window.__airiAudioInputE2E = state
const originalConsoleInfo = console.info.bind(console)
console.info = (...values: unknown[]) => {
if (typeof values[0] === 'string' && values[0].startsWith('[Voice Input] vad-ready:'))
state.vadReady = true
originalConsoleInfo(...values)
}
const originalFetch = window.fetch.bind(window)
/**
* Copies each ASR upload and response while it forwards the request.
*
* Triggering workflow:
*
* `window.fetch`
* -> `POST /audio/transcriptions`
* -> captureFetch
*
* Upstream:
* - The OpenAI-compatible transcription Provider.
*
* Downstream:
* - `window.__airiAudioInputE2E`
* - The original `window.fetch` function.
*/
const captureFetch: typeof window.fetch = async (input, init) => {
const requestUrl = typeof input === 'string'
? input
: input instanceof URL
? input.href
: input.url
const capturesTranscription = new URL(requestUrl, window.location.href).pathname.endsWith('/audio/transcriptions')
if (capturesTranscription && init?.body instanceof FormData) {
const file = init.body.get('file')
if (file instanceof Blob) {
state.transcriptionAudio.push({
base64: new Uint8Array(await file.arrayBuffer()).toBase64(),
format: 'wav',
})
}
}
const response = await originalFetch(input, init)
if (capturesTranscription && response.ok) {
const payload = await response.clone().json() as { text?: unknown }
if (typeof payload.text === 'string')
state.transcriptionResults.push(payload.text)
}
return response
}
window.fetch = captureFetch
const OriginalWebSocket = window.WebSocket
const audioChunksBySocket = new WeakMap<WebSocket, Uint8Array[]>()
const capturesAliyunNlsBySocket = new WeakMap<WebSocket, boolean>()
const capturedSockets = new WeakSet<WebSocket>()
function capturePcmAudio(socket: WebSocket) {
const audioChunks = audioChunksBySocket.get(socket)
if (!audioChunks || capturedSockets.has(socket) || !audioChunks.length)
return
const byteLength = audioChunks.reduce((total, chunk) => total + chunk.byteLength, 0)
if (byteLength < 8192)
return
capturedSockets.add(socket)
const audio = new Uint8Array(byteLength)
let offset = 0
for (const chunk of audioChunks) {
audio.set(chunk, offset)
offset += chunk.byteLength
}
state.transcriptionAudio.push({ base64: audio.toBase64(), format: 'pcm' })
}
class CaptureAliyunNlsWebSocket extends OriginalWebSocket {
constructor(url: string | URL, protocols?: string | string[]) {
super(url, protocols)
const target = new URL(url.toString(), window.location.href)
const capturesAliyunNls = target.hostname.startsWith('nls-gateway-') && target.hostname.endsWith('.aliyuncs.com')
capturesAliyunNlsBySocket.set(this, capturesAliyunNls)
if (capturesAliyunNls) {
audioChunksBySocket.set(this, [])
this.addEventListener('open', () => {
state.streamingTranscriptionReady = true
})
this.addEventListener('message', (event) => {
if (typeof event.data !== 'string')
return
try {
const payload = JSON.parse(event.data) as { header?: { name?: string }, payload?: { result?: unknown } }
if (payload.header?.name === 'SentenceEnd' && typeof payload.payload?.result === 'string')
state.transcriptionResults.push(payload.payload.result)
}
catch {
// NLS can send non-transcription frames. The Provider handles those frames.
}
})
}
}
send(data: string | ArrayBufferLike | Blob | ArrayBufferView): void {
const audioChunks = audioChunksBySocket.get(this)
if (capturesAliyunNlsBySocket.get(this) && audioChunks && typeof data !== 'string') {
if (data instanceof Blob) {
void data.arrayBuffer().then((buffer) => {
audioChunks.push(new Uint8Array(buffer))
})
}
else if (ArrayBuffer.isView(data)) {
audioChunks.push(new Uint8Array(data.buffer, data.byteOffset, data.byteLength))
}
else {
audioChunks.push(new Uint8Array(data))
}
capturePcmAudio(this)
}
let outboundData: string | Blob | BufferSource
if (typeof data === 'string' || data instanceof Blob || data instanceof ArrayBuffer) {
outboundData = data
}
else if (ArrayBuffer.isView(data)) {
// WebSocket does not accept views backed by SharedArrayBuffer. The copy uses a regular ArrayBuffer.
outboundData = new Uint8Array(data.buffer, data.byteOffset, data.byteLength).slice()
}
else {
// ArrayBufferLike also includes SharedArrayBuffer. The copy keeps the original bytes in a supported buffer.
outboundData = new Uint8Array(data).slice()
}
super.send(outboundData)
}
close(code?: number, reason?: string): void {
const audioChunks = audioChunksBySocket.get(this)
if (capturesAliyunNlsBySocket.get(this) && audioChunks && !capturedSockets.has(this) && audioChunks.length) {
capturePcmAudio(this)
}
super.close(code, reason)
}
}
window.WebSocket = CaptureAliyunNlsWebSocket
const channel = new BroadcastChannel('io-tracer-channel')
/**
* Stores completed I/O spans for the case artifact.
*
* Triggering workflow:
*
* `BroadcastChannel('io-tracer-channel')`
* -> `message`
* -> captureSpan
*
* Upstream:
* - The AIRI I/O trace exporter.
*
* Downstream:
* - `window.__airiAudioInputE2E.spans`
*/
const captureSpan = (event: MessageEvent) => {
if (event.data?.type === 'span' && event.data.span?.ended)
state.spans.push(event.data.span as SerializedIOSpan)
}
channel.addEventListener('message', captureSpan)
}
/** Returns the completed browser spans that match the optional span name. */
export function readCompletedSpans(name?: string) {
const spans = window.__airiAudioInputE2E?.spans ?? []
return name ? spans.filter(span => span.name === name) : spans
}
+116
View File
@@ -0,0 +1,116 @@
import type { AudioCapture } from '@proj-airi/vitest-plugin-fakemic'
import type { ElectronApplication, Page } from 'playwright'
import type { AudioInputSession, AudioInputTarget } from '../types'
import { Buffer } from 'node:buffer'
import { readCompletedSpans } from './browser-probe'
/** Creates the runtime session and records its page diagnostics. */
export function createSession(options: {
electronApp?: ElectronApplication
page: Page
target: AudioInputTarget
close: () => Promise<void>
transcriptionCaptureFormat?: AudioCapture['format']
}): AudioInputSession {
const diagnostics: string[] = []
const observedPages = new WeakSet<Page>()
function observePage(page: Page) {
if (observedPages.has(page))
return
observedPages.add(page)
page.on('console', (message) => {
const text = message.text()
const isAudioPipelineInfo = message.type() === 'info'
&& (text.includes('[Hearing Pipeline]') || text.includes('[Voice Input]') || text.includes('transcription'))
if (['error', 'warning'].includes(message.type()) || isAudioPipelineInfo)
diagnostics.push(`[console:${message.type()}] ${text}`)
})
page.on('pageerror', error => diagnostics.push(`[pageerror] ${error.message}`))
}
observePage(options.page)
const session: AudioInputSession = {
electronApp: options.electronApp,
page: options.page,
runtimePage: options.page,
target: options.target,
transcriptionCaptureFormat: options.transcriptionCaptureFormat,
activatePage(page) {
observePage(page)
session.page = page
},
async capturedTranscriptionAudio(count) {
try {
await options.page.waitForFunction(expectedCount => (
(window.__airiAudioInputE2E?.transcriptionAudio.length ?? 0) >= expectedCount
), count, { timeout: 60_000 })
}
catch (error) {
const runtimeState = await options.page.evaluate(async () => ({
activeModel: localStorage.getItem('settings/hearing/active-model'),
activeProvider: localStorage.getItem('settings/hearing/active-provider'),
devices: (await navigator.mediaDevices.enumerateDevices()).map(device => ({
deviceId: device.deviceId,
kind: device.kind,
label: device.label,
})),
microphoneEnabled: localStorage.getItem('settings/audio/input/enabled'),
microphoneInput: localStorage.getItem('settings/audio/input'),
microphoneOffIconVisible: Boolean(document.querySelector('[i-ph\\:microphone-slash]')),
probeInstalled: Boolean(window.__airiAudioInputE2E),
streamingTranscriptionReady: window.__airiAudioInputE2E?.streamingTranscriptionReady ?? false,
url: window.location.href,
vadReady: window.__airiAudioInputE2E?.vadReady ?? false,
}))
throw new Error(`Timed out waiting for captured transcription audio: ${JSON.stringify({ diagnostics, runtimeState })}`, { cause: error })
}
const capturedAudio = await options.page.evaluate(() => window.__airiAudioInputE2E?.transcriptionAudio ?? [])
return capturedAudio.map(audio => ({
format: audio.format,
data: Buffer.from(audio.base64, 'base64'),
}))
},
streamingTranscriptionUpdates: () => session.page.evaluate(() => window.__airiAudioInputE2E?.streamingTranscriptionUpdates ?? []),
async transcriptionResults(count) {
await options.page.waitForFunction(expectedCount => (
(window.__airiAudioInputE2E?.transcriptionResults.length ?? 0) >= expectedCount
), count, { timeout: 60_000 })
return options.page.evaluate(() => window.__airiAudioInputE2E?.transcriptionResults ?? [])
},
async completedSpans(name) {
const runtimeSpans = await options.page.evaluate(readCompletedSpans, name)
if (session.page === options.page)
return runtimeSpans
const interactionSpans = await session.page.evaluate(readCompletedSpans, name)
return [...runtimeSpans, ...interactionSpans]
},
async waitForVadReady() {
await options.page.waitForFunction(() => window.__airiAudioInputE2E?.vadReady === true, undefined, { timeout: 30_000 })
},
async waitForStreamingTranscriptionReady() {
await options.page.waitForFunction(() => window.__airiAudioInputE2E?.streamingTranscriptionReady === true, undefined, { timeout: 30_000 })
},
async snapshot() {
const runtimeState = await options.page.evaluate(() => window.__airiAudioInputE2E)
const interactionState = session.page === options.page
? runtimeState
: await session.page.evaluate(() => window.__airiAudioInputE2E)
return {
spans: runtimeState?.spans ?? [],
streamingTranscriptionUpdates: interactionState?.streamingTranscriptionUpdates ?? [],
transcriptionResults: runtimeState?.transcriptionResults ?? [],
diagnostics: [...diagnostics],
}
},
close: options.close,
}
return session
}
+60
View File
@@ -0,0 +1,60 @@
import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
import type { AudioCapture, AudioCaptureFormat, AudioTestCase, AudioTestPreflightCallback, AudioTestSession } from '@proj-airi/vitest-plugin-fakemic'
import type { ElectronApplication, Page } from 'playwright'
export const audioInputTargets = ['web', 'electron'] as const
export type AudioInputTarget = (typeof audioInputTargets)[number]
/** Values available when one case resolves its preflight callbacks. */
export interface AudioInputPreflightContext {
/** Environment variables loaded for this case. */
env: Readonly<NodeJS.ProcessEnv>
/** Clean runtime that the configuration callback can prepare. */
runtime: AudioInputSession
/** Skips this case when its environment does not satisfy a case constraint. */
skip: (condition: unknown, note?: string) => void
}
/** One optional case callback that resolves configuration from its environment. */
export type AudioInputPreflightCallback = AudioTestPreflightCallback<AudioInputPreflightContext>
/** One AIRI audio-input test definition. */
export type AudioInputTestCase = AudioTestCase<AudioInputPreflightContext>
/** Snapshot of the observable AIRI audio pipeline state. */
export interface AudioInputSnapshot {
spans: SerializedIOSpan[]
streamingTranscriptionUpdates: string[]
transcriptionResults: string[]
diagnostics: string[]
}
/** Observable audio values used by AIRI matchers. */
export interface AudioInputObservations {
/** Capture format used by the active transcription Provider. */
transcriptionCaptureFormat?: AudioCaptureFormat
capturedTranscriptionAudio: (count: number) => Promise<AudioCapture[]>
streamingTranscriptionUpdates: () => Promise<string[]>
transcriptionResults: (count: number) => Promise<string[]>
completedSpans: (name?: string) => Promise<SerializedIOSpan[]>
/** Waits until the VAD audio graph is connected to the microphone stream. */
waitForVadReady: () => Promise<void>
/** Waits until a streaming transcription transport accepts microphone audio. */
waitForStreamingTranscriptionReady: () => Promise<void>
}
/** Runtime handle for one AIRI audio-input test. */
export interface AudioInputSession extends AudioInputObservations, AudioTestSession {
/** Electron application for Electron tasks. */
electronApp?: ElectronApplication
/** Page used by case interactions. */
page: Page
/** Page that owns the audio-input pipeline. */
runtimePage: Page
/** Runtime selected for this concrete task. */
target: AudioInputTarget
/** Selects the page used by subsequent case interactions. */
activatePage: (page: Page) => void
snapshot: () => Promise<AudioInputSnapshot>
}
+30
View File
@@ -0,0 +1,30 @@
import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
declare global {
interface BrowserAudioInputState {
spans: SerializedIOSpan[]
streamingTranscriptionReady: boolean
streamingTranscriptionUpdates: string[]
transcriptionAudio: Array<{ base64: string, format: 'pcm' | 'wav' }>
transcriptionResults: string[]
vadReady: boolean
}
interface Window {
__airiAudioInputE2E?: BrowserAudioInputState
}
// NOTICE:
// TypeScript 5.9 does not declare the Baseline 2025 Uint8Array Base64 methods.
// The test runs in current Playwright and Electron Chromium runtimes that implement this API.
// Source: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Uint8Array/toBase64
// Remove this declaration when the TypeScript standard library includes the method.
interface Uint8Array<TArrayBuffer extends ArrayBufferLike = ArrayBufferLike> {
toBase64: (options?: {
alphabet?: 'base64' | 'base64url'
omitPadding?: boolean
}) => string
}
}
export {}
+25
View File
@@ -0,0 +1,25 @@
{
"compilerOptions": {
"target": "ESNext",
"lib": [
"ESNext",
"DOM",
"DOM.Iterable"
],
"module": "ESNext",
"moduleResolution": "Bundler",
"types": [
"node"
],
"strict": true,
"noEmit": true,
"isolatedModules": true,
"verbatimModuleSyntax": true,
"skipLibCheck": true
},
"include": [
"cases/**/*.ts",
"src/**/*.ts",
"vitest.config.ts"
]
}
+48
View File
@@ -0,0 +1,48 @@
import { join } from 'node:path'
import fakemic, { electron, web } from '@proj-airi/vitest-plugin-fakemic'
import { defineConfig } from 'vitest/config'
export default defineConfig({
test: {
projects: [
fakemic({
name: 'audio-web',
include: ['cases/**/*.audio.test.ts', 'cases/**/*.audio.web.test.ts'],
runtime: web({
name: 'web',
prepare: new URL('./src/runtimes/prepare-web.ts', import.meta.url).href,
url: 'http://127.0.0.1:4173/',
context: { permissions: ['microphone'] },
preview: {
configFile: join(import.meta.dirname, '../../apps/stage-web/vite.config.ts'),
root: join(import.meta.dirname, '../../apps/stage-web'),
},
}),
}),
fakemic({
name: 'audio-electron',
include: ['cases/**/*.audio.test.ts', 'cases/**/*.audio.electron.test.ts'],
runtime: electron({
name: 'electron',
prepare: new URL('./src/runtimes/prepare-electron.ts', import.meta.url).href,
entry: join(import.meta.dirname, '../../apps/stage-tamagotchi/out/main/index.js'),
args: ['--no-sandbox'],
cwd: join(import.meta.dirname, '../..'),
temporaryUserData: {
env: 'APP_USER_DATA_PATH',
prefix: 'airi-testing-audio-',
},
}),
}),
{
extends: true,
test: {
name: 'unit',
include: ['src/**/*.test.ts'],
},
},
],
},
})
+32
View File
@@ -0,0 +1,32 @@
# Vitest plugin for fake microphones
This package supplies fake-microphone Web and Electron runtimes with Vitest project integration.
Import the Vitest interface from the package root:
```ts
import fakemic, {
createAudioTestAPI,
electron,
runAudioTestSession,
web,
} from '@proj-airi/vitest-plugin-fakemic'
```
`src/index.ts` owns project configuration, runtime launch, task collection, and preflight scheduling. `src/runner.ts` owns task execution.
Runtime configuration contains only serializable values. A `prepare` module adapts the launched Playwright runtime into an application session.
```ts
fakemic({
name: 'audio-web',
include: ['cases/**/*.audio.test.ts', 'cases/**/*.audio.web.test.ts'],
runtime: web({
name: 'web',
prepare: new URL('./prepare-web.ts', import.meta.url).href,
url: 'http://127.0.0.1:4173/',
}),
})
```
Application packages own selectors, routes, probes, Provider settings, and prepare modules.
@@ -1,22 +1,24 @@
{
"name": "@proj-airi/audio-input-e2e",
"name": "@proj-airi/vitest-plugin-fakemic",
"type": "module",
"version": "0.11.3",
"private": true,
"description": "End-to-end audio input contracts for AIRI Web and Electron",
"description": "Fake microphone test runtime and Vitest integration",
"author": {
"name": "Moeru AI Project AIRI Team",
"email": "airi@moeru.ai",
"url": "https://github.com/moeru-ai"
},
"license": "MIT",
"exports": "./src/index.ts",
"scripts": {
"test": "vitest run",
"typecheck": "tsc --noEmit"
},
"dependencies": {
"@pnpm/find-workspace-dir": "catalog:",
"@proj-airi/stage-shared": "workspace:^",
"playwright": "catalog:",
"valibot": "catalog:"
"vite": "catalog:",
"vitest": "catalog:vitest"
},
"devDependencies": {
"@types/node": "catalog:"
@@ -0,0 +1,91 @@
import {
createAudioTestAPI,
createAudioTestTask,
runAudioTestSession,
} from '@proj-airi/vitest-plugin-fakemic'
import { describe, expect, it, vi } from 'vitest'
const calls: string[] = []
const audio = createAudioTestAPI<
{ value: string, preflight?: readonly ((context: { value: string }) => void)[] },
{ value: string },
{ capturedValue: string },
{ value: string }
>({
createPlans: (name, definition) => [{
name: `mock: ${name}`,
definition,
metadata: {
input: '/fixtures/input.wav',
runtime: 'mock',
},
}],
preflight: definition => definition.preflight,
async execute({ plan, task, invokeHandler, runPreflight }) {
await runPreflight({ value: 'preflight' })
Object.assign(task.context, {
capturedValue: plan.definition.value,
})
await invokeHandler()
},
})
audio.describe('createAudioTestAPI', () => {
audio.it('runs preflight before a registered task', {
value: 'captured',
preflight: [({ value }) => calls.push(value)],
}, ({ capturedValue }) => {
expect(calls).toEqual(['preflight'])
expect(capturedValue).toBe('captured')
})
})
describe('audio test tasks', () => {
it('creates one task for the current runtime project', () => {
const input = new URL('file:///audio/input.wav')
const task = createAudioTestTask('captures speech', { input })
expect(task).toEqual({ name: 'captures speech', input })
})
it('records artifacts before it closes the session', async () => {
const calls: string[] = []
const session = {
close: vi.fn(async () => {
calls.push('close')
}),
}
await runAudioTestSession({
start: async () => session,
execute: async () => {
calls.push('execute')
},
recordArtifacts: async () => {
calls.push('record')
},
})
expect(calls).toEqual(['execute', 'record', 'close'])
})
it('closes the session and keeps execution and cleanup failures', async () => {
const executionError = new Error('execution failed')
const closeError = new Error('close failed')
const result = runAudioTestSession({
start: async () => ({
close: async () => {
throw closeError
},
}),
execute: async () => {
throw executionError
},
})
await expect(result).rejects.toMatchObject({
errors: [executionError, closeError],
})
})
})
+420
View File
@@ -0,0 +1,420 @@
import type { Browser, BrowserContext, ElectronApplication } from 'playwright'
import type { RunnerTestCase, TestAPI, TestContext } from 'vitest'
import type { UserWorkspaceConfig } from 'vitest/config'
import { mkdtemp, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { env } from 'node:process'
import { fileURLToPath } from 'node:url'
import { chromium, _electron as playwrightElectron } from 'playwright'
import { preview } from 'vite'
import { describe, inject, TestRunner } from 'vitest'
/** Supported encodings for captured audio. */
export type AudioCaptureFormat = 'pcm' | 'wav'
/** One audio payload captured from a tested runtime. */
export interface AudioCapture {
format: AudioCaptureFormat
data: Uint8Array
}
/** A session that owns one audio test runtime. */
export interface AudioTestSession {
/** Releases the runtime and all resources that belong to the session. */
close: () => Promise<void>
}
/** One callback that runs after a runtime starts and before its test handler. */
export type AudioTestPreflightCallback<Context> = (context: Context) => void | Promise<void>
/** A runner-neutral audio test definition. */
export interface AudioTestCase<PreflightContext = never> {
/** File-backed microphone input for the test. */
input: URL
/** @default [] */
preflight?: readonly AudioTestPreflightCallback<PreflightContext>[]
}
/** One runnable task derived from an audio test case. */
export interface AudioTestTask {
name: string
input: URL
}
/** Lifecycle operations for one audio test session. */
export interface RunAudioTestSessionOptions<Session extends AudioTestSession> {
start: () => Promise<Session>
execute: (session: Session) => Promise<void>
recordArtifacts?: (session: Session) => Promise<void>
}
/** Metadata that identifies an audio task in Vitest reports. */
export interface AudioVitestTaskMetadata {
input: string
runtime: string
}
declare module 'vitest' {
interface TaskMeta {
audioTest?: AudioVitestTaskMetadata
}
}
/**
* One concrete audio task that the custom runner can execute.
*
* @param Definition - The concrete definition stored for this task.
*/
export interface AudioVitestPlan<Definition> {
name: string
definition: Definition
metadata: AudioVitestTaskMetadata
}
/**
* Test context exposed to an audio test callback.
*
* @param Context - Fields added by the concrete audio framework.
*/
export type AudioVitestTaskContext<Context extends object> = TestContext & Context
/**
* Callback for an audio task.
*
* @param Context - Fields added by the concrete audio framework.
*/
export type AudioTestHandler<Context extends object> = (
context: AudioVitestTaskContext<Context>,
) => void | Promise<void>
/**
* A Vitest-like test function that accepts an audio definition.
*
* @param Definition - The definition supplied by each test.
* @param Context - Fields exposed to the test callback.
*/
export interface AudioTestAPI<Definition, Context extends object> {
(name: string, definition: Definition, handler: AudioTestHandler<Context>): void
only: AudioTestAPI<Definition, Context>
skip: AudioTestAPI<Definition, Context>
todo: AudioTestAPI<Definition, Context>
fails: AudioTestAPI<Definition, Context>
}
/**
* Configuration for a package-owned audio test interface.
*
* @param Definition - The definition supplied by each test.
* @param Plan - One concrete task produced from that definition.
*/
export interface CreateAudioTestAPIOptions<Definition, Plan, PreflightContext> {
createPlans: (name: string, definition: Definition) => Array<AudioVitestPlan<Plan>>
execute: (options: {
plan: AudioVitestPlan<Plan>
task: RunnerTestCase
invokeHandler: () => Promise<void>
runPreflight: (context: PreflightContext) => Promise<void>
}) => Promise<void>
preflight?: (definition: Definition) => readonly AudioTestPreflightCallback<PreflightContext>[] | undefined
}
/** Serializable Web runtime configuration. */
export interface FakemicWebRuntime {
kind: 'web'
name: string
prepare: string
url: string
launch?: Parameters<typeof chromium.launch>[0]
context?: Parameters<Browser['newContext']>[0]
preview?: {
configFile: string
root: string
host?: string
port?: number
}
}
/** Serializable Electron runtime configuration. */
export interface FakemicElectronRuntime {
kind: 'electron'
name: string
prepare: string
entry: string
args?: string[]
cwd?: string
env?: Record<string, string>
temporaryUserData?: {
env: string
prefix?: string
}
}
/** Runtime selected by one Fakemic Vitest project. */
export type FakemicRuntime = FakemicElectronRuntime | FakemicWebRuntime
/** Context supplied to a Web prepare module. */
export interface FakemicWebPrepareContext {
browser: Browser
context: BrowserContext
close: () => Promise<void>
runtime: FakemicWebRuntime
}
/** Context supplied to an Electron prepare module. */
export interface FakemicElectronPrepareContext {
app: ElectronApplication
close: () => Promise<void>
runtime: FakemicElectronRuntime
}
/** Module that adapts a launched runtime into the application session. */
export interface FakemicPrepareModule<Context, Session extends AudioTestSession> {
default: (context: Context) => Promise<Session>
}
/** Configuration for one Fakemic Vitest project. */
export interface FakemicPluginOptions {
include: string[]
name: string
runtime: FakemicRuntime
/** @default 180000 */
testTimeout?: number
/** @default 120000 */
hookTimeout?: number
}
declare module 'vitest' {
interface ProvidedContext {
fakemicRuntime: FakemicRuntime
}
}
interface FakemicTaskExecution {
run: (task: RunnerTestCase, invokeHandler: () => Promise<void>) => Promise<void>
}
const registryKey = Symbol.for('airi.vitest-plugin-fakemic.executions')
const registryHost = globalThis as typeof globalThis & Record<typeof registryKey, WeakMap<RunnerTestCase, FakemicTaskExecution> | undefined>
// NOTICE:
// Vitest can load the runner and collected test modules through different module IDs.
// The global symbol gives both module instances access to the same task registry.
// Source: the Vitest ModuleRunner seam between the runner and collected test files.
// Remove this registry when Vitest provides a public task execution registry.
const fakemicTaskExecutions = registryHost[registryKey] ??= new WeakMap()
/** Creates package-owned `describe` and `it` functions for an audio framework. */
export function createAudioTestAPI<Definition, Plan, Context extends object, PreflightContext = never>(
options: CreateAudioTestAPIOptions<Definition, Plan, PreflightContext>,
): {
describe: typeof describe
it: AudioTestAPI<Definition, Context>
} {
const collector = TestRunner.createTaskCollector(function (
this: object,
name: string,
definition: Definition,
handler: AudioTestHandler<Context>,
) {
const plans = options.createPlans(name, definition)
const preflight = options.preflight?.(definition) ?? []
for (const plan of plans) {
const task = TestRunner.getCurrentSuite<Context>().task(plan.name, {
...this,
meta: {
audioTest: plan.metadata,
},
handler: async (context) => {
await handler(context as AudioVitestTaskContext<Context>)
},
})
fakemicTaskExecutions.set(task, {
run: (runnerTask, invokeHandler) => options.execute({
plan,
task: runnerTask,
invokeHandler,
async runPreflight(context) {
for (const callback of preflight)
await callback(context)
},
}),
})
}
})
return {
describe,
it: collector as TestAPI as AudioTestAPI<Definition, Context>,
}
}
/** Creates a Web runtime descriptor for one Fakemic project. */
export function web(options: Omit<FakemicWebRuntime, 'kind'>): FakemicWebRuntime {
return { kind: 'web', ...options }
}
/** Creates an Electron runtime descriptor for one Fakemic project. */
export function electron(options: Omit<FakemicElectronRuntime, 'kind'>): FakemicElectronRuntime {
return { kind: 'electron', ...options }
}
/** Configures a serial Node Vitest project for package-owned audio tests. */
export default function fakemic(options: FakemicPluginOptions): UserWorkspaceConfig {
return {
test: {
name: options.name,
include: options.include,
environment: 'node',
runner: fileURLToPath(new URL('./runner.ts', import.meta.url)),
fileParallelism: false,
maxWorkers: 1,
testTimeout: options.testTimeout ?? 180_000,
hookTimeout: options.hookTimeout ?? 120_000,
provide: {
fakemicRuntime: options.runtime,
},
},
}
}
/** Creates Chromium arguments for a non-looping file-backed microphone. */
export function createChromiumFileMicrophoneArguments(microphoneInput: string): string[] {
return [
'--use-fake-ui-for-media-stream',
'--use-fake-device-for-media-stream',
`--use-file-for-fake-audio-capture=${microphoneInput}%noloop`,
'--autoplay-policy=no-user-gesture-required',
]
}
/**
* Creates one runnable task from an audio case.
*
* @example
* createAudioTestTask('greets the user', { input })
* // => { name: 'greets the user', input }
*/
export function createAudioTestTask<PreflightContext = never>(
name: string,
testCase: AudioTestCase<PreflightContext>,
): AudioTestTask {
return {
name,
input: testCase.input,
}
}
/** Launches the runtime selected by the current Vitest project. */
export async function startFakemicRuntime<Session extends AudioTestSession>(microphoneInput: string): Promise<Session> {
const runtime = inject('fakemicRuntime')
if (runtime.kind === 'electron')
return startElectronFakemicRuntime<Session>(runtime, microphoneInput)
return startWebFakemicRuntime<Session>(runtime, microphoneInput)
}
async function startWebFakemicRuntime<Session extends AudioTestSession>(runtime: FakemicWebRuntime, microphoneInput: string): Promise<Session> {
const server = runtime.preview
? await preview({
configFile: runtime.preview.configFile,
root: runtime.preview.root,
preview: {
host: runtime.preview.host ?? '127.0.0.1',
port: runtime.preview.port ?? 4173,
strictPort: true,
},
})
: undefined
let browser: Browser | undefined
const close = async () => {
await browser?.close()
await server?.close()
}
try {
browser = await chromium.launch({
...runtime.launch,
args: [...(runtime.launch?.args ?? []), ...createChromiumFileMicrophoneArguments(microphoneInput)],
})
const context = await browser.newContext(runtime.context)
const module = await import(runtime.prepare) as FakemicPrepareModule<FakemicWebPrepareContext, Session>
return await module.default({ browser, context, close, runtime })
}
catch (error) {
await close()
throw error
}
}
async function startElectronFakemicRuntime<Session extends AudioTestSession>(runtime: FakemicElectronRuntime, microphoneInput: string): Promise<Session> {
const temporaryUserData = runtime.temporaryUserData
const userDataPath = temporaryUserData
? await mkdtemp(join(tmpdir(), temporaryUserData.prefix ?? 'fakemic-electron-'))
: undefined
let app: ElectronApplication | undefined
const close = async () => {
await app?.close()
if (userDataPath)
await rm(userDataPath, { recursive: true, force: true })
}
try {
const launchEnvironment = Object.fromEntries(
Object.entries({ ...env, ...runtime.env })
.filter((entry): entry is [string, string] => entry[1] !== undefined),
)
if (temporaryUserData && userDataPath)
launchEnvironment[temporaryUserData.env] = userDataPath
app = await playwrightElectron.launch({
args: [runtime.entry, ...(runtime.args ?? []), ...createChromiumFileMicrophoneArguments(microphoneInput)],
cwd: runtime.cwd,
env: launchEnvironment,
})
const launchedApp = app
const module = await import(runtime.prepare) as FakemicPrepareModule<FakemicElectronPrepareContext, Session>
return await module.default({ app: launchedApp, close, runtime })
}
catch (error) {
await close()
throw error
}
}
/** Runs one audio session and preserves execution and cleanup failures. */
export async function runAudioTestSession<Session extends AudioTestSession>(
options: RunAudioTestSessionOptions<Session>,
): Promise<void> {
const session = await options.start()
const errors: unknown[] = []
try {
await options.execute(session)
}
catch (error) {
errors.push(error)
}
try {
await options.recordArtifacts?.(session)
}
catch (error) {
errors.push(error)
}
try {
await session.close()
}
catch (error) {
errors.push(error)
}
if (errors.length === 1)
throw errors[0]
if (errors.length > 1)
throw new AggregateError(errors, 'The audio test and its cleanup produced multiple errors')
}
@@ -0,0 +1,38 @@
import type { RunnerTestCase } from 'vitest'
import { TestRunner } from 'vitest'
interface FakemicTaskExecution {
run: (task: RunnerTestCase, invokeHandler: () => Promise<void>) => Promise<void>
}
const registryKey = Symbol.for('airi.vitest-plugin-fakemic.executions')
const registryHost = globalThis as typeof globalThis & Record<typeof registryKey, WeakMap<RunnerTestCase, FakemicTaskExecution> | undefined>
const fakemicTaskExecutions = registryHost[registryKey] ??= new WeakMap()
/**
* Runs package-owned audio tasks and delegates normal tasks to Vitest.
*
* Call stack:
*
* FakemicVitestRunner.runTask
* -> fakemic task execution
* -> {@link TestRunner.getTestFn}
*/
export default class FakemicVitestRunner extends TestRunner {
async runTask(task: RunnerTestCase): Promise<void> {
const handler = TestRunner.getTestFn(task)
if (!handler)
throw new Error(`Vitest did not collect a handler for "${task.name}"`)
const execution = fakemicTaskExecutions.get(task)
if (!execution) {
await handler()
return
}
await execution.run(task, async () => {
await handler()
})
}
}
@@ -0,0 +1,23 @@
{
"compilerOptions": {
"target": "ESNext",
"lib": [
"ESNext"
],
"rootDir": ".",
"module": "ESNext",
"moduleResolution": "Bundler",
"types": [
"node"
],
"strict": true,
"noEmit": true,
"isolatedModules": true,
"verbatimModuleSyntax": true,
"skipLibCheck": true
},
"include": [
"src/**/*.ts",
"vitest.config.ts"
]
}
@@ -0,0 +1,8 @@
import { defineConfig } from 'vitest/config'
export default defineConfig({
test: {
include: ['src/**/*.test.ts'],
runner: './src/runner.ts',
},
})
+1741 -282
View File
File diff suppressed because it is too large Load Diff
+1 -1
View File
@@ -115,7 +115,7 @@ catalog:
'@mediapipe/tasks-vision': ^0.10.34
'@modelcontextprotocol/sdk': ^1.29.0
'@moeru/eslint-config': 0.1.0-beta.19
'@moeru/eventa': 1.0.0-beta.13
'@moeru/eventa': 1.0.0-beta.15
'@moeru/std': 0.1.0-beta.17
'@moeru/three-mmd': 0.1.0-beta.7
'@moeru/three-mmd-physics-ammo': 0.1.0-beta.7
+1
View File
@@ -19,6 +19,7 @@ export default defineConfig({
'packages/server-runtime',
'packages/server-sdk',
'packages/stage-shared',
'packages/vitest-plugin-fakemic',
],
},
})