feat(stage-tamagotchi): add Apple Speech transcription

This commit is contained in:
Neko Ayaka
2026-08-12 01:43:06 +08:00
parent b04cc02a5a
commit b6f7dfa4c8
27 changed files with 1924 additions and 5 deletions
+4
View File
@@ -12,14 +12,17 @@
"homepage": "https://airi.moeru.ai/docs/",
"main": "./out/main/index.js",
"scripts": {
"apple-speech:build": "pnpm -F @proj-airi/apple-speech-transcription build",
"lint": "eslint --cache .",
"typecheck": "vue-tsc --noEmit",
"app:dev": "pnpm run dev",
"app:build": "pnpm run build",
"start": "electron-vite preview",
"start:xwayland": "electron-vite preview -- --ozone-platform=x11",
"predev": "pnpm run apple-speech:build",
"dev": "electron-vite dev",
"dev:xwayland": "electron-vite dev -- --ozone-platform=x11",
"prebuild": "pnpm run apple-speech:build",
"build": "electron-vite build",
"postinstall": "electron-builder install-app-deps",
"build:unpack": "pnpm run build && electron-builder --dir",
@@ -55,6 +58,7 @@
"@moeru/eventa": "catalog:",
"@moeru/std": "catalog:",
"@pinia/colada": "catalog:",
"@proj-airi/apple-speech-transcription": "workspace:^",
"@proj-airi/audio": "workspace:^",
"@proj-airi/ccc": "workspace:^",
"@proj-airi/drizzle-duckdb-wasm": "catalog:",
@@ -0,0 +1,135 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
type InvokeHandler = (payload: unknown) => unknown
type StreamHandler = (
input: unknown,
options?: { abortController?: AbortController },
) => AsyncGenerator<unknown>
const mocks = vi.hoisted(() => ({
getCapabilities: vi.fn(),
invokeHandlers: [] as InvokeHandler[],
streamHandler: undefined as StreamHandler | undefined,
transcribeAudio: vi.fn(),
transcribePcmStream: vi.fn(),
}))
vi.mock('@moeru/eventa', async (importOriginal) => {
const original = await importOriginal<typeof import('@moeru/eventa')>()
return {
...original,
defineInvokeHandler: vi.fn((
_context: unknown,
_event: unknown,
handler: InvokeHandler,
) => {
mocks.invokeHandlers.push(handler)
return vi.fn()
}),
defineStreamInvokeHandler: vi.fn((
_context: unknown,
_event: unknown,
handler: StreamHandler,
) => {
mocks.streamHandler = handler
return vi.fn()
}),
}
})
vi.mock('@proj-airi/apple-speech-transcription', () => ({
getCapabilities: mocks.getCapabilities,
transcribeAudio: mocks.transcribeAudio,
transcribePcmStream: mocks.transcribePcmStream,
}))
const { createAppleSpeechTranscriptionService } = await import('./apple-speech-transcription')
beforeEach(() => {
mocks.getCapabilities.mockReset()
mocks.invokeHandlers.length = 0
mocks.streamHandler = undefined
mocks.transcribeAudio.mockReset()
mocks.transcribePcmStream.mockReset()
createAppleSpeechTranscriptionService({ context: {} as never })
})
describe('apple Speech transcription service', () => {
it('forwards capability and file transcription invokes to the native package', async () => {
const capabilities = { available: true, installedLocales: [], supportedLocales: ['en_US'] }
const transcript = { durationMilliseconds: 12, isFinal: true, locale: 'en_US', text: 'hello' }
mocks.getCapabilities.mockResolvedValue(capabilities)
mocks.transcribeAudio.mockResolvedValue(transcript)
const audio = new Uint8Array([1, 2])
await expect(mocks.invokeHandlers[0]?.(undefined)).resolves.toEqual(capabilities)
await expect(mocks.invokeHandlers[1]?.({ audio, fileExtension: 'wav', locale: 'en_US' })).resolves.toEqual(transcript)
expect(mocks.transcribeAudio).toHaveBeenCalledWith(audio, 'en_US', 'wav')
})
it('reads the start frame and forwards PCM audio to the native stream', async () => {
const receivedAudio: Uint8Array[] = []
mocks.transcribePcmStream.mockImplementation(async function* (
audioFrames: AsyncIterable<Uint8Array>,
locale: string,
sampleRate: number,
) {
for await (const audio of audioFrames)
receivedAudio.push(audio)
yield { locale, sampleRate, text: 'hello' }
})
const input = new ReadableStream({
start(controller) {
controller.enqueue({ type: 'start', locale: 'en_US', sampleRate: 16000 })
controller.enqueue({ type: 'audio', audio: new Uint8Array([1, 2]) })
controller.close()
},
})
const handler = mocks.streamHandler
if (!handler)
throw new Error('The service did not register its Eventa stream handler.')
const updates = []
for await (const update of handler(input))
updates.push(update)
expect(receivedAudio).toEqual([new Uint8Array([1, 2])])
expect(updates).toEqual([{ locale: 'en_US', sampleRate: 16000, text: 'hello' }])
})
it('cancels a pending input read before it releases the reader', async () => {
const abortController = new AbortController()
let cancelReason: unknown
const input = new ReadableStream({
start(controller) {
controller.enqueue({ type: 'start', locale: 'en_US', sampleRate: 16000 })
},
cancel(reason) {
cancelReason = reason
},
})
mocks.transcribePcmStream.mockImplementation(async function* (
_audioFrames: AsyncIterable<Uint8Array>,
_locale: string,
_sampleRate: number,
options: { signal?: AbortSignal },
) {
await new Promise<void>((_resolve, reject) => {
options.signal?.addEventListener('abort', () => reject(options.signal?.reason), { once: true })
})
yield undefined
})
const handler = mocks.streamHandler
if (!handler)
throw new Error('The service did not register its Eventa stream handler.')
const output = handler(input, { abortController })
const nextUpdate = output.next()
await vi.waitFor(() => expect(mocks.transcribePcmStream).toHaveBeenCalledOnce())
abortController.abort('stop')
await expect(nextUpdate).rejects.toBe('stop')
expect(cancelReason).toBe('stop')
})
})
@@ -0,0 +1,81 @@
import type { createContext } from '@moeru/eventa/adapters/electron/main'
import type { ElectronAppleSpeechStreamInput } from '../../../shared/eventa'
import { defineInvokeHandler, defineStreamInvokeHandler, isReadableStream } from '@moeru/eventa'
import { getCapabilities, transcribeAudio, transcribePcmStream } from '@proj-airi/apple-speech-transcription'
import {
electronAppleSpeechGetCapabilities,
electronAppleSpeechTranscribe,
electronAppleSpeechTranscribeStream,
} from '../../../shared/eventa'
/** Registers the macOS Apple Speech control boundary for one Electron window. */
export function createAppleSpeechTranscriptionService(params: {
context: ReturnType<typeof createContext>['context']
}) {
const disposeCapabilities = defineInvokeHandler(
params.context,
electronAppleSpeechGetCapabilities,
() => getCapabilities(),
)
const disposeTranscription = defineInvokeHandler(
params.context,
electronAppleSpeechTranscribe,
payload => transcribeAudio(payload.audio, payload.locale, payload.fileExtension),
)
const disposeStreamingTranscription = defineStreamInvokeHandler(
params.context,
electronAppleSpeechTranscribeStream,
async function* (input, options) {
if (!isReadableStream<ElectronAppleSpeechStreamInput>(input))
throw new TypeError('Apple Speech streaming transcription requires a readable input stream.')
const reader = input.getReader()
const abortSignal = options?.abortController?.signal
const cancelInput = () => {
void reader.cancel(abortSignal?.reason).catch(() => undefined)
}
abortSignal?.addEventListener('abort', cancelInput, { once: true })
try {
const firstFrame = await reader.read()
if (firstFrame.done || firstFrame.value.type !== 'start')
throw new TypeError('Apple Speech streaming transcription requires a start frame before audio.')
async function* audioFrames() {
while (true) {
const frame = await reader.read()
if (frame.done)
return
if (frame.value.type !== 'audio')
throw new TypeError('Apple Speech received more than one start frame.')
yield frame.value.audio
}
}
yield* transcribePcmStream(
audioFrames(),
firstFrame.value.locale,
firstFrame.value.sampleRate,
{ signal: abortSignal },
)
}
finally {
abortSignal?.removeEventListener('abort', cancelInput)
// Cancel pending reads before release so the native input pump cannot
// retain this reader after the Eventa invocation stops.
await reader.cancel().catch(() => undefined)
reader.releaseLock()
}
},
)
return () => {
disposeCapabilities()
disposeTranscription()
disposeStreamingTranscription()
}
}
@@ -12,6 +12,7 @@ import { isMacOS } from 'std-env'
import { createServerChannelService } from '../../services/airi/channel-server'
import { createI18nService } from '../../services/airi/i18n'
import { createAppService, createPowerMonitorService, createScreenService, createSystemPreferencesService, createWindowService } from '../../services/electron'
import { createAppleSpeechTranscriptionService } from '../../services/electron/apple-speech-transcription'
export function toggleWindowShow(window?: BrowserWindow | null): void {
if (!window) {
@@ -142,6 +143,7 @@ export async function setupBaseWindowElectronInvokes(params: {
createAppService({ context: params.context, window: params.window })
createPowerMonitorService({ context: params.context, window: params.window })
createSystemPreferencesService({ context: params.context, window: params.window })
createAppleSpeechTranscriptionService({ context: params.context })
await createI18nService({ context: params.context, window: params.window, i18n: params.i18n })
@@ -19,6 +19,7 @@ import App from './App.vue'
import { i18n } from './modules/i18n'
import './providers/apple-speech-transcription'
import '@unocss/reset/tailwind.css'
import 'splitpanes/dist/splitpanes.css'
import 'vue-sonner/style.css'
@@ -0,0 +1,17 @@
<script setup lang="ts">
import { WIP } from '@proj-airi/stage-ui/components'
</script>
<template>
<WIP />
</template>
<route lang="yaml">
meta:
layout: settings
titleKey: settings.pages.providers.provider.apple-speech-transcription.title
subtitleKey: settings.title
descriptionKey: settings.pages.providers.provider.apple-speech-transcription.description
stageTransition:
name: slide
</route>
@@ -0,0 +1,117 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
const electron = vi.hoisted(() => ({
getCapabilities: vi.fn(),
transcribe: vi.fn(),
transcribeStream: vi.fn(),
}))
vi.mock('@proj-airi/electron-vueuse', () => ({
getElectronEventaContext: vi.fn(() => ({})),
useElectronEventaInvoke: vi.fn()
.mockReturnValueOnce(electron.getCapabilities)
.mockReturnValueOnce(electron.transcribe),
}))
vi.mock('@moeru/eventa', async (importOriginal) => {
const original = await importOriginal<typeof import('@moeru/eventa')>()
return {
...original,
defineStreamInvoke: vi.fn(() => electron.transcribeStream),
}
})
const { providerAppleSpeechTranscription } = await import('./apple-speech-transcription')
afterEach(() => {
vi.clearAllMocks()
})
describe('apple Speech transcription provider', () => {
it('lists supported locales and puts the preferred locale first', async () => {
electron.getCapabilities.mockResolvedValue({
available: true,
installedLocales: ['zh_CN'],
supportedLocales: ['zh_CN', 'en_US'],
})
const models = await providerAppleSpeechTranscription.extraMethods?.listModels?.(
{},
providerAppleSpeechTranscription.createProvider({}),
)
expect(models).toEqual([
expect.objectContaining({ id: 'en_US', name: 'en-US' }),
expect.objectContaining({ id: 'zh_CN', name: 'zh-CN' }),
])
})
it('sends encoded file audio through the unary Eventa boundary', async () => {
electron.transcribe.mockResolvedValue({
durationMilliseconds: 12,
isFinal: true,
locale: 'en_US',
text: 'hello',
})
const speech = createSpeech('en_US')
const form = new FormData()
form.set('model', 'en_US')
form.set('file', new File([new Uint8Array([1, 2, 3])], 'sample.wav', { type: 'audio/wav' }))
const response = await speech.fetch?.(new URL('https://example.invalid/transcription'), { body: form })
expect(electron.transcribe).toHaveBeenCalledWith({
audio: new Uint8Array([1, 2, 3]),
fileExtension: 'wav',
locale: 'en_US',
})
await expect(response?.json()).resolves.toEqual({ text: 'hello' })
})
it('streams a start frame, PCM audio, and replaceable snapshots through Eventa', async () => {
electron.transcribeStream.mockReturnValue(new ReadableStream({
start(controller) {
controller.enqueue({
durationMilliseconds: 500,
isFinal: false,
locale: 'zh_CN',
startMilliseconds: 0,
text: '你好',
})
controller.close()
},
}))
const speech = createSpeech('zh_CN')
const audio = new ReadableStream<ArrayBuffer>({
start(controller) {
controller.enqueue(new Uint8Array([1, 2]).buffer)
controller.close()
},
})
const response = await speech.fetch?.(
new URL('https://example.invalid/transcription'),
{ body: audio as BodyInit },
)
const request = electron.transcribeStream.mock.calls[0]?.[0] as ReadableStream<unknown> | undefined
if (!request)
throw new Error('The provider did not open an Eventa request stream.')
const frames = []
for await (const frame of request)
frames.push(frame)
expect(frames).toEqual([
{ type: 'start', locale: 'zh_CN', sampleRate: 16000 },
{ type: 'audio', audio: new Uint8Array([1, 2]) },
])
await expect(response?.text()).resolves.toContain('"type":"transcript.text.snapshot"')
})
})
function createSpeech(locale: string) {
const provider = providerAppleSpeechTranscription.createProvider({})
if (!('transcription' in provider))
throw new Error('The Apple Speech provider does not support transcription.')
return provider.transcription(locale)
}
@@ -0,0 +1,159 @@
import type { TranscriptionProvider } from '@xsai-ext/providers/utils'
import { defineStreamInvoke } from '@moeru/eventa'
import { getElectronEventaContext, useElectronEventaInvoke } from '@proj-airi/electron-vueuse'
import { defineProvider } from '@proj-airi/stage-ui/libs/providers/providers/registry'
import { z } from 'zod'
import {
electronAppleSpeechGetCapabilities,
electronAppleSpeechTranscribe,
electronAppleSpeechTranscribeStream,
} from '../../shared/eventa'
export const appleSpeechTranscriptionProviderId = 'apple-speech-transcription'
const getCapabilities = useElectronEventaInvoke(electronAppleSpeechGetCapabilities)
const transcribe = useElectronEventaInvoke(electronAppleSpeechTranscribe)
const transcribeStream = defineStreamInvoke(getElectronEventaContext(), electronAppleSpeechTranscribeStream)
const sseEncoder = new TextEncoder()
function toByteArray(chunk: ArrayBuffer | ArrayBufferView) {
if (chunk instanceof ArrayBuffer)
return new Uint8Array(chunk)
return new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength)
}
function createAppleSpeechStreamResponse(
audioStream: ReadableStream<ArrayBuffer | ArrayBufferView>,
locale: string,
signal?: AbortSignal | null,
) {
const audioReader = audioStream.getReader()
let sentStart = false
const request = new ReadableStream({
async pull(controller) {
if (!sentStart) {
sentStart = true
controller.enqueue({ type: 'start' as const, locale, sampleRate: 16000 })
return
}
const chunk = await audioReader.read()
if (chunk.done) {
controller.close()
return
}
controller.enqueue({ type: 'audio' as const, audio: toByteArray(chunk.value) })
},
async cancel(reason) {
await audioReader.cancel(reason)
},
})
const updates = transcribeStream(request, { signal: signal ?? undefined })
const body = updates.pipeThrough(new TransformStream({
transform(update, controller) {
controller.enqueue(sseEncoder.encode(`data: ${JSON.stringify({
...update,
type: 'transcript.text.snapshot',
})}\n\n`))
},
}))
return new Response(body, {
headers: {
'Cache-Control': 'no-cache',
'Content-Type': 'text/event-stream',
},
})
}
function fileExtension(file: File) {
const extension = file.name.split('.').at(-1)?.toLowerCase()
if (extension && /^[a-z0-9]+$/.test(extension))
return extension
if (file.type === 'audio/mp4')
return 'm4a'
if (file.type === 'audio/aiff')
return 'aiff'
return 'wav'
}
const providerAppleSpeechTranscription = defineProvider({
id: appleSpeechTranscriptionProviderId,
name: 'Apple Speech',
nameLocalize: ({ t }) => t('settings.pages.providers.provider.apple-speech-transcription.title'),
description: 'On-device transcription provided by macOS.',
descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.apple-speech-transcription.description'),
tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt', 'streaming-transcription'],
icon: 'i-simple-icons:apple',
requiresCredentials: false,
autoConfigureWhenAvailable: true,
isAvailableBy: async () => (await getCapabilities()).available,
capabilities: {
transcription: {
protocol: 'http',
generateOutput: true,
streamOutput: true,
streamInput: true,
},
},
createProviderConfig: () => z.object({}),
createProvider() {
return {
transcription: (model) => {
const fetch: typeof globalThis.fetch = async (_input, init) => {
if (init?.body instanceof ReadableStream)
return createAppleSpeechStreamResponse(init.body, model, init.signal)
if (!(init?.body instanceof FormData))
throw new TypeError('Apple Speech transcription requires multipart audio input.')
const requestedModel = init.body.get('model')
const file = init.body.get('file')
if (typeof requestedModel !== 'string' || !(file instanceof File))
throw new TypeError('Apple Speech transcription requires a locale model and audio file.')
const result = await transcribe({
audio: new Uint8Array(await file.arrayBuffer()),
fileExtension: fileExtension(file),
locale: requestedModel,
})
return Response.json({ text: result.text })
}
return {
apiKey: '',
baseURL: 'apple-speech://local/',
fetch,
model,
}
},
} satisfies TranscriptionProvider
},
validationRequiredWhen: () => false,
extraMethods: {
listModels: async () => {
const capabilities = await getCapabilities()
const installedLocales = new Set(capabilities.installedLocales)
const preferredLocale = navigator.language.replace('-', '_')
return capabilities.supportedLocales.toSorted((left, right) => {
if (left === preferredLocale)
return -1
if (right === preferredLocale)
return 1
return Number(installedLocales.has(right)) - Number(installedLocales.has(left))
|| left.localeCompare(right)
}).map(locale => ({
id: locale,
name: locale.replaceAll('_', '-'),
provider: appleSpeechTranscriptionProviderId,
}))
},
},
})
export { providerAppleSpeechTranscription }
@@ -46,6 +46,60 @@ export const electronSpotlightShortcutSet = defineInvokeEventa<ShortcutRegistrat
export const electronOpenSettingsDevtools = defineInvokeEventa('eventa:invoke:electron:windows:settings:devtools:open')
export const electronOpenDevtoolsWindow = defineInvokeEventa<void, { key: string, route?: string, width?: number, height?: number, x?: number, y?: number }>('eventa:invoke:electron:windows:devtools:open')
/** Runtime support and locale inventory reported by the macOS Speech framework. */
export interface ElectronAppleSpeechCapabilities {
available: boolean
installedLocales: string[]
reason?: string
supportedLocales: string[]
}
/** Final result returned by one on-device Apple Speech transcription request. */
export interface ElectronAppleSpeechTranscriptionResult {
durationMilliseconds: number
isFinal: true
locale: string
text: string
}
/** One control or PCM frame in an Apple Speech streaming request. */
export type ElectronAppleSpeechStreamInput
= | {
/** Opens the native analyzer before any audio frames arrive. */
type: 'start'
locale: string
sampleRate: number
}
| {
/** Mono PCM16 audio in native byte order. */
type: 'audio'
audio: Uint8Array
}
/** A replaceable transcript snapshot from Apple Speech. */
export interface ElectronAppleSpeechStreamUpdate {
/** Duration of the Apple result range that caused this snapshot. */
durationMilliseconds: number
/** Whether the Apple result range is final. Other text can still be volatile. */
isFinal: boolean
locale: string
/** Start of the Apple result range that caused this snapshot. */
startMilliseconds: number
/** Current transcript for all received audio. This value replaces the previous snapshot. */
text: string
}
export const electronAppleSpeechGetCapabilities = defineInvokeEventa<ElectronAppleSpeechCapabilities>('eventa:invoke:electron:apple-speech:get-capabilities')
export const electronAppleSpeechTranscribe = defineInvokeEventa<ElectronAppleSpeechTranscriptionResult, {
audio: Uint8Array
fileExtension: string
locale: string
}>('eventa:invoke:electron:apple-speech:transcribe')
export const electronAppleSpeechTranscribeStream = defineInvokeEventa<
ElectronAppleSpeechStreamUpdate,
ElectronAppleSpeechStreamInput
>('eventa:invoke:electron:apple-speech:transcribe-stream')
export interface ElectronServerChannelConfig {
tlsConfig?: ServerOptions['tlsConfig'] | null
authToken: string
@@ -0,0 +1,3 @@
.turbo/
build/
native/**/*.node
@@ -0,0 +1,37 @@
# Apple Speech Transcription
This package is a macOS-only proof of concept for Apple system speech transcription in Electron. It uses `SpeechAnalyzer` and `SpeechTranscriber` through a Swift static library and a raw Node-API bridge.
## When to use it
Use this package in the Electron main process on macOS 26 or later. It transcribes files and live mono PCM16 audio. The first request for a locale can download a system-managed model.
Do not import the native binding in a renderer process. Do not show this provider on Windows, Linux, older macOS versions, or Macs where `SpeechTranscriber` reports that it is unavailable.
## Build
```bash
pnpm -F @proj-airi/apple-speech-transcription build
```
The build uses the installed Swift compiler, Clang, and Node headers. Set `NODE_INCLUDE_DIR` only when the active Node installation does not keep `node_api.h` under its prefix.
## Try a file
```bash
pnpm -F @proj-airi/apple-speech-transcription smoke -- /absolute/path/to/audio.wav en-US
```
The command prints runtime capabilities and the final transcript. It also reports the native processing time. Model download time is included when the locale asset is not installed.
## Stream live audio
Call `transcribePcmStream()` with 16 kHz, mono PCM16 audio chunks. The function yields complete transcript snapshots. Replace the prior snapshot with each new `text` value.
Do not append these snapshots. Apple can revise volatile words and punctuation before it finalizes an audio range.
## AIRI integration
The Electron renderer keeps AIRI's microphone selection and VAD pipeline. Each VAD segment crosses the Eventa boundary as a PCM stream. The Electron main process sends the stream to this package and returns replaceable transcript snapshots.
This package does not own microphone capture. The provider is available only in the Electron app on supported macOS systems. Web, Windows, Linux, and older macOS systems do not show it.
@@ -0,0 +1,489 @@
import AVFAudio
import CoreMedia
import Foundation
import Speech
public typealias AppleSpeechStreamUpdate = @Sendable (NSString?, NSString?, Bool) -> Void
@available(macOS 26.0, *)
private actor AppleSpeechStreamRegistry {
private var sessions: [String: AppleSpeechStreamSession] = [:]
func start(
localeIdentifier: String,
sampleRate: Int,
update: @escaping AppleSpeechStreamUpdate
) async throws -> String {
let identifier = UUID().uuidString
let session = try await AppleSpeechStreamSession(
localeIdentifier: localeIdentifier,
sampleRate: sampleRate,
update: update
)
sessions[identifier] = session
return identifier
}
func append(identifier: String, audio: Data) async throws {
guard let session = sessions[identifier] else {
throw AppleSpeechError.unknownStream(identifier)
}
try await session.append(audio)
}
func finish(identifier: String) async throws {
guard let session = sessions.removeValue(forKey: identifier) else {
throw AppleSpeechError.unknownStream(identifier)
}
try await session.finish()
}
func cancel(identifier: String) async {
guard let session = sessions.removeValue(forKey: identifier) else {
return
}
await session.cancel()
}
}
@available(macOS 26.0, *)
private actor AppleSpeechStreamSession {
private struct Segment {
let range: CMTimeRange
let text: String
let isFinal: Bool
}
private let analyzer: SpeechAnalyzer
private let audioFormat: AVAudioFormat
private let inputContinuation: AsyncStream<AnalyzerInput>.Continuation
private let inputSequence: AsyncStream<AnalyzerInput>
private let locale: Locale
private let transcriber: SpeechTranscriber
private let update: AppleSpeechStreamUpdate
private var resultTask: Task<Void, Error>?
private var segments: [Segment] = []
private var completed = false
private var stopped = false
init(
localeIdentifier: String,
sampleRate: Int,
update: @escaping AppleSpeechStreamUpdate
) async throws {
guard SpeechTranscriber.isAvailable else {
throw AppleSpeechError.unavailable
}
guard sampleRate > 0 else {
throw AppleSpeechError.invalidSampleRate(sampleRate)
}
let requestedLocale = Locale(identifier: localeIdentifier)
guard let locale = await SpeechTranscriber.supportedLocale(equivalentTo: requestedLocale) else {
throw AppleSpeechError.unsupportedLocale(localeIdentifier)
}
guard let audioFormat = AVAudioFormat(
commonFormat: .pcmFormatInt16,
sampleRate: Double(sampleRate),
channels: 1,
interleaved: false
) else {
throw AppleSpeechError.invalidSampleRate(sampleRate)
}
let transcriber = SpeechTranscriber(locale: locale, preset: .timeIndexedProgressiveTranscription)
if let installationRequest = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) {
try await installationRequest.downloadAndInstall()
}
let input = AsyncStream<AnalyzerInput>.makeStream()
self.analyzer = SpeechAnalyzer(modules: [transcriber])
self.audioFormat = audioFormat
self.inputContinuation = input.continuation
self.inputSequence = input.stream
self.locale = locale
self.transcriber = transcriber
self.update = update
try await analyzer.prepareToAnalyze(in: audioFormat)
resultTask = Task { [weak self, transcriber] in
do {
for try await result in transcriber.results {
try Task.checkCancellation()
await self?.accept(result)
}
} catch is CancellationError {
return
} catch {
await self?.fail(error)
throw error
}
}
try await analyzer.start(inputSequence: inputSequence)
}
func append(_ audio: Data) throws {
guard !stopped else {
throw AppleSpeechError.streamStopped
}
guard audio.count.isMultiple(of: MemoryLayout<Int16>.size) else {
throw AppleSpeechError.invalidPCMByteCount(audio.count)
}
let frameCount = audio.count / MemoryLayout<Int16>.size
guard frameCount > 0 else {
return
}
guard let buffer = AVAudioPCMBuffer(
pcmFormat: audioFormat,
frameCapacity: AVAudioFrameCount(frameCount)
), let samples = buffer.int16ChannelData?[0] else {
throw AppleSpeechError.cannotCreatePCMBuffer
}
audio.withUnsafeBytes { bytes in
guard let source = bytes.baseAddress else {
return
}
memcpy(samples, source, audio.count)
}
buffer.frameLength = AVAudioFrameCount(frameCount)
inputContinuation.yield(AnalyzerInput(buffer: buffer))
}
func finish() async throws {
guard !stopped else {
return
}
stopped = true
inputContinuation.finish()
do {
try await analyzer.finalizeAndFinishThroughEndOfInput()
try await resultTask?.value
complete()
} catch {
fail(error)
throw error
}
}
func cancel() async {
guard !stopped else {
return
}
stopped = true
inputContinuation.finish()
resultTask?.cancel()
await analyzer.cancelAndFinishNow()
complete()
}
private func accept(_ result: SpeechTranscriber.Result) {
let text = String(result.text.characters)
segments.removeAll { segment in
rangesOverlap(segment.range, result.range)
&& (result.isFinal || !segment.isFinal)
}
if !text.isEmpty {
segments.append(Segment(range: result.range, text: text, isFinal: result.isFinal))
}
segments.sort { CMTimeCompare($0.range.start, $1.range.start) < 0 }
do {
let payload: [String: Any] = [
"durationMilliseconds": milliseconds(result.range.duration),
"isFinal": result.isFinal,
"locale": locale.identifier,
"startMilliseconds": milliseconds(result.range.start),
"text": segments.map(\.text).joined().trimmingCharacters(in: .whitespacesAndNewlines),
]
update(try jsonString(payload) as NSString, nil, false)
} catch {
fail(error)
}
}
private func fail(_ error: Error) {
guard !completed else {
return
}
completed = true
stopped = true
update(nil, String(describing: error) as NSString, true)
}
private func complete() {
guard !completed else {
return
}
completed = true
update(nil, nil, true)
}
private func rangesOverlap(_ left: CMTimeRange, _ right: CMTimeRange) -> Bool {
CMTimeCompare(left.start, CMTimeRangeGetEnd(right)) < 0
&& CMTimeCompare(right.start, CMTimeRangeGetEnd(left)) < 0
}
private func milliseconds(_ time: CMTime) -> Double {
let seconds = CMTimeGetSeconds(time)
return seconds.isFinite ? seconds * 1_000 : 0
}
}
private func jsonString(_ value: [String: Any]) throws -> String {
let data = try JSONSerialization.data(withJSONObject: value, options: [.sortedKeys])
guard let json = String(data: data, encoding: .utf8) else {
throw AppleSpeechError.invalidJSON
}
return json
}
@available(macOS 26.0, *)
@objc public final class AppleSpeechBridge: NSObject {
private static let streamRegistry = AppleSpeechStreamRegistry()
@objc public static func getCapabilities(
completion: @escaping (NSString?, NSString?) -> Void
) {
Task {
guard SpeechTranscriber.isAvailable else {
completeJSON(
[
"available": false,
"installedLocales": [],
"reason": "SpeechTranscriber is unavailable on this Mac.",
"supportedLocales": [],
],
completion: completion
)
return
}
let supportedLocales = await SpeechTranscriber.supportedLocales
let installedLocales = await SpeechTranscriber.installedLocales
completeJSON(
[
"available": true,
"installedLocales": installedLocales.map(\.identifier).sorted(),
"supportedLocales": supportedLocales.map(\.identifier).sorted(),
],
completion: completion
)
}
}
@objc public static func transcribeFile(
_ path: NSString,
localeIdentifier: NSString,
completion: @escaping (NSString?, NSString?) -> Void
) {
Task {
do {
let result = try await transcribeFile(
path: path as String,
localeIdentifier: localeIdentifier as String
)
completeJSON(result, completion: completion)
} catch {
completion(nil, String(describing: error) as NSString)
}
}
}
@objc public static func transcribeAudio(
_ audio: NSData,
localeIdentifier: NSString,
fileExtension: NSString,
completion: @escaping (NSString?, NSString?) -> Void
) {
Task {
let safeExtension = (fileExtension as String).filter { $0.isLetter || $0.isNumber }
let temporaryURL = FileManager.default.temporaryDirectory
.appendingPathComponent(UUID().uuidString)
.appendingPathExtension(safeExtension.isEmpty ? "wav" : safeExtension)
do {
try (audio as Data).write(to: temporaryURL, options: .atomic)
defer { try? FileManager.default.removeItem(at: temporaryURL) }
let result = try await transcribeFile(
path: temporaryURL.path,
localeIdentifier: localeIdentifier as String
)
completeJSON(result, completion: completion)
} catch {
completion(nil, String(describing: error) as NSString)
}
}
}
@objc public static func startStreaming(
localeIdentifier: NSString,
sampleRate: Int,
update: @escaping AppleSpeechStreamUpdate,
completion: @escaping (NSString?, NSString?) -> Void
) {
let locale = localeIdentifier as String
Task {
do {
let identifier = try await streamRegistry.start(
localeIdentifier: locale,
sampleRate: sampleRate,
update: update
)
completion(identifier as NSString, nil)
} catch {
completion(nil, String(describing: error) as NSString)
}
}
}
@objc public static func appendStreamingAudio(
sessionIdentifier: NSString,
audio: NSData,
completion: @escaping (NSString?, NSString?) -> Void
) {
let identifier = sessionIdentifier as String
let audioData = audio as Data
Task {
do {
try await streamRegistry.append(
identifier: identifier,
audio: audioData
)
completion("{}" as NSString, nil)
} catch {
completion(nil, String(describing: error) as NSString)
}
}
}
@objc public static func finishStreaming(
sessionIdentifier: NSString,
completion: @escaping (NSString?, NSString?) -> Void
) {
let identifier = sessionIdentifier as String
Task {
do {
try await streamRegistry.finish(identifier: identifier)
completion("{}" as NSString, nil)
} catch {
completion(nil, String(describing: error) as NSString)
}
}
}
@objc public static func cancelStreaming(
sessionIdentifier: NSString,
completion: @escaping (NSString?, NSString?) -> Void
) {
let identifier = sessionIdentifier as String
Task {
await streamRegistry.cancel(identifier: identifier)
completion("{}" as NSString, nil)
}
}
private static func transcribeFile(
path: String,
localeIdentifier: String
) async throws -> [String: Any] {
guard SpeechTranscriber.isAvailable else {
throw AppleSpeechError.unavailable
}
let requestedLocale = Locale(identifier: localeIdentifier)
guard let locale = await SpeechTranscriber.supportedLocale(equivalentTo: requestedLocale) else {
throw AppleSpeechError.unsupportedLocale(localeIdentifier)
}
let transcriber = SpeechTranscriber(locale: locale, preset: .transcription)
if let installationRequest = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) {
try await installationRequest.downloadAndInstall()
}
let file = try AVAudioFile(forReading: URL(fileURLWithPath: path))
let analyzer = SpeechAnalyzer(modules: [transcriber])
let startedAt = ContinuousClock.now
let resultTask = Task { () throws -> [String] in
var segments: [String] = []
for try await result in transcriber.results where result.isFinal {
let text = String(result.text.characters).trimmingCharacters(in: .whitespacesAndNewlines)
if !text.isEmpty {
segments.append(text)
}
}
return segments
}
defer { resultTask.cancel() }
let lastSampleTime = try await analyzer.analyzeSequence(from: file)
if let lastSampleTime {
try await analyzer.finalizeAndFinish(through: lastSampleTime)
} else {
await analyzer.cancelAndFinishNow()
}
let segments = try await resultTask.value
let duration = startedAt.duration(to: .now).components
let durationMilliseconds = Double(duration.seconds) * 1_000
+ Double(duration.attoseconds) / 1_000_000_000_000_000
return [
"durationMilliseconds": durationMilliseconds,
"isFinal": true,
"locale": locale.identifier,
"text": segments.joined(separator: " "),
]
}
private static func completeJSON(
_ value: [String: Any],
completion: @escaping (NSString?, NSString?) -> Void
) {
do {
completion(try jsonString(value) as NSString, nil)
} catch {
completion(nil, String(describing: error) as NSString)
}
}
}
private enum AppleSpeechError: LocalizedError {
case cannotCreatePCMBuffer
case invalidPCMByteCount(Int)
case invalidJSON
case invalidSampleRate(Int)
case streamStopped
case unknownStream(String)
case unavailable
case unsupportedLocale(String)
var errorDescription: String? {
switch self {
case .cannotCreatePCMBuffer:
return "Apple Speech could not create a PCM audio buffer."
case let .invalidPCMByteCount(count):
return "Apple Speech received \(count) PCM bytes. The byte count must be even."
case .invalidJSON:
return "Apple Speech returned a result that cannot be encoded as JSON."
case let .invalidSampleRate(sampleRate):
return "Apple Speech received invalid sample rate \(sampleRate)."
case .streamStopped:
return "Apple Speech received audio after the stream stopped."
case let .unknownStream(identifier):
return "Apple Speech stream \(identifier) does not exist."
case .unavailable:
return "SpeechTranscriber is unavailable on this Mac."
case let .unsupportedLocale(locale):
return "Apple Speech does not support locale \(locale)."
}
}
}
+52
View File
@@ -0,0 +1,52 @@
/** Runtime support and locale inventory reported by the macOS Speech framework. */
export interface AppleSpeechCapabilities {
available: boolean
installedLocales: string[]
reason?: string
supportedLocales: string[]
}
/** Final result returned by one on-device Apple Speech transcription request. */
export interface AppleSpeechTranscriptionResult {
durationMilliseconds: number
isFinal: true
locale: string
text: string
}
/** A full live transcript snapshot that replaces the previous snapshot. */
export interface AppleSpeechStreamingUpdate {
/** Duration of the Apple result range that caused this snapshot. */
durationMilliseconds: number
/** Whether the Apple result range is final. Other ranges in `text` can still be volatile. */
isFinal: boolean
locale: string
/** Start of the Apple result range that caused this snapshot. */
startMilliseconds: number
/** Current transcript for all received audio. This value replaces the previous snapshot. */
text: string
}
export interface AppleSpeechStreamingOptions {
/** Cancels the analyzer and closes the native session. */
signal?: AbortSignal
}
export type AppleSpeechAudioChunk = ArrayBuffer | ArrayBufferView
/** Returns whether Apple Speech transcription is available on this Mac. */
export function getCapabilities(): Promise<AppleSpeechCapabilities>
/** Transcribes encoded audio bytes with Apple's on-device speech model. */
export function transcribeAudio(audio: Uint8Array, locale: string, fileExtension: string): Promise<AppleSpeechTranscriptionResult>
/** Transcribes one local audio file with Apple's on-device speech model. */
export function transcribeFile(path: string, locale: string): Promise<AppleSpeechTranscriptionResult>
/** Transcribes live mono PCM16 audio and yields replaceable text snapshots. */
export function transcribePcmStream(
audioStream: AsyncIterable<AppleSpeechAudioChunk>,
locale: string,
sampleRate: number,
options?: AppleSpeechStreamingOptions,
): AsyncGenerator<AppleSpeechStreamingUpdate>
@@ -0,0 +1,190 @@
import process from 'node:process'
import { createRequire } from 'node:module'
import { release } from 'node:os'
import { errorMessageFrom } from '@moeru/std'
const require = createRequire(import.meta.url)
function supportsAppleSpeechAnalyzer() {
if (process.platform !== 'darwin')
return false
const darwinMajor = Number.parseInt(release().split('.')[0] ?? '', 10)
return Number.isFinite(darwinMajor) && darwinMajor >= 25
}
function loadBinding() {
if (!supportsAppleSpeechAnalyzer())
return undefined
return require(`./native/darwin-${process.arch}/apple-speech-transcription.node`)
}
function createAsyncQueue() {
const values = []
const waiters = []
let failure
let stopped = false
function settleWaiter(waiter) {
if (failure) {
waiter.reject(failure)
return
}
if (values.length > 0) {
waiter.resolve({ done: false, value: values.shift() })
return
}
if (stopped)
waiter.resolve({ done: true, value: undefined })
}
return {
push(value) {
if (stopped)
return
const waiter = waiters.shift()
if (waiter) {
waiter.resolve({ done: false, value })
return
}
values.push(value)
},
close() {
if (stopped)
return
stopped = true
while (waiters.length > 0)
settleWaiter(waiters.shift())
},
fail(error) {
if (stopped)
return
failure = error instanceof Error ? error : new Error(errorMessageFrom(error) ?? 'Apple Speech transcription failed.')
stopped = true
while (waiters.length > 0)
settleWaiter(waiters.shift())
},
iterable: {
[Symbol.asyncIterator]() {
return {
next() {
if (failure)
return Promise.reject(failure)
if (values.length > 0)
return Promise.resolve({ done: false, value: values.shift() })
if (stopped)
return Promise.resolve({ done: true, value: undefined })
return new Promise((resolve, reject) => waiters.push({ reject, resolve }))
},
}
},
},
}
}
function toByteArray(chunk) {
if (chunk instanceof Uint8Array)
return chunk
if (chunk instanceof ArrayBuffer)
return new Uint8Array(chunk)
if (ArrayBuffer.isView(chunk))
return new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength)
throw new TypeError('Apple Speech streaming input must contain ArrayBuffer or ArrayBufferView chunks.')
}
/** Returns whether Apple Speech transcription is available on this Mac. */
export async function getCapabilities() {
const binding = loadBinding()
if (!binding) {
return {
available: false,
installedLocales: [],
reason: 'Apple Speech transcription requires macOS 26 or later.',
supportedLocales: [],
}
}
return JSON.parse(await binding.getCapabilities())
}
/** Transcribes encoded audio bytes with Apple's on-device speech model. */
export async function transcribeAudio(audio, locale, fileExtension) {
const binding = loadBinding()
if (!binding)
throw new Error('Apple Speech transcription requires macOS 26 or later.')
return JSON.parse(await binding.transcribeAudio(audio, locale, fileExtension))
}
/** Transcribes one local audio file with Apple's on-device speech model. */
export async function transcribeFile(path, locale) {
const binding = loadBinding()
if (!binding)
throw new Error('Apple Speech transcription requires macOS 26 or later.')
return JSON.parse(await binding.transcribeFile(path, locale))
}
/** Transcribes a live mono PCM16 stream and yields replaceable text snapshots. */
export async function* transcribePcmStream(audioStream, locale, sampleRate, options = {}) {
const binding = loadBinding()
if (!binding)
throw new Error('Apple Speech transcription requires macOS 26 or later.')
if (!audioStream || typeof audioStream[Symbol.asyncIterator] !== 'function')
throw new TypeError('Apple Speech streaming transcription requires an async audio stream.')
const queue = createAsyncQueue()
let streamComplete = false
const onUpdate = (json, error, complete) => {
if (error) {
streamComplete = true
queue.fail(new Error(error))
return
}
if (json)
queue.push(JSON.parse(json))
if (complete) {
streamComplete = true
queue.close()
}
}
const sessionIdentifier = await binding.startStreaming(locale, sampleRate, onUpdate)
const abort = () => {
if (!streamComplete)
queue.fail(options.signal?.reason ?? new DOMException('Aborted', 'AbortError'))
}
options.signal?.addEventListener('abort', abort, { once: true })
const inputTask = (async () => {
try {
for await (const chunk of audioStream) {
if (options.signal?.aborted)
throw options.signal.reason ?? new DOMException('Aborted', 'AbortError')
await binding.appendStreamingAudio(sessionIdentifier, toByteArray(chunk))
}
await binding.finishStreaming(sessionIdentifier)
}
catch (error) {
await binding.cancelStreaming(sessionIdentifier).catch(() => undefined)
queue.fail(error)
}
})()
try {
for await (const update of queue.iterable)
yield update
await inputTask
}
finally {
options.signal?.removeEventListener('abort', abort)
if (!streamComplete)
await binding.cancelStreaming(sessionIdentifier).catch(() => undefined)
}
}
@@ -0,0 +1,395 @@
#include <node_api.h>
#include <string>
#import <Foundation/Foundation.h>
#import "AppleSpeechBridge-Swift.h"
namespace {
struct PromiseContext {
napi_deferred deferred;
napi_env env;
};
struct PromiseResult {
NSString* error;
NSString* json;
};
struct StreamUpdateResult {
bool complete;
NSString* error;
NSString* json;
};
void callJavaScript(napi_env env, napi_value, void* context, void* data) {
auto* promiseContext = static_cast<PromiseContext*>(context);
auto* result = static_cast<PromiseResult*>(data);
if (result->error != nil) {
napi_value message;
napi_value error;
napi_create_string_utf8(env, result->error.UTF8String, NAPI_AUTO_LENGTH, &message);
napi_create_error(env, nullptr, message, &error);
napi_reject_deferred(env, promiseContext->deferred, error);
} else {
napi_value json;
napi_create_string_utf8(env, result->json.UTF8String, NAPI_AUTO_LENGTH, &json);
napi_resolve_deferred(env, promiseContext->deferred, json);
}
delete result;
}
void finalizeThreadsafeFunction(napi_env, void* data, void*) {
delete static_cast<PromiseContext*>(data);
}
void callStreamUpdateJavaScript(napi_env env, napi_value callback, void*, void* data) {
auto* result = static_cast<StreamUpdateResult*>(data);
if (env == nullptr) {
delete result;
return;
}
napi_value undefined;
napi_get_undefined(env, &undefined);
napi_value arguments[3] = {undefined, undefined, undefined};
if (result->json != nil)
napi_create_string_utf8(env, result->json.UTF8String, NAPI_AUTO_LENGTH, &arguments[0]);
if (result->error != nil)
napi_create_string_utf8(env, result->error.UTF8String, NAPI_AUTO_LENGTH, &arguments[1]);
napi_get_boolean(env, result->complete, &arguments[2]);
napi_value ignored;
napi_call_function(env, undefined, callback, 3, arguments, &ignored);
delete result;
}
napi_threadsafe_function createThreadsafeFunction(napi_env env, napi_deferred deferred) {
auto* context = new PromiseContext{deferred, env};
napi_value resourceName;
napi_create_string_utf8(env, "apple-speech-transcription", NAPI_AUTO_LENGTH, &resourceName);
napi_threadsafe_function function;
napi_create_threadsafe_function(
env,
nullptr,
nullptr,
resourceName,
0,
1,
context,
finalizeThreadsafeFunction,
context,
callJavaScript,
&function);
return function;
}
napi_threadsafe_function createStreamThreadsafeFunction(napi_env env, napi_value callback) {
napi_value resourceName;
napi_create_string_utf8(env, "apple-speech-transcription-stream", NAPI_AUTO_LENGTH, &resourceName);
napi_threadsafe_function function;
napi_create_threadsafe_function(
env,
callback,
nullptr,
resourceName,
0,
1,
nullptr,
nullptr,
nullptr,
callStreamUpdateJavaScript,
&function);
return function;
}
void complete(napi_threadsafe_function function, NSString* json, NSString* error) {
auto* result = new PromiseResult{[error copy], [json copy]};
napi_call_threadsafe_function(function, result, napi_tsfn_nonblocking);
napi_release_threadsafe_function(function, napi_tsfn_release);
}
void sendStreamUpdate(
napi_threadsafe_function function,
NSString* json,
NSString* error,
bool isComplete) {
auto* result = new StreamUpdateResult{isComplete, [error copy], [json copy]};
napi_call_threadsafe_function(function, result, napi_tsfn_nonblocking);
if (isComplete)
napi_release_threadsafe_function(function, napi_tsfn_release);
}
napi_value getCapabilities(napi_env env, napi_callback_info) {
napi_value promise;
napi_deferred deferred;
napi_create_promise(env, &deferred, &promise);
napi_threadsafe_function function = createThreadsafeFunction(env, deferred);
if (@available(macOS 26.0, *)) {
[AppleSpeechBridge getCapabilitiesWithCompletion:^(NSString* json, NSString* error) {
complete(function, json, error);
}];
} else {
complete(function, nil, @"Apple Speech transcription requires macOS 26 or later.");
}
return promise;
}
bool readString(napi_env env, napi_value value, std::string& output) {
size_t length = 0;
if (napi_get_value_string_utf8(env, value, nullptr, 0, &length) != napi_ok)
return false;
output.resize(length + 1);
size_t copied = 0;
if (napi_get_value_string_utf8(env, value, output.data(), output.size(), &copied) != napi_ok)
return false;
output.resize(copied);
return true;
}
bool readByteArray(napi_env env, napi_value value, NSData** output) {
bool isTypedArray = false;
if (napi_is_typedarray(env, value, &isTypedArray) != napi_ok || !isTypedArray)
return false;
napi_typedarray_type arrayType;
size_t length;
void* data;
napi_value arrayBuffer;
size_t byteOffset;
if (napi_get_typedarray_info(
env,
value,
&arrayType,
&length,
&data,
&arrayBuffer,
&byteOffset) != napi_ok)
return false;
if (arrayType != napi_uint8_array && arrayType != napi_uint8_clamped_array)
return false;
*output = [NSData dataWithBytes:data length:length];
return true;
}
napi_value transcribeFile(napi_env env, napi_callback_info info) {
size_t argumentCount = 2;
napi_value arguments[2];
napi_get_cb_info(env, info, &argumentCount, arguments, nullptr, nullptr);
std::string path;
std::string locale;
if (argumentCount != 2 || !readString(env, arguments[0], path) || !readString(env, arguments[1], locale)) {
napi_throw_type_error(env, nullptr, "transcribeFile expects an audio path and locale string.");
return nullptr;
}
napi_value promise;
napi_deferred deferred;
napi_create_promise(env, &deferred, &promise);
napi_threadsafe_function function = createThreadsafeFunction(env, deferred);
if (@available(macOS 26.0, *)) {
[AppleSpeechBridge transcribeFile:[NSString stringWithUTF8String:path.c_str()]
localeIdentifier:[NSString stringWithUTF8String:locale.c_str()]
completion:^(NSString* json, NSString* error) {
complete(function, json, error);
}];
} else {
complete(function, nil, @"Apple Speech transcription requires macOS 26 or later.");
}
return promise;
}
napi_value transcribeAudio(napi_env env, napi_callback_info info) {
size_t argumentCount = 3;
napi_value arguments[3];
napi_get_cb_info(env, info, &argumentCount, arguments, nullptr, nullptr);
NSData* audio;
std::string locale;
std::string fileExtension;
if (argumentCount != 3 || !readByteArray(env, arguments[0], &audio) || !readString(env, arguments[1], locale) || !readString(env, arguments[2], fileExtension)) {
napi_throw_type_error(env, nullptr, "transcribeAudio expects Uint8Array audio, a locale, and a file extension.");
return nullptr;
}
napi_value promise;
napi_deferred deferred;
napi_create_promise(env, &deferred, &promise);
napi_threadsafe_function function = createThreadsafeFunction(env, deferred);
if (@available(macOS 26.0, *)) {
[AppleSpeechBridge transcribeAudio:audio
localeIdentifier:[NSString stringWithUTF8String:locale.c_str()]
fileExtension:[NSString stringWithUTF8String:fileExtension.c_str()]
completion:^(NSString* json, NSString* error) {
complete(function, json, error);
}];
} else {
complete(function, nil, @"Apple Speech transcription requires macOS 26 or later.");
}
return promise;
}
napi_value startStreaming(napi_env env, napi_callback_info info) {
size_t argumentCount = 3;
napi_value arguments[3];
napi_get_cb_info(env, info, &argumentCount, arguments, nullptr, nullptr);
std::string locale;
int64_t sampleRate;
napi_valuetype callbackType;
if (argumentCount != 3
|| !readString(env, arguments[0], locale)
|| napi_get_value_int64(env, arguments[1], &sampleRate) != napi_ok
|| napi_typeof(env, arguments[2], &callbackType) != napi_ok
|| callbackType != napi_function) {
napi_throw_type_error(env, nullptr, "startStreaming expects a locale, sample rate, and update callback.");
return nullptr;
}
napi_value promise;
napi_deferred deferred;
napi_create_promise(env, &deferred, &promise);
napi_threadsafe_function promiseFunction = createThreadsafeFunction(env, deferred);
napi_threadsafe_function updateFunction = createStreamThreadsafeFunction(env, arguments[2]);
if (@available(macOS 26.0, *)) {
[AppleSpeechBridge startStreamingWithLocaleIdentifier:[NSString stringWithUTF8String:locale.c_str()]
sampleRate:static_cast<NSInteger>(sampleRate)
update:^(NSString* json, NSString* error, BOOL isComplete) {
sendStreamUpdate(updateFunction, json, error, isComplete);
}
completion:^(NSString* identifier, NSString* error) {
complete(promiseFunction, identifier, error);
if (error != nil)
sendStreamUpdate(updateFunction, nil, error, true);
}];
} else {
NSString* error = @"Apple Speech transcription requires macOS 26 or later.";
complete(promiseFunction, nil, error);
sendStreamUpdate(updateFunction, nil, error, true);
}
return promise;
}
napi_value appendStreamingAudio(napi_env env, napi_callback_info info) {
size_t argumentCount = 2;
napi_value arguments[2];
napi_get_cb_info(env, info, &argumentCount, arguments, nullptr, nullptr);
std::string identifier;
NSData* audio;
if (argumentCount != 2
|| !readString(env, arguments[0], identifier)
|| !readByteArray(env, arguments[1], &audio)) {
napi_throw_type_error(env, nullptr, "appendStreamingAudio expects a session identifier and PCM byte array.");
return nullptr;
}
napi_value promise;
napi_deferred deferred;
napi_create_promise(env, &deferred, &promise);
napi_threadsafe_function function = createThreadsafeFunction(env, deferred);
if (@available(macOS 26.0, *)) {
[AppleSpeechBridge appendStreamingAudioWithSessionIdentifier:[NSString stringWithUTF8String:identifier.c_str()]
audio:audio
completion:^(NSString* json, NSString* error) {
complete(function, json, error);
}];
} else {
complete(function, nil, @"Apple Speech transcription requires macOS 26 or later.");
}
return promise;
}
napi_value finishStreaming(napi_env env, napi_callback_info info) {
size_t argumentCount = 1;
napi_value arguments[1];
napi_get_cb_info(env, info, &argumentCount, arguments, nullptr, nullptr);
std::string identifier;
if (argumentCount != 1 || !readString(env, arguments[0], identifier)) {
napi_throw_type_error(env, nullptr, "finishStreaming expects a session identifier.");
return nullptr;
}
napi_value promise;
napi_deferred deferred;
napi_create_promise(env, &deferred, &promise);
napi_threadsafe_function function = createThreadsafeFunction(env, deferred);
if (@available(macOS 26.0, *)) {
[AppleSpeechBridge finishStreamingWithSessionIdentifier:[NSString stringWithUTF8String:identifier.c_str()]
completion:^(NSString* json, NSString* error) {
complete(function, json, error);
}];
} else {
complete(function, nil, @"Apple Speech transcription requires macOS 26 or later.");
}
return promise;
}
napi_value cancelStreaming(napi_env env, napi_callback_info info) {
size_t argumentCount = 1;
napi_value arguments[1];
napi_get_cb_info(env, info, &argumentCount, arguments, nullptr, nullptr);
std::string identifier;
if (argumentCount != 1 || !readString(env, arguments[0], identifier)) {
napi_throw_type_error(env, nullptr, "cancelStreaming expects a session identifier.");
return nullptr;
}
napi_value promise;
napi_deferred deferred;
napi_create_promise(env, &deferred, &promise);
napi_threadsafe_function function = createThreadsafeFunction(env, deferred);
if (@available(macOS 26.0, *)) {
[AppleSpeechBridge cancelStreamingWithSessionIdentifier:[NSString stringWithUTF8String:identifier.c_str()]
completion:^(NSString* json, NSString* error) {
complete(function, json, error);
}];
} else {
complete(function, nil, @"Apple Speech transcription requires macOS 26 or later.");
}
return promise;
}
napi_value initialize(napi_env env, napi_value exports) {
napi_property_descriptor properties[] = {
{"appendStreamingAudio", nullptr, appendStreamingAudio, nullptr, nullptr, nullptr, napi_default, nullptr},
{"cancelStreaming", nullptr, cancelStreaming, nullptr, nullptr, nullptr, napi_default, nullptr},
{"finishStreaming", nullptr, finishStreaming, nullptr, nullptr, nullptr, napi_default, nullptr},
{"getCapabilities", nullptr, getCapabilities, nullptr, nullptr, nullptr, napi_default, nullptr},
{"startStreaming", nullptr, startStreaming, nullptr, nullptr, nullptr, napi_default, nullptr},
{"transcribeAudio", nullptr, transcribeAudio, nullptr, nullptr, nullptr, napi_default, nullptr},
{"transcribeFile", nullptr, transcribeFile, nullptr, nullptr, nullptr, napi_default, nullptr},
};
napi_define_properties(env, exports, 7, properties);
return exports;
}
} // namespace
NAPI_MODULE(NODE_GYP_MODULE_NAME, initialize)
@@ -0,0 +1,24 @@
{
"name": "@proj-airi/apple-speech-transcription",
"type": "module",
"version": "0.11.3",
"private": true,
"description": "macOS 26 Apple Speech transcription bridge for Electron",
"license": "MIT",
"exports": {
".": {
"types": "./index.d.ts",
"default": "./index.mjs"
}
},
"engines": {
"node": ">=22"
},
"scripts": {
"build": "node ./scripts/build.mjs",
"smoke": "node ./scripts/smoke.mjs"
},
"dependencies": {
"@moeru/std": "catalog:"
}
}
@@ -0,0 +1,13 @@
import process from 'node:process'
import { execFileSync } from 'node:child_process'
import { fileURLToPath } from 'node:url'
if (process.platform !== 'darwin') {
console.info('Skipping Apple Speech native build outside macOS.')
process.exit(0)
}
execFileSync(fileURLToPath(new URL('./build.sh', import.meta.url)), {
stdio: 'inherit',
})
+63
View File
@@ -0,0 +1,63 @@
#!/bin/sh
set -eu
package_dir=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
architecture=$(uname -m)
case "$architecture" in
arm64) node_arch=arm64 ;;
x86_64) node_arch=x64 ;;
*) echo "Unsupported macOS architecture: $architecture" >&2; exit 1 ;;
esac
node_executable=$(node -p 'process.execPath')
node_prefix=$(dirname -- "$(dirname -- "$node_executable")")
node_include_dir=${NODE_INCLUDE_DIR:-"$node_prefix/include/node"}
if [ ! -f "$node_include_dir/node_api.h" ]; then
echo "node_api.h was not found under $node_include_dir. Set NODE_INCLUDE_DIR." >&2
exit 1
fi
build_dir=$(mktemp -d "${TMPDIR:-/tmp}/apple-speech-transcription.XXXXXX")
output_dir="$package_dir/native/darwin-$node_arch"
trap 'rm -rf -- "$build_dir"' EXIT HUP INT TERM
mkdir -p "$output_dir"
target="$architecture-apple-macosx26.0"
swift_header="$build_dir/AppleSpeechBridge-Swift.h"
swift_library="$build_dir/libAppleSpeechBridge.a"
swiftc \
-parse-as-library \
-target "$target" \
-module-name AppleSpeechBridge \
-emit-module \
-emit-objc-header \
-emit-objc-header-path "$swift_header" \
-emit-library \
-static \
"$package_dir/Sources/AppleSpeechBridge.swift" \
-o "$swift_library"
clang++ \
-std=c++17 \
-fobjc-arc \
-mmacosx-version-min=26.0 \
-I"$node_include_dir" \
-I"$build_dir" \
-c "$package_dir/native/addon.mm" \
-o "$build_dir/addon.o"
swiftc \
-target "$target" \
-emit-library \
"$build_dir/addon.o" \
"$swift_library" \
-framework AVFAudio \
-framework Foundation \
-framework Speech \
-Xlinker -lc++ \
-Xlinker -undefined \
-Xlinker dynamic_lookup \
-o "$output_dir/apple-speech-transcription.node"
echo "Built $output_dir/apple-speech-transcription.node"
@@ -0,0 +1,17 @@
import process from 'node:process'
import { resolve } from 'node:path'
import { getCapabilities, transcribeFile } from '@proj-airi/apple-speech-transcription'
const argumentsWithoutSeparator = process.argv.slice(2).filter(argument => argument !== '--')
const inputPath = argumentsWithoutSeparator[0]
const locale = argumentsWithoutSeparator[1] ?? 'en-US'
const capabilities = await getCapabilities()
console.info(JSON.stringify({ capabilities }, null, 2))
if (inputPath) {
const result = await transcribeFile(resolve(process.env.INIT_CWD ?? process.cwd(), inputPath), locale)
console.info(JSON.stringify({ result }, null, 2))
}
@@ -1062,6 +1062,9 @@ pages:
OpenAI, Azure Speech
description: LLMs, speech providers, etc.
provider:
apple-speech-transcription:
title: Apple Speech
description: On-device transcription provided by macOS 26.
app-local-audio-transcription:
title: App (Local)
description: https://github.com/moeru-ai/xsai-transformers
@@ -1011,6 +1011,9 @@ pages:
转录(语音转文本)模型服务来源,例如 Whisper.cpp, OpenAI, Azure Speech
description: LLM,语音合成,语音识别服务来源等
provider:
apple-speech-transcription:
title: Apple 语音识别
description: macOS 26 提供的设备端语音转写。
app-local-audio-transcription:
title: 应用内(本地)
description: https://github.com/moeru-ai/xsai-transformers
@@ -194,6 +194,14 @@ export interface ProviderDefinition<TConfig extends any = any> {
*/
requiresCredentials?: boolean
/**
* Makes an available provider selectable without persisted validation state.
* Use this only when the availability probe is sufficient configuration.
*
* @default false
*/
autoConfigureWhenAvailable?: boolean
createProviderConfig: (contextOptions: { t: ComposerTranslation }) => $ZodType<TConfig>
onboardingFields?: (ctx: { t: ComposerTranslation }) => ProviderOnboardingField[]
createProvider: (config: TConfig) => ProviderInstance
@@ -234,6 +234,7 @@ export function resolveTranscriptionFileName(file: File, explicitFileName?: stri
const STREAM_TRANSCRIPTION_EXECUTORS: Record<string, StreamTranscription> = {
'aliyun-nls-transcription': streamTranscription,
'apple-speech-transcription': streamTranscription,
[OFFICIAL_TRANSCRIPTION_PROVIDER_ID]: streamTranscription,
// Web Speech API is handled specially in transcribeForMediaStream since it works directly with MediaStream
}
@@ -1,6 +1,8 @@
import { createPinia, setActivePinia } from 'pinia'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { z } from 'zod'
import { defineProvider } from '../../libs/providers/providers/registry'
import { useProviderStore } from './provider'
vi.mock('vue-i18n', () => ({
@@ -64,4 +66,40 @@ describe('provider store synchronization boundary', () => {
expect.objectContaining({ id: 'auto' }),
])
})
// ROOT CAUSE:
//
// The configured-provider lists read only persisted validation state. A
// local provider whose availability probe is its complete configuration
// never entered the list that Hearing uses for transcription selection.
it('lists an available provider that opts into automatic configuration', async () => {
const providerId = 'test-local-transcription'
defineProvider({
id: providerId,
name: 'Test Local Transcription',
nameLocalize: () => 'Test Local Transcription',
description: 'Local transcription for this test.',
descriptionLocalize: () => 'Local transcription for this test.',
tasks: ['speech-to-text'],
requiresCredentials: false,
autoConfigureWhenAvailable: true,
isAvailableBy: () => true,
createProviderConfig: () => z.object({}),
createProvider: () => ({
transcription: (model: string) => ({
baseURL: 'https://example.invalid/',
model,
}),
}),
})
setActivePinia(createPinia())
const store = useProviderStore()
await vi.waitFor(() => {
expect(store.configuredTranscriptionProvidersMetadata).toContainEqual(
expect.objectContaining({ configured: true, id: providerId }),
)
})
})
})
@@ -771,6 +771,7 @@ export const useProviderStore = defineStore('provider', () => {
function projectProvider(providerId: string): ProviderMetadata | undefined {
const configuredProvider = providerConfigStore.getProvider(providerId)
const definition = findProviderDefinition(providerId)
const metadata = providerMetadata[providerId]
?? providerMetadata[configuredProvider?.definitionId ?? '']
@@ -786,7 +787,7 @@ export const useProviderStore = defineStore('provider', () => {
localizedDescription: metadata.descriptionKey === metadata.description
? metadata.description
: t(metadata.descriptionKey, metadata.description),
configured: configuredProvider?.status === 'configured',
configured: configuredProvider?.status === 'configured' || definition?.autoConfigureWhenAvailable === true,
}
}
@@ -912,19 +913,19 @@ export const useProviderStore = defineStore('provider', () => {
})
const configuredChatProvidersMetadata = computed(() => {
return allChatProvidersMetadata.value.filter(metadata => providerConfigStore.configuredProviders[metadata.id])
return allChatProvidersMetadata.value.filter(metadata => metadata.configured)
})
const configuredSpeechProvidersMetadata = computed(() => {
return allAudioSpeechProvidersMetadata.value.filter(metadata => providerConfigStore.configuredProviders[metadata.id])
return allAudioSpeechProvidersMetadata.value.filter(metadata => metadata.configured)
})
const configuredTranscriptionProvidersMetadata = computed(() => {
return allAudioTranscriptionProvidersMetadata.value.filter(metadata => providerConfigStore.configuredProviders[metadata.id])
return allAudioTranscriptionProvidersMetadata.value.filter(metadata => metadata.configured)
})
const configuredVisionProvidersMetadata = computed(() => {
return allVisionProvidersMetadata.value.filter(metadata => providerConfigStore.configuredProviders[metadata.id])
return allVisionProvidersMetadata.value.filter(metadata => metadata.configured)
})
function isProviderConfigDirty(providerId: string) {
+9
View File
@@ -1860,6 +1860,9 @@ importers:
'@pinia/colada':
specifier: 'catalog:'
version: 1.2.1(pinia@3.0.4(typescript@5.9.3)(vue@3.5.32(typescript@5.9.3)))(vue@3.5.32(typescript@5.9.3))
'@proj-airi/apple-speech-transcription':
specifier: workspace:^
version: link:../../packages/apple-speech-transcription
'@proj-airi/audio':
specifier: workspace:^
version: link:../../packages/audio
@@ -3424,6 +3427,12 @@ importers:
specifier: 'catalog:'
version: 1.6.0
packages/apple-speech-transcription:
dependencies:
'@moeru/std':
specifier: 'catalog:'
version: 0.1.0-beta.17
packages/audio:
dependencies:
'@alexanderolsen/libsamplerate-js':
+3
View File
@@ -5,6 +5,9 @@
"dependsOn": ["^build"],
"outputs": ["dist/**"]
},
"@proj-airi/apple-speech-transcription#build": {
"outputs": ["native/**/*.node"]
},
"@proj-airi/drizzle-migration#build": {
"cache": false
},