feat: streamable audio

Signed-off-by: Neko Ayaka <neko@ayaka.moe>
2026-08-14 00:48:06 +00:00 · 2024-12-02 00:31:39 +08:00
parent d9ae0aae38
commit d791c8315a
8 changed files with 376 additions and 81 deletions
@@ -0,0 +1,9 @@
+root = true
+
+[*]
+charset = utf-8
+indent_style = space
+indent_size = 2
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
@@ -12,6 +12,7 @@ import Avatar from '../assets/live2d/models/hiyori_free_zh/avatar.png'
 import Live2DViewer from '../components/Live2DViewer.vue'
 import BasicTextarea from '../components/BasicTextarea.vue'
 import { useLLM } from '../stores/llm'
+import { useQueue } from '../composables/queue'
 import AudioWaveform from './AudioWaveform.vue'

 interface Message {
@@ -22,8 +23,8 @@ interface Message {
 const llm = useLLM()
 const { audioContext } = useAudioContext()

-const openAIAPIKey = useLocalStorage('openai-api-key', '')
-const openAIAPIBaseURL = useLocalStorage('openai-api-base-url', 'https://api.openai.com/v1')
+const openAiApiKey = useLocalStorage('openai-api-key', '')
+const openAiApiBaseURL = useLocalStorage('openai-api-base-url', 'https://api.openai.com/v1')
 const openAIModel = useLocalStorage('openai-model', '')

 const mouthOpenSize = ref(0)
@@ -52,6 +53,81 @@ const model = computed<string>({
  },
 })

+const temp = ref<string>('')
+
+const audioQueue = useQueue<{ audioBuffer: AudioBuffer, text: string }>({
+  handlers: [
+    (ctx) => {
+      return new Promise((resolve) => {
+        // Create an AudioBufferSourceNode
+        const source = audioContext.createBufferSource()
+        source.buffer = ctx.data.audioBuffer
+
+        // Connect the source to the AudioContext's destination (the speakers)
+        source.connect(audioContext.destination)
+        // Connect the source to the analyzer
+        source.connect(audioWaveformRef.value!.analyser())
+
+        // Start playing the audio
+        speaking.value = true
+        source.start(0)
+        source.onended = () => {
+          speaking.value = false
+          resolve()
+        }
+      })
+    },
+  ],
+})
+
+const ttsQueue = useQueue<string>({
+  handlers: [
+    async (ctx) => {
+      const audioBuffer = await streamSpeech(ctx.data)
+      audioQueue.add({ audioBuffer, text: ctx.data })
+    },
+  ],
+})
+
+const messageContentQueue = useQueue<string>({
+  handlers: [
+    async (ctx) => {
+      if (ctx.data === '|<llm_inference_end>|') {
+        const content = temp.value.trim()
+        if (content)
+          ttsQueue.add(content)
+
+        temp.value = ''
+        return
+      }
+
+      const endMarker = ['.', '?', '!']
+
+      let newEndPartDiscovered = false
+
+      for (const marker of endMarker) {
+        if (!ctx.data.includes(marker))
+          continue
+
+        // find the end of the sentence and push it to the queue with temp
+        const periodIndex = ctx.data.indexOf(marker)
+        // split
+        const beforePeriod = ctx.data.slice(0, periodIndex + 1)
+        const afterPeriod = ctx.data.slice(periodIndex + 1)
+
+        temp.value += beforePeriod
+        ttsQueue.add(temp.value.trim())
+        temp.value = afterPeriod
+
+        newEndPartDiscovered = true
+      }
+
+      if (!newEndPartDiscovered)
+        temp.value += ctx.data
+    },
+  ],
+})
+
 async function streamSpeech(text: string) {
  const res = await ofetch('/api/v1/llm/voice/text-to-speech', {
    body: {
@@ -63,23 +139,7 @@ async function streamSpeech(text: string) {
  })

  // Decode the ArrayBuffer into an AudioBuffer
-  const audioBuffer = await audioContext.decodeAudioData(res)
-
-  // Create an AudioBufferSourceNode
-  const source = audioContext.createBufferSource()
-  source.buffer = audioBuffer
-
-  // Connect the source to the AudioContext's destination (the speakers)
-  source.connect(audioContext.destination)
-  // Connect the source to the analyzer
-  source.connect(audioWaveformRef.value!.analyser())
-
-  // Start playing the audio
-  speaking.value = true
-  source.start(0)
-  source.onended = () => {
-    speaking.value = false
-  }
+  return await audioContext.decodeAudioData(res)
 }

 function getVolumeWithLinearNormalize() {
@@ -155,12 +215,16 @@ function onSendMessage(sendingMessage: string) {
  messages.value.push({ role: 'user', content: sendingMessage })
  messages.value.push(message)
  const index = messages.value.length - 1
+  const textParts: string[] = []

  llm.stream(model.value, sendingMessage).then(async (res) => {
-    for await (const textPart of res.textStream)
+    for await (const textPart of res.textStream) {
      messages.value[index].content += textPart
+      messageContentQueue.add(textPart)
+      textParts.push(textPart)
+    }

-    await streamSpeech(messages.value[index].content)
+    messageContentQueue.add('|<llm_inference_end>|')
  })

  input.value = ''
@@ -175,20 +239,20 @@ function fromMarkdownToHTML(markdown: string) {
    .toString()
 }

-watch(openAIAPIKey, (value) => {
+watch(openAiApiKey, (value) => {
  llm.setupOpenAI({
    apiKey: value,
-    baseURL: openAIAPIBaseURL.value,
+    baseURL: openAiApiBaseURL.value,
  })
 })

 onMounted(async () => {
-  if (!openAIAPIKey.value)
+  if (!openAiApiKey.value)
    return

  llm.setupOpenAI({
-    apiKey: openAIAPIKey.value,
-    baseURL: openAIAPIBaseURL.value,
+    apiKey: openAiApiKey.value,
+    baseURL: openAiApiBaseURL.value,
  })

  const fetchedModels = await llm.models()
@@ -205,14 +269,14 @@ onUnmounted(() => {
    <div space-x="2" flex="~ row" w-full>
      <div flex="~ row" w-full>
        <input
-          v-model="openAIAPIKey"
+          v-model="openAiApiKey"
          placeholder="Input your API key"
          p="2" bg="zinc-100 dark:zinc-800" w-full rounded-lg outline-none
        >
      </div>
      <div flex="~ row" w-full>
        <input
-          v-model="openAIAPIBaseURL"
+          v-model="openAiApiBaseURL"
          placeholder="Input your API base URL"
          p="2" bg="zinc-100 dark:zinc-800" w-full rounded-lg outline-none
        >
@@ -220,11 +284,11 @@ onUnmounted(() => {
    </div>
    <div flex="~ row 1" w-full items-end space-x-2>
      <div w-full>
+        <Live2DViewer :mouth-open-size="mouthOpenSize" />
        <div>
+          <input v-model.number="mouthOpenSize" type="range" max="1" min="0" step="0.01">
          <span>{{ mouthOpenSize }}</span>
        </div>
-        <Live2DViewer :mouth-open-size="mouthOpenSize" />
-        <input v-model.number="mouthOpenSize" type="range" max="1" min="0" step="0.01">
        <AudioWaveform ref="audioWaveformRef" />
      </div>
      <div my="2" w-full space-y-2>
@@ -1,16 +0,0 @@
-export function useCount() {
-  const count = useState('count', () => Math.round(Math.random() * 20))
-
-  function inc() {
-    count.value += 1
-  }
-  function dec() {
-    count.value -= 1
-  }
-
-  return {
-    count,
-    inc,
-    dec,
-  }
-}
@@ -0,0 +1,94 @@
+import { ref } from 'vue'
+import type { Ref } from 'vue'
+
+export interface HandlerContext<T> {
+  data: T
+  itemsToBeProcessed: () => number
+}
+
+interface Events<T> {
+  add: Array<(payload: T) => void>
+  pick: Array<(payload: T) => void>
+  processing: Array<(payload: T, handler: (param: HandlerContext<T>) => Promise<any>) => void>
+  error: Array<(payload: T, error: Error, handler: (param: HandlerContext<T>) => Promise<any>) => void>
+  processed: Array<<R>(payload: T, result: R, handler: (param: HandlerContext<T>) => Promise<any>) => void>
+  done: Array<(payload: T) => void>
+}
+
+export function useQueue<T>(options: {
+  handlers: Array<(param: {
+    data: T
+    itemsToBeProcessed: () => number
+  }) => Promise<void>>
+}) {
+  const queue = ref<T[]>([]) as Ref<T[]>
+  const isProcessing = ref(false)
+  const internalEventHandler: Events<T> = {
+    add: [],
+    pick: [],
+    processing: [],
+    error: [],
+    processed: [],
+    done: [],
+  }
+
+  function on<E extends keyof Events<T>>(eventName: E, handler: Events<T>[E][number]) {
+    internalEventHandler[eventName].push(handler as any)
+  }
+
+  function emit<E extends keyof Events<T>>(eventName: E, ...params: Parameters<Events<T>[E][number]>) {
+    const handlers = internalEventHandler[eventName] as Events<T>[E]
+    handlers.forEach((handler) => {
+      (handler as any)(...params)
+    })
+  }
+
+  function add(payload: T) {
+    queue.value.push(payload)
+    emit('add', payload)
+  }
+
+  function pick() {
+    const payload = queue.value.shift()
+    if (!payload)
+      return
+
+    emit('pick', payload)
+    return payload
+  }
+
+  async function handleItem() {
+    if (isProcessing.value)
+      return
+
+    const payload = pick()
+    if (!payload)
+      return
+
+    isProcessing.value = true
+
+    for (const handler of options.handlers) {
+      emit('processing', payload, handler)
+      try {
+        const result = await handler({ data: payload, itemsToBeProcessed: () => queue.value.length })
+        emit('processed', payload, result, handler)
+      }
+      catch (err) {
+        emit('error', payload, err as Error, handler)
+        continue
+      }
+    }
+
+    isProcessing.value = false
+    emit('done', payload)
+  }
+
+  on('add', handleItem)
+  on('done', handleItem)
+
+  return {
+    add,
+    on,
+    queue,
+  }
+}
@@ -1,34 +0,0 @@
-import { acceptHMRUpdate, defineStore } from 'pinia'
-
-export const useUserStore = defineStore('user', () => {
-  /**
-   * Current named of the user.
-   */
-  const savedName = ref('')
-  const previousNames = ref(new Set<string>())
-
-  const usedNames = computed(() => Array.from(previousNames.value))
-  const otherNames = computed(() => usedNames.value.filter(name => name !== savedName.value))
-
-  /**
-   * Changes the current name of the user and saves the one that was used
-   * before.
-   *
-   * @param name - new name to set
-   */
-  function setNewName(name: string) {
-    if (savedName.value)
-      previousNames.value.add(savedName.value)
-
-    savedName.value = name
-  }
-
-  return {
-    setNewName,
-    otherNames,
-    savedName,
-  }
-})
-
-if (import.meta.hot)
-  import.meta.hot.accept(acceptHMRUpdate(useUserStore, import.meta.hot))
@@ -0,0 +1,15 @@
+version: "0.2"
+ignorePaths: []
+dictionaryDefinitions: []
+dictionaries: []
+words:
+  - composables
+  - hiyori
+  - Neuro
+  - ofetch
+  - openai
+  - pixi
+  - rehype
+  - vueuse
+ignoreWords: []
+import: []
@@ -0,0 +1,161 @@
+<script setup lang="ts">
+import { onMounted, ref, watch } from 'vue'
+import { useQueue } from '../composables/queue'
+
+function sleep(ms: number): Promise<void> {
+  return new Promise(resolve => setTimeout(resolve, ms))
+}
+
+const temp = ref<string>('')
+
+const audioQueue = useQueue<string>({
+  handlers: [
+    async (text) => {
+      // eslint-disable-next-line no-console
+      console.log('ready to play speech audio for', text)
+    },
+  ],
+})
+const ttsQueue = useQueue<string>({
+  handlers: [
+    async (ctx) => {
+      // eslint-disable-next-line no-console
+      console.log('ready to stream speech audio for', ctx)
+      audioQueue.add(ctx.data)
+    },
+  ],
+})
+const textQueue = useQueue<string>({
+  handlers: [
+    async (ctx) => {
+      const endMarker = ['.', '?', '!']
+
+      let newEndPartDiscovered = false
+
+      for (const marker of endMarker) {
+        if (!ctx.data.includes(marker))
+          continue
+
+        // find the end of the sentence and push it to the queue with temp
+        const periodIndex = ctx.data.indexOf(marker)
+        // split
+        const beforePeriod = ctx.data.slice(0, periodIndex + 1)
+        const afterPeriod = ctx.data.slice(periodIndex + 1)
+
+        temp.value += beforePeriod
+        ttsQueue.add(temp.value.trim())
+        temp.value = afterPeriod
+
+        newEndPartDiscovered = true
+      }
+
+      if (!newEndPartDiscovered)
+        temp.value += ctx.data
+    },
+  ],
+})
+
+const textParts = [
+  'Hello',
+  ' N',
+  'eko',
+  '! I',
+  ' am',
+  ' an',
+  ' AI',
+  ' assistant',
+  ' trained',
+  ' to',
+  ' help',
+  ' with',
+  ' a',
+  ' variety',
+  ' of',
+  ' tasks',
+  ' such',
+  ' as',
+  ' answering',
+  ' questions',
+  ',',
+  ' providing',
+  ' information',
+  ',',
+  ' giving',
+  ' recommendations',
+  ',',
+  ' and',
+  ' more',
+  '. How',
+  ' can',
+  ' I',
+  ' assist',
+  ' you',
+  ' today',
+  '?',
+  'Hello',
+  ' N',
+  'eko',
+  ',',
+  ' I',
+  ' am',
+  ' an',
+  ' AI',
+  ' assistant',
+  '.',
+  ' I',
+  ' can',
+  ' help',
+  ' answer',
+  ' questions',
+  ',',
+  ' provide',
+  ' information',
+  ',',
+  ' assist',
+  ' with',
+  ' tasks',
+  ',',
+  ' and',
+  ' engage',
+  ' in',
+  ' conversation',
+  '.',
+  ' How',
+  ' can',
+  ' I',
+  ' assist',
+  ' you',
+  ' today',
+  '?',
+]
+
+async function mockTextPartsStreamHandler() {
+  for (const part of textParts) {
+    await sleep(100)
+    textQueue.add(part)
+  }
+}
+
+async function handler() {
+  mockTextPartsStreamHandler()
+}
+
+onMounted(() => {
+  handler()
+})
+</script>
+
+<template>
+  <div>
+    <Suspense>
+      <ClientOnly>
+        <div />
+      </ClientOnly>
+      <template #fallback>
+        <div italic op50>
+          <span animate-pulse>Loading...</span>
+        </div>
+      </template>
+    </Suspense>
+  </div>
+</template>
@@ -7,7 +7,9 @@ export default defineEventHandler(async (event) => {
  })

  const res = await client.generate({
-    voice: 'Beatrice',
+    voice: 'Myriam',
+    // Beatrice is not 'childish' like the others
+    // voice: 'Beatrice',
    text: body.text,
    stream: true,
    model_id: 'eleven_multilingual_v2',