feat(app): vad-stt demo

kwaa · kwaa · commit 579f43eb8ebb · 2025-05-23T22:08:29.000+08:00
diff --git a/app/package.json b/app/package.json
@@ -32,6 +32,7 @@
     "@ricky0123/vad-react": "^0.0.30",
     "@xsai/generate-speech": "catalog:xsai",
     "@xsai/generate-text": "catalog:xsai",
+    "@xsai/generate-transcription": "catalog:xsai",
     "@xsai/shared": "catalog:xsai",
     "@xsai/shared-chat": "catalog:xsai",
     "foxact": "catalog:",
diff --git a/app/src/pages/debug/vad-stt.tsx b/app/src/pages/debug/vad-stt.tsx
@@ -0,0 +1,42 @@
+import { Text } from '@react-three/drei'
+import { useMicVAD, utils } from '@ricky0123/vad-react'
+import { generateTranscription } from '@xsai/generate-transcription'
+import { useEffect, useState } from 'react'
+
+import { useSTTProvider } from '~/hooks/use-providers'
+
+const DebugVadStt = () => {
+  const [sttProvider] = useSTTProvider()
+  const [file, setFile] = useState<Blob>()
+  const [text, setText] = useState('')
+
+  useMicVAD({
+    onSpeechEnd: (audio) => {
+      console.warn('onSpeechEnd')
+
+      setFile(new Blob([utils.encodeWAV(audio)], { type: 'audio/wav' }))
+    },
+    onSpeechStart: () => console.warn('onSpeechStart'),
+    startOnLoad: true,
+  })
+
+  useEffect(() => {
+    if (!file)
+      return
+
+    const getTranscription = async () => {
+      const { text } = await generateTranscription({ ...sttProvider, file })
+
+      console.warn('Transcription:', text)
+      setText(text)
+    }
+
+    void getTranscription()
+  }, [file, sttProvider])
+
+  return (
+    <Text>{text}</Text>
+  )
+}
+
+export default DebugVadStt
diff --git a/app/src/router.ts b/app/src/router.ts
@@ -15,6 +15,7 @@ export type Path =
   | `/debug/speech-recognition`
   | `/debug/tablet`
   | `/debug/text`
+  | `/debug/vad-stt`
 
 export type Params = {
   
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
@@ -39,6 +39,7 @@ catalogs:
   xsai:
     '@xsai/generate-speech': &xsai ^0.2.2
     '@xsai/generate-text': *xsai
+    '@xsai/generate-transcription': *xsai
     '@xsai/shared': *xsai
     '@xsai/shared-chat': *xsai
 
