feat(app): positional-audio demo

kwaa · kwaa · commit a35df7df31b8 · 2025-05-16T16:47:42.000+08:00
diff --git a/app/package.json b/app/package.json
@@ -29,6 +29,7 @@
     "@react-three/xr": "catalog:three",
     "@recast-navigation/core": "catalog:yuka",
     "@recast-navigation/three": "catalog:yuka",
+    "@xsai/generate-speech": "catalog:xsai",
     "@xsai/generate-text": "catalog:xsai",
     "@xsai/shared": "catalog:xsai",
     "@xsai/shared-chat": "catalog:xsai",
diff --git a/app/src/pages/debug/positional-audio.tsx b/app/src/pages/debug/positional-audio.tsx
@@ -0,0 +1,63 @@
+import { Container, Root, Text } from '@react-three/uikit'
+import { Button, Input } from '@react-three/uikit-default'
+import { generateSpeech } from '@xsai/generate-speech'
+import { useMemo, useState } from 'react'
+
+import { Galatea } from '~/components/vrm/galatea'
+import { useSetAudioBuffer } from '~/context/audio-buffer'
+import { useTTSProvider } from '~/hooks/use-providers'
+
+const DebugPositionalAudio = () => {
+  const [disabled, setDisabled] = useState(false)
+  const [value, setValue] = useState<string>('')
+  const [ttsProvider] = useTTSProvider()
+  const setAudioBuffer = useSetAudioBuffer()
+
+  const audioContext = useMemo(() => new AudioContext({ latencyHint: 'interactive' }), [])
+
+  const handleSubmit = async () => {
+    setDisabled(true)
+
+    const arrayBuffer = await generateSpeech({
+      ...ttsProvider,
+      input: value,
+    })
+
+    const audioBuffer = await audioContext.decodeAudioData(arrayBuffer)
+
+    setAudioBuffer(audioBuffer)
+    setValue('')
+    setDisabled(false)
+  }
+
+  return (
+    <>
+      <Galatea />
+      <group position={[0, 1, 0]}>
+        <Root>
+          <Container flexDirection="column" gap={4}>
+            <Input
+              data-test-id="debug-input"
+              disabled={disabled}
+              onValueChange={value => setValue(value)}
+              placeholder="Write a message..."
+              value={value}
+              width={200}
+            />
+            <Button
+              data-test-id="debug-button"
+              disabled={disabled}
+              // eslint-disable-next-line ts/no-misused-promises
+              onClick={handleSubmit}
+              variant="secondary"
+            >
+              <Text>Submit</Text>
+            </Button>
+          </Container>
+        </Root>
+      </group>
+    </>
+  )
+}
+
+export default DebugPositionalAudio
diff --git a/app/src/router.ts b/app/src/router.ts
@@ -10,6 +10,7 @@ export type Path =
   | `/debug/input`
   | `/debug/menu`
   | `/debug/planes`
+  | `/debug/positional-audio`
   | `/debug/settings`
   | `/debug/speech-recognition`
   | `/debug/tablet`
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
@@ -34,7 +34,8 @@ catalogs:
     three-stdlib: ^2.36.0
 
   xsai:
-    '@xsai/generate-text': &xsai ^0.2.2
+    '@xsai/generate-speech': &xsai ^0.2.2
+    '@xsai/generate-text': *xsai
     '@xsai/shared': *xsai
     '@xsai/shared-chat': *xsai
 
