Skip to content

Commit 28b8627

Browse files
kwaakwaa
committed
feat: lip-sync
1 parent ca934a7 commit 28b8627

File tree

10 files changed

+108
-30
lines changed

10 files changed

+108
-30
lines changed

‎app/package.json‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@
1616
"@n3p6/react-three-yuka": "workspace:",
1717
"@n3p6/use-illuminance": "workspace:",
1818
"@n3p6/use-speech-recognition": "workspace:",
19-
"@pixiv/three-vrm": "catalog:three",
20-
"@pixiv/three-vrm-animation": "catalog:three",
19+
"@pixiv/three-vrm": "catalog:vrm",
20+
"@pixiv/three-vrm-animation": "catalog:vrm",
2121
"@pmndrs/xr": "catalog:three",
2222
"@react-three/drei": "catalog:three",
2323
"@react-three/fiber": "catalog:three",
@@ -40,6 +40,7 @@
4040
"swr": "^2.3.3",
4141
"three": "catalog:three",
4242
"three-stdlib": "catalog:three",
43+
"wlipsync": "catalog:vrm",
4344
"yuka": "catalog:yuka",
4445
"zustand": "^5.0.4"
4546
},

‎app/src/assets/lip-sync/profile.json‎

Lines changed: 1 addition & 0 deletions
Large diffs are not rendered by default.

‎app/src/components/vrm/galatea-tts.tsx‎

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,27 @@
1+
import type { VRM } from '@pixiv/three-vrm'
12
import type { PositionalAudio } from 'three'
23

34
import { useThree } from '@react-three/fiber'
45
import { useEffect, useMemo, useRef } from 'react'
56
import { AudioListener } from 'three'
67

78
import { useAudioBuffer } from '~/context/audio-buffer'
9+
import { useAudioContext } from '~/context/audio-context'
10+
import { useLipSync } from '~/hooks/use-lip-sync'
811

9-
export const GalateaTTS = () => {
12+
export const GalateaTTS = ({ vrm }: { vrm: VRM }) => {
1013
const { camera } = useThree()
1114
const sound = useRef<PositionalAudio>(null)
1215
const listener = useMemo(() => new AudioListener(), [])
1316
const audioBuffer = useAudioBuffer()
17+
const audioContext = useAudioContext()
18+
const audioBufferSource = useMemo(() => {
19+
const audioBufferSource = audioContext.createBufferSource()
20+
audioBufferSource.buffer = audioBuffer ?? null
21+
return audioBufferSource
22+
}, [audioContext, audioBuffer])
23+
24+
useLipSync(audioBufferSource, vrm)
1425

1526
useEffect(() => {
1627
const _sound = sound.current
@@ -20,16 +31,21 @@ export const GalateaTTS = () => {
2031
// _sound.setRefDistance(1)
2132
_sound.setLoop(false)
2233
_sound.play()
34+
audioBufferSource.start()
2335
}
2436

2537
return () => {
26-
if (!_sound)
27-
return
28-
29-
_sound.stop()
30-
_sound.clear()
38+
try {
39+
audioBufferSource.stop()
40+
}
41+
catch {}
42+
43+
if (_sound) {
44+
_sound.stop()
45+
_sound.clear()
46+
}
3147
}
32-
}, [sound, audioBuffer])
48+
}, [sound, audioBuffer, audioBufferSource])
3349

3450
useEffect(() => {
3551
camera.add(listener)

‎app/src/components/vrm/galatea.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ export const Galatea = () => {
7272
rotation={[0, Math.PI, 0]}
7373
scale={1.05}
7474
/>
75-
<GalateaTTS />
75+
<GalateaTTS vrm={galateaVRM} />
7676
</group>
7777
<group ref={playerRef}></group>
7878
</>

‎app/src/hooks/use-lip-sync.ts‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
import type { VRM } from '@pixiv/three-vrm'
2+
import type { Profile } from 'wlipsync'
3+
4+
import { useFrame } from '@react-three/fiber'
5+
import { useEffect } from 'react'
6+
import useSWR from 'swr'
7+
import { createWLipSyncNode } from 'wlipsync'
8+
9+
import profile from '~/assets/lip-sync/profile.json' with { type: 'json' }
10+
import { useAudioContext } from '~/context/audio-context'
11+
12+
// https://github.com/mrxz/wLipSync/blob/c3bc4b321dc7e1ca333d75f7aa1e9e746cbbb23a/example/index.js#L50-L66
13+
const lipSyncMap = {
14+
A: 'aa',
15+
E: 'ee',
16+
I: 'ih',
17+
O: 'oh',
18+
U: 'ou',
19+
}
20+
21+
export const useLipSync = (audioBufferSource: AudioBufferSourceNode, vrm: VRM) => {
22+
const audioContext = useAudioContext()
23+
24+
const { data: lipSyncNode } = useSWR('wlipsync/createWLipSyncNode', async () => createWLipSyncNode(audioContext, profile as Profile))
25+
26+
useEffect(() => {
27+
if (lipSyncNode)
28+
audioBufferSource.connect(lipSyncNode)
29+
30+
return () => {
31+
audioBufferSource.disconnect()
32+
}
33+
}, [audioBufferSource, lipSyncNode])
34+
35+
useFrame(() => {
36+
if (lipSyncNode) {
37+
for (const key of Object.keys(lipSyncNode.weights)) {
38+
const weight = lipSyncNode.weights[key] * lipSyncNode.volume
39+
vrm.expressionManager?.setValue(lipSyncMap[key as keyof typeof lipSyncMap], weight)
40+
}
41+
}
42+
})
43+
}

‎app/vite.config.ts‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,12 @@ import tsconfigPaths from 'vite-tsconfig-paths'
77
export default defineConfig(({ mode }) => ({
88
assetsInclude: ['./assets/*'],
99
build: { target: 'esnext' },
10+
optimizeDeps: {
11+
esbuildOptions: {
12+
target: 'esnext',
13+
},
14+
// exclude: ['sqlocal'],
15+
},
1016
plugins: [
1117
react({
1218
babel: { plugins: [
@@ -17,15 +23,10 @@ export default defineConfig(({ mode }) => ({
1723
tsconfigPaths(),
1824
],
1925
publicDir: mode === 'development' ? 'public' : false,
20-
// optimizeDeps: {
21-
// esbuildOptions: {
22-
// target: 'esnext',
23-
// },
24-
// exclude: ['sqlocal'],
25-
// },
2626
resolve: {
2727
dedupe: ['react', 'three'],
2828
},
29+
rollupOptions: { target: 'esnext' },
2930
// server: {
3031
// headers: {
3132
// 'Cross-Origin-Embedder-Policy': 'require-corp',

‎cspell.config.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ words:
4242
- verts
4343
- vrma
4444
- vrmc
45+
- wlipsync
4546
- Xiaoyi
4647
- xsai
4748
- yuka

‎packages/react-three-vrm/package.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,8 @@
3535
"build": "pkgroll"
3636
},
3737
"peerDependencies": {
38-
"@pixiv/three-vrm": "catalog:three",
39-
"@pixiv/three-vrm-animation": "catalog:three",
38+
"@pixiv/three-vrm": "catalog:vrm",
39+
"@pixiv/three-vrm-animation": "catalog:vrm",
4040
"@react-three/drei": "catalog:three",
4141
"@react-three/fiber": "catalog:three",
4242
"@types/react": "catalog:react",

‎pnpm-lock.yaml‎

Lines changed: 22 additions & 10 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pnpm-workspace.yaml‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,6 @@ catalogs:
1818
react-router: ^7.5.3
1919

2020
three:
21-
'@pixiv/three-vrm': ^3.4.0
22-
'@pixiv/three-vrm-animation': ^3.4.0
2321
'@pmndrs/xr': &xr ^6.6.16
2422
'@react-three/drei': ^10.0.7
2523
'@react-three/fiber': ^9.1.2
@@ -33,6 +31,11 @@ catalogs:
3331
three: ^0.176.0
3432
three-stdlib: ^2.36.0
3533

34+
vrm:
35+
'@pixiv/three-vrm': &three-vrm ^3.4.0
36+
'@pixiv/three-vrm-animation': *three-vrm
37+
wlipsync: ^1.2.0
38+
3639
xsai:
3740
'@xsai/generate-speech': &xsai ^0.2.2
3841
'@xsai/generate-text': *xsai

0 commit comments

Comments
 (0)