|
| 1 | +import { GoogleGenAI } from "@google/genai"; |
| 2 | +import { extensionForMime } from "openclaw/plugin-sdk/msteams"; |
| 3 | +import type { |
| 4 | + GeneratedMusicAsset, |
| 5 | + MusicGenerationProvider, |
| 6 | + MusicGenerationRequest, |
| 7 | +} from "openclaw/plugin-sdk/music-generation"; |
| 8 | +import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth"; |
| 9 | +import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime"; |
| 10 | +import { normalizeGoogleApiBaseUrl } from "./api.js"; |
| 11 | + |
| 12 | +const DEFAULT_GOOGLE_MUSIC_MODEL = "lyria-3-clip-preview"; |
| 13 | +const GOOGLE_PRO_MUSIC_MODEL = "lyria-3-pro-preview"; |
| 14 | +const DEFAULT_TIMEOUT_MS = 180_000; |
| 15 | +const GOOGLE_MAX_INPUT_IMAGES = 10; |
| 16 | + |
| 17 | +type GoogleInlineDataPart = { |
| 18 | + mimeType?: string; |
| 19 | + mime_type?: string; |
| 20 | + data?: string; |
| 21 | +}; |
| 22 | + |
| 23 | +type GoogleGenerateMusicResponse = { |
| 24 | + candidates?: Array<{ |
| 25 | + content?: { |
| 26 | + parts?: Array<{ |
| 27 | + text?: string; |
| 28 | + inlineData?: GoogleInlineDataPart; |
| 29 | + inline_data?: GoogleInlineDataPart; |
| 30 | + }>; |
| 31 | + }; |
| 32 | + }>; |
| 33 | +}; |
| 34 | + |
| 35 | +function resolveConfiguredGoogleMusicBaseUrl(req: MusicGenerationRequest): string | undefined { |
| 36 | + const configured = req.cfg?.models?.providers?.google?.baseUrl?.trim(); |
| 37 | + return configured ? normalizeGoogleApiBaseUrl(configured) : undefined; |
| 38 | +} |
| 39 | + |
| 40 | +function buildMusicPrompt(req: MusicGenerationRequest): string { |
| 41 | + const parts = [req.prompt.trim()]; |
| 42 | + const lyrics = req.lyrics?.trim(); |
| 43 | + if (req.instrumental === true) { |
| 44 | + parts.push("Instrumental only. No vocals, no sung lyrics, no spoken word."); |
| 45 | + } |
| 46 | + if (lyrics) { |
| 47 | + parts.push(`Lyrics:\n${lyrics}`); |
| 48 | + } |
| 49 | + return parts.join("\n\n"); |
| 50 | +} |
| 51 | + |
| 52 | +function resolveSupportedFormats(model: string): readonly string[] { |
| 53 | + return model === GOOGLE_PRO_MUSIC_MODEL ? ["mp3", "wav"] : ["mp3"]; |
| 54 | +} |
| 55 | + |
| 56 | +function resolveTrackFileName(params: { index: number; mimeType: string; model: string }): string { |
| 57 | + const ext = |
| 58 | + extensionForMime(params.mimeType)?.replace(/^\./u, "") || |
| 59 | + (params.model === GOOGLE_PRO_MUSIC_MODEL ? "wav" : "mp3"); |
| 60 | + return `track-${params.index + 1}.${ext}`; |
| 61 | +} |
| 62 | + |
| 63 | +function extractTracks(params: { payload: GoogleGenerateMusicResponse; model: string }): { |
| 64 | + tracks: GeneratedMusicAsset[]; |
| 65 | + lyrics: string[]; |
| 66 | +} { |
| 67 | + const lyrics: string[] = []; |
| 68 | + const tracks: GeneratedMusicAsset[] = []; |
| 69 | + for (const part of params.payload.candidates?.[0]?.content?.parts ?? []) { |
| 70 | + if (part.text?.trim()) { |
| 71 | + lyrics.push(part.text.trim()); |
| 72 | + continue; |
| 73 | + } |
| 74 | + const inline = part.inlineData ?? part.inline_data; |
| 75 | + const data = inline?.data?.trim(); |
| 76 | + if (!data) { |
| 77 | + continue; |
| 78 | + } |
| 79 | + const mimeType = inline?.mimeType?.trim() || inline?.mime_type?.trim() || "audio/mpeg"; |
| 80 | + tracks.push({ |
| 81 | + buffer: Buffer.from(data, "base64"), |
| 82 | + mimeType, |
| 83 | + fileName: resolveTrackFileName({ |
| 84 | + index: tracks.length, |
| 85 | + mimeType, |
| 86 | + model: params.model, |
| 87 | + }), |
| 88 | + }); |
| 89 | + } |
| 90 | + return { tracks, lyrics }; |
| 91 | +} |
| 92 | + |
| 93 | +export function buildGoogleMusicGenerationProvider(): MusicGenerationProvider { |
| 94 | + return { |
| 95 | + id: "google", |
| 96 | + label: "Google", |
| 97 | + defaultModel: DEFAULT_GOOGLE_MUSIC_MODEL, |
| 98 | + models: [DEFAULT_GOOGLE_MUSIC_MODEL, GOOGLE_PRO_MUSIC_MODEL], |
| 99 | + isConfigured: ({ agentDir }) => |
| 100 | + isProviderApiKeyConfigured({ |
| 101 | + provider: "google", |
| 102 | + agentDir, |
| 103 | + }), |
| 104 | + capabilities: { |
| 105 | + maxTracks: 1, |
| 106 | + maxInputImages: GOOGLE_MAX_INPUT_IMAGES, |
| 107 | + supportsLyrics: true, |
| 108 | + supportsInstrumental: true, |
| 109 | + supportsFormat: true, |
| 110 | + supportedFormatsByModel: { |
| 111 | + [DEFAULT_GOOGLE_MUSIC_MODEL]: ["mp3"], |
| 112 | + [GOOGLE_PRO_MUSIC_MODEL]: ["mp3", "wav"], |
| 113 | + }, |
| 114 | + }, |
| 115 | + async generateMusic(req) { |
| 116 | + if ((req.inputImages?.length ?? 0) > GOOGLE_MAX_INPUT_IMAGES) { |
| 117 | + throw new Error( |
| 118 | + `Google music generation supports at most ${GOOGLE_MAX_INPUT_IMAGES} reference images.`, |
| 119 | + ); |
| 120 | + } |
| 121 | + const auth = await resolveApiKeyForProvider({ |
| 122 | + provider: "google", |
| 123 | + cfg: req.cfg, |
| 124 | + agentDir: req.agentDir, |
| 125 | + store: req.authStore, |
| 126 | + }); |
| 127 | + if (!auth.apiKey) { |
| 128 | + throw new Error("Google API key missing"); |
| 129 | + } |
| 130 | + |
| 131 | + const model = req.model?.trim() || DEFAULT_GOOGLE_MUSIC_MODEL; |
| 132 | + if (req.format) { |
| 133 | + const supportedFormats = resolveSupportedFormats(model); |
| 134 | + if (!supportedFormats.includes(req.format)) { |
| 135 | + throw new Error( |
| 136 | + `Google music generation model ${model} supports ${supportedFormats.join(", ")} output.`, |
| 137 | + ); |
| 138 | + } |
| 139 | + } |
| 140 | + |
| 141 | + const client = new GoogleGenAI({ |
| 142 | + apiKey: auth.apiKey, |
| 143 | + httpOptions: { |
| 144 | + ...(resolveConfiguredGoogleMusicBaseUrl(req) |
| 145 | + ? { baseUrl: resolveConfiguredGoogleMusicBaseUrl(req) } |
| 146 | + : {}), |
| 147 | + timeout: req.timeoutMs ?? DEFAULT_TIMEOUT_MS, |
| 148 | + }, |
| 149 | + }); |
| 150 | + const response = (await client.models.generateContent({ |
| 151 | + model, |
| 152 | + contents: [ |
| 153 | + { text: buildMusicPrompt(req) }, |
| 154 | + ...(req.inputImages ?? []).map((image) => ({ |
| 155 | + inlineData: { |
| 156 | + mimeType: image.mimeType?.trim() || "image/png", |
| 157 | + data: image.buffer?.toString("base64") ?? "", |
| 158 | + }, |
| 159 | + })), |
| 160 | + ], |
| 161 | + config: { |
| 162 | + responseModalities: ["AUDIO", "TEXT"], |
| 163 | + }, |
| 164 | + })) as GoogleGenerateMusicResponse; |
| 165 | + |
| 166 | + const { tracks, lyrics } = extractTracks({ |
| 167 | + payload: response, |
| 168 | + model, |
| 169 | + }); |
| 170 | + if (tracks.length === 0) { |
| 171 | + throw new Error("Google music generation response missing audio data"); |
| 172 | + } |
| 173 | + return { |
| 174 | + tracks, |
| 175 | + ...(lyrics.length > 0 ? { lyrics } : {}), |
| 176 | + model, |
| 177 | + metadata: { |
| 178 | + inputImageCount: req.inputImages?.length ?? 0, |
| 179 | + instrumental: req.instrumental === true, |
| 180 | + ...(req.lyrics?.trim() ? { requestedLyrics: true } : {}), |
| 181 | + ...(req.format ? { requestedFormat: req.format } : {}), |
| 182 | + }, |
| 183 | + }; |
| 184 | + }, |
| 185 | + }; |
| 186 | +} |
0 commit comments