refactor: reuse realtime output activity in google meet (#86665)

steipete · web-flow · commit bef0ba8f5a7e · 2026-05-26T00:19:35.000+01:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,7 @@ Docs: https://docs.openclaw.ai
 ### Changes
 
 - Voice: expose shared realtime turn-context tracking through the realtime voice SDK and reuse it for Discord speaker attribution and wake-name context recovery.
+- Voice: reuse shared realtime output activity tracking in Google Meet command and node audio bridges, including recent-output checks for local barge-in detection.
 - Voice: expose shared realtime output activity tracking through the realtime voice SDK and reuse it for Discord playback activity and barge-in decisions.
 - Voice: expose shared realtime consult question matching, speakable-result extraction, and alias-aware forced-consult coordination through the realtime voice SDK, then reuse it in Gateway Talk, Voice Call, and Discord voice paths.
 - Voice: share activation-name matching and consult-transcript screening through the realtime voice SDK so Discord, browser voice, and meeting surfaces can reuse one implementation.
diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256
@@ -1,2 +1,2 @@
-8712c831d993980a72d7a628cd235654a5e7dcf74edda0c9f0c9c9c5ba250afc  plugin-sdk-api-baseline.json
-f6bf4178e5429f64943f5404f3085c360a7f390c15d15b5b9c23239c7a134ca6  plugin-sdk-api-baseline.jsonl
+5ea1c7850cd69d5cfb6817148ffff622a52c6e0a2306e0ae71b6f451ad54ac2c  plugin-sdk-api-baseline.json
+df2ca60d91db5c5b0225286938a175c3b56feb3190b613a6524864471efa4588  plugin-sdk-api-baseline.jsonl
diff --git a/extensions/google-meet/src/realtime-node.ts b/extensions/google-meet/src/realtime-node.ts
@@ -9,6 +9,7 @@ import {
   createRealtimeVoiceAgentTalkbackQueue,
   createTalkSessionController,
   createRealtimeVoiceBridgeSession,
+  createRealtimeVoiceOutputActivityTracker,
   recordTalkObservabilityEvent,
   type RealtimeVoiceAgentTalkbackQueue,
   type RealtimeVoiceBridgeSession,
@@ -27,7 +28,7 @@ import {
   getGoogleMeetRealtimeTranscriptHealth,
   buildGoogleMeetSpeakExactUserMessage,
   GOOGLE_MEET_AGENT_TRANSCRIPT_DEBOUNCE_MS,
-  extendGoogleMeetOutputEchoSuppression,
+  recordGoogleMeetOutputActivity,
   getGoogleMeetRealtimeEventHealth,
   recordGoogleMeetRealtimeTranscript,
   recordGoogleMeetRealtimeEvent,
@@ -97,7 +98,7 @@ export async function startNodeAgentAudioBridge(params: {
   let lastInputAt: string | undefined;
   let lastOutputAt: string | undefined;
   let lastInputBytes = 0;
-  let lastOutputBytes = 0;
+  const outputActivity = createRealtimeVoiceOutputActivityTracker();
   let suppressedInputBytes = 0;
   let lastSuppressedInputAt: string | undefined;
   let suppressInputUntil = 0;
@@ -148,7 +149,8 @@ export async function startNodeAgentAudioBridge(params: {
   };
 
   const pushOutputAudio = async (audio: Buffer) => {
-    const suppression = extendGoogleMeetOutputEchoSuppression({
+    const suppression = recordGoogleMeetOutputActivity({
+      tracker: outputActivity,
       audio,
       audioFormat: params.config.chrome.audioFormat,
       nowMs: Date.now(),
@@ -158,7 +160,6 @@ export async function startNodeAgentAudioBridge(params: {
     suppressInputUntil = suppression.suppressInputUntilMs;
     lastOutputPlayableUntilMs = suppression.lastOutputPlayableUntilMs;
     lastOutputAt = new Date().toISOString();
-    lastOutputBytes += audio.byteLength;
     await params.runtime.nodes.invoke({
       nodeId: params.nodeId,
       command: "googlemeet.chrome",
@@ -317,12 +318,12 @@ export async function startNodeAgentAudioBridge(params: {
       providerConnected: sttSession?.isConnected() ?? false,
       realtimeReady,
       audioInputActive: lastInputBytes > 0,
-      audioOutputActive: lastOutputBytes > 0,
+      audioOutputActive: outputActivity.isActive(),
       lastInputAt,
       lastOutputAt,
       lastSuppressedInputAt,
       lastInputBytes,
-      lastOutputBytes,
+      lastOutputBytes: outputActivity.snapshot().sinkAudioBytes,
       suppressedInputBytes,
       ...getGoogleMeetRealtimeTranscriptHealth(transcript),
       consecutiveInputErrors,
@@ -351,7 +352,7 @@ export async function startNodeRealtimeAudioBridge(params: {
   let lastOutputAt: string | undefined;
   let lastClearAt: string | undefined;
   let lastInputBytes = 0;
-  let lastOutputBytes = 0;
+  const outputActivity = createRealtimeVoiceOutputActivityTracker();
   let suppressedInputBytes = 0;
   let lastSuppressedInputAt: string | undefined;
   let suppressInputUntil = 0;
@@ -505,7 +506,8 @@ export async function startNodeRealtimeAudioBridge(params: {
           turnId,
           payload: { byteLength: audio.byteLength },
         });
-        const suppression = extendGoogleMeetOutputEchoSuppression({
+        const suppression = recordGoogleMeetOutputActivity({
+          tracker: outputActivity,
           audio,
           audioFormat: params.config.chrome.audioFormat,
           nowMs: Date.now(),
@@ -515,7 +517,6 @@ export async function startNodeRealtimeAudioBridge(params: {
         suppressInputUntil = suppression.suppressInputUntilMs;
         lastOutputPlayableUntilMs = suppression.lastOutputPlayableUntilMs;
         lastOutputAt = new Date().toISOString();
-        lastOutputBytes += audio.byteLength;
         void params.runtime.nodes
           .invoke({
             nodeId: params.nodeId,
@@ -759,13 +760,13 @@ export async function startNodeRealtimeAudioBridge(params: {
       providerConnected: bridge?.bridge.isConnected() ?? false,
       realtimeReady,
       audioInputActive: lastInputBytes > 0,
-      audioOutputActive: lastOutputBytes > 0,
+      audioOutputActive: outputActivity.isActive(),
       lastInputAt,
       lastOutputAt,
       lastSuppressedInputAt,
       lastClearAt,
       lastInputBytes,
-      lastOutputBytes,
+      lastOutputBytes: outputActivity.snapshot().sinkAudioBytes,
       suppressedInputBytes,
       ...getGoogleMeetRealtimeTranscriptHealth(transcript),
       ...getGoogleMeetRealtimeEventHealth(realtimeEvents),
diff --git a/extensions/google-meet/src/realtime.ts b/extensions/google-meet/src/realtime.ts
@@ -13,6 +13,7 @@ import {
 import {
   createRealtimeVoiceAgentTalkbackQueue,
   createRealtimeVoiceBridgeSession,
+  createRealtimeVoiceOutputActivityTracker,
   createTalkSessionController,
   convertPcmToMulaw8k,
   extendRealtimeVoiceOutputEchoSuppression,
@@ -30,6 +31,7 @@ import {
   type RealtimeVoiceAgentTalkbackQueue,
   type RealtimeVoiceBridgeEventLogEntry,
   type RealtimeVoiceBridgeSession,
+  type RealtimeVoiceOutputActivityTracker,
   type RealtimeVoiceProviderConfig,
   type RealtimeVoiceProviderPlugin,
   type RealtimeVoiceTranscriptEntry,
@@ -163,6 +165,24 @@ export function extendGoogleMeetOutputEchoSuppression(params: {
   });
 }
 
+export function recordGoogleMeetOutputActivity(params: {
+  tracker: RealtimeVoiceOutputActivityTracker;
+  audio: Buffer;
+  audioFormat: GoogleMeetConfig["chrome"]["audioFormat"];
+  nowMs: number;
+  lastOutputPlayableUntilMs: number;
+  suppressInputUntilMs: number;
+}): { lastOutputPlayableUntilMs: number; suppressInputUntilMs: number; durationMs: number } {
+  const suppression = extendGoogleMeetOutputEchoSuppression(params);
+  params.tracker.markPlaybackStarted();
+  params.tracker.markAudio({
+    audioMs: suppression.durationMs,
+    sourceAudioBytes: params.audio.byteLength,
+    sinkAudioBytes: params.audio.byteLength,
+  });
+  return suppression;
+}
+
 export function resolveGoogleMeetRealtimeAudioFormat(config: GoogleMeetConfig) {
   return config.chrome.audioFormat === "g711-ulaw-8khz"
     ? REALTIME_VOICE_AUDIO_FORMAT_G711_ULAW_8KHZ
@@ -477,7 +497,7 @@ export async function startCommandAgentAudioBridge(params: {
   let lastInputAt: string | undefined;
   let lastOutputAt: string | undefined;
   let lastInputBytes = 0;
-  let lastOutputBytes = 0;
+  const outputActivity = createRealtimeVoiceOutputActivityTracker();
   let suppressedInputBytes = 0;
   let lastSuppressedInputAt: string | undefined;
   let suppressInputUntil = 0;
@@ -606,7 +626,8 @@ export async function startCommandAgentAudioBridge(params: {
   });
 
   const writeOutputAudio = (audio: Buffer) => {
-    const suppression = extendGoogleMeetOutputEchoSuppression({
+    const suppression = recordGoogleMeetOutputActivity({
+      tracker: outputActivity,
       audio,
       audioFormat: params.config.chrome.audioFormat,
       nowMs: Date.now(),
@@ -616,7 +637,6 @@ export async function startCommandAgentAudioBridge(params: {
     suppressInputUntil = suppression.suppressInputUntilMs;
     lastOutputPlayableUntilMs = suppression.lastOutputPlayableUntilMs;
     lastOutputAt = new Date().toISOString();
-    lastOutputBytes += audio.byteLength;
     emitTalkEvent({
       type: "output.audio.delta",
       turnId: ensureTalkTurn(),
@@ -787,12 +807,12 @@ export async function startCommandAgentAudioBridge(params: {
       providerConnected: sttSession?.isConnected() ?? false,
       realtimeReady,
       audioInputActive: lastInputBytes > 0,
-      audioOutputActive: lastOutputBytes > 0,
+      audioOutputActive: outputActivity.isActive(),
       lastInputAt,
       lastOutputAt,
       lastSuppressedInputAt,
       lastInputBytes,
-      lastOutputBytes,
+      lastOutputBytes: outputActivity.snapshot().sinkAudioBytes,
       suppressedInputBytes,
       ...getGoogleMeetRealtimeTranscriptHealth(transcript),
       recentTalkEvents: summarizeGoogleMeetTalkEvents(recentTalkEvents),
@@ -833,19 +853,19 @@ export async function startCommandRealtimeAudioBridge(params: {
   let lastInputAt: string | undefined;
   let lastOutputAt: string | undefined;
   let lastInputBytes = 0;
-  let lastOutputBytes = 0;
+  const outputActivity = createRealtimeVoiceOutputActivityTracker();
   let lastClearAt: string | undefined;
   let clearCount = 0;
   let suppressedInputBytes = 0;
   let lastSuppressedInputAt: string | undefined;
   let suppressInputUntil = 0;
-  let lastOutputAtMs = 0;
   let lastOutputPlayableUntilMs = 0;
   let bargeInInputProcess: BridgeProcess | undefined;
   let agentTalkback: RealtimeVoiceAgentTalkbackQueue | undefined;
 
   const suppressInputForOutput = (audio: Buffer) => {
-    const suppression = extendGoogleMeetOutputEchoSuppression({
+    const suppression = recordGoogleMeetOutputActivity({
+      tracker: outputActivity,
       audio,
       audioFormat: params.config.chrome.audioFormat,
       nowMs: Date.now(),
@@ -970,12 +990,13 @@ export async function startCommandRealtimeAudioBridge(params: {
       stdio: ["ignore", "pipe", "pipe"],
     });
     bargeInInputProcess.stdout?.on("data", (chunk) => {
-      if (stopped || lastOutputAtMs === 0) {
+      if (stopped || !outputActivity.isInterruptible()) {
         return;
       }
       const now = Date.now();
       const playbackActive = now <= Math.max(lastOutputPlayableUntilMs, suppressInputUntil);
-      if (!playbackActive && now - lastOutputAtMs > 1000) {
+      const lastOutputAudioAt = outputActivity.snapshot().lastAudioAt;
+      if (!playbackActive && (lastOutputAudioAt === undefined || now - lastOutputAudioAt > 1_000)) {
         return;
       }
       if (now - lastBargeInAt < params.config.chrome.bargeInCooldownMs) {
@@ -1141,9 +1162,7 @@ export async function startCommandRealtimeAudioBridge(params: {
           turnId,
           payload: { byteLength: audio.byteLength },
         });
-        lastOutputAtMs = Date.now();
         lastOutputAt = new Date().toISOString();
-        lastOutputBytes += audio.byteLength;
         suppressInputForOutput(audio);
         writeOutputAudio(audio);
       },
@@ -1315,12 +1334,12 @@ export async function startCommandRealtimeAudioBridge(params: {
       providerConnected: bridge?.bridge.isConnected() ?? false,
       realtimeReady,
       audioInputActive: lastInputBytes > 0,
-      audioOutputActive: lastOutputBytes > 0,
+      audioOutputActive: outputActivity.isActive(),
       lastInputAt,
       lastOutputAt,
       lastSuppressedInputAt,
       lastInputBytes,
-      lastOutputBytes,
+      lastOutputBytes: outputActivity.snapshot().sinkAudioBytes,
       suppressedInputBytes,
       ...getGoogleMeetRealtimeTranscriptHealth(transcript),
       ...getGoogleMeetRealtimeEventHealth(realtimeEvents),
diff --git a/src/talk/output-activity-tracker.test.ts b/src/talk/output-activity-tracker.test.ts
@@ -7,6 +7,7 @@ describe("realtime voice output activity tracker", () => {
 
     expect(tracker.isActive(false)).toBe(false);
     expect(tracker.isInterruptible(false)).toBe(false);
+    expect(tracker.snapshot().lastAudioAt).toBeUndefined();
 
     tracker.markAudio({ audioMs: 10, sourceAudioBytes: 480, sinkAudioBytes: 1_920 });
 
@@ -17,6 +18,7 @@ describe("realtime voice output activity tracker", () => {
       chunks: 1,
       sourceAudioBytes: 480,
       sinkAudioBytes: 1_920,
+      lastAudioAt: expect.any(Number),
     });
   });
 
@@ -43,6 +45,7 @@ describe("realtime voice output activity tracker", () => {
       playbackStarted: true,
       playbackStartedAt: 1_000,
       streamEnding: true,
+      lastAudioAt: 1_000,
     });
   });
 
diff --git a/src/talk/output-activity-tracker.ts b/src/talk/output-activity-tracker.ts
@@ -15,6 +15,7 @@ export type RealtimeVoiceOutputActivitySnapshot = {
   sinkAudioBytes: number;
   playbackStarted: boolean;
   streamEnding: boolean;
+  lastAudioAt?: number;
   playbackStartedAt?: number;
 };
 
@@ -41,6 +42,7 @@ export function createRealtimeVoiceOutputActivityTracker(
   let sinkAudioBytes = 0;
   let playbackStarted = false;
   let streamEnding = false;
+  let lastAudioAt: number | undefined;
   let playbackStartedAt: number | undefined;
 
   const snapshot = (): RealtimeVoiceOutputActivitySnapshot => ({
@@ -50,6 +52,7 @@ export function createRealtimeVoiceOutputActivityTracker(
     sinkAudioBytes,
     playbackStarted,
     streamEnding,
+    ...(lastAudioAt === undefined ? {} : { lastAudioAt }),
     ...(playbackStartedAt === undefined ? {} : { playbackStartedAt }),
   });
 
@@ -58,6 +61,7 @@ export function createRealtimeVoiceOutputActivityTracker(
       streamEnding = false;
       playbackStarted = false;
       playbackStartedAt = undefined;
+      lastAudioAt = undefined;
     },
     markStreamEnding() {
       streamEnding = true;
@@ -74,6 +78,7 @@ export function createRealtimeVoiceOutputActivityTracker(
       sourceAudioBytes += Math.max(0, delta.sourceAudioBytes ?? 0);
       sinkAudioBytes += Math.max(0, delta.sinkAudioBytes ?? 0);
       chunks += 1;
+      lastAudioAt = now();
     },
     reset() {
       audioMs = 0;
@@ -82,6 +87,7 @@ export function createRealtimeVoiceOutputActivityTracker(
       sinkAudioBytes = 0;
       playbackStarted = false;
       streamEnding = false;
+      lastAudioAt = undefined;
       playbackStartedAt = undefined;
     },
     isActive(sinkActive = false) {
