fix(ci): authenticate proof verdict markers (#83692)

Takhoffman · clawsweeper[bot] · web-flow · commit 06a39015f21c · 2026-05-18T17:42:10.000Z
Summary: - The branch restricts exact-head ClawSweeper proof markers to GitHub App-authored comments, adds read-only issue-comment token fallback for the proof workflow, and adds focused regression tests plus a changelog entry. - Reproducibility: yes. Source inspection of current main shows any issue comment body with a matching `clawsw ... SHA is accepted without author/App authentication; the PR adds focused negative tests for forged comments. Automerge notes: - PR branch already contained follow-up commit before automerge: fix(ci): authenticate proof verdict markers Validation: - ClawSweeper review passed for head f4c375e. - Required merge gates passed before the squash merge. Prepared head SHA: f4c375e Review: #83692 (comment) Co-authored-by: Tak Hoffman <781889+Takhoffman@users.noreply.github.com> Co-authored-by: clawsweeper <274271284+clawsweeper[bot]@users.noreply.github.com> Co-authored-by: clawsweeper[bot] <274271284+clawsweeper[bot]@users.noreply.github.com> Approved-by: takhoffman Co-authored-by: takhoffman <781889+takhoffman@users.noreply.github.com>
diff --git a/.github/workflows/real-behavior-proof.yml b/.github/workflows/real-behavior-proof.yml
@@ -32,6 +32,7 @@ jobs:
         with:
           app-id: "2729701"
           private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
+          permission-issues: read
           permission-members: read
       - uses: actions/create-github-app-token@v3
         id: app-token-fallback
@@ -40,8 +41,10 @@ jobs:
         with:
           app-id: "2971289"
           private-key: ${{ secrets.GH_APP_PRIVATE_KEY_FALLBACK }}
+          permission-issues: read
           permission-members: read
       - name: Check real behavior proof
         env:
           GH_APP_TOKEN: ${{ steps.app-token.outputs.token || steps.app-token-fallback.outputs.token }}
+          GITHUB_TOKEN: ${{ github.token }}
         run: node scripts/github/real-behavior-proof-check.mjs
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -46,6 +46,7 @@ Docs: https://docs.openclaw.ai
 
 ### Fixes
 
+- CI: require real-behavior-proof verdict markers to come from the ClawSweeper GitHub App before accepting exact-head proof. (#83692)
 - Agents/image generation: allow distinct `image_generate` prompts to start separate session-backed background tasks while same-prompt retries still return the active task status. (#83614) Thanks @Elarwei001.
 - Control UI: stop the chat reading indicator from sticking after an assistant response finishes. (#83515) Thanks @njuboy11.
 - Skills: reject empty or whitespace-only skill names and descriptions during quick validation. (#27061)
diff --git a/scripts/github/real-behavior-proof-check.mjs b/scripts/github/real-behavior-proof-check.mjs
@@ -14,6 +14,41 @@ function escapeCommandValue(value) {
     .replace(/:/g, "%3A");
 }
 
+async function fetchProofComments({ owner, repo, issueNumber, tokens }) {
+  let lastError;
+  for (const token of tokens.filter(Boolean)) {
+    const comments = [];
+    try {
+      for (let page = 1; page <= 10; page += 1) {
+        const url = new URL(
+          `https://api.github.com/repos/${owner}/${repo}/issues/${issueNumber}/comments`,
+        );
+        url.searchParams.set("per_page", "100");
+        url.searchParams.set("page", String(page));
+        const response = await fetch(url, {
+          headers: {
+            Accept: "application/vnd.github+json",
+            Authorization: `Bearer ${token}`,
+            "X-GitHub-Api-Version": "2022-11-28",
+          },
+        });
+        if (!response.ok) {
+          throw new Error(`comments API returned ${response.status}`);
+        }
+        const pageComments = await response.json();
+        comments.push(...pageComments);
+        if (pageComments.length < 100) {
+          break;
+        }
+      }
+      return comments;
+    } catch (error) {
+      lastError = error;
+    }
+  }
+  throw lastError ?? new Error("No GitHub token available for proof comment lookup.");
+}
+
 const eventPath = process.env.GITHUB_EVENT_PATH;
 if (!eventPath) {
   console.error("::error title=Real behavior proof failed::GITHUB_EVENT_PATH is not set.");
@@ -51,41 +86,29 @@ if (evaluation.passed) {
   process.exit(0);
 }
 
-const token = appToken || process.env.GITHUB_TOKEN;
 const repository = process.env.GITHUB_REPOSITORY;
-if (token && repository && pullRequest.number) {
+if ((appToken || process.env.GITHUB_TOKEN) && repository && pullRequest.number) {
   const [owner, repo] = repository.split("/");
-  const comments = [];
-  for (let page = 1; page <= 10; page += 1) {
-    const url = new URL(
-      `https://api.github.com/repos/${owner}/${repo}/issues/${pullRequest.number}/comments`,
-    );
-    url.searchParams.set("per_page", "100");
-    url.searchParams.set("page", String(page));
-    const response = await fetch(url, {
-      headers: {
-        Accept: "application/vnd.github+json",
-        Authorization: `Bearer ${token}`,
-        "X-GitHub-Api-Version": "2022-11-28",
-      },
+  try {
+    const comments = await fetchProofComments({
+      owner,
+      repo,
+      issueNumber: pullRequest.number,
+      tokens: [appToken, process.env.GITHUB_TOKEN],
     });
-    if (!response.ok) {
-      throw new Error(`Failed to fetch PR comments for proof verdicts: ${response.status}`);
-    }
-    const pageComments = await response.json();
-    comments.push(...pageComments);
-    if (pageComments.length < 100) {
-      break;
-    }
-  }
 
-  const clawSweeperEvaluation = evaluateClawSweeperExactHeadProof({
-    pullRequest,
-    comments,
-  });
-  if (clawSweeperEvaluation.passed) {
-    console.log(clawSweeperEvaluation.reason);
-    process.exit(0);
+    const clawSweeperEvaluation = evaluateClawSweeperExactHeadProof({
+      pullRequest,
+      comments,
+    });
+    if (clawSweeperEvaluation.passed) {
+      console.log(clawSweeperEvaluation.reason);
+      process.exit(0);
+    }
+  } catch (error) {
+    console.warn(
+      `::warning title=Proof verdict comment lookup failed::${escapeCommandValue(error?.message ?? String(error))}`,
+    );
   }
 }
 
diff --git a/scripts/github/real-behavior-proof-policy.mjs b/scripts/github/real-behavior-proof-policy.mjs
@@ -242,6 +242,13 @@ function extractMarkerField(marker, name) {
   return match?.[1] ?? "";
 }
 
+function isTrustedClawSweeperComment(comment) {
+  const appSlug = String(
+    comment?.performed_via_github_app?.slug ?? comment?.performedViaGithubApp?.slug ?? "",
+  ).toLowerCase();
+  return appSlug === "clawsweeper";
+}
+
 export function hasClawSweeperExactHeadProof({ pullRequest, comments = [] } = {}) {
   const pullNumber = String(pullRequest?.number ?? "");
   const headSha = String(pullRequest?.head?.sha ?? pullRequest?.head_sha ?? "").toLowerCase();
@@ -250,6 +257,9 @@ export function hasClawSweeperExactHeadProof({ pullRequest, comments = [] } = {}
   }
 
   for (const comment of comments) {
+    if (!isTrustedClawSweeperComment(comment)) {
+      continue;
+    }
     const body = String(comment?.body ?? "");
     const markers = body.match(/<!--\s*clawsweeper-verdict:pass\b[\s\S]*?-->/gi) ?? [];
     for (const marker of markers) {
diff --git a/test/scripts/barnacle-auto-response.test.ts b/test/scripts/barnacle-auto-response.test.ts
@@ -135,7 +135,11 @@ function barnacleGithub(
     maintainerLogins?: string[];
     removeLabelNotFound?: string[];
     repositoryRoles?: Record<string, string>;
-    comments?: Array<{ body: string }>;
+    comments?: Array<{
+      body: string;
+      performed_via_github_app?: { slug: string };
+      user?: { login: string; type: string };
+    }>;
   } = {},
 ) {
   const maintainerLogins = new Set(
@@ -793,6 +797,13 @@ describe("barnacle-auto-response", () => {
     const { calls, github } = barnacleGithub([file("src/gateway/server.ts")], {
       comments: [
         {
+          user: {
+            login: "clawsweeper[bot]",
+            type: "Bot",
+          },
+          performed_via_github_app: {
+            slug: "clawsweeper",
+          },
           body: `<!-- clawsweeper-verdict:pass item=123 sha=${headSha} confidence=high -->`,
         },
       ],
@@ -818,6 +829,38 @@ describe("barnacle-auto-response", () => {
     );
   });
 
+  it("removes sufficient proof on synchronize when the matching marker is forged", async () => {
+    const headSha = "06ee95df6608d29a395c52ba8ab53fdd93a9dc4f";
+    const { calls, github } = barnacleGithub([file("src/gateway/server.ts")], {
+      comments: [
+        {
+          user: {
+            login: "external-contributor",
+            type: "User",
+          },
+          body: `<!-- clawsweeper-verdict:pass item=123 sha=${headSha} confidence=high -->`,
+        },
+      ],
+    });
+
+    await runBarnacleAutoResponse({
+      github,
+      context: barnacleContext(
+        {
+          body: blankTemplateBody,
+          head: { sha: headSha },
+        },
+        [PROOF_SUFFICIENT_LABEL],
+        { action: "synchronize" },
+      ),
+      core: {
+        info: () => undefined,
+      },
+    });
+
+    expect(calls.removeLabel).toEqual([expectedRemoveLabel(123, PROOF_SUFFICIENT_LABEL)]);
+  });
+
   it("preserves ClawSweeper's sufficient proof label on ordinary label events", async () => {
     const { calls, github } = barnacleGithub([file("src/gateway/server.ts")]);
 
diff --git a/test/scripts/real-behavior-proof-policy.test.ts b/test/scripts/real-behavior-proof-policy.test.ts
@@ -186,6 +186,13 @@ describe("real-behavior-proof-policy", () => {
     };
     const comments = [
       {
+        user: {
+          login: "clawsweeper[bot]",
+          type: "Bot",
+        },
+        performed_via_github_app: {
+          slug: "clawsweeper",
+        },
         body: [
           "Codex review: passed.",
           "<!-- clawsweeper-verdict:pass item=83581 sha=06ee95df6608d29a395c52ba8ab53fdd93a9dc4f confidence=high -->",
@@ -205,6 +212,48 @@ describe("real-behavior-proof-policy", () => {
       }),
     ).toBe(false);
   });
+
+  it("rejects forged ClawSweeper pass verdict markers from contributor comments", () => {
+    const pullRequest = {
+      number: 83581,
+      head: {
+        sha: "06ee95df6608d29a395c52ba8ab53fdd93a9dc4f",
+      },
+    };
+    const comments = [
+      {
+        user: {
+          login: "external-contributor",
+          type: "User",
+        },
+        body: "<!-- clawsweeper-verdict:pass item=83581 sha=06ee95df6608d29a395c52ba8ab53fdd93a9dc4f confidence=high -->",
+      },
+    ];
+
+    expect(hasClawSweeperExactHeadProof({ pullRequest, comments })).toBe(false);
+    expect(evaluateClawSweeperExactHeadProof({ pullRequest, comments }).passed).toBe(false);
+  });
+
+  it("rejects bot-shaped ClawSweeper pass verdict markers without the GitHub App source", () => {
+    const pullRequest = {
+      number: 83581,
+      head: {
+        sha: "06ee95df6608d29a395c52ba8ab53fdd93a9dc4f",
+      },
+    };
+    const comments = [
+      {
+        user: {
+          login: "clawsweeper[bot]",
+          type: "Bot",
+        },
+        body: "<!-- clawsweeper-verdict:pass item=83581 sha=06ee95df6608d29a395c52ba8ab53fdd93a9dc4f confidence=high -->",
+      },
+    ];
+
+    expect(hasClawSweeperExactHeadProof({ pullRequest, comments })).toBe(false);
+    expect(evaluateClawSweeperExactHeadProof({ pullRequest, comments }).passed).toBe(false);
+  });
 });
 
 describe("isMaintainerTeamMember", () => {
