fix(net): harden trusted env proxy fetch guard and add web_fetch opt-in

cosmicnet · cosmicnet · commit b93c8f64dc99 · 2026-04-28T19:07:49.000+01:00
diff --git a/docs/.generated/config-baseline.sha256 b/docs/.generated/config-baseline.sha256
@@ -1,4 +1,4 @@
-39c5c0620611f355f20d5e9d2ddd74e198c344c63d5551a987e4b7538833ceac  config-baseline.json
-805bd3f63ff7327da45c01b78dbc990ed53bd13b89e0cbf50f319aa99334ba92  config-baseline.core.json
+7c92086c3dccfca80812ac65a6d9f994de8db6022bd3cc0f504de747ec2ffaad  config-baseline.json
+17e0eeb204084f0f5204dbe8fc0b30ac60cf1cb6119bdd9e2a31c7a23f75e05a  config-baseline.core.json
 323a9fd49a669951ca5b3442d95aad243bd1330083f9857e83a8dcfae2bbc9d0  config-baseline.channel.json
-1f5592bfd141ba1e982ce31763a253c10afb080ab4ea2b6538299b114e29cee1  config-baseline.plugin.json
+0dd6583fafae6c9134e46c4cf9bddee9822d6436436dcb1a6dcba6d012962e51  config-baseline.plugin.json
diff --git a/docs/tools/web-fetch.md b/docs/tools/web-fetch.md
@@ -72,6 +72,7 @@ Truncate output to this many characters.
         timeoutSeconds: 30,
         cacheTtlMinutes: 15,
         maxRedirects: 3,
+        useTrustedEnvProxy: false, // let a trusted HTTP(S) env proxy resolve DNS
         readability: true, // use Readability extraction
         userAgent: "Mozilla/5.0 ...", // override User-Agent
       },
@@ -134,13 +135,32 @@ Current runtime behavior:
 - If Readability is disabled, `web_fetch` skips straight to the selected
   provider fallback. If no provider is available, it fails closed.
 
+## Trusted env proxy mode
+
+If your deployment requires `web_fetch` to go through a trusted outbound HTTP(S)
+proxy, set `tools.web.fetch.useTrustedEnvProxy: true`.
+
+In this mode, OpenClaw still applies hostname-based SSRF checks before sending
+the request, but it lets the proxy resolve DNS instead of doing local DNS
+pinning. Enable this only when the proxy is operator-controlled and enforces
+outbound policy after DNS resolution.
+
+<Note>
+  If no HTTP(S) proxy env var is configured, or the target host is excluded by
+  `NO_PROXY`, `web_fetch` falls back to the normal strict path with local DNS
+  pinning.
+</Note>
+
 ## Limits and safety
 
 - `maxChars` is clamped to `tools.web.fetch.maxCharsCap`
 - Response body is capped at `maxResponseBytes` before parsing; oversized
   responses are truncated with a warning
 - Private/internal hostnames are blocked
 - Redirects are checked and limited by `maxRedirects`
+- `useTrustedEnvProxy` is an explicit opt-in and should only be enabled for
+  operator-controlled proxies that still enforce outbound policy after DNS
+  resolution
 - `web_fetch` is best-effort -- some sites need the [Web Browser](/tools/browser)
 
 ## Tool profiles
diff --git a/src/agents/tools/web-fetch.ssrf.test.ts b/src/agents/tools/web-fetch.ssrf.test.ts
@@ -36,6 +36,7 @@ function setMockFetch(
 
 function createWebFetchToolForTest(params?: {
   firecrawlApiKey?: string;
+  useTrustedEnvProxy?: boolean;
   ssrfPolicy?: { allowRfc2544BenchmarkRange?: boolean };
   cacheTtlMinutes?: number;
 }) {
@@ -58,6 +59,7 @@ function createWebFetchToolForTest(params?: {
         web: {
           fetch: {
             cacheTtlMinutes: params?.cacheTtlMinutes ?? 0,
+            useTrustedEnvProxy: params?.useTrustedEnvProxy,
             ssrfPolicy: params?.ssrfPolicy,
             ...(params?.firecrawlApiKey ? { provider: "firecrawl" } : {}),
           },
@@ -89,6 +91,7 @@ describe("web_fetch SSRF protection", () => {
     global.fetch = priorFetch;
     lookupMock.mockClear();
     vi.restoreAllMocks();
+    vi.unstubAllEnvs();
   });
 
   it("blocks localhost hostnames before fetch/firecrawl", async () => {
@@ -178,4 +181,18 @@ describe("web_fetch SSRF protection", () => {
     const stricterTool = createWebFetchToolForTest({ cacheTtlMinutes: 1 });
     await expectBlockedUrl(stricterTool, url, /private|internal|blocked/i);
   });
+
+  it("still blocks dangerous hostnames when trusted env proxy is explicitly enabled", async () => {
+    vi.stubEnv("HTTP_PROXY", "http://127.0.0.1:7890");
+    vi.stubEnv("http_proxy", "http://127.0.0.1:7890");
+    const fetchSpy = setMockFetch();
+    const tool = createWebFetchToolForTest({
+      useTrustedEnvProxy: true,
+      cacheTtlMinutes: 1,
+    });
+
+    await expectBlockedUrl(tool, "http://localhost/test", /Blocked hostname/i);
+    expect(fetchSpy).not.toHaveBeenCalled();
+    expect(lookupMock).not.toHaveBeenCalled();
+  });
 });
diff --git a/src/agents/tools/web-fetch.ts b/src/agents/tools/web-fetch.ts
@@ -116,6 +116,10 @@ function resolveFetchReadabilityEnabled(fetch?: WebFetchConfig): boolean {
   return true;
 }
 
+function resolveFetchUseTrustedEnvProxy(fetch?: WebFetchConfig): boolean {
+  return fetch?.useTrustedEnvProxy === true;
+}
+
 function resolveFetchMaxCharsCap(fetch?: WebFetchConfig): number {
   const raw =
     fetch && "maxCharsCap" in fetch && typeof fetch.maxCharsCap === "number"
@@ -272,6 +276,7 @@ type WebFetchRuntimeParams = {
   userAgent: string;
   readabilityEnabled: boolean;
   config?: OpenClawConfig;
+  useTrustedEnvProxy: boolean;
   ssrfPolicy?: {
     allowRfc2544BenchmarkRange?: boolean;
   };
@@ -389,8 +394,9 @@ async function maybeFetchProviderWebFetchPayload(
 
 async function runWebFetch(params: WebFetchRuntimeParams): Promise<Record<string, unknown>> {
   const allowRfc2544BenchmarkRange = params.ssrfPolicy?.allowRfc2544BenchmarkRange === true;
+  const useTrustedEnvProxy = params.useTrustedEnvProxy ?? false;
   const cacheKey = normalizeCacheKey(
-    `fetch:${params.url}:${params.extractMode}:${params.maxChars}${allowRfc2544BenchmarkRange ? ":allow-rfc2544" : ""}`,
+    `fetch:${params.url}:${params.extractMode}:${params.maxChars}${allowRfc2544BenchmarkRange ? ":allow-rfc2544" : ""}${useTrustedEnvProxy ? ":trusted-env-proxy" : ""}`,
   );
   const cached = readCache(FETCH_CACHE, cacheKey);
   if (cached) {
@@ -418,6 +424,7 @@ async function runWebFetch(params: WebFetchRuntimeParams): Promise<Record<string
       maxRedirects: params.maxRedirects,
       timeoutSeconds: params.timeoutSeconds,
       lookupFn: params.lookupFn,
+      useEnvProxy: useTrustedEnvProxy,
       policy: allowRfc2544BenchmarkRange ? { allowRfc2544BenchmarkRange } : undefined,
       init: {
         headers: {
@@ -651,6 +658,7 @@ export function createWebFetchTool(options?: {
         userAgent,
         readabilityEnabled,
         config: options?.config,
+        useTrustedEnvProxy: resolveFetchUseTrustedEnvProxy(fetch),
         ssrfPolicy: fetch?.ssrfPolicy,
         lookupFn: options?.lookupFn,
         resolveProviderFallback,
diff --git a/src/agents/tools/web-tools.fetch.test.ts b/src/agents/tools/web-tools.fetch.test.ts
@@ -331,7 +331,7 @@ describe("web_fetch extraction fallbacks", () => {
     expect(details?.warning).toContain("Response body truncated");
   });
 
-  it("keeps DNS pinning for untrusted web_fetch URLs even when HTTP_PROXY is configured", async () => {
+  it("keeps DNS pinning for web_fetch by default even when HTTP_PROXY is configured", async () => {
     vi.stubEnv("HTTP_PROXY", "http://127.0.0.1:7890");
     const mockFetch = installMockFetch((input: RequestInfo | URL) =>
       Promise.resolve({
@@ -353,6 +353,31 @@ describe("web_fetch extraction fallbacks", () => {
     expect(requestInit?.dispatcher).not.toBeInstanceOf(EnvHttpProxyAgent);
   });
 
+  it("uses env proxy dispatch for web_fetch when trusted env proxy is explicitly enabled", async () => {
+    vi.stubEnv("HTTP_PROXY", "http://127.0.0.1:7890");
+    const mockFetch = installMockFetch((input: RequestInfo | URL) =>
+      Promise.resolve({
+        ok: true,
+        status: 200,
+        headers: makeFetchHeaders({ "content-type": "text/plain" }),
+        text: async () => "proxy body",
+        url: resolveRequestUrl(input),
+      } as Response),
+    );
+    const tool = createFetchTool({
+      firecrawl: { enabled: false },
+      useTrustedEnvProxy: true,
+    });
+
+    await tool?.execute?.("call", { url: "https://example.com/proxy" });
+
+    const requestInit = mockFetch.mock.calls[0]?.[1] as
+      | (RequestInit & { dispatcher?: unknown })
+      | undefined;
+    expect(requestInit?.dispatcher).toBeDefined();
+    expect(requestInit?.dispatcher).toBeInstanceOf(EnvHttpProxyAgent);
+  });
+
   // NOTE: Test for wrapping url/finalUrl/warning fields requires DNS mocking.
   // The sanitization of these fields is verified by external-content.test.ts tests.
 
diff --git a/src/config/schema.base.generated.ts b/src/config/schema.base.generated.ts
@@ -8601,6 +8601,12 @@ export const GENERATED_BASE_CONFIG_SCHEMA: BaseConfigSchemaResponse = {
                     description:
                       "Use Readability to extract main content from HTML (fallbacks to basic HTML cleanup).",
                   },
+                  useTrustedEnvProxy: {
+                    type: "boolean",
+                    title: "Web Fetch Trusted Env Proxy",
+                    description:
+                      "Route web_fetch through a trusted HTTP(S) env proxy and let the proxy resolve DNS. Enable only when that proxy is operator-controlled and enforces outbound policy after DNS resolution.",
+                  },
                   ssrfPolicy: {
                     type: "object",
                     properties: {
@@ -25644,6 +25650,11 @@ export const GENERATED_BASE_CONFIG_SCHEMA: BaseConfigSchemaResponse = {
       help: "Use Readability to extract main content from HTML (fallbacks to basic HTML cleanup).",
       tags: ["tools"],
     },
+    "tools.web.fetch.useTrustedEnvProxy": {
+      label: "Web Fetch Trusted Env Proxy",
+      help: "Route web_fetch through a trusted HTTP(S) env proxy and let the proxy resolve DNS. Enable only when that proxy is operator-controlled and enforces outbound policy after DNS resolution.",
+      tags: ["tools"],
+    },
     "tools.web.fetch.ssrfPolicy": {
       label: "Web Fetch SSRF Policy",
       help: "Scoped SSRF policy overrides for web_fetch. Keep this narrow and opt in only for known local-network proxy environments.",
diff --git a/src/config/schema.help.ts b/src/config/schema.help.ts
@@ -820,6 +820,8 @@ export const FIELD_HELP: Record<string, string> = {
   "tools.web.fetch.userAgent": "Override User-Agent header for web_fetch requests.",
   "tools.web.fetch.readability":
     "Use Readability to extract main content from HTML (fallbacks to basic HTML cleanup).",
+  "tools.web.fetch.useTrustedEnvProxy":
+    "Route web_fetch through a trusted HTTP(S) env proxy and let the proxy resolve DNS. Enable only when that proxy is operator-controlled and enforces outbound policy after DNS resolution.",
   "tools.web.fetch.ssrfPolicy":
     "Scoped SSRF policy overrides for web_fetch. Keep this narrow and opt in only for known local-network proxy environments.",
   "tools.web.fetch.ssrfPolicy.allowRfc2544BenchmarkRange":
diff --git a/src/config/schema.labels.ts b/src/config/schema.labels.ts
@@ -296,6 +296,7 @@ export const FIELD_LABELS: Record<string, string> = {
   "tools.web.fetch.maxRedirects": "Web Fetch Max Redirects",
   "tools.web.fetch.userAgent": "Web Fetch User-Agent",
   "tools.web.fetch.readability": "Web Fetch Readability Extraction",
+  "tools.web.fetch.useTrustedEnvProxy": "Web Fetch Trusted Env Proxy",
   "tools.web.fetch.ssrfPolicy": "Web Fetch SSRF Policy",
   "tools.web.fetch.ssrfPolicy.allowRfc2544BenchmarkRange":
     "Web Fetch Allow RFC 2544 Benchmark Range",
diff --git a/src/config/schema.test.ts b/src/config/schema.test.ts
@@ -345,6 +345,18 @@ describe("config schema", () => {
     ).toThrow();
   });
 
+  it("accepts web fetch trusted env proxy opt-in in the runtime zod schema", () => {
+    const parsed = ToolsSchema.parse({
+      web: {
+        fetch: {
+          useTrustedEnvProxy: true,
+        },
+      },
+    });
+
+    expect(parsed?.web?.fetch?.useTrustedEnvProxy).toBe(true);
+  });
+
   it("rejects unknown keys inside web fetch firecrawl config", () => {
     expect(() =>
       ToolsSchema.parse({
diff --git a/src/config/types.tools.ts b/src/config/types.tools.ts
@@ -575,6 +575,8 @@ export type ToolsConfig = {
       userAgent?: string;
       /** Use Readability to extract main content (default: true). */
       readability?: boolean;
+      /** Route web_fetch through a trusted HTTP(S) env proxy and let the proxy resolve DNS. Enable only when that proxy enforces outbound policy. */
+      useTrustedEnvProxy?: boolean;
       /** SSRF policy configuration for web_fetch. */
       ssrfPolicy?: {
         /** Allow RFC 2544 benchmark range IPs (198.18.0.0/15) for fake-IP proxy compatibility (e.g., Clash TUN mode, Surge). */
diff --git a/src/config/zod-schema.agent-runtime.ts b/src/config/zod-schema.agent-runtime.ts
@@ -350,6 +350,7 @@ export const ToolsWebFetchSchema = z
     maxRedirects: z.number().int().nonnegative().optional(),
     userAgent: z.string().optional(),
     readability: z.boolean().optional(),
+    useTrustedEnvProxy: z.boolean().optional(),
     ssrfPolicy: z
       .object({
         allowRfc2544BenchmarkRange: z.boolean().optional(),
diff --git a/src/infra/net/fetch-guard.ssrf.test.ts b/src/infra/net/fetch-guard.ssrf.test.ts
diff --git a/src/infra/net/fetch-guard.ts b/src/infra/net/fetch-guard.ts
diff --git a/src/infra/net/proxy-env.test.ts b/src/infra/net/proxy-env.test.ts
diff --git a/src/infra/net/proxy-env.ts b/src/infra/net/proxy-env.ts
