mirror of
https://github.com/openclaw/openclaw.git
synced 2026-04-19 06:57:26 +00:00
refactor(web): split trusted and strict web tool fetch paths
This commit is contained in:
@@ -5,13 +5,14 @@ import {
|
|||||||
} from "../../infra/net/fetch-guard.js";
|
} from "../../infra/net/fetch-guard.js";
|
||||||
import type { SsrFPolicy } from "../../infra/net/ssrf.js";
|
import type { SsrFPolicy } from "../../infra/net/ssrf.js";
|
||||||
|
|
||||||
export const WEB_TOOLS_TRUSTED_NETWORK_SSRF_POLICY: SsrFPolicy = {
|
const WEB_TOOLS_TRUSTED_NETWORK_SSRF_POLICY: SsrFPolicy = {
|
||||||
dangerouslyAllowPrivateNetwork: true,
|
dangerouslyAllowPrivateNetwork: true,
|
||||||
};
|
};
|
||||||
|
|
||||||
type WebToolGuardedFetchOptions = Omit<GuardedFetchOptions, "proxy"> & {
|
type WebToolGuardedFetchOptions = Omit<GuardedFetchOptions, "proxy"> & {
|
||||||
timeoutSeconds?: number;
|
timeoutSeconds?: number;
|
||||||
};
|
};
|
||||||
|
type WebToolEndpointFetchOptions = Omit<WebToolGuardedFetchOptions, "policy">;
|
||||||
|
|
||||||
function resolveTimeoutMs(params: {
|
function resolveTimeoutMs(params: {
|
||||||
timeoutMs?: number;
|
timeoutMs?: number;
|
||||||
@@ -37,7 +38,7 @@ export async function fetchWithWebToolsNetworkGuard(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function withWebToolsNetworkGuard<T>(
|
async function withWebToolsNetworkGuard<T>(
|
||||||
params: WebToolGuardedFetchOptions,
|
params: WebToolGuardedFetchOptions,
|
||||||
run: (result: { response: Response; finalUrl: string }) => Promise<T>,
|
run: (result: { response: Response; finalUrl: string }) => Promise<T>,
|
||||||
): Promise<T> {
|
): Promise<T> {
|
||||||
@@ -48,3 +49,23 @@ export async function withWebToolsNetworkGuard<T>(
|
|||||||
await release();
|
await release();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function withTrustedWebToolsEndpoint<T>(
|
||||||
|
params: WebToolEndpointFetchOptions,
|
||||||
|
run: (result: { response: Response; finalUrl: string }) => Promise<T>,
|
||||||
|
): Promise<T> {
|
||||||
|
return await withWebToolsNetworkGuard(
|
||||||
|
{
|
||||||
|
...params,
|
||||||
|
policy: WEB_TOOLS_TRUSTED_NETWORK_SSRF_POLICY,
|
||||||
|
},
|
||||||
|
run,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function withStrictWebToolsEndpoint<T>(
|
||||||
|
params: WebToolEndpointFetchOptions,
|
||||||
|
run: (result: { response: Response; finalUrl: string }) => Promise<T>,
|
||||||
|
): Promise<T> {
|
||||||
|
return await withWebToolsNetworkGuard(params, run);
|
||||||
|
}
|
||||||
|
|||||||
22
src/agents/tools/web-search-citation-redirect.ts
Normal file
22
src/agents/tools/web-search-citation-redirect.ts
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
import { withStrictWebToolsEndpoint } from "./web-guarded-fetch.js";
|
||||||
|
|
||||||
|
const REDIRECT_TIMEOUT_MS = 5000;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resolve a citation redirect URL to its final destination using a HEAD request.
|
||||||
|
* Returns the original URL if resolution fails or times out.
|
||||||
|
*/
|
||||||
|
export async function resolveCitationRedirectUrl(url: string): Promise<string> {
|
||||||
|
try {
|
||||||
|
return await withStrictWebToolsEndpoint(
|
||||||
|
{
|
||||||
|
url,
|
||||||
|
init: { method: "HEAD" },
|
||||||
|
timeoutMs: REDIRECT_TIMEOUT_MS,
|
||||||
|
},
|
||||||
|
async ({ finalUrl }) => finalUrl || url,
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
return url;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -6,10 +6,8 @@ import { wrapWebContent } from "../../security/external-content.js";
|
|||||||
import { normalizeSecretInput } from "../../utils/normalize-secret-input.js";
|
import { normalizeSecretInput } from "../../utils/normalize-secret-input.js";
|
||||||
import type { AnyAgentTool } from "./common.js";
|
import type { AnyAgentTool } from "./common.js";
|
||||||
import { jsonResult, readNumberParam, readStringParam } from "./common.js";
|
import { jsonResult, readNumberParam, readStringParam } from "./common.js";
|
||||||
import {
|
import { withTrustedWebToolsEndpoint } from "./web-guarded-fetch.js";
|
||||||
WEB_TOOLS_TRUSTED_NETWORK_SSRF_POLICY,
|
import { resolveCitationRedirectUrl } from "./web-search-citation-redirect.js";
|
||||||
withWebToolsNetworkGuard,
|
|
||||||
} from "./web-guarded-fetch.js";
|
|
||||||
import {
|
import {
|
||||||
CacheEntry,
|
CacheEntry,
|
||||||
DEFAULT_CACHE_TTL_MINUTES,
|
DEFAULT_CACHE_TTL_MINUTES,
|
||||||
@@ -609,12 +607,11 @@ async function withTrustedWebSearchEndpoint<T>(
|
|||||||
},
|
},
|
||||||
run: (response: Response) => Promise<T>,
|
run: (response: Response) => Promise<T>,
|
||||||
): Promise<T> {
|
): Promise<T> {
|
||||||
return withWebToolsNetworkGuard(
|
return withTrustedWebToolsEndpoint(
|
||||||
{
|
{
|
||||||
url: params.url,
|
url: params.url,
|
||||||
init: params.init,
|
init: params.init,
|
||||||
timeoutSeconds: params.timeoutSeconds,
|
timeoutSeconds: params.timeoutSeconds,
|
||||||
policy: WEB_TOOLS_TRUSTED_NETWORK_SSRF_POLICY,
|
|
||||||
},
|
},
|
||||||
async ({ response }) => run(response),
|
async ({ response }) => run(response),
|
||||||
);
|
);
|
||||||
@@ -696,7 +693,7 @@ async function runGeminiSearch(params: {
|
|||||||
const batch = rawCitations.slice(i, i + MAX_CONCURRENT_REDIRECTS);
|
const batch = rawCitations.slice(i, i + MAX_CONCURRENT_REDIRECTS);
|
||||||
const resolved = await Promise.all(
|
const resolved = await Promise.all(
|
||||||
batch.map(async (citation) => {
|
batch.map(async (citation) => {
|
||||||
const resolvedUrl = await resolveRedirectUrl(citation.url);
|
const resolvedUrl = await resolveCitationRedirectUrl(citation.url);
|
||||||
return { ...citation, url: resolvedUrl };
|
return { ...citation, url: resolvedUrl };
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
@@ -708,27 +705,6 @@ async function runGeminiSearch(params: {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const REDIRECT_TIMEOUT_MS = 5000;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Resolve a redirect URL to its final destination using a HEAD request.
|
|
||||||
* Returns the original URL if resolution fails or times out.
|
|
||||||
*/
|
|
||||||
async function resolveRedirectUrl(url: string): Promise<string> {
|
|
||||||
try {
|
|
||||||
return await withWebToolsNetworkGuard(
|
|
||||||
{
|
|
||||||
url,
|
|
||||||
init: { method: "HEAD" },
|
|
||||||
timeoutMs: REDIRECT_TIMEOUT_MS,
|
|
||||||
},
|
|
||||||
async ({ finalUrl }) => finalUrl || url,
|
|
||||||
);
|
|
||||||
} catch {
|
|
||||||
return url;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function resolveSearchCount(value: unknown, fallback: number): number {
|
function resolveSearchCount(value: unknown, fallback: number): number {
|
||||||
const parsed = typeof value === "number" && Number.isFinite(value) ? value : fallback;
|
const parsed = typeof value === "number" && Number.isFinite(value) ? value : fallback;
|
||||||
const clamped = Math.max(1, Math.min(MAX_SEARCH_COUNT, Math.floor(parsed)));
|
const clamped = Math.max(1, Math.min(MAX_SEARCH_COUNT, Math.floor(parsed)));
|
||||||
@@ -1492,5 +1468,5 @@ export const __testing = {
|
|||||||
resolveKimiModel,
|
resolveKimiModel,
|
||||||
resolveKimiBaseUrl,
|
resolveKimiBaseUrl,
|
||||||
extractKimiCitations,
|
extractKimiCitations,
|
||||||
resolveRedirectUrl,
|
resolveRedirectUrl: resolveCitationRedirectUrl,
|
||||||
} as const;
|
} as const;
|
||||||
|
|||||||
@@ -5,17 +5,21 @@ export type DiscordGatewayHandle = {
|
|||||||
disconnect?: () => void;
|
disconnect?: () => void;
|
||||||
};
|
};
|
||||||
|
|
||||||
export function getDiscordGatewayEmitter(gateway?: unknown): EventEmitter | undefined {
|
export type WaitForDiscordGatewayStopParams = {
|
||||||
return (gateway as { emitter?: EventEmitter } | undefined)?.emitter;
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function waitForDiscordGatewayStop(params: {
|
|
||||||
gateway?: DiscordGatewayHandle;
|
gateway?: DiscordGatewayHandle;
|
||||||
abortSignal?: AbortSignal;
|
abortSignal?: AbortSignal;
|
||||||
onGatewayError?: (err: unknown) => void;
|
onGatewayError?: (err: unknown) => void;
|
||||||
shouldStopOnError?: (err: unknown) => boolean;
|
shouldStopOnError?: (err: unknown) => boolean;
|
||||||
registerForceStop?: (forceStop: (err: unknown) => void) => void;
|
registerForceStop?: (forceStop: (err: unknown) => void) => void;
|
||||||
}): Promise<void> {
|
};
|
||||||
|
|
||||||
|
export function getDiscordGatewayEmitter(gateway?: unknown): EventEmitter | undefined {
|
||||||
|
return (gateway as { emitter?: EventEmitter } | undefined)?.emitter;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function waitForDiscordGatewayStop(
|
||||||
|
params: WaitForDiscordGatewayStopParams,
|
||||||
|
): Promise<void> {
|
||||||
const { gateway, abortSignal, onGatewayError, shouldStopOnError } = params;
|
const { gateway, abortSignal, onGatewayError, shouldStopOnError } = params;
|
||||||
const emitter = gateway?.emitter;
|
const emitter = gateway?.emitter;
|
||||||
return await new Promise<void>((resolve, reject) => {
|
return await new Promise<void>((resolve, reject) => {
|
||||||
|
|||||||
@@ -2,9 +2,7 @@ import { EventEmitter } from "node:events";
|
|||||||
import type { Client } from "@buape/carbon";
|
import type { Client } from "@buape/carbon";
|
||||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
import type { RuntimeEnv } from "../../runtime.js";
|
import type { RuntimeEnv } from "../../runtime.js";
|
||||||
import type { waitForDiscordGatewayStop } from "../monitor.gateway.js";
|
import type { WaitForDiscordGatewayStopParams } from "../monitor.gateway.js";
|
||||||
|
|
||||||
type WaitForDiscordGatewayStopParams = Parameters<typeof waitForDiscordGatewayStop>[0];
|
|
||||||
|
|
||||||
const {
|
const {
|
||||||
attachDiscordGatewayLoggingMock,
|
attachDiscordGatewayLoggingMock,
|
||||||
|
|||||||
Reference in New Issue
Block a user