mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-08 02:31:22 +00:00
refactor(web): unify proxy-guarded fetch path for web tools
This commit is contained in:
@@ -1,6 +1,5 @@
|
|||||||
import { Type } from "@sinclair/typebox";
|
import { Type } from "@sinclair/typebox";
|
||||||
import type { OpenClawConfig } from "../../config/config.js";
|
import type { OpenClawConfig } from "../../config/config.js";
|
||||||
import { fetchWithSsrFGuard } from "../../infra/net/fetch-guard.js";
|
|
||||||
import { SsrFBlockedError } from "../../infra/net/ssrf.js";
|
import { SsrFBlockedError } from "../../infra/net/ssrf.js";
|
||||||
import { logDebug } from "../../logger.js";
|
import { logDebug } from "../../logger.js";
|
||||||
import { wrapExternalContent, wrapWebContent } from "../../security/external-content.js";
|
import { wrapExternalContent, wrapWebContent } from "../../security/external-content.js";
|
||||||
@@ -15,6 +14,7 @@ import {
|
|||||||
truncateText,
|
truncateText,
|
||||||
type ExtractMode,
|
type ExtractMode,
|
||||||
} from "./web-fetch-utils.js";
|
} from "./web-fetch-utils.js";
|
||||||
|
import { fetchWithWebToolsNetworkGuard } from "./web-guarded-fetch.js";
|
||||||
import {
|
import {
|
||||||
CacheEntry,
|
CacheEntry,
|
||||||
DEFAULT_CACHE_TTL_MINUTES,
|
DEFAULT_CACHE_TTL_MINUTES,
|
||||||
@@ -523,11 +523,10 @@ async function runWebFetch(params: WebFetchRuntimeParams): Promise<Record<string
|
|||||||
let release: (() => Promise<void>) | null = null;
|
let release: (() => Promise<void>) | null = null;
|
||||||
let finalUrl = params.url;
|
let finalUrl = params.url;
|
||||||
try {
|
try {
|
||||||
const result = await fetchWithSsrFGuard({
|
const result = await fetchWithWebToolsNetworkGuard({
|
||||||
url: params.url,
|
url: params.url,
|
||||||
maxRedirects: params.maxRedirects,
|
maxRedirects: params.maxRedirects,
|
||||||
timeoutMs: params.timeoutSeconds * 1000,
|
timeoutSeconds: params.timeoutSeconds,
|
||||||
proxy: "env",
|
|
||||||
init: {
|
init: {
|
||||||
headers: {
|
headers: {
|
||||||
Accept: "text/markdown, text/html;q=0.9, */*;q=0.1",
|
Accept: "text/markdown, text/html;q=0.9, */*;q=0.1",
|
||||||
|
|||||||
50
src/agents/tools/web-guarded-fetch.ts
Normal file
50
src/agents/tools/web-guarded-fetch.ts
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
import {
|
||||||
|
fetchWithSsrFGuard,
|
||||||
|
type GuardedFetchOptions,
|
||||||
|
type GuardedFetchResult,
|
||||||
|
} from "../../infra/net/fetch-guard.js";
|
||||||
|
import type { SsrFPolicy } from "../../infra/net/ssrf.js";
|
||||||
|
|
||||||
|
export const WEB_TOOLS_TRUSTED_NETWORK_SSRF_POLICY: SsrFPolicy = {
|
||||||
|
dangerouslyAllowPrivateNetwork: true,
|
||||||
|
};
|
||||||
|
|
||||||
|
type WebToolGuardedFetchOptions = Omit<GuardedFetchOptions, "proxy"> & {
|
||||||
|
timeoutSeconds?: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
function resolveTimeoutMs(params: {
|
||||||
|
timeoutMs?: number;
|
||||||
|
timeoutSeconds?: number;
|
||||||
|
}): number | undefined {
|
||||||
|
if (typeof params.timeoutMs === "number" && Number.isFinite(params.timeoutMs)) {
|
||||||
|
return params.timeoutMs;
|
||||||
|
}
|
||||||
|
if (typeof params.timeoutSeconds === "number" && Number.isFinite(params.timeoutSeconds)) {
|
||||||
|
return params.timeoutSeconds * 1000;
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function fetchWithWebToolsNetworkGuard(
|
||||||
|
params: WebToolGuardedFetchOptions,
|
||||||
|
): Promise<GuardedFetchResult> {
|
||||||
|
const { timeoutSeconds, ...rest } = params;
|
||||||
|
return fetchWithSsrFGuard({
|
||||||
|
...rest,
|
||||||
|
timeoutMs: resolveTimeoutMs({ timeoutMs: rest.timeoutMs, timeoutSeconds }),
|
||||||
|
proxy: "env",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function withWebToolsNetworkGuard<T>(
|
||||||
|
params: WebToolGuardedFetchOptions,
|
||||||
|
run: (result: { response: Response; finalUrl: string }) => Promise<T>,
|
||||||
|
): Promise<T> {
|
||||||
|
const { response, finalUrl, release } = await fetchWithWebToolsNetworkGuard(params);
|
||||||
|
try {
|
||||||
|
return await run({ response, finalUrl });
|
||||||
|
} finally {
|
||||||
|
await release();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -33,6 +33,7 @@ describe("web_search redirect resolution hardening", () => {
|
|||||||
timeoutMs: 5000,
|
timeoutMs: 5000,
|
||||||
init: { method: "HEAD" },
|
init: { method: "HEAD" },
|
||||||
policy: { dangerouslyAllowPrivateNetwork: true },
|
policy: { dangerouslyAllowPrivateNetwork: true },
|
||||||
|
proxy: "env",
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
expect(release).toHaveBeenCalledTimes(1);
|
expect(release).toHaveBeenCalledTimes(1);
|
||||||
|
|||||||
@@ -2,11 +2,14 @@ import { Type } from "@sinclair/typebox";
|
|||||||
import { formatCliCommand } from "../../cli/command-format.js";
|
import { formatCliCommand } from "../../cli/command-format.js";
|
||||||
import type { OpenClawConfig } from "../../config/config.js";
|
import type { OpenClawConfig } from "../../config/config.js";
|
||||||
import { logVerbose } from "../../globals.js";
|
import { logVerbose } from "../../globals.js";
|
||||||
import { fetchWithSsrFGuard } from "../../infra/net/fetch-guard.js";
|
|
||||||
import { wrapWebContent } from "../../security/external-content.js";
|
import { wrapWebContent } from "../../security/external-content.js";
|
||||||
import { normalizeSecretInput } from "../../utils/normalize-secret-input.js";
|
import { normalizeSecretInput } from "../../utils/normalize-secret-input.js";
|
||||||
import type { AnyAgentTool } from "./common.js";
|
import type { AnyAgentTool } from "./common.js";
|
||||||
import { jsonResult, readNumberParam, readStringParam } from "./common.js";
|
import { jsonResult, readNumberParam, readStringParam } from "./common.js";
|
||||||
|
import {
|
||||||
|
WEB_TOOLS_TRUSTED_NETWORK_SSRF_POLICY,
|
||||||
|
withWebToolsNetworkGuard,
|
||||||
|
} from "./web-guarded-fetch.js";
|
||||||
import {
|
import {
|
||||||
CacheEntry,
|
CacheEntry,
|
||||||
DEFAULT_CACHE_TTL_MINUTES,
|
DEFAULT_CACHE_TTL_MINUTES,
|
||||||
@@ -44,7 +47,6 @@ const BRAVE_FRESHNESS_SHORTCUTS = new Set(["pd", "pw", "pm", "py"]);
|
|||||||
const BRAVE_FRESHNESS_RANGE = /^(\d{4}-\d{2}-\d{2})to(\d{4}-\d{2}-\d{2})$/;
|
const BRAVE_FRESHNESS_RANGE = /^(\d{4}-\d{2}-\d{2})to(\d{4}-\d{2}-\d{2})$/;
|
||||||
const BRAVE_SEARCH_LANG_CODE = /^[a-z]{2}$/i;
|
const BRAVE_SEARCH_LANG_CODE = /^[a-z]{2}$/i;
|
||||||
const BRAVE_UI_LANG_LOCALE = /^([a-z]{2})-([a-z]{2})$/i;
|
const BRAVE_UI_LANG_LOCALE = /^([a-z]{2})-([a-z]{2})$/i;
|
||||||
const TRUSTED_NETWORK_SSRF_POLICY = { dangerouslyAllowPrivateNetwork: true } as const;
|
|
||||||
|
|
||||||
const WebSearchSchema = Type.Object({
|
const WebSearchSchema = Type.Object({
|
||||||
query: Type.String({ description: "Search query string." }),
|
query: Type.String({ description: "Search query string." }),
|
||||||
@@ -599,19 +601,23 @@ function resolveGeminiModel(gemini?: GeminiConfig): string {
|
|||||||
return fromConfig || DEFAULT_GEMINI_MODEL;
|
return fromConfig || DEFAULT_GEMINI_MODEL;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchTrustedWebSearchEndpoint(params: {
|
async function withTrustedWebSearchEndpoint<T>(
|
||||||
url: string;
|
params: {
|
||||||
timeoutSeconds: number;
|
url: string;
|
||||||
init: RequestInit;
|
timeoutSeconds: number;
|
||||||
}): Promise<{ response: Response; release: () => Promise<void> }> {
|
init: RequestInit;
|
||||||
const { response, release } = await fetchWithSsrFGuard({
|
},
|
||||||
url: params.url,
|
run: (response: Response) => Promise<T>,
|
||||||
init: params.init,
|
): Promise<T> {
|
||||||
timeoutMs: params.timeoutSeconds * 1000,
|
return withWebToolsNetworkGuard(
|
||||||
policy: TRUSTED_NETWORK_SSRF_POLICY,
|
{
|
||||||
proxy: "env",
|
url: params.url,
|
||||||
});
|
init: params.init,
|
||||||
return { response, release };
|
timeoutSeconds: params.timeoutSeconds,
|
||||||
|
policy: WEB_TOOLS_TRUSTED_NETWORK_SSRF_POLICY,
|
||||||
|
},
|
||||||
|
async ({ response }) => run(response),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function runGeminiSearch(params: {
|
async function runGeminiSearch(params: {
|
||||||
@@ -622,81 +628,84 @@ async function runGeminiSearch(params: {
|
|||||||
}): Promise<{ content: string; citations: Array<{ url: string; title?: string }> }> {
|
}): Promise<{ content: string; citations: Array<{ url: string; title?: string }> }> {
|
||||||
const endpoint = `${GEMINI_API_BASE}/models/${params.model}:generateContent`;
|
const endpoint = `${GEMINI_API_BASE}/models/${params.model}:generateContent`;
|
||||||
|
|
||||||
const { response: res, release } = await fetchTrustedWebSearchEndpoint({
|
return withTrustedWebSearchEndpoint(
|
||||||
url: endpoint,
|
{
|
||||||
timeoutSeconds: params.timeoutSeconds,
|
url: endpoint,
|
||||||
init: {
|
timeoutSeconds: params.timeoutSeconds,
|
||||||
method: "POST",
|
init: {
|
||||||
headers: {
|
method: "POST",
|
||||||
"Content-Type": "application/json",
|
headers: {
|
||||||
"x-goog-api-key": params.apiKey,
|
"Content-Type": "application/json",
|
||||||
},
|
"x-goog-api-key": params.apiKey,
|
||||||
body: JSON.stringify({
|
},
|
||||||
contents: [
|
body: JSON.stringify({
|
||||||
{
|
contents: [
|
||||||
parts: [{ text: params.query }],
|
{
|
||||||
},
|
parts: [{ text: params.query }],
|
||||||
],
|
},
|
||||||
tools: [{ google_search: {} }],
|
],
|
||||||
}),
|
tools: [{ google_search: {} }],
|
||||||
},
|
|
||||||
});
|
|
||||||
try {
|
|
||||||
if (!res.ok) {
|
|
||||||
const detailResult = await readResponseText(res, { maxBytes: 64_000 });
|
|
||||||
// Strip API key from any error detail to prevent accidental key leakage in logs
|
|
||||||
const safeDetail = (detailResult.text || res.statusText).replace(/key=[^&\s]+/gi, "key=***");
|
|
||||||
throw new Error(`Gemini API error (${res.status}): ${safeDetail}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
let data: GeminiGroundingResponse;
|
|
||||||
try {
|
|
||||||
data = (await res.json()) as GeminiGroundingResponse;
|
|
||||||
} catch (err) {
|
|
||||||
const safeError = String(err).replace(/key=[^&\s]+/gi, "key=***");
|
|
||||||
throw new Error(`Gemini API returned invalid JSON: ${safeError}`, { cause: err });
|
|
||||||
}
|
|
||||||
|
|
||||||
if (data.error) {
|
|
||||||
const rawMsg = data.error.message || data.error.status || "unknown";
|
|
||||||
const safeMsg = rawMsg.replace(/key=[^&\s]+/gi, "key=***");
|
|
||||||
throw new Error(`Gemini API error (${data.error.code}): ${safeMsg}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const candidate = data.candidates?.[0];
|
|
||||||
const content =
|
|
||||||
candidate?.content?.parts
|
|
||||||
?.map((p) => p.text)
|
|
||||||
.filter(Boolean)
|
|
||||||
.join("\n") ?? "No response";
|
|
||||||
|
|
||||||
const groundingChunks = candidate?.groundingMetadata?.groundingChunks ?? [];
|
|
||||||
const rawCitations = groundingChunks
|
|
||||||
.filter((chunk) => chunk.web?.uri)
|
|
||||||
.map((chunk) => ({
|
|
||||||
url: chunk.web!.uri!,
|
|
||||||
title: chunk.web?.title || undefined,
|
|
||||||
}));
|
|
||||||
|
|
||||||
// Resolve Google grounding redirect URLs to direct URLs with concurrency cap.
|
|
||||||
// Gemini typically returns 3-8 citations; cap at 10 concurrent to be safe.
|
|
||||||
const MAX_CONCURRENT_REDIRECTS = 10;
|
|
||||||
const citations: Array<{ url: string; title?: string }> = [];
|
|
||||||
for (let i = 0; i < rawCitations.length; i += MAX_CONCURRENT_REDIRECTS) {
|
|
||||||
const batch = rawCitations.slice(i, i + MAX_CONCURRENT_REDIRECTS);
|
|
||||||
const resolved = await Promise.all(
|
|
||||||
batch.map(async (citation) => {
|
|
||||||
const resolvedUrl = await resolveRedirectUrl(citation.url);
|
|
||||||
return { ...citation, url: resolvedUrl };
|
|
||||||
}),
|
}),
|
||||||
);
|
},
|
||||||
citations.push(...resolved);
|
},
|
||||||
}
|
async (res) => {
|
||||||
|
if (!res.ok) {
|
||||||
|
const detailResult = await readResponseText(res, { maxBytes: 64_000 });
|
||||||
|
// Strip API key from any error detail to prevent accidental key leakage in logs
|
||||||
|
const safeDetail = (detailResult.text || res.statusText).replace(
|
||||||
|
/key=[^&\s]+/gi,
|
||||||
|
"key=***",
|
||||||
|
);
|
||||||
|
throw new Error(`Gemini API error (${res.status}): ${safeDetail}`);
|
||||||
|
}
|
||||||
|
|
||||||
return { content, citations };
|
let data: GeminiGroundingResponse;
|
||||||
} finally {
|
try {
|
||||||
await release();
|
data = (await res.json()) as GeminiGroundingResponse;
|
||||||
}
|
} catch (err) {
|
||||||
|
const safeError = String(err).replace(/key=[^&\s]+/gi, "key=***");
|
||||||
|
throw new Error(`Gemini API returned invalid JSON: ${safeError}`, { cause: err });
|
||||||
|
}
|
||||||
|
|
||||||
|
if (data.error) {
|
||||||
|
const rawMsg = data.error.message || data.error.status || "unknown";
|
||||||
|
const safeMsg = rawMsg.replace(/key=[^&\s]+/gi, "key=***");
|
||||||
|
throw new Error(`Gemini API error (${data.error.code}): ${safeMsg}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const candidate = data.candidates?.[0];
|
||||||
|
const content =
|
||||||
|
candidate?.content?.parts
|
||||||
|
?.map((p) => p.text)
|
||||||
|
.filter(Boolean)
|
||||||
|
.join("\n") ?? "No response";
|
||||||
|
|
||||||
|
const groundingChunks = candidate?.groundingMetadata?.groundingChunks ?? [];
|
||||||
|
const rawCitations = groundingChunks
|
||||||
|
.filter((chunk) => chunk.web?.uri)
|
||||||
|
.map((chunk) => ({
|
||||||
|
url: chunk.web!.uri!,
|
||||||
|
title: chunk.web?.title || undefined,
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Resolve Google grounding redirect URLs to direct URLs with concurrency cap.
|
||||||
|
// Gemini typically returns 3-8 citations; cap at 10 concurrent to be safe.
|
||||||
|
const MAX_CONCURRENT_REDIRECTS = 10;
|
||||||
|
const citations: Array<{ url: string; title?: string }> = [];
|
||||||
|
for (let i = 0; i < rawCitations.length; i += MAX_CONCURRENT_REDIRECTS) {
|
||||||
|
const batch = rawCitations.slice(i, i + MAX_CONCURRENT_REDIRECTS);
|
||||||
|
const resolved = await Promise.all(
|
||||||
|
batch.map(async (citation) => {
|
||||||
|
const resolvedUrl = await resolveRedirectUrl(citation.url);
|
||||||
|
return { ...citation, url: resolvedUrl };
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
citations.push(...resolved);
|
||||||
|
}
|
||||||
|
|
||||||
|
return { content, citations };
|
||||||
|
},
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const REDIRECT_TIMEOUT_MS = 5000;
|
const REDIRECT_TIMEOUT_MS = 5000;
|
||||||
@@ -707,18 +716,15 @@ const REDIRECT_TIMEOUT_MS = 5000;
|
|||||||
*/
|
*/
|
||||||
async function resolveRedirectUrl(url: string): Promise<string> {
|
async function resolveRedirectUrl(url: string): Promise<string> {
|
||||||
try {
|
try {
|
||||||
const { finalUrl, release } = await fetchWithSsrFGuard({
|
return await withWebToolsNetworkGuard(
|
||||||
url,
|
{
|
||||||
init: { method: "HEAD" },
|
url,
|
||||||
timeoutMs: REDIRECT_TIMEOUT_MS,
|
init: { method: "HEAD" },
|
||||||
policy: TRUSTED_NETWORK_SSRF_POLICY,
|
timeoutMs: REDIRECT_TIMEOUT_MS,
|
||||||
proxy: "env",
|
policy: WEB_TOOLS_TRUSTED_NETWORK_SSRF_POLICY,
|
||||||
});
|
},
|
||||||
try {
|
async ({ finalUrl }) => finalUrl || url,
|
||||||
return finalUrl || url;
|
);
|
||||||
} finally {
|
|
||||||
await release();
|
|
||||||
}
|
|
||||||
} catch {
|
} catch {
|
||||||
return url;
|
return url;
|
||||||
}
|
}
|
||||||
@@ -892,33 +898,33 @@ async function runPerplexitySearch(params: {
|
|||||||
body.search_recency_filter = recencyFilter;
|
body.search_recency_filter = recencyFilter;
|
||||||
}
|
}
|
||||||
|
|
||||||
const { response: res, release } = await fetchTrustedWebSearchEndpoint({
|
return withTrustedWebSearchEndpoint(
|
||||||
url: endpoint,
|
{
|
||||||
timeoutSeconds: params.timeoutSeconds,
|
url: endpoint,
|
||||||
init: {
|
timeoutSeconds: params.timeoutSeconds,
|
||||||
method: "POST",
|
init: {
|
||||||
headers: {
|
method: "POST",
|
||||||
"Content-Type": "application/json",
|
headers: {
|
||||||
Authorization: `Bearer ${params.apiKey}`,
|
"Content-Type": "application/json",
|
||||||
"HTTP-Referer": "https://openclaw.ai",
|
Authorization: `Bearer ${params.apiKey}`,
|
||||||
"X-Title": "OpenClaw Web Search",
|
"HTTP-Referer": "https://openclaw.ai",
|
||||||
|
"X-Title": "OpenClaw Web Search",
|
||||||
|
},
|
||||||
|
body: JSON.stringify(body),
|
||||||
},
|
},
|
||||||
body: JSON.stringify(body),
|
|
||||||
},
|
},
|
||||||
});
|
async (res) => {
|
||||||
try {
|
if (!res.ok) {
|
||||||
if (!res.ok) {
|
return await throwWebSearchApiError(res, "Perplexity");
|
||||||
return await throwWebSearchApiError(res, "Perplexity");
|
}
|
||||||
}
|
|
||||||
|
|
||||||
const data = (await res.json()) as PerplexitySearchResponse;
|
const data = (await res.json()) as PerplexitySearchResponse;
|
||||||
const content = data.choices?.[0]?.message?.content ?? "No response";
|
const content = data.choices?.[0]?.message?.content ?? "No response";
|
||||||
const citations = data.citations ?? [];
|
const citations = data.citations ?? [];
|
||||||
|
|
||||||
return { content, citations };
|
return { content, citations };
|
||||||
} finally {
|
},
|
||||||
await release();
|
);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function runGrokSearch(params: {
|
async function runGrokSearch(params: {
|
||||||
@@ -948,34 +954,34 @@ async function runGrokSearch(params: {
|
|||||||
// citations are returned automatically when available — we just parse
|
// citations are returned automatically when available — we just parse
|
||||||
// them from the response without requesting them explicitly (#12910).
|
// them from the response without requesting them explicitly (#12910).
|
||||||
|
|
||||||
const { response: res, release } = await fetchTrustedWebSearchEndpoint({
|
return withTrustedWebSearchEndpoint(
|
||||||
url: XAI_API_ENDPOINT,
|
{
|
||||||
timeoutSeconds: params.timeoutSeconds,
|
url: XAI_API_ENDPOINT,
|
||||||
init: {
|
timeoutSeconds: params.timeoutSeconds,
|
||||||
method: "POST",
|
init: {
|
||||||
headers: {
|
method: "POST",
|
||||||
"Content-Type": "application/json",
|
headers: {
|
||||||
Authorization: `Bearer ${params.apiKey}`,
|
"Content-Type": "application/json",
|
||||||
|
Authorization: `Bearer ${params.apiKey}`,
|
||||||
|
},
|
||||||
|
body: JSON.stringify(body),
|
||||||
},
|
},
|
||||||
body: JSON.stringify(body),
|
|
||||||
},
|
},
|
||||||
});
|
async (res) => {
|
||||||
try {
|
if (!res.ok) {
|
||||||
if (!res.ok) {
|
return await throwWebSearchApiError(res, "xAI");
|
||||||
return await throwWebSearchApiError(res, "xAI");
|
}
|
||||||
}
|
|
||||||
|
|
||||||
const data = (await res.json()) as GrokSearchResponse;
|
const data = (await res.json()) as GrokSearchResponse;
|
||||||
const { text: extractedText, annotationCitations } = extractGrokContent(data);
|
const { text: extractedText, annotationCitations } = extractGrokContent(data);
|
||||||
const content = extractedText ?? "No response";
|
const content = extractedText ?? "No response";
|
||||||
// Prefer top-level citations; fall back to annotation-derived ones
|
// Prefer top-level citations; fall back to annotation-derived ones
|
||||||
const citations = (data.citations ?? []).length > 0 ? data.citations! : annotationCitations;
|
const citations = (data.citations ?? []).length > 0 ? data.citations! : annotationCitations;
|
||||||
const inlineCitations = data.inline_citations;
|
const inlineCitations = data.inline_citations;
|
||||||
|
|
||||||
return { content, citations, inlineCitations };
|
return { content, citations, inlineCitations };
|
||||||
} finally {
|
},
|
||||||
await release();
|
);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function extractKimiMessageText(message: KimiMessage | undefined): string | undefined {
|
function extractKimiMessageText(message: KimiMessage | undefined): string | undefined {
|
||||||
@@ -1047,71 +1053,79 @@ async function runKimiSearch(params: {
|
|||||||
const MAX_ROUNDS = 3;
|
const MAX_ROUNDS = 3;
|
||||||
|
|
||||||
for (let round = 0; round < MAX_ROUNDS; round += 1) {
|
for (let round = 0; round < MAX_ROUNDS; round += 1) {
|
||||||
const { response: res, release } = await fetchTrustedWebSearchEndpoint({
|
const nextResult = await withTrustedWebSearchEndpoint(
|
||||||
url: endpoint,
|
{
|
||||||
timeoutSeconds: params.timeoutSeconds,
|
url: endpoint,
|
||||||
init: {
|
timeoutSeconds: params.timeoutSeconds,
|
||||||
method: "POST",
|
init: {
|
||||||
headers: {
|
method: "POST",
|
||||||
"Content-Type": "application/json",
|
headers: {
|
||||||
Authorization: `Bearer ${params.apiKey}`,
|
"Content-Type": "application/json",
|
||||||
|
Authorization: `Bearer ${params.apiKey}`,
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
model: params.model,
|
||||||
|
messages,
|
||||||
|
tools: [KIMI_WEB_SEARCH_TOOL],
|
||||||
|
}),
|
||||||
},
|
},
|
||||||
body: JSON.stringify({
|
|
||||||
model: params.model,
|
|
||||||
messages,
|
|
||||||
tools: [KIMI_WEB_SEARCH_TOOL],
|
|
||||||
}),
|
|
||||||
},
|
},
|
||||||
});
|
async (
|
||||||
try {
|
res,
|
||||||
if (!res.ok) {
|
): Promise<{ done: true; content: string; citations: string[] } | { done: false }> => {
|
||||||
return await throwWebSearchApiError(res, "Kimi");
|
if (!res.ok) {
|
||||||
}
|
return await throwWebSearchApiError(res, "Kimi");
|
||||||
|
|
||||||
const data = (await res.json()) as KimiSearchResponse;
|
|
||||||
for (const citation of extractKimiCitations(data)) {
|
|
||||||
collectedCitations.add(citation);
|
|
||||||
}
|
|
||||||
const choice = data.choices?.[0];
|
|
||||||
const message = choice?.message;
|
|
||||||
const text = extractKimiMessageText(message);
|
|
||||||
const toolCalls = message?.tool_calls ?? [];
|
|
||||||
|
|
||||||
if (choice?.finish_reason !== "tool_calls" || toolCalls.length === 0) {
|
|
||||||
return { content: text ?? "No response", citations: [...collectedCitations] };
|
|
||||||
}
|
|
||||||
|
|
||||||
messages.push({
|
|
||||||
role: "assistant",
|
|
||||||
content: message?.content ?? "",
|
|
||||||
...(message?.reasoning_content
|
|
||||||
? {
|
|
||||||
reasoning_content: message.reasoning_content,
|
|
||||||
}
|
|
||||||
: {}),
|
|
||||||
tool_calls: toolCalls,
|
|
||||||
});
|
|
||||||
|
|
||||||
const toolContent = buildKimiToolResultContent(data);
|
|
||||||
let pushedToolResult = false;
|
|
||||||
for (const toolCall of toolCalls) {
|
|
||||||
const toolCallId = toolCall.id?.trim();
|
|
||||||
if (!toolCallId) {
|
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
pushedToolResult = true;
|
|
||||||
messages.push({
|
|
||||||
role: "tool",
|
|
||||||
tool_call_id: toolCallId,
|
|
||||||
content: toolContent,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!pushedToolResult) {
|
const data = (await res.json()) as KimiSearchResponse;
|
||||||
return { content: text ?? "No response", citations: [...collectedCitations] };
|
for (const citation of extractKimiCitations(data)) {
|
||||||
}
|
collectedCitations.add(citation);
|
||||||
} finally {
|
}
|
||||||
await release();
|
const choice = data.choices?.[0];
|
||||||
|
const message = choice?.message;
|
||||||
|
const text = extractKimiMessageText(message);
|
||||||
|
const toolCalls = message?.tool_calls ?? [];
|
||||||
|
|
||||||
|
if (choice?.finish_reason !== "tool_calls" || toolCalls.length === 0) {
|
||||||
|
return { done: true, content: text ?? "No response", citations: [...collectedCitations] };
|
||||||
|
}
|
||||||
|
|
||||||
|
messages.push({
|
||||||
|
role: "assistant",
|
||||||
|
content: message?.content ?? "",
|
||||||
|
...(message?.reasoning_content
|
||||||
|
? {
|
||||||
|
reasoning_content: message.reasoning_content,
|
||||||
|
}
|
||||||
|
: {}),
|
||||||
|
tool_calls: toolCalls,
|
||||||
|
});
|
||||||
|
|
||||||
|
const toolContent = buildKimiToolResultContent(data);
|
||||||
|
let pushedToolResult = false;
|
||||||
|
for (const toolCall of toolCalls) {
|
||||||
|
const toolCallId = toolCall.id?.trim();
|
||||||
|
if (!toolCallId) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
pushedToolResult = true;
|
||||||
|
messages.push({
|
||||||
|
role: "tool",
|
||||||
|
tool_call_id: toolCallId,
|
||||||
|
content: toolContent,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!pushedToolResult) {
|
||||||
|
return { done: true, content: text ?? "No response", citations: [...collectedCitations] };
|
||||||
|
}
|
||||||
|
|
||||||
|
return { done: false };
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
if (nextResult.done) {
|
||||||
|
return { content: nextResult.content, citations: nextResult.citations };
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1287,49 +1301,42 @@ async function runWebSearch(params: {
|
|||||||
url.searchParams.set("freshness", params.freshness);
|
url.searchParams.set("freshness", params.freshness);
|
||||||
}
|
}
|
||||||
|
|
||||||
const { response: res, release } = await fetchTrustedWebSearchEndpoint({
|
const mapped = await withTrustedWebSearchEndpoint(
|
||||||
url: url.toString(),
|
{
|
||||||
timeoutSeconds: params.timeoutSeconds,
|
url: url.toString(),
|
||||||
init: {
|
timeoutSeconds: params.timeoutSeconds,
|
||||||
method: "GET",
|
init: {
|
||||||
headers: {
|
method: "GET",
|
||||||
Accept: "application/json",
|
headers: {
|
||||||
"X-Subscription-Token": params.apiKey,
|
Accept: "application/json",
|
||||||
|
"X-Subscription-Token": params.apiKey,
|
||||||
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
});
|
async (res) => {
|
||||||
let mapped: Array<{
|
if (!res.ok) {
|
||||||
title: string;
|
const detailResult = await readResponseText(res, { maxBytes: 64_000 });
|
||||||
url: string;
|
const detail = detailResult.text;
|
||||||
description: string;
|
throw new Error(`Brave Search API error (${res.status}): ${detail || res.statusText}`);
|
||||||
published?: string;
|
}
|
||||||
siteName?: string;
|
|
||||||
}> = [];
|
|
||||||
try {
|
|
||||||
if (!res.ok) {
|
|
||||||
const detailResult = await readResponseText(res, { maxBytes: 64_000 });
|
|
||||||
const detail = detailResult.text;
|
|
||||||
throw new Error(`Brave Search API error (${res.status}): ${detail || res.statusText}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const data = (await res.json()) as BraveSearchResponse;
|
const data = (await res.json()) as BraveSearchResponse;
|
||||||
const results = Array.isArray(data.web?.results) ? (data.web?.results ?? []) : [];
|
const results = Array.isArray(data.web?.results) ? (data.web?.results ?? []) : [];
|
||||||
mapped = results.map((entry) => {
|
return results.map((entry) => {
|
||||||
const description = entry.description ?? "";
|
const description = entry.description ?? "";
|
||||||
const title = entry.title ?? "";
|
const title = entry.title ?? "";
|
||||||
const url = entry.url ?? "";
|
const url = entry.url ?? "";
|
||||||
const rawSiteName = resolveSiteName(url);
|
const rawSiteName = resolveSiteName(url);
|
||||||
return {
|
return {
|
||||||
title: title ? wrapWebContent(title, "web_search") : "",
|
title: title ? wrapWebContent(title, "web_search") : "",
|
||||||
url, // Keep raw for tool chaining
|
url, // Keep raw for tool chaining
|
||||||
description: description ? wrapWebContent(description, "web_search") : "",
|
description: description ? wrapWebContent(description, "web_search") : "",
|
||||||
published: entry.age || undefined,
|
published: entry.age || undefined,
|
||||||
siteName: rawSiteName || undefined,
|
siteName: rawSiteName || undefined,
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
} finally {
|
},
|
||||||
await release();
|
);
|
||||||
}
|
|
||||||
|
|
||||||
const payload = {
|
const payload = {
|
||||||
query: params.query,
|
query: params.query,
|
||||||
|
|||||||
@@ -46,6 +46,29 @@ function createKimiSearchTool(kimiConfig?: { apiKey?: string; baseUrl?: string;
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function createProviderSearchTool(provider: "brave" | "perplexity" | "grok" | "gemini" | "kimi") {
|
||||||
|
const searchConfig =
|
||||||
|
provider === "perplexity"
|
||||||
|
? { provider, perplexity: { apiKey: "pplx-config-test" } }
|
||||||
|
: provider === "grok"
|
||||||
|
? { provider, grok: { apiKey: "xai-config-test" } }
|
||||||
|
: provider === "gemini"
|
||||||
|
? { provider, gemini: { apiKey: "gemini-config-test" } }
|
||||||
|
: provider === "kimi"
|
||||||
|
? { provider, kimi: { apiKey: "moonshot-config-test" } }
|
||||||
|
: { provider, apiKey: "brave-config-test" };
|
||||||
|
return createWebSearchTool({
|
||||||
|
config: {
|
||||||
|
tools: {
|
||||||
|
web: {
|
||||||
|
search: searchConfig,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
sandboxed: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
function parseFirstRequestBody(mockFetch: ReturnType<typeof installMockFetch>) {
|
function parseFirstRequestBody(mockFetch: ReturnType<typeof installMockFetch>) {
|
||||||
const request = mockFetch.mock.calls[0]?.[1] as RequestInit | undefined;
|
const request = mockFetch.mock.calls[0]?.[1] as RequestInit | undefined;
|
||||||
const requestBody = request?.body;
|
const requestBody = request?.body;
|
||||||
@@ -62,6 +85,34 @@ function installPerplexitySuccessFetch() {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function createProviderSuccessPayload(
|
||||||
|
provider: "brave" | "perplexity" | "grok" | "gemini" | "kimi",
|
||||||
|
) {
|
||||||
|
if (provider === "brave") {
|
||||||
|
return { web: { results: [] } };
|
||||||
|
}
|
||||||
|
if (provider === "perplexity") {
|
||||||
|
return { choices: [{ message: { content: "ok" } }], citations: [] };
|
||||||
|
}
|
||||||
|
if (provider === "grok") {
|
||||||
|
return { output_text: "ok", citations: [] };
|
||||||
|
}
|
||||||
|
if (provider === "gemini") {
|
||||||
|
return {
|
||||||
|
candidates: [
|
||||||
|
{
|
||||||
|
content: { parts: [{ text: "ok" }] },
|
||||||
|
groundingMetadata: { groundingChunks: [] },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
choices: [{ finish_reason: "stop", message: { role: "assistant", content: "ok" } }],
|
||||||
|
search_results: [],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
async function executePerplexitySearch(
|
async function executePerplexitySearch(
|
||||||
query: string,
|
query: string,
|
||||||
options?: {
|
options?: {
|
||||||
@@ -159,6 +210,32 @@ describe("web_search country and language parameters", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("web_search provider proxy dispatch", () => {
|
||||||
|
const priorFetch = global.fetch;
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.unstubAllEnvs();
|
||||||
|
global.fetch = priorFetch;
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each(["brave", "perplexity", "grok", "gemini", "kimi"] as const)(
|
||||||
|
"uses proxy-aware dispatcher for %s provider when HTTP_PROXY is configured",
|
||||||
|
async (provider) => {
|
||||||
|
vi.stubEnv("HTTP_PROXY", "http://127.0.0.1:7890");
|
||||||
|
const mockFetch = installMockFetch(createProviderSuccessPayload(provider));
|
||||||
|
const tool = createProviderSearchTool(provider);
|
||||||
|
expect(tool).not.toBeNull();
|
||||||
|
|
||||||
|
await tool?.execute?.("call-1", { query: `proxy-${provider}-test` });
|
||||||
|
|
||||||
|
const requestInit = mockFetch.mock.calls[0]?.[1] as
|
||||||
|
| (RequestInit & { dispatcher?: unknown })
|
||||||
|
| undefined;
|
||||||
|
expect(requestInit?.dispatcher).toBeInstanceOf(EnvHttpProxyAgent);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
describe("web_search perplexity baseUrl defaults", () => {
|
describe("web_search perplexity baseUrl defaults", () => {
|
||||||
const priorFetch = global.fetch;
|
const priorFetch = global.fetch;
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user