mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-08 01:51:24 +00:00
fix: guard remote media fetches with SSRF checks
This commit is contained in:
170
src/infra/net/fetch-guard.ts
Normal file
170
src/infra/net/fetch-guard.ts
Normal file
@@ -0,0 +1,170 @@
|
||||
import type { Dispatcher } from "undici";
|
||||
import {
|
||||
closeDispatcher,
|
||||
createPinnedDispatcher,
|
||||
resolvePinnedHostname,
|
||||
resolvePinnedHostnameWithPolicy,
|
||||
type LookupFn,
|
||||
type SsrFPolicy,
|
||||
} from "./ssrf.js";
|
||||
|
||||
type FetchLike = (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
|
||||
export type GuardedFetchOptions = {
|
||||
url: string;
|
||||
fetchImpl?: FetchLike;
|
||||
method?: string;
|
||||
headers?: HeadersInit;
|
||||
maxRedirects?: number;
|
||||
timeoutMs?: number;
|
||||
signal?: AbortSignal;
|
||||
policy?: SsrFPolicy;
|
||||
lookupFn?: LookupFn;
|
||||
};
|
||||
|
||||
export type GuardedFetchResult = {
|
||||
response: Response;
|
||||
finalUrl: string;
|
||||
release: () => Promise<void>;
|
||||
};
|
||||
|
||||
const DEFAULT_MAX_REDIRECTS = 3;
|
||||
|
||||
function isRedirectStatus(status: number): boolean {
|
||||
return status === 301 || status === 302 || status === 303 || status === 307 || status === 308;
|
||||
}
|
||||
|
||||
function buildAbortSignal(params: { timeoutMs?: number; signal?: AbortSignal }): {
|
||||
signal?: AbortSignal;
|
||||
cleanup: () => void;
|
||||
} {
|
||||
const { timeoutMs, signal } = params;
|
||||
if (!timeoutMs && !signal) {
|
||||
return { signal: undefined, cleanup: () => {} };
|
||||
}
|
||||
|
||||
if (!timeoutMs) {
|
||||
return { signal, cleanup: () => {} };
|
||||
}
|
||||
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
||||
const onAbort = () => controller.abort();
|
||||
if (signal) {
|
||||
if (signal.aborted) {
|
||||
controller.abort();
|
||||
} else {
|
||||
signal.addEventListener("abort", onAbort, { once: true });
|
||||
}
|
||||
}
|
||||
|
||||
const cleanup = () => {
|
||||
clearTimeout(timeoutId);
|
||||
if (signal) {
|
||||
signal.removeEventListener("abort", onAbort);
|
||||
}
|
||||
};
|
||||
|
||||
return { signal: controller.signal, cleanup };
|
||||
}
|
||||
|
||||
export async function fetchWithSsrFGuard(params: GuardedFetchOptions): Promise<GuardedFetchResult> {
|
||||
const fetcher: FetchLike | undefined = params.fetchImpl ?? globalThis.fetch;
|
||||
if (!fetcher) {
|
||||
throw new Error("fetch is not available");
|
||||
}
|
||||
|
||||
const maxRedirects =
|
||||
typeof params.maxRedirects === "number" && Number.isFinite(params.maxRedirects)
|
||||
? Math.max(0, Math.floor(params.maxRedirects))
|
||||
: DEFAULT_MAX_REDIRECTS;
|
||||
|
||||
const { signal, cleanup } = buildAbortSignal({
|
||||
timeoutMs: params.timeoutMs,
|
||||
signal: params.signal,
|
||||
});
|
||||
|
||||
let released = false;
|
||||
const release = async (dispatcher?: Dispatcher | null) => {
|
||||
if (released) {
|
||||
return;
|
||||
}
|
||||
released = true;
|
||||
cleanup();
|
||||
await closeDispatcher(dispatcher ?? undefined);
|
||||
};
|
||||
|
||||
const visited = new Set<string>();
|
||||
let currentUrl = params.url;
|
||||
let redirectCount = 0;
|
||||
|
||||
while (true) {
|
||||
let parsedUrl: URL;
|
||||
try {
|
||||
parsedUrl = new URL(currentUrl);
|
||||
} catch {
|
||||
await release();
|
||||
throw new Error("Invalid URL: must be http or https");
|
||||
}
|
||||
if (!["http:", "https:"].includes(parsedUrl.protocol)) {
|
||||
await release();
|
||||
throw new Error("Invalid URL: must be http or https");
|
||||
}
|
||||
|
||||
let dispatcher: Dispatcher | null = null;
|
||||
try {
|
||||
const usePolicy = Boolean(
|
||||
params.policy?.allowPrivateNetwork || params.policy?.allowedHostnames?.length,
|
||||
);
|
||||
const pinned = usePolicy
|
||||
? await resolvePinnedHostnameWithPolicy(parsedUrl.hostname, {
|
||||
lookupFn: params.lookupFn,
|
||||
policy: params.policy,
|
||||
})
|
||||
: await resolvePinnedHostname(parsedUrl.hostname, params.lookupFn);
|
||||
dispatcher = createPinnedDispatcher(pinned);
|
||||
|
||||
const init: RequestInit & { dispatcher?: Dispatcher } = {
|
||||
redirect: "manual",
|
||||
dispatcher,
|
||||
...(params.method ? { method: params.method } : {}),
|
||||
...(params.headers ? { headers: params.headers } : {}),
|
||||
...(signal ? { signal } : {}),
|
||||
};
|
||||
|
||||
const response = await fetcher(parsedUrl.toString(), init);
|
||||
|
||||
if (isRedirectStatus(response.status)) {
|
||||
const location = response.headers.get("location");
|
||||
if (!location) {
|
||||
await release(dispatcher);
|
||||
throw new Error(`Redirect missing location header (${response.status})`);
|
||||
}
|
||||
redirectCount += 1;
|
||||
if (redirectCount > maxRedirects) {
|
||||
await release(dispatcher);
|
||||
throw new Error(`Too many redirects (limit: ${maxRedirects})`);
|
||||
}
|
||||
const nextUrl = new URL(location, parsedUrl).toString();
|
||||
if (visited.has(nextUrl)) {
|
||||
await release(dispatcher);
|
||||
throw new Error("Redirect loop detected");
|
||||
}
|
||||
visited.add(nextUrl);
|
||||
void response.body?.cancel();
|
||||
await closeDispatcher(dispatcher);
|
||||
currentUrl = nextUrl;
|
||||
continue;
|
||||
}
|
||||
|
||||
return {
|
||||
response,
|
||||
finalUrl: currentUrl,
|
||||
release: async () => release(dispatcher),
|
||||
};
|
||||
} catch (err) {
|
||||
await release(dispatcher);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -15,7 +15,12 @@ export class SsrFBlockedError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
type LookupFn = typeof dnsLookup;
|
||||
export type LookupFn = typeof dnsLookup;
|
||||
|
||||
export type SsrFPolicy = {
|
||||
allowPrivateNetwork?: boolean;
|
||||
allowedHostnames?: string[];
|
||||
};
|
||||
|
||||
const PRIVATE_IPV6_PREFIXES = ["fe80:", "fec0:", "fc", "fd"];
|
||||
const BLOCKED_HOSTNAMES = new Set(["localhost", "metadata.google.internal"]);
|
||||
@@ -28,6 +33,13 @@ function normalizeHostname(hostname: string): string {
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function normalizeHostnameSet(values?: string[]): Set<string> {
|
||||
if (!values || values.length === 0) {
|
||||
return new Set<string>();
|
||||
}
|
||||
return new Set(values.map((value) => normalizeHostname(value)).filter(Boolean));
|
||||
}
|
||||
|
||||
function parseIpv4(address: string): number[] | null {
|
||||
const parts = address.split(".");
|
||||
if (parts.length !== 4) {
|
||||
@@ -206,31 +218,40 @@ export type PinnedHostname = {
|
||||
lookup: typeof dnsLookupCb;
|
||||
};
|
||||
|
||||
export async function resolvePinnedHostname(
|
||||
export async function resolvePinnedHostnameWithPolicy(
|
||||
hostname: string,
|
||||
lookupFn: LookupFn = dnsLookup,
|
||||
params: { lookupFn?: LookupFn; policy?: SsrFPolicy } = {},
|
||||
): Promise<PinnedHostname> {
|
||||
const normalized = normalizeHostname(hostname);
|
||||
if (!normalized) {
|
||||
throw new Error("Invalid hostname");
|
||||
}
|
||||
|
||||
if (isBlockedHostname(normalized)) {
|
||||
throw new SsrFBlockedError(`Blocked hostname: ${hostname}`);
|
||||
}
|
||||
|
||||
if (isPrivateIpAddress(normalized)) {
|
||||
throw new SsrFBlockedError("Blocked: private/internal IP address");
|
||||
const allowPrivateNetwork = Boolean(params.policy?.allowPrivateNetwork);
|
||||
const allowedHostnames = normalizeHostnameSet(params.policy?.allowedHostnames);
|
||||
const isExplicitAllowed = allowedHostnames.has(normalized);
|
||||
|
||||
if (!allowPrivateNetwork && !isExplicitAllowed) {
|
||||
if (isBlockedHostname(normalized)) {
|
||||
throw new SsrFBlockedError(`Blocked hostname: ${hostname}`);
|
||||
}
|
||||
|
||||
if (isPrivateIpAddress(normalized)) {
|
||||
throw new SsrFBlockedError("Blocked: private/internal IP address");
|
||||
}
|
||||
}
|
||||
|
||||
const lookupFn = params.lookupFn ?? dnsLookup;
|
||||
const results = await lookupFn(normalized, { all: true });
|
||||
if (results.length === 0) {
|
||||
throw new Error(`Unable to resolve hostname: ${hostname}`);
|
||||
}
|
||||
|
||||
for (const entry of results) {
|
||||
if (isPrivateIpAddress(entry.address)) {
|
||||
throw new SsrFBlockedError("Blocked: resolves to private/internal IP address");
|
||||
if (!allowPrivateNetwork && !isExplicitAllowed) {
|
||||
for (const entry of results) {
|
||||
if (isPrivateIpAddress(entry.address)) {
|
||||
throw new SsrFBlockedError("Blocked: resolves to private/internal IP address");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -246,6 +267,13 @@ export async function resolvePinnedHostname(
|
||||
};
|
||||
}
|
||||
|
||||
export async function resolvePinnedHostname(
|
||||
hostname: string,
|
||||
lookupFn: LookupFn = dnsLookup,
|
||||
): Promise<PinnedHostname> {
|
||||
return await resolvePinnedHostnameWithPolicy(hostname, { lookupFn });
|
||||
}
|
||||
|
||||
export function createPinnedDispatcher(pinned: PinnedHostname): Dispatcher {
|
||||
return new Agent({
|
||||
connect: {
|
||||
|
||||
Reference in New Issue
Block a user