github/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-05-30 22:36:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(memory): route batch APIs through guarded remote HTTP

Browse Source
This commit is contained in:
Peter Steinberger
2026-02-22 18:14:00 +01:00
parent f87db7c627
commit eb041daee2
5 changed files with 178 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
src/memory/batch-voyage.ts
View File

@@ -7,6 +7,7 @@ import { runEmbeddingBatchGroups } from "./batch-runner.js";
import { uploadBatchJsonlFile } from "./batch-upload.js";
import { buildBatchHeaders, normalizeBatchBaseUrl } from "./batch-utils.js";
import type { VoyageEmbeddingClient } from "./embeddings-voyage.js";
import { withRemoteHttpResponse } from "./remote-http.js";
/**
* Voyage Batch API Input Line format.
@@ -58,6 +59,7 @@ async function submitVoyageBatch(params: {
return await postJsonWithRetry<VoyageBatchStatus>({
url: `${baseUrl}/batches`,
headers: buildBatchHeaders(params.client, { json: true }),
ssrfPolicy: params.client.ssrfPolicy,
body: {
input_file_id: inputFileId,
endpoint: VOYAGE_BATCH_ENDPOINT,
@@ -80,14 +82,20 @@ async function fetchVoyageBatchStatus(params: {
batchId: string;
}): Promise<VoyageBatchStatus> {
const baseUrl = normalizeBatchBaseUrl(params.client);
const res = await fetch(`${baseUrl}/batches/${params.batchId}`, {
headers: buildBatchHeaders(params.client, { json: true }),
return await withRemoteHttpResponse({
url: `${baseUrl}/batches/${params.batchId}`,
ssrfPolicy: params.client.ssrfPolicy,
init: {
headers: buildBatchHeaders(params.client, { json: true }),
},
onResponse: async (res) => {
if (!res.ok) {
const text = await res.text();
throw new Error(`voyage batch status failed: ${res.status} ${text}`);
}
return (await res.json()) as VoyageBatchStatus;
},
});
if (!res.ok) {
const text = await res.text();
throw new Error(`voyage batch status failed: ${res.status} ${text}`);
}
return (await res.json()) as VoyageBatchStatus;
}
async function readVoyageBatchError(params: {
@@ -96,23 +104,29 @@ async function readVoyageBatchError(params: {
}): Promise<string | undefined> {
try {
const baseUrl = normalizeBatchBaseUrl(params.client);
const res = await fetch(`${baseUrl}/files/${params.errorFileId}/content`, {
headers: buildBatchHeaders(params.client, { json: true }),
return await withRemoteHttpResponse({
url: `${baseUrl}/files/${params.errorFileId}/content`,
ssrfPolicy: params.client.ssrfPolicy,
init: {
headers: buildBatchHeaders(params.client, { json: true }),
},
onResponse: async (res) => {
if (!res.ok) {
const text = await res.text();
throw new Error(`voyage batch error file content failed: ${res.status} ${text}`);
}
const text = await res.text();
if (!text.trim()) {
return undefined;
}
const lines = text
.split("\n")
.map((line) => line.trim())
.filter(Boolean)
.map((line) => JSON.parse(line) as VoyageBatchOutputLine);
return extractBatchErrorMessage(lines);
},
});
if (!res.ok) {
const text = await res.text();
throw new Error(`voyage batch error file content failed: ${res.status} ${text}`);
}
const text = await res.text();
if (!text.trim()) {
return undefined;
}
const lines = text
.split("\n")
.map((line) => line.trim())
.filter(Boolean)
.map((line) => JSON.parse(line) as VoyageBatchOutputLine);
return extractBatchErrorMessage(lines);
} catch (err) {
return formatUnavailableBatchError(err);
}
@@ -228,33 +242,40 @@ export async function runVoyageEmbeddingBatches(params: {
}
const baseUrl = normalizeBatchBaseUrl(params.client);
const contentRes = await fetch(`${baseUrl}/files/${completed.outputFileId}/content`, {
headers: buildBatchHeaders(params.client, { json: true }),
});
if (!contentRes.ok) {
const text = await contentRes.text();
throw new Error(`voyage batch file content failed: ${contentRes.status} ${text}`);
}
const errors: string[] = [];
const remaining = new Set(group.map((request) => request.custom_id));
if (contentRes.body) {
const reader = createInterface({
input: Readable.fromWeb(
contentRes.body as unknown as import("stream/web").ReadableStream,
),
terminal: false,
});
for await (const rawLine of reader) {
if (!rawLine.trim()) {
continue;
await withRemoteHttpResponse({
url: `${baseUrl}/files/${completed.outputFileId}/content`,
ssrfPolicy: params.client.ssrfPolicy,
init: {
headers: buildBatchHeaders(params.client, { json: true }),
},
onResponse: async (contentRes) => {
if (!contentRes.ok) {
const text = await contentRes.text();
throw new Error(`voyage batch file content failed: ${contentRes.status} ${text}`);
}
const line = JSON.parse(rawLine) as VoyageBatchOutputLine;
applyEmbeddingBatchOutputLine({ line, remaining, errors, byCustomId });
}
}
if (!contentRes.body) {
return;
}
const reader = createInterface({
input: Readable.fromWeb(
contentRes.body as unknown as import("stream/web").ReadableStream,
),
terminal: false,
});
for await (const rawLine of reader) {
if (!rawLine.trim()) {
continue;
}
const line = JSON.parse(rawLine) as VoyageBatchOutputLine;
applyEmbeddingBatchOutputLine({ line, remaining, errors, byCustomId });
}
},
});
if (errors.length > 0) {
throw new Error(`voyage batch ${batchInfo.id} failed: ${errors.join("; ")}`);