mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-10 23:44:33 +00:00
perf(test): prebuild download archives and cache apply module
This commit is contained in:
@@ -1,6 +1,5 @@
|
|||||||
import fs from "node:fs/promises";
|
import fs from "node:fs/promises";
|
||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
import JSZip from "jszip";
|
|
||||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
import { withTempWorkspace, writeDownloadSkill } from "./skills-install.download-test-utils.js";
|
import { withTempWorkspace, writeDownloadSkill } from "./skills-install.download-test-utils.js";
|
||||||
import { installSkill } from "./skills-install.js";
|
import { installSkill } from "./skills-install.js";
|
||||||
@@ -34,23 +33,18 @@ async function fileExists(filePath: string): Promise<boolean> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function createZipBuffer(
|
const SAFE_ZIP_BUFFER = Buffer.from(
|
||||||
entries: Array<{ name: string; contents: string }>,
|
"UEsDBAoAAAAAAMOJVlysKpPYAgAAAAIAAAAJAAAAaGVsbG8udHh0aGlQSwECFAAKAAAAAADDiVZcrCqT2AIAAAACAAAACQAAAAAAAAAAAAAAAAAAAAAAaGVsbG8udHh0UEsFBgAAAAABAAEANwAAACkAAAAAAA==",
|
||||||
): Promise<Buffer> {
|
"base64",
|
||||||
const zip = new JSZip();
|
);
|
||||||
for (const entry of entries) {
|
const STRIP_COMPONENTS_ZIP_BUFFER = Buffer.from(
|
||||||
zip.file(entry.name, entry.contents);
|
"UEsDBAoAAAAAAMOJVlwAAAAAAAAAAAAAAAAIAAAAcGFja2FnZS9QSwMECgAAAAAAw4lWXKwqk9gCAAAAAgAAABEAAABwYWNrYWdlL2hlbGxvLnR4dGhpUEsBAhQACgAAAAAAw4lWXAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAQAAAAAAAAAHBhY2thZ2UvUEsBAhQACgAAAAAAw4lWXKwqk9gCAAAAAgAAABEAAAAAAAAAAAAAAAAAJgAAAHBhY2thZ2UvaGVsbG8udHh0UEsFBgAAAAACAAIAdQAAAFcAAAAAAA==",
|
||||||
}
|
"base64",
|
||||||
return zip.generateAsync({ type: "nodebuffer" });
|
);
|
||||||
}
|
const ZIP_SLIP_BUFFER = Buffer.from(
|
||||||
|
"UEsDBAoAAAAAAMOJVlwAAAAAAAAAAAAAAAADAAAALi4vUEsDBAoAAAAAAMOJVlwAAAAAAAAAAAAAAAARAAAALi4vb3V0c2lkZS13cml0ZS9QSwMECgAAAAAAw4lWXD3iZKoEAAAABAAAABoAAAAuLi9vdXRzaWRlLXdyaXRlL3B3bmVkLnR4dHB3bmRQSwECFAAKAAAAAADDiVZcAAAAAAAAAAAAAAAAAwAAAAAAAAAAABAAAAAAAAAALi4vUEsBAhQACgAAAAAAw4lWXAAAAAAAAAAAAAAAABEAAAAAAAAAAAAQAAAAIQAAAC4uL291dHNpZGUtd3JpdGUvUEsBAhQACgAAAAAAw4lWXD3iZKoEAAAABAAAABoAAAAAAAAAAAAAAAAAUAAAAC4uL291dHNpZGUtd3JpdGUvcHduZWQudHh0UEsFBgAAAAADAAMAuAAAAIwAAAAAAA==",
|
||||||
const SAFE_ZIP_BUFFER_PROMISE = createZipBuffer([{ name: "hello.txt", contents: "hi" }]);
|
"base64",
|
||||||
const STRIP_COMPONENTS_ZIP_BUFFER_PROMISE = createZipBuffer([
|
);
|
||||||
{ name: "package/hello.txt", contents: "hi" },
|
|
||||||
]);
|
|
||||||
const ZIP_SLIP_BUFFER_PROMISE = createZipBuffer([
|
|
||||||
{ name: "../outside-write/pwned.txt", contents: "pwnd" },
|
|
||||||
]);
|
|
||||||
const TAR_GZ_TRAVERSAL_BUFFER = Buffer.from(
|
const TAR_GZ_TRAVERSAL_BUFFER = Buffer.from(
|
||||||
// Prebuilt archive containing ../outside-write/pwned.txt.
|
// Prebuilt archive containing ../outside-write/pwned.txt.
|
||||||
"H4sIAK4xm2kAA+2VvU7DMBDH3UoIUWaYLXbcS5PYZegQEKhBRUBbIT4GZBpXCqJNSFySlSdgZed1eCgcUvFRaMsQgVD9k05nW3eWz8nfR0g1GMnY98RmEvlSVMllmAyFR2QqUUEAALUsnHlG7VcPtXwO+djEhm1YlJpAbYrBYAYDhKGoA8xiFEseqaPEUvihkGJanArr92fsk5eC3/x/YWl9GZUROuA9fNjBp3hMtoZWlNWU3SrL5k8/29LpdtvjYZbxqGx1IqT0vr7WCwaEh+GNIGEU3IkhH/YEKpXRxv3FQznsPxdQpGYaZFL/RzxtCu6JqFrYOzBX/wZ81n8NmEERTosocB4Lrn8T8ED6A9EwmHp0Wd1idQK2ZVIAm1ZshlvuttPeabonuyTlUkbkO7k2nGPXcYO9q+tkPzmPk4q1hTsqqXU2K+mDxit/fQ+Lyhf9F9795+tf/WoT/Z8yi+n+/xuoz+1p8Wk0Gs3i8QJSs3VlABAAAA==",
|
"H4sIAK4xm2kAA+2VvU7DMBDH3UoIUWaYLXbcS5PYZegQEKhBRUBbIT4GZBpXCqJNSFySlSdgZed1eCgcUvFRaMsQgVD9k05nW3eWz8nfR0g1GMnY98RmEvlSVMllmAyFR2QqUUEAALUsnHlG7VcPtXwO+djEhm1YlJpAbYrBYAYDhKGoA8xiFEseqaPEUvihkGJanArr92fsk5eC3/x/YWl9GZUROuA9fNjBp3hMtoZWlNWU3SrL5k8/29LpdtvjYZbxqGx1IqT0vr7WCwaEh+GNIGEU3IkhH/YEKpXRxv3FQznsPxdQpGYaZFL/RzxtCu6JqFrYOzBX/wZ81n8NmEERTosocB4Lrn8T8ED6A9EwmHp0Wd1idQK2ZVIAm1ZshlvuttPeabonuyTlUkbkO7k2nGPXcYO9q+tkPzmPk4q1hTsqqXU2K+mDxit/fQ+Lyhf9F9795+tf/WoT/Z8yi+n+/xuoz+1p8Wk0Gs3i8QJSs3VlABAAAA==",
|
||||||
@@ -98,9 +92,8 @@ function mockTarExtractionFlow(params: {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async function seedZipDownloadResponse() {
|
function seedZipDownloadResponse() {
|
||||||
const buffer = await SAFE_ZIP_BUFFER_PROMISE;
|
mockArchiveResponse(new Uint8Array(SAFE_ZIP_BUFFER));
|
||||||
mockArchiveResponse(new Uint8Array(buffer));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function installZipDownloadSkill(params: {
|
async function installZipDownloadSkill(params: {
|
||||||
@@ -109,7 +102,7 @@ async function installZipDownloadSkill(params: {
|
|||||||
targetDir: string;
|
targetDir: string;
|
||||||
}) {
|
}) {
|
||||||
const url = "https://example.invalid/good.zip";
|
const url = "https://example.invalid/good.zip";
|
||||||
await seedZipDownloadResponse();
|
seedZipDownloadResponse();
|
||||||
await writeDownloadSkill({
|
await writeDownloadSkill({
|
||||||
workspaceDir: params.workspaceDir,
|
workspaceDir: params.workspaceDir,
|
||||||
name: params.name,
|
name: params.name,
|
||||||
@@ -168,8 +161,7 @@ describe("installSkill download extraction safety", () => {
|
|||||||
const outsideWritePath = path.join(outsideWriteDir, "pwned.txt");
|
const outsideWritePath = path.join(outsideWriteDir, "pwned.txt");
|
||||||
const url = "https://example.invalid/evil.zip";
|
const url = "https://example.invalid/evil.zip";
|
||||||
|
|
||||||
const buffer = await ZIP_SLIP_BUFFER_PROMISE;
|
mockArchiveResponse(new Uint8Array(ZIP_SLIP_BUFFER));
|
||||||
mockArchiveResponse(new Uint8Array(buffer));
|
|
||||||
|
|
||||||
await writeDownloadSkill({
|
await writeDownloadSkill({
|
||||||
workspaceDir,
|
workspaceDir,
|
||||||
@@ -213,8 +205,7 @@ describe("installSkill download extraction safety", () => {
|
|||||||
const targetDir = path.join(stateDir, "tools", "zip-good", "target");
|
const targetDir = path.join(stateDir, "tools", "zip-good", "target");
|
||||||
const url = "https://example.invalid/good.zip";
|
const url = "https://example.invalid/good.zip";
|
||||||
|
|
||||||
const buffer = await STRIP_COMPONENTS_ZIP_BUFFER_PROMISE;
|
mockArchiveResponse(new Uint8Array(STRIP_COMPONENTS_ZIP_BUFFER));
|
||||||
mockArchiveResponse(new Uint8Array(buffer));
|
|
||||||
|
|
||||||
await writeDownloadSkill({
|
await writeDownloadSkill({
|
||||||
workspaceDir,
|
workspaceDir,
|
||||||
@@ -237,8 +228,7 @@ describe("installSkill download extraction safety", () => {
|
|||||||
const targetDir = path.join(workspaceDir, "outside");
|
const targetDir = path.join(workspaceDir, "outside");
|
||||||
const url = "https://example.invalid/good.zip";
|
const url = "https://example.invalid/good.zip";
|
||||||
|
|
||||||
const buffer = await SAFE_ZIP_BUFFER_PROMISE;
|
mockArchiveResponse(new Uint8Array(SAFE_ZIP_BUFFER));
|
||||||
mockArchiveResponse(new Uint8Array(buffer));
|
|
||||||
|
|
||||||
await writeDownloadSkill({
|
await writeDownloadSkill({
|
||||||
workspaceDir,
|
workspaceDir,
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import fs from "node:fs/promises";
|
import fs from "node:fs/promises";
|
||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
import { resolveApiKeyForProvider } from "../agents/model-auth.js";
|
import { resolveApiKeyForProvider } from "../agents/model-auth.js";
|
||||||
import type { MsgContext } from "../auto-reply/templating.js";
|
import type { MsgContext } from "../auto-reply/templating.js";
|
||||||
import type { OpenClawConfig } from "../config/config.js";
|
import type { OpenClawConfig } from "../config/config.js";
|
||||||
@@ -29,9 +29,7 @@ vi.mock("../process/exec.js", () => ({
|
|||||||
runExec: vi.fn(),
|
runExec: vi.fn(),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
async function loadApply() {
|
let applyMediaUnderstanding: typeof import("./apply.js").applyMediaUnderstanding;
|
||||||
return await import("./apply.js");
|
|
||||||
}
|
|
||||||
|
|
||||||
const TEMP_MEDIA_PREFIX = "openclaw-media-";
|
const TEMP_MEDIA_PREFIX = "openclaw-media-";
|
||||||
const tempMediaDirs: string[] = [];
|
const tempMediaDirs: string[] = [];
|
||||||
@@ -137,7 +135,6 @@ async function applyWithDisabledMedia(params: {
|
|||||||
mediaType?: string;
|
mediaType?: string;
|
||||||
cfg?: OpenClawConfig;
|
cfg?: OpenClawConfig;
|
||||||
}) {
|
}) {
|
||||||
const { applyMediaUnderstanding } = await loadApply();
|
|
||||||
const ctx: MsgContext = {
|
const ctx: MsgContext = {
|
||||||
Body: params.body,
|
Body: params.body,
|
||||||
MediaPath: params.mediaPath,
|
MediaPath: params.mediaPath,
|
||||||
@@ -164,6 +161,10 @@ describe("applyMediaUnderstanding", () => {
|
|||||||
const mockedResolveApiKey = vi.mocked(resolveApiKeyForProvider);
|
const mockedResolveApiKey = vi.mocked(resolveApiKeyForProvider);
|
||||||
const mockedFetchRemoteMedia = vi.mocked(fetchRemoteMedia);
|
const mockedFetchRemoteMedia = vi.mocked(fetchRemoteMedia);
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
({ applyMediaUnderstanding } = await import("./apply.js"));
|
||||||
|
});
|
||||||
|
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
mockedResolveApiKey.mockClear();
|
mockedResolveApiKey.mockClear();
|
||||||
mockedFetchRemoteMedia.mockClear();
|
mockedFetchRemoteMedia.mockClear();
|
||||||
@@ -183,7 +184,6 @@ describe("applyMediaUnderstanding", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("sets Transcript and replaces Body when audio transcription succeeds", async () => {
|
it("sets Transcript and replaces Body when audio transcription succeeds", async () => {
|
||||||
const { applyMediaUnderstanding } = await loadApply();
|
|
||||||
const ctx = await createAudioCtx();
|
const ctx = await createAudioCtx();
|
||||||
const result = await applyMediaUnderstanding({
|
const result = await applyMediaUnderstanding({
|
||||||
ctx,
|
ctx,
|
||||||
@@ -202,7 +202,6 @@ describe("applyMediaUnderstanding", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("skips file blocks for text-like audio when transcription succeeds", async () => {
|
it("skips file blocks for text-like audio when transcription succeeds", async () => {
|
||||||
const { applyMediaUnderstanding } = await loadApply();
|
|
||||||
const ctx = await createAudioCtx({
|
const ctx = await createAudioCtx({
|
||||||
fileName: "data.mp3",
|
fileName: "data.mp3",
|
||||||
mediaType: "audio/mpeg",
|
mediaType: "audio/mpeg",
|
||||||
@@ -221,7 +220,6 @@ describe("applyMediaUnderstanding", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("keeps caption for command parsing when audio has user text", async () => {
|
it("keeps caption for command parsing when audio has user text", async () => {
|
||||||
const { applyMediaUnderstanding } = await loadApply();
|
|
||||||
const ctx = await createAudioCtx({
|
const ctx = await createAudioCtx({
|
||||||
body: "<media:audio> /capture status",
|
body: "<media:audio> /capture status",
|
||||||
});
|
});
|
||||||
@@ -241,7 +239,6 @@ describe("applyMediaUnderstanding", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("handles URL-only attachments for audio transcription", async () => {
|
it("handles URL-only attachments for audio transcription", async () => {
|
||||||
const { applyMediaUnderstanding } = await loadApply();
|
|
||||||
const ctx: MsgContext = {
|
const ctx: MsgContext = {
|
||||||
Body: "<media:audio>",
|
Body: "<media:audio>",
|
||||||
MediaUrl: "https://example.com/note.ogg",
|
MediaUrl: "https://example.com/note.ogg",
|
||||||
@@ -281,7 +278,6 @@ describe("applyMediaUnderstanding", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("skips audio transcription when attachment exceeds maxBytes", async () => {
|
it("skips audio transcription when attachment exceeds maxBytes", async () => {
|
||||||
const { applyMediaUnderstanding } = await loadApply();
|
|
||||||
const ctx = await createAudioCtx({
|
const ctx = await createAudioCtx({
|
||||||
fileName: "large.wav",
|
fileName: "large.wav",
|
||||||
mediaType: "audio/wav",
|
mediaType: "audio/wav",
|
||||||
@@ -312,7 +308,6 @@ describe("applyMediaUnderstanding", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("falls back to CLI model when provider fails", async () => {
|
it("falls back to CLI model when provider fails", async () => {
|
||||||
const { applyMediaUnderstanding } = await loadApply();
|
|
||||||
const ctx = await createAudioCtx();
|
const ctx = await createAudioCtx();
|
||||||
const cfg: OpenClawConfig = {
|
const cfg: OpenClawConfig = {
|
||||||
tools: {
|
tools: {
|
||||||
@@ -357,7 +352,6 @@ describe("applyMediaUnderstanding", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("uses CLI image understanding and preserves caption for commands", async () => {
|
it("uses CLI image understanding and preserves caption for commands", async () => {
|
||||||
const { applyMediaUnderstanding } = await loadApply();
|
|
||||||
const imagePath = await createTempMediaFile({
|
const imagePath = await createTempMediaFile({
|
||||||
fileName: "photo.jpg",
|
fileName: "photo.jpg",
|
||||||
content: "image-bytes",
|
content: "image-bytes",
|
||||||
@@ -405,7 +399,6 @@ describe("applyMediaUnderstanding", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("uses shared media models list when capability config is missing", async () => {
|
it("uses shared media models list when capability config is missing", async () => {
|
||||||
const { applyMediaUnderstanding } = await loadApply();
|
|
||||||
const imagePath = await createTempMediaFile({
|
const imagePath = await createTempMediaFile({
|
||||||
fileName: "shared.jpg",
|
fileName: "shared.jpg",
|
||||||
content: "image-bytes",
|
content: "image-bytes",
|
||||||
@@ -447,7 +440,6 @@ describe("applyMediaUnderstanding", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("uses active model when enabled and models are missing", async () => {
|
it("uses active model when enabled and models are missing", async () => {
|
||||||
const { applyMediaUnderstanding } = await loadApply();
|
|
||||||
const audioPath = await createTempMediaFile({
|
const audioPath = await createTempMediaFile({
|
||||||
fileName: "fallback.ogg",
|
fileName: "fallback.ogg",
|
||||||
content: Buffer.from([0, 255, 0, 1, 2, 3, 4, 5, 6]),
|
content: Buffer.from([0, 255, 0, 1, 2, 3, 4, 5, 6]),
|
||||||
@@ -485,7 +477,6 @@ describe("applyMediaUnderstanding", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("handles multiple audio attachments when attachment mode is all", async () => {
|
it("handles multiple audio attachments when attachment mode is all", async () => {
|
||||||
const { applyMediaUnderstanding } = await loadApply();
|
|
||||||
const dir = await createTempMediaDir();
|
const dir = await createTempMediaDir();
|
||||||
const audioBytes = Buffer.from([200, 201, 202, 203, 204, 205, 206, 207, 208]);
|
const audioBytes = Buffer.from([200, 201, 202, 203, 204, 205, 206, 207, 208]);
|
||||||
const audioPathA = path.join(dir, "note-a.ogg");
|
const audioPathA = path.join(dir, "note-a.ogg");
|
||||||
@@ -529,7 +520,6 @@ describe("applyMediaUnderstanding", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("orders mixed media outputs as image, audio, video", async () => {
|
it("orders mixed media outputs as image, audio, video", async () => {
|
||||||
const { applyMediaUnderstanding } = await loadApply();
|
|
||||||
const dir = await createTempMediaDir();
|
const dir = await createTempMediaDir();
|
||||||
const imagePath = path.join(dir, "photo.jpg");
|
const imagePath = path.join(dir, "photo.jpg");
|
||||||
const audioPath = path.join(dir, "note.ogg");
|
const audioPath = path.join(dir, "note.ogg");
|
||||||
|
|||||||
Reference in New Issue
Block a user