perf(test): reduce fixture churn in hot suites

This commit is contained in:
Peter Steinberger
2026-02-13 22:40:20 +00:00
parent dac8f5ba3f
commit e324cb5b94
5 changed files with 237 additions and 133 deletions

View File

@@ -1,6 +1,7 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path"; import path from "node:path";
import { beforeEach, describe, expect, it, vi } from "vitest"; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { withTempHome as withTempHomeBase } from "../../test/helpers/temp-home.js";
import { loadModelCatalog } from "../agents/model-catalog.js"; import { loadModelCatalog } from "../agents/model-catalog.js";
import { getReplyFromConfig } from "./reply.js"; import { getReplyFromConfig } from "./reply.js";
@@ -22,11 +23,69 @@ vi.mock("../agents/model-catalog.js", () => ({
loadModelCatalog: vi.fn(), loadModelCatalog: vi.fn(),
})); }));
type HomeEnvSnapshot = {
HOME: string | undefined;
USERPROFILE: string | undefined;
HOMEDRIVE: string | undefined;
HOMEPATH: string | undefined;
OPENCLAW_STATE_DIR: string | undefined;
};
function snapshotHomeEnv(): HomeEnvSnapshot {
return {
HOME: process.env.HOME,
USERPROFILE: process.env.USERPROFILE,
HOMEDRIVE: process.env.HOMEDRIVE,
HOMEPATH: process.env.HOMEPATH,
OPENCLAW_STATE_DIR: process.env.OPENCLAW_STATE_DIR,
};
}
function restoreHomeEnv(snapshot: HomeEnvSnapshot) {
for (const [key, value] of Object.entries(snapshot)) {
if (value === undefined) {
delete process.env[key];
} else {
process.env[key] = value;
}
}
}
let fixtureRoot = "";
let caseId = 0;
async function withTempHome<T>(fn: (home: string) => Promise<T>): Promise<T> { async function withTempHome<T>(fn: (home: string) => Promise<T>): Promise<T> {
return withTempHomeBase(fn, { prefix: "openclaw-stream-" }); const home = path.join(fixtureRoot, `case-${++caseId}`);
await fs.mkdir(path.join(home, ".openclaw", "agents", "main", "sessions"), { recursive: true });
const envSnapshot = snapshotHomeEnv();
process.env.HOME = home;
process.env.USERPROFILE = home;
process.env.OPENCLAW_STATE_DIR = path.join(home, ".openclaw");
if (process.platform === "win32") {
const match = home.match(/^([A-Za-z]:)(.*)$/);
if (match) {
process.env.HOMEDRIVE = match[1];
process.env.HOMEPATH = match[2] || "\\";
}
}
try {
return await fn(home);
} finally {
restoreHomeEnv(envSnapshot);
}
} }
describe("block streaming", () => { describe("block streaming", () => {
beforeAll(async () => {
fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-stream-"));
});
afterAll(async () => {
await fs.rm(fixtureRoot, { recursive: true, force: true });
});
beforeEach(() => { beforeEach(() => {
piEmbeddedMock.abortEmbeddedPiRun.mockReset().mockReturnValue(false); piEmbeddedMock.abortEmbeddedPiRun.mockReset().mockReturnValue(false);
piEmbeddedMock.queueEmbeddedPiMessage.mockReset().mockReturnValue(false); piEmbeddedMock.queueEmbeddedPiMessage.mockReset().mockReturnValue(false);

View File

@@ -1,7 +1,7 @@
import fs from "node:fs/promises"; import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path"; import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { withTempHome as withTempHomeBase } from "../../test/helpers/temp-home.js";
import { loadModelCatalog } from "../agents/model-catalog.js"; import { loadModelCatalog } from "../agents/model-catalog.js";
import { runEmbeddedPiAgent } from "../agents/pi-embedded.js"; import { runEmbeddedPiAgent } from "../agents/pi-embedded.js";
import { saveSessionStore } from "../config/sessions.js"; import { saveSessionStore } from "../config/sessions.js";
@@ -19,22 +19,78 @@ vi.mock("../agents/model-catalog.js", () => ({
loadModelCatalog: vi.fn(), loadModelCatalog: vi.fn(),
})); }));
type HomeEnvSnapshot = {
HOME: string | undefined;
USERPROFILE: string | undefined;
HOMEDRIVE: string | undefined;
HOMEPATH: string | undefined;
OPENCLAW_STATE_DIR: string | undefined;
OPENCLAW_AGENT_DIR: string | undefined;
PI_CODING_AGENT_DIR: string | undefined;
};
function snapshotHomeEnv(): HomeEnvSnapshot {
return {
HOME: process.env.HOME,
USERPROFILE: process.env.USERPROFILE,
HOMEDRIVE: process.env.HOMEDRIVE,
HOMEPATH: process.env.HOMEPATH,
OPENCLAW_STATE_DIR: process.env.OPENCLAW_STATE_DIR,
OPENCLAW_AGENT_DIR: process.env.OPENCLAW_AGENT_DIR,
PI_CODING_AGENT_DIR: process.env.PI_CODING_AGENT_DIR,
};
}
function restoreHomeEnv(snapshot: HomeEnvSnapshot) {
for (const [key, value] of Object.entries(snapshot)) {
if (value === undefined) {
delete process.env[key];
} else {
process.env[key] = value;
}
}
}
let fixtureRoot = "";
let caseId = 0;
async function withTempHome<T>(fn: (home: string) => Promise<T>): Promise<T> { async function withTempHome<T>(fn: (home: string) => Promise<T>): Promise<T> {
return withTempHomeBase( const home = path.join(fixtureRoot, `case-${++caseId}`);
async (home) => { await fs.mkdir(path.join(home, ".openclaw", "agents", "main", "sessions"), { recursive: true });
return await fn(home); const envSnapshot = snapshotHomeEnv();
}, process.env.HOME = home;
{ process.env.USERPROFILE = home;
env: { process.env.OPENCLAW_STATE_DIR = path.join(home, ".openclaw");
OPENCLAW_AGENT_DIR: (home) => path.join(home, ".openclaw", "agent"), process.env.OPENCLAW_AGENT_DIR = path.join(home, ".openclaw", "agent");
PI_CODING_AGENT_DIR: (home) => path.join(home, ".openclaw", "agent"), process.env.PI_CODING_AGENT_DIR = path.join(home, ".openclaw", "agent");
},
prefix: "openclaw-rawbody-", if (process.platform === "win32") {
}, const match = home.match(/^([A-Za-z]:)(.*)$/);
); if (match) {
process.env.HOMEDRIVE = match[1];
process.env.HOMEPATH = match[2] || "\\";
}
}
try {
return await fn(home);
} finally {
restoreHomeEnv(envSnapshot);
}
} }
describe("RawBody directive parsing", () => { describe("RawBody directive parsing", () => {
type ReplyMessage = Parameters<typeof getReplyFromConfig>[0];
type ReplyConfig = Parameters<typeof getReplyFromConfig>[2];
beforeAll(async () => {
fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-rawbody-"));
});
afterAll(async () => {
await fs.rm(fixtureRoot, { recursive: true, force: true });
});
beforeEach(() => { beforeEach(() => {
vi.mocked(runEmbeddedPiAgent).mockReset(); vi.mocked(runEmbeddedPiAgent).mockReset();
vi.mocked(loadModelCatalog).mockResolvedValue([ vi.mocked(loadModelCatalog).mockResolvedValue([
@@ -46,147 +102,116 @@ describe("RawBody directive parsing", () => {
vi.clearAllMocks(); vi.clearAllMocks();
}); });
it("/model, /think, /verbose directives detected from RawBody even when Body has structural wrapper", async () => { it("detects command directives from RawBody/CommandBody in wrapped group messages", async () => {
await withTempHome(async (home) => { await withTempHome(async (home) => {
vi.mocked(runEmbeddedPiAgent).mockReset(); const assertCommandReply = async (input: {
message: ReplyMessage;
const groupMessageCtx = { config: ReplyConfig;
Body: `[Chat messages since your last reply - for context]\\n[WhatsApp ...] Someone: hello\\n\\n[Current message - respond to this]\\n[WhatsApp ...] Jake: /think:high\\n[from: Jake McInteer (+6421807830)]`, expectedIncludes: string[];
RawBody: "/think:high", }) => {
From: "+1222", vi.mocked(runEmbeddedPiAgent).mockReset();
To: "+1222", const res = await getReplyFromConfig(input.message, {}, input.config);
ChatType: "group", const text = Array.isArray(res) ? res[0]?.text : res?.text;
CommandAuthorized: true, for (const expected of input.expectedIncludes) {
expect(text).toContain(expected);
}
expect(runEmbeddedPiAgent).not.toHaveBeenCalled();
}; };
const res = await getReplyFromConfig( await assertCommandReply({
groupMessageCtx, message: {
{}, Body: `[Chat messages since your last reply - for context]\\n[WhatsApp ...] Someone: hello\\n\\n[Current message - respond to this]\\n[WhatsApp ...] Jake: /think:high\\n[from: Jake McInteer (+6421807830)]`,
{ RawBody: "/think:high",
From: "+1222",
To: "+1222",
ChatType: "group",
CommandAuthorized: true,
},
config: {
agents: { agents: {
defaults: { defaults: {
model: "anthropic/claude-opus-4-5", model: "anthropic/claude-opus-4-5",
workspace: path.join(home, "openclaw"), workspace: path.join(home, "openclaw-1"),
}, },
}, },
channels: { whatsapp: { allowFrom: ["*"] } }, channels: { whatsapp: { allowFrom: ["*"] } },
session: { store: path.join(home, "sessions.json") }, session: { store: path.join(home, "sessions-1.json") },
}, },
); expectedIncludes: ["Thinking level set to high."],
});
const text = Array.isArray(res) ? res[0]?.text : res?.text; await assertCommandReply({
expect(text).toContain("Thinking level set to high."); message: {
expect(runEmbeddedPiAgent).not.toHaveBeenCalled(); Body: "[Context]\nJake: /model status\n[from: Jake]",
}); RawBody: "/model status",
}); From: "+1222",
To: "+1222",
it("/model status detected from RawBody", async () => { ChatType: "group",
await withTempHome(async (home) => { CommandAuthorized: true,
vi.mocked(runEmbeddedPiAgent).mockReset(); },
config: {
const groupMessageCtx = {
Body: `[Context]\nJake: /model status\n[from: Jake]`,
RawBody: "/model status",
From: "+1222",
To: "+1222",
ChatType: "group",
CommandAuthorized: true,
};
const res = await getReplyFromConfig(
groupMessageCtx,
{},
{
agents: { agents: {
defaults: { defaults: {
model: "anthropic/claude-opus-4-5", model: "anthropic/claude-opus-4-5",
workspace: path.join(home, "openclaw"), workspace: path.join(home, "openclaw-2"),
models: { models: {
"anthropic/claude-opus-4-5": {}, "anthropic/claude-opus-4-5": {},
}, },
}, },
}, },
channels: { whatsapp: { allowFrom: ["*"] } }, channels: { whatsapp: { allowFrom: ["*"] } },
session: { store: path.join(home, "sessions.json") }, session: { store: path.join(home, "sessions-2.json") },
}, },
); expectedIncludes: ["anthropic/claude-opus-4-5"],
});
const text = Array.isArray(res) ? res[0]?.text : res?.text; await assertCommandReply({
expect(text).toContain("anthropic/claude-opus-4-5"); message: {
expect(runEmbeddedPiAgent).not.toHaveBeenCalled(); Body: "[Context]\nJake: /verbose on\n[from: Jake]",
}); CommandBody: "/verbose on",
}); From: "+1222",
To: "+1222",
it("CommandBody is honored when RawBody is missing", async () => { ChatType: "group",
await withTempHome(async (home) => { CommandAuthorized: true,
vi.mocked(runEmbeddedPiAgent).mockReset(); },
config: {
const groupMessageCtx = {
Body: `[Context]\nJake: /verbose on\n[from: Jake]`,
CommandBody: "/verbose on",
From: "+1222",
To: "+1222",
ChatType: "group",
CommandAuthorized: true,
};
const res = await getReplyFromConfig(
groupMessageCtx,
{},
{
agents: { agents: {
defaults: { defaults: {
model: "anthropic/claude-opus-4-5", model: "anthropic/claude-opus-4-5",
workspace: path.join(home, "openclaw"), workspace: path.join(home, "openclaw-3"),
}, },
}, },
channels: { whatsapp: { allowFrom: ["*"] } }, channels: { whatsapp: { allowFrom: ["*"] } },
session: { store: path.join(home, "sessions.json") }, session: { store: path.join(home, "sessions-3.json") },
}, },
); expectedIncludes: ["Verbose logging enabled."],
});
const text = Array.isArray(res) ? res[0]?.text : res?.text; await assertCommandReply({
expect(text).toContain("Verbose logging enabled."); message: {
expect(runEmbeddedPiAgent).not.toHaveBeenCalled(); Body: `[Chat messages since your last reply - for context]\\n[WhatsApp ...] Someone: hello\\n\\n[Current message - respond to this]\\n[WhatsApp ...] Jake: /status\\n[from: Jake McInteer (+6421807830)]`,
}); RawBody: "/status",
}); ChatType: "group",
From: "+1222",
it("Integration: WhatsApp group message with structural wrapper and RawBody command", async () => { To: "+1222",
await withTempHome(async (home) => { SessionKey: "agent:main:whatsapp:group:g1",
vi.mocked(runEmbeddedPiAgent).mockReset(); Provider: "whatsapp",
Surface: "whatsapp",
const groupMessageCtx = { SenderE164: "+1222",
Body: `[Chat messages since your last reply - for context]\\n[WhatsApp ...] Someone: hello\\n\\n[Current message - respond to this]\\n[WhatsApp ...] Jake: /status\\n[from: Jake McInteer (+6421807830)]`, CommandAuthorized: true,
RawBody: "/status", },
ChatType: "group", config: {
From: "+1222",
To: "+1222",
SessionKey: "agent:main:whatsapp:group:g1",
Provider: "whatsapp",
Surface: "whatsapp",
SenderE164: "+1222",
CommandAuthorized: true,
};
const res = await getReplyFromConfig(
groupMessageCtx,
{},
{
agents: { agents: {
defaults: { defaults: {
model: "anthropic/claude-opus-4-5", model: "anthropic/claude-opus-4-5",
workspace: path.join(home, "openclaw"), workspace: path.join(home, "openclaw-4"),
}, },
}, },
channels: { whatsapp: { allowFrom: ["+1222"] } }, channels: { whatsapp: { allowFrom: ["+1222"] } },
session: { store: path.join(home, "sessions.json") }, session: { store: path.join(home, "sessions-4.json") },
}, },
); expectedIncludes: ["Session: agent:main:whatsapp:group:g1", "anthropic/claude-opus-4-5"],
});
const text = Array.isArray(res) ? res[0]?.text : res?.text;
expect(text).toContain("Session: agent:main:whatsapp:group:g1");
expect(text).toContain("anthropic/claude-opus-4-5");
expect(runEmbeddedPiAgent).not.toHaveBeenCalled();
}); });
}); });

View File

@@ -144,6 +144,7 @@ describe("memory index", () => {
throw new Error("manager missing"); throw new Error("manager missing");
} }
await first.manager.sync({ force: true }); await first.manager.sync({ force: true });
const callsAfterFirstSync = embedBatchCalls;
await first.manager.close(); await first.manager.close();
const second = await getMemorySearchManager({ const second = await getMemorySearchManager({
@@ -168,8 +169,9 @@ describe("memory index", () => {
} }
manager = second.manager; manager = second.manager;
await second.manager.sync({ reason: "test" }); await second.manager.sync({ reason: "test" });
const results = await second.manager.search("alpha"); expect(embedBatchCalls).toBeGreaterThan(callsAfterFirstSync);
expect(results.length).toBeGreaterThan(0); const status = second.manager.status();
expect(status.files).toBeGreaterThan(0);
}); });
it("reuses cached embeddings on forced reindex", async () => { it("reuses cached embeddings on forced reindex", async () => {
@@ -280,7 +282,7 @@ describe("memory index", () => {
}); });
it("hybrid weights can favor vector-only matches over keyword-only matches", async () => { it("hybrid weights can favor vector-only matches over keyword-only matches", async () => {
const manyAlpha = Array.from({ length: 80 }, () => "Alpha").join(" "); const manyAlpha = Array.from({ length: 50 }, () => "Alpha").join(" ");
await fs.writeFile( await fs.writeFile(
path.join(workspaceDir, "memory", "vector-only.md"), path.join(workspaceDir, "memory", "vector-only.md"),
"Alpha beta. Alpha beta. Alpha beta. Alpha beta.", "Alpha beta. Alpha beta. Alpha beta. Alpha beta.",
@@ -338,7 +340,7 @@ describe("memory index", () => {
}); });
it("hybrid weights can favor keyword matches when text weight dominates", async () => { it("hybrid weights can favor keyword matches when text weight dominates", async () => {
const manyAlpha = Array.from({ length: 80 }, () => "Alpha").join(" "); const manyAlpha = Array.from({ length: 50 }, () => "Alpha").join(" ");
await fs.writeFile( await fs.writeFile(
path.join(workspaceDir, "memory", "vector-only.md"), path.join(workspaceDir, "memory", "vector-only.md"),
"Alpha beta. Alpha beta. Alpha beta. Alpha beta.", "Alpha beta. Alpha beta. Alpha beta. Alpha beta.",

View File

@@ -1,7 +1,7 @@
import fs from "node:fs/promises"; import fs from "node:fs/promises";
import os from "node:os"; import os from "node:os";
import path from "node:path"; import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { getMemorySearchManager, type MemoryIndexManager } from "./index.js"; import { getMemorySearchManager, type MemoryIndexManager } from "./index.js";
const embedBatch = vi.fn(async () => []); const embedBatch = vi.fn(async () => []);
@@ -25,11 +25,21 @@ vi.mock("./embeddings.js", () => ({
})); }));
describe("memory indexing with OpenAI batches", () => { describe("memory indexing with OpenAI batches", () => {
let fixtureRoot: string;
let caseId = 0;
let workspaceDir: string; let workspaceDir: string;
let indexPath: string; let indexPath: string;
let manager: MemoryIndexManager | null = null; let manager: MemoryIndexManager | null = null;
let setTimeoutSpy: ReturnType<typeof vi.spyOn>; let setTimeoutSpy: ReturnType<typeof vi.spyOn>;
beforeAll(async () => {
fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-mem-batch-"));
});
afterAll(async () => {
await fs.rm(fixtureRoot, { recursive: true, force: true });
});
beforeEach(async () => { beforeEach(async () => {
embedBatch.mockClear(); embedBatch.mockClear();
embedQuery.mockClear(); embedQuery.mockClear();
@@ -48,9 +58,9 @@ describe("memory indexing with OpenAI batches", () => {
} }
return realSetTimeout(handler, delay, ...args); return realSetTimeout(handler, delay, ...args);
}) as typeof setTimeout); }) as typeof setTimeout);
workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-mem-batch-")); workspaceDir = path.join(fixtureRoot, `case-${++caseId}`);
indexPath = path.join(workspaceDir, "index.sqlite"); indexPath = path.join(workspaceDir, "index.sqlite");
await fs.mkdir(path.join(workspaceDir, "memory")); await fs.mkdir(path.join(workspaceDir, "memory"), { recursive: true });
}); });
afterEach(async () => { afterEach(async () => {
@@ -60,7 +70,6 @@ describe("memory indexing with OpenAI batches", () => {
await manager.close(); await manager.close();
manager = null; manager = null;
} }
await fs.rm(workspaceDir, { recursive: true, force: true });
}); });
it("uses OpenAI batch uploads when enabled", async () => { it("uses OpenAI batch uploads when enabled", async () => {

View File

@@ -1,7 +1,7 @@
import fs from "node:fs/promises"; import fs from "node:fs/promises";
import os from "node:os"; import os from "node:os";
import path from "node:path"; import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { getMemorySearchManager, type MemoryIndexManager } from "./index.js"; import { getMemorySearchManager, type MemoryIndexManager } from "./index.js";
const embedBatch = vi.fn(async (texts: string[]) => texts.map(() => [0, 1, 0])); const embedBatch = vi.fn(async (texts: string[]) => texts.map(() => [0, 1, 0]));
@@ -20,16 +20,26 @@ vi.mock("./embeddings.js", () => ({
})); }));
describe("memory embedding batches", () => { describe("memory embedding batches", () => {
let fixtureRoot: string;
let caseId = 0;
let workspaceDir: string; let workspaceDir: string;
let indexPath: string; let indexPath: string;
let manager: MemoryIndexManager | null = null; let manager: MemoryIndexManager | null = null;
beforeAll(async () => {
fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-mem-"));
});
afterAll(async () => {
await fs.rm(fixtureRoot, { recursive: true, force: true });
});
beforeEach(async () => { beforeEach(async () => {
embedBatch.mockClear(); embedBatch.mockClear();
embedQuery.mockClear(); embedQuery.mockClear();
workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-mem-")); workspaceDir = path.join(fixtureRoot, `case-${++caseId}`);
indexPath = path.join(workspaceDir, "index.sqlite"); indexPath = path.join(workspaceDir, "index.sqlite");
await fs.mkdir(path.join(workspaceDir, "memory")); await fs.mkdir(path.join(workspaceDir, "memory"), { recursive: true });
}); });
afterEach(async () => { afterEach(async () => {
@@ -37,7 +47,6 @@ describe("memory embedding batches", () => {
await manager.close(); await manager.close();
manager = null; manager = null;
} }
await fs.rm(workspaceDir, { recursive: true, force: true });
}); });
it("splits large files across multiple embedding batches", async () => { it("splits large files across multiple embedding batches", async () => {