github/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-05-08 09:01:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(agents): dedupe auth profile rotation fixture setup

Browse Source
This commit is contained in:
Peter Steinberger
2026-02-22 09:39:24 +00:00
parent 6b5c20055b
commit 7d13227d41
1 changed files with 81 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

181
src/agents/pi-embedded-runner.run-embedded-pi-agent.auth-profile-rotation.e2e.test.ts
View File

@@ -12,6 +12,14 @@ vi.mock("./pi-embedded-runner/run/attempt.js", () => ({
runEmbeddedAttempt: (params: unknown) => runEmbeddedAttemptMock(params), runEmbeddedAttempt: (params: unknown) => runEmbeddedAttemptMock(params),
})); }));
vi.mock("./models-config.js", async (importOriginal) => {
const mod = await importOriginal<typeof import("./models-config.js")>();
return {
...mod,
ensureOpenClawModelsJson: vi.fn(async () => ({ wrote: false })),
};
});
let runEmbeddedPiAgent: typeof import("./pi-embedded-runner.js").runEmbeddedPiAgent; let runEmbeddedPiAgent: typeof import("./pi-embedded-runner.js").runEmbeddedPiAgent;
beforeAll(async () => { beforeAll(async () => {
@@ -235,6 +243,19 @@ async function withTimedAgentWorkspace<T>(
} }
} }
async function withAgentWorkspace<T>(
run: (ctx: { agentDir: string; workspaceDir: string }) => Promise<T>,
) {
const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-agent-"));
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-workspace-"));
try {
return await run({ agentDir, workspaceDir });
} finally {
await fs.rm(agentDir, { recursive: true, force: true });
await fs.rm(workspaceDir, { recursive: true, force: true });
}
}
async function runTurnWithCooldownSeed(params: { async function runTurnWithCooldownSeed(params: {
sessionKey: string; sessionKey: string;
runId: string; runId: string;
@@ -288,9 +309,7 @@ describe("runEmbeddedPiAgent auth profile rotation", () => {
for (const testCase of cases) { for (const testCase of cases) {
runEmbeddedAttemptMock.mockClear(); runEmbeddedAttemptMock.mockClear();
const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-agent-")); await withAgentWorkspace(async ({ agentDir, workspaceDir }) => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-workspace-"));
try {
await writeAuthStore(agentDir); await writeAuthStore(agentDir);
mockFailedThenSuccessfulAttempt(testCase.errorMessage); mockFailedThenSuccessfulAttempt(testCase.errorMessage);
await runAutoPinnedOpenAiTurn({ await runAutoPinnedOpenAiTurn({
@@ -302,17 +321,12 @@ describe("runEmbeddedPiAgent auth profile rotation", () => {
expect(runEmbeddedAttemptMock).toHaveBeenCalledTimes(2); expect(runEmbeddedAttemptMock).toHaveBeenCalledTimes(2);
await expectProfileP2UsageUpdated(agentDir); await expectProfileP2UsageUpdated(agentDir);
} finally { });
await fs.rm(agentDir, { recursive: true, force: true });
await fs.rm(workspaceDir, { recursive: true, force: true });
}
} }
}); });
it("does not rotate for compaction timeouts", async () => { it("does not rotate for compaction timeouts", async () => {
const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-agent-")); await withAgentWorkspace(async ({ agentDir, workspaceDir }) => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-workspace-"));
try {
await writeAuthStore(agentDir); await writeAuthStore(agentDir);
runEmbeddedAttemptMock.mockResolvedValueOnce( runEmbeddedAttemptMock.mockResolvedValueOnce(
@@ -348,16 +362,11 @@ describe("runEmbeddedPiAgent auth profile rotation", () => {
expect(result.meta.aborted).toBe(true); expect(result.meta.aborted).toBe(true);
await expectProfileP2UsageUnchanged(agentDir); await expectProfileP2UsageUnchanged(agentDir);
} finally { });
await fs.rm(agentDir, { recursive: true, force: true });
await fs.rm(workspaceDir, { recursive: true, force: true });
}
}); });
it("does not rotate for user-pinned profiles", async () => { it("does not rotate for user-pinned profiles", async () => {
const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-agent-")); await withAgentWorkspace(async ({ agentDir, workspaceDir }) => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-workspace-"));
try {
await writeAuthStore(agentDir); await writeAuthStore(agentDir);
mockSingleErrorAttempt({ errorMessage: "rate limit" }); mockSingleErrorAttempt({ errorMessage: "rate limit" });
@@ -380,10 +389,7 @@ describe("runEmbeddedPiAgent auth profile rotation", () => {
expect(runEmbeddedAttemptMock).toHaveBeenCalledTimes(1); expect(runEmbeddedAttemptMock).toHaveBeenCalledTimes(1);
await expectProfileP2UsageUnchanged(agentDir); await expectProfileP2UsageUnchanged(agentDir);
} finally { });
await fs.rm(agentDir, { recursive: true, force: true });
await fs.rm(workspaceDir, { recursive: true, force: true });
}
}); });
it("honors user-pinned profiles even when in cooldown", async () => { it("honors user-pinned profiles even when in cooldown", async () => {
@@ -400,9 +406,7 @@ describe("runEmbeddedPiAgent auth profile rotation", () => {
}); });
it("ignores user-locked profile when provider mismatches", async () => { it("ignores user-locked profile when provider mismatches", async () => {
const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-agent-")); await withAgentWorkspace(async ({ agentDir, workspaceDir }) => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-workspace-"));
try {
await writeAuthStore(agentDir, { includeAnthropic: true }); await writeAuthStore(agentDir, { includeAnthropic: true });
runEmbeddedAttemptMock.mockResolvedValueOnce( runEmbeddedAttemptMock.mockResolvedValueOnce(
@@ -432,10 +436,7 @@ describe("runEmbeddedPiAgent auth profile rotation", () => {
}); });
expect(runEmbeddedAttemptMock).toHaveBeenCalledTimes(1); expect(runEmbeddedAttemptMock).toHaveBeenCalledTimes(1);
} finally { });
await fs.rm(agentDir, { recursive: true, force: true });
await fs.rm(workspaceDir, { recursive: true, force: true });
}
}); });
it("skips profiles in cooldown during initial selection", async () => { it("skips profiles in cooldown during initial selection", async () => {
@@ -486,47 +487,43 @@ describe("runEmbeddedPiAgent auth profile rotation", () => {
}); });
it("fails over when auth is unavailable and fallbacks are configured", async () => { it("fails over when auth is unavailable and fallbacks are configured", async () => {
const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-agent-"));
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-workspace-"));
const previousOpenAiKey = process.env.OPENAI_API_KEY; const previousOpenAiKey = process.env.OPENAI_API_KEY;
delete process.env.OPENAI_API_KEY; delete process.env.OPENAI_API_KEY;
try { try {
const authPath = path.join(agentDir, "auth-profiles.json"); await withAgentWorkspace(async ({ agentDir, workspaceDir }) => {
await fs.writeFile(authPath, JSON.stringify({ version: 1, profiles: {}, usageStats: {} })); const authPath = path.join(agentDir, "auth-profiles.json");
await fs.writeFile(authPath, JSON.stringify({ version: 1, profiles: {}, usageStats: {} }));
await expect( await expect(
runEmbeddedPiAgent({ runEmbeddedPiAgent({
sessionId: "session:test", sessionId: "session:test",
sessionKey: "agent:test:auth-unavailable", sessionKey: "agent:test:auth-unavailable",
sessionFile: path.join(workspaceDir, "session.jsonl"), sessionFile: path.join(workspaceDir, "session.jsonl"),
workspaceDir, workspaceDir,
agentDir, agentDir,
config: makeConfig({ fallbacks: ["openai/mock-2"], apiKey: "" }), config: makeConfig({ fallbacks: ["openai/mock-2"], apiKey: "" }),
prompt: "hello", prompt: "hello",
provider: "openai", provider: "openai",
model: "mock-1", model: "mock-1",
authProfileIdSource: "auto", authProfileIdSource: "auto",
timeoutMs: 5_000, timeoutMs: 5_000,
runId: "run:auth-unavailable", runId: "run:auth-unavailable",
}), }),
).rejects.toMatchObject({ name: "FailoverError", reason: "auth" }); ).rejects.toMatchObject({ name: "FailoverError", reason: "auth" });
expect(runEmbeddedAttemptMock).not.toHaveBeenCalled(); expect(runEmbeddedAttemptMock).not.toHaveBeenCalled();
});
} finally { } finally {
if (previousOpenAiKey === undefined) { if (previousOpenAiKey === undefined) {
delete process.env.OPENAI_API_KEY; delete process.env.OPENAI_API_KEY;
} else { } else {
process.env.OPENAI_API_KEY = previousOpenAiKey; process.env.OPENAI_API_KEY = previousOpenAiKey;
} }
await fs.rm(agentDir, { recursive: true, force: true });
await fs.rm(workspaceDir, { recursive: true, force: true });
} }
}); });
it("uses the active erroring model in billing failover errors", async () => { it("uses the active erroring model in billing failover errors", async () => {
const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-agent-")); await withAgentWorkspace(async ({ agentDir, workspaceDir }) => {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-workspace-"));
try {
await writeAuthStore(agentDir); await writeAuthStore(agentDir);
mockSingleErrorAttempt({ mockSingleErrorAttempt({
errorMessage: "insufficient credits", errorMessage: "insufficient credits",
@@ -564,56 +561,40 @@ describe("runEmbeddedPiAgent auth profile rotation", () => {
expect(thrown).toBeInstanceOf(Error); expect(thrown).toBeInstanceOf(Error);
expect((thrown as Error).message).toContain("openai (mock-rotated) returned a billing error"); expect((thrown as Error).message).toContain("openai (mock-rotated) returned a billing error");
expect(runEmbeddedAttemptMock).toHaveBeenCalledTimes(1); expect(runEmbeddedAttemptMock).toHaveBeenCalledTimes(1);
} finally { });
await fs.rm(agentDir, { recursive: true, force: true });
await fs.rm(workspaceDir, { recursive: true, force: true });
}
}); });
it("skips profiles in cooldown when rotating after failure", async () => { it("skips profiles in cooldown when rotating after failure", async () => {
vi.useFakeTimers(); await withTimedAgentWorkspace(async ({ agentDir, workspaceDir, now }) => {
try { const authPath = path.join(agentDir, "auth-profiles.json");
const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-agent-")); const payload = {
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-workspace-")); version: 1,
const now = Date.now(); profiles: {
vi.setSystemTime(now); "openai:p1": { type: "api_key", provider: "openai", key: "sk-one" },
"openai:p2": { type: "api_key", provider: "openai", key: "sk-two" },
"openai:p3": { type: "api_key", provider: "openai", key: "sk-three" },
},
usageStats: {
"openai:p1": { lastUsed: 1 },
"openai:p2": { cooldownUntil: now + 60 * 60 * 1000 }, // p2 in cooldown
"openai:p3": { lastUsed: 3 },
},
};
await fs.writeFile(authPath, JSON.stringify(payload));
try { mockFailedThenSuccessfulAttempt("rate limit");
const authPath = path.join(agentDir, "auth-profiles.json"); await runAutoPinnedOpenAiTurn({
const payload = { agentDir,
version: 1, workspaceDir,
profiles: { sessionKey: "agent:test:rotate-skip-cooldown",
"openai:p1": { type: "api_key", provider: "openai", key: "sk-one" }, runId: "run:rotate-skip-cooldown",
"openai:p2": { type: "api_key", provider: "openai", key: "sk-two" }, });
"openai:p3": { type: "api_key", provider: "openai", key: "sk-three" },
},
usageStats: {
"openai:p1": { lastUsed: 1 },
"openai:p2": { cooldownUntil: now + 60 * 60 * 1000 }, // p2 in cooldown
"openai:p3": { lastUsed: 3 },
},
};
await fs.writeFile(authPath, JSON.stringify(payload));
mockFailedThenSuccessfulAttempt("rate limit"); expect(runEmbeddedAttemptMock).toHaveBeenCalledTimes(2);
await runAutoPinnedOpenAiTurn({ const usageStats = await readUsageStats(agentDir);
agentDir, expect(typeof usageStats["openai:p1"]?.lastUsed).toBe("number");
workspaceDir, expect(typeof usageStats["openai:p3"]?.lastUsed).toBe("number");
sessionKey: "agent:test:rotate-skip-cooldown", expect(usageStats["openai:p2"]?.cooldownUntil).toBe(now + 60 * 60 * 1000);
runId: "run:rotate-skip-cooldown", });
});
expect(runEmbeddedAttemptMock).toHaveBeenCalledTimes(2);
const usageStats = await readUsageStats(agentDir);
expect(typeof usageStats["openai:p1"]?.lastUsed).toBe("number");
expect(typeof usageStats["openai:p3"]?.lastUsed).toBe("number");
expect(usageStats["openai:p2"]?.cooldownUntil).toBe(now + 60 * 60 * 1000);
} finally {
await fs.rm(agentDir, { recursive: true, force: true });
await fs.rm(workspaceDir, { recursive: true, force: true });
}
} finally {
vi.useRealTimers();
}
}); });
}); });