diff --git a/src/agents/model-auth.e2e.test.ts b/src/agents/model-auth.e2e.test.ts index 40c483f91e3..71fba9d177b 100644 --- a/src/agents/model-auth.e2e.test.ts +++ b/src/agents/model-auth.e2e.test.ts @@ -67,6 +67,25 @@ async function resolveBedrockProvider() { }); } +async function withEnvUpdates( + updates: Record, + run: () => Promise, +): Promise { + const snapshot = captureEnv(Object.keys(updates)); + try { + for (const [key, value] of Object.entries(updates)) { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } + return await run(); + } finally { + snapshot.restore(); + } +} + describe("getApiKeyForModel", () => { it("migrates legacy oauth.json into auth-profiles.json", async () => { const envSnapshot = captureEnv([ @@ -187,127 +206,75 @@ describe("getApiKeyForModel", () => { }); it("throws when ZAI API key is missing", async () => { - const previousZai = process.env.ZAI_API_KEY; - const previousLegacy = process.env.Z_AI_API_KEY; + await withEnvUpdates( + { + ZAI_API_KEY: undefined, + Z_AI_API_KEY: undefined, + }, + async () => { + let error: unknown = null; + try { + await resolveApiKeyForProvider({ + provider: "zai", + store: { version: 1, profiles: {} }, + }); + } catch (err) { + error = err; + } - try { - delete process.env.ZAI_API_KEY; - delete process.env.Z_AI_API_KEY; - - let error: unknown = null; - try { - await resolveApiKeyForProvider({ - provider: "zai", - store: { version: 1, profiles: {} }, - }); - } catch (err) { - error = err; - } - - expect(String(error)).toContain('No API key found for provider "zai".'); - } finally { - if (previousZai === undefined) { - delete process.env.ZAI_API_KEY; - } else { - process.env.ZAI_API_KEY = previousZai; - } - if (previousLegacy === undefined) { - delete process.env.Z_AI_API_KEY; - } else { - process.env.Z_AI_API_KEY = previousLegacy; - } - } + expect(String(error)).toContain('No API key found for provider "zai".'); + }, + ); }); it("accepts legacy Z_AI_API_KEY for zai", async () => { - const previousZai = process.env.ZAI_API_KEY; - const previousLegacy = process.env.Z_AI_API_KEY; - - try { - delete process.env.ZAI_API_KEY; - process.env.Z_AI_API_KEY = "zai-test-key"; - - const resolved = await resolveApiKeyForProvider({ - provider: "zai", - store: { version: 1, profiles: {} }, - }); - expect(resolved.apiKey).toBe("zai-test-key"); - expect(resolved.source).toContain("Z_AI_API_KEY"); - } finally { - if (previousZai === undefined) { - delete process.env.ZAI_API_KEY; - } else { - process.env.ZAI_API_KEY = previousZai; - } - if (previousLegacy === undefined) { - delete process.env.Z_AI_API_KEY; - } else { - process.env.Z_AI_API_KEY = previousLegacy; - } - } + await withEnvUpdates( + { + ZAI_API_KEY: undefined, + Z_AI_API_KEY: "zai-test-key", + }, + async () => { + const resolved = await resolveApiKeyForProvider({ + provider: "zai", + store: { version: 1, profiles: {} }, + }); + expect(resolved.apiKey).toBe("zai-test-key"); + expect(resolved.source).toContain("Z_AI_API_KEY"); + }, + ); }); it("resolves Synthetic API key from env", async () => { - const previousSynthetic = process.env.SYNTHETIC_API_KEY; - - try { - process.env.SYNTHETIC_API_KEY = "synthetic-test-key"; - + await withEnvUpdates({ SYNTHETIC_API_KEY: "synthetic-test-key" }, async () => { const resolved = await resolveApiKeyForProvider({ provider: "synthetic", store: { version: 1, profiles: {} }, }); expect(resolved.apiKey).toBe("synthetic-test-key"); expect(resolved.source).toContain("SYNTHETIC_API_KEY"); - } finally { - if (previousSynthetic === undefined) { - delete process.env.SYNTHETIC_API_KEY; - } else { - process.env.SYNTHETIC_API_KEY = previousSynthetic; - } - } + }); }); it("resolves Qianfan API key from env", async () => { - const previous = process.env.QIANFAN_API_KEY; - - try { - process.env.QIANFAN_API_KEY = "qianfan-test-key"; - + await withEnvUpdates({ QIANFAN_API_KEY: "qianfan-test-key" }, async () => { const resolved = await resolveApiKeyForProvider({ provider: "qianfan", store: { version: 1, profiles: {} }, }); expect(resolved.apiKey).toBe("qianfan-test-key"); expect(resolved.source).toContain("QIANFAN_API_KEY"); - } finally { - if (previous === undefined) { - delete process.env.QIANFAN_API_KEY; - } else { - process.env.QIANFAN_API_KEY = previous; - } - } + }); }); it("resolves Vercel AI Gateway API key from env", async () => { - const previousGatewayKey = process.env.AI_GATEWAY_API_KEY; - - try { - process.env.AI_GATEWAY_API_KEY = "gateway-test-key"; - + await withEnvUpdates({ AI_GATEWAY_API_KEY: "gateway-test-key" }, async () => { const resolved = await resolveApiKeyForProvider({ provider: "vercel-ai-gateway", store: { version: 1, profiles: {} }, }); expect(resolved.apiKey).toBe("gateway-test-key"); expect(resolved.source).toContain("AI_GATEWAY_API_KEY"); - } finally { - if (previousGatewayKey === undefined) { - delete process.env.AI_GATEWAY_API_KEY; - } else { - process.env.AI_GATEWAY_API_KEY = previousGatewayKey; - } - } + }); }); it("prefers Bedrock bearer token over access keys and profile", async () => { @@ -368,113 +335,63 @@ describe("getApiKeyForModel", () => { }); it("accepts VOYAGE_API_KEY for voyage", async () => { - const previous = process.env.VOYAGE_API_KEY; - - try { - process.env.VOYAGE_API_KEY = "voyage-test-key"; - + await withEnvUpdates({ VOYAGE_API_KEY: "voyage-test-key" }, async () => { const resolved = await resolveApiKeyForProvider({ provider: "voyage", store: { version: 1, profiles: {} }, }); expect(resolved.apiKey).toBe("voyage-test-key"); expect(resolved.source).toContain("VOYAGE_API_KEY"); - } finally { - if (previous === undefined) { - delete process.env.VOYAGE_API_KEY; - } else { - process.env.VOYAGE_API_KEY = previous; - } - } + }); }); it("strips embedded CR/LF from ANTHROPIC_API_KEY", async () => { - const previous = process.env.ANTHROPIC_API_KEY; - - try { - process.env.ANTHROPIC_API_KEY = "sk-ant-test-\r\nkey"; - + await withEnvUpdates({ ANTHROPIC_API_KEY: "sk-ant-test-\r\nkey" }, async () => { const resolved = resolveEnvApiKey("anthropic"); expect(resolved?.apiKey).toBe("sk-ant-test-key"); expect(resolved?.source).toContain("ANTHROPIC_API_KEY"); - } finally { - if (previous === undefined) { - delete process.env.ANTHROPIC_API_KEY; - } else { - process.env.ANTHROPIC_API_KEY = previous; - } - } + }); }); it("resolveEnvApiKey('huggingface') returns HUGGINGFACE_HUB_TOKEN when set", async () => { - const prevHub = process.env.HUGGINGFACE_HUB_TOKEN; - const prevHf = process.env.HF_TOKEN; - try { - delete process.env.HF_TOKEN; - process.env.HUGGINGFACE_HUB_TOKEN = "hf_hub_xyz"; - - const resolved = resolveEnvApiKey("huggingface"); - expect(resolved?.apiKey).toBe("hf_hub_xyz"); - expect(resolved?.source).toContain("HUGGINGFACE_HUB_TOKEN"); - } finally { - if (prevHub === undefined) { - delete process.env.HUGGINGFACE_HUB_TOKEN; - } else { - process.env.HUGGINGFACE_HUB_TOKEN = prevHub; - } - if (prevHf === undefined) { - delete process.env.HF_TOKEN; - } else { - process.env.HF_TOKEN = prevHf; - } - } + await withEnvUpdates( + { + HUGGINGFACE_HUB_TOKEN: "hf_hub_xyz", + HF_TOKEN: undefined, + }, + async () => { + const resolved = resolveEnvApiKey("huggingface"); + expect(resolved?.apiKey).toBe("hf_hub_xyz"); + expect(resolved?.source).toContain("HUGGINGFACE_HUB_TOKEN"); + }, + ); }); it("resolveEnvApiKey('huggingface') prefers HUGGINGFACE_HUB_TOKEN over HF_TOKEN when both set", async () => { - const prevHub = process.env.HUGGINGFACE_HUB_TOKEN; - const prevHf = process.env.HF_TOKEN; - try { - process.env.HUGGINGFACE_HUB_TOKEN = "hf_hub_first"; - process.env.HF_TOKEN = "hf_second"; - - const resolved = resolveEnvApiKey("huggingface"); - expect(resolved?.apiKey).toBe("hf_hub_first"); - expect(resolved?.source).toContain("HUGGINGFACE_HUB_TOKEN"); - } finally { - if (prevHub === undefined) { - delete process.env.HUGGINGFACE_HUB_TOKEN; - } else { - process.env.HUGGINGFACE_HUB_TOKEN = prevHub; - } - if (prevHf === undefined) { - delete process.env.HF_TOKEN; - } else { - process.env.HF_TOKEN = prevHf; - } - } + await withEnvUpdates( + { + HUGGINGFACE_HUB_TOKEN: "hf_hub_first", + HF_TOKEN: "hf_second", + }, + async () => { + const resolved = resolveEnvApiKey("huggingface"); + expect(resolved?.apiKey).toBe("hf_hub_first"); + expect(resolved?.source).toContain("HUGGINGFACE_HUB_TOKEN"); + }, + ); }); it("resolveEnvApiKey('huggingface') returns HF_TOKEN when only HF_TOKEN set", async () => { - const prevHub = process.env.HUGGINGFACE_HUB_TOKEN; - const prevHf = process.env.HF_TOKEN; - try { - delete process.env.HUGGINGFACE_HUB_TOKEN; - process.env.HF_TOKEN = "hf_abc123"; - - const resolved = resolveEnvApiKey("huggingface"); - expect(resolved?.apiKey).toBe("hf_abc123"); - expect(resolved?.source).toContain("HF_TOKEN"); - } finally { - if (prevHub === undefined) { - delete process.env.HUGGINGFACE_HUB_TOKEN; - } else { - process.env.HUGGINGFACE_HUB_TOKEN = prevHub; - } - if (prevHf === undefined) { - delete process.env.HF_TOKEN; - } else { - process.env.HF_TOKEN = prevHf; - } - } + await withEnvUpdates( + { + HUGGINGFACE_HUB_TOKEN: undefined, + HF_TOKEN: "hf_abc123", + }, + async () => { + const resolved = resolveEnvApiKey("huggingface"); + expect(resolved?.apiKey).toBe("hf_abc123"); + expect(resolved?.source).toContain("HF_TOKEN"); + }, + ); }); }); diff --git a/src/agents/model-auth.test.ts b/src/agents/model-auth.test.ts new file mode 100644 index 00000000000..2c93ee0723d --- /dev/null +++ b/src/agents/model-auth.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, it } from "vitest"; +import type { AuthProfileStore } from "./auth-profiles.js"; +import { requireApiKey, resolveAwsSdkEnvVarName, resolveModelAuthMode } from "./model-auth.js"; + +describe("resolveAwsSdkEnvVarName", () => { + it("prefers bearer token over access keys and profile", () => { + const env = { + AWS_BEARER_TOKEN_BEDROCK: "bearer", + AWS_ACCESS_KEY_ID: "access", + AWS_SECRET_ACCESS_KEY: "secret", + AWS_PROFILE: "default", + } as NodeJS.ProcessEnv; + + expect(resolveAwsSdkEnvVarName(env)).toBe("AWS_BEARER_TOKEN_BEDROCK"); + }); + + it("uses access keys when bearer token is missing", () => { + const env = { + AWS_ACCESS_KEY_ID: "access", + AWS_SECRET_ACCESS_KEY: "secret", + AWS_PROFILE: "default", + } as NodeJS.ProcessEnv; + + expect(resolveAwsSdkEnvVarName(env)).toBe("AWS_ACCESS_KEY_ID"); + }); + + it("uses profile when no bearer token or access keys exist", () => { + const env = { + AWS_PROFILE: "default", + } as NodeJS.ProcessEnv; + + expect(resolveAwsSdkEnvVarName(env)).toBe("AWS_PROFILE"); + }); + + it("returns undefined when no AWS auth env is set", () => { + expect(resolveAwsSdkEnvVarName({} as NodeJS.ProcessEnv)).toBeUndefined(); + }); +}); + +describe("resolveModelAuthMode", () => { + it("returns mixed when provider has both token and api key profiles", () => { + const store: AuthProfileStore = { + version: 1, + profiles: { + "openai:token": { + type: "token", + provider: "openai", + token: "token-value", + }, + "openai:key": { + type: "api_key", + provider: "openai", + key: "api-key", + }, + }, + }; + + expect(resolveModelAuthMode("openai", undefined, store)).toBe("mixed"); + }); + + it("returns aws-sdk when provider auth is overridden", () => { + expect( + resolveModelAuthMode( + "amazon-bedrock", + { + models: { + providers: { + "amazon-bedrock": { + baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com", + models: [], + auth: "aws-sdk", + }, + }, + }, + }, + { version: 1, profiles: {} }, + ), + ).toBe("aws-sdk"); + }); +}); + +describe("requireApiKey", () => { + it("normalizes line breaks in resolved API keys", () => { + const key = requireApiKey( + { + apiKey: "\n sk-test-abc\r\n", + source: "env: OPENAI_API_KEY", + mode: "api-key", + }, + "openai", + ); + + expect(key).toBe("sk-test-abc"); + }); + + it("throws when no API key is present", () => { + expect(() => + requireApiKey( + { + source: "env: OPENAI_API_KEY", + mode: "api-key", + }, + "openai", + ), + ).toThrow('No API key resolved for provider "openai"'); + }); +}); diff --git a/src/agents/model-fallback.e2e.test.ts b/src/agents/model-fallback.e2e.test.ts index 5eb47349092..ef4555654a2 100644 --- a/src/agents/model-fallback.e2e.test.ts +++ b/src/agents/model-fallback.e2e.test.ts @@ -23,6 +23,61 @@ function makeCfg(overrides: Partial = {}): OpenClawConfig { } as OpenClawConfig; } +function makeFallbacksOnlyCfg(): OpenClawConfig { + return { + agents: { + defaults: { + model: { + fallbacks: ["openai/gpt-5.2"], + }, + }, + }, + } as OpenClawConfig; +} + +function makeProviderFallbackCfg(provider: string): OpenClawConfig { + return makeCfg({ + agents: { + defaults: { + model: { + primary: `${provider}/m1`, + fallbacks: ["fallback/ok-model"], + }, + }, + }, + }); +} + +async function withTempAuthStore( + store: AuthProfileStore, + run: (tempDir: string) => Promise, +): Promise { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-")); + saveAuthProfileStore(store, tempDir); + try { + return await run(tempDir); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +} + +async function runWithStoredAuth(params: { + cfg: OpenClawConfig; + store: AuthProfileStore; + provider: string; + run: (provider: string, model: string) => Promise; +}) { + return withTempAuthStore(params.store, async (tempDir) => + runWithModelFallback({ + cfg: params.cfg, + provider: params.provider, + model: "m1", + agentDir: tempDir, + run: params.run, + }), + ); +} + async function expectFallsBackToHaiku(params: { provider: string; model: string; @@ -121,7 +176,6 @@ describe("runWithModelFallback", () => { }); it("skips providers when all profiles are in cooldown", async () => { - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-")); const provider = `cooldown-test-${crypto.randomUUID()}`; const profileId = `${provider}:default`; @@ -136,23 +190,12 @@ describe("runWithModelFallback", () => { }, usageStats: { [profileId]: { - cooldownUntil: Date.now() + 60_000, + cooldownUntil: Date.now() + 5 * 60_000, }, }, }; - saveAuthProfileStore(store, tempDir); - - const cfg = makeCfg({ - agents: { - defaults: { - model: { - primary: `${provider}/m1`, - fallbacks: ["fallback/ok-model"], - }, - }, - }, - }); + const cfg = makeProviderFallbackCfg(provider); const run = vi.fn().mockImplementation(async (providerId, modelId) => { if (providerId === "fallback") { return "ok"; @@ -160,25 +203,19 @@ describe("runWithModelFallback", () => { throw new Error(`unexpected provider: ${providerId}/${modelId}`); }); - try { - const result = await runWithModelFallback({ - cfg, - provider, - model: "m1", - agentDir: tempDir, - run, - }); + const result = await runWithStoredAuth({ + cfg, + store, + provider, + run, + }); - expect(result.result).toBe("ok"); - expect(run.mock.calls).toEqual([["fallback", "ok-model"]]); - expect(result.attempts[0]?.reason).toBe("rate_limit"); - } finally { - await fs.rm(tempDir, { recursive: true, force: true }); - } + expect(result.result).toBe("ok"); + expect(run.mock.calls).toEqual([["fallback", "ok-model"]]); + expect(result.attempts[0]?.reason).toBe("rate_limit"); }); it("does not skip when any profile is available", async () => { - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-")); const provider = `cooldown-mixed-${crypto.randomUUID()}`; const profileA = `${provider}:a`; const profileB = `${provider}:b`; @@ -204,18 +241,7 @@ describe("runWithModelFallback", () => { }, }; - saveAuthProfileStore(store, tempDir); - - const cfg = makeCfg({ - agents: { - defaults: { - model: { - primary: `${provider}/m1`, - fallbacks: ["fallback/ok-model"], - }, - }, - }, - }); + const cfg = makeProviderFallbackCfg(provider); const run = vi.fn().mockImplementation(async (providerId) => { if (providerId === provider) { return "ok"; @@ -223,21 +249,16 @@ describe("runWithModelFallback", () => { return "unexpected"; }); - try { - const result = await runWithModelFallback({ - cfg, - provider, - model: "m1", - agentDir: tempDir, - run, - }); + const result = await runWithStoredAuth({ + cfg, + store, + provider, + run, + }); - expect(result.result).toBe("ok"); - expect(run.mock.calls).toEqual([[provider, "m1"]]); - expect(result.attempts).toEqual([]); - } finally { - await fs.rm(tempDir, { recursive: true, force: true }); - } + expect(result.result).toBe("ok"); + expect(run.mock.calls).toEqual([[provider, "m1"]]); + expect(result.attempts).toEqual([]); }); it("does not append configured primary when fallbacksOverride is set", async () => { @@ -271,15 +292,7 @@ describe("runWithModelFallback", () => { }); it("uses fallbacksOverride instead of agents.defaults.model.fallbacks", async () => { - const cfg = { - agents: { - defaults: { - model: { - fallbacks: ["openai/gpt-5.2"], - }, - }, - }, - } as OpenClawConfig; + const cfg = makeFallbacksOnlyCfg(); const calls: Array<{ provider: string; model: string }> = []; @@ -308,15 +321,7 @@ describe("runWithModelFallback", () => { }); it("treats an empty fallbacksOverride as disabling global fallbacks", async () => { - const cfg = { - agents: { - defaults: { - model: { - fallbacks: ["openai/gpt-5.2"], - }, - }, - }, - } as OpenClawConfig; + const cfg = makeFallbacksOnlyCfg(); const calls: Array<{ provider: string; model: string }> = [];