Secrets: harden SecretRef-safe models.json persistence (#38955)

This commit is contained in:
Josh Avant
2026-03-07 11:28:39 -06:00
committed by GitHub
parent b08337b902
commit 8e20dd22d8
66 changed files with 2713 additions and 299 deletions

View File

@@ -0,0 +1,42 @@
export const PROVIDER_ENV_API_KEY_CANDIDATES: Record<string, string[]> = {
"github-copilot": ["COPILOT_GITHUB_TOKEN", "GH_TOKEN", "GITHUB_TOKEN"],
anthropic: ["ANTHROPIC_OAUTH_TOKEN", "ANTHROPIC_API_KEY"],
chutes: ["CHUTES_OAUTH_TOKEN", "CHUTES_API_KEY"],
zai: ["ZAI_API_KEY", "Z_AI_API_KEY"],
opencode: ["OPENCODE_API_KEY", "OPENCODE_ZEN_API_KEY"],
"qwen-portal": ["QWEN_OAUTH_TOKEN", "QWEN_PORTAL_API_KEY"],
volcengine: ["VOLCANO_ENGINE_API_KEY"],
"volcengine-plan": ["VOLCANO_ENGINE_API_KEY"],
byteplus: ["BYTEPLUS_API_KEY"],
"byteplus-plan": ["BYTEPLUS_API_KEY"],
"minimax-portal": ["MINIMAX_OAUTH_TOKEN", "MINIMAX_API_KEY"],
"kimi-coding": ["KIMI_API_KEY", "KIMICODE_API_KEY"],
huggingface: ["HUGGINGFACE_HUB_TOKEN", "HF_TOKEN"],
openai: ["OPENAI_API_KEY"],
google: ["GEMINI_API_KEY"],
voyage: ["VOYAGE_API_KEY"],
groq: ["GROQ_API_KEY"],
deepgram: ["DEEPGRAM_API_KEY"],
cerebras: ["CEREBRAS_API_KEY"],
xai: ["XAI_API_KEY"],
openrouter: ["OPENROUTER_API_KEY"],
litellm: ["LITELLM_API_KEY"],
"vercel-ai-gateway": ["AI_GATEWAY_API_KEY"],
"cloudflare-ai-gateway": ["CLOUDFLARE_AI_GATEWAY_API_KEY"],
moonshot: ["MOONSHOT_API_KEY"],
minimax: ["MINIMAX_API_KEY"],
nvidia: ["NVIDIA_API_KEY"],
xiaomi: ["XIAOMI_API_KEY"],
synthetic: ["SYNTHETIC_API_KEY"],
venice: ["VENICE_API_KEY"],
mistral: ["MISTRAL_API_KEY"],
together: ["TOGETHER_API_KEY"],
qianfan: ["QIANFAN_API_KEY"],
ollama: ["OLLAMA_API_KEY"],
vllm: ["VLLM_API_KEY"],
kilocode: ["KILOCODE_API_KEY"],
};
export function listKnownProviderEnvApiKeyNames(): string[] {
return [...new Set(Object.values(PROVIDER_ENV_API_KEY_CANDIDATES).flat())];
}

View File

@@ -0,0 +1,26 @@
import { describe, expect, it } from "vitest";
import { listKnownProviderEnvApiKeyNames } from "./model-auth-env-vars.js";
import { isNonSecretApiKeyMarker, NON_ENV_SECRETREF_MARKER } from "./model-auth-markers.js";
describe("model auth markers", () => {
it("recognizes explicit non-secret markers", () => {
expect(isNonSecretApiKeyMarker(NON_ENV_SECRETREF_MARKER)).toBe(true);
expect(isNonSecretApiKeyMarker("qwen-oauth")).toBe(true);
expect(isNonSecretApiKeyMarker("ollama-local")).toBe(true);
});
it("recognizes known env marker names but not arbitrary all-caps keys", () => {
expect(isNonSecretApiKeyMarker("OPENAI_API_KEY")).toBe(true);
expect(isNonSecretApiKeyMarker("ALLCAPS_EXAMPLE")).toBe(false);
});
it("recognizes all built-in provider env marker names", () => {
for (const envVarName of listKnownProviderEnvApiKeyNames()) {
expect(isNonSecretApiKeyMarker(envVarName)).toBe(true);
}
});
it("can exclude env marker-name interpretation for display-only paths", () => {
expect(isNonSecretApiKeyMarker("OPENAI_API_KEY", { includeEnvVarName: false })).toBe(false);
});
});

View File

@@ -0,0 +1,80 @@
import type { SecretRefSource } from "../config/types.secrets.js";
import { listKnownProviderEnvApiKeyNames } from "./model-auth-env-vars.js";
export const MINIMAX_OAUTH_MARKER = "minimax-oauth";
export const QWEN_OAUTH_MARKER = "qwen-oauth";
export const OLLAMA_LOCAL_AUTH_MARKER = "ollama-local";
export const NON_ENV_SECRETREF_MARKER = "secretref-managed"; // pragma: allowlist secret
export const SECRETREF_ENV_HEADER_MARKER_PREFIX = "secretref-env:"; // pragma: allowlist secret
const AWS_SDK_ENV_MARKERS = new Set([
"AWS_BEARER_TOKEN_BEDROCK",
"AWS_ACCESS_KEY_ID",
"AWS_PROFILE",
]);
// Legacy marker names kept for backward compatibility with existing models.json files.
const LEGACY_ENV_API_KEY_MARKERS = [
"GOOGLE_API_KEY",
"DEEPSEEK_API_KEY",
"PERPLEXITY_API_KEY",
"FIREWORKS_API_KEY",
"NOVITA_API_KEY",
"AZURE_OPENAI_API_KEY",
"AZURE_API_KEY",
"MINIMAX_CODE_PLAN_KEY",
];
const KNOWN_ENV_API_KEY_MARKERS = new Set([
...listKnownProviderEnvApiKeyNames(),
...LEGACY_ENV_API_KEY_MARKERS,
...AWS_SDK_ENV_MARKERS,
]);
export function isAwsSdkAuthMarker(value: string): boolean {
return AWS_SDK_ENV_MARKERS.has(value.trim());
}
export function resolveNonEnvSecretRefApiKeyMarker(_source: SecretRefSource): string {
return NON_ENV_SECRETREF_MARKER;
}
export function resolveNonEnvSecretRefHeaderValueMarker(_source: SecretRefSource): string {
return NON_ENV_SECRETREF_MARKER;
}
export function resolveEnvSecretRefHeaderValueMarker(envVarName: string): string {
return `${SECRETREF_ENV_HEADER_MARKER_PREFIX}${envVarName.trim()}`;
}
export function isSecretRefHeaderValueMarker(value: string): boolean {
const trimmed = value.trim();
return (
trimmed === NON_ENV_SECRETREF_MARKER || trimmed.startsWith(SECRETREF_ENV_HEADER_MARKER_PREFIX)
);
}
export function isNonSecretApiKeyMarker(
value: string,
opts?: { includeEnvVarName?: boolean },
): boolean {
const trimmed = value.trim();
if (!trimmed) {
return false;
}
const isKnownMarker =
trimmed === MINIMAX_OAUTH_MARKER ||
trimmed === QWEN_OAUTH_MARKER ||
trimmed === OLLAMA_LOCAL_AUTH_MARKER ||
trimmed === NON_ENV_SECRETREF_MARKER ||
isAwsSdkAuthMarker(trimmed);
if (isKnownMarker) {
return true;
}
if (opts?.includeEnvVarName === false) {
return false;
}
// Do not treat arbitrary ALL_CAPS values as markers; only recognize the
// known env-var markers we intentionally persist for compatibility.
return KNOWN_ENV_API_KEY_MARKERS.has(trimmed);
}

View File

@@ -16,6 +16,8 @@ import {
resolveAuthProfileOrder,
resolveAuthStorePathForDisplay,
} from "./auth-profiles.js";
import { PROVIDER_ENV_API_KEY_CANDIDATES } from "./model-auth-env-vars.js";
import { OLLAMA_LOCAL_AUTH_MARKER } from "./model-auth-markers.js";
import { normalizeProviderId } from "./model-selection.js";
export { ensureAuthProfileStore, resolveAuthProfileOrder } from "./auth-profiles.js";
@@ -90,7 +92,7 @@ function resolveSyntheticLocalProviderAuth(params: {
}
return {
apiKey: "ollama-local", // pragma: allowlist secret
apiKey: OLLAMA_LOCAL_AUTH_MARKER,
source: "models.providers.ollama (synthetic local key)",
mode: "api-key",
};
@@ -281,20 +283,14 @@ export function resolveEnvApiKey(provider: string): EnvApiKeyResult | null {
return { apiKey: value, source };
};
if (normalized === "github-copilot") {
return pick("COPILOT_GITHUB_TOKEN") ?? pick("GH_TOKEN") ?? pick("GITHUB_TOKEN");
}
if (normalized === "anthropic") {
return pick("ANTHROPIC_OAUTH_TOKEN") ?? pick("ANTHROPIC_API_KEY");
}
if (normalized === "chutes") {
return pick("CHUTES_OAUTH_TOKEN") ?? pick("CHUTES_API_KEY");
}
if (normalized === "zai") {
return pick("ZAI_API_KEY") ?? pick("Z_AI_API_KEY");
const candidates = PROVIDER_ENV_API_KEY_CANDIDATES[normalized];
if (candidates) {
for (const envVar of candidates) {
const resolved = pick(envVar);
if (resolved) {
return resolved;
}
}
}
if (normalized === "google-vertex") {
@@ -304,65 +300,7 @@ export function resolveEnvApiKey(provider: string): EnvApiKeyResult | null {
}
return { apiKey: envKey, source: "gcloud adc" };
}
if (normalized === "opencode") {
return pick("OPENCODE_API_KEY") ?? pick("OPENCODE_ZEN_API_KEY");
}
if (normalized === "qwen-portal") {
return pick("QWEN_OAUTH_TOKEN") ?? pick("QWEN_PORTAL_API_KEY");
}
if (normalized === "volcengine" || normalized === "volcengine-plan") {
return pick("VOLCANO_ENGINE_API_KEY");
}
if (normalized === "byteplus" || normalized === "byteplus-plan") {
return pick("BYTEPLUS_API_KEY");
}
if (normalized === "minimax-portal") {
return pick("MINIMAX_OAUTH_TOKEN") ?? pick("MINIMAX_API_KEY");
}
if (normalized === "kimi-coding") {
return pick("KIMI_API_KEY") ?? pick("KIMICODE_API_KEY");
}
if (normalized === "huggingface") {
return pick("HUGGINGFACE_HUB_TOKEN") ?? pick("HF_TOKEN");
}
const envMap: Record<string, string> = {
openai: "OPENAI_API_KEY",
google: "GEMINI_API_KEY",
voyage: "VOYAGE_API_KEY",
groq: "GROQ_API_KEY",
deepgram: "DEEPGRAM_API_KEY",
cerebras: "CEREBRAS_API_KEY",
xai: "XAI_API_KEY",
openrouter: "OPENROUTER_API_KEY",
litellm: "LITELLM_API_KEY",
"vercel-ai-gateway": "AI_GATEWAY_API_KEY",
"cloudflare-ai-gateway": "CLOUDFLARE_AI_GATEWAY_API_KEY",
moonshot: "MOONSHOT_API_KEY",
minimax: "MINIMAX_API_KEY",
nvidia: "NVIDIA_API_KEY",
xiaomi: "XIAOMI_API_KEY",
synthetic: "SYNTHETIC_API_KEY",
venice: "VENICE_API_KEY",
mistral: "MISTRAL_API_KEY",
opencode: "OPENCODE_API_KEY",
together: "TOGETHER_API_KEY",
qianfan: "QIANFAN_API_KEY",
ollama: "OLLAMA_API_KEY",
vllm: "VLLM_API_KEY",
kilocode: "KILOCODE_API_KEY",
};
const envVar = envMap[normalized];
if (!envVar) {
return null;
}
return pick(envVar);
return null;
}
export function resolveModelAuthMode(

View File

@@ -0,0 +1,43 @@
import fs from "node:fs/promises";
import path from "node:path";
import { describe, expect, it } from "vitest";
import { resolveOpenClawAgentDir } from "./agent-paths.js";
import {
CUSTOM_PROXY_MODELS_CONFIG,
installModelsConfigTestHooks,
withModelsTempHome as withTempHome,
} from "./models-config.e2e-harness.js";
import { ensureOpenClawModelsJson } from "./models-config.js";
installModelsConfigTestHooks();
describe("models-config file mode", () => {
it("writes models.json with mode 0600", async () => {
if (process.platform === "win32") {
return;
}
await withTempHome(async () => {
await ensureOpenClawModelsJson(CUSTOM_PROXY_MODELS_CONFIG);
const modelsPath = path.join(resolveOpenClawAgentDir(), "models.json");
const stat = await fs.stat(modelsPath);
expect(stat.mode & 0o777).toBe(0o600);
});
});
it("repairs models.json mode to 0600 on no-content-change paths", async () => {
if (process.platform === "win32") {
return;
}
await withTempHome(async () => {
await ensureOpenClawModelsJson(CUSTOM_PROXY_MODELS_CONFIG);
const modelsPath = path.join(resolveOpenClawAgentDir(), "models.json");
await fs.chmod(modelsPath, 0o644);
const result = await ensureOpenClawModelsJson(CUSTOM_PROXY_MODELS_CONFIG);
expect(result.wrote).toBe(false);
const stat = await fs.stat(modelsPath);
expect(stat.mode & 0o777).toBe(0o600);
});
});
});

View File

@@ -4,6 +4,7 @@ import { describe, expect, it } from "vitest";
import type { OpenClawConfig } from "../config/config.js";
import { validateConfigObject } from "../config/validation.js";
import { resolveOpenClawAgentDir } from "./agent-paths.js";
import { NON_ENV_SECRETREF_MARKER } from "./model-auth-markers.js";
import {
CUSTOM_PROXY_MODELS_CONFIG,
installModelsConfigTestHooks,
@@ -166,7 +167,7 @@ describe("models-config", () => {
const parsed = await readGeneratedModelsJson<{
providers: Record<string, { apiKey?: string; models?: Array<{ id: string }> }>;
}>();
expect(parsed.providers.minimax?.apiKey).toBe("MINIMAX_API_KEY");
expect(parsed.providers.minimax?.apiKey).toBe("MINIMAX_API_KEY"); // pragma: allowlist secret
const ids = parsed.providers.minimax?.models?.map((model) => model.id);
expect(ids).toContain("MiniMax-VL-01");
});
@@ -220,6 +221,117 @@ describe("models-config", () => {
});
});
it("replaces stale merged apiKey when provider is SecretRef-managed in current config", async () => {
await withTempHome(async () => {
await writeAgentModelsJson({
providers: {
custom: {
baseUrl: "https://agent.example/v1",
apiKey: "STALE_AGENT_KEY", // pragma: allowlist secret
api: "openai-responses",
models: [{ id: "agent-model", name: "Agent model", input: ["text"] }],
},
},
});
await ensureOpenClawModelsJson({
models: {
mode: "merge",
providers: {
custom: {
...createMergeConfigProvider(),
apiKey: { source: "env", provider: "default", id: "CUSTOM_PROVIDER_API_KEY" }, // pragma: allowlist secret
},
},
},
});
const parsed = await readGeneratedModelsJson<{
providers: Record<string, { apiKey?: string; baseUrl?: string }>;
}>();
expect(parsed.providers.custom?.apiKey).toBe("CUSTOM_PROVIDER_API_KEY"); // pragma: allowlist secret
expect(parsed.providers.custom?.baseUrl).toBe("https://agent.example/v1");
});
});
it("replaces stale merged apiKey when provider is SecretRef-managed via auth-profiles", async () => {
await withTempHome(async () => {
const agentDir = resolveOpenClawAgentDir();
await fs.mkdir(agentDir, { recursive: true });
await fs.writeFile(
path.join(agentDir, "auth-profiles.json"),
`${JSON.stringify(
{
version: 1,
profiles: {
"minimax:default": {
type: "api_key",
provider: "minimax",
keyRef: { source: "env", provider: "default", id: "MINIMAX_API_KEY" }, // pragma: allowlist secret
},
},
},
null,
2,
)}\n`,
"utf8",
);
await writeAgentModelsJson({
providers: {
minimax: {
baseUrl: "https://api.minimax.io/anthropic",
apiKey: "STALE_AGENT_KEY", // pragma: allowlist secret
api: "anthropic-messages",
models: [{ id: "MiniMax-M2.5", name: "MiniMax M2.5", input: ["text"] }],
},
},
});
await ensureOpenClawModelsJson({
models: {
mode: "merge",
providers: {},
},
});
const parsed = await readGeneratedModelsJson<{
providers: Record<string, { apiKey?: string }>;
}>();
expect(parsed.providers.minimax?.apiKey).toBe("MINIMAX_API_KEY"); // pragma: allowlist secret
});
});
it("replaces stale non-env marker when provider transitions back to plaintext config", async () => {
await withTempHome(async () => {
await writeAgentModelsJson({
providers: {
custom: {
baseUrl: "https://agent.example/v1",
apiKey: NON_ENV_SECRETREF_MARKER,
api: "openai-responses",
models: [{ id: "agent-model", name: "Agent model", input: ["text"] }],
},
},
});
await ensureOpenClawModelsJson({
models: {
mode: "merge",
providers: {
custom: {
...createMergeConfigProvider(),
apiKey: "ALLCAPS_SAMPLE", // pragma: allowlist secret
},
},
},
});
const parsed = await readGeneratedModelsJson<{
providers: Record<string, { apiKey?: string }>;
}>();
expect(parsed.providers.custom?.apiKey).toBe("ALLCAPS_SAMPLE");
});
});
it("uses config apiKey/baseUrl when existing agent values are empty", async () => {
await withTempHome(async () => {
const parsed = await runCustomProviderMergeTest({

View File

@@ -0,0 +1,121 @@
import { mkdtempSync } from "node:fs";
import { writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { captureEnv } from "../test-utils/env.js";
import {
MINIMAX_OAUTH_MARKER,
NON_ENV_SECRETREF_MARKER,
QWEN_OAUTH_MARKER,
} from "./model-auth-markers.js";
import { resolveImplicitProviders } from "./models-config.providers.js";
describe("models-config provider auth provenance", () => {
it("persists env keyRef and tokenRef auth profiles as env var markers", async () => {
const agentDir = mkdtempSync(join(tmpdir(), "openclaw-test-"));
const envSnapshot = captureEnv(["VOLCANO_ENGINE_API_KEY", "TOGETHER_API_KEY"]);
delete process.env.VOLCANO_ENGINE_API_KEY;
delete process.env.TOGETHER_API_KEY;
await writeFile(
join(agentDir, "auth-profiles.json"),
JSON.stringify(
{
version: 1,
profiles: {
"volcengine:default": {
type: "api_key",
provider: "volcengine",
keyRef: { source: "env", provider: "default", id: "VOLCANO_ENGINE_API_KEY" },
},
"together:default": {
type: "token",
provider: "together",
tokenRef: { source: "env", provider: "default", id: "TOGETHER_API_KEY" },
},
},
},
null,
2,
),
"utf8",
);
try {
const providers = await resolveImplicitProviders({ agentDir });
expect(providers?.volcengine?.apiKey).toBe("VOLCANO_ENGINE_API_KEY");
expect(providers?.["volcengine-plan"]?.apiKey).toBe("VOLCANO_ENGINE_API_KEY");
expect(providers?.together?.apiKey).toBe("TOGETHER_API_KEY");
} finally {
envSnapshot.restore();
}
});
it("uses non-env marker for ref-managed profiles even when runtime plaintext is present", async () => {
const agentDir = mkdtempSync(join(tmpdir(), "openclaw-test-"));
await writeFile(
join(agentDir, "auth-profiles.json"),
JSON.stringify(
{
version: 1,
profiles: {
"byteplus:default": {
type: "api_key",
provider: "byteplus",
key: "sk-runtime-resolved-byteplus",
keyRef: { source: "file", provider: "vault", id: "/byteplus/apiKey" },
},
"together:default": {
type: "token",
provider: "together",
token: "tok-runtime-resolved-together",
tokenRef: { source: "exec", provider: "vault", id: "providers/together/token" },
},
},
},
null,
2,
),
"utf8",
);
const providers = await resolveImplicitProviders({ agentDir });
expect(providers?.byteplus?.apiKey).toBe(NON_ENV_SECRETREF_MARKER);
expect(providers?.["byteplus-plan"]?.apiKey).toBe(NON_ENV_SECRETREF_MARKER);
expect(providers?.together?.apiKey).toBe(NON_ENV_SECRETREF_MARKER);
});
it("keeps oauth compatibility markers for minimax-portal and qwen-portal", async () => {
const agentDir = mkdtempSync(join(tmpdir(), "openclaw-test-"));
await writeFile(
join(agentDir, "auth-profiles.json"),
JSON.stringify(
{
version: 1,
profiles: {
"minimax-portal:default": {
type: "oauth",
provider: "minimax-portal",
access: "access-token",
refresh: "refresh-token",
expires: Date.now() + 60_000,
},
"qwen-portal:default": {
type: "oauth",
provider: "qwen-portal",
access: "access-token",
refresh: "refresh-token",
expires: Date.now() + 60_000,
},
},
},
null,
2,
),
"utf8",
);
const providers = await resolveImplicitProviders({ agentDir });
expect(providers?.["minimax-portal"]?.apiKey).toBe(MINIMAX_OAUTH_MARKER);
expect(providers?.["qwen-portal"]?.apiKey).toBe(QWEN_OAUTH_MARKER);
});
});

View File

@@ -0,0 +1,76 @@
import { mkdtempSync } from "node:fs";
import { writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { captureEnv } from "../test-utils/env.js";
import { NON_ENV_SECRETREF_MARKER } from "./model-auth-markers.js";
import { resolveImplicitProviders } from "./models-config.providers.js";
describe("cloudflare-ai-gateway profile provenance", () => {
it("prefers env keyRef marker over runtime plaintext for persistence", async () => {
const agentDir = mkdtempSync(join(tmpdir(), "openclaw-test-"));
const envSnapshot = captureEnv(["CLOUDFLARE_AI_GATEWAY_API_KEY"]);
delete process.env.CLOUDFLARE_AI_GATEWAY_API_KEY;
await writeFile(
join(agentDir, "auth-profiles.json"),
JSON.stringify(
{
version: 1,
profiles: {
"cloudflare-ai-gateway:default": {
type: "api_key",
provider: "cloudflare-ai-gateway",
key: "sk-runtime-cloudflare",
keyRef: { source: "env", provider: "default", id: "CLOUDFLARE_AI_GATEWAY_API_KEY" },
metadata: {
accountId: "acct_123",
gatewayId: "gateway_456",
},
},
},
},
null,
2,
),
"utf8",
);
try {
const providers = await resolveImplicitProviders({ agentDir });
expect(providers?.["cloudflare-ai-gateway"]?.apiKey).toBe("CLOUDFLARE_AI_GATEWAY_API_KEY");
} finally {
envSnapshot.restore();
}
});
it("uses non-env marker for non-env keyRef cloudflare profiles", async () => {
const agentDir = mkdtempSync(join(tmpdir(), "openclaw-test-"));
await writeFile(
join(agentDir, "auth-profiles.json"),
JSON.stringify(
{
version: 1,
profiles: {
"cloudflare-ai-gateway:default": {
type: "api_key",
provider: "cloudflare-ai-gateway",
key: "sk-runtime-cloudflare",
keyRef: { source: "file", provider: "vault", id: "/cloudflare/apiKey" },
metadata: {
accountId: "acct_123",
gatewayId: "gateway_456",
},
},
},
},
null,
2,
),
"utf8",
);
const providers = await resolveImplicitProviders({ agentDir });
expect(providers?.["cloudflare-ai-gateway"]?.apiKey).toBe(NON_ENV_SECRETREF_MARKER);
});
});

View File

@@ -0,0 +1,140 @@
import { mkdtempSync } from "node:fs";
import { writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import { NON_ENV_SECRETREF_MARKER } from "./model-auth-markers.js";
import { resolveImplicitProviders } from "./models-config.providers.js";
describe("provider discovery auth marker guardrails", () => {
let originalVitest: string | undefined;
let originalNodeEnv: string | undefined;
let originalFetch: typeof globalThis.fetch | undefined;
afterEach(() => {
if (originalVitest !== undefined) {
process.env.VITEST = originalVitest;
} else {
delete process.env.VITEST;
}
if (originalNodeEnv !== undefined) {
process.env.NODE_ENV = originalNodeEnv;
} else {
delete process.env.NODE_ENV;
}
if (originalFetch) {
globalThis.fetch = originalFetch;
}
});
function enableDiscovery() {
originalVitest = process.env.VITEST;
originalNodeEnv = process.env.NODE_ENV;
originalFetch = globalThis.fetch;
delete process.env.VITEST;
delete process.env.NODE_ENV;
}
it("does not send marker value as vLLM bearer token during discovery", async () => {
enableDiscovery();
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({ data: [] }),
});
globalThis.fetch = fetchMock as unknown as typeof fetch;
const agentDir = mkdtempSync(join(tmpdir(), "openclaw-test-"));
await writeFile(
join(agentDir, "auth-profiles.json"),
JSON.stringify(
{
version: 1,
profiles: {
"vllm:default": {
type: "api_key",
provider: "vllm",
keyRef: { source: "file", provider: "vault", id: "/vllm/apiKey" },
},
},
},
null,
2,
),
"utf8",
);
const providers = await resolveImplicitProviders({ agentDir });
expect(providers?.vllm?.apiKey).toBe(NON_ENV_SECRETREF_MARKER);
const request = fetchMock.mock.calls[0]?.[1] as
| { headers?: Record<string, string> }
| undefined;
expect(request?.headers?.Authorization).toBeUndefined();
});
it("does not call Hugging Face discovery with marker-backed credentials", async () => {
enableDiscovery();
const fetchMock = vi.fn();
globalThis.fetch = fetchMock as unknown as typeof fetch;
const agentDir = mkdtempSync(join(tmpdir(), "openclaw-test-"));
await writeFile(
join(agentDir, "auth-profiles.json"),
JSON.stringify(
{
version: 1,
profiles: {
"huggingface:default": {
type: "api_key",
provider: "huggingface",
keyRef: { source: "exec", provider: "vault", id: "providers/hf/token" },
},
},
},
null,
2,
),
"utf8",
);
const providers = await resolveImplicitProviders({ agentDir });
expect(providers?.huggingface?.apiKey).toBe(NON_ENV_SECRETREF_MARKER);
const huggingfaceCalls = fetchMock.mock.calls.filter(([url]) =>
String(url).includes("router.huggingface.co"),
);
expect(huggingfaceCalls).toHaveLength(0);
});
it("keeps all-caps plaintext API keys for authenticated discovery", async () => {
enableDiscovery();
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({ data: [{ id: "vllm/test-model" }] }),
});
globalThis.fetch = fetchMock as unknown as typeof fetch;
const agentDir = mkdtempSync(join(tmpdir(), "openclaw-test-"));
await writeFile(
join(agentDir, "auth-profiles.json"),
JSON.stringify(
{
version: 1,
profiles: {
"vllm:default": {
type: "api_key",
provider: "vllm",
key: "ALLCAPS_SAMPLE",
},
},
},
null,
2,
),
"utf8",
);
await resolveImplicitProviders({ agentDir });
const vllmCall = fetchMock.mock.calls.find(([url]) => String(url).includes(":8000"));
const request = vllmCall?.[1] as { headers?: Record<string, string> } | undefined;
expect(request?.headers?.Authorization).toBe("Bearer ALLCAPS_SAMPLE");
});
});

View File

@@ -3,6 +3,7 @@ import os from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
import type { OpenClawConfig } from "../config/config.js";
import { NON_ENV_SECRETREF_MARKER } from "./model-auth-markers.js";
import { normalizeProviders } from "./models-config.providers.js";
describe("normalizeProviders", () => {
@@ -73,4 +74,30 @@ describe("normalizeProviders", () => {
await fs.rm(agentDir, { recursive: true, force: true });
}
});
it("normalizes SecretRef-backed provider headers to non-secret marker values", async () => {
const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-agent-"));
try {
const providers: NonNullable<NonNullable<OpenClawConfig["models"]>["providers"]> = {
openai: {
baseUrl: "https://api.openai.com/v1",
api: "openai-completions",
headers: {
Authorization: { source: "env", provider: "default", id: "OPENAI_HEADER_TOKEN" },
"X-Tenant-Token": { source: "file", provider: "vault", id: "/openai/token" },
},
models: [],
},
};
const normalized = normalizeProviders({
providers,
agentDir,
});
expect(normalized?.openai?.headers?.Authorization).toBe("secretref-env:OPENAI_HEADER_TOKEN");
expect(normalized?.openai?.headers?.["X-Tenant-Token"]).toBe(NON_ENV_SECRETREF_MARKER);
} finally {
await fs.rm(agentDir, { recursive: true, force: true });
}
});
});

View File

@@ -1,6 +1,6 @@
import type { OpenClawConfig } from "../config/config.js";
import type { ModelDefinitionConfig } from "../config/types.models.js";
import { coerceSecretRef } from "../config/types.secrets.js";
import { coerceSecretRef, resolveSecretInputRef } from "../config/types.secrets.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
import {
DEFAULT_COPILOT_API_BASE_URL,
@@ -41,6 +41,15 @@ import {
buildHuggingfaceModelDefinition,
} from "./huggingface-models.js";
import { discoverKilocodeModels } from "./kilocode-models.js";
import {
MINIMAX_OAUTH_MARKER,
OLLAMA_LOCAL_AUTH_MARKER,
QWEN_OAUTH_MARKER,
isNonSecretApiKeyMarker,
resolveNonEnvSecretRefApiKeyMarker,
resolveNonEnvSecretRefHeaderValueMarker,
resolveEnvSecretRefHeaderValueMarker,
} from "./model-auth-markers.js";
import { resolveAwsSdkEnvVarName, resolveEnvApiKey } from "./model-auth.js";
import { OLLAMA_NATIVE_BASE_URL } from "./ollama-stream.js";
import {
@@ -63,7 +72,6 @@ const MINIMAX_DEFAULT_MODEL_ID = "MiniMax-M2.5";
const MINIMAX_DEFAULT_VISION_MODEL_ID = "MiniMax-VL-01";
const MINIMAX_DEFAULT_CONTEXT_WINDOW = 200000;
const MINIMAX_DEFAULT_MAX_TOKENS = 8192;
const MINIMAX_OAUTH_PLACEHOLDER = "minimax-oauth";
// Pricing per 1M tokens (USD) — https://platform.minimaxi.com/document/Price
const MINIMAX_API_COST = {
input: 0.3,
@@ -133,7 +141,6 @@ const KIMI_CODING_DEFAULT_COST = {
};
const QWEN_PORTAL_BASE_URL = "https://portal.qwen.ai/v1";
const QWEN_PORTAL_OAUTH_PLACEHOLDER = "qwen-oauth";
const QWEN_PORTAL_DEFAULT_CONTEXT_WINDOW = 128000;
const QWEN_PORTAL_DEFAULT_MAX_TOKENS = 8192;
const QWEN_PORTAL_DEFAULT_COST = {
@@ -404,35 +411,125 @@ function resolveAwsSdkApiKeyVarName(): string {
return resolveAwsSdkEnvVarName() ?? "AWS_PROFILE";
}
function normalizeHeaderValues(params: {
headers: ProviderConfig["headers"] | undefined;
secretDefaults:
| {
env?: string;
file?: string;
exec?: string;
}
| undefined;
}): { headers: ProviderConfig["headers"] | undefined; mutated: boolean } {
const { headers } = params;
if (!headers) {
return { headers, mutated: false };
}
let mutated = false;
const nextHeaders: Record<string, NonNullable<ProviderConfig["headers"]>[string]> = {};
for (const [headerName, headerValue] of Object.entries(headers)) {
const resolvedRef = resolveSecretInputRef({
value: headerValue,
defaults: params.secretDefaults,
}).ref;
if (!resolvedRef || !resolvedRef.id.trim()) {
nextHeaders[headerName] = headerValue;
continue;
}
mutated = true;
nextHeaders[headerName] =
resolvedRef.source === "env"
? resolveEnvSecretRefHeaderValueMarker(resolvedRef.id)
: resolveNonEnvSecretRefHeaderValueMarker(resolvedRef.source);
}
if (!mutated) {
return { headers, mutated: false };
}
return { headers: nextHeaders, mutated: true };
}
type ProfileApiKeyResolution = {
apiKey: string;
source: "plaintext" | "env-ref" | "non-env-ref";
/** Optional secret value that may be used for provider discovery only. */
discoveryApiKey?: string;
};
function toDiscoveryApiKey(value: string | undefined): string | undefined {
const trimmed = value?.trim();
if (!trimmed || isNonSecretApiKeyMarker(trimmed)) {
return undefined;
}
return trimmed;
}
function resolveApiKeyFromCredential(
cred: ReturnType<typeof ensureAuthProfileStore>["profiles"][string] | undefined,
): ProfileApiKeyResolution | undefined {
if (!cred) {
return undefined;
}
if (cred.type === "api_key") {
const keyRef = coerceSecretRef(cred.keyRef);
if (keyRef && keyRef.id.trim()) {
if (keyRef.source === "env") {
const envVar = keyRef.id.trim();
return {
apiKey: envVar,
source: "env-ref",
discoveryApiKey: toDiscoveryApiKey(process.env[envVar]),
};
}
return {
apiKey: resolveNonEnvSecretRefApiKeyMarker(keyRef.source),
source: "non-env-ref",
};
}
if (cred.key?.trim()) {
return {
apiKey: cred.key,
source: "plaintext",
discoveryApiKey: toDiscoveryApiKey(cred.key),
};
}
return undefined;
}
if (cred.type === "token") {
const tokenRef = coerceSecretRef(cred.tokenRef);
if (tokenRef && tokenRef.id.trim()) {
if (tokenRef.source === "env") {
const envVar = tokenRef.id.trim();
return {
apiKey: envVar,
source: "env-ref",
discoveryApiKey: toDiscoveryApiKey(process.env[envVar]),
};
}
return {
apiKey: resolveNonEnvSecretRefApiKeyMarker(tokenRef.source),
source: "non-env-ref",
};
}
if (cred.token?.trim()) {
return {
apiKey: cred.token,
source: "plaintext",
discoveryApiKey: toDiscoveryApiKey(cred.token),
};
}
}
return undefined;
}
function resolveApiKeyFromProfiles(params: {
provider: string;
store: ReturnType<typeof ensureAuthProfileStore>;
}): string | undefined {
}): ProfileApiKeyResolution | undefined {
const ids = listProfilesForProvider(params.store, params.provider);
for (const id of ids) {
const cred = params.store.profiles[id];
if (!cred) {
continue;
}
if (cred.type === "api_key") {
if (cred.key?.trim()) {
return cred.key;
}
const keyRef = coerceSecretRef(cred.keyRef);
if (keyRef?.source === "env" && keyRef.id.trim()) {
return keyRef.id.trim();
}
continue;
}
if (cred.type === "token") {
if (cred.token?.trim()) {
return cred.token;
}
const tokenRef = coerceSecretRef(cred.tokenRef);
if (tokenRef?.source === "env" && tokenRef.id.trim()) {
return tokenRef.id.trim();
}
continue;
const resolved = resolveApiKeyFromCredential(params.store.profiles[id]);
if (resolved) {
return resolved;
}
}
return undefined;
@@ -484,6 +581,12 @@ function normalizeAntigravityProvider(provider: ProviderConfig): ProviderConfig
export function normalizeProviders(params: {
providers: ModelsConfig["providers"];
agentDir: string;
secretDefaults?: {
env?: string;
file?: string;
exec?: string;
};
secretRefManagedProviders?: Set<string>;
}): ModelsConfig["providers"] {
const { providers } = params;
if (!providers) {
@@ -505,18 +608,51 @@ export function normalizeProviders(params: {
mutated = true;
}
let normalizedProvider = provider;
const configuredApiKey = normalizedProvider.apiKey;
// Fix common misconfig: apiKey set to "${ENV_VAR}" instead of "ENV_VAR".
if (
typeof configuredApiKey === "string" &&
normalizeApiKeyConfig(configuredApiKey) !== configuredApiKey
) {
const normalizedHeaders = normalizeHeaderValues({
headers: normalizedProvider.headers,
secretDefaults: params.secretDefaults,
});
if (normalizedHeaders.mutated) {
mutated = true;
normalizedProvider = {
...normalizedProvider,
apiKey: normalizeApiKeyConfig(configuredApiKey),
};
normalizedProvider = { ...normalizedProvider, headers: normalizedHeaders.headers };
}
const configuredApiKey = normalizedProvider.apiKey;
const configuredApiKeyRef = resolveSecretInputRef({
value: configuredApiKey,
defaults: params.secretDefaults,
}).ref;
const profileApiKey = resolveApiKeyFromProfiles({
provider: normalizedKey,
store: authStore,
});
if (configuredApiKeyRef && configuredApiKeyRef.id.trim()) {
const marker =
configuredApiKeyRef.source === "env"
? configuredApiKeyRef.id.trim()
: resolveNonEnvSecretRefApiKeyMarker(configuredApiKeyRef.source);
if (normalizedProvider.apiKey !== marker) {
mutated = true;
normalizedProvider = { ...normalizedProvider, apiKey: marker };
}
params.secretRefManagedProviders?.add(normalizedKey);
} else if (typeof configuredApiKey === "string") {
// Fix common misconfig: apiKey set to "${ENV_VAR}" instead of "ENV_VAR".
const normalizedConfiguredApiKey = normalizeApiKeyConfig(configuredApiKey);
if (normalizedConfiguredApiKey !== configuredApiKey) {
mutated = true;
normalizedProvider = {
...normalizedProvider,
apiKey: normalizedConfiguredApiKey,
};
}
if (
profileApiKey &&
profileApiKey.source !== "plaintext" &&
normalizedConfiguredApiKey === profileApiKey.apiKey
) {
params.secretRefManagedProviders?.add(normalizedKey);
}
}
// If a provider defines models, pi's ModelRegistry requires apiKey to be set.
@@ -534,12 +670,11 @@ export function normalizeProviders(params: {
normalizedProvider = { ...normalizedProvider, apiKey };
} else {
const fromEnv = resolveEnvApiKeyVarName(normalizedKey);
const fromProfiles = resolveApiKeyFromProfiles({
provider: normalizedKey,
store: authStore,
});
const apiKey = fromEnv ?? fromProfiles;
const apiKey = fromEnv ?? profileApiKey?.apiKey;
if (apiKey?.trim()) {
if (profileApiKey && profileApiKey.source !== "plaintext") {
params.secretRefManagedProviders?.add(normalizedKey);
}
mutated = true;
normalizedProvider = { ...normalizedProvider, apiKey };
}
@@ -778,14 +913,8 @@ async function buildOllamaProvider(
};
}
async function buildHuggingfaceProvider(apiKey?: string): Promise<ProviderConfig> {
// Resolve env var name to value for discovery (GET /v1/models requires Bearer token).
const resolvedSecret =
apiKey?.trim() !== ""
? /^[A-Z][A-Z0-9_]*$/.test(apiKey!.trim())
? (process.env[apiKey!.trim()] ?? "").trim()
: apiKey!.trim()
: "";
async function buildHuggingfaceProvider(discoveryApiKey?: string): Promise<ProviderConfig> {
const resolvedSecret = toDiscoveryApiKey(discoveryApiKey) ?? "";
const models =
resolvedSecret !== ""
? await discoverHuggingfaceModels(resolvedSecret)
@@ -946,10 +1075,24 @@ export async function resolveImplicitProviders(params: {
const authStore = ensureAuthProfileStore(params.agentDir, {
allowKeychainPrompt: false,
});
const resolveProviderApiKey = (
provider: string,
): { apiKey: string | undefined; discoveryApiKey?: string } => {
const envVar = resolveEnvApiKeyVarName(provider);
if (envVar) {
return {
apiKey: envVar,
discoveryApiKey: toDiscoveryApiKey(process.env[envVar]),
};
}
const fromProfiles = resolveApiKeyFromProfiles({ provider, store: authStore });
return {
apiKey: fromProfiles?.apiKey,
discoveryApiKey: fromProfiles?.discoveryApiKey,
};
};
const minimaxKey =
resolveEnvApiKeyVarName("minimax") ??
resolveApiKeyFromProfiles({ provider: "minimax", store: authStore });
const minimaxKey = resolveProviderApiKey("minimax").apiKey;
if (minimaxKey) {
providers.minimax = { ...buildMinimaxProvider(), apiKey: minimaxKey };
}
@@ -958,34 +1101,26 @@ export async function resolveImplicitProviders(params: {
if (minimaxOauthProfile.length > 0) {
providers["minimax-portal"] = {
...buildMinimaxPortalProvider(),
apiKey: MINIMAX_OAUTH_PLACEHOLDER,
apiKey: MINIMAX_OAUTH_MARKER,
};
}
const moonshotKey =
resolveEnvApiKeyVarName("moonshot") ??
resolveApiKeyFromProfiles({ provider: "moonshot", store: authStore });
const moonshotKey = resolveProviderApiKey("moonshot").apiKey;
if (moonshotKey) {
providers.moonshot = { ...buildMoonshotProvider(), apiKey: moonshotKey };
}
const kimiCodingKey =
resolveEnvApiKeyVarName("kimi-coding") ??
resolveApiKeyFromProfiles({ provider: "kimi-coding", store: authStore });
const kimiCodingKey = resolveProviderApiKey("kimi-coding").apiKey;
if (kimiCodingKey) {
providers["kimi-coding"] = { ...buildKimiCodingProvider(), apiKey: kimiCodingKey };
}
const syntheticKey =
resolveEnvApiKeyVarName("synthetic") ??
resolveApiKeyFromProfiles({ provider: "synthetic", store: authStore });
const syntheticKey = resolveProviderApiKey("synthetic").apiKey;
if (syntheticKey) {
providers.synthetic = { ...buildSyntheticProvider(), apiKey: syntheticKey };
}
const veniceKey =
resolveEnvApiKeyVarName("venice") ??
resolveApiKeyFromProfiles({ provider: "venice", store: authStore });
const veniceKey = resolveProviderApiKey("venice").apiKey;
if (veniceKey) {
providers.venice = { ...(await buildVeniceProvider()), apiKey: veniceKey };
}
@@ -994,13 +1129,11 @@ export async function resolveImplicitProviders(params: {
if (qwenProfiles.length > 0) {
providers["qwen-portal"] = {
...buildQwenPortalProvider(),
apiKey: QWEN_PORTAL_OAUTH_PLACEHOLDER,
apiKey: QWEN_OAUTH_MARKER,
};
}
const volcengineKey =
resolveEnvApiKeyVarName("volcengine") ??
resolveApiKeyFromProfiles({ provider: "volcengine", store: authStore });
const volcengineKey = resolveProviderApiKey("volcengine").apiKey;
if (volcengineKey) {
providers.volcengine = { ...buildDoubaoProvider(), apiKey: volcengineKey };
providers["volcengine-plan"] = {
@@ -1009,9 +1142,7 @@ export async function resolveImplicitProviders(params: {
};
}
const byteplusKey =
resolveEnvApiKeyVarName("byteplus") ??
resolveApiKeyFromProfiles({ provider: "byteplus", store: authStore });
const byteplusKey = resolveProviderApiKey("byteplus").apiKey;
if (byteplusKey) {
providers.byteplus = { ...buildBytePlusProvider(), apiKey: byteplusKey };
providers["byteplus-plan"] = {
@@ -1020,9 +1151,7 @@ export async function resolveImplicitProviders(params: {
};
}
const xiaomiKey =
resolveEnvApiKeyVarName("xiaomi") ??
resolveApiKeyFromProfiles({ provider: "xiaomi", store: authStore });
const xiaomiKey = resolveProviderApiKey("xiaomi").apiKey;
if (xiaomiKey) {
providers.xiaomi = { ...buildXiaomiProvider(), apiKey: xiaomiKey };
}
@@ -1042,7 +1171,9 @@ export async function resolveImplicitProviders(params: {
if (!baseUrl) {
continue;
}
const apiKey = resolveEnvApiKeyVarName("cloudflare-ai-gateway") ?? cred.key?.trim() ?? "";
const envVarApiKey = resolveEnvApiKeyVarName("cloudflare-ai-gateway");
const profileApiKey = resolveApiKeyFromCredential(cred)?.apiKey;
const apiKey = envVarApiKey ?? profileApiKey ?? "";
if (!apiKey) {
continue;
}
@@ -1059,9 +1190,7 @@ export async function resolveImplicitProviders(params: {
// Use the user's configured baseUrl (from explicit providers) for model
// discovery so that remote / non-default Ollama instances are reachable.
// Skip discovery when explicit models are already defined.
const ollamaKey =
resolveEnvApiKeyVarName("ollama") ??
resolveApiKeyFromProfiles({ provider: "ollama", store: authStore });
const ollamaKey = resolveProviderApiKey("ollama").apiKey;
const explicitOllama = params.explicitProviders?.ollama;
const hasExplicitModels =
Array.isArray(explicitOllama?.models) && explicitOllama.models.length > 0;
@@ -1070,7 +1199,7 @@ export async function resolveImplicitProviders(params: {
...explicitOllama,
baseUrl: resolveOllamaApiBase(explicitOllama.baseUrl),
api: explicitOllama.api ?? "ollama",
apiKey: ollamaKey ?? explicitOllama.apiKey ?? "ollama-local",
apiKey: ollamaKey ?? explicitOllama.apiKey ?? OLLAMA_LOCAL_AUTH_MARKER,
};
} else {
const ollamaBaseUrl = explicitOllama?.baseUrl;
@@ -1083,7 +1212,7 @@ export async function resolveImplicitProviders(params: {
if (ollamaProvider.models.length > 0 || ollamaKey || explicitOllama?.apiKey) {
providers.ollama = {
...ollamaProvider,
apiKey: ollamaKey ?? explicitOllama?.apiKey ?? "ollama-local",
apiKey: ollamaKey ?? explicitOllama?.apiKey ?? OLLAMA_LOCAL_AUTH_MARKER,
};
}
}
@@ -1091,23 +1220,16 @@ export async function resolveImplicitProviders(params: {
// vLLM provider - OpenAI-compatible local server (opt-in via env/profile).
// If explicitly configured, keep user-defined models/settings as-is.
if (!params.explicitProviders?.vllm) {
const vllmEnvVar = resolveEnvApiKeyVarName("vllm");
const vllmProfileKey = resolveApiKeyFromProfiles({ provider: "vllm", store: authStore });
const vllmKey = vllmEnvVar ?? vllmProfileKey;
const { apiKey: vllmKey, discoveryApiKey } = resolveProviderApiKey("vllm");
if (vllmKey) {
const discoveryApiKey = vllmEnvVar
? (process.env[vllmEnvVar]?.trim() ?? "")
: (vllmProfileKey ?? "");
providers.vllm = {
...(await buildVllmProvider({ apiKey: discoveryApiKey || undefined })),
...(await buildVllmProvider({ apiKey: discoveryApiKey })),
apiKey: vllmKey,
};
}
}
const togetherKey =
resolveEnvApiKeyVarName("together") ??
resolveApiKeyFromProfiles({ provider: "together", store: authStore });
const togetherKey = resolveProviderApiKey("together").apiKey;
if (togetherKey) {
providers.together = {
...buildTogetherProvider(),
@@ -1115,41 +1237,32 @@ export async function resolveImplicitProviders(params: {
};
}
const huggingfaceKey =
resolveEnvApiKeyVarName("huggingface") ??
resolveApiKeyFromProfiles({ provider: "huggingface", store: authStore });
const { apiKey: huggingfaceKey, discoveryApiKey: huggingfaceDiscoveryApiKey } =
resolveProviderApiKey("huggingface");
if (huggingfaceKey) {
const hfProvider = await buildHuggingfaceProvider(huggingfaceKey);
const hfProvider = await buildHuggingfaceProvider(huggingfaceDiscoveryApiKey);
providers.huggingface = {
...hfProvider,
apiKey: huggingfaceKey,
};
}
const qianfanKey =
resolveEnvApiKeyVarName("qianfan") ??
resolveApiKeyFromProfiles({ provider: "qianfan", store: authStore });
const qianfanKey = resolveProviderApiKey("qianfan").apiKey;
if (qianfanKey) {
providers.qianfan = { ...buildQianfanProvider(), apiKey: qianfanKey };
}
const openrouterKey =
resolveEnvApiKeyVarName("openrouter") ??
resolveApiKeyFromProfiles({ provider: "openrouter", store: authStore });
const openrouterKey = resolveProviderApiKey("openrouter").apiKey;
if (openrouterKey) {
providers.openrouter = { ...buildOpenrouterProvider(), apiKey: openrouterKey };
}
const nvidiaKey =
resolveEnvApiKeyVarName("nvidia") ??
resolveApiKeyFromProfiles({ provider: "nvidia", store: authStore });
const nvidiaKey = resolveProviderApiKey("nvidia").apiKey;
if (nvidiaKey) {
providers.nvidia = { ...buildNvidiaProvider(), apiKey: nvidiaKey };
}
const kilocodeKey =
resolveEnvApiKeyVarName("kilocode") ??
resolveApiKeyFromProfiles({ provider: "kilocode", store: authStore });
const kilocodeKey = resolveProviderApiKey("kilocode").apiKey;
if (kilocodeKey) {
providers.kilocode = { ...(await buildKilocodeProviderWithDiscovery()), apiKey: kilocodeKey };
}

View File

@@ -0,0 +1,162 @@
import { describe, expect, it } from "vitest";
import type { OpenClawConfig } from "../config/config.js";
import {
clearConfigCache,
clearRuntimeConfigSnapshot,
loadConfig,
setRuntimeConfigSnapshot,
} from "../config/config.js";
import { NON_ENV_SECRETREF_MARKER } from "./model-auth-markers.js";
import {
installModelsConfigTestHooks,
withModelsTempHome as withTempHome,
} from "./models-config.e2e-harness.js";
import { ensureOpenClawModelsJson } from "./models-config.js";
import { readGeneratedModelsJson } from "./models-config.test-utils.js";
installModelsConfigTestHooks();
describe("models-config runtime source snapshot", () => {
it("uses runtime source snapshot markers when passed the active runtime config", async () => {
await withTempHome(async () => {
const sourceConfig: OpenClawConfig = {
models: {
providers: {
openai: {
baseUrl: "https://api.openai.com/v1",
apiKey: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, // pragma: allowlist secret
api: "openai-completions" as const,
models: [],
},
},
},
};
const runtimeConfig: OpenClawConfig = {
models: {
providers: {
openai: {
baseUrl: "https://api.openai.com/v1",
apiKey: "sk-runtime-resolved", // pragma: allowlist secret
api: "openai-completions" as const,
models: [],
},
},
},
};
try {
setRuntimeConfigSnapshot(runtimeConfig, sourceConfig);
await ensureOpenClawModelsJson(loadConfig());
const parsed = await readGeneratedModelsJson<{
providers: Record<string, { apiKey?: string }>;
}>();
expect(parsed.providers.openai?.apiKey).toBe("OPENAI_API_KEY"); // pragma: allowlist secret
} finally {
clearRuntimeConfigSnapshot();
clearConfigCache();
}
});
});
it("uses non-env marker from runtime source snapshot for file refs", async () => {
await withTempHome(async () => {
const sourceConfig: OpenClawConfig = {
models: {
providers: {
moonshot: {
baseUrl: "https://api.moonshot.ai/v1",
apiKey: { source: "file", provider: "vault", id: "/moonshot/apiKey" },
api: "openai-completions" as const,
models: [],
},
},
},
};
const runtimeConfig: OpenClawConfig = {
models: {
providers: {
moonshot: {
baseUrl: "https://api.moonshot.ai/v1",
apiKey: "sk-runtime-moonshot", // pragma: allowlist secret
api: "openai-completions" as const,
models: [],
},
},
},
};
try {
setRuntimeConfigSnapshot(runtimeConfig, sourceConfig);
await ensureOpenClawModelsJson(loadConfig());
const parsed = await readGeneratedModelsJson<{
providers: Record<string, { apiKey?: string }>;
}>();
expect(parsed.providers.moonshot?.apiKey).toBe(NON_ENV_SECRETREF_MARKER);
} finally {
clearRuntimeConfigSnapshot();
clearConfigCache();
}
});
});
it("uses header markers from runtime source snapshot instead of resolved runtime values", async () => {
await withTempHome(async () => {
const sourceConfig: OpenClawConfig = {
models: {
providers: {
openai: {
baseUrl: "https://api.openai.com/v1",
api: "openai-completions" as const,
headers: {
Authorization: {
source: "env",
provider: "default",
id: "OPENAI_HEADER_TOKEN", // pragma: allowlist secret
},
"X-Tenant-Token": {
source: "file",
provider: "vault",
id: "/providers/openai/tenantToken",
},
},
models: [],
},
},
},
};
const runtimeConfig: OpenClawConfig = {
models: {
providers: {
openai: {
baseUrl: "https://api.openai.com/v1",
api: "openai-completions" as const,
headers: {
Authorization: "Bearer runtime-openai-token",
"X-Tenant-Token": "runtime-tenant-token",
},
models: [],
},
},
},
};
try {
setRuntimeConfigSnapshot(runtimeConfig, sourceConfig);
await ensureOpenClawModelsJson(loadConfig());
const parsed = await readGeneratedModelsJson<{
providers: Record<string, { headers?: Record<string, string> }>;
}>();
expect(parsed.providers.openai?.headers?.Authorization).toBe(
"secretref-env:OPENAI_HEADER_TOKEN", // pragma: allowlist secret
);
expect(parsed.providers.openai?.headers?.["X-Tenant-Token"]).toBe(NON_ENV_SECRETREF_MARKER);
} finally {
clearRuntimeConfigSnapshot();
clearConfigCache();
}
});
});
});

View File

@@ -1,9 +1,15 @@
import fs from "node:fs/promises";
import path from "node:path";
import { type OpenClawConfig, loadConfig } from "../config/config.js";
import {
getRuntimeConfigSnapshot,
getRuntimeConfigSourceSnapshot,
type OpenClawConfig,
loadConfig,
} from "../config/config.js";
import { applyConfigEnvVars } from "../config/env-vars.js";
import { isRecord } from "../utils.js";
import { resolveOpenClawAgentDir } from "./agent-paths.js";
import { isNonSecretApiKeyMarker } from "./model-auth-markers.js";
import {
normalizeProviders,
type ProviderConfig,
@@ -15,6 +21,7 @@ import {
type ModelsConfig = NonNullable<OpenClawConfig["models"]>;
const DEFAULT_MODE: NonNullable<ModelsConfig["mode"]> = "merge";
const MODELS_JSON_WRITE_LOCKS = new Map<string, Promise<void>>();
function resolvePreferredTokenLimit(explicitValue: number, implicitValue: number): number {
// Keep catalog refresh behavior for stale low values while preserving
@@ -141,8 +148,9 @@ async function resolveProvidersForModelsJson(params: {
function mergeWithExistingProviderSecrets(params: {
nextProviders: Record<string, ProviderConfig>;
existingProviders: Record<string, NonNullable<ModelsConfig["providers"]>[string]>;
secretRefManagedProviders: ReadonlySet<string>;
}): Record<string, ProviderConfig> {
const { nextProviders, existingProviders } = params;
const { nextProviders, existingProviders, secretRefManagedProviders } = params;
const mergedProviders: Record<string, ProviderConfig> = {};
for (const [key, entry] of Object.entries(existingProviders)) {
mergedProviders[key] = entry;
@@ -159,7 +167,12 @@ function mergeWithExistingProviderSecrets(params: {
continue;
}
const preserved: Record<string, unknown> = {};
if (typeof existing.apiKey === "string" && existing.apiKey) {
if (
!secretRefManagedProviders.has(key) &&
typeof existing.apiKey === "string" &&
existing.apiKey &&
!isNonSecretApiKeyMarker(existing.apiKey, { includeEnvVarName: false })
) {
preserved.apiKey = existing.apiKey;
}
if (typeof existing.baseUrl === "string" && existing.baseUrl) {
@@ -174,6 +187,7 @@ async function resolveProvidersForMode(params: {
mode: NonNullable<ModelsConfig["mode"]>;
targetPath: string;
providers: Record<string, ProviderConfig>;
secretRefManagedProviders: ReadonlySet<string>;
}): Promise<Record<string, ProviderConfig>> {
if (params.mode !== "merge") {
return params.providers;
@@ -189,6 +203,7 @@ async function resolveProvidersForMode(params: {
return mergeWithExistingProviderSecrets({
nextProviders: params.providers,
existingProviders,
secretRefManagedProviders: params.secretRefManagedProviders,
});
}
@@ -200,45 +215,94 @@ async function readRawFile(pathname: string): Promise<string> {
}
}
async function ensureModelsFileMode(pathname: string): Promise<void> {
await fs.chmod(pathname, 0o600).catch(() => {
// best-effort
});
}
function resolveModelsConfigInput(config?: OpenClawConfig): OpenClawConfig {
const runtimeSource = getRuntimeConfigSourceSnapshot();
if (!runtimeSource) {
return config ?? loadConfig();
}
if (!config) {
return runtimeSource;
}
const runtimeResolved = getRuntimeConfigSnapshot();
if (runtimeResolved && config === runtimeResolved) {
return runtimeSource;
}
return config;
}
async function withModelsJsonWriteLock<T>(targetPath: string, run: () => Promise<T>): Promise<T> {
const prior = MODELS_JSON_WRITE_LOCKS.get(targetPath) ?? Promise.resolve();
let release: () => void = () => {};
const gate = new Promise<void>((resolve) => {
release = resolve;
});
const pending = prior.then(() => gate);
MODELS_JSON_WRITE_LOCKS.set(targetPath, pending);
try {
await prior;
return await run();
} finally {
release();
if (MODELS_JSON_WRITE_LOCKS.get(targetPath) === pending) {
MODELS_JSON_WRITE_LOCKS.delete(targetPath);
}
}
}
export async function ensureOpenClawModelsJson(
config?: OpenClawConfig,
agentDirOverride?: string,
): Promise<{ agentDir: string; wrote: boolean }> {
const cfg = config ?? loadConfig();
const cfg = resolveModelsConfigInput(config);
const agentDir = agentDirOverride?.trim() ? agentDirOverride.trim() : resolveOpenClawAgentDir();
// Ensure config env vars (e.g. AWS_PROFILE, AWS_ACCESS_KEY_ID) are
// available in process.env before implicit provider discovery. Some
// callers (agent runner, tools) pass config objects that haven't gone
// through the full loadConfig() pipeline which applies these.
applyConfigEnvVars(cfg);
const providers = await resolveProvidersForModelsJson({ cfg, agentDir });
if (Object.keys(providers).length === 0) {
return { agentDir, wrote: false };
}
const mode = cfg.models?.mode ?? DEFAULT_MODE;
const targetPath = path.join(agentDir, "models.json");
const mergedProviders = await resolveProvidersForMode({
mode,
targetPath,
providers,
return await withModelsJsonWriteLock(targetPath, async () => {
// Ensure config env vars (e.g. AWS_PROFILE, AWS_ACCESS_KEY_ID) are
// available in process.env before implicit provider discovery. Some
// callers (agent runner, tools) pass config objects that haven't gone
// through the full loadConfig() pipeline which applies these.
applyConfigEnvVars(cfg);
const providers = await resolveProvidersForModelsJson({ cfg, agentDir });
if (Object.keys(providers).length === 0) {
return { agentDir, wrote: false };
}
const mode = cfg.models?.mode ?? DEFAULT_MODE;
const secretRefManagedProviders = new Set<string>();
const normalizedProviders =
normalizeProviders({
providers,
agentDir,
secretDefaults: cfg.secrets?.defaults,
secretRefManagedProviders,
}) ?? providers;
const mergedProviders = await resolveProvidersForMode({
mode,
targetPath,
providers: normalizedProviders,
secretRefManagedProviders,
});
const next = `${JSON.stringify({ providers: mergedProviders }, null, 2)}\n`;
const existingRaw = await readRawFile(targetPath);
if (existingRaw === next) {
await ensureModelsFileMode(targetPath);
return { agentDir, wrote: false };
}
await fs.mkdir(agentDir, { recursive: true, mode: 0o700 });
await fs.writeFile(targetPath, next, { mode: 0o600 });
await ensureModelsFileMode(targetPath);
return { agentDir, wrote: true };
});
const normalizedProviders = normalizeProviders({
providers: mergedProviders,
agentDir,
});
const next = `${JSON.stringify({ providers: normalizedProviders }, null, 2)}\n`;
const existingRaw = await readRawFile(targetPath);
if (existingRaw === next) {
return { agentDir, wrote: false };
}
await fs.mkdir(agentDir, { recursive: true, mode: 0o700 });
await fs.writeFile(targetPath, next, { mode: 0o600 });
return { agentDir, wrote: true };
}

View File

@@ -0,0 +1,55 @@
import fs from "node:fs/promises";
import { describe, expect, it, vi } from "vitest";
import {
CUSTOM_PROXY_MODELS_CONFIG,
installModelsConfigTestHooks,
withModelsTempHome,
} from "./models-config.e2e-harness.js";
import { ensureOpenClawModelsJson } from "./models-config.js";
import { readGeneratedModelsJson } from "./models-config.test-utils.js";
installModelsConfigTestHooks();
describe("models-config write serialization", () => {
it("serializes concurrent models.json writes to avoid overlap", async () => {
await withModelsTempHome(async () => {
const first = structuredClone(CUSTOM_PROXY_MODELS_CONFIG);
const second = structuredClone(CUSTOM_PROXY_MODELS_CONFIG);
const firstModel = first.models?.providers?.["custom-proxy"]?.models?.[0];
const secondModel = second.models?.providers?.["custom-proxy"]?.models?.[0];
if (!firstModel || !secondModel) {
throw new Error("custom-proxy fixture missing expected model entries");
}
firstModel.name = "Proxy A";
secondModel.name = "Proxy B with longer name";
const originalWriteFile = fs.writeFile.bind(fs);
let inFlightWrites = 0;
let maxInFlightWrites = 0;
const writeSpy = vi.spyOn(fs, "writeFile").mockImplementation(async (...args) => {
inFlightWrites += 1;
if (inFlightWrites > maxInFlightWrites) {
maxInFlightWrites = inFlightWrites;
}
await new Promise((resolve) => setTimeout(resolve, 20));
try {
return await originalWriteFile(...args);
} finally {
inFlightWrites -= 1;
}
});
try {
await Promise.all([ensureOpenClawModelsJson(first), ensureOpenClawModelsJson(second)]);
} finally {
writeSpy.mockRestore();
}
expect(maxInFlightWrites).toBe(1);
const parsed = await readGeneratedModelsJson<{
providers: { "custom-proxy"?: { models?: Array<{ name?: string }> } };
}>();
expect(parsed.providers["custom-proxy"]?.models?.[0]?.name).toBe("Proxy B with longer name");
});
});
});

View File

@@ -179,6 +179,28 @@ describe("buildInlineProviderModels", () => {
expect(result).toHaveLength(1);
expect(result[0].headers).toBeUndefined();
});
it("preserves literal marker-shaped headers in inline provider models", () => {
const providers: Parameters<typeof buildInlineProviderModels>[0] = {
custom: {
headers: {
Authorization: "secretref-env:OPENAI_HEADER_TOKEN",
"X-Managed": "secretref-managed",
"X-Static": "tenant-a",
},
models: [makeModel("custom-model")],
},
};
const result = buildInlineProviderModels(providers);
expect(result).toHaveLength(1);
expect(result[0].headers).toEqual({
Authorization: "secretref-env:OPENAI_HEADER_TOKEN",
"X-Managed": "secretref-managed",
"X-Static": "tenant-a",
});
});
});
describe("resolveModel", () => {
@@ -223,6 +245,56 @@ describe("resolveModel", () => {
});
});
it("preserves literal marker-shaped provider headers in fallback models", () => {
const cfg = {
models: {
providers: {
custom: {
baseUrl: "http://localhost:9000",
headers: {
Authorization: "secretref-env:OPENAI_HEADER_TOKEN",
"X-Managed": "secretref-managed",
"X-Custom-Auth": "token-123",
},
models: [makeModel("listed-model")],
},
},
},
} as OpenClawConfig;
const result = resolveModel("custom", "missing-model", "/tmp/agent", cfg);
expect(result.error).toBeUndefined();
expect((result.model as unknown as { headers?: Record<string, string> }).headers).toEqual({
Authorization: "secretref-env:OPENAI_HEADER_TOKEN",
"X-Managed": "secretref-managed",
"X-Custom-Auth": "token-123",
});
});
it("drops marker headers from discovered models.json entries", () => {
mockDiscoveredModel({
provider: "custom",
modelId: "listed-model",
templateModel: {
...makeModel("listed-model"),
provider: "custom",
headers: {
Authorization: "secretref-env:OPENAI_HEADER_TOKEN",
"X-Managed": "secretref-managed",
"X-Static": "tenant-a",
},
},
});
const result = resolveModel("custom", "listed-model", "/tmp/agent");
expect(result.error).toBeUndefined();
expect((result.model as unknown as { headers?: Record<string, string> }).headers).toEqual({
"X-Static": "tenant-a",
});
});
it("prefers matching configured model metadata for fallback token limits", () => {
const cfg = {
models: {

View File

@@ -5,6 +5,7 @@ import type { ModelDefinitionConfig } from "../../config/types.js";
import { resolveOpenClawAgentDir } from "../agent-paths.js";
import { DEFAULT_CONTEXT_TOKENS } from "../defaults.js";
import { buildModelAliasLines } from "../model-alias-lines.js";
import { isSecretRefHeaderValueMarker } from "../model-auth-markers.js";
import { normalizeModelCompat } from "../model-compat.js";
import { resolveForwardCompatModel } from "../model-forward-compat.js";
import { findNormalizedProviderValue, normalizeProviderId } from "../model-selection.js";
@@ -19,9 +20,29 @@ type InlineProviderConfig = {
baseUrl?: string;
api?: ModelDefinitionConfig["api"];
models?: ModelDefinitionConfig[];
headers?: Record<string, string>;
headers?: unknown;
};
function sanitizeModelHeaders(
headers: unknown,
opts?: { stripSecretRefMarkers?: boolean },
): Record<string, string> | undefined {
if (!headers || typeof headers !== "object" || Array.isArray(headers)) {
return undefined;
}
const next: Record<string, string> = {};
for (const [headerName, headerValue] of Object.entries(headers)) {
if (typeof headerValue !== "string") {
continue;
}
if (opts?.stripSecretRefMarkers && isSecretRefHeaderValueMarker(headerValue)) {
continue;
}
next[headerName] = headerValue;
}
return Object.keys(next).length > 0 ? next : undefined;
}
export { buildModelAliasLines };
function resolveConfiguredProviderConfig(
@@ -46,16 +67,23 @@ function applyConfiguredProviderOverrides(params: {
}): Model<Api> {
const { discoveredModel, providerConfig, modelId } = params;
if (!providerConfig) {
return discoveredModel;
return {
...discoveredModel,
// Discovered models originate from models.json and may contain persistence markers.
headers: sanitizeModelHeaders(discoveredModel.headers, { stripSecretRefMarkers: true }),
};
}
const configuredModel = providerConfig.models?.find((candidate) => candidate.id === modelId);
if (
!configuredModel &&
!providerConfig.baseUrl &&
!providerConfig.api &&
!providerConfig.headers
) {
return discoveredModel;
const discoveredHeaders = sanitizeModelHeaders(discoveredModel.headers, {
stripSecretRefMarkers: true,
});
const providerHeaders = sanitizeModelHeaders(providerConfig.headers);
const configuredHeaders = sanitizeModelHeaders(configuredModel?.headers);
if (!configuredModel && !providerConfig.baseUrl && !providerConfig.api && !providerHeaders) {
return {
...discoveredModel,
headers: discoveredHeaders,
};
}
return {
...discoveredModel,
@@ -67,13 +95,13 @@ function applyConfiguredProviderOverrides(params: {
contextWindow: configuredModel?.contextWindow ?? discoveredModel.contextWindow,
maxTokens: configuredModel?.maxTokens ?? discoveredModel.maxTokens,
headers:
providerConfig.headers || configuredModel?.headers
discoveredHeaders || providerHeaders || configuredHeaders
? {
...discoveredModel.headers,
...providerConfig.headers,
...configuredModel?.headers,
...discoveredHeaders,
...providerHeaders,
...configuredHeaders,
}
: discoveredModel.headers,
: undefined,
compat: configuredModel?.compat ?? discoveredModel.compat,
};
}
@@ -86,15 +114,22 @@ export function buildInlineProviderModels(
if (!trimmed) {
return [];
}
const providerHeaders = sanitizeModelHeaders(entry?.headers);
return (entry?.models ?? []).map((model) => ({
...model,
provider: trimmed,
baseUrl: entry?.baseUrl,
api: model.api ?? entry?.api,
headers:
entry?.headers || (model as InlineModelEntry).headers
? { ...entry?.headers, ...(model as InlineModelEntry).headers }
: undefined,
headers: (() => {
const modelHeaders = sanitizeModelHeaders((model as InlineModelEntry).headers);
if (!providerHeaders && !modelHeaders) {
return undefined;
}
return {
...providerHeaders,
...modelHeaders,
};
})(),
}));
});
}
@@ -161,6 +196,8 @@ export function resolveModelWithRegistry(params: {
}
const configuredModel = providerConfig?.models?.find((candidate) => candidate.id === modelId);
const providerHeaders = sanitizeModelHeaders(providerConfig?.headers);
const modelHeaders = sanitizeModelHeaders(configuredModel?.headers);
if (providerConfig || modelId.startsWith("mock-")) {
return normalizeModelCompat({
id: modelId,
@@ -180,9 +217,7 @@ export function resolveModelWithRegistry(params: {
providerConfig?.models?.[0]?.maxTokens ??
DEFAULT_CONTEXT_TOKENS,
headers:
providerConfig?.headers || configuredModel?.headers
? { ...providerConfig?.headers, ...configuredModel?.headers }
: undefined,
providerHeaders || modelHeaders ? { ...providerHeaders, ...modelHeaders } : undefined,
} as Model<Api>);
}