feat(security): add provider-based external secrets management

This commit is contained in:
joshavant
2026-02-25 17:39:31 -06:00
committed by Peter Steinberger
parent bb60cab76d
commit 4e7a833a24
35 changed files with 1779 additions and 669 deletions

View File

@@ -177,15 +177,18 @@ describe("ensureApiKeyFromEnvOrPrompt", () => {
});
expect(result).toBe("env-key");
expect(setCredential).toHaveBeenCalledWith({ source: "env", id: "MINIMAX_API_KEY" }, "ref");
expect(setCredential).toHaveBeenCalledWith(
{ source: "env", provider: "default", id: "MINIMAX_API_KEY" },
"ref",
);
expect(text).not.toHaveBeenCalled();
});
it("re-prompts after sops ref validation failure and succeeds with env ref", async () => {
it("re-prompts after provider ref validation failure and succeeds with env ref", async () => {
process.env.MINIMAX_API_KEY = "env-key";
delete process.env.MINIMAX_OAUTH_TOKEN;
const selectValues: Array<"file" | "env"> = ["file", "env"];
const selectValues: Array<"provider" | "env" | "filemain"> = ["provider", "filemain", "env"];
const select = vi.fn(async () => selectValues.shift() ?? "env") as WizardPrompter["select"];
const text = vi
.fn<WizardPrompter["text"]>()
@@ -195,7 +198,17 @@ describe("ensureApiKeyFromEnvOrPrompt", () => {
const setCredential = vi.fn(async () => undefined);
const result = await ensureApiKeyFromEnvOrPrompt({
config: {},
config: {
secrets: {
providers: {
filemain: {
source: "file",
path: "/tmp/does-not-exist-secrets.json",
mode: "jsonPointer",
},
},
},
},
provider: "minimax",
envLabel: "MINIMAX_API_KEY",
promptMessage: "Enter key",
@@ -207,9 +220,12 @@ describe("ensureApiKeyFromEnvOrPrompt", () => {
});
expect(result).toBe("env-key");
expect(setCredential).toHaveBeenCalledWith({ source: "env", id: "MINIMAX_API_KEY" }, "ref");
expect(setCredential).toHaveBeenCalledWith(
{ source: "env", provider: "default", id: "MINIMAX_API_KEY" },
"ref",
);
expect(note).toHaveBeenCalledWith(
expect.stringContaining("Could not validate this encrypted file reference."),
expect.stringContaining("Could not validate provider reference"),
"Reference check failed",
);
});

View File

@@ -1,6 +1,10 @@
import { resolveEnvApiKey } from "../agents/model-auth.js";
import type { OpenClawConfig } from "../config/types.js";
import type { SecretInput, SecretRef } from "../config/types.secrets.js";
import {
DEFAULT_SECRET_PROVIDER_ALIAS,
type SecretInput,
type SecretRef,
} from "../config/types.secrets.js";
import { encodeJsonPointerToken } from "../secrets/json-pointer.js";
import { PROVIDER_ENV_VARS } from "../secrets/provider-env-vars.js";
import { resolveSecretRefString } from "../secrets/resolve.js";
@@ -14,7 +18,7 @@ const ENV_SOURCE_LABEL_RE = /(?:^|:\s)([A-Z][A-Z0-9_]*)$/;
const ENV_SECRET_REF_ID_RE = /^[A-Z][A-Z0-9_]{0,127}$/;
const FILE_SECRET_REF_SEGMENT_RE = /^(?:[^~]|~0|~1)*$/;
type SecretRefSourceChoice = "env" | "file";
type SecretRefChoice = "env" | "provider";
function isValidFileSecretRefId(value: string): boolean {
if (!value.startsWith("/")) {
@@ -43,11 +47,36 @@ function resolveDefaultProviderEnvVar(provider: string): string | undefined {
return envVars?.find((candidate) => candidate.trim().length > 0);
}
function resolveDefaultSopsPointerId(provider: string): string {
function resolveDefaultFilePointerId(provider: string): string {
return `/providers/${encodeJsonPointerToken(provider)}/apiKey`;
}
function resolveDefaultProviderAlias(
config: OpenClawConfig,
source: "env" | "file" | "exec",
): string {
const configured =
source === "env"
? config.secrets?.defaults?.env
: source === "file"
? config.secrets?.defaults?.file
: config.secrets?.defaults?.exec;
if (configured?.trim()) {
return configured.trim();
}
const providers = config.secrets?.providers;
if (providers) {
for (const [providerName, provider] of Object.entries(providers)) {
if (provider?.source === source) {
return providerName;
}
}
}
return DEFAULT_SECRET_PROVIDER_ALIAS;
}
function resolveRefFallbackInput(params: {
config: OpenClawConfig;
provider: string;
preferredEnvVar?: string;
envKeyValue?: string;
@@ -57,7 +86,11 @@ function resolveRefFallbackInput(params: {
const value = process.env[fallbackEnvVar]?.trim();
if (value) {
return {
input: { source: "env", id: fallbackEnvVar },
input: {
source: "env",
provider: resolveDefaultProviderAlias(params.config, "env"),
id: fallbackEnvVar,
},
resolvedValue: value,
};
}
@@ -81,11 +114,11 @@ async function resolveApiKeyRefForOnboarding(params: {
}): Promise<{ ref: SecretRef; resolvedValue: string }> {
const defaultEnvVar =
params.preferredEnvVar ?? resolveDefaultProviderEnvVar(params.provider) ?? "";
const defaultFilePointer = resolveDefaultSopsPointerId(params.provider);
let sourceChoice: SecretRefSourceChoice = "env";
const defaultFilePointer = resolveDefaultFilePointerId(params.provider);
let sourceChoice: SecretRefChoice = "env";
while (true) {
const sourceRaw: SecretRefSourceChoice = await params.prompter.select<SecretRefSourceChoice>({
const sourceRaw: SecretRefChoice = await params.prompter.select<SecretRefChoice>({
message: "Where is this API key stored?",
initialValue: sourceChoice,
options: [
@@ -95,13 +128,13 @@ async function resolveApiKeyRefForOnboarding(params: {
hint: "Reference a variable from your runtime environment",
},
{
value: "file",
label: "Encrypted sops file",
hint: "Reference a JSON pointer from secrets.sources.file",
value: "provider",
label: "Configured secret provider",
hint: "Use a configured file or exec secret provider",
},
],
});
const source: SecretRefSourceChoice = sourceRaw === "file" ? "file" : "env";
const source: SecretRefChoice = sourceRaw === "provider" ? "provider" : "env";
sourceChoice = source;
if (source === "env") {
@@ -128,7 +161,11 @@ async function resolveApiKeyRefForOnboarding(params: {
`No valid environment variable name provided for provider "${params.provider}".`,
);
}
const ref: SecretRef = { source: "env", id: envVar };
const ref: SecretRef = {
source: "env",
provider: resolveDefaultProviderAlias(params.config, "env"),
id: envVar,
};
const resolvedValue = await resolveSecretRefString(ref, {
config: params.config,
env: process.env,
@@ -140,36 +177,94 @@ async function resolveApiKeyRefForOnboarding(params: {
return { ref, resolvedValue };
}
const pointerRaw = await params.prompter.text({
message: "JSON pointer inside encrypted secrets file",
initialValue: defaultFilePointer,
placeholder: "/providers/openai/apiKey",
const externalProviders = Object.entries(params.config.secrets?.providers ?? {}).filter(
([, provider]) => provider?.source === "file" || provider?.source === "exec",
);
if (externalProviders.length === 0) {
await params.prompter.note(
"No file/exec secret providers are configured yet. Add one under secrets.providers, or select Environment variable.",
"No providers configured",
);
continue;
}
const defaultProvider = resolveDefaultProviderAlias(params.config, "file");
const selectedProvider = await params.prompter.select<string>({
message: "Select secret provider",
initialValue:
externalProviders.find(([providerName]) => providerName === defaultProvider)?.[0] ??
externalProviders[0]?.[0],
options: externalProviders.map(([providerName, provider]) => ({
value: providerName,
label: providerName,
hint: provider?.source === "exec" ? "Exec provider" : "File provider",
})),
});
const providerEntry = params.config.secrets?.providers?.[selectedProvider];
if (!providerEntry || (providerEntry.source !== "file" && providerEntry.source !== "exec")) {
await params.prompter.note(
`Provider "${selectedProvider}" is not a file/exec provider.`,
"Invalid provider",
);
continue;
}
const idPrompt =
providerEntry.source === "file"
? "Secret id (JSON pointer for jsonPointer mode, or 'value' for raw mode)"
: "Secret id for the exec provider";
const idDefault =
providerEntry.source === "file"
? providerEntry.mode === "raw"
? "value"
: defaultFilePointer
: `${params.provider}/apiKey`;
const idRaw = await params.prompter.text({
message: idPrompt,
initialValue: idDefault,
placeholder: providerEntry.source === "file" ? "/providers/openai/apiKey" : "openai/api-key",
validate: (value) => {
const candidate = value.trim();
if (!isValidFileSecretRefId(candidate)) {
if (!candidate) {
return "Secret id cannot be empty.";
}
if (
providerEntry.source === "file" &&
providerEntry.mode !== "raw" &&
!isValidFileSecretRefId(candidate)
) {
return 'Use an absolute JSON pointer like "/providers/openai/apiKey".';
}
if (
providerEntry.source === "file" &&
providerEntry.mode === "raw" &&
candidate !== "value"
) {
return 'Raw file mode expects id "value".';
}
return undefined;
},
});
const pointer = String(pointerRaw ?? "").trim() || defaultFilePointer;
const ref: SecretRef = { source: "file", id: pointer };
const id = String(idRaw ?? "").trim() || idDefault;
const ref: SecretRef = {
source: providerEntry.source,
provider: selectedProvider,
id,
};
try {
const resolvedValue = await resolveSecretRefString(ref, {
config: params.config,
env: process.env,
});
await params.prompter.note(
`Validated encrypted file reference ${pointer}. OpenClaw will store a reference, not the key value.`,
`Validated ${providerEntry.source} reference ${selectedProvider}:${id}. OpenClaw will store a reference, not the key value.`,
"Reference validated",
);
return { ref, resolvedValue };
} catch (error) {
await params.prompter.note(
[
"Could not validate this encrypted file reference.",
`Could not validate provider reference ${selectedProvider}:${id}.`,
formatErrorMessage(error),
"Check secrets.sources.file configuration and sops key access, then try again.",
"Check your provider configuration and try again.",
].join("\n"),
"Reference check failed",
);
@@ -287,7 +382,7 @@ export async function resolveSecretInputModeForEnvSelection(params: {
{
value: "ref",
label: "Use secret reference",
hint: "Stores a reference to env or encrypted sops secrets",
hint: "Stores a reference to env or configured external secret providers",
},
],
});
@@ -379,6 +474,7 @@ export async function ensureApiKeyFromEnvOrPrompt(params: {
if (selectedMode === "ref") {
if (typeof params.prompter.select !== "function") {
const fallback = resolveRefFallbackInput({
config: params.config,
provider: params.provider,
preferredEnvVar: envKey?.source ? extractEnvVarFromSourceLabel(envKey.source) : undefined,
envKeyValue: envKey?.apiKey,

View File

@@ -181,6 +181,7 @@ describe("applyAuthChoiceMiniMax", () => {
const parsed = await readAuthProfiles(agentDir);
expect(parsed.profiles?.["minimax-cn:default"]?.keyRef).toEqual({
source: "env",
provider: "default",
id: "MINIMAX_API_KEY",
});
expect(parsed.profiles?.["minimax-cn:default"]?.key).toBeUndefined();

View File

@@ -82,7 +82,7 @@ describe("applyAuthChoiceOpenAI", () => {
profiles?: Record<string, { key?: string; keyRef?: unknown }>;
}>(agentDir);
expect(parsed.profiles?.["openai:default"]).toMatchObject({
keyRef: { source: "env", id: "OPENAI_API_KEY" },
keyRef: { source: "env", provider: "default", id: "OPENAI_API_KEY" },
});
expect(parsed.profiles?.["openai:default"]?.key).toBeUndefined();
});

View File

@@ -88,7 +88,7 @@ describe("volcengine/byteplus auth choice", () => {
profiles?: Record<string, { key?: string; keyRef?: unknown }>;
}>(agentDir);
expect(parsed.profiles?.["volcengine:default"]).toMatchObject({
keyRef: { source: "env", id: "VOLCANO_ENGINE_API_KEY" },
keyRef: { source: "env", provider: "default", id: "VOLCANO_ENGINE_API_KEY" },
});
expect(parsed.profiles?.["volcengine:default"]?.key).toBeUndefined();
});
@@ -153,7 +153,7 @@ describe("volcengine/byteplus auth choice", () => {
profiles?: Record<string, { key?: string; keyRef?: unknown }>;
}>(agentDir);
expect(parsed.profiles?.["byteplus:default"]).toMatchObject({
keyRef: { source: "env", id: "BYTEPLUS_API_KEY" },
keyRef: { source: "env", provider: "default", id: "BYTEPLUS_API_KEY" },
});
expect(parsed.profiles?.["byteplus:default"]?.key).toBeUndefined();
});

View File

@@ -46,7 +46,7 @@ vi.mock("./zai-endpoint-detect.js", () => ({
type StoredAuthProfile = {
key?: string;
keyRef?: { source: string; id: string };
keyRef?: { source: string; provider: string; id: string };
access?: string;
refresh?: string;
provider?: string;
@@ -633,7 +633,7 @@ describe("applyAuthChoice", () => {
expectEnvPrompt: boolean;
expectedTextCalls: number;
expectedKey?: string;
expectedKeyRef?: { source: "env"; id: string };
expectedKeyRef?: { source: "env"; provider: string; id: string };
expectedModel?: string;
expectedModelPrefix?: string;
}> = [
@@ -679,7 +679,7 @@ describe("applyAuthChoice", () => {
opts: { secretInputMode: "ref" },
expectEnvPrompt: false,
expectedTextCalls: 1,
expectedKeyRef: { source: "env", id: "AI_GATEWAY_API_KEY" },
expectedKeyRef: { source: "env", provider: "default", id: "AI_GATEWAY_API_KEY" },
expectedModel: "vercel-ai-gateway/anthropic/claude-opus-4.6",
},
];
@@ -740,14 +740,16 @@ describe("applyAuthChoice", () => {
}
});
it("retries ref setup when sops preflight fails and can switch to env ref", async () => {
it("retries ref setup when provider preflight fails and can switch to env ref", async () => {
await setupTempState();
process.env.OPENAI_API_KEY = "sk-openai-env";
const selectValues: Array<"file" | "env"> = ["file", "env"];
const selectValues: Array<"provider" | "env" | "filemain"> = ["provider", "filemain", "env"];
const select = vi.fn(async (params: Parameters<WizardPrompter["select"]>[0]) => {
if (params.options.some((option) => option.value === "file")) {
return (selectValues.shift() ?? "env") as never;
const next = selectValues[0];
if (next && params.options.some((option) => option.value === next)) {
selectValues.shift();
return next as never;
}
return (params.options[0]?.value ?? "env") as never;
});
@@ -767,7 +769,17 @@ describe("applyAuthChoice", () => {
const result = await applyAuthChoice({
authChoice: "openai-api-key",
config: {},
config: {
secrets: {
providers: {
filemain: {
source: "file",
path: "/tmp/openclaw-missing-secrets.json",
mode: "jsonPointer",
},
},
},
},
prompter,
runtime,
setDefaultModel: false,
@@ -779,7 +791,7 @@ describe("applyAuthChoice", () => {
mode: "api_key",
});
expect(note).toHaveBeenCalledWith(
expect.stringContaining("Could not validate this encrypted file reference."),
expect.stringContaining("Could not validate provider reference"),
"Reference check failed",
);
expect(note).toHaveBeenCalledWith(
@@ -787,7 +799,7 @@ describe("applyAuthChoice", () => {
"Reference validated",
);
expect(await readAuthProfile("openai:default")).toMatchObject({
keyRef: { source: "env", id: "OPENAI_API_KEY" },
keyRef: { source: "env", provider: "default", id: "OPENAI_API_KEY" },
});
});
@@ -1014,7 +1026,7 @@ describe("applyAuthChoice", () => {
expectEnvPrompt: boolean;
expectedTextCalls: number;
expectedKey?: string;
expectedKeyRef?: { source: string; id: string };
expectedKeyRef?: { source: string; provider: string; id: string };
expectedMetadata: { accountId: string; gatewayId: string };
}> = [
{
@@ -1038,10 +1050,7 @@ describe("applyAuthChoice", () => {
},
expectEnvPrompt: false,
expectedTextCalls: 3,
expectedKeyRef: {
source: "env",
id: "CLOUDFLARE_AI_GATEWAY_API_KEY",
},
expectedKeyRef: { source: "env", provider: "default", id: "CLOUDFLARE_AI_GATEWAY_API_KEY" },
expectedMetadata: {
accountId: "cf-account-id-ref",
gatewayId: "cf-gateway-id-ref",

View File

@@ -12,7 +12,7 @@ describe("resolveProviderAuthOverview", () => {
"github-copilot:default": {
type: "token",
provider: "github-copilot",
tokenRef: { source: "env", id: "GITHUB_TOKEN" },
tokenRef: { source: "env", provider: "default", id: "GITHUB_TOKEN" },
},
},
} as never,

View File

@@ -55,7 +55,7 @@ describe("onboard auth credentials secret refs", () => {
profiles?: Record<string, { key?: string; keyRef?: unknown }>;
}>(env.agentDir);
expect(parsed.profiles?.["moonshot:default"]).toMatchObject({
keyRef: { source: "env", id: "MOONSHOT_API_KEY" },
keyRef: { source: "env", provider: "default", id: "MOONSHOT_API_KEY" },
});
expect(parsed.profiles?.["moonshot:default"]?.key).toBeUndefined();
});
@@ -70,7 +70,7 @@ describe("onboard auth credentials secret refs", () => {
profiles?: Record<string, { key?: string; keyRef?: unknown }>;
}>(env.agentDir);
expect(parsed.profiles?.["moonshot:default"]).toMatchObject({
keyRef: { source: "env", id: "MOONSHOT_API_KEY" },
keyRef: { source: "env", provider: "default", id: "MOONSHOT_API_KEY" },
});
expect(parsed.profiles?.["moonshot:default"]?.key).toBeUndefined();
});
@@ -104,7 +104,7 @@ describe("onboard auth credentials secret refs", () => {
profiles?: Record<string, { key?: string; keyRef?: unknown; metadata?: unknown }>;
}>(env.agentDir);
expect(parsed.profiles?.["cloudflare-ai-gateway:default"]).toMatchObject({
keyRef: { source: "env", id: "CLOUDFLARE_AI_GATEWAY_API_KEY" },
keyRef: { source: "env", provider: "default", id: "CLOUDFLARE_AI_GATEWAY_API_KEY" },
metadata: { accountId: "account-1", gatewayId: "gateway-1" },
});
expect(parsed.profiles?.["cloudflare-ai-gateway:default"]?.key).toBeUndefined();
@@ -137,7 +137,7 @@ describe("onboard auth credentials secret refs", () => {
profiles?: Record<string, { key?: string; keyRef?: unknown }>;
}>(env.agentDir);
expect(parsed.profiles?.["openai:default"]).toMatchObject({
keyRef: { source: "env", id: "OPENAI_API_KEY" },
keyRef: { source: "env", provider: "default", id: "OPENAI_API_KEY" },
});
expect(parsed.profiles?.["openai:default"]?.key).toBeUndefined();
});
@@ -156,12 +156,12 @@ describe("onboard auth credentials secret refs", () => {
}>(env.agentDir);
expect(parsed.profiles?.["volcengine:default"]).toMatchObject({
keyRef: { source: "env", id: "VOLCANO_ENGINE_API_KEY" },
keyRef: { source: "env", provider: "default", id: "VOLCANO_ENGINE_API_KEY" },
});
expect(parsed.profiles?.["volcengine:default"]?.key).toBeUndefined();
expect(parsed.profiles?.["byteplus:default"]).toMatchObject({
keyRef: { source: "env", id: "BYTEPLUS_API_KEY" },
keyRef: { source: "env", provider: "default", id: "BYTEPLUS_API_KEY" },
});
expect(parsed.profiles?.["byteplus:default"]?.key).toBeUndefined();
});

View File

@@ -4,7 +4,12 @@ import type { OAuthCredentials } from "@mariozechner/pi-ai";
import { resolveOpenClawAgentDir } from "../agents/agent-paths.js";
import { upsertAuthProfile } from "../agents/auth-profiles.js";
import { resolveStateDir } from "../config/paths.js";
import { isSecretRef, type SecretInput, type SecretRef } from "../config/types.secrets.js";
import {
coerceSecretRef,
DEFAULT_SECRET_PROVIDER_ALIAS,
type SecretInput,
type SecretRef,
} from "../config/types.secrets.js";
import { KILOCODE_DEFAULT_MODEL_REF } from "../providers/kilocode-shared.js";
import { PROVIDER_ENV_VARS } from "../secrets/provider-env-vars.js";
import { normalizeSecretInput } from "../utils/normalize-secret-input.js";
@@ -22,7 +27,7 @@ export type ApiKeyStorageOptions = {
};
function buildEnvSecretRef(id: string): SecretRef {
return { source: "env", id };
return { source: "env", provider: DEFAULT_SECRET_PROVIDER_ALIAS, id };
}
function parseEnvSecretRef(value: string): SecretRef | null {
@@ -49,8 +54,9 @@ function resolveApiKeySecretInput(
input: SecretInput,
options?: ApiKeyStorageOptions,
): SecretInput {
if (isSecretRef(input)) {
return input;
const coercedRef = coerceSecretRef(input);
if (coercedRef) {
return coercedRef;
}
const normalized = normalizeSecretInput(input);
const inlineEnvRef = parseEnvSecretRef(normalized);

View File

@@ -238,6 +238,7 @@ describe("promptCustomApiConfig", () => {
expect(result.config.models?.providers?.custom?.apiKey).toEqual({
source: "env",
provider: "default",
id: "CUSTOM_PROVIDER_API_KEY",
});
const firstCall = fetchMock.mock.calls[0]?.[1] as
@@ -246,7 +247,7 @@ describe("promptCustomApiConfig", () => {
expect(firstCall?.headers?.Authorization).toBe("Bearer test-env-key");
});
it("re-prompts source after encrypted file ref preflight fails and succeeds with env ref", async () => {
it("re-prompts source after provider ref preflight fails and succeeds with env ref", async () => {
vi.stubEnv("CUSTOM_PROVIDER_API_KEY", "test-env-key");
const prompter = createTestPrompter({
text: [
@@ -257,18 +258,29 @@ describe("promptCustomApiConfig", () => {
"custom",
"",
],
select: ["ref", "file", "env", "openai"],
select: ["ref", "provider", "filemain", "env", "openai"],
});
stubFetchSequence([{ ok: true }]);
const result = await runPromptCustomApi(prompter);
const result = await runPromptCustomApi(prompter, {
secrets: {
providers: {
filemain: {
source: "file",
path: "/tmp/openclaw-missing-provider.json",
mode: "jsonPointer",
},
},
},
});
expect(prompter.note).toHaveBeenCalledWith(
expect.stringContaining("Could not validate this encrypted file reference."),
expect.stringContaining("Could not validate provider reference"),
"Reference check failed",
);
expect(result.config.models?.providers?.custom?.apiKey).toEqual({
source: "env",
provider: "default",
id: "CUSTOM_PROVIDER_API_KEY",
});
});

View File

@@ -611,6 +611,7 @@ describe("onboard (non-interactive): provider auth", () => {
});
expect(await readCustomLocalProviderApiKeyInput(configPath)).toEqual({
source: "env",
provider: "default",
id: "CUSTOM_API_KEY",
});
},

View File

@@ -814,7 +814,7 @@ export async function applyNonInteractiveAuthChoice(params: {
});
const customApiKeyInput: SecretInput | undefined =
requestedSecretInputMode === "ref" && resolvedCustomApiKey?.source === "env"
? { source: "env", id: "CUSTOM_API_KEY" }
? { source: "env", provider: "default", id: "CUSTOM_API_KEY" }
: resolvedCustomApiKey?.key;
const result = applyCustomApiConfig({
config: nextConfig,