Secrets: add migrate rollback and skill ref support

This commit is contained in:
joshavant
2026-02-21 14:23:44 -08:00
committed by Peter Steinberger
parent 2e53033f22
commit f6a854bd37
11 changed files with 1450 additions and 5 deletions

View File

@@ -65,4 +65,52 @@ describe("discoverAuthStorage", () => {
await fs.rm(agentDir, { recursive: true, force: true });
}
});
it("scrubs static api_key entries from legacy auth.json and keeps oauth entries", async () => {
const agentDir = await createAgentDir();
try {
saveAuthProfileStore(
{
version: 1,
profiles: {
"openrouter:default": {
type: "api_key",
provider: "openrouter",
key: "sk-or-v1-runtime",
},
},
},
agentDir,
);
await fs.writeFile(
path.join(agentDir, "auth.json"),
JSON.stringify(
{
openrouter: { type: "api_key", key: "legacy-static-key" },
"openai-codex": {
type: "oauth",
access: "oauth-access",
refresh: "oauth-refresh",
expires: Date.now() + 60_000,
},
},
null,
2,
),
);
discoverAuthStorage(agentDir);
const parsed = JSON.parse(await fs.readFile(path.join(agentDir, "auth.json"), "utf8")) as {
[key: string]: unknown;
};
expect(parsed.openrouter).toBeUndefined();
expect(parsed["openai-codex"]).toMatchObject({
type: "oauth",
access: "oauth-access",
});
} finally {
await fs.rm(agentDir, { recursive: true, force: true });
}
});
});

View File

@@ -105,9 +105,10 @@ function applySkillConfigEnvOverrides(params: {
}
}
if (normalizedPrimaryEnv && skillConfig.apiKey && !process.env[normalizedPrimaryEnv]) {
const resolvedApiKey = typeof skillConfig.apiKey === "string" ? skillConfig.apiKey.trim() : "";
if (normalizedPrimaryEnv && resolvedApiKey && !process.env[normalizedPrimaryEnv]) {
if (!pendingOverrides[normalizedPrimaryEnv]) {
pendingOverrides[normalizedPrimaryEnv] = skillConfig.apiKey;
pendingOverrides[normalizedPrimaryEnv] = resolvedApiKey;
}
}

View File

@@ -3,6 +3,8 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
import { createCliRuntimeCapture } from "./test-runtime-capture.js";
const callGatewayFromCli = vi.fn();
const runSecretsMigration = vi.fn();
const rollbackSecretsMigration = vi.fn();
const { defaultRuntime, runtimeLogs, runtimeErrors, resetRuntimeCapture } =
createCliRuntimeCapture();
@@ -17,6 +19,11 @@ vi.mock("../runtime.js", () => ({
defaultRuntime,
}));
vi.mock("../secrets/migrate.js", () => ({
runSecretsMigration: (options: unknown) => runSecretsMigration(options),
rollbackSecretsMigration: (options: unknown) => rollbackSecretsMigration(options),
}));
const { registerSecretsCli } = await import("./secrets-cli.js");
describe("secrets CLI", () => {
@@ -30,6 +37,8 @@ describe("secrets CLI", () => {
beforeEach(() => {
resetRuntimeCapture();
callGatewayFromCli.mockReset();
runSecretsMigration.mockReset();
rollbackSecretsMigration.mockReset();
});
it("calls secrets.reload and prints human output", async () => {
@@ -50,4 +59,38 @@ describe("secrets CLI", () => {
await createProgram().parseAsync(["secrets", "reload", "--json"], { from: "user" });
expect(runtimeLogs.at(-1)).toContain('"ok": true');
});
it("runs secrets migrate as dry-run by default", async () => {
runSecretsMigration.mockResolvedValue({
mode: "dry-run",
changed: true,
secretsFilePath: "/tmp/secrets.enc.json",
counters: { secretsWritten: 3 },
changedFiles: ["/tmp/openclaw.json"],
});
await createProgram().parseAsync(["secrets", "migrate"], { from: "user" });
expect(runSecretsMigration).toHaveBeenCalledWith(
expect.objectContaining({ write: false, scrubEnv: true }),
);
expect(runtimeLogs.at(-1)).toContain("dry run");
});
it("runs rollback when --rollback is provided", async () => {
rollbackSecretsMigration.mockResolvedValue({
backupId: "20260221T010203Z",
restoredFiles: 2,
deletedFiles: 1,
});
await createProgram().parseAsync(["secrets", "migrate", "--rollback", "20260221T010203Z"], {
from: "user",
});
expect(rollbackSecretsMigration).toHaveBeenCalledWith({
backupId: "20260221T010203Z",
});
expect(runtimeLogs.at(-1)).toContain("rollback complete");
});
});

View File

@@ -1,11 +1,59 @@
import type { Command } from "commander";
import { danger } from "../globals.js";
import { defaultRuntime } from "../runtime.js";
import {
rollbackSecretsMigration,
runSecretsMigration,
type SecretsMigrationRollbackResult,
type SecretsMigrationRunResult,
} from "../secrets/migrate.js";
import { formatDocsLink } from "../terminal/links.js";
import { theme } from "../terminal/theme.js";
import { addGatewayClientOptions, callGatewayFromCli, type GatewayRpcOpts } from "./gateway-rpc.js";
type SecretsReloadOptions = GatewayRpcOpts & { json?: boolean };
type SecretsMigrateOptions = {
write?: boolean;
rollback?: string;
scrubEnv?: boolean;
json?: boolean;
};
function printMigrationResult(
result: SecretsMigrationRunResult | SecretsMigrationRollbackResult,
json: boolean,
): void {
if (json) {
defaultRuntime.log(JSON.stringify(result, null, 2));
return;
}
if ("restoredFiles" in result) {
defaultRuntime.log(
`Secrets rollback complete for backup ${result.backupId}. Restored ${result.restoredFiles} file(s), deleted ${result.deletedFiles} file(s).`,
);
return;
}
if (result.mode === "dry-run") {
if (!result.changed) {
defaultRuntime.log("Secrets migrate dry run: no changes needed.");
return;
}
defaultRuntime.log(
`Secrets migrate dry run: ${result.changedFiles.length} file(s) would change, ${result.counters.secretsWritten} secret value(s) would move to ${result.secretsFilePath}.`,
);
return;
}
if (!result.changed) {
defaultRuntime.log("Secrets migrate: no changes applied.");
return;
}
defaultRuntime.log(
`Secrets migrated. Backup: ${result.backupId}. Moved ${result.counters.secretsWritten} secret value(s) into ${result.secretsFilePath}.`,
);
}
export function registerSecretsCli(program: Command) {
const secrets = program
@@ -44,4 +92,30 @@ export function registerSecretsCli(program: Command) {
defaultRuntime.exit(1);
}
});
secrets
.command("migrate")
.description("Migrate plaintext secrets to file-backed SecretRefs (sops)")
.option("--write", "Apply migration changes (default is dry-run)", false)
.option("--rollback <backup-id>", "Rollback a previous migration backup id")
.option("--no-scrub-env", "Keep matching plaintext values in ~/.openclaw/.env")
.option("--json", "Output JSON", false)
.action(async (opts: SecretsMigrateOptions) => {
try {
if (typeof opts.rollback === "string" && opts.rollback.trim()) {
const result = await rollbackSecretsMigration({ backupId: opts.rollback.trim() });
printMigrationResult(result, Boolean(opts.json));
return;
}
const result = await runSecretsMigration({
write: Boolean(opts.write),
scrubEnv: opts.scrubEnv ?? true,
});
printMigrationResult(result, Boolean(opts.json));
} catch (err) {
defaultRuntime.error(danger(String(err)));
defaultRuntime.exit(1);
}
});
}

View File

@@ -36,6 +36,21 @@ describe("config secret refs schema", () => {
expect(result.ok).toBe(true);
});
it("accepts skills entry apiKey refs", () => {
const result = validateConfigObjectRaw({
skills: {
entries: {
"review-pr": {
enabled: true,
apiKey: { source: "env", id: "SKILL_REVIEW_PR_API_KEY" },
},
},
},
});
expect(result.ok).toBe(true);
});
it("rejects invalid secret ref id", () => {
const result = validateConfigObjectRaw({
models: {

View File

@@ -1,6 +1,8 @@
import type { SecretInput } from "./types.secrets.js";
export type SkillConfig = {
enabled?: boolean;
apiKey?: string;
apiKey?: SecretInput;
env?: Record<string, string>;
config?: Record<string, unknown>;
};

View File

@@ -4,7 +4,12 @@ import { parseDurationMs } from "../cli/parse-duration.js";
import { ToolsSchema } from "./zod-schema.agent-runtime.js";
import { AgentsSchema, AudioSchema, BindingsSchema, BroadcastSchema } from "./zod-schema.agents.js";
import { ApprovalsSchema } from "./zod-schema.approvals.js";
import { HexColorSchema, ModelsConfigSchema, SecretsConfigSchema } from "./zod-schema.core.js";
import {
HexColorSchema,
ModelsConfigSchema,
SecretInputSchema,
SecretsConfigSchema,
} from "./zod-schema.core.js";
import { HookMappingSchema, HooksGmailSchema, InternalHooksSchema } from "./zod-schema.hooks.js";
import { InstallRecordShape } from "./zod-schema.installs.js";
import { ChannelsSchema } from "./zod-schema.providers.js";
@@ -722,7 +727,7 @@ export const OpenClawSchema = z
z
.object({
enabled: z.boolean().optional(),
apiKey: z.string().optional().register(sensitive),
apiKey: SecretInputSchema.optional().register(sensitive),
env: z.record(z.string(), z.string()).optional(),
config: z.record(z.string(), z.unknown()).optional(),
})

204
src/secrets/migrate.test.ts Normal file
View File

@@ -0,0 +1,204 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const runExecMock = vi.hoisted(() => vi.fn());
vi.mock("../process/exec.js", () => ({
runExec: runExecMock,
}));
const { rollbackSecretsMigration, runSecretsMigration } = await import("./migrate.js");
describe("secrets migrate", () => {
let baseDir = "";
let stateDir = "";
let configPath = "";
let env: NodeJS.ProcessEnv;
let authStorePath = "";
let envPath = "";
beforeEach(async () => {
baseDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-secrets-migrate-"));
stateDir = path.join(baseDir, ".openclaw");
configPath = path.join(stateDir, "openclaw.json");
authStorePath = path.join(stateDir, "agents", "main", "agent", "auth-profiles.json");
envPath = path.join(stateDir, ".env");
env = {
...process.env,
OPENCLAW_STATE_DIR: stateDir,
OPENCLAW_CONFIG_PATH: configPath,
};
await fs.mkdir(path.dirname(configPath), { recursive: true });
await fs.mkdir(path.dirname(authStorePath), { recursive: true });
await fs.writeFile(
configPath,
`${JSON.stringify(
{
models: {
providers: {
openai: {
baseUrl: "https://api.openai.com/v1",
apiKey: "sk-openai-plaintext",
models: [{ id: "gpt-5", name: "gpt-5" }],
},
},
},
skills: {
entries: {
"review-pr": {
enabled: true,
apiKey: "sk-skill-plaintext",
},
},
},
channels: {
googlechat: {
serviceAccount: '{"type":"service_account","client_email":"bot@example.com"}',
},
},
},
null,
2,
)}\n`,
"utf8",
);
await fs.writeFile(
authStorePath,
`${JSON.stringify(
{
version: 1,
profiles: {
"openai:default": {
type: "api_key",
provider: "openai",
key: "sk-profile-plaintext",
},
},
},
null,
2,
)}\n`,
"utf8",
);
await fs.writeFile(
envPath,
"OPENAI_API_KEY=sk-openai-plaintext\nSKILL_KEY=sk-skill-plaintext\nUNRELATED=value\n",
"utf8",
);
runExecMock.mockReset();
runExecMock.mockImplementation(async (_cmd: string, args: string[]) => {
if (args[0] === "--encrypt") {
const outputPath = args[args.indexOf("--output") + 1];
const inputPath = args.at(-1);
if (!outputPath || !inputPath) {
throw new Error("missing sops encrypt paths");
}
await fs.copyFile(inputPath, outputPath);
return { stdout: "", stderr: "" };
}
if (args[0] === "--decrypt") {
const sourcePath = args.at(-1);
if (!sourcePath) {
throw new Error("missing sops decrypt source");
}
const raw = await fs.readFile(sourcePath, "utf8");
return { stdout: raw, stderr: "" };
}
throw new Error(`unexpected sops invocation: ${args.join(" ")}`);
});
});
afterEach(async () => {
await fs.rm(baseDir, { recursive: true, force: true });
});
it("reports a dry-run without mutating files", async () => {
const beforeConfig = await fs.readFile(configPath, "utf8");
const beforeAuthStore = await fs.readFile(authStorePath, "utf8");
const result = await runSecretsMigration({ env });
expect(result.mode).toBe("dry-run");
expect(result.changed).toBe(true);
expect(result.counters.secretsWritten).toBeGreaterThanOrEqual(3);
expect(await fs.readFile(configPath, "utf8")).toBe(beforeConfig);
expect(await fs.readFile(authStorePath, "utf8")).toBe(beforeAuthStore);
});
it("migrates plaintext to file-backed refs and can rollback", async () => {
const applyResult = await runSecretsMigration({ env, write: true });
expect(applyResult.mode).toBe("write");
expect(applyResult.changed).toBe(true);
expect(applyResult.backupId).toBeTruthy();
const migratedConfig = JSON.parse(await fs.readFile(configPath, "utf8")) as {
models: { providers: { openai: { apiKey: unknown } } };
skills: { entries: { "review-pr": { apiKey: unknown } } };
channels: { googlechat: { serviceAccount?: unknown; serviceAccountRef?: unknown } };
secrets: { sources: { file: { type: string; path: string } } };
};
expect(migratedConfig.models.providers.openai.apiKey).toEqual({
source: "file",
id: "/providers/openai/apiKey",
});
expect(migratedConfig.skills.entries["review-pr"].apiKey).toEqual({
source: "file",
id: "/skills/entries/review-pr/apiKey",
});
expect(migratedConfig.channels.googlechat.serviceAccount).toBeUndefined();
expect(migratedConfig.channels.googlechat.serviceAccountRef).toEqual({
source: "file",
id: "/channels/googlechat/serviceAccount",
});
expect(migratedConfig.secrets.sources.file.type).toBe("sops");
const migratedAuth = JSON.parse(await fs.readFile(authStorePath, "utf8")) as {
profiles: { "openai:default": { key?: string; keyRef?: unknown } };
};
expect(migratedAuth.profiles["openai:default"].key).toBeUndefined();
expect(migratedAuth.profiles["openai:default"].keyRef).toEqual({
source: "file",
id: "/auth-profiles/main/openai:default/key",
});
const migratedEnv = await fs.readFile(envPath, "utf8");
expect(migratedEnv).not.toContain("sk-openai-plaintext");
expect(migratedEnv).not.toContain("sk-skill-plaintext");
expect(migratedEnv).toContain("UNRELATED=value");
const secretsPath = path.join(stateDir, "secrets.enc.json");
const secretsPayload = JSON.parse(await fs.readFile(secretsPath, "utf8")) as {
providers: { openai: { apiKey: string } };
skills: { entries: { "review-pr": { apiKey: string } } };
channels: { googlechat: { serviceAccount: string } };
"auth-profiles": { main: { "openai:default": { key: string } } };
};
expect(secretsPayload.providers.openai.apiKey).toBe("sk-openai-plaintext");
expect(secretsPayload.skills.entries["review-pr"].apiKey).toBe("sk-skill-plaintext");
expect(secretsPayload.channels.googlechat.serviceAccount).toContain("service_account");
expect(secretsPayload["auth-profiles"].main["openai:default"].key).toBe("sk-profile-plaintext");
const rollbackResult = await rollbackSecretsMigration({ env, backupId: applyResult.backupId! });
expect(rollbackResult.restoredFiles).toBeGreaterThan(0);
const rolledBackConfig = await fs.readFile(configPath, "utf8");
expect(rolledBackConfig).toContain("sk-openai-plaintext");
expect(rolledBackConfig).toContain("sk-skill-plaintext");
const rolledBackAuth = await fs.readFile(authStorePath, "utf8");
expect(rolledBackAuth).toContain("sk-profile-plaintext");
await expect(fs.stat(secretsPath)).rejects.toThrow();
const rolledBackEnv = await fs.readFile(envPath, "utf8");
expect(rolledBackEnv).toContain("OPENAI_API_KEY=sk-openai-plaintext");
});
});

1023
src/secrets/migrate.ts Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -33,6 +33,14 @@ describe("secrets runtime snapshot", () => {
},
},
},
skills: {
entries: {
"review-pr": {
enabled: true,
apiKey: { source: "env", id: "REVIEW_SKILL_API_KEY" },
},
},
},
};
const snapshot = await prepareSecretsRuntimeSnapshot({
@@ -40,6 +48,7 @@ describe("secrets runtime snapshot", () => {
env: {
OPENAI_API_KEY: "sk-env-openai",
GITHUB_TOKEN: "ghp-env-token",
REVIEW_SKILL_API_KEY: "sk-skill-ref",
},
agentDirs: ["/tmp/openclaw-agent-main"],
loadAuthStore: () => ({
@@ -62,6 +71,7 @@ describe("secrets runtime snapshot", () => {
});
expect(snapshot.config.models?.providers?.openai?.apiKey).toBe("sk-env-openai");
expect(snapshot.config.skills?.entries?.["review-pr"]?.apiKey).toBe("sk-skill-ref");
expect(snapshot.warnings).toHaveLength(2);
expect(snapshot.authStores[0]?.store.profiles["openai:default"]).toMatchObject({
type: "api_key",

View File

@@ -40,6 +40,10 @@ type ProviderLike = {
apiKey?: unknown;
};
type SkillEntryLike = {
apiKey?: unknown;
};
type GoogleChatAccountLike = {
serviceAccount?: unknown;
serviceAccountRef?: unknown;
@@ -127,6 +131,22 @@ async function resolveConfigSecretRefs(params: {
}
}
const skillEntries = resolved.skills?.entries as Record<string, SkillEntryLike> | undefined;
if (skillEntries) {
for (const [skillKey, entry] of Object.entries(skillEntries)) {
if (!isSecretRef(entry.apiKey)) {
continue;
}
const resolvedValue = await resolveSecretRefValue(entry.apiKey, params.context);
if (!isNonEmptyString(resolvedValue)) {
throw new Error(
`skills.entries.${skillKey}.apiKey resolved to a non-string or empty value.`,
);
}
entry.apiKey = resolvedValue;
}
}
const googleChat = resolved.channels?.googlechat as GoogleChatAccountLike | undefined;
if (googleChat) {
await resolveGoogleChatServiceAccount(