Secrets: preserve runtime snapshot source refs on write

src/config/io.runtime-snapshot-write.test.ts

@@ -0,0 +1,63 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { describe, expect, it } from "vitest";
+import { withTempHome } from "./home-env.test-harness.js";
+import {
+  clearConfigCache,
+  clearRuntimeConfigSnapshot,
+  loadConfig,
+  setRuntimeConfigSnapshot,
+  writeConfigFile,
+} from "./io.js";
+import type { OpenClawConfig } from "./types.js";
+
+describe("runtime config snapshot writes", () => {
+  it("preserves source secret refs when writeConfigFile receives runtime-resolved config", async () => {
+    await withTempHome("openclaw-config-runtime-write-", async (home) => {
+      const configPath = path.join(home, ".openclaw", "openclaw.json");
+      const sourceConfig: OpenClawConfig = {
+        models: {
+          providers: {
+            openai: {
+              baseUrl: "https://api.openai.com/v1",
+              apiKey: { source: "env", id: "OPENAI_API_KEY" },
+              models: [],
+            },
+          },
+        },
+      };
+      const runtimeConfig: OpenClawConfig = {
+        models: {
+          providers: {
+            openai: {
+              baseUrl: "https://api.openai.com/v1",
+              apiKey: "sk-runtime-resolved",
+              models: [],
+            },
+          },
+        },
+      };
+
+      await fs.mkdir(path.dirname(configPath), { recursive: true });
+      await fs.writeFile(configPath, `${JSON.stringify(sourceConfig, null, 2)}\n`, "utf8");
+
+      try {
+        setRuntimeConfigSnapshot(runtimeConfig, sourceConfig);
+        expect(loadConfig().models?.providers?.openai?.apiKey).toBe("sk-runtime-resolved");
+
+        await writeConfigFile(loadConfig());
+
+        const persisted = JSON.parse(await fs.readFile(configPath, "utf8")) as {
+          models?: { providers?: { openai?: { apiKey?: unknown } } };
+        };
+        expect(persisted.models?.providers?.openai?.apiKey).toEqual({
+          source: "env",
+          id: "OPENAI_API_KEY",
+        });
+      } finally {
+        clearRuntimeConfigSnapshot();
+        clearConfigCache();
+      }
+    });
+  });
+});

src/config/io.ts

@@ -1299,6 +1299,7 @@ let configCache: {
   config: OpenClawConfig;
 } | null = null;
 let runtimeConfigSnapshot: OpenClawConfig | null = null;
+let runtimeConfigSourceSnapshot: OpenClawConfig | null = null;
 
 function resolveConfigCacheMs(env: NodeJS.ProcessEnv): number {
   const raw = env.OPENCLAW_CONFIG_CACHE_MS?.trim();
@@ -1326,13 +1327,18 @@ export function clearConfigCache(): void {
   configCache = null;
 }
 
-export function setRuntimeConfigSnapshot(config: OpenClawConfig): void {
+export function setRuntimeConfigSnapshot(
+  config: OpenClawConfig,
+  sourceConfig?: OpenClawConfig,
+): void {
   runtimeConfigSnapshot = config;
+  runtimeConfigSourceSnapshot = sourceConfig ?? null;
   clearConfigCache();
 }
 
 export function clearRuntimeConfigSnapshot(): void {
   runtimeConfigSnapshot = null;
+  runtimeConfigSourceSnapshot = null;
   clearConfigCache();
 }
 
@@ -1380,9 +1386,14 @@ export async function writeConfigFile(
   options: ConfigWriteOptions = {},
 ): Promise<void> {
   const io = createConfigIO();
+  let nextCfg = cfg;
+  if (runtimeConfigSnapshot && runtimeConfigSourceSnapshot) {
+    const runtimePatch = createMergePatch(runtimeConfigSnapshot, cfg);
+    nextCfg = coerceConfig(applyMergePatch(runtimeConfigSourceSnapshot, runtimePatch));
+  }
   const sameConfigPath =
     options.expectedConfigPath === undefined || options.expectedConfigPath === io.configPath;
-  await io.writeConfigFile(cfg, {
+  await io.writeConfigFile(nextCfg, {
     envSnapshotForRestore: sameConfigPath ? options.envSnapshotForRestore : undefined,
     unsetPaths: options.unsetPaths,
   });

src/config/types.secrets.ts

@@ -13,6 +13,24 @@ export type SecretRef = {
 
 export type SecretInput = string | SecretRef;
 
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+export function isSecretRef(value: unknown): value is SecretRef {
+  if (!isRecord(value)) {
+    return false;
+  }
+  if (Object.keys(value).length !== 2) {
+    return false;
+  }
+  return (
+    (value.source === "env" || value.source === "file") &&
+    typeof value.id === "string" &&
+    value.id.trim().length > 0
+  );
+}
+
 export type EnvSecretSourceConfig = {
   type?: "env";
 };

src/secrets/json-pointer.ts

@@ -0,0 +1,94 @@
+function failOrUndefined(params: { onMissing: "throw" | "undefined"; message: string }): undefined {
+  if (params.onMissing === "throw") {
+    throw new Error(params.message);
+  }
+  return undefined;
+}
+
+export function decodeJsonPointerToken(token: string): string {
+  return token.replace(/~1/g, "/").replace(/~0/g, "~");
+}
+
+export function encodeJsonPointerToken(token: string): string {
+  return token.replace(/~/g, "~0").replace(/\//g, "~1");
+}
+
+export function readJsonPointer(
+  root: unknown,
+  pointer: string,
+  options: { onMissing?: "throw" | "undefined" } = {},
+): unknown {
+  const onMissing = options.onMissing ?? "throw";
+  if (!pointer.startsWith("/")) {
+    return failOrUndefined({
+      onMissing,
+      message:
+        'File-backed secret ids must be absolute JSON pointers (for example: "/providers/openai/apiKey").',
+    });
+  }
+
+  const tokens = pointer
+    .slice(1)
+    .split("/")
+    .map((token) => decodeJsonPointerToken(token));
+
+  let current: unknown = root;
+  for (const token of tokens) {
+    if (Array.isArray(current)) {
+      const index = Number.parseInt(token, 10);
+      if (!Number.isFinite(index) || index < 0 || index >= current.length) {
+        return failOrUndefined({
+          onMissing,
+          message: `JSON pointer segment "${token}" is out of bounds.`,
+        });
+      }
+      current = current[index];
+      continue;
+    }
+    if (typeof current !== "object" || current === null || Array.isArray(current)) {
+      return failOrUndefined({
+        onMissing,
+        message: `JSON pointer segment "${token}" does not exist.`,
+      });
+    }
+    const record = current as Record<string, unknown>;
+    if (!Object.hasOwn(record, token)) {
+      return failOrUndefined({
+        onMissing,
+        message: `JSON pointer segment "${token}" does not exist.`,
+      });
+    }
+    current = record[token];
+  }
+  return current;
+}
+
+export function setJsonPointer(
+  root: Record<string, unknown>,
+  pointer: string,
+  value: unknown,
+): void {
+  if (!pointer.startsWith("/")) {
+    throw new Error(`Invalid JSON pointer "${pointer}".`);
+  }
+
+  const tokens = pointer
+    .slice(1)
+    .split("/")
+    .map((token) => decodeJsonPointerToken(token));
+
+  let current: Record<string, unknown> = root;
+  for (let index = 0; index < tokens.length; index += 1) {
+    const token = tokens[index];
+    const isLast = index === tokens.length - 1;
+    if (isLast) {
+      current[token] = value;
+      return;
+    }
+    const child = current[token];
+    if (typeof child !== "object" || child === null || Array.isArray(child)) {
+      current[token] = {};
+    }
+    current = current[token] as Record<string, unknown>;
+  }
+}

src/secrets/provider-env-vars.ts

@@ -0,0 +1,28 @@
+export const PROVIDER_ENV_VARS: Record<string, readonly string[]> = {
+  openai: ["OPENAI_API_KEY"],
+  anthropic: ["ANTHROPIC_API_KEY"],
+  google: ["GEMINI_API_KEY"],
+  minimax: ["MINIMAX_API_KEY"],
+  "minimax-cn": ["MINIMAX_API_KEY"],
+  moonshot: ["MOONSHOT_API_KEY"],
+  "kimi-coding": ["KIMI_API_KEY", "KIMICODE_API_KEY"],
+  synthetic: ["SYNTHETIC_API_KEY"],
+  venice: ["VENICE_API_KEY"],
+  zai: ["ZAI_API_KEY", "Z_AI_API_KEY"],
+  xiaomi: ["XIAOMI_API_KEY"],
+  openrouter: ["OPENROUTER_API_KEY"],
+  "cloudflare-ai-gateway": ["CLOUDFLARE_AI_GATEWAY_API_KEY"],
+  litellm: ["LITELLM_API_KEY"],
+  "vercel-ai-gateway": ["AI_GATEWAY_API_KEY"],
+  opencode: ["OPENCODE_API_KEY", "OPENCODE_ZEN_API_KEY"],
+  together: ["TOGETHER_API_KEY"],
+  huggingface: ["HUGGINGFACE_HUB_TOKEN", "HF_TOKEN"],
+  qianfan: ["QIANFAN_API_KEY"],
+  xai: ["XAI_API_KEY"],
+  volcengine: ["VOLCANO_ENGINE_API_KEY"],
+  byteplus: ["BYTEPLUS_API_KEY"],
+};
+
+export function listKnownSecretEnvVarNames(): string[] {
+  return [...new Set(Object.values(PROVIDER_ENV_VARS).flatMap((keys) => keys))];
+}

src/secrets/runtime.ts

@@ -10,13 +10,11 @@ import {
   clearRuntimeConfigSnapshot,
   setRuntimeConfigSnapshot,
   type OpenClawConfig,
-  type SecretRef,
 } from "../config/config.js";
-import { runExec } from "../process/exec.js";
+import { isSecretRef, type SecretRef } from "../config/types.secrets.js";
 import { resolveUserPath } from "../utils.js";
-
+import { readJsonPointer } from "./json-pointer.js";
-const DEFAULT_SOPS_TIMEOUT_MS = 5_000;
+import { decryptSopsJsonFile, DEFAULT_SOPS_TIMEOUT_MS } from "./sops.js";
-const MAX_SOPS_OUTPUT_BYTES = 10 * 1024 * 1024;
 
 type SecretResolverWarningCode = "SECRETS_REF_OVERRIDES_PLAINTEXT";
 
@@ -77,61 +75,10 @@ function isRecord(value: unknown): value is Record<string, unknown> {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 
-function isSecretRef(value: unknown): value is SecretRef {
-  if (!isRecord(value)) {
-    return false;
-  }
-  if (Object.keys(value).length !== 2) {
-    return false;
-  }
-  return (
-    (value.source === "env" || value.source === "file") &&
-    typeof value.id === "string" &&
-    value.id.trim().length > 0
-  );
-}
-
 function isNonEmptyString(value: unknown): value is string {
   return typeof value === "string" && value.trim().length > 0;
 }
 
-function decodeJsonPointerToken(token: string): string {
-  return token.replace(/~1/g, "/").replace(/~0/g, "~");
-}
-
-function readJsonPointer(root: unknown, pointer: string): unknown {
-  if (!pointer.startsWith("/")) {
-    throw new Error(
-      `File-backed secret ids must be absolute JSON pointers (for example: /providers/openai/apiKey).`,
-    );
-  }
-
-  const tokens = pointer
-    .slice(1)
-    .split("/")
-    .map((token) => decodeJsonPointerToken(token));
-
-  let current: unknown = root;
-  for (const token of tokens) {
-    if (Array.isArray(current)) {
-      const index = Number.parseInt(token, 10);
-      if (!Number.isFinite(index) || index < 0 || index >= current.length) {
-        throw new Error(`JSON pointer segment "${token}" is out of bounds.`);
-      }
-      current = current[index];
-      continue;
-    }
-    if (!isRecord(current)) {
-      throw new Error(`JSON pointer segment "${token}" does not exist.`);
-    }
-    if (!Object.hasOwn(current, token)) {
-      throw new Error(`JSON pointer segment "${token}" does not exist.`);
-    }
-    current = current[token];
-  }
-  return current;
-}
-
 async function decryptSopsFile(config: OpenClawConfig): Promise<unknown> {
   const fileSource = config.secrets?.sources?.file;
   if (!fileSource) {
@@ -148,30 +95,12 @@ async function decryptSopsFile(config: OpenClawConfig): Promise<unknown> {
     typeof fileSource.timeoutMs === "number" && Number.isFinite(fileSource.timeoutMs)
       ? Math.max(1, Math.floor(fileSource.timeoutMs))
       : DEFAULT_SOPS_TIMEOUT_MS;
-
+  return await decryptSopsJsonFile({
-  try {
+    path: resolvedPath,
-    const { stdout } = await runExec("sops", ["--decrypt", "--output-type", "json", resolvedPath], {
+    timeoutMs,
-      timeoutMs,
+    missingBinaryMessage:
-      maxBuffer: MAX_SOPS_OUTPUT_BYTES,
+      "sops binary not found in PATH. Install sops >= 3.9.0 or disable secrets.sources.file.",
-    });
+  });
-    return JSON.parse(stdout) as unknown;
-  } catch (err) {
-    const error = err as NodeJS.ErrnoException & { message?: string };
-    if (error.code === "ENOENT") {
-      throw new Error(
-        `sops binary not found in PATH. Install sops >= 3.9.0 or disable secrets.sources.file.`,
-        { cause: err },
-      );
-    }
-    if (typeof error.message === "string" && error.message.toLowerCase().includes("timed out")) {
-      throw new Error(`sops decrypt timed out after ${timeoutMs}ms for ${resolvedPath}.`, {
-        cause: err,
-      });
-    }
-    throw new Error(`sops decrypt failed for ${resolvedPath}: ${String(error.message ?? err)}`, {
-      cause: err,
-    });
-  }
 }
 
 async function resolveSecretRefValue(ref: SecretRef, context: ResolverContext): Promise<unknown> {
@@ -191,7 +120,7 @@ async function resolveSecretRefValue(ref: SecretRef, context: ResolverContext):
   if (ref.source === "file") {
     context.fileSecretsPromise ??= decryptSopsFile(context.config);
     const payload = await context.fileSecretsPromise;
-    return readJsonPointer(payload, id);
+    return readJsonPointer(payload, id, { onMissing: "throw" });
   }
 
   throw new Error(`Unsupported secret source "${String((ref as { source?: unknown }).source)}".`);
@@ -369,7 +298,7 @@ export async function prepareSecretsRuntimeSnapshot(params: {
 
 export function activateSecretsRuntimeSnapshot(snapshot: PreparedSecretsRuntimeSnapshot): void {
   const next = cloneSnapshot(snapshot);
-  setRuntimeConfigSnapshot(next.config);
+  setRuntimeConfigSnapshot(next.config, next.sourceConfig);
   replaceRuntimeAuthProfileStoreSnapshots(next.authStores);
   activeSnapshot = next;
 }

src/secrets/sops.ts

@@ -0,0 +1,120 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { runExec } from "../process/exec.js";
+
+export const DEFAULT_SOPS_TIMEOUT_MS = 5_000;
+const MAX_SOPS_OUTPUT_BYTES = 10 * 1024 * 1024;
+
+function normalizeTimeoutMs(value: number | undefined): number {
+  if (typeof value === "number" && Number.isFinite(value)) {
+    return Math.max(1, Math.floor(value));
+  }
+  return DEFAULT_SOPS_TIMEOUT_MS;
+}
+
+function isTimeoutError(message: string | undefined): boolean {
+  return typeof message === "string" && message.toLowerCase().includes("timed out");
+}
+
+type SopsErrorContext = {
+  missingBinaryMessage: string;
+  operationLabel: string;
+};
+
+function toSopsError(err: unknown, params: SopsErrorContext): Error {
+  const error = err as NodeJS.ErrnoException & { message?: string };
+  if (error.code === "ENOENT") {
+    return new Error(params.missingBinaryMessage, { cause: err });
+  }
+  return new Error(`${params.operationLabel}: ${String(error.message ?? err)}`, {
+    cause: err,
+  });
+}
+
+function ensureDirForFile(filePath: string): void {
+  fs.mkdirSync(path.dirname(filePath), { recursive: true, mode: 0o700 });
+}
+
+export async function decryptSopsJsonFile(params: {
+  path: string;
+  timeoutMs?: number;
+  missingBinaryMessage: string;
+}): Promise<unknown> {
+  const timeoutMs = normalizeTimeoutMs(params.timeoutMs);
+  try {
+    const { stdout } = await runExec("sops", ["--decrypt", "--output-type", "json", params.path], {
+      timeoutMs,
+      maxBuffer: MAX_SOPS_OUTPUT_BYTES,
+    });
+    return JSON.parse(stdout) as unknown;
+  } catch (err) {
+    const error = err as NodeJS.ErrnoException & { message?: string };
+    if (isTimeoutError(error.message)) {
+      throw new Error(`sops decrypt timed out after ${timeoutMs}ms for ${params.path}.`, {
+        cause: err,
+      });
+    }
+    throw toSopsError(err, {
+      missingBinaryMessage: params.missingBinaryMessage,
+      operationLabel: `sops decrypt failed for ${params.path}`,
+    });
+  }
+}
+
+export async function encryptSopsJsonFile(params: {
+  path: string;
+  payload: Record<string, unknown>;
+  timeoutMs?: number;
+  missingBinaryMessage: string;
+}): Promise<void> {
+  ensureDirForFile(params.path);
+  const timeoutMs = normalizeTimeoutMs(params.timeoutMs);
+  const tmpPlain = path.join(
+    path.dirname(params.path),
+    `${path.basename(params.path)}.${process.pid}.${crypto.randomUUID()}.plain.tmp`,
+  );
+  const tmpEncrypted = path.join(
+    path.dirname(params.path),
+    `${path.basename(params.path)}.${process.pid}.${crypto.randomUUID()}.enc.tmp`,
+  );
+
+  fs.writeFileSync(tmpPlain, `${JSON.stringify(params.payload, null, 2)}\n`, "utf8");
+  fs.chmodSync(tmpPlain, 0o600);
+
+  try {
+    await runExec(
+      "sops",
+      [
+        "--encrypt",
+        "--input-type",
+        "json",
+        "--output-type",
+        "json",
+        "--output",
+        tmpEncrypted,
+        tmpPlain,
+      ],
+      {
+        timeoutMs,
+        maxBuffer: MAX_SOPS_OUTPUT_BYTES,
+      },
+    );
+    fs.renameSync(tmpEncrypted, params.path);
+    fs.chmodSync(params.path, 0o600);
+  } catch (err) {
+    const error = err as NodeJS.ErrnoException & { message?: string };
+    if (isTimeoutError(error.message)) {
+      throw new Error(`sops encrypt timed out after ${timeoutMs}ms for ${params.path}.`, {
+        cause: err,
+      });
+    }
+    throw toSopsError(err, {
+      missingBinaryMessage: params.missingBinaryMessage,
+      operationLabel: `sops encrypt failed for ${params.path}`,
+    });
+  } finally {
+    fs.rmSync(tmpPlain, { force: true });
+    fs.rmSync(tmpEncrypted, { force: true });
+  }
+}