mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-11 23:23:43 +00:00
Secrets migrate: split plan/apply/backup modules
This commit is contained in:
joshavant
committed by
Peter Steinberger
Peter Steinberger
parent
4807e40cbd
commit
363334253b
@@ -1,770 +1,25 @@
|
|||||||
import crypto from "node:crypto";
|
import { applyMigrationPlan } from "./migrate/apply.js";
|
||||||
import fs from "node:fs";
|
|
||||||
import os from "node:os";
|
|
||||||
import path from "node:path";
|
|
||||||
import { isDeepStrictEqual } from "node:util";
|
|
||||||
import { listAgentIds, resolveAgentDir } from "../agents/agent-scope.js";
|
|
||||||
import { resolveAuthStorePath } from "../agents/auth-profiles/paths.js";
|
|
||||||
import { createConfigIO, resolveStateDir, type OpenClawConfig } from "../config/config.js";
|
|
||||||
import { isSecretRef } from "../config/types.secrets.js";
|
|
||||||
import { resolveConfigDir, resolveUserPath } from "../utils.js";
|
|
||||||
import {
|
import {
|
||||||
encodeJsonPointerToken,
|
listSecretsMigrationBackups,
|
||||||
readJsonPointer as readJsonPointerRaw,
|
readBackupManifest,
|
||||||
setJsonPointer,
|
resolveSecretsMigrationBackupRoot,
|
||||||
} from "./json-pointer.js";
|
restoreFromManifest,
|
||||||
import { listKnownSecretEnvVarNames } from "./provider-env-vars.js";
|
} from "./migrate/backup.js";
|
||||||
import { decryptSopsJsonFile, encryptSopsJsonFile, DEFAULT_SOPS_TIMEOUT_MS } from "./sops.js";
|
import { buildMigrationPlan } from "./migrate/plan.js";
|
||||||
|
import type {
|
||||||
|
SecretsMigrationRollbackOptions,
|
||||||
|
SecretsMigrationRollbackResult,
|
||||||
|
SecretsMigrationRunOptions,
|
||||||
|
SecretsMigrationRunResult,
|
||||||
|
} from "./migrate/types.js";
|
||||||
|
|
||||||
const DEFAULT_SECRETS_FILE_PATH = "~/.openclaw/secrets.enc.json";
|
export type {
|
||||||
const BACKUP_DIRNAME = "secrets-migrate";
|
SecretsMigrationRollbackOptions,
|
||||||
const BACKUP_MANIFEST_FILENAME = "manifest.json";
|
SecretsMigrationRollbackResult,
|
||||||
const BACKUP_RETENTION = 20;
|
SecretsMigrationRunOptions,
|
||||||
|
SecretsMigrationRunResult,
|
||||||
type MigrationCounters = {
|
|
||||||
configRefs: number;
|
|
||||||
authProfileRefs: number;
|
|
||||||
plaintextRemoved: number;
|
|
||||||
secretsWritten: number;
|
|
||||||
envEntriesRemoved: number;
|
|
||||||
authStoresChanged: number;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
type AuthStoreChange = {
|
|
||||||
path: string;
|
|
||||||
nextStore: Record<string, unknown>;
|
|
||||||
};
|
|
||||||
|
|
||||||
type EnvChange = {
|
|
||||||
path: string;
|
|
||||||
nextRaw: string;
|
|
||||||
};
|
|
||||||
|
|
||||||
type BackupManifestEntry = {
|
|
||||||
path: string;
|
|
||||||
existed: boolean;
|
|
||||||
backupPath?: string;
|
|
||||||
mode?: number;
|
|
||||||
};
|
|
||||||
|
|
||||||
type BackupManifest = {
|
|
||||||
version: 1;
|
|
||||||
backupId: string;
|
|
||||||
createdAt: string;
|
|
||||||
entries: BackupManifestEntry[];
|
|
||||||
};
|
|
||||||
|
|
||||||
type MigrationPlan = {
|
|
||||||
changed: boolean;
|
|
||||||
counters: MigrationCounters;
|
|
||||||
stateDir: string;
|
|
||||||
configChanged: boolean;
|
|
||||||
nextConfig: OpenClawConfig;
|
|
||||||
configWriteOptions: Awaited<
|
|
||||||
ReturnType<ReturnType<typeof createConfigIO>["readConfigFileSnapshotForWrite"]>
|
|
||||||
>["writeOptions"];
|
|
||||||
authStoreChanges: AuthStoreChange[];
|
|
||||||
payloadChanged: boolean;
|
|
||||||
nextPayload: Record<string, unknown>;
|
|
||||||
secretsFilePath: string;
|
|
||||||
secretsFileTimeoutMs: number;
|
|
||||||
envChange: EnvChange | null;
|
|
||||||
backupTargets: string[];
|
|
||||||
};
|
|
||||||
|
|
||||||
export type SecretsMigrationRunOptions = {
|
|
||||||
write?: boolean;
|
|
||||||
scrubEnv?: boolean;
|
|
||||||
env?: NodeJS.ProcessEnv;
|
|
||||||
now?: Date;
|
|
||||||
};
|
|
||||||
|
|
||||||
export type SecretsMigrationRunResult = {
|
|
||||||
mode: "dry-run" | "write";
|
|
||||||
changed: boolean;
|
|
||||||
backupId?: string;
|
|
||||||
backupDir?: string;
|
|
||||||
secretsFilePath: string;
|
|
||||||
counters: MigrationCounters;
|
|
||||||
changedFiles: string[];
|
|
||||||
};
|
|
||||||
|
|
||||||
export type SecretsMigrationRollbackOptions = {
|
|
||||||
backupId: string;
|
|
||||||
env?: NodeJS.ProcessEnv;
|
|
||||||
};
|
|
||||||
|
|
||||||
export type SecretsMigrationRollbackResult = {
|
|
||||||
backupId: string;
|
|
||||||
restoredFiles: number;
|
|
||||||
deletedFiles: number;
|
|
||||||
};
|
|
||||||
|
|
||||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
||||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
||||||
}
|
|
||||||
|
|
||||||
function isNonEmptyString(value: unknown): value is string {
|
|
||||||
return typeof value === "string" && value.trim().length > 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
function normalizeSopsTimeoutMs(value: unknown): number {
|
|
||||||
if (typeof value === "number" && Number.isFinite(value)) {
|
|
||||||
return Math.max(1, Math.floor(value));
|
|
||||||
}
|
|
||||||
return DEFAULT_SOPS_TIMEOUT_MS;
|
|
||||||
}
|
|
||||||
|
|
||||||
function readJsonPointer(root: unknown, pointer: string): unknown {
|
|
||||||
return readJsonPointerRaw(root, pointer, { onMissing: "undefined" });
|
|
||||||
}
|
|
||||||
|
|
||||||
function formatBackupId(now: Date): string {
|
|
||||||
const year = now.getUTCFullYear();
|
|
||||||
const month = String(now.getUTCMonth() + 1).padStart(2, "0");
|
|
||||||
const day = String(now.getUTCDate()).padStart(2, "0");
|
|
||||||
const hour = String(now.getUTCHours()).padStart(2, "0");
|
|
||||||
const minute = String(now.getUTCMinutes()).padStart(2, "0");
|
|
||||||
const second = String(now.getUTCSeconds()).padStart(2, "0");
|
|
||||||
return `${year}${month}${day}T${hour}${minute}${second}Z`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function resolveUniqueBackupId(stateDir: string, now: Date): string {
|
|
||||||
const backupRoot = resolveBackupRoot(stateDir);
|
|
||||||
const base = formatBackupId(now);
|
|
||||||
let candidate = base;
|
|
||||||
let attempt = 0;
|
|
||||||
|
|
||||||
while (fs.existsSync(path.join(backupRoot, candidate))) {
|
|
||||||
attempt += 1;
|
|
||||||
const suffix = `${String(attempt).padStart(2, "0")}-${crypto.randomBytes(2).toString("hex")}`;
|
|
||||||
candidate = `${base}-${suffix}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
return candidate;
|
|
||||||
}
|
|
||||||
|
|
||||||
function parseEnvValue(raw: string): string {
|
|
||||||
const trimmed = raw.trim();
|
|
||||||
if (
|
|
||||||
(trimmed.startsWith('"') && trimmed.endsWith('"')) ||
|
|
||||||
(trimmed.startsWith("'") && trimmed.endsWith("'"))
|
|
||||||
) {
|
|
||||||
return trimmed.slice(1, -1);
|
|
||||||
}
|
|
||||||
return trimmed;
|
|
||||||
}
|
|
||||||
|
|
||||||
function scrubEnvRaw(
|
|
||||||
raw: string,
|
|
||||||
migratedValues: Set<string>,
|
|
||||||
allowedEnvKeys: Set<string>,
|
|
||||||
): {
|
|
||||||
nextRaw: string;
|
|
||||||
removed: number;
|
|
||||||
} {
|
|
||||||
if (migratedValues.size === 0 || allowedEnvKeys.size === 0) {
|
|
||||||
return { nextRaw: raw, removed: 0 };
|
|
||||||
}
|
|
||||||
const lines = raw.split(/\r?\n/);
|
|
||||||
const nextLines: string[] = [];
|
|
||||||
let removed = 0;
|
|
||||||
for (const line of lines) {
|
|
||||||
const match = line.match(/^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/);
|
|
||||||
if (!match) {
|
|
||||||
nextLines.push(line);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
const envKey = match[1] ?? "";
|
|
||||||
if (!allowedEnvKeys.has(envKey)) {
|
|
||||||
nextLines.push(line);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
const parsedValue = parseEnvValue(match[2] ?? "");
|
|
||||||
if (migratedValues.has(parsedValue)) {
|
|
||||||
removed += 1;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
nextLines.push(line);
|
|
||||||
}
|
|
||||||
const hadTrailingNewline = raw.endsWith("\n");
|
|
||||||
const joined = nextLines.join("\n");
|
|
||||||
return {
|
|
||||||
nextRaw:
|
|
||||||
hadTrailingNewline || joined.length === 0
|
|
||||||
? `${joined}${joined.endsWith("\n") ? "" : "\n"}`
|
|
||||||
: joined,
|
|
||||||
removed,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function ensureDirForFile(filePath: string): void {
|
|
||||||
fs.mkdirSync(path.dirname(filePath), { recursive: true, mode: 0o700 });
|
|
||||||
}
|
|
||||||
|
|
||||||
function saveJsonFile(pathname: string, value: unknown): void {
|
|
||||||
ensureDirForFile(pathname);
|
|
||||||
fs.writeFileSync(pathname, `${JSON.stringify(value, null, 2)}\n`, "utf8");
|
|
||||||
fs.chmodSync(pathname, 0o600);
|
|
||||||
}
|
|
||||||
|
|
||||||
function resolveFileSource(
|
|
||||||
config: OpenClawConfig,
|
|
||||||
env: NodeJS.ProcessEnv,
|
|
||||||
): {
|
|
||||||
path: string;
|
|
||||||
timeoutMs: number;
|
|
||||||
hadConfiguredSource: boolean;
|
|
||||||
} {
|
|
||||||
const source = config.secrets?.sources?.file;
|
|
||||||
if (source && source.type === "sops" && isNonEmptyString(source.path)) {
|
|
||||||
return {
|
|
||||||
path: resolveUserPath(source.path),
|
|
||||||
timeoutMs: normalizeSopsTimeoutMs(source.timeoutMs),
|
|
||||||
hadConfiguredSource: true,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
path: resolveUserPath(resolveDefaultSecretsConfigPath(env)),
|
|
||||||
timeoutMs: DEFAULT_SOPS_TIMEOUT_MS,
|
|
||||||
hadConfiguredSource: false,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function resolveDefaultSecretsConfigPath(env: NodeJS.ProcessEnv): string {
|
|
||||||
if (env.OPENCLAW_STATE_DIR?.trim() || env.CLAWDBOT_STATE_DIR?.trim()) {
|
|
||||||
return path.join(resolveStateDir(env, os.homedir), "secrets.enc.json");
|
|
||||||
}
|
|
||||||
return DEFAULT_SECRETS_FILE_PATH;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function decryptSopsJson(
|
|
||||||
pathname: string,
|
|
||||||
timeoutMs: number,
|
|
||||||
): Promise<Record<string, unknown>> {
|
|
||||||
if (!fs.existsSync(pathname)) {
|
|
||||||
return {};
|
|
||||||
}
|
|
||||||
const parsed = await decryptSopsJsonFile({
|
|
||||||
path: pathname,
|
|
||||||
timeoutMs,
|
|
||||||
missingBinaryMessage:
|
|
||||||
"sops binary not found in PATH. Install sops >= 3.9.0 to run secrets migrate.",
|
|
||||||
});
|
|
||||||
if (!isRecord(parsed)) {
|
|
||||||
throw new Error("sops decrypt failed: decrypted payload is not a JSON object");
|
|
||||||
}
|
|
||||||
return parsed;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function encryptSopsJson(params: {
|
|
||||||
pathname: string;
|
|
||||||
timeoutMs: number;
|
|
||||||
payload: Record<string, unknown>;
|
|
||||||
}): Promise<void> {
|
|
||||||
await encryptSopsJsonFile({
|
|
||||||
path: params.pathname,
|
|
||||||
payload: params.payload,
|
|
||||||
timeoutMs: params.timeoutMs,
|
|
||||||
missingBinaryMessage:
|
|
||||||
"sops binary not found in PATH. Install sops >= 3.9.0 to run secrets migrate.",
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function migrateModelProviderSecrets(params: {
|
|
||||||
config: OpenClawConfig;
|
|
||||||
payload: Record<string, unknown>;
|
|
||||||
counters: MigrationCounters;
|
|
||||||
migratedValues: Set<string>;
|
|
||||||
}): void {
|
|
||||||
const providers = params.config.models?.providers as
|
|
||||||
| Record<string, { apiKey?: unknown }>
|
|
||||||
| undefined;
|
|
||||||
if (!providers) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
for (const [providerId, provider] of Object.entries(providers)) {
|
|
||||||
if (isSecretRef(provider.apiKey)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (!isNonEmptyString(provider.apiKey)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
const value = provider.apiKey.trim();
|
|
||||||
const id = `/providers/${encodeJsonPointerToken(providerId)}/apiKey`;
|
|
||||||
const existing = readJsonPointer(params.payload, id);
|
|
||||||
if (!isDeepStrictEqual(existing, value)) {
|
|
||||||
setJsonPointer(params.payload, id, value);
|
|
||||||
params.counters.secretsWritten += 1;
|
|
||||||
}
|
|
||||||
provider.apiKey = { source: "file", id };
|
|
||||||
params.counters.configRefs += 1;
|
|
||||||
params.migratedValues.add(value);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function migrateSkillEntrySecrets(params: {
|
|
||||||
config: OpenClawConfig;
|
|
||||||
payload: Record<string, unknown>;
|
|
||||||
counters: MigrationCounters;
|
|
||||||
migratedValues: Set<string>;
|
|
||||||
}): void {
|
|
||||||
const entries = params.config.skills?.entries as Record<string, { apiKey?: unknown }> | undefined;
|
|
||||||
if (!entries) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
for (const [skillKey, entry] of Object.entries(entries)) {
|
|
||||||
if (!isRecord(entry) || isSecretRef(entry.apiKey)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (!isNonEmptyString(entry.apiKey)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
const value = entry.apiKey.trim();
|
|
||||||
const id = `/skills/entries/${encodeJsonPointerToken(skillKey)}/apiKey`;
|
|
||||||
const existing = readJsonPointer(params.payload, id);
|
|
||||||
if (!isDeepStrictEqual(existing, value)) {
|
|
||||||
setJsonPointer(params.payload, id, value);
|
|
||||||
params.counters.secretsWritten += 1;
|
|
||||||
}
|
|
||||||
entry.apiKey = { source: "file", id };
|
|
||||||
params.counters.configRefs += 1;
|
|
||||||
params.migratedValues.add(value);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function migrateGoogleChatServiceAccount(params: {
|
|
||||||
account: Record<string, unknown>;
|
|
||||||
pointerId: string;
|
|
||||||
counters: MigrationCounters;
|
|
||||||
payload: Record<string, unknown>;
|
|
||||||
}): void {
|
|
||||||
const explicitRef = isSecretRef(params.account.serviceAccountRef)
|
|
||||||
? params.account.serviceAccountRef
|
|
||||||
: null;
|
|
||||||
const inlineRef = isSecretRef(params.account.serviceAccount)
|
|
||||||
? params.account.serviceAccount
|
|
||||||
: null;
|
|
||||||
if (explicitRef || inlineRef) {
|
|
||||||
if (
|
|
||||||
params.account.serviceAccount !== undefined &&
|
|
||||||
!isSecretRef(params.account.serviceAccount)
|
|
||||||
) {
|
|
||||||
delete params.account.serviceAccount;
|
|
||||||
params.counters.plaintextRemoved += 1;
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const value = params.account.serviceAccount;
|
|
||||||
const hasStringValue = isNonEmptyString(value);
|
|
||||||
const hasObjectValue = isRecord(value) && Object.keys(value).length > 0;
|
|
||||||
if (!hasStringValue && !hasObjectValue) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const id = `${params.pointerId}/serviceAccount`;
|
|
||||||
const normalizedValue = hasStringValue ? value.trim() : structuredClone(value);
|
|
||||||
const existing = readJsonPointer(params.payload, id);
|
|
||||||
if (!isDeepStrictEqual(existing, normalizedValue)) {
|
|
||||||
setJsonPointer(params.payload, id, normalizedValue);
|
|
||||||
params.counters.secretsWritten += 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
params.account.serviceAccountRef = { source: "file", id };
|
|
||||||
delete params.account.serviceAccount;
|
|
||||||
params.counters.configRefs += 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
function migrateGoogleChatSecrets(params: {
|
|
||||||
config: OpenClawConfig;
|
|
||||||
payload: Record<string, unknown>;
|
|
||||||
counters: MigrationCounters;
|
|
||||||
}): void {
|
|
||||||
const googlechat = params.config.channels?.googlechat;
|
|
||||||
if (!isRecord(googlechat)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
migrateGoogleChatServiceAccount({
|
|
||||||
account: googlechat,
|
|
||||||
pointerId: "/channels/googlechat",
|
|
||||||
payload: params.payload,
|
|
||||||
counters: params.counters,
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!isRecord(googlechat.accounts)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
for (const [accountId, accountValue] of Object.entries(googlechat.accounts)) {
|
|
||||||
if (!isRecord(accountValue)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
migrateGoogleChatServiceAccount({
|
|
||||||
account: accountValue,
|
|
||||||
pointerId: `/channels/googlechat/accounts/${encodeJsonPointerToken(accountId)}`,
|
|
||||||
payload: params.payload,
|
|
||||||
counters: params.counters,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function collectAuthStorePaths(config: OpenClawConfig, stateDir: string): string[] {
|
|
||||||
const paths = new Set<string>();
|
|
||||||
paths.add(resolveUserPath(resolveAuthStorePath()));
|
|
||||||
|
|
||||||
const agentsRoot = path.join(resolveUserPath(stateDir), "agents");
|
|
||||||
if (fs.existsSync(agentsRoot)) {
|
|
||||||
for (const entry of fs.readdirSync(agentsRoot, { withFileTypes: true })) {
|
|
||||||
if (!entry.isDirectory()) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
paths.add(path.join(agentsRoot, entry.name, "agent", "auth-profiles.json"));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const agentId of listAgentIds(config)) {
|
|
||||||
const agentDir = resolveAgentDir(config, agentId);
|
|
||||||
paths.add(resolveUserPath(resolveAuthStorePath(agentDir)));
|
|
||||||
}
|
|
||||||
|
|
||||||
return [...paths];
|
|
||||||
}
|
|
||||||
|
|
||||||
function deriveAuthStoreScope(authStorePath: string, stateDir: string): string {
|
|
||||||
const agentsRoot = path.join(resolveUserPath(stateDir), "agents");
|
|
||||||
const relative = path.relative(agentsRoot, authStorePath);
|
|
||||||
if (!relative.startsWith("..")) {
|
|
||||||
const segments = relative.split(path.sep);
|
|
||||||
if (segments.length >= 3 && segments[1] === "agent" && segments[2] === "auth-profiles.json") {
|
|
||||||
const candidate = segments[0]?.trim();
|
|
||||||
if (candidate) {
|
|
||||||
return candidate;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const digest = crypto.createHash("sha1").update(authStorePath).digest("hex").slice(0, 8);
|
|
||||||
return `path-${digest}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function migrateAuthStoreSecrets(params: {
|
|
||||||
store: Record<string, unknown>;
|
|
||||||
scope: string;
|
|
||||||
payload: Record<string, unknown>;
|
|
||||||
counters: MigrationCounters;
|
|
||||||
migratedValues: Set<string>;
|
|
||||||
}): boolean {
|
|
||||||
const profiles = params.store.profiles;
|
|
||||||
if (!isRecord(profiles)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
let changed = false;
|
|
||||||
for (const [profileId, profileValue] of Object.entries(profiles)) {
|
|
||||||
if (!isRecord(profileValue)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (profileValue.type === "api_key") {
|
|
||||||
const keyRef = isSecretRef(profileValue.keyRef) ? profileValue.keyRef : null;
|
|
||||||
const key = isNonEmptyString(profileValue.key) ? profileValue.key.trim() : "";
|
|
||||||
if (keyRef) {
|
|
||||||
if (key) {
|
|
||||||
delete profileValue.key;
|
|
||||||
params.counters.plaintextRemoved += 1;
|
|
||||||
changed = true;
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (!key) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
const id = `/auth-profiles/${encodeJsonPointerToken(params.scope)}/${encodeJsonPointerToken(profileId)}/key`;
|
|
||||||
const existing = readJsonPointer(params.payload, id);
|
|
||||||
if (!isDeepStrictEqual(existing, key)) {
|
|
||||||
setJsonPointer(params.payload, id, key);
|
|
||||||
params.counters.secretsWritten += 1;
|
|
||||||
}
|
|
||||||
profileValue.keyRef = { source: "file", id };
|
|
||||||
delete profileValue.key;
|
|
||||||
params.counters.authProfileRefs += 1;
|
|
||||||
params.migratedValues.add(key);
|
|
||||||
changed = true;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (profileValue.type === "token") {
|
|
||||||
const tokenRef = isSecretRef(profileValue.tokenRef) ? profileValue.tokenRef : null;
|
|
||||||
const token = isNonEmptyString(profileValue.token) ? profileValue.token.trim() : "";
|
|
||||||
if (tokenRef) {
|
|
||||||
if (token) {
|
|
||||||
delete profileValue.token;
|
|
||||||
params.counters.plaintextRemoved += 1;
|
|
||||||
changed = true;
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (!token) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
const id = `/auth-profiles/${encodeJsonPointerToken(params.scope)}/${encodeJsonPointerToken(profileId)}/token`;
|
|
||||||
const existing = readJsonPointer(params.payload, id);
|
|
||||||
if (!isDeepStrictEqual(existing, token)) {
|
|
||||||
setJsonPointer(params.payload, id, token);
|
|
||||||
params.counters.secretsWritten += 1;
|
|
||||||
}
|
|
||||||
profileValue.tokenRef = { source: "file", id };
|
|
||||||
delete profileValue.token;
|
|
||||||
params.counters.authProfileRefs += 1;
|
|
||||||
params.migratedValues.add(token);
|
|
||||||
changed = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return changed;
|
|
||||||
}
|
|
||||||
|
|
||||||
function resolveBackupRoot(stateDir: string): string {
|
|
||||||
return path.join(resolveUserPath(stateDir), "backups", BACKUP_DIRNAME);
|
|
||||||
}
|
|
||||||
|
|
||||||
function createBackupManifest(params: {
|
|
||||||
stateDir: string;
|
|
||||||
targets: string[];
|
|
||||||
backupId: string;
|
|
||||||
now: Date;
|
|
||||||
}): { backupDir: string; manifestPath: string; manifest: BackupManifest } {
|
|
||||||
const backupDir = path.join(resolveBackupRoot(params.stateDir), params.backupId);
|
|
||||||
fs.mkdirSync(backupDir, { recursive: true, mode: 0o700 });
|
|
||||||
|
|
||||||
const entries: BackupManifestEntry[] = [];
|
|
||||||
let index = 0;
|
|
||||||
for (const target of params.targets) {
|
|
||||||
const normalized = resolveUserPath(target);
|
|
||||||
const exists = fs.existsSync(normalized);
|
|
||||||
if (!exists) {
|
|
||||||
entries.push({ path: normalized, existed: false });
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
const backupName = `file-${String(index).padStart(4, "0")}.bak`;
|
|
||||||
const backupPath = path.join(backupDir, backupName);
|
|
||||||
fs.copyFileSync(normalized, backupPath);
|
|
||||||
const stats = fs.statSync(normalized);
|
|
||||||
entries.push({
|
|
||||||
path: normalized,
|
|
||||||
existed: true,
|
|
||||||
backupPath,
|
|
||||||
mode: stats.mode & 0o777,
|
|
||||||
});
|
|
||||||
index += 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
const manifest: BackupManifest = {
|
|
||||||
version: 1,
|
|
||||||
backupId: params.backupId,
|
|
||||||
createdAt: params.now.toISOString(),
|
|
||||||
entries,
|
|
||||||
};
|
|
||||||
const manifestPath = path.join(backupDir, BACKUP_MANIFEST_FILENAME);
|
|
||||||
fs.writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf8");
|
|
||||||
fs.chmodSync(manifestPath, 0o600);
|
|
||||||
|
|
||||||
return { backupDir, manifestPath, manifest };
|
|
||||||
}
|
|
||||||
|
|
||||||
function restoreFromManifest(manifest: BackupManifest): {
|
|
||||||
restoredFiles: number;
|
|
||||||
deletedFiles: number;
|
|
||||||
} {
|
|
||||||
let restoredFiles = 0;
|
|
||||||
let deletedFiles = 0;
|
|
||||||
|
|
||||||
for (const entry of manifest.entries) {
|
|
||||||
if (!entry.existed) {
|
|
||||||
if (fs.existsSync(entry.path)) {
|
|
||||||
fs.rmSync(entry.path, { force: true });
|
|
||||||
deletedFiles += 1;
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!entry.backupPath || !fs.existsSync(entry.backupPath)) {
|
|
||||||
throw new Error(`Backup file is missing for ${entry.path}.`);
|
|
||||||
}
|
|
||||||
ensureDirForFile(entry.path);
|
|
||||||
fs.copyFileSync(entry.backupPath, entry.path);
|
|
||||||
fs.chmodSync(entry.path, entry.mode ?? 0o600);
|
|
||||||
restoredFiles += 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
return { restoredFiles, deletedFiles };
|
|
||||||
}
|
|
||||||
|
|
||||||
function pruneOldBackups(stateDir: string): void {
|
|
||||||
const backupRoot = resolveBackupRoot(stateDir);
|
|
||||||
if (!fs.existsSync(backupRoot)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const dirs = fs
|
|
||||||
.readdirSync(backupRoot, { withFileTypes: true })
|
|
||||||
.filter((entry) => entry.isDirectory())
|
|
||||||
.map((entry) => entry.name)
|
|
||||||
.toSorted();
|
|
||||||
|
|
||||||
if (dirs.length <= BACKUP_RETENTION) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const toDelete = dirs.slice(0, Math.max(0, dirs.length - BACKUP_RETENTION));
|
|
||||||
for (const dir of toDelete) {
|
|
||||||
fs.rmSync(path.join(backupRoot, dir), { recursive: true, force: true });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function buildMigrationPlan(params: {
|
|
||||||
env: NodeJS.ProcessEnv;
|
|
||||||
scrubEnv: boolean;
|
|
||||||
}): Promise<MigrationPlan> {
|
|
||||||
const io = createConfigIO({ env: params.env });
|
|
||||||
const { snapshot, writeOptions } = await io.readConfigFileSnapshotForWrite();
|
|
||||||
if (!snapshot.valid) {
|
|
||||||
const issues =
|
|
||||||
snapshot.issues.length > 0
|
|
||||||
? snapshot.issues.map((issue) => `${issue.path || "<root>"}: ${issue.message}`).join("\n")
|
|
||||||
: "Unknown validation issue.";
|
|
||||||
throw new Error(`Cannot migrate secrets because config is invalid:\n${issues}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const stateDir = resolveStateDir(params.env, os.homedir);
|
|
||||||
const nextConfig = structuredClone(snapshot.config);
|
|
||||||
const fileSource = resolveFileSource(nextConfig, params.env);
|
|
||||||
const previousPayload = await decryptSopsJson(fileSource.path, fileSource.timeoutMs);
|
|
||||||
const nextPayload = structuredClone(previousPayload);
|
|
||||||
|
|
||||||
const counters: MigrationCounters = {
|
|
||||||
configRefs: 0,
|
|
||||||
authProfileRefs: 0,
|
|
||||||
plaintextRemoved: 0,
|
|
||||||
secretsWritten: 0,
|
|
||||||
envEntriesRemoved: 0,
|
|
||||||
authStoresChanged: 0,
|
|
||||||
};
|
|
||||||
|
|
||||||
const migratedValues = new Set<string>();
|
|
||||||
|
|
||||||
migrateModelProviderSecrets({
|
|
||||||
config: nextConfig,
|
|
||||||
payload: nextPayload,
|
|
||||||
counters,
|
|
||||||
migratedValues,
|
|
||||||
});
|
|
||||||
migrateSkillEntrySecrets({
|
|
||||||
config: nextConfig,
|
|
||||||
payload: nextPayload,
|
|
||||||
counters,
|
|
||||||
migratedValues,
|
|
||||||
});
|
|
||||||
migrateGoogleChatSecrets({
|
|
||||||
config: nextConfig,
|
|
||||||
payload: nextPayload,
|
|
||||||
counters,
|
|
||||||
});
|
|
||||||
|
|
||||||
const authStoreChanges: AuthStoreChange[] = [];
|
|
||||||
for (const authStorePath of collectAuthStorePaths(nextConfig, stateDir)) {
|
|
||||||
if (!fs.existsSync(authStorePath)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
const raw = fs.readFileSync(authStorePath, "utf8");
|
|
||||||
let parsed: unknown;
|
|
||||||
try {
|
|
||||||
parsed = JSON.parse(raw) as unknown;
|
|
||||||
} catch {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (!isRecord(parsed)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
const nextStore = structuredClone(parsed);
|
|
||||||
const scope = deriveAuthStoreScope(authStorePath, stateDir);
|
|
||||||
const changed = migrateAuthStoreSecrets({
|
|
||||||
store: nextStore,
|
|
||||||
scope,
|
|
||||||
payload: nextPayload,
|
|
||||||
counters,
|
|
||||||
migratedValues,
|
|
||||||
});
|
|
||||||
if (!changed) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
authStoreChanges.push({ path: authStorePath, nextStore });
|
|
||||||
}
|
|
||||||
counters.authStoresChanged = authStoreChanges.length;
|
|
||||||
|
|
||||||
if (counters.secretsWritten > 0 && !fileSource.hadConfiguredSource) {
|
|
||||||
const defaultConfigPath = resolveDefaultSecretsConfigPath(params.env);
|
|
||||||
nextConfig.secrets ??= {};
|
|
||||||
nextConfig.secrets.sources ??= {};
|
|
||||||
nextConfig.secrets.sources.file = {
|
|
||||||
type: "sops",
|
|
||||||
path: defaultConfigPath,
|
|
||||||
timeoutMs: DEFAULT_SOPS_TIMEOUT_MS,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
const configChanged = !isDeepStrictEqual(snapshot.config, nextConfig);
|
|
||||||
const payloadChanged = !isDeepStrictEqual(previousPayload, nextPayload);
|
|
||||||
|
|
||||||
let envChange: EnvChange | null = null;
|
|
||||||
if (params.scrubEnv && migratedValues.size > 0) {
|
|
||||||
const envPath = path.join(resolveConfigDir(params.env, os.homedir), ".env");
|
|
||||||
if (fs.existsSync(envPath)) {
|
|
||||||
const rawEnv = fs.readFileSync(envPath, "utf8");
|
|
||||||
const scrubbed = scrubEnvRaw(rawEnv, migratedValues, new Set(listKnownSecretEnvVarNames()));
|
|
||||||
if (scrubbed.removed > 0 && scrubbed.nextRaw !== rawEnv) {
|
|
||||||
counters.envEntriesRemoved = scrubbed.removed;
|
|
||||||
envChange = {
|
|
||||||
path: envPath,
|
|
||||||
nextRaw: scrubbed.nextRaw,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const backupTargets = new Set<string>();
|
|
||||||
if (configChanged) {
|
|
||||||
backupTargets.add(io.configPath);
|
|
||||||
}
|
|
||||||
if (payloadChanged) {
|
|
||||||
backupTargets.add(fileSource.path);
|
|
||||||
}
|
|
||||||
for (const change of authStoreChanges) {
|
|
||||||
backupTargets.add(change.path);
|
|
||||||
}
|
|
||||||
if (envChange) {
|
|
||||||
backupTargets.add(envChange.path);
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
changed: configChanged || payloadChanged || authStoreChanges.length > 0 || Boolean(envChange),
|
|
||||||
counters,
|
|
||||||
stateDir,
|
|
||||||
configChanged,
|
|
||||||
nextConfig,
|
|
||||||
configWriteOptions: writeOptions,
|
|
||||||
authStoreChanges,
|
|
||||||
payloadChanged,
|
|
||||||
nextPayload,
|
|
||||||
secretsFilePath: fileSource.path,
|
|
||||||
secretsFileTimeoutMs: fileSource.timeoutMs,
|
|
||||||
envChange,
|
|
||||||
backupTargets: [...backupTargets],
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function runSecretsMigration(
|
export async function runSecretsMigration(
|
||||||
options: SecretsMigrationRunOptions = {},
|
options: SecretsMigrationRunOptions = {},
|
||||||
): Promise<SecretsMigrationRunResult> {
|
): Promise<SecretsMigrationRunResult> {
|
||||||
@@ -782,116 +37,23 @@ export async function runSecretsMigration(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!plan.changed) {
|
return await applyMigrationPlan({
|
||||||
return {
|
plan,
|
||||||
mode: "write",
|
env,
|
||||||
changed: false,
|
now: options.now ?? new Date(),
|
||||||
secretsFilePath: plan.secretsFilePath,
|
|
||||||
counters: plan.counters,
|
|
||||||
changedFiles: [],
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
const now = options.now ?? new Date();
|
|
||||||
const backupId = resolveUniqueBackupId(plan.stateDir, now);
|
|
||||||
const backup = createBackupManifest({
|
|
||||||
stateDir: plan.stateDir,
|
|
||||||
targets: plan.backupTargets,
|
|
||||||
backupId,
|
|
||||||
now,
|
|
||||||
});
|
});
|
||||||
|
|
||||||
try {
|
|
||||||
if (plan.payloadChanged) {
|
|
||||||
await encryptSopsJson({
|
|
||||||
pathname: plan.secretsFilePath,
|
|
||||||
timeoutMs: plan.secretsFileTimeoutMs,
|
|
||||||
payload: plan.nextPayload,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
if (plan.configChanged) {
|
|
||||||
const io = createConfigIO({ env });
|
|
||||||
await io.writeConfigFile(plan.nextConfig, plan.configWriteOptions);
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const change of plan.authStoreChanges) {
|
|
||||||
saveJsonFile(change.path, change.nextStore);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (plan.envChange) {
|
|
||||||
ensureDirForFile(plan.envChange.path);
|
|
||||||
fs.writeFileSync(plan.envChange.path, plan.envChange.nextRaw, "utf8");
|
|
||||||
fs.chmodSync(plan.envChange.path, 0o600);
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
restoreFromManifest(backup.manifest);
|
|
||||||
throw new Error(
|
|
||||||
`Secrets migration failed and was rolled back from backup ${backupId}: ${String(err)}`,
|
|
||||||
{
|
|
||||||
cause: err,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
pruneOldBackups(plan.stateDir);
|
|
||||||
|
|
||||||
return {
|
|
||||||
mode: "write",
|
|
||||||
changed: true,
|
|
||||||
backupId,
|
|
||||||
backupDir: backup.backupDir,
|
|
||||||
secretsFilePath: plan.secretsFilePath,
|
|
||||||
counters: plan.counters,
|
|
||||||
changedFiles: plan.backupTargets,
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export function resolveSecretsMigrationBackupRoot(env: NodeJS.ProcessEnv = process.env): string {
|
export { resolveSecretsMigrationBackupRoot, listSecretsMigrationBackups };
|
||||||
return resolveBackupRoot(resolveStateDir(env, os.homedir));
|
|
||||||
}
|
|
||||||
|
|
||||||
export function listSecretsMigrationBackups(env: NodeJS.ProcessEnv = process.env): string[] {
|
|
||||||
const root = resolveSecretsMigrationBackupRoot(env);
|
|
||||||
if (!fs.existsSync(root)) {
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
return fs
|
|
||||||
.readdirSync(root, { withFileTypes: true })
|
|
||||||
.filter((entry) => entry.isDirectory())
|
|
||||||
.map((entry) => entry.name)
|
|
||||||
.toSorted();
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function rollbackSecretsMigration(
|
export async function rollbackSecretsMigration(
|
||||||
options: SecretsMigrationRollbackOptions,
|
options: SecretsMigrationRollbackOptions,
|
||||||
): Promise<SecretsMigrationRollbackResult> {
|
): Promise<SecretsMigrationRollbackResult> {
|
||||||
const env = options.env ?? process.env;
|
const env = options.env ?? process.env;
|
||||||
const backupDir = path.join(resolveSecretsMigrationBackupRoot(env), options.backupId);
|
const manifest = readBackupManifest({
|
||||||
const manifestPath = path.join(backupDir, BACKUP_MANIFEST_FILENAME);
|
backupId: options.backupId,
|
||||||
if (!fs.existsSync(manifestPath)) {
|
env,
|
||||||
const available = listSecretsMigrationBackups(env);
|
});
|
||||||
const suffix =
|
|
||||||
available.length > 0
|
|
||||||
? ` Available backups: ${available.slice(-10).join(", ")}`
|
|
||||||
: " No backups were found.";
|
|
||||||
throw new Error(`Backup "${options.backupId}" was not found.${suffix}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
let parsed: unknown;
|
|
||||||
try {
|
|
||||||
parsed = JSON.parse(fs.readFileSync(manifestPath, "utf8")) as unknown;
|
|
||||||
} catch (err) {
|
|
||||||
throw new Error(`Failed to read backup manifest at ${manifestPath}: ${String(err)}`, {
|
|
||||||
cause: err,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!isRecord(parsed) || !Array.isArray(parsed.entries)) {
|
|
||||||
throw new Error(`Backup manifest at ${manifestPath} is invalid.`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const manifest = parsed as BackupManifest;
|
|
||||||
const restored = restoreFromManifest(manifest);
|
const restored = restoreFromManifest(manifest);
|
||||||
return {
|
return {
|
||||||
backupId: options.backupId,
|
backupId: options.backupId,
|
||||||
|
|||||||
95
src/secrets/migrate/apply.ts
Normal file
95
src/secrets/migrate/apply.ts
Normal file
@@ -0,0 +1,95 @@
|
|||||||
|
import fs from "node:fs";
|
||||||
|
import { createConfigIO } from "../../config/config.js";
|
||||||
|
import { ensureDirForFile, writeJsonFileSecure } from "../shared.js";
|
||||||
|
import { encryptSopsJsonFile } from "../sops.js";
|
||||||
|
import {
|
||||||
|
createBackupManifest,
|
||||||
|
pruneOldBackups,
|
||||||
|
resolveUniqueBackupId,
|
||||||
|
restoreFromManifest,
|
||||||
|
} from "./backup.js";
|
||||||
|
import type { MigrationPlan, SecretsMigrationRunResult } from "./types.js";
|
||||||
|
|
||||||
|
async function encryptSopsJson(params: {
|
||||||
|
pathname: string;
|
||||||
|
timeoutMs: number;
|
||||||
|
payload: Record<string, unknown>;
|
||||||
|
}): Promise<void> {
|
||||||
|
await encryptSopsJsonFile({
|
||||||
|
path: params.pathname,
|
||||||
|
payload: params.payload,
|
||||||
|
timeoutMs: params.timeoutMs,
|
||||||
|
missingBinaryMessage:
|
||||||
|
"sops binary not found in PATH. Install sops >= 3.9.0 to run secrets migrate.",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function applyMigrationPlan(params: {
|
||||||
|
plan: MigrationPlan;
|
||||||
|
env: NodeJS.ProcessEnv;
|
||||||
|
now: Date;
|
||||||
|
}): Promise<SecretsMigrationRunResult> {
|
||||||
|
const { plan } = params;
|
||||||
|
if (!plan.changed) {
|
||||||
|
return {
|
||||||
|
mode: "write",
|
||||||
|
changed: false,
|
||||||
|
secretsFilePath: plan.secretsFilePath,
|
||||||
|
counters: plan.counters,
|
||||||
|
changedFiles: [],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const backupId = resolveUniqueBackupId(plan.stateDir, params.now);
|
||||||
|
const backup = createBackupManifest({
|
||||||
|
stateDir: plan.stateDir,
|
||||||
|
targets: plan.backupTargets,
|
||||||
|
backupId,
|
||||||
|
now: params.now,
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (plan.payloadChanged) {
|
||||||
|
await encryptSopsJson({
|
||||||
|
pathname: plan.secretsFilePath,
|
||||||
|
timeoutMs: plan.secretsFileTimeoutMs,
|
||||||
|
payload: plan.nextPayload,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (plan.configChanged) {
|
||||||
|
const io = createConfigIO({ env: params.env });
|
||||||
|
await io.writeConfigFile(plan.nextConfig, plan.configWriteOptions);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const change of plan.authStoreChanges) {
|
||||||
|
writeJsonFileSecure(change.path, change.nextStore);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (plan.envChange) {
|
||||||
|
ensureDirForFile(plan.envChange.path);
|
||||||
|
fs.writeFileSync(plan.envChange.path, plan.envChange.nextRaw, "utf8");
|
||||||
|
fs.chmodSync(plan.envChange.path, 0o600);
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
restoreFromManifest(backup.manifest);
|
||||||
|
throw new Error(
|
||||||
|
`Secrets migration failed and was rolled back from backup ${backupId}: ${String(err)}`,
|
||||||
|
{
|
||||||
|
cause: err,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
pruneOldBackups(plan.stateDir);
|
||||||
|
|
||||||
|
return {
|
||||||
|
mode: "write",
|
||||||
|
changed: true,
|
||||||
|
backupId,
|
||||||
|
backupDir: backup.backupDir,
|
||||||
|
secretsFilePath: plan.secretsFilePath,
|
||||||
|
counters: plan.counters,
|
||||||
|
changedFiles: plan.backupTargets,
|
||||||
|
};
|
||||||
|
}
|
||||||
182
src/secrets/migrate/backup.ts
Normal file
182
src/secrets/migrate/backup.ts
Normal file
@@ -0,0 +1,182 @@
|
|||||||
|
import crypto from "node:crypto";
|
||||||
|
import fs from "node:fs";
|
||||||
|
import os from "node:os";
|
||||||
|
import path from "node:path";
|
||||||
|
import { resolveStateDir } from "../../config/config.js";
|
||||||
|
import { resolveUserPath } from "../../utils.js";
|
||||||
|
import { ensureDirForFile, isRecord } from "../shared.js";
|
||||||
|
import type { BackupManifest } from "./types.js";
|
||||||
|
|
||||||
|
export const BACKUP_DIRNAME = "secrets-migrate";
|
||||||
|
export const BACKUP_MANIFEST_FILENAME = "manifest.json";
|
||||||
|
export const BACKUP_RETENTION = 20;
|
||||||
|
|
||||||
|
export function resolveBackupRoot(stateDir: string): string {
|
||||||
|
return path.join(resolveUserPath(stateDir), "backups", BACKUP_DIRNAME);
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatBackupId(now: Date): string {
|
||||||
|
const year = now.getUTCFullYear();
|
||||||
|
const month = String(now.getUTCMonth() + 1).padStart(2, "0");
|
||||||
|
const day = String(now.getUTCDate()).padStart(2, "0");
|
||||||
|
const hour = String(now.getUTCHours()).padStart(2, "0");
|
||||||
|
const minute = String(now.getUTCMinutes()).padStart(2, "0");
|
||||||
|
const second = String(now.getUTCSeconds()).padStart(2, "0");
|
||||||
|
return `${year}${month}${day}T${hour}${minute}${second}Z`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveUniqueBackupId(stateDir: string, now: Date): string {
|
||||||
|
const backupRoot = resolveBackupRoot(stateDir);
|
||||||
|
const base = formatBackupId(now);
|
||||||
|
let candidate = base;
|
||||||
|
let attempt = 0;
|
||||||
|
|
||||||
|
while (fs.existsSync(path.join(backupRoot, candidate))) {
|
||||||
|
attempt += 1;
|
||||||
|
const suffix = `${String(attempt).padStart(2, "0")}-${crypto.randomBytes(2).toString("hex")}`;
|
||||||
|
candidate = `${base}-${suffix}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
return candidate;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createBackupManifest(params: {
|
||||||
|
stateDir: string;
|
||||||
|
targets: string[];
|
||||||
|
backupId: string;
|
||||||
|
now: Date;
|
||||||
|
}): { backupDir: string; manifestPath: string; manifest: BackupManifest } {
|
||||||
|
const backupDir = path.join(resolveBackupRoot(params.stateDir), params.backupId);
|
||||||
|
fs.mkdirSync(backupDir, { recursive: true, mode: 0o700 });
|
||||||
|
|
||||||
|
const entries: BackupManifest["entries"] = [];
|
||||||
|
let index = 0;
|
||||||
|
for (const target of params.targets) {
|
||||||
|
const normalized = resolveUserPath(target);
|
||||||
|
const exists = fs.existsSync(normalized);
|
||||||
|
if (!exists) {
|
||||||
|
entries.push({ path: normalized, existed: false });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const backupName = `file-${String(index).padStart(4, "0")}.bak`;
|
||||||
|
const backupPath = path.join(backupDir, backupName);
|
||||||
|
fs.copyFileSync(normalized, backupPath);
|
||||||
|
const stats = fs.statSync(normalized);
|
||||||
|
entries.push({
|
||||||
|
path: normalized,
|
||||||
|
existed: true,
|
||||||
|
backupPath,
|
||||||
|
mode: stats.mode & 0o777,
|
||||||
|
});
|
||||||
|
index += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
const manifest: BackupManifest = {
|
||||||
|
version: 1,
|
||||||
|
backupId: params.backupId,
|
||||||
|
createdAt: params.now.toISOString(),
|
||||||
|
entries,
|
||||||
|
};
|
||||||
|
const manifestPath = path.join(backupDir, BACKUP_MANIFEST_FILENAME);
|
||||||
|
fs.writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf8");
|
||||||
|
fs.chmodSync(manifestPath, 0o600);
|
||||||
|
|
||||||
|
return { backupDir, manifestPath, manifest };
|
||||||
|
}
|
||||||
|
|
||||||
|
export function restoreFromManifest(manifest: BackupManifest): {
|
||||||
|
restoredFiles: number;
|
||||||
|
deletedFiles: number;
|
||||||
|
} {
|
||||||
|
let restoredFiles = 0;
|
||||||
|
let deletedFiles = 0;
|
||||||
|
|
||||||
|
for (const entry of manifest.entries) {
|
||||||
|
if (!entry.existed) {
|
||||||
|
if (fs.existsSync(entry.path)) {
|
||||||
|
fs.rmSync(entry.path, { force: true });
|
||||||
|
deletedFiles += 1;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!entry.backupPath || !fs.existsSync(entry.backupPath)) {
|
||||||
|
throw new Error(`Backup file is missing for ${entry.path}.`);
|
||||||
|
}
|
||||||
|
ensureDirForFile(entry.path);
|
||||||
|
fs.copyFileSync(entry.backupPath, entry.path);
|
||||||
|
fs.chmodSync(entry.path, entry.mode ?? 0o600);
|
||||||
|
restoredFiles += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
return { restoredFiles, deletedFiles };
|
||||||
|
}
|
||||||
|
|
||||||
|
export function pruneOldBackups(stateDir: string): void {
|
||||||
|
const backupRoot = resolveBackupRoot(stateDir);
|
||||||
|
if (!fs.existsSync(backupRoot)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const dirs = fs
|
||||||
|
.readdirSync(backupRoot, { withFileTypes: true })
|
||||||
|
.filter((entry) => entry.isDirectory())
|
||||||
|
.map((entry) => entry.name)
|
||||||
|
.toSorted();
|
||||||
|
|
||||||
|
if (dirs.length <= BACKUP_RETENTION) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const toDelete = dirs.slice(0, Math.max(0, dirs.length - BACKUP_RETENTION));
|
||||||
|
for (const dir of toDelete) {
|
||||||
|
fs.rmSync(path.join(backupRoot, dir), { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveSecretsMigrationBackupRoot(env: NodeJS.ProcessEnv = process.env): string {
|
||||||
|
return resolveBackupRoot(resolveStateDir(env, os.homedir));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function listSecretsMigrationBackups(env: NodeJS.ProcessEnv = process.env): string[] {
|
||||||
|
const root = resolveSecretsMigrationBackupRoot(env);
|
||||||
|
if (!fs.existsSync(root)) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
return fs
|
||||||
|
.readdirSync(root, { withFileTypes: true })
|
||||||
|
.filter((entry) => entry.isDirectory())
|
||||||
|
.map((entry) => entry.name)
|
||||||
|
.toSorted();
|
||||||
|
}
|
||||||
|
|
||||||
|
export function readBackupManifest(params: {
|
||||||
|
backupId: string;
|
||||||
|
env: NodeJS.ProcessEnv;
|
||||||
|
}): BackupManifest {
|
||||||
|
const backupDir = path.join(resolveSecretsMigrationBackupRoot(params.env), params.backupId);
|
||||||
|
const manifestPath = path.join(backupDir, BACKUP_MANIFEST_FILENAME);
|
||||||
|
if (!fs.existsSync(manifestPath)) {
|
||||||
|
const available = listSecretsMigrationBackups(params.env);
|
||||||
|
const suffix =
|
||||||
|
available.length > 0
|
||||||
|
? ` Available backups: ${available.slice(-10).join(", ")}`
|
||||||
|
: " No backups were found.";
|
||||||
|
throw new Error(`Backup "${params.backupId}" was not found.${suffix}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
let parsed: unknown;
|
||||||
|
try {
|
||||||
|
parsed = JSON.parse(fs.readFileSync(manifestPath, "utf8")) as unknown;
|
||||||
|
} catch (err) {
|
||||||
|
throw new Error(`Failed to read backup manifest at ${manifestPath}: ${String(err)}`, {
|
||||||
|
cause: err,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!isRecord(parsed) || !Array.isArray(parsed.entries)) {
|
||||||
|
throw new Error(`Backup manifest at ${manifestPath} is invalid.`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return parsed as BackupManifest;
|
||||||
|
}
|
||||||
524
src/secrets/migrate/plan.ts
Normal file
524
src/secrets/migrate/plan.ts
Normal file
@@ -0,0 +1,524 @@
|
|||||||
|
import crypto from "node:crypto";
|
||||||
|
import fs from "node:fs";
|
||||||
|
import os from "node:os";
|
||||||
|
import path from "node:path";
|
||||||
|
import { isDeepStrictEqual } from "node:util";
|
||||||
|
import { listAgentIds, resolveAgentDir } from "../../agents/agent-scope.js";
|
||||||
|
import { resolveAuthStorePath } from "../../agents/auth-profiles/paths.js";
|
||||||
|
import { createConfigIO, resolveStateDir, type OpenClawConfig } from "../../config/config.js";
|
||||||
|
import { isSecretRef } from "../../config/types.secrets.js";
|
||||||
|
import { resolveConfigDir, resolveUserPath } from "../../utils.js";
|
||||||
|
import {
|
||||||
|
encodeJsonPointerToken,
|
||||||
|
readJsonPointer as readJsonPointerRaw,
|
||||||
|
setJsonPointer,
|
||||||
|
} from "../json-pointer.js";
|
||||||
|
import { listKnownSecretEnvVarNames } from "../provider-env-vars.js";
|
||||||
|
import { isNonEmptyString, isRecord, normalizePositiveInt } from "../shared.js";
|
||||||
|
import { decryptSopsJsonFile, DEFAULT_SOPS_TIMEOUT_MS } from "../sops.js";
|
||||||
|
import type { AuthStoreChange, EnvChange, MigrationCounters, MigrationPlan } from "./types.js";
|
||||||
|
|
||||||
|
const DEFAULT_SECRETS_FILE_PATH = "~/.openclaw/secrets.enc.json";
|
||||||
|
|
||||||
|
function readJsonPointer(root: unknown, pointer: string): unknown {
|
||||||
|
return readJsonPointerRaw(root, pointer, { onMissing: "undefined" });
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseEnvValue(raw: string): string {
|
||||||
|
const trimmed = raw.trim();
|
||||||
|
if (
|
||||||
|
(trimmed.startsWith('"') && trimmed.endsWith('"')) ||
|
||||||
|
(trimmed.startsWith("'") && trimmed.endsWith("'"))
|
||||||
|
) {
|
||||||
|
return trimmed.slice(1, -1);
|
||||||
|
}
|
||||||
|
return trimmed;
|
||||||
|
}
|
||||||
|
|
||||||
|
function scrubEnvRaw(
|
||||||
|
raw: string,
|
||||||
|
migratedValues: Set<string>,
|
||||||
|
allowedEnvKeys: Set<string>,
|
||||||
|
): {
|
||||||
|
nextRaw: string;
|
||||||
|
removed: number;
|
||||||
|
} {
|
||||||
|
if (migratedValues.size === 0 || allowedEnvKeys.size === 0) {
|
||||||
|
return { nextRaw: raw, removed: 0 };
|
||||||
|
}
|
||||||
|
const lines = raw.split(/\r?\n/);
|
||||||
|
const nextLines: string[] = [];
|
||||||
|
let removed = 0;
|
||||||
|
for (const line of lines) {
|
||||||
|
const match = line.match(/^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/);
|
||||||
|
if (!match) {
|
||||||
|
nextLines.push(line);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const envKey = match[1] ?? "";
|
||||||
|
if (!allowedEnvKeys.has(envKey)) {
|
||||||
|
nextLines.push(line);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const parsedValue = parseEnvValue(match[2] ?? "");
|
||||||
|
if (migratedValues.has(parsedValue)) {
|
||||||
|
removed += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
nextLines.push(line);
|
||||||
|
}
|
||||||
|
const hadTrailingNewline = raw.endsWith("\n");
|
||||||
|
const joined = nextLines.join("\n");
|
||||||
|
return {
|
||||||
|
nextRaw:
|
||||||
|
hadTrailingNewline || joined.length === 0
|
||||||
|
? `${joined}${joined.endsWith("\n") ? "" : "\n"}`
|
||||||
|
: joined,
|
||||||
|
removed,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveFileSource(
|
||||||
|
config: OpenClawConfig,
|
||||||
|
env: NodeJS.ProcessEnv,
|
||||||
|
): {
|
||||||
|
path: string;
|
||||||
|
timeoutMs: number;
|
||||||
|
hadConfiguredSource: boolean;
|
||||||
|
} {
|
||||||
|
const source = config.secrets?.sources?.file;
|
||||||
|
if (source && source.type === "sops" && isNonEmptyString(source.path)) {
|
||||||
|
return {
|
||||||
|
path: resolveUserPath(source.path),
|
||||||
|
timeoutMs: normalizePositiveInt(source.timeoutMs, DEFAULT_SOPS_TIMEOUT_MS),
|
||||||
|
hadConfiguredSource: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
path: resolveUserPath(resolveDefaultSecretsConfigPath(env)),
|
||||||
|
timeoutMs: DEFAULT_SOPS_TIMEOUT_MS,
|
||||||
|
hadConfiguredSource: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveDefaultSecretsConfigPath(env: NodeJS.ProcessEnv): string {
|
||||||
|
if (env.OPENCLAW_STATE_DIR?.trim() || env.CLAWDBOT_STATE_DIR?.trim()) {
|
||||||
|
return path.join(resolveStateDir(env, os.homedir), "secrets.enc.json");
|
||||||
|
}
|
||||||
|
return DEFAULT_SECRETS_FILE_PATH;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function decryptSopsJson(
|
||||||
|
pathname: string,
|
||||||
|
timeoutMs: number,
|
||||||
|
): Promise<Record<string, unknown>> {
|
||||||
|
if (!fs.existsSync(pathname)) {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
const parsed = await decryptSopsJsonFile({
|
||||||
|
path: pathname,
|
||||||
|
timeoutMs,
|
||||||
|
missingBinaryMessage:
|
||||||
|
"sops binary not found in PATH. Install sops >= 3.9.0 to run secrets migrate.",
|
||||||
|
});
|
||||||
|
if (!isRecord(parsed)) {
|
||||||
|
throw new Error("sops decrypt failed: decrypted payload is not a JSON object");
|
||||||
|
}
|
||||||
|
return parsed;
|
||||||
|
}
|
||||||
|
|
||||||
|
function migrateModelProviderSecrets(params: {
|
||||||
|
config: OpenClawConfig;
|
||||||
|
payload: Record<string, unknown>;
|
||||||
|
counters: MigrationCounters;
|
||||||
|
migratedValues: Set<string>;
|
||||||
|
}): void {
|
||||||
|
const providers = params.config.models?.providers as
|
||||||
|
| Record<string, { apiKey?: unknown }>
|
||||||
|
| undefined;
|
||||||
|
if (!providers) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for (const [providerId, provider] of Object.entries(providers)) {
|
||||||
|
if (isSecretRef(provider.apiKey)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!isNonEmptyString(provider.apiKey)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const value = provider.apiKey.trim();
|
||||||
|
const id = `/providers/${encodeJsonPointerToken(providerId)}/apiKey`;
|
||||||
|
const existing = readJsonPointer(params.payload, id);
|
||||||
|
if (!isDeepStrictEqual(existing, value)) {
|
||||||
|
setJsonPointer(params.payload, id, value);
|
||||||
|
params.counters.secretsWritten += 1;
|
||||||
|
}
|
||||||
|
provider.apiKey = { source: "file", id };
|
||||||
|
params.counters.configRefs += 1;
|
||||||
|
params.migratedValues.add(value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function migrateSkillEntrySecrets(params: {
|
||||||
|
config: OpenClawConfig;
|
||||||
|
payload: Record<string, unknown>;
|
||||||
|
counters: MigrationCounters;
|
||||||
|
migratedValues: Set<string>;
|
||||||
|
}): void {
|
||||||
|
const entries = params.config.skills?.entries as Record<string, { apiKey?: unknown }> | undefined;
|
||||||
|
if (!entries) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for (const [skillKey, entry] of Object.entries(entries)) {
|
||||||
|
if (!isRecord(entry) || isSecretRef(entry.apiKey)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!isNonEmptyString(entry.apiKey)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const value = entry.apiKey.trim();
|
||||||
|
const id = `/skills/entries/${encodeJsonPointerToken(skillKey)}/apiKey`;
|
||||||
|
const existing = readJsonPointer(params.payload, id);
|
||||||
|
if (!isDeepStrictEqual(existing, value)) {
|
||||||
|
setJsonPointer(params.payload, id, value);
|
||||||
|
params.counters.secretsWritten += 1;
|
||||||
|
}
|
||||||
|
entry.apiKey = { source: "file", id };
|
||||||
|
params.counters.configRefs += 1;
|
||||||
|
params.migratedValues.add(value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function migrateGoogleChatServiceAccount(params: {
|
||||||
|
account: Record<string, unknown>;
|
||||||
|
pointerId: string;
|
||||||
|
counters: MigrationCounters;
|
||||||
|
payload: Record<string, unknown>;
|
||||||
|
}): void {
|
||||||
|
const explicitRef = isSecretRef(params.account.serviceAccountRef)
|
||||||
|
? params.account.serviceAccountRef
|
||||||
|
: null;
|
||||||
|
const inlineRef = isSecretRef(params.account.serviceAccount)
|
||||||
|
? params.account.serviceAccount
|
||||||
|
: null;
|
||||||
|
if (explicitRef || inlineRef) {
|
||||||
|
if (
|
||||||
|
params.account.serviceAccount !== undefined &&
|
||||||
|
!isSecretRef(params.account.serviceAccount)
|
||||||
|
) {
|
||||||
|
delete params.account.serviceAccount;
|
||||||
|
params.counters.plaintextRemoved += 1;
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const value = params.account.serviceAccount;
|
||||||
|
const hasStringValue = isNonEmptyString(value);
|
||||||
|
const hasObjectValue = isRecord(value) && Object.keys(value).length > 0;
|
||||||
|
if (!hasStringValue && !hasObjectValue) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const id = `${params.pointerId}/serviceAccount`;
|
||||||
|
const normalizedValue = hasStringValue ? value.trim() : structuredClone(value);
|
||||||
|
const existing = readJsonPointer(params.payload, id);
|
||||||
|
if (!isDeepStrictEqual(existing, normalizedValue)) {
|
||||||
|
setJsonPointer(params.payload, id, normalizedValue);
|
||||||
|
params.counters.secretsWritten += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
params.account.serviceAccountRef = { source: "file", id };
|
||||||
|
delete params.account.serviceAccount;
|
||||||
|
params.counters.configRefs += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
function migrateGoogleChatSecrets(params: {
|
||||||
|
config: OpenClawConfig;
|
||||||
|
payload: Record<string, unknown>;
|
||||||
|
counters: MigrationCounters;
|
||||||
|
}): void {
|
||||||
|
const googlechat = params.config.channels?.googlechat;
|
||||||
|
if (!isRecord(googlechat)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
migrateGoogleChatServiceAccount({
|
||||||
|
account: googlechat,
|
||||||
|
pointerId: "/channels/googlechat",
|
||||||
|
payload: params.payload,
|
||||||
|
counters: params.counters,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!isRecord(googlechat.accounts)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for (const [accountId, accountValue] of Object.entries(googlechat.accounts)) {
|
||||||
|
if (!isRecord(accountValue)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
migrateGoogleChatServiceAccount({
|
||||||
|
account: accountValue,
|
||||||
|
pointerId: `/channels/googlechat/accounts/${encodeJsonPointerToken(accountId)}`,
|
||||||
|
payload: params.payload,
|
||||||
|
counters: params.counters,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function collectAuthStorePaths(config: OpenClawConfig, stateDir: string): string[] {
|
||||||
|
const paths = new Set<string>();
|
||||||
|
paths.add(resolveUserPath(resolveAuthStorePath()));
|
||||||
|
|
||||||
|
const agentsRoot = path.join(resolveUserPath(stateDir), "agents");
|
||||||
|
if (fs.existsSync(agentsRoot)) {
|
||||||
|
for (const entry of fs.readdirSync(agentsRoot, { withFileTypes: true })) {
|
||||||
|
if (!entry.isDirectory()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
paths.add(path.join(agentsRoot, entry.name, "agent", "auth-profiles.json"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const agentId of listAgentIds(config)) {
|
||||||
|
const agentDir = resolveAgentDir(config, agentId);
|
||||||
|
paths.add(resolveUserPath(resolveAuthStorePath(agentDir)));
|
||||||
|
}
|
||||||
|
|
||||||
|
return [...paths];
|
||||||
|
}
|
||||||
|
|
||||||
|
function deriveAuthStoreScope(authStorePath: string, stateDir: string): string {
|
||||||
|
const agentsRoot = path.join(resolveUserPath(stateDir), "agents");
|
||||||
|
const relative = path.relative(agentsRoot, authStorePath);
|
||||||
|
if (!relative.startsWith("..")) {
|
||||||
|
const segments = relative.split(path.sep);
|
||||||
|
if (segments.length >= 3 && segments[1] === "agent" && segments[2] === "auth-profiles.json") {
|
||||||
|
const candidate = segments[0]?.trim();
|
||||||
|
if (candidate) {
|
||||||
|
return candidate;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const digest = crypto.createHash("sha1").update(authStorePath).digest("hex").slice(0, 8);
|
||||||
|
return `path-${digest}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function migrateAuthStoreSecrets(params: {
|
||||||
|
store: Record<string, unknown>;
|
||||||
|
scope: string;
|
||||||
|
payload: Record<string, unknown>;
|
||||||
|
counters: MigrationCounters;
|
||||||
|
migratedValues: Set<string>;
|
||||||
|
}): boolean {
|
||||||
|
const profiles = params.store.profiles;
|
||||||
|
if (!isRecord(profiles)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
let changed = false;
|
||||||
|
for (const [profileId, profileValue] of Object.entries(profiles)) {
|
||||||
|
if (!isRecord(profileValue)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (profileValue.type === "api_key") {
|
||||||
|
const keyRef = isSecretRef(profileValue.keyRef) ? profileValue.keyRef : null;
|
||||||
|
const key = isNonEmptyString(profileValue.key) ? profileValue.key.trim() : "";
|
||||||
|
if (keyRef) {
|
||||||
|
if (key) {
|
||||||
|
delete profileValue.key;
|
||||||
|
params.counters.plaintextRemoved += 1;
|
||||||
|
changed = true;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!key) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const id = `/auth-profiles/${encodeJsonPointerToken(params.scope)}/${encodeJsonPointerToken(profileId)}/key`;
|
||||||
|
const existing = readJsonPointer(params.payload, id);
|
||||||
|
if (!isDeepStrictEqual(existing, key)) {
|
||||||
|
setJsonPointer(params.payload, id, key);
|
||||||
|
params.counters.secretsWritten += 1;
|
||||||
|
}
|
||||||
|
profileValue.keyRef = { source: "file", id };
|
||||||
|
delete profileValue.key;
|
||||||
|
params.counters.authProfileRefs += 1;
|
||||||
|
params.migratedValues.add(key);
|
||||||
|
changed = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (profileValue.type === "token") {
|
||||||
|
const tokenRef = isSecretRef(profileValue.tokenRef) ? profileValue.tokenRef : null;
|
||||||
|
const token = isNonEmptyString(profileValue.token) ? profileValue.token.trim() : "";
|
||||||
|
if (tokenRef) {
|
||||||
|
if (token) {
|
||||||
|
delete profileValue.token;
|
||||||
|
params.counters.plaintextRemoved += 1;
|
||||||
|
changed = true;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!token) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const id = `/auth-profiles/${encodeJsonPointerToken(params.scope)}/${encodeJsonPointerToken(profileId)}/token`;
|
||||||
|
const existing = readJsonPointer(params.payload, id);
|
||||||
|
if (!isDeepStrictEqual(existing, token)) {
|
||||||
|
setJsonPointer(params.payload, id, token);
|
||||||
|
params.counters.secretsWritten += 1;
|
||||||
|
}
|
||||||
|
profileValue.tokenRef = { source: "file", id };
|
||||||
|
delete profileValue.token;
|
||||||
|
params.counters.authProfileRefs += 1;
|
||||||
|
params.migratedValues.add(token);
|
||||||
|
changed = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return changed;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function buildMigrationPlan(params: {
|
||||||
|
env: NodeJS.ProcessEnv;
|
||||||
|
scrubEnv: boolean;
|
||||||
|
}): Promise<MigrationPlan> {
|
||||||
|
const io = createConfigIO({ env: params.env });
|
||||||
|
const { snapshot, writeOptions } = await io.readConfigFileSnapshotForWrite();
|
||||||
|
if (!snapshot.valid) {
|
||||||
|
const issues =
|
||||||
|
snapshot.issues.length > 0
|
||||||
|
? snapshot.issues.map((issue) => `${issue.path || "<root>"}: ${issue.message}`).join("\n")
|
||||||
|
: "Unknown validation issue.";
|
||||||
|
throw new Error(`Cannot migrate secrets because config is invalid:\n${issues}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const stateDir = resolveStateDir(params.env, os.homedir);
|
||||||
|
const nextConfig = structuredClone(snapshot.config);
|
||||||
|
const fileSource = resolveFileSource(nextConfig, params.env);
|
||||||
|
const previousPayload = await decryptSopsJson(fileSource.path, fileSource.timeoutMs);
|
||||||
|
const nextPayload = structuredClone(previousPayload);
|
||||||
|
|
||||||
|
const counters: MigrationCounters = {
|
||||||
|
configRefs: 0,
|
||||||
|
authProfileRefs: 0,
|
||||||
|
plaintextRemoved: 0,
|
||||||
|
secretsWritten: 0,
|
||||||
|
envEntriesRemoved: 0,
|
||||||
|
authStoresChanged: 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
const migratedValues = new Set<string>();
|
||||||
|
|
||||||
|
migrateModelProviderSecrets({
|
||||||
|
config: nextConfig,
|
||||||
|
payload: nextPayload,
|
||||||
|
counters,
|
||||||
|
migratedValues,
|
||||||
|
});
|
||||||
|
migrateSkillEntrySecrets({
|
||||||
|
config: nextConfig,
|
||||||
|
payload: nextPayload,
|
||||||
|
counters,
|
||||||
|
migratedValues,
|
||||||
|
});
|
||||||
|
migrateGoogleChatSecrets({
|
||||||
|
config: nextConfig,
|
||||||
|
payload: nextPayload,
|
||||||
|
counters,
|
||||||
|
});
|
||||||
|
|
||||||
|
const authStoreChanges: AuthStoreChange[] = [];
|
||||||
|
for (const authStorePath of collectAuthStorePaths(nextConfig, stateDir)) {
|
||||||
|
if (!fs.existsSync(authStorePath)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const raw = fs.readFileSync(authStorePath, "utf8");
|
||||||
|
let parsed: unknown;
|
||||||
|
try {
|
||||||
|
parsed = JSON.parse(raw) as unknown;
|
||||||
|
} catch {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!isRecord(parsed)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const nextStore = structuredClone(parsed);
|
||||||
|
const scope = deriveAuthStoreScope(authStorePath, stateDir);
|
||||||
|
const changed = migrateAuthStoreSecrets({
|
||||||
|
store: nextStore,
|
||||||
|
scope,
|
||||||
|
payload: nextPayload,
|
||||||
|
counters,
|
||||||
|
migratedValues,
|
||||||
|
});
|
||||||
|
if (!changed) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
authStoreChanges.push({ path: authStorePath, nextStore });
|
||||||
|
}
|
||||||
|
counters.authStoresChanged = authStoreChanges.length;
|
||||||
|
|
||||||
|
if (counters.secretsWritten > 0 && !fileSource.hadConfiguredSource) {
|
||||||
|
const defaultConfigPath = resolveDefaultSecretsConfigPath(params.env);
|
||||||
|
nextConfig.secrets ??= {};
|
||||||
|
nextConfig.secrets.sources ??= {};
|
||||||
|
nextConfig.secrets.sources.file = {
|
||||||
|
type: "sops",
|
||||||
|
path: defaultConfigPath,
|
||||||
|
timeoutMs: DEFAULT_SOPS_TIMEOUT_MS,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const configChanged = !isDeepStrictEqual(snapshot.config, nextConfig);
|
||||||
|
const payloadChanged = !isDeepStrictEqual(previousPayload, nextPayload);
|
||||||
|
|
||||||
|
let envChange: EnvChange | null = null;
|
||||||
|
if (params.scrubEnv && migratedValues.size > 0) {
|
||||||
|
const envPath = path.join(resolveConfigDir(params.env, os.homedir), ".env");
|
||||||
|
if (fs.existsSync(envPath)) {
|
||||||
|
const rawEnv = fs.readFileSync(envPath, "utf8");
|
||||||
|
const scrubbed = scrubEnvRaw(rawEnv, migratedValues, new Set(listKnownSecretEnvVarNames()));
|
||||||
|
if (scrubbed.removed > 0 && scrubbed.nextRaw !== rawEnv) {
|
||||||
|
counters.envEntriesRemoved = scrubbed.removed;
|
||||||
|
envChange = {
|
||||||
|
path: envPath,
|
||||||
|
nextRaw: scrubbed.nextRaw,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const backupTargets = new Set<string>();
|
||||||
|
if (configChanged) {
|
||||||
|
backupTargets.add(io.configPath);
|
||||||
|
}
|
||||||
|
if (payloadChanged) {
|
||||||
|
backupTargets.add(fileSource.path);
|
||||||
|
}
|
||||||
|
for (const change of authStoreChanges) {
|
||||||
|
backupTargets.add(change.path);
|
||||||
|
}
|
||||||
|
if (envChange) {
|
||||||
|
backupTargets.add(envChange.path);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
changed: configChanged || payloadChanged || authStoreChanges.length > 0 || Boolean(envChange),
|
||||||
|
counters,
|
||||||
|
stateDir,
|
||||||
|
configChanged,
|
||||||
|
nextConfig,
|
||||||
|
configWriteOptions: writeOptions,
|
||||||
|
authStoreChanges,
|
||||||
|
payloadChanged,
|
||||||
|
nextPayload,
|
||||||
|
secretsFilePath: fileSource.path,
|
||||||
|
secretsFileTimeoutMs: fileSource.timeoutMs,
|
||||||
|
envChange,
|
||||||
|
backupTargets: [...backupTargets],
|
||||||
|
};
|
||||||
|
}
|
||||||
79
src/secrets/migrate/types.ts
Normal file
79
src/secrets/migrate/types.ts
Normal file
@@ -0,0 +1,79 @@
|
|||||||
|
import type { OpenClawConfig } from "../../config/config.js";
|
||||||
|
import type { ConfigWriteOptions } from "../../config/io.js";
|
||||||
|
|
||||||
|
export type MigrationCounters = {
|
||||||
|
configRefs: number;
|
||||||
|
authProfileRefs: number;
|
||||||
|
plaintextRemoved: number;
|
||||||
|
secretsWritten: number;
|
||||||
|
envEntriesRemoved: number;
|
||||||
|
authStoresChanged: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type AuthStoreChange = {
|
||||||
|
path: string;
|
||||||
|
nextStore: Record<string, unknown>;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type EnvChange = {
|
||||||
|
path: string;
|
||||||
|
nextRaw: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type BackupManifestEntry = {
|
||||||
|
path: string;
|
||||||
|
existed: boolean;
|
||||||
|
backupPath?: string;
|
||||||
|
mode?: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type BackupManifest = {
|
||||||
|
version: 1;
|
||||||
|
backupId: string;
|
||||||
|
createdAt: string;
|
||||||
|
entries: BackupManifestEntry[];
|
||||||
|
};
|
||||||
|
|
||||||
|
export type MigrationPlan = {
|
||||||
|
changed: boolean;
|
||||||
|
counters: MigrationCounters;
|
||||||
|
stateDir: string;
|
||||||
|
configChanged: boolean;
|
||||||
|
nextConfig: OpenClawConfig;
|
||||||
|
configWriteOptions: ConfigWriteOptions;
|
||||||
|
authStoreChanges: AuthStoreChange[];
|
||||||
|
payloadChanged: boolean;
|
||||||
|
nextPayload: Record<string, unknown>;
|
||||||
|
secretsFilePath: string;
|
||||||
|
secretsFileTimeoutMs: number;
|
||||||
|
envChange: EnvChange | null;
|
||||||
|
backupTargets: string[];
|
||||||
|
};
|
||||||
|
|
||||||
|
export type SecretsMigrationRunOptions = {
|
||||||
|
write?: boolean;
|
||||||
|
scrubEnv?: boolean;
|
||||||
|
env?: NodeJS.ProcessEnv;
|
||||||
|
now?: Date;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type SecretsMigrationRunResult = {
|
||||||
|
mode: "dry-run" | "write";
|
||||||
|
changed: boolean;
|
||||||
|
backupId?: string;
|
||||||
|
backupDir?: string;
|
||||||
|
secretsFilePath: string;
|
||||||
|
counters: MigrationCounters;
|
||||||
|
changedFiles: string[];
|
||||||
|
};
|
||||||
|
|
||||||
|
export type SecretsMigrationRollbackOptions = {
|
||||||
|
backupId: string;
|
||||||
|
env?: NodeJS.ProcessEnv;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type SecretsMigrationRollbackResult = {
|
||||||
|
backupId: string;
|
||||||
|
restoredFiles: number;
|
||||||
|
deletedFiles: number;
|
||||||
|
};
|
||||||
@@ -1,3 +1,6 @@
|
|||||||
|
import fs from "node:fs";
|
||||||
|
import path from "node:path";
|
||||||
|
|
||||||
export function isRecord(value: unknown): value is Record<string, unknown> {
|
export function isRecord(value: unknown): value is Record<string, unknown> {
|
||||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||||
}
|
}
|
||||||
@@ -12,3 +15,13 @@ export function normalizePositiveInt(value: unknown, fallback: number): number {
|
|||||||
}
|
}
|
||||||
return Math.max(1, Math.floor(fallback));
|
return Math.max(1, Math.floor(fallback));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function ensureDirForFile(filePath: string): void {
|
||||||
|
fs.mkdirSync(path.dirname(filePath), { recursive: true, mode: 0o700 });
|
||||||
|
}
|
||||||
|
|
||||||
|
export function writeJsonFileSecure(pathname: string, value: unknown): void {
|
||||||
|
ensureDirForFile(pathname);
|
||||||
|
fs.writeFileSync(pathname, `${JSON.stringify(value, null, 2)}\n`, "utf8");
|
||||||
|
fs.chmodSync(pathname, 0o600);
|
||||||
|
}
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import crypto from "node:crypto";
|
|||||||
import fs from "node:fs";
|
import fs from "node:fs";
|
||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
import { runExec } from "../process/exec.js";
|
import { runExec } from "../process/exec.js";
|
||||||
import { normalizePositiveInt } from "./shared.js";
|
import { ensureDirForFile, normalizePositiveInt } from "./shared.js";
|
||||||
|
|
||||||
export const DEFAULT_SOPS_TIMEOUT_MS = 5_000;
|
export const DEFAULT_SOPS_TIMEOUT_MS = 5_000;
|
||||||
const MAX_SOPS_OUTPUT_BYTES = 10 * 1024 * 1024;
|
const MAX_SOPS_OUTPUT_BYTES = 10 * 1024 * 1024;
|
||||||
@@ -30,10 +30,6 @@ function toSopsError(err: unknown, params: SopsErrorContext): Error {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function ensureDirForFile(filePath: string): void {
|
|
||||||
fs.mkdirSync(path.dirname(filePath), { recursive: true, mode: 0o700 });
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function decryptSopsJsonFile(params: {
|
export async function decryptSopsJsonFile(params: {
|
||||||
path: string;
|
path: string;
|
||||||
timeoutMs?: number;
|
timeoutMs?: number;
|
||||||
|
|||||||
Reference in New Issue
Block a user