feat(agents): add generic provider api key rotation (#19587)

src/agents/api-key-rotation.ts

@@ -0,0 +1,72 @@
+import { formatErrorMessage } from "../infra/errors.js";
+import { collectProviderApiKeys, isApiKeyRateLimitError } from "./live-auth-keys.js";
+
+type ApiKeyRetryParams = {
+  apiKey: string;
+  error: unknown;
+  attempt: number;
+};
+
+type ExecuteWithApiKeyRotationOptions<T> = {
+  provider: string;
+  apiKeys: string[];
+  execute: (apiKey: string) => Promise<T>;
+  shouldRetry?: (params: ApiKeyRetryParams & { message: string }) => boolean;
+  onRetry?: (params: ApiKeyRetryParams & { message: string }) => void;
+};
+
+function dedupeApiKeys(raw: string[]): string[] {
+  const seen = new Set<string>();
+  const keys: string[] = [];
+  for (const value of raw) {
+    const apiKey = value.trim();
+    if (!apiKey || seen.has(apiKey)) {
+      continue;
+    }
+    seen.add(apiKey);
+    keys.push(apiKey);
+  }
+  return keys;
+}
+
+export function collectProviderApiKeysForExecution(params: {
+  provider: string;
+  primaryApiKey?: string;
+}): string[] {
+  const { primaryApiKey, provider } = params;
+  return dedupeApiKeys([primaryApiKey?.trim() ?? "", ...collectProviderApiKeys(provider)]);
+}
+
+export async function executeWithApiKeyRotation<T>(
+  params: ExecuteWithApiKeyRotationOptions<T>,
+): Promise<T> {
+  const keys = dedupeApiKeys(params.apiKeys);
+  if (keys.length === 0) {
+    throw new Error(`No API keys configured for provider "${params.provider}".`);
+  }
+
+  let lastError: unknown;
+  for (let attempt = 0; attempt < keys.length; attempt += 1) {
+    const apiKey = keys[attempt];
+    try {
+      return await params.execute(apiKey);
+    } catch (error) {
+      lastError = error;
+      const message = formatErrorMessage(error);
+      const retryable = params.shouldRetry
+        ? params.shouldRetry({ apiKey, error, attempt, message })
+        : isApiKeyRateLimitError(message);
+
+      if (!retryable || attempt + 1 >= keys.length) {
+        break;
+      }
+
+      params.onRetry?.({ apiKey, error, attempt, message });
+    }
+  }
+
+  if (lastError === undefined) {
+    throw new Error(`Failed to run API request for ${params.provider}.`);
+  }
+  throw lastError;
+}

src/agents/live-auth-keys.ts

@@ -1,4 +1,47 @@
+import { normalizeProviderId } from "./model-selection.js";
+
 const KEY_SPLIT_RE = /[\s,;]+/g;
+const GOOGLE_LIVE_SINGLE_KEY = "OPENCLAW_LIVE_GEMINI_KEY";
+
+const PROVIDER_PREFIX_OVERRIDES: Record<string, string> = {
+  google: "GEMINI",
+  "google-vertex": "GEMINI",
+};
+
+type ProviderApiKeyConfig = {
+  liveSingle?: string;
+  listVar?: string;
+  primaryVar?: string;
+  prefixedVar?: string;
+  fallbackVars: string[];
+};
+
+const PROVIDER_API_KEY_CONFIG: Record<string, Omit<ProviderApiKeyConfig, "fallbackVars">> = {
+  anthropic: {
+    liveSingle: "OPENCLAW_LIVE_ANTHROPIC_KEY",
+    listVar: "OPENCLAW_LIVE_ANTHROPIC_KEYS",
+    primaryVar: "ANTHROPIC_API_KEY",
+    prefixedVar: "ANTHROPIC_API_KEY_",
+  },
+  google: {
+    liveSingle: GOOGLE_LIVE_SINGLE_KEY,
+    listVar: "GEMINI_API_KEYS",
+    primaryVar: "GEMINI_API_KEY",
+    prefixedVar: "GEMINI_API_KEY_",
+  },
+  "google-vertex": {
+    liveSingle: GOOGLE_LIVE_SINGLE_KEY,
+    listVar: "GEMINI_API_KEYS",
+    primaryVar: "GEMINI_API_KEY",
+    prefixedVar: "GEMINI_API_KEY_",
+  },
+  openai: {
+    liveSingle: "OPENCLAW_LIVE_OPENAI_KEY",
+    listVar: "OPENAI_API_KEYS",
+    primaryVar: "OPENAI_API_KEY",
+    prefixedVar: "OPENAI_API_KEY_",
+  },
+};
 
 function parseKeyList(raw?: string | null): string[] {
   if (!raw) {
@@ -25,17 +68,53 @@ function collectEnvPrefixedKeys(prefix: string): string[] {
   return keys;
 }
 
-export function collectAnthropicApiKeys(): string[] {
-  const forcedSingle = process.env.OPENCLAW_LIVE_ANTHROPIC_KEY?.trim();
+function resolveProviderApiKeyConfig(provider: string): ProviderApiKeyConfig {
+  const normalized = normalizeProviderId(provider);
+  const custom = PROVIDER_API_KEY_CONFIG[normalized];
+  const base = PROVIDER_PREFIX_OVERRIDES[normalized] ?? normalized.toUpperCase().replace(/-/g, "_");
+
+  const liveSingle = custom?.liveSingle ?? `OPENCLAW_LIVE_${base}_KEY`;
+  const listVar = custom?.listVar ?? `${base}_API_KEYS`;
+  const primaryVar = custom?.primaryVar ?? `${base}_API_KEY`;
+  const prefixedVar = custom?.prefixedVar ?? `${base}_API_KEY_`;
+
+  if (normalized === "google" || normalized === "google-vertex") {
+    return {
+      liveSingle,
+      listVar,
+      primaryVar,
+      prefixedVar,
+      fallbackVars: ["GOOGLE_API_KEY"],
+    };
+  }
+
+  return {
+    liveSingle,
+    listVar,
+    primaryVar,
+    prefixedVar,
+    fallbackVars: [],
+  };
+}
+
+export function collectProviderApiKeys(provider: string): string[] {
+  const config = resolveProviderApiKeyConfig(provider);
+
+  const forcedSingle = config.liveSingle ? process.env[config.liveSingle]?.trim() : undefined;
   if (forcedSingle) {
     return [forcedSingle];
   }
 
-  const fromList = parseKeyList(process.env.OPENCLAW_LIVE_ANTHROPIC_KEYS);
-  const fromEnv = collectEnvPrefixedKeys("ANTHROPIC_API_KEY");
-  const primary = process.env.ANTHROPIC_API_KEY?.trim();
+  const fromList = parseKeyList(config.listVar ? process.env[config.listVar] : undefined);
+  const primary = config.primaryVar ? process.env[config.primaryVar]?.trim() : undefined;
+  const fromPrefixed = config.prefixedVar ? collectEnvPrefixedKeys(config.prefixedVar) : [];
+
+  const fallback = config.fallbackVars
+    .map((envVar) => process.env[envVar]?.trim())
+    .filter(Boolean) as string[];
 
   const seen = new Set<string>();
+
   const add = (value?: string) => {
     if (!value) {
       return;
@@ -49,17 +128,26 @@ export function collectAnthropicApiKeys(): string[] {
   for (const value of fromList) {
     add(value);
   }
-  if (primary) {
-    add(primary);
+  add(primary);
+  for (const value of fromPrefixed) {
+    add(value);
   }
-  for (const value of fromEnv) {
+  for (const value of fallback) {
     add(value);
   }
 
   return Array.from(seen);
 }
 
-export function isAnthropicRateLimitError(message: string): boolean {
+export function collectAnthropicApiKeys(): string[] {
+  return collectProviderApiKeys("anthropic");
+}
+
+export function collectGeminiApiKeys(): string[] {
+  return collectProviderApiKeys("google");
+}
+
+export function isApiKeyRateLimitError(message: string): boolean {
   const lower = message.toLowerCase();
   if (lower.includes("rate_limit")) {
     return true;
@@ -70,9 +158,22 @@ export function isAnthropicRateLimitError(message: string): boolean {
   if (lower.includes("429")) {
     return true;
   }
+  if (lower.includes("quota exceeded") || lower.includes("quota_exceeded")) {
+    return true;
+  }
+  if (lower.includes("resource exhausted") || lower.includes("resource_exhausted")) {
+    return true;
+  }
+  if (lower.includes("too many requests")) {
+    return true;
+  }
   return false;
 }
 
+export function isAnthropicRateLimitError(message: string): boolean {
+  return isApiKeyRateLimitError(message);
+}
+
 export function isAnthropicBillingError(message: string): boolean {
   const lower = message.toLowerCase();
   if (lower.includes("credit balance")) {
@@ -91,7 +192,7 @@ export function isAnthropicBillingError(message: string): boolean {
     return true;
   }
   if (
-    /["']?(?:status|code)["']?\s*[:=]\s*402\b|\bhttp\s*402\b|\berror(?:\s+code)?\s*[:=]?\s*402\b|\b(?:got|returned|received)\s+(?:a\s+)?402\b|^\s*402\s+payment/i.test(
+    /["']?(?:status|code)["']?\s*[:=]\s*402\b|\bhttp\s*402\b|\berror(?:\s+code)?\s*[:=]?\s*402\b|\b(?:got|returned|received)\s+(?:a\s+)?402\b|^\s*402\spayment/i.test(
       lower,
     )
   ) {