feat(agents): add generic provider api key rotation (#19587)

src/agents/live-auth-keys.ts

@@ -1,4 +1,47 @@
+import { normalizeProviderId } from "./model-selection.js";
+
 const KEY_SPLIT_RE = /[\s,;]+/g;
+const GOOGLE_LIVE_SINGLE_KEY = "OPENCLAW_LIVE_GEMINI_KEY";
+
+const PROVIDER_PREFIX_OVERRIDES: Record<string, string> = {
+  google: "GEMINI",
+  "google-vertex": "GEMINI",
+};
+
+type ProviderApiKeyConfig = {
+  liveSingle?: string;
+  listVar?: string;
+  primaryVar?: string;
+  prefixedVar?: string;
+  fallbackVars: string[];
+};
+
+const PROVIDER_API_KEY_CONFIG: Record<string, Omit<ProviderApiKeyConfig, "fallbackVars">> = {
+  anthropic: {
+    liveSingle: "OPENCLAW_LIVE_ANTHROPIC_KEY",
+    listVar: "OPENCLAW_LIVE_ANTHROPIC_KEYS",
+    primaryVar: "ANTHROPIC_API_KEY",
+    prefixedVar: "ANTHROPIC_API_KEY_",
+  },
+  google: {
+    liveSingle: GOOGLE_LIVE_SINGLE_KEY,
+    listVar: "GEMINI_API_KEYS",
+    primaryVar: "GEMINI_API_KEY",
+    prefixedVar: "GEMINI_API_KEY_",
+  },
+  "google-vertex": {
+    liveSingle: GOOGLE_LIVE_SINGLE_KEY,
+    listVar: "GEMINI_API_KEYS",
+    primaryVar: "GEMINI_API_KEY",
+    prefixedVar: "GEMINI_API_KEY_",
+  },
+  openai: {
+    liveSingle: "OPENCLAW_LIVE_OPENAI_KEY",
+    listVar: "OPENAI_API_KEYS",
+    primaryVar: "OPENAI_API_KEY",
+    prefixedVar: "OPENAI_API_KEY_",
+  },
+};
 
 function parseKeyList(raw?: string | null): string[] {
   if (!raw) {
@@ -25,17 +68,53 @@ function collectEnvPrefixedKeys(prefix: string): string[] {
   return keys;
 }
 
-export function collectAnthropicApiKeys(): string[] {
-  const forcedSingle = process.env.OPENCLAW_LIVE_ANTHROPIC_KEY?.trim();
+function resolveProviderApiKeyConfig(provider: string): ProviderApiKeyConfig {
+  const normalized = normalizeProviderId(provider);
+  const custom = PROVIDER_API_KEY_CONFIG[normalized];
+  const base = PROVIDER_PREFIX_OVERRIDES[normalized] ?? normalized.toUpperCase().replace(/-/g, "_");
+
+  const liveSingle = custom?.liveSingle ?? `OPENCLAW_LIVE_${base}_KEY`;
+  const listVar = custom?.listVar ?? `${base}_API_KEYS`;
+  const primaryVar = custom?.primaryVar ?? `${base}_API_KEY`;
+  const prefixedVar = custom?.prefixedVar ?? `${base}_API_KEY_`;
+
+  if (normalized === "google" || normalized === "google-vertex") {
+    return {
+      liveSingle,
+      listVar,
+      primaryVar,
+      prefixedVar,
+      fallbackVars: ["GOOGLE_API_KEY"],
+    };
+  }
+
+  return {
+    liveSingle,
+    listVar,
+    primaryVar,
+    prefixedVar,
+    fallbackVars: [],
+  };
+}
+
+export function collectProviderApiKeys(provider: string): string[] {
+  const config = resolveProviderApiKeyConfig(provider);
+
+  const forcedSingle = config.liveSingle ? process.env[config.liveSingle]?.trim() : undefined;
   if (forcedSingle) {
     return [forcedSingle];
   }
 
-  const fromList = parseKeyList(process.env.OPENCLAW_LIVE_ANTHROPIC_KEYS);
-  const fromEnv = collectEnvPrefixedKeys("ANTHROPIC_API_KEY");
-  const primary = process.env.ANTHROPIC_API_KEY?.trim();
+  const fromList = parseKeyList(config.listVar ? process.env[config.listVar] : undefined);
+  const primary = config.primaryVar ? process.env[config.primaryVar]?.trim() : undefined;
+  const fromPrefixed = config.prefixedVar ? collectEnvPrefixedKeys(config.prefixedVar) : [];
+
+  const fallback = config.fallbackVars
+    .map((envVar) => process.env[envVar]?.trim())
+    .filter(Boolean) as string[];
 
   const seen = new Set<string>();
+
   const add = (value?: string) => {
     if (!value) {
       return;
@@ -49,17 +128,26 @@ export function collectAnthropicApiKeys(): string[] {
   for (const value of fromList) {
     add(value);
   }
-  if (primary) {
-    add(primary);
+  add(primary);
+  for (const value of fromPrefixed) {
+    add(value);
   }
-  for (const value of fromEnv) {
+  for (const value of fallback) {
     add(value);
   }
 
   return Array.from(seen);
 }
 
-export function isAnthropicRateLimitError(message: string): boolean {
+export function collectAnthropicApiKeys(): string[] {
+  return collectProviderApiKeys("anthropic");
+}
+
+export function collectGeminiApiKeys(): string[] {
+  return collectProviderApiKeys("google");
+}
+
+export function isApiKeyRateLimitError(message: string): boolean {
   const lower = message.toLowerCase();
   if (lower.includes("rate_limit")) {
     return true;
@@ -70,9 +158,22 @@ export function isAnthropicRateLimitError(message: string): boolean {
   if (lower.includes("429")) {
     return true;
   }
+  if (lower.includes("quota exceeded") || lower.includes("quota_exceeded")) {
+    return true;
+  }
+  if (lower.includes("resource exhausted") || lower.includes("resource_exhausted")) {
+    return true;
+  }
+  if (lower.includes("too many requests")) {
+    return true;
+  }
   return false;
 }
 
+export function isAnthropicRateLimitError(message: string): boolean {
+  return isApiKeyRateLimitError(message);
+}
+
 export function isAnthropicBillingError(message: string): boolean {
   const lower = message.toLowerCase();
   if (lower.includes("credit balance")) {
@@ -91,7 +192,7 @@ export function isAnthropicBillingError(message: string): boolean {
     return true;
   }
   if (
-    /["']?(?:status|code)["']?\s*[:=]\s*402\b|\bhttp\s*402\b|\berror(?:\s+code)?\s*[:=]?\s*402\b|\b(?:got|returned|received)\s+(?:a\s+)?402\b|^\s*402\s+payment/i.test(
+    /["']?(?:status|code)["']?\s*[:=]\s*402\b|\bhttp\s*402\b|\berror(?:\s+code)?\s*[:=]?\s*402\b|\b(?:got|returned|received)\s+(?:a\s+)?402\b|^\s*402\spayment/i.test(
       lower,
     )
   ) {