github/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-03-30 08:51:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(agents): add generic provider api key rotation (#19587)

Browse Source
This commit is contained in:
Peter Steinberger
2026-02-18 01:31:11 +01:00
committed by GitHub
parent 9cce40d123
commit 2e91552f09
8 changed files with 318 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
src/agents/api-key-rotation.ts Normal file
View File

@@ -0,0 +1,72 @@
import { formatErrorMessage } from "../infra/errors.js";
import { collectProviderApiKeys, isApiKeyRateLimitError } from "./live-auth-keys.js";
type ApiKeyRetryParams = {
apiKey: string;
error: unknown;
attempt: number;
};
type ExecuteWithApiKeyRotationOptions<T> = {
provider: string;
apiKeys: string[];
execute: (apiKey: string) => Promise<T>;
shouldRetry?: (params: ApiKeyRetryParams & { message: string }) => boolean;
onRetry?: (params: ApiKeyRetryParams & { message: string }) => void;
};
function dedupeApiKeys(raw: string[]): string[] {
const seen = new Set<string>();
const keys: string[] = [];
for (const value of raw) {
const apiKey = value.trim();
if (!apiKey || seen.has(apiKey)) {
continue;
}
seen.add(apiKey);
keys.push(apiKey);
}
return keys;
}
export function collectProviderApiKeysForExecution(params: {
provider: string;
primaryApiKey?: string;
}): string[] {
const { primaryApiKey, provider } = params;
return dedupeApiKeys([primaryApiKey?.trim() ?? "", ...collectProviderApiKeys(provider)]);
}
export async function executeWithApiKeyRotation<T>(
params: ExecuteWithApiKeyRotationOptions<T>,
): Promise<T> {
const keys = dedupeApiKeys(params.apiKeys);
if (keys.length === 0) {
throw new Error(`No API keys configured for provider "${params.provider}".`);
}
let lastError: unknown;
for (let attempt = 0; attempt < keys.length; attempt += 1) {
const apiKey = keys[attempt];
try {
return await params.execute(apiKey);
} catch (error) {
lastError = error;
const message = formatErrorMessage(error);
const retryable = params.shouldRetry
? params.shouldRetry({ apiKey, error, attempt, message })
: isApiKeyRateLimitError(message);
if (!retryable || attempt + 1 >= keys.length) {
break;
}
params.onRetry?.({ apiKey, error, attempt, message });
}
}
if (lastError === undefined) {
throw new Error(`Failed to run API request for ${params.provider}.`);
}
throw lastError;
}

121
src/agents/live-auth-keys.ts
View File

@@ -1,4 +1,47 @@
import { normalizeProviderId } from "./model-selection.js";
const KEY_SPLIT_RE = /[\s,;]+/g;
const GOOGLE_LIVE_SINGLE_KEY = "OPENCLAW_LIVE_GEMINI_KEY";
const PROVIDER_PREFIX_OVERRIDES: Record<string, string> = {
google: "GEMINI",
"google-vertex": "GEMINI",
};
type ProviderApiKeyConfig = {
liveSingle?: string;
listVar?: string;
primaryVar?: string;
prefixedVar?: string;
fallbackVars: string[];
};
const PROVIDER_API_KEY_CONFIG: Record<string, Omit<ProviderApiKeyConfig, "fallbackVars">> = {
anthropic: {
liveSingle: "OPENCLAW_LIVE_ANTHROPIC_KEY",
listVar: "OPENCLAW_LIVE_ANTHROPIC_KEYS",
primaryVar: "ANTHROPIC_API_KEY",
prefixedVar: "ANTHROPIC_API_KEY_",
},
google: {
liveSingle: GOOGLE_LIVE_SINGLE_KEY,
listVar: "GEMINI_API_KEYS",
primaryVar: "GEMINI_API_KEY",
prefixedVar: "GEMINI_API_KEY_",
},
"google-vertex": {
liveSingle: GOOGLE_LIVE_SINGLE_KEY,
listVar: "GEMINI_API_KEYS",
primaryVar: "GEMINI_API_KEY",
prefixedVar: "GEMINI_API_KEY_",
},
openai: {
liveSingle: "OPENCLAW_LIVE_OPENAI_KEY",
listVar: "OPENAI_API_KEYS",
primaryVar: "OPENAI_API_KEY",
prefixedVar: "OPENAI_API_KEY_",
},
};
function parseKeyList(raw?: string | null): string[] {
if (!raw) {
@@ -25,17 +68,53 @@ function collectEnvPrefixedKeys(prefix: string): string[] {
return keys;
}
export function collectAnthropicApiKeys(): string[] {
const forcedSingle = process.env.OPENCLAW_LIVE_ANTHROPIC_KEY?.trim();
function resolveProviderApiKeyConfig(provider: string): ProviderApiKeyConfig {
const normalized = normalizeProviderId(provider);
const custom = PROVIDER_API_KEY_CONFIG[normalized];
const base = PROVIDER_PREFIX_OVERRIDES[normalized] ?? normalized.toUpperCase().replace(/-/g, "_");
const liveSingle = custom?.liveSingle ?? `OPENCLAW_LIVE_${base}_KEY`;
const listVar = custom?.listVar ?? `${base}_API_KEYS`;
const primaryVar = custom?.primaryVar ?? `${base}_API_KEY`;
const prefixedVar = custom?.prefixedVar ?? `${base}_API_KEY_`;
if (normalized === "google" || normalized === "google-vertex") {
return {
liveSingle,
listVar,
primaryVar,
prefixedVar,
fallbackVars: ["GOOGLE_API_KEY"],
};
}
return {
liveSingle,
listVar,
primaryVar,
prefixedVar,
fallbackVars: [],
};
}
export function collectProviderApiKeys(provider: string): string[] {
const config = resolveProviderApiKeyConfig(provider);
const forcedSingle = config.liveSingle ? process.env[config.liveSingle]?.trim() : undefined;
if (forcedSingle) {
return [forcedSingle];
}
const fromList = parseKeyList(process.env.OPENCLAW_LIVE_ANTHROPIC_KEYS);
const fromEnv = collectEnvPrefixedKeys("ANTHROPIC_API_KEY");
const primary = process.env.ANTHROPIC_API_KEY?.trim();
const fromList = parseKeyList(config.listVar ? process.env[config.listVar] : undefined);
const primary = config.primaryVar ? process.env[config.primaryVar]?.trim() : undefined;
const fromPrefixed = config.prefixedVar ? collectEnvPrefixedKeys(config.prefixedVar) : [];
const fallback = config.fallbackVars
.map((envVar) => process.env[envVar]?.trim())
.filter(Boolean) as string[];
const seen = new Set<string>();
const add = (value?: string) => {
if (!value) {
return;
@@ -49,17 +128,26 @@ export function collectAnthropicApiKeys(): string[] {
for (const value of fromList) {
add(value);
}
if (primary) {
add(primary);
add(primary);
for (const value of fromPrefixed) {
add(value);
}
for (const value of fromEnv) {
for (const value of fallback) {
add(value);
}
return Array.from(seen);
}
export function isAnthropicRateLimitError(message: string): boolean {
export function collectAnthropicApiKeys(): string[] {
return collectProviderApiKeys("anthropic");
}
export function collectGeminiApiKeys(): string[] {
return collectProviderApiKeys("google");
}
export function isApiKeyRateLimitError(message: string): boolean {
const lower = message.toLowerCase();
if (lower.includes("rate_limit")) {
return true;
@@ -70,9 +158,22 @@ export function isAnthropicRateLimitError(message: string): boolean {
if (lower.includes("429")) {
return true;
}
if (lower.includes("quota exceeded") || lower.includes("quota_exceeded")) {
return true;
}
if (lower.includes("resource exhausted") || lower.includes("resource_exhausted")) {
return true;
}
if (lower.includes("too many requests")) {
return true;
}
return false;
}
export function isAnthropicRateLimitError(message: string): boolean {
return isApiKeyRateLimitError(message);
}
export function isAnthropicBillingError(message: string): boolean {
const lower = message.toLowerCase();
if (lower.includes("credit balance")) {
@@ -91,7 +192,7 @@ export function isAnthropicBillingError(message: string): boolean {
return true;
}
if (
/["']?(?:status|code)["']?\s*[:=]\s*402\b|\bhttp\s*402\b|\berror(?:\s+code)?\s*[:=]?\s*402\b|\b(?:got|returned|received)\s+(?:a\s+)?402\b|^\s*402\s+payment/i.test(
/["']?(?:status|code)["']?\s*[:=]\s*402\b|\bhttp\s*402\b|\berror(?:\s+code)?\s*[:=]?\s*402\b|\b(?:got|returned|received)\s+(?:a\s+)?402\b|^\s*402\spayment/i.test(
lower,
)
) {

68
src/media-understanding/runner.entries.ts
View File

@@ -2,6 +2,10 @@ import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { requireApiKey, resolveApiKeyForProvider } from "../agents/model-auth.js";
import {
collectProviderApiKeysForExecution,
executeWithApiKeyRotation,
} from "../agents/api-key-rotation.js";
import type { MsgContext } from "../auto-reply/templating.js";
import { applyTemplate } from "../auto-reply/templating.js";
import type { OpenClawConfig } from "../config/config.js";
@@ -408,7 +412,10 @@ export async function runProviderEntry(params: {
preferredProfile: entry.preferredProfile,
agentDir: params.agentDir,
});
const apiKey = requireApiKey(auth, providerId);
const apiKeys = collectProviderApiKeysForExecution({
provider: providerId,
primaryApiKey: requireApiKey(auth, providerId),
});
const providerConfig = cfg.models?.providers?.[providerId];
const baseUrl = entry.baseUrl ?? params.config?.baseUrl ?? providerConfig?.baseUrl;
const mergedHeaders = {
@@ -423,18 +430,23 @@ export async function runProviderEntry(params: {
entry,
});
const model = entry.model?.trim() || DEFAULT_AUDIO_MODELS[providerId] || entry.model;
const result = await provider.transcribeAudio({
buffer: media.buffer,
fileName: media.fileName,
mime: media.mime,
apiKey,
baseUrl,
headers,
model,
language: entry.language ?? params.config?.language ?? cfg.tools?.media?.audio?.language,
prompt,
query: providerQuery,
timeoutMs,
const result = await executeWithApiKeyRotation({
provider: providerId,
apiKeys,
execute: async (apiKey) =>
provider.transcribeAudio({
buffer: media.buffer,
fileName: media.fileName,
mime: media.mime,
apiKey,
baseUrl,
headers,
model,
language: entry.language ?? params.config?.language ?? cfg.tools?.media?.audio?.language,
prompt,
query: providerQuery,
timeoutMs,
}),
});
return {
kind: "audio.transcription",
@@ -468,18 +480,26 @@ export async function runProviderEntry(params: {
preferredProfile: entry.preferredProfile,
agentDir: params.agentDir,
});
const apiKey = requireApiKey(auth, providerId);
const apiKeys = collectProviderApiKeysForExecution({
provider: providerId,
primaryApiKey: requireApiKey(auth, providerId),
});
const providerConfig = cfg.models?.providers?.[providerId];
const result = await provider.describeVideo({
buffer: media.buffer,
fileName: media.fileName,
mime: media.mime,
apiKey,
baseUrl: providerConfig?.baseUrl,
headers: providerConfig?.headers,
model: entry.model,
prompt,
timeoutMs,
const result = await executeWithApiKeyRotation({
provider: providerId,
apiKeys,
execute: (apiKey) =>
provider.describeVideo({
buffer: media.buffer,
fileName: media.fileName,
mime: media.mime,
apiKey,
baseUrl: providerConfig?.baseUrl,
headers: providerConfig?.headers,
model: entry.model,
prompt,
timeoutMs,
}),
});
return {
kind: "video.description",

70
src/memory/embeddings-gemini.ts
View File

@@ -1,13 +1,18 @@
import type { EmbeddingProvider, EmbeddingProviderOptions } from "./embeddings.js";
import {
collectProviderApiKeysForExecution,
executeWithApiKeyRotation,
} from "../agents/api-key-rotation.js";
import { requireApiKey, resolveApiKeyForProvider } from "../agents/model-auth.js";
import { parseGeminiAuth } from "../infra/gemini-auth.js";
import { debugEmbeddingsLog } from "./embeddings-debug.js";
import type { EmbeddingProvider, EmbeddingProviderOptions } from "./embeddings.js";
export type GeminiEmbeddingClient = {
baseUrl: string;
headers: Record<string, string>;
model: string;
modelPath: string;
apiKeys: string[];
};
const DEFAULT_GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta";
@@ -62,23 +67,40 @@ export async function createGeminiEmbeddingProvider(
const embedUrl = `${baseUrl}/${client.modelPath}:embedContent`;
const batchUrl = `${baseUrl}/${client.modelPath}:batchEmbedContents`;
const embedQuery = async (text: string): Promise<number[]> => {
if (!text.trim()) {
return [];
}
const res = await fetch(embedUrl, {
const fetchWithGeminiAuth = async (apiKey: string, endpoint: string, body: unknown) => {
const authHeaders = parseGeminiAuth(apiKey);
const headers = {
...authHeaders.headers,
...client.headers,
};
const res = await fetch(endpoint, {
method: "POST",
headers: client.headers,
body: JSON.stringify({
content: { parts: [{ text }] },
taskType: "RETRIEVAL_QUERY",
}),
headers,
body: JSON.stringify(body),
});
if (!res.ok) {
const payload = await res.text();
throw new Error(`gemini embeddings failed: ${res.status} ${payload}`);
}
const payload = (await res.json()) as { embedding?: { values?: number[] } };
return (await res.json()) as {
embedding?: { values?: number[] };
embeddings?: Array<{ values?: number[] }>;
};
};
const embedQuery = async (text: string): Promise<number[]> => {
if (!text.trim()) {
return [];
}
const payload = await executeWithApiKeyRotation({
provider: "google",
apiKeys: client.apiKeys,
execute: (apiKey) =>
fetchWithGeminiAuth(apiKey, embedUrl, {
content: { parts: [{ text }] },
taskType: "RETRIEVAL_QUERY",
}),
});
return payload.embedding?.values ?? [];
};
@@ -91,16 +113,14 @@ export async function createGeminiEmbeddingProvider(
content: { parts: [{ text }] },
taskType: "RETRIEVAL_DOCUMENT",
}));
const res = await fetch(batchUrl, {
method: "POST",
headers: client.headers,
body: JSON.stringify({ requests }),
const payload = await executeWithApiKeyRotation({
provider: "google",
apiKeys: client.apiKeys,
execute: (apiKey) =>
fetchWithGeminiAuth(apiKey, batchUrl, {
requests,
}),
});
if (!res.ok) {
const payload = await res.text();
throw new Error(`gemini embeddings failed: ${res.status} ${payload}`);
}
const payload = (await res.json()) as { embeddings?: Array<{ values?: number[] }> };
const embeddings = Array.isArray(payload.embeddings) ? payload.embeddings : [];
return texts.map((_, index) => embeddings[index]?.values ?? []);
};
@@ -139,11 +159,13 @@ export async function resolveGeminiEmbeddingClient(
const rawBaseUrl = remoteBaseUrl || providerConfig?.baseUrl?.trim() || DEFAULT_GEMINI_BASE_URL;
const baseUrl = normalizeGeminiBaseUrl(rawBaseUrl);
const headerOverrides = Object.assign({}, providerConfig?.headers, remote?.headers);
const authHeaders = parseGeminiAuth(apiKey);
const headers: Record<string, string> = {
...authHeaders.headers,
...headerOverrides,
};
const apiKeys = collectProviderApiKeysForExecution({
provider: "google",
primaryApiKey: apiKey,
});
const model = normalizeGeminiModel(options.model);
const modelPath = buildGeminiModelPath(model);
debugEmbeddingsLog("memory embeddings: gemini client", {
@@ -154,5 +176,5 @@ export async function resolveGeminiEmbeddingClient(
embedEndpoint: `${baseUrl}/${modelPath}:embedContent`,
batchEndpoint: `${baseUrl}/${modelPath}:batchEmbedContents`,
});
return { baseUrl, headers, model, modelPath };
return { baseUrl, headers, model, modelPath, apiKeys };
}