mirror of
https://github.com/openclaw/openclaw.git
synced 2026-06-02 15:07:15 +00:00
refactor(agents): extract cooldown decision helper
This commit is contained in:
@@ -143,11 +143,15 @@ async function expectSkippedUnavailableProvider(params: {
|
|||||||
}) {
|
}) {
|
||||||
const provider = `${params.providerPrefix}-${crypto.randomUUID()}`;
|
const provider = `${params.providerPrefix}-${crypto.randomUUID()}`;
|
||||||
const cfg = makeProviderFallbackCfg(provider);
|
const cfg = makeProviderFallbackCfg(provider);
|
||||||
|
const primaryStore = makeSingleProviderStore({
|
||||||
|
provider,
|
||||||
|
usageStat: params.usageStat,
|
||||||
|
});
|
||||||
// Include fallback provider profile so the fallback is attempted (not skipped as no-profile).
|
// Include fallback provider profile so the fallback is attempted (not skipped as no-profile).
|
||||||
const store: AuthProfileStore = {
|
const store: AuthProfileStore = {
|
||||||
...makeSingleProviderStore({ provider, usageStat: params.usageStat }),
|
...primaryStore,
|
||||||
profiles: {
|
profiles: {
|
||||||
...makeSingleProviderStore({ provider, usageStat: params.usageStat }).profiles,
|
...primaryStore.profiles,
|
||||||
"fallback:default": {
|
"fallback:default": {
|
||||||
type: "api_key",
|
type: "api_key",
|
||||||
provider: "fallback",
|
provider: "fallback",
|
||||||
|
|||||||
@@ -306,6 +306,76 @@ export const _probeThrottleInternals = {
|
|||||||
resolveProbeThrottleKey,
|
resolveProbeThrottleKey,
|
||||||
} as const;
|
} as const;
|
||||||
|
|
||||||
|
type CooldownDecision =
|
||||||
|
| {
|
||||||
|
type: "skip";
|
||||||
|
reason: FailoverReason;
|
||||||
|
error: string;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
type: "attempt";
|
||||||
|
reason: FailoverReason;
|
||||||
|
markProbe: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
function resolveCooldownDecision(params: {
|
||||||
|
candidate: ModelCandidate;
|
||||||
|
isPrimary: boolean;
|
||||||
|
requestedModel: boolean;
|
||||||
|
hasFallbackCandidates: boolean;
|
||||||
|
now: number;
|
||||||
|
probeThrottleKey: string;
|
||||||
|
authStore: ReturnType<typeof ensureAuthProfileStore>;
|
||||||
|
profileIds: string[];
|
||||||
|
}): CooldownDecision {
|
||||||
|
const shouldProbe = shouldProbePrimaryDuringCooldown({
|
||||||
|
isPrimary: params.isPrimary,
|
||||||
|
hasFallbackCandidates: params.hasFallbackCandidates,
|
||||||
|
now: params.now,
|
||||||
|
throttleKey: params.probeThrottleKey,
|
||||||
|
authStore: params.authStore,
|
||||||
|
profileIds: params.profileIds,
|
||||||
|
});
|
||||||
|
|
||||||
|
const inferredReason =
|
||||||
|
resolveProfilesUnavailableReason({
|
||||||
|
store: params.authStore,
|
||||||
|
profileIds: params.profileIds,
|
||||||
|
now: params.now,
|
||||||
|
}) ?? "rate_limit";
|
||||||
|
const isPersistentIssue =
|
||||||
|
inferredReason === "auth" ||
|
||||||
|
inferredReason === "auth_permanent" ||
|
||||||
|
inferredReason === "billing";
|
||||||
|
if (isPersistentIssue) {
|
||||||
|
return {
|
||||||
|
type: "skip",
|
||||||
|
reason: inferredReason,
|
||||||
|
error: `Provider ${params.candidate.provider} has ${inferredReason} issue (skipping all models)`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// For primary: try when requested model or when probe allows.
|
||||||
|
// For same-provider fallbacks: only relax cooldown on rate_limit, which
|
||||||
|
// is commonly model-scoped and can recover on a sibling model.
|
||||||
|
const shouldAttemptDespiteCooldown =
|
||||||
|
(params.isPrimary && (!params.requestedModel || shouldProbe)) ||
|
||||||
|
(!params.isPrimary && inferredReason === "rate_limit");
|
||||||
|
if (!shouldAttemptDespiteCooldown) {
|
||||||
|
return {
|
||||||
|
type: "skip",
|
||||||
|
reason: inferredReason,
|
||||||
|
error: `Provider ${params.candidate.provider} is in cooldown (all profiles unavailable)`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
type: "attempt",
|
||||||
|
reason: inferredReason,
|
||||||
|
markProbe: params.isPrimary && shouldProbe,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export async function runWithModelFallback<T>(params: {
|
export async function runWithModelFallback<T>(params: {
|
||||||
cfg: OpenClawConfig | undefined;
|
cfg: OpenClawConfig | undefined;
|
||||||
provider: string;
|
provider: string;
|
||||||
@@ -347,51 +417,28 @@ export async function runWithModelFallback<T>(params: {
|
|||||||
params.provider === candidate.provider && params.model === candidate.model;
|
params.provider === candidate.provider && params.model === candidate.model;
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
const probeThrottleKey = resolveProbeThrottleKey(candidate.provider, params.agentDir);
|
const probeThrottleKey = resolveProbeThrottleKey(candidate.provider, params.agentDir);
|
||||||
const shouldProbe = shouldProbePrimaryDuringCooldown({
|
const decision = resolveCooldownDecision({
|
||||||
|
candidate,
|
||||||
isPrimary,
|
isPrimary,
|
||||||
|
requestedModel,
|
||||||
hasFallbackCandidates,
|
hasFallbackCandidates,
|
||||||
now,
|
now,
|
||||||
throttleKey: probeThrottleKey,
|
probeThrottleKey,
|
||||||
authStore,
|
authStore,
|
||||||
profileIds,
|
profileIds,
|
||||||
});
|
});
|
||||||
|
|
||||||
const inferredReason =
|
if (decision.type === "skip") {
|
||||||
resolveProfilesUnavailableReason({
|
|
||||||
store: authStore,
|
|
||||||
profileIds,
|
|
||||||
now,
|
|
||||||
}) ?? "rate_limit";
|
|
||||||
const isPersistentIssue =
|
|
||||||
inferredReason === "auth" ||
|
|
||||||
inferredReason === "auth_permanent" ||
|
|
||||||
inferredReason === "billing";
|
|
||||||
if (isPersistentIssue) {
|
|
||||||
attempts.push({
|
attempts.push({
|
||||||
provider: candidate.provider,
|
provider: candidate.provider,
|
||||||
model: candidate.model,
|
model: candidate.model,
|
||||||
error: `Provider ${candidate.provider} has ${inferredReason} issue (skipping all models)`,
|
error: decision.error,
|
||||||
reason: inferredReason,
|
reason: decision.reason,
|
||||||
});
|
});
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// For primary: try when requested model or when probe allows.
|
if (decision.markProbe) {
|
||||||
// For same-provider fallbacks: only relax cooldown on rate_limit, which
|
|
||||||
// is commonly model-scoped and can recover on a sibling model.
|
|
||||||
const shouldAttemptDespiteCooldown =
|
|
||||||
(isPrimary && (!requestedModel || shouldProbe)) ||
|
|
||||||
(!isPrimary && inferredReason === "rate_limit");
|
|
||||||
if (!shouldAttemptDespiteCooldown) {
|
|
||||||
attempts.push({
|
|
||||||
provider: candidate.provider,
|
|
||||||
model: candidate.model,
|
|
||||||
error: `Provider ${candidate.provider} is in cooldown (all profiles unavailable)`,
|
|
||||||
reason: inferredReason,
|
|
||||||
});
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (isPrimary && shouldProbe) {
|
|
||||||
lastProbeAttempt.set(probeThrottleKey, now);
|
lastProbeAttempt.set(probeThrottleKey, now);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user