refactor(agents): extract cooldown decision helper

This commit is contained in:
Gustavo Madeira Santana
2026-02-25 20:26:28 -05:00
parent 353b3dfddb
commit e6f2b4742b
2 changed files with 84 additions and 33 deletions

View File

@@ -143,11 +143,15 @@ async function expectSkippedUnavailableProvider(params: {
}) { }) {
const provider = `${params.providerPrefix}-${crypto.randomUUID()}`; const provider = `${params.providerPrefix}-${crypto.randomUUID()}`;
const cfg = makeProviderFallbackCfg(provider); const cfg = makeProviderFallbackCfg(provider);
const primaryStore = makeSingleProviderStore({
provider,
usageStat: params.usageStat,
});
// Include fallback provider profile so the fallback is attempted (not skipped as no-profile). // Include fallback provider profile so the fallback is attempted (not skipped as no-profile).
const store: AuthProfileStore = { const store: AuthProfileStore = {
...makeSingleProviderStore({ provider, usageStat: params.usageStat }), ...primaryStore,
profiles: { profiles: {
...makeSingleProviderStore({ provider, usageStat: params.usageStat }).profiles, ...primaryStore.profiles,
"fallback:default": { "fallback:default": {
type: "api_key", type: "api_key",
provider: "fallback", provider: "fallback",

View File

@@ -306,6 +306,76 @@ export const _probeThrottleInternals = {
resolveProbeThrottleKey, resolveProbeThrottleKey,
} as const; } as const;
type CooldownDecision =
| {
type: "skip";
reason: FailoverReason;
error: string;
}
| {
type: "attempt";
reason: FailoverReason;
markProbe: boolean;
};
function resolveCooldownDecision(params: {
candidate: ModelCandidate;
isPrimary: boolean;
requestedModel: boolean;
hasFallbackCandidates: boolean;
now: number;
probeThrottleKey: string;
authStore: ReturnType<typeof ensureAuthProfileStore>;
profileIds: string[];
}): CooldownDecision {
const shouldProbe = shouldProbePrimaryDuringCooldown({
isPrimary: params.isPrimary,
hasFallbackCandidates: params.hasFallbackCandidates,
now: params.now,
throttleKey: params.probeThrottleKey,
authStore: params.authStore,
profileIds: params.profileIds,
});
const inferredReason =
resolveProfilesUnavailableReason({
store: params.authStore,
profileIds: params.profileIds,
now: params.now,
}) ?? "rate_limit";
const isPersistentIssue =
inferredReason === "auth" ||
inferredReason === "auth_permanent" ||
inferredReason === "billing";
if (isPersistentIssue) {
return {
type: "skip",
reason: inferredReason,
error: `Provider ${params.candidate.provider} has ${inferredReason} issue (skipping all models)`,
};
}
// For primary: try when requested model or when probe allows.
// For same-provider fallbacks: only relax cooldown on rate_limit, which
// is commonly model-scoped and can recover on a sibling model.
const shouldAttemptDespiteCooldown =
(params.isPrimary && (!params.requestedModel || shouldProbe)) ||
(!params.isPrimary && inferredReason === "rate_limit");
if (!shouldAttemptDespiteCooldown) {
return {
type: "skip",
reason: inferredReason,
error: `Provider ${params.candidate.provider} is in cooldown (all profiles unavailable)`,
};
}
return {
type: "attempt",
reason: inferredReason,
markProbe: params.isPrimary && shouldProbe,
};
}
export async function runWithModelFallback<T>(params: { export async function runWithModelFallback<T>(params: {
cfg: OpenClawConfig | undefined; cfg: OpenClawConfig | undefined;
provider: string; provider: string;
@@ -347,51 +417,28 @@ export async function runWithModelFallback<T>(params: {
params.provider === candidate.provider && params.model === candidate.model; params.provider === candidate.provider && params.model === candidate.model;
const now = Date.now(); const now = Date.now();
const probeThrottleKey = resolveProbeThrottleKey(candidate.provider, params.agentDir); const probeThrottleKey = resolveProbeThrottleKey(candidate.provider, params.agentDir);
const shouldProbe = shouldProbePrimaryDuringCooldown({ const decision = resolveCooldownDecision({
candidate,
isPrimary, isPrimary,
requestedModel,
hasFallbackCandidates, hasFallbackCandidates,
now, now,
throttleKey: probeThrottleKey, probeThrottleKey,
authStore, authStore,
profileIds, profileIds,
}); });
const inferredReason = if (decision.type === "skip") {
resolveProfilesUnavailableReason({
store: authStore,
profileIds,
now,
}) ?? "rate_limit";
const isPersistentIssue =
inferredReason === "auth" ||
inferredReason === "auth_permanent" ||
inferredReason === "billing";
if (isPersistentIssue) {
attempts.push({ attempts.push({
provider: candidate.provider, provider: candidate.provider,
model: candidate.model, model: candidate.model,
error: `Provider ${candidate.provider} has ${inferredReason} issue (skipping all models)`, error: decision.error,
reason: inferredReason, reason: decision.reason,
}); });
continue; continue;
} }
// For primary: try when requested model or when probe allows. if (decision.markProbe) {
// For same-provider fallbacks: only relax cooldown on rate_limit, which
// is commonly model-scoped and can recover on a sibling model.
const shouldAttemptDespiteCooldown =
(isPrimary && (!requestedModel || shouldProbe)) ||
(!isPrimary && inferredReason === "rate_limit");
if (!shouldAttemptDespiteCooldown) {
attempts.push({
provider: candidate.provider,
model: candidate.model,
error: `Provider ${candidate.provider} is in cooldown (all profiles unavailable)`,
reason: inferredReason,
});
continue;
}
if (isPrimary && shouldProbe) {
lastProbeAttempt.set(probeThrottleKey, now); lastProbeAttempt.set(probeThrottleKey, now);
} }
} }