refactor(auth-profiles): centralize active-window logic + strengthen regression coverage

This commit is contained in:
Peter Steinberger
2026-02-22 13:22:55 +01:00
parent 1152b25866
commit 07527e22ce
3 changed files with 138 additions and 114 deletions

View File

@@ -83,6 +83,32 @@ describe("markAuthProfileFailure", () => {
expectCooldownInRange(remainingMs, 0.8 * 60 * 60 * 1000, 1.2 * 60 * 60 * 1000); expectCooldownInRange(remainingMs, 0.8 * 60 * 60 * 1000, 1.2 * 60 * 60 * 1000);
}); });
}); });
it("keeps persisted cooldownUntil unchanged across mid-window retries", async () => {
await withAuthProfileStore(async ({ agentDir, store }) => {
await markAuthProfileFailure({
store,
profileId: "anthropic:default",
reason: "rate_limit",
agentDir,
});
const firstCooldownUntil = store.usageStats?.["anthropic:default"]?.cooldownUntil;
expect(typeof firstCooldownUntil).toBe("number");
await markAuthProfileFailure({
store,
profileId: "anthropic:default",
reason: "rate_limit",
agentDir,
});
const secondCooldownUntil = store.usageStats?.["anthropic:default"]?.cooldownUntil;
expect(secondCooldownUntil).toBe(firstCooldownUntil);
const reloaded = ensureAuthProfileStore(agentDir);
expect(reloaded.usageStats?.["anthropic:default"]?.cooldownUntil).toBe(firstCooldownUntil);
});
});
it("resets backoff counters outside the failure window", async () => { it("resets backoff counters outside the failure window", async () => {
const agentDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-auth-")); const agentDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-auth-"));
try { try {

View File

@@ -1,5 +1,5 @@
import { describe, expect, it, vi } from "vitest"; import { describe, expect, it, vi } from "vitest";
import type { AuthProfileStore } from "./types.js"; import type { AuthProfileStore, ProfileUsageStats } from "./types.js";
import { import {
clearAuthProfileCooldown, clearAuthProfileCooldown,
clearExpiredCooldowns, clearExpiredCooldowns,
@@ -353,118 +353,111 @@ describe("markAuthProfileFailure — active windows do not extend on retry", ()
// Regression for https://github.com/openclaw/openclaw/issues/23516 // Regression for https://github.com/openclaw/openclaw/issues/23516
// When all providers are at saturation backoff (60 min) and retries fire every 30 min, // When all providers are at saturation backoff (60 min) and retries fire every 30 min,
// each retry was resetting cooldownUntil to now+60m, preventing recovery. // each retry was resetting cooldownUntil to now+60m, preventing recovery.
type WindowStats = ProfileUsageStats;
it("keeps an active cooldownUntil unchanged on a mid-window retry", async () => { async function markFailureAt(params: {
const now = 1_000_000; store: ReturnType<typeof makeStore>;
// Profile already has 50 min remaining on its cooldown now: number;
const existingCooldownUntil = now + 50 * 60 * 1000; reason: "rate_limit" | "billing";
const store = makeStore({ }): Promise<void> {
"anthropic:default": { vi.useFakeTimers();
cooldownUntil: existingCooldownUntil, vi.setSystemTime(params.now);
errorCount: 3, // already at saturation (60-min backoff) try {
await markAuthProfileFailure({
store: params.store,
profileId: "anthropic:default",
reason: params.reason,
});
} finally {
vi.useRealTimers();
}
}
const activeWindowCases = [
{
label: "cooldownUntil",
reason: "rate_limit" as const,
buildUsageStats: (now: number): WindowStats => ({
cooldownUntil: now + 50 * 60 * 1000,
errorCount: 3,
lastFailureAt: now - 10 * 60 * 1000, lastFailureAt: now - 10 * 60 * 1000,
}, }),
}); readUntil: (stats: WindowStats | undefined) => stats?.cooldownUntil,
},
vi.useFakeTimers(); {
vi.setSystemTime(now); label: "disabledUntil",
try { reason: "billing" as const,
await markAuthProfileFailure({ buildUsageStats: (now: number): WindowStats => ({
store, disabledUntil: now + 20 * 60 * 60 * 1000,
profileId: "anthropic:default",
reason: "rate_limit",
});
} finally {
vi.useRealTimers();
}
const stats = store.usageStats?.["anthropic:default"];
expect(stats?.cooldownUntil).toBe(existingCooldownUntil);
});
it("recomputes cooldownUntil when the previous window already expired", async () => {
const now = 1_000_000;
const expiredCooldownUntil = now - 60_000;
const store = makeStore({
"anthropic:default": {
cooldownUntil: expiredCooldownUntil,
errorCount: 3, // saturated 60-min backoff
lastFailureAt: now - 60_000,
},
});
vi.useFakeTimers();
vi.setSystemTime(now);
try {
await markAuthProfileFailure({
store,
profileId: "anthropic:default",
reason: "rate_limit",
});
} finally {
vi.useRealTimers();
}
const stats = store.usageStats?.["anthropic:default"];
expect(stats?.cooldownUntil).toBe(now + 60 * 60 * 1000);
});
it("keeps an active disabledUntil unchanged on a billing retry", async () => {
const now = 1_000_000;
// Profile already has 20 hours remaining on a billing disable
const existingDisabledUntil = now + 20 * 60 * 60 * 1000;
const store = makeStore({
"anthropic:default": {
disabledUntil: existingDisabledUntil,
disabledReason: "billing", disabledReason: "billing",
errorCount: 5, errorCount: 5,
failureCounts: { billing: 5 }, failureCounts: { billing: 5 },
lastFailureAt: now - 60_000, lastFailureAt: now - 60_000,
}, }),
}); readUntil: (stats: WindowStats | undefined) => stats?.disabledUntil,
},
];
vi.useFakeTimers(); for (const testCase of activeWindowCases) {
vi.setSystemTime(now); it(`keeps active ${testCase.label} unchanged on retry`, async () => {
try { const now = 1_000_000;
await markAuthProfileFailure({ const existingStats = testCase.buildUsageStats(now);
const existingUntil = testCase.readUntil(existingStats);
const store = makeStore({ "anthropic:default": existingStats });
await markFailureAt({
store, store,
profileId: "anthropic:default", now,
reason: "billing", reason: testCase.reason,
}); });
} finally {
vi.useRealTimers();
}
const stats = store.usageStats?.["anthropic:default"]; const stats = store.usageStats?.["anthropic:default"];
expect(stats?.disabledUntil).toBe(existingDisabledUntil); expect(testCase.readUntil(stats)).toBe(existingUntil);
}); });
}
it("recomputes disabledUntil when the previous billing window already expired", async () => { const expiredWindowCases = [
const now = 1_000_000; {
const expiredDisabledUntil = now - 60_000; label: "cooldownUntil",
const store = makeStore({ reason: "rate_limit" as const,
"anthropic:default": { buildUsageStats: (now: number): WindowStats => ({
disabledUntil: expiredDisabledUntil, cooldownUntil: now - 60_000,
errorCount: 3,
lastFailureAt: now - 60_000,
}),
expectedUntil: (now: number) => now + 60 * 60 * 1000,
readUntil: (stats: WindowStats | undefined) => stats?.cooldownUntil,
},
{
label: "disabledUntil",
reason: "billing" as const,
buildUsageStats: (now: number): WindowStats => ({
disabledUntil: now - 60_000,
disabledReason: "billing", disabledReason: "billing",
errorCount: 5, errorCount: 5,
failureCounts: { billing: 2 }, // next billing backoff: 20h failureCounts: { billing: 2 },
lastFailureAt: now - 60_000, lastFailureAt: now - 60_000,
}, }),
}); expectedUntil: (now: number) => now + 20 * 60 * 60 * 1000,
readUntil: (stats: WindowStats | undefined) => stats?.disabledUntil,
},
];
vi.useFakeTimers(); for (const testCase of expiredWindowCases) {
vi.setSystemTime(now); it(`recomputes ${testCase.label} after the previous window expires`, async () => {
try { const now = 1_000_000;
await markAuthProfileFailure({ const store = makeStore({
store, "anthropic:default": testCase.buildUsageStats(now),
profileId: "anthropic:default",
reason: "billing",
}); });
} finally {
vi.useRealTimers();
}
const stats = store.usageStats?.["anthropic:default"]; await markFailureAt({
expect(stats?.disabledUntil).toBe(now + 20 * 60 * 60 * 1000); store,
}); now,
reason: testCase.reason,
});
const stats = store.usageStats?.["anthropic:default"];
expect(testCase.readUntil(stats)).toBe(testCase.expectedUntil(now));
});
}
}); });

View File

@@ -256,6 +256,17 @@ export function resolveProfileUnusableUntilForDisplay(
return resolveProfileUnusableUntil(stats); return resolveProfileUnusableUntil(stats);
} }
function keepActiveWindowOrRecompute(params: {
existingUntil: number | undefined;
now: number;
recomputedUntil: number;
}): number {
const { existingUntil, now, recomputedUntil } = params;
const hasActiveWindow =
typeof existingUntil === "number" && Number.isFinite(existingUntil) && existingUntil > now;
return hasActiveWindow ? existingUntil : recomputedUntil;
}
function computeNextProfileUsageStats(params: { function computeNextProfileUsageStats(params: {
existing: ProfileUsageStats; existing: ProfileUsageStats;
now: number; now: number;
@@ -287,29 +298,23 @@ function computeNextProfileUsageStats(params: {
baseMs: params.cfgResolved.billingBackoffMs, baseMs: params.cfgResolved.billingBackoffMs,
maxMs: params.cfgResolved.billingMaxMs, maxMs: params.cfgResolved.billingMaxMs,
}); });
const existingDisabledUntil = params.existing.disabledUntil;
const hasActiveDisabledWindow =
typeof existingDisabledUntil === "number" &&
Number.isFinite(existingDisabledUntil) &&
existingDisabledUntil > params.now;
// Keep active disable windows immutable so retries within the window cannot // Keep active disable windows immutable so retries within the window cannot
// extend recovery time indefinitely. // extend recovery time indefinitely.
updatedStats.disabledUntil = hasActiveDisabledWindow updatedStats.disabledUntil = keepActiveWindowOrRecompute({
? existingDisabledUntil existingUntil: params.existing.disabledUntil,
: params.now + backoffMs; now: params.now,
recomputedUntil: params.now + backoffMs,
});
updatedStats.disabledReason = "billing"; updatedStats.disabledReason = "billing";
} else { } else {
const backoffMs = calculateAuthProfileCooldownMs(nextErrorCount); const backoffMs = calculateAuthProfileCooldownMs(nextErrorCount);
const existingCooldownUntil = params.existing.cooldownUntil;
const hasActiveCooldownWindow =
typeof existingCooldownUntil === "number" &&
Number.isFinite(existingCooldownUntil) &&
existingCooldownUntil > params.now;
// Keep active cooldown windows immutable so retries within the window // Keep active cooldown windows immutable so retries within the window
// cannot push recovery further out. // cannot push recovery further out.
updatedStats.cooldownUntil = hasActiveCooldownWindow updatedStats.cooldownUntil = keepActiveWindowOrRecompute({
? existingCooldownUntil existingUntil: params.existing.cooldownUntil,
: params.now + backoffMs; now: params.now,
recomputedUntil: params.now + backoffMs,
});
} }
return updatedStats; return updatedStats;