mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-08 12:41:23 +00:00
refactor(auth-profiles): centralize active-window logic + strengthen regression coverage
This commit is contained in:
@@ -83,6 +83,32 @@ describe("markAuthProfileFailure", () => {
|
|||||||
expectCooldownInRange(remainingMs, 0.8 * 60 * 60 * 1000, 1.2 * 60 * 60 * 1000);
|
expectCooldownInRange(remainingMs, 0.8 * 60 * 60 * 1000, 1.2 * 60 * 60 * 1000);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
it("keeps persisted cooldownUntil unchanged across mid-window retries", async () => {
|
||||||
|
await withAuthProfileStore(async ({ agentDir, store }) => {
|
||||||
|
await markAuthProfileFailure({
|
||||||
|
store,
|
||||||
|
profileId: "anthropic:default",
|
||||||
|
reason: "rate_limit",
|
||||||
|
agentDir,
|
||||||
|
});
|
||||||
|
|
||||||
|
const firstCooldownUntil = store.usageStats?.["anthropic:default"]?.cooldownUntil;
|
||||||
|
expect(typeof firstCooldownUntil).toBe("number");
|
||||||
|
|
||||||
|
await markAuthProfileFailure({
|
||||||
|
store,
|
||||||
|
profileId: "anthropic:default",
|
||||||
|
reason: "rate_limit",
|
||||||
|
agentDir,
|
||||||
|
});
|
||||||
|
|
||||||
|
const secondCooldownUntil = store.usageStats?.["anthropic:default"]?.cooldownUntil;
|
||||||
|
expect(secondCooldownUntil).toBe(firstCooldownUntil);
|
||||||
|
|
||||||
|
const reloaded = ensureAuthProfileStore(agentDir);
|
||||||
|
expect(reloaded.usageStats?.["anthropic:default"]?.cooldownUntil).toBe(firstCooldownUntil);
|
||||||
|
});
|
||||||
|
});
|
||||||
it("resets backoff counters outside the failure window", async () => {
|
it("resets backoff counters outside the failure window", async () => {
|
||||||
const agentDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-auth-"));
|
const agentDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-auth-"));
|
||||||
try {
|
try {
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { describe, expect, it, vi } from "vitest";
|
import { describe, expect, it, vi } from "vitest";
|
||||||
import type { AuthProfileStore } from "./types.js";
|
import type { AuthProfileStore, ProfileUsageStats } from "./types.js";
|
||||||
import {
|
import {
|
||||||
clearAuthProfileCooldown,
|
clearAuthProfileCooldown,
|
||||||
clearExpiredCooldowns,
|
clearExpiredCooldowns,
|
||||||
@@ -353,118 +353,111 @@ describe("markAuthProfileFailure — active windows do not extend on retry", ()
|
|||||||
// Regression for https://github.com/openclaw/openclaw/issues/23516
|
// Regression for https://github.com/openclaw/openclaw/issues/23516
|
||||||
// When all providers are at saturation backoff (60 min) and retries fire every 30 min,
|
// When all providers are at saturation backoff (60 min) and retries fire every 30 min,
|
||||||
// each retry was resetting cooldownUntil to now+60m, preventing recovery.
|
// each retry was resetting cooldownUntil to now+60m, preventing recovery.
|
||||||
|
type WindowStats = ProfileUsageStats;
|
||||||
|
|
||||||
it("keeps an active cooldownUntil unchanged on a mid-window retry", async () => {
|
async function markFailureAt(params: {
|
||||||
const now = 1_000_000;
|
store: ReturnType<typeof makeStore>;
|
||||||
// Profile already has 50 min remaining on its cooldown
|
now: number;
|
||||||
const existingCooldownUntil = now + 50 * 60 * 1000;
|
reason: "rate_limit" | "billing";
|
||||||
const store = makeStore({
|
}): Promise<void> {
|
||||||
"anthropic:default": {
|
vi.useFakeTimers();
|
||||||
cooldownUntil: existingCooldownUntil,
|
vi.setSystemTime(params.now);
|
||||||
errorCount: 3, // already at saturation (60-min backoff)
|
try {
|
||||||
|
await markAuthProfileFailure({
|
||||||
|
store: params.store,
|
||||||
|
profileId: "anthropic:default",
|
||||||
|
reason: params.reason,
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
vi.useRealTimers();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const activeWindowCases = [
|
||||||
|
{
|
||||||
|
label: "cooldownUntil",
|
||||||
|
reason: "rate_limit" as const,
|
||||||
|
buildUsageStats: (now: number): WindowStats => ({
|
||||||
|
cooldownUntil: now + 50 * 60 * 1000,
|
||||||
|
errorCount: 3,
|
||||||
lastFailureAt: now - 10 * 60 * 1000,
|
lastFailureAt: now - 10 * 60 * 1000,
|
||||||
},
|
}),
|
||||||
});
|
readUntil: (stats: WindowStats | undefined) => stats?.cooldownUntil,
|
||||||
|
},
|
||||||
vi.useFakeTimers();
|
{
|
||||||
vi.setSystemTime(now);
|
label: "disabledUntil",
|
||||||
try {
|
reason: "billing" as const,
|
||||||
await markAuthProfileFailure({
|
buildUsageStats: (now: number): WindowStats => ({
|
||||||
store,
|
disabledUntil: now + 20 * 60 * 60 * 1000,
|
||||||
profileId: "anthropic:default",
|
|
||||||
reason: "rate_limit",
|
|
||||||
});
|
|
||||||
} finally {
|
|
||||||
vi.useRealTimers();
|
|
||||||
}
|
|
||||||
|
|
||||||
const stats = store.usageStats?.["anthropic:default"];
|
|
||||||
expect(stats?.cooldownUntil).toBe(existingCooldownUntil);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("recomputes cooldownUntil when the previous window already expired", async () => {
|
|
||||||
const now = 1_000_000;
|
|
||||||
const expiredCooldownUntil = now - 60_000;
|
|
||||||
const store = makeStore({
|
|
||||||
"anthropic:default": {
|
|
||||||
cooldownUntil: expiredCooldownUntil,
|
|
||||||
errorCount: 3, // saturated 60-min backoff
|
|
||||||
lastFailureAt: now - 60_000,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
vi.useFakeTimers();
|
|
||||||
vi.setSystemTime(now);
|
|
||||||
try {
|
|
||||||
await markAuthProfileFailure({
|
|
||||||
store,
|
|
||||||
profileId: "anthropic:default",
|
|
||||||
reason: "rate_limit",
|
|
||||||
});
|
|
||||||
} finally {
|
|
||||||
vi.useRealTimers();
|
|
||||||
}
|
|
||||||
|
|
||||||
const stats = store.usageStats?.["anthropic:default"];
|
|
||||||
expect(stats?.cooldownUntil).toBe(now + 60 * 60 * 1000);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("keeps an active disabledUntil unchanged on a billing retry", async () => {
|
|
||||||
const now = 1_000_000;
|
|
||||||
// Profile already has 20 hours remaining on a billing disable
|
|
||||||
const existingDisabledUntil = now + 20 * 60 * 60 * 1000;
|
|
||||||
const store = makeStore({
|
|
||||||
"anthropic:default": {
|
|
||||||
disabledUntil: existingDisabledUntil,
|
|
||||||
disabledReason: "billing",
|
disabledReason: "billing",
|
||||||
errorCount: 5,
|
errorCount: 5,
|
||||||
failureCounts: { billing: 5 },
|
failureCounts: { billing: 5 },
|
||||||
lastFailureAt: now - 60_000,
|
lastFailureAt: now - 60_000,
|
||||||
},
|
}),
|
||||||
});
|
readUntil: (stats: WindowStats | undefined) => stats?.disabledUntil,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
vi.useFakeTimers();
|
for (const testCase of activeWindowCases) {
|
||||||
vi.setSystemTime(now);
|
it(`keeps active ${testCase.label} unchanged on retry`, async () => {
|
||||||
try {
|
const now = 1_000_000;
|
||||||
await markAuthProfileFailure({
|
const existingStats = testCase.buildUsageStats(now);
|
||||||
|
const existingUntil = testCase.readUntil(existingStats);
|
||||||
|
const store = makeStore({ "anthropic:default": existingStats });
|
||||||
|
|
||||||
|
await markFailureAt({
|
||||||
store,
|
store,
|
||||||
profileId: "anthropic:default",
|
now,
|
||||||
reason: "billing",
|
reason: testCase.reason,
|
||||||
});
|
});
|
||||||
} finally {
|
|
||||||
vi.useRealTimers();
|
|
||||||
}
|
|
||||||
|
|
||||||
const stats = store.usageStats?.["anthropic:default"];
|
const stats = store.usageStats?.["anthropic:default"];
|
||||||
expect(stats?.disabledUntil).toBe(existingDisabledUntil);
|
expect(testCase.readUntil(stats)).toBe(existingUntil);
|
||||||
});
|
});
|
||||||
|
}
|
||||||
|
|
||||||
it("recomputes disabledUntil when the previous billing window already expired", async () => {
|
const expiredWindowCases = [
|
||||||
const now = 1_000_000;
|
{
|
||||||
const expiredDisabledUntil = now - 60_000;
|
label: "cooldownUntil",
|
||||||
const store = makeStore({
|
reason: "rate_limit" as const,
|
||||||
"anthropic:default": {
|
buildUsageStats: (now: number): WindowStats => ({
|
||||||
disabledUntil: expiredDisabledUntil,
|
cooldownUntil: now - 60_000,
|
||||||
|
errorCount: 3,
|
||||||
|
lastFailureAt: now - 60_000,
|
||||||
|
}),
|
||||||
|
expectedUntil: (now: number) => now + 60 * 60 * 1000,
|
||||||
|
readUntil: (stats: WindowStats | undefined) => stats?.cooldownUntil,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: "disabledUntil",
|
||||||
|
reason: "billing" as const,
|
||||||
|
buildUsageStats: (now: number): WindowStats => ({
|
||||||
|
disabledUntil: now - 60_000,
|
||||||
disabledReason: "billing",
|
disabledReason: "billing",
|
||||||
errorCount: 5,
|
errorCount: 5,
|
||||||
failureCounts: { billing: 2 }, // next billing backoff: 20h
|
failureCounts: { billing: 2 },
|
||||||
lastFailureAt: now - 60_000,
|
lastFailureAt: now - 60_000,
|
||||||
},
|
}),
|
||||||
});
|
expectedUntil: (now: number) => now + 20 * 60 * 60 * 1000,
|
||||||
|
readUntil: (stats: WindowStats | undefined) => stats?.disabledUntil,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
vi.useFakeTimers();
|
for (const testCase of expiredWindowCases) {
|
||||||
vi.setSystemTime(now);
|
it(`recomputes ${testCase.label} after the previous window expires`, async () => {
|
||||||
try {
|
const now = 1_000_000;
|
||||||
await markAuthProfileFailure({
|
const store = makeStore({
|
||||||
store,
|
"anthropic:default": testCase.buildUsageStats(now),
|
||||||
profileId: "anthropic:default",
|
|
||||||
reason: "billing",
|
|
||||||
});
|
});
|
||||||
} finally {
|
|
||||||
vi.useRealTimers();
|
|
||||||
}
|
|
||||||
|
|
||||||
const stats = store.usageStats?.["anthropic:default"];
|
await markFailureAt({
|
||||||
expect(stats?.disabledUntil).toBe(now + 20 * 60 * 60 * 1000);
|
store,
|
||||||
});
|
now,
|
||||||
|
reason: testCase.reason,
|
||||||
|
});
|
||||||
|
|
||||||
|
const stats = store.usageStats?.["anthropic:default"];
|
||||||
|
expect(testCase.readUntil(stats)).toBe(testCase.expectedUntil(now));
|
||||||
|
});
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -256,6 +256,17 @@ export function resolveProfileUnusableUntilForDisplay(
|
|||||||
return resolveProfileUnusableUntil(stats);
|
return resolveProfileUnusableUntil(stats);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function keepActiveWindowOrRecompute(params: {
|
||||||
|
existingUntil: number | undefined;
|
||||||
|
now: number;
|
||||||
|
recomputedUntil: number;
|
||||||
|
}): number {
|
||||||
|
const { existingUntil, now, recomputedUntil } = params;
|
||||||
|
const hasActiveWindow =
|
||||||
|
typeof existingUntil === "number" && Number.isFinite(existingUntil) && existingUntil > now;
|
||||||
|
return hasActiveWindow ? existingUntil : recomputedUntil;
|
||||||
|
}
|
||||||
|
|
||||||
function computeNextProfileUsageStats(params: {
|
function computeNextProfileUsageStats(params: {
|
||||||
existing: ProfileUsageStats;
|
existing: ProfileUsageStats;
|
||||||
now: number;
|
now: number;
|
||||||
@@ -287,29 +298,23 @@ function computeNextProfileUsageStats(params: {
|
|||||||
baseMs: params.cfgResolved.billingBackoffMs,
|
baseMs: params.cfgResolved.billingBackoffMs,
|
||||||
maxMs: params.cfgResolved.billingMaxMs,
|
maxMs: params.cfgResolved.billingMaxMs,
|
||||||
});
|
});
|
||||||
const existingDisabledUntil = params.existing.disabledUntil;
|
|
||||||
const hasActiveDisabledWindow =
|
|
||||||
typeof existingDisabledUntil === "number" &&
|
|
||||||
Number.isFinite(existingDisabledUntil) &&
|
|
||||||
existingDisabledUntil > params.now;
|
|
||||||
// Keep active disable windows immutable so retries within the window cannot
|
// Keep active disable windows immutable so retries within the window cannot
|
||||||
// extend recovery time indefinitely.
|
// extend recovery time indefinitely.
|
||||||
updatedStats.disabledUntil = hasActiveDisabledWindow
|
updatedStats.disabledUntil = keepActiveWindowOrRecompute({
|
||||||
? existingDisabledUntil
|
existingUntil: params.existing.disabledUntil,
|
||||||
: params.now + backoffMs;
|
now: params.now,
|
||||||
|
recomputedUntil: params.now + backoffMs,
|
||||||
|
});
|
||||||
updatedStats.disabledReason = "billing";
|
updatedStats.disabledReason = "billing";
|
||||||
} else {
|
} else {
|
||||||
const backoffMs = calculateAuthProfileCooldownMs(nextErrorCount);
|
const backoffMs = calculateAuthProfileCooldownMs(nextErrorCount);
|
||||||
const existingCooldownUntil = params.existing.cooldownUntil;
|
|
||||||
const hasActiveCooldownWindow =
|
|
||||||
typeof existingCooldownUntil === "number" &&
|
|
||||||
Number.isFinite(existingCooldownUntil) &&
|
|
||||||
existingCooldownUntil > params.now;
|
|
||||||
// Keep active cooldown windows immutable so retries within the window
|
// Keep active cooldown windows immutable so retries within the window
|
||||||
// cannot push recovery further out.
|
// cannot push recovery further out.
|
||||||
updatedStats.cooldownUntil = hasActiveCooldownWindow
|
updatedStats.cooldownUntil = keepActiveWindowOrRecompute({
|
||||||
? existingCooldownUntil
|
existingUntil: params.existing.cooldownUntil,
|
||||||
: params.now + backoffMs;
|
now: params.now,
|
||||||
|
recomputedUntil: params.now + backoffMs,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
return updatedStats;
|
return updatedStats;
|
||||||
|
|||||||
Reference in New Issue
Block a user