chore: merge origin/main into main

src/agents/auth-profiles/usage.test.ts

@@ -1,9 +1,10 @@
 import { describe, expect, it, vi } from "vitest";
-import type { AuthProfileStore } from "./types.js";
+import type { AuthProfileStore, ProfileUsageStats } from "./types.js";
 import {
   clearAuthProfileCooldown,
   clearExpiredCooldowns,
   isProfileInCooldown,
+  markAuthProfileFailure,
   resolveProfileUnusableUntil,
 } from "./usage.js";
 
@@ -347,3 +348,116 @@ describe("clearAuthProfileCooldown", () => {
     expect(store.usageStats).toBeUndefined();
   });
 });
+
+describe("markAuthProfileFailure — active windows do not extend on retry", () => {
+  // Regression for https://github.com/openclaw/openclaw/issues/23516
+  // When all providers are at saturation backoff (60 min) and retries fire every 30 min,
+  // each retry was resetting cooldownUntil to now+60m, preventing recovery.
+  type WindowStats = ProfileUsageStats;
+
+  async function markFailureAt(params: {
+    store: ReturnType<typeof makeStore>;
+    now: number;
+    reason: "rate_limit" | "billing";
+  }): Promise<void> {
+    vi.useFakeTimers();
+    vi.setSystemTime(params.now);
+    try {
+      await markAuthProfileFailure({
+        store: params.store,
+        profileId: "anthropic:default",
+        reason: params.reason,
+      });
+    } finally {
+      vi.useRealTimers();
+    }
+  }
+
+  const activeWindowCases = [
+    {
+      label: "cooldownUntil",
+      reason: "rate_limit" as const,
+      buildUsageStats: (now: number): WindowStats => ({
+        cooldownUntil: now + 50 * 60 * 1000,
+        errorCount: 3,
+        lastFailureAt: now - 10 * 60 * 1000,
+      }),
+      readUntil: (stats: WindowStats | undefined) => stats?.cooldownUntil,
+    },
+    {
+      label: "disabledUntil",
+      reason: "billing" as const,
+      buildUsageStats: (now: number): WindowStats => ({
+        disabledUntil: now + 20 * 60 * 60 * 1000,
+        disabledReason: "billing",
+        errorCount: 5,
+        failureCounts: { billing: 5 },
+        lastFailureAt: now - 60_000,
+      }),
+      readUntil: (stats: WindowStats | undefined) => stats?.disabledUntil,
+    },
+  ];
+
+  for (const testCase of activeWindowCases) {
+    it(`keeps active ${testCase.label} unchanged on retry`, async () => {
+      const now = 1_000_000;
+      const existingStats = testCase.buildUsageStats(now);
+      const existingUntil = testCase.readUntil(existingStats);
+      const store = makeStore({ "anthropic:default": existingStats });
+
+      await markFailureAt({
+        store,
+        now,
+        reason: testCase.reason,
+      });
+
+      const stats = store.usageStats?.["anthropic:default"];
+      expect(testCase.readUntil(stats)).toBe(existingUntil);
+    });
+  }
+
+  const expiredWindowCases = [
+    {
+      label: "cooldownUntil",
+      reason: "rate_limit" as const,
+      buildUsageStats: (now: number): WindowStats => ({
+        cooldownUntil: now - 60_000,
+        errorCount: 3,
+        lastFailureAt: now - 60_000,
+      }),
+      expectedUntil: (now: number) => now + 60 * 60 * 1000,
+      readUntil: (stats: WindowStats | undefined) => stats?.cooldownUntil,
+    },
+    {
+      label: "disabledUntil",
+      reason: "billing" as const,
+      buildUsageStats: (now: number): WindowStats => ({
+        disabledUntil: now - 60_000,
+        disabledReason: "billing",
+        errorCount: 5,
+        failureCounts: { billing: 2 },
+        lastFailureAt: now - 60_000,
+      }),
+      expectedUntil: (now: number) => now + 20 * 60 * 60 * 1000,
+      readUntil: (stats: WindowStats | undefined) => stats?.disabledUntil,
+    },
+  ];
+
+  for (const testCase of expiredWindowCases) {
+    it(`recomputes ${testCase.label} after the previous window expires`, async () => {
+      const now = 1_000_000;
+      const store = makeStore({
+        "anthropic:default": testCase.buildUsageStats(now),
+      });
+
+      await markFailureAt({
+        store,
+        now,
+        reason: testCase.reason,
+      });
+
+      const stats = store.usageStats?.["anthropic:default"];
+      expect(testCase.readUntil(stats)).toBe(testCase.expectedUntil(now));
+    });
+  }
+});

src/agents/auth-profiles/usage.ts

@@ -256,6 +256,17 @@ export function resolveProfileUnusableUntilForDisplay(
   return resolveProfileUnusableUntil(stats);
 }
 
+function keepActiveWindowOrRecompute(params: {
+  existingUntil: number | undefined;
+  now: number;
+  recomputedUntil: number;
+}): number {
+  const { existingUntil, now, recomputedUntil } = params;
+  const hasActiveWindow =
+    typeof existingUntil === "number" && Number.isFinite(existingUntil) && existingUntil > now;
+  return hasActiveWindow ? existingUntil : recomputedUntil;
+}
+
 function computeNextProfileUsageStats(params: {
   existing: ProfileUsageStats;
   now: number;
@@ -287,11 +298,23 @@ function computeNextProfileUsageStats(params: {
       baseMs: params.cfgResolved.billingBackoffMs,
       maxMs: params.cfgResolved.billingMaxMs,
     });
-    updatedStats.disabledUntil = params.now + backoffMs;
+    // Keep active disable windows immutable so retries within the window cannot
+    // extend recovery time indefinitely.
+    updatedStats.disabledUntil = keepActiveWindowOrRecompute({
+      existingUntil: params.existing.disabledUntil,
+      now: params.now,
+      recomputedUntil: params.now + backoffMs,
+    });
     updatedStats.disabledReason = "billing";
   } else {
     const backoffMs = calculateAuthProfileCooldownMs(nextErrorCount);
-    updatedStats.cooldownUntil = params.now + backoffMs;
+    // Keep active cooldown windows immutable so retries within the window
+    // cannot push recovery further out.
+    updatedStats.cooldownUntil = keepActiveWindowOrRecompute({
+      existingUntil: params.existing.cooldownUntil,
+      now: params.now,
+      recomputedUntil: params.now + backoffMs,
+    });
   }
 
   return updatedStats;