fix(auth): distinguish revoked API keys from transient auth errors (#25754)

Merged via /review-pr -> /prepare-pr -> /merge-pr.

Prepared head SHA: 8f9c07a200
Co-authored-by: rrenamed <87486610+rrenamed@users.noreply.github.com>
Co-authored-by: gumadeiras <5599352+gumadeiras@users.noreply.github.com>
Reviewed-by: @gumadeiras
This commit is contained in:
Aleksandrs Tihenko
2026-02-26 02:47:16 +02:00
committed by GitHub
parent f312222159
commit c0026274d9
15 changed files with 247 additions and 18 deletions

View File

@@ -114,6 +114,22 @@ describe("markAuthProfileFailure", () => {
expect(reloaded.usageStats?.["anthropic:default"]?.cooldownUntil).toBe(firstCooldownUntil);
});
});
it("disables auth_permanent failures via disabledUntil (like billing)", async () => {
await withAuthProfileStore(async ({ agentDir, store }) => {
await markAuthProfileFailure({
store,
profileId: "anthropic:default",
reason: "auth_permanent",
agentDir,
});
const stats = store.usageStats?.["anthropic:default"];
expect(typeof stats?.disabledUntil).toBe("number");
expect(stats?.disabledReason).toBe("auth_permanent");
// Should NOT set cooldownUntil (that's for transient errors)
expect(stats?.cooldownUntil).toBeUndefined();
});
});
it("resets backoff counters outside the failure window", async () => {
const agentDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-auth-"));
try {

View File

@@ -34,6 +34,7 @@ export type AuthProfileCredential = ApiKeyCredential | TokenCredential | OAuthCr
export type AuthProfileFailureReason =
| "auth"
| "auth_permanent"
| "format"
| "rate_limit"
| "billing"

View File

@@ -141,6 +141,24 @@ describe("resolveProfilesUnavailableReason", () => {
).toBe("billing");
});
it("returns auth_permanent for active permanent auth disables", () => {
const now = Date.now();
const store = makeStore({
"anthropic:default": {
disabledUntil: now + 60_000,
disabledReason: "auth_permanent",
},
});
expect(
resolveProfilesUnavailableReason({
store,
profileIds: ["anthropic:default"],
now,
}),
).toBe("auth_permanent");
});
it("uses recorded non-rate-limit failure counts for active cooldown windows", () => {
const now = Date.now();
const store = makeStore({
@@ -490,7 +508,7 @@ describe("markAuthProfileFailure — active windows do not extend on retry", ()
async function markFailureAt(params: {
store: ReturnType<typeof makeStore>;
now: number;
reason: "rate_limit" | "billing";
reason: "rate_limit" | "billing" | "auth_permanent";
}): Promise<void> {
vi.useFakeTimers();
vi.setSystemTime(params.now);
@@ -528,6 +546,18 @@ describe("markAuthProfileFailure — active windows do not extend on retry", ()
}),
readUntil: (stats: WindowStats | undefined) => stats?.disabledUntil,
},
{
label: "disabledUntil(auth_permanent)",
reason: "auth_permanent" as const,
buildUsageStats: (now: number): WindowStats => ({
disabledUntil: now + 20 * 60 * 60 * 1000,
disabledReason: "auth_permanent",
errorCount: 5,
failureCounts: { auth_permanent: 5 },
lastFailureAt: now - 60_000,
}),
readUntil: (stats: WindowStats | undefined) => stats?.disabledUntil,
},
];
for (const testCase of activeWindowCases) {
@@ -573,6 +603,19 @@ describe("markAuthProfileFailure — active windows do not extend on retry", ()
expectedUntil: (now: number) => now + 20 * 60 * 60 * 1000,
readUntil: (stats: WindowStats | undefined) => stats?.disabledUntil,
},
{
label: "disabledUntil(auth_permanent)",
reason: "auth_permanent" as const,
buildUsageStats: (now: number): WindowStats => ({
disabledUntil: now - 60_000,
disabledReason: "auth_permanent",
errorCount: 5,
failureCounts: { auth_permanent: 2 },
lastFailureAt: now - 60_000,
}),
expectedUntil: (now: number) => now + 20 * 60 * 60 * 1000,
readUntil: (stats: WindowStats | undefined) => stats?.disabledUntil,
},
];
for (const testCase of expiredWindowCases) {

View File

@@ -4,6 +4,7 @@ import { saveAuthProfileStore, updateAuthProfileStoreWithLock } from "./store.js
import type { AuthProfileFailureReason, AuthProfileStore, ProfileUsageStats } from "./types.js";
const FAILURE_REASON_PRIORITY: AuthProfileFailureReason[] = [
"auth_permanent",
"auth",
"billing",
"format",
@@ -394,8 +395,8 @@ function computeNextProfileUsageStats(params: {
lastFailureAt: params.now,
};
if (params.reason === "billing") {
const billingCount = failureCounts.billing ?? 1;
if (params.reason === "billing" || params.reason === "auth_permanent") {
const billingCount = failureCounts[params.reason] ?? 1;
const backoffMs = calculateAuthProfileBillingDisableMsWithConfig({
errorCount: billingCount,
baseMs: params.cfgResolved.billingBackoffMs,
@@ -408,7 +409,7 @@ function computeNextProfileUsageStats(params: {
now: params.now,
recomputedUntil: params.now + backoffMs,
});
updatedStats.disabledReason = "billing";
updatedStats.disabledReason = params.reason;
} else {
const backoffMs = calculateAuthProfileCooldownMs(nextErrorCount);
// Keep active cooldown windows immutable so retries within the window
@@ -424,8 +425,9 @@ function computeNextProfileUsageStats(params: {
}
/**
* Mark a profile as failed for a specific reason. Billing failures are treated
* as "disabled" (longer backoff) vs the regular cooldown window.
* Mark a profile as failed for a specific reason. Billing and permanent-auth
* failures are treated as "disabled" (longer backoff) vs the regular cooldown
* window.
*/
export async function markAuthProfileFailure(params: {
store: AuthProfileStore;

View File

@@ -4,6 +4,7 @@ import {
describeFailoverError,
isTimeoutError,
resolveFailoverReasonFromError,
resolveFailoverStatus,
} from "./failover-error.js";
describe("failover-error", () => {
@@ -69,6 +70,36 @@ describe("failover-error", () => {
expect(err?.status).toBe(400);
});
it("401/403 with generic message still returns auth (backward compat)", () => {
expect(resolveFailoverReasonFromError({ status: 401, message: "Unauthorized" })).toBe("auth");
expect(resolveFailoverReasonFromError({ status: 403, message: "Forbidden" })).toBe("auth");
});
it("401 with permanent auth message returns auth_permanent", () => {
expect(resolveFailoverReasonFromError({ status: 401, message: "invalid_api_key" })).toBe(
"auth_permanent",
);
});
it("403 with revoked key message returns auth_permanent", () => {
expect(resolveFailoverReasonFromError({ status: 403, message: "api key revoked" })).toBe(
"auth_permanent",
);
});
it("resolveFailoverStatus maps auth_permanent to 403", () => {
expect(resolveFailoverStatus("auth_permanent")).toBe(403);
});
it("coerces permanent auth error with correct reason", () => {
const err = coerceToFailoverError(
{ status: 401, message: "invalid_api_key" },
{ provider: "anthropic", model: "claude-opus-4-6" },
);
expect(err?.reason).toBe("auth_permanent");
expect(err?.provider).toBe("anthropic");
});
it("describes non-Error values consistently", () => {
const described = describeFailoverError(123);
expect(described.message).toBe("123");

View File

@@ -1,4 +1,8 @@
import { classifyFailoverReason, type FailoverReason } from "./pi-embedded-helpers.js";
import {
classifyFailoverReason,
isAuthPermanentErrorMessage,
type FailoverReason,
} from "./pi-embedded-helpers.js";
const TIMEOUT_HINT_RE =
/timeout|timed out|deadline exceeded|context deadline exceeded|stop reason:\s*abort|reason:\s*abort|unhandled stop reason:\s*abort/i;
@@ -47,6 +51,8 @@ export function resolveFailoverStatus(reason: FailoverReason): number | undefine
return 429;
case "auth":
return 401;
case "auth_permanent":
return 403;
case "timeout":
return 408;
case "format":
@@ -158,6 +164,10 @@ export function resolveFailoverReasonFromError(err: unknown): FailoverReason | n
return "rate_limit";
}
if (status === 401 || status === 403) {
const msg = getErrorMessage(err);
if (msg && isAuthPermanentErrorMessage(msg)) {
return "auth_permanent";
}
return "auth";
}
if (status === 408) {

View File

@@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";
import {
classifyFailoverReason,
isAuthErrorMessage,
isAuthPermanentErrorMessage,
isBillingErrorMessage,
isCloudCodeAssistFormatError,
isCloudflareOrHtmlErrorPage,
@@ -16,6 +17,39 @@ import {
parseImageSizeError,
} from "./pi-embedded-helpers.js";
describe("isAuthPermanentErrorMessage", () => {
it("matches permanent auth failure patterns", () => {
const samples = [
"invalid_api_key",
"api key revoked",
"api key deactivated",
"key has been disabled",
"key has been revoked",
"account has been deactivated",
"could not authenticate api key",
"could not validate credentials",
"API_KEY_REVOKED",
"api_key_deleted",
];
for (const sample of samples) {
expect(isAuthPermanentErrorMessage(sample)).toBe(true);
}
});
it("does not match transient auth errors", () => {
const samples = [
"unauthorized",
"invalid token",
"authentication failed",
"forbidden",
"access denied",
"token has expired",
];
for (const sample of samples) {
expect(isAuthPermanentErrorMessage(sample)).toBe(false);
}
});
});
describe("isAuthErrorMessage", () => {
it("matches credential validation errors", () => {
const samples = [
@@ -480,6 +514,12 @@ describe("classifyFailoverReason", () => {
),
).toBe("rate_limit");
});
it("classifies permanent auth errors as auth_permanent", () => {
expect(classifyFailoverReason("invalid_api_key")).toBe("auth_permanent");
expect(classifyFailoverReason("Your api key has been revoked")).toBe("auth_permanent");
expect(classifyFailoverReason("key has been disabled")).toBe("auth_permanent");
expect(classifyFailoverReason("account has been deactivated")).toBe("auth_permanent");
});
it("classifies JSON api_error internal server failures as timeout", () => {
expect(
classifyFailoverReason(

View File

@@ -16,6 +16,7 @@ export {
getApiErrorPayloadFingerprint,
isAuthAssistantError,
isAuthErrorMessage,
isAuthPermanentErrorMessage,
isModelNotFoundErrorMessage,
isBillingAssistantError,
parseApiErrorInfo,

View File

@@ -649,6 +649,14 @@ const ERROR_PATTERNS = {
"plans & billing",
"insufficient balance",
],
authPermanent: [
/api[_ ]?key[_ ]?(?:revoked|invalid|deactivated|deleted)/i,
"invalid_api_key",
"key has been disabled",
"key has been revoked",
"account has been deactivated",
/could not (?:authenticate|validate).*(?:api[_ ]?key|credentials)/i,
],
auth: [
/invalid[_ ]?api[_ ]?key/,
"incorrect api key",
@@ -755,6 +763,10 @@ export function isBillingAssistantError(msg: AssistantMessage | undefined): bool
return isBillingErrorMessage(msg.errorMessage ?? "");
}
export function isAuthPermanentErrorMessage(raw: string): boolean {
return matchesErrorPatterns(raw, ERROR_PATTERNS.authPermanent);
}
export function isAuthErrorMessage(raw: string): boolean {
return matchesErrorPatterns(raw, ERROR_PATTERNS.auth);
}
@@ -899,6 +911,9 @@ export function classifyFailoverReason(raw: string): FailoverReason | null {
if (isTimeoutErrorMessage(raw)) {
return "timeout";
}
if (isAuthPermanentErrorMessage(raw)) {
return "auth_permanent";
}
if (isAuthErrorMessage(raw)) {
return "auth";
}

View File

@@ -2,6 +2,7 @@ export type EmbeddedContextFile = { path: string; content: string };
export type FailoverReason =
| "auth"
| "auth_permanent"
| "format"
| "rate_limit"
| "billing"