fix: trigger fallback on auth profile exhaustion

src/agents/pi-embedded-runner.run-embedded-pi-agent.auth-profile-rotation.test.ts

@@ -63,12 +63,12 @@ const makeAttempt = (overrides: Partial<EmbeddedRunAttemptResult>): EmbeddedRunA
   ...overrides,
 });
 
-const makeConfig = (): ClawdbotConfig =>
+const makeConfig = (opts?: { fallbacks?: string[]; apiKey?: string }): ClawdbotConfig =>
   ({
     agents: {
       defaults: {
         model: {
-          fallbacks: [],
+          fallbacks: opts?.fallbacks ?? [],
         },
       },
     },
@@ -76,7 +76,7 @@ const makeConfig = (): ClawdbotConfig =>
       providers: {
         openai: {
           api: "openai-responses",
-          apiKey: "sk-test",
+          apiKey: opts?.apiKey ?? "sk-test",
           baseUrl: "https://example.com",
           models: [
             {
@@ -94,7 +94,13 @@ const makeConfig = (): ClawdbotConfig =>
     },
   }) satisfies ClawdbotConfig;
 
-const writeAuthStore = async (agentDir: string, opts?: { includeAnthropic?: boolean }) => {
+const writeAuthStore = async (
+  agentDir: string,
+  opts?: {
+    includeAnthropic?: boolean;
+    usageStats?: Record<string, { lastUsed?: number; cooldownUntil?: number }>;
+  },
+) => {
   const authPath = path.join(agentDir, "auth-profiles.json");
   const payload = {
     version: 1,
@@ -105,10 +111,12 @@ const writeAuthStore = async (agentDir: string, opts?: { includeAnthropic?: bool
         ? { "anthropic:default": { type: "api_key", provider: "anthropic", key: "sk-anth" } }
         : {}),
     },
-    usageStats: {
-      "openai:p1": { lastUsed: 1 },
-      "openai:p2": { lastUsed: 2 },
-    },
+    usageStats:
+      opts?.usageStats ??
+      ({
+        "openai:p1": { lastUsed: 1 },
+        "openai:p2": { lastUsed: 2 },
+      } as Record<string, { lastUsed?: number }>),
   };
   await fs.writeFile(authPath, JSON.stringify(payload));
 };
@@ -384,6 +392,92 @@ describe("runEmbeddedPiAgent auth profile rotation", () => {
     }
   });
 
+  it("fails over when all profiles are in cooldown and fallbacks are configured", async () => {
+    vi.useFakeTimers();
+    try {
+      const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "clawdbot-agent-"));
+      const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "clawdbot-workspace-"));
+      const now = Date.now();
+      vi.setSystemTime(now);
+
+      try {
+        await writeAuthStore(agentDir, {
+          usageStats: {
+            "openai:p1": { lastUsed: 1, cooldownUntil: now + 60 * 60 * 1000 },
+            "openai:p2": { lastUsed: 2, cooldownUntil: now + 60 * 60 * 1000 },
+          },
+        });
+
+        await expect(
+          runEmbeddedPiAgent({
+            sessionId: "session:test",
+            sessionKey: "agent:test:cooldown-failover",
+            sessionFile: path.join(workspaceDir, "session.jsonl"),
+            workspaceDir,
+            agentDir,
+            config: makeConfig({ fallbacks: ["openai/mock-2"] }),
+            prompt: "hello",
+            provider: "openai",
+            model: "mock-1",
+            authProfileIdSource: "auto",
+            timeoutMs: 5_000,
+            runId: "run:cooldown-failover",
+          }),
+        ).rejects.toMatchObject({
+          name: "FailoverError",
+          reason: "rate_limit",
+          provider: "openai",
+          model: "mock-1",
+        });
+
+        expect(runEmbeddedAttemptMock).not.toHaveBeenCalled();
+      } finally {
+        await fs.rm(agentDir, { recursive: true, force: true });
+        await fs.rm(workspaceDir, { recursive: true, force: true });
+      }
+    } finally {
+      vi.useRealTimers();
+    }
+  });
+
+  it("fails over when auth is unavailable and fallbacks are configured", async () => {
+    const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "clawdbot-agent-"));
+    const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "clawdbot-workspace-"));
+    const previousOpenAiKey = process.env.OPENAI_API_KEY;
+    delete process.env.OPENAI_API_KEY;
+    try {
+      const authPath = path.join(agentDir, "auth-profiles.json");
+      await fs.writeFile(authPath, JSON.stringify({ version: 1, profiles: {}, usageStats: {} }));
+
+      await expect(
+        runEmbeddedPiAgent({
+          sessionId: "session:test",
+          sessionKey: "agent:test:auth-unavailable",
+          sessionFile: path.join(workspaceDir, "session.jsonl"),
+          workspaceDir,
+          agentDir,
+          config: makeConfig({ fallbacks: ["openai/mock-2"], apiKey: "" }),
+          prompt: "hello",
+          provider: "openai",
+          model: "mock-1",
+          authProfileIdSource: "auto",
+          timeoutMs: 5_000,
+          runId: "run:auth-unavailable",
+        }),
+      ).rejects.toMatchObject({ name: "FailoverError", reason: "auth" });
+
+      expect(runEmbeddedAttemptMock).not.toHaveBeenCalled();
+    } finally {
+      if (previousOpenAiKey === undefined) {
+        delete process.env.OPENAI_API_KEY;
+      } else {
+        process.env.OPENAI_API_KEY = previousOpenAiKey;
+      }
+      await fs.rm(agentDir, { recursive: true, force: true });
+      await fs.rm(workspaceDir, { recursive: true, force: true });
+    }
+  });
+
   it("skips profiles in cooldown when rotating after failure", async () => {
     vi.useFakeTimers();
     try {