test: streamline auto-reply and tts suites

This commit is contained in:
Peter Steinberger
2026-02-21 21:43:16 +00:00
parent 5d9e7c942c
commit 52ddb6ae18
7 changed files with 1530 additions and 1809 deletions

View File

@@ -31,6 +31,9 @@ const state = vi.hoisted(() => ({
runCliAgentMock: vi.fn(),
}));
let modelFallbackModule: typeof import("../../agents/model-fallback.js");
let onAgentEvent: typeof import("../../infra/agent-events.js").onAgentEvent;
let runReplyAgentPromise:
| Promise<(typeof import("./agent-runner.js"))["runReplyAgent"]>
| undefined;
@@ -75,6 +78,8 @@ vi.mock("./queue.js", () => ({
beforeAll(async () => {
// Avoid attributing the initial agent-runner import cost to the first test case.
modelFallbackModule = await import("../../agents/model-fallback.js");
({ onAgentEvent } = await import("../../infra/agent-events.js"));
await getRunReplyAgent();
});
@@ -629,83 +634,70 @@ describe("runReplyAgent typing (heartbeat)", () => {
});
});
it("announces model fallback in verbose mode", async () => {
const sessionEntry: SessionEntry = {
sessionId: "session",
updatedAt: Date.now(),
};
const sessionStore = { main: sessionEntry };
state.runEmbeddedPiAgentMock.mockResolvedValueOnce({ payloads: [{ text: "final" }], meta: {} });
const modelFallback = await import("../../agents/model-fallback.js");
vi.spyOn(modelFallback, "runWithModelFallback").mockImplementationOnce(
async ({ run }: { run: (provider: string, model: string) => Promise<unknown> }) => ({
result: await run("deepinfra", "moonshotai/Kimi-K2.5"),
provider: "deepinfra",
model: "moonshotai/Kimi-K2.5",
attempts: [
{
provider: "fireworks",
model: "fireworks/minimax-m2p5",
error: "Provider fireworks is in cooldown (all profiles unavailable)",
reason: "rate_limit",
},
],
}),
);
it("announces model fallback only when verbose mode is enabled", async () => {
const cases = [
{ name: "verbose on", verbose: "on" as const, expectNotice: true },
{ name: "verbose off", verbose: "off" as const, expectNotice: false },
] as const;
for (const testCase of cases) {
const sessionEntry: SessionEntry = {
sessionId: "session",
updatedAt: Date.now(),
};
const sessionStore = { main: sessionEntry };
state.runEmbeddedPiAgentMock.mockResolvedValueOnce({
payloads: [{ text: "final" }],
meta: {},
});
vi.spyOn(modelFallbackModule, "runWithModelFallback").mockImplementationOnce(
async ({ run }: { run: (provider: string, model: string) => Promise<unknown> }) => ({
result: await run("deepinfra", "moonshotai/Kimi-K2.5"),
provider: "deepinfra",
model: "moonshotai/Kimi-K2.5",
attempts: [
{
provider: "fireworks",
model: "fireworks/minimax-m2p5",
error: "Provider fireworks is in cooldown (all profiles unavailable)",
reason: "rate_limit",
},
],
}),
);
const { run } = createMinimalRun({
resolvedVerboseLevel: "on",
sessionEntry,
sessionStore,
sessionKey: "main",
});
const res = await run();
expect(Array.isArray(res)).toBe(true);
const payloads = res as { text?: string }[];
expect(payloads[0]?.text).toContain("Model Fallback:");
expect(payloads[0]?.text).toContain("deepinfra/moonshotai/Kimi-K2.5");
expect(sessionEntry.fallbackNoticeReason).toBe("rate limit");
});
it("does not announce model fallback when verbose is off", async () => {
const { onAgentEvent } = await import("../../infra/agent-events.js");
state.runEmbeddedPiAgentMock.mockResolvedValueOnce({ payloads: [{ text: "final" }], meta: {} });
const modelFallback = await import("../../agents/model-fallback.js");
vi.spyOn(modelFallback, "runWithModelFallback").mockImplementationOnce(
async ({ run }: { run: (provider: string, model: string) => Promise<unknown> }) => ({
result: await run("deepinfra", "moonshotai/Kimi-K2.5"),
provider: "deepinfra",
model: "moonshotai/Kimi-K2.5",
attempts: [
{
provider: "fireworks",
model: "fireworks/minimax-m2p5",
error: "Provider fireworks is in cooldown (all profiles unavailable)",
reason: "rate_limit",
},
],
}),
);
const { run } = createMinimalRun({
resolvedVerboseLevel: "off",
});
const phases: string[] = [];
const off = onAgentEvent((evt) => {
const phase = typeof evt.data?.phase === "string" ? evt.data.phase : null;
if (evt.stream === "lifecycle" && phase) {
phases.push(phase);
const { run } = createMinimalRun({
resolvedVerboseLevel: testCase.verbose,
sessionEntry,
sessionStore,
sessionKey: "main",
});
const phases: string[] = [];
const off = onAgentEvent((evt) => {
const phase = typeof evt.data?.phase === "string" ? evt.data.phase : null;
if (evt.stream === "lifecycle" && phase) {
phases.push(phase);
}
});
const res = await run();
off();
const payload = Array.isArray(res)
? (res[0] as { text?: string })
: (res as { text?: string });
if (testCase.expectNotice) {
expect(payload.text, testCase.name).toContain("Model Fallback:");
expect(payload.text, testCase.name).toContain("deepinfra/moonshotai/Kimi-K2.5");
expect(sessionEntry.fallbackNoticeReason, testCase.name).toBe("rate limit");
continue;
}
});
const res = await run();
off();
const payload = Array.isArray(res) ? (res[0] as { text?: string }) : (res as { text?: string });
expect(payload.text).not.toContain("Model Fallback:");
expect(phases.filter((phase) => phase === "fallback")).toHaveLength(1);
expect(payload.text, testCase.name).not.toContain("Model Fallback:");
expect(
phases.filter((phase) => phase === "fallback"),
testCase.name,
).toHaveLength(1);
}
});
it("announces model fallback only once per active fallback state", async () => {
const { onAgentEvent } = await import("../../infra/agent-events.js");
const sessionEntry: SessionEntry = {
sessionId: "session",
updatedAt: Date.now(),
@@ -716,9 +708,8 @@ describe("runReplyAgent typing (heartbeat)", () => {
payloads: [{ text: "final" }],
meta: {},
});
const modelFallback = await import("../../agents/model-fallback.js");
const fallbackSpy = vi
.spyOn(modelFallback, "runWithModelFallback")
.spyOn(modelFallbackModule, "runWithModelFallback")
.mockImplementation(
async ({ run }: { run: (provider: string, model: string) => Promise<unknown> }) => ({
result: await run("deepinfra", "moonshotai/Kimi-K2.5"),
@@ -773,9 +764,8 @@ describe("runReplyAgent typing (heartbeat)", () => {
payloads: [{ text: "final" }],
meta: {},
});
const modelFallback = await import("../../agents/model-fallback.js");
const fallbackSpy = vi
.spyOn(modelFallback, "runWithModelFallback")
.spyOn(modelFallbackModule, "runWithModelFallback")
.mockImplementation(
async ({
provider,
@@ -833,7 +823,6 @@ describe("runReplyAgent typing (heartbeat)", () => {
});
it("announces fallback-cleared once when runtime returns to selected model", async () => {
const { onAgentEvent } = await import("../../infra/agent-events.js");
const sessionEntry: SessionEntry = {
sessionId: "session",
updatedAt: Date.now(),
@@ -845,9 +834,8 @@ describe("runReplyAgent typing (heartbeat)", () => {
payloads: [{ text: "final" }],
meta: {},
});
const modelFallback = await import("../../agents/model-fallback.js");
const fallbackSpy = vi
.spyOn(modelFallback, "runWithModelFallback")
.spyOn(modelFallbackModule, "runWithModelFallback")
.mockImplementation(
async ({
provider,
@@ -915,7 +903,6 @@ describe("runReplyAgent typing (heartbeat)", () => {
});
it("emits fallback lifecycle events while verbose is off", async () => {
const { onAgentEvent } = await import("../../infra/agent-events.js");
const sessionEntry: SessionEntry = {
sessionId: "session",
updatedAt: Date.now(),
@@ -927,9 +914,8 @@ describe("runReplyAgent typing (heartbeat)", () => {
payloads: [{ text: "final" }],
meta: {},
});
const modelFallback = await import("../../agents/model-fallback.js");
const fallbackSpy = vi
.spyOn(modelFallback, "runWithModelFallback")
.spyOn(modelFallbackModule, "runWithModelFallback")
.mockImplementation(
async ({
provider,
@@ -1008,9 +994,8 @@ describe("runReplyAgent typing (heartbeat)", () => {
payloads: [{ text: "final" }],
meta: {},
});
const modelFallback = await import("../../agents/model-fallback.js");
const fallbackSpy = vi
.spyOn(modelFallback, "runWithModelFallback")
.spyOn(modelFallbackModule, "runWithModelFallback")
.mockImplementation(
async ({ run }: { run: (provider: string, model: string) => Promise<unknown> }) => ({
result: await run("deepinfra", "moonshotai/Kimi-K2.5"),
@@ -1058,9 +1043,8 @@ describe("runReplyAgent typing (heartbeat)", () => {
payloads: [{ text: "final" }],
meta: {},
});
const modelFallback = await import("../../agents/model-fallback.js");
const fallbackSpy = vi
.spyOn(modelFallback, "runWithModelFallback")
.spyOn(modelFallbackModule, "runWithModelFallback")
.mockImplementation(
async ({ run }: { run: (provider: string, model: string) => Promise<unknown> }) => ({
result: await run("deepinfra", "moonshotai/Kimi-K2.5"),