mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-08 00:11:23 +00:00
fix(cron): keep watchdog timer armed during ticks
This commit is contained in:
@@ -1,9 +1,12 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
createCronStoreHarness,
|
||||
createNoopLogger,
|
||||
createCronStoreHarness,
|
||||
createRunningCronServiceState,
|
||||
} from "./service.test-harness.js";
|
||||
import { createCronServiceState } from "./service/state.js";
|
||||
import { onTimer } from "./service/timer.js";
|
||||
import type { CronJob } from "./types.js";
|
||||
|
||||
@@ -31,6 +34,14 @@ function createDueRecurringJob(params: {
|
||||
};
|
||||
}
|
||||
|
||||
function createDeferred<T>() {
|
||||
let resolve!: (value: T) => void;
|
||||
const promise = new Promise<T>((res) => {
|
||||
resolve = res;
|
||||
});
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
describe("CronService - timer re-arm when running (#12025)", () => {
|
||||
beforeEach(() => {
|
||||
noopLogger.debug.mockClear();
|
||||
@@ -81,4 +92,64 @@ describe("CronService - timer re-arm when running (#12025)", () => {
|
||||
timeoutSpy.mockRestore();
|
||||
await store.cleanup();
|
||||
});
|
||||
|
||||
it("arms a watchdog timer while a timer tick is still executing", async () => {
|
||||
const timeoutSpy = vi.spyOn(globalThis, "setTimeout");
|
||||
const store = await makeStorePath();
|
||||
const now = Date.parse("2026-02-06T10:05:00.000Z");
|
||||
const deferredRun = createDeferred<{ status: "ok"; summary: string }>();
|
||||
|
||||
await fs.mkdir(path.dirname(store.storePath), { recursive: true });
|
||||
await fs.writeFile(
|
||||
store.storePath,
|
||||
JSON.stringify(
|
||||
{
|
||||
version: 1,
|
||||
jobs: [
|
||||
createDueRecurringJob({
|
||||
id: "long-running-job",
|
||||
nowMs: now,
|
||||
nextRunAtMs: now,
|
||||
}),
|
||||
],
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
const state = createCronServiceState({
|
||||
storePath: store.storePath,
|
||||
cronEnabled: true,
|
||||
log: noopLogger,
|
||||
nowMs: () => now,
|
||||
enqueueSystemEvent: vi.fn(),
|
||||
requestHeartbeatNow: vi.fn(),
|
||||
runIsolatedAgentJob: vi.fn(async () => await deferredRun.promise),
|
||||
});
|
||||
|
||||
let settled = false;
|
||||
const timerPromise = onTimer(state);
|
||||
void timerPromise.finally(() => {
|
||||
settled = true;
|
||||
});
|
||||
|
||||
await Promise.resolve();
|
||||
expect(settled).toBe(false);
|
||||
expect(state.running).toBe(true);
|
||||
expect(state.timer).not.toBeNull();
|
||||
|
||||
const delays = timeoutSpy.mock.calls
|
||||
.map(([, delay]) => delay)
|
||||
.filter((d): d is number => typeof d === "number");
|
||||
expect(delays).toContain(60_000);
|
||||
|
||||
deferredRun.resolve({ status: "ok", summary: "done" });
|
||||
await timerPromise;
|
||||
expect(state.running).toBe(false);
|
||||
|
||||
timeoutSpy.mockRestore();
|
||||
await store.cleanup();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -221,6 +221,17 @@ export function armTimer(state: CronServiceState) {
|
||||
);
|
||||
}
|
||||
|
||||
function armRunningRecheckTimer(state: CronServiceState) {
|
||||
if (state.timer) {
|
||||
clearTimeout(state.timer);
|
||||
}
|
||||
state.timer = setTimeout(() => {
|
||||
void onTimer(state).catch((err) => {
|
||||
state.deps.log.error({ err: String(err) }, "cron: timer tick failed");
|
||||
});
|
||||
}, MAX_TIMER_DELAY_MS);
|
||||
}
|
||||
|
||||
export async function onTimer(state: CronServiceState) {
|
||||
if (state.running) {
|
||||
// Re-arm the timer so the scheduler keeps ticking even when a job is
|
||||
@@ -233,17 +244,13 @@ export async function onTimer(state: CronServiceState) {
|
||||
// zero-delay hot-loop when past-due jobs are waiting for the current
|
||||
// execution to finish.
|
||||
// See: https://github.com/openclaw/openclaw/issues/12025
|
||||
if (state.timer) {
|
||||
clearTimeout(state.timer);
|
||||
}
|
||||
state.timer = setTimeout(() => {
|
||||
void onTimer(state).catch((err) => {
|
||||
state.deps.log.error({ err: String(err) }, "cron: timer tick failed");
|
||||
});
|
||||
}, MAX_TIMER_DELAY_MS);
|
||||
armRunningRecheckTimer(state);
|
||||
return;
|
||||
}
|
||||
state.running = true;
|
||||
// Keep a watchdog timer armed while a tick is executing. If execution hangs
|
||||
// (for example in a provider call), the scheduler still wakes to re-check.
|
||||
armRunningRecheckTimer(state);
|
||||
try {
|
||||
const dueJobs = await locked(state, async () => {
|
||||
await ensureLoaded(state, { forceReload: true, skipRecompute: true });
|
||||
|
||||
Reference in New Issue
Block a user