github/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-05-08 15:58:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(telegram): simplify polling restart flow

Browse Source
This commit is contained in:
Peter Steinberger
2026-02-26 03:32:36 +01:00
parent 069bbf9741
commit b786d11fea
2 changed files with 184 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files

173
src/telegram/monitor.test.ts
View File

@@ -67,6 +67,36 @@ const { startTelegramWebhookSpy } = vi.hoisted(() => ({
startTelegramWebhookSpy: vi.fn(async () => ({ server: { close: vi.fn() }, stop: vi.fn() })), startTelegramWebhookSpy: vi.fn(async () => ({ server: { close: vi.fn() }, stop: vi.fn() })),
})); }));
type RunnerStub = {
task: () => Promise<void>;
stop: ReturnType<typeof vi.fn<() => void | Promise<void>>>;
isRunning: () => boolean;
};
const makeRunnerStub = (overrides: Partial<RunnerStub> = {}): RunnerStub => ({
task: overrides.task ?? (() => Promise.resolve()),
stop: overrides.stop ?? vi.fn<() => void | Promise<void>>(),
isRunning: overrides.isRunning ?? (() => false),
});
async function monitorWithAutoAbort(
opts: Omit<Parameters<typeof monitorTelegramProvider>[0], "abortSignal"> = {},
) {
const abort = new AbortController();
runSpy.mockImplementationOnce(() =>
makeRunnerStub({
task: async () => {
abort.abort();
},
}),
);
await monitorTelegramProvider({
token: "tok",
...opts,
abortSignal: abort.signal,
});
}
vi.mock("../config/config.js", async (importOriginal) => { vi.mock("../config/config.js", async (importOriginal) => {
const actual = await importOriginal<typeof import("../config/config.js")>(); const actual = await importOriginal<typeof import("../config/config.js")>();
return { return {
@@ -149,7 +179,7 @@ describe("monitorTelegramProvider (grammY)", () => {
Object.values(api).forEach((fn) => { Object.values(api).forEach((fn) => {
fn?.mockReset?.(); fn?.mockReset?.();
}); });
await monitorTelegramProvider({ token: "tok" }); await monitorWithAutoAbort();
expect(handlers.message).toBeDefined(); expect(handlers.message).toBeDefined();
await handlers.message?.({ await handlers.message?.({
message: { message: {
@@ -172,7 +202,7 @@ describe("monitorTelegramProvider (grammY)", () => {
channels: { telegram: {} }, channels: { telegram: {} },
}); });
await monitorTelegramProvider({ token: "tok" }); await monitorWithAutoAbort();
expect(runSpy).toHaveBeenCalledWith( expect(runSpy).toHaveBeenCalledWith(
expect.anything(), expect.anything(),
@@ -180,7 +210,7 @@ describe("monitorTelegramProvider (grammY)", () => {
sink: { concurrency: 3 }, sink: { concurrency: 3 },
runner: expect.objectContaining({ runner: expect.objectContaining({
silent: true, silent: true,
maxRetryTime: 5 * 60 * 1000, maxRetryTime: 60 * 60 * 1000,
retryInterval: "exponential", retryInterval: "exponential",
}), }),
}), }),
@@ -191,7 +221,7 @@ describe("monitorTelegramProvider (grammY)", () => {
Object.values(api).forEach((fn) => { Object.values(api).forEach((fn) => {
fn?.mockReset?.(); fn?.mockReset?.();
}); });
await monitorTelegramProvider({ token: "tok" }); await monitorWithAutoAbort();
await handlers.message?.({ await handlers.message?.({
message: { message: {
message_id: 2, message_id: 2,
@@ -205,24 +235,27 @@ describe("monitorTelegramProvider (grammY)", () => {
}); });
it("retries on recoverable undici fetch errors", async () => { it("retries on recoverable undici fetch errors", async () => {
const abort = new AbortController();
const networkError = Object.assign(new TypeError("fetch failed"), { const networkError = Object.assign(new TypeError("fetch failed"), {
cause: Object.assign(new Error("connect timeout"), { cause: Object.assign(new Error("connect timeout"), {
code: "UND_ERR_CONNECT_TIMEOUT", code: "UND_ERR_CONNECT_TIMEOUT",
}), }),
}); });
runSpy runSpy
.mockImplementationOnce(() => ({ .mockImplementationOnce(() =>
task: () => Promise.reject(networkError), makeRunnerStub({
stop: vi.fn(), task: () => Promise.reject(networkError),
isRunning: (): boolean => false, }),
})) )
.mockImplementationOnce(() => ({ .mockImplementationOnce(() =>
task: () => Promise.resolve(), makeRunnerStub({
stop: vi.fn(), task: async () => {
isRunning: (): boolean => false, abort.abort();
})); },
}),
);
await monitorTelegramProvider({ token: "tok" }); await monitorTelegramProvider({ token: "tok", abortSignal: abort.signal });
expect(computeBackoff).toHaveBeenCalled(); expect(computeBackoff).toHaveBeenCalled();
expect(sleepWithAbort).toHaveBeenCalled(); expect(sleepWithAbort).toHaveBeenCalled();
@@ -230,6 +263,7 @@ describe("monitorTelegramProvider (grammY)", () => {
}); });
it("deletes webhook before starting polling", async () => { it("deletes webhook before starting polling", async () => {
const abort = new AbortController();
const order: string[] = []; const order: string[] = [];
api.deleteWebhook.mockReset(); api.deleteWebhook.mockReset();
api.deleteWebhook.mockImplementationOnce(async () => { api.deleteWebhook.mockImplementationOnce(async () => {
@@ -238,20 +272,21 @@ describe("monitorTelegramProvider (grammY)", () => {
}); });
runSpy.mockImplementationOnce(() => { runSpy.mockImplementationOnce(() => {
order.push("run"); order.push("run");
return { return makeRunnerStub({
task: () => Promise.resolve(), task: async () => {
stop: vi.fn(), abort.abort();
isRunning: () => false, },
}; });
}); });
await monitorTelegramProvider({ token: "tok" }); await monitorTelegramProvider({ token: "tok", abortSignal: abort.signal });
expect(api.deleteWebhook).toHaveBeenCalledWith({ drop_pending_updates: false }); expect(api.deleteWebhook).toHaveBeenCalledWith({ drop_pending_updates: false });
expect(order).toEqual(["deleteWebhook", "run"]); expect(order).toEqual(["deleteWebhook", "run"]);
}); });
it("retries recoverable deleteWebhook failures before polling", async () => { it("retries recoverable deleteWebhook failures before polling", async () => {
const abort = new AbortController();
const cleanupError = Object.assign(new TypeError("fetch failed"), { const cleanupError = Object.assign(new TypeError("fetch failed"), {
cause: Object.assign(new Error("connect timeout"), { cause: Object.assign(new Error("connect timeout"), {
code: "UND_ERR_CONNECT_TIMEOUT", code: "UND_ERR_CONNECT_TIMEOUT",
@@ -259,13 +294,15 @@ describe("monitorTelegramProvider (grammY)", () => {
}); });
api.deleteWebhook.mockReset(); api.deleteWebhook.mockReset();
api.deleteWebhook.mockRejectedValueOnce(cleanupError).mockResolvedValueOnce(true); api.deleteWebhook.mockRejectedValueOnce(cleanupError).mockResolvedValueOnce(true);
runSpy.mockImplementationOnce(() => ({ runSpy.mockImplementationOnce(() =>
task: () => Promise.resolve(), makeRunnerStub({
stop: vi.fn(), task: async () => {
isRunning: () => false, abort.abort();
})); },
}),
);
await monitorTelegramProvider({ token: "tok" }); await monitorTelegramProvider({ token: "tok", abortSignal: abort.signal });
expect(api.deleteWebhook).toHaveBeenCalledTimes(2); expect(api.deleteWebhook).toHaveBeenCalledTimes(2);
expect(computeBackoff).toHaveBeenCalled(); expect(computeBackoff).toHaveBeenCalled();
@@ -274,6 +311,7 @@ describe("monitorTelegramProvider (grammY)", () => {
}); });
it("retries setup-time recoverable errors before starting polling", async () => { it("retries setup-time recoverable errors before starting polling", async () => {
const abort = new AbortController();
const setupError = Object.assign(new TypeError("fetch failed"), { const setupError = Object.assign(new TypeError("fetch failed"), {
cause: Object.assign(new Error("connect timeout"), { cause: Object.assign(new Error("connect timeout"), {
code: "UND_ERR_CONNECT_TIMEOUT", code: "UND_ERR_CONNECT_TIMEOUT",
@@ -281,13 +319,15 @@ describe("monitorTelegramProvider (grammY)", () => {
}); });
createTelegramBotErrors.push(setupError); createTelegramBotErrors.push(setupError);
runSpy.mockImplementationOnce(() => ({ runSpy.mockImplementationOnce(() =>
task: () => Promise.resolve(), makeRunnerStub({
stop: vi.fn(), task: async () => {
isRunning: () => false, abort.abort();
})); },
}),
);
await monitorTelegramProvider({ token: "tok" }); await monitorTelegramProvider({ token: "tok", abortSignal: abort.signal });
expect(computeBackoff).toHaveBeenCalled(); expect(computeBackoff).toHaveBeenCalled();
expect(sleepWithAbort).toHaveBeenCalled(); expect(sleepWithAbort).toHaveBeenCalled();
@@ -295,6 +335,7 @@ describe("monitorTelegramProvider (grammY)", () => {
}); });
it("awaits runner.stop before retrying after recoverable polling error", async () => { it("awaits runner.stop before retrying after recoverable polling error", async () => {
const abort = new AbortController();
const recoverableError = Object.assign(new TypeError("fetch failed"), { const recoverableError = Object.assign(new TypeError("fetch failed"), {
cause: Object.assign(new Error("connect timeout"), { cause: Object.assign(new Error("connect timeout"), {
code: "UND_ERR_CONNECT_TIMEOUT", code: "UND_ERR_CONNECT_TIMEOUT",
@@ -307,21 +348,22 @@ describe("monitorTelegramProvider (grammY)", () => {
}); });
runSpy runSpy
.mockImplementationOnce(() => ({ .mockImplementationOnce(() =>
task: () => Promise.reject(recoverableError), makeRunnerStub({
stop: firstStop, task: () => Promise.reject(recoverableError),
isRunning: () => false, stop: firstStop,
})) }),
)
.mockImplementationOnce(() => { .mockImplementationOnce(() => {
expect(firstStopped).toBe(true); expect(firstStopped).toBe(true);
return { return makeRunnerStub({
task: () => Promise.resolve(), task: async () => {
stop: vi.fn(), abort.abort();
isRunning: () => false, },
}; });
}); });
await monitorTelegramProvider({ token: "tok" }); await monitorTelegramProvider({ token: "tok", abortSignal: abort.signal });
expect(firstStop).toHaveBeenCalled(); expect(firstStop).toHaveBeenCalled();
expect(computeBackoff).toHaveBeenCalled(); expect(computeBackoff).toHaveBeenCalled();
@@ -330,16 +372,17 @@ describe("monitorTelegramProvider (grammY)", () => {
}); });
it("surfaces non-recoverable errors", async () => { it("surfaces non-recoverable errors", async () => {
runSpy.mockImplementationOnce(() => ({ runSpy.mockImplementationOnce(() =>
task: () => Promise.reject(new Error("bad token")), makeRunnerStub({
stop: vi.fn(), task: () => Promise.reject(new Error("bad token")),
isRunning: (): boolean => false, }),
})); );
await expect(monitorTelegramProvider({ token: "tok" })).rejects.toThrow("bad token"); await expect(monitorTelegramProvider({ token: "tok" })).rejects.toThrow("bad token");
}); });
it("force-restarts polling when unhandled network rejection stalls runner", async () => { it("force-restarts polling when unhandled network rejection stalls runner", async () => {
const abort = new AbortController();
let running = true; let running = true;
let releaseTask: (() => void) | undefined; let releaseTask: (() => void) | undefined;
const stop = vi.fn(async () => { const stop = vi.fn(async () => {
@@ -348,21 +391,25 @@ describe("monitorTelegramProvider (grammY)", () => {
}); });
runSpy runSpy
.mockImplementationOnce(() => ({ .mockImplementationOnce(() =>
task: () => makeRunnerStub({
new Promise<void>((resolve) => { task: () =>
releaseTask = resolve; new Promise<void>((resolve) => {
}), releaseTask = resolve;
stop, }),
isRunning: () => running, stop,
})) isRunning: () => running,
.mockImplementationOnce(() => ({ }),
task: () => Promise.resolve(), )
stop: vi.fn(), .mockImplementationOnce(() =>
isRunning: () => false, makeRunnerStub({
})); task: async () => {
abort.abort();
},
}),
);
const monitor = monitorTelegramProvider({ token: "tok" }); const monitor = monitorTelegramProvider({ token: "tok", abortSignal: abort.signal });
await vi.waitFor(() => expect(runSpy).toHaveBeenCalledTimes(1)); await vi.waitFor(() => expect(runSpy).toHaveBeenCalledTimes(1));
expect(emitUnhandledRejection(new TypeError("fetch failed"))).toBe(true); expect(emitUnhandledRejection(new TypeError("fetch failed"))).toBe(true);

136
src/telegram/monitor.ts
View File

@@ -45,9 +45,8 @@ export function createTelegramRunnerOptions(cfg: OpenClawConfig): RunOptions<unk
}, },
// Suppress grammY getUpdates stack traces; we log concise errors ourselves. // Suppress grammY getUpdates stack traces; we log concise errors ourselves.
silent: true, silent: true,
// Retry transient failures before surfacing errors. Use a generous // Keep grammY retrying for a long outage window. If polling still
// window so the runner survives prolonged outages (e.g. scheduled // stops, the outer monitor loop restarts it with backoff.
// internet downtime) without the outer loop needing to restart it.
maxRetryTime: 60 * 60 * 1000, maxRetryTime: 60 * 60 * 1000,
retryInterval: "exponential", retryInterval: "exponential",
}, },
@@ -61,6 +60,8 @@ const TELEGRAM_POLL_RESTART_POLICY = {
jitter: 0.25, jitter: 0.25,
}; };
type TelegramBot = ReturnType<typeof createTelegramBot>;
const isGetUpdatesConflict = (err: unknown) => { const isGetUpdatesConflict = (err: unknown) => {
if (!err || typeof err !== "object") { if (!err || typeof err !== "object") {
return false; return false;
@@ -188,21 +189,11 @@ export async function monitorTelegramProvider(opts: MonitorTelegramOpts = {}) {
let restartAttempts = 0; let restartAttempts = 0;
let webhookCleared = false; let webhookCleared = false;
const runnerOptions = createTelegramRunnerOptions(cfg); const runnerOptions = createTelegramRunnerOptions(cfg);
const waitBeforeRetryOnRecoverableSetupError = async ( const waitBeforeRestart = async (buildLine: (delay: string) => string): Promise<boolean> => {
err: unknown,
logPrefix: string,
): Promise<boolean> => {
if (opts.abortSignal?.aborted) {
return false;
}
if (!isRecoverableTelegramNetworkError(err, { context: "unknown" })) {
throw err;
}
restartAttempts += 1; restartAttempts += 1;
const delayMs = computeBackoff(TELEGRAM_POLL_RESTART_POLICY, restartAttempts); const delayMs = computeBackoff(TELEGRAM_POLL_RESTART_POLICY, restartAttempts);
(opts.runtime?.error ?? console.error)( const delay = formatDurationPrecise(delayMs);
`${logPrefix}: ${formatErrorMessage(err)}; retrying in ${formatDurationPrecise(delayMs)}.`, log(buildLine(delay));
);
try { try {
await sleepWithAbort(delayMs, opts.abortSignal); await sleepWithAbort(delayMs, opts.abortSignal);
} catch (sleepErr) { } catch (sleepErr) {
@@ -214,10 +205,24 @@ export async function monitorTelegramProvider(opts: MonitorTelegramOpts = {}) {
return true; return true;
}; };
while (!opts.abortSignal?.aborted) { const waitBeforeRetryOnRecoverableSetupError = async (
let bot; err: unknown,
logPrefix: string,
): Promise<boolean> => {
if (opts.abortSignal?.aborted) {
return false;
}
if (!isRecoverableTelegramNetworkError(err, { context: "unknown" })) {
throw err;
}
return waitBeforeRestart(
(delay) => `${logPrefix}: ${formatErrorMessage(err)}; retrying in ${delay}.`,
);
};
const createPollingBot = async (): Promise<TelegramBot | undefined> => {
try { try {
bot = createTelegramBot({ return createTelegramBot({
token, token,
runtime: opts.runtime, runtime: opts.runtime,
proxyFetch, proxyFetch,
@@ -234,31 +239,34 @@ export async function monitorTelegramProvider(opts: MonitorTelegramOpts = {}) {
"Telegram setup network error", "Telegram setup network error",
); );
if (!shouldRetry) { if (!shouldRetry) {
return; return undefined;
} }
continue; return undefined;
} }
};
if (!webhookCleared) { const ensureWebhookCleanup = async (bot: TelegramBot): Promise<"ready" | "retry" | "exit"> => {
try { if (webhookCleared) {
await withTelegramApiErrorLogging({ return "ready";
operation: "deleteWebhook",
runtime: opts.runtime,
fn: () => bot.api.deleteWebhook({ drop_pending_updates: false }),
});
webhookCleared = true;
} catch (err) {
const shouldRetry = await waitBeforeRetryOnRecoverableSetupError(
err,
"Telegram webhook cleanup failed",
);
if (!shouldRetry) {
return;
}
continue;
}
} }
try {
await withTelegramApiErrorLogging({
operation: "deleteWebhook",
runtime: opts.runtime,
fn: () => bot.api.deleteWebhook({ drop_pending_updates: false }),
});
webhookCleared = true;
return "ready";
} catch (err) {
const shouldRetry = await waitBeforeRetryOnRecoverableSetupError(
err,
"Telegram webhook cleanup failed",
);
return shouldRetry ? "retry" : "exit";
}
};
const runPollingCycle = async (bot: TelegramBot): Promise<"continue" | "exit"> => {
const runner = run(bot, runnerOptions); const runner = run(bot, runnerOptions);
activeRunner = runner; activeRunner = runner;
let stopPromise: Promise<void> | undefined; let stopPromise: Promise<void> | undefined;
@@ -280,23 +288,16 @@ export async function monitorTelegramProvider(opts: MonitorTelegramOpts = {}) {
// runner.task() returns a promise that resolves when the runner stops // runner.task() returns a promise that resolves when the runner stops
await runner.task(); await runner.task();
if (opts.abortSignal?.aborted) { if (opts.abortSignal?.aborted) {
return; return "exit";
} }
// The runner stopped on its own. This can happen when grammY's
// maxRetryTime is exceeded (e.g. prolonged network outage).
// Instead of exiting permanently, restart with backoff so polling
// recovers once connectivity is restored.
restartAttempts += 1;
const delayMs = computeBackoff(TELEGRAM_POLL_RESTART_POLICY, restartAttempts);
const reason = forceRestarted const reason = forceRestarted
? "unhandled network error" ? "unhandled network error"
: "runner stopped (maxRetryTime exceeded or graceful stop)"; : "runner stopped (maxRetryTime exceeded or graceful stop)";
forceRestarted = false; forceRestarted = false;
log( const shouldRestart = await waitBeforeRestart(
`Telegram polling runner stopped (${reason}); restarting in ${formatDurationPrecise(delayMs)}.`, (delay) => `Telegram polling runner stopped (${reason}); restarting in ${delay}.`,
); );
await sleepWithAbort(delayMs, opts.abortSignal); return shouldRestart ? "continue" : "exit";
continue;
} catch (err) { } catch (err) {
forceRestarted = false; forceRestarted = false;
if (opts.abortSignal?.aborted) { if (opts.abortSignal?.aborted) {
@@ -307,25 +308,36 @@ export async function monitorTelegramProvider(opts: MonitorTelegramOpts = {}) {
if (!isConflict && !isRecoverable) { if (!isConflict && !isRecoverable) {
throw err; throw err;
} }
restartAttempts += 1;
const delayMs = computeBackoff(TELEGRAM_POLL_RESTART_POLICY, restartAttempts);
const reason = isConflict ? "getUpdates conflict" : "network error"; const reason = isConflict ? "getUpdates conflict" : "network error";
const errMsg = formatErrorMessage(err); const errMsg = formatErrorMessage(err);
(opts.runtime?.error ?? console.error)( const shouldRestart = await waitBeforeRestart(
`Telegram ${reason}: ${errMsg}; retrying in ${formatDurationPrecise(delayMs)}.`, (delay) => `Telegram ${reason}: ${errMsg}; retrying in ${delay}.`,
); );
try { return shouldRestart ? "continue" : "exit";
await sleepWithAbort(delayMs, opts.abortSignal);
} catch (sleepErr) {
if (opts.abortSignal?.aborted) {
return;
}
throw sleepErr;
}
} finally { } finally {
opts.abortSignal?.removeEventListener("abort", stopOnAbort); opts.abortSignal?.removeEventListener("abort", stopOnAbort);
await stopRunner(); await stopRunner();
} }
};
while (!opts.abortSignal?.aborted) {
const bot = await createPollingBot();
if (!bot) {
continue;
}
const cleanupState = await ensureWebhookCleanup(bot);
if (cleanupState === "retry") {
continue;
}
if (cleanupState === "exit") {
return;
}
const state = await runPollingCycle(bot);
if (state === "exit") {
return;
}
} }
} finally { } finally {
unregisterHandler(); unregisterHandler();