mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-11 11:21:41 +00:00
feat(cron): add failure destination support to failed cron jobs (#31059)
* feat(cron): add failure destination support with webhook mode and bestEffort handling Extends PR #24789 failure alerts with features from PR #29145: - Add webhook delivery mode for failure alerts (mode: 'webhook') - Add accountId support for multi-account channel configurations - Add bestEffort handling to skip alerts when job has bestEffort=true - Add separate failureDestination config (global + per-job in delivery) - Add duplicate prevention (prevents sending to same as primary delivery) - Add CLI flags: --failure-alert-mode, --failure-alert-account-id - Add UI fields for new options in web cron editor * fix(cron): merge failureAlert mode/accountId and preserve failureDestination on updates - Fix mergeCronFailureAlert to merge mode and accountId fields - Fix mergeCronDelivery to preserve failureDestination on updates - Fix isSameDeliveryTarget to use 'announce' as default instead of 'none' to properly detect duplicates when delivery.mode is undefined * fix(cron): validate webhook mode requires URL in resolveFailureDestination When mode is 'webhook' but no 'to' URL is provided, return null instead of creating an invalid plan that silently fails later. * fix(cron): fail closed on webhook mode without URL and make failureDestination fields clearable - sendCronFailureAlert: fail closed when mode is webhook but URL is missing - mergeCronDelivery: use per-key presence checks so callers can clear nested failureDestination fields via cron.update Note: protocol:check shows missing internalEvents in Swift models - this is a pre-existing issue unrelated to these changes (upstream sync needed). * fix(cron): use separate schema for failureDestination and fix type cast - Create CronFailureDestinationSchema excluding after/cooldownMs fields - Fix type cast in sendFailureNotificationAnnounce to use CronMessageChannel * fix(cron): merge global failureDestination with partial job overrides When job has partial failureDestination config, fall back to global config for unset fields instead of treating it as a full override. * fix(cron): avoid forcing announce mode and clear inherited to on mode change - UI: only include mode in patch if explicitly set to non-default - delivery.ts: clear inherited 'to' when job overrides mode, since URL semantics differ between announce and webhook modes * fix(cron): preserve explicit to on mode override and always include mode in UI patches - delivery.ts: preserve job-level explicit 'to' when overriding mode - UI: always include mode in failureAlert patch so users can switch between announce/webhook * fix(cron): allow clearing accountId and treat undefined global mode as announce - UI: always include accountId in patch so users can clear it - delivery.ts: treat undefined global mode as announce when comparing for clearing inherited 'to' * Cron: harden failure destination routing and add regression coverage * Cron: resolve failure destination review feedback * Cron: drop unrelated timeout assertions from conflict resolution * Cron: format cron CLI regression test * Cron: align gateway cron test mock types --------- Co-authored-by: Tak Hoffman <781889+Takhoffman@users.noreply.github.com>
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { resolveCronDeliveryPlan } from "./delivery.js";
|
||||
import { resolveCronDeliveryPlan, resolveFailureDestination } from "./delivery.js";
|
||||
import type { CronJob } from "./types.js";
|
||||
|
||||
function makeJob(overrides: Partial<CronJob>): CronJob {
|
||||
@@ -85,3 +85,96 @@ describe("resolveCronDeliveryPlan", () => {
|
||||
expect(plan.accountId).toBe("bot-a");
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveFailureDestination", () => {
|
||||
it("merges global defaults with job-level overrides", () => {
|
||||
const plan = resolveFailureDestination(
|
||||
makeJob({
|
||||
delivery: {
|
||||
mode: "announce",
|
||||
channel: "telegram",
|
||||
to: "111",
|
||||
failureDestination: { channel: "signal", mode: "announce" },
|
||||
},
|
||||
}),
|
||||
{
|
||||
channel: "telegram",
|
||||
to: "222",
|
||||
mode: "announce",
|
||||
accountId: "global-account",
|
||||
},
|
||||
);
|
||||
expect(plan).toEqual({
|
||||
mode: "announce",
|
||||
channel: "signal",
|
||||
to: "222",
|
||||
accountId: "global-account",
|
||||
});
|
||||
});
|
||||
|
||||
it("returns null for webhook mode without destination URL", () => {
|
||||
const plan = resolveFailureDestination(
|
||||
makeJob({
|
||||
delivery: {
|
||||
mode: "announce",
|
||||
channel: "telegram",
|
||||
to: "111",
|
||||
failureDestination: { mode: "webhook" },
|
||||
},
|
||||
}),
|
||||
undefined,
|
||||
);
|
||||
expect(plan).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null when failure destination matches primary delivery target", () => {
|
||||
const plan = resolveFailureDestination(
|
||||
makeJob({
|
||||
delivery: {
|
||||
mode: "announce",
|
||||
channel: "telegram",
|
||||
to: "111",
|
||||
accountId: "bot-a",
|
||||
failureDestination: {
|
||||
mode: "announce",
|
||||
channel: "telegram",
|
||||
to: "111",
|
||||
accountId: "bot-a",
|
||||
},
|
||||
},
|
||||
}),
|
||||
undefined,
|
||||
);
|
||||
expect(plan).toBeNull();
|
||||
});
|
||||
|
||||
it("allows job-level failure destination fields to clear inherited global values", () => {
|
||||
const plan = resolveFailureDestination(
|
||||
makeJob({
|
||||
delivery: {
|
||||
mode: "announce",
|
||||
channel: "telegram",
|
||||
to: "111",
|
||||
failureDestination: {
|
||||
mode: "announce",
|
||||
channel: undefined as never,
|
||||
to: undefined as never,
|
||||
accountId: undefined as never,
|
||||
},
|
||||
},
|
||||
}),
|
||||
{
|
||||
channel: "signal",
|
||||
to: "group-abc",
|
||||
accountId: "global-account",
|
||||
mode: "announce",
|
||||
},
|
||||
);
|
||||
expect(plan).toEqual({
|
||||
mode: "announce",
|
||||
channel: "last",
|
||||
to: undefined,
|
||||
accountId: undefined,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,4 +1,14 @@
|
||||
import type { CronDeliveryMode, CronJob, CronMessageChannel } from "./types.js";
|
||||
import type { CliDeps } from "../cli/deps.js";
|
||||
import { createOutboundSendDeps } from "../cli/outbound-send-deps.js";
|
||||
import type { CronFailureDestinationConfig } from "../config/types.cron.js";
|
||||
import type { OpenClawConfig } from "../config/types.js";
|
||||
import { formatErrorMessage } from "../infra/errors.js";
|
||||
import { deliverOutboundPayloads } from "../infra/outbound/deliver.js";
|
||||
import { resolveAgentOutboundIdentity } from "../infra/outbound/identity.js";
|
||||
import { buildOutboundSessionContext } from "../infra/outbound/session-context.js";
|
||||
import { getChildLogger } from "../logging.js";
|
||||
import { resolveDeliveryTarget } from "./isolated-agent/delivery-target.js";
|
||||
import type { CronDelivery, CronDeliveryMode, CronJob, CronMessageChannel } from "./types.js";
|
||||
|
||||
export type CronDeliveryPlan = {
|
||||
mode: CronDeliveryMode;
|
||||
@@ -90,3 +100,202 @@ export function resolveCronDeliveryPlan(job: CronJob): CronDeliveryPlan {
|
||||
requested,
|
||||
};
|
||||
}
|
||||
|
||||
export type CronFailureDeliveryPlan = {
|
||||
mode: "announce" | "webhook";
|
||||
channel?: CronMessageChannel;
|
||||
to?: string;
|
||||
accountId?: string;
|
||||
};
|
||||
|
||||
export type CronFailureDestinationInput = {
|
||||
channel?: CronMessageChannel;
|
||||
to?: string;
|
||||
accountId?: string;
|
||||
mode?: "announce" | "webhook";
|
||||
};
|
||||
|
||||
function normalizeFailureMode(value: unknown): "announce" | "webhook" | undefined {
|
||||
if (typeof value !== "string") {
|
||||
return undefined;
|
||||
}
|
||||
const trimmed = value.trim().toLowerCase();
|
||||
if (trimmed === "announce" || trimmed === "webhook") {
|
||||
return trimmed;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function resolveFailureDestination(
|
||||
job: CronJob,
|
||||
globalConfig?: CronFailureDestinationConfig,
|
||||
): CronFailureDeliveryPlan | null {
|
||||
const delivery = job.delivery;
|
||||
const jobFailureDest = delivery?.failureDestination as CronFailureDestinationInput | undefined;
|
||||
const hasJobFailureDest = jobFailureDest && typeof jobFailureDest === "object";
|
||||
|
||||
let channel: CronMessageChannel | undefined;
|
||||
let to: string | undefined;
|
||||
let accountId: string | undefined;
|
||||
let mode: "announce" | "webhook" | undefined;
|
||||
|
||||
// Start with global config as base
|
||||
if (globalConfig) {
|
||||
channel = normalizeChannel(globalConfig.channel);
|
||||
to = normalizeTo(globalConfig.to);
|
||||
accountId = normalizeAccountId(globalConfig.accountId);
|
||||
mode = normalizeFailureMode(globalConfig.mode);
|
||||
}
|
||||
|
||||
// Override with job-level values if present
|
||||
if (hasJobFailureDest) {
|
||||
const jobChannel = normalizeChannel(jobFailureDest.channel);
|
||||
const jobTo = normalizeTo(jobFailureDest.to);
|
||||
const jobAccountId = normalizeAccountId(jobFailureDest.accountId);
|
||||
const jobMode = normalizeFailureMode(jobFailureDest.mode);
|
||||
const hasJobChannelField = "channel" in jobFailureDest;
|
||||
const hasJobToField = "to" in jobFailureDest;
|
||||
const hasJobAccountIdField = "accountId" in jobFailureDest;
|
||||
|
||||
// Track if 'to' was explicitly set at job level
|
||||
const jobToExplicitValue = hasJobToField && jobTo !== undefined;
|
||||
|
||||
// Respect explicit clears from partial patches.
|
||||
if (hasJobChannelField) {
|
||||
channel = jobChannel;
|
||||
}
|
||||
if (hasJobToField) {
|
||||
to = jobTo;
|
||||
}
|
||||
if (hasJobAccountIdField) {
|
||||
accountId = jobAccountId;
|
||||
}
|
||||
if (jobMode !== undefined) {
|
||||
// Mode was explicitly overridden - clear inherited 'to' since URL semantics differ
|
||||
// between announce (channel recipient) and webhook (HTTP endpoint)
|
||||
// But preserve explicit 'to' that was set at job level
|
||||
// Treat undefined global mode as "announce" for comparison
|
||||
const globalMode = globalConfig?.mode ?? "announce";
|
||||
if (!jobToExplicitValue && globalMode !== jobMode) {
|
||||
to = undefined;
|
||||
}
|
||||
mode = jobMode;
|
||||
}
|
||||
}
|
||||
|
||||
if (!channel && !to && !accountId && !mode) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const resolvedMode = mode ?? "announce";
|
||||
|
||||
// Webhook mode requires a URL
|
||||
if (resolvedMode === "webhook" && !to) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const result: CronFailureDeliveryPlan = {
|
||||
mode: resolvedMode,
|
||||
channel: resolvedMode === "announce" ? (channel ?? "last") : undefined,
|
||||
to,
|
||||
accountId,
|
||||
};
|
||||
|
||||
if (delivery && isSameDeliveryTarget(delivery, result)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function isSameDeliveryTarget(
|
||||
delivery: CronDelivery,
|
||||
failurePlan: CronFailureDeliveryPlan,
|
||||
): boolean {
|
||||
const primaryMode = delivery.mode ?? "announce";
|
||||
if (primaryMode === "none") {
|
||||
return false;
|
||||
}
|
||||
|
||||
const primaryChannel = delivery.channel;
|
||||
const primaryTo = delivery.to;
|
||||
const primaryAccountId = delivery.accountId;
|
||||
|
||||
if (failurePlan.mode === "webhook") {
|
||||
return primaryMode === "webhook" && primaryTo === failurePlan.to;
|
||||
}
|
||||
|
||||
const primaryChannelNormalized = primaryChannel ?? "last";
|
||||
const failureChannelNormalized = failurePlan.channel ?? "last";
|
||||
|
||||
return (
|
||||
failureChannelNormalized === primaryChannelNormalized &&
|
||||
failurePlan.to === primaryTo &&
|
||||
failurePlan.accountId === primaryAccountId
|
||||
);
|
||||
}
|
||||
|
||||
const FAILURE_NOTIFICATION_TIMEOUT_MS = 30_000;
|
||||
const cronDeliveryLogger = getChildLogger({ subsystem: "cron-delivery" });
|
||||
|
||||
export async function sendFailureNotificationAnnounce(
|
||||
deps: CliDeps,
|
||||
cfg: OpenClawConfig,
|
||||
agentId: string,
|
||||
jobId: string,
|
||||
target: { channel?: string; to?: string; accountId?: string },
|
||||
message: string,
|
||||
): Promise<void> {
|
||||
const resolvedTarget = await resolveDeliveryTarget(cfg, agentId, {
|
||||
channel: target.channel as CronMessageChannel | undefined,
|
||||
to: target.to,
|
||||
accountId: target.accountId,
|
||||
});
|
||||
|
||||
if (!resolvedTarget.ok) {
|
||||
cronDeliveryLogger.warn(
|
||||
{ error: resolvedTarget.error.message },
|
||||
"cron: failed to resolve failure destination target",
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const identity = resolveAgentOutboundIdentity(cfg, agentId);
|
||||
const session = buildOutboundSessionContext({
|
||||
cfg,
|
||||
agentId,
|
||||
sessionKey: `cron:${jobId}:failure`,
|
||||
});
|
||||
|
||||
const abortController = new AbortController();
|
||||
const timeout = setTimeout(() => {
|
||||
abortController.abort();
|
||||
}, FAILURE_NOTIFICATION_TIMEOUT_MS);
|
||||
|
||||
try {
|
||||
await deliverOutboundPayloads({
|
||||
cfg,
|
||||
channel: resolvedTarget.channel,
|
||||
to: resolvedTarget.to,
|
||||
accountId: resolvedTarget.accountId,
|
||||
threadId: resolvedTarget.threadId,
|
||||
payloads: [{ text: message }],
|
||||
session,
|
||||
identity,
|
||||
bestEffort: false,
|
||||
deps: createOutboundSendDeps(deps),
|
||||
abortSignal: abortController.signal,
|
||||
});
|
||||
} catch (err) {
|
||||
cronDeliveryLogger.warn(
|
||||
{
|
||||
err: formatErrorMessage(err),
|
||||
channel: resolvedTarget.channel,
|
||||
to: resolvedTarget.to,
|
||||
},
|
||||
"cron: failure destination announce failed",
|
||||
);
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -195,4 +195,72 @@ describe("CronService failure alerts", () => {
|
||||
cron.stop();
|
||||
await store.cleanup();
|
||||
});
|
||||
|
||||
it("threads failure alert mode/accountId and skips best-effort jobs", async () => {
|
||||
const store = await makeStorePath();
|
||||
const sendCronFailureAlert = vi.fn(async () => undefined);
|
||||
const runIsolatedAgentJob = vi.fn(async () => ({
|
||||
status: "error" as const,
|
||||
error: "temporary upstream error",
|
||||
}));
|
||||
|
||||
const cron = new CronService({
|
||||
storePath: store.storePath,
|
||||
cronEnabled: true,
|
||||
cronConfig: {
|
||||
failureAlert: {
|
||||
enabled: true,
|
||||
after: 1,
|
||||
mode: "webhook",
|
||||
accountId: "global-account",
|
||||
},
|
||||
},
|
||||
log: noopLogger,
|
||||
enqueueSystemEvent: vi.fn(),
|
||||
requestHeartbeatNow: vi.fn(),
|
||||
runIsolatedAgentJob,
|
||||
sendCronFailureAlert,
|
||||
});
|
||||
|
||||
await cron.start();
|
||||
const normalJob = await cron.add({
|
||||
name: "normal alert job",
|
||||
enabled: true,
|
||||
schedule: { kind: "every", everyMs: 60_000 },
|
||||
sessionTarget: "isolated",
|
||||
wakeMode: "next-heartbeat",
|
||||
payload: { kind: "agentTurn", message: "run report" },
|
||||
delivery: { mode: "announce", channel: "telegram", to: "19098680" },
|
||||
});
|
||||
const bestEffortJob = await cron.add({
|
||||
name: "best effort alert job",
|
||||
enabled: true,
|
||||
schedule: { kind: "every", everyMs: 60_000 },
|
||||
sessionTarget: "isolated",
|
||||
wakeMode: "next-heartbeat",
|
||||
payload: { kind: "agentTurn", message: "run report" },
|
||||
delivery: {
|
||||
mode: "announce",
|
||||
channel: "telegram",
|
||||
to: "19098680",
|
||||
bestEffort: true,
|
||||
},
|
||||
});
|
||||
|
||||
await cron.run(normalJob.id, "force");
|
||||
expect(sendCronFailureAlert).toHaveBeenCalledTimes(1);
|
||||
expect(sendCronFailureAlert).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
mode: "webhook",
|
||||
accountId: "global-account",
|
||||
to: undefined,
|
||||
}),
|
||||
);
|
||||
|
||||
await cron.run(bestEffortJob.id, "force");
|
||||
expect(sendCronFailureAlert).toHaveBeenCalledTimes(1);
|
||||
|
||||
cron.stop();
|
||||
await store.cleanup();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -222,6 +222,51 @@ describe("applyJobPatch", () => {
|
||||
expect(job.delivery).toEqual({ mode: "webhook", to: "https://example.invalid/trim" });
|
||||
});
|
||||
|
||||
it("rejects failureDestination on main jobs without webhook delivery mode", () => {
|
||||
const job = createMainSystemEventJob("job-main-failure-dest", {
|
||||
mode: "announce",
|
||||
channel: "telegram",
|
||||
to: "123",
|
||||
failureDestination: {
|
||||
mode: "announce",
|
||||
channel: "telegram",
|
||||
to: "999",
|
||||
},
|
||||
});
|
||||
|
||||
expect(() => applyJobPatch(job, { enabled: true })).toThrow(
|
||||
'cron delivery.failureDestination is only supported for sessionTarget="isolated" unless delivery.mode="webhook"',
|
||||
);
|
||||
});
|
||||
|
||||
it("validates and trims webhook failureDestination target URLs", () => {
|
||||
const expectedError =
|
||||
"cron failure destination webhook requires delivery.failureDestination.to to be a valid http(s) URL";
|
||||
const job = createIsolatedAgentTurnJob("job-failure-webhook-target", {
|
||||
mode: "announce",
|
||||
channel: "telegram",
|
||||
to: "123",
|
||||
failureDestination: {
|
||||
mode: "webhook",
|
||||
to: "not-a-url",
|
||||
},
|
||||
});
|
||||
|
||||
expect(() => applyJobPatch(job, { enabled: true })).toThrow(expectedError);
|
||||
|
||||
job.delivery = {
|
||||
mode: "announce",
|
||||
channel: "telegram",
|
||||
to: "123",
|
||||
failureDestination: {
|
||||
mode: "webhook",
|
||||
to: " https://example.invalid/failure ",
|
||||
},
|
||||
};
|
||||
expect(() => applyJobPatch(job, { enabled: true })).not.toThrow();
|
||||
expect(job.delivery?.failureDestination?.to).toBe("https://example.invalid/failure");
|
||||
});
|
||||
|
||||
it("rejects Telegram delivery with invalid target (chatId/topicId format)", () => {
|
||||
const job = createIsolatedAgentTurnJob("job-telegram-invalid", {
|
||||
mode: "announce",
|
||||
@@ -365,6 +410,25 @@ describe("createJob rejects sessionTarget main for non-default agents", () => {
|
||||
}),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("rejects failureDestination on main jobs without webhook delivery mode", () => {
|
||||
const state = createMockState(now, { defaultAgentId: "main" });
|
||||
expect(() =>
|
||||
createJob(state, {
|
||||
...mainJobInput("main"),
|
||||
delivery: {
|
||||
mode: "announce",
|
||||
channel: "telegram",
|
||||
to: "123",
|
||||
failureDestination: {
|
||||
mode: "announce",
|
||||
channel: "signal",
|
||||
to: "+15550001111",
|
||||
},
|
||||
},
|
||||
}),
|
||||
).toThrow('cron channel delivery config is only supported for sessionTarget="isolated"');
|
||||
});
|
||||
});
|
||||
|
||||
describe("applyJobPatch rejects sessionTarget main for non-default agents", () => {
|
||||
|
||||
@@ -151,6 +151,27 @@ function assertDeliverySupport(job: Pick<CronJob, "sessionTarget" | "delivery">)
|
||||
}
|
||||
}
|
||||
|
||||
function assertFailureDestinationSupport(job: Pick<CronJob, "sessionTarget" | "delivery">) {
|
||||
const failureDestination = job.delivery?.failureDestination;
|
||||
if (!failureDestination) {
|
||||
return;
|
||||
}
|
||||
if (job.sessionTarget === "main" && job.delivery?.mode !== "webhook") {
|
||||
throw new Error(
|
||||
'cron delivery.failureDestination is only supported for sessionTarget="isolated" unless delivery.mode="webhook"',
|
||||
);
|
||||
}
|
||||
if (failureDestination.mode === "webhook") {
|
||||
const target = normalizeHttpWebhookUrl(failureDestination.to);
|
||||
if (!target) {
|
||||
throw new Error(
|
||||
"cron failure destination webhook requires delivery.failureDestination.to to be a valid http(s) URL",
|
||||
);
|
||||
}
|
||||
failureDestination.to = target;
|
||||
}
|
||||
}
|
||||
|
||||
export function findJobOrThrow(state: CronServiceState, id: string) {
|
||||
const job = state.store?.jobs.find((j) => j.id === id);
|
||||
if (!job) {
|
||||
@@ -452,6 +473,7 @@ export function createJob(state: CronServiceState, input: CronJobCreate): CronJo
|
||||
assertSupportedJobSpec(job);
|
||||
assertMainSessionAgentId(job, state.deps.defaultAgentId);
|
||||
assertDeliverySupport(job);
|
||||
assertFailureDestinationSupport(job);
|
||||
job.state.nextRunAtMs = computeJobNextRunAtMs(job, now);
|
||||
return job;
|
||||
}
|
||||
@@ -517,6 +539,15 @@ export function applyJobPatch(
|
||||
if ("failureAlert" in patch) {
|
||||
job.failureAlert = mergeCronFailureAlert(job.failureAlert, patch.failureAlert);
|
||||
}
|
||||
if (
|
||||
job.sessionTarget === "main" &&
|
||||
job.delivery?.mode !== "webhook" &&
|
||||
job.delivery?.failureDestination
|
||||
) {
|
||||
throw new Error(
|
||||
'cron delivery.failureDestination is only supported for sessionTarget="isolated" unless delivery.mode="webhook"',
|
||||
);
|
||||
}
|
||||
if (job.sessionTarget === "main" && job.delivery?.mode !== "webhook") {
|
||||
job.delivery = undefined;
|
||||
}
|
||||
@@ -532,6 +563,7 @@ export function applyJobPatch(
|
||||
assertSupportedJobSpec(job);
|
||||
assertMainSessionAgentId(job, opts?.defaultAgentId);
|
||||
assertDeliverySupport(job);
|
||||
assertFailureDestinationSupport(job);
|
||||
}
|
||||
|
||||
function mergeCronPayload(existing: CronPayload, patch: CronPayloadPatch): CronPayload {
|
||||
@@ -668,6 +700,7 @@ function mergeCronDelivery(
|
||||
to: existing?.to,
|
||||
accountId: existing?.accountId,
|
||||
bestEffort: existing?.bestEffort,
|
||||
failureDestination: existing?.failureDestination,
|
||||
};
|
||||
|
||||
if (typeof patch.mode === "string") {
|
||||
@@ -685,6 +718,39 @@ function mergeCronDelivery(
|
||||
if (typeof patch.bestEffort === "boolean") {
|
||||
next.bestEffort = patch.bestEffort;
|
||||
}
|
||||
if ("failureDestination" in patch) {
|
||||
if (patch.failureDestination === undefined) {
|
||||
next.failureDestination = undefined;
|
||||
} else {
|
||||
const existingFd = next.failureDestination;
|
||||
const patchFd = patch.failureDestination;
|
||||
const nextFd: typeof next.failureDestination = {
|
||||
channel: existingFd?.channel,
|
||||
to: existingFd?.to,
|
||||
accountId: existingFd?.accountId,
|
||||
mode: existingFd?.mode,
|
||||
};
|
||||
if (patchFd) {
|
||||
if ("channel" in patchFd) {
|
||||
const channel = typeof patchFd.channel === "string" ? patchFd.channel.trim() : "";
|
||||
nextFd.channel = channel ? channel : undefined;
|
||||
}
|
||||
if ("to" in patchFd) {
|
||||
const to = typeof patchFd.to === "string" ? patchFd.to.trim() : "";
|
||||
nextFd.to = to ? to : undefined;
|
||||
}
|
||||
if ("accountId" in patchFd) {
|
||||
const accountId = typeof patchFd.accountId === "string" ? patchFd.accountId.trim() : "";
|
||||
nextFd.accountId = accountId ? accountId : undefined;
|
||||
}
|
||||
if ("mode" in patchFd) {
|
||||
const mode = typeof patchFd.mode === "string" ? patchFd.mode.trim() : "";
|
||||
nextFd.mode = mode === "announce" || mode === "webhook" ? mode : undefined;
|
||||
}
|
||||
}
|
||||
next.failureDestination = nextFd;
|
||||
}
|
||||
}
|
||||
|
||||
return next;
|
||||
}
|
||||
@@ -719,6 +785,14 @@ function mergeCronFailureAlert(
|
||||
: -1;
|
||||
next.cooldownMs = cooldownMs >= 0 ? Math.floor(cooldownMs) : undefined;
|
||||
}
|
||||
if ("mode" in patch) {
|
||||
const mode = typeof patch.mode === "string" ? patch.mode.trim() : "";
|
||||
next.mode = mode === "announce" || mode === "webhook" ? mode : undefined;
|
||||
}
|
||||
if ("accountId" in patch) {
|
||||
const accountId = typeof patch.accountId === "string" ? patch.accountId.trim() : "";
|
||||
next.accountId = accountId ? accountId : undefined;
|
||||
}
|
||||
|
||||
return next;
|
||||
}
|
||||
|
||||
@@ -96,6 +96,8 @@ export type CronServiceDeps = {
|
||||
text: string;
|
||||
channel: CronMessageChannel;
|
||||
to?: string;
|
||||
mode?: "announce" | "webhook";
|
||||
accountId?: string;
|
||||
}) => Promise<void>;
|
||||
onEvent?: (evt: CronEvent) => void;
|
||||
};
|
||||
|
||||
@@ -188,7 +188,14 @@ function clampNonNegativeInt(value: unknown, fallback: number): number {
|
||||
function resolveFailureAlert(
|
||||
state: CronServiceState,
|
||||
job: CronJob,
|
||||
): { after: number; cooldownMs: number; channel: CronMessageChannel; to?: string } | null {
|
||||
): {
|
||||
after: number;
|
||||
cooldownMs: number;
|
||||
channel: CronMessageChannel;
|
||||
to?: string;
|
||||
mode?: "announce" | "webhook";
|
||||
accountId?: string;
|
||||
} | null {
|
||||
const globalConfig = state.deps.cronConfig?.failureAlert;
|
||||
const jobConfig = job.failureAlert === false ? undefined : job.failureAlert;
|
||||
|
||||
@@ -199,6 +206,9 @@ function resolveFailureAlert(
|
||||
return null;
|
||||
}
|
||||
|
||||
const mode = jobConfig?.mode ?? globalConfig?.mode;
|
||||
const explicitTo = normalizeTo(jobConfig?.to);
|
||||
|
||||
return {
|
||||
after: clampPositiveInt(jobConfig?.after ?? globalConfig?.after, DEFAULT_FAILURE_ALERT_AFTER),
|
||||
cooldownMs: clampNonNegativeInt(
|
||||
@@ -209,7 +219,9 @@ function resolveFailureAlert(
|
||||
normalizeCronMessageChannel(jobConfig?.channel) ??
|
||||
normalizeCronMessageChannel(job.delivery?.channel) ??
|
||||
"last",
|
||||
to: normalizeTo(jobConfig?.to) ?? normalizeTo(job.delivery?.to),
|
||||
to: mode === "webhook" ? explicitTo : (explicitTo ?? normalizeTo(job.delivery?.to)),
|
||||
mode,
|
||||
accountId: jobConfig?.accountId ?? globalConfig?.accountId,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -221,6 +233,8 @@ function emitFailureAlert(
|
||||
consecutiveErrors: number;
|
||||
channel: CronMessageChannel;
|
||||
to?: string;
|
||||
mode?: "announce" | "webhook";
|
||||
accountId?: string;
|
||||
},
|
||||
) {
|
||||
const safeJobName = params.job.name || params.job.id;
|
||||
@@ -237,6 +251,8 @@ function emitFailureAlert(
|
||||
text,
|
||||
channel: params.channel,
|
||||
to: params.to,
|
||||
mode: params.mode,
|
||||
accountId: params.accountId,
|
||||
})
|
||||
.catch((err) => {
|
||||
state.deps.log.warn(
|
||||
@@ -287,19 +303,26 @@ export function applyJobResult(
|
||||
job.state.consecutiveErrors = (job.state.consecutiveErrors ?? 0) + 1;
|
||||
const alertConfig = resolveFailureAlert(state, job);
|
||||
if (alertConfig && job.state.consecutiveErrors >= alertConfig.after) {
|
||||
const now = state.deps.nowMs();
|
||||
const lastAlert = job.state.lastFailureAlertAtMs;
|
||||
const inCooldown =
|
||||
typeof lastAlert === "number" && now - lastAlert < Math.max(0, alertConfig.cooldownMs);
|
||||
if (!inCooldown) {
|
||||
emitFailureAlert(state, {
|
||||
job,
|
||||
error: result.error,
|
||||
consecutiveErrors: job.state.consecutiveErrors,
|
||||
channel: alertConfig.channel,
|
||||
to: alertConfig.to,
|
||||
});
|
||||
job.state.lastFailureAlertAtMs = now;
|
||||
const isBestEffort =
|
||||
job.delivery?.bestEffort === true ||
|
||||
(job.payload.kind === "agentTurn" && job.payload.bestEffortDeliver === true);
|
||||
if (!isBestEffort) {
|
||||
const now = state.deps.nowMs();
|
||||
const lastAlert = job.state.lastFailureAlertAtMs;
|
||||
const inCooldown =
|
||||
typeof lastAlert === "number" && now - lastAlert < Math.max(0, alertConfig.cooldownMs);
|
||||
if (!inCooldown) {
|
||||
emitFailureAlert(state, {
|
||||
job,
|
||||
error: result.error,
|
||||
consecutiveErrors: job.state.consecutiveErrors,
|
||||
channel: alertConfig.channel,
|
||||
to: alertConfig.to,
|
||||
mode: alertConfig.mode,
|
||||
accountId: alertConfig.accountId,
|
||||
});
|
||||
job.state.lastFailureAlertAtMs = now;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
|
||||
@@ -25,6 +25,15 @@ export type CronDelivery = {
|
||||
/** Explicit channel account id for multi-account setups (e.g. multiple Telegram bots). */
|
||||
accountId?: string;
|
||||
bestEffort?: boolean;
|
||||
/** Separate destination for failure notifications. */
|
||||
failureDestination?: CronFailureDestination;
|
||||
};
|
||||
|
||||
export type CronFailureDestination = {
|
||||
channel?: CronMessageChannel;
|
||||
to?: string;
|
||||
accountId?: string;
|
||||
mode?: "announce" | "webhook";
|
||||
};
|
||||
|
||||
export type CronDeliveryPatch = Partial<CronDelivery>;
|
||||
@@ -61,6 +70,10 @@ export type CronFailureAlert = {
|
||||
channel?: CronMessageChannel;
|
||||
to?: string;
|
||||
cooldownMs?: number;
|
||||
/** Delivery mode: announce (via messaging channels) or webhook (HTTP POST). */
|
||||
mode?: "announce" | "webhook";
|
||||
/** Account ID for multi-account channel configurations. */
|
||||
accountId?: string;
|
||||
};
|
||||
|
||||
export type CronPayload =
|
||||
|
||||
Reference in New Issue
Block a user