mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-08 09:01:22 +00:00
fix: add exponential backoff to announce queue drain on failure (#24783)
When the gateway rejects connections (e.g. scope-upgrade 'pairing required'), the announce queue drain loop would retry every ~1s indefinitely because the only delay was the fixed debounceMs (default 1000ms). This adds a consecutiveFailures counter with exponential backoff: 2s, 4s, 8s, 16s, 32s, 60s (capped). The counter resets on successful drain. The backoff is applied by shifting lastEnqueuedAt forward so that waitForQueueDebounce naturally delays the next attempt. Fixes #24777 Co-authored-by: Knut <knut@Knut-sin-Mac-mini.local>
This commit is contained in:
@@ -48,6 +48,8 @@ type AnnounceQueueState = {
|
||||
droppedCount: number;
|
||||
summaryLines: string[];
|
||||
send: (item: AnnounceQueueItem) => Promise<void>;
|
||||
/** Consecutive drain failures — drives exponential backoff on errors. */
|
||||
consecutiveFailures: number;
|
||||
};
|
||||
|
||||
const ANNOUNCE_QUEUES = new Map<string, AnnounceQueueState>();
|
||||
@@ -89,6 +91,7 @@ function getAnnounceQueue(
|
||||
droppedCount: 0,
|
||||
summaryLines: [],
|
||||
send,
|
||||
consecutiveFailures: 0,
|
||||
};
|
||||
applyQueueRuntimeSettings({
|
||||
target: created,
|
||||
@@ -174,10 +177,16 @@ function scheduleAnnounceDrain(key: string) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Drain succeeded — reset failure counter.
|
||||
queue.consecutiveFailures = 0;
|
||||
} catch (err) {
|
||||
// Keep items in queue and retry after debounce; avoid hot-loop retries.
|
||||
queue.lastEnqueuedAt = Date.now();
|
||||
defaultRuntime.error?.(`announce queue drain failed for ${key}: ${String(err)}`);
|
||||
queue.consecutiveFailures++;
|
||||
// Exponential backoff on consecutive failures: 2s, 4s, 8s, ... capped at 60s.
|
||||
const errorBackoffMs = Math.min(1000 * Math.pow(2, queue.consecutiveFailures), 60_000);
|
||||
queue.lastEnqueuedAt = Date.now() + errorBackoffMs - queue.debounceMs;
|
||||
defaultRuntime.error?.(
|
||||
`announce queue drain failed for ${key} (attempt ${queue.consecutiveFailures}, retry in ${Math.round(errorBackoffMs / 1000)}s): ${String(err)}`,
|
||||
);
|
||||
} finally {
|
||||
queue.draining = false;
|
||||
if (queue.items.length === 0 && queue.droppedCount === 0) {
|
||||
|
||||
Reference in New Issue
Block a user