fix: add exponential backoff to announce queue drain on failure (#24783)

When the gateway rejects connections (e.g. scope-upgrade 'pairing required'),
the announce queue drain loop would retry every ~1s indefinitely because
the only delay was the fixed debounceMs (default 1000ms).

This adds a consecutiveFailures counter with exponential backoff:
2s, 4s, 8s, 16s, 32s, 60s (capped). The counter resets on successful drain.

The backoff is applied by shifting lastEnqueuedAt forward so that
waitForQueueDebounce naturally delays the next attempt.

Fixes #24777

Co-authored-by: Knut <knut@Knut-sin-Mac-mini.local>
This commit is contained in:
banna-commits
2026-02-24 04:33:34 +01:00
committed by GitHub
parent 52ac7634db
commit e3da57d956

View File

@@ -48,6 +48,8 @@ type AnnounceQueueState = {
droppedCount: number;
summaryLines: string[];
send: (item: AnnounceQueueItem) => Promise<void>;
/** Consecutive drain failures — drives exponential backoff on errors. */
consecutiveFailures: number;
};
const ANNOUNCE_QUEUES = new Map<string, AnnounceQueueState>();
@@ -89,6 +91,7 @@ function getAnnounceQueue(
droppedCount: 0,
summaryLines: [],
send,
consecutiveFailures: 0,
};
applyQueueRuntimeSettings({
target: created,
@@ -174,10 +177,16 @@ function scheduleAnnounceDrain(key: string) {
break;
}
}
// Drain succeeded — reset failure counter.
queue.consecutiveFailures = 0;
} catch (err) {
// Keep items in queue and retry after debounce; avoid hot-loop retries.
queue.lastEnqueuedAt = Date.now();
defaultRuntime.error?.(`announce queue drain failed for ${key}: ${String(err)}`);
queue.consecutiveFailures++;
// Exponential backoff on consecutive failures: 2s, 4s, 8s, ... capped at 60s.
const errorBackoffMs = Math.min(1000 * Math.pow(2, queue.consecutiveFailures), 60_000);
queue.lastEnqueuedAt = Date.now() + errorBackoffMs - queue.debounceMs;
defaultRuntime.error?.(
`announce queue drain failed for ${key} (attempt ${queue.consecutiveFailures}, retry in ${Math.round(errorBackoffMs / 1000)}s): ${String(err)}`,
);
} finally {
queue.draining = false;
if (queue.items.length === 0 && queue.droppedCount === 0) {