feat: 添加用户消息串行队列功能，防止同账户并发请求触发限流

- 新增 userMessageQueueService.js 实现基于 Redis 的队列锁机制 - 在 claudeRelayService、claudeConsoleRelayService、bedrockRelayService、ccrRelayService 中集成队列锁 - 添加 Redis 原子性 Lua 脚本：acquireUserMessageLock、releaseUserMessageLock、refreshUserMessageLock - 支持锁续租机制，防止长时间请求锁过期 - 添加可配置参数：USER_MESSAGE_QUEUE_ENABLED、USER_MESSAGE_QUEUE_DELAY_MS、USER_MESSAGE_QUEUE_TIMEOUT_MS - 添加 Web 管理界面配置入口 - 添加 logger.performance 方法用于结构化性能日志 - 添加完整单元测试 (tests/userMessageQueue.test.js)
2026-01-23 09:38:02 +00:00 · 2025-12-09 17:04:01 +08:00
parent 95870883a1
commit f5d1c25295
14 changed files with 2048 additions and 18 deletions
--- a/src/models/redis.js
+++ b/src/models/redis.js
@@ -2556,4 +2556,249 @@ redisClient.getDateStringInTimezone = getDateStringInTimezone
 redisClient.getHourInTimezone = getHourInTimezone
 redisClient.getWeekStringInTimezone = getWeekStringInTimezone

+// ============== 用户消息队列相关方法 ==============
+
+/**
+ * 尝试获取用户消息队列锁
+ * 使用 Lua 脚本保证原子性
+ * @param {string} accountId - 账户ID
+ * @param {string} requestId - 请求ID
+ * @param {number} lockTtlMs - 锁 TTL（毫秒）
+ * @param {number} delayMs - 请求间隔（毫秒）
+ * @returns {Promise<{acquired: boolean, waitMs: number}>}
+ *   - acquired: 是否成功获取锁
+ *   - waitMs: 需要等待的毫秒数（-1表示被占用需等待，>=0表示需要延迟的毫秒数）
+ */
+redisClient.acquireUserMessageLock = async function (accountId, requestId, lockTtlMs, delayMs) {
+  const lockKey = `user_msg_queue_lock:${accountId}`
+  const lastTimeKey = `user_msg_queue_last:${accountId}`
+
+  const script = `
+    local lockKey = KEYS[1]
+    local lastTimeKey = KEYS[2]
+    local requestId = ARGV[1]
+    local lockTtl = tonumber(ARGV[2])
+    local delayMs = tonumber(ARGV[3])
+
+    -- 检查锁是否空闲
+    local currentLock = redis.call('GET', lockKey)
+    if currentLock == false then
+      -- 检查是否需要延迟
+      local lastTime = redis.call('GET', lastTimeKey)
+      local now = redis.call('TIME')
+      local nowMs = tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000)
+
+      if lastTime then
+        local elapsed = nowMs - tonumber(lastTime)
+        if elapsed < delayMs then
+          -- 需要等待的毫秒数
+          return {0, delayMs - elapsed}
+        end
+      end
+
+      -- 获取锁
+      redis.call('SET', lockKey, requestId, 'PX', lockTtl)
+      return {1, 0}
+    end
+
+    -- 锁被占用，返回等待
+    return {0, -1}
+  `
+
+  try {
+    const result = await this.client.eval(
+      script,
+      2,
+      lockKey,
+      lastTimeKey,
+      requestId,
+      lockTtlMs,
+      delayMs
+    )
+    return {
+      acquired: result[0] === 1,
+      waitMs: result[1]
+    }
+  } catch (error) {
+    logger.error(`Failed to acquire user message lock for account ${accountId}:`, error)
+    // 返回 redisError 标记，让上层能区分 Redis 故障和正常锁占用
+    return { acquired: false, waitMs: -1, redisError: true, errorMessage: error.message }
+  }
+}
+
+/**
+ * 续租用户消息队列锁（仅锁持有者可续租）
+ * @param {string} accountId - 账户ID
+ * @param {string} requestId - 请求ID
+ * @param {number} lockTtlMs - 锁 TTL（毫秒）
+ * @returns {Promise<boolean>} 是否续租成功（只有锁持有者才能续租）
+ */
+redisClient.refreshUserMessageLock = async function (accountId, requestId, lockTtlMs) {
+  const lockKey = `user_msg_queue_lock:${accountId}`
+
+  const script = `
+    local lockKey = KEYS[1]
+    local requestId = ARGV[1]
+    local lockTtl = tonumber(ARGV[2])
+
+    local currentLock = redis.call('GET', lockKey)
+    if currentLock == requestId then
+      redis.call('PEXPIRE', lockKey, lockTtl)
+      return 1
+    end
+    return 0
+  `
+
+  try {
+    const result = await this.client.eval(script, 1, lockKey, requestId, lockTtlMs)
+    return result === 1
+  } catch (error) {
+    logger.error(`Failed to refresh user message lock for account ${accountId}:`, error)
+    return false
+  }
+}
+
+/**
+ * 释放用户消息队列锁并记录完成时间
+ * @param {string} accountId - 账户ID
+ * @param {string} requestId - 请求ID
+ * @returns {Promise<boolean>} 是否成功释放
+ */
+redisClient.releaseUserMessageLock = async function (accountId, requestId) {
+  const lockKey = `user_msg_queue_lock:${accountId}`
+  const lastTimeKey = `user_msg_queue_last:${accountId}`
+
+  const script = `
+    local lockKey = KEYS[1]
+    local lastTimeKey = KEYS[2]
+    local requestId = ARGV[1]
+
+    -- 验证锁持有者
+    local currentLock = redis.call('GET', lockKey)
+    if currentLock == requestId then
+      -- 记录完成时间
+      local now = redis.call('TIME')
+      local nowMs = tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000)
+      redis.call('SET', lastTimeKey, nowMs, 'EX', 60)  -- 60秒后过期
+
+      -- 删除锁
+      redis.call('DEL', lockKey)
+      return 1
+    end
+    return 0
+  `
+
+  try {
+    const result = await this.client.eval(script, 2, lockKey, lastTimeKey, requestId)
+    return result === 1
+  } catch (error) {
+    logger.error(`Failed to release user message lock for account ${accountId}:`, error)
+    return false
+  }
+}
+
+/**
+ * 强制释放用户消息队列锁（用于清理孤儿锁）
+ * @param {string} accountId - 账户ID
+ * @returns {Promise<boolean>} 是否成功释放
+ */
+redisClient.forceReleaseUserMessageLock = async function (accountId) {
+  const lockKey = `user_msg_queue_lock:${accountId}`
+
+  try {
+    await this.client.del(lockKey)
+    return true
+  } catch (error) {
+    logger.error(`Failed to force release user message lock for account ${accountId}:`, error)
+    return false
+  }
+}
+
+/**
+ * 获取用户消息队列统计信息（用于调试）
+ * @param {string} accountId - 账户ID
+ * @returns {Promise<Object>} 队列统计
+ */
+redisClient.getUserMessageQueueStats = async function (accountId) {
+  const lockKey = `user_msg_queue_lock:${accountId}`
+  const lastTimeKey = `user_msg_queue_last:${accountId}`
+
+  try {
+    const [lockHolder, lastTime, lockTtl] = await Promise.all([
+      this.client.get(lockKey),
+      this.client.get(lastTimeKey),
+      this.client.pttl(lockKey)
+    ])
+
+    return {
+      accountId,
+      isLocked: !!lockHolder,
+      lockHolder,
+      lockTtlMs: lockTtl > 0 ? lockTtl : 0,
+      lockTtlRaw: lockTtl, // 原始 PTTL 值：>0 有TTL，-1 无过期时间，-2 键不存在
+      lastCompletedAt: lastTime ? new Date(parseInt(lastTime)).toISOString() : null
+    }
+  } catch (error) {
+    logger.error(`Failed to get user message queue stats for account ${accountId}:`, error)
+    return {
+      accountId,
+      isLocked: false,
+      lockHolder: null,
+      lockTtlMs: 0,
+      lockTtlRaw: -2,
+      lastCompletedAt: null
+    }
+  }
+}
+
+/**
+ * 扫描所有用户消息队列锁（用于清理任务）
+ * @returns {Promise<string[]>} 账户ID列表
+ */
+redisClient.scanUserMessageQueueLocks = async function () {
+  const accountIds = []
+  let cursor = '0'
+  let iterations = 0
+  const MAX_ITERATIONS = 1000 // 防止无限循环
+
+  try {
+    do {
+      const [newCursor, keys] = await this.client.scan(
+        cursor,
+        'MATCH',
+        'user_msg_queue_lock:*',
+        'COUNT',
+        100
+      )
+      cursor = newCursor
+      iterations++
+
+      for (const key of keys) {
+        const accountId = key.replace('user_msg_queue_lock:', '')
+        accountIds.push(accountId)
+      }
+
+      // 防止无限循环
+      if (iterations >= MAX_ITERATIONS) {
+        logger.warn(
+          `📬 User message queue: SCAN reached max iterations (${MAX_ITERATIONS}), stopping early`,
+          { foundLocks: accountIds.length }
+        )
+        break
+      }
+    } while (cursor !== '0')
+
+    if (accountIds.length > 0) {
+      logger.debug(
+        `📬 User message queue: scanned ${accountIds.length} lock(s) in ${iterations} iteration(s)`
+      )
+    }
+
+    return accountIds
+  } catch (error) {
+    logger.error('Failed to scan user message queue locks:', error)
+    return []
+  }
+}
+
 module.exports = redisClient