feat: droid apikey异常自动移除

src/services/droidAccountService.js

@@ -308,6 +308,46 @@ class DroidAccountService {
     }
   }
 
+  /**
+   * 删除指定的 Droid API Key 条目
+   */
+  async removeApiKeyEntry(accountId, keyId) {
+    if (!accountId || !keyId) {
+      return { removed: false, remainingCount: 0 }
+    }
+
+    try {
+      const accountData = await redis.getDroidAccount(accountId)
+      if (!accountData) {
+        return { removed: false, remainingCount: 0 }
+      }
+
+      const entries = this._parseApiKeyEntries(accountData.apiKeys)
+      if (!entries || entries.length === 0) {
+        return { removed: false, remainingCount: 0 }
+      }
+
+      const filtered = entries.filter((entry) => entry && entry.id !== keyId)
+      if (filtered.length === entries.length) {
+        return { removed: false, remainingCount: entries.length }
+      }
+
+      accountData.apiKeys = filtered.length ? JSON.stringify(filtered) : ''
+      accountData.apiKeyCount = String(filtered.length)
+
+      await redis.setDroidAccount(accountId, accountData)
+
+      logger.warn(
+        `🚫 已删除 Droid API Key ${keyId}（Account: ${accountId}），剩余 ${filtered.length}`
+      )
+
+      return { removed: true, remainingCount: filtered.length }
+    } catch (error) {
+      logger.error(`❌ 删除 Droid API Key 失败：${keyId}（Account: ${accountId}）`, error)
+      return { removed: false, remainingCount: 0, error }
+    }
+  }
+
   /**
    * 使用 WorkOS Refresh Token 刷新并验证凭证
    */

src/services/droidRelayService.js

@@ -192,8 +192,12 @@ class DroidRelayService {
       disableStreaming = false
     } = options
     const keyInfo = apiKeyData || {}
+    const clientApiKeyId = keyInfo.id || null
     const normalizedEndpoint = this._normalizeEndpointType(endpointType)
     const normalizedRequestBody = this._normalizeRequestBody(requestBody, normalizedEndpoint)
+    let account = null
+    let selectedApiKey = null
+    let accessToken = null
 
     try {
       logger.info(
@@ -203,16 +207,13 @@ class DroidRelayService {
       )
 
       // 选择一个可用的 Droid 账户（支持粘性会话和分组调度）
-      const account = await droidScheduler.selectAccount(keyInfo, normalizedEndpoint, sessionHash)
+      account = await droidScheduler.selectAccount(keyInfo, normalizedEndpoint, sessionHash)
 
       if (!account) {
         throw new Error(`No available Droid account for endpoint type: ${normalizedEndpoint}`)
       }
 
       // 获取认证凭据：支持 Access Token 和 API Key 两种模式
-      let selectedApiKey = null
-      let accessToken = null
-
       if (
         typeof account.authenticationMethod === 'string' &&
         account.authenticationMethod.toLowerCase().trim() === 'api_key'
@@ -281,7 +282,10 @@ class DroidRelayService {
           keyInfo,
           normalizedRequestBody,
           normalizedEndpoint,
-          skipUsageRecord
+          skipUsageRecord,
+          selectedApiKey,
+          sessionHash,
+          clientApiKeyId
         )
       } else {
         // 非流式响应：使用 axios
@@ -316,6 +320,21 @@ class DroidRelayService {
     } catch (error) {
       logger.error(`❌ Droid relay error: ${error.message}`, error)
 
+      const status = error?.response?.status
+      if (status >= 400 && status < 500) {
+        try {
+          await this._handleUpstreamClientError(status, {
+            account,
+            selectedAccountApiKey: selectedApiKey,
+            endpointType: normalizedEndpoint,
+            sessionHash,
+            clientApiKeyId
+          })
+        } catch (handlingError) {
+          logger.error('❌ 处理 Droid 4xx 异常失败:', handlingError)
+        }
+      }
+
       if (error.response) {
         // HTTP 错误响应
         return {
@@ -354,7 +373,10 @@ class DroidRelayService {
     apiKeyData,
     requestBody,
     endpointType,
-    skipUsageRecord = false
+    skipUsageRecord = false,
+    selectedAccountApiKey = null,
+    sessionHash = null,
+    clientApiKeyId = null
   ) {
     return new Promise((resolve, reject) => {
       const url = new URL(apiUrl)
@@ -470,6 +492,17 @@ class DroidRelayService {
             logger.info('✅ res.end() reached')
             const body = Buffer.concat(chunks).toString()
             logger.error(`❌ Factory.ai error response body: ${body || '(empty)'}`)
+            if (res.statusCode >= 400 && res.statusCode < 500) {
+              this._handleUpstreamClientError(res.statusCode, {
+                account,
+                selectedAccountApiKey,
+                endpointType,
+                sessionHash,
+                clientApiKeyId
+              }).catch((handlingError) => {
+                logger.error('❌ 处理 Droid 流式4xx 异常失败:', handlingError)
+              })
+            }
             if (!clientResponse.headersSent) {
               clientResponse.status(res.statusCode).json({
                 error: 'upstream_error',
@@ -1123,6 +1156,152 @@ class DroidRelayService {
     }
   }
 
+  /**
+   * 处理上游 4xx 响应，移除问题 API Key 或停止账号调度
+   */
+  async _handleUpstreamClientError(statusCode, context = {}) {
+    if (!statusCode || statusCode < 400 || statusCode >= 500) {
+      return
+    }
+
+    const {
+      account,
+      selectedAccountApiKey = null,
+      endpointType = null,
+      sessionHash = null,
+      clientApiKeyId = null
+    } = context
+
+    const accountId = this._extractAccountId(account)
+    if (!accountId) {
+      logger.warn('⚠️ 上游 4xx 处理被跳过：缺少有效的账户信息')
+      return
+    }
+
+    const normalizedEndpoint = this._normalizeEndpointType(
+      endpointType || account?.endpointType || 'anthropic'
+    )
+    const authMethod =
+      typeof account?.authenticationMethod === 'string'
+        ? account.authenticationMethod.toLowerCase().trim()
+        : ''
+
+    if (authMethod === 'api_key') {
+      if (selectedAccountApiKey?.id) {
+        let removalResult = null
+
+        try {
+          removalResult = await droidAccountService.removeApiKeyEntry(
+            accountId,
+            selectedAccountApiKey.id
+          )
+        } catch (error) {
+          logger.error(
+            `❌ 移除 Droid API Key ${selectedAccountApiKey.id}（Account: ${accountId}）失败：`,
+            error
+          )
+        }
+
+        await this._clearApiKeyStickyMapping(accountId, normalizedEndpoint, sessionHash)
+
+        if (removalResult?.removed) {
+          logger.warn(
+            `🚫 上游返回 ${statusCode}，已移除 Droid API Key ${selectedAccountApiKey.id}（Account: ${accountId}）`
+          )
+        } else {
+          logger.warn(
+            `⚠️ 上游返回 ${statusCode}，但未能移除 Droid API Key ${selectedAccountApiKey.id}（Account: ${accountId}）`
+          )
+        }
+
+        if (!removalResult || removalResult.remainingCount === 0) {
+          await this._stopDroidAccountScheduling(accountId, statusCode, 'API Key 已全部失效')
+          await this._clearAccountStickyMapping(normalizedEndpoint, sessionHash, clientApiKeyId)
+        } else {
+          logger.info(
+            `ℹ️ Droid 账号 ${accountId} 仍有 ${removalResult.remainingCount} 个 API Key 可用`
+          )
+        }
+
+        return
+      }
+
+      logger.warn(
+        `⚠️ 上游返回 ${statusCode}，但未获取到对应的 Droid API Key（Account: ${accountId}）`
+      )
+      await this._stopDroidAccountScheduling(accountId, statusCode, '缺少可用 API Key')
+      await this._clearAccountStickyMapping(normalizedEndpoint, sessionHash, clientApiKeyId)
+      return
+    }
+
+    await this._stopDroidAccountScheduling(accountId, statusCode, '凭证不可用')
+    await this._clearAccountStickyMapping(normalizedEndpoint, sessionHash, clientApiKeyId)
+  }
+
+  /**
+   * 停止指定 Droid 账号的调度
+   */
+  async _stopDroidAccountScheduling(accountId, statusCode, reason = '') {
+    if (!accountId) {
+      return
+    }
+
+    const message = reason ? `${reason}` : '上游返回 4xx 错误'
+
+    try {
+      await droidAccountService.updateAccount(accountId, {
+        schedulable: 'false',
+        status: 'error',
+        errorMessage: `上游返回 ${statusCode}：${message}`
+      })
+      logger.warn(`🚫 已停止调度 Droid 账号 ${accountId}（状态码 ${statusCode}，原因：${message}）`)
+    } catch (error) {
+      logger.error(`❌ 停止调度 Droid 账号失败：${accountId}`, error)
+    }
+  }
+
+  /**
+   * 清理账号层面的粘性调度映射
+   */
+  async _clearAccountStickyMapping(endpointType, sessionHash, clientApiKeyId) {
+    if (!sessionHash) {
+      return
+    }
+
+    const normalizedEndpoint = this._normalizeEndpointType(endpointType)
+    const apiKeyPart = clientApiKeyId || 'default'
+    const stickyKey = `droid:${normalizedEndpoint}:${apiKeyPart}:${sessionHash}`
+
+    try {
+      await redis.deleteSessionAccountMapping(stickyKey)
+      logger.debug(`🧹 已清理 Droid 粘性会话映射：${stickyKey}`)
+    } catch (error) {
+      logger.warn(`⚠️ 清理 Droid 粘性会话映射失败：${stickyKey}`, error)
+    }
+  }
+
+  /**
+   * 清理 API Key 级别的粘性映射
+   */
+  async _clearApiKeyStickyMapping(accountId, endpointType, sessionHash) {
+    if (!accountId || !sessionHash) {
+      return
+    }
+
+    try {
+      const stickyKey = this._composeApiKeyStickyKey(accountId, endpointType, sessionHash)
+      if (stickyKey) {
+        await redis.deleteSessionAccountMapping(stickyKey)
+        logger.debug(`🧹 已清理 Droid API Key 粘性映射：${stickyKey}`)
+      }
+    } catch (error) {
+      logger.warn(
+        `⚠️ 清理 Droid API Key 粘性映射失败：${accountId}（endpoint: ${endpointType}）`,
+        error
+      )
+    }
+  }
+
   _mapNetworkErrorStatus(error) {
     const code = (error && error.code ? String(error.code) : '').toUpperCase()
 

web/admin-spa/src/components/accounts/AccountForm.vue

@@ -1773,6 +1773,9 @@
                 <ul class="mt-1 list-disc space-y-1 pl-4">
                   <li>新会话将随机命中一个 Key，并在会话有效期内保持粘性。</li>
                   <li>若某 Key 失效，会自动切换到剩余可用 Key，最大化成功率。</li>
+                  <li>
+                    若上游返回 4xx 错误码，该 Key 会被自动移除；全部 Key 清空后账号将暂停调度。
+                  </li>
                 </ul>
               </div>
             </div>