diff --git a/src/app.js b/src/app.js index 31d7b7f3..67f26bfe 100644 --- a/src/app.js +++ b/src/app.js @@ -537,6 +537,15 @@ class Application { logger.info( `🔄 Cleanup tasks scheduled every ${config.system.cleanupInterval / 1000 / 60} minutes` ) + + // 🚨 启动限流状态自动清理服务 + // 每5分钟检查一次过期的限流状态,确保账号能及时恢复调度 + const rateLimitCleanupService = require('./services/rateLimitCleanupService') + const cleanupIntervalMinutes = config.system.rateLimitCleanupInterval || 5 // 默认5分钟 + rateLimitCleanupService.start(cleanupIntervalMinutes) + logger.info( + `🚨 Rate limit cleanup service started (checking every ${cleanupIntervalMinutes} minutes)` + ) } setupGracefulShutdown() { @@ -554,6 +563,15 @@ class Application { } catch (error) { logger.error('❌ Error cleaning up pricing service:', error) } + + // 停止限流清理服务 + try { + const rateLimitCleanupService = require('./services/rateLimitCleanupService') + rateLimitCleanupService.stop() + logger.info('🚨 Rate limit cleanup service stopped') + } catch (error) { + logger.error('❌ Error stopping rate limit cleanup service:', error) + } try { await redis.disconnect() diff --git a/src/middleware/auth.js b/src/middleware/auth.js index aadcf0d9..4d4364ac 100644 --- a/src/middleware/auth.js +++ b/src/middleware/auth.js @@ -1092,7 +1092,7 @@ const globalRateLimit = async (req, res, next) => // 📊 请求大小限制中间件 const requestSizeLimit = (req, res, next) => { - const maxSize = 10 * 1024 * 1024 // 10MB + const maxSize = 60 * 1024 * 1024 // 60MB const contentLength = parseInt(req.headers['content-length'] || '0') if (contentLength > maxSize) { diff --git a/src/routes/admin.js b/src/routes/admin.js index c26e613c..90024d0a 100644 --- a/src/routes/admin.js +++ b/src/routes/admin.js @@ -2015,7 +2015,9 @@ router.post('/claude-accounts', authenticateAdmin, async (req, res) => { groupId, groupIds, autoStopOnWarning, - useUnifiedUserAgent + useUnifiedUserAgent, + useUnifiedClientId, + unifiedClientId } = req.body if (!name) { @@ -2056,7 +2058,9 @@ router.post('/claude-accounts', authenticateAdmin, async (req, res) => { platform, priority: priority || 50, // 默认优先级为50 autoStopOnWarning: autoStopOnWarning === true, // 默认为false - useUnifiedUserAgent: useUnifiedUserAgent === true // 默认为false + useUnifiedUserAgent: useUnifiedUserAgent === true, // 默认为false + useUnifiedClientId: useUnifiedClientId === true, // 默认为false + unifiedClientId: unifiedClientId || '' // 统一的客户端标识 }) // 如果是分组类型,将账户添加到分组 @@ -2659,6 +2663,23 @@ router.post( } ) +// 重置Claude Console账户状态(清除所有异常状态) +router.post( + '/claude-console-accounts/:accountId/reset-status', + authenticateAdmin, + async (req, res) => { + try { + const { accountId } = req.params + const result = await claudeConsoleAccountService.resetAccountStatus(accountId) + logger.success(`✅ Admin reset status for Claude Console account: ${accountId}`) + return res.json({ success: true, data: result }) + } catch (error) { + logger.error('❌ Failed to reset Claude Console account status:', error) + return res.status(500).json({ error: 'Failed to reset status', message: error.message }) + } + } +) + // 手动重置所有Claude Console账户的每日使用量 router.post('/claude-console-accounts/reset-all-usage', authenticateAdmin, async (req, res) => { try { @@ -6146,6 +6167,21 @@ router.put('/openai-accounts/:id/toggle', authenticateAdmin, async (req, res) => } }) +// 重置 OpenAI 账户状态(清除所有异常状态) +router.post('/openai-accounts/:accountId/reset-status', authenticateAdmin, async (req, res) => { + try { + const { accountId } = req.params + + const result = await openaiAccountService.resetAccountStatus(accountId) + + logger.success(`✅ Admin reset status for OpenAI account: ${accountId}`) + return res.json({ success: true, data: result }) + } catch (error) { + logger.error('❌ Failed to reset OpenAI account status:', error) + return res.status(500).json({ error: 'Failed to reset status', message: error.message }) + } +}) + // 切换 OpenAI 账户调度状态 router.put( '/openai-accounts/:accountId/toggle-schedulable', diff --git a/src/routes/openaiRoutes.js b/src/routes/openaiRoutes.js index 283ab896..eb4a6b59 100644 --- a/src/routes/openaiRoutes.js +++ b/src/routes/openaiRoutes.js @@ -87,7 +87,8 @@ async function getOpenAIAuthToken(apiKeyData, sessionId = null, requestedModel = } } -router.post('/responses', authenticateApiKey, async (req, res) => { +// 主处理函数,供两个路由共享 +const handleResponses = async (req, res) => { let upstream = null try { // 从中间件获取 API Key 数据 @@ -205,6 +206,96 @@ router.post('/responses', authenticateApiKey, async (req, res) => { axiosConfig ) } + + // 处理 429 限流错误 + if (upstream.status === 429) { + logger.warn(`🚫 Rate limit detected for OpenAI account ${accountId} (Codex API)`) + + // 解析响应体中的限流信息 + let resetsInSeconds = null + let errorData = null + + try { + // 对于429错误,无论是否是流式请求,响应都会是完整的JSON错误对象 + if (isStream && upstream.data) { + // 流式响应需要先收集数据 + const chunks = [] + await new Promise((resolve, reject) => { + upstream.data.on('data', (chunk) => chunks.push(chunk)) + upstream.data.on('end', resolve) + upstream.data.on('error', reject) + // 设置超时防止无限等待 + setTimeout(resolve, 5000) + }) + + const fullResponse = Buffer.concat(chunks).toString() + try { + errorData = JSON.parse(fullResponse) + } catch (e) { + logger.error('Failed to parse 429 error response:', e) + logger.debug('Raw response:', fullResponse) + } + } else { + // 非流式响应直接使用data + errorData = upstream.data + } + + // 提取重置时间 + if (errorData && errorData.error && errorData.error.resets_in_seconds) { + resetsInSeconds = errorData.error.resets_in_seconds + logger.info( + `🕐 Codex rate limit will reset in ${resetsInSeconds} seconds (${Math.ceil(resetsInSeconds / 60)} minutes / ${Math.ceil(resetsInSeconds / 3600)} hours)` + ) + } else { + logger.warn( + '⚠️ Could not extract resets_in_seconds from 429 response, using default 60 minutes' + ) + } + } catch (e) { + logger.error('⚠️ Failed to parse rate limit error:', e) + } + + // 标记账户为限流状态 + await unifiedOpenAIScheduler.markAccountRateLimited( + accountId, + 'openai', + sessionId ? crypto.createHash('sha256').update(sessionId).digest('hex') : null, + resetsInSeconds + ) + + // 返回错误响应给客户端 + const errorResponse = errorData || { + error: { + type: 'usage_limit_reached', + message: 'The usage limit has been reached', + resets_in_seconds: resetsInSeconds + } + } + + if (isStream) { + // 流式响应也需要设置正确的状态码 + res.status(429) + res.setHeader('Content-Type', 'text/event-stream') + res.setHeader('Cache-Control', 'no-cache') + res.setHeader('Connection', 'keep-alive') + res.write(`data: ${JSON.stringify(errorResponse)}\n\n`) + res.end() + } else { + res.status(429).json(errorResponse) + } + + return + } else if (upstream.status === 200 || upstream.status === 201) { + // 请求成功,检查并移除限流状态 + const isRateLimited = await unifiedOpenAIScheduler.isAccountRateLimited(accountId) + if (isRateLimited) { + logger.info( + `✅ Removing rate limit for OpenAI account ${accountId} after successful request` + ) + await unifiedOpenAIScheduler.removeAccountRateLimit(accountId, 'openai') + } + } + res.status(upstream.status) if (isStream) { @@ -239,6 +330,8 @@ router.post('/responses', authenticateApiKey, async (req, res) => { let usageData = null let actualModel = null let usageReported = false + let rateLimitDetected = false + let rateLimitResetsInSeconds = null if (!isStream) { // 非流式响应处理 @@ -317,6 +410,17 @@ router.post('/responses', authenticateApiKey, async (req, res) => { logger.debug('📊 Captured OpenAI usage data:', usageData) } } + + // 检查是否有限流错误 + if (eventData.error && eventData.error.type === 'usage_limit_reached') { + rateLimitDetected = true + if (eventData.error.resets_in_seconds) { + rateLimitResetsInSeconds = eventData.error.resets_in_seconds + logger.warn( + `🚫 Rate limit detected in stream, resets in ${rateLimitResetsInSeconds} seconds` + ) + } + } } catch (e) { // 忽略解析错误 } @@ -388,6 +492,26 @@ router.post('/responses', authenticateApiKey, async (req, res) => { } } + // 如果在流式响应中检测到限流 + if (rateLimitDetected) { + logger.warn(`🚫 Processing rate limit for OpenAI account ${accountId} from stream`) + await unifiedOpenAIScheduler.markAccountRateLimited( + accountId, + 'openai', + sessionId ? crypto.createHash('sha256').update(sessionId).digest('hex') : null, + rateLimitResetsInSeconds + ) + } else if (upstream.status === 200) { + // 流式请求成功,检查并移除限流状态 + const isRateLimited = await unifiedOpenAIScheduler.isAccountRateLimited(accountId) + if (isRateLimited) { + logger.info( + `✅ Removing rate limit for OpenAI account ${accountId} after successful stream` + ) + await unifiedOpenAIScheduler.removeAccountRateLimit(accountId, 'openai') + } + } + res.end() }) @@ -419,7 +543,11 @@ router.post('/responses', authenticateApiKey, async (req, res) => { res.status(status).json({ error: { message } }) } } -}) +} + +// 注册两个路由路径,都使用相同的处理函数 +router.post('/responses', authenticateApiKey, handleResponses) +router.post('/v1/responses', authenticateApiKey, handleResponses) // 使用情况统计端点 router.get('/usage', authenticateApiKey, async (req, res) => { diff --git a/src/services/claudeAccountService.js b/src/services/claudeAccountService.js index 86e595e5..ee67b0ad 100644 --- a/src/services/claudeAccountService.js +++ b/src/services/claudeAccountService.js @@ -60,7 +60,9 @@ class ClaudeAccountService { schedulable = true, // 是否可被调度 subscriptionInfo = null, // 手动设置的订阅信息 autoStopOnWarning = false, // 5小时使用量接近限制时自动停止调度 - useUnifiedUserAgent = false // 是否使用统一Claude Code版本的User-Agent + useUnifiedUserAgent = false, // 是否使用统一Claude Code版本的User-Agent + useUnifiedClientId = false, // 是否使用统一的客户端标识 + unifiedClientId = '' // 统一的客户端标识 } = options const accountId = uuidv4() @@ -93,6 +95,8 @@ class ClaudeAccountService { schedulable: schedulable.toString(), // 是否可被调度 autoStopOnWarning: autoStopOnWarning.toString(), // 5小时使用量接近限制时自动停止调度 useUnifiedUserAgent: useUnifiedUserAgent.toString(), // 是否使用统一Claude Code版本的User-Agent + useUnifiedClientId: useUnifiedClientId.toString(), // 是否使用统一的客户端标识 + unifiedClientId: unifiedClientId || '', // 统一的客户端标识 // 优先使用手动设置的订阅信息,否则使用OAuth数据中的,否则默认为空 subscriptionInfo: subscriptionInfo ? JSON.stringify(subscriptionInfo) @@ -166,7 +170,10 @@ class ClaudeAccountService { createdAt: accountData.createdAt, expiresAt: accountData.expiresAt, scopes: claudeAiOauth ? claudeAiOauth.scopes : [], - autoStopOnWarning + autoStopOnWarning, + useUnifiedUserAgent, + useUnifiedClientId, + unifiedClientId } } @@ -492,6 +499,9 @@ class ClaudeAccountService { autoStopOnWarning: account.autoStopOnWarning === 'true', // 默认为false // 添加统一User-Agent设置 useUnifiedUserAgent: account.useUnifiedUserAgent === 'true', // 默认为false + // 添加统一客户端标识设置 + useUnifiedClientId: account.useUnifiedClientId === 'true', // 默认为false + unifiedClientId: account.unifiedClientId || '', // 统一的客户端标识 // 添加停止原因 stoppedReason: account.stoppedReason || null } @@ -528,7 +538,9 @@ class ClaudeAccountService { 'schedulable', 'subscriptionInfo', 'autoStopOnWarning', - 'useUnifiedUserAgent' + 'useUnifiedUserAgent', + 'useUnifiedClientId', + 'unifiedClientId' ] const updatedData = { ...accountData } @@ -1067,6 +1079,8 @@ class ClaudeAccountService { const updatedAccountData = { ...accountData } updatedAccountData.rateLimitedAt = new Date().toISOString() updatedAccountData.rateLimitStatus = 'limited' + // 限流时停止调度,与 OpenAI 账号保持一致 + updatedAccountData.schedulable = false // 如果提供了准确的限流重置时间戳(来自API响应头) if (rateLimitResetTimestamp) { @@ -1151,9 +1165,33 @@ class ClaudeAccountService { delete accountData.rateLimitedAt delete accountData.rateLimitStatus delete accountData.rateLimitEndAt // 清除限流结束时间 + // 恢复可调度状态,与 OpenAI 账号保持一致 + accountData.schedulable = true await redis.setClaudeAccount(accountId, accountData) - logger.success(`✅ Rate limit removed for account: ${accountData.name} (${accountId})`) + logger.success( + `✅ Rate limit removed for account: ${accountData.name} (${accountId}), schedulable restored` + ) + + // 发送 Webhook 通知限流已解除 + try { + const webhookNotifier = require('../utils/webhookNotifier') + await webhookNotifier.sendAccountAnomalyNotification({ + accountId, + accountName: accountData.name || 'Claude Account', + platform: 'claude-oauth', + status: 'recovered', + errorCode: 'CLAUDE_OAUTH_RATE_LIMIT_CLEARED', + reason: 'Rate limit has been cleared and account is now schedulable', + timestamp: getISOStringWithTimezone(new Date()) + }) + logger.info( + `📢 Webhook notification sent for Claude account ${accountData.name} rate limit cleared` + ) + } catch (webhookError) { + logger.error('Failed to send rate limit cleared webhook notification:', webhookError) + } + return { success: true } } catch (error) { logger.error(`❌ Failed to remove rate limit for account: ${accountId}`, error) diff --git a/src/services/claudeConsoleAccountService.js b/src/services/claudeConsoleAccountService.js index 34c9a5c7..cd4f4f72 100644 --- a/src/services/claudeConsoleAccountService.js +++ b/src/services/claudeConsoleAccountService.js @@ -400,6 +400,7 @@ class ClaudeConsoleAccountService { rateLimitedAt: new Date().toISOString(), rateLimitStatus: 'limited', isActive: 'false', // 禁用账户 + schedulable: 'false', // 停止调度,与其他平台保持一致 errorMessage: `Rate limited at ${new Date().toISOString()}` } @@ -468,6 +469,7 @@ class ClaudeConsoleAccountService { // 没有额度限制,完全恢复 await client.hset(accountKey, { isActive: 'true', + schedulable: 'true', // 恢复调度,与其他平台保持一致 status: 'active', errorMessage: '' }) @@ -1131,6 +1133,66 @@ class ClaudeConsoleAccountService { return null } } + + // 🔄 重置账户所有异常状态 + async resetAccountStatus(accountId) { + try { + const accountData = await this.getAccount(accountId) + if (!accountData) { + throw new Error('Account not found') + } + + const client = redis.getClientSafe() + const accountKey = `${this.ACCOUNT_KEY_PREFIX}${accountId}` + + // 准备要更新的字段 + const updates = { + status: 'active', + errorMessage: '', + schedulable: 'true', + isActive: 'true' // 重要:必须恢复isActive状态 + } + + // 删除所有异常状态相关的字段 + const fieldsToDelete = [ + 'rateLimitedAt', + 'rateLimitStatus', + 'unauthorizedAt', + 'unauthorizedCount', + 'overloadedAt', + 'overloadStatus', + 'blockedAt', + 'quotaStoppedAt' + ] + + // 执行更新 + await client.hset(accountKey, updates) + await client.hdel(accountKey, ...fieldsToDelete) + + logger.success(`✅ Reset all error status for Claude Console account ${accountId}`) + + // 发送 Webhook 通知 + try { + const webhookNotifier = require('../utils/webhookNotifier') + await webhookNotifier.sendAccountAnomalyNotification({ + accountId, + accountName: accountData.name || accountId, + platform: 'claude-console', + status: 'recovered', + errorCode: 'STATUS_RESET', + reason: 'Account status manually reset', + timestamp: new Date().toISOString() + }) + } catch (webhookError) { + logger.warn('Failed to send webhook notification:', webhookError) + } + + return { success: true, accountId } + } catch (error) { + logger.error(`❌ Failed to reset Claude Console account status: ${accountId}`, error) + throw error + } + } } module.exports = new ClaudeConsoleAccountService() diff --git a/src/services/claudeRelayService.js b/src/services/claudeRelayService.js index 6c91f731..eed4e365 100644 --- a/src/services/claudeRelayService.js +++ b/src/services/claudeRelayService.js @@ -126,8 +126,11 @@ class ClaudeRelayService { // 获取有效的访问token const accessToken = await claudeAccountService.getValidAccessToken(accountId) + // 获取账户信息 + const account = await claudeAccountService.getAccount(accountId) + // 处理请求体(传递 clientHeaders 以判断是否需要设置 Claude Code 系统提示词) - const processedBody = this._processRequestBody(requestBody, clientHeaders) + const processedBody = this._processRequestBody(requestBody, clientHeaders, account) // 获取代理配置 const proxyAgent = await this._getProxyAgent(accountId) @@ -356,7 +359,7 @@ class ClaudeRelayService { } // 🔄 处理请求体 - _processRequestBody(body, clientHeaders = {}) { + _processRequestBody(body, clientHeaders = {}, account = null) { if (!body) { return body } @@ -458,9 +461,31 @@ class ClaudeRelayService { delete processedBody.top_p } + // 处理统一的客户端标识 + if (account && account.useUnifiedClientId && account.unifiedClientId) { + this._replaceClientId(processedBody, account.unifiedClientId) + } + return processedBody } + // 🔄 替换请求中的客户端标识 + _replaceClientId(body, unifiedClientId) { + if (!body || !body.metadata || !body.metadata.user_id || !unifiedClientId) { + return + } + + const userId = body.metadata.user_id + // user_id格式:user_{64位十六进制}_account__session_{uuid} + // 只替换第一个下划线后到_account之前的部分(客户端标识) + const match = userId.match(/^user_[a-f0-9]{64}(_account__session_[a-f0-9-]{36})$/) + if (match && match[1]) { + // 替换客户端标识部分 + body.metadata.user_id = `user_${unifiedClientId}${match[1]}` + logger.info(`🔄 Replaced client ID with unified ID: ${body.metadata.user_id}`) + } + } + // 🔢 验证并限制max_tokens参数 _validateAndLimitMaxTokens(body) { if (!body || !body.max_tokens) { @@ -844,8 +869,11 @@ class ClaudeRelayService { // 获取有效的访问token const accessToken = await claudeAccountService.getValidAccessToken(accountId) + // 获取账户信息 + const account = await claudeAccountService.getAccount(accountId) + // 处理请求体(传递 clientHeaders 以判断是否需要设置 Claude Code 系统提示词) - const processedBody = this._processRequestBody(requestBody, clientHeaders) + const processedBody = this._processRequestBody(requestBody, clientHeaders, account) // 获取代理配置 const proxyAgent = await this._getProxyAgent(accountId) diff --git a/src/services/openaiAccountService.js b/src/services/openaiAccountService.js index eb13ac1a..628d92c5 100644 --- a/src/services/openaiAccountService.js +++ b/src/services/openaiAccountService.js @@ -814,14 +814,37 @@ function isRateLimited(account) { } // 设置账户限流状态 -async function setAccountRateLimited(accountId, isLimited) { +async function setAccountRateLimited(accountId, isLimited, resetsInSeconds = null) { const updates = { rateLimitStatus: isLimited ? 'limited' : 'normal', - rateLimitedAt: isLimited ? new Date().toISOString() : null + rateLimitedAt: isLimited ? new Date().toISOString() : null, + // 限流时停止调度,解除限流时恢复调度 + schedulable: isLimited ? 'false' : 'true' + } + + // 如果提供了重置时间(秒数),计算重置时间戳 + if (isLimited && resetsInSeconds !== null && resetsInSeconds > 0) { + const resetTime = new Date(Date.now() + resetsInSeconds * 1000).toISOString() + updates.rateLimitResetAt = resetTime + logger.info( + `🕐 Account ${accountId} will be reset at ${resetTime} (in ${resetsInSeconds} seconds / ${Math.ceil(resetsInSeconds / 60)} minutes)` + ) + } else if (isLimited) { + // 如果没有提供重置时间,使用默认的60分钟 + const defaultResetSeconds = 60 * 60 // 1小时 + const resetTime = new Date(Date.now() + defaultResetSeconds * 1000).toISOString() + updates.rateLimitResetAt = resetTime + logger.warn( + `⚠️ No reset time provided for account ${accountId}, using default 60 minutes. Reset at ${resetTime}` + ) + } else if (!isLimited) { + updates.rateLimitResetAt = null } await updateAccount(accountId, updates) - logger.info(`Set rate limit status for OpenAI account ${accountId}: ${updates.rateLimitStatus}`) + logger.info( + `Set rate limit status for OpenAI account ${accountId}: ${updates.rateLimitStatus}, schedulable: ${updates.schedulable}` + ) // 如果被限流,发送 Webhook 通知 if (isLimited) { @@ -834,7 +857,9 @@ async function setAccountRateLimited(accountId, isLimited) { platform: 'openai', status: 'blocked', errorCode: 'OPENAI_RATE_LIMITED', - reason: 'Account rate limited (429 error). Estimated reset in 1 hour', + reason: resetsInSeconds + ? `Account rate limited (429 error). Reset in ${Math.ceil(resetsInSeconds / 60)} minutes` + : 'Account rate limited (429 error). Estimated reset in 1 hour', timestamp: new Date().toISOString() }) logger.info(`📢 Webhook notification sent for OpenAI account ${account.name} rate limit`) @@ -844,6 +869,48 @@ async function setAccountRateLimited(accountId, isLimited) { } } +// 🔄 重置账户所有异常状态 +async function resetAccountStatus(accountId) { + const account = await getAccount(accountId) + if (!account) { + throw new Error('Account not found') + } + + const updates = { + // 根据是否有有效的 accessToken 来设置 status + status: account.accessToken ? 'active' : 'created', + // 恢复可调度状态 + schedulable: 'true', + // 清除错误相关字段 + errorMessage: null, + rateLimitedAt: null, + rateLimitStatus: 'normal', + rateLimitResetAt: null + } + + await updateAccount(accountId, updates) + logger.info(`✅ Reset all error status for OpenAI account ${accountId}`) + + // 发送 Webhook 通知 + try { + const webhookNotifier = require('../utils/webhookNotifier') + await webhookNotifier.sendAccountAnomalyNotification({ + accountId, + accountName: account.name || accountId, + platform: 'openai', + status: 'recovered', + errorCode: 'STATUS_RESET', + reason: 'Account status manually reset', + timestamp: new Date().toISOString() + }) + logger.info(`📢 Webhook notification sent for OpenAI account ${account.name} status reset`) + } catch (webhookError) { + logger.error('Failed to send status reset webhook notification:', webhookError) + } + + return { success: true, message: 'Account status reset successfully' } +} + // 切换账户调度状态 async function toggleSchedulable(accountId) { const account = await getAccount(accountId) @@ -873,15 +940,26 @@ async function getAccountRateLimitInfo(accountId) { return null } - if (account.rateLimitStatus === 'limited' && account.rateLimitedAt) { - const limitedAt = new Date(account.rateLimitedAt).getTime() + if (account.rateLimitStatus === 'limited') { const now = Date.now() - const limitDuration = 60 * 60 * 1000 // 1小时 - const remainingTime = Math.max(0, limitedAt + limitDuration - now) + let remainingTime = 0 + + // 优先使用 rateLimitResetAt 字段(精确的重置时间) + if (account.rateLimitResetAt) { + const resetAt = new Date(account.rateLimitResetAt).getTime() + remainingTime = Math.max(0, resetAt - now) + } + // 回退到使用 rateLimitedAt + 默认1小时 + else if (account.rateLimitedAt) { + const limitedAt = new Date(account.rateLimitedAt).getTime() + const limitDuration = 60 * 60 * 1000 // 默认1小时 + remainingTime = Math.max(0, limitedAt + limitDuration - now) + } return { isRateLimited: remainingTime > 0, rateLimitedAt: account.rateLimitedAt, + rateLimitResetAt: account.rateLimitResetAt, minutesRemaining: Math.ceil(remainingTime / (60 * 1000)) } } @@ -889,6 +967,7 @@ async function getAccountRateLimitInfo(accountId) { return { isRateLimited: false, rateLimitedAt: null, + rateLimitResetAt: null, minutesRemaining: 0 } } @@ -926,6 +1005,7 @@ module.exports = { refreshAccountToken, isTokenExpired, setAccountRateLimited, + resetAccountStatus, toggleSchedulable, getAccountRateLimitInfo, updateAccountUsage, diff --git a/src/services/rateLimitCleanupService.js b/src/services/rateLimitCleanupService.js new file mode 100644 index 00000000..6230e7c9 --- /dev/null +++ b/src/services/rateLimitCleanupService.js @@ -0,0 +1,351 @@ +/** + * 限流状态自动清理服务 + * 定期检查并清理所有类型账号的过期限流状态 + */ + +const logger = require('../utils/logger') +const openaiAccountService = require('./openaiAccountService') +const claudeAccountService = require('./claudeAccountService') +const claudeConsoleAccountService = require('./claudeConsoleAccountService') +const unifiedOpenAIScheduler = require('./unifiedOpenAIScheduler') +const webhookService = require('./webhookService') + +class RateLimitCleanupService { + constructor() { + this.cleanupInterval = null + this.isRunning = false + // 默认每5分钟检查一次 + this.intervalMs = 5 * 60 * 1000 + // 存储已清理的账户信息,用于发送恢复通知 + this.clearedAccounts = [] + } + + /** + * 启动自动清理服务 + * @param {number} intervalMinutes - 检查间隔(分钟),默认5分钟 + */ + start(intervalMinutes = 5) { + if (this.cleanupInterval) { + logger.warn('⚠️ Rate limit cleanup service is already running') + return + } + + this.intervalMs = intervalMinutes * 60 * 1000 + + logger.info(`🧹 Starting rate limit cleanup service (interval: ${intervalMinutes} minutes)`) + + // 立即执行一次清理 + this.performCleanup() + + // 设置定期执行 + this.cleanupInterval = setInterval(() => { + this.performCleanup() + }, this.intervalMs) + } + + /** + * 停止自动清理服务 + */ + stop() { + if (this.cleanupInterval) { + clearInterval(this.cleanupInterval) + this.cleanupInterval = null + logger.info('🛑 Rate limit cleanup service stopped') + } + } + + /** + * 执行一次清理检查 + */ + async performCleanup() { + if (this.isRunning) { + logger.debug('⏭️ Cleanup already in progress, skipping this cycle') + return + } + + this.isRunning = true + const startTime = Date.now() + + try { + logger.debug('🔍 Starting rate limit cleanup check...') + + const results = { + openai: { checked: 0, cleared: 0, errors: [] }, + claude: { checked: 0, cleared: 0, errors: [] }, + claudeConsole: { checked: 0, cleared: 0, errors: [] } + } + + // 清理 OpenAI 账号 + await this.cleanupOpenAIAccounts(results.openai) + + // 清理 Claude 账号 + await this.cleanupClaudeAccounts(results.claude) + + // 清理 Claude Console 账号 + await this.cleanupClaudeConsoleAccounts(results.claudeConsole) + + const totalChecked = + results.openai.checked + results.claude.checked + results.claudeConsole.checked + const totalCleared = + results.openai.cleared + results.claude.cleared + results.claudeConsole.cleared + const duration = Date.now() - startTime + + if (totalCleared > 0) { + logger.info( + `✅ Rate limit cleanup completed: ${totalCleared} accounts cleared out of ${totalChecked} checked (${duration}ms)` + ) + logger.info(` OpenAI: ${results.openai.cleared}/${results.openai.checked}`) + logger.info(` Claude: ${results.claude.cleared}/${results.claude.checked}`) + logger.info( + ` Claude Console: ${results.claudeConsole.cleared}/${results.claudeConsole.checked}` + ) + + // 发送 webhook 恢复通知 + if (this.clearedAccounts.length > 0) { + await this.sendRecoveryNotifications() + } + } else { + logger.debug( + `🔍 Rate limit cleanup check completed: no expired limits found (${duration}ms)` + ) + } + + // 清空已清理账户列表 + this.clearedAccounts = [] + + // 记录错误 + const allErrors = [ + ...results.openai.errors, + ...results.claude.errors, + ...results.claudeConsole.errors + ] + if (allErrors.length > 0) { + logger.warn(`⚠️ Encountered ${allErrors.length} errors during cleanup:`, allErrors) + } + } catch (error) { + logger.error('❌ Rate limit cleanup failed:', error) + } finally { + this.isRunning = false + } + } + + /** + * 清理 OpenAI 账号的过期限流 + */ + async cleanupOpenAIAccounts(result) { + try { + const accounts = await openaiAccountService.getAllAccounts() + + for (const account of accounts) { + // 只检查标记为限流的账号 + if (account.rateLimitStatus === 'limited') { + result.checked++ + + try { + // 使用 unifiedOpenAIScheduler 的检查方法,它会自动清除过期的限流 + const isStillLimited = await unifiedOpenAIScheduler.isAccountRateLimited(account.id) + + if (!isStillLimited) { + result.cleared++ + logger.info( + `🧹 Auto-cleared expired rate limit for OpenAI account: ${account.name} (${account.id})` + ) + + // 记录已清理的账户信息 + this.clearedAccounts.push({ + platform: 'OpenAI', + accountId: account.id, + accountName: account.name, + previousStatus: 'rate_limited', + currentStatus: 'active' + }) + } + } catch (error) { + result.errors.push({ + accountId: account.id, + accountName: account.name, + error: error.message + }) + } + } + } + } catch (error) { + logger.error('Failed to cleanup OpenAI accounts:', error) + result.errors.push({ error: error.message }) + } + } + + /** + * 清理 Claude 账号的过期限流 + */ + async cleanupClaudeAccounts(result) { + try { + const accounts = await claudeAccountService.getAllAccounts() + + for (const account of accounts) { + // 只检查标记为限流的账号 + if (account.rateLimitStatus === 'limited' || account.rateLimitedAt) { + result.checked++ + + try { + // 使用 claudeAccountService 的检查方法,它会自动清除过期的限流 + const isStillLimited = await claudeAccountService.isAccountRateLimited(account.id) + + if (!isStillLimited) { + result.cleared++ + logger.info( + `🧹 Auto-cleared expired rate limit for Claude account: ${account.name} (${account.id})` + ) + + // 记录已清理的账户信息 + this.clearedAccounts.push({ + platform: 'Claude', + accountId: account.id, + accountName: account.name, + previousStatus: 'rate_limited', + currentStatus: 'active' + }) + } + } catch (error) { + result.errors.push({ + accountId: account.id, + accountName: account.name, + error: error.message + }) + } + } + } + } catch (error) { + logger.error('Failed to cleanup Claude accounts:', error) + result.errors.push({ error: error.message }) + } + } + + /** + * 清理 Claude Console 账号的过期限流 + */ + async cleanupClaudeConsoleAccounts(result) { + try { + const accounts = await claudeConsoleAccountService.getAllAccounts() + + for (const account of accounts) { + // 检查两种状态字段:rateLimitStatus 和 status + const hasRateLimitStatus = account.rateLimitStatus === 'limited' + const hasStatusRateLimited = account.status === 'rate_limited' + + if (hasRateLimitStatus || hasStatusRateLimited) { + result.checked++ + + try { + // 使用 claudeConsoleAccountService 的检查方法,它会自动清除过期的限流 + const isStillLimited = await claudeConsoleAccountService.isAccountRateLimited( + account.id + ) + + if (!isStillLimited) { + result.cleared++ + + // 如果 status 字段是 rate_limited,需要额外清理 + if (hasStatusRateLimited && !hasRateLimitStatus) { + await claudeConsoleAccountService.updateAccount(account.id, { + status: 'active' + }) + } + + logger.info( + `🧹 Auto-cleared expired rate limit for Claude Console account: ${account.name} (${account.id})` + ) + + // 记录已清理的账户信息 + this.clearedAccounts.push({ + platform: 'Claude Console', + accountId: account.id, + accountName: account.name, + previousStatus: 'rate_limited', + currentStatus: 'active' + }) + } + } catch (error) { + result.errors.push({ + accountId: account.id, + accountName: account.name, + error: error.message + }) + } + } + } + } catch (error) { + logger.error('Failed to cleanup Claude Console accounts:', error) + result.errors.push({ error: error.message }) + } + } + + /** + * 手动触发一次清理(供 API 或 CLI 调用) + */ + async manualCleanup() { + logger.info('🧹 Manual rate limit cleanup triggered') + await this.performCleanup() + } + + /** + * 发送限流恢复通知 + */ + async sendRecoveryNotifications() { + try { + // 按平台分组账户 + const groupedAccounts = {} + for (const account of this.clearedAccounts) { + if (!groupedAccounts[account.platform]) { + groupedAccounts[account.platform] = [] + } + groupedAccounts[account.platform].push(account) + } + + // 构建通知消息 + const platforms = Object.keys(groupedAccounts) + const totalAccounts = this.clearedAccounts.length + + let message = `🎉 共有 ${totalAccounts} 个账户的限流状态已恢复\n\n` + + for (const platform of platforms) { + const accounts = groupedAccounts[platform] + message += `**${platform}** (${accounts.length} 个):\n` + for (const account of accounts) { + message += `• ${account.accountName} (ID: ${account.accountId})\n` + } + message += '\n' + } + + // 发送 webhook 通知 + await webhookService.sendNotification('rateLimitRecovery', { + title: '限流恢复通知', + message, + totalAccounts, + platforms: Object.keys(groupedAccounts), + accounts: this.clearedAccounts, + timestamp: new Date().toISOString() + }) + + logger.info(`📢 已发送限流恢复通知,涉及 ${totalAccounts} 个账户`) + } catch (error) { + logger.error('❌ 发送限流恢复通知失败:', error) + } + } + + /** + * 获取服务状态 + */ + getStatus() { + return { + running: !!this.cleanupInterval, + intervalMinutes: this.intervalMs / (60 * 1000), + isProcessing: this.isRunning + } + } +} + +// 创建单例实例 +const rateLimitCleanupService = new RateLimitCleanupService() + +module.exports = rateLimitCleanupService diff --git a/src/services/unifiedOpenAIScheduler.js b/src/services/unifiedOpenAIScheduler.js index 85404543..cbc7a8ff 100644 --- a/src/services/unifiedOpenAIScheduler.js +++ b/src/services/unifiedOpenAIScheduler.js @@ -301,10 +301,10 @@ class UnifiedOpenAIScheduler { } // 🚫 标记账户为限流状态 - async markAccountRateLimited(accountId, accountType, sessionHash = null) { + async markAccountRateLimited(accountId, accountType, sessionHash = null, resetsInSeconds = null) { try { if (accountType === 'openai') { - await openaiAccountService.setAccountRateLimited(accountId, true) + await openaiAccountService.setAccountRateLimited(accountId, true, resetsInSeconds) } // 删除会话映射 @@ -347,12 +347,30 @@ class UnifiedOpenAIScheduler { return false } - if (account.rateLimitStatus === 'limited' && account.rateLimitedAt) { - const limitedAt = new Date(account.rateLimitedAt).getTime() - const now = Date.now() - const limitDuration = 60 * 60 * 1000 // 1小时 + if (account.rateLimitStatus === 'limited') { + // 如果有具体的重置时间,使用它 + if (account.rateLimitResetAt) { + const resetTime = new Date(account.rateLimitResetAt).getTime() + const now = Date.now() + const isStillLimited = now < resetTime - return now < limitedAt + limitDuration + // 如果已经过了重置时间,自动清除限流状态 + if (!isStillLimited) { + logger.info(`✅ Auto-clearing rate limit for account ${accountId} (reset time reached)`) + await openaiAccountService.setAccountRateLimited(accountId, false) + return false + } + + return isStillLimited + } + + // 如果没有具体的重置时间,使用默认的1小时 + if (account.rateLimitedAt) { + const limitedAt = new Date(account.rateLimitedAt).getTime() + const now = Date.now() + const limitDuration = 60 * 60 * 1000 // 1小时 + return now < limitedAt + limitDuration + } } return false } catch (error) { diff --git a/src/services/webhookService.js b/src/services/webhookService.js index d791ce68..c0d049c5 100755 --- a/src/services/webhookService.js +++ b/src/services/webhookService.js @@ -375,6 +375,7 @@ class WebhookService { quotaWarning: '📊 配额警告', systemError: '❌ 系统错误', securityAlert: '🔒 安全警报', + rateLimitRecovery: '🎉 限流恢复通知', test: '🧪 测试通知' } @@ -390,6 +391,7 @@ class WebhookService { quotaWarning: 'active', systemError: 'critical', securityAlert: 'critical', + rateLimitRecovery: 'active', test: 'passive' } @@ -405,6 +407,7 @@ class WebhookService { quotaWarning: 'bell', systemError: 'alert', securityAlert: 'alarm', + rateLimitRecovery: 'success', test: 'default' } @@ -470,6 +473,14 @@ class WebhookService { lines.push(`**平台**: ${data.platform}`) } + if (data.platforms) { + lines.push(`**涉及平台**: ${data.platforms.join(', ')}`) + } + + if (data.totalAccounts) { + lines.push(`**恢复账户数**: ${data.totalAccounts}`) + } + if (data.status) { lines.push(`**状态**: ${data.status}`) } @@ -539,6 +550,7 @@ class WebhookService { quotaWarning: 'yellow', systemError: 'red', securityAlert: 'red', + rateLimitRecovery: 'green', test: 'blue' } @@ -554,6 +566,7 @@ class WebhookService { quotaWarning: ':chart_with_downwards_trend:', systemError: ':x:', securityAlert: ':lock:', + rateLimitRecovery: ':tada:', test: ':test_tube:' } @@ -569,6 +582,7 @@ class WebhookService { quotaWarning: 0xffeb3b, // 黄色 systemError: 0xf44336, // 红色 securityAlert: 0xf44336, // 红色 + rateLimitRecovery: 0x4caf50, // 绿色 test: 0x2196f3 // 蓝色 } diff --git a/web/admin-spa/src/components/accounts/AccountForm.vue b/web/admin-spa/src/components/accounts/AccountForm.vue index d47a4ba5..4edd47b3 100644 --- a/web/admin-spa/src/components/accounts/AccountForm.vue +++ b/web/admin-spa/src/components/accounts/AccountForm.vue @@ -934,6 +934,64 @@ + +
+ +
+
+ +
+ +
+