feat: 添加账户状态监控和自动恢复机制

- 实现账户健康度监控系统,支持30分钟内错误率检测
- 添加自动恢复机制,失败账户在30分钟后自动尝试恢复
- 优化账户选择策略,优先选择健康账户
- 增强Redis键管理,添加账户状态和错误追踪功能
- 改进Gemini服务错误处理和重试逻辑
- 新增standardGeminiRoutes标准化路由支持

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
shaw
2025-09-11 22:02:53 +08:00
parent b6b16d05f0
commit 7c4feec5aa
8 changed files with 912 additions and 10 deletions

View File

@@ -2292,6 +2292,178 @@ class ClaudeAccountService {
// 不抛出错误,移除过载状态失败不应该影响主流程
}
}
/**
* 检查并恢复因5小时限制被自动停止的账号
* 用于定时任务自动恢复
* @returns {Promise<{checked: number, recovered: number, accounts: Array}>}
*/
async checkAndRecoverFiveHourStoppedAccounts() {
const result = {
checked: 0,
recovered: 0,
accounts: []
}
try {
const accounts = await this.getAllAccounts()
const now = new Date()
for (const account of accounts) {
// 只检查因5小时限制被自动停止的账号
// 重要不恢复手动停止的账号没有fiveHourAutoStopped标记的
if (account.fiveHourAutoStopped === 'true' && account.schedulable === 'false') {
result.checked++
// 使用分布式锁防止并发修改
const lockKey = `lock:account:${account.id}:recovery`
const lockValue = `${Date.now()}_${Math.random()}`
const lockTTL = 5000 // 5秒锁超时
try {
// 尝试获取锁
const lockAcquired = await redis.setAccountLock(lockKey, lockValue, lockTTL)
if (!lockAcquired) {
logger.debug(
`⏭️ Account ${account.name} (${account.id}) is being processed by another instance`
)
continue
}
// 重新获取账号数据,确保是最新的
const latestAccount = await redis.getClaudeAccount(account.id)
if (
!latestAccount ||
latestAccount.fiveHourAutoStopped !== 'true' ||
latestAccount.schedulable !== 'false'
) {
// 账号状态已变化,跳过
await redis.releaseAccountLock(lockKey, lockValue)
continue
}
// 检查当前时间是否已经进入新的5小时窗口
let shouldRecover = false
let newWindowStart = null
let newWindowEnd = null
if (latestAccount.sessionWindowEnd) {
const windowEnd = new Date(latestAccount.sessionWindowEnd)
// 使用严格的时间比较添加1分钟缓冲避免边界问题
if (now.getTime() > windowEnd.getTime() + 60000) {
shouldRecover = true
// 计算新的窗口时间(基于窗口结束时间,而不是当前时间)
// 这样可以保证窗口时间的连续性
newWindowStart = new Date(windowEnd)
newWindowStart.setMilliseconds(newWindowStart.getMilliseconds() + 1)
newWindowEnd = new Date(newWindowStart)
newWindowEnd.setHours(newWindowEnd.getHours() + 5)
logger.info(
`🔄 Account ${latestAccount.name} (${latestAccount.id}) has entered new session window. ` +
`Old window: ${latestAccount.sessionWindowStart} - ${latestAccount.sessionWindowEnd}, ` +
`New window: ${newWindowStart.toISOString()} - ${newWindowEnd.toISOString()}`
)
}
} else {
// 如果没有窗口结束时间但有停止时间检查是否已经过了5小时
if (latestAccount.fiveHourStoppedAt) {
const stoppedAt = new Date(latestAccount.fiveHourStoppedAt)
const hoursSinceStopped = (now.getTime() - stoppedAt.getTime()) / (1000 * 60 * 60)
// 使用严格的5小时判断加上1分钟缓冲
if (hoursSinceStopped > 5.017) {
// 5小时1分钟
shouldRecover = true
newWindowStart = this._calculateSessionWindowStart(now)
newWindowEnd = this._calculateSessionWindowEnd(newWindowStart)
logger.info(
`🔄 Account ${latestAccount.name} (${latestAccount.id}) stopped ${hoursSinceStopped.toFixed(2)} hours ago, recovering`
)
}
}
}
if (shouldRecover) {
// 恢复账号调度
const updatedAccountData = { ...latestAccount }
// 恢复调度状态
updatedAccountData.schedulable = 'true'
delete updatedAccountData.fiveHourAutoStopped
delete updatedAccountData.fiveHourStoppedAt
// 更新会话窗口(如果有新窗口)
if (newWindowStart && newWindowEnd) {
updatedAccountData.sessionWindowStart = newWindowStart.toISOString()
updatedAccountData.sessionWindowEnd = newWindowEnd.toISOString()
// 清除会话窗口状态
delete updatedAccountData.sessionWindowStatus
delete updatedAccountData.sessionWindowStatusUpdatedAt
}
// 保存更新
await redis.setClaudeAccount(account.id, updatedAccountData)
result.recovered++
result.accounts.push({
id: latestAccount.id,
name: latestAccount.name,
oldWindow: latestAccount.sessionWindowEnd
? {
start: latestAccount.sessionWindowStart,
end: latestAccount.sessionWindowEnd
}
: null,
newWindow:
newWindowStart && newWindowEnd
? {
start: newWindowStart.toISOString(),
end: newWindowEnd.toISOString()
}
: null
})
logger.info(
`✅ Auto-resumed scheduling for account ${latestAccount.name} (${latestAccount.id}) - 5-hour limit expired`
)
}
// 释放锁
await redis.releaseAccountLock(lockKey, lockValue)
} catch (error) {
// 确保释放锁
if (lockKey && lockValue) {
try {
await redis.releaseAccountLock(lockKey, lockValue)
} catch (unlockError) {
logger.error(`Failed to release lock for account ${account.id}:`, unlockError)
}
}
logger.error(
`❌ Failed to check/recover 5-hour stopped account ${account.name} (${account.id}):`,
error
)
}
}
}
if (result.recovered > 0) {
logger.info(
`🔄 5-hour limit recovery completed: ${result.recovered}/${result.checked} accounts recovered`
)
}
return result
} catch (error) {
logger.error('❌ Failed to check and recover 5-hour stopped accounts:', error)
throw error
}
}
}
module.exports = new ClaudeAccountService()

View File

@@ -1290,13 +1290,17 @@ async function generateContent(
// 按照 gemini-cli 的转换格式构造请求
const request = {
model: requestData.model,
user_prompt_id: userPromptId,
request: {
...requestData.request,
session_id: sessionId
}
}
// 只有当 userPromptId 存在时才添加
if (userPromptId) {
request.user_prompt_id = userPromptId
}
// 只有当projectId存在时才添加project字段
if (projectId) {
request.project = projectId
@@ -1309,6 +1313,12 @@ async function generateContent(
sessionId
})
// 添加详细的请求日志
logger.info('📦 generateContent 请求详情', {
url: `${CODE_ASSIST_ENDPOINT}/${CODE_ASSIST_API_VERSION}:generateContent`,
requestBody: JSON.stringify(request, null, 2)
})
const axiosConfig = {
url: `${CODE_ASSIST_ENDPOINT}/${CODE_ASSIST_API_VERSION}:generateContent`,
method: 'POST',
@@ -1356,13 +1366,17 @@ async function generateContentStream(
// 按照 gemini-cli 的转换格式构造请求
const request = {
model: requestData.model,
user_prompt_id: userPromptId,
request: {
...requestData.request,
session_id: sessionId
}
}
// 只有当 userPromptId 存在时才添加
if (userPromptId) {
request.user_prompt_id = userPromptId
}
// 只有当projectId存在时才添加project字段
if (projectId) {
request.project = projectId

View File

@@ -215,6 +215,39 @@ class RateLimitCleanupService {
}
}
}
// 检查并恢复因5小时限制被自动停止的账号
try {
const fiveHourResult = await claudeAccountService.checkAndRecoverFiveHourStoppedAccounts()
if (fiveHourResult.recovered > 0) {
// 将5小时限制恢复的账号也加入到已清理账户列表中用于发送通知
for (const account of fiveHourResult.accounts) {
this.clearedAccounts.push({
platform: 'Claude',
accountId: account.id,
accountName: account.name,
previousStatus: '5hour_limited',
currentStatus: 'active',
windowInfo: account.newWindow
})
}
// 更新统计数据
result.checked += fiveHourResult.checked
result.cleared += fiveHourResult.recovered
logger.info(
`🕐 Claude 5-hour limit recovery: ${fiveHourResult.recovered}/${fiveHourResult.checked} accounts recovered`
)
}
} catch (error) {
logger.error('Failed to check and recover 5-hour stopped Claude accounts:', error)
result.errors.push({
type: '5hour_recovery',
error: error.message
})
}
} catch (error) {
logger.error('Failed to cleanup Claude accounts:', error)
result.errors.push({ error: error.message })