From a119cb1744aafc1a8cbff919110cedc75c1f9899 Mon Sep 17 00:00:00 2001 From: SunSeekerX Date: Mon, 9 Feb 2026 18:13:45 +0800 Subject: [PATCH] 1 --- .env.example | 6 + CLAUDE.md | 724 +++--------------- ...de-relay-servicelogs.claude-relay-audit.log.json | 35 + ...vicelogs.claude-relay-auth-detail-audit.log.json | 35 + ...ay-servicelogs.claude-relay-error-audit.log.json | 35 + ...servicelogs.claude-relay-security-audit.log.json | 35 + cli/index.js | 4 +- config/config.example.js | 8 + config/models.js | 21 + scripts/manage-session-windows.js | 2 +- scripts/test-api-response.js | 4 +- scripts/test-bedrock-models.js | 2 +- scripts/test-gemini-refresh.js | 2 +- scripts/test-group-scheduling.js | 6 +- scripts/test-model-mapping.js | 2 +- src/app.js | 15 +- src/handlers/geminiHandlers.js | 192 ++++- src/middleware/auth.js | 103 +-- src/routes/admin/accountBalance.js | 2 +- src/routes/admin/accountGroups.js | 10 +- src/routes/admin/azureOpenaiAccounts.js | 24 +- src/routes/admin/bedrockAccounts.js | 15 +- src/routes/admin/ccrAccounts.js | 5 +- src/routes/admin/claudeAccounts.js | 4 +- src/routes/admin/claudeConsoleAccounts.js | 4 +- src/routes/admin/dashboard.js | 26 +- src/routes/admin/droidAccounts.js | 18 +- src/routes/admin/geminiAccounts.js | 9 +- src/routes/admin/geminiApiAccounts.js | 2 +- src/routes/admin/openaiAccounts.js | 2 +- src/routes/admin/openaiResponsesAccounts.js | 40 +- src/routes/admin/sync.js | 8 +- src/routes/admin/system.js | 42 +- src/routes/admin/usageStats.js | 18 +- src/routes/api.js | 20 +- src/routes/apiStats.js | 26 +- src/routes/azureOpenaiRoutes.js | 89 ++- src/routes/droidRoutes.js | 2 +- src/routes/openaiClaudeRoutes.js | 6 +- src/routes/openaiGeminiRoutes.js | 6 +- src/routes/openaiRoutes.js | 8 +- .../{ => account}/accountBalanceService.js | 10 +- .../azureOpenaiAccountService.js | 119 ++- .../{ => account}/bedrockAccountService.js | 82 +- .../{ => account}/ccrAccountService.js | 31 +- .../{ => account}/claudeAccountService.js | 44 +- .../claudeConsoleAccountService.js | 32 +- .../{ => account}/droidAccountService.js | 77 +- .../{ => account}/geminiAccountService.js | 42 +- .../{ => account}/geminiApiAccountService.js | 33 +- .../{ => account}/openaiAccountService.js | 41 +- .../openaiResponsesAccountService.js | 27 +- src/services/accountNameCacheService.js | 20 +- src/services/accountTestSchedulerService.js | 2 +- src/services/anthropicGeminiBridgeService.js | 4 +- src/services/claudeRelayConfigService.js | 8 +- src/services/rateLimitCleanupService.js | 8 +- .../{ => relay}/antigravityRelayService.js | 6 +- .../{ => relay}/azureOpenaiRelayService.js | 17 +- .../{ => relay}/bedrockRelayService.js | 35 +- src/services/{ => relay}/ccrRelayService.js | 90 ++- .../{ => relay}/claudeConsoleRelayService.js | 104 ++- .../{ => relay}/claudeRelayService.js | 105 ++- src/services/{ => relay}/droidRelayService.js | 47 +- .../{ => relay}/geminiRelayService.js | 8 +- .../openaiResponsesRelayService.js | 146 ++-- .../{ => scheduler}/droidScheduler.js | 55 +- .../{ => scheduler}/unifiedClaudeScheduler.js | 39 +- .../{ => scheduler}/unifiedGeminiScheduler.js | 80 +- .../{ => scheduler}/unifiedOpenAIScheduler.js | 248 +++--- src/utils/logger.js | 139 ++-- src/utils/testPayloadHelper.js | 82 +- src/utils/upstreamErrorHelper.js | 255 ++++++ tests/accountBalanceService.test.js | 2 +- .../src/components/accounts/AccountForm.vue | 119 +-- .../accounts/AccountScheduledTestModal.vue | 46 +- .../components/accounts/AccountTestModal.vue | 654 ---------------- .../components/apikeys/ApiKeyTestModal.vue | 629 --------------- .../src/components/common/ModelSelector.vue | 69 ++ .../components/common/UnifiedTestModal.vue | 589 ++++++++++++++ .../settings/ModelPricingSection.vue | 345 +++++++++ web/admin-spa/src/utils/http_apis.js | 9 + web/admin-spa/src/utils/useTestState.js | 208 +++++ web/admin-spa/src/views/AccountsView.vue | 75 +- web/admin-spa/src/views/ApiStatsView.vue | 5 +- web/admin-spa/src/views/SettingsView.vue | 18 + 86 files changed, 3803 insertions(+), 2618 deletions(-) create mode 100644 Dcodegithubclaude-relay-servicelogs.claude-relay-audit.log.json create mode 100644 Dcodegithubclaude-relay-servicelogs.claude-relay-auth-detail-audit.log.json create mode 100644 Dcodegithubclaude-relay-servicelogs.claude-relay-error-audit.log.json create mode 100644 Dcodegithubclaude-relay-servicelogs.claude-relay-security-audit.log.json rename src/services/{ => account}/accountBalanceService.js (98%) rename src/services/{ => account}/azureOpenaiAccountService.js (81%) rename src/services/{ => account}/bedrockAccountService.js (91%) rename src/services/{ => account}/ccrAccountService.js (95%) rename src/services/{ => account}/claudeAccountService.js (98%) rename src/services/{ => account}/claudeConsoleAccountService.js (97%) rename src/services/{ => account}/droidAccountService.js (95%) rename src/services/{ => account}/geminiAccountService.js (97%) rename src/services/{ => account}/geminiApiAccountService.js (94%) rename src/services/{ => account}/openaiAccountService.js (96%) rename src/services/{ => account}/openaiResponsesAccountService.js (95%) rename src/services/{ => relay}/antigravityRelayService.js (95%) rename src/services/{ => relay}/azureOpenaiRelayService.js (97%) rename src/services/{ => relay}/bedrockRelayService.js (94%) rename src/services/{ => relay}/ccrRelayService.js (89%) rename src/services/{ => relay}/claudeConsoleRelayService.js (93%) rename src/services/{ => relay}/claudeRelayService.js (97%) rename src/services/{ => relay}/droidRelayService.js (96%) rename src/services/{ => relay}/geminiRelayService.js (98%) rename src/services/{ => relay}/openaiResponsesRelayService.js (87%) rename src/services/{ => scheduler}/droidScheduler.js (72%) rename src/services/{ => scheduler}/unifiedClaudeScheduler.js (98%) rename src/services/{ => scheduler}/unifiedGeminiScheduler.js (91%) rename src/services/{ => scheduler}/unifiedOpenAIScheduler.js (83%) create mode 100644 src/utils/upstreamErrorHelper.js delete mode 100644 web/admin-spa/src/components/accounts/AccountTestModal.vue delete mode 100644 web/admin-spa/src/components/apikeys/ApiKeyTestModal.vue create mode 100644 web/admin-spa/src/components/common/ModelSelector.vue create mode 100644 web/admin-spa/src/components/common/UnifiedTestModal.vue create mode 100644 web/admin-spa/src/components/settings/ModelPricingSection.vue create mode 100644 web/admin-spa/src/utils/useTestState.js diff --git a/.env.example b/.env.example index 7c5f0603..c67b80a0 100644 --- a/.env.example +++ b/.env.example @@ -155,6 +155,12 @@ DEBUG_HTTP_TRAFFIC=false # 启用HTTP请求/响应调试日志（仅开发环 ENABLE_CORS=true TRUST_PROXY=true +# ⏱️ 上游错误自动暂停配置（秒） +# UPSTREAM_ERROR_5XX_TTL_SECONDS=300 # 5xx错误暂停时间（默认5分钟） +# UPSTREAM_ERROR_OVERLOAD_TTL_SECONDS=600 # 529过载暂停时间（默认10分钟） +# UPSTREAM_ERROR_AUTH_TTL_SECONDS=1800 # 401/403认证错误暂停时间（默认30分钟） +# UPSTREAM_ERROR_TIMEOUT_TTL_SECONDS=300 # 504超时暂停时间（默认5分钟） + # 🔒 客户端限制（可选） # ALLOW_CUSTOM_CLIENTS=false diff --git a/CLAUDE.md b/CLAUDE.md index 892b4758..8681d1d2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,637 +1,167 @@ -# CLAUDE.md +# CLAUDE.md <260209.0> -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -这个文件为 Claude Code (claude.ai/code) 提供在此代码库中工作的指导。 +This file provides guidance to Claude Code when working with this repository. ## 项目概述 -Claude Relay Service 是一个多平台 AI API 中转服务，支持 **Claude (官方/Console)、Gemini、OpenAI Responses (Codex)、AWS Bedrock、Azure OpenAI、Droid (Factory.ai)、CCR** 等多种账户类型。提供完整的多账户管理、API Key 认证、代理配置、用户管理、LDAP认证、Webhook通知和现代化 Web 管理界面。该服务作为客户端（如 Claude Code、Gemini CLI、Codex、Droid CLI、Cherry Studio 等）与 AI API 之间的中间件，提供认证、限流、监控、定价计算、成本统计等功能。 +Claude Relay Service — 多平台 AI API 中转服务，作为客户端与上游 AI API 之间的中间件。 +支持 Claude (官方/Console)、Gemini、OpenAI Responses、AWS Bedrock、Azure OpenAI、Droid、CCR 等账户类型。 +核心能力：多账户管理、API Key 认证、统一调度、代理配置、限流、成本统计。 -## 核心架构 +## 架构原则 -### 关键架构概念 +### Clean Architecture 分层映射 -- **统一调度系统**: 使用 unifiedClaudeScheduler、unifiedGeminiScheduler、unifiedOpenAIScheduler、droidScheduler 实现跨账户类型的智能调度 -- **多账户类型支持**: 支持 claude-official、claude-console、bedrock、ccr、droid、gemini、openai-responses、azure-openai 等账户类型 -- **代理认证流**: 客户端用自建API Key → 验证 → 统一调度器选择账户 → 获取账户token → 转发到对应API -- **Token管理**: 自动监控OAuth token过期并刷新，支持10秒提前刷新策略 -- **代理支持**: 每个账户支持独立代理配置，OAuth token交换也通过代理进行 -- **数据加密**: 敏感数据（refreshToken, accessToken, credentials）使用AES加密存储在Redis -- **粘性会话**: 支持会话级别的账户绑定，同一会话使用同一账户，确保上下文连续性 -- **权限控制**: API Key支持权限配置（all/claude/gemini/openai等），控制可访问的服务类型 -- **客户端限制**: 基于User-Agent的客户端识别和限制，支持ClaudeCode、Gemini-CLI等预定义客户端 -- **模型黑名单**: 支持API Key级别的模型访问限制 -- **并发请求排队**: 当API Key并发数超限时，请求进入队列等待而非立即返回429，支持配置最大排队数、超时时间，适用于Claude Code Agent并行工具调用场景 +| 层级 | 目录 | 职责 | +|------|------|------| +| **框架层** | `src/routes/`, `src/middleware/` | HTTP 路由、请求验证、响应格式化 | +| **接口适配层** | `src/handlers/`, `src/services/openaiToClaude.js` | 请求/响应格式转换 | +| **用例层** | `src/services/*Scheduler.js`, `*RelayService.js` | 调度逻辑、转发编排 | +| **实体层** | `src/services/*AccountService.js`, `src/models/` | 账户管理、数据模型 | +| **基础设施层** | `src/utils/`, `config/` | 日志、缓存、加密、代理 | -### 主要服务组件 +### 开发原则 -#### 核心转发服务 +- **依赖方向**: 外层 → 内层，内层不知道外层存在 +- **新增路由**: 只做参数提取和响应格式化，业务逻辑放 service +- **新增服务**: 先确定属于哪一层，遵循该层职责边界 +- **格式转换**: 不同 API 格式的转换放 handlers 或专用转换服务 +- **数据访问**: 通过 `src/models/redis.js` 统一访问 -- **claudeRelayService.js**: Claude官方API转发，处理OAuth认证和流式响应 -- **claudeConsoleRelayService.js**: Claude Console账户转发服务 -- **geminiRelayService.js**: Gemini API转发服务 -- **bedrockRelayService.js**: AWS Bedrock API转发服务 -- **azureOpenaiRelayService.js**: Azure OpenAI API转发服务 -- **droidRelayService.js**: Droid (Factory.ai) API转发服务 -- **ccrRelayService.js**: CCR账户转发服务 -- **openaiResponsesRelayService.js**: OpenAI Responses (Codex) 转发服务 +### 安全约束 -#### 账户管理服务 +- 敏感数据（OAuth token、refreshToken、credentials）必须 AES 加密存储（参考 `claudeAccountService.js`） +- API Key 使用 SHA-256 哈希存储，禁止明文 +- 每个请求必须经过完整认证链（API Key → 权限 → 客户端限制 → 模型黑名单） +- 客户端断开时必须通过 AbortController 清理资源和并发计数 +- 日志中禁止输出完整 token，使用 `tokenMask.js` 脱敏 -- **claudeAccountService.js**: Claude官方账户管理，OAuth token刷新和账户选择 -- **claudeConsoleAccountService.js**: Claude Console账户管理 -- **geminiAccountService.js**: Gemini账户管理，Google OAuth token刷新 -- **bedrockAccountService.js**: AWS Bedrock账户管理 -- **azureOpenaiAccountService.js**: Azure OpenAI账户管理 -- **droidAccountService.js**: Droid账户管理 -- **ccrAccountService.js**: CCR账户管理 -- **openaiResponsesAccountService.js**: OpenAI Responses账户管理 -- **openaiAccountService.js**: OpenAI兼容账户管理 -- **accountGroupService.js**: 账户组管理，支持账户分组和优先级 +## 项目结构 -#### 统一调度器 - -- **unifiedClaudeScheduler.js**: Claude多账户类型统一调度（claude-official/console/bedrock/ccr） -- **unifiedGeminiScheduler.js**: Gemini账户统一调度 -- **unifiedOpenAIScheduler.js**: OpenAI兼容服务统一调度 -- **droidScheduler.js**: Droid账户调度 - -#### 核心功能服务 - -- **apiKeyService.js**: API Key管理，验证、限流、使用统计、成本计算 -- **userService.js**: 用户管理系统，支持用户注册、登录、API Key管理 -- **userMessageQueueService.js**: 用户消息串行队列，防止同账户并发用户消息触发限流 -- **pricingService.js**: 定价服务，模型价格管理和成本计算 -- **costInitService.js**: 成本数据初始化服务 -- **webhookService.js**: Webhook通知服务 -- **webhookConfigService.js**: Webhook配置管理 -- **ldapService.js**: LDAP认证服务 -- **tokenRefreshService.js**: Token自动刷新服务 -- **rateLimitCleanupService.js**: 速率限制状态清理服务 -- **claudeCodeHeadersService.js**: Claude Code客户端请求头处理 - -#### 工具服务 - -- **oauthHelper.js**: OAuth工具，PKCE流程实现和代理支持 -- **workosOAuthHelper.js**: WorkOS OAuth集成 -- **openaiToClaude.js**: OpenAI格式到Claude格式的转换 - -### 认证和代理流程 - -1. 客户端使用自建API Key（cr\_前缀格式）发送请求到对应路由（/api、/claude、/gemini、/openai、/droid等） -2. **authenticateApiKey中间件**验证API Key有效性、速率限制、权限、客户端限制、模型黑名单 -3. **统一调度器**（如 unifiedClaudeScheduler）根据请求模型、会话hash、API Key权限选择最优账户 -4. 检查选中账户的token有效性，过期则自动刷新（使用代理） -5. 根据账户类型调用对应的转发服务（claudeRelayService、geminiRelayService等） -6. 移除客户端API Key，使用账户凭据（OAuth Bearer token、API Key等）转发请求 -7. 通过账户配置的代理发送到目标API（Anthropic、Google、AWS等） -8. 流式或非流式返回响应，捕获真实usage数据 -9. 记录使用统计（input/output/cache_create/cache_read tokens）和成本计算 -10. 更新速率限制计数器和并发控制 - -### OAuth集成 - -- **PKCE流程**: 完整的OAuth 2.0 PKCE实现，支持代理 -- **自动刷新**: 智能token过期检测和自动刷新机制 -- **代理支持**: OAuth授权和token交换全程支持代理配置 -- **安全存储**: claudeAiOauth数据加密存储，包含accessToken、refreshToken、scopes - -## 新增功能概览（相比旧版本） - -### 多平台支持 - -- ✅ **Claude Console账户**: 支持Claude Console类型账户 -- ✅ **AWS Bedrock**: 完整的AWS Bedrock API支持 -- ✅ **Azure OpenAI**: Azure OpenAI服务支持 -- ✅ **Droid (Factory.ai)**: Factory.ai API支持 -- ✅ **CCR账户**: CCR凭据支持 -- ✅ **OpenAI兼容**: OpenAI格式转换和Responses格式支持 - -### 用户和权限系统 - -- ✅ **用户管理**: 完整的用户注册、登录、API Key管理系统 -- ✅ **LDAP认证**: 企业级LDAP/Active Directory集成 -- ✅ **权限控制**: API Key级别的服务权限（all/claude/gemini/openai） -- ✅ **客户端限制**: 基于User-Agent的客户端识别和限制 -- ✅ **模型黑名单**: API Key级别的模型访问控制 - -### 统一调度和会话管理 - -- ✅ **统一调度器**: 跨账户类型的智能调度系统 -- ✅ **粘性会话**: 会话级账户绑定，支持自动续期 -- ✅ **并发控制**: Redis Sorted Set实现的并发限制 -- ✅ **负载均衡**: 自动账户选择和故障转移 - -### 成本和监控 - -- ✅ **定价服务**: 模型价格管理和自动成本计算 -- ✅ **成本统计**: 详细的token使用和费用统计 -- ✅ **缓存监控**: 全局缓存统计和命中率分析 -- ✅ **实时指标**: 可配置窗口的实时统计（METRICS_WINDOW） - -### Webhook和通知 - -- ✅ **Webhook系统**: 事件通知和Webhook配置管理 -- ✅ **多URL支持**: 支持多个Webhook URL（逗号分隔） - -### 高级功能 - -- ✅ **529错误处理**: 自动识别Claude过载状态并暂时排除账户 -- ✅ **HTTP调试**: DEBUG_HTTP_TRAFFIC模式详细记录HTTP请求/响应 -- ✅ **数据迁移**: 完整的数据导入导出工具（含加密/脱敏） -- ✅ **自动清理**: 并发计数、速率限制、临时错误状态自动清理 - -## 常用命令 - -### 基本开发命令 - -````bash -# 安装依赖和初始化 -npm install -npm run setup # 生成配置和管理员凭据 -npm run install:web # 安装Web界面依赖 - -# 开发和运行 -npm run dev # 开发模式（热重载） -npm start # 生产模式 -npm test # 运行测试 -npm run lint # 代码检查 - -# Docker部署 -docker-compose up -d # 推荐方式 -docker-compose --profile monitoring up -d # 包含监控 - -# 服务管理 -npm run service:start:daemon # 后台启动（推荐） -npm run service:status # 查看服务状态 -npm run service:logs # 查看日志 -npm run service:stop # 停止服务 - -### 开发环境配置 - -#### 必须配置的环境变量 -- `JWT_SECRET`: JWT密钥（32字符以上随机字符串） -- `ENCRYPTION_KEY`: 数据加密密钥（32字符固定长度） -- `REDIS_HOST`: Redis主机地址（默认localhost） -- `REDIS_PORT`: Redis端口（默认6379） -- `REDIS_PASSWORD`: Redis密码（可选） - -#### 新增重要环境变量（可选） -- `USER_MANAGEMENT_ENABLED`: 启用用户管理系统（默认false） -- `LDAP_ENABLED`: 启用LDAP认证（默认false） -- `LDAP_URL`: LDAP服务器地址（如 ldaps://ldap.example.com:636） -- `LDAP_TLS_REJECT_UNAUTHORIZED`: LDAP证书验证（默认true） -- `WEBHOOK_ENABLED`: 启用Webhook通知（默认true） -- `WEBHOOK_URLS`: Webhook通知URL列表（逗号分隔） -- `CLAUDE_OVERLOAD_HANDLING_MINUTES`: Claude 529错误处理持续时间（分钟，0表示禁用） -- `STICKY_SESSION_TTL_HOURS`: 粘性会话TTL（小时，默认1） -- `STICKY_SESSION_RENEWAL_THRESHOLD_MINUTES`: 粘性会话续期阈值（分钟，默认0） -- `USER_MESSAGE_QUEUE_ENABLED`: 启用用户消息串行队列（默认false） -- `USER_MESSAGE_QUEUE_DELAY_MS`: 用户消息请求间隔（毫秒，默认200） -- `USER_MESSAGE_QUEUE_TIMEOUT_MS`: 队列等待超时（毫秒，默认5000，锁持有时间短无需长等待） -- `USER_MESSAGE_QUEUE_LOCK_TTL_MS`: 锁TTL（毫秒，默认5000，请求发送后立即释放无需长TTL） -- `METRICS_WINDOW`: 实时指标统计窗口（分钟，1-60，默认5） -- `MAX_API_KEYS_PER_USER`: 每用户最大API Key数量（默认1） -- `ALLOW_USER_DELETE_API_KEYS`: 允许用户删除自己的API Keys（默认false） -- `DEBUG_HTTP_TRAFFIC`: 启用HTTP请求/响应调试日志（默认false，仅开发环境） -- `PROXY_USE_IPV4`: 代理使用IPv4（默认true） -- `REQUEST_TIMEOUT`: 请求超时时间（毫秒，默认600000即10分钟） -- `CLEAR_CONCURRENCY_QUEUES_ON_STARTUP`: 启动时清理残留的并发排队计数器（默认true，多实例部署时建议设为false） - -#### AWS Bedrock配置（可选） -- `CLAUDE_CODE_USE_BEDROCK`: 启用Bedrock（设置为1启用） -- `AWS_REGION`: AWS默认区域（默认us-east-1） -- `ANTHROPIC_MODEL`: Bedrock默认模型 -- `ANTHROPIC_SMALL_FAST_MODEL`: Bedrock小型快速模型 -- `ANTHROPIC_SMALL_FAST_MODEL_AWS_REGION`: 小型模型区域 -- `CLAUDE_CODE_MAX_OUTPUT_TOKENS`: 最大输出tokens（默认4096） -- `MAX_THINKING_TOKENS`: 最大思考tokens（默认1024） -- `DISABLE_PROMPT_CACHING`: 禁用提示缓存（设置为1禁用） - -#### 初始化命令 -```bash -cp config/config.example.js config/config.js -cp .env.example .env -npm run setup # 自动生成密钥并创建管理员账户 +``` +src/ +├── routes/ # HTTP 路由 +│ ├── api.js # Claude API 主路由 +│ ├── admin/ # 管理后台路由（24个子文件） +│ ├── geminiRoutes.js, standardGeminiRoutes.js +│ ├── openaiRoutes.js, openaiClaudeRoutes.js, openaiGeminiRoutes.js +│ ├── azureOpenaiRoutes.js, droidRoutes.js +│ ├── userRoutes.js, webhook.js, unified.js, apiStats.js, web.js +├── middleware/ # auth.js(认证/权限/限流), browserFallback.js +├── handlers/ # geminiHandlers.js +├── services/ # 业务服务 +│ ├── relay/ # 各平台转发服务（9个） +│ ├── account/ # 各平台账户管理（11个） +│ ├── scheduler/ # 统一调度器（4个） +│ ├── apiKeyService.js # API Key 管理 +│ ├── pricingService.js # 定价和成本 +│ └── ... # 其余 ~30 个业务服务 +├── models/redis.js # Redis 数据模型 +├── utils/ # 35+ 工具文件（logger, proxy, oauth, cache, stream...） +config/config.js # 主配置 +scripts/ # 运维脚本 +cli/ # CLI 工具 +web/admin-spa/ # Vue SPA 管理界面 +data/init.json # 管理员凭据 ``` -## Web界面功能 +## 核心请求流程 -### OAuth账户添加流程 +``` +客户端(cr_前缀Key) → 路由 → auth中间件(验证/权限/限流/模型黑名单) + → 统一调度器(选账户/粘性会话) → Token检查/刷新 + → 转发服务(通过代理发送) → 上游API + → 流式/非流式响应 → Usage捕获 → 成本计算 → 返回客户端 +``` -1. **基本信息和代理设置**: 配置账户名称、描述和代理参数 -2. **OAuth授权**: - - 生成授权URL → 用户打开链接并登录Claude Code账号 - - 授权后会显示Authorization Code → 复制并粘贴到输入框 - - 系统自动交换token并创建账户 +关键机制： +- **粘性会话**: 基于请求内容 hash 绑定账户，同一会话用同一账户 +- **并发控制**: Redis Sorted Set 实现，支持排队等待（非直接 429） +- **529 处理**: 自动标记过载账户，配置时长内排除 +- **加密存储**: 敏感数据（OAuth token、credentials）AES 加密存于 Redis +- **流式响应**: SSE 传输，实时捕获 usage，客户端断开时 AbortController 清理资源 -### 核心管理功能 +## 开发规范 -- **实时仪表板**: 系统统计、账户状态、使用量监控、实时指标（METRICS_WINDOW配置窗口） -- **API Key管理**: 创建、配额设置、使用统计查看、权限配置、客户端限制、模型黑名单 -- **多平台账户管理**: - - Claude账户（官方/Console）: OAuth账户添加、代理配置、状态监控 - - Gemini账户: Google OAuth授权、代理配置 - - OpenAI Responses (Codex)账户: API Key配置 - - AWS Bedrock账户: AWS凭据配置 - - Azure OpenAI账户: Azure凭据和端点配置 - - Droid账户: Factory.ai API Key配置 - - CCR账户: CCR凭据配置 -- **用户管理**: 用户注册、登录、API Key分配（USER_MANAGEMENT_ENABLED启用时） -- **系统日志**: 实时日志查看，多级别过滤，HTTP调试日志（DEBUG_HTTP_TRAFFIC启用时） -- **Webhook配置**: Webhook URL管理、事件配置 -- **主题系统**: 支持明亮/暗黑模式切换，自动保存用户偏好设置 -- **成本分析**: 详细的token使用和成本统计（基于pricingService） -- **缓存监控**: 解密缓存统计和性能监控 +### 代码格式化 -## 重要端点 - -### API转发端点（多路由支持） - -#### Claude服务路由 -- `POST /api/v1/messages` - Claude消息处理（支持流式） -- `POST /claude/v1/messages` - Claude消息处理（别名路由） -- `POST /v1/messages/count_tokens` - Token计数Beta API -- `GET /api/v1/models` - 模型列表 -- `GET /api/v1/usage` - 使用统计查询 -- `GET /api/v1/key-info` - API Key信息 -- `GET /v1/me` - 用户信息（Claude Code客户端需要） -- `GET /v1/organizations/:org_id/usage` - 组织使用统计 - -#### Gemini服务路由 -- `POST /gemini/v1/models/:model:generateContent` - 标准Gemini API格式 -- `POST /gemini/v1/models/:model:streamGenerateContent` - Gemini流式 -- `GET /gemini/v1/models` - Gemini模型列表 -- 其他Gemini兼容路由（保持向后兼容） - -#### OpenAI兼容路由 -- `POST /openai/v1/chat/completions` - OpenAI格式转发（支持responses格式） -- `POST /openai/claude/v1/chat/completions` - OpenAI格式转Claude -- `POST /openai/gemini/v1/chat/completions` - OpenAI格式转Gemini -- `GET /openai/v1/models` - OpenAI格式模型列表 - -#### Droid (Factory.ai) 路由 -- `POST /droid/claude/v1/messages` - Droid Claude转发 -- `POST /droid/openai/v1/chat/completions` - Droid OpenAI转发 - -#### Azure OpenAI 路由 -- `POST /azure/...` - Azure OpenAI API转发 - -### 管理端点 - -#### OAuth和账户管理 -- `POST /admin/claude-accounts/generate-auth-url` - 生成OAuth授权URL（含代理） -- `POST /admin/claude-accounts/exchange-code` - 交换authorization code -- `POST /admin/claude-accounts` - 创建Claude OAuth账户 -- 各平台账户CRUD端点（gemini、openai、bedrock、azure、droid、ccr） - -#### 用户管理（USER_MANAGEMENT_ENABLED启用时） -- `POST /users/register` - 用户注册 -- `POST /users/login` - 用户登录 -- `GET /users/profile` - 用户资料 -- `POST /users/api-keys` - 创建用户API Key - -#### Webhook管理 -- `GET /admin/webhook/configs` - 获取Webhook配置 -- `POST /admin/webhook/configs` - 创建Webhook配置 -- `PUT /admin/webhook/configs/:id` - 更新Webhook配置 -- `DELETE /admin/webhook/configs/:id` - 删除Webhook配置 - -### 系统端点 - -- `GET /health` - 健康检查（包含组件状态、版本、内存等） -- `GET /metrics` - 系统指标（使用统计、uptime、内存） -- `GET /web` - 传统Web管理界面 -- `GET /admin-next/` - 新版SPA管理界面（主界面） -- `GET /admin/dashboard` - 系统概览数据 - -## 故障排除 - -### OAuth相关问题 - -1. **代理配置错误**: 检查代理设置是否正确，OAuth token交换也需要代理 -2. **授权码无效**: 确保复制了完整的Authorization Code，没有遗漏字符 -3. **Token刷新失败**: 检查refreshToken有效性和代理配置 - -### Gemini Token刷新问题 - -1. **刷新失败**: 确保 refresh_token 有效且未过期 -2. **错误日志**: 查看 `logs/token-refresh-error.log` 获取详细错误信息 -3. **测试脚本**: 运行 `node scripts/test-gemini-refresh.js` 测试 token 刷新 - -### 常见开发问题 - -1. **Redis连接失败**: 确认Redis服务运行，检查REDIS_HOST、REDIS_PORT、REDIS_PASSWORD配置 -2. **管理员登录失败**: 检查data/init.json存在，运行npm run setup重新初始化 -3. **API Key格式错误**: 确保使用cr\_前缀格式（可通过API_KEY_PREFIX配置修改） -4. **代理连接问题**: 验证SOCKS5/HTTP代理配置和认证信息，检查PROXY_USE_IPV4设置 -5. **粘性会话失效**: 检查Redis中session数据，确认STICKY_SESSION_TTL_HOURS配置，通过Nginx代理时需添加 `underscores_in_headers on;` -6. **LDAP认证失败**: - - 检查LDAP_URL、LDAP_BIND_DN、LDAP_BIND_PASSWORD配置 - - 自签名证书问题：设置 LDAP_TLS_REJECT_UNAUTHORIZED=false - - 查看日志中的LDAP连接错误详情 -7. **用户管理功能不可用**: 确认USER_MANAGEMENT_ENABLED=true，检查userService初始化 -8. **Webhook通知失败**: - - 确认WEBHOOK_ENABLED=true - - 检查WEBHOOK_URLS格式（逗号分隔） - - 查看logs/webhook-*.log日志 -9. **统一调度器选择账户失败**: - - 检查账户状态（status: 'active'） - - 确认账户类型与请求路由匹配 - - 查看粘性会话绑定情况 -10. **并发计数泄漏**: 系统每分钟自动清理过期并发计数（concurrency cleanup task），重启时也会自动清理 -11. **速率限制未清理**: rateLimitCleanupService每5分钟自动清理过期限流状态 -12. **成本统计不准确**: 运行 `npm run init:costs` 初始化成本数据，检查pricingService是否正确加载模型价格 -13. **缓存命中率低**: 查看缓存监控统计，调整LRU缓存大小配置 -14. **用户消息队列超时**: 优化后锁持有时间已从分钟级降到毫秒级（请求发送后立即释放），默认 `USER_MESSAGE_QUEUE_TIMEOUT_MS=5000` 已足够。如仍有超时，检查网络延迟或禁用此功能（`USER_MESSAGE_QUEUE_ENABLED=false`） -15. **并发请求排队问题**: - - 排队超时：检查 `concurrentRequestQueueTimeoutMs` 配置是否合理（默认10秒） - - 排队数过多：调整 `concurrentRequestQueueMaxSize` 和 `concurrentRequestQueueMaxSizeMultiplier` - - 查看排队统计：访问 `/admin/concurrency-queue/stats` 接口查看 entered/success/timeout/cancelled/socket_changed/rejected_overload 统计 - - 排队计数泄漏：系统重启时自动清理，或访问 `/admin/concurrency-queue` DELETE 接口手动清理 - - Socket 身份验证失败：查看 `socket_changed` 统计，如果频繁发生，检查代理配置或客户端连接稳定性 - - 健康检查拒绝：查看 `rejected_overload` 统计，表示队列过载时的快速失败次数 - -### 代理配置要求（并发请求排队） - -使用并发请求排队功能时，需要正确配置代理（如 Nginx）的超时参数： - -- **推荐配置**: `proxy_read_timeout >= max(2 × concurrentRequestQueueTimeoutMs, 60s)` - - 当前默认排队超时 10 秒，Nginx 默认 `proxy_read_timeout = 60s` 已满足要求 - - 如果调整排队超时到 60 秒，推荐代理超时 ≥ 120 秒 -- **Nginx 配置示例**: - ```nginx - location /api/ { - proxy_read_timeout 120s; # 排队超时 60s 时推荐 120s - proxy_connect_timeout 10s; - # ...其他配置 - } - ``` -- **企业防火墙环境**: - - 某些企业防火墙可能静默关闭长时间无数据的连接（20-40 秒） - - 如遇此问题，联系网络管理员调整空闲连接超时策略 - - 或降低 `concurrentRequestQueueTimeoutMs` 配置 -- **后续升级说明**: 如有需要，后续版本可能提供可选的轻量级心跳机制 - -### 调试工具 - -- **日志系统**: Winston结构化日志，支持不同级别，logs/目录下分类存储 - - `logs/claude-relay-*.log` - 应用主日志 - - `logs/token-refresh-error.log` - Token刷新错误 - - `logs/webhook-*.log` - Webhook通知日志 - - `logs/http-debug-*.log` - HTTP调试日志（DEBUG_HTTP_TRAFFIC=true时） -- **CLI工具**: 命令行状态查看和管理（npm run cli） -- **Web界面**: 实时日志查看和系统监控（/admin-next/） -- **健康检查**: /health端点提供系统状态（redis、logger、内存、版本等） -- **系统指标**: /metrics端点提供详细的使用统计和性能指标 -- **缓存监控**: cacheMonitor提供全局缓存统计和命中率分析 -- **数据导出工具**: npm run data:export 导出Redis数据进行调试 -- **Redis Key调试**: npm run data:debug 查看所有Redis键 - -## 开发最佳实践 - -### 代码格式化要求 - -- **必须使用 Prettier 格式化所有代码** -- 后端代码（src/）：运行 `npx prettier --write ` 格式化 -- 前端代码（web/admin-spa/）：已安装 `prettier-plugin-tailwindcss`，运行 `npx prettier --write ` 格式化 -- 提交前检查格式：`npx prettier --check ` -- 格式化所有文件：`npm run format`（如果配置了此脚本） - -### 前端开发特殊要求 - -- **响应式设计**: 必须兼容不同设备尺寸（手机、平板、桌面），使用 Tailwind CSS 响应式前缀（sm:、md:、lg:、xl:） -- **暗黑模式兼容**: 项目已集成完整的暗黑模式支持，所有新增/修改的UI组件都必须同时兼容明亮模式和暗黑模式 - - 使用 Tailwind CSS 的 `dark:` 前缀为暗黑模式提供样式 - - 文本颜色：`text-gray-700 dark:text-gray-200` - - 背景颜色：`bg-white dark:bg-gray-800` - - 边框颜色：`border-gray-200 dark:border-gray-700` - - 状态颜色保持一致：`text-blue-500`、`text-green-600`、`text-red-500` 等 -- **主题切换**: 使用 `stores/theme.js` 中的 `useThemeStore()` 来实现主题切换功能 -- **玻璃态效果**: 保持现有的玻璃态设计风格，在暗黑模式下调整透明度和背景色 -- **图标和交互**: 确保所有图标、按钮、交互元素在两种模式下都清晰可见且易于操作 - -### 代码修改原则 - -- 对现有文件进行修改时，首先检查代码库的现有模式和风格 -- 尽可能重用现有的服务和工具函数，避免重复代码 -- 遵循项目现有的错误处理和日志记录模式 -- 敏感数据必须使用加密存储（参考 claudeAccountService.js 中的加密实现） - -### 测试和质量保证 - -- 运行 `npm run lint` 进行代码风格检查（使用 ESLint） -- 运行 `npm test` 执行测试套件（Jest + SuperTest 配置） -- 在修改核心服务后，使用 CLI 工具验证功能：`npm run cli status` -- 检查日志文件 `logs/claude-relay-*.log` 确认服务正常运行 -- 注意：当前项目缺少实际测试文件，建议补充单元测试和集成测试 +- **必须使用 Prettier**: `npx prettier --write ` +- 前端额外安装了 `prettier-plugin-tailwindcss` +- 提交前检查：`npx prettier --check ` ### 开发工作流 -- **功能开发**: 始终从理解现有代码开始，重用已有的服务和模式 -- **调试流程**: 使用 Winston 日志 + Web 界面实时日志查看 + CLI 状态工具 -- **代码审查**: 关注安全性（加密存储）、性能（异步处理）、错误处理 -- **部署前检查**: 运行 lint → 测试 CLI 功能 → 检查日志 → Docker 构建 +1. **理解现有代码** → 读相关文件，了解现有模式 +2. **编写代码** → 重用已有服务和工具函数 +3. **格式化** → `npx prettier --write <修改的文件>` +4. **检查** → `npm run lint` +5. **测试** → `npm test` +6. **验证** → `npm run cli status` 确认服务正常 -### 常见文件位置 +### 前端要求 -- 核心服务逻辑：`src/services/` 目录（30+服务文件） -- 路由处理：`src/routes/` 目录（api.js、admin.js、geminiRoutes.js、openaiRoutes.js等13个路由文件） -- 中间件：`src/middleware/` 目录（auth.js、browserFallback.js、debugInterceptor.js等） -- 配置管理：`config/config.js`（完整的多平台配置） -- Redis 模型：`src/models/redis.js` -- 工具函数：`src/utils/` 目录 - - `logger.js` - 日志系统 - - `oauthHelper.js` - OAuth工具 - - `proxyHelper.js` - 代理工具 - - `sessionHelper.js` - 会话管理 - - `cacheMonitor.js` - 缓存监控 - - `costCalculator.js` - 成本计算 - - `rateLimitHelper.js` - 速率限制 - - `webhookNotifier.js` - Webhook通知 - - `tokenMask.js` - Token脱敏 - - `workosOAuthHelper.js` - WorkOS OAuth - - `modelHelper.js` - 模型工具 - - `inputValidator.js` - 输入验证 -- CLI工具：`cli/index.js` 和 `src/cli/` 目录 -- 脚本目录：`scripts/` 目录 - - `setup.js` - 初始化脚本 - - `manage.js` - 服务管理 - - `migrate-apikey-expiry.js` - API Key过期迁移 - - `fix-usage-stats.js` - 使用统计修复 - - `data-transfer.js` / `data-transfer-enhanced.js` - 数据导入导出 - - `update-model-pricing.js` - 模型价格更新 - - `test-pricing-fallback.js` - 价格回退测试 - - `debug-redis-keys.js` - Redis调试 -- 前端主题管理：`web/admin-spa/src/stores/theme.js` -- 前端组件：`web/admin-spa/src/components/` 目录 -- 前端页面：`web/admin-spa/src/views/` 目录 -- 初始化数据：`data/init.json`（管理员凭据存储） -- 日志目录：`logs/`（各类日志文件） +- 响应式设计：Tailwind CSS 响应式前缀（sm:、md:、lg:、xl:） +- 暗黑模式：所有组件必须兼容，使用 `dark:` 前缀 +- 主题切换：`web/admin-spa/src/stores/theme.js` 的 `useThemeStore()` +- 保持现有玻璃态设计风格 -### 重要架构决策 +暗黑模式配色对照： -- **统一调度系统**: 使用统一调度器（unifiedClaudeScheduler等）实现跨账户类型的智能调度，支持粘性会话、负载均衡、故障转移 -- **多账户类型支持**: 支持8种账户类型（claude-official、claude-console、bedrock、ccr、droid、gemini、openai-responses、azure-openai） -- **加密存储**: 所有敏感数据（OAuth token、refreshToken、credentials）都使用 AES 加密存储在 Redis -- **独立代理**: 每个账户支持独立的代理配置（SOCKS5/HTTP），包括OAuth授权流程 -- **API Key哈希**: 使用SHA-256哈希存储，支持自定义前缀（默认 `cr_`） -- **权限系统**: API Key支持细粒度权限控制（all/claude/gemini/openai等） -- **请求流程**: API Key验证（含权限、客户端、模型黑名单） → 统一调度器选择账户 → Token刷新（如需）→ 请求转发 → Usage捕获 → 成本计算 -- **流式响应**: 支持SSE流式响应，实时捕获真实usage数据，客户端断开时自动清理资源（AbortController） -- **粘性会话**: 基于请求内容hash的会话绑定，同一会话始终使用同一账户，支持自动续期 -- **自动清理**: 定时清理任务（过期Key、错误账户、临时错误、并发计数、速率限制状态） -- **缓存优化**: 多层LRU缓存（解密缓存、账户缓存），全局缓存监控和统计 -- **成本追踪**: 实时token使用统计（input/output/cache_create/cache_read）和成本计算（基于pricingService） -- **并发控制**: Redis Sorted Set实现的并发计数，支持自动过期清理 -- **并发请求排队**: 当API Key并发超限时，请求进入队列等待而非直接返回429 - - **工作原理**: 采用「先占后检查」模式，每次轮询尝试占位，超限则释放继续等待 - - **指数退避**: 初始200ms，指数增长至最大2秒，带±20%抖动防惊群效应 - - **智能清理**: 排队计数有TTL保护（超时+30秒），进程崩溃也能自动清理 - - **Socket身份验证**: 使用UUID token + socket对象引用双重验证，避免HTTP Keep-Alive连接复用导致的身份混淆 - - **健康检查**: P90等待时间超过阈值时快速失败（返回429），避免新请求在过载时继续排队 - - **配置参数**: `concurrentRequestQueueEnabled`（默认false）、`concurrentRequestQueueMaxSize`（默认3）、`concurrentRequestQueueMaxSizeMultiplier`（默认0）、`concurrentRequestQueueTimeoutMs`（默认10秒）、`concurrentRequestQueueMaxRedisFailCount`（默认5）、`concurrentRequestQueueHealthCheckEnabled`（默认true）、`concurrentRequestQueueHealthThreshold`（默认0.8） - - **最大排队数**: max(固定值, 并发限制×倍数)，例如并发限制=10、倍数=2时最大排队数=20 - - **适用场景**: Claude Code Agent并行工具调用、批量请求处理 -- **客户端识别**: 基于User-Agent的客户端限制，支持预定义客户端（ClaudeCode、Gemini-CLI等） -- **错误处理**: 529错误自动标记账户过载状态，配置时长内自动排除该账户 +| 元素 | 明亮模式 | 暗黑模式 | +|------|----------|----------| +| 文本 | `text-gray-700` | `dark:text-gray-200` | +| 背景 | `bg-white` | `dark:bg-gray-800` | +| 边框 | `border-gray-200` | `dark:border-gray-700` | +| 状态色 | `text-blue-500` / `text-green-600` / `text-red-500` | 保持一致 | -### 核心数据流和性能优化 +### 代码修改原则 -- **哈希映射优化**: API Key 验证从 O(n) 优化到 O(1) 查找 -- **智能 Usage 捕获**: 从 SSE 流中解析真实的 token 使用数据 -- **多维度统计**: 支持按时间、模型、用户的实时使用统计 -- **异步处理**: 非阻塞的统计记录和日志写入 -- **原子操作**: Redis 管道操作确保数据一致性 +- 先检查现有模式和风格，重用已有服务和工具函数 +- 敏感数据必须加密存储（参考 claudeAccountService.js） +- 遵循现有的错误处理和日志记录模式 -### 安全和容错机制 - -- **多层加密**: API Key 哈希 + OAuth Token AES 加密 -- **零信任验证**: 每个请求都需要完整的认证链 -- **优雅降级**: Redis 连接失败时的回退机制 -- **自动重试**: 指数退避重试策略和错误隔离 -- **资源清理**: 客户端断开时的自动清理机制 - -## 项目特定注意事项 - -### Redis 数据结构 - -- **API Keys**: - - `api_key:{id}` - API Key详细信息（含权限、客户端限制、模型黑名单等） - - `api_key_hash:{hash}` - 哈希到ID的快速映射 - - `api_key_usage:{keyId}` - 使用统计数据 - - `api_key_cost:{keyId}` - 成本统计数据 -- **账户数据**（多类型）: - - `claude_account:{id}` - Claude官方账户（加密的OAuth数据） - - `claude_console_account:{id}` - Claude Console账户 - - `gemini_account:{id}` - Gemini账户 - - `openai_responses_account:{id}` - OpenAI Responses账户 - - `bedrock_account:{id}` - AWS Bedrock账户 - - `azure_openai_account:{id}` - Azure OpenAI账户 - - `droid_account:{id}` - Droid账户 - - `ccr_account:{id}` - CCR账户 -- **用户管理**: - - `user:{id}` - 用户信息 - - `user_email:{email}` - 邮箱到用户ID映射 - - `user_session:{token}` - 用户会话 -- **管理员**: - - `admin:{id}` - 管理员信息 - - `admin_username:{username}` - 用户名映射 - - `admin_credentials` - 管理员凭据（从data/init.json同步） -- **会话管理**: - - `session:{token}` - JWT会话管理 - - `sticky_session:{sessionHash}` - 粘性会话账户绑定 - - `session_window:{accountId}` - 账户会话窗口 -- **使用统计**: - - `usage:daily:{date}:{key}:{model}` - 按日期、Key、模型的使用统计 - - `usage:account:{accountId}:{date}` - 按账户的使用统计 - - `usage:global:{date}` - 全局使用统计 -- **速率限制**: - - `rate_limit:{keyId}:{window}` - 速率限制计数器 - - `rate_limit_state:{accountId}` - 账户限流状态 - - `overload:{accountId}` - 账户过载状态（529错误） -- **并发控制**: - - `concurrency:{accountId}` - Redis Sorted Set实现的并发计数 -- **并发请求排队**: - - `concurrency:queue:{apiKeyId}` - API Key级别的排队计数器（TTL由 `concurrentRequestQueueTimeoutMs` + 30秒缓冲决定） - - `concurrency:queue:stats:{apiKeyId}` - 排队统计（entered/success/timeout/cancelled） - - `concurrency:queue:wait_times:{apiKeyId}` - 按API Key的等待时间记录（用于P50/P90/P99计算） - - `concurrency:queue:wait_times:global` - 全局等待时间记录 -- **Webhook配置**: - - `webhook_config:{id}` - Webhook配置 -- **用户消息队列**: - - `user_msg_queue_lock:{accountId}` - 用户消息队列锁（当前持有者requestId） - - `user_msg_queue_last:{accountId}` - 上次请求完成时间戳（用于延迟计算） -- **系统信息**: - - `system_info` - 系统状态缓存 - - `model_pricing` - 模型价格数据（pricingService） - -### 流式响应处理 - -- 支持 SSE (Server-Sent Events) 流式传输，实时推送响应数据 -- 自动从SSE流中解析真实usage数据（input/output/cache_create/cache_read tokens） -- 客户端断开时通过 AbortController 清理资源和并发计数 -- 错误时发送适当的 SSE 错误事件（带时间戳和错误类型） -- 支持大文件流式传输（REQUEST_TIMEOUT配置超时时间） -- 禁用Nagle算法确保数据立即发送（socket.setNoDelay） -- 设置 `X-Accel-Buffering: no` 禁用Nginx缓冲 - -### CLI 工具使用示例 +## 常用命令 ```bash -# API Key管理 -npm run cli keys create -- --name "MyApp" --limit 1000 -npm run cli keys list -npm run cli keys delete -- --id -npm run cli keys update -- --id --limit 2000 - -# 系统状态查看 -npm run cli status # 查看系统概况 -npm run status # 统一状态脚本 -npm run status:detail # 详细状态 - -# Claude账户管理 -npm run cli accounts list -npm run cli accounts refresh -npm run cli accounts add -- --name "Account1" - -# Gemini账户管理 -npm run cli gemini list -npm run cli gemini add -- --name "Gemini1" - -# 管理员操作 -npm run cli admin create -- --username admin2 -npm run cli admin reset-password -- --username admin -npm run cli admin list - -# 数据管理 -npm run data:export # 导出Redis数据 -npm run data:export:sanitized # 导出脱敏数据 -npm run data:export:enhanced # 增强导出（含解密） -npm run data:export:encrypted # 导出加密数据 -npm run data:import # 导入数据 -npm run data:import:enhanced # 增强导入 -npm run data:debug # 调试Redis键 - -# 数据迁移和修复 -npm run migrate:apikey-expiry # API Key过期时间迁移 -npm run migrate:apikey-expiry:dry # 干跑模式 -npm run migrate:fix-usage-stats # 修复使用统计 - -# 成本和定价 -npm run init:costs # 初始化成本数据 -npm run update:pricing # 更新模型价格 -npm run test:pricing-fallback # 测试价格回退 - -# 监控 -npm run monitor # 增强监控脚本 +npm install && npm run setup # 初始化 +npm run dev # 开发模式（热重载） +npm start # 生产模式 +npm run lint # ESLint 检查 +npm test # Jest + SuperTest +docker-compose up -d # Docker 部署 +npm run cli status # 系统状态 +npm run data:export # 导出 Redis 数据 +npm run data:debug # 调试 Redis 键 ``` +## 环境变量（必须） + +- `JWT_SECRET` — JWT 密钥（32字符+） +- `ENCRYPTION_KEY` — AES 加密密钥（32字符固定） +- `REDIS_HOST` / `REDIS_PORT` / `REDIS_PASSWORD` — Redis 连接 + +其他可选环境变量见 `.env.example`。 + +## 故障排除 + +| 问题 | 排查方向 | +|------|----------| +| Redis 连接失败 | 检查 REDIS_HOST/PORT/PASSWORD | +| 管理员登录失败 | 检查 data/init.json，运行 `npm run setup` | +| API Key 格式错误 | 确保使用 `cr_` 前缀格式（可通过 API_KEY_PREFIX 配置） | +| Token 刷新失败 | 检查 refreshToken 有效性和代理配置，查看 `logs/token-refresh-error.log` | +| 调度器选账户失败 | 检查账户 status:'active'，确认类型与路由匹配，查看粘性会话绑定 | +| 并发计数泄漏 | 系统每分钟自动清理，重启也会清理 | +| 粘性会话失效 | 检查 Redis 中 session 数据，Nginx 代理需添加 `underscores_in_headers on` | +| LDAP 认证失败 | 检查 LDAP_URL/BIND_DN/BIND_PASSWORD，自签名证书设 `LDAP_TLS_REJECT_UNAUTHORIZED=false` | +| Webhook 通知失败 | 确认 WEBHOOK_ENABLED=true，检查 WEBHOOK_URLS 格式，查看 `logs/webhook-*.log` | +| 成本统计不准确 | 运行 `npm run init:costs`，检查 pricingService 模型价格 | + +日志：`logs/` 目录。Web 界面 `/admin-next/` 可实时查看。 + # important-instruction-reminders Do what has been asked; nothing more, nothing less. NEVER create files unless they're absolutely necessary for achieving your goal. ALWAYS prefer editing an existing file to creating a new one. NEVER proactively create documentation files (\*.md) or README files. Only create documentation files if explicitly requested by the User. -```` diff --git a/Dcodegithubclaude-relay-servicelogs.claude-relay-audit.log.json b/Dcodegithubclaude-relay-servicelogs.claude-relay-audit.log.json new file mode 100644 index 00000000..2ea414e3 --- /dev/null +++ b/Dcodegithubclaude-relay-servicelogs.claude-relay-audit.log.json @@ -0,0 +1,35 @@ +{ + "keep": { + "days": false, + "amount": 5 + }, + "auditLog": "D:\\code\\github\\claude-relay-service\\logs\\.claude-relay-audit.log.json", + "files": [ + { + "date": 1769443203308, + "name": "D:\\code\\github\\claude-relay-service\\logs\\claude-relay-2026-01-27.log", + "hash": "2d09cc308d32c20207a0bf4eb0ae7aa7f25485b5cf2fee4dfce4be9ff2db8055" + }, + { + "date": 1770353006077, + "name": "D:\\code\\github\\claude-relay-service\\logs\\claude-relay-2026-02-06.log", + "hash": "b370804c9b7f59563bcbbdb8bb2a920bd5711af9f25af77d50cb78e11c482e34" + }, + { + "date": 1770535480737, + "name": "D:\\code\\github\\claude-relay-service\\logs\\claude-relay-2026-02-08.log", + "hash": "641ed3aaf8fe8003a5ed1ebb2331bf18f6957f26bd084e74d33b54aa69a5ff69" + }, + { + "date": 1770566683359, + "name": "D:\\code\\github\\claude-relay-service\\logs\\claude-relay-2026-02-09.log", + "hash": "66a23ce0a7addb428d1f03594856b909a5660cf19574ab4b7cedf42aa48574f5" + }, + { + "date": 1770623291843, + "name": "/mnt/d/code/github/claude-relay-service/logs/claude-relay-2026-02-09.log", + "hash": "261a1c288f8f0a2b394c970ece8d005240366bccac0cfe9f7f90c44deb8ea8da" + } + ], + "hashType": "sha256" +} \ No newline at end of file diff --git a/Dcodegithubclaude-relay-servicelogs.claude-relay-auth-detail-audit.log.json b/Dcodegithubclaude-relay-servicelogs.claude-relay-auth-detail-audit.log.json new file mode 100644 index 00000000..fa5e9e59 --- /dev/null +++ b/Dcodegithubclaude-relay-servicelogs.claude-relay-auth-detail-audit.log.json @@ -0,0 +1,35 @@ +{ + "keep": { + "days": false, + "amount": 5 + }, + "auditLog": "D:\\code\\github\\claude-relay-service\\logs\\.claude-relay-auth-detail-audit.log.json", + "files": [ + { + "date": 1769440584327, + "name": "D:\\code\\github\\claude-relay-service\\logs\\claude-relay-auth-detail-2026-01-26.log", + "hash": "08a424abf9d6edc0047328d0c2ca6c1f377b61c71ce13d4bf5226719ea0ac4a6" + }, + { + "date": 1770353006082, + "name": "D:\\code\\github\\claude-relay-service\\logs\\claude-relay-auth-detail-2026-02-06.log", + "hash": "94960d294f44af312596a8c363a42f21d4ac14d6641bf0d41ce6a7892a72777d" + }, + { + "date": 1770535480742, + "name": "D:\\code\\github\\claude-relay-service\\logs\\claude-relay-auth-detail-2026-02-08.log", + "hash": "2c8ab04264ba558d4f288851d4ecc293f3ca6f1f44a6b1a7fa8572372cc222bc" + }, + { + "date": 1770568187072, + "name": "D:\\code\\github\\claude-relay-service\\logs\\claude-relay-auth-detail-2026-02-09.log", + "hash": "2e53b7737b1c159bb0767a53ec6840e5fb40632bc09e75d68b436e709ae77e48" + }, + { + "date": 1770623291858, + "name": "/mnt/d/code/github/claude-relay-service/logs/claude-relay-auth-detail-2026-02-09.log", + "hash": "59fdda7c9d8d3fead2b5caef37285278e5b2f7d761aabeaf7809b0e312fde2bc" + } + ], + "hashType": "sha256" +} \ No newline at end of file diff --git a/Dcodegithubclaude-relay-servicelogs.claude-relay-error-audit.log.json b/Dcodegithubclaude-relay-servicelogs.claude-relay-error-audit.log.json new file mode 100644 index 00000000..1a409cc1 --- /dev/null +++ b/Dcodegithubclaude-relay-servicelogs.claude-relay-error-audit.log.json @@ -0,0 +1,35 @@ +{ + "keep": { + "days": false, + "amount": 5 + }, + "auditLog": "D:\\code\\github\\claude-relay-service\\logs\\.claude-relay-error-audit.log.json", + "files": [ + { + "date": 1769440584323, + "name": "D:\\code\\github\\claude-relay-service\\logs\\claude-relay-error-2026-01-26.log", + "hash": "8370bcc61ba4622611d3962dabf9e174600f5d3ac541e6ec3e3fab9f25557d39" + }, + { + "date": 1770353006079, + "name": "D:\\code\\github\\claude-relay-service\\logs\\claude-relay-error-2026-02-06.log", + "hash": "7ef9cbd6923d8439948c9933ccdcc1969bf574b5bf3ce363df613278e9fe0ff7" + }, + { + "date": 1770535480739, + "name": "D:\\code\\github\\claude-relay-service\\logs\\claude-relay-error-2026-02-08.log", + "hash": "003cd6b8b6d1b690ecc5db32d12c6bf928ac063292d0411191d38b5e353e03f7" + }, + { + "date": 1770568187069, + "name": "D:\\code\\github\\claude-relay-service\\logs\\claude-relay-error-2026-02-09.log", + "hash": "3e1b5ef731712188fb42b1f8dfdfcfb3938ce4c735007d064586286bad026978" + }, + { + "date": 1770623291848, + "name": "/mnt/d/code/github/claude-relay-service/logs/claude-relay-error-2026-02-09.log", + "hash": "0ab72bd08dff65ef26beb0df51cabfd29a6d71bd4b65c609496a06c00e939718" + } + ], + "hashType": "sha256" +} \ No newline at end of file diff --git a/Dcodegithubclaude-relay-servicelogs.claude-relay-security-audit.log.json b/Dcodegithubclaude-relay-servicelogs.claude-relay-security-audit.log.json new file mode 100644 index 00000000..c3a9f90f --- /dev/null +++ b/Dcodegithubclaude-relay-servicelogs.claude-relay-security-audit.log.json @@ -0,0 +1,35 @@ +{ + "keep": { + "days": false, + "amount": 5 + }, + "auditLog": "D:\\code\\github\\claude-relay-service\\logs\\.claude-relay-security-audit.log.json", + "files": [ + { + "date": 1769443932546, + "name": "D:\\code\\github\\claude-relay-service\\logs\\claude-relay-security-2026-01-27.log", + "hash": "8fd27ee13e9c6f8c3f20ce2dd84292d0ba2b132ae3c5a67e86c3ebb212efe36d" + }, + { + "date": 1770353006080, + "name": "D:\\code\\github\\claude-relay-service\\logs\\claude-relay-security-2026-02-06.log", + "hash": "4efd558cc9ecacc592809085db8e0014db2285fa05dd06a4476fdefc66fcef06" + }, + { + "date": 1770535480741, + "name": "D:\\code\\github\\claude-relay-service\\logs\\claude-relay-security-2026-02-08.log", + "hash": "59e5ca0add7eccb588c701630ef109e319fa617a9eed87986a9c8ea117fecb4b" + }, + { + "date": 1770567313766, + "name": "D:\\code\\github\\claude-relay-service\\logs\\claude-relay-security-2026-02-09.log", + "hash": "5f58e9eb8dda2a1bde5e2573152c310e76716e8856072d9d32746e0561cdeb23" + }, + { + "date": 1770623291852, + "name": "/mnt/d/code/github/claude-relay-service/logs/claude-relay-security-2026-02-09.log", + "hash": "4f0265e9f8b3322ae3929aec511e4baf4473f784f5e75cbfa110a9e1f6e3e6f4" + } + ], + "hashType": "sha256" +} \ No newline at end of file diff --git a/cli/index.js b/cli/index.js index cbee5076..5514c30a 100644 --- a/cli/index.js +++ b/cli/index.js @@ -11,8 +11,8 @@ const path = require('path') const redis = require('../src/models/redis') const apiKeyService = require('../src/services/apiKeyService') -const claudeAccountService = require('../src/services/claudeAccountService') -const bedrockAccountService = require('../src/services/bedrockAccountService') +const claudeAccountService = require('../src/services/account/claudeAccountService') +const bedrockAccountService = require('../src/services/account/bedrockAccountService') const program = new Command() diff --git a/config/config.example.js b/config/config.example.js index eb6e4494..dda050bc 100644 --- a/config/config.example.js +++ b/config/config.example.js @@ -228,6 +228,14 @@ const config = { enabled: process.env.QUOTA_CARD_LIMITS_ENABLED !== 'false', // 默认启用 maxExpiryDays: parseInt(process.env.QUOTA_CARD_MAX_EXPIRY_DAYS) || 90, // 最大有效期距今天数 maxTotalCostLimit: parseFloat(process.env.QUOTA_CARD_MAX_TOTAL_COST_LIMIT) || 1000 // 最大总额度（美元） + }, + + // ⏱️ 上游错误自动暂停配置 + upstreamError: { + serverErrorTtlSeconds: parseInt(process.env.UPSTREAM_ERROR_5XX_TTL_SECONDS) || 300, // 5xx错误暂停秒数 + overloadTtlSeconds: parseInt(process.env.UPSTREAM_ERROR_OVERLOAD_TTL_SECONDS) || 600, // 529过载暂停秒数 + authErrorTtlSeconds: parseInt(process.env.UPSTREAM_ERROR_AUTH_TTL_SECONDS) || 1800, // 401/403认证错误暂停秒数 + timeoutTtlSeconds: parseInt(process.env.UPSTREAM_ERROR_TIMEOUT_TTL_SECONDS) || 300 // 504超时暂停秒数 } } diff --git a/config/models.js b/config/models.js index c1dddf72..595c4403 100644 --- a/config/models.js +++ b/config/models.js @@ -35,6 +35,13 @@ const OPENAI_MODELS = [ { value: 'codex-mini', label: 'Codex Mini' } ] +const BEDROCK_MODELS = [ + { value: 'us.anthropic.claude-opus-4-6-20250610-v1:0', label: 'Claude Opus 4.6' }, + { value: 'us.anthropic.claude-sonnet-4-5-20250929-v1:0', label: 'Claude Sonnet 4.5' }, + { value: 'us.anthropic.claude-sonnet-4-20250514-v1:0', label: 'Claude Sonnet 4' }, + { value: 'us.anthropic.claude-3-5-haiku-20241022-v1:0', label: 'Claude 3.5 Haiku' } +] + // 其他模型（用于账户编辑的模型映射） const OTHER_MODELS = [ { value: 'deepseek-chat', label: 'DeepSeek Chat' }, @@ -43,11 +50,25 @@ const OTHER_MODELS = [ { value: 'GLM', label: 'GLM' } ] +// 各平台测试可用模型 +const PLATFORM_TEST_MODELS = { + claude: CLAUDE_MODELS, + 'claude-console': CLAUDE_MODELS, + bedrock: BEDROCK_MODELS, + gemini: GEMINI_MODELS, + 'openai-responses': OPENAI_MODELS, + 'azure-openai': [], + droid: CLAUDE_MODELS, + ccr: CLAUDE_MODELS +} + module.exports = { CLAUDE_MODELS, GEMINI_MODELS, OPENAI_MODELS, + BEDROCK_MODELS, OTHER_MODELS, + PLATFORM_TEST_MODELS, // 按服务分组 getModelsByService: (service) => { switch (service) { diff --git a/scripts/manage-session-windows.js b/scripts/manage-session-windows.js index a8ed7a2d..781af5cc 100644 --- a/scripts/manage-session-windows.js +++ b/scripts/manage-session-windows.js @@ -6,7 +6,7 @@ */ const redis = require('../src/models/redis') -const claudeAccountService = require('../src/services/claudeAccountService') +const claudeAccountService = require('../src/services/account/claudeAccountService') const readline = require('readline') // 创建readline接口 diff --git a/scripts/test-api-response.js b/scripts/test-api-response.js index 8131e0f9..7ed3f0ee 100644 --- a/scripts/test-api-response.js +++ b/scripts/test-api-response.js @@ -3,8 +3,8 @@ */ const redis = require('../src/models/redis') -const claudeAccountService = require('../src/services/claudeAccountService') -const claudeConsoleAccountService = require('../src/services/claudeConsoleAccountService') +const claudeAccountService = require('../src/services/account/claudeAccountService') +const claudeConsoleAccountService = require('../src/services/account/claudeConsoleAccountService') const accountGroupService = require('../src/services/accountGroupService') async function testApiResponse() { diff --git a/scripts/test-bedrock-models.js b/scripts/test-bedrock-models.js index e0d62eff..ad325b81 100644 --- a/scripts/test-bedrock-models.js +++ b/scripts/test-bedrock-models.js @@ -1,6 +1,6 @@ #!/usr/bin/env node -const bedrockRelayService = require('../src/services/bedrockRelayService') +const bedrockRelayService = require('../src/services/relay/bedrockRelayService') async function testBedrockModels() { try { diff --git a/scripts/test-gemini-refresh.js b/scripts/test-gemini-refresh.js index 6de3ee78..2ac6386a 100644 --- a/scripts/test-gemini-refresh.js +++ b/scripts/test-gemini-refresh.js @@ -11,7 +11,7 @@ const dotenv = require('dotenv') dotenv.config({ path: path.join(__dirname, '..', '.env') }) const redis = require('../src/models/redis') -const geminiAccountService = require('../src/services/geminiAccountService') +const geminiAccountService = require('../src/services/account/geminiAccountService') const crypto = require('crypto') const config = require('../config/config') diff --git a/scripts/test-group-scheduling.js b/scripts/test-group-scheduling.js index e22a20e1..a604e6d0 100644 --- a/scripts/test-group-scheduling.js +++ b/scripts/test-group-scheduling.js @@ -7,10 +7,10 @@ require('dotenv').config() const { v4: uuidv4 } = require('uuid') const redis = require('../src/models/redis') const accountGroupService = require('../src/services/accountGroupService') -const claudeAccountService = require('../src/services/claudeAccountService') -const claudeConsoleAccountService = require('../src/services/claudeConsoleAccountService') +const claudeAccountService = require('../src/services/account/claudeAccountService') +const claudeConsoleAccountService = require('../src/services/account/claudeConsoleAccountService') const apiKeyService = require('../src/services/apiKeyService') -const unifiedClaudeScheduler = require('../src/services/unifiedClaudeScheduler') +const unifiedClaudeScheduler = require('../src/services/scheduler/unifiedClaudeScheduler') // 测试配置 const TEST_PREFIX = 'test_group_' diff --git a/scripts/test-model-mapping.js b/scripts/test-model-mapping.js index 7719f9d5..ade02e7f 100644 --- a/scripts/test-model-mapping.js +++ b/scripts/test-model-mapping.js @@ -1,6 +1,6 @@ #!/usr/bin/env node -const bedrockRelayService = require('../src/services/bedrockRelayService') +const bedrockRelayService = require('../src/services/relay/bedrockRelayService') function testModelMapping() { console.log('🧪 测试模型映射功能...') diff --git a/src/app.js b/src/app.js index 576fa796..e6875b4c 100644 --- a/src/app.js +++ b/src/app.js @@ -86,7 +86,7 @@ class Application { // 💳 初始化账户余额查询服务（Provider 注册） try { - const accountBalanceService = require('./services/accountBalanceService') + const accountBalanceService = require('./services/account/accountBalanceService') const { registerAllProviders } = require('./services/balanceProviders') registerAllProviders(accountBalanceService) logger.info('✅ 账户余额查询服务已初始化') @@ -137,7 +137,7 @@ class Application { // 🕐 初始化Claude账户会话窗口 logger.info('🕐 Initializing Claude account session windows...') - const claudeAccountService = require('./services/claudeAccountService') + const claudeAccountService = require('./services/account/claudeAccountService') await claudeAccountService.initializeSessionWindows() // 📊 初始化费用排序索引服务 @@ -639,9 +639,12 @@ class Application { // 注册各个服务的缓存实例 const services = [ - { name: 'claudeAccount', service: require('./services/claudeAccountService') }, - { name: 'claudeConsole', service: require('./services/claudeConsoleAccountService') }, - { name: 'bedrockAccount', service: require('./services/bedrockAccountService') } + { name: 'claudeAccount', service: require('./services/account/claudeAccountService') }, + { + name: 'claudeConsole', + service: require('./services/account/claudeConsoleAccountService') + }, + { name: 'bedrockAccount', service: require('./services/account/bedrockAccountService') } ] // 注册已加载的服务缓存 @@ -673,7 +676,7 @@ class Application { logger.info('🧹 Starting scheduled cleanup...') const apiKeyService = require('./services/apiKeyService') - const claudeAccountService = require('./services/claudeAccountService') + const claudeAccountService = require('./services/account/claudeAccountService') const [expiredKeys, errorAccounts] = await Promise.all([ apiKeyService.cleanupExpiredKeys(), diff --git a/src/handlers/geminiHandlers.js b/src/handlers/geminiHandlers.js index d12f2f1c..0d79d612 100644 --- a/src/handlers/geminiHandlers.js +++ b/src/handlers/geminiHandlers.js @@ -6,13 +6,13 @@ */ const logger = require('../utils/logger') -const geminiAccountService = require('../services/geminiAccountService') -const geminiApiAccountService = require('../services/geminiApiAccountService') -const { sendGeminiRequest, getAvailableModels } = require('../services/geminiRelayService') -const { sendAntigravityRequest } = require('../services/antigravityRelayService') +const geminiAccountService = require('../services/account/geminiAccountService') +const geminiApiAccountService = require('../services/account/geminiApiAccountService') +const { sendGeminiRequest, getAvailableModels } = require('../services/relay/geminiRelayService') +const { sendAntigravityRequest } = require('../services/relay/antigravityRelayService') const crypto = require('crypto') const sessionHelper = require('../utils/sessionHelper') -const unifiedGeminiScheduler = require('../services/unifiedGeminiScheduler') +const unifiedGeminiScheduler = require('../services/scheduler/unifiedGeminiScheduler') const apiKeyService = require('../services/apiKeyService') const redis = require('../models/redis') const { updateRateLimitCounters } = require('../utils/rateLimitHelper') @@ -20,6 +20,52 @@ const { parseSSELine } = require('../utils/sseParser') const axios = require('axios') const { getSafeMessage } = require('../utils/errorSanitizer') const ProxyHelper = require('../utils/proxyHelper') +const upstreamErrorHelper = require('../utils/upstreamErrorHelper') + +// 处理 Gemini 上游错误，标记账户为临时不可用 +const handleGeminiUpstreamError = async ( + errorStatus, + accountId, + accountType, + sessionHash, + headers, + disableAutoProtection = false +) => { + if (!accountId || !errorStatus) { + return + } + const autoProtectionDisabled = disableAutoProtection === true || disableAutoProtection === 'true' + try { + if (errorStatus === 429) { + if (!autoProtectionDisabled) { + const ttl = upstreamErrorHelper.parseRetryAfter(headers) + await upstreamErrorHelper.markTempUnavailable(accountId, accountType || 'gemini', 429, ttl) + // 同时设置 rate-limit 状态，保持与 /messages handler 一致 + await unifiedGeminiScheduler + .markAccountRateLimited(accountId, accountType || 'gemini', sessionHash) + .catch((e) => logger.warn('Failed to mark account as rate limited:', e)) + } + if (sessionHash) { + await unifiedGeminiScheduler._deleteSessionMapping(sessionHash) + } + return + } + if (errorStatus >= 500 || errorStatus === 401 || errorStatus === 403) { + if (!autoProtectionDisabled) { + await upstreamErrorHelper.markTempUnavailable( + accountId, + accountType || 'gemini', + errorStatus + ) + } + } + if (sessionHash) { + await unifiedGeminiScheduler._deleteSessionMapping(sessionHash) + } + } catch (e) { + logger.warn('[UpstreamError] Failed to handle Gemini upstream error:', e) + } +} // ============================================================================ // 工具函数 @@ -44,28 +90,65 @@ function buildGeminiApiUrl(baseUrl, model, action, apiKey, options = {}) { // 移除末尾的斜杠（如果有） const normalizedBaseUrl = baseUrl.replace(/\/+$/, '') - // 检查是否为新格式（以 /models 结尾） - const isNewFormat = normalizedBaseUrl.endsWith('/models') + // 模式 3: URL 模板（包含 {model} 占位符） + const isTemplate = normalizedBaseUrl.includes('{model}') + // 模式 2: 以 /models 结尾 + const isModelsFormat = normalizedBaseUrl.endsWith('/models') + + // 模板校验: 有 {model} 但没有 {action} 且 {model} 后面没有 : 开头的固定 action + if (isTemplate && !listModels && !normalizedBaseUrl.includes('{action}')) { + const afterModel = normalizedBaseUrl.split('{model}')[1] || '' + if (!afterModel.startsWith(':')) { + const err = new Error( + `Gemini baseUrl 模板配置错误: 包含 {model} 但缺少 :{action} 或固定 action。` + + `当前: ${baseUrl}，示例: https://proxy.com/v1beta/models/{model}:{action}` + ) + err.statusCode = 400 + throw err + } + } let url if (listModels) { - // 获取模型列表 - if (isNewFormat) { - // 新格式: baseUrl 已包含 /v1beta/models，直接添加查询参数 + if (isTemplate) { + // 模板模式: 分离 path 和 query，分别剔除含 {model}/{action} 的部分 + const [pathPart, queryPart] = normalizedBaseUrl.split('?') + let cleanPath = pathPart.split('{model}')[0].replace(/\/+$/, '') + let cleanQuery = '' + if (queryPart) { + cleanQuery = queryPart + .split('&') + .filter((p) => !p.includes('{model}') && !p.includes('{action}')) + .join('&') + } + // 如果 {model} 在 query 里（path 未变），path 可能缺少 /models + if (cleanPath === pathPart.replace(/\/+$/, '') && !cleanPath.endsWith('/models')) { + const logger = require('../utils/logger') + logger.warn( + 'Gemini 模板 {model} 在 query 中，listModels 路径可能不正确，自动追加 /v1beta/models', + { baseUrl } + ) + cleanPath += '/v1beta/models' + } + const base = cleanQuery ? `${cleanPath}?${cleanQuery}` : cleanPath + const separator = base.includes('?') ? '&' : '?' + url = `${base}${separator}key=${apiKey}` + } else if (isModelsFormat) { url = `${normalizedBaseUrl}?key=${apiKey}` } else { - // 旧格式: 需要拼接 /v1beta/models url = `${normalizedBaseUrl}/v1beta/models?key=${apiKey}` } } else { - // 模型操作 (generateContent, streamGenerateContent, countTokens) const streamParam = stream ? '&alt=sse' : '' - if (isNewFormat) { - // 新格式: baseUrl 已包含 /v1beta/models，直接拼接 /{model}:action + if (isTemplate) { + // 模板模式: 直接替换占位符（{action} 可选，用户可硬编码 action） + url = normalizedBaseUrl.replace('{model}', model).replace('{action}', action) + const separator = url.includes('?') ? '&' : '?' + url += `${separator}key=${apiKey}${streamParam}` + } else if (isModelsFormat) { url = `${normalizedBaseUrl}/${model}:${action}?key=${apiKey}${streamParam}` } else { - // 旧格式: 需要拼接 /v1beta/models/{model}:action url = `${normalizedBaseUrl}/v1beta/models/${model}:${action}?key=${apiKey}${streamParam}` } } @@ -664,6 +747,16 @@ async function handleMessages(req, res) { } } + // 处理其他上游错误（5xx/401/403） + await handleGeminiUpstreamError( + errorStatus, + accountId, + accountType, + sessionHash, + error.response?.headers, + account?.disableAutoProtection + ) + // 返回错误响应 const status = errorStatus || 500 const errorResponse = { @@ -1429,6 +1522,10 @@ async function handleCountTokens(req, res) { * 处理 generateContent 请求（v1internal 格式） */ async function handleGenerateContent(req, res) { + let accountId = null + let accountType = null + let sessionHash = null + try { if (!ensureGeminiPermission(req, res)) { return undefined @@ -1437,7 +1534,7 @@ async function handleGenerateContent(req, res) { const { project, user_prompt_id, request: requestData } = req.body // 从路径参数或请求体中获取模型名 const model = req.body.model || req.params.modelName || 'gemini-2.5-flash' - const sessionHash = sessionHelper.generateSessionHash(req.body) + sessionHash = sessionHelper.generateSessionHash(req.body) // 处理不同格式的请求 let actualRequestData = requestData @@ -1478,7 +1575,7 @@ async function handleGenerateContent(req, res) { sessionHash, model ) - const { accountId, accountType } = schedulerResult + ;({ accountId, accountType } = schedulerResult) // v1internal 路由只支持 OAuth 账户，不支持 API Key 账户 if (accountType === 'gemini-api') { @@ -1638,6 +1735,14 @@ async function handleGenerateContent(req, res) { requestMethod: error.config?.method, stack: error.stack }) + await handleGeminiUpstreamError( + error.response?.status, + accountId, + accountType, + sessionHash, + error.response?.headers, + account?.disableAutoProtection + ) res.status(500).json({ error: { message: getSafeMessage(error) || 'Internal server error', @@ -1653,6 +1758,9 @@ async function handleGenerateContent(req, res) { */ async function handleStreamGenerateContent(req, res) { let abortController = null + let accountId = null + let accountType = null + let sessionHash = null try { if (!ensureGeminiPermission(req, res)) { @@ -1662,7 +1770,7 @@ async function handleStreamGenerateContent(req, res) { const { project, user_prompt_id, request: requestData } = req.body // 从路径参数或请求体中获取模型名 const model = req.body.model || req.params.modelName || 'gemini-2.5-flash' - const sessionHash = sessionHelper.generateSessionHash(req.body) + sessionHash = sessionHelper.generateSessionHash(req.body) // 处理不同格式的请求 let actualRequestData = requestData @@ -1703,7 +1811,7 @@ async function handleStreamGenerateContent(req, res) { sessionHash, model ) - const { accountId, accountType } = schedulerResult + ;({ accountId, accountType } = schedulerResult) // v1internal 路由只支持 OAuth 账户，不支持 API Key 账户 if (accountType === 'gemini-api') { @@ -1997,6 +2105,14 @@ async function handleStreamGenerateContent(req, res) { requestMethod: error.config?.method, stack: error.stack }) + await handleGeminiUpstreamError( + error.response?.status, + accountId, + accountType, + sessionHash, + error.response?.headers, + account?.disableAutoProtection + ) if (!res.headersSent) { res.status(500).json({ @@ -2025,6 +2141,7 @@ async function handleStandardGenerateContent(req, res) { let account = null let sessionHash = null let accountId = null + let accountType = null let isApiAccount = false try { @@ -2102,8 +2219,7 @@ async function handleStandardGenerateContent(req, res) { model, { allowApiAccounts: true } ) - ;({ accountId } = schedulerResult) - const { accountType } = schedulerResult + ;({ accountId, accountType } = schedulerResult) isApiAccount = accountType === 'gemini-api' const actualAccountId = accountId @@ -2148,6 +2264,12 @@ async function handleStandardGenerateContent(req, res) { // Gemini API 账户：直接使用 API Key 请求 const apiUrl = buildGeminiApiUrl(account.baseUrl, model, 'generateContent', account.apiKey) + logger.info('📤 Gemini upstream request', { + targetUrl: apiUrl.replace(/key=[^&]+/, 'key=***'), + model, + accountId: account.id + }) + const axiosConfig = { method: 'POST', url: apiUrl, @@ -2282,6 +2404,14 @@ async function handleStandardGenerateContent(req, res) { responseData: error.response?.data, stack: error.stack }) + await handleGeminiUpstreamError( + error.response?.status, + accountId, + accountType, + sessionHash, + error.response?.headers, + account?.disableAutoProtection + ) res.status(500).json({ error: { @@ -2300,6 +2430,7 @@ async function handleStandardStreamGenerateContent(req, res) { let account = null let sessionHash = null let accountId = null + let accountType = null let isApiAccount = false try { @@ -2375,8 +2506,7 @@ async function handleStandardStreamGenerateContent(req, res) { model, { allowApiAccounts: true } ) - ;({ accountId } = schedulerResult) - const { accountType } = schedulerResult + ;({ accountId, accountType } = schedulerResult) isApiAccount = accountType === 'gemini-api' const actualAccountId = accountId @@ -2446,6 +2576,12 @@ async function handleStandardStreamGenerateContent(req, res) { } ) + logger.info('📤 Gemini upstream request', { + targetUrl: apiUrl.replace(/key=[^&]+/, 'key=***'), + model, + accountId: actualAccountId + }) + const axiosConfig = { method: 'POST', url: apiUrl, @@ -2755,9 +2891,17 @@ async function handleStandardStreamGenerateContent(req, res) { responseData: normalizedError.parsedBody || normalizedError.rawBody, stack: error.stack }) + await handleGeminiUpstreamError( + normalizedError.status || error.response?.status, + accountId, + accountType, + sessionHash, + error.response?.headers, + account?.disableAutoProtection + ) if (!res.headersSent) { - const statusCode = normalizedError.status || 500 + const statusCode = error.statusCode || normalizedError.status || 500 const responseBody = { error: { message: normalizedError.message, diff --git a/src/middleware/auth.js b/src/middleware/auth.js index 787a0eb9..cf76b0b7 100644 --- a/src/middleware/auth.js +++ b/src/middleware/auth.js @@ -1451,6 +1451,7 @@ const authenticateAdmin = async (req, res, next) => { } const authDuration = Date.now() - startTime + req._authInfo = `${adminSession.username} ${authDuration}ms` logger.security(`Admin authenticated: ${adminSession.username} in ${authDuration}ms`) return next() @@ -1593,6 +1594,7 @@ const authenticateUserOrAdmin = async (req, res, next) => { req.userType = 'admin' const authDuration = Date.now() - startTime + req._authInfo = `${adminSession.username} ${authDuration}ms` logger.security(`Admin authenticated: ${adminSession.username} in ${authDuration}ms`) return next() } @@ -1773,67 +1775,80 @@ const requestLogger = (req, res, next) => { const userAgent = req.get('User-Agent') || 'unknown' const referer = req.get('Referer') || 'none' - // 记录请求开始 + // 请求开始 → debug 级别（减少正常请求的日志量） const isDebugRoute = req.originalUrl.includes('event_logging') if (req.originalUrl !== '/health') { - if (isDebugRoute) { - logger.debug(`▶️ [${requestId}] ${req.method} ${req.originalUrl} | IP: ${clientIP}`) - } else { - logger.info(`▶️ [${requestId}] ${req.method} ${req.originalUrl} | IP: ${clientIP}`) - } + logger.debug(`▶ [${requestId}] ${req.method} ${req.originalUrl}`, { + ip: clientIP, + body: req.body && Object.keys(req.body).length > 0 ? req.body : undefined + }) + } + + // 拦截 res.json() 捕获响应体 + const originalJson = res.json.bind(res) + res.json = (body) => { + res._responseBody = body + return originalJson(body) } res.on('finish', () => { + if (req.originalUrl === '/health') { + return + } const duration = Date.now() - start const contentLength = res.get('Content-Length') || '0' + const status = res.statusCode - // 构建日志元数据 - const logMetadata = { - requestId, - method: req.method, - url: req.originalUrl, - status: res.statusCode, - duration, - contentLength, - ip: clientIP, - userAgent, - referer + // 状态 emoji + const emoji = status >= 500 ? '❌' : status >= 400 ? '⚠️ ' : '🟢' + const level = status >= 500 ? 'error' : status >= 400 ? 'warn' : 'info' + + // 主消息行 + const msg = `${emoji} ${status} ${req.method} ${req.originalUrl} ${duration}ms ${contentLength}B` + + // 构建树形 metadata + const meta = { requestId } + + // 请求体（非 GET 且有内容时显示） + if (req.method !== 'GET' && req.body && Object.keys(req.body).length > 0) { + meta.req = req.body } - // 根据状态码选择日志级别 - if (res.statusCode >= 500) { - logger.error( - `◀️ [${requestId}] ${req.method} ${req.originalUrl} | ${res.statusCode} | ${duration}ms | ${contentLength}B`, - logMetadata - ) - } else if (res.statusCode >= 400) { - logger.warn( - `◀️ [${requestId}] ${req.method} ${req.originalUrl} | ${res.statusCode} | ${duration}ms | ${contentLength}B`, - logMetadata - ) - } else if (req.originalUrl !== '/health') { - if (isDebugRoute) { - logger.debug( - `🟢 ${req.method} ${req.originalUrl} - ${res.statusCode} (${duration}ms)`, - logMetadata - ) - } else { - logger.request(req.method, req.originalUrl, res.statusCode, duration, logMetadata) - } + // 查询参数（GET 请求且有查询参数时单独显示） + const queryIdx = req.originalUrl.indexOf('?') + if (queryIdx > -1) { + meta.query = req.originalUrl.substring(queryIdx + 1) } - // API Key相关日志 + // 响应体 + if (res._responseBody) { + meta.res = res._responseBody + } + + // API Key 信息（合并到同一条日志） if (req.apiKey) { - logger.api( - `📱 [${requestId}] Request from ${req.apiKey.name} (${req.apiKey.id}) | ${duration}ms` - ) + meta.key = `${req.apiKey.name} (${req.apiKey.id})` + } + + // 认证信息 + if (req._authInfo) { + meta.auth = req._authInfo + } + + // 完整信息写入文件 + meta.ip = clientIP + meta.ua = userAgent + meta.referer = referer + + if (isDebugRoute) { + logger.debug(msg, meta) + } else { + logger[level](msg, meta) } // 慢请求警告 if (duration > 5000) { - logger.warn( - `🐌 [${requestId}] Slow request detected: ${duration}ms for ${req.method} ${req.originalUrl}` - ) + logger.warn(`🐌 Slow request: ${duration}ms ${req.method} ${req.originalUrl}`) } }) diff --git a/src/routes/admin/accountBalance.js b/src/routes/admin/accountBalance.js index 7f1d18db..b23c9a7e 100644 --- a/src/routes/admin/accountBalance.js +++ b/src/routes/admin/accountBalance.js @@ -1,7 +1,7 @@ const express = require('express') const { authenticateAdmin } = require('../../middleware/auth') const logger = require('../../utils/logger') -const accountBalanceService = require('../../services/accountBalanceService') +const accountBalanceService = require('../../services/account/accountBalanceService') const balanceScriptService = require('../../services/balanceScriptService') const { isBalanceScriptEnabled } = require('../../utils/featureFlags') diff --git a/src/routes/admin/accountGroups.js b/src/routes/admin/accountGroups.js index f927aae7..74616bbe 100644 --- a/src/routes/admin/accountGroups.js +++ b/src/routes/admin/accountGroups.js @@ -1,10 +1,10 @@ const express = require('express') const accountGroupService = require('../../services/accountGroupService') -const claudeAccountService = require('../../services/claudeAccountService') -const claudeConsoleAccountService = require('../../services/claudeConsoleAccountService') -const geminiAccountService = require('../../services/geminiAccountService') -const openaiAccountService = require('../../services/openaiAccountService') -const droidAccountService = require('../../services/droidAccountService') +const claudeAccountService = require('../../services/account/claudeAccountService') +const claudeConsoleAccountService = require('../../services/account/claudeConsoleAccountService') +const geminiAccountService = require('../../services/account/geminiAccountService') +const openaiAccountService = require('../../services/account/openaiAccountService') +const droidAccountService = require('../../services/account/droidAccountService') const { authenticateAdmin } = require('../../middleware/auth') const logger = require('../../utils/logger') diff --git a/src/routes/admin/azureOpenaiAccounts.js b/src/routes/admin/azureOpenaiAccounts.js index bf3ae7e0..f3a0525b 100644 --- a/src/routes/admin/azureOpenaiAccounts.js +++ b/src/routes/admin/azureOpenaiAccounts.js @@ -1,5 +1,5 @@ const express = require('express') -const azureOpenaiAccountService = require('../../services/azureOpenaiAccountService') +const azureOpenaiAccountService = require('../../services/account/azureOpenaiAccountService') const accountGroupService = require('../../services/accountGroupService') const apiKeyService = require('../../services/apiKeyService') const redis = require('../../models/redis') @@ -433,13 +433,16 @@ router.post('/azure-openai-accounts/:accountId/test', authenticateAdmin, async ( } // 构造测试请求 - const { createOpenAITestPayload } = require('../../utils/testPayloadHelper') + const { + createChatCompletionsTestPayload, + extractErrorMessage + } = require('../../utils/testPayloadHelper') const { getProxyAgent } = require('../../utils/proxyHelper') const deploymentName = account.deploymentName || 'gpt-4o-mini' const apiVersion = account.apiVersion || '2024-02-15-preview' const apiUrl = `${account.endpoint}/openai/deployments/${deploymentName}/chat/completions?api-version=${apiVersion}` - const payload = createOpenAITestPayload(deploymentName) + const payload = createChatCompletionsTestPayload(deploymentName) const requestConfig = { headers: { @@ -488,10 +491,23 @@ router.post('/azure-openai-accounts/:accountId/test', authenticateAdmin, async ( return res.status(500).json({ success: false, error: 'Test failed', - message: error.response?.data?.error?.message || error.message, + message: extractErrorMessage(error.response?.data, error.message), latency }) } }) +// 重置 Azure OpenAI 账户状态 +router.post('/:accountId/reset-status', authenticateAdmin, async (req, res) => { + try { + const { accountId } = req.params + const result = await azureOpenaiAccountService.resetAccountStatus(accountId) + logger.success(`Admin reset status for Azure OpenAI account: ${accountId}`) + return res.json({ success: true, data: result }) + } catch (error) { + logger.error('❌ Failed to reset Azure OpenAI account status:', error) + return res.status(500).json({ error: 'Failed to reset status', message: error.message }) + } +}) + module.exports = router diff --git a/src/routes/admin/bedrockAccounts.js b/src/routes/admin/bedrockAccounts.js index 4b6a365b..0c119219 100644 --- a/src/routes/admin/bedrockAccounts.js +++ b/src/routes/admin/bedrockAccounts.js @@ -5,7 +5,7 @@ const express = require('express') const router = express.Router() -const bedrockAccountService = require('../../services/bedrockAccountService') +const bedrockAccountService = require('../../services/account/bedrockAccountService') const apiKeyService = require('../../services/apiKeyService') const accountGroupService = require('../../services/accountGroupService') const redis = require('../../models/redis') @@ -363,4 +363,17 @@ router.post('/:accountId/test', authenticateAdmin, async (req, res) => { } }) +// 重置 Bedrock 账户状态 +router.post('/:accountId/reset-status', authenticateAdmin, async (req, res) => { + try { + const { accountId } = req.params + const result = await bedrockAccountService.resetAccountStatus(accountId) + logger.success(`Admin reset status for Bedrock account: ${accountId}`) + return res.json({ success: true, data: result }) + } catch (error) { + logger.error('❌ Failed to reset Bedrock account status:', error) + return res.status(500).json({ error: 'Failed to reset status', message: error.message }) + } +}) + module.exports = router diff --git a/src/routes/admin/ccrAccounts.js b/src/routes/admin/ccrAccounts.js index 558ba8d3..66657a64 100644 --- a/src/routes/admin/ccrAccounts.js +++ b/src/routes/admin/ccrAccounts.js @@ -1,5 +1,5 @@ const express = require('express') -const ccrAccountService = require('../../services/ccrAccountService') +const ccrAccountService = require('../../services/account/ccrAccountService') const accountGroupService = require('../../services/accountGroupService') const apiKeyService = require('../../services/apiKeyService') const redis = require('../../models/redis') @@ -7,6 +7,7 @@ const { authenticateAdmin } = require('../../middleware/auth') const logger = require('../../utils/logger') const webhookNotifier = require('../../utils/webhookNotifier') const { formatAccountExpiry, mapExpiryField } = require('./utils') +const { extractErrorMessage } = require('../../utils/testPayloadHelper') const router = express.Router() @@ -492,7 +493,7 @@ router.post('/:accountId/test', authenticateAdmin, async (req, res) => { return res.status(500).json({ success: false, error: 'Test failed', - message: error.response?.data?.error?.message || error.message, + message: extractErrorMessage(error.response?.data, error.message), latency }) } diff --git a/src/routes/admin/claudeAccounts.js b/src/routes/admin/claudeAccounts.js index 590e919d..6d5884fd 100644 --- a/src/routes/admin/claudeAccounts.js +++ b/src/routes/admin/claudeAccounts.js @@ -6,8 +6,8 @@ const express = require('express') const router = express.Router() -const claudeAccountService = require('../../services/claudeAccountService') -const claudeRelayService = require('../../services/claudeRelayService') +const claudeAccountService = require('../../services/account/claudeAccountService') +const claudeRelayService = require('../../services/relay/claudeRelayService') const accountGroupService = require('../../services/accountGroupService') const accountTestSchedulerService = require('../../services/accountTestSchedulerService') const apiKeyService = require('../../services/apiKeyService') diff --git a/src/routes/admin/claudeConsoleAccounts.js b/src/routes/admin/claudeConsoleAccounts.js index 5ed7f6fa..2f13f030 100644 --- a/src/routes/admin/claudeConsoleAccounts.js +++ b/src/routes/admin/claudeConsoleAccounts.js @@ -6,8 +6,8 @@ const express = require('express') const router = express.Router() -const claudeConsoleAccountService = require('../../services/claudeConsoleAccountService') -const claudeConsoleRelayService = require('../../services/claudeConsoleRelayService') +const claudeConsoleAccountService = require('../../services/account/claudeConsoleAccountService') +const claudeConsoleRelayService = require('../../services/relay/claudeConsoleRelayService') const accountGroupService = require('../../services/accountGroupService') const apiKeyService = require('../../services/apiKeyService') const redis = require('../../models/redis') diff --git a/src/routes/admin/dashboard.js b/src/routes/admin/dashboard.js index fb47f98e..cf7e8d9c 100644 --- a/src/routes/admin/dashboard.js +++ b/src/routes/admin/dashboard.js @@ -1,16 +1,17 @@ const express = require('express') const apiKeyService = require('../../services/apiKeyService') -const claudeAccountService = require('../../services/claudeAccountService') -const claudeConsoleAccountService = require('../../services/claudeConsoleAccountService') -const bedrockAccountService = require('../../services/bedrockAccountService') -const ccrAccountService = require('../../services/ccrAccountService') -const geminiAccountService = require('../../services/geminiAccountService') -const droidAccountService = require('../../services/droidAccountService') -const openaiResponsesAccountService = require('../../services/openaiResponsesAccountService') +const claudeAccountService = require('../../services/account/claudeAccountService') +const claudeConsoleAccountService = require('../../services/account/claudeConsoleAccountService') +const bedrockAccountService = require('../../services/account/bedrockAccountService') +const ccrAccountService = require('../../services/account/ccrAccountService') +const geminiAccountService = require('../../services/account/geminiAccountService') +const droidAccountService = require('../../services/account/droidAccountService') +const openaiResponsesAccountService = require('../../services/account/openaiResponsesAccountService') const redis = require('../../models/redis') const { authenticateAdmin } = require('../../middleware/auth') const logger = require('../../utils/logger') const CostCalculator = require('../../utils/costCalculator') +const upstreamErrorHelper = require('../../utils/upstreamErrorHelper') const config = require('../../../config/config') const router = express.Router() @@ -352,6 +353,17 @@ router.get('/dashboard', authenticateAdmin, async (req, res) => { } }) +// 获取所有临时不可用账户状态 +router.get('/temp-unavailable', authenticateAdmin, async (req, res) => { + try { + const statuses = await upstreamErrorHelper.getAllTempUnavailable() + return res.json({ success: true, data: statuses }) + } catch (error) { + logger.error('❌ Failed to get temp unavailable statuses:', error) + return res.status(500).json({ error: 'Failed to get temp unavailable statuses' }) + } +}) + // 获取使用统计 router.get('/usage-stats', authenticateAdmin, async (req, res) => { try { diff --git a/src/routes/admin/droidAccounts.js b/src/routes/admin/droidAccounts.js index 532a07b9..de0dd02f 100644 --- a/src/routes/admin/droidAccounts.js +++ b/src/routes/admin/droidAccounts.js @@ -1,6 +1,6 @@ const express = require('express') const crypto = require('crypto') -const droidAccountService = require('../../services/droidAccountService') +const droidAccountService = require('../../services/account/droidAccountService') const accountGroupService = require('../../services/accountGroupService') const apiKeyService = require('../../services/apiKeyService') const redis = require('../../models/redis') @@ -13,6 +13,7 @@ const { } = require('../../utils/workosOAuthHelper') const webhookNotifier = require('../../utils/webhookNotifier') const { formatAccountExpiry, mapExpiryField } = require('./utils') +const { extractErrorMessage } = require('../../utils/testPayloadHelper') const router = express.Router() @@ -683,10 +684,23 @@ router.post('/droid-accounts/:accountId/test', authenticateAdmin, async (req, re return res.status(500).json({ success: false, error: 'Test failed', - message: error.response?.data?.error?.message || error.message, + message: extractErrorMessage(error.response?.data, error.message), latency }) } }) +// 重置 Droid 账户状态 +router.post('/:accountId/reset-status', authenticateAdmin, async (req, res) => { + try { + const { accountId } = req.params + const result = await droidAccountService.resetAccountStatus(accountId) + logger.success(`Admin reset status for Droid account: ${accountId}`) + return res.json({ success: true, data: result }) + } catch (error) { + logger.error('❌ Failed to reset Droid account status:', error) + return res.status(500).json({ error: 'Failed to reset status', message: error.message }) + } +}) + module.exports = router diff --git a/src/routes/admin/geminiAccounts.js b/src/routes/admin/geminiAccounts.js index 07a81218..fa0f046f 100644 --- a/src/routes/admin/geminiAccounts.js +++ b/src/routes/admin/geminiAccounts.js @@ -1,5 +1,5 @@ const express = require('express') -const geminiAccountService = require('../../services/geminiAccountService') +const geminiAccountService = require('../../services/account/geminiAccountService') const accountGroupService = require('../../services/accountGroupService') const apiKeyService = require('../../services/apiKeyService') const redis = require('../../models/redis') @@ -532,7 +532,10 @@ router.post('/:accountId/test', authenticateAdmin, async (req, res) => { // 构造测试请求 const axios = require('axios') - const { createGeminiTestPayload } = require('../../utils/testPayloadHelper') + const { + createGeminiTestPayload, + extractErrorMessage + } = require('../../utils/testPayloadHelper') const { getProxyAgent } = require('../../utils/proxyHelper') const apiUrl = `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent` @@ -585,7 +588,7 @@ router.post('/:accountId/test', authenticateAdmin, async (req, res) => { return res.status(500).json({ success: false, error: 'Test failed', - message: error.response?.data?.error?.message || error.message, + message: extractErrorMessage(error.response?.data, error.message), latency }) } diff --git a/src/routes/admin/geminiApiAccounts.js b/src/routes/admin/geminiApiAccounts.js index 9c773de6..b2bb5309 100644 --- a/src/routes/admin/geminiApiAccounts.js +++ b/src/routes/admin/geminiApiAccounts.js @@ -1,5 +1,5 @@ const express = require('express') -const geminiApiAccountService = require('../../services/geminiApiAccountService') +const geminiApiAccountService = require('../../services/account/geminiApiAccountService') const apiKeyService = require('../../services/apiKeyService') const accountGroupService = require('../../services/accountGroupService') const redis = require('../../models/redis') diff --git a/src/routes/admin/openaiAccounts.js b/src/routes/admin/openaiAccounts.js index 50f46c44..6d98e682 100644 --- a/src/routes/admin/openaiAccounts.js +++ b/src/routes/admin/openaiAccounts.js @@ -6,7 +6,7 @@ const express = require('express') const crypto = require('crypto') const axios = require('axios') -const openaiAccountService = require('../../services/openaiAccountService') +const openaiAccountService = require('../../services/account/openaiAccountService') const accountGroupService = require('../../services/accountGroupService') const apiKeyService = require('../../services/apiKeyService') const redis = require('../../models/redis') diff --git a/src/routes/admin/openaiResponsesAccounts.js b/src/routes/admin/openaiResponsesAccounts.js index 3c7f91a0..4efab59d 100644 --- a/src/routes/admin/openaiResponsesAccounts.js +++ b/src/routes/admin/openaiResponsesAccounts.js @@ -4,7 +4,8 @@ */ const express = require('express') -const openaiResponsesAccountService = require('../../services/openaiResponsesAccountService') +const axios = require('axios') +const openaiResponsesAccountService = require('../../services/account/openaiResponsesAccountService') const apiKeyService = require('../../services/apiKeyService') const accountGroupService = require('../../services/accountGroupService') const redis = require('../../models/redis') @@ -12,6 +13,8 @@ const { authenticateAdmin } = require('../../middleware/auth') const logger = require('../../utils/logger') const webhookNotifier = require('../../utils/webhookNotifier') const { formatAccountExpiry, mapExpiryField } = require('./utils') +const { createOpenAITestPayload, extractErrorMessage } = require('../../utils/testPayloadHelper') +const { getProxyAgent } = require('../../utils/proxyHelper') const router = express.Router() @@ -459,31 +462,25 @@ router.post('/openai-responses-accounts/:accountId/test', authenticateAdmin, asy const startTime = Date.now() try { - // 获取账户信息 + // 获取账户信息（apiKey 已自动解密） const account = await openaiResponsesAccountService.getAccount(accountId) if (!account) { return res.status(404).json({ error: 'Account not found' }) } - // 获取解密后的 API Key - const apiKey = await openaiResponsesAccountService.getDecryptedApiKey(accountId) - if (!apiKey) { + if (!account.apiKey) { return res.status(401).json({ error: 'API Key not found or decryption failed' }) } // 构造测试请求 - const axios = require('axios') - const { createOpenAITestPayload } = require('../../utils/testPayloadHelper') - const { getProxyAgent } = require('../../utils/proxyHelper') - - const baseUrl = account.baseUrl || 'https://api.openai.com' - const apiUrl = `${baseUrl}/v1/chat/completions` - const payload = createOpenAITestPayload(model) + const baseUrl = account.baseApi || 'https://api.openai.com' + const apiUrl = `${baseUrl}/responses` + const payload = createOpenAITestPayload(model, { stream: false }) const requestConfig = { headers: { 'Content-Type': 'application/json', - Authorization: `Bearer ${apiKey}` + Authorization: `Bearer ${account.apiKey}` }, timeout: 30000 } @@ -500,10 +497,19 @@ router.post('/openai-responses-accounts/:accountId/test', authenticateAdmin, asy const response = await axios.post(apiUrl, payload, requestConfig) const latency = Date.now() - startTime - // 提取响应文本 + // 提取响应文本（Responses API 格式） let responseText = '' - if (response.data?.choices?.[0]?.message?.content) { - responseText = response.data.choices[0].message.content + const output = response.data?.output + if (Array.isArray(output)) { + for (const item of output) { + if (item.type === 'message' && Array.isArray(item.content)) { + for (const block of item.content) { + if (block.type === 'output_text' && block.text) { + responseText += block.text + } + } + } + } } logger.success( @@ -527,7 +533,7 @@ router.post('/openai-responses-accounts/:accountId/test', authenticateAdmin, asy return res.status(500).json({ success: false, error: 'Test failed', - message: error.response?.data?.error?.message || error.message, + message: extractErrorMessage(error.response?.data, error.message), latency }) } diff --git a/src/routes/admin/sync.js b/src/routes/admin/sync.js index ab0ef219..a194f0e6 100644 --- a/src/routes/admin/sync.js +++ b/src/routes/admin/sync.js @@ -8,10 +8,10 @@ const router = express.Router() const { authenticateAdmin } = require('../../middleware/auth') const redis = require('../../models/redis') -const claudeAccountService = require('../../services/claudeAccountService') -const claudeConsoleAccountService = require('../../services/claudeConsoleAccountService') -const openaiAccountService = require('../../services/openaiAccountService') -const openaiResponsesAccountService = require('../../services/openaiResponsesAccountService') +const claudeAccountService = require('../../services/account/claudeAccountService') +const claudeConsoleAccountService = require('../../services/account/claudeConsoleAccountService') +const openaiAccountService = require('../../services/account/openaiAccountService') +const openaiResponsesAccountService = require('../../services/account/openaiResponsesAccountService') const logger = require('../../utils/logger') function toBool(value, defaultValue = false) { diff --git a/src/routes/admin/system.js b/src/routes/admin/system.js index 6304c86d..2eba07bc 100644 --- a/src/routes/admin/system.js +++ b/src/routes/admin/system.js @@ -3,7 +3,7 @@ const fs = require('fs') const path = require('path') const axios = require('axios') const claudeCodeHeadersService = require('../../services/claudeCodeHeadersService') -const claudeAccountService = require('../../services/claudeAccountService') +const claudeAccountService = require('../../services/account/claudeAccountService') const redis = require('../../models/redis') const { authenticateAdmin } = require('../../middleware/auth') const logger = require('../../utils/logger') @@ -408,4 +408,44 @@ router.post('/claude-code-version/clear', authenticateAdmin, async (req, res) => } }) +// ==================== 模型价格管理 ==================== + +const pricingService = require('../../services/pricingService') + +// 获取所有模型价格数据 +router.get('/models/pricing', authenticateAdmin, async (req, res) => { + try { + const data = pricingService.pricingData + res.json({ + success: true, + data: data || {} + }) + } catch (error) { + logger.error('Failed to get model pricing:', error) + res.status(500).json({ error: 'Failed to get model pricing', message: error.message }) + } +}) + +// 获取价格服务状态 +router.get('/models/pricing/status', authenticateAdmin, async (req, res) => { + try { + const status = pricingService.getStatus() + res.json({ success: true, data: status }) + } catch (error) { + logger.error('Failed to get pricing status:', error) + res.status(500).json({ error: 'Failed to get pricing status', message: error.message }) + } +}) + +// 强制刷新价格数据 +router.post('/models/pricing/refresh', authenticateAdmin, async (req, res) => { + try { + const result = await pricingService.forceUpdate() + res.json({ success: result.success, message: result.message }) + } catch (error) { + logger.error('Failed to refresh pricing:', error) + res.status(500).json({ error: 'Failed to refresh pricing', message: error.message }) + } +}) + module.exports = router diff --git a/src/routes/admin/usageStats.js b/src/routes/admin/usageStats.js index cfa61bd4..521ad2e7 100644 --- a/src/routes/admin/usageStats.js +++ b/src/routes/admin/usageStats.js @@ -1,14 +1,14 @@ const express = require('express') const apiKeyService = require('../../services/apiKeyService') -const ccrAccountService = require('../../services/ccrAccountService') -const claudeAccountService = require('../../services/claudeAccountService') -const claudeConsoleAccountService = require('../../services/claudeConsoleAccountService') -const geminiAccountService = require('../../services/geminiAccountService') -const geminiApiAccountService = require('../../services/geminiApiAccountService') -const openaiAccountService = require('../../services/openaiAccountService') -const openaiResponsesAccountService = require('../../services/openaiResponsesAccountService') -const droidAccountService = require('../../services/droidAccountService') -const bedrockAccountService = require('../../services/bedrockAccountService') +const ccrAccountService = require('../../services/account/ccrAccountService') +const claudeAccountService = require('../../services/account/claudeAccountService') +const claudeConsoleAccountService = require('../../services/account/claudeConsoleAccountService') +const geminiAccountService = require('../../services/account/geminiAccountService') +const geminiApiAccountService = require('../../services/account/geminiApiAccountService') +const openaiAccountService = require('../../services/account/openaiAccountService') +const openaiResponsesAccountService = require('../../services/account/openaiResponsesAccountService') +const droidAccountService = require('../../services/account/droidAccountService') +const bedrockAccountService = require('../../services/account/bedrockAccountService') const redis = require('../../models/redis') const { authenticateAdmin } = require('../../middleware/auth') const logger = require('../../utils/logger') diff --git a/src/routes/api.js b/src/routes/api.js index 4d5647e4..c6bb7ad3 100644 --- a/src/routes/api.js +++ b/src/routes/api.js @@ -1,10 +1,10 @@ const express = require('express') -const claudeRelayService = require('../services/claudeRelayService') -const claudeConsoleRelayService = require('../services/claudeConsoleRelayService') -const bedrockRelayService = require('../services/bedrockRelayService') -const ccrRelayService = require('../services/ccrRelayService') -const bedrockAccountService = require('../services/bedrockAccountService') -const unifiedClaudeScheduler = require('../services/unifiedClaudeScheduler') +const claudeRelayService = require('../services/relay/claudeRelayService') +const claudeConsoleRelayService = require('../services/relay/claudeConsoleRelayService') +const bedrockRelayService = require('../services/relay/bedrockRelayService') +const ccrRelayService = require('../services/relay/ccrRelayService') +const bedrockAccountService = require('../services/account/bedrockAccountService') +const unifiedClaudeScheduler = require('../services/scheduler/unifiedClaudeScheduler') const apiKeyService = require('../services/apiKeyService') const { authenticateApiKey } = require('../middleware/auth') const logger = require('../utils/logger') @@ -12,8 +12,8 @@ const { getEffectiveModel, parseVendorPrefixedModel } = require('../utils/modelH const sessionHelper = require('../utils/sessionHelper') const { updateRateLimitCounters } = require('../utils/rateLimitHelper') const claudeRelayConfigService = require('../services/claudeRelayConfigService') -const claudeAccountService = require('../services/claudeAccountService') -const claudeConsoleAccountService = require('../services/claudeConsoleAccountService') +const claudeAccountService = require('../services/account/claudeAccountService') +const claudeConsoleAccountService = require('../services/account/claudeConsoleAccountService') const { isWarmupRequest, buildMockWarmupResponse, @@ -1289,8 +1289,8 @@ router.get('/v1/models', authenticateApiKey, async (req, res) => { }) } - const unifiedGeminiScheduler = require('../services/unifiedGeminiScheduler') - const geminiAccountService = require('../services/geminiAccountService') + const unifiedGeminiScheduler = require('../services/scheduler/unifiedGeminiScheduler') + const geminiAccountService = require('../services/account/geminiAccountService') let accountSelection try { diff --git a/src/routes/apiStats.js b/src/routes/apiStats.js index 9d7b2ca6..481e1154 100644 --- a/src/routes/apiStats.js +++ b/src/routes/apiStats.js @@ -3,10 +3,14 @@ const redis = require('../models/redis') const logger = require('../utils/logger') const apiKeyService = require('../services/apiKeyService') const CostCalculator = require('../utils/costCalculator') -const claudeAccountService = require('../services/claudeAccountService') -const openaiAccountService = require('../services/openaiAccountService') +const claudeAccountService = require('../services/account/claudeAccountService') +const openaiAccountService = require('../services/account/openaiAccountService') const serviceRatesService = require('../services/serviceRatesService') -const { createClaudeTestPayload } = require('../utils/testPayloadHelper') +const { + createClaudeTestPayload, + extractErrorMessage, + sanitizeErrorMsg +} = require('../utils/testPayloadHelper') const modelsConfig = require('../../config/models') const { getSafeMessage } = require('../utils/errorSanitizer') @@ -25,7 +29,7 @@ router.get('/models', (req, res) => { }) } - // 返回所有模型（按服务分组） + // 返回所有模型（按服务分组 + 平台维度） res.json({ success: true, data: { @@ -33,7 +37,8 @@ router.get('/models', (req, res) => { gemini: modelsConfig.GEMINI_MODELS, openai: modelsConfig.OPENAI_MODELS, other: modelsConfig.OTHER_MODELS, - all: modelsConfig.getAllModels() + all: modelsConfig.getAllModels(), + platforms: modelsConfig.PLATFORM_TEST_MODELS } }) }) @@ -920,7 +925,8 @@ router.post('/api-key/test', async (req, res) => { responseStream: res, payload: createClaudeTestPayload(model, { stream: true, prompt, maxTokens }), timeout: 60000, - extraHeaders: { 'x-api-key': apiKey } + extraHeaders: { 'x-api-key': apiKey }, + sanitize: true }) } catch (error) { logger.error('❌ API Key test failed:', error) @@ -1015,14 +1021,14 @@ router.post('/api-key/test-gemini', async (req, res) => { let errorMsg = `API Error: ${response.status}` try { const json = JSON.parse(errorData) - errorMsg = json.message || json.error?.message || json.error || errorMsg + errorMsg = extractErrorMessage(json, errorMsg) } catch { if (errorData.length < 200) { errorMsg = errorData || errorMsg } } res.write( - `data: ${JSON.stringify({ type: 'test_complete', success: false, error: errorMsg })}\n\n` + `data: ${JSON.stringify({ type: 'test_complete', success: false, error: sanitizeErrorMsg(errorMsg) })}\n\n` ) res.end() }) @@ -1168,14 +1174,14 @@ router.post('/api-key/test-openai', async (req, res) => { let errorMsg = `API Error: ${response.status}` try { const json = JSON.parse(errorData) - errorMsg = json.message || json.error?.message || json.error || errorMsg + errorMsg = extractErrorMessage(json, errorMsg) } catch { if (errorData.length < 200) { errorMsg = errorData || errorMsg } } res.write( - `data: ${JSON.stringify({ type: 'test_complete', success: false, error: errorMsg })}\n\n` + `data: ${JSON.stringify({ type: 'test_complete', success: false, error: sanitizeErrorMsg(errorMsg) })}\n\n` ) res.end() }) diff --git a/src/routes/azureOpenaiRoutes.js b/src/routes/azureOpenaiRoutes.js index ce476c09..7993a887 100644 --- a/src/routes/azureOpenaiRoutes.js +++ b/src/routes/azureOpenaiRoutes.js @@ -2,10 +2,11 @@ const express = require('express') const router = express.Router() const logger = require('../utils/logger') const { authenticateApiKey } = require('../middleware/auth') -const azureOpenaiAccountService = require('../services/azureOpenaiAccountService') -const azureOpenaiRelayService = require('../services/azureOpenaiRelayService') +const azureOpenaiAccountService = require('../services/account/azureOpenaiAccountService') +const azureOpenaiRelayService = require('../services/relay/azureOpenaiRelayService') const apiKeyService = require('../services/apiKeyService') const crypto = require('crypto') +const upstreamErrorHelper = require('../utils/upstreamErrorHelper') // 支持的模型列表 - 基于真实的 Azure OpenAI 模型 const ALLOWED_MODELS = { @@ -163,6 +164,16 @@ router.post('/chat/completions', authenticateApiKey, async (req, res) => { let account = null if (req.apiKey?.azureOpenaiAccountId) { account = await azureOpenaiAccountService.getAccount(req.apiKey.azureOpenaiAccountId) + if (account) { + const isTempUnavailable = await upstreamErrorHelper.isTempUnavailable( + account.id, + 'azure-openai' + ) + if (isTempUnavailable) { + logger.warn(`⏱️ Bound Azure OpenAI account temporarily unavailable, falling back to pool`) + account = null + } + } if (!account) { logger.warn(`Bound Azure OpenAI account not found: ${req.apiKey.azureOpenaiAccountId}`) } @@ -182,6 +193,24 @@ router.post('/chat/completions', authenticateApiKey, async (req, res) => { endpoint: 'chat/completions' }) + // 检查上游响应状态码（仅对认证/限流/服务端错误暂停，不对 400/404 等客户端错误暂停） + const azureAutoProtectionDisabled = + account?.disableAutoProtection === true || account?.disableAutoProtection === 'true' + const shouldPause = + account?.id && + !azureAutoProtectionDisabled && + (response.status === 401 || + response.status === 403 || + response.status === 429 || + response.status >= 500) + if (shouldPause) { + const customTtl = + response.status === 429 ? upstreamErrorHelper.parseRetryAfter(response.headers) : null + await upstreamErrorHelper + .markTempUnavailable(account.id, 'azure-openai', response.status, customTtl) + .catch(() => {}) + } + // 处理流式响应 if (req.body.stream) { await azureOpenaiRelayService.handleStreamResponse(response, res, { @@ -256,6 +285,16 @@ router.post('/responses', authenticateApiKey, async (req, res) => { let account = null if (req.apiKey?.azureOpenaiAccountId) { account = await azureOpenaiAccountService.getAccount(req.apiKey.azureOpenaiAccountId) + if (account) { + const isTempUnavailable = await upstreamErrorHelper.isTempUnavailable( + account.id, + 'azure-openai' + ) + if (isTempUnavailable) { + logger.warn(`⏱️ Bound Azure OpenAI account temporarily unavailable, falling back to pool`) + account = null + } + } if (!account) { logger.warn(`Bound Azure OpenAI account not found: ${req.apiKey.azureOpenaiAccountId}`) } @@ -275,6 +314,24 @@ router.post('/responses', authenticateApiKey, async (req, res) => { endpoint: 'responses' }) + // 检查上游响应状态码（仅对认证/限流/服务端错误暂停，不对 400/404 等客户端错误暂停） + const azureAutoProtectionDisabled = + account?.disableAutoProtection === true || account?.disableAutoProtection === 'true' + const shouldPause = + account?.id && + !azureAutoProtectionDisabled && + (response.status === 401 || + response.status === 403 || + response.status === 429 || + response.status >= 500) + if (shouldPause) { + const customTtl = + response.status === 429 ? upstreamErrorHelper.parseRetryAfter(response.headers) : null + await upstreamErrorHelper + .markTempUnavailable(account.id, 'azure-openai', response.status, customTtl) + .catch(() => {}) + } + // 处理流式响应 if (req.body.stream) { await azureOpenaiRelayService.handleStreamResponse(response, res, { @@ -348,6 +405,16 @@ router.post('/embeddings', authenticateApiKey, async (req, res) => { let account = null if (req.apiKey?.azureOpenaiAccountId) { account = await azureOpenaiAccountService.getAccount(req.apiKey.azureOpenaiAccountId) + if (account) { + const isTempUnavailable = await upstreamErrorHelper.isTempUnavailable( + account.id, + 'azure-openai' + ) + if (isTempUnavailable) { + logger.warn(`⏱️ Bound Azure OpenAI account temporarily unavailable, falling back to pool`) + account = null + } + } if (!account) { logger.warn(`Bound Azure OpenAI account not found: ${req.apiKey.azureOpenaiAccountId}`) } @@ -367,6 +434,24 @@ router.post('/embeddings', authenticateApiKey, async (req, res) => { endpoint: 'embeddings' }) + // 检查上游响应状态码（仅对认证/限流/服务端错误暂停，不对 400/404 等客户端错误暂停） + const azureAutoProtectionDisabled = + account?.disableAutoProtection === true || account?.disableAutoProtection === 'true' + const shouldPause = + account?.id && + !azureAutoProtectionDisabled && + (response.status === 401 || + response.status === 403 || + response.status === 429 || + response.status >= 500) + if (shouldPause) { + const customTtl = + response.status === 429 ? upstreamErrorHelper.parseRetryAfter(response.headers) : null + await upstreamErrorHelper + .markTempUnavailable(account.id, 'azure-openai', response.status, customTtl) + .catch(() => {}) + } + // 处理响应 const { usageData, actualModel } = azureOpenaiRelayService.handleNonStreamResponse( response, diff --git a/src/routes/droidRoutes.js b/src/routes/droidRoutes.js index b6d9932a..7e471cee 100644 --- a/src/routes/droidRoutes.js +++ b/src/routes/droidRoutes.js @@ -1,7 +1,7 @@ const crypto = require('crypto') const express = require('express') const { authenticateApiKey } = require('../middleware/auth') -const droidRelayService = require('../services/droidRelayService') +const droidRelayService = require('../services/relay/droidRelayService') const sessionHelper = require('../utils/sessionHelper') const logger = require('../utils/logger') const apiKeyService = require('../services/apiKeyService') diff --git a/src/routes/openaiClaudeRoutes.js b/src/routes/openaiClaudeRoutes.js index ae791b56..a98237f3 100644 --- a/src/routes/openaiClaudeRoutes.js +++ b/src/routes/openaiClaudeRoutes.js @@ -7,11 +7,11 @@ const express = require('express') const router = express.Router() const logger = require('../utils/logger') const { authenticateApiKey } = require('../middleware/auth') -const claudeRelayService = require('../services/claudeRelayService') -const claudeConsoleRelayService = require('../services/claudeConsoleRelayService') +const claudeRelayService = require('../services/relay/claudeRelayService') +const claudeConsoleRelayService = require('../services/relay/claudeConsoleRelayService') const openaiToClaude = require('../services/openaiToClaude') const apiKeyService = require('../services/apiKeyService') -const unifiedClaudeScheduler = require('../services/unifiedClaudeScheduler') +const unifiedClaudeScheduler = require('../services/scheduler/unifiedClaudeScheduler') const claudeCodeHeadersService = require('../services/claudeCodeHeadersService') const { getSafeMessage } = require('../utils/errorSanitizer') const sessionHelper = require('../utils/sessionHelper') diff --git a/src/routes/openaiGeminiRoutes.js b/src/routes/openaiGeminiRoutes.js index d4c39146..57f13090 100644 --- a/src/routes/openaiGeminiRoutes.js +++ b/src/routes/openaiGeminiRoutes.js @@ -2,9 +2,9 @@ const express = require('express') const router = express.Router() const logger = require('../utils/logger') const { authenticateApiKey } = require('../middleware/auth') -const geminiAccountService = require('../services/geminiAccountService') -const unifiedGeminiScheduler = require('../services/unifiedGeminiScheduler') -const { getAvailableModels } = require('../services/geminiRelayService') +const geminiAccountService = require('../services/account/geminiAccountService') +const unifiedGeminiScheduler = require('../services/scheduler/unifiedGeminiScheduler') +const { getAvailableModels } = require('../services/relay/geminiRelayService') const crypto = require('crypto') const apiKeyService = require('../services/apiKeyService') diff --git a/src/routes/openaiRoutes.js b/src/routes/openaiRoutes.js index 6610bd98..363255fc 100644 --- a/src/routes/openaiRoutes.js +++ b/src/routes/openaiRoutes.js @@ -4,10 +4,10 @@ const router = express.Router() const logger = require('../utils/logger') const config = require('../../config/config') const { authenticateApiKey } = require('../middleware/auth') -const unifiedOpenAIScheduler = require('../services/unifiedOpenAIScheduler') -const openaiAccountService = require('../services/openaiAccountService') -const openaiResponsesAccountService = require('../services/openaiResponsesAccountService') -const openaiResponsesRelayService = require('../services/openaiResponsesRelayService') +const unifiedOpenAIScheduler = require('../services/scheduler/unifiedOpenAIScheduler') +const openaiAccountService = require('../services/account/openaiAccountService') +const openaiResponsesAccountService = require('../services/account/openaiResponsesAccountService') +const openaiResponsesRelayService = require('../services/relay/openaiResponsesRelayService') const apiKeyService = require('../services/apiKeyService') const redis = require('../models/redis') const crypto = require('crypto') diff --git a/src/services/accountBalanceService.js b/src/services/account/accountBalanceService.js similarity index 98% rename from src/services/accountBalanceService.js rename to src/services/account/accountBalanceService.js index ec25f171..4a468d4a 100644 --- a/src/services/accountBalanceService.js +++ b/src/services/account/accountBalanceService.js @@ -1,8 +1,8 @@ -const redis = require('../models/redis') -const balanceScriptService = require('./balanceScriptService') -const logger = require('../utils/logger') -const CostCalculator = require('../utils/costCalculator') -const { isBalanceScriptEnabled } = require('../utils/featureFlags') +const redis = require('../../models/redis') +const balanceScriptService = require('../balanceScriptService') +const logger = require('../../utils/logger') +const CostCalculator = require('../../utils/costCalculator') +const { isBalanceScriptEnabled } = require('../../utils/featureFlags') class AccountBalanceService { constructor(options = {}) { diff --git a/src/services/azureOpenaiAccountService.js b/src/services/account/azureOpenaiAccountService.js similarity index 81% rename from src/services/azureOpenaiAccountService.js rename to src/services/account/azureOpenaiAccountService.js index 25624142..6f611d82 100644 --- a/src/services/azureOpenaiAccountService.js +++ b/src/services/account/azureOpenaiAccountService.js @@ -1,8 +1,9 @@ -const redisClient = require('../models/redis') +const redisClient = require('../../models/redis') const { v4: uuidv4 } = require('uuid') const crypto = require('crypto') -const config = require('../../config/config') -const logger = require('../utils/logger') +const config = require('../../../config/config') +const logger = require('../../utils/logger') +const upstreamErrorHelper = require('../../utils/upstreamErrorHelper') // 加密相关常量 const ALGORITHM = 'aes-256-cbc' @@ -138,6 +139,10 @@ async function createAccount(accountData) { isActive: accountData.isActive !== false ? 'true' : 'false', status: 'active', schedulable: accountData.schedulable !== false ? 'true' : 'false', + disableAutoProtection: + accountData.disableAutoProtection === true || accountData.disableAutoProtection === 'true' + ? 'true' + : 'false', // 关闭自动防护 createdAt: now, updatedAt: now } @@ -230,6 +235,14 @@ async function updateAccount(accountId, updates) { // 直接保存，不做任何调整 } + // 自动防护开关 + if (updates.disableAutoProtection !== undefined) { + updates.disableAutoProtection = + updates.disableAutoProtection === true || updates.disableAutoProtection === 'true' + ? 'true' + : 'false' + } + // 更新账户类型时处理共享账户集合 const client = redisClient.getClientSafe() if (updates.accountType && updates.accountType !== existingAccount.accountType) { @@ -262,7 +275,7 @@ async function updateAccount(accountId, updates) { // 删除账户 async function deleteAccount(accountId) { // 首先从所有分组中移除此账户 - const accountGroupService = require('./accountGroupService') + const accountGroupService = require('../accountGroupService') await accountGroupService.removeAccountFromAllGroups(accountId) const client = redisClient.getClientSafe() @@ -380,8 +393,14 @@ async function selectAvailableAccount(sessionId = null) { if (accountId) { const account = await getAccount(accountId) if (account && account.isActive === 'true' && account.schedulable === 'true') { - logger.debug(`Reusing Azure OpenAI account ${accountId} for session ${sessionId}`) - return account + const isTempUnavail = await upstreamErrorHelper.isTempUnavailable(accountId, 'azure-openai') + if (!isTempUnavail) { + logger.debug(`Reusing Azure OpenAI account ${accountId} for session ${sessionId}`) + return account + } + logger.warn( + `⏱️ Session-bound Azure OpenAI account ${accountId} temporarily unavailable, falling back to pool` + ) } } } @@ -389,18 +408,30 @@ async function selectAvailableAccount(sessionId = null) { // 获取所有共享账户 const sharedAccounts = await getSharedAccounts() - // 过滤出可用的账户 - const availableAccounts = sharedAccounts.filter((acc) => { - // ✅ 检查账户订阅是否过期 + // 过滤出可用的账户（异步过滤，包含临时不可用检查） + const availableAccounts = [] + for (const acc of sharedAccounts) { + // 检查账户订阅是否过期 if (isSubscriptionExpired(acc)) { logger.debug( `⏰ Skipping expired Azure OpenAI account: ${acc.name}, expired at ${acc.subscriptionExpiresAt}` ) - return false + continue } - return acc.isActive === 'true' && acc.schedulable === 'true' - }) + if (acc.isActive !== 'true' || acc.schedulable !== 'true') { + continue + } + + // 检查临时不可用状态 + const isTempUnavail = await upstreamErrorHelper.isTempUnavailable(acc.id, 'azure-openai') + if (isTempUnavail) { + logger.debug(`⏱️ Skipping temporarily unavailable Azure OpenAI account: ${acc.name}`) + continue + } + + availableAccounts.push(acc) + } if (availableAccounts.length === 0) { throw new Error('No available Azure OpenAI accounts') @@ -515,6 +546,69 @@ async function migrateApiKeysForAzureSupport() { return migratedCount } +// 🔄 重置Azure OpenAI账户所有异常状态 +async function resetAccountStatus(accountId) { + try { + const accountData = await getAccount(accountId) + if (!accountData) { + throw new Error('Account not found') + } + + const client = redisClient.getClientSafe() + const accountKey = `azure_openai:account:${accountId}` + + const updates = { + status: 'active', + errorMessage: '', + schedulable: 'true', + isActive: 'true' + } + + const fieldsToDelete = [ + 'rateLimitedAt', + 'rateLimitStatus', + 'unauthorizedAt', + 'unauthorizedCount', + 'overloadedAt', + 'overloadStatus', + 'blockedAt', + 'quotaStoppedAt' + ] + + await client.hset(accountKey, updates) + await client.hdel(accountKey, ...fieldsToDelete) + + logger.success(`Reset all error status for Azure OpenAI account ${accountId}`) + + // 清除临时不可用状态 + await upstreamErrorHelper.clearTempUnavailable(accountId, 'azure-openai').catch(() => {}) + + // 异步发送 Webhook 通知（忽略错误） + try { + const webhookNotifier = require('../../utils/webhookNotifier') + await webhookNotifier.sendAccountAnomalyNotification({ + accountId, + accountName: accountData.name || accountId, + platform: 'azure-openai', + status: 'recovered', + errorCode: 'STATUS_RESET', + reason: 'Account status manually reset', + timestamp: new Date().toISOString() + }) + } catch (webhookError) { + logger.warn( + 'Failed to send webhook notification for Azure OpenAI status reset:', + webhookError + ) + } + + return { success: true, accountId } + } catch (error) { + logger.error(`❌ Failed to reset Azure OpenAI account status: ${accountId}`, error) + throw error + } +} + module.exports = { createAccount, getAccount, @@ -528,6 +622,7 @@ module.exports = { performHealthChecks, toggleSchedulable, migrateApiKeysForAzureSupport, + resetAccountStatus, encrypt, decrypt } diff --git a/src/services/bedrockAccountService.js b/src/services/account/bedrockAccountService.js similarity index 91% rename from src/services/bedrockAccountService.js rename to src/services/account/bedrockAccountService.js index 0bb270c0..f1c7bfde 100644 --- a/src/services/bedrockAccountService.js +++ b/src/services/account/bedrockAccountService.js @@ -1,10 +1,11 @@ const { v4: uuidv4 } = require('uuid') const crypto = require('crypto') -const redis = require('../models/redis') -const logger = require('../utils/logger') -const config = require('../../config/config') -const bedrockRelayService = require('./bedrockRelayService') -const LRUCache = require('../utils/lruCache') +const redis = require('../../models/redis') +const logger = require('../../utils/logger') +const config = require('../../../config/config') +const bedrockRelayService = require('../relay/bedrockRelayService') +const LRUCache = require('../../utils/lruCache') +const upstreamErrorHelper = require('../../utils/upstreamErrorHelper') class BedrockAccountService { constructor() { @@ -41,7 +42,8 @@ class BedrockAccountService { accountType = 'shared', // 'dedicated' or 'shared' priority = 50, // 调度优先级 (1-100，数字越小优先级越高) schedulable = true, // 是否可被调度 - credentialType = 'access_key' // 'access_key', 'bearer_token'（默认为 access_key） + credentialType = 'access_key', // 'access_key', 'bearer_token'（默认为 access_key） + disableAutoProtection = false // 是否关闭自动防护（429/401/400/529 不自动禁用） } = options const accountId = uuidv4() @@ -64,7 +66,8 @@ class BedrockAccountService { createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(), - type: 'bedrock' // 标识这是Bedrock账户 + type: 'bedrock', // 标识这是Bedrock账户 + disableAutoProtection // 关闭自动防护 } // 加密存储AWS凭证 @@ -343,6 +346,11 @@ class BedrockAccountService { account.subscriptionExpiresAt = updates.subscriptionExpiresAt } + // 自动防护开关 + if (updates.disableAutoProtection !== undefined) { + account.disableAutoProtection = updates.disableAutoProtection + } + account.updatedAt = new Date().toISOString() await client.set(`bedrock_account:${accountId}`, JSON.stringify(account)) @@ -776,6 +784,66 @@ class BedrockAccountService { return { success: false, error: error.message } } } + + // 🔄 重置Bedrock账户所有异常状态 + async resetAccountStatus(accountId) { + try { + const accountData = await this.getAccount(accountId) + if (!accountData) { + throw new Error('Account not found') + } + + const client = redis.getClientSafe() + const accountKey = `bedrock:account:${accountId}` + + const updates = { + status: 'active', + errorMessage: '', + schedulable: 'true', + isActive: 'true' + } + + const fieldsToDelete = [ + 'rateLimitedAt', + 'rateLimitStatus', + 'unauthorizedAt', + 'unauthorizedCount', + 'overloadedAt', + 'overloadStatus', + 'blockedAt', + 'quotaStoppedAt' + ] + + await client.hset(accountKey, updates) + await client.hdel(accountKey, ...fieldsToDelete) + + logger.success(`Reset all error status for Bedrock account ${accountId}`) + + // 清除临时不可用状态 + await upstreamErrorHelper.clearTempUnavailable(accountId, 'bedrock').catch(() => {}) + + // 异步发送 Webhook 通知（忽略错误） + try { + const webhookNotifier = require('../../utils/webhookNotifier') + await webhookNotifier.sendAccountAnomalyNotification({ + accountId, + accountName: accountData.name || accountId, + platform: 'bedrock', + status: 'recovered', + errorCode: 'STATUS_RESET', + reason: 'Account status manually reset', + timestamp: new Date().toISOString() + }) + } catch (webhookError) { + logger.warn('Failed to send webhook notification for Bedrock status reset:', webhookError) + } + + return { success: true, accountId } + } catch (error) { + logger.error(`❌ Failed to reset Bedrock account status: ${accountId}`, error) + throw error + } + } } module.exports = new BedrockAccountService() diff --git a/src/services/ccrAccountService.js b/src/services/account/ccrAccountService.js similarity index 95% rename from src/services/ccrAccountService.js rename to src/services/account/ccrAccountService.js index ca19100b..e957cb53 100644 --- a/src/services/ccrAccountService.js +++ b/src/services/account/ccrAccountService.js @@ -1,8 +1,9 @@ const { v4: uuidv4 } = require('uuid') -const ProxyHelper = require('../utils/proxyHelper') -const redis = require('../models/redis') -const logger = require('../utils/logger') -const { createEncryptor } = require('../utils/commonHelper') +const ProxyHelper = require('../../utils/proxyHelper') +const redis = require('../../models/redis') +const logger = require('../../utils/logger') +const { createEncryptor } = require('../../utils/commonHelper') +const upstreamErrorHelper = require('../../utils/upstreamErrorHelper') class CcrAccountService { constructor() { @@ -39,7 +40,8 @@ class CcrAccountService { accountType = 'shared', // 'dedicated' or 'shared' schedulable = true, // 是否可被调度 dailyQuota = 0, // 每日额度限制（美元），0表示不限制 - quotaResetTime = '00:00' // 额度重置时间（HH:mm格式） + quotaResetTime = '00:00', // 额度重置时间（HH:mm格式） + disableAutoProtection = false // 是否关闭自动防护（429/401/400/529 不自动禁用） } = options // 验证必填字段 @@ -86,7 +88,8 @@ class CcrAccountService { // 使用与统计一致的时区日期，避免边界问题 lastResetDate: redis.getDateStringInTimezone(), // 最后重置日期（按配置时区） quotaResetTime, // 额度重置时间 - quotaStoppedAt: '' // 因额度停用的时间 + quotaStoppedAt: '', // 因额度停用的时间 + disableAutoProtection: disableAutoProtection.toString() // 关闭自动防护 } const client = redis.getClientSafe() @@ -175,7 +178,8 @@ class CcrAccountService { dailyUsage: parseFloat(accountData.dailyUsage || '0'), lastResetDate: accountData.lastResetDate || '', quotaResetTime: accountData.quotaResetTime || '00:00', - quotaStoppedAt: accountData.quotaStoppedAt || null + quotaStoppedAt: accountData.quotaStoppedAt || null, + disableAutoProtection: accountData.disableAutoProtection === 'true' }) } } @@ -221,6 +225,7 @@ class CcrAccountService { } accountData.isActive = accountData.isActive === 'true' accountData.schedulable = accountData.schedulable !== 'false' // 默认为true + accountData.disableAutoProtection = accountData.disableAutoProtection === 'true' if (accountData.proxy) { accountData.proxy = JSON.parse(accountData.proxy) @@ -299,6 +304,11 @@ class CcrAccountService { updatedData.subscriptionExpiresAt = updates.subscriptionExpiresAt } + // 自动防护开关 + if (updates.disableAutoProtection !== undefined) { + updatedData.disableAutoProtection = updates.disableAutoProtection.toString() + } + await client.hset(`${this.ACCOUNT_KEY_PREFIX}${accountId}`, updatedData) // 处理共享账户集合变更 @@ -691,7 +701,7 @@ class CcrAccountService { // 发送 Webhook 通知 try { - const webhookNotifier = require('../utils/webhookNotifier') + const webhookNotifier = require('../../utils/webhookNotifier') await webhookNotifier.sendAccountAnomalyNotification({ accountId, accountName: account.name || accountId, @@ -858,9 +868,12 @@ class CcrAccountService { logger.success(`Reset all error status for CCR account ${accountId}`) + // 清除临时不可用状态 + await upstreamErrorHelper.clearTempUnavailable(accountId, 'ccr').catch(() => {}) + // 异步发送 Webhook 通知（忽略错误） try { - const webhookNotifier = require('../utils/webhookNotifier') + const webhookNotifier = require('../../utils/webhookNotifier') await webhookNotifier.sendAccountAnomalyNotification({ accountId, accountName: accountData.name || accountId, diff --git a/src/services/claudeAccountService.js b/src/services/account/claudeAccountService.js similarity index 98% rename from src/services/claudeAccountService.js rename to src/services/account/claudeAccountService.js index 710b7bf4..2155eb5e 100644 --- a/src/services/claudeAccountService.js +++ b/src/services/account/claudeAccountService.js @@ -1,22 +1,23 @@ const { v4: uuidv4 } = require('uuid') const crypto = require('crypto') -const ProxyHelper = require('../utils/proxyHelper') +const ProxyHelper = require('../../utils/proxyHelper') const axios = require('axios') -const redis = require('../models/redis') -const config = require('../../config/config') -const logger = require('../utils/logger') -const { maskToken } = require('../utils/tokenMask') +const redis = require('../../models/redis') +const config = require('../../../config/config') +const logger = require('../../utils/logger') +const { maskToken } = require('../../utils/tokenMask') +const upstreamErrorHelper = require('../../utils/upstreamErrorHelper') const { logRefreshStart, logRefreshSuccess, logRefreshError, logTokenUsage, logRefreshSkipped -} = require('../utils/tokenRefreshLogger') -const tokenRefreshService = require('./tokenRefreshService') -const LRUCache = require('../utils/lruCache') -const { formatDateWithTimezone, getISOStringWithTimezone } = require('../utils/dateHelper') -const { isOpus45OrNewer } = require('../utils/modelHelper') +} = require('../../utils/tokenRefreshLogger') +const tokenRefreshService = require('../tokenRefreshService') +const LRUCache = require('../../utils/lruCache') +const { formatDateWithTimezone, getISOStringWithTimezone } = require('../../utils/dateHelper') +const { isOpus45OrNewer } = require('../../utils/modelHelper') /** * Check if account is Pro (not Max) @@ -401,7 +402,7 @@ class ClaudeAccountService { // 发送Webhook通知 try { - const webhookNotifier = require('../utils/webhookNotifier') + const webhookNotifier = require('../../utils/webhookNotifier') await webhookNotifier.sendAccountAnomalyNotification({ accountId, accountName: accountData.name, @@ -787,7 +788,7 @@ class ClaudeAccountService { // 检查是否手动禁用了账号，如果是则发送webhook通知 if (updates.isActive === 'false' && accountData.isActive === 'true') { try { - const webhookNotifier = require('../utils/webhookNotifier') + const webhookNotifier = require('../../utils/webhookNotifier') await webhookNotifier.sendAccountAnomalyNotification({ accountId, accountName: updatedData.name || 'Unknown Account', @@ -831,7 +832,7 @@ class ClaudeAccountService { async deleteAccount(accountId) { try { // 首先从所有分组中移除此账户 - const accountGroupService = require('./accountGroupService') + const accountGroupService = require('../accountGroupService') await accountGroupService.removeAccountFromAllGroups(accountId) const result = await redis.deleteClaudeAccount(accountId) @@ -1387,7 +1388,7 @@ class ClaudeAccountService { // 发送Webhook通知 try { - const webhookNotifier = require('../utils/webhookNotifier') + const webhookNotifier = require('../../utils/webhookNotifier') await webhookNotifier.sendAccountAnomalyNotification({ accountId, accountName: accountData.name || 'Claude Account', @@ -1742,7 +1743,7 @@ class ClaudeAccountService { // 发送Webhook通知 try { - const webhookNotifier = require('../utils/webhookNotifier') + const webhookNotifier = require('../../utils/webhookNotifier') await webhookNotifier.sendAccountAnomalyNotification({ accountId, accountName: accountData.name || 'Claude Account', @@ -2386,7 +2387,7 @@ class ClaudeAccountService { // 发送Webhook通知 try { - const webhookNotifier = require('../utils/webhookNotifier') + const webhookNotifier = require('../../utils/webhookNotifier') await webhookNotifier.sendAccountAnomalyNotification({ accountId, accountName: accountData.name, @@ -2500,6 +2501,13 @@ class ClaudeAccountService { const serverErrorKey = `claude_account:${accountId}:5xx_errors` await redis.client.del(serverErrorKey) + // 清除过载状态 + const overloadKey = `account:overload:${accountId}` + await redis.client.del(overloadKey) + + // 清除临时不可用状态 + await upstreamErrorHelper.clearTempUnavailable(accountId, 'claude-official').catch(() => {}) + logger.info( `✅ Successfully reset all error states for account ${accountData.name} (${accountId})` ) @@ -2704,7 +2712,7 @@ class ClaudeAccountService { // 发送Webhook通知 try { - const webhookNotifier = require('../utils/webhookNotifier') + const webhookNotifier = require('../../utils/webhookNotifier') await webhookNotifier.sendAccountAnomalyNotification({ accountId, accountName: accountData.name, @@ -2795,7 +2803,7 @@ class ClaudeAccountService { if (canSendWarning) { // 发送Webhook通知 try { - const webhookNotifier = require('../utils/webhookNotifier') + const webhookNotifier = require('../../utils/webhookNotifier') await webhookNotifier.sendAccountAnomalyNotification({ accountId, accountName: accountData.name || 'Claude Account', diff --git a/src/services/claudeConsoleAccountService.js b/src/services/account/claudeConsoleAccountService.js similarity index 97% rename from src/services/claudeConsoleAccountService.js rename to src/services/account/claudeConsoleAccountService.js index e6c25c24..8d68dbcb 100644 --- a/src/services/claudeConsoleAccountService.js +++ b/src/services/account/claudeConsoleAccountService.js @@ -1,10 +1,11 @@ const { v4: uuidv4 } = require('uuid') const crypto = require('crypto') -const ProxyHelper = require('../utils/proxyHelper') -const redis = require('../models/redis') -const logger = require('../utils/logger') -const config = require('../../config/config') -const LRUCache = require('../utils/lruCache') +const ProxyHelper = require('../../utils/proxyHelper') +const redis = require('../../models/redis') +const logger = require('../../utils/logger') +const config = require('../../../config/config') +const LRUCache = require('../../utils/lruCache') +const upstreamErrorHelper = require('../../utils/upstreamErrorHelper') class ClaudeConsoleAccountService { constructor() { @@ -414,7 +415,7 @@ class ClaudeConsoleAccountService { // 检查是否手动禁用了账号，如果是则发送webhook通知 if (updates.isActive === false && existingAccount.isActive === true) { try { - const webhookNotifier = require('../utils/webhookNotifier') + const webhookNotifier = require('../../utils/webhookNotifier') await webhookNotifier.sendAccountAnomalyNotification({ accountId, accountName: updatedData.name || existingAccount.name || 'Unknown Account', @@ -512,8 +513,8 @@ class ClaudeConsoleAccountService { // 发送Webhook通知 try { - const webhookNotifier = require('../utils/webhookNotifier') - const { getISOStringWithTimezone } = require('../utils/dateHelper') + const webhookNotifier = require('../../utils/webhookNotifier') + const { getISOStringWithTimezone } = require('../../utils/dateHelper') await webhookNotifier.sendAccountAnomalyNotification({ accountId, accountName: account.name || 'Claude Console Account', @@ -726,7 +727,7 @@ class ClaudeConsoleAccountService { // 发送Webhook通知 try { - const webhookNotifier = require('../utils/webhookNotifier') + const webhookNotifier = require('../../utils/webhookNotifier') await webhookNotifier.sendAccountAnomalyNotification({ accountId, accountName: account.name || 'Claude Console Account', @@ -793,7 +794,7 @@ class ClaudeConsoleAccountService { // 发送Webhook通知，包含完整错误详情 try { - const webhookNotifier = require('../utils/webhookNotifier') + const webhookNotifier = require('../../utils/webhookNotifier') await webhookNotifier.sendAccountAnomalyNotification({ accountId, accountName: account.name || 'Claude Console Account', @@ -947,7 +948,7 @@ class ClaudeConsoleAccountService { // 发送Webhook通知 try { - const webhookNotifier = require('../utils/webhookNotifier') + const webhookNotifier = require('../../utils/webhookNotifier') await webhookNotifier.sendAccountAnomalyNotification({ accountId, accountName: account.name || 'Claude Console Account', @@ -1040,7 +1041,7 @@ class ClaudeConsoleAccountService { // 发送Webhook通知 if (accountData && Object.keys(accountData).length > 0) { try { - const webhookNotifier = require('../utils/webhookNotifier') + const webhookNotifier = require('../../utils/webhookNotifier') await webhookNotifier.sendAccountAnomalyNotification({ accountId, accountName: accountData.name || 'Unknown Account', @@ -1329,7 +1330,7 @@ class ClaudeConsoleAccountService { // 发送webhook通知 try { - const webhookNotifier = require('../utils/webhookNotifier') + const webhookNotifier = require('../../utils/webhookNotifier') await webhookNotifier.sendAccountAnomalyNotification({ accountId, accountName: accountData.name || 'Unknown Account', @@ -1479,9 +1480,12 @@ class ClaudeConsoleAccountService { logger.success(`Reset all error status for Claude Console account ${accountId}`) + // 清除临时不可用状态 + await upstreamErrorHelper.clearTempUnavailable(accountId, 'claude-console').catch(() => {}) + // 发送 Webhook 通知 try { - const webhookNotifier = require('../utils/webhookNotifier') + const webhookNotifier = require('../../utils/webhookNotifier') await webhookNotifier.sendAccountAnomalyNotification({ accountId, accountName: accountData.name || accountId, diff --git a/src/services/droidAccountService.js b/src/services/account/droidAccountService.js similarity index 95% rename from src/services/droidAccountService.js rename to src/services/account/droidAccountService.js index 406fe789..cd9b7980 100644 --- a/src/services/droidAccountService.js +++ b/src/services/account/droidAccountService.js @@ -1,11 +1,12 @@ const { v4: uuidv4 } = require('uuid') const crypto = require('crypto') const axios = require('axios') -const redis = require('../models/redis') -const logger = require('../utils/logger') -const { maskToken } = require('../utils/tokenMask') -const ProxyHelper = require('../utils/proxyHelper') -const { createEncryptor, isTruthy } = require('../utils/commonHelper') +const redis = require('../../models/redis') +const logger = require('../../utils/logger') +const { maskToken } = require('../../utils/tokenMask') +const ProxyHelper = require('../../utils/proxyHelper') +const { createEncryptor, isTruthy } = require('../../utils/commonHelper') +const upstreamErrorHelper = require('../../utils/upstreamErrorHelper') /** * Droid 账户管理服务 @@ -476,7 +477,8 @@ class DroidAccountService { authenticationMethod = '', expiresIn = null, apiKeys = [], - userAgent = '' // 自定义 User-Agent + userAgent = '', // 自定义 User-Agent + disableAutoProtection = false // 是否关闭自动防护（429/401/400/529 不自动禁用） } = options const accountId = uuidv4() @@ -753,7 +755,8 @@ class DroidAccountService { apiKeys: hasApiKeys ? JSON.stringify(apiKeyEntries) : '', apiKeyCount: hasApiKeys ? String(apiKeyEntries.length) : '0', apiKeyStrategy: hasApiKeys ? 'random_sticky' : '', - userAgent: userAgent || '' // 自定义 User-Agent + userAgent: userAgent || '', // 自定义 User-Agent + disableAutoProtection: disableAutoProtection.toString() // 关闭自动防护 } await redis.setDroidAccount(accountId, accountData) @@ -1494,6 +1497,66 @@ class DroidAccountService { logger.warn(`⚠️ Failed to update lastUsedAt for Droid account ${accountId}:`, error) } } + + // 🔄 重置Droid账户所有异常状态 + async resetAccountStatus(accountId) { + try { + const accountData = await this.getAccount(accountId) + if (!accountData) { + throw new Error('Account not found') + } + + const client = redis.getClientSafe() + const accountKey = `droid:account:${accountId}` + + const updates = { + status: 'active', + errorMessage: '', + schedulable: 'true', + isActive: 'true' + } + + const fieldsToDelete = [ + 'rateLimitedAt', + 'rateLimitStatus', + 'unauthorizedAt', + 'unauthorizedCount', + 'overloadedAt', + 'overloadStatus', + 'blockedAt', + 'quotaStoppedAt' + ] + + await client.hset(accountKey, updates) + await client.hdel(accountKey, ...fieldsToDelete) + + logger.success(`Reset all error status for Droid account ${accountId}`) + + // 清除临时不可用状态 + await upstreamErrorHelper.clearTempUnavailable(accountId, 'droid').catch(() => {}) + + // 异步发送 Webhook 通知（忽略错误） + try { + const webhookNotifier = require('../../utils/webhookNotifier') + await webhookNotifier.sendAccountAnomalyNotification({ + accountId, + accountName: accountData.name || accountId, + platform: 'droid', + status: 'recovered', + errorCode: 'STATUS_RESET', + reason: 'Account status manually reset', + timestamp: new Date().toISOString() + }) + } catch (webhookError) { + logger.warn('Failed to send webhook notification for Droid status reset:', webhookError) + } + + return { success: true, accountId } + } catch (error) { + logger.error(`❌ Failed to reset Droid account status: ${accountId}`, error) + throw error + } + } } // 导出单例 diff --git a/src/services/geminiAccountService.js b/src/services/account/geminiAccountService.js similarity index 97% rename from src/services/geminiAccountService.js rename to src/services/account/geminiAccountService.js index 9850d2b2..7dc0f419 100644 --- a/src/services/geminiAccountService.js +++ b/src/services/account/geminiAccountService.js @@ -1,21 +1,22 @@ -const redisClient = require('../models/redis') +const redisClient = require('../../models/redis') const { v4: uuidv4 } = require('uuid') const crypto = require('crypto') const https = require('https') -const logger = require('../utils/logger') +const logger = require('../../utils/logger') const { OAuth2Client } = require('google-auth-library') -const { maskToken } = require('../utils/tokenMask') -const ProxyHelper = require('../utils/proxyHelper') +const { maskToken } = require('../../utils/tokenMask') +const ProxyHelper = require('../../utils/proxyHelper') +const upstreamErrorHelper = require('../../utils/upstreamErrorHelper') const { logRefreshStart, logRefreshSuccess, logRefreshError, logTokenUsage, logRefreshSkipped -} = require('../utils/tokenRefreshLogger') -const tokenRefreshService = require('./tokenRefreshService') -const { createEncryptor } = require('../utils/commonHelper') -const antigravityClient = require('./antigravityClient') +} = require('../../utils/tokenRefreshLogger') +const tokenRefreshService = require('../tokenRefreshService') +const { createEncryptor } = require('../../utils/commonHelper') +const antigravityClient = require('../antigravityClient') // Gemini 账户键前缀 const GEMINI_ACCOUNT_KEY_PREFIX = 'gemini_account:' @@ -134,7 +135,7 @@ async function fetchAvailableModelsAntigravity( getAntigravityModelAlias, getAntigravityModelMetadata, normalizeAntigravityModelInput - } = require('../utils/antigravityModel') + } = require('../../utils/antigravityModel') const pushModel = (modelId) => { if (!modelId || seen.has(modelId)) { @@ -523,6 +524,12 @@ async function createAccount(accountData) { // 支持的模型列表（可选） supportedModels: accountData.supportedModels || [], // 空数组表示支持所有模型 + // 自动防护开关 + disableAutoProtection: + accountData.disableAutoProtection === true || accountData.disableAutoProtection === 'true' + ? 'true' + : 'false', + // 时间戳 createdAt: now, updatedAt: now, @@ -666,6 +673,14 @@ async function updateAccount(accountId, updates) { // 直接保存，不做任何调整 } + // 自动防护开关 + if (updates.disableAutoProtection !== undefined) { + updates.disableAutoProtection = + updates.disableAutoProtection === true || updates.disableAutoProtection === 'true' + ? 'true' + : 'false' + } + // 如果通过 geminiOauth 更新，也要检查是否新增了 refresh token if (updates.geminiOauth && !oldRefreshToken) { const oauthData = @@ -692,7 +707,7 @@ async function updateAccount(accountId, updates) { // 检查是否手动禁用了账号，如果是则发送webhook通知 if (updates.isActive === 'false' && existingAccount.isActive !== 'false') { try { - const webhookNotifier = require('../utils/webhookNotifier') + const webhookNotifier = require('../../utils/webhookNotifier') await webhookNotifier.sendAccountAnomalyNotification({ accountId, accountName: updates.name || existingAccount.name || 'Unknown Account', @@ -1076,7 +1091,7 @@ async function refreshAccountToken(accountId) { // 发送Webhook通知 try { - const webhookNotifier = require('../utils/webhookNotifier') + const webhookNotifier = require('../../utils/webhookNotifier') await webhookNotifier.sendAccountAnomalyNotification({ accountId, accountName: account.name, @@ -1843,9 +1858,12 @@ async function resetAccountStatus(accountId) { await updateAccount(accountId, updates) logger.info(`✅ Reset all error status for Gemini account ${accountId}`) + // 清除临时不可用状态 + await upstreamErrorHelper.clearTempUnavailable(accountId, 'gemini').catch(() => {}) + // 发送 Webhook 通知 try { - const webhookNotifier = require('../utils/webhookNotifier') + const webhookNotifier = require('../../utils/webhookNotifier') await webhookNotifier.sendAccountAnomalyNotification({ accountId, accountName: account.name || accountId, diff --git a/src/services/geminiApiAccountService.js b/src/services/account/geminiApiAccountService.js similarity index 94% rename from src/services/geminiApiAccountService.js rename to src/services/account/geminiApiAccountService.js index 663269ca..35812fb8 100644 --- a/src/services/geminiApiAccountService.js +++ b/src/services/account/geminiApiAccountService.js @@ -1,9 +1,10 @@ const { v4: uuidv4 } = require('uuid') const crypto = require('crypto') -const redis = require('../models/redis') -const logger = require('../utils/logger') -const config = require('../../config/config') -const LRUCache = require('../utils/lruCache') +const redis = require('../../models/redis') +const logger = require('../../utils/logger') +const config = require('../../../config/config') +const LRUCache = require('../../utils/lruCache') +const upstreamErrorHelper = require('../../utils/upstreamErrorHelper') class GeminiApiAccountService { constructor() { @@ -44,7 +45,8 @@ class GeminiApiAccountService { accountType = 'shared', // 'dedicated' or 'shared' schedulable = true, // 是否可被调度 supportedModels = [], // 支持的模型列表 - rateLimitDuration = 60 // 限流时间（分钟） + rateLimitDuration = 60, // 限流时间（分钟） + disableAutoProtection = false } = options // 验证必填字段 @@ -79,7 +81,11 @@ class GeminiApiAccountService { // 限流相关 rateLimitedAt: '', rateLimitStatus: '', - rateLimitDuration: rateLimitDuration.toString() + rateLimitDuration: rateLimitDuration.toString(), + + // 自动防护开关 + disableAutoProtection: + disableAutoProtection === true || disableAutoProtection === 'true' ? 'true' : 'false' } // 保存到 Redis @@ -154,6 +160,14 @@ class GeminiApiAccountService { : updates.baseUrl } + // 处理 disableAutoProtection 布尔值转字符串 + if (updates.disableAutoProtection !== undefined) { + updates.disableAutoProtection = + updates.disableAutoProtection === true || updates.disableAutoProtection === 'true' + ? 'true' + : 'false' + } + // 更新 Redis const client = redis.getClientSafe() const key = `${this.ACCOUNT_KEY_PREFIX}${accountId}` @@ -363,7 +377,7 @@ class GeminiApiAccountService { ) try { - const webhookNotifier = require('../utils/webhookNotifier') + const webhookNotifier = require('../../utils/webhookNotifier') await webhookNotifier.sendAccountAnomalyNotification({ accountId, accountName: account.name || accountId, @@ -456,9 +470,12 @@ class GeminiApiAccountService { await this.updateAccount(accountId, updates) logger.info(`✅ Reset all error status for Gemini-API account ${accountId}`) + // 清除临时不可用状态 + await upstreamErrorHelper.clearTempUnavailable(accountId, 'gemini-api').catch(() => {}) + // 发送 Webhook 通知 try { - const webhookNotifier = require('../utils/webhookNotifier') + const webhookNotifier = require('../../utils/webhookNotifier') await webhookNotifier.sendAccountAnomalyNotification({ accountId, accountName: account.name || accountId, diff --git a/src/services/openaiAccountService.js b/src/services/account/openaiAccountService.js similarity index 96% rename from src/services/openaiAccountService.js rename to src/services/account/openaiAccountService.js index 32f0590b..7fd41e1a 100644 --- a/src/services/openaiAccountService.js +++ b/src/services/account/openaiAccountService.js @@ -1,19 +1,20 @@ -const redisClient = require('../models/redis') +const redisClient = require('../../models/redis') const { v4: uuidv4 } = require('uuid') const axios = require('axios') -const ProxyHelper = require('../utils/proxyHelper') -const config = require('../../config/config') -const logger = require('../utils/logger') -// const { maskToken } = require('../utils/tokenMask') +const ProxyHelper = require('../../utils/proxyHelper') +const config = require('../../../config/config') +const logger = require('../../utils/logger') +const upstreamErrorHelper = require('../../utils/upstreamErrorHelper') +// const { maskToken } = require('../../utils/tokenMask') const { logRefreshStart, logRefreshSuccess, logRefreshError, logTokenUsage, logRefreshSkipped -} = require('../utils/tokenRefreshLogger') -const tokenRefreshService = require('./tokenRefreshService') -const { createEncryptor } = require('../utils/commonHelper') +} = require('../../utils/tokenRefreshLogger') +const tokenRefreshService = require('../tokenRefreshService') +const { createEncryptor } = require('../../utils/commonHelper') // 使用 commonHelper 的加密器 const encryptor = createEncryptor('openai-account-salt') @@ -405,7 +406,7 @@ async function refreshAccountToken(accountId) { // 发送 Webhook 通知（如果启用） try { - const webhookNotifier = require('../utils/webhookNotifier') + const webhookNotifier = require('../../utils/webhookNotifier') await webhookNotifier.sendAccountAnomalyNotification({ accountId, accountName: account?.name || accountName, @@ -496,6 +497,11 @@ async function createAccount(accountData) { isActive: accountData.isActive !== false ? 'true' : 'false', status: 'active', schedulable: accountData.schedulable !== false ? 'true' : 'false', + // 自动防护开关 + disableAutoProtection: + accountData.disableAutoProtection === true || accountData.disableAutoProtection === 'true' + ? 'true' + : 'false', lastRefresh: now, createdAt: now, updatedAt: now @@ -605,6 +611,14 @@ async function updateAccount(accountId, updates) { // 直接保存，不做任何调整 } + // 处理 disableAutoProtection 布尔值转字符串 + if (updates.disableAutoProtection !== undefined) { + updates.disableAutoProtection = + updates.disableAutoProtection === true || updates.disableAutoProtection === 'true' + ? 'true' + : 'false' + } + // 更新账户类型时处理共享账户集合 const client = redisClient.getClientSafe() if (updates.accountType && updates.accountType !== existingAccount.accountType) { @@ -961,7 +975,7 @@ async function setAccountRateLimited(accountId, isLimited, resetsInSeconds = nul if (isLimited) { try { const account = await getAccount(accountId) - const webhookNotifier = require('../utils/webhookNotifier') + const webhookNotifier = require('../../utils/webhookNotifier') await webhookNotifier.sendAccountAnomalyNotification({ accountId, accountName: account.name || accountId, @@ -1005,7 +1019,7 @@ async function markAccountUnauthorized(accountId, reason = 'OpenAI账号认证 ) try { - const webhookNotifier = require('../utils/webhookNotifier') + const webhookNotifier = require('../../utils/webhookNotifier') await webhookNotifier.sendAccountAnomalyNotification({ accountId, accountName: account.name || accountId, @@ -1045,9 +1059,12 @@ async function resetAccountStatus(accountId) { await updateAccount(accountId, updates) logger.info(`✅ Reset all error status for OpenAI account ${accountId}`) + // 清除临时不可用状态 + await upstreamErrorHelper.clearTempUnavailable(accountId, 'openai').catch(() => {}) + // 发送 Webhook 通知 try { - const webhookNotifier = require('../utils/webhookNotifier') + const webhookNotifier = require('../../utils/webhookNotifier') await webhookNotifier.sendAccountAnomalyNotification({ accountId, accountName: account.name || accountId, diff --git a/src/services/openaiResponsesAccountService.js b/src/services/account/openaiResponsesAccountService.js similarity index 95% rename from src/services/openaiResponsesAccountService.js rename to src/services/account/openaiResponsesAccountService.js index ca62a1ac..c2cae7a8 100644 --- a/src/services/openaiResponsesAccountService.js +++ b/src/services/account/openaiResponsesAccountService.js @@ -1,9 +1,10 @@ const { v4: uuidv4 } = require('uuid') const crypto = require('crypto') -const redis = require('../models/redis') -const logger = require('../utils/logger') -const config = require('../../config/config') -const LRUCache = require('../utils/lruCache') +const redis = require('../../models/redis') +const logger = require('../../utils/logger') +const config = require('../../../config/config') +const LRUCache = require('../../utils/lruCache') +const upstreamErrorHelper = require('../../utils/upstreamErrorHelper') class OpenAIResponsesAccountService { constructor() { @@ -49,7 +50,8 @@ class OpenAIResponsesAccountService { schedulable = true, // 是否可被调度 dailyQuota = 0, // 每日额度限制（美元），0表示不限制 quotaResetTime = '00:00', // 额度重置时间（HH:mm格式） - rateLimitDuration = 60 // 限流时间（分钟） + rateLimitDuration = 60, // 限流时间（分钟） + disableAutoProtection = false // 是否关闭自动防护（429/401/400/529 不自动禁用） } = options // 验证必填字段 @@ -93,7 +95,8 @@ class OpenAIResponsesAccountService { dailyUsage: '0', lastResetDate: redis.getDateStringInTimezone(), quotaResetTime, - quotaStoppedAt: '' + quotaStoppedAt: '', + disableAutoProtection: disableAutoProtection.toString() // 关闭自动防护 } // 保存到 Redis @@ -162,6 +165,11 @@ class OpenAIResponsesAccountService { // 直接保存，不做任何调整 } + // 自动防护开关 + if (updates.disableAutoProtection !== undefined) { + updates.disableAutoProtection = updates.disableAutoProtection.toString() + } + // 更新 Redis const client = redis.getClientSafe() const key = `${this.ACCOUNT_KEY_PREFIX}${accountId}` @@ -310,7 +318,7 @@ class OpenAIResponsesAccountService { ) try { - const webhookNotifier = require('../utils/webhookNotifier') + const webhookNotifier = require('../../utils/webhookNotifier') await webhookNotifier.sendAccountAnomalyNotification({ accountId, accountName: account.name || accountId, @@ -475,9 +483,12 @@ class OpenAIResponsesAccountService { await this.updateAccount(accountId, updates) logger.info(`✅ Reset all error status for OpenAI-Responses account ${accountId}`) + // 清除临时不可用状态 + await upstreamErrorHelper.clearTempUnavailable(accountId, 'openai-responses').catch(() => {}) + // 发送 Webhook 通知 try { - const webhookNotifier = require('../utils/webhookNotifier') + const webhookNotifier = require('../../utils/webhookNotifier') await webhookNotifier.sendAccountAnomalyNotification({ accountId, accountName: account.name || accountId, diff --git a/src/services/accountNameCacheService.js b/src/services/accountNameCacheService.js index 564c8080..f218b746 100644 --- a/src/services/accountNameCacheService.js +++ b/src/services/accountNameCacheService.js @@ -49,26 +49,26 @@ class AccountNameCacheService { const newGroupCache = new Map() // 延迟加载服务，避免循环依赖 - const claudeAccountService = require('./claudeAccountService') - const claudeConsoleAccountService = require('./claudeConsoleAccountService') - const geminiAccountService = require('./geminiAccountService') - const openaiAccountService = require('./openaiAccountService') - const azureOpenaiAccountService = require('./azureOpenaiAccountService') - const bedrockAccountService = require('./bedrockAccountService') - const droidAccountService = require('./droidAccountService') - const ccrAccountService = require('./ccrAccountService') + const claudeAccountService = require('./account/claudeAccountService') + const claudeConsoleAccountService = require('./account/claudeConsoleAccountService') + const geminiAccountService = require('./account/geminiAccountService') + const openaiAccountService = require('./account/openaiAccountService') + const azureOpenaiAccountService = require('./account/azureOpenaiAccountService') + const bedrockAccountService = require('./account/bedrockAccountService') + const droidAccountService = require('./account/droidAccountService') + const ccrAccountService = require('./account/ccrAccountService') const accountGroupService = require('./accountGroupService') // 可选服务（可能不存在） let geminiApiAccountService = null let openaiResponsesAccountService = null try { - geminiApiAccountService = require('./geminiApiAccountService') + geminiApiAccountService = require('./account/geminiApiAccountService') } catch (e) { // 服务不存在，忽略 } try { - openaiResponsesAccountService = require('./openaiResponsesAccountService') + openaiResponsesAccountService = require('./account/openaiResponsesAccountService') } catch (e) { // 服务不存在，忽略 } diff --git a/src/services/accountTestSchedulerService.js b/src/services/accountTestSchedulerService.js index 59b4c6af..c0a21ceb 100644 --- a/src/services/accountTestSchedulerService.js +++ b/src/services/accountTestSchedulerService.js @@ -269,7 +269,7 @@ class AccountTestSchedulerService { * @private */ async _testClaudeAccount(accountId, model) { - const claudeRelayService = require('./claudeRelayService') + const claudeRelayService = require('./relay/claudeRelayService') return await claudeRelayService.testAccountConnectionSync(accountId, model) } diff --git a/src/services/anthropicGeminiBridgeService.js b/src/services/anthropicGeminiBridgeService.js index 422c6e1c..6a868de2 100644 --- a/src/services/anthropicGeminiBridgeService.js +++ b/src/services/anthropicGeminiBridgeService.js @@ -34,8 +34,8 @@ const fs = require('fs') const path = require('path') const logger = require('../utils/logger') const { getProjectRoot } = require('../utils/projectPaths') -const geminiAccountService = require('./geminiAccountService') -const unifiedGeminiScheduler = require('./unifiedGeminiScheduler') +const geminiAccountService = require('./account/geminiAccountService') +const unifiedGeminiScheduler = require('./scheduler/unifiedGeminiScheduler') const sessionHelper = require('../utils/sessionHelper') const signatureCache = require('../utils/signatureCache') const apiKeyService = require('./apiKeyService') diff --git a/src/services/claudeRelayConfigService.js b/src/services/claudeRelayConfigService.js index 8a6f26a3..066ba7d3 100644 --- a/src/services/claudeRelayConfigService.js +++ b/src/services/claudeRelayConfigService.js @@ -280,16 +280,16 @@ class ClaudeRelayConfigService { let accountService switch (accountType) { case 'claude-official': - accountService = require('./claudeAccountService') + accountService = require('./account/claudeAccountService') break case 'claude-console': - accountService = require('./claudeConsoleAccountService') + accountService = require('./account/claudeConsoleAccountService') break case 'bedrock': - accountService = require('./bedrockAccountService') + accountService = require('./account/bedrockAccountService') break case 'ccr': - accountService = require('./ccrAccountService') + accountService = require('./account/ccrAccountService') break default: logger.warn(`Unknown account type for validation: ${accountType}`) diff --git a/src/services/rateLimitCleanupService.js b/src/services/rateLimitCleanupService.js index d3b90df6..1107c90b 100644 --- a/src/services/rateLimitCleanupService.js +++ b/src/services/rateLimitCleanupService.js @@ -4,10 +4,10 @@ */ const logger = require('../utils/logger') -const openaiAccountService = require('./openaiAccountService') -const claudeAccountService = require('./claudeAccountService') -const claudeConsoleAccountService = require('./claudeConsoleAccountService') -const unifiedOpenAIScheduler = require('./unifiedOpenAIScheduler') +const openaiAccountService = require('./account/openaiAccountService') +const claudeAccountService = require('./account/claudeAccountService') +const claudeConsoleAccountService = require('./account/claudeConsoleAccountService') +const unifiedOpenAIScheduler = require('./scheduler/unifiedOpenAIScheduler') const webhookService = require('./webhookService') class RateLimitCleanupService { diff --git a/src/services/antigravityRelayService.js b/src/services/relay/antigravityRelayService.js similarity index 95% rename from src/services/antigravityRelayService.js rename to src/services/relay/antigravityRelayService.js index 6a3378f1..971722d4 100644 --- a/src/services/antigravityRelayService.js +++ b/src/services/relay/antigravityRelayService.js @@ -1,7 +1,7 @@ -const apiKeyService = require('./apiKeyService') +const apiKeyService = require('../apiKeyService') const { convertMessagesToGemini, convertGeminiResponse } = require('./geminiRelayService') -const { normalizeAntigravityModelInput } = require('../utils/antigravityModel') -const antigravityClient = require('./antigravityClient') +const { normalizeAntigravityModelInput } = require('../../utils/antigravityModel') +const antigravityClient = require('../antigravityClient') function buildRequestData({ messages, model, temperature, maxTokens, sessionId }) { const requestedModel = normalizeAntigravityModelInput(model) diff --git a/src/services/azureOpenaiRelayService.js b/src/services/relay/azureOpenaiRelayService.js similarity index 97% rename from src/services/azureOpenaiRelayService.js rename to src/services/relay/azureOpenaiRelayService.js index e1a09e02..ed6e5dc9 100644 --- a/src/services/azureOpenaiRelayService.js +++ b/src/services/relay/azureOpenaiRelayService.js @@ -1,7 +1,8 @@ const axios = require('axios') -const ProxyHelper = require('../utils/proxyHelper') -const logger = require('../utils/logger') -const config = require('../../config/config') +const ProxyHelper = require('../../utils/proxyHelper') +const logger = require('../../utils/logger') +const config = require('../../../config/config') +const upstreamErrorHelper = require('../../utils/upstreamErrorHelper') // 转换模型名称（去掉 azure/ 前缀） function normalizeModelName(model) { @@ -212,6 +213,16 @@ async function handleAzureOpenAIRequest({ logger.error('Azure OpenAI Request Failed', errorDetails) } + // 网络错误标记临时不可用 + const azureAutoProtectionDisabled = + account?.disableAutoProtection === true || account?.disableAutoProtection === 'true' + if (account?.id && !azureAutoProtectionDisabled) { + const statusCode = error.response?.status || 503 + await upstreamErrorHelper + .markTempUnavailable(account.id, 'azure-openai', statusCode) + .catch(() => {}) + } + throw error } } diff --git a/src/services/bedrockRelayService.js b/src/services/relay/bedrockRelayService.js similarity index 94% rename from src/services/bedrockRelayService.js rename to src/services/relay/bedrockRelayService.js index 87e65105..3a479fdf 100644 --- a/src/services/bedrockRelayService.js +++ b/src/services/relay/bedrockRelayService.js @@ -4,9 +4,10 @@ const { InvokeModelWithResponseStreamCommand } = require('@aws-sdk/client-bedrock-runtime') const { fromEnv } = require('@aws-sdk/credential-providers') -const logger = require('../utils/logger') -const config = require('../../config/config') -const userMessageQueueService = require('./userMessageQueueService') +const logger = require('../../utils/logger') +const config = require('../../../config/config') +const userMessageQueueService = require('../userMessageQueueService') +const upstreamErrorHelper = require('../../utils/upstreamErrorHelper') class BedrockRelayService { constructor() { @@ -188,7 +189,7 @@ class BedrockRelayService { } } catch (error) { logger.error('❌ Bedrock非流式请求失败:', error) - throw this._handleBedrockError(error) + throw this._handleBedrockError(error, accountId, bedrockAccount) } finally { // 📬 释放用户消息队列锁（兜底，正常情况下已在请求发送后提前释放） if (queueLockAcquired && queueRequestId && accountId) { @@ -376,10 +377,12 @@ class BedrockRelayService { } res.write('event: error\n') - res.write(`data: ${JSON.stringify({ error: this._handleBedrockError(error).message })}\n\n`) + res.write( + `data: ${JSON.stringify({ error: this._handleBedrockError(error, accountId, bedrockAccount).message })}\n\n` + ) res.end() - throw this._handleBedrockError(error) + throw this._handleBedrockError(error, accountId, bedrockAccount) } finally { // 📬 释放用户消息队列锁（兜底，正常情况下已在请求发送后提前释放） if (queueLockAcquired && queueRequestId && accountId) { @@ -647,7 +650,25 @@ class BedrockRelayService { } // 处理Bedrock错误 - _handleBedrockError(error) { + _handleBedrockError(error, accountId = null, bedrockAccount = null) { + const autoProtectionDisabled = + bedrockAccount?.disableAutoProtection === true || + bedrockAccount?.disableAutoProtection === 'true' + if (accountId && !autoProtectionDisabled) { + if (error.name === 'ThrottlingException') { + upstreamErrorHelper.markTempUnavailable(accountId, 'bedrock', 429).catch(() => {}) + } else if (error.name === 'AccessDeniedException') { + upstreamErrorHelper.markTempUnavailable(accountId, 'bedrock', 403).catch(() => {}) + } else if ( + error.name === 'ServiceUnavailableException' || + error.name === 'InternalServerException' + ) { + upstreamErrorHelper.markTempUnavailable(accountId, 'bedrock', 500).catch(() => {}) + } else if (error.name === 'ModelNotReadyException') { + upstreamErrorHelper.markTempUnavailable(accountId, 'bedrock', 503).catch(() => {}) + } + } + const errorMessage = error.message || 'Unknown Bedrock error' if (error.name === 'ValidationException') { diff --git a/src/services/ccrRelayService.js b/src/services/relay/ccrRelayService.js similarity index 89% rename from src/services/ccrRelayService.js rename to src/services/relay/ccrRelayService.js index d5f97c9f..c6cd76df 100644 --- a/src/services/ccrRelayService.js +++ b/src/services/relay/ccrRelayService.js @@ -1,10 +1,11 @@ const axios = require('axios') -const ccrAccountService = require('./ccrAccountService') -const logger = require('../utils/logger') -const config = require('../../config/config') -const { parseVendorPrefixedModel } = require('../utils/modelHelper') -const userMessageQueueService = require('./userMessageQueueService') -const { isStreamWritable } = require('../utils/streamHelper') +const ccrAccountService = require('../account/ccrAccountService') +const logger = require('../../utils/logger') +const config = require('../../../config/config') +const { parseVendorPrefixedModel } = require('../../utils/modelHelper') +const userMessageQueueService = require('../userMessageQueueService') +const { isStreamWritable } = require('../../utils/streamHelper') +const upstreamErrorHelper = require('../../utils/upstreamErrorHelper') class CcrRelayService { constructor() { @@ -261,7 +262,11 @@ class CcrRelayService { // 检查错误状态并相应处理 if (response.status === 401) { logger.warn(`🚫 Unauthorized error detected for CCR account ${accountId}`) - await ccrAccountService.markAccountUnauthorized(accountId) + const autoProtectionDisabled = + account?.disableAutoProtection === true || account?.disableAutoProtection === 'true' + if (!autoProtectionDisabled) { + await upstreamErrorHelper.markTempUnavailable(accountId, 'ccr', 401).catch(() => {}) + } } else if (response.status === 429) { logger.warn(`🚫 Rate limit detected for CCR account ${accountId}`) // 收到429先检查是否因为超过了手动配置的每日额度 @@ -270,9 +275,35 @@ class CcrRelayService { }) await ccrAccountService.markAccountRateLimited(accountId) + const autoProtectionDisabled = + account?.disableAutoProtection === true || account?.disableAutoProtection === 'true' + if (!autoProtectionDisabled) { + await upstreamErrorHelper + .markTempUnavailable( + accountId, + 'ccr', + 429, + upstreamErrorHelper.parseRetryAfter(response.headers) + ) + .catch(() => {}) + } } else if (response.status === 529) { logger.warn(`🚫 Overload error detected for CCR account ${accountId}`) await ccrAccountService.markAccountOverloaded(accountId) + const autoProtectionDisabled = + account?.disableAutoProtection === true || account?.disableAutoProtection === 'true' + if (!autoProtectionDisabled) { + await upstreamErrorHelper.markTempUnavailable(accountId, 'ccr', 529).catch(() => {}) + } + } else if (response.status >= 500) { + logger.warn(`🔥 Server error (${response.status}) detected for CCR account ${accountId}`) + const autoProtectionDisabled = + account?.disableAutoProtection === true || account?.disableAutoProtection === 'true' + if (!autoProtectionDisabled) { + await upstreamErrorHelper + .markTempUnavailable(accountId, 'ccr', response.status) + .catch(() => {}) + } } else if (response.status === 200 || response.status === 201) { // 如果请求成功，检查并移除错误状态 const isRateLimited = await ccrAccountService.isAccountRateLimited(accountId) @@ -310,6 +341,15 @@ class CcrRelayService { error.message ) + // 网络错误标记临时不可用 + if (accountId && !error.response) { + const autoProtectionDisabled = + account?.disableAutoProtection === true || account?.disableAutoProtection === 'true' + if (!autoProtectionDisabled) { + await upstreamErrorHelper.markTempUnavailable(accountId, 'ccr', 503).catch(() => {}) + } + } + throw error } finally { // 📬 释放用户消息队列锁（兜底，正常情况下已在请求发送后提前释放） @@ -489,6 +529,14 @@ class CcrRelayService { ) } else { logger.error(`❌ CCR stream relay failed (Account: ${account?.name || accountId}):`, error) + // 网络错误标记临时不可用 + if (accountId && !error.response) { + const autoProtectionDisabled = + account?.disableAutoProtection === true || account?.disableAutoProtection === 'true' + if (!autoProtectionDisabled) { + await upstreamErrorHelper.markTempUnavailable(accountId, 'ccr', 503).catch(() => {}) + } + } } throw error } finally { @@ -596,16 +644,40 @@ class CcrRelayService { `❌ CCR API returned error status: ${response.status} | Account: ${account?.name || accountId}` ) + const autoProtectionDisabled = + account?.disableAutoProtection === true || account?.disableAutoProtection === 'true' + if (response.status === 401) { - ccrAccountService.markAccountUnauthorized(accountId) + if (!autoProtectionDisabled) { + upstreamErrorHelper.markTempUnavailable(accountId, 'ccr', 401).catch(() => {}) + } } else if (response.status === 429) { ccrAccountService.markAccountRateLimited(accountId) + if (!autoProtectionDisabled) { + upstreamErrorHelper + .markTempUnavailable( + accountId, + 'ccr', + 429, + upstreamErrorHelper.parseRetryAfter(response.headers) + ) + .catch(() => {}) + } // 检查是否因为超过每日额度 ccrAccountService.checkQuotaUsage(accountId).catch((err) => { logger.error('❌ Failed to check quota after 429 error:', err) }) } else if (response.status === 529) { ccrAccountService.markAccountOverloaded(accountId) + if (!autoProtectionDisabled) { + upstreamErrorHelper.markTempUnavailable(accountId, 'ccr', 529).catch(() => {}) + } + } else if (response.status >= 500) { + if (!autoProtectionDisabled) { + upstreamErrorHelper + .markTempUnavailable(accountId, 'ccr', response.status) + .catch(() => {}) + } } // 设置错误响应的状态码和响应头 @@ -885,7 +957,7 @@ class CcrRelayService { // ⏰ 更新账户最后使用时间 async _updateLastUsedTime(accountId) { try { - const redis = require('../models/redis') + const redis = require('../../models/redis') const client = redis.getClientSafe() await client.hset(`ccr_account:${accountId}`, 'lastUsedAt', new Date().toISOString()) } catch (error) { diff --git a/src/services/claudeConsoleRelayService.js b/src/services/relay/claudeConsoleRelayService.js similarity index 93% rename from src/services/claudeConsoleRelayService.js rename to src/services/relay/claudeConsoleRelayService.js index 31e8af83..d7dcabc1 100644 --- a/src/services/claudeConsoleRelayService.js +++ b/src/services/relay/claudeConsoleRelayService.js @@ -1,17 +1,18 @@ const axios = require('axios') const { v4: uuidv4 } = require('uuid') -const claudeConsoleAccountService = require('./claudeConsoleAccountService') -const redis = require('../models/redis') -const logger = require('../utils/logger') -const config = require('../../config/config') +const claudeConsoleAccountService = require('../account/claudeConsoleAccountService') +const redis = require('../../models/redis') +const logger = require('../../utils/logger') +const config = require('../../../config/config') const { sanitizeUpstreamError, sanitizeErrorMessage, isAccountDisabledError -} = require('../utils/errorSanitizer') -const userMessageQueueService = require('./userMessageQueueService') -const { isStreamWritable } = require('../utils/streamHelper') -const { filterForClaude } = require('../utils/headerFilter') +} = require('../../utils/errorSanitizer') +const upstreamErrorHelper = require('../../utils/upstreamErrorHelper') +const userMessageQueueService = require('../userMessageQueueService') +const { isStreamWritable } = require('../../utils/streamHelper') +const { filterForClaude } = require('../../utils/headerFilter') class ClaudeConsoleRelayService { constructor() { @@ -334,7 +335,9 @@ class ClaudeConsoleRelayService { `🚫 Unauthorized error detected for Claude Console account ${accountId}${autoProtectionDisabled ? ' (auto-protection disabled, skipping status change)' : ''}` ) if (!autoProtectionDisabled) { - await claudeConsoleAccountService.markAccountUnauthorized(accountId) + await upstreamErrorHelper + .markTempUnavailable(accountId, 'claude-console', 401) + .catch(() => {}) } } else if (accountDisabledError) { logger.error( @@ -357,6 +360,14 @@ class ClaudeConsoleRelayService { if (!autoProtectionDisabled) { await claudeConsoleAccountService.markAccountRateLimited(accountId) + await upstreamErrorHelper + .markTempUnavailable( + accountId, + 'claude-console', + 429, + upstreamErrorHelper.parseRetryAfter(response.headers) + ) + .catch(() => {}) } } else if (response.status === 529) { logger.warn( @@ -364,6 +375,18 @@ class ClaudeConsoleRelayService { ) if (!autoProtectionDisabled) { await claudeConsoleAccountService.markAccountOverloaded(accountId) + await upstreamErrorHelper + .markTempUnavailable(accountId, 'claude-console', 529) + .catch(() => {}) + } + } else if (response.status >= 500) { + logger.warn( + `🔥 Server error (${response.status}) detected for Claude Console account ${accountId}${autoProtectionDisabled ? ' (auto-protection disabled, skipping status change)' : ''}` + ) + if (!autoProtectionDisabled) { + await upstreamErrorHelper + .markTempUnavailable(accountId, 'claude-console', response.status) + .catch(() => {}) } } else if (response.status === 200 || response.status === 201) { // 如果请求成功，检查并移除错误状态 @@ -831,7 +854,9 @@ class ClaudeConsoleRelayService { `🚫 [Stream] Unauthorized error detected for Claude Console account ${accountId}${autoProtectionDisabled ? ' (auto-protection disabled, skipping status change)' : ''}` ) if (!autoProtectionDisabled) { - await claudeConsoleAccountService.markAccountUnauthorized(accountId) + await upstreamErrorHelper + .markTempUnavailable(accountId, 'claude-console', 401) + .catch(() => {}) } } else if (accountDisabledError) { logger.error( @@ -854,6 +879,14 @@ class ClaudeConsoleRelayService { }) if (!autoProtectionDisabled) { await claudeConsoleAccountService.markAccountRateLimited(accountId) + await upstreamErrorHelper + .markTempUnavailable( + accountId, + 'claude-console', + 429, + upstreamErrorHelper.parseRetryAfter(response.headers) + ) + .catch(() => {}) } } else if (response.status === 529) { logger.warn( @@ -861,6 +894,18 @@ class ClaudeConsoleRelayService { ) if (!autoProtectionDisabled) { await claudeConsoleAccountService.markAccountOverloaded(accountId) + await upstreamErrorHelper + .markTempUnavailable(accountId, 'claude-console', 529) + .catch(() => {}) + } + } else if (response.status >= 500) { + logger.warn( + `🔥 [Stream] Server error (${response.status}) detected for Claude Console account ${accountId}${autoProtectionDisabled ? ' (auto-protection disabled, skipping status change)' : ''}` + ) + if (!autoProtectionDisabled) { + await upstreamErrorHelper + .markTempUnavailable(accountId, 'claude-console', response.status) + .catch(() => {}) } } @@ -1246,16 +1291,37 @@ class ClaudeConsoleRelayService { // 检查错误状态 if (error.response) { + const catchAutoProtectionDisabled = + account?.disableAutoProtection === true || account?.disableAutoProtection === 'true' if (error.response.status === 401) { - claudeConsoleAccountService.markAccountUnauthorized(accountId) + if (!catchAutoProtectionDisabled) { + upstreamErrorHelper + .markTempUnavailable(accountId, 'claude-console', 401) + .catch(() => {}) + } } else if (error.response.status === 429) { - claudeConsoleAccountService.markAccountRateLimited(accountId) - // 检查是否因为超过每日额度 - claudeConsoleAccountService.checkQuotaUsage(accountId).catch((err) => { - logger.error('❌ Failed to check quota after 429 error:', err) - }) + if (!catchAutoProtectionDisabled) { + claudeConsoleAccountService.markAccountRateLimited(accountId) + // 检查是否因为超过每日额度 + claudeConsoleAccountService.checkQuotaUsage(accountId).catch((err) => { + logger.error('❌ Failed to check quota after 429 error:', err) + }) + upstreamErrorHelper + .markTempUnavailable( + accountId, + 'claude-console', + 429, + upstreamErrorHelper.parseRetryAfter(error.response.headers) + ) + .catch(() => {}) + } } else if (error.response.status === 529) { - claudeConsoleAccountService.markAccountOverloaded(accountId) + if (!catchAutoProtectionDisabled) { + claudeConsoleAccountService.markAccountOverloaded(accountId) + upstreamErrorHelper + .markTempUnavailable(accountId, 'claude-console', 529) + .catch(() => {}) + } } } @@ -1311,7 +1377,7 @@ class ClaudeConsoleRelayService { // 🕐 更新最后使用时间 async _updateLastUsedTime(accountId) { try { - const client = require('../models/redis').getClientSafe() + const client = require('../../models/redis').getClientSafe() const accountKey = `claude_console_account:${accountId}` const exists = await client.exists(accountKey) @@ -1390,7 +1456,7 @@ class ClaudeConsoleRelayService { // 🧪 测试账号连接（供Admin API使用） async testAccountConnection(accountId, responseStream) { - const { sendStreamTestRequest } = require('../utils/testPayloadHelper') + const { sendStreamTestRequest } = require('../../utils/testPayloadHelper') try { const account = await claudeConsoleAccountService.getAccount(accountId) diff --git a/src/services/claudeRelayService.js b/src/services/relay/claudeRelayService.js similarity index 97% rename from src/services/claudeRelayService.js rename to src/services/relay/claudeRelayService.js index cab37ee3..11a0bbad 100644 --- a/src/services/claudeRelayService.js +++ b/src/services/relay/claudeRelayService.js @@ -1,26 +1,27 @@ const https = require('https') const zlib = require('zlib') const path = require('path') -const ProxyHelper = require('../utils/proxyHelper') -const { filterForClaude } = require('../utils/headerFilter') -const claudeAccountService = require('./claudeAccountService') -const unifiedClaudeScheduler = require('./unifiedClaudeScheduler') -const sessionHelper = require('../utils/sessionHelper') -const logger = require('../utils/logger') -const config = require('../../config/config') -const claudeCodeHeadersService = require('./claudeCodeHeadersService') -const redis = require('../models/redis') -const ClaudeCodeValidator = require('../validators/clients/claudeCodeValidator') -const { formatDateWithTimezone } = require('../utils/dateHelper') -const requestIdentityService = require('./requestIdentityService') -const { createClaudeTestPayload } = require('../utils/testPayloadHelper') -const userMessageQueueService = require('./userMessageQueueService') -const { isStreamWritable } = require('../utils/streamHelper') +const ProxyHelper = require('../../utils/proxyHelper') +const { filterForClaude } = require('../../utils/headerFilter') +const claudeAccountService = require('../account/claudeAccountService') +const unifiedClaudeScheduler = require('../scheduler/unifiedClaudeScheduler') +const sessionHelper = require('../../utils/sessionHelper') +const logger = require('../../utils/logger') +const config = require('../../../config/config') +const claudeCodeHeadersService = require('../claudeCodeHeadersService') +const redis = require('../../models/redis') +const ClaudeCodeValidator = require('../../validators/clients/claudeCodeValidator') +const { formatDateWithTimezone } = require('../../utils/dateHelper') +const requestIdentityService = require('../requestIdentityService') +const { createClaudeTestPayload } = require('../../utils/testPayloadHelper') +const userMessageQueueService = require('../userMessageQueueService') +const { isStreamWritable } = require('../../utils/streamHelper') +const upstreamErrorHelper = require('../../utils/upstreamErrorHelper') const { getHttpsAgentForStream, getHttpsAgentForNonStream, getPricingData -} = require('../utils/performanceOptimizer') +} = require('../../utils/performanceOptimizer') // structuredClone polyfill for Node < 17 const safeClone = @@ -693,22 +694,26 @@ class ClaudeRelayService { if (errorCount >= 1) { logger.error( - `❌ Account ${accountId} encountered 401 error (${errorCount} errors), marking as unauthorized` - ) - await unifiedClaudeScheduler.markAccountUnauthorized( - accountId, - accountType, - sessionHash + `❌ Account ${accountId} encountered 401 error (${errorCount} errors), temporarily pausing` ) } + await upstreamErrorHelper.markTempUnavailable(accountId, accountType, 401).catch(() => {}) + // 清除粘性会话，让后续请求路由到其他账户 + if (sessionHash) { + await unifiedClaudeScheduler.clearSessionMapping(sessionHash).catch(() => {}) + } } // 检查是否为403状态码（禁止访问） // 注意：如果进行了重试，retryCount > 0；这里的 403 是重试后最终的结果 else if (response.statusCode === 403) { logger.error( - `🚫 Forbidden error (403) detected for account ${accountId}${retryCount > 0 ? ` after ${retryCount} retries` : ''}, marking as blocked` + `🚫 Forbidden error (403) detected for account ${accountId}${retryCount > 0 ? ` after ${retryCount} retries` : ''}, temporarily pausing` ) - await unifiedClaudeScheduler.markAccountBlocked(accountId, accountType, sessionHash) + await upstreamErrorHelper.markTempUnavailable(accountId, accountType, 403).catch(() => {}) + // 清除粘性会话，让后续请求路由到其他账户 + if (sessionHash) { + await unifiedClaudeScheduler.clearSessionMapping(sessionHash).catch(() => {}) + } } // 检查是否返回组织被禁用错误（400状态码） else if (organizationDisabledError) { @@ -734,6 +739,7 @@ class ClaudeRelayService { } else { logger.info(`🚫 529 error handling is disabled, skipping account overload marking`) } + await upstreamErrorHelper.markTempUnavailable(accountId, accountType, 529).catch(() => {}) } // 检查是否为5xx状态码 else if (response.statusCode >= 500 && response.statusCode < 600) { @@ -819,6 +825,14 @@ class ClaudeRelayService { sessionHash, rateLimitResetTimestamp ) + await upstreamErrorHelper + .markTempUnavailable( + accountId, + accountType, + 429, + upstreamErrorHelper.parseRetryAfter(response.headers) + ) + .catch(() => {}) if (dedicatedRateLimitMessage) { return { @@ -1935,6 +1949,14 @@ class ClaudeRelayService { sessionHash, rateLimitResetTimestamp ) + await upstreamErrorHelper + .markTempUnavailable( + accountId, + accountType, + 429, + upstreamErrorHelper.parseRetryAfter(res.headers) + ) + .catch(() => {}) logger.warn(`🚫 [Stream] Rate limit detected for account ${accountId}, status 429`) if (isDedicatedOfficialAccount) { @@ -2032,21 +2054,29 @@ class ClaudeRelayService { if (errorCount >= 1) { logger.error( - `❌ [Stream] Account ${accountId} encountered 401 error (${errorCount} errors), marking as unauthorized` - ) - await unifiedClaudeScheduler.markAccountUnauthorized( - accountId, - accountType, - sessionHash + `❌ [Stream] Account ${accountId} encountered 401 error (${errorCount} errors), temporarily pausing` ) } + await upstreamErrorHelper + .markTempUnavailable(accountId, accountType, 401) + .catch(() => {}) + // 清除粘性会话，让后续请求路由到其他账户 + if (sessionHash) { + await unifiedClaudeScheduler.clearSessionMapping(sessionHash).catch(() => {}) + } } else if (res.statusCode === 403) { // 403 处理：走到这里说明重试已用尽或不适用重试，直接标记 blocked // 注意：重试逻辑已在 handleErrorResponse 外部提前处理 logger.error( - `🚫 [Stream] Forbidden error (403) detected for account ${accountId}${retryCount > 0 ? ` after ${retryCount} retries` : ''}, marking as blocked` + `🚫 [Stream] Forbidden error (403) detected for account ${accountId}${retryCount > 0 ? ` after ${retryCount} retries` : ''}, temporarily pausing` ) - await unifiedClaudeScheduler.markAccountBlocked(accountId, accountType, sessionHash) + await upstreamErrorHelper + .markTempUnavailable(accountId, accountType, 403) + .catch(() => {}) + // 清除粘性会话，让后续请求路由到其他账户 + if (sessionHash) { + await unifiedClaudeScheduler.clearSessionMapping(sessionHash).catch(() => {}) + } } else if (res.statusCode === 529) { logger.warn(`🚫 [Stream] Overload error (529) detected for account ${accountId}`) @@ -2068,6 +2098,9 @@ class ClaudeRelayService { `🚫 [Stream] 529 error handling is disabled, skipping account overload marking` ) } + await upstreamErrorHelper + .markTempUnavailable(accountId, accountType, 529) + .catch(() => {}) } else if (res.statusCode >= 500 && res.statusCode < 600) { logger.warn( `🔥 [Stream] Server error (${res.statusCode}) detected for account ${accountId}` @@ -2506,6 +2539,14 @@ class ClaudeRelayService { sessionHash, rateLimitResetTimestamp ) + await upstreamErrorHelper + .markTempUnavailable( + accountId, + accountType, + 429, + upstreamErrorHelper.parseRetryAfter(res.headers) + ) + .catch(() => {}) } } else if (res.statusCode === 200) { // 请求成功，清除401和500错误计数 diff --git a/src/services/droidRelayService.js b/src/services/relay/droidRelayService.js similarity index 96% rename from src/services/droidRelayService.js rename to src/services/relay/droidRelayService.js index 8e663611..6b16aafd 100644 --- a/src/services/droidRelayService.js +++ b/src/services/relay/droidRelayService.js @@ -1,13 +1,14 @@ const https = require('https') const axios = require('axios') -const ProxyHelper = require('../utils/proxyHelper') -const droidScheduler = require('./droidScheduler') -const droidAccountService = require('./droidAccountService') -const apiKeyService = require('./apiKeyService') -const redis = require('../models/redis') -const { updateRateLimitCounters } = require('../utils/rateLimitHelper') -const logger = require('../utils/logger') -const runtimeAddon = require('../utils/runtimeAddon') +const ProxyHelper = require('../../utils/proxyHelper') +const droidScheduler = require('../scheduler/droidScheduler') +const droidAccountService = require('../account/droidAccountService') +const apiKeyService = require('../apiKeyService') +const redis = require('../../models/redis') +const { updateRateLimitCounters } = require('../../utils/rateLimitHelper') +const logger = require('../../utils/logger') +const runtimeAddon = require('../../utils/runtimeAddon') +const upstreamErrorHelper = require('../../utils/upstreamErrorHelper') const SYSTEM_PROMPT = 'You are Droid, an AI software engineering agent built by Factory.' const RUNTIME_EVENT_FMT_PAYLOAD = 'fmtPayload' @@ -346,6 +347,21 @@ class DroidRelayService { } const status = error?.response?.status + const droidAutoProtectionDisabled = + account?.disableAutoProtection === true || account?.disableAutoProtection === 'true' + // 5xx 错误 + if (status >= 500 && account?.id && !droidAutoProtectionDisabled) { + await upstreamErrorHelper.markTempUnavailable(account.id, 'droid', status).catch(() => {}) + } else if ( + !status && + account?.id && + error.message !== 'Client disconnected' && + !droidAutoProtectionDisabled + ) { + // 网络错误（非客户端断开），临时不可用 + await upstreamErrorHelper.markTempUnavailable(account.id, 'droid', 503).catch(() => {}) + } + if (status >= 400 && status < 500) { try { await this._handleUpstreamClientError(status, { @@ -518,6 +534,15 @@ class DroidRelayService { logger.info('✅ res.end() reached') const body = Buffer.concat(chunks).toString() logger.error(`❌ Factory.ai error response body: ${body || '(empty)'}`) + if (res.statusCode >= 500) { + const streamAutoProtectionDisabled = + account?.disableAutoProtection === true || account?.disableAutoProtection === 'true' + if (!streamAutoProtectionDisabled) { + upstreamErrorHelper + .markTempUnavailable(account.id, 'droid', res.statusCode) + .catch(() => {}) + } + } if (res.statusCode >= 400 && res.statusCode < 500) { this._handleUpstreamClientError(res.statusCode, { account, @@ -1380,7 +1405,11 @@ class DroidRelayService { return } - await this._stopDroidAccountScheduling(accountId, statusCode, '凭证不可用') + const clientErrorAutoProtectionDisabled = + account?.disableAutoProtection === true || account?.disableAutoProtection === 'true' + if (!clientErrorAutoProtectionDisabled) { + await upstreamErrorHelper.markTempUnavailable(accountId, 'droid', statusCode) + } await this._clearAccountStickyMapping(normalizedEndpoint, sessionHash, clientApiKeyId) } diff --git a/src/services/geminiRelayService.js b/src/services/relay/geminiRelayService.js similarity index 98% rename from src/services/geminiRelayService.js rename to src/services/relay/geminiRelayService.js index 8a4aad03..e68cda1f 100644 --- a/src/services/geminiRelayService.js +++ b/src/services/relay/geminiRelayService.js @@ -1,8 +1,8 @@ const axios = require('axios') -const ProxyHelper = require('../utils/proxyHelper') -const logger = require('../utils/logger') -const config = require('../../config/config') -const apiKeyService = require('./apiKeyService') +const ProxyHelper = require('../../utils/proxyHelper') +const logger = require('../../utils/logger') +const config = require('../../../config/config') +const apiKeyService = require('../apiKeyService') // Gemini API 配置 const GEMINI_API_BASE = 'https://cloudcode.googleapis.com/v1' diff --git a/src/services/openaiResponsesRelayService.js b/src/services/relay/openaiResponsesRelayService.js similarity index 87% rename from src/services/openaiResponsesRelayService.js rename to src/services/relay/openaiResponsesRelayService.js index 23711718..1cb06e13 100644 --- a/src/services/openaiResponsesRelayService.js +++ b/src/services/relay/openaiResponsesRelayService.js @@ -1,13 +1,14 @@ const axios = require('axios') -const ProxyHelper = require('../utils/proxyHelper') -const logger = require('../utils/logger') -const { filterForOpenAI } = require('../utils/headerFilter') -const openaiResponsesAccountService = require('./openaiResponsesAccountService') -const apiKeyService = require('./apiKeyService') -const unifiedOpenAIScheduler = require('./unifiedOpenAIScheduler') -const config = require('../../config/config') +const ProxyHelper = require('../../utils/proxyHelper') +const logger = require('../../utils/logger') +const { filterForOpenAI } = require('../../utils/headerFilter') +const openaiResponsesAccountService = require('../account/openaiResponsesAccountService') +const apiKeyService = require('../apiKeyService') +const unifiedOpenAIScheduler = require('../scheduler/unifiedOpenAIScheduler') +const config = require('../../../config/config') const crypto = require('crypto') -const LRUCache = require('../utils/lruCache') +const LRUCache = require('../../utils/lruCache') +const upstreamErrorHelper = require('../../utils/upstreamErrorHelper') // lastUsedAt 更新节流（每账户 60 秒内最多更新一次，使用 LRU 防止内存泄漏） const lastUsedAtThrottle = new LRUCache(1000) // 最多缓存 1000 个账户 @@ -160,6 +161,19 @@ class OpenAIResponsesRelayService { sessionHash ) + const oaiAutoProtectionDisabled = + account?.disableAutoProtection === true || account?.disableAutoProtection === 'true' + if (!oaiAutoProtectionDisabled) { + await upstreamErrorHelper + .markTempUnavailable( + account.id, + 'openai-responses', + 429, + resetsInSeconds || upstreamErrorHelper.parseRetryAfter(response.headers) + ) + .catch(() => {}) + } + // 返回错误响应（使用处理后的数据，避免循环引用） const errorResponse = errorData || { error: { @@ -218,31 +232,23 @@ class OpenAIResponsesRelayService { }) if (response.status === 401) { - let reason = 'OpenAI Responses账号认证失败（401错误）' - if (errorData) { - if (typeof errorData === 'string' && errorData.trim()) { - reason = `OpenAI Responses账号认证失败（401错误）：${errorData.trim()}` - } else if ( - errorData.error && - typeof errorData.error.message === 'string' && - errorData.error.message.trim() - ) { - reason = `OpenAI Responses账号认证失败（401错误）：${errorData.error.message.trim()}` - } else if (typeof errorData.message === 'string' && errorData.message.trim()) { - reason = `OpenAI Responses账号认证失败（401错误）：${errorData.message.trim()}` - } - } + logger.warn(`🚫 OpenAI Responses账号认证失败（401错误）for account ${account?.id}`) try { - await unifiedOpenAIScheduler.markAccountUnauthorized( - account.id, - 'openai-responses', - sessionHash, - reason - ) + // 仅临时暂停，不永久禁用 + const oaiAutoProtectionDisabled = + account?.disableAutoProtection === true || account?.disableAutoProtection === 'true' + if (!oaiAutoProtectionDisabled) { + await upstreamErrorHelper + .markTempUnavailable(account.id, 'openai-responses', 401) + .catch(() => {}) + } + if (sessionHash) { + await unifiedOpenAIScheduler._deleteSessionMapping(sessionHash).catch(() => {}) + } } catch (markError) { logger.error( - '❌ Failed to mark OpenAI-Responses account unauthorized after 401:', + '❌ Failed to mark OpenAI-Responses account temporarily unavailable after 401:', markError ) } @@ -272,11 +278,36 @@ class OpenAIResponsesRelayService { return res.status(401).json(unauthorizedResponse) } + // 处理 5xx 上游错误 + if (response.status >= 500 && account?.id) { + try { + const oaiAutoProtectionDisabled = + account?.disableAutoProtection === true || account?.disableAutoProtection === 'true' + if (!oaiAutoProtectionDisabled) { + await upstreamErrorHelper.markTempUnavailable( + account.id, + 'openai-responses', + response.status + ) + } + if (sessionHash) { + await unifiedOpenAIScheduler._deleteSessionMapping(sessionHash).catch(() => {}) + } + } catch (markError) { + logger.warn( + 'Failed to mark OpenAI-Responses account temporarily unavailable:', + markError + ) + } + } + // 清理监听器 req.removeListener('close', handleClientDisconnect) res.removeListener('close', handleClientDisconnect) - return res.status(response.status).json(errorData) + return res + .status(response.status) + .json(upstreamErrorHelper.sanitizeErrorForClient(errorData)) } // 更新最后使用时间（节流） @@ -314,10 +345,15 @@ class OpenAIResponsesRelayService { // 检查是否是网络错误 if (error.code === 'ECONNREFUSED' || error.code === 'ETIMEDOUT') { - await openaiResponsesAccountService.updateAccount(account.id, { - status: 'error', - errorMessage: `Connection error: ${error.code}` - }) + if (account?.id) { + const oaiAutoProtectionDisabled = + account?.disableAutoProtection === true || account?.disableAutoProtection === 'true' + if (!oaiAutoProtectionDisabled) { + await upstreamErrorHelper + .markTempUnavailable(account.id, 'openai-responses', 503) + .catch(() => {}) + } + } } // 如果已经发送了响应头，直接结束 @@ -352,31 +388,25 @@ class OpenAIResponsesRelayService { } if (status === 401) { - let reason = 'OpenAI Responses账号认证失败（401错误）' - if (errorData) { - if (typeof errorData === 'string' && errorData.trim()) { - reason = `OpenAI Responses账号认证失败（401错误）：${errorData.trim()}` - } else if ( - errorData.error && - typeof errorData.error.message === 'string' && - errorData.error.message.trim() - ) { - reason = `OpenAI Responses账号认证失败（401错误）：${errorData.error.message.trim()}` - } else if (typeof errorData.message === 'string' && errorData.message.trim()) { - reason = `OpenAI Responses账号认证失败（401错误）：${errorData.message.trim()}` - } - } + logger.warn( + `🚫 OpenAI Responses账号认证失败（401错误）for account ${account?.id} (catch handler)` + ) try { - await unifiedOpenAIScheduler.markAccountUnauthorized( - account.id, - 'openai-responses', - sessionHash, - reason - ) + // 仅临时暂停，不永久禁用 + const oaiAutoProtectionDisabled = + account?.disableAutoProtection === true || account?.disableAutoProtection === 'true' + if (!oaiAutoProtectionDisabled) { + await upstreamErrorHelper + .markTempUnavailable(account.id, 'openai-responses', 401) + .catch(() => {}) + } + if (sessionHash) { + await unifiedOpenAIScheduler._deleteSessionMapping(sessionHash).catch(() => {}) + } } catch (markError) { logger.error( - '❌ Failed to mark OpenAI-Responses account unauthorized in catch handler:', + '❌ Failed to mark OpenAI-Responses account temporarily unavailable in catch handler:', markError ) } @@ -402,7 +432,7 @@ class OpenAIResponsesRelayService { return res.status(401).json(unauthorizedResponse) } - return res.status(status).json(errorData) + return res.status(status).json(upstreamErrorHelper.sanitizeErrorForClient(errorData)) } // 其他错误 @@ -571,7 +601,7 @@ class OpenAIResponsesRelayService { // 更新账户使用额度（如果设置了额度限制） if (parseFloat(account.dailyQuota) > 0) { // 使用CostCalculator正确计算费用（考虑缓存token的不同价格） - const CostCalculator = require('../utils/costCalculator') + const CostCalculator = require('../../utils/costCalculator') const costInfo = CostCalculator.calculateCost( { input_tokens: actualInputTokens, // 实际输入（不含缓存） @@ -700,7 +730,7 @@ class OpenAIResponsesRelayService { // 更新账户使用额度（如果设置了额度限制） if (parseFloat(account.dailyQuota) > 0) { // 使用CostCalculator正确计算费用（考虑缓存token的不同价格） - const CostCalculator = require('../utils/costCalculator') + const CostCalculator = require('../../utils/costCalculator') const costInfo = CostCalculator.calculateCost( { input_tokens: actualInputTokens, // 实际输入（不含缓存） diff --git a/src/services/droidScheduler.js b/src/services/scheduler/droidScheduler.js similarity index 72% rename from src/services/droidScheduler.js rename to src/services/scheduler/droidScheduler.js index 14adcd40..ba6796aa 100644 --- a/src/services/droidScheduler.js +++ b/src/services/scheduler/droidScheduler.js @@ -1,13 +1,14 @@ -const droidAccountService = require('./droidAccountService') -const accountGroupService = require('./accountGroupService') -const redis = require('../models/redis') -const logger = require('../utils/logger') +const droidAccountService = require('../account/droidAccountService') +const accountGroupService = require('../accountGroupService') +const redis = require('../../models/redis') +const logger = require('../../utils/logger') +const upstreamErrorHelper = require('../../utils/upstreamErrorHelper') const { isTruthy, isAccountHealthy, sortAccountsByPriority, normalizeEndpointType -} = require('../utils/commonHelper') +} = require('../../utils/commonHelper') class DroidScheduler { constructor() { @@ -57,9 +58,21 @@ class DroidScheduler { }) ) - return accounts.filter( - (account) => account && isAccountHealthy(account) && this._isAccountSchedulable(account) - ) + const result = [] + for (const account of accounts) { + if (!account || !isAccountHealthy(account) || !this._isAccountSchedulable(account)) { + continue + } + const isTempUnavailable = await upstreamErrorHelper.isTempUnavailable(account.id, 'droid') + if (isTempUnavailable) { + logger.debug( + `⏭️ Skipping Droid group member ${account.name || account.id} - temporarily unavailable` + ) + continue + } + result.push(account) + } + return result } async _ensureLastUsedUpdated(accountId) { @@ -99,8 +112,15 @@ class DroidScheduler { } else { const account = await droidAccountService.getAccount(binding) if (account) { - candidates = [account] - isDedicatedBinding = true + const isTempUnavailable = await upstreamErrorHelper.isTempUnavailable(account.id, 'droid') + if (isTempUnavailable) { + logger.warn( + `⏱️ Bound Droid account ${account.name || account.id} temporarily unavailable, falling back to pool` + ) + } else { + candidates = [account] + isDedicatedBinding = true + } } } } @@ -109,13 +129,26 @@ class DroidScheduler { candidates = await droidAccountService.getSchedulableAccounts(normalizedEndpoint) } - const filtered = candidates.filter( + const syncFiltered = candidates.filter( (account) => account && isAccountHealthy(account) && this._isAccountSchedulable(account) && this._matchesEndpoint(account, normalizedEndpoint) ) + const filteredResults = await Promise.all( + syncFiltered.map(async (account) => { + const isTempUnavailable = await upstreamErrorHelper.isTempUnavailable(account.id, 'droid') + if (isTempUnavailable) { + logger.debug( + `⏭️ Skipping Droid account ${account.name || account.id} - temporarily unavailable` + ) + return null + } + return account + }) + ) + const filtered = filteredResults.filter(Boolean) if (filtered.length === 0) { throw new Error( diff --git a/src/services/unifiedClaudeScheduler.js b/src/services/scheduler/unifiedClaudeScheduler.js similarity index 98% rename from src/services/unifiedClaudeScheduler.js rename to src/services/scheduler/unifiedClaudeScheduler.js index f56e4e7c..cb9c819c 100644 --- a/src/services/unifiedClaudeScheduler.js +++ b/src/services/scheduler/unifiedClaudeScheduler.js @@ -1,12 +1,13 @@ -const claudeAccountService = require('./claudeAccountService') -const claudeConsoleAccountService = require('./claudeConsoleAccountService') -const bedrockAccountService = require('./bedrockAccountService') -const ccrAccountService = require('./ccrAccountService') -const accountGroupService = require('./accountGroupService') -const redis = require('../models/redis') -const logger = require('../utils/logger') -const { parseVendorPrefixedModel, isOpus45OrNewer } = require('../utils/modelHelper') -const { isSchedulable, sortAccountsByPriority } = require('../utils/commonHelper') +const claudeAccountService = require('../account/claudeAccountService') +const claudeConsoleAccountService = require('../account/claudeConsoleAccountService') +const bedrockAccountService = require('../account/bedrockAccountService') +const ccrAccountService = require('../account/ccrAccountService') +const accountGroupService = require('../accountGroupService') +const redis = require('../../models/redis') +const logger = require('../../utils/logger') +const { parseVendorPrefixedModel, isOpus45OrNewer } = require('../../utils/modelHelper') +const { isSchedulable, sortAccountsByPriority } = require('../../utils/commonHelper') +const upstreamErrorHelper = require('../../utils/upstreamErrorHelper') /** * Check if account is Pro (not Max) @@ -1175,7 +1176,7 @@ class UnifiedClaudeScheduler { const client = redis.getClientSafe() const mappingData = JSON.stringify({ accountId, accountType }) // 依据配置设置TTL（小时） - const appConfig = require('../../config/config') + const appConfig = require('../../../config/config') const ttlHours = appConfig.session?.stickyTtlHours || 1 const ttlSeconds = Math.max(1, Math.floor(ttlHours * 60 * 60)) await client.setex(`${this.SESSION_MAPPING_PREFIX}${sessionHash}`, ttlSeconds, mappingData) @@ -1224,7 +1225,7 @@ class UnifiedClaudeScheduler { return true } - const appConfig = require('../../config/config') + const appConfig = require('../../../config/config') const ttlHours = appConfig.session?.stickyTtlHours || 1 const renewalThresholdMinutes = appConfig.session?.renewalThresholdMinutes || 0 @@ -1261,15 +1262,10 @@ class UnifiedClaudeScheduler { ttlSeconds = 300 ) { try { - const client = redis.getClientSafe() - const key = `temp_unavailable:${accountType}:${accountId}` - await client.setex(key, ttlSeconds, '1') + await upstreamErrorHelper.markTempUnavailable(accountId, accountType, 500, ttlSeconds) if (sessionHash) { await this._deleteSessionMapping(sessionHash) } - logger.warn( - `⏱️ Account ${accountId} (${accountType}) marked temporarily unavailable for ${ttlSeconds}s` - ) return { success: true } } catch (error) { logger.error(`❌ Failed to mark account temporarily unavailable: ${accountId}`, error) @@ -1279,14 +1275,7 @@ class UnifiedClaudeScheduler { // 🔍 检查账户是否临时不可用 async isAccountTemporarilyUnavailable(accountId, accountType) { - try { - const client = redis.getClientSafe() - const key = `temp_unavailable:${accountType}:${accountId}` - return (await client.exists(key)) === 1 - } catch (error) { - logger.error(`❌ Failed to check temp unavailable status: ${accountId}`, error) - return false - } + return upstreamErrorHelper.isTempUnavailable(accountId, accountType) } // 🚫 标记账户为限流状态 diff --git a/src/services/unifiedGeminiScheduler.js b/src/services/scheduler/unifiedGeminiScheduler.js similarity index 91% rename from src/services/unifiedGeminiScheduler.js rename to src/services/scheduler/unifiedGeminiScheduler.js index 7a1f65c6..80f1bc74 100644 --- a/src/services/unifiedGeminiScheduler.js +++ b/src/services/scheduler/unifiedGeminiScheduler.js @@ -1,9 +1,10 @@ -const geminiAccountService = require('./geminiAccountService') -const geminiApiAccountService = require('./geminiApiAccountService') -const accountGroupService = require('./accountGroupService') -const redis = require('../models/redis') -const logger = require('../utils/logger') -const { isSchedulable, isActive, sortAccountsByPriority } = require('../utils/commonHelper') +const geminiAccountService = require('../account/geminiAccountService') +const geminiApiAccountService = require('../account/geminiApiAccountService') +const accountGroupService = require('../accountGroupService') +const redis = require('../../models/redis') +const logger = require('../../utils/logger') +const { isSchedulable, isActive, sortAccountsByPriority } = require('../../utils/commonHelper') +const upstreamErrorHelper = require('../../utils/upstreamErrorHelper') const OAUTH_PROVIDER_GEMINI_CLI = 'gemini-cli' const OAUTH_PROVIDER_ANTIGRAVITY = 'antigravity' @@ -241,8 +242,17 @@ class UnifiedGeminiScheduler { const accountId = apiKeyData.geminiAccountId.replace('api:', '') const boundAccount = await geminiApiAccountService.getAccount(accountId) if (boundAccount && isActive(boundAccount.isActive) && boundAccount.status !== 'error') { + const isTempUnavailable = await upstreamErrorHelper.isTempUnavailable( + accountId, + 'gemini-api' + ) + if (isTempUnavailable) { + logger.warn( + `⏱️ Bound Gemini-API account ${boundAccount.name} (${accountId}) temporarily unavailable, falling back to pool` + ) + } const isRateLimited = await this.isAccountRateLimited(accountId) - if (!isRateLimited) { + if (!isRateLimited && !isTempUnavailable) { // 检查模型支持 if ( requestedModel && @@ -298,8 +308,17 @@ class UnifiedGeminiScheduler { ) { return availableAccounts } + const isTempUnavailable = await upstreamErrorHelper.isTempUnavailable( + boundAccount.id, + 'gemini' + ) + if (isTempUnavailable) { + logger.warn( + `⏱️ Bound Gemini account ${boundAccount.name} (${boundAccount.id}) temporarily unavailable, falling back to pool` + ) + } const isRateLimited = await this.isAccountRateLimited(boundAccount.id) - if (!isRateLimited) { + if (!isRateLimited && !isTempUnavailable) { // 检查模型支持 if ( requestedModel && @@ -364,6 +383,13 @@ class UnifiedGeminiScheduler { continue } + // 检查临时不可用 + const isTempUnavailable = await upstreamErrorHelper.isTempUnavailable(account.id, 'gemini') + if (isTempUnavailable) { + logger.debug(`⏭️ Skipping Gemini account ${account.name} - temporarily unavailable`) + continue + } + // 检查模型支持 if (requestedModel && account.supportedModels && account.supportedModels.length > 0) { // 处理可能带有 models/ 前缀的模型名 @@ -417,6 +443,16 @@ class UnifiedGeminiScheduler { } } + // 检查临时不可用 + const isTempUnavailable = await upstreamErrorHelper.isTempUnavailable( + account.id, + 'gemini-api' + ) + if (isTempUnavailable) { + logger.debug(`⏭️ Skipping Gemini-API account ${account.name} - temporarily unavailable`) + continue + } + // 检查是否被限流 const isRateLimited = await this.isAccountRateLimited(account.id) if (!isRateLimited) { @@ -451,6 +487,14 @@ class UnifiedGeminiScheduler { logger.info(`🚫 Gemini account ${accountId} is not schedulable`) return false } + const isTempUnavailable = await upstreamErrorHelper.isTempUnavailable( + accountId, + accountType + ) + if (isTempUnavailable) { + logger.info(`⏱️ Gemini account ${accountId} is temporarily unavailable`) + return false + } return !(await this.isAccountRateLimited(accountId)) } else if (accountType === 'gemini-api') { const account = await geminiApiAccountService.getAccount(accountId) @@ -462,6 +506,14 @@ class UnifiedGeminiScheduler { logger.info(`🚫 Gemini-API account ${accountId} is not schedulable`) return false } + const isTempUnavailable = await upstreamErrorHelper.isTempUnavailable( + accountId, + accountType + ) + if (isTempUnavailable) { + logger.info(`⏱️ Gemini account ${accountId} is temporarily unavailable`) + return false + } return !(await this.isAccountRateLimited(accountId)) } return false @@ -494,7 +546,7 @@ class UnifiedGeminiScheduler { const client = redis.getClientSafe() const mappingData = JSON.stringify({ accountId, accountType }) // 依据配置设置TTL（小时） - const appConfig = require('../../config/config') + const appConfig = require('../../../config/config') const ttlHours = appConfig.session?.stickyTtlHours || 1 const ttlSeconds = Math.max(1, Math.floor(ttlHours * 60 * 60)) const key = this._getSessionMappingKey(sessionHash, oauthProvider) @@ -535,7 +587,7 @@ class UnifiedGeminiScheduler { return true } - const appConfig = require('../../config/config') + const appConfig = require('../../../config/config') const ttlHours = appConfig.session?.stickyTtlHours || 1 const renewalThresholdMinutes = appConfig.session?.renewalThresholdMinutes || 0 if (!renewalThresholdMinutes) { @@ -749,6 +801,14 @@ class UnifiedGeminiScheduler { // 检查是否被限流 const isRateLimited = await this.isAccountRateLimited(account.id, accountType) if (!isRateLimited) { + const isTempUnavailable = await upstreamErrorHelper.isTempUnavailable( + account.id, + accountType + ) + if (isTempUnavailable) { + logger.debug(`⏭️ Skipping group member ${account.name} - temporarily unavailable`) + continue + } availableAccounts.push({ ...account, accountId: account.id, diff --git a/src/services/unifiedOpenAIScheduler.js b/src/services/scheduler/unifiedOpenAIScheduler.js similarity index 83% rename from src/services/unifiedOpenAIScheduler.js rename to src/services/scheduler/unifiedOpenAIScheduler.js index 981a3aa9..45195f02 100644 --- a/src/services/unifiedOpenAIScheduler.js +++ b/src/services/scheduler/unifiedOpenAIScheduler.js @@ -1,9 +1,10 @@ -const openaiAccountService = require('./openaiAccountService') -const openaiResponsesAccountService = require('./openaiResponsesAccountService') -const accountGroupService = require('./accountGroupService') -const redis = require('../models/redis') -const logger = require('../utils/logger') -const { isSchedulable, sortAccountsByPriority } = require('../utils/commonHelper') +const openaiAccountService = require('../account/openaiAccountService') +const openaiResponsesAccountService = require('../account/openaiResponsesAccountService') +const accountGroupService = require('../accountGroupService') +const redis = require('../../models/redis') +const logger = require('../../utils/logger') +const { isSchedulable, sortAccountsByPriority } = require('../../utils/commonHelper') +const upstreamErrorHelper = require('../../utils/upstreamErrorHelper') class UnifiedOpenAIScheduler { constructor() { @@ -153,91 +154,102 @@ class UnifiedOpenAIScheduler { boundAccount.status !== 'unauthorized' if (isActiveBoundAccount) { - if (accountType === 'openai') { - const readiness = await this._ensureAccountReadyForScheduling( - boundAccount, - boundAccount.id, - { sanitized: false } - ) - - if (!readiness.canUse) { - const isRateLimited = readiness.reason === 'rate_limited' - const errorMsg = isRateLimited - ? `Dedicated account ${boundAccount.name} is currently rate limited` - : `Dedicated account ${boundAccount.name} is not schedulable` - logger.warn(`⚠️ ${errorMsg}`) - const error = new Error(errorMsg) - error.statusCode = isRateLimited ? 429 : 403 - throw error - } - } else { - const hasRateLimitFlag = this._isRateLimited(boundAccount.rateLimitStatus) - if (hasRateLimitFlag) { - const isRateLimitCleared = await openaiResponsesAccountService.checkAndClearRateLimit( - boundAccount.id - ) - if (!isRateLimitCleared) { - const errorMsg = `Dedicated account ${boundAccount.name} is currently rate limited` - logger.warn(`⚠️ ${errorMsg}`) - const error = new Error(errorMsg) - error.statusCode = 429 // Too Many Requests - 限流 - throw error - } - // 限流已解除，刷新账户最新状态，确保后续调度信息准确 - boundAccount = await openaiResponsesAccountService.getAccount(boundAccount.id) - if (!boundAccount) { - const errorMsg = `Dedicated account ${apiKeyData.openaiAccountId} not found after rate limit reset` - logger.warn(`⚠️ ${errorMsg}`) - const error = new Error(errorMsg) - error.statusCode = 404 - throw error - } - } - - if (!isSchedulable(boundAccount.schedulable)) { - const errorMsg = `Dedicated account ${boundAccount.name} is not schedulable` - logger.warn(`⚠️ ${errorMsg}`) - const error = new Error(errorMsg) - error.statusCode = 403 // Forbidden - 调度被禁止 - throw error - } - - // ⏰ 检查 OpenAI-Responses 专属账户订阅是否过期 - if (openaiResponsesAccountService.isSubscriptionExpired(boundAccount)) { - const errorMsg = `Dedicated account ${boundAccount.name} subscription has expired` - logger.warn(`⚠️ ${errorMsg}`) - const error = new Error(errorMsg) - error.statusCode = 403 // Forbidden - 订阅已过期 - throw error - } - } - - // 专属账户：可选的模型检查（只有明确配置了supportedModels且不为空才检查） - // OpenAI-Responses 账户默认支持所有模型 - if ( - accountType === 'openai' && - requestedModel && - boundAccount.supportedModels && - boundAccount.supportedModels.length > 0 - ) { - const modelSupported = boundAccount.supportedModels.includes(requestedModel) - if (!modelSupported) { - const errorMsg = `Dedicated account ${boundAccount.name} does not support model ${requestedModel}` - logger.warn(`⚠️ ${errorMsg}`) - const error = new Error(errorMsg) - error.statusCode = 400 // Bad Request - 请求参数错误 - throw error - } - } - - logger.info( - `🎯 Using bound dedicated ${accountType} account: ${boundAccount.name} (${boundAccount.id}) for API key ${apiKeyData.name}` - ) - // 更新账户的最后使用时间 - await this.updateAccountLastUsed(boundAccount.id, accountType) - return { - accountId: boundAccount.id, + // 检查是否临时不可用 + const isTempUnavailable = await upstreamErrorHelper.isTempUnavailable( + boundAccount.id, accountType + ) + if (isTempUnavailable) { + logger.warn( + `⏱️ Bound ${accountType} account ${boundAccount.name} temporarily unavailable, falling back to pool` + ) + // 不 throw，让代码继续走到共享池选择 + } else { + if (accountType === 'openai') { + const readiness = await this._ensureAccountReadyForScheduling( + boundAccount, + boundAccount.id, + { sanitized: false } + ) + + if (!readiness.canUse) { + const isRateLimited = readiness.reason === 'rate_limited' + const errorMsg = isRateLimited + ? `Dedicated account ${boundAccount.name} is currently rate limited` + : `Dedicated account ${boundAccount.name} is not schedulable` + logger.warn(`⚠️ ${errorMsg}`) + const error = new Error(errorMsg) + error.statusCode = isRateLimited ? 429 : 403 + throw error + } + } else { + const hasRateLimitFlag = this._isRateLimited(boundAccount.rateLimitStatus) + if (hasRateLimitFlag) { + const isRateLimitCleared = + await openaiResponsesAccountService.checkAndClearRateLimit(boundAccount.id) + if (!isRateLimitCleared) { + const errorMsg = `Dedicated account ${boundAccount.name} is currently rate limited` + logger.warn(`⚠️ ${errorMsg}`) + const error = new Error(errorMsg) + error.statusCode = 429 // Too Many Requests - 限流 + throw error + } + // 限流已解除，刷新账户最新状态，确保后续调度信息准确 + boundAccount = await openaiResponsesAccountService.getAccount(boundAccount.id) + if (!boundAccount) { + const errorMsg = `Dedicated account ${apiKeyData.openaiAccountId} not found after rate limit reset` + logger.warn(`⚠️ ${errorMsg}`) + const error = new Error(errorMsg) + error.statusCode = 404 + throw error + } + } + + if (!isSchedulable(boundAccount.schedulable)) { + const errorMsg = `Dedicated account ${boundAccount.name} is not schedulable` + logger.warn(`⚠️ ${errorMsg}`) + const error = new Error(errorMsg) + error.statusCode = 403 // Forbidden - 调度被禁止 + throw error + } + + // ⏰ 检查 OpenAI-Responses 专属账户订阅是否过期 + if (openaiResponsesAccountService.isSubscriptionExpired(boundAccount)) { + const errorMsg = `Dedicated account ${boundAccount.name} subscription has expired` + logger.warn(`⚠️ ${errorMsg}`) + const error = new Error(errorMsg) + error.statusCode = 403 // Forbidden - 订阅已过期 + throw error + } + } + + // 专属账户：可选的模型检查（只有明确配置了supportedModels且不为空才检查） + // OpenAI-Responses 账户默认支持所有模型 + if ( + accountType === 'openai' && + requestedModel && + boundAccount.supportedModels && + boundAccount.supportedModels.length > 0 + ) { + const modelSupported = boundAccount.supportedModels.includes(requestedModel) + if (!modelSupported) { + const errorMsg = `Dedicated account ${boundAccount.name} does not support model ${requestedModel}` + logger.warn(`⚠️ ${errorMsg}`) + const error = new Error(errorMsg) + error.statusCode = 400 // Bad Request - 请求参数错误 + throw error + } + } + + logger.info( + `🎯 Using bound dedicated ${accountType} account: ${boundAccount.name} (${boundAccount.id}) for API key ${apiKeyData.name}` + ) + // 更新账户的最后使用时间 + await this.updateAccountLastUsed(boundAccount.id, accountType) + return { + accountId: boundAccount.id, + accountType + } } } else { // 专属账户不可用时直接报错，不降级到共享池 @@ -370,6 +382,12 @@ class UnifiedOpenAIScheduler { continue } + const isTempUnavailable = await upstreamErrorHelper.isTempUnavailable(accountId, 'openai') + if (isTempUnavailable) { + logger.debug(`⏭️ Skipping openai account ${account.name} - temporarily unavailable`) + continue + } + // 检查token是否过期并自动刷新 const isExpired = openaiAccountService.isTokenExpired(account) if (isExpired) { @@ -465,6 +483,17 @@ class UnifiedOpenAIScheduler { } } + const isTempUnavailable = await upstreamErrorHelper.isTempUnavailable( + account.id, + 'openai-responses' + ) + if (isTempUnavailable) { + logger.debug( + `⏭️ Skipping openai-responses account ${account.name} - temporarily unavailable` + ) + continue + } + // ⏰ 检查订阅是否过期 if (openaiResponsesAccountService.isSubscriptionExpired(account)) { logger.debug( @@ -517,6 +546,15 @@ class UnifiedOpenAIScheduler { return false } + const isTempUnavailable = await upstreamErrorHelper.isTempUnavailable( + accountId, + accountType + ) + if (isTempUnavailable) { + logger.info(`⏱️ OpenAI account ${accountId} (${accountType}) is temporarily unavailable`) + return false + } + return true } else if (accountType === 'openai-responses') { const account = await openaiResponsesAccountService.getAccount(accountId) @@ -541,7 +579,20 @@ class UnifiedOpenAIScheduler { // 检查并清除过期的限流状态 const isRateLimitCleared = await openaiResponsesAccountService.checkAndClearRateLimit(accountId) - return !this._isRateLimited(account.rateLimitStatus) || isRateLimitCleared + if (this._isRateLimited(account.rateLimitStatus) && !isRateLimitCleared) { + return false + } + + const isTempUnavailable = await upstreamErrorHelper.isTempUnavailable( + accountId, + accountType + ) + if (isTempUnavailable) { + logger.info(`⏱️ OpenAI account ${accountId} (${accountType}) is temporarily unavailable`) + return false + } + + return true } return false } catch (error) { @@ -572,7 +623,7 @@ class UnifiedOpenAIScheduler { const client = redis.getClientSafe() const mappingData = JSON.stringify({ accountId, accountType }) // 依据配置设置TTL（小时） - const appConfig = require('../../config/config') + const appConfig = require('../../../config/config') const ttlHours = appConfig.session?.stickyTtlHours || 1 const ttlSeconds = Math.max(1, Math.floor(ttlHours * 60 * 60)) await client.setex(`${this.SESSION_MAPPING_PREFIX}${sessionHash}`, ttlSeconds, mappingData) @@ -598,7 +649,7 @@ class UnifiedOpenAIScheduler { return true } - const appConfig = require('../../config/config') + const appConfig = require('../../../config/config') const ttlHours = appConfig.session?.stickyTtlHours || 1 const renewalThresholdMinutes = appConfig.session?.renewalThresholdMinutes || 0 if (!renewalThresholdMinutes) { @@ -849,6 +900,17 @@ class UnifiedOpenAIScheduler { continue } + const isTempUnavailable = await upstreamErrorHelper.isTempUnavailable( + account.id, + accountType + ) + if (isTempUnavailable) { + logger.debug( + `⏭️ Skipping group member ${accountType} account ${account.name} - temporarily unavailable` + ) + continue + } + // 检查token是否过期（仅对 OpenAI OAuth 账户检查） if (accountType === 'openai') { const isExpired = openaiAccountService.isTokenExpired(account) diff --git a/src/utils/logger.js b/src/utils/logger.js index f0202e89..5515075f 100644 --- a/src/utils/logger.js +++ b/src/utils/logger.js @@ -7,13 +7,11 @@ const fs = require('fs') const os = require('os') // 安全的 JSON 序列化函数，处理循环引用和特殊字符 -const safeStringify = (obj, maxDepth = 3, fullDepth = false) => { +const safeStringify = (obj, maxDepth = Infinity) => { const seen = new WeakSet() - // 如果是fullDepth模式，增加深度限制 - const actualMaxDepth = fullDepth ? 10 : maxDepth const replacer = (key, value, depth = 0) => { - if (depth > actualMaxDepth) { + if (depth > maxDepth) { return '[Max Depth Reached]' } @@ -21,18 +19,13 @@ const safeStringify = (obj, maxDepth = 3, fullDepth = false) => { if (typeof value === 'string') { try { // 移除或转义可能导致JSON解析错误的字符 - let cleanValue = value + const cleanValue = value // eslint-disable-next-line no-control-regex .replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F]/g, '') // 移除控制字符 .replace(/[\uD800-\uDFFF]/g, '') // 移除孤立的代理对字符 // eslint-disable-next-line no-control-regex .replace(/\u0000/g, '') // 移除NUL字节 - // 如果字符串过长，截断并添加省略号 - if (cleanValue.length > 1000) { - cleanValue = `${cleanValue.substring(0, 997)}...` - } - return cleanValue } catch (error) { return '[Invalid String Data]' @@ -77,7 +70,37 @@ const safeStringify = (obj, maxDepth = 3, fullDepth = false) => { try { const processed = replacer('', obj) - return JSON.stringify(processed) + const result = JSON.stringify(processed) + // 体积保护: 超过 50KB 时对大字段做截断，保留顶层结构 + if (result.length > 50000 && processed && typeof processed === 'object') { + const truncated = { ...processed, _truncated: true, _totalChars: result.length } + // 第一轮: 截断单个大字段 + for (const [k, v] of Object.entries(truncated)) { + if (k.startsWith('_')) { + continue + } + const fieldStr = typeof v === 'string' ? v : JSON.stringify(v) + if (fieldStr && fieldStr.length > 10000) { + truncated[k] = `${fieldStr.substring(0, 10000)}...[truncated]` + } + } + // 第二轮: 如果总长度仍超 50KB，逐字段缩减到 2KB + let secondResult = JSON.stringify(truncated) + if (secondResult.length > 50000) { + for (const [k, v] of Object.entries(truncated)) { + if (k.startsWith('_')) { + continue + } + const fieldStr = typeof v === 'string' ? v : JSON.stringify(v) + if (fieldStr && fieldStr.length > 2000) { + truncated[k] = `${fieldStr.substring(0, 2000)}...[truncated]` + } + } + secondResult = JSON.stringify(truncated) + } + return secondResult + } + return result } catch (error) { // 如果JSON.stringify仍然失败，使用更保守的方法 try { @@ -93,50 +116,64 @@ const safeStringify = (obj, maxDepth = 3, fullDepth = false) => { } } -// 📝 增强的日志格式 -const createLogFormat = (colorize = false) => { - const formats = [ +// 控制台不显示的 metadata 字段（已在 message 中或低价值） +const CONSOLE_SKIP_KEYS = new Set(['type', 'level', 'message', 'timestamp', 'stack']) + +// 控制台格式: 树形展示 metadata +const createConsoleFormat = () => + winston.format.combine( winston.format.timestamp({ format: () => formatDateWithTimezone(new Date(), false) }), - winston.format.errors({ stack: true }) - // 移除 winston.format.metadata() 来避免自动包装 - ] - - if (colorize) { - formats.push(winston.format.colorize()) - } - - formats.push( + winston.format.errors({ stack: true }), + winston.format.colorize(), winston.format.printf(({ level, message, timestamp, stack, ...rest }) => { - const emoji = { - error: '❌', - warn: '⚠️ ', - info: 'ℹ️ ', - debug: '🐛', - verbose: '📝' + // 时间戳只取时分秒 + const shortTime = timestamp ? timestamp.split(' ').pop() : '' + + let logMessage = `${shortTime} ${message}` + + // 收集要显示的 metadata + const entries = Object.entries(rest).filter(([k]) => !CONSOLE_SKIP_KEYS.has(k)) + + if (entries.length > 0) { + const indent = ' '.repeat(shortTime.length + 1) + entries.forEach(([key, value], i) => { + const isLast = i === entries.length - 1 + const branch = isLast ? '└─' : '├─' + const displayValue = + value !== null && typeof value === 'object' ? safeStringify(value) : String(value) + logMessage += `\n${indent}${branch} ${key}: ${displayValue}` + }) } - let logMessage = `${emoji[level] || '📝'} [${timestamp}] ${level.toUpperCase()}: ${message}` - - // 直接处理额外数据，不需要metadata包装 - const additionalData = { ...rest } - delete additionalData.level - delete additionalData.message - delete additionalData.timestamp - delete additionalData.stack - - if (Object.keys(additionalData).length > 0) { - logMessage += ` | ${safeStringify(additionalData)}` + if (stack) { + logMessage += `\n${stack}` } - - return stack ? `${logMessage}\n${stack}` : logMessage + return logMessage }) ) - return winston.format.combine(...formats) -} +// 文件格式: NDJSON（完整结构化数据） +const createFileFormat = () => + winston.format.combine( + winston.format.timestamp({ format: () => formatDateWithTimezone(new Date(), false) }), + winston.format.errors({ stack: true }), + winston.format.printf(({ level, message, timestamp, stack, ...rest }) => { + const entry = { ts: timestamp, lvl: level, msg: message } + // 合并所有 metadata + for (const [k, v] of Object.entries(rest)) { + if (k !== 'level' && k !== 'message' && k !== 'timestamp' && k !== 'stack') { + entry[k] = v + } + } + if (stack) { + entry.stack = stack + } + return safeStringify(entry) + }) + ) -const logFormat = createLogFormat(false) -const consoleFormat = createLogFormat(true) +const fileFormat = createFileFormat() +const consoleFormat = createConsoleFormat() const isTestEnv = process.env.NODE_ENV === 'test' || process.env.JEST_WORKER_ID // 📁 确保日志目录存在并设置权限 @@ -153,7 +190,7 @@ const createRotateTransport = (filename, level = null) => { maxSize: config.logging.maxSize, maxFiles: config.logging.maxFiles, auditFile: path.join(config.logging.dirname, `.${filename.replace('%DATE%', 'audit')}.json`), - format: logFormat + format: fileFormat }) if (level) { @@ -184,7 +221,7 @@ const errorFileTransport = createRotateTransport('claude-relay-error-%DATE%.log' // 🔒 创建专门的安全日志记录器 const securityLogger = winston.createLogger({ level: 'warn', - format: logFormat, + format: fileFormat, transports: [createRotateTransport('claude-relay-security-%DATE%.log', 'warn')], silent: false }) @@ -207,7 +244,7 @@ const authDetailLogger = winston.createLogger({ // 🌟 增强的 Winston logger const logger = winston.createLogger({ level: process.env.LOG_LEVEL || config.logging.level, - format: logFormat, + format: fileFormat, transports: [ // 📄 文件输出 dailyRotateFileTransport, @@ -225,7 +262,7 @@ const logger = winston.createLogger({ exceptionHandlers: [ new winston.transports.File({ filename: path.join(config.logging.dirname, 'exceptions.log'), - format: logFormat, + format: fileFormat, maxsize: 10485760, // 10MB maxFiles: 5 }), @@ -238,7 +275,7 @@ const logger = winston.createLogger({ rejectionHandlers: [ new winston.transports.File({ filename: path.join(config.logging.dirname, 'rejections.log'), - format: logFormat, + format: fileFormat, maxsize: 10485760, // 10MB maxFiles: 5 }), diff --git a/src/utils/testPayloadHelper.js b/src/utils/testPayloadHelper.js index 656f6ba9..4a2187b5 100644 --- a/src/utils/testPayloadHelper.js +++ b/src/utils/testPayloadHelper.js @@ -1,4 +1,11 @@ const crypto = require('crypto') +const { mapToErrorCode } = require('./errorSanitizer') + +// 将原始错误信息映射为安全的标准错误码消息 +const sanitizeErrorMsg = (msg) => { + const mapped = mapToErrorCode({ message: msg }, { logOriginal: false }) + return `[${mapped.code}] ${mapped.message}` +} /** * 生成随机十六进制字符串 @@ -92,7 +99,8 @@ async function sendStreamTestRequest(options) { payload = createClaudeTestPayload('claude-sonnet-4-5-20250929', { stream: true }), proxyAgent = null, timeout = 30000, - extraHeaders = {} + extraHeaders = {}, + sanitize = false } = options const sendSSE = (type, data = {}) => { @@ -166,17 +174,17 @@ async function sendStreamTestRequest(options) { let errorMsg = `API Error: ${response.status}` try { const json = JSON.parse(errorData) - errorMsg = json.message || json.error?.message || json.error || errorMsg + errorMsg = extractErrorMessage(json, errorMsg) } catch { if (errorData.length < 200) { errorMsg = errorData || errorMsg } } - endTest(false, errorMsg) + endTest(false, sanitize ? sanitizeErrorMsg(errorMsg) : errorMsg) resolve() }) response.data.on('error', (err) => { - endTest(false, err.message) + endTest(false, sanitize ? sanitizeErrorMsg(err.message) : err.message) resolve() }) }) @@ -270,7 +278,7 @@ function createGeminiTestPayload(_model = 'gemini-2.5-pro', options = {}) { * @returns {object} 测试请求体 */ function createOpenAITestPayload(model = 'gpt-5', options = {}) { - const { prompt = 'hi', maxTokens = 100 } = options + const { prompt = 'hi', maxTokens = 100, stream = true } = options return { model, input: [ @@ -280,15 +288,77 @@ function createOpenAITestPayload(model = 'gpt-5', options = {}) { } ], max_output_tokens: maxTokens, - stream: true + stream } } +/** + * 生成 Chat Completions 测试请求体（用于 Azure OpenAI 等 Chat Completions 端点） + * @param {string} model - 模型名称 + * @param {object} options - 可选配置 + * @param {string} options.prompt - 自定义提示词（默认 'hi'） + * @param {number} options.maxTokens - 最大输出 token（默认 100） + * @returns {object} 测试请求体 + */ +function createChatCompletionsTestPayload(model = 'gpt-4o-mini', options = {}) { + const { prompt = 'hi', maxTokens = 100 } = options + return { + model, + messages: [ + { + role: 'user', + content: prompt + } + ], + max_tokens: maxTokens + } +} + +/** + * 从各种格式的错误响应中提取可读错误信息 + * 支持格式: {message}, {error:{message}}, {msg:{error:{message}}}, {error:"string"} 等 + * @param {object} json - 解析后的 JSON 错误响应 + * @param {string} fallback - 提取失败时的回退信息 + * @returns {string} 错误信息 + */ +function extractErrorMessage(json, fallback) { + if (!json || typeof json !== 'object') { + return fallback + } + // 直接 message + if (json.message && typeof json.message === 'string') { + return json.message + } + // {error: {message: "..."}} + if (json.error?.message) { + return json.error.message + } + // {msg: {error: {message: "..."}}} (relay 包装格式) + if (json.msg?.error?.message) { + return json.msg.error.message + } + if (json.msg?.message) { + return json.msg.message + } + // {error: "string"} + if (typeof json.error === 'string') { + return json.error + } + // {msg: "string"} + if (typeof json.msg === 'string') { + return json.msg + } + return fallback +} + module.exports = { randomHex, generateSessionString, createClaudeTestPayload, createGeminiTestPayload, createOpenAITestPayload, + createChatCompletionsTestPayload, + extractErrorMessage, + sanitizeErrorMsg, sendStreamTestRequest } diff --git a/src/utils/upstreamErrorHelper.js b/src/utils/upstreamErrorHelper.js new file mode 100644 index 00000000..6ddc2ca5 --- /dev/null +++ b/src/utils/upstreamErrorHelper.js @@ -0,0 +1,255 @@ +const logger = require('./logger') + +const TEMP_UNAVAILABLE_PREFIX = 'temp_unavailable' + +// 默认 TTL（秒） +const DEFAULT_TTL = { + server_error: 300, // 5xx: 5分钟 + overload: 600, // 529: 10分钟 + auth_error: 1800, // 401/403: 30分钟 + timeout: 300, // 504/网络超时: 5分钟 + rate_limit: 300 // 429: 5分钟（优先使用响应头解析值） +} + +// 延迟加载配置，避免循环依赖 +let _configCache = null +const getConfig = () => { + if (!_configCache) { + try { + _configCache = require('../../config/config') + } catch { + _configCache = {} + } + } + return _configCache +} + +const getTtlConfig = () => { + const config = getConfig() + return { + server_error: config.upstreamError?.serverErrorTtlSeconds ?? DEFAULT_TTL.server_error, + overload: config.upstreamError?.overloadTtlSeconds ?? DEFAULT_TTL.overload, + auth_error: config.upstreamError?.authErrorTtlSeconds ?? DEFAULT_TTL.auth_error, + timeout: config.upstreamError?.timeoutTtlSeconds ?? DEFAULT_TTL.timeout, + rate_limit: DEFAULT_TTL.rate_limit + } +} + +// 延迟加载 redis，避免循环依赖 +let _redis = null +const getRedis = () => { + if (!_redis) { + _redis = require('../models/redis') + } + return _redis +} + +// 根据 HTTP 状态码分类错误类型 +const classifyError = (statusCode) => { + if (statusCode === 529) { + return 'overload' + } + if (statusCode === 504) { + return 'timeout' + } + if (statusCode === 401 || statusCode === 403) { + return 'auth_error' + } + if (statusCode === 429) { + return 'rate_limit' + } + if (statusCode >= 500) { + return 'server_error' + } + return null +} + +// 解析 429 响应头中的重置时间（返回秒数） +const parseRetryAfter = (headers) => { + if (!headers) { + return null + } + + // 标准 Retry-After 头（秒数或 HTTP 日期） + const retryAfter = headers['retry-after'] + if (retryAfter) { + const seconds = parseInt(retryAfter, 10) + if (!isNaN(seconds) && seconds > 0) { + return seconds + } + const date = new Date(retryAfter) + if (!isNaN(date.getTime())) { + const diff = Math.ceil((date.getTime() - Date.now()) / 1000) + if (diff > 0) { + return diff + } + } + } + + // Anthropic 限流重置头（ISO 时间） + const anthropicReset = headers['anthropic-ratelimit-unified-reset'] + if (anthropicReset) { + const date = new Date(anthropicReset) + if (!isNaN(date.getTime())) { + const diff = Math.ceil((date.getTime() - Date.now()) / 1000) + if (diff > 0) { + return diff + } + } + } + + // OpenAI/Codex 限流重置头 + const xReset = headers['x-ratelimit-reset-requests'] || headers['x-codex-ratelimit-reset'] + if (xReset) { + const seconds = parseInt(xReset, 10) + if (!isNaN(seconds) && seconds > 0) { + return seconds + } + } + + return null +} + +// 标记账户为临时不可用 +const markTempUnavailable = async (accountId, accountType, statusCode, customTtl = null) => { + try { + const errorType = classifyError(statusCode) + if (!errorType) { + return { success: false, reason: 'not_a_pausable_error' } + } + + const ttlConfig = getTtlConfig() + const ttlSeconds = customTtl ?? ttlConfig[errorType] + + const redis = getRedis() + const client = redis.getClientSafe() + const key = `${TEMP_UNAVAILABLE_PREFIX}:${accountType}:${accountId}` + await client.setex( + key, + ttlSeconds, + JSON.stringify({ + statusCode, + errorType, + markedAt: new Date().toISOString() + }) + ) + + logger.warn( + `⏱️ [UpstreamError] Account ${accountId} (${accountType}) marked temporarily unavailable for ${ttlSeconds}s (${statusCode} ${errorType})` + ) + + return { success: true, ttlSeconds, errorType } + } catch (error) { + logger.error( + `❌ [UpstreamError] Failed to mark account ${accountId} temporarily unavailable:`, + error + ) + return { success: false } + } +} + +// 检查账户是否临时不可用 +const isTempUnavailable = async (accountId, accountType) => { + try { + const redis = getRedis() + const client = redis.getClientSafe() + const key = `${TEMP_UNAVAILABLE_PREFIX}:${accountType}:${accountId}` + return (await client.exists(key)) === 1 + } catch (error) { + logger.error( + `❌ [UpstreamError] Failed to check temp unavailable status for ${accountId}:`, + error + ) + return false + } +} + +// 清除临时不可用状态 +const clearTempUnavailable = async (accountId, accountType) => { + try { + const redis = getRedis() + const client = redis.getClientSafe() + const key = `${TEMP_UNAVAILABLE_PREFIX}:${accountType}:${accountId}` + await client.del(key) + } catch (error) { + logger.error(`❌ [UpstreamError] Failed to clear temp unavailable for ${accountId}:`, error) + } +} + +// 批量查询所有临时不可用状态（用于前端展示） +const getAllTempUnavailable = async () => { + try { + const redis = getRedis() + const client = redis.getClientSafe() + const pattern = `${TEMP_UNAVAILABLE_PREFIX}:*` + const keys = await client.keys(pattern) + if (!keys.length) { + return {} + } + + const pipeline = client.pipeline() + for (const key of keys) { + pipeline.get(key) + pipeline.ttl(key) + } + const results = await pipeline.exec() + + const statuses = {} + for (let i = 0; i < keys.length; i++) { + const key = keys[i] + // key format: temp_unavailable:{accountType}:{accountId} + const parts = key.split(':') + const accountType = parts[1] + const accountId = parts.slice(2).join(':') + const [getErr, value] = results[i * 2] + const [ttlErr, ttl] = results[i * 2 + 1] + if (getErr || ttlErr || !value) { + continue + } + + try { + const data = JSON.parse(value) + const compositeKey = `${accountType}:${accountId}` + statuses[compositeKey] = { + accountId, + accountType, + statusCode: data.statusCode, + errorType: data.errorType, + markedAt: data.markedAt, + ttl: ttl > 0 ? ttl : 0 + } + } catch { + // ignore parse errors + } + } + return statuses + } catch (error) { + logger.error('❌ [UpstreamError] Failed to get all temp unavailable statuses:', error) + return {} + } +} + +// 清洗上游错误数据，去除内部路由标识（如 [codex/codex]） +const sanitizeErrorForClient = (errorData) => { + if (!errorData || typeof errorData !== 'object') { + return errorData + } + try { + const str = JSON.stringify(errorData) + const cleaned = str.replace(/ \[[^\]\/]+\/[^\]]+\]/g, '') + return JSON.parse(cleaned) + } catch { + return errorData + } +} + +module.exports = { + markTempUnavailable, + isTempUnavailable, + clearTempUnavailable, + getAllTempUnavailable, + classifyError, + parseRetryAfter, + sanitizeErrorForClient, + TEMP_UNAVAILABLE_PREFIX +} diff --git a/tests/accountBalanceService.test.js b/tests/accountBalanceService.test.js index c2a9c3a8..479424a5 100644 --- a/tests/accountBalanceService.test.js +++ b/tests/accountBalanceService.test.js @@ -6,7 +6,7 @@ jest.mock('../src/utils/logger', () => ({ error: jest.fn() })) -const accountBalanceServiceModule = require('../src/services/accountBalanceService') +const accountBalanceServiceModule = require('../src/services/account/accountBalanceService') const { AccountBalanceService } = accountBalanceServiceModule diff --git a/web/admin-spa/src/components/accounts/AccountForm.vue b/web/admin-spa/src/components/accounts/AccountForm.vue index 80da6659..ce2eb71e 100644 --- a/web/admin-spa/src/components/accounts/AccountForm.vue +++ b/web/admin-spa/src/components/accounts/AccountForm.vue @@ -1614,7 +1614,7 @@ -
+
@@ -1714,24 +1714,26 @@ {{ errors.baseUrl }}

- 填写 API 基础地址，必须以 - /models - 结尾。系统会自动拼接 + 支持三种格式，系统自动识别： +

+

+ 以 /models 结尾: /{model}:generateContenthttps://proxy.com/v1beta/models

- 官方: + 模板模式: https://generativelanguage.googleapis.com/v1beta/modelshttps://proxy.com/api/{model}:{action}

- 上游为 CRS: + 域名: https://your-crs.com/gemini/v1beta/modelshttps://generativelanguage.googleapis.com + (自动拼接 /v1beta/models)

@@ -3371,26 +3373,24 @@

账号被限流后暂停调度的时间（分钟）

+ - -
- - -

- 勾选后遇到 401/400/429/529 等上游错误仅记录日志并透传，不自动禁用或限流 -

-
+ +
+ + +

+ 勾选后遇到 401/400/429/529 等上游错误仅记录日志并透传，不自动禁用或限流 +

@@ -3505,24 +3505,26 @@ {{ errors.baseUrl }}

- 填写 API 基础地址，必须以 - /models - 结尾。系统会自动拼接 + 支持三种格式，系统自动识别： +

+

+ 以 /models 结尾: /{model}:generateContenthttps://proxy.com/v1beta/models

- 官方: + 模板模式: https://generativelanguage.googleapis.com/v1beta/modelshttps://proxy.com/api/{model}:{action}

- 上游为 CRS: + 域名: https://your-crs.com/gemini/v1beta/modelshttps://generativelanguage.googleapis.com + (自动拼接 /v1beta/models)

@@ -4039,6 +4041,20 @@ const handleCancel = () => { const isEdit = computed(() => !!props.account) const show = ref(true) +// 支持 disableAutoProtection 的平台白名单 +const autoProtectionPlatforms = [ + 'claude-console', + 'ccr', + 'droid', + 'bedrock', + 'azure-openai', + 'azure_openai', + 'gemini', + 'gemini-api', + 'openai', + 'openai-responses' +] + // OAuthFlow 组件引用 const oauthFlowRef = ref(null) @@ -4274,7 +4290,9 @@ const form = ref({ })(), userAgent: props.account?.userAgent || '', enableRateLimit: props.account ? props.account.rateLimitDuration > 0 : true, - disableAutoProtection: props.account?.disableAutoProtection === true, + disableAutoProtection: + props.account?.disableAutoProtection === true || + props.account?.disableAutoProtection === 'true', // 额度管理字段 dailyQuota: props.account?.dailyQuota || 0, dailyUsage: props.account?.dailyUsage || 0, @@ -5242,13 +5260,9 @@ const createAccount = async () => { errors.value.apiKey = '请填写 API Key' hasError = true } - // 验证 baseUrl 必须以 /models 结尾 if (!form.value.baseUrl || form.value.baseUrl.trim() === '') { errors.value.baseUrl = '请填写 API 基础地址' hasError = true - } else if (!form.value.baseUrl.trim().endsWith('/models')) { - errors.value.baseUrl = 'API 基础地址必须以 /models 结尾' - hasError = true } } else { // 其他平台（如 Droid）使用多 API Key 输入 @@ -5407,9 +5421,7 @@ const createAccount = async () => { data.userAgent = form.value.userAgent || null // 如果不启用限流，传递 0 表示不限流 data.rateLimitDuration = form.value.enableRateLimit ? form.value.rateLimitDuration || 60 : 0 - // 上游错误处理（仅 Claude Console） if (form.value.platform === 'claude-console') { - data.disableAutoProtection = !!form.value.disableAutoProtection data.interceptWarmup = !!form.value.interceptWarmup } // 额度管理字段 @@ -5474,6 +5486,11 @@ const createAccount = async () => { data.schedulable = form.value.schedulable !== false } + // 支持 disableAutoProtection 的平台才写入 + if (autoProtectionPlatforms.includes(form.value.platform)) { + data.disableAutoProtection = !!form.value.disableAutoProtection + } + let result if (form.value.platform === 'claude') { result = await accountsStore.createClaudeAccount(data) @@ -5540,17 +5557,13 @@ const updateAccount = async () => { return } - // Gemini API 的 baseUrl 验证（必须以 /models 结尾） + // Gemini API 的 baseUrl 验证 if (form.value.platform === 'gemini-api') { const baseUrl = form.value.baseUrl?.trim() || '' if (!baseUrl) { errors.value.baseUrl = '请填写 API 基础地址' return } - if (!baseUrl.endsWith('/models')) { - errors.value.baseUrl = 'API 基础地址必须以 /models 结尾' - return - } } // 分组类型验证 - 更新账户流程修复 @@ -5755,8 +5768,6 @@ const updateAccount = async () => { data.userAgent = form.value.userAgent || null // 如果不启用限流，传递 0 表示不限流 data.rateLimitDuration = form.value.enableRateLimit ? form.value.rateLimitDuration || 60 : 0 - // 上游错误处理 - data.disableAutoProtection = !!form.value.disableAutoProtection // 拦截预热请求 data.interceptWarmup = !!form.value.interceptWarmup // 额度管理字段 @@ -5847,6 +5858,11 @@ const updateAccount = async () => { : [] } + // 支持 disableAutoProtection 的平台才写入 + if (autoProtectionPlatforms.includes(props.account.platform)) { + data.disableAutoProtection = !!form.value.disableAutoProtection + } + if (props.account.platform === 'claude') { await accountsStore.updateClaudeAccount(props.account.id, data) } else if (props.account.platform === 'claude-console') { @@ -6399,7 +6415,8 @@ watch( // 并发控制字段 maxConcurrentTasks: newAccount.maxConcurrentTasks || 0, // 上游错误处理 - disableAutoProtection: newAccount.disableAutoProtection === true + disableAutoProtection: + newAccount.disableAutoProtection === true || newAccount.disableAutoProtection === 'true' } // 如果是Claude Console账户，加载实时使用情况 diff --git a/web/admin-spa/src/components/accounts/AccountScheduledTestModal.vue b/web/admin-spa/src/components/accounts/AccountScheduledTestModal.vue index 8b9f5b1c..d66dbf34 100644 --- a/web/admin-spa/src/components/accounts/AccountScheduledTestModal.vue +++ b/web/admin-spa/src/components/accounts/AccountScheduledTestModal.vue @@ -111,30 +111,12 @@ - -
- -
@@ -219,9 +201,11 @@ diff --git a/web/admin-spa/src/components/apikeys/ApiKeyTestModal.vue b/web/admin-spa/src/components/apikeys/ApiKeyTestModal.vue deleted file mode 100644 index e7f5fdac..00000000 --- a/web/admin-spa/src/components/apikeys/ApiKeyTestModal.vue +++ /dev/null @@ -1,629 +0,0 @@ -