From 584fa8c9c17c50a50c299e821331bc20f1fa7e7f Mon Sep 17 00:00:00 2001 From: SunSeekerX Date: Wed, 31 Dec 2025 02:08:47 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=A4=A7=E8=A7=84=E6=A8=A1=E6=80=A7?= =?UTF-8?q?=E8=83=BD=E4=BC=98=E5=8C=96=20-=20Redis=20Pipeline=20=E6=89=B9?= =?UTF-8?q?=E9=87=8F=E6=93=8D=E4=BD=9C=E3=80=81=E7=B4=A2=E5=BC=95=E7=B3=BB?= =?UTF-8?q?=E7=BB=9F=E3=80=81=E8=BF=9E=E6=8E=A5=E6=B1=A0=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 10 + cli/index.js | 16 +- scripts/generate-test-data.js | 2 +- scripts/migrate-apikey-expiry.js | 3 +- scripts/migrate-usage-index.js | 124 + src/app.js | 53 +- src/middleware/auth.js | 24 +- src/models/redis.js | 1053 ++++++- src/routes/admin/apiKeys.js | 122 +- src/routes/admin/ccrAccounts.js | 6 +- src/routes/admin/claudeAccounts.js | 10 +- src/routes/admin/claudeConsoleAccounts.js | 6 +- src/routes/admin/dashboard.js | 509 ++-- src/routes/admin/droidAccounts.js | 162 +- src/routes/admin/geminiAccounts.js | 6 +- src/routes/admin/geminiApiAccounts.js | 139 +- src/routes/admin/openaiAccounts.js | 12 +- src/routes/admin/openaiResponsesAccounts.js | 165 +- src/routes/admin/sync.js | 20 +- src/routes/admin/usageStats.js | 1233 +++++--- src/routes/apiStats.js | 25 +- src/routes/openaiRoutes.js | 94 +- src/routes/web.js | 12 +- src/services/accountGroupService.js | 198 +- src/services/apiKeyIndexService.js | 642 +++++ src/services/apiKeyService.js | 389 ++- src/services/azureOpenaiAccountService.js | 20 +- src/services/bedrockAccountService.js | 14 +- src/services/billingEventPublisher.js | 2 +- src/services/ccrAccountService.js | 106 +- src/services/claudeAccountService.js | 14 +- src/services/claudeCodeHeadersService.js | 28 +- src/services/claudeConsoleAccountService.js | 29 +- src/services/claudeRelayConfigService.js | 4 +- src/services/claudeRelayService.js | 49 +- src/services/costInitService.js | 280 +- src/services/costRankService.js | 35 +- src/services/droidAccountService.js | 113 +- src/services/droidRelayService.js | 2 +- src/services/droidScheduler.js | 94 +- src/services/geminiAccountService.js | 127 +- src/services/geminiApiAccountService.js | 22 +- src/services/modelService.js | 2 +- src/services/openaiAccountService.js | 130 +- src/services/openaiResponsesAccountService.js | 139 +- src/services/openaiResponsesRelayService.js | 26 +- src/services/pricingService.js | 6 +- src/services/unifiedClaudeScheduler.js | 64 +- src/services/unifiedGeminiScheduler.js | 64 +- src/services/unifiedOpenAIScheduler.js | 68 +- src/services/userMessageQueueService.js | 26 +- src/services/userService.js | 44 +- src/utils/commonHelper.js | 300 ++ src/utils/oauthHelper.js | 10 +- src/utils/performanceOptimizer.js | 168 ++ src/utils/sseParser.js | 70 +- src/utils/workosOAuthHelper.js | 2 +- .../tutorial/ClaudeCodeTutorial.vue | 495 ++++ .../src/components/tutorial/CodexTutorial.vue | 354 +++ .../components/tutorial/DroidCliTutorial.vue | 98 + .../components/tutorial/GeminiCliTutorial.vue | 183 ++ .../tutorial/NodeInstallTutorial.vue | 234 ++ .../src/components/tutorial/VerifyInstall.vue | 30 + .../components/user/UserApiKeysManager.vue | 4 +- .../src/composables/useTutorialUrls.js | 52 + web/admin-spa/src/stores/auth.js | 14 +- web/admin-spa/src/views/ApiKeysView.vue | 5 +- web/admin-spa/src/views/TutorialView.vue | 2515 +---------------- 68 files changed, 6541 insertions(+), 4536 deletions(-) create mode 100644 scripts/migrate-usage-index.js create mode 100644 src/services/apiKeyIndexService.js create mode 100644 src/utils/commonHelper.js create mode 100644 src/utils/performanceOptimizer.js create mode 100644 web/admin-spa/src/components/tutorial/ClaudeCodeTutorial.vue create mode 100644 web/admin-spa/src/components/tutorial/CodexTutorial.vue create mode 100644 web/admin-spa/src/components/tutorial/DroidCliTutorial.vue create mode 100644 web/admin-spa/src/components/tutorial/GeminiCliTutorial.vue create mode 100644 web/admin-spa/src/components/tutorial/NodeInstallTutorial.vue create mode 100644 web/admin-spa/src/components/tutorial/VerifyInstall.vue create mode 100644 web/admin-spa/src/composables/useTutorialUrls.js diff --git a/.env.example b/.env.example index eeb10de0..e767fa25 100644 --- a/.env.example +++ b/.env.example @@ -61,6 +61,16 @@ PROXY_USE_IPV4=true # ⏱️ 请求超时配置 REQUEST_TIMEOUT=600000 # 请求超时设置（毫秒），默认10分钟 +# 🔗 HTTP 连接池配置（keep-alive） +# 流式请求最大连接数（默认65535） +# HTTPS_MAX_SOCKETS_STREAM=65535 +# 非流式请求最大连接数（默认16384） +# HTTPS_MAX_SOCKETS_NON_STREAM=16384 +# 空闲连接数（默认2048） +# HTTPS_MAX_FREE_SOCKETS=2048 +# 空闲连接超时（毫秒，默认30000） +# HTTPS_FREE_SOCKET_TIMEOUT=30000 + # 🔧 请求体大小配置 REQUEST_MAX_SIZE_MB=60 diff --git a/cli/index.js b/cli/index.js index 908a9311..cbee5076 100644 --- a/cli/index.js +++ b/cli/index.js @@ -103,7 +103,7 @@ program try { const [, apiKeys, accounts] = await Promise.all([ redis.getSystemStats(), - apiKeyService.getAllApiKeys(), + apiKeyService.getAllApiKeysFast(), claudeAccountService.getAllAccounts() ]) @@ -284,7 +284,7 @@ async function listApiKeys() { const spinner = ora('正在获取 API Keys...').start() try { - const apiKeys = await apiKeyService.getAllApiKeys() + const apiKeys = await apiKeyService.getAllApiKeysFast() spinner.succeed(`找到 ${apiKeys.length} 个 API Keys`) if (apiKeys.length === 0) { @@ -314,7 +314,7 @@ async function listApiKeys() { tableData.push([ key.name, - key.apiKey ? `${key.apiKey.substring(0, 20)}...` : '-', + key.maskedKey || '-', key.isActive ? '🟢 活跃' : '🔴 停用', expiryStatus, `${(key.usage?.total?.tokens || 0).toLocaleString()}`, @@ -333,7 +333,7 @@ async function listApiKeys() { async function updateApiKeyExpiry() { try { // 获取所有 API Keys - const apiKeys = await apiKeyService.getAllApiKeys() + const apiKeys = await apiKeyService.getAllApiKeysFast() if (apiKeys.length === 0) { console.log(styles.warning('没有找到任何 API Keys')) @@ -347,7 +347,7 @@ async function updateApiKeyExpiry() { name: 'selectedKey', message: '选择要修改的 API Key:', choices: apiKeys.map((key) => ({ - name: `${key.name} (${key.apiKey?.substring(0, 20)}...) - ${key.expiresAt ? new Date(key.expiresAt).toLocaleDateString() : '永不过期'}`, + name: `${key.name} (${key.maskedKey || key.id.substring(0, 8)}) - ${key.expiresAt ? new Date(key.expiresAt).toLocaleDateString() : '永不过期'}`, value: key })) } @@ -463,7 +463,7 @@ async function renewApiKeys() { const spinner = ora('正在查找即将过期的 API Keys...').start() try { - const apiKeys = await apiKeyService.getAllApiKeys() + const apiKeys = await apiKeyService.getAllApiKeysFast() const now = new Date() const sevenDaysLater = new Date(now.getTime() + 7 * 24 * 60 * 60 * 1000) @@ -562,7 +562,7 @@ async function renewApiKeys() { async function deleteApiKey() { try { - const apiKeys = await apiKeyService.getAllApiKeys() + const apiKeys = await apiKeyService.getAllApiKeysFast() if (apiKeys.length === 0) { console.log(styles.warning('没有找到任何 API Keys')) @@ -575,7 +575,7 @@ async function deleteApiKey() { name: 'selectedKeys', message: '选择要删除的 API Keys (空格选择，回车确认):', choices: apiKeys.map((key) => ({ - name: `${key.name} (${key.apiKey?.substring(0, 20)}...)`, + name: `${key.name} (${key.maskedKey || key.id.substring(0, 8)})`, value: key.id })) } diff --git a/scripts/generate-test-data.js b/scripts/generate-test-data.js index a1b181bd..6429f90a 100755 --- a/scripts/generate-test-data.js +++ b/scripts/generate-test-data.js @@ -141,7 +141,7 @@ async function cleanTestData() { logger.info('🧹 Cleaning test data...') // 获取所有API Keys - const allKeys = await apiKeyService.getAllApiKeys() + const allKeys = await apiKeyService.getAllApiKeysFast() // 找出所有测试 API Keys const testKeys = allKeys.filter((key) => key.name && key.name.startsWith('Test API Key')) diff --git a/scripts/migrate-apikey-expiry.js b/scripts/migrate-apikey-expiry.js index 1be8a82e..b418169f 100644 --- a/scripts/migrate-apikey-expiry.js +++ b/scripts/migrate-apikey-expiry.js @@ -12,6 +12,7 @@ */ const redis = require('../src/models/redis') +const apiKeyService = require('../src/services/apiKeyService') const logger = require('../src/utils/logger') const readline = require('readline') @@ -51,7 +52,7 @@ async function migrateApiKeys() { logger.success('✅ Connected to Redis') // 获取所有 API Keys - const apiKeys = await redis.getAllApiKeys() + const apiKeys = await apiKeyService.getAllApiKeysFast() logger.info(`📊 Found ${apiKeys.length} API Keys in total`) // 统计信息 diff --git a/scripts/migrate-usage-index.js b/scripts/migrate-usage-index.js new file mode 100644 index 00000000..77c2db0c --- /dev/null +++ b/scripts/migrate-usage-index.js @@ -0,0 +1,124 @@ +/** + * 历史数据索引迁移脚本 + * 为现有的 usage 数据建立索引，加速查询 + */ +const Redis = require('ioredis') +const config = require('../config/config') + +const redis = new Redis({ + host: config.redis.host, + port: config.redis.port, + password: config.redis.password, + db: config.redis.db || 0 +}) + +async function migrate() { + console.log('开始迁移历史数据索引...') + console.log('Redis DB:', config.redis.db || 0) + + const stats = { + dailyIndex: 0, + hourlyIndex: 0, + modelDailyIndex: 0, + modelHourlyIndex: 0 + } + + // 1. 迁移 usage:daily:{keyId}:{date} 索引 + console.log('\n1. 迁移 usage:daily 索引...') + let cursor = '0' + do { + const [newCursor, keys] = await redis.scan(cursor, 'MATCH', 'usage:daily:*', 'COUNT', 500) + cursor = newCursor + + const pipeline = redis.pipeline() + for (const key of keys) { + // usage:daily:{keyId}:{date} + const match = key.match(/^usage:daily:([^:]+):(\d{4}-\d{2}-\d{2})$/) + if (match) { + const [, keyId, date] = match + pipeline.sadd(`usage:daily:index:${date}`, keyId) + pipeline.expire(`usage:daily:index:${date}`, 86400 * 32) + stats.dailyIndex++ + } + } + if (keys.length > 0) await pipeline.exec() + } while (cursor !== '0') + console.log(` 已处理 ${stats.dailyIndex} 条`) + + // 2. 迁移 usage:hourly:{keyId}:{date}:{hour} 索引 + console.log('\n2. 迁移 usage:hourly 索引...') + cursor = '0' + do { + const [newCursor, keys] = await redis.scan(cursor, 'MATCH', 'usage:hourly:*', 'COUNT', 500) + cursor = newCursor + + const pipeline = redis.pipeline() + for (const key of keys) { + // usage:hourly:{keyId}:{date}:{hour} + const match = key.match(/^usage:hourly:([^:]+):(\d{4}-\d{2}-\d{2}:\d{2})$/) + if (match) { + const [, keyId, hourKey] = match + pipeline.sadd(`usage:hourly:index:${hourKey}`, keyId) + pipeline.expire(`usage:hourly:index:${hourKey}`, 86400 * 7) + stats.hourlyIndex++ + } + } + if (keys.length > 0) await pipeline.exec() + } while (cursor !== '0') + console.log(` 已处理 ${stats.hourlyIndex} 条`) + + // 3. 迁移 usage:model:daily:{model}:{date} 索引 + console.log('\n3. 迁移 usage:model:daily 索引...') + cursor = '0' + do { + const [newCursor, keys] = await redis.scan(cursor, 'MATCH', 'usage:model:daily:*', 'COUNT', 500) + cursor = newCursor + + const pipeline = redis.pipeline() + for (const key of keys) { + // usage:model:daily:{model}:{date} + const match = key.match(/^usage:model:daily:([^:]+):(\d{4}-\d{2}-\d{2})$/) + if (match) { + const [, model, date] = match + pipeline.sadd(`usage:model:daily:index:${date}`, model) + pipeline.expire(`usage:model:daily:index:${date}`, 86400 * 32) + stats.modelDailyIndex++ + } + } + if (keys.length > 0) await pipeline.exec() + } while (cursor !== '0') + console.log(` 已处理 ${stats.modelDailyIndex} 条`) + + // 4. 迁移 usage:model:hourly:{model}:{date}:{hour} 索引 + console.log('\n4. 迁移 usage:model:hourly 索引...') + cursor = '0' + do { + const [newCursor, keys] = await redis.scan(cursor, 'MATCH', 'usage:model:hourly:*', 'COUNT', 500) + cursor = newCursor + + const pipeline = redis.pipeline() + for (const key of keys) { + // usage:model:hourly:{model}:{date}:{hour} + const match = key.match(/^usage:model:hourly:([^:]+):(\d{4}-\d{2}-\d{2}:\d{2})$/) + if (match) { + const [, model, hourKey] = match + pipeline.sadd(`usage:model:hourly:index:${hourKey}`, model) + pipeline.expire(`usage:model:hourly:index:${hourKey}`, 86400 * 7) + stats.modelHourlyIndex++ + } + } + if (keys.length > 0) await pipeline.exec() + } while (cursor !== '0') + console.log(` 已处理 ${stats.modelHourlyIndex} 条`) + + console.log('\n迁移完成!') + console.log('统计:', stats) + + redis.disconnect() +} + +migrate().catch((err) => { + console.error('迁移失败:', err) + redis.disconnect() + process.exit(1) +}) diff --git a/src/app.js b/src/app.js index db15df2e..48ec9709 100644 --- a/src/app.js +++ b/src/app.js @@ -50,7 +50,12 @@ class Application { // 🔗 连接Redis logger.info('🔄 Connecting to Redis...') await redis.connect() - logger.success('✅ Redis connected successfully') + logger.success('Redis connected successfully') + + // 📊 后台异步迁移 usage 索引（不阻塞启动） + redis.migrateUsageIndex().catch((err) => { + logger.error('📊 Background usage index migration failed:', err) + }) // 💰 初始化价格服务 logger.info('🔄 Initializing pricing service...') @@ -94,6 +99,18 @@ class Application { const costRankService = require('./services/costRankService') await costRankService.initialize() + // 🔍 初始化 API Key 索引服务（用于分页查询优化） + logger.info('🔍 Initializing API Key index service...') + const apiKeyIndexService = require('./services/apiKeyIndexService') + apiKeyIndexService.init(redis) + await apiKeyIndexService.checkAndRebuild() + + // 📁 确保账户分组反向索引存在（后台执行，不阻塞启动） + const accountGroupService = require('./services/accountGroupService') + accountGroupService.ensureReverseIndexes().catch((err) => { + logger.error('📁 Account group reverse index migration failed:', err) + }) + // 超早期拦截 /admin-next/ 请求 - 在所有中间件之前 this.app.use((req, res, next) => { if (req.path === '/admin-next/' && req.method === 'GET') { @@ -384,7 +401,7 @@ class Application { // 🚨 错误处理 this.app.use(errorHandler) - logger.success('✅ Application initialized successfully') + logger.success('Application initialized successfully') } catch (error) { logger.error('💥 Application initialization failed:', error) throw error @@ -419,7 +436,7 @@ class Application { await redis.setSession('admin_credentials', adminCredentials) - logger.success('✅ Admin credentials loaded from init.json (single source of truth)') + logger.success('Admin credentials loaded from init.json (single source of truth)') logger.info(`📋 Admin username: ${adminCredentials.username}`) } catch (error) { logger.error('❌ Failed to initialize admin credentials:', { @@ -436,22 +453,24 @@ class Application { const client = redis.getClient() // 获取所有 session:* 键 - const sessionKeys = await client.keys('session:*') + const sessionKeys = await redis.scanKeys('session:*') + const dataList = await redis.batchHgetallChunked(sessionKeys) let validCount = 0 let invalidCount = 0 - for (const key of sessionKeys) { + for (let i = 0; i < sessionKeys.length; i++) { + const key = sessionKeys[i] // 跳过 admin_credentials（系统凭据） if (key === 'session:admin_credentials') { continue } - const sessionData = await client.hgetall(key) + const sessionData = dataList[i] // 检查会话完整性：必须有 username 和 loginTime - const hasUsername = !!sessionData.username - const hasLoginTime = !!sessionData.loginTime + const hasUsername = !!sessionData?.username + const hasLoginTime = !!sessionData?.loginTime if (!hasUsername || !hasLoginTime) { // 无效会话 - 可能是漏洞利用创建的伪造会话 @@ -466,11 +485,11 @@ class Application { } if (invalidCount > 0) { - logger.security(`🔒 Startup security check: Removed ${invalidCount} invalid sessions`) + logger.security(`Startup security check: Removed ${invalidCount} invalid sessions`) } logger.success( - `✅ Session cleanup completed: ${validCount} valid, ${invalidCount} invalid removed` + `Session cleanup completed: ${validCount} valid, ${invalidCount} invalid removed` ) } catch (error) { // 清理失败不应阻止服务启动 @@ -520,9 +539,7 @@ class Application { await this.initialize() this.server = this.app.listen(config.server.port, config.server.host, () => { - logger.start( - `🚀 Claude Relay Service started on ${config.server.host}:${config.server.port}` - ) + logger.start(`Claude Relay Service started on ${config.server.host}:${config.server.port}`) logger.info( `🌐 Web interface: http://${config.server.host}:${config.server.port}/admin-next/api-stats` ) @@ -577,7 +594,7 @@ class Application { logger.info(`📊 Cache System - Registered: ${stats.cacheCount} caches`) }, 5000) - logger.success('✅ Cache monitoring initialized') + logger.success('Cache monitoring initialized') } catch (error) { logger.error('❌ Failed to initialize cache monitoring:', error) // 不阻止应用启动 @@ -626,7 +643,7 @@ class Application { // 每分钟主动清理所有过期的并发项，不依赖请求触发 setInterval(async () => { try { - const keys = await redis.keys('concurrency:*') + const keys = await redis.scanKeys('concurrency:*') if (keys.length === 0) { return } @@ -808,9 +825,9 @@ class Application { // 🔢 清理所有并发计数（Phase 1 修复：防止重启泄漏） try { logger.info('🔢 Cleaning up all concurrency counters...') - const keys = await redis.keys('concurrency:*') + const keys = await redis.scanKeys('concurrency:*') if (keys.length > 0) { - await redis.client.del(...keys) + await redis.batchDelChunked(keys) logger.info(`✅ Cleaned ${keys.length} concurrency keys`) } else { logger.info('✅ No concurrency keys to clean') @@ -827,7 +844,7 @@ class Application { logger.error('❌ Error disconnecting Redis:', error) } - logger.success('✅ Graceful shutdown completed') + logger.success('Graceful shutdown completed') process.exit(0) }) diff --git a/src/middleware/auth.js b/src/middleware/auth.js index f051a266..39feb5a8 100644 --- a/src/middleware/auth.js +++ b/src/middleware/auth.js @@ -451,7 +451,7 @@ const authenticateApiKey = async (req, res, next) => { } if (!apiKey) { - logger.security(`🔒 Missing API key attempt from ${req.ip || 'unknown'}`) + logger.security(`Missing API key attempt from ${req.ip || 'unknown'}`) return res.status(401).json({ error: 'Missing API key', message: @@ -461,7 +461,7 @@ const authenticateApiKey = async (req, res, next) => { // 基本API Key格式验证 if (typeof apiKey !== 'string' || apiKey.length < 10 || apiKey.length > 512) { - logger.security(`🔒 Invalid API key format from ${req.ip || 'unknown'}`) + logger.security(`Invalid API key format from ${req.ip || 'unknown'}`) return res.status(401).json({ error: 'Invalid API key format', message: 'API key format is invalid' @@ -473,7 +473,7 @@ const authenticateApiKey = async (req, res, next) => { if (!validation.valid) { const clientIP = req.ip || req.connection?.remoteAddress || 'unknown' - logger.security(`🔒 Invalid API key attempt: ${validation.error} from ${clientIP}`) + logger.security(`Invalid API key attempt: ${validation.error} from ${clientIP}`) return res.status(401).json({ error: 'Invalid API key', message: validation.error @@ -1357,7 +1357,7 @@ const authenticateAdmin = async (req, res, next) => { req.headers['x-admin-token'] if (!token) { - logger.security(`🔒 Missing admin token attempt from ${req.ip || 'unknown'}`) + logger.security(`Missing admin token attempt from ${req.ip || 'unknown'}`) return res.status(401).json({ error: 'Missing admin token', message: 'Please provide an admin token' @@ -1366,7 +1366,7 @@ const authenticateAdmin = async (req, res, next) => { // 基本token格式验证 if (typeof token !== 'string' || token.length < 32 || token.length > 512) { - logger.security(`🔒 Invalid admin token format from ${req.ip || 'unknown'}`) + logger.security(`Invalid admin token format from ${req.ip || 'unknown'}`) return res.status(401).json({ error: 'Invalid admin token format', message: 'Admin token format is invalid' @@ -1382,7 +1382,7 @@ const authenticateAdmin = async (req, res, next) => { ]) if (!adminSession || Object.keys(adminSession).length === 0) { - logger.security(`🔒 Invalid admin token attempt from ${req.ip || 'unknown'}`) + logger.security(`Invalid admin token attempt from ${req.ip || 'unknown'}`) return res.status(401).json({ error: 'Invalid admin token', message: 'Invalid or expired admin session' @@ -1440,7 +1440,7 @@ const authenticateAdmin = async (req, res, next) => { } const authDuration = Date.now() - startTime - logger.security(`🔐 Admin authenticated: ${adminSession.username} in ${authDuration}ms`) + logger.security(`Admin authenticated: ${adminSession.username} in ${authDuration}ms`) return next() } catch (error) { @@ -1471,7 +1471,7 @@ const authenticateUser = async (req, res, next) => { req.headers['x-user-token'] if (!sessionToken) { - logger.security(`🔒 Missing user session token attempt from ${req.ip || 'unknown'}`) + logger.security(`Missing user session token attempt from ${req.ip || 'unknown'}`) return res.status(401).json({ error: 'Missing user session token', message: 'Please login to access this resource' @@ -1480,7 +1480,7 @@ const authenticateUser = async (req, res, next) => { // 基本token格式验证 if (typeof sessionToken !== 'string' || sessionToken.length < 32 || sessionToken.length > 128) { - logger.security(`🔒 Invalid user session token format from ${req.ip || 'unknown'}`) + logger.security(`Invalid user session token format from ${req.ip || 'unknown'}`) return res.status(401).json({ error: 'Invalid session token format', message: 'Session token format is invalid' @@ -1491,7 +1491,7 @@ const authenticateUser = async (req, res, next) => { const sessionValidation = await userService.validateUserSession(sessionToken) if (!sessionValidation) { - logger.security(`🔒 Invalid user session token attempt from ${req.ip || 'unknown'}`) + logger.security(`Invalid user session token attempt from ${req.ip || 'unknown'}`) return res.status(401).json({ error: 'Invalid session token', message: 'Invalid or expired user session' @@ -1582,7 +1582,7 @@ const authenticateUserOrAdmin = async (req, res, next) => { req.userType = 'admin' const authDuration = Date.now() - startTime - logger.security(`🔐 Admin authenticated: ${adminSession.username} in ${authDuration}ms`) + logger.security(`Admin authenticated: ${adminSession.username} in ${authDuration}ms`) return next() } } @@ -1623,7 +1623,7 @@ const authenticateUserOrAdmin = async (req, res, next) => { } // 如果都失败了，返回未授权 - logger.security(`🔒 Authentication failed from ${req.ip || 'unknown'}`) + logger.security(`Authentication failed from ${req.ip || 'unknown'}`) return res.status(401).json({ error: 'Authentication required', message: 'Please login as user or admin to access this resource' diff --git a/src/models/redis.js b/src/models/redis.js index 8b41cabc..debfbc7c 100644 --- a/src/models/redis.js +++ b/src/models/redis.js @@ -122,6 +122,171 @@ class RedisClient { } } + // 🔄 自动迁移 usage 索引（启动时调用） + async migrateUsageIndex() { + const migrationKey = 'system:migration:usage_index_v2' // v2: 添加 keymodel 迁移 + const migrated = await this.client.get(migrationKey) + if (migrated) { + logger.debug('📊 Usage index migration already completed') + return + } + + logger.info('📊 Starting usage index migration...') + const stats = { daily: 0, hourly: 0, modelDaily: 0, modelHourly: 0 } + + try { + // 迁移 usage:daily + let cursor = '0' + do { + const [newCursor, keys] = await this.client.scan( + cursor, + 'MATCH', + 'usage:daily:*', + 'COUNT', + 500 + ) + cursor = newCursor + const pipeline = this.client.pipeline() + for (const key of keys) { + const match = key.match(/^usage:daily:([^:]+):(\d{4}-\d{2}-\d{2})$/) + if (match) { + pipeline.sadd(`usage:daily:index:${match[2]}`, match[1]) + pipeline.expire(`usage:daily:index:${match[2]}`, 86400 * 32) + stats.daily++ + } + } + if (keys.length > 0) await pipeline.exec() + } while (cursor !== '0') + + // 迁移 usage:hourly + cursor = '0' + do { + const [newCursor, keys] = await this.client.scan( + cursor, + 'MATCH', + 'usage:hourly:*', + 'COUNT', + 500 + ) + cursor = newCursor + const pipeline = this.client.pipeline() + for (const key of keys) { + const match = key.match(/^usage:hourly:([^:]+):(\d{4}-\d{2}-\d{2}:\d{2})$/) + if (match) { + pipeline.sadd(`usage:hourly:index:${match[2]}`, match[1]) + pipeline.expire(`usage:hourly:index:${match[2]}`, 86400 * 7) + stats.hourly++ + } + } + if (keys.length > 0) await pipeline.exec() + } while (cursor !== '0') + + // 迁移 usage:model:daily + cursor = '0' + do { + const [newCursor, keys] = await this.client.scan( + cursor, + 'MATCH', + 'usage:model:daily:*', + 'COUNT', + 500 + ) + cursor = newCursor + const pipeline = this.client.pipeline() + for (const key of keys) { + const match = key.match(/^usage:model:daily:([^:]+):(\d{4}-\d{2}-\d{2})$/) + if (match) { + pipeline.sadd(`usage:model:daily:index:${match[2]}`, match[1]) + pipeline.expire(`usage:model:daily:index:${match[2]}`, 86400 * 32) + stats.modelDaily++ + } + } + if (keys.length > 0) await pipeline.exec() + } while (cursor !== '0') + + // 迁移 usage:model:hourly + cursor = '0' + do { + const [newCursor, keys] = await this.client.scan( + cursor, + 'MATCH', + 'usage:model:hourly:*', + 'COUNT', + 500 + ) + cursor = newCursor + const pipeline = this.client.pipeline() + for (const key of keys) { + const match = key.match(/^usage:model:hourly:([^:]+):(\d{4}-\d{2}-\d{2}:\d{2})$/) + if (match) { + pipeline.sadd(`usage:model:hourly:index:${match[2]}`, match[1]) + pipeline.expire(`usage:model:hourly:index:${match[2]}`, 86400 * 7) + stats.modelHourly++ + } + } + if (keys.length > 0) await pipeline.exec() + } while (cursor !== '0') + + // 迁移 usage:keymodel:daily (usage:{keyId}:model:daily:{model}:{date}) + cursor = '0' + do { + const [newCursor, keys] = await this.client.scan( + cursor, + 'MATCH', + 'usage:*:model:daily:*', + 'COUNT', + 500 + ) + cursor = newCursor + const pipeline = this.client.pipeline() + for (const key of keys) { + // usage:{keyId}:model:daily:{model}:{date} + const match = key.match(/^usage:([^:]+):model:daily:(.+):(\d{4}-\d{2}-\d{2})$/) + if (match) { + const [, keyId, model, date] = match + pipeline.sadd(`usage:keymodel:daily:index:${date}`, `${keyId}:${model}`) + pipeline.expire(`usage:keymodel:daily:index:${date}`, 86400 * 32) + stats.keymodelDaily = (stats.keymodelDaily || 0) + 1 + } + } + if (keys.length > 0) await pipeline.exec() + } while (cursor !== '0') + + // 迁移 usage:keymodel:hourly (usage:{keyId}:model:hourly:{model}:{hour}) + cursor = '0' + do { + const [newCursor, keys] = await this.client.scan( + cursor, + 'MATCH', + 'usage:*:model:hourly:*', + 'COUNT', + 500 + ) + cursor = newCursor + const pipeline = this.client.pipeline() + for (const key of keys) { + // usage:{keyId}:model:hourly:{model}:{hour} + const match = key.match(/^usage:([^:]+):model:hourly:(.+):(\d{4}-\d{2}-\d{2}:\d{2})$/) + if (match) { + const [, keyId, model, hour] = match + pipeline.sadd(`usage:keymodel:hourly:index:${hour}`, `${keyId}:${model}`) + pipeline.expire(`usage:keymodel:hourly:index:${hour}`, 86400 * 7) + stats.keymodelHourly = (stats.keymodelHourly || 0) + 1 + } + } + if (keys.length > 0) await pipeline.exec() + } while (cursor !== '0') + + // 标记迁移完成 + await this.client.set(migrationKey, Date.now().toString()) + logger.info( + `📊 Usage index migration completed: daily=${stats.daily}, hourly=${stats.hourly}, modelDaily=${stats.modelDaily}, modelHourly=${stats.modelHourly}, keymodelDaily=${stats.keymodelDaily || 0}, keymodelHourly=${stats.keymodelHourly || 0}` + ) + } catch (error) { + logger.error('📊 Usage index migration failed:', error) + } + } + async disconnect() { if (this.client) { await this.client.quit() @@ -180,15 +345,18 @@ class RedisClient { } async getAllApiKeys() { - const keys = await this.client.keys('apikey:*') + const keys = await this.scanKeys('apikey:*') const apiKeys = [] - for (const key of keys) { + const dataList = await this.batchHgetallChunked(keys) + + for (let i = 0; i < keys.length; i++) { + const key = keys[i] // 过滤掉hash_map，它不是真正的API Key if (key === 'apikey:hash_map') { continue } - const keyData = await this.client.hgetall(key) + const keyData = dataList[i] if (keyData && Object.keys(keyData).length > 0) { apiKeys.push({ id: key.replace('apikey:', ''), ...keyData }) } @@ -198,24 +366,155 @@ class RedisClient { /** * 使用 SCAN 获取所有 API Key ID（避免 KEYS 命令阻塞） - * @returns {Promise} API Key ID 列表 + * @returns {Promise} API Key ID 列表（已去重） */ async scanApiKeyIds() { - const keyIds = [] + const keyIds = new Set() let cursor = '0' + // 排除索引 key 的前缀 + const excludePrefixes = [ + 'apikey:hash_map', + 'apikey:idx:', + 'apikey:set:', + 'apikey:tags:', + 'apikey:index:' + ] do { const [newCursor, keys] = await this.client.scan(cursor, 'MATCH', 'apikey:*', 'COUNT', 100) cursor = newCursor for (const key of keys) { - if (key !== 'apikey:hash_map') { - keyIds.push(key.replace('apikey:', '')) + // 只接受 apikey: 形态，排除索引 key + if (excludePrefixes.some((prefix) => key.startsWith(prefix))) continue + // 确保是 apikey: 格式（只有一个冒号） + if (key.split(':').length !== 2) continue + keyIds.add(key.replace('apikey:', '')) + } + } while (cursor !== '0') + + return [...keyIds] + } + + /** + * 使用索引获取所有 API Key 的标签（优化版本） + * 优先级：索引就绪时用 apikey:tags:all > apikey:idx:all + pipeline > SCAN + * @returns {Promise} 去重排序后的标签列表 + */ + async scanAllApiKeyTags() { + // 检查索引是否就绪（非重建中且版本号正确） + const isIndexReady = await this._checkIndexReady() + + if (isIndexReady) { + // 方案1：直接读取索引服务维护的标签集合 + const cachedTags = await this.client.smembers('apikey:tags:all') + if (cachedTags && cachedTags.length > 0) { + // 保持 trim 一致性 + return cachedTags + .map((t) => (t ? t.trim() : '')) + .filter((t) => t) + .sort() + } + + // 方案2：使用索引的 key ID 列表 + pipeline + const indexedKeyIds = await this.client.smembers('apikey:idx:all') + if (indexedKeyIds && indexedKeyIds.length > 0) { + return this._extractTagsFromKeyIds(indexedKeyIds) + } + } + + // 方案3：回退到 SCAN（索引未就绪或重建中） + return this._scanTagsFallback() + } + + /** + * 检查索引是否就绪 + */ + async _checkIndexReady() { + try { + const version = await this.client.get('apikey:index:version') + // 版本号 >= 2 表示索引就绪 + return parseInt(version) >= 2 + } catch { + return false + } + } + + async _extractTagsFromKeyIds(keyIds) { + const tagSet = new Set() + const pipeline = this.client.pipeline() + for (const keyId of keyIds) { + pipeline.hmget(`apikey:${keyId}`, 'tags', 'isDeleted') + } + + const results = await pipeline.exec() + if (!results) return [] + + for (const result of results) { + if (!result) continue + const [err, values] = result + if (err || !values) continue + const [tags, isDeleted] = values + if (isDeleted === 'true' || !tags) continue + + try { + const parsed = JSON.parse(tags) + if (Array.isArray(parsed)) { + for (const tag of parsed) { + if (tag && typeof tag === 'string' && tag.trim()) { + tagSet.add(tag.trim()) + } + } + } + } catch { + // 忽略解析错误 + } + } + return Array.from(tagSet).sort() + } + + async _scanTagsFallback() { + const tagSet = new Set() + let cursor = '0' + + do { + const [newCursor, keys] = await this.client.scan(cursor, 'MATCH', 'apikey:*', 'COUNT', 100) + cursor = newCursor + + const validKeys = keys.filter((k) => k !== 'apikey:hash_map' && k.split(':').length === 2) + if (validKeys.length === 0) continue + + const pipeline = this.client.pipeline() + for (const key of validKeys) { + pipeline.hmget(key, 'tags', 'isDeleted') + } + + const results = await pipeline.exec() + if (!results) continue + + for (const result of results) { + if (!result) continue + const [err, values] = result + if (err || !values) continue + const [tags, isDeleted] = values + if (isDeleted === 'true' || !tags) continue + + try { + const parsed = JSON.parse(tags) + if (Array.isArray(parsed)) { + for (const tag of parsed) { + if (tag && typeof tag === 'string' && tag.trim()) { + tagSet.add(tag.trim()) + } + } + } + } catch { + // 忽略解析错误 } } } while (cursor !== '0') - return keyIds + return Array.from(tagSet).sort() } /** @@ -318,7 +617,40 @@ class RedisClient { modelFilter = [] } = options - // 1. 使用 SCAN 获取所有 apikey:* 的 ID 列表（避免阻塞） + // 尝试使用索引查询（性能优化） + const apiKeyIndexService = require('../services/apiKeyIndexService') + const indexReady = await apiKeyIndexService.isIndexReady() + + // 索引路径支持的条件： + // - 无模型筛选（需要查询使用记录） + // - 非 bindingAccount 搜索模式（索引不支持） + // - 非 status/expiresAt 排序（索引不支持） + // - 无搜索关键词（索引只搜 name，旧逻辑搜 name+owner，不一致） + const canUseIndex = + indexReady && + modelFilter.length === 0 && + searchMode !== 'bindingAccount' && + !['status', 'expiresAt'].includes(sortBy) && + !search + + if (canUseIndex) { + // 使用索引查询 + try { + return await apiKeyIndexService.queryWithIndex({ + page, + pageSize, + sortBy, + sortOrder, + isActive: isActive === '' ? undefined : isActive === 'true' || isActive === true, + tag, + excludeDeleted + }) + } catch (error) { + logger.warn('⚠️ 索引查询失败，降级到全量扫描:', error.message) + } + } + + // 降级：使用 SCAN 获取所有 apikey:* 的 ID 列表（避免阻塞） const keyIds = await this.scanApiKeyIds() // 2. 使用 Pipeline 批量获取基础数据 @@ -436,7 +768,18 @@ class RedisClient { // 🔍 通过哈希值查找API Key（性能优化） async findApiKeyByHash(hashedKey) { // 使用反向映射表：hash -> keyId - const keyId = await this.client.hget('apikey:hash_map', hashedKey) + let keyId = await this.client.hget('apikey:hash_map', hashedKey) + + // 回退：查旧结构 apikey_hash:*（启动回填未完成时兼容） + if (!keyId) { + const oldData = await this.client.hgetall(`apikey_hash:${hashedKey}`) + if (oldData && oldData.id) { + keyId = oldData.id + // 回填到 hash_map + await this.client.hset('apikey:hash_map', hashedKey, keyId) + } + } + if (!keyId) { return null } @@ -668,6 +1011,32 @@ class RedisClient { const { metricsWindow } = configLocal.system pipeline.expire(systemMinuteKey, metricsWindow * 60 * 2) + // 添加索引（用于快速查询，避免 SCAN） + pipeline.sadd(`usage:daily:index:${today}`, keyId) + pipeline.sadd(`usage:hourly:index:${currentHour}`, keyId) + pipeline.sadd(`usage:model:daily:index:${today}`, normalizedModel) + pipeline.sadd(`usage:model:hourly:index:${currentHour}`, normalizedModel) + pipeline.sadd(`usage:model:monthly:index:${currentMonth}`, normalizedModel) + pipeline.sadd('usage:model:monthly:months', currentMonth) // 全局月份索引 + pipeline.sadd(`usage:keymodel:daily:index:${today}`, `${keyId}:${normalizedModel}`) + pipeline.sadd(`usage:keymodel:hourly:index:${currentHour}`, `${keyId}:${normalizedModel}`) + // 清理空标记（有新数据时） + pipeline.del(`usage:daily:index:${today}:empty`) + pipeline.del(`usage:hourly:index:${currentHour}:empty`) + pipeline.del(`usage:model:daily:index:${today}:empty`) + pipeline.del(`usage:model:hourly:index:${currentHour}:empty`) + pipeline.del(`usage:model:monthly:index:${currentMonth}:empty`) + pipeline.del(`usage:keymodel:daily:index:${today}:empty`) + pipeline.del(`usage:keymodel:hourly:index:${currentHour}:empty`) + // 索引过期时间 + pipeline.expire(`usage:daily:index:${today}`, 86400 * 32) + pipeline.expire(`usage:hourly:index:${currentHour}`, 86400 * 7) + pipeline.expire(`usage:model:daily:index:${today}`, 86400 * 32) + pipeline.expire(`usage:model:hourly:index:${currentHour}`, 86400 * 7) + pipeline.expire(`usage:model:monthly:index:${currentMonth}`, 86400 * 365) + pipeline.expire(`usage:keymodel:daily:index:${today}`, 86400 * 32) + pipeline.expire(`usage:keymodel:hourly:index:${currentHour}`, 86400 * 7) + // 执行Pipeline await pipeline.exec() } @@ -803,7 +1172,29 @@ class RedisClient { this.client.expire(accountHourly, 86400 * 7), // 7天过期 this.client.expire(accountModelDaily, 86400 * 32), // 32天过期 this.client.expire(accountModelMonthly, 86400 * 365), // 1年过期 - this.client.expire(accountModelHourly, 86400 * 7) // 7天过期 + this.client.expire(accountModelHourly, 86400 * 7), // 7天过期 + + // 添加索引 + this.client.sadd(`account_usage:hourly:index:${currentHour}`, accountId), + this.client.sadd( + `account_usage:model:hourly:index:${currentHour}`, + `${accountId}:${normalizedModel}` + ), + this.client.expire(`account_usage:hourly:index:${currentHour}`, 86400 * 7), + this.client.expire(`account_usage:model:hourly:index:${currentHour}`, 86400 * 7), + // daily 索引 + this.client.sadd(`account_usage:daily:index:${today}`, accountId), + this.client.sadd( + `account_usage:model:daily:index:${today}`, + `${accountId}:${normalizedModel}` + ), + this.client.expire(`account_usage:daily:index:${today}`, 86400 * 32), + this.client.expire(`account_usage:model:daily:index:${today}`, 86400 * 32), + // 清理空标记 + this.client.del(`account_usage:hourly:index:${currentHour}:empty`), + this.client.del(`account_usage:model:hourly:index:${currentHour}:empty`), + this.client.del(`account_usage:daily:index:${today}:empty`), + this.client.del(`account_usage:model:daily:index:${today}:empty`) ] // 如果是 1M 上下文请求，添加额外的统计 @@ -1111,31 +1502,38 @@ class RedisClient { logger.debug(`💰 Opus cost incremented successfully, new weekly total: $${results[0][1]}`) } - // 💰 计算账户的每日费用（基于模型使用） + // 💰 计算账户的每日费用（基于模型使用，使用索引集合替代 KEYS） async getAccountDailyCost(accountId) { const CostCalculator = require('../utils/costCalculator') const today = getDateStringInTimezone() - // 获取账户今日所有模型的使用数据 - const pattern = `account_usage:model:daily:${accountId}:*:${today}` - const modelKeys = await this.client.keys(pattern) + // 使用索引集合替代 KEYS 命令 + const indexKey = `account_usage:model:daily:index:${today}` + const allEntries = await this.client.smembers(indexKey) - if (!modelKeys || modelKeys.length === 0) { + // 过滤出当前账户的条目（格式：accountId:model） + const accountPrefix = `${accountId}:` + const accountModels = allEntries + .filter((entry) => entry.startsWith(accountPrefix)) + .map((entry) => entry.substring(accountPrefix.length)) + + if (accountModels.length === 0) { return 0 } + // Pipeline 批量获取所有模型数据 + const pipeline = this.client.pipeline() + for (const model of accountModels) { + pipeline.hgetall(`account_usage:model:daily:${accountId}:${model}:${today}`) + } + const results = await pipeline.exec() + let totalCost = 0 + for (let i = 0; i < accountModels.length; i++) { + const model = accountModels[i] + const [err, modelUsage] = results[i] - for (const key of modelKeys) { - // 从key中解析模型名称 - // 格式：account_usage:model:daily:{accountId}:{model}:{date} - const parts = key.split(':') - const model = parts[4] // 模型名在第5个位置（索引4） - - // 获取该模型的使用数据 - const modelUsage = await this.client.hgetall(key) - - if (modelUsage && (modelUsage.inputTokens || modelUsage.outputTokens)) { + if (!err && modelUsage && (modelUsage.inputTokens || modelUsage.outputTokens)) { const usage = { input_tokens: parseInt(modelUsage.inputTokens || 0), output_tokens: parseInt(modelUsage.outputTokens || 0), @@ -1143,7 +1541,6 @@ class RedisClient { cache_read_input_tokens: parseInt(modelUsage.cacheReadTokens || 0) } - // 使用CostCalculator计算费用 const costResult = CostCalculator.calculateCost(usage, model) totalCost += costResult.costs.total @@ -1157,6 +1554,124 @@ class RedisClient { return totalCost } + // 💰 批量计算多个账户的每日费用 + async batchGetAccountDailyCost(accountIds) { + if (!accountIds || accountIds.length === 0) { + return new Map() + } + + const CostCalculator = require('../utils/costCalculator') + const today = getDateStringInTimezone() + + // 一次获取索引 + const indexKey = `account_usage:model:daily:index:${today}` + const allEntries = await this.client.smembers(indexKey) + + // 按 accountId 分组 + const accountIdSet = new Set(accountIds) + const entriesByAccount = new Map() + for (const entry of allEntries) { + const colonIndex = entry.indexOf(':') + if (colonIndex === -1) continue + const accountId = entry.substring(0, colonIndex) + const model = entry.substring(colonIndex + 1) + if (accountIdSet.has(accountId)) { + if (!entriesByAccount.has(accountId)) entriesByAccount.set(accountId, []) + entriesByAccount.get(accountId).push(model) + } + } + + const costMap = new Map(accountIds.map((id) => [id, 0])) + + // 如果索引为空，回退到 KEYS 命令（兼容旧数据） + if (allEntries.length === 0) { + logger.debug('💰 Daily cost index empty, falling back to KEYS for batch cost calculation') + for (const accountId of accountIds) { + try { + const cost = await this.getAccountDailyCostFallback(accountId, today, CostCalculator) + costMap.set(accountId, cost) + } catch { + // 忽略单个账户的错误 + } + } + return costMap + } + + // Pipeline 批量获取所有模型数据 + const pipeline = this.client.pipeline() + const queryOrder = [] + for (const [accountId, models] of entriesByAccount) { + for (const model of models) { + pipeline.hgetall(`account_usage:model:daily:${accountId}:${model}:${today}`) + queryOrder.push({ accountId, model }) + } + } + + if (queryOrder.length === 0) { + return costMap + } + + const results = await pipeline.exec() + + for (let i = 0; i < queryOrder.length; i++) { + const { accountId, model } = queryOrder[i] + const [err, modelUsage] = results[i] + + if (!err && modelUsage && (modelUsage.inputTokens || modelUsage.outputTokens)) { + const usage = { + input_tokens: parseInt(modelUsage.inputTokens || 0), + output_tokens: parseInt(modelUsage.outputTokens || 0), + cache_creation_input_tokens: parseInt(modelUsage.cacheCreateTokens || 0), + cache_read_input_tokens: parseInt(modelUsage.cacheReadTokens || 0) + } + + const costResult = CostCalculator.calculateCost(usage, model) + costMap.set(accountId, costMap.get(accountId) + costResult.costs.total) + } + } + + return costMap + } + + // 💰 回退方法：使用 KEYS 命令计算单个账户的每日费用 + async getAccountDailyCostFallback(accountId, today, CostCalculator) { + const pattern = `account_usage:model:daily:${accountId}:*:${today}` + const modelKeys = await this.client.keys(pattern) + + if (!modelKeys || modelKeys.length === 0) { + return 0 + } + + let totalCost = 0 + const pipeline = this.client.pipeline() + for (const key of modelKeys) { + pipeline.hgetall(key) + } + const results = await pipeline.exec() + + for (let i = 0; i < modelKeys.length; i++) { + const key = modelKeys[i] + const [err, modelUsage] = results[i] + if (err || !modelUsage) continue + + const parts = key.split(':') + const model = parts[4] + + if (modelUsage.inputTokens || modelUsage.outputTokens) { + const usage = { + input_tokens: parseInt(modelUsage.inputTokens || 0), + output_tokens: parseInt(modelUsage.outputTokens || 0), + cache_creation_input_tokens: parseInt(modelUsage.cacheCreateTokens || 0), + cache_read_input_tokens: parseInt(modelUsage.cacheReadTokens || 0) + } + const costResult = CostCalculator.calculateCost(usage, model) + totalCost += costResult.costs.total + } + } + + return totalCost + } + // 📊 获取账户使用统计 async getAccountUsageStats(accountId, accountType = null) { const accountKey = `account_usage:${accountId}` @@ -1367,6 +1882,8 @@ class RedisClient { async setClaudeAccount(accountId, accountData) { const key = `claude:account:${accountId}` await this.client.hset(key, accountData) + await this.client.sadd('claude:account:index', accountId) + await this.client.del('claude:account:index:empty') } async getClaudeAccount(accountId) { @@ -1375,19 +1892,30 @@ class RedisClient { } async getAllClaudeAccounts() { - const keys = await this.client.keys('claude:account:*') + const accountIds = await this.getAllIdsByIndex( + 'claude:account:index', + 'claude:account:*', + /^claude:account:(.+)$/ + ) + if (accountIds.length === 0) return [] + + const keys = accountIds.map((id) => `claude:account:${id}`) + const pipeline = this.client.pipeline() + keys.forEach((key) => pipeline.hgetall(key)) + const results = await pipeline.exec() + const accounts = [] - for (const key of keys) { - const accountData = await this.client.hgetall(key) - if (accountData && Object.keys(accountData).length > 0) { - accounts.push({ id: key.replace('claude:account:', ''), ...accountData }) + results.forEach(([err, accountData], index) => { + if (!err && accountData && Object.keys(accountData).length > 0) { + accounts.push({ id: accountIds[index], ...accountData }) } - } + }) return accounts } async deleteClaudeAccount(accountId) { const key = `claude:account:${accountId}` + await this.client.srem('claude:account:index', accountId) return await this.client.del(key) } @@ -1395,6 +1923,8 @@ class RedisClient { async setDroidAccount(accountId, accountData) { const key = `droid:account:${accountId}` await this.client.hset(key, accountData) + await this.client.sadd('droid:account:index', accountId) + await this.client.del('droid:account:index:empty') } async getDroidAccount(accountId) { @@ -1403,25 +1933,39 @@ class RedisClient { } async getAllDroidAccounts() { - const keys = await this.client.keys('droid:account:*') + const accountIds = await this.getAllIdsByIndex( + 'droid:account:index', + 'droid:account:*', + /^droid:account:(.+)$/ + ) + if (accountIds.length === 0) return [] + + const keys = accountIds.map((id) => `droid:account:${id}`) + const pipeline = this.client.pipeline() + keys.forEach((key) => pipeline.hgetall(key)) + const results = await pipeline.exec() + const accounts = [] - for (const key of keys) { - const accountData = await this.client.hgetall(key) - if (accountData && Object.keys(accountData).length > 0) { - accounts.push({ id: key.replace('droid:account:', ''), ...accountData }) + results.forEach(([err, accountData], index) => { + if (!err && accountData && Object.keys(accountData).length > 0) { + accounts.push({ id: accountIds[index], ...accountData }) } - } + }) return accounts } async deleteDroidAccount(accountId) { const key = `droid:account:${accountId}` + // 从索引中移除 + await this.client.srem('droid:account:index', accountId) return await this.client.del(key) } async setOpenAiAccount(accountId, accountData) { const key = `openai:account:${accountId}` await this.client.hset(key, accountData) + await this.client.sadd('openai:account:index', accountId) + await this.client.del('openai:account:index:empty') } async getOpenAiAccount(accountId) { const key = `openai:account:${accountId}` @@ -1429,18 +1973,29 @@ class RedisClient { } async deleteOpenAiAccount(accountId) { const key = `openai:account:${accountId}` + await this.client.srem('openai:account:index', accountId) return await this.client.del(key) } async getAllOpenAIAccounts() { - const keys = await this.client.keys('openai:account:*') + const accountIds = await this.getAllIdsByIndex( + 'openai:account:index', + 'openai:account:*', + /^openai:account:(.+)$/ + ) + if (accountIds.length === 0) return [] + + const keys = accountIds.map((id) => `openai:account:${id}`) + const pipeline = this.client.pipeline() + keys.forEach((key) => pipeline.hgetall(key)) + const results = await pipeline.exec() + const accounts = [] - for (const key of keys) { - const accountData = await this.client.hgetall(key) - if (accountData && Object.keys(accountData).length > 0) { - accounts.push({ id: key.replace('openai:account:', ''), ...accountData }) + results.forEach(([err, accountData], index) => { + if (!err && accountData && Object.keys(accountData).length > 0) { + accounts.push({ id: accountIds[index], ...accountData }) } - } + }) return accounts } @@ -1461,13 +2016,18 @@ class RedisClient { return await this.client.del(key) } - // 🗝️ API Key哈希索引管理 + // 🗝️ API Key哈希索引管理（兼容旧结构 apikey_hash:* 和新结构 apikey:hash_map） async setApiKeyHash(hashedKey, keyData, ttl = 0) { + // 写入旧结构（兼容） const key = `apikey_hash:${hashedKey}` await this.client.hset(key, keyData) if (ttl > 0) { await this.client.expire(key, ttl) } + // 同时写入新结构 hash_map（认证使用此结构） + if (keyData.id) { + await this.client.hset('apikey:hash_map', hashedKey, keyData.id) + } } async getApiKeyHash(hashedKey) { @@ -1476,8 +2036,11 @@ class RedisClient { } async deleteApiKeyHash(hashedKey) { - const key = `apikey_hash:${hashedKey}` - return await this.client.del(key) + // 同时清理旧结构和新结构，确保 Key 轮换/删除后旧 Key 失效 + const oldKey = `apikey_hash:${hashedKey}` + await this.client.del(oldKey) + // 从新的 hash_map 中移除（认证使用此结构） + await this.client.hdel('apikey:hash_map', hashedKey) } // 🔗 OAuth会话管理 @@ -1536,11 +2099,34 @@ class RedisClient { } } + // 🔍 通过索引获取 key 列表（替代 SCAN） + async getKeysByIndex(indexKey, keyPattern) { + const members = await this.client.smembers(indexKey) + if (!members || members.length === 0) return [] + return members.map((id) => keyPattern.replace('{id}', id)) + } + + // 🔍 批量通过索引获取数据 + async getDataByIndex(indexKey, keyPattern) { + const keys = await this.getKeysByIndex(indexKey, keyPattern) + if (keys.length === 0) return [] + return await this.batchHgetallChunked(keys) + } + // 📊 获取今日系统统计 async getTodayStats() { try { const today = getDateStringInTimezone() - const dailyKeys = await this.client.keys(`usage:daily:*:${today}`) + // 优先使用索引查询，回退到 SCAN + let dailyKeys = [] + const indexKey = `usage:daily:index:${today}` + const indexMembers = await this.client.smembers(indexKey) + if (indexMembers && indexMembers.length > 0) { + dailyKeys = indexMembers.map((keyId) => `usage:daily:${keyId}:${today}`) + } else { + // 回退到 SCAN（兼容历史数据） + dailyKeys = await this.scanKeys(`usage:daily:*:${today}`) + } let totalRequestsToday = 0 let totalTokensToday = 0 @@ -1551,12 +2137,10 @@ class RedisClient { // 批量获取所有今日数据，提高性能 if (dailyKeys.length > 0) { - const pipeline = this.client.pipeline() - dailyKeys.forEach((key) => pipeline.hgetall(key)) - const results = await pipeline.exec() + const results = await this.batchHgetallChunked(dailyKeys) - for (const [error, dailyData] of results) { - if (error || !dailyData) { + for (const dailyData of results) { + if (!dailyData) { continue } @@ -1588,7 +2172,7 @@ class RedisClient { } // 获取今日创建的API Key数量（批量优化） - const allApiKeys = await this.client.keys('apikey:*') + const allApiKeys = await this.scanKeys('apikey:*') let apiKeysCreatedToday = 0 if (allApiKeys.length > 0) { @@ -1629,7 +2213,7 @@ class RedisClient { // 📈 获取系统总的平均RPM和TPM async getSystemAverages() { try { - const allApiKeys = await this.client.keys('apikey:*') + const allApiKeys = await this.scanKeys('apikey:*') let totalRequests = 0 let totalTokens = 0 let totalInputTokens = 0 @@ -3518,4 +4102,361 @@ redisClient.getAccountLastTestTime = async function (accountId, platform) { } } +/** + * 使用 SCAN 获取匹配模式的所有 keys（避免 KEYS 命令阻塞 Redis） + * @param {string} pattern - 匹配模式，如 'usage:model:daily:*:2025-01-01' + * @param {number} batchSize - 每次 SCAN 的数量，默认 200 + * @returns {Promise} 匹配的 key 列表 + */ +redisClient.scanKeys = async function (pattern, batchSize = 200) { + const keys = [] + let cursor = '0' + const client = this.getClientSafe() + + do { + const [newCursor, batch] = await client.scan(cursor, 'MATCH', pattern, 'COUNT', batchSize) + cursor = newCursor + keys.push(...batch) + } while (cursor !== '0') + + // 去重（SCAN 可能返回重复 key） + return [...new Set(keys)] +} + +/** + * 批量 HGETALL（使用 Pipeline 减少网络往返） + * @param {string[]} keys - 要获取的 key 列表 + * @returns {Promise} 每个 key 对应的数据，失败的返回 null + */ +redisClient.batchHgetall = async function (keys) { + if (!keys || keys.length === 0) { + return [] + } + + const client = this.getClientSafe() + const pipeline = client.pipeline() + keys.forEach((k) => pipeline.hgetall(k)) + const results = await pipeline.exec() + + return results.map(([err, data]) => (err ? null : data)) +} + +/** + * 使用 SCAN + Pipeline 获取匹配模式的所有数据 + * @param {string} pattern - 匹配模式 + * @param {number} batchSize - SCAN 批次大小 + * @returns {Promise<{key: string, data: Object}[]>} key 和数据的数组 + */ +redisClient.scanAndGetAll = async function (pattern, batchSize = 200) { + const keys = await this.scanKeys(pattern, batchSize) + if (keys.length === 0) { + return [] + } + + const dataList = await this.batchHgetall(keys) + return keys.map((key, i) => ({ key, data: dataList[i] })).filter((item) => item.data !== null) +} + +/** + * 批量获取多个 API Key 的使用统计、费用、并发等数据 + * @param {string[]} keyIds - API Key ID 列表 + * @returns {Promise>} keyId -> 统计数据的映射 + */ +redisClient.batchGetApiKeyStats = async function (keyIds) { + if (!keyIds || keyIds.length === 0) { + return new Map() + } + + const client = this.getClientSafe() + const today = getDateStringInTimezone() + const tzDate = getDateInTimezone() + const currentMonth = `${tzDate.getUTCFullYear()}-${String(tzDate.getUTCMonth() + 1).padStart(2, '0')}` + const currentWeek = getWeekStringInTimezone() + const currentHour = `${today}:${String(getHourInTimezone(new Date())).padStart(2, '0')}` + + const pipeline = client.pipeline() + + // 为每个 keyId 添加所有需要的查询 + for (const keyId of keyIds) { + // usage stats (3 hgetall) + pipeline.hgetall(`usage:${keyId}`) + pipeline.hgetall(`usage:daily:${keyId}:${today}`) + pipeline.hgetall(`usage:monthly:${keyId}:${currentMonth}`) + // cost stats (4 get) + pipeline.get(`usage:cost:daily:${keyId}:${today}`) + pipeline.get(`usage:cost:monthly:${keyId}:${currentMonth}`) + pipeline.get(`usage:cost:hourly:${keyId}:${currentHour}`) + pipeline.get(`usage:cost:total:${keyId}`) + // concurrency (1 zcard) + pipeline.zcard(`concurrency:${keyId}`) + // weekly opus cost (1 get) + pipeline.get(`usage:opus:weekly:${keyId}:${currentWeek}`) + // rate limit (4 get) + pipeline.get(`rate_limit:requests:${keyId}`) + pipeline.get(`rate_limit:tokens:${keyId}`) + pipeline.get(`rate_limit:cost:${keyId}`) + pipeline.get(`rate_limit:window_start:${keyId}`) + // apikey data for createdAt (1 hgetall) + pipeline.hgetall(`apikey:${keyId}`) + } + + const results = await pipeline.exec() + const statsMap = new Map() + const FIELDS_PER_KEY = 14 + + for (let i = 0; i < keyIds.length; i++) { + const keyId = keyIds[i] + const offset = i * FIELDS_PER_KEY + + const [ + [, usageTotal], + [, usageDaily], + [, usageMonthly], + [, costDaily], + [, costMonthly], + [, costHourly], + [, costTotal], + [, concurrency], + [, weeklyOpusCost], + [, rateLimitRequests], + [, rateLimitTokens], + [, rateLimitCost], + [, rateLimitWindowStart], + [, keyData] + ] = results.slice(offset, offset + FIELDS_PER_KEY) + + statsMap.set(keyId, { + usageTotal: usageTotal || {}, + usageDaily: usageDaily || {}, + usageMonthly: usageMonthly || {}, + costStats: { + daily: parseFloat(costDaily || 0), + monthly: parseFloat(costMonthly || 0), + hourly: parseFloat(costHourly || 0), + total: parseFloat(costTotal || 0) + }, + concurrency: concurrency || 0, + dailyCost: parseFloat(costDaily || 0), + weeklyOpusCost: parseFloat(weeklyOpusCost || 0), + rateLimit: { + requests: parseInt(rateLimitRequests || 0), + tokens: parseInt(rateLimitTokens || 0), + cost: parseFloat(rateLimitCost || 0), + windowStart: rateLimitWindowStart ? parseInt(rateLimitWindowStart) : null + }, + createdAt: keyData?.createdAt || null + }) + } + + return statsMap +} + +/** + * 分批 HGETALL（避免单次 pipeline 体积过大导致内存峰值） + * @param {string[]} keys - 要获取的 key 列表 + * @param {number} chunkSize - 每批大小，默认 500 + * @returns {Promise} 每个 key 对应的数据，失败的返回 null + */ +redisClient.batchHgetallChunked = async function (keys, chunkSize = 500) { + if (!keys || keys.length === 0) return [] + if (keys.length <= chunkSize) return this.batchHgetall(keys) + + const results = [] + for (let i = 0; i < keys.length; i += chunkSize) { + const chunk = keys.slice(i, i + chunkSize) + const chunkResults = await this.batchHgetall(chunk) + results.push(...chunkResults) + } + return results +} + +/** + * 分批 GET（避免单次 pipeline 体积过大） + * @param {string[]} keys - 要获取的 key 列表 + * @param {number} chunkSize - 每批大小，默认 500 + * @returns {Promise<(string|null)[]>} 每个 key 对应的值 + */ +redisClient.batchGetChunked = async function (keys, chunkSize = 500) { + if (!keys || keys.length === 0) return [] + + const client = this.getClientSafe() + if (keys.length <= chunkSize) { + const pipeline = client.pipeline() + keys.forEach((k) => pipeline.get(k)) + const results = await pipeline.exec() + return results.map(([err, val]) => (err ? null : val)) + } + + const results = [] + for (let i = 0; i < keys.length; i += chunkSize) { + const chunk = keys.slice(i, i + chunkSize) + const pipeline = client.pipeline() + chunk.forEach((k) => pipeline.get(k)) + const chunkResults = await pipeline.exec() + results.push(...chunkResults.map(([err, val]) => (err ? null : val))) + } + return results +} + +/** + * SCAN + 分批处理（边扫描边处理，避免全量 keys 堆内存） + * @param {string} pattern - 匹配模式 + * @param {Function} processor - 处理函数 (keys: string[], dataList: Object[]) => void + * @param {Object} options - 配置选项 + * @param {number} options.scanBatchSize - SCAN 每次返回数量，默认 200 + * @param {number} options.processBatchSize - 处理批次大小，默认 500 + * @param {string} options.fetchType - 获取类型：'hgetall' | 'get' | 'none'，默认 'hgetall' + */ +redisClient.scanAndProcess = async function (pattern, processor, options = {}) { + const { scanBatchSize = 200, processBatchSize = 500, fetchType = 'hgetall' } = options + const client = this.getClientSafe() + + let cursor = '0' + let pendingKeys = [] + const processedKeys = new Set() // 全程去重 + + const processBatch = async (keys) => { + if (keys.length === 0) return + + // 过滤已处理的 key + const uniqueKeys = keys.filter((k) => !processedKeys.has(k)) + if (uniqueKeys.length === 0) return + + uniqueKeys.forEach((k) => processedKeys.add(k)) + + let dataList = [] + if (fetchType === 'hgetall') { + dataList = await this.batchHgetall(uniqueKeys) + } else if (fetchType === 'get') { + const pipeline = client.pipeline() + uniqueKeys.forEach((k) => pipeline.get(k)) + const results = await pipeline.exec() + dataList = results.map(([err, val]) => (err ? null : val)) + } else { + dataList = uniqueKeys.map(() => null) // fetchType === 'none' + } + + await processor(uniqueKeys, dataList) + } + + do { + const [newCursor, batch] = await client.scan(cursor, 'MATCH', pattern, 'COUNT', scanBatchSize) + cursor = newCursor + pendingKeys.push(...batch) + + // 达到处理批次大小时处理 + while (pendingKeys.length >= processBatchSize) { + const toProcess = pendingKeys.slice(0, processBatchSize) + pendingKeys = pendingKeys.slice(processBatchSize) + await processBatch(toProcess) + } + } while (cursor !== '0') + + // 处理剩余的 keys + if (pendingKeys.length > 0) { + await processBatch(pendingKeys) + } +} + +/** + * SCAN + 分批获取所有数据（返回结果，适合需要聚合的场景） + * @param {string} pattern - 匹配模式 + * @param {Object} options - 配置选项 + * @returns {Promise<{key: string, data: Object}[]>} key 和数据的数组 + */ +redisClient.scanAndGetAllChunked = async function (pattern, options = {}) { + const results = [] + await this.scanAndProcess( + pattern, + (keys, dataList) => { + keys.forEach((key, i) => { + if (dataList[i] !== null) { + results.push({ key, data: dataList[i] }) + } + }) + }, + { ...options, fetchType: 'hgetall' } + ) + return results +} + +/** + * 分批删除 keys（避免大量 DEL 阻塞） + * @param {string[]} keys - 要删除的 key 列表 + * @param {number} chunkSize - 每批大小，默认 500 + * @returns {Promise} 删除的 key 数量 + */ +redisClient.batchDelChunked = async function (keys, chunkSize = 500) { + if (!keys || keys.length === 0) return 0 + + const client = this.getClientSafe() + let deleted = 0 + + for (let i = 0; i < keys.length; i += chunkSize) { + const chunk = keys.slice(i, i + chunkSize) + const pipeline = client.pipeline() + chunk.forEach((k) => pipeline.del(k)) + const results = await pipeline.exec() + deleted += results.filter(([err, val]) => !err && val > 0).length + } + + return deleted +} + +/** + * 通用索引辅助函数：获取所有 ID（优先索引，回退 SCAN） + * @param {string} indexKey - 索引 Set 的 key + * @param {string} scanPattern - SCAN 的 pattern + * @param {RegExp} extractRegex - 从 key 中提取 ID 的正则 + * @returns {Promise} ID 列表 + */ +redisClient.getAllIdsByIndex = async function (indexKey, scanPattern, extractRegex) { + const client = this.getClientSafe() + // 检查是否已标记为空（避免重复 SCAN） + const emptyMarker = await client.get(`${indexKey}:empty`) + if (emptyMarker === '1') { + return [] + } + let ids = await client.smembers(indexKey) + if (ids && ids.length > 0) { + return ids + } + // 回退到 SCAN（仅首次） + const keys = await this.scanKeys(scanPattern) + if (keys.length === 0) { + // 标记为空，避免重复 SCAN（1小时过期，允许新数据写入后重新检测） + await client.setex(`${indexKey}:empty`, 3600, '1') + return [] + } + ids = keys + .map((k) => { + const match = k.match(extractRegex) + return match ? match[1] : null + }) + .filter(Boolean) + // 建立索引 + if (ids.length > 0) { + await client.sadd(indexKey, ...ids) + } + return ids +} + +/** + * 添加到索引 + */ +redisClient.addToIndex = async function (indexKey, id) { + const client = this.getClientSafe() + await client.sadd(indexKey, id) + // 清除空标记（如果存在） + await client.del(`${indexKey}:empty`) +} + +/** + * 从索引移除 + */ +redisClient.removeFromIndex = async function (indexKey, id) { + const client = this.getClientSafe() + await client.srem(indexKey, id) +} + module.exports = redisClient diff --git a/src/routes/admin/apiKeys.js b/src/routes/admin/apiKeys.js index d88444bd..4a74cfda 100644 --- a/src/routes/admin/apiKeys.js +++ b/src/routes/admin/apiKeys.js @@ -82,11 +82,12 @@ router.get('/api-keys/:keyId/cost-debug', authenticateAdmin, async (req, res) => const client = redis.getClientSafe() // 获取所有相关的Redis键 - const costKeys = await client.keys(`usage:cost:*:${keyId}:*`) + const costKeys = await redis.scanKeys(`usage:cost:*:${keyId}:*`) + const costValues = await redis.batchGetChunked(costKeys) const keyValues = {} - for (const key of costKeys) { - keyValues[key] = await client.get(key) + for (let i = 0; i < costKeys.length; i++) { + keyValues[costKeys[i]] = costValues[i] } return res.json({ @@ -287,20 +288,30 @@ router.get('/api-keys', authenticateAdmin, async (req, res) => { }) } - // 为每个API Key添加owner的displayName + // 为每个API Key添加owner的displayName（批量获取优化） + const userIdsToFetch = [ + ...new Set(result.items.filter((k) => k.userId).map((k) => k.userId)) + ] + const userMap = new Map() + + if (userIdsToFetch.length > 0) { + // 批量获取用户信息 + const users = await Promise.all( + userIdsToFetch.map((id) => + userService.getUserById(id, false).catch(() => null) + ) + ) + userIdsToFetch.forEach((id, i) => { + if (users[i]) userMap.set(id, users[i]) + }) + } + for (const apiKey of result.items) { - if (apiKey.userId) { - try { - const user = await userService.getUserById(apiKey.userId, false) - if (user) { - apiKey.ownerDisplayName = user.displayName || user.username || 'Unknown User' - } else { - apiKey.ownerDisplayName = 'Unknown User' - } - } catch (error) { - logger.debug(`无法获取用户 ${apiKey.userId} 的信息:`, error) - apiKey.ownerDisplayName = 'Unknown User' - } + if (apiKey.userId && userMap.has(apiKey.userId)) { + const user = userMap.get(apiKey.userId) + apiKey.ownerDisplayName = user.displayName || user.username || 'Unknown User' + } else if (apiKey.userId) { + apiKey.ownerDisplayName = 'Unknown User' } else { apiKey.ownerDisplayName = apiKey.createdBy === 'admin' ? 'Admin' : apiKey.createdBy || 'Admin' @@ -571,6 +582,56 @@ router.get('/api-keys/cost-sort-status', authenticateAdmin, async (req, res) => } }) +// 获取 API Key 索引状态 +router.get('/api-keys/index-status', authenticateAdmin, async (req, res) => { + try { + const apiKeyIndexService = require('../../services/apiKeyIndexService') + const status = await apiKeyIndexService.getStatus() + return res.json({ success: true, data: status }) + } catch (error) { + logger.error('❌ Failed to get API Key index status:', error) + return res.status(500).json({ + success: false, + error: 'Failed to get index status', + message: error.message + }) + } +}) + +// 手动重建 API Key 索引 +router.post('/api-keys/index-rebuild', authenticateAdmin, async (req, res) => { + try { + const apiKeyIndexService = require('../../services/apiKeyIndexService') + const status = await apiKeyIndexService.getStatus() + + if (status.building) { + return res.status(409).json({ + success: false, + error: 'INDEX_BUILDING', + message: '索引正在重建中，请稍后再试', + progress: status.progress + }) + } + + // 异步重建，不等待完成 + apiKeyIndexService.rebuildIndexes().catch((err) => { + logger.error('❌ Failed to rebuild API Key index:', err) + }) + + return res.json({ + success: true, + message: 'API Key 索引重建已开始' + }) + } catch (error) { + logger.error('❌ Failed to trigger API Key index rebuild:', error) + return res.status(500).json({ + success: false, + error: 'Failed to trigger rebuild', + message: error.message + }) + } +}) + // 强制刷新费用排序索引 router.post('/api-keys/cost-sort-refresh', authenticateAdmin, async (req, res) => { try { @@ -636,22 +697,7 @@ router.get('/supported-clients', authenticateAdmin, async (req, res) => { // 获取已存在的标签列表 router.get('/api-keys/tags', authenticateAdmin, async (req, res) => { try { - const apiKeys = await apiKeyService.getAllApiKeys() - const tagSet = new Set() - - // 收集所有API Keys的标签 - for (const apiKey of apiKeys) { - if (apiKey.tags && Array.isArray(apiKey.tags)) { - apiKey.tags.forEach((tag) => { - if (tag && tag.trim()) { - tagSet.add(tag.trim()) - } - }) - } - } - - // 转换为数组并排序 - const tags = Array.from(tagSet).sort() + const tags = await apiKeyService.getAllTags() logger.info(`📋 Retrieved ${tags.length} unique tags from API keys`) return res.json({ success: true, data: tags }) @@ -1725,7 +1771,7 @@ router.put('/api-keys/batch', authenticateAdmin, async (req, res) => { // 执行更新 await apiKeyService.updateApiKey(keyId, finalUpdates) results.successCount++ - logger.success(`✅ Batch edit: API key ${keyId} updated successfully`) + logger.success(`Batch edit: API key ${keyId} updated successfully`) } catch (error) { results.failedCount++ results.errors.push(`Failed to update key ${keyId}: ${error.message}`) @@ -2176,7 +2222,7 @@ router.delete('/api-keys/batch', authenticateAdmin, async (req, res) => { await apiKeyService.deleteApiKey(keyId) results.successCount++ - logger.success(`✅ Batch delete: API key ${keyId} deleted successfully`) + logger.success(`Batch delete: API key ${keyId} deleted successfully`) } catch (error) { results.failedCount++ results.errors.push({ @@ -2231,13 +2277,13 @@ router.delete('/api-keys/:keyId', authenticateAdmin, async (req, res) => { // 📋 获取已删除的API Keys router.get('/api-keys/deleted', authenticateAdmin, async (req, res) => { try { - const deletedApiKeys = await apiKeyService.getAllApiKeys(true) // Include deleted - const onlyDeleted = deletedApiKeys.filter((key) => key.isDeleted === 'true') + const deletedApiKeys = await apiKeyService.getAllApiKeysFast(true) // Include deleted + const onlyDeleted = deletedApiKeys.filter((key) => key.isDeleted === true) // Add additional metadata for deleted keys const enrichedKeys = onlyDeleted.map((key) => ({ ...key, - isDeleted: key.isDeleted === 'true', + isDeleted: key.isDeleted === true, deletedAt: key.deletedAt, deletedBy: key.deletedBy, deletedByType: key.deletedByType, @@ -2264,7 +2310,7 @@ router.post('/api-keys/:keyId/restore', authenticateAdmin, async (req, res) => { const result = await apiKeyService.restoreApiKey(keyId, adminUsername, 'admin') if (result.success) { - logger.success(`✅ Admin ${adminUsername} restored API key: ${keyId}`) + logger.success(`Admin ${adminUsername} restored API key: ${keyId}`) return res.json({ success: true, message: 'API Key 已成功恢复', diff --git a/src/routes/admin/ccrAccounts.js b/src/routes/admin/ccrAccounts.js index 172b196f..fccbca4e 100644 --- a/src/routes/admin/ccrAccounts.js +++ b/src/routes/admin/ccrAccounts.js @@ -377,7 +377,7 @@ router.post('/:accountId/reset-usage', authenticateAdmin, async (req, res) => { const { accountId } = req.params await ccrAccountService.resetDailyUsage(accountId) - logger.success(`✅ Admin manually reset daily usage for CCR account: ${accountId}`) + logger.success(`Admin manually reset daily usage for CCR account: ${accountId}`) return res.json({ success: true, message: 'Daily usage reset successfully' }) } catch (error) { logger.error('❌ Failed to reset CCR account daily usage:', error) @@ -390,7 +390,7 @@ router.post('/:accountId/reset-status', authenticateAdmin, async (req, res) => { try { const { accountId } = req.params const result = await ccrAccountService.resetAccountStatus(accountId) - logger.success(`✅ Admin reset status for CCR account: ${accountId}`) + logger.success(`Admin reset status for CCR account: ${accountId}`) return res.json({ success: true, data: result }) } catch (error) { logger.error('❌ Failed to reset CCR account status:', error) @@ -403,7 +403,7 @@ router.post('/reset-all-usage', authenticateAdmin, async (req, res) => { try { await ccrAccountService.resetAllDailyUsage() - logger.success('✅ Admin manually reset daily usage for all CCR accounts') + logger.success('Admin manually reset daily usage for all CCR accounts') return res.json({ success: true, message: 'All daily usage reset successfully' }) } catch (error) { logger.error('❌ Failed to reset all CCR accounts daily usage:', error) diff --git a/src/routes/admin/claudeAccounts.js b/src/routes/admin/claudeAccounts.js index d079e346..590e919d 100644 --- a/src/routes/admin/claudeAccounts.js +++ b/src/routes/admin/claudeAccounts.js @@ -36,7 +36,7 @@ router.post('/claude-accounts/generate-auth-url', authenticateAdmin, async (req, expiresAt: new Date(Date.now() + 10 * 60 * 1000).toISOString() // 10分钟过期 }) - logger.success('🔗 Generated OAuth authorization URL with proxy support') + logger.success('Generated OAuth authorization URL with proxy support') return res.json({ success: true, data: { @@ -152,7 +152,7 @@ router.post('/claude-accounts/generate-setup-token-url', authenticateAdmin, asyn expiresAt: new Date(Date.now() + 10 * 60 * 1000).toISOString() // 10分钟过期 }) - logger.success('🔗 Generated Setup Token authorization URL with proxy support') + logger.success('Generated Setup Token authorization URL with proxy support') return res.json({ success: true, data: { @@ -786,7 +786,7 @@ router.post('/claude-accounts/:accountId/update-profile', authenticateAdmin, asy const profileInfo = await claudeAccountService.fetchAndUpdateAccountProfile(accountId) - logger.success(`✅ Updated profile for Claude account: ${accountId}`) + logger.success(`Updated profile for Claude account: ${accountId}`) return res.json({ success: true, message: 'Account profile updated successfully', @@ -805,7 +805,7 @@ router.post('/claude-accounts/update-all-profiles', authenticateAdmin, async (re try { const result = await claudeAccountService.updateAllAccountProfiles() - logger.success('✅ Batch profile update completed') + logger.success('Batch profile update completed') return res.json({ success: true, message: 'Batch profile update completed', @@ -841,7 +841,7 @@ router.post('/claude-accounts/:accountId/reset-status', authenticateAdmin, async const result = await claudeAccountService.resetAccountStatus(accountId) - logger.success(`✅ Admin reset status for Claude account: ${accountId}`) + logger.success(`Admin reset status for Claude account: ${accountId}`) return res.json({ success: true, data: result }) } catch (error) { logger.error('❌ Failed to reset Claude account status:', error) diff --git a/src/routes/admin/claudeConsoleAccounts.js b/src/routes/admin/claudeConsoleAccounts.js index fc0fcf62..5ed7f6fa 100644 --- a/src/routes/admin/claudeConsoleAccounts.js +++ b/src/routes/admin/claudeConsoleAccounts.js @@ -441,7 +441,7 @@ router.post( const { accountId } = req.params await claudeConsoleAccountService.resetDailyUsage(accountId) - logger.success(`✅ Admin manually reset daily usage for Claude Console account: ${accountId}`) + logger.success(`Admin manually reset daily usage for Claude Console account: ${accountId}`) return res.json({ success: true, message: 'Daily usage reset successfully' }) } catch (error) { logger.error('❌ Failed to reset Claude Console account daily usage:', error) @@ -458,7 +458,7 @@ router.post( try { const { accountId } = req.params const result = await claudeConsoleAccountService.resetAccountStatus(accountId) - logger.success(`✅ Admin reset status for Claude Console account: ${accountId}`) + logger.success(`Admin reset status for Claude Console account: ${accountId}`) return res.json({ success: true, data: result }) } catch (error) { logger.error('❌ Failed to reset Claude Console account status:', error) @@ -472,7 +472,7 @@ router.post('/claude-console-accounts/reset-all-usage', authenticateAdmin, async try { await claudeConsoleAccountService.resetAllDailyUsage() - logger.success('✅ Admin manually reset daily usage for all Claude Console accounts') + logger.success('Admin manually reset daily usage for all Claude Console accounts') return res.json({ success: true, message: 'All daily usage reset successfully' }) } catch (error) { logger.error('❌ Failed to reset all Claude Console accounts daily usage:', error) diff --git a/src/routes/admin/dashboard.js b/src/routes/admin/dashboard.js index fe2cb440..21e84e6c 100644 --- a/src/routes/admin/dashboard.js +++ b/src/routes/admin/dashboard.js @@ -23,7 +23,6 @@ const router = express.Router() router.get('/dashboard', authenticateAdmin, async (req, res) => { try { const [ - , apiKeys, claudeAccounts, claudeConsoleAccounts, @@ -37,8 +36,7 @@ router.get('/dashboard', authenticateAdmin, async (req, res) => { systemAverages, realtimeMetrics ] = await Promise.all([ - redis.getSystemStats(), - apiKeyService.getAllApiKeys(), + apiKeyService.getAllApiKeysFast(), claudeAccountService.getAllAccounts(), claudeConsoleAccountService.getAllAccounts(), geminiAccountService.getAllAccounts(), @@ -68,246 +66,95 @@ router.get('/dashboard', authenticateAdmin, async (req, res) => { return false } - const normalDroidAccounts = droidAccounts.filter( - (acc) => - normalizeBoolean(acc.isActive) && - acc.status !== 'blocked' && - acc.status !== 'unauthorized' && - normalizeBoolean(acc.schedulable) && - !isRateLimitedFlag(acc.rateLimitStatus) - ).length - const abnormalDroidAccounts = droidAccounts.filter( - (acc) => - !normalizeBoolean(acc.isActive) || acc.status === 'blocked' || acc.status === 'unauthorized' - ).length - const pausedDroidAccounts = droidAccounts.filter( - (acc) => - !normalizeBoolean(acc.schedulable) && - normalizeBoolean(acc.isActive) && - acc.status !== 'blocked' && - acc.status !== 'unauthorized' - ).length - const rateLimitedDroidAccounts = droidAccounts.filter((acc) => - isRateLimitedFlag(acc.rateLimitStatus) - ).length + // 通用账户统计函数 - 单次遍历完成所有统计 + const countAccountStats = (accounts, opts = {}) => { + const { isStringType = false, checkGeminiRateLimit = false } = opts + let normal = 0, + abnormal = 0, + paused = 0, + rateLimited = 0 - // 计算使用统计（统一使用allTokens） - const totalTokensUsed = apiKeys.reduce( - (sum, key) => sum + (key.usage?.total?.allTokens || 0), - 0 - ) - const totalRequestsUsed = apiKeys.reduce( - (sum, key) => sum + (key.usage?.total?.requests || 0), - 0 - ) - const totalInputTokensUsed = apiKeys.reduce( - (sum, key) => sum + (key.usage?.total?.inputTokens || 0), - 0 - ) - const totalOutputTokensUsed = apiKeys.reduce( - (sum, key) => sum + (key.usage?.total?.outputTokens || 0), - 0 - ) - const totalCacheCreateTokensUsed = apiKeys.reduce( - (sum, key) => sum + (key.usage?.total?.cacheCreateTokens || 0), - 0 - ) - const totalCacheReadTokensUsed = apiKeys.reduce( - (sum, key) => sum + (key.usage?.total?.cacheReadTokens || 0), - 0 - ) - const totalAllTokensUsed = apiKeys.reduce( - (sum, key) => sum + (key.usage?.total?.allTokens || 0), - 0 - ) + for (const acc of accounts) { + const isActive = isStringType + ? acc.isActive === 'true' || + acc.isActive === true || + (!acc.isActive && acc.isActive !== 'false' && acc.isActive !== false) + : acc.isActive + const isBlocked = acc.status === 'blocked' || acc.status === 'unauthorized' + const isSchedulable = isStringType + ? acc.schedulable !== 'false' && acc.schedulable !== false + : acc.schedulable !== false + const isRateLimited = checkGeminiRateLimit + ? acc.rateLimitStatus === 'limited' || + (acc.rateLimitStatus && acc.rateLimitStatus.isRateLimited) + : acc.rateLimitStatus && acc.rateLimitStatus.isRateLimited - const activeApiKeys = apiKeys.filter((key) => key.isActive).length + if (!isActive || isBlocked) { + abnormal++ + } else if (!isSchedulable) { + paused++ + } else if (isRateLimited) { + rateLimited++ + } else { + normal++ + } + } + return { normal, abnormal, paused, rateLimited } + } - // Claude账户统计 - 根据账户管理页面的判断逻辑 - const normalClaudeAccounts = claudeAccounts.filter( - (acc) => - acc.isActive && - acc.status !== 'blocked' && - acc.status !== 'unauthorized' && - acc.schedulable !== false && - !(acc.rateLimitStatus && acc.rateLimitStatus.isRateLimited) - ).length - const abnormalClaudeAccounts = claudeAccounts.filter( - (acc) => !acc.isActive || acc.status === 'blocked' || acc.status === 'unauthorized' - ).length - const pausedClaudeAccounts = claudeAccounts.filter( - (acc) => - acc.schedulable === false && - acc.isActive && - acc.status !== 'blocked' && - acc.status !== 'unauthorized' - ).length - const rateLimitedClaudeAccounts = claudeAccounts.filter( - (acc) => acc.rateLimitStatus && acc.rateLimitStatus.isRateLimited - ).length + // Droid 账户统计（特殊逻辑） + let normalDroidAccounts = 0, + abnormalDroidAccounts = 0, + pausedDroidAccounts = 0, + rateLimitedDroidAccounts = 0 + for (const acc of droidAccounts) { + const isActive = normalizeBoolean(acc.isActive) + const isBlocked = acc.status === 'blocked' || acc.status === 'unauthorized' + const isSchedulable = normalizeBoolean(acc.schedulable) + const isRateLimited = isRateLimitedFlag(acc.rateLimitStatus) - // Claude Console账户统计 - const normalClaudeConsoleAccounts = claudeConsoleAccounts.filter( - (acc) => - acc.isActive && - acc.status !== 'blocked' && - acc.status !== 'unauthorized' && - acc.schedulable !== false && - !(acc.rateLimitStatus && acc.rateLimitStatus.isRateLimited) - ).length - const abnormalClaudeConsoleAccounts = claudeConsoleAccounts.filter( - (acc) => !acc.isActive || acc.status === 'blocked' || acc.status === 'unauthorized' - ).length - const pausedClaudeConsoleAccounts = claudeConsoleAccounts.filter( - (acc) => - acc.schedulable === false && - acc.isActive && - acc.status !== 'blocked' && - acc.status !== 'unauthorized' - ).length - const rateLimitedClaudeConsoleAccounts = claudeConsoleAccounts.filter( - (acc) => acc.rateLimitStatus && acc.rateLimitStatus.isRateLimited - ).length + if (!isActive || isBlocked) { + abnormalDroidAccounts++ + } else if (!isSchedulable) { + pausedDroidAccounts++ + } else if (isRateLimited) { + rateLimitedDroidAccounts++ + } else { + normalDroidAccounts++ + } + } - // Gemini账户统计 - const normalGeminiAccounts = geminiAccounts.filter( - (acc) => - acc.isActive && - acc.status !== 'blocked' && - acc.status !== 'unauthorized' && - acc.schedulable !== false && - !( - acc.rateLimitStatus === 'limited' || - (acc.rateLimitStatus && acc.rateLimitStatus.isRateLimited) - ) - ).length - const abnormalGeminiAccounts = geminiAccounts.filter( - (acc) => !acc.isActive || acc.status === 'blocked' || acc.status === 'unauthorized' - ).length - const pausedGeminiAccounts = geminiAccounts.filter( - (acc) => - acc.schedulable === false && - acc.isActive && - acc.status !== 'blocked' && - acc.status !== 'unauthorized' - ).length - const rateLimitedGeminiAccounts = geminiAccounts.filter( - (acc) => - acc.rateLimitStatus === 'limited' || - (acc.rateLimitStatus && acc.rateLimitStatus.isRateLimited) - ).length + // 计算使用统计（单次遍历） + let totalTokensUsed = 0, + totalRequestsUsed = 0, + totalInputTokensUsed = 0, + totalOutputTokensUsed = 0, + totalCacheCreateTokensUsed = 0, + totalCacheReadTokensUsed = 0, + totalAllTokensUsed = 0, + activeApiKeys = 0 + for (const key of apiKeys) { + const usage = key.usage?.total + if (usage) { + totalTokensUsed += usage.allTokens || 0 + totalRequestsUsed += usage.requests || 0 + totalInputTokensUsed += usage.inputTokens || 0 + totalOutputTokensUsed += usage.outputTokens || 0 + totalCacheCreateTokensUsed += usage.cacheCreateTokens || 0 + totalCacheReadTokensUsed += usage.cacheReadTokens || 0 + totalAllTokensUsed += usage.allTokens || 0 + } + if (key.isActive) activeApiKeys++ + } - // Bedrock账户统计 - const normalBedrockAccounts = bedrockAccounts.filter( - (acc) => - acc.isActive && - acc.status !== 'blocked' && - acc.status !== 'unauthorized' && - acc.schedulable !== false && - !(acc.rateLimitStatus && acc.rateLimitStatus.isRateLimited) - ).length - const abnormalBedrockAccounts = bedrockAccounts.filter( - (acc) => !acc.isActive || acc.status === 'blocked' || acc.status === 'unauthorized' - ).length - const pausedBedrockAccounts = bedrockAccounts.filter( - (acc) => - acc.schedulable === false && - acc.isActive && - acc.status !== 'blocked' && - acc.status !== 'unauthorized' - ).length - const rateLimitedBedrockAccounts = bedrockAccounts.filter( - (acc) => acc.rateLimitStatus && acc.rateLimitStatus.isRateLimited - ).length - - // OpenAI账户统计 - // 注意：OpenAI账户的isActive和schedulable是字符串类型，默认值为'true' - const normalOpenAIAccounts = openaiAccounts.filter( - (acc) => - (acc.isActive === 'true' || - acc.isActive === true || - (!acc.isActive && acc.isActive !== 'false' && acc.isActive !== false)) && - acc.status !== 'blocked' && - acc.status !== 'unauthorized' && - acc.schedulable !== 'false' && - acc.schedulable !== false && // 包括'true'、true和undefined - !(acc.rateLimitStatus && acc.rateLimitStatus.isRateLimited) - ).length - const abnormalOpenAIAccounts = openaiAccounts.filter( - (acc) => - acc.isActive === 'false' || - acc.isActive === false || - acc.status === 'blocked' || - acc.status === 'unauthorized' - ).length - const pausedOpenAIAccounts = openaiAccounts.filter( - (acc) => - (acc.schedulable === 'false' || acc.schedulable === false) && - (acc.isActive === 'true' || - acc.isActive === true || - (!acc.isActive && acc.isActive !== 'false' && acc.isActive !== false)) && - acc.status !== 'blocked' && - acc.status !== 'unauthorized' - ).length - const rateLimitedOpenAIAccounts = openaiAccounts.filter( - (acc) => acc.rateLimitStatus && acc.rateLimitStatus.isRateLimited - ).length - - // CCR账户统计 - const normalCcrAccounts = ccrAccounts.filter( - (acc) => - acc.isActive && - acc.status !== 'blocked' && - acc.status !== 'unauthorized' && - acc.schedulable !== false && - !(acc.rateLimitStatus && acc.rateLimitStatus.isRateLimited) - ).length - const abnormalCcrAccounts = ccrAccounts.filter( - (acc) => !acc.isActive || acc.status === 'blocked' || acc.status === 'unauthorized' - ).length - const pausedCcrAccounts = ccrAccounts.filter( - (acc) => - acc.schedulable === false && - acc.isActive && - acc.status !== 'blocked' && - acc.status !== 'unauthorized' - ).length - const rateLimitedCcrAccounts = ccrAccounts.filter( - (acc) => acc.rateLimitStatus && acc.rateLimitStatus.isRateLimited - ).length - - // OpenAI-Responses账户统计 - // 注意：OpenAI-Responses账户的isActive和schedulable也是字符串类型 - const normalOpenAIResponsesAccounts = openaiResponsesAccounts.filter( - (acc) => - (acc.isActive === 'true' || - acc.isActive === true || - (!acc.isActive && acc.isActive !== 'false' && acc.isActive !== false)) && - acc.status !== 'blocked' && - acc.status !== 'unauthorized' && - acc.schedulable !== 'false' && - acc.schedulable !== false && - !(acc.rateLimitStatus && acc.rateLimitStatus.isRateLimited) - ).length - const abnormalOpenAIResponsesAccounts = openaiResponsesAccounts.filter( - (acc) => - acc.isActive === 'false' || - acc.isActive === false || - acc.status === 'blocked' || - acc.status === 'unauthorized' - ).length - const pausedOpenAIResponsesAccounts = openaiResponsesAccounts.filter( - (acc) => - (acc.schedulable === 'false' || acc.schedulable === false) && - (acc.isActive === 'true' || - acc.isActive === true || - (!acc.isActive && acc.isActive !== 'false' && acc.isActive !== false)) && - acc.status !== 'blocked' && - acc.status !== 'unauthorized' - ).length - const rateLimitedOpenAIResponsesAccounts = openaiResponsesAccounts.filter( - (acc) => acc.rateLimitStatus && acc.rateLimitStatus.isRateLimited - ).length + // 各平台账户统计（单次遍历） + const claudeStats = countAccountStats(claudeAccounts) + const claudeConsoleStats = countAccountStats(claudeConsoleAccounts) + const geminiStats = countAccountStats(geminiAccounts, { checkGeminiRateLimit: true }) + const bedrockStats = countAccountStats(bedrockAccounts) + const openaiStats = countAccountStats(openaiAccounts, { isStringType: true }) + const ccrStats = countAccountStats(ccrAccounts) + const openaiResponsesStats = countAccountStats(openaiResponsesAccounts, { isStringType: true }) const dashboard = { overview: { @@ -323,90 +170,90 @@ router.get('/dashboard', authenticateAdmin, async (req, res) => { openaiResponsesAccounts.length + ccrAccounts.length, normalAccounts: - normalClaudeAccounts + - normalClaudeConsoleAccounts + - normalGeminiAccounts + - normalBedrockAccounts + - normalOpenAIAccounts + - normalOpenAIResponsesAccounts + - normalCcrAccounts, + claudeStats.normal + + claudeConsoleStats.normal + + geminiStats.normal + + bedrockStats.normal + + openaiStats.normal + + openaiResponsesStats.normal + + ccrStats.normal, abnormalAccounts: - abnormalClaudeAccounts + - abnormalClaudeConsoleAccounts + - abnormalGeminiAccounts + - abnormalBedrockAccounts + - abnormalOpenAIAccounts + - abnormalOpenAIResponsesAccounts + - abnormalCcrAccounts + + claudeStats.abnormal + + claudeConsoleStats.abnormal + + geminiStats.abnormal + + bedrockStats.abnormal + + openaiStats.abnormal + + openaiResponsesStats.abnormal + + ccrStats.abnormal + abnormalDroidAccounts, pausedAccounts: - pausedClaudeAccounts + - pausedClaudeConsoleAccounts + - pausedGeminiAccounts + - pausedBedrockAccounts + - pausedOpenAIAccounts + - pausedOpenAIResponsesAccounts + - pausedCcrAccounts + + claudeStats.paused + + claudeConsoleStats.paused + + geminiStats.paused + + bedrockStats.paused + + openaiStats.paused + + openaiResponsesStats.paused + + ccrStats.paused + pausedDroidAccounts, rateLimitedAccounts: - rateLimitedClaudeAccounts + - rateLimitedClaudeConsoleAccounts + - rateLimitedGeminiAccounts + - rateLimitedBedrockAccounts + - rateLimitedOpenAIAccounts + - rateLimitedOpenAIResponsesAccounts + - rateLimitedCcrAccounts + + claudeStats.rateLimited + + claudeConsoleStats.rateLimited + + geminiStats.rateLimited + + bedrockStats.rateLimited + + openaiStats.rateLimited + + openaiResponsesStats.rateLimited + + ccrStats.rateLimited + rateLimitedDroidAccounts, // 各平台详细统计 accountsByPlatform: { claude: { total: claudeAccounts.length, - normal: normalClaudeAccounts, - abnormal: abnormalClaudeAccounts, - paused: pausedClaudeAccounts, - rateLimited: rateLimitedClaudeAccounts + normal: claudeStats.normal, + abnormal: claudeStats.abnormal, + paused: claudeStats.paused, + rateLimited: claudeStats.rateLimited }, 'claude-console': { total: claudeConsoleAccounts.length, - normal: normalClaudeConsoleAccounts, - abnormal: abnormalClaudeConsoleAccounts, - paused: pausedClaudeConsoleAccounts, - rateLimited: rateLimitedClaudeConsoleAccounts + normal: claudeConsoleStats.normal, + abnormal: claudeConsoleStats.abnormal, + paused: claudeConsoleStats.paused, + rateLimited: claudeConsoleStats.rateLimited }, gemini: { total: geminiAccounts.length, - normal: normalGeminiAccounts, - abnormal: abnormalGeminiAccounts, - paused: pausedGeminiAccounts, - rateLimited: rateLimitedGeminiAccounts + normal: geminiStats.normal, + abnormal: geminiStats.abnormal, + paused: geminiStats.paused, + rateLimited: geminiStats.rateLimited }, bedrock: { total: bedrockAccounts.length, - normal: normalBedrockAccounts, - abnormal: abnormalBedrockAccounts, - paused: pausedBedrockAccounts, - rateLimited: rateLimitedBedrockAccounts + normal: bedrockStats.normal, + abnormal: bedrockStats.abnormal, + paused: bedrockStats.paused, + rateLimited: bedrockStats.rateLimited }, openai: { total: openaiAccounts.length, - normal: normalOpenAIAccounts, - abnormal: abnormalOpenAIAccounts, - paused: pausedOpenAIAccounts, - rateLimited: rateLimitedOpenAIAccounts + normal: openaiStats.normal, + abnormal: openaiStats.abnormal, + paused: openaiStats.paused, + rateLimited: openaiStats.rateLimited }, ccr: { total: ccrAccounts.length, - normal: normalCcrAccounts, - abnormal: abnormalCcrAccounts, - paused: pausedCcrAccounts, - rateLimited: rateLimitedCcrAccounts + normal: ccrStats.normal, + abnormal: ccrStats.abnormal, + paused: ccrStats.paused, + rateLimited: ccrStats.rateLimited }, 'openai-responses': { total: openaiResponsesAccounts.length, - normal: normalOpenAIResponsesAccounts, - abnormal: abnormalOpenAIResponsesAccounts, - paused: pausedOpenAIResponsesAccounts, - rateLimited: rateLimitedOpenAIResponsesAccounts + normal: openaiResponsesStats.normal, + abnormal: openaiResponsesStats.abnormal, + paused: openaiResponsesStats.paused, + rateLimited: openaiResponsesStats.rateLimited }, droid: { total: droidAccounts.length, @@ -418,20 +265,20 @@ router.get('/dashboard', authenticateAdmin, async (req, res) => { }, // 保留旧字段以兼容 activeAccounts: - normalClaudeAccounts + - normalClaudeConsoleAccounts + - normalGeminiAccounts + - normalBedrockAccounts + - normalOpenAIAccounts + - normalOpenAIResponsesAccounts + - normalCcrAccounts + + claudeStats.normal + + claudeConsoleStats.normal + + geminiStats.normal + + bedrockStats.normal + + openaiStats.normal + + openaiResponsesStats.normal + + ccrStats.normal + normalDroidAccounts, totalClaudeAccounts: claudeAccounts.length + claudeConsoleAccounts.length, - activeClaudeAccounts: normalClaudeAccounts + normalClaudeConsoleAccounts, - rateLimitedClaudeAccounts: rateLimitedClaudeAccounts + rateLimitedClaudeConsoleAccounts, + activeClaudeAccounts: claudeStats.normal + claudeConsoleStats.normal, + rateLimitedClaudeAccounts: claudeStats.rateLimited + claudeConsoleStats.rateLimited, totalGeminiAccounts: geminiAccounts.length, - activeGeminiAccounts: normalGeminiAccounts, - rateLimitedGeminiAccounts, + activeGeminiAccounts: geminiStats.normal, + rateLimitedGeminiAccounts: geminiStats.rateLimited, totalTokensUsed, totalRequestsUsed, totalInputTokensUsed, @@ -461,8 +308,8 @@ router.get('/dashboard', authenticateAdmin, async (req, res) => { }, systemHealth: { redisConnected: redis.isConnected, - claudeAccountsHealthy: normalClaudeAccounts + normalClaudeConsoleAccounts > 0, - geminiAccountsHealthy: normalGeminiAccounts > 0, + claudeAccountsHealthy: claudeStats.normal + claudeConsoleStats.normal > 0, + geminiAccountsHealthy: geminiStats.normal > 0, droidAccountsHealthy: normalDroidAccounts > 0, uptime: process.uptime() }, @@ -482,7 +329,7 @@ router.get('/usage-stats', authenticateAdmin, async (req, res) => { const { period = 'daily' } = req.query // daily, monthly // 获取基础API Key统计 - const apiKeys = await apiKeyService.getAllApiKeys() + const apiKeys = await apiKeyService.getAllApiKeysFast() const stats = apiKeys.map((key) => ({ keyId: key.id, @@ -512,55 +359,48 @@ router.get('/model-stats', authenticateAdmin, async (req, res) => { `📊 Getting global model stats, period: ${period}, startDate: ${startDate}, endDate: ${endDate}, today: ${today}, currentMonth: ${currentMonth}` ) - const client = redis.getClientSafe() - - // 获取所有模型的统计数据 - let searchPatterns = [] + // 收集所有需要扫描的日期 + const datePatterns = [] if (startDate && endDate) { - // 自定义日期范围，生成多个日期的搜索模式 + // 自定义日期范围 const start = new Date(startDate) const end = new Date(endDate) - // 确保日期范围有效 if (start > end) { return res.status(400).json({ error: 'Start date must be before or equal to end date' }) } - // 限制最大范围为365天 const daysDiff = Math.ceil((end - start) / (1000 * 60 * 60 * 24)) + 1 if (daysDiff > 365) { return res.status(400).json({ error: 'Date range cannot exceed 365 days' }) } - // 生成日期范围内所有日期的搜索模式 const currentDate = new Date(start) while (currentDate <= end) { const dateStr = redis.getDateStringInTimezone(currentDate) - searchPatterns.push(`usage:model:daily:*:${dateStr}`) + datePatterns.push({ dateStr, pattern: `usage:model:daily:*:${dateStr}` }) currentDate.setDate(currentDate.getDate() + 1) } - logger.info(`📊 Generated ${searchPatterns.length} search patterns for date range`) + logger.info(`📊 Generated ${datePatterns.length} search patterns for date range`) } else { // 使用默认的period const pattern = period === 'daily' ? `usage:model:daily:*:${today}` : `usage:model:monthly:*:${currentMonth}` - searchPatterns = [pattern] + datePatterns.push({ dateStr: period === 'daily' ? today : currentMonth, pattern }) } - logger.info('📊 Searching patterns:', searchPatterns) - - // 获取所有匹配的keys - const allKeys = [] - for (const pattern of searchPatterns) { - const keys = await client.keys(pattern) - allKeys.push(...keys) + // 按日期集合扫描，串行避免并行触发多次全库 SCAN + const allResults = [] + for (const { pattern } of datePatterns) { + const results = await redis.scanAndGetAllChunked(pattern) + allResults.push(...results) } - logger.info(`📊 Found ${allKeys.length} matching keys in total`) + logger.info(`📊 Found ${allResults.length} matching keys in total`) // 模型名标准化函数（与redis.js保持一致） const normalizeModelName = (model) => { @@ -570,23 +410,23 @@ router.get('/model-stats', authenticateAdmin, async (req, res) => { // 对于Bedrock模型，去掉区域前缀进行统一 if (model.includes('.anthropic.') || model.includes('.claude')) { - // 匹配所有AWS区域格式：region.anthropic.model-name-v1:0 -> claude-model-name - // 支持所有AWS区域格式，如：us-east-1, eu-west-1, ap-southeast-1, ca-central-1等 - let normalized = model.replace(/^[a-z0-9-]+\./, '') // 去掉任何区域前缀（更通用） - normalized = normalized.replace('anthropic.', '') // 去掉anthropic前缀 - normalized = normalized.replace(/-v\d+:\d+$/, '') // 去掉版本后缀（如-v1:0, -v2:1等） + let normalized = model.replace(/^[a-z0-9-]+\./, '') + normalized = normalized.replace('anthropic.', '') + normalized = normalized.replace(/-v\d+:\d+$/, '') return normalized } - // 对于其他模型，去掉常见的版本后缀 return model.replace(/-v\d+:\d+$|:latest$/, '') } // 聚合相同模型的数据 const modelStatsMap = new Map() - for (const key of allKeys) { - const match = key.match(/usage:model:daily:(.+):\d{4}-\d{2}-\d{2}$/) + for (const { key, data } of allResults) { + // 支持 daily 和 monthly 两种格式 + const match = + key.match(/usage:model:daily:(.+):\d{4}-\d{2}-\d{2}$/) || + key.match(/usage:model:monthly:(.+):\d{4}-\d{2}$/) if (!match) { logger.warn(`📊 Pattern mismatch for key: ${key}`) @@ -595,7 +435,6 @@ router.get('/model-stats', authenticateAdmin, async (req, res) => { const rawModel = match[1] const normalizedModel = normalizeModelName(rawModel) - const data = await client.hgetall(key) if (data && Object.keys(data).length > 0) { const stats = modelStatsMap.get(normalizedModel) || { diff --git a/src/routes/admin/droidAccounts.js b/src/routes/admin/droidAccounts.js index 1c012947..ba045de2 100644 --- a/src/routes/admin/droidAccounts.js +++ b/src/routes/admin/droidAccounts.js @@ -2,6 +2,7 @@ const express = require('express') const crypto = require('crypto') const droidAccountService = require('../../services/droidAccountService') const accountGroupService = require('../../services/accountGroupService') +const apiKeyService = require('../../services/apiKeyService') const redis = require('../../models/redis') const { authenticateAdmin } = require('../../middleware/auth') const logger = require('../../utils/logger') @@ -142,67 +143,112 @@ router.post('/droid-accounts/exchange-code', authenticateAdmin, async (req, res) router.get('/droid-accounts', authenticateAdmin, async (req, res) => { try { const accounts = await droidAccountService.getAllAccounts() - const allApiKeys = await redis.getAllApiKeys() + const accountIds = accounts.map((a) => a.id) - // 添加使用统计 - const accountsWithStats = await Promise.all( - accounts.map(async (account) => { - try { - const usageStats = await redis.getAccountUsageStats(account.id, 'droid') - let groupInfos = [] - try { - groupInfos = await accountGroupService.getAccountGroups(account.id) - } catch (groupError) { - logger.debug(`Failed to get group infos for Droid account ${account.id}:`, groupError) - groupInfos = [] - } + // 并行获取：轻量 API Keys + 分组信息 + daily cost + const [allApiKeys, allGroupInfosMap, dailyCostMap] = await Promise.all([ + apiKeyService.getAllApiKeysLite(), + accountGroupService.batchGetAccountGroupsByIndex(accountIds, 'droid'), + redis.batchGetAccountDailyCost(accountIds) + ]) - const groupIds = groupInfos.map((group) => group.id) - const boundApiKeysCount = allApiKeys.reduce((count, key) => { - const binding = key.droidAccountId - if (!binding) { - return count - } - if (binding === account.id) { - return count + 1 - } - if (binding.startsWith('group:')) { - const groupId = binding.substring('group:'.length) - if (groupIds.includes(groupId)) { - return count + 1 - } - } - return count - }, 0) + // 构建绑定数映射（droid 需要展开 group 绑定） + // 1. 先构建 groupId -> accountIds 映射 + const groupToAccountIds = new Map() + for (const [accountId, groups] of allGroupInfosMap) { + for (const group of groups) { + if (!groupToAccountIds.has(group.id)) groupToAccountIds.set(group.id, []) + groupToAccountIds.get(group.id).push(accountId) + } + } - const formattedAccount = formatAccountExpiry(account) - return { - ...formattedAccount, - schedulable: account.schedulable === 'true', - boundApiKeysCount, - groupInfos, - usage: { - daily: usageStats.daily, - total: usageStats.total, - averages: usageStats.averages - } - } - } catch (error) { - logger.warn(`Failed to get stats for Droid account ${account.id}:`, error.message) - const formattedAccount = formatAccountExpiry(account) - return { - ...formattedAccount, - boundApiKeysCount: 0, - groupInfos: [], - usage: { - daily: { tokens: 0, requests: 0 }, - total: { tokens: 0, requests: 0 }, - averages: { rpm: 0, tpm: 0 } - } - } - } + // 2. 单次遍历构建绑定数 + const directBindingCount = new Map() + const groupBindingCount = new Map() + for (const key of allApiKeys) { + const binding = key.droidAccountId + if (!binding) continue + if (binding.startsWith('group:')) { + const groupId = binding.substring('group:'.length) + groupBindingCount.set(groupId, (groupBindingCount.get(groupId) || 0) + 1) + } else { + directBindingCount.set(binding, (directBindingCount.get(binding) || 0) + 1) + } + } + + // 批量获取使用统计 + const client = redis.getClientSafe() + const today = redis.getDateStringInTimezone() + const tzDate = redis.getDateInTimezone() + const currentMonth = `${tzDate.getUTCFullYear()}-${String(tzDate.getUTCMonth() + 1).padStart(2, '0')}` + + const statsPipeline = client.pipeline() + for (const accountId of accountIds) { + statsPipeline.hgetall(`account_usage:${accountId}`) + statsPipeline.hgetall(`account_usage:daily:${accountId}:${today}`) + statsPipeline.hgetall(`account_usage:monthly:${accountId}:${currentMonth}`) + } + const statsResults = await statsPipeline.exec() + + // 处理统计数据 + const allUsageStatsMap = new Map() + for (let i = 0; i < accountIds.length; i++) { + const accountId = accountIds[i] + const [errTotal, total] = statsResults[i * 3] + const [errDaily, daily] = statsResults[i * 3 + 1] + const [errMonthly, monthly] = statsResults[i * 3 + 2] + + const parseUsage = (data) => ({ + requests: parseInt(data?.totalRequests || data?.requests) || 0, + tokens: parseInt(data?.totalTokens || data?.tokens) || 0, + inputTokens: parseInt(data?.totalInputTokens || data?.inputTokens) || 0, + outputTokens: parseInt(data?.totalOutputTokens || data?.outputTokens) || 0, + cacheCreateTokens: parseInt(data?.totalCacheCreateTokens || data?.cacheCreateTokens) || 0, + cacheReadTokens: parseInt(data?.totalCacheReadTokens || data?.cacheReadTokens) || 0, + allTokens: + parseInt(data?.totalAllTokens || data?.allTokens) || + (parseInt(data?.totalInputTokens || data?.inputTokens) || 0) + + (parseInt(data?.totalOutputTokens || data?.outputTokens) || 0) + + (parseInt(data?.totalCacheCreateTokens || data?.cacheCreateTokens) || 0) + + (parseInt(data?.totalCacheReadTokens || data?.cacheReadTokens) || 0) }) - ) + + allUsageStatsMap.set(accountId, { + total: errTotal ? {} : parseUsage(total), + daily: errDaily ? {} : parseUsage(daily), + monthly: errMonthly ? {} : parseUsage(monthly) + }) + } + + // 处理账户数据 + const accountsWithStats = accounts.map((account) => { + const groupInfos = allGroupInfosMap.get(account.id) || [] + const usageStats = allUsageStatsMap.get(account.id) || { + daily: { tokens: 0, requests: 0 }, + total: { tokens: 0, requests: 0 }, + monthly: { tokens: 0, requests: 0 } + } + const dailyCost = dailyCostMap.get(account.id) || 0 + + // 计算绑定数：直接绑定 + 通过 group 绑定 + let boundApiKeysCount = directBindingCount.get(account.id) || 0 + for (const group of groupInfos) { + boundApiKeysCount += groupBindingCount.get(group.id) || 0 + } + + const formattedAccount = formatAccountExpiry(account) + return { + ...formattedAccount, + schedulable: account.schedulable === 'true', + boundApiKeysCount, + groupInfos, + usage: { + daily: { ...usageStats.daily, cost: dailyCost }, + total: usageStats.total, + monthly: usageStats.monthly + } + } + }) return res.json({ success: true, data: accountsWithStats }) } catch (error) { @@ -434,7 +480,7 @@ router.get('/droid-accounts/:id', authenticateAdmin, async (req, res) => { } // 获取绑定的 API Key 数量 - const allApiKeys = await redis.getAllApiKeys() + const allApiKeys = await apiKeyService.getAllApiKeysFast() const groupIds = groupInfos.map((group) => group.id) const boundApiKeysCount = allApiKeys.reduce((count, key) => { const binding = key.droidAccountId diff --git a/src/routes/admin/geminiAccounts.js b/src/routes/admin/geminiAccounts.js index 35419ef8..b2fee5e7 100644 --- a/src/routes/admin/geminiAccounts.js +++ b/src/routes/admin/geminiAccounts.js @@ -66,7 +66,7 @@ router.post('/poll-auth-status', authenticateAdmin, async (req, res) => { const result = await geminiAccountService.pollAuthorizationStatus(sessionId) if (result.success) { - logger.success(`✅ Gemini OAuth authorization successful for session: ${sessionId}`) + logger.success(`Gemini OAuth authorization successful for session: ${sessionId}`) return res.json({ success: true, data: { tokens: result.tokens } }) } else { return res.json({ success: false, error: result.error }) @@ -128,7 +128,7 @@ router.post('/exchange-code', authenticateAdmin, async (req, res) => { await redis.deleteOAuthSession(sessionId) } - logger.success('✅ Successfully exchanged Gemini authorization code') + logger.success('Successfully exchanged Gemini authorization code') return res.json({ success: true, data: { tokens } }) } catch (error) { logger.error('❌ Failed to exchange Gemini authorization code:', error) @@ -483,7 +483,7 @@ router.post('/:id/reset-status', authenticateAdmin, async (req, res) => { const result = await geminiAccountService.resetAccountStatus(id) - logger.success(`✅ Admin reset status for Gemini account: ${id}`) + logger.success(`Admin reset status for Gemini account: ${id}`) return res.json({ success: true, data: result }) } catch (error) { logger.error('❌ Failed to reset Gemini account status:', error) diff --git a/src/routes/admin/geminiApiAccounts.js b/src/routes/admin/geminiApiAccounts.js index df49efa0..598adc91 100644 --- a/src/routes/admin/geminiApiAccounts.js +++ b/src/routes/admin/geminiApiAccounts.js @@ -31,53 +31,106 @@ router.get('/gemini-api-accounts', authenticateAdmin, async (req, res) => { } } - // 处理使用统计和绑定的 API Key 数量 - const accountsWithStats = await Promise.all( - accounts.map(async (account) => { - // 检查并清除过期的限流状态 - await geminiApiAccountService.checkAndClearRateLimit(account.id) + const accountIds = accounts.map((a) => a.id) - // 获取使用统计信息 - let usageStats - try { - usageStats = await redis.getAccountUsageStats(account.id, 'gemini-api') - } catch (error) { - logger.debug(`Failed to get usage stats for Gemini-API account ${account.id}:`, error) - usageStats = { - daily: { requests: 0, tokens: 0, allTokens: 0 }, - total: { requests: 0, tokens: 0, allTokens: 0 }, - monthly: { requests: 0, tokens: 0, allTokens: 0 } - } - } + // 并行获取：轻量 API Keys + 分组信息 + daily cost + 清除限流状态 + const [allApiKeys, allGroupInfosMap, dailyCostMap] = await Promise.all([ + apiKeyService.getAllApiKeysLite(), + accountGroupService.batchGetAccountGroupsByIndex(accountIds, 'gemini'), + redis.batchGetAccountDailyCost(accountIds), + // 批量清除限流状态 + Promise.all(accountIds.map((id) => geminiApiAccountService.checkAndClearRateLimit(id))) + ]) - // 计算绑定的API Key数量（支持 api: 前缀） - const allKeys = await redis.getAllApiKeys() - let boundCount = 0 + // 单次遍历构建绑定数映射（只算直连，不算 group） + const bindingCountMap = new Map() + for (const key of allApiKeys) { + const binding = key.geminiAccountId + if (!binding) continue + // 处理 api: 前缀 + const accountId = binding.startsWith('api:') ? binding.substring(4) : binding + bindingCountMap.set(accountId, (bindingCountMap.get(accountId) || 0) + 1) + } - for (const key of allKeys) { - if (key.geminiAccountId) { - // 检查是否绑定了此 Gemini-API 账户（支持 api: 前缀） - if (key.geminiAccountId === `api:${account.id}`) { - boundCount++ - } - } - } + // 批量获取使用统计 + const client = redis.getClientSafe() + const today = redis.getDateStringInTimezone() + const tzDate = redis.getDateInTimezone() + const currentMonth = `${tzDate.getUTCFullYear()}-${String(tzDate.getUTCMonth() + 1).padStart(2, '0')}` - // 获取分组信息 - const groupInfos = await accountGroupService.getAccountGroups(account.id) + const statsPipeline = client.pipeline() + for (const accountId of accountIds) { + statsPipeline.hgetall(`account_usage:${accountId}`) + statsPipeline.hgetall(`account_usage:daily:${accountId}:${today}`) + statsPipeline.hgetall(`account_usage:monthly:${accountId}:${currentMonth}`) + } + const statsResults = await statsPipeline.exec() - return { - ...account, - groupInfos, - usage: { - daily: usageStats.daily, - total: usageStats.total, - averages: usageStats.averages || usageStats.monthly - }, - boundApiKeys: boundCount - } + // 处理统计数据 + const allUsageStatsMap = new Map() + for (let i = 0; i < accountIds.length; i++) { + const accountId = accountIds[i] + const [errTotal, total] = statsResults[i * 3] + const [errDaily, daily] = statsResults[i * 3 + 1] + const [errMonthly, monthly] = statsResults[i * 3 + 2] + + const parseUsage = (data) => ({ + requests: parseInt(data?.totalRequests || data?.requests) || 0, + tokens: parseInt(data?.totalTokens || data?.tokens) || 0, + inputTokens: parseInt(data?.totalInputTokens || data?.inputTokens) || 0, + outputTokens: parseInt(data?.totalOutputTokens || data?.outputTokens) || 0, + cacheCreateTokens: parseInt(data?.totalCacheCreateTokens || data?.cacheCreateTokens) || 0, + cacheReadTokens: parseInt(data?.totalCacheReadTokens || data?.cacheReadTokens) || 0, + allTokens: + parseInt(data?.totalAllTokens || data?.allTokens) || + (parseInt(data?.totalInputTokens || data?.inputTokens) || 0) + + (parseInt(data?.totalOutputTokens || data?.outputTokens) || 0) + + (parseInt(data?.totalCacheCreateTokens || data?.cacheCreateTokens) || 0) + + (parseInt(data?.totalCacheReadTokens || data?.cacheReadTokens) || 0) }) - ) + + allUsageStatsMap.set(accountId, { + total: errTotal ? {} : parseUsage(total), + daily: errDaily ? {} : parseUsage(daily), + monthly: errMonthly ? {} : parseUsage(monthly) + }) + } + + // 处理账户数据 + const accountsWithStats = accounts.map((account) => { + const groupInfos = allGroupInfosMap.get(account.id) || [] + const usageStats = allUsageStatsMap.get(account.id) || { + daily: { requests: 0, tokens: 0, allTokens: 0 }, + total: { requests: 0, tokens: 0, allTokens: 0 }, + monthly: { requests: 0, tokens: 0, allTokens: 0 } + } + const dailyCost = dailyCostMap.get(account.id) || 0 + const boundCount = bindingCountMap.get(account.id) || 0 + + // 计算 averages（rpm/tpm） + const createdAt = account.createdAt ? new Date(account.createdAt) : new Date() + const daysSinceCreated = Math.max( + 1, + Math.ceil((Date.now() - createdAt.getTime()) / (1000 * 60 * 60 * 24)) + ) + const totalMinutes = daysSinceCreated * 24 * 60 + const totalRequests = usageStats.total.requests || 0 + const totalTokens = usageStats.total.tokens || usageStats.total.allTokens || 0 + + return { + ...account, + groupInfos, + usage: { + daily: { ...usageStats.daily, cost: dailyCost }, + total: usageStats.total, + averages: { + rpm: Math.round((totalRequests / totalMinutes) * 100) / 100, + tpm: Math.round((totalTokens / totalMinutes) * 100) / 100 + } + }, + boundApiKeys: boundCount + } + }) res.json({ success: true, data: accountsWithStats }) } catch (error) { @@ -275,7 +328,7 @@ router.delete('/gemini-api-accounts/:id', authenticateAdmin, async (req, res) => message += `，${unboundCount} 个 API Key 已切换为共享池模式` } - logger.success(`✅ ${message}`) + logger.success(`${message}`) res.json({ success: true, @@ -389,7 +442,7 @@ router.post('/gemini-api-accounts/:id/reset-status', authenticateAdmin, async (r const result = await geminiApiAccountService.resetAccountStatus(id) - logger.success(`✅ Admin reset status for Gemini-API account: ${id}`) + logger.success(`Admin reset status for Gemini-API account: ${id}`) return res.json({ success: true, data: result }) } catch (error) { logger.error('❌ Failed to reset Gemini-API account status:', error) diff --git a/src/routes/admin/openaiAccounts.js b/src/routes/admin/openaiAccounts.js index 64575160..50f46c44 100644 --- a/src/routes/admin/openaiAccounts.js +++ b/src/routes/admin/openaiAccounts.js @@ -80,7 +80,7 @@ router.post('/generate-auth-url', authenticateAdmin, async (req, res) => { const authUrl = `${OPENAI_CONFIG.BASE_URL}/oauth/authorize?${params.toString()}` - logger.success('🔗 Generated OpenAI OAuth authorization URL') + logger.success('Generated OpenAI OAuth authorization URL') return res.json({ success: true, @@ -191,7 +191,7 @@ router.post('/exchange-code', authenticateAdmin, async (req, res) => { // 清理 Redis 会话 await redis.deleteOAuthSession(sessionId) - logger.success('✅ OpenAI OAuth token exchange successful') + logger.success('OpenAI OAuth token exchange successful') return res.json({ success: true, @@ -386,7 +386,7 @@ router.post('/', authenticateAdmin, async (req, res) => { delete refreshedAccount.accessToken delete refreshedAccount.refreshToken - logger.success(`✅ 创建并验证 OpenAI 账户成功: ${name} (ID: ${tempAccount.id})`) + logger.success(`创建并验证 OpenAI 账户成功: ${name} (ID: ${tempAccount.id})`) return res.json({ success: true, @@ -450,7 +450,7 @@ router.post('/', authenticateAdmin, async (req, res) => { } } - logger.success(`✅ 创建 OpenAI 账户成功: ${name} (ID: ${createdAccount.id})`) + logger.success(`创建 OpenAI 账户成功: ${name} (ID: ${createdAccount.id})`) return res.json({ success: true, @@ -541,7 +541,7 @@ router.put('/:id', authenticateAdmin, async (req, res) => { }) } - logger.success(`✅ Token 验证成功，继续更新账户信息`) + logger.success(`Token 验证成功，继续更新账户信息`) } catch (refreshError) { // 刷新失败，恢复原始 token logger.warn(`❌ Token 验证失败，恢复原始配置: ${refreshError.message}`) @@ -755,7 +755,7 @@ router.post('/:accountId/reset-status', authenticateAdmin, async (req, res) => { const result = await openaiAccountService.resetAccountStatus(accountId) - logger.success(`✅ Admin reset status for OpenAI account: ${accountId}`) + logger.success(`Admin reset status for OpenAI account: ${accountId}`) return res.json({ success: true, data: result }) } catch (error) { logger.error('❌ Failed to reset OpenAI account status:', error) diff --git a/src/routes/admin/openaiResponsesAccounts.js b/src/routes/admin/openaiResponsesAccounts.js index d4b8f836..dc66ffd9 100644 --- a/src/routes/admin/openaiResponsesAccounts.js +++ b/src/routes/admin/openaiResponsesAccounts.js @@ -39,92 +39,95 @@ router.get('/openai-responses-accounts', authenticateAdmin, async (req, res) => } } - // 处理额度信息、使用统计和绑定的 API Key 数量 - const accountsWithStats = await Promise.all( - accounts.map(async (account) => { - try { - // 检查是否需要重置额度 - const today = redis.getDateStringInTimezone() - if (account.lastResetDate !== today) { - // 今天还没重置过，需要重置 - await openaiResponsesAccountService.updateAccount(account.id, { - dailyUsage: '0', - lastResetDate: today, - quotaStoppedAt: '' - }) - account.dailyUsage = '0' - account.lastResetDate = today - account.quotaStoppedAt = '' - } + const accountIds = accounts.map((a) => a.id) - // 检查并清除过期的限流状态 - await openaiResponsesAccountService.checkAndClearRateLimit(account.id) + // 并行获取：轻量 API Keys + 分组信息 + daily cost + 清理限流状态 + const [allApiKeys, allGroupInfosMap, dailyCostMap] = await Promise.all([ + apiKeyService.getAllApiKeysLite(), + accountGroupService.batchGetAccountGroupsByIndex(accountIds, 'openai'), + redis.batchGetAccountDailyCost(accountIds), + // 批量清理限流状态 + Promise.all(accountIds.map((id) => openaiResponsesAccountService.checkAndClearRateLimit(id))) + ]) - // 获取使用统计信息 - let usageStats - try { - usageStats = await redis.getAccountUsageStats(account.id, 'openai-responses') - } catch (error) { - logger.debug( - `Failed to get usage stats for OpenAI-Responses account ${account.id}:`, - error - ) - usageStats = { - daily: { requests: 0, tokens: 0, allTokens: 0 }, - total: { requests: 0, tokens: 0, allTokens: 0 }, - monthly: { requests: 0, tokens: 0, allTokens: 0 } - } - } + // 单次遍历构建绑定数映射（只算直连，不算 group） + const bindingCountMap = new Map() + for (const key of allApiKeys) { + const binding = key.openaiAccountId + if (!binding) continue + // 处理 responses: 前缀 + const accountId = binding.startsWith('responses:') ? binding.substring(10) : binding + bindingCountMap.set(accountId, (bindingCountMap.get(accountId) || 0) + 1) + } - // 计算绑定的API Key数量（支持 responses: 前缀） - const allKeys = await redis.getAllApiKeys() - let boundCount = 0 + // 批量获取使用统计（不含 daily cost，已单独获取） + const client = redis.getClientSafe() + const today = redis.getDateStringInTimezone() + const tzDate = redis.getDateInTimezone() + const currentMonth = `${tzDate.getUTCFullYear()}-${String(tzDate.getUTCMonth() + 1).padStart(2, '0')}` - for (const key of allKeys) { - // 检查是否绑定了该账户（包括 responses: 前缀） - if ( - key.openaiAccountId === account.id || - key.openaiAccountId === `responses:${account.id}` - ) { - boundCount++ - } - } + const statsPipeline = client.pipeline() + for (const accountId of accountIds) { + statsPipeline.hgetall(`account_usage:${accountId}`) + statsPipeline.hgetall(`account_usage:daily:${accountId}:${today}`) + statsPipeline.hgetall(`account_usage:monthly:${accountId}:${currentMonth}`) + } + const statsResults = await statsPipeline.exec() - // 调试日志：检查绑定计数 - if (boundCount > 0) { - logger.info(`OpenAI-Responses account ${account.id} has ${boundCount} bound API keys`) - } + // 处理统计数据 + const allUsageStatsMap = new Map() + for (let i = 0; i < accountIds.length; i++) { + const accountId = accountIds[i] + const [errTotal, total] = statsResults[i * 3] + const [errDaily, daily] = statsResults[i * 3 + 1] + const [errMonthly, monthly] = statsResults[i * 3 + 2] - // 获取分组信息 - const groupInfos = await accountGroupService.getAccountGroups(account.id) - - const formattedAccount = formatAccountExpiry(account) - return { - ...formattedAccount, - groupInfos, - boundApiKeysCount: boundCount, - usage: { - daily: usageStats.daily, - total: usageStats.total, - monthly: usageStats.monthly - } - } - } catch (error) { - logger.error(`Failed to process OpenAI-Responses account ${account.id}:`, error) - const formattedAccount = formatAccountExpiry(account) - return { - ...formattedAccount, - groupInfos: [], - boundApiKeysCount: 0, - usage: { - daily: { requests: 0, tokens: 0, allTokens: 0 }, - total: { requests: 0, tokens: 0, allTokens: 0 }, - monthly: { requests: 0, tokens: 0, allTokens: 0 } - } - } - } + const parseUsage = (data) => ({ + requests: parseInt(data?.totalRequests || data?.requests) || 0, + tokens: parseInt(data?.totalTokens || data?.tokens) || 0, + inputTokens: parseInt(data?.totalInputTokens || data?.inputTokens) || 0, + outputTokens: parseInt(data?.totalOutputTokens || data?.outputTokens) || 0, + cacheCreateTokens: parseInt(data?.totalCacheCreateTokens || data?.cacheCreateTokens) || 0, + cacheReadTokens: parseInt(data?.totalCacheReadTokens || data?.cacheReadTokens) || 0, + allTokens: + parseInt(data?.totalAllTokens || data?.allTokens) || + (parseInt(data?.totalInputTokens || data?.inputTokens) || 0) + + (parseInt(data?.totalOutputTokens || data?.outputTokens) || 0) + + (parseInt(data?.totalCacheCreateTokens || data?.cacheCreateTokens) || 0) + + (parseInt(data?.totalCacheReadTokens || data?.cacheReadTokens) || 0) }) - ) + + allUsageStatsMap.set(accountId, { + total: errTotal ? {} : parseUsage(total), + daily: errDaily ? {} : parseUsage(daily), + monthly: errMonthly ? {} : parseUsage(monthly) + }) + } + + // 处理额度信息、使用统计和绑定的 API Key 数量 + const accountsWithStats = accounts.map((account) => { + const usageStats = allUsageStatsMap.get(account.id) || { + daily: { requests: 0, tokens: 0, allTokens: 0 }, + total: { requests: 0, tokens: 0, allTokens: 0 }, + monthly: { requests: 0, tokens: 0, allTokens: 0 } + } + + const groupInfos = allGroupInfosMap.get(account.id) || [] + const boundCount = bindingCountMap.get(account.id) || 0 + const dailyCost = dailyCostMap.get(account.id) || 0 + + const formattedAccount = formatAccountExpiry(account) + return { + ...formattedAccount, + groupInfos, + boundApiKeysCount: boundCount, + usage: { + daily: { ...usageStats.daily, cost: dailyCost }, + total: usageStats.total, + monthly: usageStats.monthly + } + } + }) res.json({ success: true, data: accountsWithStats }) } catch (error) { @@ -413,7 +416,7 @@ router.post('/openai-responses-accounts/:id/reset-status', authenticateAdmin, as const result = await openaiResponsesAccountService.resetAccountStatus(id) - logger.success(`✅ Admin reset status for OpenAI-Responses account: ${id}`) + logger.success(`Admin reset status for OpenAI-Responses account: ${id}`) return res.json({ success: true, data: result }) } catch (error) { logger.error('❌ Failed to reset OpenAI-Responses account status:', error) @@ -432,7 +435,7 @@ router.post('/openai-responses-accounts/:id/reset-usage', authenticateAdmin, asy quotaStoppedAt: '' }) - logger.success(`✅ Admin manually reset daily usage for OpenAI-Responses account ${id}`) + logger.success(`Admin manually reset daily usage for OpenAI-Responses account ${id}`) res.json({ success: true, diff --git a/src/routes/admin/sync.js b/src/routes/admin/sync.js index 6345e810..ab0ef219 100644 --- a/src/routes/admin/sync.js +++ b/src/routes/admin/sync.js @@ -288,10 +288,12 @@ router.get('/sync/export-accounts', authenticateAdmin, async (req, res) => { // ===== OpenAI OAuth accounts ===== const openaiOAuthAccounts = [] { - const client = redis.getClientSafe() - const openaiKeys = await client.keys('openai:account:*') - for (const key of openaiKeys) { - const id = key.split(':').slice(2).join(':') + const openaiIds = await redis.getAllIdsByIndex( + 'openai:account:index', + 'openai:account:*', + /^openai:account:(.+)$/ + ) + for (const id of openaiIds) { const account = await openaiAccountService.getAccount(id) if (!account) { continue @@ -390,10 +392,12 @@ router.get('/sync/export-accounts', authenticateAdmin, async (req, res) => { // ===== OpenAI Responses API Key accounts ===== const openaiResponsesAccounts = [] - const client = redis.getClientSafe() - const openaiResponseKeys = await client.keys('openai_responses_account:*') - for (const key of openaiResponseKeys) { - const id = key.split(':').slice(1).join(':') + const openaiResponseIds = await redis.getAllIdsByIndex( + 'openai_responses_account:index', + 'openai_responses_account:*', + /^openai_responses_account:(.+)$/ + ) + for (const id of openaiResponseIds) { const full = await openaiResponsesAccountService.getAccount(id) if (!full) { continue diff --git a/src/routes/admin/usageStats.js b/src/routes/admin/usageStats.js index bf35f8a0..7264a387 100644 --- a/src/routes/admin/usageStats.js +++ b/src/routes/admin/usageStats.js @@ -16,6 +16,81 @@ const pricingService = require('../../services/pricingService') const router = express.Router() +// 辅助函数：通过索引获取数据，回退到 SCAN +// keyPattern 支持占位符：{id}、{keyId}+{model}、{accountId}+{model} +async function getUsageDataByIndex(indexKey, keyPattern, scanPattern) { + const members = await redis.client.smembers(indexKey) + if (members && members.length > 0) { + const keys = members.map((id) => { + // 检查是否是 keymodel 格式 (keyId:model) + if (keyPattern.includes('{keyId}') && keyPattern.includes('{model}')) { + const [keyId, ...modelParts] = id.split(':') + const model = modelParts.join(':') + return keyPattern.replace('{keyId}', keyId).replace('{model}', model) + } + // 检查是否是 accountId:model 格式 + if (keyPattern.includes('{accountId}') && keyPattern.includes('{model}')) { + const [accountId, ...modelParts] = id.split(':') + const model = modelParts.join(':') + return keyPattern.replace('{accountId}', accountId).replace('{model}', model) + } + return keyPattern.replace('{id}', id) + }) + const dataList = await redis.batchHgetallChunked(keys) + const result = [] + keys.forEach((key, i) => { + if (dataList[i] && Object.keys(dataList[i]).length > 0) { + result.push({ key, data: dataList[i] }) + } + }) + return result + } + // 索引为空，检查空标记 + const emptyMarker = await redis.client.get(`${indexKey}:empty`) + if (emptyMarker === '1') { + return [] + } + // 回退到 SCAN（兼容历史数据） + const keys = await redis.scanKeys(scanPattern) + if (keys.length === 0) { + // 设置空标记，1小时过期 + await redis.client.setex(`${indexKey}:empty`, 3600, '1') + return [] + } + // 建立索引 + const ids = keys.map((k) => { + if (keyPattern.includes('{keyId}') && keyPattern.includes('{model}')) { + // keymodel 格式：usage:{keyId}:model:daily:{model}:{date} 或 hourly + const match = + k.match(/usage:([^:]+):model:daily:(.+):\d{4}-\d{2}-\d{2}$/) || + k.match(/usage:([^:]+):model:hourly:(.+):\d{4}-\d{2}-\d{2}:\d{2}$/) + if (match) return `${match[1]}:${match[2]}` + } + if (keyPattern.includes('{accountId}') && keyPattern.includes('{model}')) { + // account_usage:model:daily 或 hourly + const match = + k.match(/account_usage:model:daily:([^:]+):(.+):\d{4}-\d{2}-\d{2}$/) || + k.match(/account_usage:model:hourly:([^:]+):(.+):\d{4}-\d{2}-\d{2}:\d{2}$/) + if (match) return `${match[1]}:${match[2]}` + } + // 通用格式：提取最后一个 : 前的 id + const parts = k.split(':') + return parts[parts.length - 2] + }) + const validIds = ids.filter(Boolean) + if (validIds.length > 0) { + await redis.client.sadd(indexKey, ...validIds) + } + const dataList = await redis.batchHgetallChunked(keys) + const result = [] + keys.forEach((key, i) => { + if (dataList[i] && Object.keys(dataList[i]).length > 0) { + result.push({ key, data: dataList[i] }) + } + }) + return result +} + const accountTypeNames = { claude: 'Claude官方', 'claude-console': 'Claude Console', @@ -244,17 +319,16 @@ router.get('/accounts/:accountId/usage-history', authenticateAdmin, async (req, const sumModelCostsForDay = async (dateKey) => { const modelPattern = `account_usage:model:daily:${accountId}:*:${dateKey}` - const modelKeys = await client.keys(modelPattern) + const modelResults = await redis.scanAndGetAllChunked(modelPattern) let summedCost = 0 - if (modelKeys.length === 0) { + if (modelResults.length === 0) { return summedCost } - for (const modelKey of modelKeys) { + for (const { key: modelKey, data: modelData } of modelResults) { const modelParts = modelKey.split(':') const modelName = modelParts[4] || 'unknown' - const modelData = await client.hgetall(modelKey) if (!modelData || Object.keys(modelData).length === 0) { continue } @@ -410,7 +484,6 @@ router.get('/accounts/:accountId/usage-history', authenticateAdmin, async (req, router.get('/usage-trend', authenticateAdmin, async (req, res) => { try { const { days = 7, granularity = 'day', startDate, endDate } = req.query - const client = redis.getClientSafe() const trendData = [] @@ -419,21 +492,9 @@ router.get('/usage-trend', authenticateAdmin, async (req, res) => { let startTime, endTime if (startDate && endDate) { - // 使用自定义时间范围 startTime = new Date(startDate) endTime = new Date(endDate) - - // 调试日志 - logger.info('📊 Usage trend hour granularity - received times:') - logger.info(` startDate (raw): ${startDate}`) - logger.info(` endDate (raw): ${endDate}`) - logger.info(` startTime (parsed): ${startTime.toISOString()}`) - logger.info(` endTime (parsed): ${endTime.toISOString()}`) - logger.info( - ` System timezone offset: ${require('../../../config/config').system.timezoneOffset || 8}` - ) } else { - // 默认最近24小时 endTime = new Date() startTime = new Date(endTime.getTime() - 24 * 60 * 60 * 1000) } @@ -446,21 +507,87 @@ router.get('/usage-trend', authenticateAdmin, async (req, res) => { }) } - // 按小时遍历 + // 收集所有小时的元数据和涉及的日期 + const hourInfos = [] + const dateSet = new Set() const currentHour = new Date(startTime) currentHour.setMinutes(0, 0, 0) while (currentHour <= endTime) { - // 注意：前端发送的时间已经是UTC时间，不需要再次转换 - // 直接从currentHour生成对应系统时区的日期和小时 const tzCurrentHour = redis.getDateInTimezone(currentHour) const dateStr = redis.getDateStringInTimezone(currentHour) const hour = String(tzCurrentHour.getUTCHours()).padStart(2, '0') const hourKey = `${dateStr}:${hour}` - // 获取当前小时的模型统计数据 - const modelPattern = `usage:model:hourly:*:${hourKey}` - const modelKeys = await client.keys(modelPattern) + dateSet.add(dateStr) + + const tzDateForLabel = redis.getDateInTimezone(currentHour) + const month = String(tzDateForLabel.getUTCMonth() + 1).padStart(2, '0') + const day = String(tzDateForLabel.getUTCDate()).padStart(2, '0') + const hourStr = String(tzDateForLabel.getUTCHours()).padStart(2, '0') + + hourInfos.push({ + hourKey, + dateStr, + isoTime: currentHour.toISOString(), + label: `${month}/${day} ${hourStr}:00` + }) + + currentHour.setHours(currentHour.getHours() + 1) + } + + // 使用索引获取数据，按小时批量查询 + const dates = [...dateSet] + const modelDataMap = new Map() + const usageDataMap = new Map() + + // 并行获取所有小时的数据 + const fetchPromises = hourInfos.map(async (hourInfo) => { + const [modelResults, usageResults] = await Promise.all([ + getUsageDataByIndex( + `usage:model:hourly:index:${hourInfo.hourKey}`, + `usage:model:hourly:{id}:${hourInfo.hourKey}`, + `usage:model:hourly:*:${hourInfo.hourKey}` + ), + getUsageDataByIndex( + `usage:hourly:index:${hourInfo.hourKey}`, + `usage:hourly:{id}:${hourInfo.hourKey}`, + `usage:hourly:*:${hourInfo.hourKey}` + ) + ]) + return { modelResults, usageResults } + }) + + const allResults = await Promise.all(fetchPromises) + allResults.forEach(({ modelResults, usageResults }) => { + modelResults.forEach(({ key, data }) => modelDataMap.set(key, data)) + usageResults.forEach(({ key, data }) => usageDataMap.set(key, data)) + }) + + // 按 hourKey 分组 + const modelKeysByHour = new Map() + const usageKeysByHour = new Map() + for (const key of modelDataMap.keys()) { + const match = key.match(/usage:model:hourly:.+?:(\d{4}-\d{2}-\d{2}:\d{2})/) + if (match) { + const hourKey = match[1] + if (!modelKeysByHour.has(hourKey)) modelKeysByHour.set(hourKey, []) + modelKeysByHour.get(hourKey).push(key) + } + } + for (const key of usageDataMap.keys()) { + const match = key.match(/usage:hourly:.+?:(\d{4}-\d{2}-\d{2}:\d{2})/) + if (match) { + const hourKey = match[1] + if (!usageKeysByHour.has(hourKey)) usageKeysByHour.set(hourKey, []) + usageKeysByHour.get(hourKey).push(key) + } + } + + // 处理每个小时的数据 + for (const hourInfo of hourInfos) { + const modelKeys = modelKeysByHour.get(hourInfo.hourKey) || [] + const usageKeys = usageKeysByHour.get(hourInfo.hourKey) || [] let hourInputTokens = 0 let hourOutputTokens = 0 @@ -469,46 +596,41 @@ router.get('/usage-trend', authenticateAdmin, async (req, res) => { let hourCacheReadTokens = 0 let hourCost = 0 + // 处理模型级别数据 for (const modelKey of modelKeys) { - const modelMatch = modelKey.match(/usage:model:hourly:(.+):\d{4}-\d{2}-\d{2}:\d{2}$/) - if (!modelMatch) { - continue - } + const modelMatch = modelKey.match(/usage:model:hourly:(.+?):\d{4}-\d{2}-\d{2}:\d{2}/) + if (!modelMatch) continue const model = modelMatch[1] - const data = await client.hgetall(modelKey) + const data = modelDataMap.get(modelKey) + if (!data || Object.keys(data).length === 0) continue - if (data && Object.keys(data).length > 0) { - const modelInputTokens = parseInt(data.inputTokens) || 0 - const modelOutputTokens = parseInt(data.outputTokens) || 0 - const modelCacheCreateTokens = parseInt(data.cacheCreateTokens) || 0 - const modelCacheReadTokens = parseInt(data.cacheReadTokens) || 0 - const modelRequests = parseInt(data.requests) || 0 + const modelInputTokens = parseInt(data.inputTokens) || 0 + const modelOutputTokens = parseInt(data.outputTokens) || 0 + const modelCacheCreateTokens = parseInt(data.cacheCreateTokens) || 0 + const modelCacheReadTokens = parseInt(data.cacheReadTokens) || 0 + const modelRequests = parseInt(data.requests) || 0 - hourInputTokens += modelInputTokens - hourOutputTokens += modelOutputTokens - hourCacheCreateTokens += modelCacheCreateTokens - hourCacheReadTokens += modelCacheReadTokens - hourRequests += modelRequests + hourInputTokens += modelInputTokens + hourOutputTokens += modelOutputTokens + hourCacheCreateTokens += modelCacheCreateTokens + hourCacheReadTokens += modelCacheReadTokens + hourRequests += modelRequests - const modelUsage = { - input_tokens: modelInputTokens, - output_tokens: modelOutputTokens, - cache_creation_input_tokens: modelCacheCreateTokens, - cache_read_input_tokens: modelCacheReadTokens - } - const modelCostResult = CostCalculator.calculateCost(modelUsage, model) - hourCost += modelCostResult.costs.total + const modelUsage = { + input_tokens: modelInputTokens, + output_tokens: modelOutputTokens, + cache_creation_input_tokens: modelCacheCreateTokens, + cache_read_input_tokens: modelCacheReadTokens } + const modelCostResult = CostCalculator.calculateCost(modelUsage, model) + hourCost += modelCostResult.costs.total } // 如果没有模型级别的数据，尝试API Key级别的数据 if (modelKeys.length === 0) { - const pattern = `usage:hourly:*:${hourKey}` - const keys = await client.keys(pattern) - - for (const key of keys) { - const data = await client.hgetall(key) + for (const key of usageKeys) { + const data = usageDataMap.get(key) if (data) { hourInputTokens += parseInt(data.inputTokens) || 0 hourOutputTokens += parseInt(data.outputTokens) || 0 @@ -528,16 +650,9 @@ router.get('/usage-trend', authenticateAdmin, async (req, res) => { hourCost = costResult.costs.total } - // 格式化时间标签 - 使用系统时区的显示 - const tzDateForLabel = redis.getDateInTimezone(currentHour) - const month = String(tzDateForLabel.getUTCMonth() + 1).padStart(2, '0') - const day = String(tzDateForLabel.getUTCDate()).padStart(2, '0') - const hourStr = String(tzDateForLabel.getUTCHours()).padStart(2, '0') - trendData.push({ - // 对于小时粒度，只返回hour字段，不返回date字段 - hour: currentHour.toISOString(), // 保留原始ISO时间用于排序 - label: `${month}/${day} ${hourStr}:00`, // 添加格式化的标签 + hour: hourInfo.isoTime, + label: hourInfo.label, inputTokens: hourInputTokens, outputTokens: hourOutputTokens, requests: hourRequests, @@ -547,24 +662,71 @@ router.get('/usage-trend', authenticateAdmin, async (req, res) => { hourInputTokens + hourOutputTokens + hourCacheCreateTokens + hourCacheReadTokens, cost: hourCost }) - - // 移到下一个小时 - currentHour.setHours(currentHour.getHours() + 1) } } else { - // 天粒度统计（保持原有逻辑） + // 天粒度统计（按日期集合扫描） const daysCount = parseInt(days) || 7 const today = new Date() - // 获取过去N天的数据 + // 收集所有天的元数据 + const dayInfos = [] for (let i = 0; i < daysCount; i++) { const date = new Date(today) date.setDate(date.getDate() - i) const dateStr = redis.getDateStringInTimezone(date) + dayInfos.push({ dateStr }) + } - // 汇总当天所有API Key的使用数据 - const pattern = `usage:daily:*:${dateStr}` - const keys = await client.keys(pattern) + // 使用索引获取数据，按日期批量查询 + const modelDataMap = new Map() + const usageDataMap = new Map() + + const fetchPromises = dayInfos.map(async (dayInfo) => { + const [modelResults, usageResults] = await Promise.all([ + getUsageDataByIndex( + `usage:model:daily:index:${dayInfo.dateStr}`, + `usage:model:daily:{id}:${dayInfo.dateStr}`, + `usage:model:daily:*:${dayInfo.dateStr}` + ), + getUsageDataByIndex( + `usage:daily:index:${dayInfo.dateStr}`, + `usage:daily:{id}:${dayInfo.dateStr}`, + `usage:daily:*:${dayInfo.dateStr}` + ) + ]) + return { modelResults, usageResults } + }) + + const allResults = await Promise.all(fetchPromises) + allResults.forEach(({ modelResults, usageResults }) => { + modelResults.forEach(({ key, data }) => modelDataMap.set(key, data)) + usageResults.forEach(({ key, data }) => usageDataMap.set(key, data)) + }) + + // 按 dateStr 分组 + const modelKeysByDate = new Map() + const usageKeysByDate = new Map() + for (const key of modelDataMap.keys()) { + const match = key.match(/usage:model:daily:.+?:(\d{4}-\d{2}-\d{2})/) + if (match) { + const dateStr = match[1] + if (!modelKeysByDate.has(dateStr)) modelKeysByDate.set(dateStr, []) + modelKeysByDate.get(dateStr).push(key) + } + } + for (const key of usageDataMap.keys()) { + const match = key.match(/usage:daily:.+?:(\d{4}-\d{2}-\d{2})/) + if (match) { + const dateStr = match[1] + if (!usageKeysByDate.has(dateStr)) usageKeysByDate.set(dateStr, []) + usageKeysByDate.get(dateStr).push(key) + } + } + + // 处理每天的数据 + for (const dayInfo of dayInfos) { + const modelKeys = modelKeysByDate.get(dayInfo.dateStr) || [] + const usageKeys = usageKeysByDate.get(dayInfo.dateStr) || [] let dayInputTokens = 0 let dayOutputTokens = 0 @@ -573,53 +735,41 @@ router.get('/usage-trend', authenticateAdmin, async (req, res) => { let dayCacheReadTokens = 0 let dayCost = 0 - // 按模型统计使用量 - // const modelUsageMap = new Map(); - - // 获取当天所有模型的使用数据 - const modelPattern = `usage:model:daily:*:${dateStr}` - const modelKeys = await client.keys(modelPattern) - + // 处理模型级别数据 for (const modelKey of modelKeys) { - // 解析模型名称 - const modelMatch = modelKey.match(/usage:model:daily:(.+):\d{4}-\d{2}-\d{2}$/) - if (!modelMatch) { - continue - } + const modelMatch = modelKey.match(/usage:model:daily:(.+?):\d{4}-\d{2}-\d{2}/) + if (!modelMatch) continue const model = modelMatch[1] - const data = await client.hgetall(modelKey) + const data = modelDataMap.get(modelKey) + if (!data || Object.keys(data).length === 0) continue - if (data && Object.keys(data).length > 0) { - const modelInputTokens = parseInt(data.inputTokens) || 0 - const modelOutputTokens = parseInt(data.outputTokens) || 0 - const modelCacheCreateTokens = parseInt(data.cacheCreateTokens) || 0 - const modelCacheReadTokens = parseInt(data.cacheReadTokens) || 0 - const modelRequests = parseInt(data.requests) || 0 + const modelInputTokens = parseInt(data.inputTokens) || 0 + const modelOutputTokens = parseInt(data.outputTokens) || 0 + const modelCacheCreateTokens = parseInt(data.cacheCreateTokens) || 0 + const modelCacheReadTokens = parseInt(data.cacheReadTokens) || 0 + const modelRequests = parseInt(data.requests) || 0 - // 累加总数 - dayInputTokens += modelInputTokens - dayOutputTokens += modelOutputTokens - dayCacheCreateTokens += modelCacheCreateTokens - dayCacheReadTokens += modelCacheReadTokens - dayRequests += modelRequests + dayInputTokens += modelInputTokens + dayOutputTokens += modelOutputTokens + dayCacheCreateTokens += modelCacheCreateTokens + dayCacheReadTokens += modelCacheReadTokens + dayRequests += modelRequests - // 按模型计算费用 - const modelUsage = { - input_tokens: modelInputTokens, - output_tokens: modelOutputTokens, - cache_creation_input_tokens: modelCacheCreateTokens, - cache_read_input_tokens: modelCacheReadTokens - } - const modelCostResult = CostCalculator.calculateCost(modelUsage, model) - dayCost += modelCostResult.costs.total + const modelUsage = { + input_tokens: modelInputTokens, + output_tokens: modelOutputTokens, + cache_creation_input_tokens: modelCacheCreateTokens, + cache_read_input_tokens: modelCacheReadTokens } + const modelCostResult = CostCalculator.calculateCost(modelUsage, model) + dayCost += modelCostResult.costs.total } // 如果没有模型级别的数据，回退到原始方法 - if (modelKeys.length === 0 && keys.length > 0) { - for (const key of keys) { - const data = await client.hgetall(key) + if (modelKeys.length === 0 && usageKeys.length > 0) { + for (const key of usageKeys) { + const data = usageDataMap.get(key) if (data) { dayInputTokens += parseInt(data.inputTokens) || 0 dayOutputTokens += parseInt(data.outputTokens) || 0 @@ -629,7 +779,6 @@ router.get('/usage-trend', authenticateAdmin, async (req, res) => { } } - // 使用默认模型价格计算 const usage = { input_tokens: dayInputTokens, output_tokens: dayOutputTokens, @@ -641,7 +790,7 @@ router.get('/usage-trend', authenticateAdmin, async (req, res) => { } trendData.push({ - date: dateStr, + date: dayInfo.dateStr, inputTokens: dayInputTokens, outputTokens: dayOutputTokens, requests: dayRequests, @@ -727,25 +876,29 @@ router.get('/api-keys/:keyId/model-stats', authenticateAdmin, async (req, res) = const modelStatsMap = new Map() const modelStats = [] // 定义结果数组 - for (const pattern of searchPatterns) { - const keys = await client.keys(pattern) - logger.info(`📊 Pattern ${pattern} found ${keys.length} keys`) - - for (const key of keys) { - const match = - key.match(/usage:.+:model:daily:(.+):\d{4}-\d{2}-\d{2}$/) || - key.match(/usage:.+:model:monthly:(.+):\d{4}-\d{2}$/) - - if (!match) { - logger.warn(`📊 Pattern mismatch for key: ${key}`) - continue - } - - const model = match[1] - const data = await client.hgetall(key) - - if (data && Object.keys(data).length > 0) { - // 累加同一模型的数据 + if (period === 'custom' && startDate && endDate) { + // 自定义日期范围，使用索引 + const start = new Date(startDate) + const end = new Date(endDate) + const fetchPromises = [] + for (let d = new Date(start); d <= end; d.setDate(d.getDate() + 1)) { + const dateStr = redis.getDateStringInTimezone(d) + fetchPromises.push( + getUsageDataByIndex( + `usage:keymodel:daily:index:${dateStr}`, + `usage:{keyId}:model:daily:{model}:${dateStr}`, + `usage:*:model:daily:*:${dateStr}` + ) + ) + } + const allResults = await Promise.all(fetchPromises) + for (const results of allResults) { + for (const { key, data } of results) { + // 过滤出属于该 keyId 的记录 + if (!key.startsWith(`usage:${keyId}:model:`)) continue + const match = key.match(/usage:.+:model:daily:(.+):\d{4}-\d{2}-\d{2}$/) + if (!match) continue + const model = match[1] if (!modelStatsMap.has(model)) { modelStatsMap.set(model, { requests: 0, @@ -756,7 +909,6 @@ router.get('/api-keys/:keyId/model-stats', authenticateAdmin, async (req, res) = allTokens: 0 }) } - const stats = modelStatsMap.get(model) stats.requests += parseInt(data.requests) || 0 stats.inputTokens += parseInt(data.inputTokens) || 0 @@ -766,6 +918,45 @@ router.get('/api-keys/:keyId/model-stats', authenticateAdmin, async (req, res) = stats.allTokens += parseInt(data.allTokens) || 0 } } + } else { + // 预设期间，使用索引 + let results + if (period === 'daily') { + results = await getUsageDataByIndex( + `usage:keymodel:daily:index:${today}`, + `usage:{keyId}:model:daily:{model}:${today}`, + `usage:*:model:daily:*:${today}` + ) + } else { + // monthly - 需要月度 keymodel 索引，暂时回退到 SCAN + const pattern = `usage:${keyId}:model:monthly:*:${currentMonth}` + results = await redis.scanAndGetAllChunked(pattern) + } + for (const { key, data } of results) { + if (!key.startsWith(`usage:${keyId}:model:`)) continue + const match = + key.match(/usage:.+:model:daily:(.+):\d{4}-\d{2}-\d{2}$/) || + key.match(/usage:.+:model:monthly:(.+):\d{4}-\d{2}$/) + if (!match) continue + const model = match[1] + if (!modelStatsMap.has(model)) { + modelStatsMap.set(model, { + requests: 0, + inputTokens: 0, + outputTokens: 0, + cacheCreateTokens: 0, + cacheReadTokens: 0, + allTokens: 0 + }) + } + const stats = modelStatsMap.get(model) + stats.requests += parseInt(data.requests) || 0 + stats.inputTokens += parseInt(data.inputTokens) || 0 + stats.outputTokens += parseInt(data.outputTokens) || 0 + stats.cacheCreateTokens += parseInt(data.cacheCreateTokens) || 0 + stats.cacheReadTokens += parseInt(data.cacheReadTokens) || 0 + stats.allTokens += parseInt(data.allTokens) || 0 + } } // 将汇总的数据转换为最终结果 @@ -806,7 +997,7 @@ router.get('/api-keys/:keyId/model-stats', authenticateAdmin, async (req, res) = // 尝试从API Keys列表中获取usage数据作为备选方案 try { - const apiKeys = await apiKeyService.getAllApiKeys() + const apiKeys = await apiKeyService.getAllApiKeysFast() const targetApiKey = apiKeys.find((key) => key.id === keyId) if (targetApiKey && targetApiKey.usage) { @@ -1019,41 +1210,9 @@ router.get('/account-usage-trend', authenticateAdmin, async (req, res) => { } const fallbackModel = fallbackModelByGroup[group] || 'unknown' - const client = redis.getClientSafe() const trendData = [] const accountCostTotals = new Map() - const sumModelCosts = async (accountId, period, timeKey) => { - const modelPattern = `account_usage:model:${period}:${accountId}:*:${timeKey}` - const modelKeys = await client.keys(modelPattern) - let totalCost = 0 - - for (const modelKey of modelKeys) { - const modelData = await client.hgetall(modelKey) - if (!modelData) { - continue - } - - const parts = modelKey.split(':') - if (parts.length < 5) { - continue - } - - const modelName = parts[4] - const usage = { - input_tokens: parseInt(modelData.inputTokens) || 0, - output_tokens: parseInt(modelData.outputTokens) || 0, - cache_creation_input_tokens: parseInt(modelData.cacheCreateTokens) || 0, - cache_read_input_tokens: parseInt(modelData.cacheReadTokens) || 0 - } - - const costResult = CostCalculator.calculateCost(usage, modelName) - totalCost += costResult.costs.total - } - - return totalCost - } - if (granularity === 'hour') { let startTime let endTime @@ -1066,6 +1225,9 @@ router.get('/account-usage-trend', authenticateAdmin, async (req, res) => { startTime = new Date(endTime.getTime() - 24 * 60 * 60 * 1000) } + // 收集所有小时的元数据和涉及的日期 + const hourInfos = [] + const dateSet = new Set() const currentHour = new Date(startTime) currentHour.setMinutes(0, 0, 0) @@ -1075,35 +1237,92 @@ router.get('/account-usage-trend', authenticateAdmin, async (req, res) => { const hour = String(tzCurrentHour.getUTCHours()).padStart(2, '0') const hourKey = `${dateStr}:${hour}` + dateSet.add(dateStr) + const tzDateForLabel = redis.getDateInTimezone(currentHour) const monthLabel = String(tzDateForLabel.getUTCMonth() + 1).padStart(2, '0') const dayLabel = String(tzDateForLabel.getUTCDate()).padStart(2, '0') const hourLabel = String(tzDateForLabel.getUTCHours()).padStart(2, '0') + hourInfos.push({ + hourKey, + dateStr, + isoTime: currentHour.toISOString(), + label: `${monthLabel}/${dayLabel} ${hourLabel}:00` + }) + + currentHour.setHours(currentHour.getHours() + 1) + } + + // 按小时获取 account_usage 数据（避免全库扫描） + const dates = [...dateSet] + const usageDataMap = new Map() + const modelDataMap = new Map() + + // 并行获取每个小时的数据 + const fetchPromises = hourInfos.map(async (hourInfo) => { + const [usageResults, modelResults] = await Promise.all([ + getUsageDataByIndex( + `account_usage:hourly:index:${hourInfo.hourKey}`, + `account_usage:hourly:{id}:${hourInfo.hourKey}`, + `account_usage:hourly:*:${hourInfo.hourKey}` + ), + getUsageDataByIndex( + `account_usage:model:hourly:index:${hourInfo.hourKey}`, + `account_usage:model:hourly:{accountId}:{model}:${hourInfo.hourKey}`, + `account_usage:model:hourly:*:${hourInfo.hourKey}` + ) + ]) + return { usageResults, modelResults } + }) + + const allResults = await Promise.all(fetchPromises) + allResults.forEach(({ usageResults, modelResults }) => { + usageResults.forEach(({ key, data }) => usageDataMap.set(key, data)) + modelResults.forEach(({ key, data }) => modelDataMap.set(key, data)) + }) + + // 按 hourKey 分组 + const usageKeysByHour = new Map() + const modelKeysByHour = new Map() + for (const key of usageDataMap.keys()) { + const match = key.match(/account_usage:hourly:.+?:(\d{4}-\d{2}-\d{2}:\d{2})/) + if (match) { + const hourKey = match[1] + if (!usageKeysByHour.has(hourKey)) usageKeysByHour.set(hourKey, []) + usageKeysByHour.get(hourKey).push(key) + } + } + for (const key of modelDataMap.keys()) { + const match = key.match(/account_usage:model:hourly:(.+?):.+?:(\d{4}-\d{2}-\d{2}:\d{2})/) + if (match) { + const accountId = match[1] + const hourKey = match[2] + const mapKey = `${accountId}:${hourKey}` + if (!modelKeysByHour.has(mapKey)) modelKeysByHour.set(mapKey, []) + modelKeysByHour.get(mapKey).push(key) + } + } + + // 处理每个小时的数据 + for (const hourInfo of hourInfos) { + const usageKeys = usageKeysByHour.get(hourInfo.hourKey) || [] + const hourData = { - hour: currentHour.toISOString(), - label: `${monthLabel}/${dayLabel} ${hourLabel}:00`, + hour: hourInfo.isoTime, + label: hourInfo.label, accounts: {} } - const pattern = `account_usage:hourly:*:${hourKey}` - const keys = await client.keys(pattern) - - for (const key of keys) { + for (const key of usageKeys) { const match = key.match(/account_usage:hourly:(.+?):\d{4}-\d{2}-\d{2}:\d{2}/) - if (!match) { - continue - } + if (!match) continue const accountId = match[1] - if (!accountIdSet.has(accountId)) { - continue - } + if (!accountIdSet.has(accountId)) continue - const data = await client.hgetall(key) - if (!data) { - continue - } + const data = usageDataMap.get(key) + if (!data) continue const inputTokens = parseInt(data.inputTokens) || 0 const outputTokens = parseInt(data.outputTokens) || 0 @@ -1114,7 +1333,27 @@ router.get('/account-usage-trend', authenticateAdmin, async (req, res) => { inputTokens + outputTokens + cacheCreateTokens + cacheReadTokens const requests = parseInt(data.requests) || 0 - let cost = await sumModelCosts(accountId, 'hourly', hourKey) + // 计算模型费用（从预加载的数据中） + let cost = 0 + const modelKeys = modelKeysByHour.get(`${accountId}:${hourInfo.hourKey}`) || [] + for (const modelKey of modelKeys) { + const modelData = modelDataMap.get(modelKey) + if (!modelData) continue + + const parts = modelKey.split(':') + if (parts.length < 5) continue + + const modelName = parts[4] + const usage = { + input_tokens: parseInt(modelData.inputTokens) || 0, + output_tokens: parseInt(modelData.outputTokens) || 0, + cache_creation_input_tokens: parseInt(modelData.cacheCreateTokens) || 0, + cache_read_input_tokens: parseInt(modelData.cacheReadTokens) || 0 + } + + const costResult = CostCalculator.calculateCost(usage, modelName) + cost += costResult.costs.total + } if (cost === 0 && allTokens > 0) { const fallbackUsage = { @@ -1141,40 +1380,93 @@ router.get('/account-usage-trend', authenticateAdmin, async (req, res) => { } trendData.push(hourData) - currentHour.setHours(currentHour.getHours() + 1) } } else { const daysCount = parseInt(days) || 7 const today = new Date() + // 收集所有天的元数据 + const dayInfos = [] for (let i = 0; i < daysCount; i++) { const date = new Date(today) date.setDate(date.getDate() - i) const dateStr = redis.getDateStringInTimezone(date) + dayInfos.push({ dateStr }) + } + + // 使用索引获取数据 + const usagePromises = dayInfos.map((d) => + getUsageDataByIndex( + `account_usage:daily:index:${d.dateStr}`, + `account_usage:daily:{id}:${d.dateStr}`, + `account_usage:daily:*:${d.dateStr}` + ) + ) + const modelPromises = dayInfos.map((d) => + getUsageDataByIndex( + `account_usage:model:daily:index:${d.dateStr}`, + `account_usage:model:daily:{accountId}:{model}:${d.dateStr}`, + `account_usage:model:daily:*:${d.dateStr}` + ) + ) + const [usageResultsArr, modelResultsArr] = await Promise.all([ + Promise.all(usagePromises), + Promise.all(modelPromises) + ]) + + const usageDataMap = new Map() + const modelDataMap = new Map() + for (const results of usageResultsArr) { + for (const { key, data } of results) { + usageDataMap.set(key, data) + } + } + for (const results of modelResultsArr) { + for (const { key, data } of results) { + modelDataMap.set(key, data) + } + } + + // 按 dateStr 分组 + const usageKeysByDate = new Map() + const modelKeysByDate = new Map() + for (const key of usageDataMap.keys()) { + const match = key.match(/account_usage:daily:.+?:(\d{4}-\d{2}-\d{2})/) + if (match) { + const dateStr = match[1] + if (!usageKeysByDate.has(dateStr)) usageKeysByDate.set(dateStr, []) + usageKeysByDate.get(dateStr).push(key) + } + } + for (const key of modelDataMap.keys()) { + const match = key.match(/account_usage:model:daily:(.+?):.+?:(\d{4}-\d{2}-\d{2})/) + if (match) { + const accountId = match[1] + const dateStr = match[2] + const mapKey = `${accountId}:${dateStr}` + if (!modelKeysByDate.has(mapKey)) modelKeysByDate.set(mapKey, []) + modelKeysByDate.get(mapKey).push(key) + } + } + + // 处理每天的数据 + for (const dayInfo of dayInfos) { + const usageKeys = usageKeysByDate.get(dayInfo.dateStr) || [] const dayData = { - date: dateStr, + date: dayInfo.dateStr, accounts: {} } - const pattern = `account_usage:daily:*:${dateStr}` - const keys = await client.keys(pattern) - - for (const key of keys) { + for (const key of usageKeys) { const match = key.match(/account_usage:daily:(.+?):\d{4}-\d{2}-\d{2}/) - if (!match) { - continue - } + if (!match) continue const accountId = match[1] - if (!accountIdSet.has(accountId)) { - continue - } + if (!accountIdSet.has(accountId)) continue - const data = await client.hgetall(key) - if (!data) { - continue - } + const data = usageDataMap.get(key) + if (!data) continue const inputTokens = parseInt(data.inputTokens) || 0 const outputTokens = parseInt(data.outputTokens) || 0 @@ -1185,7 +1477,27 @@ router.get('/account-usage-trend', authenticateAdmin, async (req, res) => { inputTokens + outputTokens + cacheCreateTokens + cacheReadTokens const requests = parseInt(data.requests) || 0 - let cost = await sumModelCosts(accountId, 'daily', dateStr) + // 计算模型费用（从预加载的数据中） + let cost = 0 + const modelKeys = modelKeysByDate.get(`${accountId}:${dayInfo.dateStr}`) || [] + for (const modelKey of modelKeys) { + const modelData = modelDataMap.get(modelKey) + if (!modelData) continue + + const parts = modelKey.split(':') + if (parts.length < 5) continue + + const modelName = parts[4] + const usage = { + input_tokens: parseInt(modelData.inputTokens) || 0, + output_tokens: parseInt(modelData.outputTokens) || 0, + cache_creation_input_tokens: parseInt(modelData.cacheCreateTokens) || 0, + cache_read_input_tokens: parseInt(modelData.cacheReadTokens) || 0 + } + + const costResult = CostCalculator.calculateCost(usage, modelName) + cost += costResult.costs.total + } if (cost === 0 && allTokens > 0) { const fallbackUsage = { @@ -1250,121 +1562,169 @@ router.get('/api-keys-usage-trend', authenticateAdmin, async (req, res) => { logger.info(`📊 Getting API keys usage trend, granularity: ${granularity}, days: ${days}`) - const client = redis.getClientSafe() const trendData = [] - // 获取所有API Keys - const apiKeys = await apiKeyService.getAllApiKeys() - const apiKeyMap = new Map(apiKeys.map((key) => [key.id, key])) + // 获取所有API Keys（只需要 id 和 name，过滤已删除的） + const apiKeyIds = await redis.scanApiKeyIds() + const apiKeyBasicData = await redis.batchGetApiKeys(apiKeyIds) + const apiKeyMap = new Map( + apiKeyBasicData.filter((key) => !key.isDeleted).map((key) => [key.id, key]) + ) if (granularity === 'hour') { // 小时粒度统计 let endTime, startTime if (startDate && endDate) { - // 自定义时间范围 startTime = new Date(startDate) endTime = new Date(endDate) } else { - // 默认近24小时 endTime = new Date() startTime = new Date(endTime.getTime() - 24 * 60 * 60 * 1000) } - // 按小时遍历 + // 收集所有小时的元数据和涉及的日期 + const hourInfos = [] + const dateSet = new Set() const currentHour = new Date(startTime) currentHour.setMinutes(0, 0, 0) while (currentHour <= endTime) { - // 使用时区转换后的时间来生成键 const tzCurrentHour = redis.getDateInTimezone(currentHour) const dateStr = redis.getDateStringInTimezone(currentHour) const hour = String(tzCurrentHour.getUTCHours()).padStart(2, '0') const hourKey = `${dateStr}:${hour}` - // 获取这个小时所有API Key的数据 - const pattern = `usage:hourly:*:${hourKey}` - const keys = await client.keys(pattern) + dateSet.add(dateStr) - // 格式化时间标签 const tzDateForLabel = redis.getDateInTimezone(currentHour) const monthLabel = String(tzDateForLabel.getUTCMonth() + 1).padStart(2, '0') const dayLabel = String(tzDateForLabel.getUTCDate()).padStart(2, '0') const hourLabel = String(tzDateForLabel.getUTCHours()).padStart(2, '0') + hourInfos.push({ + hourKey, + dateStr, + isoTime: currentHour.toISOString(), + label: `${monthLabel}/${dayLabel} ${hourLabel}:00` + }) + + currentHour.setHours(currentHour.getHours() + 1) + } + + // 使用索引获取数据，按小时批量查询 + const dates = [...dateSet] + const usageDataMap = new Map() + const modelDataMap = new Map() + + const fetchPromises = hourInfos.map(async (hourInfo) => { + const [usageResults, modelResults] = await Promise.all([ + getUsageDataByIndex( + `usage:hourly:index:${hourInfo.hourKey}`, + `usage:hourly:{id}:${hourInfo.hourKey}`, + `usage:hourly:*:${hourInfo.hourKey}` + ), + getUsageDataByIndex( + `usage:keymodel:hourly:index:${hourInfo.hourKey}`, + `usage:{keyId}:model:hourly:{model}:${hourInfo.hourKey}`, + `usage:*:model:hourly:*:${hourInfo.hourKey}` + ) + ]) + return { usageResults, modelResults } + }) + + const allResults = await Promise.all(fetchPromises) + allResults.forEach(({ usageResults, modelResults }) => { + usageResults.forEach(({ key, data }) => usageDataMap.set(key, data)) + modelResults.forEach(({ key, data }) => modelDataMap.set(key, data)) + }) + + // 按 hourKey 分组 keys + const usageKeysByHour = new Map() + const modelKeysByHour = new Map() + for (const key of usageDataMap.keys()) { + const match = key.match(/usage:hourly:.+?:(\d{4}-\d{2}-\d{2}:\d{2})/) + if (match) { + const hourKey = match[1] + if (!usageKeysByHour.has(hourKey)) usageKeysByHour.set(hourKey, []) + usageKeysByHour.get(hourKey).push(key) + } + } + for (const key of modelDataMap.keys()) { + const match = key.match(/usage:.+?:model:hourly:.+?:(\d{4}-\d{2}-\d{2}:\d{2})/) + if (match) { + const hourKey = match[1] + if (!modelKeysByHour.has(hourKey)) modelKeysByHour.set(hourKey, []) + modelKeysByHour.get(hourKey).push(key) + } + } + + // 处理每个小时的数据 + for (const hourInfo of hourInfos) { + const hourUsageKeys = usageKeysByHour.get(hourInfo.hourKey) || [] + const hourModelKeys = modelKeysByHour.get(hourInfo.hourKey) || [] + const hourData = { - hour: currentHour.toISOString(), // 使用原始时间，不进行时区转换 - label: `${monthLabel}/${dayLabel} ${hourLabel}:00`, // 添加格式化的标签 + hour: hourInfo.isoTime, + label: hourInfo.label, apiKeys: {} } - // 先收集基础数据 + // 处理 usage 数据 const apiKeyDataMap = new Map() - for (const key of keys) { + for (const key of hourUsageKeys) { const match = key.match(/usage:hourly:(.+?):\d{4}-\d{2}-\d{2}:\d{2}/) - if (!match) { - continue - } + if (!match) continue const apiKeyId = match[1] - const data = await client.hgetall(key) + const data = usageDataMap.get(key) + if (!data || !apiKeyMap.has(apiKeyId)) continue - if (data && apiKeyMap.has(apiKeyId)) { - const inputTokens = parseInt(data.inputTokens) || 0 - const outputTokens = parseInt(data.outputTokens) || 0 - const cacheCreateTokens = parseInt(data.cacheCreateTokens) || 0 - const cacheReadTokens = parseInt(data.cacheReadTokens) || 0 - const totalTokens = inputTokens + outputTokens + cacheCreateTokens + cacheReadTokens + const inputTokens = parseInt(data.inputTokens) || 0 + const outputTokens = parseInt(data.outputTokens) || 0 + const cacheCreateTokens = parseInt(data.cacheCreateTokens) || 0 + const cacheReadTokens = parseInt(data.cacheReadTokens) || 0 - apiKeyDataMap.set(apiKeyId, { - name: apiKeyMap.get(apiKeyId).name, - tokens: totalTokens, - requests: parseInt(data.requests) || 0, - inputTokens, - outputTokens, - cacheCreateTokens, - cacheReadTokens - }) - } + apiKeyDataMap.set(apiKeyId, { + name: apiKeyMap.get(apiKeyId).name, + tokens: inputTokens + outputTokens + cacheCreateTokens + cacheReadTokens, + requests: parseInt(data.requests) || 0, + inputTokens, + outputTokens, + cacheCreateTokens, + cacheReadTokens + }) } - // 获取该小时的模型级别数据来计算准确费用 - const modelPattern = `usage:*:model:hourly:*:${hourKey}` - const modelKeys = await client.keys(modelPattern) + // 处理 model 数据计算费用 const apiKeyCostMap = new Map() - - for (const modelKey of modelKeys) { + for (const modelKey of hourModelKeys) { const match = modelKey.match(/usage:(.+?):model:hourly:(.+?):\d{4}-\d{2}-\d{2}:\d{2}/) - if (!match) { - continue - } + if (!match) continue const apiKeyId = match[1] const model = match[2] - const modelData = await client.hgetall(modelKey) + const modelData = modelDataMap.get(modelKey) + if (!modelData || !apiKeyDataMap.has(apiKeyId)) continue - if (modelData && apiKeyDataMap.has(apiKeyId)) { - const usage = { - input_tokens: parseInt(modelData.inputTokens) || 0, - output_tokens: parseInt(modelData.outputTokens) || 0, - cache_creation_input_tokens: parseInt(modelData.cacheCreateTokens) || 0, - cache_read_input_tokens: parseInt(modelData.cacheReadTokens) || 0 - } - - const costResult = CostCalculator.calculateCost(usage, model) - const currentCost = apiKeyCostMap.get(apiKeyId) || 0 - apiKeyCostMap.set(apiKeyId, currentCost + costResult.costs.total) + const usage = { + input_tokens: parseInt(modelData.inputTokens) || 0, + output_tokens: parseInt(modelData.outputTokens) || 0, + cache_creation_input_tokens: parseInt(modelData.cacheCreateTokens) || 0, + cache_read_input_tokens: parseInt(modelData.cacheReadTokens) || 0 } + + const costResult = CostCalculator.calculateCost(usage, model) + const currentCost = apiKeyCostMap.get(apiKeyId) || 0 + apiKeyCostMap.set(apiKeyId, currentCost + costResult.costs.total) } // 组合数据 for (const [apiKeyId, data] of apiKeyDataMap) { - const cost = apiKeyCostMap.get(apiKeyId) || 0 - - // 如果没有模型级别数据，使用默认模型计算（降级方案） - let finalCost = cost + let cost = apiKeyCostMap.get(apiKeyId) || 0 let formattedCost = CostCalculator.formatCost(cost) + // 降级方案 if (cost === 0 && data.tokens > 0) { const usage = { input_tokens: data.inputTokens, @@ -1373,7 +1733,7 @@ router.get('/api-keys-usage-trend', authenticateAdmin, async (req, res) => { cache_read_input_tokens: data.cacheReadTokens } const fallbackResult = CostCalculator.calculateCost(usage, 'claude-3-5-sonnet-20241022') - finalCost = fallbackResult.costs.total + cost = fallbackResult.costs.total formattedCost = fallbackResult.formatted.total } @@ -1381,101 +1741,138 @@ router.get('/api-keys-usage-trend', authenticateAdmin, async (req, res) => { name: data.name, tokens: data.tokens, requests: data.requests, - cost: finalCost, + cost, formattedCost } } trendData.push(hourData) - currentHour.setHours(currentHour.getHours() + 1) } } else { - // 天粒度统计 + // 天粒度统计（按日期集合扫描） const daysCount = parseInt(days) || 7 const today = new Date() - // 获取过去N天的数据 + // 收集所有天的元数据 + const dayInfos = [] for (let i = 0; i < daysCount; i++) { const date = new Date(today) date.setDate(date.getDate() - i) const dateStr = redis.getDateStringInTimezone(date) + dayInfos.push({ dateStr }) + } - // 获取这一天所有API Key的数据 - const pattern = `usage:daily:*:${dateStr}` - const keys = await client.keys(pattern) + // 使用索引获取数据，按日期批量查询 + const usageDataMap = new Map() + const modelDataMap = new Map() + + const fetchPromises = dayInfos.map(async (dayInfo) => { + const [usageResults, modelResults] = await Promise.all([ + getUsageDataByIndex( + `usage:daily:index:${dayInfo.dateStr}`, + `usage:daily:{id}:${dayInfo.dateStr}`, + `usage:daily:*:${dayInfo.dateStr}` + ), + getUsageDataByIndex( + `usage:keymodel:daily:index:${dayInfo.dateStr}`, + `usage:{keyId}:model:daily:{model}:${dayInfo.dateStr}`, + `usage:*:model:daily:*:${dayInfo.dateStr}` + ) + ]) + return { usageResults, modelResults } + }) + + const allResults = await Promise.all(fetchPromises) + allResults.forEach(({ usageResults, modelResults }) => { + usageResults.forEach(({ key, data }) => usageDataMap.set(key, data)) + modelResults.forEach(({ key, data }) => modelDataMap.set(key, data)) + }) + + // 按 dateStr 分组 keys + const usageKeysByDate = new Map() + const modelKeysByDate = new Map() + for (const key of usageDataMap.keys()) { + const match = key.match(/usage:daily:.+?:(\d{4}-\d{2}-\d{2})/) + if (match) { + const dateStr = match[1] + if (!usageKeysByDate.has(dateStr)) usageKeysByDate.set(dateStr, []) + usageKeysByDate.get(dateStr).push(key) + } + } + for (const key of modelDataMap.keys()) { + const match = key.match(/usage:.+?:model:daily:.+?:(\d{4}-\d{2}-\d{2})/) + if (match) { + const dateStr = match[1] + if (!modelKeysByDate.has(dateStr)) modelKeysByDate.set(dateStr, []) + modelKeysByDate.get(dateStr).push(key) + } + } + + // 处理每天的数据 + for (const dayInfo of dayInfos) { + const dayUsageKeys = usageKeysByDate.get(dayInfo.dateStr) || [] + const dayModelKeys = modelKeysByDate.get(dayInfo.dateStr) || [] const dayData = { - date: dateStr, + date: dayInfo.dateStr, apiKeys: {} } - // 先收集基础数据 + // 处理 usage 数据 const apiKeyDataMap = new Map() - for (const key of keys) { + for (const key of dayUsageKeys) { const match = key.match(/usage:daily:(.+?):\d{4}-\d{2}-\d{2}/) - if (!match) { - continue - } + if (!match) continue const apiKeyId = match[1] - const data = await client.hgetall(key) + const data = usageDataMap.get(key) + if (!data || !apiKeyMap.has(apiKeyId)) continue - if (data && apiKeyMap.has(apiKeyId)) { - const inputTokens = parseInt(data.inputTokens) || 0 - const outputTokens = parseInt(data.outputTokens) || 0 - const cacheCreateTokens = parseInt(data.cacheCreateTokens) || 0 - const cacheReadTokens = parseInt(data.cacheReadTokens) || 0 - const totalTokens = inputTokens + outputTokens + cacheCreateTokens + cacheReadTokens + const inputTokens = parseInt(data.inputTokens) || 0 + const outputTokens = parseInt(data.outputTokens) || 0 + const cacheCreateTokens = parseInt(data.cacheCreateTokens) || 0 + const cacheReadTokens = parseInt(data.cacheReadTokens) || 0 - apiKeyDataMap.set(apiKeyId, { - name: apiKeyMap.get(apiKeyId).name, - tokens: totalTokens, - requests: parseInt(data.requests) || 0, - inputTokens, - outputTokens, - cacheCreateTokens, - cacheReadTokens - }) - } + apiKeyDataMap.set(apiKeyId, { + name: apiKeyMap.get(apiKeyId).name, + tokens: inputTokens + outputTokens + cacheCreateTokens + cacheReadTokens, + requests: parseInt(data.requests) || 0, + inputTokens, + outputTokens, + cacheCreateTokens, + cacheReadTokens + }) } - // 获取该天的模型级别数据来计算准确费用 - const modelPattern = `usage:*:model:daily:*:${dateStr}` - const modelKeys = await client.keys(modelPattern) + // 处理 model 数据计算费用 const apiKeyCostMap = new Map() - - for (const modelKey of modelKeys) { + for (const modelKey of dayModelKeys) { const match = modelKey.match(/usage:(.+?):model:daily:(.+?):\d{4}-\d{2}-\d{2}/) - if (!match) { - continue - } + if (!match) continue const apiKeyId = match[1] const model = match[2] - const modelData = await client.hgetall(modelKey) + const modelData = modelDataMap.get(modelKey) + if (!modelData || !apiKeyDataMap.has(apiKeyId)) continue - if (modelData && apiKeyDataMap.has(apiKeyId)) { - const usage = { - input_tokens: parseInt(modelData.inputTokens) || 0, - output_tokens: parseInt(modelData.outputTokens) || 0, - cache_creation_input_tokens: parseInt(modelData.cacheCreateTokens) || 0, - cache_read_input_tokens: parseInt(modelData.cacheReadTokens) || 0 - } - - const costResult = CostCalculator.calculateCost(usage, model) - const currentCost = apiKeyCostMap.get(apiKeyId) || 0 - apiKeyCostMap.set(apiKeyId, currentCost + costResult.costs.total) + const usage = { + input_tokens: parseInt(modelData.inputTokens) || 0, + output_tokens: parseInt(modelData.outputTokens) || 0, + cache_creation_input_tokens: parseInt(modelData.cacheCreateTokens) || 0, + cache_read_input_tokens: parseInt(modelData.cacheReadTokens) || 0 } + + const costResult = CostCalculator.calculateCost(usage, model) + const currentCost = apiKeyCostMap.get(apiKeyId) || 0 + apiKeyCostMap.set(apiKeyId, currentCost + costResult.costs.total) } // 组合数据 for (const [apiKeyId, data] of apiKeyDataMap) { - const cost = apiKeyCostMap.get(apiKeyId) || 0 - - // 如果没有模型级别数据，使用默认模型计算（降级方案） - let finalCost = cost + let cost = apiKeyCostMap.get(apiKeyId) || 0 let formattedCost = CostCalculator.formatCost(cost) + // 降级方案 if (cost === 0 && data.tokens > 0) { const usage = { input_tokens: data.inputTokens, @@ -1484,7 +1881,7 @@ router.get('/api-keys-usage-trend', authenticateAdmin, async (req, res) => { cache_read_input_tokens: data.cacheReadTokens } const fallbackResult = CostCalculator.calculateCost(usage, 'claude-3-5-sonnet-20241022') - finalCost = fallbackResult.costs.total + cost = fallbackResult.costs.total formattedCost = fallbackResult.formatted.total } @@ -1492,7 +1889,7 @@ router.get('/api-keys-usage-trend', authenticateAdmin, async (req, res) => { name: data.name, tokens: data.tokens, requests: data.requests, - cost: finalCost, + cost, formattedCost } } @@ -1564,9 +1961,6 @@ router.get('/usage-costs', authenticateAdmin, async (req, res) => { return model.replace(/-v\d+:\d+$|:latest$/, '') } - // 获取所有API Keys的使用统计 - const apiKeys = await apiKeyService.getAllApiKeys() - const totalCosts = { inputCost: 0, outputCost: 0, @@ -1592,10 +1986,11 @@ router.get('/usage-costs', authenticateAdmin, async (req, res) => { } else if (period === 'monthly') { pattern = `usage:model:monthly:*:${currentMonth}` } else if (period === '7days') { - // 最近7天：汇总daily数据 + // 最近7天：汇总daily数据（使用 SCAN + Pipeline 优化） const modelUsageMap = new Map() - // 获取最近7天的所有daily统计数据 + // 收集最近7天的所有日期 + const dateStrs = [] for (let i = 0; i < 7; i++) { const date = new Date() date.setDate(date.getDate() - i) @@ -1603,37 +1998,44 @@ router.get('/usage-costs', authenticateAdmin, async (req, res) => { const dateStr = `${currentTzDate.getUTCFullYear()}-${String( currentTzDate.getUTCMonth() + 1 ).padStart(2, '0')}-${String(currentTzDate.getUTCDate()).padStart(2, '0')}` - const dayPattern = `usage:model:daily:*:${dateStr}` + dateStrs.push(dateStr) + } - const dayKeys = await client.keys(dayPattern) + // 使用索引获取数据 + const fetchPromises = dateStrs.map((dateStr) => + getUsageDataByIndex( + `usage:model:daily:index:${dateStr}`, + `usage:model:daily:{id}:${dateStr}`, + `usage:model:daily:*:${dateStr}` + ) + ) + const allResults = await Promise.all(fetchPromises) + const allData = allResults.flat() - for (const key of dayKeys) { - const modelMatch = key.match(/usage:model:daily:(.+):\d{4}-\d{2}-\d{2}$/) - if (!modelMatch) { - continue - } + // 处理数据 + for (const { key, data } of allData) { + if (!data) continue - const rawModel = modelMatch[1] - const normalizedModel = normalizeModelName(rawModel) - const data = await client.hgetall(key) + const modelMatch = key.match(/usage:model:daily:(.+):\d{4}-\d{2}-\d{2}$/) + if (!modelMatch) continue - if (data && Object.keys(data).length > 0) { - if (!modelUsageMap.has(normalizedModel)) { - modelUsageMap.set(normalizedModel, { - inputTokens: 0, - outputTokens: 0, - cacheCreateTokens: 0, - cacheReadTokens: 0 - }) - } + const rawModel = modelMatch[1] + const normalizedModel = normalizeModelName(rawModel) - const modelUsage = modelUsageMap.get(normalizedModel) - modelUsage.inputTokens += parseInt(data.inputTokens) || 0 - modelUsage.outputTokens += parseInt(data.outputTokens) || 0 - modelUsage.cacheCreateTokens += parseInt(data.cacheCreateTokens) || 0 - modelUsage.cacheReadTokens += parseInt(data.cacheReadTokens) || 0 - } + if (!modelUsageMap.has(normalizedModel)) { + modelUsageMap.set(normalizedModel, { + inputTokens: 0, + outputTokens: 0, + cacheCreateTokens: 0, + cacheReadTokens: 0 + }) } + + const modelUsage = modelUsageMap.get(normalizedModel) + modelUsage.inputTokens += parseInt(data.inputTokens) || 0 + modelUsage.outputTokens += parseInt(data.outputTokens) || 0 + modelUsage.cacheCreateTokens += parseInt(data.cacheCreateTokens) || 0 + modelUsage.cacheReadTokens += parseInt(data.cacheReadTokens) || 0 } // 计算7天统计的费用 @@ -1690,40 +2092,47 @@ router.get('/usage-costs', authenticateAdmin, async (req, res) => { } }) } else { - // 全部时间，先尝试从Redis获取所有历史模型统计数据（只使用monthly数据避免重复计算） - const allModelKeys = await client.keys('usage:model:monthly:*:*') - logger.info(`💰 Total period calculation: found ${allModelKeys.length} monthly model keys`) + // 全部时间，使用月份索引 + const months = await redis.client.smembers('usage:model:monthly:months') + const allData = [] + if (months && months.length > 0) { + const fetchPromises = months.map((month) => + getUsageDataByIndex( + `usage:model:monthly:index:${month}`, + `usage:model:monthly:{id}:${month}`, + `usage:model:monthly:*:${month}` + ) + ) + const results = await Promise.all(fetchPromises) + results.forEach((r) => allData.push(...r)) + } + logger.info(`💰 Total period calculation: found ${allData.length} monthly model keys`) - if (allModelKeys.length > 0) { - // 如果有详细的模型统计数据，使用模型级别的计算 + if (allData.length > 0) { const modelUsageMap = new Map() - for (const key of allModelKeys) { - // 解析模型名称（只处理monthly数据） + for (const { key, data } of allData) { + if (!data) continue + const modelMatch = key.match(/usage:model:monthly:(.+):(\d{4}-\d{2})$/) - if (!modelMatch) { - continue - } + if (!modelMatch) continue const model = modelMatch[1] - const data = await client.hgetall(key) - if (data && Object.keys(data).length > 0) { - if (!modelUsageMap.has(model)) { - modelUsageMap.set(model, { - inputTokens: 0, - outputTokens: 0, - cacheCreateTokens: 0, - cacheReadTokens: 0 - }) - } - - const modelUsage = modelUsageMap.get(model) - modelUsage.inputTokens += parseInt(data.inputTokens) || 0 - modelUsage.outputTokens += parseInt(data.outputTokens) || 0 - modelUsage.cacheCreateTokens += parseInt(data.cacheCreateTokens) || 0 - modelUsage.cacheReadTokens += parseInt(data.cacheReadTokens) || 0 + if (!modelUsageMap.has(model)) { + modelUsageMap.set(model, { + inputTokens: 0, + outputTokens: 0, + cacheCreateTokens: 0, + cacheReadTokens: 0 + }) } + + const modelUsage = modelUsageMap.get(model) + modelUsage.inputTokens += parseInt(data.inputTokens) || 0 + modelUsage.outputTokens += parseInt(data.outputTokens) || 0 + modelUsage.cacheCreateTokens += parseInt(data.cacheCreateTokens) || 0 + modelUsage.cacheReadTokens += parseInt(data.cacheReadTokens) || 0 } // 使用模型级别的数据计算费用 @@ -1764,8 +2173,9 @@ router.get('/usage-costs', authenticateAdmin, async (req, res) => { } } } else { - // 如果没有详细的模型统计数据，回退到API Key汇总数据 + // 如果没有详细的模型统计数据，回退到API Key汇总数据（延迟加载） logger.warn('No detailed model statistics found, falling back to API Key aggregated data') + const apiKeys = await apiKeyService.getAllApiKeysFast() for (const apiKey of apiKeys) { if (apiKey.usage && apiKey.usage.total) { @@ -1807,49 +2217,60 @@ router.get('/usage-costs', authenticateAdmin, async (req, res) => { }) } - // 对于今日或本月，从Redis获取详细的模型统计 - const keys = await client.keys(pattern) - - for (const key of keys) { - const match = key.match( - period === 'today' - ? /usage:model:daily:(.+):\d{4}-\d{2}-\d{2}$/ - : /usage:model:monthly:(.+):\d{4}-\d{2}$/ + // 对于今日或本月，使用索引查询 + let allData + if (period === 'today') { + const results = await getUsageDataByIndex( + `usage:model:daily:index:${today}`, + `usage:model:daily:{id}:${today}`, + `usage:model:daily:*:${today}` ) + allData = results + } else { + // 本月 - 使用月度索引 + const results = await getUsageDataByIndex( + `usage:model:monthly:index:${currentMonth}`, + `usage:model:monthly:{id}:${currentMonth}`, + `usage:model:monthly:*:${currentMonth}` + ) + allData = results + } + const regex = + period === 'today' + ? /usage:model:daily:(.+):\d{4}-\d{2}-\d{2}$/ + : /usage:model:monthly:(.+):\d{4}-\d{2}$/ - if (!match) { - continue - } + for (const { key, data } of allData) { + if (!data) continue + + const match = key.match(regex) + if (!match) continue const model = match[1] - const data = await client.hgetall(key) + const usage = { + input_tokens: parseInt(data.inputTokens) || 0, + output_tokens: parseInt(data.outputTokens) || 0, + cache_creation_input_tokens: parseInt(data.cacheCreateTokens) || 0, + cache_read_input_tokens: parseInt(data.cacheReadTokens) || 0 + } - if (data && Object.keys(data).length > 0) { - const usage = { - input_tokens: parseInt(data.inputTokens) || 0, - output_tokens: parseInt(data.outputTokens) || 0, - cache_creation_input_tokens: parseInt(data.cacheCreateTokens) || 0, - cache_read_input_tokens: parseInt(data.cacheReadTokens) || 0 - } + const costResult = CostCalculator.calculateCost(usage, model) - const costResult = CostCalculator.calculateCost(usage, model) + // 累加总费用 + totalCosts.inputCost += costResult.costs.input + totalCosts.outputCost += costResult.costs.output + totalCosts.cacheCreateCost += costResult.costs.cacheWrite + totalCosts.cacheReadCost += costResult.costs.cacheRead + totalCosts.totalCost += costResult.costs.total - // 累加总费用 - totalCosts.inputCost += costResult.costs.input - totalCosts.outputCost += costResult.costs.output - totalCosts.cacheCreateCost += costResult.costs.cacheWrite - totalCosts.cacheReadCost += costResult.costs.cacheRead - totalCosts.totalCost += costResult.costs.total - - // 记录模型费用 - modelCosts[model] = { - model, - requests: parseInt(data.requests) || 0, - usage, - costs: costResult.costs, - formatted: costResult.formatted, - usingDynamicPricing: costResult.usingDynamicPricing - } + // 记录模型费用 + modelCosts[model] = { + model, + requests: parseInt(data.requests) || 0, + usage, + costs: costResult.costs, + formatted: costResult.formatted, + usingDynamicPricing: costResult.usingDynamicPricing } } @@ -2260,7 +2681,7 @@ router.get('/accounts/:accountId/usage-records', authenticateAdmin, async (req, return res.status(404).json({ success: false, error: 'Account not found' }) } - const allApiKeys = await apiKeyService.getAllApiKeys(true) + const allApiKeys = await apiKeyService.getAllApiKeysFast(true) const apiKeyNameCache = new Map( allApiKeys.map((key) => [key.id, key.name || key.label || key.id]) ) diff --git a/src/routes/apiStats.js b/src/routes/apiStats.js index 62614b65..96ca590c 100644 --- a/src/routes/apiStats.js +++ b/src/routes/apiStats.js @@ -39,7 +39,7 @@ router.post('/api/get-key-id', async (req, res) => { if (!validation.valid) { const clientIP = req.ip || req.connection?.remoteAddress || 'unknown' - logger.security(`🔒 Invalid API key in get-key-id: ${validation.error} from ${clientIP}`) + logger.security(`Invalid API key in get-key-id: ${validation.error} from ${clientIP}`) return res.status(401).json({ error: 'Invalid API key', message: validation.error @@ -87,7 +87,7 @@ router.post('/api/user-stats', async (req, res) => { keyData = await redis.getApiKey(apiId) if (!keyData || Object.keys(keyData).length === 0) { - logger.security(`🔒 API key not found for ID: ${apiId} from ${req.ip || 'unknown'}`) + logger.security(`API key not found for ID: ${apiId} from ${req.ip || 'unknown'}`) return res.status(404).json({ error: 'API key not found', message: 'The specified API key does not exist' @@ -166,7 +166,7 @@ router.post('/api/user-stats', async (req, res) => { } else if (apiKey) { // 通过 apiKey 查询（保持向后兼容） if (typeof apiKey !== 'string' || apiKey.length < 10 || apiKey.length > 512) { - logger.security(`🔒 Invalid API key format in user stats query from ${req.ip || 'unknown'}`) + logger.security(`Invalid API key format in user stats query from ${req.ip || 'unknown'}`) return res.status(400).json({ error: 'Invalid API key format', message: 'API key format is invalid' @@ -191,7 +191,7 @@ router.post('/api/user-stats', async (req, res) => { keyData = validatedKeyData keyId = keyData.id } else { - logger.security(`🔒 Missing API key or ID in user stats query from ${req.ip || 'unknown'}`) + logger.security(`Missing API key or ID in user stats query from ${req.ip || 'unknown'}`) return res.status(400).json({ error: 'API Key or ID is required', message: 'Please provide your API Key or API ID' @@ -224,17 +224,16 @@ router.post('/api/user-stats', async (req, res) => { logger.debug(`📊 使用 allTimeCost 计算用户统计: ${allTimeCost}`) } else { // Fallback: 如果 allTimeCost 为空（旧键），尝试月度键 - const allModelKeys = await client.keys(`usage:${keyId}:model:monthly:*:*`) + const allModelResults = await redis.scanAndGetAllChunked(`usage:${keyId}:model:monthly:*:*`) const modelUsageMap = new Map() - for (const key of allModelKeys) { + for (const { key, data } of allModelResults) { const modelMatch = key.match(/usage:.+:model:monthly:(.+):(\d{4}-\d{2})$/) if (!modelMatch) { continue } const model = modelMatch[1] - const data = await client.hgetall(key) if (data && Object.keys(data).length > 0) { if (!modelUsageMap.has(model)) { @@ -717,9 +716,9 @@ router.post('/api/batch-model-stats', async (req, res) => { ? `usage:${apiId}:model:daily:*:${today}` : `usage:${apiId}:model:monthly:*:${currentMonth}` - const keys = await client.keys(pattern) + const results = await redis.scanAndGetAllChunked(pattern) - for (const key of keys) { + for (const { key, data } of results) { const match = key.match( period === 'daily' ? /usage:.+:model:daily:(.+):\d{4}-\d{2}-\d{2}$/ @@ -731,7 +730,6 @@ router.post('/api/batch-model-stats', async (req, res) => { } const model = match[1] - const data = await client.hgetall(key) if (data && Object.keys(data).length > 0) { if (!modelUsageMap.has(model)) { @@ -886,7 +884,7 @@ router.post('/api/user-model-stats', async (req, res) => { keyData = await redis.getApiKey(apiId) if (!keyData || Object.keys(keyData).length === 0) { - logger.security(`🔒 API key not found for ID: ${apiId} from ${req.ip || 'unknown'}`) + logger.security(`API key not found for ID: ${apiId} from ${req.ip || 'unknown'}`) return res.status(404).json({ error: 'API key not found', message: 'The specified API key does not exist' @@ -953,10 +951,10 @@ router.post('/api/user-model-stats', async (req, res) => { ? `usage:${keyId}:model:daily:*:${today}` : `usage:${keyId}:model:monthly:*:${currentMonth}` - const keys = await client.keys(pattern) + const results = await redis.scanAndGetAllChunked(pattern) const modelStats = [] - for (const key of keys) { + for (const { key, data } of results) { const match = key.match( period === 'daily' ? /usage:.+:model:daily:(.+):\d{4}-\d{2}-\d{2}$/ @@ -968,7 +966,6 @@ router.post('/api/user-model-stats', async (req, res) => { } const model = match[1] - const data = await client.hgetall(key) if (data && Object.keys(data).length > 0) { const usage = { diff --git a/src/routes/openaiRoutes.js b/src/routes/openaiRoutes.js index 7faf9e87..5ba62289 100644 --- a/src/routes/openaiRoutes.js +++ b/src/routes/openaiRoutes.js @@ -12,6 +12,7 @@ const apiKeyService = require('../services/apiKeyService') const crypto = require('crypto') const ProxyHelper = require('../utils/proxyHelper') const { updateRateLimitCounters } = require('../utils/rateLimitHelper') +const { IncrementalSSEParser } = require('../utils/sseParser') // 创建代理 Agent（使用统一的代理工具） function createProxyAgent(proxy) { @@ -576,7 +577,6 @@ const handleResponses = async (req, res) => { } // 处理响应并捕获 usage 数据和真实的 model - let buffer = '' let usageData = null let actualModel = null let usageReported = false @@ -644,74 +644,50 @@ const handleResponses = async (req, res) => { } } - // 解析 SSE 事件以捕获 usage 数据和 model - const parseSSEForUsage = (data) => { - const lines = data.split('\n') + // 使用增量 SSE 解析器 + const sseParser = new IncrementalSSEParser() - for (const line of lines) { - if (line.startsWith('event: response.completed')) { - // 下一行应该是数据 - continue + // 处理解析出的事件 + const processSSEEvent = (eventData) => { + // 检查是否是 response.completed 事件 + if (eventData.type === 'response.completed' && eventData.response) { + // 从响应中获取真实的 model + if (eventData.response.model) { + actualModel = eventData.response.model + logger.debug(`📊 Captured actual model: ${actualModel}`) } - if (line.startsWith('data: ')) { - try { - const jsonStr = line.slice(6) // 移除 'data: ' 前缀 - const eventData = JSON.parse(jsonStr) + // 获取 usage 数据 + if (eventData.response.usage) { + usageData = eventData.response.usage + logger.debug('📊 Captured OpenAI usage data:', usageData) + } + } - // 检查是否是 response.completed 事件 - if (eventData.type === 'response.completed' && eventData.response) { - // 从响应中获取真实的 model - if (eventData.response.model) { - actualModel = eventData.response.model - logger.debug(`📊 Captured actual model: ${actualModel}`) - } - - // 获取 usage 数据 - if (eventData.response.usage) { - usageData = eventData.response.usage - logger.debug('📊 Captured OpenAI usage data:', usageData) - } - } - - // 检查是否有限流错误 - if (eventData.error && eventData.error.type === 'usage_limit_reached') { - rateLimitDetected = true - if (eventData.error.resets_in_seconds) { - rateLimitResetsInSeconds = eventData.error.resets_in_seconds - logger.warn( - `🚫 Rate limit detected in stream, resets in ${rateLimitResetsInSeconds} seconds` - ) - } - } - } catch (e) { - // 忽略解析错误 - } + // 检查是否有限流错误 + if (eventData.error && eventData.error.type === 'usage_limit_reached') { + rateLimitDetected = true + if (eventData.error.resets_in_seconds) { + rateLimitResetsInSeconds = eventData.error.resets_in_seconds + logger.warn( + `🚫 Rate limit detected in stream, resets in ${rateLimitResetsInSeconds} seconds` + ) } } } upstream.data.on('data', (chunk) => { try { - const chunkStr = chunk.toString() - // 转发数据给客户端 if (!res.destroyed) { res.write(chunk) } - // 同时解析数据以捕获 usage 信息 - buffer += chunkStr - - // 处理完整的 SSE 事件 - if (buffer.includes('\n\n')) { - const events = buffer.split('\n\n') - buffer = events.pop() || '' // 保留最后一个可能不完整的事件 - - for (const event of events) { - if (event.trim()) { - parseSSEForUsage(event) - } + // 使用增量解析器处理数据 + const events = sseParser.feed(chunk.toString()) + for (const event of events) { + if (event.type === 'data' && event.data) { + processSSEEvent(event.data) } } } catch (error) { @@ -721,8 +697,14 @@ const handleResponses = async (req, res) => { upstream.data.on('end', async () => { // 处理剩余的 buffer - if (buffer.trim()) { - parseSSEForUsage(buffer) + const remaining = sseParser.getRemaining() + if (remaining.trim()) { + const events = sseParser.feed('\n\n') // 强制刷新剩余内容 + for (const event of events) { + if (event.type === 'data' && event.data) { + processSSEEvent(event.data) + } + } } // 记录使用统计 diff --git a/src/routes/web.js b/src/routes/web.js index ed3bfa57..547b9869 100644 --- a/src/routes/web.js +++ b/src/routes/web.js @@ -74,7 +74,7 @@ router.post('/auth/login', async (req, res) => { const isValidPassword = await bcrypt.compare(password, adminData.passwordHash) if (!isValidUsername || !isValidPassword) { - logger.security(`🔒 Failed login attempt for username: ${username}`) + logger.security(`Failed login attempt for username: ${username}`) return res.status(401).json({ error: 'Invalid credentials', message: 'Invalid username or password' @@ -96,7 +96,7 @@ router.post('/auth/login', async (req, res) => { // 不再更新 Redis 中的最后登录时间，因为 Redis 只是缓存 // init.json 是唯一真实数据源 - logger.success(`🔐 Admin login successful: ${username}`) + logger.success(`Admin login successful: ${username}`) return res.json({ success: true, @@ -197,7 +197,7 @@ router.post('/auth/change-password', async (req, res) => { // 验证当前密码 const isValidPassword = await bcrypt.compare(currentPassword, adminData.passwordHash) if (!isValidPassword) { - logger.security(`🔒 Invalid current password attempt for user: ${sessionData.username}`) + logger.security(`Invalid current password attempt for user: ${sessionData.username}`) return res.status(401).json({ error: 'Invalid current password', message: 'Current password is incorrect' @@ -253,7 +253,7 @@ router.post('/auth/change-password', async (req, res) => { // 清除当前会话（强制用户重新登录） await redis.deleteSession(token) - logger.success(`🔐 Admin password changed successfully for user: ${updatedUsername}`) + logger.success(`Admin password changed successfully for user: ${updatedUsername}`) return res.json({ success: true, @@ -294,7 +294,7 @@ router.get('/auth/user', async (req, res) => { // 🔒 安全修复：验证会话完整性 if (!sessionData.username || !sessionData.loginTime) { - logger.security(`🔒 Invalid session structure in /auth/user from ${req.ip || 'unknown'}`) + logger.security(`Invalid session structure in /auth/user from ${req.ip || 'unknown'}`) await redis.deleteSession(token) return res.status(401).json({ error: 'Invalid session', @@ -352,7 +352,7 @@ router.post('/auth/refresh', async (req, res) => { // 🔒 安全修复：验证会话完整性（必须有 username 和 loginTime） if (!sessionData.username || !sessionData.loginTime) { - logger.security(`🔒 Invalid session structure detected from ${req.ip || 'unknown'}`) + logger.security(`Invalid session structure detected from ${req.ip || 'unknown'}`) await redis.deleteSession(token) // 清理无效/伪造的会话 return res.status(401).json({ error: 'Invalid session', diff --git a/src/services/accountGroupService.js b/src/services/accountGroupService.js index 23293a18..f4413204 100644 --- a/src/services/accountGroupService.js +++ b/src/services/accountGroupService.js @@ -7,6 +7,56 @@ class AccountGroupService { this.GROUPS_KEY = 'account_groups' this.GROUP_PREFIX = 'account_group:' this.GROUP_MEMBERS_PREFIX = 'account_group_members:' + this.REVERSE_INDEX_PREFIX = 'account_groups_reverse:' + this.REVERSE_INDEX_MIGRATED_KEY = 'account_groups_reverse:migrated' + } + + /** + * 确保反向索引存在（启动时自动调用） + * 检查是否已迁移，如果没有则自动回填 + */ + async ensureReverseIndexes() { + try { + const client = redis.getClientSafe() + if (!client) return + + // 检查是否已迁移 + const migrated = await client.get(this.REVERSE_INDEX_MIGRATED_KEY) + if (migrated === 'true') { + logger.debug('📁 账户分组反向索引已存在，跳过回填') + return + } + + logger.info('📁 开始回填账户分组反向索引...') + + const allGroupIds = await client.smembers(this.GROUPS_KEY) + if (allGroupIds.length === 0) { + await client.set(this.REVERSE_INDEX_MIGRATED_KEY, 'true') + return + } + + let totalOperations = 0 + + for (const groupId of allGroupIds) { + const group = await client.hgetall(`${this.GROUP_PREFIX}${groupId}`) + if (!group || !group.platform) continue + + const members = await client.smembers(`${this.GROUP_MEMBERS_PREFIX}${groupId}`) + if (members.length === 0) continue + + const pipeline = client.pipeline() + for (const accountId of members) { + pipeline.sadd(`${this.REVERSE_INDEX_PREFIX}${group.platform}:${accountId}`, groupId) + } + await pipeline.exec() + totalOperations += members.length + } + + await client.set(this.REVERSE_INDEX_MIGRATED_KEY, 'true') + logger.success(`📁 账户分组反向索引回填完成，共 ${totalOperations} 条`) + } catch (error) { + logger.error('❌ 账户分组反向索引回填失败:', error) + } } /** @@ -50,7 +100,7 @@ class AccountGroupService { // 添加到分组集合 await client.sadd(this.GROUPS_KEY, groupId) - logger.success(`✅ 创建账户分组成功: ${name} (${platform})`) + logger.success(`创建账户分组成功: ${name} (${platform})`) return group } catch (error) { @@ -101,7 +151,7 @@ class AccountGroupService { // 返回更新后的完整数据 const updatedGroup = await client.hgetall(groupKey) - logger.success(`✅ 更新账户分组成功: ${updatedGroup.name}`) + logger.success(`更新账户分组成功: ${updatedGroup.name}`) return updatedGroup } catch (error) { @@ -143,7 +193,7 @@ class AccountGroupService { // 从分组集合中移除 await client.srem(this.GROUPS_KEY, groupId) - logger.success(`✅ 删除账户分组成功: ${group.name}`) + logger.success(`删除账户分组成功: ${group.name}`) } catch (error) { logger.error('❌ 删除账户分组失败:', error) throw error @@ -234,7 +284,10 @@ class AccountGroupService { // 添加到分组成员集合 await client.sadd(`${this.GROUP_MEMBERS_PREFIX}${groupId}`, accountId) - logger.success(`✅ 添加账户到分组成功: ${accountId} -> ${group.name}`) + // 维护反向索引 + await client.sadd(`account_groups_reverse:${group.platform}:${accountId}`, groupId) + + logger.success(`添加账户到分组成功: ${accountId} -> ${group.name}`) } catch (error) { logger.error('❌ 添加账户到分组失败:', error) throw error @@ -245,15 +298,26 @@ class AccountGroupService { * 从分组移除账户 * @param {string} accountId - 账户ID * @param {string} groupId - 分组ID + * @param {string} platform - 平台（可选，如果不传则从分组获取） */ - async removeAccountFromGroup(accountId, groupId) { + async removeAccountFromGroup(accountId, groupId, platform = null) { try { const client = redis.getClientSafe() // 从分组成员集合中移除 await client.srem(`${this.GROUP_MEMBERS_PREFIX}${groupId}`, accountId) - logger.success(`✅ 从分组移除账户成功: ${accountId}`) + // 维护反向索引 + let groupPlatform = platform + if (!groupPlatform) { + const group = await this.getGroup(groupId) + groupPlatform = group?.platform + } + if (groupPlatform) { + await client.srem(`account_groups_reverse:${groupPlatform}:${accountId}`, groupId) + } + + logger.success(`从分组移除账户成功: ${accountId}`) } catch (error) { logger.error('❌ 从分组移除账户失败:', error) throw error @@ -399,7 +463,7 @@ class AccountGroupService { await this.addAccountToGroup(accountId, groupId, accountPlatform) } - logger.success(`✅ 批量设置账户分组成功: ${accountId} -> [${groupIds.join(', ')}]`) + logger.success(`批量设置账户分组成功: ${accountId} -> [${groupIds.join(', ')}]`) } catch (error) { logger.error('❌ 批量设置账户分组失败:', error) throw error @@ -409,8 +473,9 @@ class AccountGroupService { /** * 从所有分组中移除账户 * @param {string} accountId - 账户ID + * @param {string} platform - 平台（可选，用于清理反向索引） */ - async removeAccountFromAllGroups(accountId) { + async removeAccountFromAllGroups(accountId, platform = null) { try { const client = redis.getClientSafe() const allGroupIds = await client.smembers(this.GROUPS_KEY) @@ -419,12 +484,127 @@ class AccountGroupService { await client.srem(`${this.GROUP_MEMBERS_PREFIX}${groupId}`, accountId) } - logger.success(`✅ 从所有分组移除账户成功: ${accountId}`) + // 清理反向索引 + if (platform) { + await client.del(`account_groups_reverse:${platform}:${accountId}`) + } else { + // 如果没有指定平台，清理所有可能的平台 + const platforms = ['claude', 'gemini', 'openai', 'droid'] + const pipeline = client.pipeline() + for (const p of platforms) { + pipeline.del(`account_groups_reverse:${p}:${accountId}`) + } + await pipeline.exec() + } + + logger.success(`从所有分组移除账户成功: ${accountId}`) } catch (error) { logger.error('❌ 从所有分组移除账户失败:', error) throw error } } + + /** + * 批量获取多个账户的分组信息（性能优化版本，使用反向索引） + * @param {Array} accountIds - 账户ID数组 + * @param {string} platform - 平台类型 + * @param {Object} options - 选项 + * @param {boolean} options.skipMemberCount - 是否跳过 memberCount（默认 true） + * @returns {Map} accountId -> 分组信息数组的映射 + */ + async batchGetAccountGroupsByIndex(accountIds, platform, options = {}) { + const { skipMemberCount = true } = options + + if (!accountIds || accountIds.length === 0) { + return new Map() + } + + try { + const client = redis.getClientSafe() + + // Pipeline 批量获取所有账户的分组ID + const pipeline = client.pipeline() + for (const accountId of accountIds) { + pipeline.smembers(`${this.REVERSE_INDEX_PREFIX}${platform}:${accountId}`) + } + const groupIdResults = await pipeline.exec() + + // 收集所有需要的分组ID + const uniqueGroupIds = new Set() + const accountGroupIdsMap = new Map() + let hasAnyGroups = false + accountIds.forEach((accountId, i) => { + const [err, groupIds] = groupIdResults[i] + const ids = err ? [] : groupIds || [] + accountGroupIdsMap.set(accountId, ids) + ids.forEach((id) => { + uniqueGroupIds.add(id) + hasAnyGroups = true + }) + }) + + // 如果反向索引全空，回退到原方法（兼容未迁移的数据） + if (!hasAnyGroups) { + const migrated = await client.get(this.REVERSE_INDEX_MIGRATED_KEY) + if (migrated !== 'true') { + logger.debug('📁 Reverse index not migrated, falling back to getAccountGroups') + const result = new Map() + for (const accountId of accountIds) { + try { + const groups = await this.getAccountGroups(accountId) + result.set(accountId, groups) + } catch { + result.set(accountId, []) + } + } + return result + } + } + + // 批量获取分组详情 + const groupDetailsMap = new Map() + if (uniqueGroupIds.size > 0) { + const detailPipeline = client.pipeline() + const groupIdArray = Array.from(uniqueGroupIds) + for (const groupId of groupIdArray) { + detailPipeline.hgetall(`${this.GROUP_PREFIX}${groupId}`) + if (!skipMemberCount) { + detailPipeline.scard(`${this.GROUP_MEMBERS_PREFIX}${groupId}`) + } + } + const detailResults = await detailPipeline.exec() + + const step = skipMemberCount ? 1 : 2 + for (let i = 0; i < groupIdArray.length; i++) { + const groupId = groupIdArray[i] + const [err1, groupData] = detailResults[i * step] + if (!err1 && groupData && Object.keys(groupData).length > 0) { + const group = { ...groupData } + if (!skipMemberCount) { + const [err2, memberCount] = detailResults[i * step + 1] + group.memberCount = err2 ? 0 : memberCount || 0 + } + groupDetailsMap.set(groupId, group) + } + } + } + + // 构建最终结果 + const result = new Map() + for (const [accountId, groupIds] of accountGroupIdsMap) { + const groups = groupIds + .map((gid) => groupDetailsMap.get(gid)) + .filter(Boolean) + .sort((a, b) => new Date(b.createdAt) - new Date(a.createdAt)) + result.set(accountId, groups) + } + + return result + } catch (error) { + logger.error('❌ 批量获取账户分组失败:', error) + return new Map(accountIds.map((id) => [id, []])) + } + } } module.exports = new AccountGroupService() diff --git a/src/services/apiKeyIndexService.js b/src/services/apiKeyIndexService.js new file mode 100644 index 00000000..cd8117e3 --- /dev/null +++ b/src/services/apiKeyIndexService.js @@ -0,0 +1,642 @@ +/** + * API Key 索引服务 + * 维护 Sorted Set 索引以支持高效分页查询 + */ + +const { randomUUID } = require('crypto') +const logger = require('../utils/logger') + +class ApiKeyIndexService { + constructor() { + this.redis = null + this.INDEX_VERSION_KEY = 'apikey:index:version' + this.CURRENT_VERSION = 2 // 版本升级，触发重建 + this.isBuilding = false + this.buildProgress = { current: 0, total: 0 } + + // 索引键名 + this.INDEX_KEYS = { + CREATED_AT: 'apikey:idx:createdAt', + LAST_USED_AT: 'apikey:idx:lastUsedAt', + NAME: 'apikey:idx:name', + ACTIVE_SET: 'apikey:set:active', + DELETED_SET: 'apikey:set:deleted', + ALL_SET: 'apikey:idx:all', + TAGS_ALL: 'apikey:tags:all' // 所有标签的集合 + } + } + + /** + * 初始化服务 + */ + init(redis) { + this.redis = redis + return this + } + + /** + * 启动时检查并重建索引 + */ + async checkAndRebuild() { + if (!this.redis) { + logger.warn('⚠️ ApiKeyIndexService: Redis not initialized') + return + } + + try { + const client = this.redis.getClientSafe() + const version = await client.get(this.INDEX_VERSION_KEY) + + // 始终检查并回填 hash_map（幂等操作，确保升级兼容） + this.rebuildHashMap().catch((err) => { + logger.error('❌ API Key hash_map 回填失败:', err) + }) + + if (parseInt(version) >= this.CURRENT_VERSION) { + logger.info('✅ API Key 索引已是最新版本') + return + } + + // 后台异步重建，不阻塞启动 + this.rebuildIndexes().catch((err) => { + logger.error('❌ API Key 索引重建失败:', err) + }) + } catch (error) { + logger.error('❌ 检查 API Key 索引版本失败:', error) + } + } + + /** + * 回填 apikey:hash_map（升级兼容） + * 扫描所有 API Key，确保 hash -> keyId 映射存在 + */ + async rebuildHashMap() { + if (!this.redis) return + + try { + const client = this.redis.getClientSafe() + const keyIds = await this.redis.scanApiKeyIds() + + let rebuilt = 0 + const BATCH_SIZE = 100 + + for (let i = 0; i < keyIds.length; i += BATCH_SIZE) { + const batch = keyIds.slice(i, i + BATCH_SIZE) + const pipeline = client.pipeline() + + // 批量获取 API Key 数据 + for (const keyId of batch) { + pipeline.hgetall(`apikey:${keyId}`) + } + const results = await pipeline.exec() + + // 检查并回填缺失的映射 + const fillPipeline = client.pipeline() + let needFill = false + + for (let j = 0; j < batch.length; j++) { + const keyData = results[j]?.[1] + if (keyData && keyData.apiKey) { + // keyData.apiKey 存储的是哈希值 + const exists = await client.hexists('apikey:hash_map', keyData.apiKey) + if (!exists) { + fillPipeline.hset('apikey:hash_map', keyData.apiKey, batch[j]) + rebuilt++ + needFill = true + } + } + } + + if (needFill) { + await fillPipeline.exec() + } + } + + if (rebuilt > 0) { + logger.info(`🔧 回填了 ${rebuilt} 个 API Key 到 hash_map`) + } + } catch (error) { + logger.error('❌ 回填 hash_map 失败:', error) + throw error + } + } + + /** + * 检查索引是否可用 + */ + async isIndexReady() { + if (!this.redis || this.isBuilding) { + return false + } + + try { + const client = this.redis.getClientSafe() + const version = await client.get(this.INDEX_VERSION_KEY) + return parseInt(version) >= this.CURRENT_VERSION + } catch { + return false + } + } + + /** + * 重建所有索引 + */ + async rebuildIndexes() { + if (this.isBuilding) { + logger.warn('⚠️ API Key 索引正在重建中，跳过') + return + } + + this.isBuilding = true + const startTime = Date.now() + + try { + const client = this.redis.getClientSafe() + logger.info('🔨 开始重建 API Key 索引...') + + // 0. 先删除版本号，让 _checkIndexReady 返回 false，查询回退到 SCAN + await client.del(this.INDEX_VERSION_KEY) + + // 1. 清除旧索引 + const indexKeys = Object.values(this.INDEX_KEYS) + for (const key of indexKeys) { + await client.del(key) + } + // 清除标签索引（用 SCAN 避免阻塞） + let cursor = '0' + do { + const [newCursor, keys] = await client.scan(cursor, 'MATCH', 'apikey:tag:*', 'COUNT', 100) + cursor = newCursor + if (keys.length > 0) { + await client.del(...keys) + } + } while (cursor !== '0') + + // 2. 扫描所有 API Key + const keyIds = await this.redis.scanApiKeyIds() + this.buildProgress = { current: 0, total: keyIds.length } + + logger.info(`📊 发现 ${keyIds.length} 个 API Key，开始建立索引...`) + + // 3. 批量处理（每批 500 个） + const BATCH_SIZE = 500 + for (let i = 0; i < keyIds.length; i += BATCH_SIZE) { + const batch = keyIds.slice(i, i + BATCH_SIZE) + const apiKeys = await this.redis.batchGetApiKeys(batch) + + const pipeline = client.pipeline() + + for (const apiKey of apiKeys) { + if (!apiKey || !apiKey.id) continue + + const keyId = apiKey.id + const createdAt = apiKey.createdAt ? new Date(apiKey.createdAt).getTime() : 0 + const lastUsedAt = apiKey.lastUsedAt ? new Date(apiKey.lastUsedAt).getTime() : 0 + const name = (apiKey.name || '').toLowerCase() + const isActive = apiKey.isActive === true || apiKey.isActive === 'true' + const isDeleted = apiKey.isDeleted === true || apiKey.isDeleted === 'true' + + // 创建时间索引 + pipeline.zadd(this.INDEX_KEYS.CREATED_AT, createdAt, keyId) + + // 最后使用时间索引 + pipeline.zadd(this.INDEX_KEYS.LAST_USED_AT, lastUsedAt, keyId) + + // 名称索引（用于排序，存储格式：name\0keyId） + pipeline.zadd(this.INDEX_KEYS.NAME, 0, `${name}\x00${keyId}`) + + // 全部集合 + pipeline.sadd(this.INDEX_KEYS.ALL_SET, keyId) + + // 状态集合 + if (isDeleted) { + pipeline.sadd(this.INDEX_KEYS.DELETED_SET, keyId) + } else if (isActive) { + pipeline.sadd(this.INDEX_KEYS.ACTIVE_SET, keyId) + } + + // 标签索引 + const tags = Array.isArray(apiKey.tags) ? apiKey.tags : [] + for (const tag of tags) { + if (tag && typeof tag === 'string') { + pipeline.sadd(`apikey:tag:${tag}`, keyId) + pipeline.sadd(this.INDEX_KEYS.TAGS_ALL, tag) // 维护标签集合 + } + } + } + + await pipeline.exec() + this.buildProgress.current = Math.min(i + BATCH_SIZE, keyIds.length) + + // 每批次后短暂让出 CPU + await new Promise((resolve) => setTimeout(resolve, 10)) + } + + // 4. 更新版本号 + await client.set(this.INDEX_VERSION_KEY, this.CURRENT_VERSION) + + const duration = ((Date.now() - startTime) / 1000).toFixed(2) + logger.success(`✅ API Key 索引重建完成，共 ${keyIds.length} 条，耗时 ${duration}s`) + } catch (error) { + logger.error('❌ API Key 索引重建失败:', error) + throw error + } finally { + this.isBuilding = false + } + } + + /** + * 添加单个 API Key 到索引 + */ + async addToIndex(apiKey) { + if (!this.redis || !apiKey || !apiKey.id) return + + try { + const client = this.redis.getClientSafe() + const keyId = apiKey.id + const createdAt = apiKey.createdAt ? new Date(apiKey.createdAt).getTime() : Date.now() + const lastUsedAt = apiKey.lastUsedAt ? new Date(apiKey.lastUsedAt).getTime() : 0 + const name = (apiKey.name || '').toLowerCase() + const isActive = apiKey.isActive === true || apiKey.isActive === 'true' + const isDeleted = apiKey.isDeleted === true || apiKey.isDeleted === 'true' + + const pipeline = client.pipeline() + + pipeline.zadd(this.INDEX_KEYS.CREATED_AT, createdAt, keyId) + pipeline.zadd(this.INDEX_KEYS.LAST_USED_AT, lastUsedAt, keyId) + pipeline.zadd(this.INDEX_KEYS.NAME, 0, `${name}\x00${keyId}`) + pipeline.sadd(this.INDEX_KEYS.ALL_SET, keyId) + + if (isDeleted) { + pipeline.sadd(this.INDEX_KEYS.DELETED_SET, keyId) + pipeline.srem(this.INDEX_KEYS.ACTIVE_SET, keyId) + } else if (isActive) { + pipeline.sadd(this.INDEX_KEYS.ACTIVE_SET, keyId) + pipeline.srem(this.INDEX_KEYS.DELETED_SET, keyId) + } else { + pipeline.srem(this.INDEX_KEYS.ACTIVE_SET, keyId) + pipeline.srem(this.INDEX_KEYS.DELETED_SET, keyId) + } + + // 标签索引 + const tags = Array.isArray(apiKey.tags) ? apiKey.tags : [] + for (const tag of tags) { + if (tag && typeof tag === 'string') { + pipeline.sadd(`apikey:tag:${tag}`, keyId) + pipeline.sadd(this.INDEX_KEYS.TAGS_ALL, tag) + } + } + + await pipeline.exec() + } catch (error) { + logger.error(`❌ 添加 API Key ${apiKey.id} 到索引失败:`, error) + } + } + + /** + * 更新索引（状态、名称、标签变化时调用） + */ + async updateIndex(keyId, updates, oldData = {}) { + if (!this.redis || !keyId) return + + try { + const client = this.redis.getClientSafe() + const pipeline = client.pipeline() + + // 更新名称索引 + if (updates.name !== undefined) { + const oldName = (oldData.name || '').toLowerCase() + const newName = (updates.name || '').toLowerCase() + if (oldName !== newName) { + pipeline.zrem(this.INDEX_KEYS.NAME, `${oldName}\x00${keyId}`) + pipeline.zadd(this.INDEX_KEYS.NAME, 0, `${newName}\x00${keyId}`) + } + } + + // 更新最后使用时间索引 + if (updates.lastUsedAt !== undefined) { + const lastUsedAt = updates.lastUsedAt ? new Date(updates.lastUsedAt).getTime() : 0 + pipeline.zadd(this.INDEX_KEYS.LAST_USED_AT, lastUsedAt, keyId) + } + + // 更新状态集合 + if (updates.isActive !== undefined || updates.isDeleted !== undefined) { + const isActive = updates.isActive ?? oldData.isActive + const isDeleted = updates.isDeleted ?? oldData.isDeleted + + if (isDeleted === true || isDeleted === 'true') { + pipeline.sadd(this.INDEX_KEYS.DELETED_SET, keyId) + pipeline.srem(this.INDEX_KEYS.ACTIVE_SET, keyId) + } else if (isActive === true || isActive === 'true') { + pipeline.sadd(this.INDEX_KEYS.ACTIVE_SET, keyId) + pipeline.srem(this.INDEX_KEYS.DELETED_SET, keyId) + } else { + pipeline.srem(this.INDEX_KEYS.ACTIVE_SET, keyId) + pipeline.srem(this.INDEX_KEYS.DELETED_SET, keyId) + } + } + + // 更新标签索引 + const removedTags = [] + if (updates.tags !== undefined) { + const oldTags = Array.isArray(oldData.tags) ? oldData.tags : [] + const newTags = Array.isArray(updates.tags) ? updates.tags : [] + + // 移除旧标签 + for (const tag of oldTags) { + if (tag && !newTags.includes(tag)) { + pipeline.srem(`apikey:tag:${tag}`, keyId) + removedTags.push(tag) + } + } + // 添加新标签 + for (const tag of newTags) { + if (tag && typeof tag === 'string') { + pipeline.sadd(`apikey:tag:${tag}`, keyId) + pipeline.sadd(this.INDEX_KEYS.TAGS_ALL, tag) + } + } + } + + await pipeline.exec() + + // 检查被移除的标签集合是否为空，为空则从 tags:all 移除 + for (const tag of removedTags) { + const count = await client.scard(`apikey:tag:${tag}`) + if (count === 0) { + await client.srem(this.INDEX_KEYS.TAGS_ALL, tag) + } + } + } catch (error) { + logger.error(`❌ 更新 API Key ${keyId} 索引失败:`, error) + } + } + + /** + * 从索引中移除 API Key + */ + async removeFromIndex(keyId, oldData = {}) { + if (!this.redis || !keyId) return + + try { + const client = this.redis.getClientSafe() + const pipeline = client.pipeline() + + const name = (oldData.name || '').toLowerCase() + + pipeline.zrem(this.INDEX_KEYS.CREATED_AT, keyId) + pipeline.zrem(this.INDEX_KEYS.LAST_USED_AT, keyId) + pipeline.zrem(this.INDEX_KEYS.NAME, `${name}\x00${keyId}`) + pipeline.srem(this.INDEX_KEYS.ALL_SET, keyId) + pipeline.srem(this.INDEX_KEYS.ACTIVE_SET, keyId) + pipeline.srem(this.INDEX_KEYS.DELETED_SET, keyId) + + // 移除标签索引 + const tags = Array.isArray(oldData.tags) ? oldData.tags : [] + for (const tag of tags) { + if (tag) { + pipeline.srem(`apikey:tag:${tag}`, keyId) + } + } + + await pipeline.exec() + + // 检查标签集合是否为空，为空则从 tags:all 移除 + for (const tag of tags) { + if (tag) { + const count = await client.scard(`apikey:tag:${tag}`) + if (count === 0) { + await client.srem(this.INDEX_KEYS.TAGS_ALL, tag) + } + } + } + } catch (error) { + logger.error(`❌ 从索引移除 API Key ${keyId} 失败:`, error) + } + } + + /** + * 使用索引进行分页查询 + * 使用 ZINTERSTORE 优化，避免全量拉回内存 + */ + async queryWithIndex(options = {}) { + const { + page = 1, + pageSize = 20, + sortBy = 'createdAt', + sortOrder = 'desc', + isActive, + tag, + excludeDeleted = true + } = options + + const client = this.redis.getClientSafe() + const tempSets = [] + + try { + // 1. 构建筛选集合 + let filterSet = this.INDEX_KEYS.ALL_SET + + // 状态筛选 + if (isActive === true || isActive === 'true') { + // 筛选活跃的 + filterSet = this.INDEX_KEYS.ACTIVE_SET + } else if (isActive === false || isActive === 'false') { + // 筛选未激活的 = ALL - ACTIVE (- DELETED if excludeDeleted) + const tempKey = `apikey:tmp:inactive:${randomUUID()}` + if (excludeDeleted) { + await client.sdiffstore( + tempKey, + this.INDEX_KEYS.ALL_SET, + this.INDEX_KEYS.ACTIVE_SET, + this.INDEX_KEYS.DELETED_SET + ) + } else { + await client.sdiffstore(tempKey, this.INDEX_KEYS.ALL_SET, this.INDEX_KEYS.ACTIVE_SET) + } + await client.expire(tempKey, 60) + filterSet = tempKey + tempSets.push(tempKey) + } else if (excludeDeleted) { + // 排除已删除：ALL - DELETED + const tempKey = `apikey:tmp:notdeleted:${randomUUID()}` + await client.sdiffstore(tempKey, this.INDEX_KEYS.ALL_SET, this.INDEX_KEYS.DELETED_SET) + await client.expire(tempKey, 60) + filterSet = tempKey + tempSets.push(tempKey) + } + + // 标签筛选 + if (tag) { + const tagSet = `apikey:tag:${tag}` + const tempKey = `apikey:tmp:tag:${randomUUID()}` + await client.sinterstore(tempKey, filterSet, tagSet) + await client.expire(tempKey, 60) + filterSet = tempKey + tempSets.push(tempKey) + } + + // 2. 获取筛选后的 keyId 集合 + const filterMembers = await client.smembers(filterSet) + if (filterMembers.length === 0) { + // 没有匹配的数据 + return { + items: [], + pagination: { page: 1, pageSize, total: 0, totalPages: 1 }, + availableTags: await this._getAvailableTags(client) + } + } + + // 3. 排序 + let sortedKeyIds + + if (sortBy === 'name') { + // 优化：只拉筛选后 keyId 的 name 字段，避免全量扫描 name 索引 + const pipeline = client.pipeline() + for (const keyId of filterMembers) { + pipeline.hget(`apikey:${keyId}`, 'name') + } + const results = await pipeline.exec() + + // 组装并排序 + const items = filterMembers.map((keyId, i) => ({ + keyId, + name: (results[i]?.[1] || '').toLowerCase() + })) + items.sort((a, b) => + sortOrder === 'desc' ? b.name.localeCompare(a.name) : a.name.localeCompare(b.name) + ) + sortedKeyIds = items.map((item) => item.keyId) + } else { + // createdAt / lastUsedAt 索引成员是 keyId，可以用 ZINTERSTORE + const sortIndex = this._getSortIndex(sortBy) + const tempSortedKey = `apikey:tmp:sorted:${randomUUID()}` + tempSets.push(tempSortedKey) + + // 将 filterSet 转换为 Sorted Set（所有分数为 0） + const filterZsetKey = `apikey:tmp:filter:${randomUUID()}` + tempSets.push(filterZsetKey) + + const zaddArgs = [] + for (const member of filterMembers) { + zaddArgs.push(0, member) + } + await client.zadd(filterZsetKey, ...zaddArgs) + await client.expire(filterZsetKey, 60) + + // ZINTERSTORE：取交集，使用排序索引的分数（WEIGHTS 0 1） + await client.zinterstore(tempSortedKey, 2, filterZsetKey, sortIndex, 'WEIGHTS', 0, 1) + await client.expire(tempSortedKey, 60) + + // 获取排序后的 keyId + sortedKeyIds = + sortOrder === 'desc' + ? await client.zrevrange(tempSortedKey, 0, -1) + : await client.zrange(tempSortedKey, 0, -1) + } + + // 4. 分页 + const total = sortedKeyIds.length + const totalPages = Math.max(Math.ceil(total / pageSize), 1) + const validPage = Math.min(Math.max(1, page), totalPages) + const start = (validPage - 1) * pageSize + const pageKeyIds = sortedKeyIds.slice(start, start + pageSize) + + // 5. 获取数据 + const items = await this.redis.batchGetApiKeys(pageKeyIds) + + // 6. 获取所有标签 + const availableTags = await this._getAvailableTags(client) + + return { + items, + pagination: { + page: validPage, + pageSize, + total, + totalPages + }, + availableTags + } + } finally { + // 7. 清理临时集合 + for (const tempKey of tempSets) { + client.del(tempKey).catch(() => {}) + } + } + } + + /** + * 获取排序索引键名 + */ + _getSortIndex(sortBy) { + switch (sortBy) { + case 'createdAt': + return this.INDEX_KEYS.CREATED_AT + case 'lastUsedAt': + return this.INDEX_KEYS.LAST_USED_AT + case 'name': + return this.INDEX_KEYS.NAME + default: + return this.INDEX_KEYS.CREATED_AT + } + } + + /** + * 获取所有可用标签（从 tags:all 集合） + */ + async _getAvailableTags(client) { + try { + const tags = await client.smembers(this.INDEX_KEYS.TAGS_ALL) + return tags.sort() + } catch { + return [] + } + } + + /** + * 更新 lastUsedAt 索引（供 recordUsage 调用） + */ + async updateLastUsedAt(keyId, lastUsedAt) { + if (!this.redis || !keyId) return + + try { + const client = this.redis.getClientSafe() + const timestamp = lastUsedAt ? new Date(lastUsedAt).getTime() : Date.now() + await client.zadd(this.INDEX_KEYS.LAST_USED_AT, timestamp, keyId) + } catch (error) { + logger.error(`❌ 更新 API Key ${keyId} lastUsedAt 索引失败:`, error) + } + } + + /** + * 获取索引状态 + */ + async getStatus() { + if (!this.redis) { + return { ready: false, building: false } + } + + try { + const client = this.redis.getClientSafe() + const version = await client.get(this.INDEX_VERSION_KEY) + const totalCount = await client.scard(this.INDEX_KEYS.ALL_SET) + + return { + ready: parseInt(version) >= this.CURRENT_VERSION, + building: this.isBuilding, + progress: this.buildProgress, + version: parseInt(version) || 0, + currentVersion: this.CURRENT_VERSION, + totalIndexed: totalCount + } + } catch { + return { ready: false, building: this.isBuilding } + } + } +} + +// 单例 +const apiKeyIndexService = new ApiKeyIndexService() + +module.exports = apiKeyIndexService diff --git a/src/services/apiKeyService.js b/src/services/apiKeyService.js index 0e9e7597..b71f4af2 100644 --- a/src/services/apiKeyService.js +++ b/src/services/apiKeyService.js @@ -166,6 +166,22 @@ class ApiKeyService { logger.warn(`Failed to add key ${keyId} to cost rank indexes:`, err.message) } + // 同步添加到 API Key 索引（用于分页查询优化） + try { + const apiKeyIndexService = require('./apiKeyIndexService') + await apiKeyIndexService.addToIndex({ + id: keyId, + name: keyData.name, + createdAt: keyData.createdAt, + lastUsedAt: keyData.lastUsedAt, + isActive: keyData.isActive === 'true', + isDeleted: false, + tags: JSON.parse(keyData.tags || '[]') + }) + } catch (err) { + logger.warn(`Failed to add key ${keyId} to API Key index:`, err.message) + } + logger.success(`🔑 Generated new API key: ${name} (${keyId})`) return { @@ -493,6 +509,11 @@ class ApiKeyService { } } + // 🏷️ 获取所有标签（轻量级，使用 SCAN + Pipeline） + async getAllTags() { + return await redis.scanAllApiKeyTags() + } + // 📋 获取所有API Keys async getAllApiKeys(includeDeleted = false) { try { @@ -657,6 +678,266 @@ class ApiKeyService { } } + /** + * 🚀 快速获取所有 API Keys（使用 Pipeline 批量操作，性能优化版） + * 适用于 dashboard、usage-costs 等需要大量 API Key 数据的场景 + * @param {boolean} includeDeleted - 是否包含已删除的 API Keys + * @returns {Promise} API Keys 列表 + */ + async getAllApiKeysFast(includeDeleted = false) { + try { + // 1. 使用 SCAN 获取所有 API Key IDs + const keyIds = await redis.scanApiKeyIds() + if (keyIds.length === 0) { + return [] + } + + // 2. 批量获取基础数据 + let apiKeys = await redis.batchGetApiKeys(keyIds) + + // 3. 过滤已删除的 + if (!includeDeleted) { + apiKeys = apiKeys.filter((key) => !key.isDeleted) + } + + // 4. 批量获取统计数据（单次 Pipeline） + const activeKeyIds = apiKeys.map((k) => k.id) + const statsMap = await redis.batchGetApiKeyStats(activeKeyIds) + + // 5. 合并数据 + for (const key of apiKeys) { + const stats = statsMap.get(key.id) || {} + + // 处理 usage 数据 + const usageTotal = stats.usageTotal || {} + const usageDaily = stats.usageDaily || {} + const usageMonthly = stats.usageMonthly || {} + + // 计算平均 RPM/TPM + const createdAt = stats.createdAt ? new Date(stats.createdAt) : new Date() + const daysSinceCreated = Math.max( + 1, + Math.ceil((Date.now() - createdAt.getTime()) / (1000 * 60 * 60 * 24)) + ) + const totalMinutes = daysSinceCreated * 24 * 60 + // 兼容旧数据格式：优先读 totalXxx，fallback 到 xxx + const totalRequests = parseInt(usageTotal.totalRequests || usageTotal.requests) || 0 + const totalTokens = parseInt(usageTotal.totalTokens || usageTotal.tokens) || 0 + let inputTokens = parseInt(usageTotal.totalInputTokens || usageTotal.inputTokens) || 0 + let outputTokens = parseInt(usageTotal.totalOutputTokens || usageTotal.outputTokens) || 0 + let cacheCreateTokens = + parseInt(usageTotal.totalCacheCreateTokens || usageTotal.cacheCreateTokens) || 0 + let cacheReadTokens = + parseInt(usageTotal.totalCacheReadTokens || usageTotal.cacheReadTokens) || 0 + + // 旧数据兼容：没有 input/output 分离时做 30/70 拆分 + const totalFromSeparate = inputTokens + outputTokens + if (totalFromSeparate === 0 && totalTokens > 0) { + inputTokens = Math.round(totalTokens * 0.3) + outputTokens = Math.round(totalTokens * 0.7) + cacheCreateTokens = 0 + cacheReadTokens = 0 + } + + // allTokens：优先读存储值，否则计算，最后 fallback 到 totalTokens + const allTokens = + parseInt(usageTotal.totalAllTokens || usageTotal.allTokens) || + inputTokens + outputTokens + cacheCreateTokens + cacheReadTokens || + totalTokens + + key.usage = { + total: { + requests: totalRequests, + tokens: allTokens, // 与 getUsageStats 语义一致：包含 cache 的总 tokens + inputTokens, + outputTokens, + cacheCreateTokens, + cacheReadTokens, + allTokens, + cost: stats.costStats?.total || 0 + }, + daily: { + requests: parseInt(usageDaily.totalRequests || usageDaily.requests) || 0, + tokens: parseInt(usageDaily.totalTokens || usageDaily.tokens) || 0 + }, + monthly: { + requests: parseInt(usageMonthly.totalRequests || usageMonthly.requests) || 0, + tokens: parseInt(usageMonthly.totalTokens || usageMonthly.tokens) || 0 + }, + averages: { + rpm: Math.round((totalRequests / totalMinutes) * 100) / 100, + tpm: Math.round((totalTokens / totalMinutes) * 100) / 100 + }, + totalCost: stats.costStats?.total || 0 + } + + // 费用统计 + key.totalCost = stats.costStats?.total || 0 + key.dailyCost = stats.dailyCost || 0 + key.weeklyOpusCost = stats.weeklyOpusCost || 0 + + // 并发 + key.currentConcurrency = stats.concurrency || 0 + + // 类型转换 + key.tokenLimit = parseInt(key.tokenLimit) || 0 + key.concurrencyLimit = parseInt(key.concurrencyLimit) || 0 + key.rateLimitWindow = parseInt(key.rateLimitWindow) || 0 + key.rateLimitRequests = parseInt(key.rateLimitRequests) || 0 + key.rateLimitCost = parseFloat(key.rateLimitCost) || 0 + key.dailyCostLimit = parseFloat(key.dailyCostLimit) || 0 + key.totalCostLimit = parseFloat(key.totalCostLimit) || 0 + key.weeklyOpusCostLimit = parseFloat(key.weeklyOpusCostLimit) || 0 + key.activationDays = parseInt(key.activationDays) || 0 + key.isActive = key.isActive === 'true' || key.isActive === true + key.enableModelRestriction = + key.enableModelRestriction === 'true' || key.enableModelRestriction === true + key.enableClientRestriction = + key.enableClientRestriction === 'true' || key.enableClientRestriction === true + key.isActivated = key.isActivated === 'true' || key.isActivated === true + key.permissions = key.permissions || 'all' + key.activationUnit = key.activationUnit || 'days' + key.expirationMode = key.expirationMode || 'fixed' + key.activatedAt = key.activatedAt || null + + // Rate limit 窗口数据 + if (key.rateLimitWindow > 0) { + const rl = stats.rateLimit || {} + key.currentWindowRequests = rl.requests || 0 + key.currentWindowTokens = rl.tokens || 0 + key.currentWindowCost = rl.cost || 0 + + if (rl.windowStart) { + const now = Date.now() + const windowDuration = key.rateLimitWindow * 60 * 1000 + const windowEndTime = rl.windowStart + windowDuration + + if (now < windowEndTime) { + key.windowStartTime = rl.windowStart + key.windowEndTime = windowEndTime + key.windowRemainingSeconds = Math.max(0, Math.floor((windowEndTime - now) / 1000)) + } else { + key.windowStartTime = null + key.windowEndTime = null + key.windowRemainingSeconds = 0 + key.currentWindowRequests = 0 + key.currentWindowTokens = 0 + key.currentWindowCost = 0 + } + } else { + key.windowStartTime = null + key.windowEndTime = null + key.windowRemainingSeconds = null + } + } else { + key.currentWindowRequests = 0 + key.currentWindowTokens = 0 + key.currentWindowCost = 0 + key.windowStartTime = null + key.windowEndTime = null + key.windowRemainingSeconds = null + } + + // JSON 字段解析（兼容已解析的数组和未解析的字符串） + if (Array.isArray(key.restrictedModels)) { + // 已解析，保持不变 + } else if (key.restrictedModels) { + try { + key.restrictedModels = JSON.parse(key.restrictedModels) + } catch { + key.restrictedModels = [] + } + } else { + key.restrictedModels = [] + } + if (Array.isArray(key.allowedClients)) { + // 已解析，保持不变 + } else if (key.allowedClients) { + try { + key.allowedClients = JSON.parse(key.allowedClients) + } catch { + key.allowedClients = [] + } + } else { + key.allowedClients = [] + } + if (Array.isArray(key.tags)) { + // 已解析，保持不变 + } else if (key.tags) { + try { + key.tags = JSON.parse(key.tags) + } catch { + key.tags = [] + } + } else { + key.tags = [] + } + + // 生成掩码key后再清理敏感字段 + if (key.apiKey) { + key.maskedKey = `${this.prefix}****${key.apiKey.slice(-4)}` + } + delete key.apiKey + delete key.ccrAccountId + + // 不获取 lastUsage（太慢），设为 null + key.lastUsage = null + } + + return apiKeys + } catch (error) { + logger.error('❌ Failed to get API keys (fast):', error) + throw error + } + } + + /** + * 获取所有 API Keys 的轻量版本（仅绑定字段，用于计算绑定数） + * @returns {Promise} 包含绑定字段的 API Keys 列表 + */ + async getAllApiKeysLite() { + try { + const client = redis.getClientSafe() + const keyIds = await redis.scanApiKeyIds() + + if (keyIds.length === 0) { + return [] + } + + // Pipeline 只获取绑定相关字段 + const pipeline = client.pipeline() + for (const keyId of keyIds) { + pipeline.hmget( + `apikey:${keyId}`, + 'claudeAccountId', + 'geminiAccountId', + 'openaiAccountId', + 'droidAccountId', + 'isDeleted' + ) + } + const results = await pipeline.exec() + + return keyIds + .map((id, i) => { + const [err, fields] = results[i] + if (err) return null + return { + id, + claudeAccountId: fields[0] || null, + geminiAccountId: fields[1] || null, + openaiAccountId: fields[2] || null, + droidAccountId: fields[3] || null, + isDeleted: fields[4] === 'true' + } + }) + .filter((k) => k && !k.isDeleted) + } catch (error) { + logger.error('❌ Failed to get API keys (lite):', error) + return [] + } + } + // 📝 更新API Key async updateApiKey(keyId, updates) { try { @@ -730,6 +1011,19 @@ class ApiKeyService { // keyData.apiKey 存储的就是 hashedKey（见generateApiKey第123行） await redis.setApiKey(keyId, updatedData, keyData.apiKey) + // 同步更新 API Key 索引 + try { + const apiKeyIndexService = require('./apiKeyIndexService') + await apiKeyIndexService.updateIndex(keyId, updates, { + name: keyData.name, + isActive: keyData.isActive === 'true', + isDeleted: keyData.isDeleted === 'true', + tags: JSON.parse(keyData.tags || '[]') + }) + } catch (err) { + logger.warn(`Failed to update API Key index for ${keyId}:`, err.message) + } + logger.success(`📝 Updated API key: ${keyId}, hashMap updated`) return { success: true } @@ -772,6 +1066,23 @@ class ApiKeyService { logger.warn(`Failed to remove key ${keyId} from cost rank indexes:`, err.message) } + // 更新 API Key 索引（标记为已删除） + try { + const apiKeyIndexService = require('./apiKeyIndexService') + await apiKeyIndexService.updateIndex( + keyId, + { isDeleted: true, isActive: false }, + { + name: keyData.name, + isActive: keyData.isActive === 'true', + isDeleted: false, + tags: JSON.parse(keyData.tags || '[]') + } + ) + } catch (err) { + logger.warn(`Failed to update API Key index for deleted key ${keyId}:`, err.message) + } + logger.success(`🗑️ Soft deleted API key: ${keyId} by ${deletedBy} (${deletedByType})`) return { success: true } @@ -831,7 +1142,24 @@ class ApiKeyService { logger.warn(`Failed to add restored key ${keyId} to cost rank indexes:`, err.message) } - logger.success(`✅ Restored API key: ${keyId} by ${restoredBy} (${restoredByType})`) + // 更新 API Key 索引（恢复为活跃状态） + try { + const apiKeyIndexService = require('./apiKeyIndexService') + await apiKeyIndexService.updateIndex( + keyId, + { isDeleted: false, isActive: true }, + { + name: keyData.name, + isActive: false, + isDeleted: true, + tags: JSON.parse(keyData.tags || '[]') + } + ) + } catch (err) { + logger.warn(`Failed to update API Key index for restored key ${keyId}:`, err.message) + } + + logger.success(`Restored API key: ${keyId} by ${restoredBy} (${restoredByType})`) return { success: true, apiKey: updatedData } } catch (error) { @@ -866,9 +1194,20 @@ class ApiKeyService { await redis.client.del(`usage:monthly:${currentMonth}:${keyId}`) // 删除所有相关的统计键（通过模式匹配） - const usageKeys = await redis.client.keys(`usage:*:${keyId}*`) + const usageKeys = await redis.scanKeys(`usage:*:${keyId}*`) if (usageKeys.length > 0) { - await redis.client.del(...usageKeys) + await redis.batchDelChunked(usageKeys) + } + + // 从 API Key 索引中移除 + try { + const apiKeyIndexService = require('./apiKeyIndexService') + await apiKeyIndexService.removeFromIndex(keyId, { + name: keyData.name, + tags: JSON.parse(keyData.tags || '[]') + }) + } catch (err) { + logger.warn(`Failed to remove key ${keyId} from API Key index:`, err.message) } // 删除API Key本身 @@ -886,8 +1225,8 @@ class ApiKeyService { // 🧹 清空所有已删除的API Keys async clearAllDeletedApiKeys() { try { - const allKeys = await this.getAllApiKeys(true) - const deletedKeys = allKeys.filter((key) => key.isDeleted === 'true') + const allKeys = await this.getAllApiKeysFast(true) + const deletedKeys = allKeys.filter((key) => key.isDeleted === true) let successCount = 0 let failedCount = 0 @@ -982,9 +1321,18 @@ class ApiKeyService { const keyData = await redis.getApiKey(keyId) if (keyData && Object.keys(keyData).length > 0) { // 更新最后使用时间 - keyData.lastUsedAt = new Date().toISOString() + const lastUsedAt = new Date().toISOString() + keyData.lastUsedAt = lastUsedAt await redis.setApiKey(keyId, keyData) + // 同步更新 lastUsedAt 索引 + try { + const apiKeyIndexService = require('./apiKeyIndexService') + await apiKeyIndexService.updateLastUsedAt(keyId, lastUsedAt) + } catch (err) { + // 索引更新失败不影响主流程 + } + // 记录账户级别的使用统计（只统计实际处理请求的账户） if (accountId) { await redis.incrementAccountUsage( @@ -1192,9 +1540,18 @@ class ApiKeyService { const keyData = await redis.getApiKey(keyId) if (keyData && Object.keys(keyData).length > 0) { // 更新最后使用时间 - keyData.lastUsedAt = new Date().toISOString() + const lastUsedAt = new Date().toISOString() + keyData.lastUsedAt = lastUsedAt await redis.setApiKey(keyId, keyData) + // 同步更新 lastUsedAt 索引 + try { + const apiKeyIndexService = require('./apiKeyIndexService') + await apiKeyIndexService.updateLastUsedAt(keyId, lastUsedAt) + } catch (err) { + // 索引更新失败不影响主流程 + } + // 记录账户级别的使用统计（只统计实际处理请求的账户） if (accountId) { await redis.incrementAccountUsage( @@ -1493,12 +1850,12 @@ class ApiKeyService { // 👤 获取用户的API Keys async getUserApiKeys(userId, includeDeleted = false) { try { - const allKeys = await redis.getAllApiKeys() + const allKeys = await this.getAllApiKeysFast(includeDeleted) let userKeys = allKeys.filter((key) => key.userId === userId) - // 默认过滤掉已删除的API Keys + // 默认过滤掉已删除的API Keys（Fast版本返回布尔值） if (!includeDeleted) { - userKeys = userKeys.filter((key) => key.isDeleted !== 'true') + userKeys = userKeys.filter((key) => !key.isDeleted) } // Populate usage stats for each user's API key (same as getAllApiKeys does) @@ -1512,9 +1869,9 @@ class ApiKeyService { id: key.id, name: key.name, description: key.description, - key: key.apiKey ? `${this.prefix}****${key.apiKey.slice(-4)}` : null, // 只显示前缀和后4位 + key: key.maskedKey || null, // Fast版本已提供maskedKey tokenLimit: parseInt(key.tokenLimit || 0), - isActive: key.isActive === 'true', + isActive: key.isActive === true, // Fast版本返回布尔值 createdAt: key.createdAt, lastUsedAt: key.lastUsedAt, expiresAt: key.expiresAt, @@ -1738,7 +2095,7 @@ class ApiKeyService { } // 获取所有API Keys - const allKeys = await this.getAllApiKeys() + const allKeys = await this.getAllApiKeysFast() // 筛选绑定到此账号的 API Keys let boundKeys = [] @@ -1788,13 +2145,13 @@ class ApiKeyService { // 🧹 清理过期的API Keys async cleanupExpiredKeys() { try { - const apiKeys = await redis.getAllApiKeys() + const apiKeys = await this.getAllApiKeysFast() const now = new Date() let cleanedCount = 0 for (const key of apiKeys) { - // 检查是否已过期且仍处于激活状态 - if (key.expiresAt && new Date(key.expiresAt) < now && key.isActive === 'true') { + // 检查是否已过期且仍处于激活状态（Fast版本返回布尔值） + if (key.expiresAt && new Date(key.expiresAt) < now && key.isActive === true) { // 将过期的 API Key 标记为禁用状态，而不是直接删除 await this.updateApiKey(key.id, { isActive: false }) logger.info(`🔒 API Key ${key.id} (${key.name}) has expired and been disabled`) diff --git a/src/services/azureOpenaiAccountService.js b/src/services/azureOpenaiAccountService.js index 1f8ded80..25624142 100644 --- a/src/services/azureOpenaiAccountService.js +++ b/src/services/azureOpenaiAccountService.js @@ -150,6 +150,7 @@ async function createAccount(accountData) { const client = redisClient.getClientSafe() await client.hset(`${AZURE_OPENAI_ACCOUNT_KEY_PREFIX}${accountId}`, account) + await redisClient.addToIndex('azure_openai:account:index', accountId) // 如果是共享账户，添加到共享账户集合 if (account.accountType === 'shared') { @@ -270,6 +271,9 @@ async function deleteAccount(accountId) { // 从Redis中删除账户数据 await client.del(accountKey) + // 从索引中移除 + await redisClient.removeFromIndex('azure_openai:account:index', accountId) + // 从共享账户集合中移除 await client.srem(SHARED_AZURE_OPENAI_ACCOUNTS_KEY, accountId) @@ -279,16 +283,22 @@ async function deleteAccount(accountId) { // 获取所有账户 async function getAllAccounts() { - const client = redisClient.getClientSafe() - const keys = await client.keys(`${AZURE_OPENAI_ACCOUNT_KEY_PREFIX}*`) + const accountIds = await redisClient.getAllIdsByIndex( + 'azure_openai:account:index', + `${AZURE_OPENAI_ACCOUNT_KEY_PREFIX}*`, + /^azure_openai:account:(.+)$/ + ) - if (!keys || keys.length === 0) { + if (!accountIds || accountIds.length === 0) { return [] } + const keys = accountIds.map((id) => `${AZURE_OPENAI_ACCOUNT_KEY_PREFIX}${id}`) const accounts = [] - for (const key of keys) { - const accountData = await client.hgetall(key) + const dataList = await redisClient.batchHgetallChunked(keys) + + for (let i = 0; i < keys.length; i++) { + const accountData = dataList[i] if (accountData && Object.keys(accountData).length > 0) { // 不返回敏感数据给前端 delete accountData.apiKey diff --git a/src/services/bedrockAccountService.js b/src/services/bedrockAccountService.js index cd404b13..42fc73e3 100644 --- a/src/services/bedrockAccountService.js +++ b/src/services/bedrockAccountService.js @@ -73,6 +73,7 @@ class BedrockAccountService { const client = redis.getClientSafe() await client.set(`bedrock_account:${accountId}`, JSON.stringify(accountData)) + await redis.addToIndex('bedrock_account:index', accountId) logger.info(`✅ 创建Bedrock账户成功 - ID: ${accountId}, 名称: ${name}, 区域: ${region}`) @@ -127,11 +128,17 @@ class BedrockAccountService { async getAllAccounts() { try { const client = redis.getClientSafe() - const keys = await client.keys('bedrock_account:*') + const accountIds = await redis.getAllIdsByIndex( + 'bedrock_account:index', + 'bedrock_account:*', + /^bedrock_account:(.+)$/ + ) + const keys = accountIds.map((id) => `bedrock_account:${id}`) const accounts = [] + const dataList = await redis.batchGetChunked(keys) - for (const key of keys) { - const accountData = await client.get(key) + for (let i = 0; i < keys.length; i++) { + const accountData = dataList[i] if (accountData) { const account = JSON.parse(accountData) @@ -280,6 +287,7 @@ class BedrockAccountService { const client = redis.getClientSafe() await client.del(`bedrock_account:${accountId}`) + await redis.removeFromIndex('bedrock_account:index', accountId) logger.info(`✅ 删除Bedrock账户成功 - ID: ${accountId}`) diff --git a/src/services/billingEventPublisher.js b/src/services/billingEventPublisher.js index 5be0a6b4..82631d40 100644 --- a/src/services/billingEventPublisher.js +++ b/src/services/billingEventPublisher.js @@ -208,7 +208,7 @@ class BillingEventPublisher { // MKSTREAM: 如果 stream 不存在则创建 await client.xgroup('CREATE', this.streamKey, groupName, '0', 'MKSTREAM') - logger.success(`✅ Created consumer group: ${groupName}`) + logger.success(`Created consumer group: ${groupName}`) return true } catch (error) { if (error.message.includes('BUSYGROUP')) { diff --git a/src/services/ccrAccountService.js b/src/services/ccrAccountService.js index eeb119c2..c074d6cf 100644 --- a/src/services/ccrAccountService.js +++ b/src/services/ccrAccountService.js @@ -1,33 +1,24 @@ const { v4: uuidv4 } = require('uuid') -const crypto = require('crypto') const ProxyHelper = require('../utils/proxyHelper') const redis = require('../models/redis') const logger = require('../utils/logger') const config = require('../../config/config') -const LRUCache = require('../utils/lruCache') +const { createEncryptor } = require('../utils/commonHelper') class CcrAccountService { constructor() { - // 加密相关常量 - this.ENCRYPTION_ALGORITHM = 'aes-256-cbc' - this.ENCRYPTION_SALT = 'ccr-account-salt' - // Redis键前缀 this.ACCOUNT_KEY_PREFIX = 'ccr_account:' this.SHARED_ACCOUNTS_KEY = 'shared_ccr_accounts' - // 🚀 性能优化：缓存派生的加密密钥，避免每次重复计算 - // scryptSync 是 CPU 密集型操作，缓存可以减少 95%+ 的 CPU 密集型操作 - this._encryptionKeyCache = null - - // 🔄 解密结果缓存，提高解密性能 - this._decryptCache = new LRUCache(500) + // 使用 commonHelper 的加密器 + this._encryptor = createEncryptor('ccr-account-salt') // 🧹 定期清理缓存（每10分钟） setInterval( () => { - this._decryptCache.cleanup() - logger.info('🧹 CCR account decrypt cache cleanup completed', this._decryptCache.getStats()) + this._encryptor.clearCache() + logger.info('🧹 CCR account decrypt cache cleanup completed', this._encryptor.getStats()) }, 10 * 60 * 1000 ) @@ -106,6 +97,7 @@ class CcrAccountService { logger.debug(`[DEBUG] CCR Account data to save: ${JSON.stringify(accountData, null, 2)}`) await client.hset(`${this.ACCOUNT_KEY_PREFIX}${accountId}`, accountData) + await redis.addToIndex('ccr_account:index', accountId) // 如果是共享账户，添加到共享账户集合 if (accountType === 'shared') { @@ -139,12 +131,17 @@ class CcrAccountService { // 📋 获取所有CCR账户 async getAllAccounts() { try { - const client = redis.getClientSafe() - const keys = await client.keys(`${this.ACCOUNT_KEY_PREFIX}*`) + const accountIds = await redis.getAllIdsByIndex( + 'ccr_account:index', + `${this.ACCOUNT_KEY_PREFIX}*`, + /^ccr_account:(.+)$/ + ) + const keys = accountIds.map((id) => `${this.ACCOUNT_KEY_PREFIX}${id}`) const accounts = [] + const dataList = await redis.batchHgetallChunked(keys) - for (const key of keys) { - const accountData = await client.hgetall(key) + for (let i = 0; i < keys.length; i++) { + const accountData = dataList[i] if (accountData && Object.keys(accountData).length > 0) { // 获取限流状态信息 const rateLimitInfo = this._getRateLimitInfo(accountData) @@ -331,6 +328,9 @@ class CcrAccountService { // 从共享账户集合中移除 await client.srem(this.SHARED_ACCOUNTS_KEY, accountId) + // 从索引中移除 + await redis.removeFromIndex('ccr_account:index', accountId) + // 删除账户数据 const result = await client.del(`${this.ACCOUNT_KEY_PREFIX}${accountId}`) @@ -403,7 +403,7 @@ class CcrAccountService { `ℹ️ CCR account ${accountId} rate limit removed but remains stopped due to quota exceeded` ) } else { - logger.success(`✅ Removed rate limit for CCR account: ${accountId}`) + logger.success(`Removed rate limit for CCR account: ${accountId}`) } await client.hmset(accountKey, { @@ -488,7 +488,7 @@ class CcrAccountService { errorMessage: '' }) - logger.success(`✅ Removed overload status for CCR account: ${accountId}`) + logger.success(`Removed overload status for CCR account: ${accountId}`) return { success: true } } catch (error) { logger.error(`❌ Failed to remove overload status for CCR account: ${accountId}`, error) @@ -606,70 +606,12 @@ class CcrAccountService { // 🔐 加密敏感数据 _encryptSensitiveData(data) { - if (!data) { - return '' - } - try { - const key = this._generateEncryptionKey() - const iv = crypto.randomBytes(16) - const cipher = crypto.createCipheriv(this.ENCRYPTION_ALGORITHM, key, iv) - let encrypted = cipher.update(data, 'utf8', 'hex') - encrypted += cipher.final('hex') - return `${iv.toString('hex')}:${encrypted}` - } catch (error) { - logger.error('❌ CCR encryption error:', error) - return data - } + return this._encryptor.encrypt(data) } // 🔓 解密敏感数据 _decryptSensitiveData(encryptedData) { - if (!encryptedData) { - return '' - } - - // 🎯 检查缓存 - const cacheKey = crypto.createHash('sha256').update(encryptedData).digest('hex') - const cached = this._decryptCache.get(cacheKey) - if (cached !== undefined) { - return cached - } - - try { - const parts = encryptedData.split(':') - if (parts.length === 2) { - const key = this._generateEncryptionKey() - const iv = Buffer.from(parts[0], 'hex') - const encrypted = parts[1] - const decipher = crypto.createDecipheriv(this.ENCRYPTION_ALGORITHM, key, iv) - let decrypted = decipher.update(encrypted, 'hex', 'utf8') - decrypted += decipher.final('utf8') - - // 💾 存入缓存（5分钟过期） - this._decryptCache.set(cacheKey, decrypted, 5 * 60 * 1000) - - return decrypted - } else { - logger.error('❌ Invalid CCR encrypted data format') - return encryptedData - } - } catch (error) { - logger.error('❌ CCR decryption error:', error) - return encryptedData - } - } - - // 🔑 生成加密密钥 - _generateEncryptionKey() { - // 性能优化：缓存密钥派生结果，避免重复的 CPU 密集计算 - if (!this._encryptionKeyCache) { - this._encryptionKeyCache = crypto.scryptSync( - config.security.encryptionKey, - this.ENCRYPTION_SALT, - 32 - ) - } - return this._encryptionKeyCache + return this._encryptor.decrypt(encryptedData) } // 🔍 获取限流状态信息 @@ -843,7 +785,7 @@ class CcrAccountService { } } - logger.success(`✅ Reset daily usage for ${resetCount} CCR accounts`) + logger.success(`Reset daily usage for ${resetCount} CCR accounts`) return { success: true, resetCount } } catch (error) { logger.error('❌ Failed to reset all CCR daily usage:', error) @@ -915,7 +857,7 @@ class CcrAccountService { await client.hset(accountKey, updates) await client.hdel(accountKey, ...fieldsToDelete) - logger.success(`✅ Reset all error status for CCR account ${accountId}`) + logger.success(`Reset all error status for CCR account ${accountId}`) // 异步发送 Webhook 通知（忽略错误） try { diff --git a/src/services/claudeAccountService.js b/src/services/claudeAccountService.js index a2f8e6d2..710b7bf4 100644 --- a/src/services/claudeAccountService.js +++ b/src/services/claudeAccountService.js @@ -1570,7 +1570,7 @@ class ClaudeAccountService { 'rateLimitAutoStopped' ) - logger.success(`✅ Rate limit removed for account: ${accountData.name} (${accountId})`) + logger.success(`Rate limit removed for account: ${accountData.name} (${accountId})`) return { success: true } } catch (error) { @@ -2242,7 +2242,7 @@ class ClaudeAccountService { await new Promise((resolve) => setTimeout(resolve, 1000)) } - logger.success(`✅ Profile update completed: ${successCount} success, ${failureCount} failed`) + logger.success(`Profile update completed: ${successCount} success, ${failureCount} failed`) return { totalAccounts: accounts.length, @@ -2310,11 +2310,11 @@ class ClaudeAccountService { } } - logger.success('✅ Session window initialization completed:') - logger.success(` 📊 Total accounts: ${accounts.length}`) - logger.success(` ✅ Valid windows: ${validWindowCount}`) - logger.success(` ⏰ Expired windows: ${expiredWindowCount}`) - logger.success(` 📭 No windows: ${noWindowCount}`) + logger.success('Session window initialization completed:') + logger.success(` Total accounts: ${accounts.length}`) + logger.success(` Valid windows: ${validWindowCount}`) + logger.success(` Expired windows: ${expiredWindowCount}`) + logger.success(` No windows: ${noWindowCount}`) return { total: accounts.length, diff --git a/src/services/claudeCodeHeadersService.js b/src/services/claudeCodeHeadersService.js index 3bbbbea0..8a8b48fb 100644 --- a/src/services/claudeCodeHeadersService.js +++ b/src/services/claudeCodeHeadersService.js @@ -5,6 +5,7 @@ const redis = require('../models/redis') const logger = require('../utils/logger') +const { getCachedConfig, setCachedConfig, deleteCachedConfig } = require('../utils/performanceOptimizer') class ClaudeCodeHeadersService { constructor() { @@ -41,6 +42,9 @@ class ClaudeCodeHeadersService { 'sec-fetch-mode', 'accept-encoding' ] + + // Headers 缓存 TTL（60秒） + this.headersCacheTtl = 60000 } /** @@ -147,6 +151,9 @@ class ClaudeCodeHeadersService { await redis.getClient().setex(key, 86400 * 7, JSON.stringify(data)) // 7天过期 + // 更新内存缓存，避免延迟 + setCachedConfig(key, extractedHeaders, this.headersCacheTtl) + logger.info(`✅ Stored Claude Code headers for account ${accountId}, version: ${version}`) } catch (error) { logger.error(`❌ Failed to store Claude Code headers for account ${accountId}:`, error) @@ -154,18 +161,27 @@ class ClaudeCodeHeadersService { } /** - * 获取账号的 Claude Code headers + * 获取账号的 Claude Code headers（带内存缓存） */ async getAccountHeaders(accountId) { + const cacheKey = `claude_code_headers:${accountId}` + + // 检查内存缓存 + const cached = getCachedConfig(cacheKey) + if (cached) { + return cached + } + try { - const key = `claude_code_headers:${accountId}` - const data = await redis.getClient().get(key) + const data = await redis.getClient().get(cacheKey) if (data) { const parsed = JSON.parse(data) logger.debug( `📋 Retrieved Claude Code headers for account ${accountId}, version: ${parsed.version}` ) + // 缓存到内存 + setCachedConfig(cacheKey, parsed.headers, this.headersCacheTtl) return parsed.headers } @@ -183,8 +199,10 @@ class ClaudeCodeHeadersService { */ async clearAccountHeaders(accountId) { try { - const key = `claude_code_headers:${accountId}` - await redis.getClient().del(key) + const cacheKey = `claude_code_headers:${accountId}` + await redis.getClient().del(cacheKey) + // 删除内存缓存 + deleteCachedConfig(cacheKey) logger.info(`🗑️ Cleared Claude Code headers for account ${accountId}`) } catch (error) { logger.error(`❌ Failed to clear Claude Code headers for account ${accountId}:`, error) diff --git a/src/services/claudeConsoleAccountService.js b/src/services/claudeConsoleAccountService.js index a46af870..c0770f34 100644 --- a/src/services/claudeConsoleAccountService.js +++ b/src/services/claudeConsoleAccountService.js @@ -129,6 +129,7 @@ class ClaudeConsoleAccountService { logger.debug(`[DEBUG] Account data to save: ${JSON.stringify(accountData, null, 2)}`) await client.hset(`${this.ACCOUNT_KEY_PREFIX}${accountId}`, accountData) + await redis.addToIndex('claude_console_account:index', accountId) // 如果是共享账户，添加到共享账户集合 if (accountType === 'shared') { @@ -167,11 +168,18 @@ class ClaudeConsoleAccountService { async getAllAccounts() { try { const client = redis.getClientSafe() - const keys = await client.keys(`${this.ACCOUNT_KEY_PREFIX}*`) + const accountIds = await redis.getAllIdsByIndex( + 'claude_console_account:index', + `${this.ACCOUNT_KEY_PREFIX}*`, + /^claude_console_account:(.+)$/ + ) + const keys = accountIds.map((id) => `${this.ACCOUNT_KEY_PREFIX}${id}`) const accounts = [] + const dataList = await redis.batchHgetallChunked(keys) - for (const key of keys) { - const accountData = await client.hgetall(key) + for (let i = 0; i < keys.length; i++) { + const key = keys[i] + const accountData = dataList[i] if (accountData && Object.keys(accountData).length > 0) { if (!accountData.id) { logger.warn(`⚠️ 检测到缺少ID的Claude Console账户数据，执行清理: ${key}`) @@ -449,6 +457,7 @@ class ClaudeConsoleAccountService { // 从Redis删除 await client.del(`${this.ACCOUNT_KEY_PREFIX}${accountId}`) + await redis.removeFromIndex('claude_console_account:index', accountId) // 从共享账户集合中移除 if (account.accountType === 'shared') { @@ -577,7 +586,7 @@ class ClaudeConsoleAccountService { } await client.hset(accountKey, updateData) - logger.success(`✅ Rate limit removed and account re-enabled: ${accountId}`) + logger.success(`Rate limit removed and account re-enabled: ${accountId}`) } } else { if (await client.hdel(accountKey, 'rateLimitAutoStopped')) { @@ -585,7 +594,7 @@ class ClaudeConsoleAccountService { `ℹ️ Removed stale auto-stop flag for Claude Console account ${accountId} during rate limit recovery` ) } - logger.success(`✅ Rate limit removed for Claude Console account: ${accountId}`) + logger.success(`Rate limit removed for Claude Console account: ${accountId}`) } return { success: true } @@ -858,7 +867,7 @@ class ClaudeConsoleAccountService { } await client.hset(accountKey, updateData) - logger.success(`✅ Blocked status removed and account re-enabled: ${accountId}`) + logger.success(`Blocked status removed and account re-enabled: ${accountId}`) } } else { if (await client.hdel(accountKey, 'blockedAutoStopped')) { @@ -866,7 +875,7 @@ class ClaudeConsoleAccountService { `ℹ️ Removed stale auto-stop flag for Claude Console account ${accountId} during blocked status recovery` ) } - logger.success(`✅ Blocked status removed for Claude Console account: ${accountId}`) + logger.success(`Blocked status removed for Claude Console account: ${accountId}`) } return { success: true } @@ -967,7 +976,7 @@ class ClaudeConsoleAccountService { await client.hdel(`${this.ACCOUNT_KEY_PREFIX}${accountId}`, 'overloadedAt', 'overloadStatus') - logger.success(`✅ Overload status removed for Claude Console account: ${accountId}`) + logger.success(`Overload status removed for Claude Console account: ${accountId}`) return { success: true } } catch (error) { logger.error( @@ -1416,7 +1425,7 @@ class ClaudeConsoleAccountService { } } - logger.success(`✅ Reset daily usage for ${resetCount} Claude Console accounts`) + logger.success(`Reset daily usage for ${resetCount} Claude Console accounts`) } catch (error) { logger.error('Failed to reset all daily usage:', error) } @@ -1489,7 +1498,7 @@ class ClaudeConsoleAccountService { await client.hset(accountKey, updates) await client.hdel(accountKey, ...fieldsToDelete) - logger.success(`✅ Reset all error status for Claude Console account ${accountId}`) + logger.success(`Reset all error status for Claude Console account ${accountId}`) // 发送 Webhook 通知 try { diff --git a/src/services/claudeRelayConfigService.js b/src/services/claudeRelayConfigService.js index 4fa2b411..49ffe166 100644 --- a/src/services/claudeRelayConfigService.js +++ b/src/services/claudeRelayConfigService.js @@ -18,8 +18,8 @@ const DEFAULT_CONFIG = { // 用户消息队列配置 userMessageQueueEnabled: false, // 是否启用用户消息队列（默认关闭） userMessageQueueDelayMs: 200, // 请求间隔（毫秒） - userMessageQueueTimeoutMs: 5000, // 队列等待超时（毫秒），优化后锁持有时间短无需长等待 - userMessageQueueLockTtlMs: 5000, // 锁TTL（毫秒），请求发送后立即释放无需长TTL + userMessageQueueTimeoutMs: 60000, // 队列等待超时（毫秒） + userMessageQueueLockTtlMs: 120000, // 锁TTL（毫秒） // 并发请求排队配置 concurrentRequestQueueEnabled: false, // 是否启用并发请求排队（默认关闭） concurrentRequestQueueMaxSize: 3, // 固定最小排队数（默认3） diff --git a/src/services/claudeRelayService.js b/src/services/claudeRelayService.js index 8fb90685..9f02af1c 100644 --- a/src/services/claudeRelayService.js +++ b/src/services/claudeRelayService.js @@ -1,6 +1,5 @@ const https = require('https') const zlib = require('zlib') -const fs = require('fs') const path = require('path') const ProxyHelper = require('../utils/proxyHelper') const { filterForClaude } = require('../utils/headerFilter') @@ -17,6 +16,17 @@ const requestIdentityService = require('./requestIdentityService') const { createClaudeTestPayload } = require('../utils/testPayloadHelper') const userMessageQueueService = require('./userMessageQueueService') const { isStreamWritable } = require('../utils/streamHelper') +const { + getHttpsAgentForStream, + getHttpsAgentForNonStream, + getPricingData +} = require('../utils/performanceOptimizer') + +// structuredClone polyfill for Node < 17 +const safeClone = + typeof structuredClone === 'function' + ? structuredClone + : (obj) => JSON.parse(JSON.stringify(obj)) class ClaudeRelayService { constructor() { @@ -684,8 +694,8 @@ class ClaudeRelayService { return body } - // 深拷贝请求体 - const processedBody = JSON.parse(JSON.stringify(body)) + // 使用 safeClone 替代 JSON.parse(JSON.stringify()) 提升性能 + const processedBody = safeClone(body) // 验证并限制max_tokens参数 this._validateAndLimitMaxTokens(processedBody) @@ -815,15 +825,15 @@ class ClaudeRelayService { } try { - // 读取模型定价配置文件 + // 使用缓存的定价数据 const pricingFilePath = path.join(__dirname, '../../data/model_pricing.json') + const pricingData = getPricingData(pricingFilePath) - if (!fs.existsSync(pricingFilePath)) { + if (!pricingData) { logger.warn('⚠️ Model pricing file not found, skipping max_tokens validation') return } - const pricingData = JSON.parse(fs.readFileSync(pricingFilePath, 'utf8')) const model = body.model || 'claude-sonnet-4-20250514' // 查找对应模型的配置 @@ -989,20 +999,20 @@ class ClaudeRelayService { } // 🌐 获取代理Agent（使用统一的代理工具） - async _getProxyAgent(accountId) { + async _getProxyAgent(accountId, account = null) { try { - const accountData = await claudeAccountService.getAllAccounts() - const account = accountData.find((acc) => acc.id === accountId) + // 优先使用传入的 account 对象，避免重复查询 + const accountData = account || (await claudeAccountService.getAccount(accountId)) - if (!account || !account.proxy) { + if (!accountData || !accountData.proxy) { logger.debug('🌐 No proxy configured for Claude account') return null } - const proxyAgent = ProxyHelper.createProxyAgent(account.proxy) + const proxyAgent = ProxyHelper.createProxyAgent(accountData.proxy) if (proxyAgent) { logger.info( - `🌐 Using proxy for Claude request: ${ProxyHelper.getProxyDescription(account.proxy)}` + `🌐 Using proxy for Claude request: ${ProxyHelper.getProxyDescription(accountData.proxy)}` ) } return proxyAgent @@ -1096,9 +1106,7 @@ class ClaudeRelayService { headers['User-Agent'] = userAgent headers['Accept'] = acceptHeader - logger.info(`🔗 指纹是这个: ${headers['User-Agent']}`) - - logger.info(`🔗 指纹是这个: ${headers['User-Agent']}`) + logger.debug(`🔗 Request User-Agent: ${headers['User-Agent']}`) // 根据模型和客户端传递的 anthropic-beta 动态设置 header const modelId = requestPayload?.model || body?.model @@ -1191,19 +1199,22 @@ class ClaudeRelayService { path: requestPath + (url.search || ''), method: 'POST', headers, - agent: proxyAgent, + agent: proxyAgent || getHttpsAgentForNonStream(), timeout: config.requestTimeout || 600000 } const req = https.request(options, (res) => { - let responseData = Buffer.alloc(0) + // 使用数组收集 chunks，避免 O(n²) 的 Buffer.concat + const chunks = [] res.on('data', (chunk) => { - responseData = Buffer.concat([responseData, chunk]) + chunks.push(chunk) }) res.on('end', () => { try { + // 一次性合并所有 chunks + const responseData = Buffer.concat(chunks) let responseBody = '' // 根据Content-Encoding处理响应数据 @@ -1586,7 +1597,7 @@ class ClaudeRelayService { path: url.pathname + (url.search || ''), method: 'POST', headers, - agent: proxyAgent, + agent: proxyAgent || getHttpsAgentForStream(), timeout: config.requestTimeout || 600000 } diff --git a/src/services/costInitService.js b/src/services/costInitService.js index 1edd65b1..c0b55850 100644 --- a/src/services/costInitService.js +++ b/src/services/costInitService.js @@ -1,9 +1,65 @@ const redis = require('../models/redis') -const apiKeyService = require('./apiKeyService') const CostCalculator = require('../utils/costCalculator') const logger = require('../utils/logger') +// HMGET 需要的字段 +const USAGE_FIELDS = [ + 'totalInputTokens', + 'inputTokens', + 'totalOutputTokens', + 'outputTokens', + 'totalCacheCreateTokens', + 'cacheCreateTokens', + 'totalCacheReadTokens', + 'cacheReadTokens' +] + class CostInitService { + /** + * 带并发限制的并行执行 + */ + async parallelLimit(items, fn, concurrency = 20) { + let index = 0 + const results = [] + + async function worker() { + while (index < items.length) { + const currentIndex = index++ + try { + results[currentIndex] = await fn(items[currentIndex], currentIndex) + } catch (error) { + results[currentIndex] = { error } + } + } + } + + await Promise.all(Array(Math.min(concurrency, items.length)).fill().map(worker)) + return results + } + + /** + * 使用 SCAN 获取匹配的 keys（带去重） + */ + async scanKeysWithDedup(client, pattern, count = 500) { + const seen = new Set() + const allKeys = [] + let cursor = '0' + + do { + const [newCursor, keys] = await client.scan(cursor, 'MATCH', pattern, 'COUNT', count) + cursor = newCursor + + for (const key of keys) { + if (!seen.has(key)) { + seen.add(key) + allKeys.push(key) + } + } + } while (cursor !== '0') + + return allKeys + } + /** * 初始化所有API Key的费用数据 * 扫描历史使用记录并计算费用 @@ -12,25 +68,55 @@ class CostInitService { try { logger.info('💰 Starting cost initialization for all API Keys...') - const apiKeys = await apiKeyService.getAllApiKeys() + // 用 scanApiKeyIds 获取 ID，然后过滤已删除的 + const allKeyIds = await redis.scanApiKeyIds() const client = redis.getClientSafe() + // 批量检查 isDeleted 状态，过滤已删除的 key + const FILTER_BATCH = 100 + const apiKeyIds = [] + + for (let i = 0; i < allKeyIds.length; i += FILTER_BATCH) { + const batch = allKeyIds.slice(i, i + FILTER_BATCH) + const pipeline = client.pipeline() + + for (const keyId of batch) { + pipeline.hget(`apikey:${keyId}`, 'isDeleted') + } + + const results = await pipeline.exec() + + for (let j = 0; j < results.length; j++) { + const [err, isDeleted] = results[j] + if (!err && isDeleted !== 'true') { + apiKeyIds.push(batch[j]) + } + } + } + + logger.info(`💰 Found ${apiKeyIds.length} active API Keys to process (filtered ${allKeyIds.length - apiKeyIds.length} deleted)`) + let processedCount = 0 let errorCount = 0 - for (const apiKey of apiKeys) { - try { - await this.initializeApiKeyCosts(apiKey.id, client) - processedCount++ + // 优化6: 并行处理 + 并发限制 + await this.parallelLimit( + apiKeyIds, + async (apiKeyId) => { + try { + await this.initializeApiKeyCosts(apiKeyId, client) + processedCount++ - if (processedCount % 10 === 0) { - logger.info(`💰 Processed ${processedCount} API Keys...`) + if (processedCount % 100 === 0) { + logger.info(`💰 Processed ${processedCount}/${apiKeyIds.length} API Keys...`) + } + } catch (error) { + errorCount++ + logger.error(`❌ Failed to initialize costs for API Key ${apiKeyId}:`, error) } - } catch (error) { - errorCount++ - logger.error(`❌ Failed to initialize costs for API Key ${apiKey.id}:`, error) - } - } + }, + 20 // 并发数 + ) logger.success( `💰 Cost initialization completed! Processed: ${processedCount}, Errors: ${errorCount}` @@ -46,32 +132,60 @@ class CostInitService { * 初始化单个API Key的费用数据 */ async initializeApiKeyCosts(apiKeyId, client) { - // 获取所有时间的模型使用统计 - const modelKeys = await client.keys(`usage:${apiKeyId}:model:*:*:*`) + // 优化4: 使用 SCAN 获取 keys（带去重） + const modelKeys = await this.scanKeysWithDedup(client, `usage:${apiKeyId}:model:*:*:*`) + + if (modelKeys.length === 0) { + return + } + + // 优化5: 使用 Pipeline + HMGET 批量获取数据 + const BATCH_SIZE = 100 + const allData = [] + + for (let i = 0; i < modelKeys.length; i += BATCH_SIZE) { + const batch = modelKeys.slice(i, i + BATCH_SIZE) + const pipeline = client.pipeline() + + for (const key of batch) { + pipeline.hmget(key, ...USAGE_FIELDS) + } + + const results = await pipeline.exec() + + for (let j = 0; j < results.length; j++) { + const [err, values] = results[j] + if (err) continue + + // 将数组转换为对象 + const data = {} + let hasData = false + for (let k = 0; k < USAGE_FIELDS.length; k++) { + if (values[k] !== null) { + data[USAGE_FIELDS[k]] = values[k] + hasData = true + } + } + + if (hasData) { + allData.push({ key: batch[j], data }) + } + } + } // 按日期分组统计 - const dailyCosts = new Map() // date -> cost - const monthlyCosts = new Map() // month -> cost - const hourlyCosts = new Map() // hour -> cost + const dailyCosts = new Map() + const monthlyCosts = new Map() + const hourlyCosts = new Map() - for (const key of modelKeys) { - // 解析key格式: usage:{keyId}:model:{period}:{model}:{date} + for (const { key, data } of allData) { const match = key.match( /usage:(.+):model:(daily|monthly|hourly):(.+):(\d{4}-\d{2}(?:-\d{2})?(?::\d{2})?)$/ ) - if (!match) { - continue - } + if (!match) continue const [, , period, model, dateStr] = match - // 获取使用数据 - const data = await client.hgetall(key) - if (!data || Object.keys(data).length === 0) { - continue - } - - // 计算费用 const usage = { input_tokens: parseInt(data.totalInputTokens) || parseInt(data.inputTokens) || 0, output_tokens: parseInt(data.totalOutputTokens) || parseInt(data.outputTokens) || 0, @@ -84,47 +198,34 @@ class CostInitService { const costResult = CostCalculator.calculateCost(usage, model) const cost = costResult.costs.total - // 根据period分组累加费用 if (period === 'daily') { - const currentCost = dailyCosts.get(dateStr) || 0 - dailyCosts.set(dateStr, currentCost + cost) + dailyCosts.set(dateStr, (dailyCosts.get(dateStr) || 0) + cost) } else if (period === 'monthly') { - const currentCost = monthlyCosts.get(dateStr) || 0 - monthlyCosts.set(dateStr, currentCost + cost) + monthlyCosts.set(dateStr, (monthlyCosts.get(dateStr) || 0) + cost) } else if (period === 'hourly') { - const currentCost = hourlyCosts.get(dateStr) || 0 - hourlyCosts.set(dateStr, currentCost + cost) + hourlyCosts.set(dateStr, (hourlyCosts.get(dateStr) || 0) + cost) } } - // 将计算出的费用写入Redis - const promises = [] + // 使用 SET NX EX 只补缺失的键，不覆盖已存在的 + const pipeline = client.pipeline() - // 写入每日费用 + // 写入每日费用（只补缺失） for (const [date, cost] of dailyCosts) { const key = `usage:cost:daily:${apiKeyId}:${date}` - promises.push( - client.set(key, cost.toString()), - client.expire(key, 86400 * 30) // 30天过期 - ) + pipeline.set(key, cost.toString(), 'EX', 86400 * 30, 'NX') } - // 写入每月费用 + // 写入每月费用（只补缺失） for (const [month, cost] of monthlyCosts) { const key = `usage:cost:monthly:${apiKeyId}:${month}` - promises.push( - client.set(key, cost.toString()), - client.expire(key, 86400 * 90) // 90天过期 - ) + pipeline.set(key, cost.toString(), 'EX', 86400 * 90, 'NX') } - // 写入每小时费用 + // 写入每小时费用（只补缺失） for (const [hour, cost] of hourlyCosts) { const key = `usage:cost:hourly:${apiKeyId}:${hour}` - promises.push( - client.set(key, cost.toString()), - client.expire(key, 86400 * 7) // 7天过期 - ) + pipeline.set(key, cost.toString(), 'EX', 86400 * 7, 'NX') } // 计算总费用 @@ -133,37 +234,25 @@ class CostInitService { totalCost += cost } - // 写入总费用 - 修复：只在总费用不存在时初始化，避免覆盖现有累计值 + // 写入总费用（只补缺失） if (totalCost > 0) { const totalKey = `usage:cost:total:${apiKeyId}` - // 先检查总费用是否已存在 const existingTotal = await client.get(totalKey) if (!existingTotal || parseFloat(existingTotal) === 0) { - // 仅在总费用不存在或为0时才初始化 - promises.push(client.set(totalKey, totalCost.toString())) + pipeline.set(totalKey, totalCost.toString()) logger.info(`💰 Initialized total cost for API Key ${apiKeyId}: $${totalCost.toFixed(6)}`) } else { - // 如果总费用已存在，保持不变，避免覆盖累计值 - // 注意：这个逻辑防止因每日费用键过期（30天）导致的错误覆盖 - // 如果需要强制重新计算，请先手动删除 usage:cost:total:{keyId} 键 const existing = parseFloat(existingTotal) - const calculated = totalCost - - if (calculated > existing * 1.1) { - // 如果计算值比现有值大 10% 以上，记录警告（可能是数据不一致） + if (totalCost > existing * 1.1) { logger.warn( - `💰 Total cost mismatch for API Key ${apiKeyId}: existing=$${existing.toFixed(6)}, calculated=$${calculated.toFixed(6)} (from last 30 days). Keeping existing value to prevent data loss.` - ) - } else { - logger.debug( - `💰 Skipping total cost initialization for API Key ${apiKeyId} - existing: $${existing.toFixed(6)}, calculated: $${calculated.toFixed(6)}` + `💰 Total cost mismatch for API Key ${apiKeyId}: existing=$${existing.toFixed(6)}, calculated=$${totalCost.toFixed(6)} (from last 30 days). Keeping existing value.` ) } } } - await Promise.all(promises) + await pipeline.exec() logger.debug( `💰 Initialized costs for API Key ${apiKeyId}: Daily entries: ${dailyCosts.size}, Total cost: $${totalCost.toFixed(2)}` @@ -172,41 +261,66 @@ class CostInitService { /** * 检查是否需要初始化费用数据 + * 使用 SCAN 代替 KEYS，正确处理 cursor */ async needsInitialization() { try { const client = redis.getClientSafe() - // 检查是否有任何费用数据 - const costKeys = await client.keys('usage:cost:*') + // 正确循环 SCAN 检查是否有任何费用数据 + let cursor = '0' + let hasCostData = false - // 如果没有费用数据，需要初始化 - if (costKeys.length === 0) { + do { + const [newCursor, keys] = await client.scan(cursor, 'MATCH', 'usage:cost:*', 'COUNT', 100) + cursor = newCursor + if (keys.length > 0) { + hasCostData = true + break + } + } while (cursor !== '0') + + if (!hasCostData) { logger.info('💰 No cost data found, initialization needed') return true } - // 检查是否有使用数据但没有对应的费用数据 - const sampleKeys = await client.keys('usage:*:model:daily:*:*') - if (sampleKeys.length > 10) { - // 抽样检查 - const sampleSize = Math.min(10, sampleKeys.length) - for (let i = 0; i < sampleSize; i++) { - const usageKey = sampleKeys[Math.floor(Math.random() * sampleKeys.length)] + // 抽样检查使用数据是否有对应的费用数据 + cursor = '0' + let samplesChecked = 0 + const maxSamples = 10 + + do { + const [newCursor, usageKeys] = await client.scan( + cursor, + 'MATCH', + 'usage:*:model:daily:*:*', + 'COUNT', + 100 + ) + cursor = newCursor + + for (const usageKey of usageKeys) { + if (samplesChecked >= maxSamples) break + const match = usageKey.match(/usage:(.+):model:daily:(.+):(\d{4}-\d{2}-\d{2})$/) if (match) { const [, keyId, , date] = match const costKey = `usage:cost:daily:${keyId}:${date}` const hasCost = await client.exists(costKey) + if (!hasCost) { logger.info( `💰 Found usage without cost data for key ${keyId} on ${date}, initialization needed` ) return true } + samplesChecked++ } } - } + + if (samplesChecked >= maxSamples) break + } while (cursor !== '0') logger.info('💰 Cost data appears to be up to date') return false diff --git a/src/services/costRankService.js b/src/services/costRankService.js index 605be48b..edd9acde 100644 --- a/src/services/costRankService.js +++ b/src/services/costRankService.js @@ -103,7 +103,7 @@ class CostRankService { } this.isInitialized = true - logger.success('✅ CostRankService initialized') + logger.success('CostRankService initialized') } catch (error) { logger.error('❌ Failed to initialize CostRankService:', error) throw error @@ -391,17 +391,32 @@ class CostRankService { return {} } - const status = {} - + // 使用 Pipeline 批量获取 + const pipeline = client.pipeline() for (const timeRange of VALID_TIME_RANGES) { - const meta = await client.hgetall(RedisKeys.metaKey(timeRange)) - status[timeRange] = { - lastUpdate: meta.lastUpdate || null, - keyCount: parseInt(meta.keyCount || 0), - status: meta.status || 'unknown', - updateDuration: parseInt(meta.updateDuration || 0) - } + pipeline.hgetall(RedisKeys.metaKey(timeRange)) } + const results = await pipeline.exec() + + const status = {} + VALID_TIME_RANGES.forEach((timeRange, i) => { + const [err, meta] = results[i] + if (err || !meta) { + status[timeRange] = { + lastUpdate: null, + keyCount: 0, + status: 'unknown', + updateDuration: 0 + } + } else { + status[timeRange] = { + lastUpdate: meta.lastUpdate || null, + keyCount: parseInt(meta.keyCount || 0), + status: meta.status || 'unknown', + updateDuration: parseInt(meta.updateDuration || 0) + } + } + }) return status } diff --git a/src/services/droidAccountService.js b/src/services/droidAccountService.js index 41dc8dc1..b8669231 100644 --- a/src/services/droidAccountService.js +++ b/src/services/droidAccountService.js @@ -6,7 +6,7 @@ const config = require('../../config/config') const logger = require('../utils/logger') const { maskToken } = require('../utils/tokenMask') const ProxyHelper = require('../utils/proxyHelper') -const LRUCache = require('../utils/lruCache') +const { createEncryptor, isTruthy } = require('../utils/commonHelper') /** * Droid 账户管理服务 @@ -26,24 +26,14 @@ class DroidAccountService { this.refreshIntervalHours = 6 // 每6小时刷新一次 this.tokenValidHours = 8 // Token 有效期8小时 - // 加密相关常量 - this.ENCRYPTION_ALGORITHM = 'aes-256-cbc' - this.ENCRYPTION_SALT = 'droid-account-salt' - - // 🚀 性能优化：缓存派生的加密密钥 - this._encryptionKeyCache = null - - // 🔄 解密结果缓存 - this._decryptCache = new LRUCache(500) + // 使用 commonHelper 的加密器 + this._encryptor = createEncryptor('droid-account-salt') // 🧹 定期清理缓存（每10分钟） - setInterval( - () => { - this._decryptCache.cleanup() - logger.info('🧹 Droid decrypt cache cleanup completed', this._decryptCache.getStats()) - }, - 10 * 60 * 1000 - ) + setInterval(() => { + this._encryptor.clearCache() + logger.info('🧹 Droid decrypt cache cleanup completed', this._encryptor.getStats()) + }, 10 * 60 * 1000) this.supportedEndpointTypes = new Set(['anthropic', 'openai', 'comm']) } @@ -69,92 +59,19 @@ class DroidAccountService { return 'anthropic' } + // 使用 commonHelper 的 isTruthy _isTruthy(value) { - if (value === undefined || value === null) { - return false - } - if (typeof value === 'boolean') { - return value - } - if (typeof value === 'string') { - const normalized = value.trim().toLowerCase() - if (normalized === 'true') { - return true - } - if (normalized === 'false') { - return false - } - return normalized.length > 0 && normalized !== '0' && normalized !== 'no' - } - return Boolean(value) + return isTruthy(value) } - /** - * 生成加密密钥（缓存优化） - */ - _generateEncryptionKey() { - if (!this._encryptionKeyCache) { - this._encryptionKeyCache = crypto.scryptSync( - config.security.encryptionKey, - this.ENCRYPTION_SALT, - 32 - ) - logger.info('🔑 Droid encryption key derived and cached for performance optimization') - } - return this._encryptionKeyCache - } - - /** - * 加密敏感数据 - */ + // 加密敏感数据 _encryptSensitiveData(text) { - if (!text) { - return '' - } - - const key = this._generateEncryptionKey() - const iv = crypto.randomBytes(16) - const cipher = crypto.createCipheriv(this.ENCRYPTION_ALGORITHM, key, iv) - - let encrypted = cipher.update(text, 'utf8', 'hex') - encrypted += cipher.final('hex') - - return `${iv.toString('hex')}:${encrypted}` + return this._encryptor.encrypt(text) } - /** - * 解密敏感数据（带缓存） - */ + // 解密敏感数据（带缓存） _decryptSensitiveData(encryptedText) { - if (!encryptedText) { - return '' - } - - // 🎯 检查缓存 - const cacheKey = crypto.createHash('sha256').update(encryptedText).digest('hex') - const cached = this._decryptCache.get(cacheKey) - if (cached !== undefined) { - return cached - } - - try { - const key = this._generateEncryptionKey() - const parts = encryptedText.split(':') - const iv = Buffer.from(parts[0], 'hex') - const encrypted = parts[1] - - const decipher = crypto.createDecipheriv(this.ENCRYPTION_ALGORITHM, key, iv) - let decrypted = decipher.update(encrypted, 'hex', 'utf8') - decrypted += decipher.final('utf8') - - // 💾 存入缓存（5分钟过期） - this._decryptCache.set(cacheKey, decrypted, 5 * 60 * 1000) - - return decrypted - } catch (error) { - logger.error('❌ Failed to decrypt Droid data:', error) - return '' - } + return this._encryptor.decrypt(encryptedText) } _parseApiKeyEntries(rawEntries) { @@ -683,7 +600,7 @@ class DroidAccountService { lastRefreshAt = new Date().toISOString() status = 'active' - logger.success(`✅ 使用 Refresh Token 成功验证并刷新 Droid 账户: ${name} (${accountId})`) + logger.success(`使用 Refresh Token 成功验证并刷新 Droid 账户: ${name} (${accountId})`) } catch (error) { logger.error('❌ 使用 Refresh Token 验证 Droid 账户失败:', error) throw new Error(`Refresh Token 验证失败：${error.message}`) @@ -1368,7 +1285,7 @@ class DroidAccountService { } } - logger.success(`✅ Droid account token refreshed successfully: ${accountId}`) + logger.success(`Droid account token refreshed successfully: ${accountId}`) return { accessToken: refreshed.accessToken, diff --git a/src/services/droidRelayService.js b/src/services/droidRelayService.js index 115be7d9..77522b78 100644 --- a/src/services/droidRelayService.js +++ b/src/services/droidRelayService.js @@ -609,7 +609,7 @@ class DroidRelayService { ' [stream]' ) - logger.success(`✅ Droid stream completed - Account: ${account.name}`) + logger.success(`Droid stream completed - Account: ${account.name}`) } else { logger.success( `✅ Droid stream completed - Account: ${account.name}, usage recording skipped` diff --git a/src/services/droidScheduler.js b/src/services/droidScheduler.js index 465cd383..37ea9e4b 100644 --- a/src/services/droidScheduler.js +++ b/src/services/droidScheduler.js @@ -2,103 +2,31 @@ const droidAccountService = require('./droidAccountService') const accountGroupService = require('./accountGroupService') const redis = require('../models/redis') const logger = require('../utils/logger') +const { isTruthy, isAccountHealthy, sortAccountsByPriority, normalizeEndpointType } = require('../utils/commonHelper') class DroidScheduler { constructor() { this.STICKY_PREFIX = 'droid' } - _normalizeEndpointType(endpointType) { - if (!endpointType) { - return 'anthropic' - } - const normalized = String(endpointType).toLowerCase() - if (normalized === 'openai') { - return 'openai' - } - if (normalized === 'comm') { - return 'comm' - } - if (normalized === 'anthropic') { - return 'anthropic' - } - return 'anthropic' - } - - _isTruthy(value) { - if (value === undefined || value === null) { - return false - } - if (typeof value === 'boolean') { - return value - } - if (typeof value === 'string') { - return value.toLowerCase() === 'true' - } - return Boolean(value) - } - - _isAccountActive(account) { - if (!account) { - return false - } - const isActive = this._isTruthy(account.isActive) - if (!isActive) { - return false - } - - const status = (account.status || 'active').toLowerCase() - const unhealthyStatuses = new Set(['error', 'unauthorized', 'blocked']) - return !unhealthyStatuses.has(status) - } - _isAccountSchedulable(account) { - return this._isTruthy(account?.schedulable ?? true) + return isTruthy(account?.schedulable ?? true) } _matchesEndpoint(account, endpointType) { - const normalizedEndpoint = this._normalizeEndpointType(endpointType) - const accountEndpoint = this._normalizeEndpointType(account?.endpointType) - if (normalizedEndpoint === accountEndpoint) { - return true - } - - // comm 端点可以使用任何类型的账户 - if (normalizedEndpoint === 'comm') { - return true - } - + const normalizedEndpoint = normalizeEndpointType(endpointType) + const accountEndpoint = normalizeEndpointType(account?.endpointType) + if (normalizedEndpoint === accountEndpoint) return true + if (normalizedEndpoint === 'comm') return true const sharedEndpoints = new Set(['anthropic', 'openai']) return sharedEndpoints.has(normalizedEndpoint) && sharedEndpoints.has(accountEndpoint) } - _sortCandidates(candidates) { - return [...candidates].sort((a, b) => { - const priorityA = parseInt(a.priority, 10) || 50 - const priorityB = parseInt(b.priority, 10) || 50 - - if (priorityA !== priorityB) { - return priorityA - priorityB - } - - const lastUsedA = a.lastUsedAt ? new Date(a.lastUsedAt).getTime() : 0 - const lastUsedB = b.lastUsedAt ? new Date(b.lastUsedAt).getTime() : 0 - - if (lastUsedA !== lastUsedB) { - return lastUsedA - lastUsedB - } - - const createdA = a.createdAt ? new Date(a.createdAt).getTime() : 0 - const createdB = b.createdAt ? new Date(b.createdAt).getTime() : 0 - return createdA - createdB - }) - } - _composeStickySessionKey(endpointType, sessionHash, apiKeyId) { if (!sessionHash) { return null } - const normalizedEndpoint = this._normalizeEndpointType(endpointType) + const normalizedEndpoint = normalizeEndpointType(endpointType) const apiKeyPart = apiKeyId || 'default' return `${this.STICKY_PREFIX}:${normalizedEndpoint}:${apiKeyPart}:${sessionHash}` } @@ -121,7 +49,7 @@ class DroidScheduler { ) return accounts.filter( - (account) => account && this._isAccountActive(account) && this._isAccountSchedulable(account) + (account) => account && isAccountHealthy(account) && this._isAccountSchedulable(account) ) } @@ -145,7 +73,7 @@ class DroidScheduler { } async selectAccount(apiKeyData, endpointType, sessionHash) { - const normalizedEndpoint = this._normalizeEndpointType(endpointType) + const normalizedEndpoint = normalizeEndpointType(endpointType) const stickyKey = this._composeStickySessionKey(normalizedEndpoint, sessionHash, apiKeyData?.id) let candidates = [] @@ -175,7 +103,7 @@ class DroidScheduler { const filtered = candidates.filter( (account) => account && - this._isAccountActive(account) && + isAccountHealthy(account) && this._isAccountSchedulable(account) && this._matchesEndpoint(account, normalizedEndpoint) ) @@ -203,7 +131,7 @@ class DroidScheduler { } } - const sorted = this._sortCandidates(filtered) + const sorted = sortAccountsByPriority(filtered) const selected = sorted[0] if (!selected) { diff --git a/src/services/geminiAccountService.js b/src/services/geminiAccountService.js index a23d81b3..c40ecf5b 100644 --- a/src/services/geminiAccountService.js +++ b/src/services/geminiAccountService.js @@ -1,6 +1,5 @@ const redisClient = require('../models/redis') const { v4: uuidv4 } = require('uuid') -const crypto = require('crypto') const https = require('https') const config = require('../../config/config') const logger = require('../utils/logger') @@ -15,7 +14,10 @@ const { logRefreshSkipped } = require('../utils/tokenRefreshLogger') const tokenRefreshService = require('./tokenRefreshService') -const LRUCache = require('../utils/lruCache') +const { createEncryptor } = require('../utils/commonHelper') + +// Gemini 账户键前缀 +const GEMINI_ACCOUNT_KEY_PREFIX = 'gemini_account:' // Gemini CLI OAuth 配置 - 这些是公开的 Gemini CLI 凭据 const OAUTH_CLIENT_ID = '681255809395-oo8ft2oprdrnp9e3aqf6av3hmdib135j.apps.googleusercontent.com' @@ -34,91 +36,15 @@ const keepAliveAgent = new https.Agent({ logger.info('🌐 Gemini HTTPS Agent initialized with TCP Keep-Alive support') -// 加密相关常量 -const ALGORITHM = 'aes-256-cbc' -const ENCRYPTION_SALT = 'gemini-account-salt' -const IV_LENGTH = 16 - -// 🚀 性能优化：缓存派生的加密密钥，避免每次重复计算 -// scryptSync 是 CPU 密集型操作，缓存可以减少 95%+ 的 CPU 占用 -let _encryptionKeyCache = null - -// 🔄 解密结果缓存，提高解密性能 -const decryptCache = new LRUCache(500) - -// 生成加密密钥（使用与 claudeAccountService 相同的方法） -function generateEncryptionKey() { - if (!_encryptionKeyCache) { - _encryptionKeyCache = crypto.scryptSync(config.security.encryptionKey, ENCRYPTION_SALT, 32) - logger.info('🔑 Gemini encryption key derived and cached for performance optimization') - } - return _encryptionKeyCache -} - -// Gemini 账户键前缀 -const GEMINI_ACCOUNT_KEY_PREFIX = 'gemini_account:' -const SHARED_GEMINI_ACCOUNTS_KEY = 'shared_gemini_accounts' -const ACCOUNT_SESSION_MAPPING_PREFIX = 'gemini_session_account_mapping:' - -// 加密函数 -function encrypt(text) { - if (!text) { - return '' - } - const key = generateEncryptionKey() - const iv = crypto.randomBytes(IV_LENGTH) - const cipher = crypto.createCipheriv(ALGORITHM, key, iv) - let encrypted = cipher.update(text) - encrypted = Buffer.concat([encrypted, cipher.final()]) - return `${iv.toString('hex')}:${encrypted.toString('hex')}` -} - -// 解密函数 -function decrypt(text) { - if (!text) { - return '' - } - - // 🎯 检查缓存 - const cacheKey = crypto.createHash('sha256').update(text).digest('hex') - const cached = decryptCache.get(cacheKey) - if (cached !== undefined) { - return cached - } - - try { - const key = generateEncryptionKey() - // IV 是固定长度的 32 个十六进制字符（16 字节） - const ivHex = text.substring(0, 32) - const encryptedHex = text.substring(33) // 跳过冒号 - - const iv = Buffer.from(ivHex, 'hex') - const encryptedText = Buffer.from(encryptedHex, 'hex') - const decipher = crypto.createDecipheriv(ALGORITHM, key, iv) - let decrypted = decipher.update(encryptedText) - decrypted = Buffer.concat([decrypted, decipher.final()]) - const result = decrypted.toString() - - // 💾 存入缓存（5分钟过期） - decryptCache.set(cacheKey, result, 5 * 60 * 1000) - - // 📊 定期打印缓存统计 - if ((decryptCache.hits + decryptCache.misses) % 1000 === 0) { - decryptCache.printStats() - } - - return result - } catch (error) { - logger.error('Decryption error:', error) - return '' - } -} +// 使用 commonHelper 的加密器 +const encryptor = createEncryptor('gemini-account-salt') +const { encrypt, decrypt } = encryptor // 🧹 定期清理缓存（每10分钟） setInterval( () => { - decryptCache.cleanup() - logger.info('🧹 Gemini decrypt cache cleanup completed', decryptCache.getStats()) + encryptor.clearCache() + logger.info('🧹 Gemini decrypt cache cleanup completed', encryptor.getStats()) }, 10 * 60 * 1000 ) @@ -426,6 +352,7 @@ async function createAccount(accountData) { // 保存到 Redis const client = redisClient.getClientSafe() await client.hset(`${GEMINI_ACCOUNT_KEY_PREFIX}${id}`, account) + await redisClient.addToIndex('gemini_account:index', id) // 如果是共享账户，添加到共享账户集合 if (account.accountType === 'shared') { @@ -623,19 +550,20 @@ async function deleteAccount(accountId) { // 从 Redis 删除 const client = redisClient.getClientSafe() await client.del(`${GEMINI_ACCOUNT_KEY_PREFIX}${accountId}`) + await redisClient.removeFromIndex('gemini_account:index', accountId) // 从共享账户集合中移除 if (account.accountType === 'shared') { await client.srem(SHARED_GEMINI_ACCOUNTS_KEY, accountId) } - // 清理会话映射 - const sessionMappings = await client.keys(`${ACCOUNT_SESSION_MAPPING_PREFIX}*`) - for (const key of sessionMappings) { - const mappedAccountId = await client.get(key) - if (mappedAccountId === accountId) { - await client.del(key) - } + // 清理会话映射（使用反向索引） + const sessionHashes = await client.smembers(`gemini_account_sessions:${accountId}`) + if (sessionHashes.length > 0) { + const pipeline = client.pipeline() + sessionHashes.forEach((hash) => pipeline.del(`${ACCOUNT_SESSION_MAPPING_PREFIX}${hash}`)) + pipeline.del(`gemini_account_sessions:${accountId}`) + await pipeline.exec() } logger.info(`Deleted Gemini account: ${accountId}`) @@ -645,11 +573,17 @@ async function deleteAccount(accountId) { // 获取所有账户 async function getAllAccounts() { const client = redisClient.getClientSafe() - const keys = await client.keys(`${GEMINI_ACCOUNT_KEY_PREFIX}*`) + const accountIds = await redisClient.getAllIdsByIndex( + 'gemini_account:index', + `${GEMINI_ACCOUNT_KEY_PREFIX}*`, + /^gemini_account:(.+)$/ + ) + const keys = accountIds.map((id) => `${GEMINI_ACCOUNT_KEY_PREFIX}${id}`) const accounts = [] + const dataList = await redisClient.batchHgetallChunked(keys) - for (const key of keys) { - const accountData = await client.hgetall(key) + for (let i = 0; i < keys.length; i++) { + const accountData = dataList[i] if (accountData && Object.keys(accountData).length > 0) { // 获取限流状态信息 const rateLimitInfo = await getAccountRateLimitInfo(accountData.id) @@ -752,6 +686,8 @@ async function selectAvailableAccount(apiKeyId, sessionHash = null) { 3600, // 1小时过期 account.id ) + await client.sadd(`gemini_account_sessions:${account.id}`, sessionHash) + await client.expire(`gemini_account_sessions:${account.id}`, 3600) } return account @@ -811,6 +747,8 @@ async function selectAvailableAccount(apiKeyId, sessionHash = null) { // 创建粘性会话映射 if (sessionHash) { await client.setex(`${ACCOUNT_SESSION_MAPPING_PREFIX}${sessionHash}`, 3600, selectedAccount.id) + await client.sadd(`gemini_account_sessions:${selectedAccount.id}`, sessionHash) + await client.expire(`gemini_account_sessions:${selectedAccount.id}`, 3600) } return selectedAccount @@ -1684,8 +1622,7 @@ module.exports = { setupUser, encrypt, decrypt, - generateEncryptionKey, - decryptCache, // 暴露缓存对象以便测试和监控 + encryptor, // 暴露加密器以便测试和监控 countTokens, generateContent, generateContentStream, diff --git a/src/services/geminiApiAccountService.js b/src/services/geminiApiAccountService.js index c03568a2..663269ca 100644 --- a/src/services/geminiApiAccountService.js +++ b/src/services/geminiApiAccountService.js @@ -85,7 +85,7 @@ class GeminiApiAccountService { // 保存到 Redis await this._saveAccount(accountId, accountData) - logger.success(`🚀 Created Gemini-API account: ${name} (${accountId})`) + logger.success(`Created Gemini-API account: ${name} (${accountId})`) return { ...accountData, @@ -172,6 +172,9 @@ class GeminiApiAccountService { // 从共享账户列表中移除 await client.srem(this.SHARED_ACCOUNTS_KEY, accountId) + // 从索引中移除 + await redis.removeFromIndex('gemini_api_account:index', accountId) + // 删除账户数据 await client.del(key) @@ -223,11 +226,17 @@ class GeminiApiAccountService { } // 直接从 Redis 获取所有账户（包括非共享账户） - const keys = await client.keys(`${this.ACCOUNT_KEY_PREFIX}*`) - for (const key of keys) { - const accountId = key.replace(this.ACCOUNT_KEY_PREFIX, '') + const allAccountIds = await redis.getAllIdsByIndex( + 'gemini_api_account:index', + `${this.ACCOUNT_KEY_PREFIX}*`, + /^gemini_api_account:(.+)$/ + ) + const keys = allAccountIds.map((id) => `${this.ACCOUNT_KEY_PREFIX}${id}`) + const dataList = await redis.batchHgetallChunked(keys) + for (let i = 0; i < allAccountIds.length; i++) { + const accountId = allAccountIds[i] if (!accountIds.includes(accountId)) { - const accountData = await client.hgetall(key) + const accountData = dataList[i] if (accountData && accountData.id) { // 过滤非活跃账户 if (includeInactive || accountData.isActive === 'true') { @@ -576,6 +585,9 @@ class GeminiApiAccountService { // 保存账户数据 await client.hset(key, accountData) + // 添加到索引 + await redis.addToIndex('gemini_api_account:index', accountId) + // 添加到共享账户列表 if (accountData.accountType === 'shared') { await client.sadd(this.SHARED_ACCOUNTS_KEY, accountId) diff --git a/src/services/modelService.js b/src/services/modelService.js index 49607f55..5a6275a0 100644 --- a/src/services/modelService.js +++ b/src/services/modelService.js @@ -18,7 +18,7 @@ class ModelService { (sum, config) => sum + config.models.length, 0 ) - logger.success(`✅ Model service initialized with ${totalModels} models`) + logger.success(`Model service initialized with ${totalModels} models`) } /** diff --git a/src/services/openaiAccountService.js b/src/services/openaiAccountService.js index 08c93387..21fd1569 100644 --- a/src/services/openaiAccountService.js +++ b/src/services/openaiAccountService.js @@ -1,6 +1,5 @@ const redisClient = require('../models/redis') const { v4: uuidv4 } = require('uuid') -const crypto = require('crypto') const axios = require('axios') const ProxyHelper = require('../utils/proxyHelper') const config = require('../../config/config') @@ -13,104 +12,23 @@ const { logTokenUsage, logRefreshSkipped } = require('../utils/tokenRefreshLogger') -const LRUCache = require('../utils/lruCache') const tokenRefreshService = require('./tokenRefreshService') +const { createEncryptor } = require('../utils/commonHelper') -// 加密相关常量 -const ALGORITHM = 'aes-256-cbc' -const ENCRYPTION_SALT = 'openai-account-salt' -const IV_LENGTH = 16 - -// 🚀 性能优化：缓存派生的加密密钥，避免每次重复计算 -// scryptSync 是 CPU 密集型操作，缓存可以减少 95%+ 的 CPU 占用 -let _encryptionKeyCache = null - -// 🔄 解密结果缓存，提高解密性能 -const decryptCache = new LRUCache(500) - -// 生成加密密钥（使用与 claudeAccountService 相同的方法） -function generateEncryptionKey() { - if (!_encryptionKeyCache) { - _encryptionKeyCache = crypto.scryptSync(config.security.encryptionKey, ENCRYPTION_SALT, 32) - logger.info('🔑 OpenAI encryption key derived and cached for performance optimization') - } - return _encryptionKeyCache -} +// 使用 commonHelper 的加密器 +const encryptor = createEncryptor('openai-account-salt') +const { encrypt, decrypt } = encryptor // OpenAI 账户键前缀 const OPENAI_ACCOUNT_KEY_PREFIX = 'openai:account:' const SHARED_OPENAI_ACCOUNTS_KEY = 'shared_openai_accounts' const ACCOUNT_SESSION_MAPPING_PREFIX = 'openai_session_account_mapping:' -// 加密函数 -function encrypt(text) { - if (!text) { - return '' - } - const key = generateEncryptionKey() - const iv = crypto.randomBytes(IV_LENGTH) - const cipher = crypto.createCipheriv(ALGORITHM, key, iv) - let encrypted = cipher.update(text) - encrypted = Buffer.concat([encrypted, cipher.final()]) - return `${iv.toString('hex')}:${encrypted.toString('hex')}` -} - -// 解密函数 -function decrypt(text) { - if (!text || text === '') { - return '' - } - - // 检查是否是有效的加密格式（至少需要 32 个字符的 IV + 冒号 + 加密文本） - if (text.length < 33 || text.charAt(32) !== ':') { - logger.warn('Invalid encrypted text format, returning empty string', { - textLength: text ? text.length : 0, - char32: text && text.length > 32 ? text.charAt(32) : 'N/A', - first50: text ? text.substring(0, 50) : 'N/A' - }) - return '' - } - - // 🎯 检查缓存 - const cacheKey = crypto.createHash('sha256').update(text).digest('hex') - const cached = decryptCache.get(cacheKey) - if (cached !== undefined) { - return cached - } - - try { - const key = generateEncryptionKey() - // IV 是固定长度的 32 个十六进制字符（16 字节） - const ivHex = text.substring(0, 32) - const encryptedHex = text.substring(33) // 跳过冒号 - - const iv = Buffer.from(ivHex, 'hex') - const encryptedText = Buffer.from(encryptedHex, 'hex') - const decipher = crypto.createDecipheriv(ALGORITHM, key, iv) - let decrypted = decipher.update(encryptedText) - decrypted = Buffer.concat([decrypted, decipher.final()]) - const result = decrypted.toString() - - // 💾 存入缓存（5分钟过期） - decryptCache.set(cacheKey, result, 5 * 60 * 1000) - - // 📊 定期打印缓存统计 - if ((decryptCache.hits + decryptCache.misses) % 1000 === 0) { - decryptCache.printStats() - } - - return result - } catch (error) { - logger.error('Decryption error:', error) - return '' - } -} - // 🧹 定期清理缓存（每10分钟） setInterval( () => { - decryptCache.cleanup() - logger.info('🧹 OpenAI decrypt cache cleanup completed', decryptCache.getStats()) + encryptor.clearCache() + logger.info('🧹 OpenAI decrypt cache cleanup completed', encryptor.getStats()) }, 10 * 60 * 1000 ) @@ -591,6 +509,7 @@ async function createAccount(accountData) { const client = redisClient.getClientSafe() await client.hset(`${OPENAI_ACCOUNT_KEY_PREFIX}${accountId}`, account) + await redisClient.addToIndex('openai:account:index', accountId) // 如果是共享账户，添加到共享账户集合 if (account.accountType === 'shared') { @@ -725,19 +644,20 @@ async function deleteAccount(accountId) { // 从 Redis 删除 const client = redisClient.getClientSafe() await client.del(`${OPENAI_ACCOUNT_KEY_PREFIX}${accountId}`) + await redisClient.removeFromIndex('openai:account:index', accountId) // 从共享账户集合中移除 if (account.accountType === 'shared') { await client.srem(SHARED_OPENAI_ACCOUNTS_KEY, accountId) } - // 清理会话映射 - const sessionMappings = await client.keys(`${ACCOUNT_SESSION_MAPPING_PREFIX}*`) - for (const key of sessionMappings) { - const mappedAccountId = await client.get(key) - if (mappedAccountId === accountId) { - await client.del(key) - } + // 清理会话映射（使用反向索引） + const sessionHashes = await client.smembers(`openai_account_sessions:${accountId}`) + if (sessionHashes.length > 0) { + const pipeline = client.pipeline() + sessionHashes.forEach((hash) => pipeline.del(`${ACCOUNT_SESSION_MAPPING_PREFIX}${hash}`)) + pipeline.del(`openai_account_sessions:${accountId}`) + await pipeline.exec() } logger.info(`Deleted OpenAI account: ${accountId}`) @@ -747,11 +667,17 @@ async function deleteAccount(accountId) { // 获取所有账户 async function getAllAccounts() { const client = redisClient.getClientSafe() - const keys = await client.keys(`${OPENAI_ACCOUNT_KEY_PREFIX}*`) + const accountIds = await redisClient.getAllIdsByIndex( + 'openai:account:index', + `${OPENAI_ACCOUNT_KEY_PREFIX}*`, + /^openai:account:(.+)$/ + ) + const keys = accountIds.map((id) => `${OPENAI_ACCOUNT_KEY_PREFIX}${id}`) const accounts = [] + const dataList = await redisClient.batchHgetallChunked(keys) - for (const key of keys) { - const accountData = await client.hgetall(key) + for (let i = 0; i < keys.length; i++) { + const accountData = dataList[i] if (accountData && Object.keys(accountData).length > 0) { const codexUsage = buildCodexUsageSnapshot(accountData) @@ -926,6 +852,9 @@ async function selectAvailableAccount(apiKeyId, sessionHash = null) { 3600, // 1小时过期 account.id ) + // 反向索引：accountId -> sessionHash（用于删除账户时快速清理） + await client.sadd(`openai_account_sessions:${account.id}`, sessionHash) + await client.expire(`openai_account_sessions:${account.id}`, 3600) } return account @@ -976,6 +905,8 @@ async function selectAvailableAccount(apiKeyId, sessionHash = null) { 3600, // 1小时过期 selectedAccount.id ) + await client.sadd(`openai_account_sessions:${selectedAccount.id}`, sessionHash) + await client.expire(`openai_account_sessions:${selectedAccount.id}`, 3600) } return selectedAccount @@ -1278,6 +1209,5 @@ module.exports = { updateCodexUsageSnapshot, encrypt, decrypt, - generateEncryptionKey, - decryptCache // 暴露缓存对象以便测试和监控 + encryptor // 暴露加密器以便测试和监控 } diff --git a/src/services/openaiResponsesAccountService.js b/src/services/openaiResponsesAccountService.js index 41e61c92..708fa8b4 100644 --- a/src/services/openaiResponsesAccountService.js +++ b/src/services/openaiResponsesAccountService.js @@ -99,7 +99,7 @@ class OpenAIResponsesAccountService { // 保存到 Redis await this._saveAccount(accountId, accountData) - logger.success(`🚀 Created OpenAI-Responses account: ${name} (${accountId})`) + logger.success(`Created OpenAI-Responses account: ${name} (${accountId})`) return { ...accountData, @@ -180,6 +180,9 @@ class OpenAIResponsesAccountService { // 从共享账户列表中移除 await client.srem(this.SHARED_ACCOUNTS_KEY, accountId) + // 从索引中移除 + await redis.removeFromIndex('openai_responses_account:index', accountId) + // 删除账户数据 await client.del(key) @@ -191,97 +194,62 @@ class OpenAIResponsesAccountService { // 获取所有账户 async getAllAccounts(includeInactive = false) { const client = redis.getClientSafe() - const accountIds = await client.smembers(this.SHARED_ACCOUNTS_KEY) + + // 使用索引获取所有账户ID + const accountIds = await redis.getAllIdsByIndex( + 'openai_responses_account:index', + `${this.ACCOUNT_KEY_PREFIX}*`, + /^openai_responses_account:(.+)$/ + ) + if (accountIds.length === 0) return [] + + const keys = accountIds.map((id) => `${this.ACCOUNT_KEY_PREFIX}${id}`) + // Pipeline 批量查询所有账户数据 + const pipeline = client.pipeline() + keys.forEach((key) => pipeline.hgetall(key)) + const results = await pipeline.exec() + const accounts = [] + results.forEach(([err, accountData], index) => { + if (err || !accountData || !accountData.id) return - for (const accountId of accountIds) { - const account = await this.getAccount(accountId) - if (account) { - // 过滤非活跃账户 - if (includeInactive || account.isActive === 'true') { - // 隐藏敏感信息 - account.apiKey = '***' + // 过滤非活跃账户 + if (!includeInactive && accountData.isActive !== 'true') return - // 获取限流状态信息（与普通OpenAI账号保持一致的格式） - const rateLimitInfo = this._getRateLimitInfo(account) + // 隐藏敏感信息 + accountData.apiKey = '***' - // 格式化 rateLimitStatus 为对象（与普通 OpenAI 账号一致） - account.rateLimitStatus = rateLimitInfo.isRateLimited - ? { - isRateLimited: true, - rateLimitedAt: account.rateLimitedAt || null, - minutesRemaining: rateLimitInfo.remainingMinutes || 0 - } - : { - isRateLimited: false, - rateLimitedAt: null, - minutesRemaining: 0 - } - - // 转换 schedulable 字段为布尔值（前端需要布尔值来判断） - account.schedulable = account.schedulable !== 'false' - // 转换 isActive 字段为布尔值 - account.isActive = account.isActive === 'true' - - // ✅ 前端显示订阅过期时间（业务字段） - account.expiresAt = account.subscriptionExpiresAt || null - account.platform = account.platform || 'openai-responses' - - accounts.push(account) + // 解析 JSON 字段 + if (accountData.proxy) { + try { + accountData.proxy = JSON.parse(accountData.proxy) + } catch { + accountData.proxy = null } } - } - // 直接从 Redis 获取所有账户（包括非共享账户） - const keys = await client.keys(`${this.ACCOUNT_KEY_PREFIX}*`) - for (const key of keys) { - const accountId = key.replace(this.ACCOUNT_KEY_PREFIX, '') - if (!accountIds.includes(accountId)) { - const accountData = await client.hgetall(key) - if (accountData && accountData.id) { - // 过滤非活跃账户 - if (includeInactive || accountData.isActive === 'true') { - // 隐藏敏感信息 - accountData.apiKey = '***' - // 解析 JSON 字段 - if (accountData.proxy) { - try { - accountData.proxy = JSON.parse(accountData.proxy) - } catch (e) { - accountData.proxy = null - } - } - - // 获取限流状态信息（与普通OpenAI账号保持一致的格式） - const rateLimitInfo = this._getRateLimitInfo(accountData) - - // 格式化 rateLimitStatus 为对象（与普通 OpenAI 账号一致） - accountData.rateLimitStatus = rateLimitInfo.isRateLimited - ? { - isRateLimited: true, - rateLimitedAt: accountData.rateLimitedAt || null, - minutesRemaining: rateLimitInfo.remainingMinutes || 0 - } - : { - isRateLimited: false, - rateLimitedAt: null, - minutesRemaining: 0 - } - - // 转换 schedulable 字段为布尔值（前端需要布尔值来判断） - accountData.schedulable = accountData.schedulable !== 'false' - // 转换 isActive 字段为布尔值 - accountData.isActive = accountData.isActive === 'true' - - // ✅ 前端显示订阅过期时间（业务字段） - accountData.expiresAt = accountData.subscriptionExpiresAt || null - accountData.platform = accountData.platform || 'openai-responses' - - accounts.push(accountData) + // 获取限流状态信息 + const rateLimitInfo = this._getRateLimitInfo(accountData) + accountData.rateLimitStatus = rateLimitInfo.isRateLimited + ? { + isRateLimited: true, + rateLimitedAt: accountData.rateLimitedAt || null, + minutesRemaining: rateLimitInfo.remainingMinutes || 0 } - } - } - } + : { + isRateLimited: false, + rateLimitedAt: null, + minutesRemaining: 0 + } + + // 转换字段类型 + accountData.schedulable = accountData.schedulable !== 'false' + accountData.isActive = accountData.isActive === 'true' + accountData.expiresAt = accountData.subscriptionExpiresAt || null + accountData.platform = accountData.platform || 'openai-responses' + + accounts.push(accountData) + }) return accounts } @@ -644,6 +612,9 @@ class OpenAIResponsesAccountService { // 保存账户数据 await client.hset(key, accountData) + // 添加到索引 + await redis.addToIndex('openai_responses_account:index', accountId) + // 添加到共享账户列表 if (accountData.accountType === 'shared') { await client.sadd(this.SHARED_ACCOUNTS_KEY, accountId) diff --git a/src/services/openaiResponsesRelayService.js b/src/services/openaiResponsesRelayService.js index 688e6ca7..d7db8eaa 100644 --- a/src/services/openaiResponsesRelayService.js +++ b/src/services/openaiResponsesRelayService.js @@ -7,6 +7,11 @@ const apiKeyService = require('./apiKeyService') const unifiedOpenAIScheduler = require('./unifiedOpenAIScheduler') const config = require('../../config/config') const crypto = require('crypto') +const LRUCache = require('../utils/lruCache') + +// lastUsedAt 更新节流（每账户 60 秒内最多更新一次，使用 LRU 防止内存泄漏） +const lastUsedAtThrottle = new LRUCache(1000) // 最多缓存 1000 个账户 +const LAST_USED_AT_THROTTLE_MS = 60000 // 抽取缓存写入 token，兼容多种字段命名 function extractCacheCreationTokens(usageData) { @@ -39,6 +44,21 @@ class OpenAIResponsesRelayService { this.defaultTimeout = config.requestTimeout || 600000 } + // 节流更新 lastUsedAt + async _throttledUpdateLastUsedAt(accountId) { + const now = Date.now() + const lastUpdate = lastUsedAtThrottle.get(accountId) + + if (lastUpdate && now - lastUpdate < LAST_USED_AT_THROTTLE_MS) { + return // 跳过更新 + } + + lastUsedAtThrottle.set(accountId, now, LAST_USED_AT_THROTTLE_MS) + await openaiResponsesAccountService.updateAccount(accountId, { + lastUsedAt: new Date().toISOString() + }) + } + // 处理请求转发 async handleRequest(req, res, account, apiKeyData) { let abortController = null @@ -259,10 +279,8 @@ class OpenAIResponsesRelayService { return res.status(response.status).json(errorData) } - // 更新最后使用时间 - await openaiResponsesAccountService.updateAccount(account.id, { - lastUsedAt: new Date().toISOString() - }) + // 更新最后使用时间（节流） + await this._throttledUpdateLastUsedAt(account.id) // 处理流式响应 if (req.body?.stream && response.data && typeof response.data.pipe === 'function') { diff --git a/src/services/pricingService.js b/src/services/pricingService.js index 0d0470cb..a8f37667 100644 --- a/src/services/pricingService.js +++ b/src/services/pricingService.js @@ -105,7 +105,7 @@ class PricingService { // 设置文件监听器 this.setupFileWatcher() - logger.success('💰 Pricing service initialized successfully') + logger.success('Pricing service initialized successfully') } catch (error) { logger.error('❌ Failed to initialize pricing service:', error) } @@ -298,7 +298,7 @@ class PricingService { this.pricingData = jsonData this.lastUpdated = new Date() - logger.success(`💰 Downloaded pricing data for ${Object.keys(jsonData).length} models`) + logger.success(`Downloaded pricing data for ${Object.keys(jsonData).length} models`) // 设置或重新设置文件监听器 this.setupFileWatcher() @@ -762,7 +762,7 @@ class PricingService { this.lastUpdated = new Date() const modelCount = Object.keys(jsonData).length - logger.success(`💰 Reloaded pricing data for ${modelCount} models from file`) + logger.success(`Reloaded pricing data for ${modelCount} models from file`) // 显示一些统计信息 const claudeModels = Object.keys(jsonData).filter((k) => k.includes('claude')).length diff --git a/src/services/unifiedClaudeScheduler.js b/src/services/unifiedClaudeScheduler.js index 750a1765..0d39ad68 100644 --- a/src/services/unifiedClaudeScheduler.js +++ b/src/services/unifiedClaudeScheduler.js @@ -6,6 +6,7 @@ const accountGroupService = require('./accountGroupService') const redis = require('../models/redis') const logger = require('../utils/logger') const { parseVendorPrefixedModel, isOpus45OrNewer } = require('../utils/modelHelper') +const { isSchedulable, sortAccountsByPriority } = require('../utils/commonHelper') /** * Check if account is Pro (not Max) @@ -38,16 +39,6 @@ class UnifiedClaudeScheduler { this.SESSION_MAPPING_PREFIX = 'unified_claude_session_mapping:' } - // 🔧 辅助方法：检查账户是否可调度（兼容字符串和布尔值） - _isSchedulable(schedulable) { - // 如果是 undefined 或 null，默认为可调度 - if (schedulable === undefined || schedulable === null) { - return true - } - // 明确设置为 false（布尔值）或 'false'（字符串）时不可调度 - return schedulable !== false && schedulable !== 'false' - } - // 🔍 检查账户是否支持请求的模型 _isModelSupportedByAccount(account, accountType, requestedModel, context = '') { if (!requestedModel) { @@ -286,7 +277,7 @@ class UnifiedClaudeScheduler { throw error } - if (!this._isSchedulable(boundAccount.schedulable)) { + if (!isSchedulable(boundAccount.schedulable)) { logger.warn( `⚠️ Bound Claude OAuth account ${apiKeyData.claudeAccountId} is not schedulable (schedulable: ${boundAccount?.schedulable}), falling back to pool` ) @@ -319,7 +310,7 @@ class UnifiedClaudeScheduler { boundConsoleAccount && boundConsoleAccount.isActive === true && boundConsoleAccount.status === 'active' && - this._isSchedulable(boundConsoleAccount.schedulable) + isSchedulable(boundConsoleAccount.schedulable) ) { // 检查是否临时不可用 const isTempUnavailable = await this.isAccountTemporarilyUnavailable( @@ -354,7 +345,7 @@ class UnifiedClaudeScheduler { if ( boundBedrockAccountResult.success && boundBedrockAccountResult.data.isActive === true && - this._isSchedulable(boundBedrockAccountResult.data.schedulable) + isSchedulable(boundBedrockAccountResult.data.schedulable) ) { // 检查是否临时不可用 const isTempUnavailable = await this.isAccountTemporarilyUnavailable( @@ -436,7 +427,7 @@ class UnifiedClaudeScheduler { } // 按优先级和最后使用时间排序 - const sortedAccounts = this._sortAccountsByPriority(availableAccounts) + const sortedAccounts = sortAccountsByPriority(availableAccounts) // 选择第一个账户 const selectedAccount = sortedAccounts[0] @@ -496,7 +487,7 @@ class UnifiedClaudeScheduler { throw error } - if (!this._isSchedulable(boundAccount.schedulable)) { + if (!isSchedulable(boundAccount.schedulable)) { logger.warn( `⚠️ Bound Claude OAuth account ${apiKeyData.claudeAccountId} is not schedulable (schedulable: ${boundAccount?.schedulable})` ) @@ -530,7 +521,7 @@ class UnifiedClaudeScheduler { boundConsoleAccount && boundConsoleAccount.isActive === true && boundConsoleAccount.status === 'active' && - this._isSchedulable(boundConsoleAccount.schedulable) + isSchedulable(boundConsoleAccount.schedulable) ) { // 主动触发一次额度检查 try { @@ -579,7 +570,7 @@ class UnifiedClaudeScheduler { if ( boundBedrockAccountResult.success && boundBedrockAccountResult.data.isActive === true && - this._isSchedulable(boundBedrockAccountResult.data.schedulable) + isSchedulable(boundBedrockAccountResult.data.schedulable) ) { logger.info( `🎯 Using bound dedicated Bedrock account: ${boundBedrockAccountResult.data.name} (${apiKeyData.bedrockAccountId})` @@ -609,7 +600,7 @@ class UnifiedClaudeScheduler { account.status !== 'blocked' && account.status !== 'temp_error' && (account.accountType === 'shared' || !account.accountType) && // 兼容旧数据 - this._isSchedulable(account.schedulable) + isSchedulable(account.schedulable) ) { // 检查是否可调度 @@ -691,7 +682,7 @@ class UnifiedClaudeScheduler { currentAccount.isActive === true && currentAccount.status === 'active' && currentAccount.accountType === 'shared' && - this._isSchedulable(currentAccount.schedulable) + isSchedulable(currentAccount.schedulable) ) { // 检查是否可调度 @@ -826,7 +817,7 @@ class UnifiedClaudeScheduler { if ( account.isActive === true && account.accountType === 'shared' && - this._isSchedulable(account.schedulable) + isSchedulable(account.schedulable) ) { // 检查是否临时不可用 const isTempUnavailable = await this.isAccountTemporarilyUnavailable( @@ -870,7 +861,7 @@ class UnifiedClaudeScheduler { account.isActive === true && account.status === 'active' && account.accountType === 'shared' && - this._isSchedulable(account.schedulable) + isSchedulable(account.schedulable) ) { // 检查模型支持 if (!this._isModelSupportedByAccount(account, 'ccr', requestedModel)) { @@ -949,21 +940,6 @@ class UnifiedClaudeScheduler { return availableAccounts } - // 🔢 按优先级和最后使用时间排序账户 - _sortAccountsByPriority(accounts) { - return accounts.sort((a, b) => { - // 首先按优先级排序（数字越小优先级越高） - if (a.priority !== b.priority) { - return a.priority - b.priority - } - - // 优先级相同时，按最后使用时间排序（最久未使用的优先） - const aLastUsed = new Date(a.lastUsedAt || 0).getTime() - const bLastUsed = new Date(b.lastUsedAt || 0).getTime() - return aLastUsed - bLastUsed - }) - } - // 🔍 检查账户是否可用 async _isAccountAvailable(accountId, accountType, requestedModel = null) { try { @@ -978,7 +954,7 @@ class UnifiedClaudeScheduler { return false } // 检查是否可调度 - if (!this._isSchedulable(account.schedulable)) { + if (!isSchedulable(account.schedulable)) { logger.info(`🚫 Account ${accountId} is not schedulable`) return false } @@ -1029,7 +1005,7 @@ class UnifiedClaudeScheduler { return false } // 检查是否可调度 - if (!this._isSchedulable(account.schedulable)) { + if (!isSchedulable(account.schedulable)) { logger.info(`🚫 Claude Console account ${accountId} is not schedulable`) return false } @@ -1093,7 +1069,7 @@ class UnifiedClaudeScheduler { return false } // 检查是否可调度 - if (!this._isSchedulable(accountResult.data.schedulable)) { + if (!isSchedulable(accountResult.data.schedulable)) { logger.info(`🚫 Bedrock account ${accountId} is not schedulable`) return false } @@ -1113,7 +1089,7 @@ class UnifiedClaudeScheduler { return false } // 检查是否可调度 - if (!this._isSchedulable(account.schedulable)) { + if (!isSchedulable(account.schedulable)) { logger.info(`🚫 CCR account ${accountId} is not schedulable`) return false } @@ -1544,7 +1520,7 @@ class UnifiedClaudeScheduler { ? account.status === 'active' : account.status === 'active' - if (isActive && status && this._isSchedulable(account.schedulable)) { + if (isActive && status && isSchedulable(account.schedulable)) { // 检查模型支持 if (!this._isModelSupportedByAccount(account, accountType, requestedModel, 'in group')) { continue @@ -1594,7 +1570,7 @@ class UnifiedClaudeScheduler { } // 使用现有的优先级排序逻辑 - const sortedAccounts = this._sortAccountsByPriority(availableAccounts) + const sortedAccounts = sortAccountsByPriority(availableAccounts) // 选择第一个账户 const selectedAccount = sortedAccounts[0] @@ -1664,7 +1640,7 @@ class UnifiedClaudeScheduler { } // 3. 按优先级和最后使用时间排序 - const sortedAccounts = this._sortAccountsByPriority(availableCcrAccounts) + const sortedAccounts = sortAccountsByPriority(availableCcrAccounts) const selectedAccount = sortedAccounts[0] // 4. 建立会话映射 @@ -1710,7 +1686,7 @@ class UnifiedClaudeScheduler { account.isActive === true && account.status === 'active' && account.accountType === 'shared' && - this._isSchedulable(account.schedulable) + isSchedulable(account.schedulable) ) { // 检查模型支持 if (!this._isModelSupportedByAccount(account, 'ccr', requestedModel)) { diff --git a/src/services/unifiedGeminiScheduler.js b/src/services/unifiedGeminiScheduler.js index 33193501..2581196b 100644 --- a/src/services/unifiedGeminiScheduler.js +++ b/src/services/unifiedGeminiScheduler.js @@ -3,28 +3,13 @@ const geminiApiAccountService = require('./geminiApiAccountService') const accountGroupService = require('./accountGroupService') const redis = require('../models/redis') const logger = require('../utils/logger') +const { isSchedulable, isActive, sortAccountsByPriority } = require('../utils/commonHelper') class UnifiedGeminiScheduler { constructor() { this.SESSION_MAPPING_PREFIX = 'unified_gemini_session_mapping:' } - // 🔧 辅助方法：检查账户是否可调度（兼容字符串和布尔值） - _isSchedulable(schedulable) { - // 如果是 undefined 或 null，默认为可调度 - if (schedulable === undefined || schedulable === null) { - return true - } - // 明确设置为 false（布尔值）或 'false'（字符串）时不可调度 - return schedulable !== false && schedulable !== 'false' - } - - // 🔧 辅助方法：检查账户是否激活（兼容字符串和布尔值） - _isActive(isActive) { - // 兼容布尔值 true 和字符串 'true' - return isActive === true || isActive === 'true' - } - // 🎯 统一调度Gemini账号 async selectAccountForApiKey( apiKeyData, @@ -43,7 +28,7 @@ class UnifiedGeminiScheduler { const boundAccount = await geminiApiAccountService.getAccount(accountId) if ( boundAccount && - this._isActive(boundAccount.isActive) && + isActive(boundAccount.isActive) && boundAccount.status !== 'error' ) { logger.info( @@ -80,7 +65,7 @@ class UnifiedGeminiScheduler { const boundAccount = await geminiAccountService.getAccount(apiKeyData.geminiAccountId) if ( boundAccount && - this._isActive(boundAccount.isActive) && + isActive(boundAccount.isActive) && boundAccount.status !== 'error' ) { logger.info( @@ -150,7 +135,7 @@ class UnifiedGeminiScheduler { } // 按优先级和最后使用时间排序 - const sortedAccounts = this._sortAccountsByPriority(availableAccounts) + const sortedAccounts = sortAccountsByPriority(availableAccounts) // 选择第一个账户 const selectedAccount = sortedAccounts[0] @@ -200,7 +185,7 @@ class UnifiedGeminiScheduler { const boundAccount = await geminiApiAccountService.getAccount(accountId) if ( boundAccount && - this._isActive(boundAccount.isActive) && + isActive(boundAccount.isActive) && boundAccount.status !== 'error' ) { const isRateLimited = await this.isAccountRateLimited(accountId) @@ -251,7 +236,7 @@ class UnifiedGeminiScheduler { const boundAccount = await geminiAccountService.getAccount(apiKeyData.geminiAccountId) if ( boundAccount && - this._isActive(boundAccount.isActive) && + isActive(boundAccount.isActive) && boundAccount.status !== 'error' ) { const isRateLimited = await this.isAccountRateLimited(boundAccount.id) @@ -298,10 +283,10 @@ class UnifiedGeminiScheduler { const geminiAccounts = await geminiAccountService.getAllAccounts() for (const account of geminiAccounts) { if ( - this._isActive(account.isActive) && + isActive(account.isActive) && account.status !== 'error' && (account.accountType === 'shared' || !account.accountType) && // 兼容旧数据 - this._isSchedulable(account.schedulable) + isSchedulable(account.schedulable) ) { // 检查是否可调度 @@ -348,10 +333,10 @@ class UnifiedGeminiScheduler { const geminiApiAccounts = await geminiApiAccountService.getAllAccounts() for (const account of geminiApiAccounts) { if ( - this._isActive(account.isActive) && + isActive(account.isActive) && account.status !== 'error' && (account.accountType === 'shared' || !account.accountType) && - this._isSchedulable(account.schedulable) + isSchedulable(account.schedulable) ) { // 检查模型支持 if (requestedModel && account.supportedModels && account.supportedModels.length > 0) { @@ -388,42 +373,27 @@ class UnifiedGeminiScheduler { return availableAccounts } - // 🔢 按优先级和最后使用时间排序账户 - _sortAccountsByPriority(accounts) { - return accounts.sort((a, b) => { - // 首先按优先级排序（数字越小优先级越高） - if (a.priority !== b.priority) { - return a.priority - b.priority - } - - // 优先级相同时，按最后使用时间排序（最久未使用的优先） - const aLastUsed = new Date(a.lastUsedAt || 0).getTime() - const bLastUsed = new Date(b.lastUsedAt || 0).getTime() - return aLastUsed - bLastUsed - }) - } - // 🔍 检查账户是否可用 async _isAccountAvailable(accountId, accountType) { try { if (accountType === 'gemini') { const account = await geminiAccountService.getAccount(accountId) - if (!account || !this._isActive(account.isActive) || account.status === 'error') { + if (!account || !isActive(account.isActive) || account.status === 'error') { return false } // 检查是否可调度 - if (!this._isSchedulable(account.schedulable)) { + if (!isSchedulable(account.schedulable)) { logger.info(`🚫 Gemini account ${accountId} is not schedulable`) return false } return !(await this.isAccountRateLimited(accountId)) } else if (accountType === 'gemini-api') { const account = await geminiApiAccountService.getAccount(accountId) - if (!account || !this._isActive(account.isActive) || account.status === 'error') { + if (!account || !isActive(account.isActive) || account.status === 'error') { return false } // 检查是否可调度 - if (!this._isSchedulable(account.schedulable)) { + if (!isSchedulable(account.schedulable)) { logger.info(`🚫 Gemini-API account ${accountId} is not schedulable`) return false } @@ -665,9 +635,9 @@ class UnifiedGeminiScheduler { // 检查账户是否可用 if ( - this._isActive(account.isActive) && + isActive(account.isActive) && account.status !== 'error' && - this._isSchedulable(account.schedulable) + isSchedulable(account.schedulable) ) { // 对于 Gemini OAuth 账户，检查 token 是否过期 if (accountType === 'gemini') { @@ -714,7 +684,7 @@ class UnifiedGeminiScheduler { } // 使用现有的优先级排序逻辑 - const sortedAccounts = this._sortAccountsByPriority(availableAccounts) + const sortedAccounts = sortAccountsByPriority(availableAccounts) // 选择第一个账户 const selectedAccount = sortedAccounts[0] diff --git a/src/services/unifiedOpenAIScheduler.js b/src/services/unifiedOpenAIScheduler.js index 6027df59..a1e8ef44 100644 --- a/src/services/unifiedOpenAIScheduler.js +++ b/src/services/unifiedOpenAIScheduler.js @@ -3,42 +3,13 @@ const openaiResponsesAccountService = require('./openaiResponsesAccountService') const accountGroupService = require('./accountGroupService') const redis = require('../models/redis') const logger = require('../utils/logger') +const { isSchedulable, sortAccountsByPriority } = require('../utils/commonHelper') class UnifiedOpenAIScheduler { constructor() { this.SESSION_MAPPING_PREFIX = 'unified_openai_session_mapping:' } - // 🔢 按优先级和最后使用时间排序账户（与 Claude/Gemini 调度保持一致） - _sortAccountsByPriority(accounts) { - return accounts.sort((a, b) => { - const aPriority = Number.parseInt(a.priority, 10) - const bPriority = Number.parseInt(b.priority, 10) - const normalizedAPriority = Number.isFinite(aPriority) ? aPriority : 50 - const normalizedBPriority = Number.isFinite(bPriority) ? bPriority : 50 - - // 首先按优先级排序（数字越小优先级越高） - if (normalizedAPriority !== normalizedBPriority) { - return normalizedAPriority - normalizedBPriority - } - - // 优先级相同时，按最后使用时间排序（最久未使用的优先） - const aLastUsed = new Date(a.lastUsedAt || 0).getTime() - const bLastUsed = new Date(b.lastUsedAt || 0).getTime() - return aLastUsed - bLastUsed - }) - } - - // 🔧 辅助方法：检查账户是否可调度（兼容字符串和布尔值） - _isSchedulable(schedulable) { - // 如果是 undefined 或 null，默认为可调度 - if (schedulable === undefined || schedulable === null) { - return true - } - // 明确设置为 false（布尔值）或 'false'（字符串）时不可调度 - return schedulable !== false && schedulable !== 'false' - } - // 🔧 辅助方法：检查账户是否被限流（兼容字符串和对象格式） _isRateLimited(rateLimitStatus) { if (!rateLimitStatus) { @@ -85,7 +56,7 @@ class UnifiedOpenAIScheduler { let rateLimitChecked = false let stillLimited = false - let isSchedulable = this._isSchedulable(account.schedulable) + let isSchedulable = isSchedulable(account.schedulable) if (!isSchedulable) { if (!hasRateLimitFlag) { @@ -224,7 +195,7 @@ class UnifiedOpenAIScheduler { } } - if (!this._isSchedulable(boundAccount.schedulable)) { + if (!isSchedulable(boundAccount.schedulable)) { const errorMsg = `Dedicated account ${boundAccount.name} is not schedulable` logger.warn(`⚠️ ${errorMsg}`) const error = new Error(errorMsg) @@ -336,7 +307,7 @@ class UnifiedOpenAIScheduler { } // 按优先级和最后使用时间排序（与 Claude/Gemini 调度保持一致） - const sortedAccounts = this._sortAccountsByPriority(availableAccounts) + const sortedAccounts = sortAccountsByPriority(availableAccounts) // 选择第一个账户 const selectedAccount = sortedAccounts[0] @@ -451,11 +422,12 @@ class UnifiedOpenAIScheduler { if ( (account.isActive === true || account.isActive === 'true') && account.status !== 'error' && - account.status !== 'rateLimited' && (account.accountType === 'shared' || !account.accountType) ) { - const hasRateLimitFlag = this._hasRateLimitFlag(account.rateLimitStatus) - const schedulable = this._isSchedulable(account.schedulable) + // 检查 rateLimitStatus 或 status === 'rateLimited' + const hasRateLimitFlag = + this._hasRateLimitFlag(account.rateLimitStatus) || account.status === 'rateLimited' + const schedulable = isSchedulable(account.schedulable) if (!schedulable && !hasRateLimitFlag) { logger.debug(`⏭️ Skipping OpenAI-Responses account ${account.name} - not schedulable`) @@ -464,9 +436,23 @@ class UnifiedOpenAIScheduler { let isRateLimitCleared = false if (hasRateLimitFlag) { - isRateLimitCleared = await openaiResponsesAccountService.checkAndClearRateLimit( - account.id - ) + // 区分正常限流和历史遗留数据 + if (this._hasRateLimitFlag(account.rateLimitStatus)) { + // 有 rateLimitStatus，走正常清理逻辑 + isRateLimitCleared = await openaiResponsesAccountService.checkAndClearRateLimit( + account.id + ) + } else { + // 只有 status=rateLimited 但没有 rateLimitStatus，是历史遗留数据，直接清除 + await openaiResponsesAccountService.updateAccount(account.id, { + status: 'active', + schedulable: 'true' + }) + isRateLimitCleared = true + logger.info( + `✅ OpenAI-Responses账号 ${account.name} 清除历史遗留限流状态（status=rateLimited 但无 rateLimitStatus）` + ) + } if (!isRateLimitCleared) { logger.debug(`⏭️ Skipping OpenAI-Responses account ${account.name} - rate limited`) @@ -544,7 +530,7 @@ class UnifiedOpenAIScheduler { return false } // 检查是否可调度 - if (!this._isSchedulable(account.schedulable)) { + if (!isSchedulable(account.schedulable)) { logger.info(`🚫 OpenAI-Responses account ${accountId} is not schedulable`) return false } @@ -905,7 +891,7 @@ class UnifiedOpenAIScheduler { } // 按优先级和最后使用时间排序（与 Claude/Gemini 调度保持一致） - const sortedAccounts = this._sortAccountsByPriority(availableAccounts) + const sortedAccounts = sortAccountsByPriority(availableAccounts) // 选择第一个账户 const selectedAccount = sortedAccounts[0] diff --git a/src/services/userMessageQueueService.js b/src/services/userMessageQueueService.js index 2b4784a2..6bf46c72 100644 --- a/src/services/userMessageQueueService.js +++ b/src/services/userMessageQueueService.js @@ -10,6 +10,7 @@ const { v4: uuidv4 } = require('uuid') const redis = require('../models/redis') const config = require('../../config/config') const logger = require('../utils/logger') +const { getCachedConfig, setCachedConfig } = require('../utils/performanceOptimizer') // 清理任务间隔 const CLEANUP_INTERVAL_MS = 60000 // 1分钟 @@ -19,6 +20,9 @@ const POLL_INTERVAL_BASE_MS = 50 // 基础轮询间隔 const POLL_INTERVAL_MAX_MS = 500 // 最大轮询间隔 const POLL_BACKOFF_FACTOR = 1.5 // 退避因子 +// 配置缓存 key +const CONFIG_CACHE_KEY = 'user_message_queue_config' + class UserMessageQueueService { constructor() { this.cleanupTimer = null @@ -64,18 +68,23 @@ class UserMessageQueueService { } /** - * 获取当前配置（支持 Web 界面配置优先） + * 获取当前配置（支持 Web 界面配置优先，带短 TTL 缓存） * @returns {Promise} 配置对象 */ async getConfig() { + // 检查缓存 + const cached = getCachedConfig(CONFIG_CACHE_KEY) + if (cached) { + return cached + } + // 默认配置（防止 config.userMessageQueue 未定义） - // 注意：优化后的默认值 - 锁持有时间从分钟级降到毫秒级，无需长等待 const queueConfig = config.userMessageQueue || {} const defaults = { enabled: queueConfig.enabled ?? false, delayMs: queueConfig.delayMs ?? 200, - timeoutMs: queueConfig.timeoutMs ?? 5000, // 从 60000 降到 5000，因为锁持有时间短 - lockTtlMs: queueConfig.lockTtlMs ?? 5000 // 从 120000 降到 5000，5秒足以覆盖请求发送 + timeoutMs: queueConfig.timeoutMs ?? 60000, + lockTtlMs: queueConfig.lockTtlMs ?? 120000 } // 尝试从 claudeRelayConfigService 获取 Web 界面配置 @@ -83,7 +92,7 @@ class UserMessageQueueService { const claudeRelayConfigService = require('./claudeRelayConfigService') const webConfig = await claudeRelayConfigService.getConfig() - return { + const result = { enabled: webConfig.userMessageQueueEnabled !== undefined ? webConfig.userMessageQueueEnabled @@ -101,8 +110,13 @@ class UserMessageQueueService { ? webConfig.userMessageQueueLockTtlMs : defaults.lockTtlMs } + + // 缓存配置 30 秒 + setCachedConfig(CONFIG_CACHE_KEY, result, 30000) + return result } catch { - // 回退到环境变量配置 + // 回退到环境变量配置，也缓存 + setCachedConfig(CONFIG_CACHE_KEY, defaults, 30000) return defaults } } diff --git a/src/services/userService.js b/src/services/userService.js index 00f0665f..c8b4b0f8 100644 --- a/src/services/userService.js +++ b/src/services/userService.js @@ -74,6 +74,7 @@ class UserService { // 保存用户信息 await redis.set(`${this.userPrefix}${user.id}`, JSON.stringify(user)) await redis.set(`${this.usernamePrefix}${username}`, user.id) + await redis.addToIndex('user:index', user.id) // 如果是新用户，尝试转移匹配的API Keys if (isNewUser) { @@ -167,8 +168,8 @@ class UserService { `📊 Calculated user ${userId} usage: ${totalUsage.requests} requests, ${totalUsage.inputTokens} input tokens, $${totalUsage.totalCost.toFixed(4)} total cost from ${userApiKeys.length} API keys` ) - // Count only non-deleted API keys for the user's active count - const activeApiKeyCount = userApiKeys.filter((key) => key.isDeleted !== 'true').length + // Count only non-deleted API keys for the user's active count（布尔值比较） + const activeApiKeyCount = userApiKeys.filter((key) => !key.isDeleted).length return { totalUsage, @@ -191,14 +192,18 @@ class UserService { // 📋 获取所有用户列表（管理员功能） async getAllUsers(options = {}) { try { - const client = redis.getClientSafe() const { page = 1, limit = 20, role, isActive } = options - const pattern = `${this.userPrefix}*` - const keys = await client.keys(pattern) + const userIds = await redis.getAllIdsByIndex( + 'user:index', + `${this.userPrefix}*`, + /^user:(.+)$/ + ) + const keys = userIds.map((id) => `${this.userPrefix}${id}`) + const dataList = await redis.batchGetChunked(keys) const users = [] - for (const key of keys) { - const userData = await client.get(key) + for (let i = 0; i < keys.length; i++) { + const userData = dataList[i] if (userData) { const user = JSON.parse(userData) @@ -398,14 +403,15 @@ class UserService { try { const client = redis.getClientSafe() const pattern = `${this.userSessionPrefix}*` - const keys = await client.keys(pattern) + const keys = await redis.scanKeys(pattern) + const dataList = await redis.batchGetChunked(keys) - for (const key of keys) { - const sessionData = await client.get(key) + for (let i = 0; i < keys.length; i++) { + const sessionData = dataList[i] if (sessionData) { const session = JSON.parse(sessionData) if (session.userId === userId) { - await client.del(key) + await client.del(keys[i]) } } } @@ -454,9 +460,13 @@ class UserService { // 📊 获取用户统计信息 async getUserStats() { try { - const client = redis.getClientSafe() - const pattern = `${this.userPrefix}*` - const keys = await client.keys(pattern) + const userIds = await redis.getAllIdsByIndex( + 'user:index', + `${this.userPrefix}*`, + /^user:(.+)$/ + ) + const keys = userIds.map((id) => `${this.userPrefix}${id}`) + const dataList = await redis.batchGetChunked(keys) const stats = { totalUsers: 0, @@ -472,8 +482,8 @@ class UserService { } } - for (const key of keys) { - const userData = await client.get(key) + for (let i = 0; i < keys.length; i++) { + const userData = dataList[i] if (userData) { const user = JSON.parse(userData) stats.totalUsers++ @@ -522,7 +532,7 @@ class UserService { const { displayName, username, email } = user // 获取所有API Keys - const allApiKeys = await apiKeyService.getAllApiKeys() + const allApiKeys = await apiKeyService.getAllApiKeysFast() // 找到没有用户ID的API Keys（即由Admin创建的） const unownedApiKeys = allApiKeys.filter((key) => !key.userId || key.userId === '') diff --git a/src/utils/commonHelper.js b/src/utils/commonHelper.js new file mode 100644 index 00000000..752166da --- /dev/null +++ b/src/utils/commonHelper.js @@ -0,0 +1,300 @@ +// 通用工具函数集合 +// 抽取自各服务的重复代码，统一管理 + +const crypto = require('crypto') +const config = require('../../config/config') +const LRUCache = require('./lruCache') + +// ============================================ +// 加密相关 - 工厂模式支持不同 salt +// ============================================ + +const ALGORITHM = 'aes-256-cbc' +const IV_LENGTH = 16 + +// 缓存不同 salt 的加密实例 +const _encryptorCache = new Map() + +// 创建加密器实例（每个 salt 独立缓存） +const createEncryptor = (salt) => { + if (_encryptorCache.has(salt)) return _encryptorCache.get(salt) + + let keyCache = null + const decryptCache = new LRUCache(500) + + const getKey = () => { + if (!keyCache) keyCache = crypto.scryptSync(config.security.encryptionKey, salt, 32) + return keyCache + } + + const encrypt = (text) => { + if (!text) return '' + const key = getKey() + const iv = crypto.randomBytes(IV_LENGTH) + const cipher = crypto.createCipheriv(ALGORITHM, key, iv) + let encrypted = cipher.update(text, 'utf8', 'hex') + encrypted += cipher.final('hex') + return iv.toString('hex') + ':' + encrypted + } + + const decrypt = (text, useCache = true) => { + if (!text) return '' + if (!text.includes(':')) return text + const cacheKey = crypto.createHash('sha256').update(text).digest('hex') + if (useCache) { + const cached = decryptCache.get(cacheKey) + if (cached !== undefined) return cached + } + try { + const key = getKey() + const [ivHex, encrypted] = text.split(':') + const iv = Buffer.from(ivHex, 'hex') + const decipher = crypto.createDecipheriv(ALGORITHM, key, iv) + let decrypted = decipher.update(encrypted, 'hex', 'utf8') + decrypted += decipher.final('utf8') + if (useCache) decryptCache.set(cacheKey, decrypted, 5 * 60 * 1000) + return decrypted + } catch (e) { + return text + } + } + + const instance = { + encrypt, + decrypt, + getKey, + clearCache: () => decryptCache.clear(), + getStats: () => decryptCache.getStats?.() || { size: decryptCache.size } + } + + _encryptorCache.set(salt, instance) + return instance +} + +// 默认加密器（向后兼容） +const defaultEncryptor = createEncryptor('claude-relay-salt') +const encrypt = defaultEncryptor.encrypt +const decrypt = defaultEncryptor.decrypt +const getEncryptionKey = defaultEncryptor.getKey +const clearDecryptCache = defaultEncryptor.clearCache +const getDecryptCacheStats = defaultEncryptor.getStats + +// ============================================ +// 布尔值处理 +// ============================================ + +// 转换为布尔值（宽松模式） +const toBoolean = (value) => value === true || value === 'true' || (typeof value === 'string' && value.toLowerCase() === 'true') + +// 检查是否为真值（null/undefined 返回 false） +const isTruthy = (value) => value != null && toBoolean(value) + +// 检查是否可调度（默认 true，只有明确 false 才返回 false） +const isSchedulable = (value) => value !== false && value !== 'false' + +// 检查是否激活 +const isActive = (value) => value === true || value === 'true' + +// 检查账户是否健康（激活且状态正常） +const isAccountHealthy = (account) => { + if (!account) return false + if (!isTruthy(account.isActive)) return false + const status = (account.status || 'active').toLowerCase() + return !['error', 'unauthorized', 'blocked', 'temp_error'].includes(status) +} + +// ============================================ +// JSON 处理 +// ============================================ + +// 安全解析 JSON +const safeParseJson = (value, fallback = null) => { + if (!value || typeof value !== 'string') return fallback + try { return JSON.parse(value) } catch { return fallback } +} + +// 安全解析 JSON 为对象 +const safeParseJsonObject = (value, fallback = null) => { + const parsed = safeParseJson(value, fallback) + return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : fallback +} + +// 安全解析 JSON 为数组 +const safeParseJsonArray = (value, fallback = []) => { + const parsed = safeParseJson(value, fallback) + return Array.isArray(parsed) ? parsed : fallback +} + +// ============================================ +// 模型名称处理 +// ============================================ + +// 规范化模型名称（用于统计聚合） +const normalizeModelName = (model) => { + if (!model || model === 'unknown') return model + // Bedrock 模型: us-east-1.anthropic.claude-3-5-sonnet-v1:0 + if (model.includes('.anthropic.') || model.includes('.claude')) { + return model.replace(/^[a-z0-9-]+\./, '').replace('anthropic.', '').replace(/-v\d+:\d+$/, '') + } + return model.replace(/-v\d+:\d+$|:latest$/, '') +} + +// 规范化端点类型 +const normalizeEndpointType = (endpointType) => { + if (!endpointType) return 'anthropic' + const normalized = String(endpointType).toLowerCase() + return ['openai', 'comm', 'anthropic'].includes(normalized) ? normalized : 'anthropic' +} + +// 检查模型是否在映射表中 +const isModelInMapping = (modelMapping, requestedModel) => { + if (!modelMapping || Object.keys(modelMapping).length === 0) return true + if (Object.prototype.hasOwnProperty.call(modelMapping, requestedModel)) return true + const lower = requestedModel.toLowerCase() + return Object.keys(modelMapping).some(k => k.toLowerCase() === lower) +} + +// 获取映射后的模型名称 +const getMappedModelName = (modelMapping, requestedModel) => { + if (!modelMapping || Object.keys(modelMapping).length === 0) return requestedModel + if (modelMapping[requestedModel]) return modelMapping[requestedModel] + const lower = requestedModel.toLowerCase() + for (const [key, value] of Object.entries(modelMapping)) { + if (key.toLowerCase() === lower) return value + } + return requestedModel +} + +// ============================================ +// 账户调度相关 +// ============================================ + +// 按优先级和最后使用时间排序账户 +const sortAccountsByPriority = (accounts) => { + return [...accounts].sort((a, b) => { + const priorityA = parseInt(a.priority, 10) || 50 + const priorityB = parseInt(b.priority, 10) || 50 + if (priorityA !== priorityB) return priorityA - priorityB + const lastUsedA = a.lastUsedAt ? new Date(a.lastUsedAt).getTime() : 0 + const lastUsedB = b.lastUsedAt ? new Date(b.lastUsedAt).getTime() : 0 + if (lastUsedA !== lastUsedB) return lastUsedA - lastUsedB + const createdA = a.createdAt ? new Date(a.createdAt).getTime() : 0 + const createdB = b.createdAt ? new Date(b.createdAt).getTime() : 0 + return createdA - createdB + }) +} + +// 生成粘性会话 Key +const composeStickySessionKey = (prefix, sessionHash, apiKeyId = null) => { + if (!sessionHash) return null + return `sticky:${prefix}:${apiKeyId || 'default'}:${sessionHash}` +} + +// 过滤可用账户（激活 + 健康 + 可调度） +const filterAvailableAccounts = (accounts) => { + return accounts.filter(acc => acc && isAccountHealthy(acc) && isSchedulable(acc.schedulable)) +} + +// ============================================ +// 字符串处理 +// ============================================ + +// 截断字符串 +const truncate = (str, maxLen = 100, suffix = '...') => { + if (!str || str.length <= maxLen) return str + return str.slice(0, maxLen - suffix.length) + suffix +} + +// 掩码敏感信息（保留前后几位） +const maskSensitive = (str, keepStart = 4, keepEnd = 4, maskChar = '*') => { + if (!str || str.length <= keepStart + keepEnd) return str + const maskLen = Math.min(str.length - keepStart - keepEnd, 8) + return str.slice(0, keepStart) + maskChar.repeat(maskLen) + str.slice(-keepEnd) +} + +// ============================================ +// 数值处理 +// ============================================ + +// 安全解析整数 +const safeParseInt = (value, fallback = 0) => { + const parsed = parseInt(value, 10) + return isNaN(parsed) ? fallback : parsed +} + +// 安全解析浮点数 +const safeParseFloat = (value, fallback = 0) => { + const parsed = parseFloat(value) + return isNaN(parsed) ? fallback : parsed +} + +// 限制数值范围 +const clamp = (value, min, max) => Math.min(Math.max(value, min), max) + +// ============================================ +// 时间处理 +// ============================================ + +// 获取时区偏移后的日期 +const getDateInTimezone = (date = new Date(), offset = config.system?.timezoneOffset || 8) => { + return new Date(date.getTime() + offset * 3600000) +} + +// 获取时区日期字符串 YYYY-MM-DD +const getDateStringInTimezone = (date = new Date()) => { + const d = getDateInTimezone(date) + return `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, '0')}-${String(d.getUTCDate()).padStart(2, '0')}` +} + +// 检查是否过期 +const isExpired = (expiresAt) => { + if (!expiresAt) return false + return new Date(expiresAt).getTime() < Date.now() +} + +// 计算剩余时间（秒） +const getTimeRemaining = (expiresAt) => { + if (!expiresAt) return Infinity + return Math.max(0, Math.floor((new Date(expiresAt).getTime() - Date.now()) / 1000)) +} + +module.exports = { + // 加密 + createEncryptor, + encrypt, + decrypt, + getEncryptionKey, + clearDecryptCache, + getDecryptCacheStats, + // 布尔值 + toBoolean, + isTruthy, + isSchedulable, + isActive, + isAccountHealthy, + // JSON + safeParseJson, + safeParseJsonObject, + safeParseJsonArray, + // 模型 + normalizeModelName, + normalizeEndpointType, + isModelInMapping, + getMappedModelName, + // 调度 + sortAccountsByPriority, + composeStickySessionKey, + filterAvailableAccounts, + // 字符串 + truncate, + maskSensitive, + // 数值 + safeParseInt, + safeParseFloat, + clamp, + // 时间 + getDateInTimezone, + getDateStringInTimezone, + isExpired, + getTimeRemaining +} diff --git a/src/utils/oauthHelper.js b/src/utils/oauthHelper.js index 556e4243..030f1b1b 100644 --- a/src/utils/oauthHelper.js +++ b/src/utils/oauthHelper.js @@ -210,7 +210,7 @@ async function exchangeCodeForTokens(authorizationCode, codeVerifier, state, pro dataKeys: response.data ? Object.keys(response.data) : [] }) - logger.success('✅ OAuth token exchange successful', { + logger.success('OAuth token exchange successful', { status: response.status, hasAccessToken: !!response.data?.access_token, hasRefreshToken: !!response.data?.refresh_token, @@ -430,7 +430,7 @@ async function exchangeSetupTokenCode(authorizationCode, codeVerifier, state, pr dataKeys: response.data ? Object.keys(response.data) : [] }) - logger.success('✅ Setup Token exchange successful', { + logger.success('Setup Token exchange successful', { status: response.status, hasAccessToken: !!response.data?.access_token, scopes: response.data?.scope, @@ -660,7 +660,7 @@ async function getOrganizationInfo(sessionKey, proxyConfig = null) { throw new Error('未找到具有chat能力的组织') } - logger.success('✅ Found organization', { + logger.success('Found organization', { uuid: bestOrg.uuid, capabilities: maxCapabilities }) @@ -777,7 +777,7 @@ async function authorizeWithCookie(sessionKey, organizationUuid, scope, proxyCon // 构建完整的授权码（包含state，如果有的话） const fullCode = responseState ? `${authorizationCode}#${responseState}` : authorizationCode - logger.success('✅ Got authorization code via Cookie', { + logger.success('Got authorization code via Cookie', { codeLength: authorizationCode.length, codePrefix: `${authorizationCode.substring(0, 10)}...` }) @@ -853,7 +853,7 @@ async function oauthWithCookie(sessionKey, proxyConfig = null, isSetupToken = fa ? await exchangeSetupTokenCode(authorizationCode, codeVerifier, state, proxyConfig) : await exchangeCodeForTokens(authorizationCode, codeVerifier, state, proxyConfig) - logger.success('✅ Cookie-based OAuth flow completed', { + logger.success('Cookie-based OAuth flow completed', { isSetupToken, organizationUuid, hasAccessToken: !!tokenData.accessToken, diff --git a/src/utils/performanceOptimizer.js b/src/utils/performanceOptimizer.js new file mode 100644 index 00000000..29b91ed5 --- /dev/null +++ b/src/utils/performanceOptimizer.js @@ -0,0 +1,168 @@ +/** + * 性能优化工具模块 + * 提供 HTTP keep-alive 连接池、定价数据缓存等优化功能 + */ + +const https = require('https') +const http = require('http') +const fs = require('fs') +const LRUCache = require('./lruCache') + +// 连接池配置（从环境变量读取） +const STREAM_MAX_SOCKETS = parseInt(process.env.HTTPS_MAX_SOCKETS_STREAM) || 65535 +const NON_STREAM_MAX_SOCKETS = parseInt(process.env.HTTPS_MAX_SOCKETS_NON_STREAM) || 16384 +const MAX_FREE_SOCKETS = parseInt(process.env.HTTPS_MAX_FREE_SOCKETS) || 2048 +const FREE_SOCKET_TIMEOUT = parseInt(process.env.HTTPS_FREE_SOCKET_TIMEOUT) || 30000 + +// 流式请求 agent：高 maxSockets，timeout=0（不限制） +const httpsAgentStream = new https.Agent({ + keepAlive: true, + maxSockets: STREAM_MAX_SOCKETS, + maxFreeSockets: MAX_FREE_SOCKETS, + timeout: 0, + freeSocketTimeout: FREE_SOCKET_TIMEOUT +}) + +// 非流式请求 agent：较小 maxSockets +const httpsAgentNonStream = new https.Agent({ + keepAlive: true, + maxSockets: NON_STREAM_MAX_SOCKETS, + maxFreeSockets: MAX_FREE_SOCKETS, + timeout: 0, // 不限制，由请求层 REQUEST_TIMEOUT 控制 + freeSocketTimeout: FREE_SOCKET_TIMEOUT +}) + +// HTTP agent（非流式） +const httpAgent = new http.Agent({ + keepAlive: true, + maxSockets: NON_STREAM_MAX_SOCKETS, + maxFreeSockets: MAX_FREE_SOCKETS, + timeout: 0, // 不限制，由请求层 REQUEST_TIMEOUT 控制 + freeSocketTimeout: FREE_SOCKET_TIMEOUT +}) + +// 定价数据缓存（按文件路径区分） +const pricingDataCache = new Map() +const PRICING_CACHE_TTL = 5 * 60 * 1000 // 5分钟 + +// Redis 配置缓存（短 TTL） +const configCache = new LRUCache(100) +const CONFIG_CACHE_TTL = 30 * 1000 // 30秒 + +/** + * 获取流式请求的 HTTPS agent + */ +function getHttpsAgentForStream() { + return httpsAgentStream +} + +/** + * 获取非流式请求的 HTTPS agent + */ +function getHttpsAgentForNonStream() { + return httpsAgentNonStream +} + +/** + * 获取定价数据（带缓存，按路径区分） + * @param {string} pricingFilePath - 定价文件路径 + * @returns {Object|null} 定价数据 + */ +function getPricingData(pricingFilePath) { + const now = Date.now() + const cached = pricingDataCache.get(pricingFilePath) + + // 检查缓存是否有效 + if (cached && now - cached.loadTime < PRICING_CACHE_TTL) { + return cached.data + } + + // 重新加载 + try { + if (!fs.existsSync(pricingFilePath)) { + return null + } + const data = JSON.parse(fs.readFileSync(pricingFilePath, 'utf8')) + pricingDataCache.set(pricingFilePath, { data, loadTime: now }) + return data + } catch (error) { + return null + } +} + +/** + * 清除定价数据缓存（用于热更新） + * @param {string} pricingFilePath - 可选，指定路径则只清除该路径缓存 + */ +function clearPricingCache(pricingFilePath = null) { + if (pricingFilePath) { + pricingDataCache.delete(pricingFilePath) + } else { + pricingDataCache.clear() + } +} + +/** + * 获取缓存的配置 + * @param {string} key - 缓存键 + * @returns {*} 缓存值 + */ +function getCachedConfig(key) { + return configCache.get(key) +} + +/** + * 设置配置缓存 + * @param {string} key - 缓存键 + * @param {*} value - 值 + * @param {number} ttl - TTL（毫秒） + */ +function setCachedConfig(key, value, ttl = CONFIG_CACHE_TTL) { + configCache.set(key, value, ttl) +} + +/** + * 删除配置缓存 + * @param {string} key - 缓存键 + */ +function deleteCachedConfig(key) { + configCache.cache.delete(key) +} + +/** + * 获取连接池统计信息 + */ +function getAgentStats() { + return { + httpsStream: { + sockets: Object.keys(httpsAgentStream.sockets).length, + freeSockets: Object.keys(httpsAgentStream.freeSockets).length, + requests: Object.keys(httpsAgentStream.requests).length, + maxSockets: STREAM_MAX_SOCKETS + }, + httpsNonStream: { + sockets: Object.keys(httpsAgentNonStream.sockets).length, + freeSockets: Object.keys(httpsAgentNonStream.freeSockets).length, + requests: Object.keys(httpsAgentNonStream.requests).length, + maxSockets: NON_STREAM_MAX_SOCKETS + }, + http: { + sockets: Object.keys(httpAgent.sockets).length, + freeSockets: Object.keys(httpAgent.freeSockets).length, + requests: Object.keys(httpAgent.requests).length + }, + configCache: configCache.getStats() + } +} + +module.exports = { + getHttpsAgentForStream, + getHttpsAgentForNonStream, + getHttpAgent: () => httpAgent, + getPricingData, + clearPricingCache, + getCachedConfig, + setCachedConfig, + deleteCachedConfig, + getAgentStats +} diff --git a/src/utils/sseParser.js b/src/utils/sseParser.js index ea3d6a9c..466e0524 100644 --- a/src/utils/sseParser.js +++ b/src/utils/sseParser.js @@ -47,6 +47,72 @@ function parseSSELine(line) { } } -module.exports = { - parseSSELine +/** + * 增量 SSE 解析器类 + * 用于处理流式数据，避免每次都 split 整个 buffer + */ +class IncrementalSSEParser { + constructor() { + this.buffer = '' + } + + /** + * 添加数据块并返回完整的事件 + * @param {string} chunk - 数据块 + * @returns {Array} 解析出的完整事件数组 + */ + feed(chunk) { + this.buffer += chunk + const events = [] + + // 查找完整的事件（以 \n\n 分隔） + let idx + while ((idx = this.buffer.indexOf('\n\n')) !== -1) { + const event = this.buffer.slice(0, idx) + this.buffer = this.buffer.slice(idx + 2) + + if (event.trim()) { + // 解析事件中的每一行 + const lines = event.split('\n') + for (const line of lines) { + if (line.startsWith('data: ')) { + const jsonStr = line.slice(6) + if (jsonStr && jsonStr !== '[DONE]') { + try { + events.push({ type: 'data', data: JSON.parse(jsonStr) }) + } catch (e) { + events.push({ type: 'invalid', raw: jsonStr, error: e }) + } + } else if (jsonStr === '[DONE]') { + events.push({ type: 'done' }) + } + } else if (line.startsWith('event: ')) { + events.push({ type: 'event', name: line.slice(7).trim() }) + } + } + } + } + + return events + } + + /** + * 获取剩余的 buffer 内容 + * @returns {string} + */ + getRemaining() { + return this.buffer + } + + /** + * 重置解析器 + */ + reset() { + this.buffer = '' + } +} + +module.exports = { + parseSSELine, + IncrementalSSEParser } diff --git a/src/utils/workosOAuthHelper.js b/src/utils/workosOAuthHelper.js index ffd0f4d6..47f603f6 100644 --- a/src/utils/workosOAuthHelper.js +++ b/src/utils/workosOAuthHelper.js @@ -61,7 +61,7 @@ async function startDeviceAuthorization(proxyConfig = null) { throw new Error('WorkOS 返回数据缺少必要字段 (device_code / verification_uri)') } - logger.success('✅ 成功获取 WorkOS 设备码授权信息', { + logger.success('成功获取 WorkOS 设备码授权信息', { verificationUri: data.verification_uri, userCode: data.user_code }) diff --git a/web/admin-spa/src/components/tutorial/ClaudeCodeTutorial.vue b/web/admin-spa/src/components/tutorial/ClaudeCodeTutorial.vue new file mode 100644 index 00000000..2cc8b38a --- /dev/null +++ b/web/admin-spa/src/components/tutorial/ClaudeCodeTutorial.vue @@ -0,0 +1,495 @@ + + + diff --git a/web/admin-spa/src/components/tutorial/CodexTutorial.vue b/web/admin-spa/src/components/tutorial/CodexTutorial.vue new file mode 100644 index 00000000..cb39c263 --- /dev/null +++ b/web/admin-spa/src/components/tutorial/CodexTutorial.vue @@ -0,0 +1,354 @@ + + + diff --git a/web/admin-spa/src/components/tutorial/DroidCliTutorial.vue b/web/admin-spa/src/components/tutorial/DroidCliTutorial.vue new file mode 100644 index 00000000..e5c9400b --- /dev/null +++ b/web/admin-spa/src/components/tutorial/DroidCliTutorial.vue @@ -0,0 +1,98 @@ + + + diff --git a/web/admin-spa/src/components/tutorial/GeminiCliTutorial.vue b/web/admin-spa/src/components/tutorial/GeminiCliTutorial.vue new file mode 100644 index 00000000..59e4a20e --- /dev/null +++ b/web/admin-spa/src/components/tutorial/GeminiCliTutorial.vue @@ -0,0 +1,183 @@ + + + diff --git a/web/admin-spa/src/components/tutorial/NodeInstallTutorial.vue b/web/admin-spa/src/components/tutorial/NodeInstallTutorial.vue new file mode 100644 index 00000000..4f6352bf --- /dev/null +++ b/web/admin-spa/src/components/tutorial/NodeInstallTutorial.vue @@ -0,0 +1,234 @@ + + + diff --git a/web/admin-spa/src/components/tutorial/VerifyInstall.vue b/web/admin-spa/src/components/tutorial/VerifyInstall.vue new file mode 100644 index 00000000..91287233 --- /dev/null +++ b/web/admin-spa/src/components/tutorial/VerifyInstall.vue @@ -0,0 +1,30 @@ + + + diff --git a/web/admin-spa/src/components/user/UserApiKeysManager.vue b/web/admin-spa/src/components/user/UserApiKeysManager.vue index 092aa73c..3a439b21 100644 --- a/web/admin-spa/src/components/user/UserApiKeysManager.vue +++ b/web/admin-spa/src/components/user/UserApiKeysManager.vue @@ -85,7 +85,7 @@

{{ apiKey.name }}

Deleted diff --git a/web/admin-spa/src/composables/useTutorialUrls.js b/web/admin-spa/src/composables/useTutorialUrls.js new file mode 100644 index 00000000..a0f7d095 --- /dev/null +++ b/web/admin-spa/src/composables/useTutorialUrls.js @@ -0,0 +1,52 @@ +import { computed } from 'vue' + +export function useTutorialUrls() { + const getBaseUrlPrefix = () => { + const customPrefix = import.meta.env.VITE_API_BASE_PREFIX + if (customPrefix) { + return customPrefix.replace(/\/$/, '') + } + + let origin = '' + if (window.location.origin) { + origin = window.location.origin + } else { + const protocol = window.location.protocol + const hostname = window.location.hostname + const port = window.location.port + origin = protocol + '//' + hostname + if ( + port && + ((protocol === 'http:' && port !== '80') || (protocol === 'https:' && port !== '443')) + ) { + origin += ':' + port + } + } + + if (!origin) { + const currentUrl = window.location.href + const pathStart = currentUrl.indexOf('/', 8) + if (pathStart !== -1) { + origin = currentUrl.substring(0, pathStart) + } else { + return '' + } + } + + return origin + } + + const currentBaseUrl = computed(() => getBaseUrlPrefix() + '/api') + const geminiBaseUrl = computed(() => getBaseUrlPrefix() + '/gemini') + const openaiBaseUrl = computed(() => getBaseUrlPrefix() + '/openai') + const droidClaudeBaseUrl = computed(() => getBaseUrlPrefix() + '/droid/claude') + const droidOpenaiBaseUrl = computed(() => getBaseUrlPrefix() + '/droid/openai') + + return { + currentBaseUrl, + geminiBaseUrl, + openaiBaseUrl, + droidClaudeBaseUrl, + droidOpenaiBaseUrl + } +} diff --git a/web/admin-spa/src/stores/auth.js b/web/admin-spa/src/stores/auth.js index bbd39c62..0962f256 100644 --- a/web/admin-spa/src/stores/auth.js +++ b/web/admin-spa/src/stores/auth.js @@ -66,18 +66,14 @@ export const useAuthStore = defineStore('auth', () => { async function verifyToken() { try { - // 获取当前用户信息 + // /web/auth/user 已做完整 token 验证（session 存在性、完整性） + // 成功返回即表示 token 有效，无需再调用 dashboard const userResult = await apiClient.get('/web/auth/user') - if (userResult.success && userResult.user) { - username.value = userResult.user.username - } - - // 使用 dashboard 端点来验证 token - // 如果 token 无效，会抛出错误 - const result = await apiClient.get('/admin/dashboard') - if (!result.success) { + if (!userResult.success || !userResult.user) { logout() + return } + username.value = userResult.user.username } catch (error) { // token 无效，需要重新登录 logout() diff --git a/web/admin-spa/src/views/ApiKeysView.vue b/web/admin-spa/src/views/ApiKeysView.vue index 52a4cf56..a7481698 100644 --- a/web/admin-spa/src/views/ApiKeysView.vue +++ b/web/admin-spa/src/views/ApiKeysView.vue @@ -2122,7 +2122,6 @@ import { ref, reactive, computed, onMounted, onUnmounted, watch } from 'vue' import { useRouter } from 'vue-router' import { showToast } from '@/utils/toast' import { apiClient } from '@/config/api' -import { useClientsStore } from '@/stores/clients' import { useAuthStore } from '@/stores/auth' import * as XLSX from 'xlsx-js-style' import CreateApiKeyModal from '@/components/apikeys/CreateApiKeyModal.vue' @@ -2139,7 +2138,6 @@ import ActionDropdown from '@/components/common/ActionDropdown.vue' // 响应式数据 const router = useRouter() -const clientsStore = useClientsStore() const authStore = useAuthStore() const apiKeys = ref([]) @@ -4803,7 +4801,8 @@ onMounted(async () => { fetchCostSortStatus() // 先加载 API Keys（优先显示列表） - await Promise.all([clientsStore.loadSupportedClients(), loadApiKeys(), loadUsedModels()]) + // supported-clients 由 Create/Edit 模态框按需加载，无需预加载 + await Promise.all([loadApiKeys(), loadUsedModels()]) // 初始化全选状态 updateSelectAllState() diff --git a/web/admin-spa/src/views/TutorialView.vue b/web/admin-spa/src/views/TutorialView.vue index 373c7019..d6f462aa 100644 --- a/web/admin-spa/src/views/TutorialView.vue +++ b/web/admin-spa/src/views/TutorialView.vue @@ -5,15 +5,15 @@ class="mb-3 flex items-center text-xl font-bold text-gray-900 dark:text-gray-100 sm:mb-4 sm:text-2xl" > - Claude Code 使用教程 + {{ currentToolTitle }} 使用教程

- 跟着这个教程，你可以轻松在自己的电脑上安装并使用 Claude Code。 + 跟着这个教程，你可以轻松在自己的电脑上安装并使用 {{ currentToolTitle }}。

-
+
- -
- -
-

+
+
+

-

- Claude Code 需要 Node.js 环境才能运行。 -

- -
-
- - Windows 安装方法 -
-
-

- 方法一：官网下载（推荐） -

-
    -
  1. - 打开浏览器访问 - https://nodejs.org/ -
    2. -
  2. 点击 "LTS" 版本进行下载（推荐长期支持版本）
    3. -
  3. - 下载完成后双击 - .msi - 文件 -
    4. -
  4. 按照安装向导完成安装，保持默认设置即可
    5. -
-
-
-

- 方法二：使用包管理器 -

-

- 如果你安装了 Chocolatey 或 Scoop，可以使用命令行安装： -

-
-
# 使用 Chocolatey
-
choco install nodejs
-
# 或使用 Scoop
-
scoop install nodejs
-
-
-
-
- Windows 注意事项 -
-
    -
  • • 建议使用 PowerShell 而不是 CMD
    • -
  • • 如果遇到权限问题，尝试以管理员身份运行
    • -
  • • 某些杀毒软件可能会误报，需要添加白名单
    • -
-
-
- - -
-
- 验证安装是否成功 -
-

- 安装完成后，打开 PowerShell 或 CMD，输入以下命令： -

-
-
node --version
-
npm --version
-
-

- 如果显示版本号，说明安装成功了！ -

-
-
- - -
-

- 2 - 安装 Claude Code -

- -
-
- - 安装 Claude Code -
-

- 打开 PowerShell 或 CMD，运行以下命令： -

-
-
# 全局安装 Claude Code
-
- npm install -g @anthropic-ai/claude-code -
-
-

- 这个命令会从 npm 官方仓库下载并安装最新版本的 Claude Code。 -

- -
-
- 提示 -
-
    -
  • • 建议使用 PowerShell 而不是 CMD，功能更强大
    • -
  • • 如果遇到权限问题，以管理员身份运行 PowerShell
    • -
-
-
- - -
-
验证 Claude Code 安装
-

- 安装完成后，输入以下命令检查是否安装成功： -

-
-
claude --version
-
-

- 如果显示版本号，恭喜你！Claude Code 已经成功安装了。 -

-
-
- - -
-

- 3 - 设置环境变量 -

- -
-
- - 配置 Claude Code 环境变量 -
-

- 为了让 Claude Code 连接到你的中转服务，需要设置两个环境变量： -

- -
-
-
- 方法一：PowerShell 临时设置（当前会话） -
-

- 在 PowerShell 中运行以下命令： -

-
-
- $env:ANTHROPIC_BASE_URL = "{{ currentBaseUrl }}" -
-
- $env:ANTHROPIC_AUTH_TOKEN = "你的API密钥" -
-
-

- 💡 记得将 "你的API密钥" 替换为在上方 "API Keys" 标签页中创建的实际密钥。 -

-
- -
-
- 方法二：PowerShell 永久设置（用户级） -
-

- 在 PowerShell 中运行以下命令设置用户级环境变量： -

-
-
# 设置用户级环境变量（永久生效）
-
- [System.Environment]::SetEnvironmentVariable("ANTHROPIC_BASE_URL", "{{ - currentBaseUrl - }}", [System.EnvironmentVariableTarget]::User) -
-
- [System.Environment]::SetEnvironmentVariable("ANTHROPIC_AUTH_TOKEN", - "你的API密钥", [System.EnvironmentVariableTarget]::User) -
-
-

查看已设置的环境变量：

-
-
# 查看用户级环境变量
-
- [System.Environment]::GetEnvironmentVariable("ANTHROPIC_BASE_URL", - [System.EnvironmentVariableTarget]::User) -
-
- [System.Environment]::GetEnvironmentVariable("ANTHROPIC_AUTH_TOKEN", - [System.EnvironmentVariableTarget]::User) -
-
-

- 💡 设置后需要重新打开 PowerShell 窗口才能生效。 -

-
-
-
- - -
-
- VSCode Claude 插件配置 -
-

- 如果使用 VSCode 的 Claude 插件，需要在配置文件中进行设置： -

-
-

- 配置文件位置： - C:\Users\你的用户名\.claude\config.json -

-

- 💡 如果该文件不存在，请手动创建。 -

-
-
-
{
-
"primaryApiKey": "crs"
-
}
-
-
- - -
-
验证环境变量设置
-

- 设置完环境变量后，可以通过以下命令验证是否设置成功： -

- -
-
-
- 在 PowerShell 中验证： -
-
-
echo $env:ANTHROPIC_BASE_URL
-
echo $env:ANTHROPIC_AUTH_TOKEN
-
-
- -
-
- 在 CMD 中验证： -
-
-
echo %ANTHROPIC_BASE_URL%
-
echo %ANTHROPIC_AUTH_TOKEN%
-
-
-
- -
-

- 预期输出示例： -

-
-
{{ currentBaseUrl }}
-
cr_xxxxxxxxxxxxxxxxxx
-
-

- 💡 如果输出为空或显示变量名本身，说明环境变量设置失败，请重新设置。 -

-
-
- - -
-
- - 配置 Gemini CLI 环境变量 -
-

- 如果你使用 Gemini CLI，需要设置以下环境变量： -

- -
-
-
- PowerShell 设置方法 -
-

- 在 PowerShell 中运行以下命令： -

-
-
- $env:GOOGLE_GEMINI_BASE_URL = "{{ geminiBaseUrl }}" -
-
- $env:GEMINI_API_KEY = "你的API密钥" -
-
- $env:GEMINI_MODEL = "gemini-2.5-pro" -
-
-

- 💡 使用与 Claude Code 相同的 API 密钥即可。 -

-
- -
-
- PowerShell 永久设置（用户级） -
-

- 在 PowerShell 中运行以下命令： -

-
-
# 设置用户级环境变量（永久生效）
-
- [System.Environment]::SetEnvironmentVariable("GOOGLE_GEMINI_BASE_URL", "{{ - geminiBaseUrl - }}", [System.EnvironmentVariableTarget]::User) -
-
- [System.Environment]::SetEnvironmentVariable("GEMINI_API_KEY", "你的API密钥", - [System.EnvironmentVariableTarget]::User) -
-
- [System.Environment]::SetEnvironmentVariable("GEMINI_MODEL", "gemini-2.5-pro", - [System.EnvironmentVariableTarget]::User) -
-
-

- 💡 设置后需要重新打开 PowerShell 窗口才能生效。 -

-
- -
-
- 验证 Gemini CLI 环境变量 -
-

在 PowerShell 中验证：

-
-
echo $env:GOOGLE_GEMINI_BASE_URL
-
echo $env:GEMINI_API_KEY
-
echo $env:GEMINI_MODEL
-
-
-
-
- - -
-
- - 配置 Codex 环境变量 -
-

- 如果你使用支持 OpenAI API 的工具（如 Codex），需要设置以下环境变量： -

- -
-
-
Codex 配置文件
-

- 在 - ~/.codex/config.toml - 文件开头添加以下配置： -

-
-
- {{ line }} -
-
-

- 在 - ~/.codex/auth.json - 文件中配置API密钥： -

-
-
- {{ line }} -
-
-
- -

{{ codexConfigContent.authInstructions.description }}

- - -
- {{ codexConfigContent.authInstructions.title }} -
- - -
-

- {{ codexConfigContent.authInstructions.platform.title }}: -

-
-
- {{ codexConfigContent.authInstructions.platform.command }} -
-
-
- - -
-

- {{ codexConfigContent.authInstructions.persistent.title }}: -

-

- {{ codexConfigContent.authInstructions.persistent.description }} -

-
-
- {{ command }} -
-
-
-
-
-
-
-
- - -
-
- - 配置 Droid CLI -
-

- Droid CLI 使用 - ~/.factory/config.json - 保存自定义模型；在 Windows 中可直接编辑 - C:\Users\你的用户名\.factory\config.json。 -

-
-
- 配置文件示例 -
-

- 将以下内容追加到配置文件中，并替换示例中的域名和 API 密钥： -

-
-
- {{ line }} -
-
-

- 💡 在 Droid CLI 中选择自定义模型即可使用新的 Droid 账号池；确保服务地址可被本地访问。 -

-
-
- -
-

- 4 - 开始使用 Claude Code -

-
-

- 现在你可以开始使用 Claude Code 了！ -

- -
-
-
- 启动 Claude Code -
-
-
claude
-
-
- -
-
- 在特定项目中使用 -
-
-
# 进入你的项目目录
-
cd C:\path\to\your\project
-
# 启动 Claude Code
-
claude
-
-
-
-
-
- - -
-

- - Windows 常见问题解决 -

-
-
- - 安装时提示 "permission denied" 错误 - -
-

这通常是权限问题，尝试以下解决方法：

-
    -
  • 以管理员身份运行 PowerShell
    • -
  • - 或者配置 npm 使用用户目录：npm config set prefix %APPDATA%\npm -
    • -
-
-
- -
- - PowerShell 执行策略错误 - -
-

如果遇到执行策略限制，运行：

-
-
- Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser -
-
-
-
- -
- - 环境变量设置后不生效 - -
-

设置永久环境变量后需要：

-
    -
  • 重新启动 PowerShell 或 CMD
    • -
  • 或者注销并重新登录 Windows
    • -
  • - 验证设置：echo $env:ANTHROPIC_BASE_URL -
    • -
-
-
-
+ + {{ tool.name }} +
- -
- -
-

- 1 - 安装 Node.js 环境 -

-

- Claude Code 需要 Node.js 环境才能运行。 -

- -
-
- - macOS 安装方法 -
-
-

方法一：使用 Homebrew（推荐）

-

- 如果你已经安装了 Homebrew，使用它安装 Node.js 会更方便： -

-
-
# 更新 Homebrew
-
brew update
-
# 安装 Node.js
-
brew install node
-
-
-
-

方法二：官网下载

-
    -
  1. - 访问 - https://nodejs.org/ -
    2. -
  2. 下载适合 macOS 的 LTS 版本
    3. -
  3. - 打开下载的 - .pkg - 文件 -
    4. -
  4. 按照安装程序指引完成安装
    5. -
-
-
-
- macOS 注意事项 -
-
    -
  • - • 如果遇到权限问题，可能需要使用 - sudo -
    • -
  • • 首次运行可能需要在系统偏好设置中允许
    • -
  • • 建议使用 Terminal 或 iTerm2
    • -
-
-
- - -
-
验证安装是否成功
-

- 安装完成后，打开 Terminal，输入以下命令： -

-
-
node --version
-
npm --version
-
-

- 如果显示版本号，说明安装成功了！ -

-
-
- - -
-

- 2 - 安装 Claude Code -

- -
-
- - 安装 Claude Code -
-

- 打开 Terminal，运行以下命令： -

-
-
# 全局安装 Claude Code
-
- npm install -g @anthropic-ai/claude-code -
-
-

- 如果遇到权限问题，可以使用 sudo： -

-
-
- sudo npm install -g @anthropic-ai/claude-code -
-
-
- - -
-
验证 Claude Code 安装
-

- 安装完成后，输入以下命令检查是否安装成功： -

-
-
claude --version
-
-

- 如果显示版本号，恭喜你！Claude Code 已经成功安装了。 -

-
-
- - -
-

- 3 - 设置环境变量 -

- -
-
- - 配置 Claude Code 环境变量 -
-

- 为了让 Claude Code 连接到你的中转服务，需要设置两个环境变量： -

- -
-
-
- 方法一：临时设置（当前会话） -
-

- 在 Terminal 中运行以下命令： -

-
-
- export ANTHROPIC_BASE_URL="{{ currentBaseUrl }}" -
-
- export ANTHROPIC_AUTH_TOKEN="你的API密钥" -
-
-

- 💡 记得将 "你的API密钥" 替换为在上方 "API Keys" 标签页中创建的实际密钥。 -

-
- -
-
- 方法二：永久设置 -
-

- 编辑你的 shell 配置文件（根据你使用的 shell）： -

-
-
# 对于 zsh (默认)
-
- echo 'export ANTHROPIC_BASE_URL="{{ currentBaseUrl }}"' >> ~/.zshrc -
-
- echo 'export ANTHROPIC_AUTH_TOKEN="你的API密钥"' >> ~/.zshrc -
-
source ~/.zshrc
-
-
-
# 对于 bash
-
- echo 'export ANTHROPIC_BASE_URL="{{ currentBaseUrl }}"' >> ~/.bash_profile -
-
- echo 'export ANTHROPIC_AUTH_TOKEN="你的API密钥"' >> ~/.bash_profile -
-
source ~/.bash_profile
-
-
-
-
- - -
-
- VSCode Claude 插件配置 -
-

- 如果使用 VSCode 的 Claude 插件，需要在配置文件中进行设置： -

-
-

- 配置文件位置： - ~/.claude/config.json -

-

- 💡 如果该文件不存在，请手动创建。 -

-
-
-
{
-
"primaryApiKey": "crs"
-
}
-
-
- - -
-
- - 配置 Gemini CLI 环境变量 -
-

- 如果你使用 Gemini CLI，需要设置以下环境变量： -

- -
-
-
- Terminal 设置方法 -
-

- 在 Terminal 中运行以下命令： -

-
-
- export GOOGLE_GEMINI_BASE_URL="{{ geminiBaseUrl }}" -
-
- export GEMINI_API_KEY="你的API密钥" -
-
- export GEMINI_MODEL="gemini-2.5-pro" -
-
-

- 💡 使用与 Claude Code 相同的 API 密钥即可。 -

-
- -
-
- 永久设置方法 -
-

- 添加到你的 shell 配置文件： -

-
-
# 对于 zsh (默认)
-
- echo 'export GOOGLE_GEMINI_BASE_URL="{{ geminiBaseUrl }}"' >> ~/.zshrc -
-
- echo 'export GEMINI_API_KEY="你的API密钥"' >> ~/.zshrc -
-
- echo 'export GEMINI_MODEL="gemini-2.5-pro"' >> ~/.zshrc -
-
source ~/.zshrc
-
-
-
# 对于 bash
-
- echo 'export GOOGLE_GEMINI_BASE_URL="{{ geminiBaseUrl }}"' >> ~/.bash_profile -
-
- echo 'export GEMINI_API_KEY="你的API密钥"' >> ~/.bash_profile -
-
- echo 'export GEMINI_MODEL="gemini-2.5-pro"' >> ~/.bash_profile -
-
source ~/.bash_profile
-
-
- -
-
- 验证 Gemini CLI 环境变量 -
-

在 Terminal 中验证：

-
-
echo $GOOGLE_GEMINI_BASE_URL
-
echo $GEMINI_API_KEY
-
echo $GEMINI_MODEL
-
-
-
-
- - -
-
- - 配置 Codex 环境变量 -
-

- 如果你使用支持 OpenAI API 的工具（如 Codex），需要设置以下环境变量： -

- -
-
-
Codex 配置文件
-

- 在 - ~/.codex/config.toml - 文件开头添加以下配置： -

-
-
- {{ line }} -
-
-

- 在 - ~/.codex/auth.json - 文件中配置API密钥： -

-
-
- {{ line }} -
-
-
- -

{{ codexConfigContent.authInstructions.description }}

- - -
- {{ codexConfigContent.authInstructions.title }} -
- - -
-

- {{ codexConfigContent.authInstructions.platform.title }}: -

-
-
- {{ codexConfigContent.authInstructions.platform.command }} -
-
-
- - -
-

- {{ codexConfigContent.authInstructions.persistent.title }}: -

-

- {{ codexConfigContent.authInstructions.persistent.description }} -

-
-
- {{ command }} -
-
-
-
-
-
-
-
- - -
-
- - 配置 Droid CLI -
-

- Droid CLI 使用 - ~/.factory/config.json - 保存自定义模型；你可以在 Finder 中按 - ⌘ + Shift + G - 并输入路径，或运行 - open ~/.factory - 快速打开配置目录。 -

-
-
- 配置文件示例 -
-

- 将以下内容追加到配置文件中，并替换示例中的域名和 API 密钥： -

-
-
- {{ line }} -
-
-

- 💡 在 Droid CLI 中选择自定义模型即可使用新的 Droid 账号池；确保服务地址可被本地访问。 -

-
-
- - -
-

- 4 - 开始使用 Claude Code -

-
-

- 现在你可以开始使用 Claude Code 了！ -

- -
-
-
- 启动 Claude Code -
-
-
claude
-
-
- -
-
- 在特定项目中使用 -
-
-
# 进入你的项目目录
-
cd /path/to/your/project
-
# 启动 Claude Code
-
claude
-
-
-
-
-
- - -
-

- - macOS 常见问题解决 -

-
-
- - 安装时提示权限错误 - -
-

尝试以下解决方法：

-
    -
  • - 使用 sudo 安装：sudo npm install -g @anthropic-ai/claude-code -
    • -
  • - 或者配置 npm 使用用户目录：npm config set prefix ~/.npm-global -
    • -
-
-
- -
- - macOS 安全设置阻止运行 - -
-

如果系统阻止运行 Claude Code：

-
    -
  • 打开"系统偏好设置" → "安全性与隐私"
    • -
  • 点击"仍要打开"或"允许"
    • -
  • - 或者在 Terminal 中运行：sudo spctl --master-disable -
    • -
-
-
- -
- - 环境变量不生效 - -
-

检查以下几点：

-
    -
  • 确认修改了正确的配置文件（.zshrc 或 .bash_profile）
    • -
  • 重新启动 Terminal
    • -
  • - 验证设置：echo $ANTHROPIC_BASE_URL -
    • -
-
-
-
-
-
- - -
- -
-

- 1 - 安装 Node.js 环境 -

-

- Claude Code 需要 Node.js 环境才能运行。 -

- -
-
- - Linux 安装方法 -
-
-

方法一：使用官方仓库（推荐）

-
-
# 添加 NodeSource 仓库
-
- curl -fsSL https://deb.nodesource.com/setup_lts.x | sudo -E bash - -
-
# 安装 Node.js
-
sudo apt-get install -y nodejs
-
-
-
-

方法二：使用系统包管理器

-

- 虽然版本可能不是最新的，但对于基本使用已经足够： -

-
-
# Ubuntu/Debian
-
sudo apt update
-
sudo apt install nodejs npm
-
# CentOS/RHEL/Fedora
-
sudo dnf install nodejs npm
-
-
-
-
- Linux 注意事项 -
-
    -
  • • 某些发行版可能需要安装额外的依赖
    • -
  • - • 如果遇到权限问题，使用 - sudo -
    • -
  • • 确保你的用户在 npm 的全局目录有写权限
    • -
-
-
- - -
-
验证安装是否成功
-

- 安装完成后，打开终端，输入以下命令： -

-
-
node --version
-
npm --version
-
-

- 如果显示版本号，说明安装成功了！ -

-
-
- - -
-

- 2 - 安装 Claude Code -

- -
-
- - 安装 Claude Code -
-

- 打开终端，运行以下命令： -

-
-
# 全局安装 Claude Code
-
- npm install -g @anthropic-ai/claude-code -
-
-

- 如果遇到权限问题，可以使用 sudo： -

-
-
- sudo npm install -g @anthropic-ai/claude-code -
-
-
- - -
-
验证 Claude Code 安装
-

- 安装完成后，输入以下命令检查是否安装成功： -

-
-
claude --version
-
-

- 如果显示版本号，恭喜你！Claude Code 已经成功安装了。 -

-
-
- - -
-

- 3 - 设置环境变量 -

- -
-
- - 配置 Claude Code 环境变量 -
-

- 为了让 Claude Code 连接到你的中转服务，需要设置两个环境变量： -

- -
-
-
- 方法一：临时设置（当前会话） -
-

在终端中运行以下命令：

-
-
- export ANTHROPIC_BASE_URL="{{ currentBaseUrl }}" -
-
- export ANTHROPIC_AUTH_TOKEN="你的API密钥" -
-
-

- 💡 记得将 "你的API密钥" 替换为在上方 "API Keys" 标签页中创建的实际密钥。 -

-
- -
-
- 方法二：永久设置 -
-

编辑你的 shell 配置文件：

-
-
# 对于 bash (默认)
-
- echo 'export ANTHROPIC_BASE_URL="{{ currentBaseUrl }}"' >> ~/.bashrc -
-
- echo 'export ANTHROPIC_AUTH_TOKEN="你的API密钥"' >> ~/.bashrc -
-
source ~/.bashrc
-
-
-
# 对于 zsh
-
- echo 'export ANTHROPIC_BASE_URL="{{ currentBaseUrl }}"' >> ~/.zshrc -
-
- echo 'export ANTHROPIC_AUTH_TOKEN="你的API密钥"' >> ~/.zshrc -
-
source ~/.zshrc
-
-
-
-
- - -
-
- - 配置 Gemini CLI 环境变量 -
-

- 如果你使用 Gemini CLI，需要设置以下环境变量： -

- -
-
-
- 终端设置方法 -
-

在终端中运行以下命令：

-
-
- export GOOGLE_GEMINI_BASE_URL="{{ geminiBaseUrl }}" -
-
- export GEMINI_API_KEY="你的API密钥" -
-
- export GEMINI_MODEL="gemini-2.5-pro" -
-
-

- 💡 使用与 Claude Code 相同的 API 密钥即可。 -

-
- -
-
- 永久设置方法 -
-

- 添加到你的 shell 配置文件： -

-
-
# 对于 bash (默认)
-
- echo 'export GOOGLE_GEMINI_BASE_URL="{{ geminiBaseUrl }}"' >> ~/.bashrc -
-
- echo 'export GEMINI_API_KEY="你的API密钥"' >> ~/.bashrc -
-
- echo 'export GEMINI_MODEL="gemini-2.5-pro"' >> ~/.bashrc -
-
source ~/.bashrc
-
-
-
# 对于 zsh
-
- echo 'export GOOGLE_GEMINI_BASE_URL="{{ geminiBaseUrl }}"' >> ~/.zshrc -
-
- echo 'export GEMINI_API_KEY="你的API密钥"' >> ~/.zshrc -
-
- echo 'export GEMINI_MODEL="gemini-2.5-pro"' >> ~/.zshrc -
-
source ~/.zshrc
-
-
- -
-
- 验证 Gemini CLI 环境变量 -
-

在终端中验证：

-
-
echo $GOOGLE_GEMINI_BASE_URL
-
echo $GEMINI_API_KEY
-
echo $GEMINI_MODEL
-
-
-
-
- - -
-
- - 配置 Codex 环境变量 -
-

- 如果你使用支持 OpenAI API 的工具（如 Codex），需要设置以下环境变量： -

- -
-
-
Codex 配置文件
-

- 在 - ~/.codex/config.toml - 文件开头添加以下配置： -

-
-
- {{ line }} -
-
-

- 在 - ~/.codex/auth.json - 文件中配置API密钥： -

-
-
- {{ line }} -
-
-
- -

{{ codexConfigContent.authInstructions.description }}

- - -
- {{ codexConfigContent.authInstructions.title }} -
- - -
-

- {{ codexConfigContent.authInstructions.platform.title }}: -

-
-
- {{ codexConfigContent.authInstructions.platform.command }} -
-
-
- - -
-

- {{ codexConfigContent.authInstructions.persistent.title }}: -

-

- {{ codexConfigContent.authInstructions.persistent.description }} -

-
-
- {{ command }} -
-
-
-
-
-
-
- -
-
- VSCode Claude 插件配置 -
-

- 如果使用 VSCode 的 Claude 插件，需要在配置文件中进行设置： -

-
-

- 配置文件位置： - ~/.claude/config.json -

-

- 💡 如果该文件不存在，请手动创建。 -

-
-
-
{
-
"primaryApiKey": "crs"
-
}
-
-
- - -
-
- - 配置 Droid CLI -
-

- Droid CLI 使用 - ~/.factory/config.json - 保存自定义模型；在 Linux 或 WSL2 中，可直接编辑 - /home/你的用户名/.factory/config.json - 或在终端运行 - xdg-open ~/.factory - 打开目录。 -

-
-
- 配置文件示例 -
-

- 将以下内容追加到配置文件中，并替换示例中的域名和 API 密钥： -

-
-
- {{ line }} -
-
-

- 💡 在 Droid CLI 中选择自定义模型即可使用新的 Droid 账号池；确保服务地址可被本地访问。 -

-
-
- - -
-

- 4 - 开始使用 Claude Code -

-
-

- 现在你可以开始使用 Claude Code 了！ -

- -
-
-
- 启动 Claude Code -
-
-
claude
-
-
- -
-
- 在特定项目中使用 -
-
-
# 进入你的项目目录
-
cd /path/to/your/project
-
# 启动 Claude Code
-
claude
-
-
-
-
-
- - -
-

- - Linux 常见问题解决 -

-
-
- - 安装时提示权限错误 - -
-

尝试以下解决方法：

-
    -
  • - 使用 sudo 安装：sudo npm install -g @anthropic-ai/claude-code -
    • -
  • - 或者配置 npm 使用用户目录：npm config set prefix ~/.npm-global -
    • -
  • - 然后添加到 PATH：export PATH=~/.npm-global/bin:$PATH -
    • -
-
-
- -
- - 缺少依赖库 - -
-

某些 Linux 发行版需要安装额外依赖：

-
-
# Ubuntu/Debian
-
- sudo apt install build-essential -
-
# CentOS/RHEL
-
- sudo dnf groupinstall "Development Tools" -
-
-
-
- -
- - 环境变量不生效 - -
-

检查以下几点：

-
    -
  • 确认修改了正确的配置文件（.bashrc 或 .zshrc）
    • -
  • - 重新启动终端或运行 - source ~/.bashrc -
    • -
  • - 验证设置：echo $ANTHROPIC_BASE_URL -
    • -
-
-
-
-
-
- - -
-
🎉 恭喜你！
-

- 你已经成功安装并配置了 Claude Code，现在可以开始享受 AI 编程助手带来的便利了。 -

-

- 如果在使用过程中遇到任何问题，可以查看官方文档或社区讨论获取帮助。 -

-
-
+ +