| |
| |
| |
| |
|
|
| const logger = require('../utils/logger') |
| const openaiAccountService = require('./openaiAccountService') |
| const claudeAccountService = require('./claudeAccountService') |
| const claudeConsoleAccountService = require('./claudeConsoleAccountService') |
| const unifiedOpenAIScheduler = require('./unifiedOpenAIScheduler') |
| const webhookService = require('./webhookService') |
|
|
| class RateLimitCleanupService { |
| constructor() { |
| this.cleanupInterval = null |
| this.isRunning = false |
| |
| this.intervalMs = 5 * 60 * 1000 |
| |
| this.clearedAccounts = [] |
| } |
|
|
| |
| |
| |
| |
| start(intervalMinutes = 5) { |
| if (this.cleanupInterval) { |
| logger.warn('⚠️ Rate limit cleanup service is already running') |
| return |
| } |
|
|
| this.intervalMs = intervalMinutes * 60 * 1000 |
|
|
| logger.info(`🧹 Starting rate limit cleanup service (interval: ${intervalMinutes} minutes)`) |
|
|
| |
| this.performCleanup() |
|
|
| |
| this.cleanupInterval = setInterval(() => { |
| this.performCleanup() |
| }, this.intervalMs) |
| } |
|
|
| |
| |
| |
| stop() { |
| if (this.cleanupInterval) { |
| clearInterval(this.cleanupInterval) |
| this.cleanupInterval = null |
| logger.info('🛑 Rate limit cleanup service stopped') |
| } |
| } |
|
|
| |
| |
| |
| async performCleanup() { |
| if (this.isRunning) { |
| logger.debug('⏭️ Cleanup already in progress, skipping this cycle') |
| return |
| } |
|
|
| this.isRunning = true |
| const startTime = Date.now() |
|
|
| try { |
| logger.debug('🔍 Starting rate limit cleanup check...') |
|
|
| const results = { |
| openai: { checked: 0, cleared: 0, errors: [] }, |
| claude: { checked: 0, cleared: 0, errors: [] }, |
| claudeConsole: { checked: 0, cleared: 0, errors: [] } |
| } |
|
|
| |
| await this.cleanupOpenAIAccounts(results.openai) |
|
|
| |
| await this.cleanupClaudeAccounts(results.claude) |
|
|
| |
| await this.cleanupClaudeConsoleAccounts(results.claudeConsole) |
|
|
| const totalChecked = |
| results.openai.checked + results.claude.checked + results.claudeConsole.checked |
| const totalCleared = |
| results.openai.cleared + results.claude.cleared + results.claudeConsole.cleared |
| const duration = Date.now() - startTime |
|
|
| if (totalCleared > 0) { |
| logger.info( |
| `✅ Rate limit cleanup completed: ${totalCleared} accounts cleared out of ${totalChecked} checked (${duration}ms)` |
| ) |
| logger.info(` OpenAI: ${results.openai.cleared}/${results.openai.checked}`) |
| logger.info(` Claude: ${results.claude.cleared}/${results.claude.checked}`) |
| logger.info( |
| ` Claude Console: ${results.claudeConsole.cleared}/${results.claudeConsole.checked}` |
| ) |
|
|
| |
| if (this.clearedAccounts.length > 0) { |
| await this.sendRecoveryNotifications() |
| } |
| } else { |
| logger.debug( |
| `🔍 Rate limit cleanup check completed: no expired limits found (${duration}ms)` |
| ) |
| } |
|
|
| |
| const allErrors = [ |
| ...results.openai.errors, |
| ...results.claude.errors, |
| ...results.claudeConsole.errors |
| ] |
| if (allErrors.length > 0) { |
| logger.warn(`⚠️ Encountered ${allErrors.length} errors during cleanup:`, allErrors) |
| } |
| } catch (error) { |
| logger.error('❌ Rate limit cleanup failed:', error) |
| } finally { |
| |
| this.clearedAccounts = [] |
| this.isRunning = false |
| } |
| } |
|
|
| |
| |
| |
| async cleanupOpenAIAccounts(result) { |
| try { |
| |
| const accounts = await openaiAccountService.getAllAccounts() |
|
|
| for (const account of accounts) { |
| const { rateLimitStatus } = account |
| const isRateLimited = |
| rateLimitStatus === 'limited' || |
| (rateLimitStatus && |
| typeof rateLimitStatus === 'object' && |
| (rateLimitStatus.status === 'limited' || rateLimitStatus.isRateLimited === true)) |
|
|
| if (isRateLimited) { |
| result.checked++ |
|
|
| try { |
| |
| const isStillLimited = await unifiedOpenAIScheduler.isAccountRateLimited(account.id) |
|
|
| if (!isStillLimited) { |
| result.cleared++ |
| logger.info( |
| `🧹 Auto-cleared expired rate limit for OpenAI account: ${account.name} (${account.id})` |
| ) |
|
|
| |
| this.clearedAccounts.push({ |
| platform: 'OpenAI', |
| accountId: account.id, |
| accountName: account.name, |
| previousStatus: 'rate_limited', |
| currentStatus: 'active' |
| }) |
| } |
| } catch (error) { |
| result.errors.push({ |
| accountId: account.id, |
| accountName: account.name, |
| error: error.message |
| }) |
| } |
| } |
| } |
| } catch (error) { |
| logger.error('Failed to cleanup OpenAI accounts:', error) |
| result.errors.push({ error: error.message }) |
| } |
| } |
|
|
| |
| |
| |
| async cleanupClaudeAccounts(result) { |
| try { |
| |
| const redis = require('../models/redis') |
| const accounts = await redis.getAllClaudeAccounts() |
|
|
| for (const account of accounts) { |
| |
| const isRateLimited = |
| account.rateLimitStatus === 'limited' || |
| (account.rateLimitStatus && |
| typeof account.rateLimitStatus === 'object' && |
| account.rateLimitStatus.status === 'limited') |
|
|
| const autoStopped = account.rateLimitAutoStopped === 'true' |
| const needsAutoStopRecovery = |
| autoStopped && (account.rateLimitEndAt || account.schedulable === 'false') |
|
|
| |
| if (isRateLimited || account.rateLimitedAt || needsAutoStopRecovery) { |
| result.checked++ |
|
|
| try { |
| |
| const isStillLimited = await claudeAccountService.isAccountRateLimited(account.id) |
|
|
| if (!isStillLimited) { |
| if (!isRateLimited && autoStopped) { |
| await claudeAccountService.removeAccountRateLimit(account.id) |
| } |
| result.cleared++ |
| logger.info( |
| `🧹 Auto-cleared expired rate limit for Claude account: ${account.name} (${account.id})` |
| ) |
|
|
| |
| this.clearedAccounts.push({ |
| platform: 'Claude', |
| accountId: account.id, |
| accountName: account.name, |
| previousStatus: 'rate_limited', |
| currentStatus: 'active' |
| }) |
| } |
| } catch (error) { |
| result.errors.push({ |
| accountId: account.id, |
| accountName: account.name, |
| error: error.message |
| }) |
| } |
| } |
| } |
|
|
| |
| try { |
| const fiveHourResult = await claudeAccountService.checkAndRecoverFiveHourStoppedAccounts() |
|
|
| if (fiveHourResult.recovered > 0) { |
| |
| for (const account of fiveHourResult.accounts) { |
| this.clearedAccounts.push({ |
| platform: 'Claude', |
| accountId: account.id, |
| accountName: account.name, |
| previousStatus: '5hour_limited', |
| currentStatus: 'active', |
| windowInfo: account.newWindow |
| }) |
| } |
|
|
| |
| result.checked += fiveHourResult.checked |
| result.cleared += fiveHourResult.recovered |
|
|
| logger.info( |
| `🕐 Claude 5-hour limit recovery: ${fiveHourResult.recovered}/${fiveHourResult.checked} accounts recovered` |
| ) |
| } |
| } catch (error) { |
| logger.error('Failed to check and recover 5-hour stopped Claude accounts:', error) |
| result.errors.push({ |
| type: '5hour_recovery', |
| error: error.message |
| }) |
| } |
| } catch (error) { |
| logger.error('Failed to cleanup Claude accounts:', error) |
| result.errors.push({ error: error.message }) |
| } |
| } |
|
|
| |
| |
| |
| async cleanupClaudeConsoleAccounts(result) { |
| try { |
| |
| const accounts = await claudeConsoleAccountService.getAllAccounts() |
|
|
| for (const account of accounts) { |
| |
| const isRateLimited = |
| account.rateLimitStatus === 'limited' || |
| (account.rateLimitStatus && |
| typeof account.rateLimitStatus === 'object' && |
| account.rateLimitStatus.status === 'limited') |
|
|
| const autoStopped = account.rateLimitAutoStopped === 'true' |
| const needsAutoStopRecovery = autoStopped && account.schedulable === 'false' |
|
|
| |
| const hasStatusRateLimited = account.status === 'rate_limited' |
|
|
| if (isRateLimited || hasStatusRateLimited || needsAutoStopRecovery) { |
| result.checked++ |
|
|
| try { |
| |
| const isStillLimited = await claudeConsoleAccountService.isAccountRateLimited( |
| account.id |
| ) |
|
|
| if (!isStillLimited) { |
| if (!isRateLimited && autoStopped) { |
| await claudeConsoleAccountService.removeAccountRateLimit(account.id) |
| } |
| result.cleared++ |
|
|
| |
| if (hasStatusRateLimited && !isRateLimited) { |
| await claudeConsoleAccountService.updateAccount(account.id, { |
| status: 'active' |
| }) |
| } |
|
|
| logger.info( |
| `🧹 Auto-cleared expired rate limit for Claude Console account: ${account.name} (${account.id})` |
| ) |
|
|
| |
| this.clearedAccounts.push({ |
| platform: 'Claude Console', |
| accountId: account.id, |
| accountName: account.name, |
| previousStatus: 'rate_limited', |
| currentStatus: 'active' |
| }) |
| } |
| } catch (error) { |
| result.errors.push({ |
| accountId: account.id, |
| accountName: account.name, |
| error: error.message |
| }) |
| } |
| } |
| } |
| } catch (error) { |
| logger.error('Failed to cleanup Claude Console accounts:', error) |
| result.errors.push({ error: error.message }) |
| } |
| } |
|
|
| |
| |
| |
| async manualCleanup() { |
| logger.info('🧹 Manual rate limit cleanup triggered') |
| await this.performCleanup() |
| } |
|
|
| |
| |
| |
| async sendRecoveryNotifications() { |
| try { |
| |
| const groupedAccounts = {} |
| for (const account of this.clearedAccounts) { |
| if (!groupedAccounts[account.platform]) { |
| groupedAccounts[account.platform] = [] |
| } |
| groupedAccounts[account.platform].push(account) |
| } |
|
|
| |
| const platforms = Object.keys(groupedAccounts) |
| const totalAccounts = this.clearedAccounts.length |
|
|
| let message = `🎉 共有 ${totalAccounts} 个账户的限流状态已恢复\n\n` |
|
|
| for (const platform of platforms) { |
| const accounts = groupedAccounts[platform] |
| message += `**${platform}** (${accounts.length} 个):\n` |
| for (const account of accounts) { |
| message += `• ${account.accountName} (ID: ${account.accountId})\n` |
| } |
| message += '\n' |
| } |
|
|
| |
| await webhookService.sendNotification('rateLimitRecovery', { |
| title: '限流恢复通知', |
| message, |
| totalAccounts, |
| platforms: Object.keys(groupedAccounts), |
| accounts: this.clearedAccounts, |
| timestamp: new Date().toISOString() |
| }) |
|
|
| logger.info(`📢 已发送限流恢复通知,涉及 ${totalAccounts} 个账户`) |
| } catch (error) { |
| logger.error('❌ 发送限流恢复通知失败:', error) |
| } |
| } |
|
|
| |
| |
| |
| getStatus() { |
| return { |
| running: !!this.cleanupInterval, |
| intervalMinutes: this.intervalMs / (60 * 1000), |
| isProcessing: this.isRunning |
| } |
| } |
| } |
|
|
| |
| const rateLimitCleanupService = new RateLimitCleanupService() |
|
|
| module.exports = rateLimitCleanupService |
|
|