diff --git a/api/cache/cacheConfig.js b/api/cache/cacheConfig.js index 87c403bae..1ca3e902e 100644 --- a/api/cache/cacheConfig.js +++ b/api/cache/cacheConfig.js @@ -44,6 +44,14 @@ const cacheConfig = { REDIS_KEY_PREFIX: process.env[REDIS_KEY_PREFIX_VAR] || REDIS_KEY_PREFIX || '', REDIS_MAX_LISTENERS: math(process.env.REDIS_MAX_LISTENERS, 40), REDIS_PING_INTERVAL: math(process.env.REDIS_PING_INTERVAL, 0), + /** Max delay between reconnection attempts in ms */ + REDIS_RETRY_MAX_DELAY: math(process.env.REDIS_RETRY_MAX_DELAY, 3000), + /** Max number of reconnection attempts (0 = infinite) */ + REDIS_RETRY_MAX_ATTEMPTS: math(process.env.REDIS_RETRY_MAX_ATTEMPTS, 10), + /** Connection timeout in ms */ + REDIS_CONNECT_TIMEOUT: math(process.env.REDIS_CONNECT_TIMEOUT, 10000), + /** Queue commands when disconnected */ + REDIS_ENABLE_OFFLINE_QUEUE: isEnabled(process.env.REDIS_ENABLE_OFFLINE_QUEUE ?? 'true'), CI: isEnabled(process.env.CI), DEBUG_MEMORY_CACHE: isEnabled(process.env.DEBUG_MEMORY_CACHE), diff --git a/api/cache/cacheFactory.js b/api/cache/cacheFactory.js index b9739f4d3..bc361d661 100644 --- a/api/cache/cacheFactory.js +++ b/api/cache/cacheFactory.js @@ -1,12 +1,13 @@ const KeyvRedis = require('@keyv/redis').default; const { Keyv } = require('keyv'); -const { cacheConfig } = require('./cacheConfig'); -const { keyvRedisClient, ioredisClient, GLOBAL_PREFIX_SEPARATOR } = require('./redisClients'); +const { RedisStore } = require('rate-limit-redis'); const { Time } = require('librechat-data-provider'); +const { logger } = require('@librechat/data-schemas'); const { RedisStore: ConnectRedis } = require('connect-redis'); const MemoryStore = require('memorystore')(require('express-session')); +const { keyvRedisClient, ioredisClient, GLOBAL_PREFIX_SEPARATOR } = require('./redisClients'); +const { cacheConfig } = require('./cacheConfig'); const { violationFile } = require('./keyvFiles'); -const { RedisStore } = require('rate-limit-redis'); /** * Creates a cache instance using Redis or a fallback store. Suitable for general caching needs. @@ -20,11 +21,21 @@ const standardCache = (namespace, ttl = undefined, fallbackStore = undefined) => cacheConfig.USE_REDIS && !cacheConfig.FORCED_IN_MEMORY_CACHE_NAMESPACES?.includes(namespace) ) { - const keyvRedis = new KeyvRedis(keyvRedisClient); - const cache = new Keyv(keyvRedis, { namespace, ttl }); - keyvRedis.namespace = cacheConfig.REDIS_KEY_PREFIX; - keyvRedis.keyPrefixSeparator = GLOBAL_PREFIX_SEPARATOR; - return cache; + try { + const keyvRedis = new KeyvRedis(keyvRedisClient); + const cache = new Keyv(keyvRedis, { namespace, ttl }); + keyvRedis.namespace = cacheConfig.REDIS_KEY_PREFIX; + keyvRedis.keyPrefixSeparator = GLOBAL_PREFIX_SEPARATOR; + + cache.on('error', (err) => { + logger.error(`Cache error in namespace ${namespace}:`, err); + }); + + return cache; + } catch (err) { + logger.error(`Failed to create Redis cache for namespace ${namespace}:`, err); + throw err; + } } if (fallbackStore) return new Keyv({ store: fallbackStore, namespace, ttl }); return new Keyv({ namespace, ttl }); @@ -50,7 +61,13 @@ const violationCache = (namespace, ttl = undefined) => { const sessionCache = (namespace, ttl = undefined) => { namespace = namespace.endsWith(':') ? namespace : `${namespace}:`; if (!cacheConfig.USE_REDIS) return new MemoryStore({ ttl, checkPeriod: Time.ONE_DAY }); - return new ConnectRedis({ client: ioredisClient, ttl, prefix: namespace }); + const store = new ConnectRedis({ client: ioredisClient, ttl, prefix: namespace }); + if (ioredisClient) { + ioredisClient.on('error', (err) => { + logger.error(`Session store Redis error for namespace ${namespace}:`, err); + }); + } + return store; }; /** @@ -62,8 +79,30 @@ const limiterCache = (prefix) => { if (!prefix) throw new Error('prefix is required'); if (!cacheConfig.USE_REDIS) return undefined; prefix = prefix.endsWith(':') ? prefix : `${prefix}:`; - return new RedisStore({ sendCommand, prefix }); + + try { + if (!ioredisClient) { + logger.warn(`Redis client not available for rate limiter with prefix ${prefix}`); + return undefined; + } + + return new RedisStore({ sendCommand, prefix }); + } catch (err) { + logger.error(`Failed to create Redis rate limiter for prefix ${prefix}:`, err); + return undefined; + } +}; + +const sendCommand = (...args) => { + if (!ioredisClient) { + logger.warn('Redis client not available for command execution'); + return Promise.reject(new Error('Redis client not available')); + } + + return ioredisClient.call(...args).catch((err) => { + logger.error('Redis command execution failed:', err); + throw err; + }); }; -const sendCommand = (...args) => ioredisClient?.call(...args); module.exports = { standardCache, sessionCache, violationCache, limiterCache }; diff --git a/api/cache/cacheFactory.spec.js b/api/cache/cacheFactory.spec.js index 76d01a915..ce364a4a3 100644 --- a/api/cache/cacheFactory.spec.js +++ b/api/cache/cacheFactory.spec.js @@ -6,13 +6,17 @@ const mockKeyvRedis = { keyPrefixSeparator: '', }; -const mockKeyv = jest.fn().mockReturnValue({ mock: 'keyv' }); +const mockKeyv = jest.fn().mockReturnValue({ + mock: 'keyv', + on: jest.fn(), +}); const mockConnectRedis = jest.fn().mockReturnValue({ mock: 'connectRedis' }); const mockMemoryStore = jest.fn().mockReturnValue({ mock: 'memoryStore' }); const mockRedisStore = jest.fn().mockReturnValue({ mock: 'redisStore' }); const mockIoredisClient = { call: jest.fn(), + on: jest.fn(), }; const mockKeyvRedisClient = {}; @@ -53,6 +57,14 @@ jest.mock('rate-limit-redis', () => ({ RedisStore: mockRedisStore, })); +jest.mock('@librechat/data-schemas', () => ({ + logger: { + error: jest.fn(), + warn: jest.fn(), + info: jest.fn(), + }, +})); + // Import after mocking const { standardCache, sessionCache, violationCache, limiterCache } = require('./cacheFactory'); const { cacheConfig } = require('./cacheConfig'); @@ -142,6 +154,28 @@ describe('cacheFactory', () => { expect(require('@keyv/redis').default).toHaveBeenCalledWith(mockKeyvRedisClient); expect(mockKeyv).toHaveBeenCalledWith(mockKeyvRedis, { namespace, ttl }); }); + + it('should throw error when Redis cache creation fails', () => { + cacheConfig.USE_REDIS = true; + const namespace = 'test-namespace'; + const ttl = 3600; + const testError = new Error('Redis connection failed'); + + const KeyvRedis = require('@keyv/redis').default; + KeyvRedis.mockImplementationOnce(() => { + throw testError; + }); + + expect(() => standardCache(namespace, ttl)).toThrow('Redis connection failed'); + + const { logger } = require('@librechat/data-schemas'); + expect(logger.error).toHaveBeenCalledWith( + `Failed to create Redis cache for namespace ${namespace}:`, + testError, + ); + + expect(mockKeyv).not.toHaveBeenCalled(); + }); }); describe('violationCache', () => { @@ -233,6 +267,86 @@ describe('cacheFactory', () => { checkPeriod: Time.ONE_DAY, }); }); + + it('should throw error when ConnectRedis constructor fails', () => { + cacheConfig.USE_REDIS = true; + const namespace = 'sessions'; + const ttl = 86400; + + // Mock ConnectRedis to throw an error during construction + const redisError = new Error('Redis connection failed'); + mockConnectRedis.mockImplementationOnce(() => { + throw redisError; + }); + + // The error should propagate up, not be caught + expect(() => sessionCache(namespace, ttl)).toThrow('Redis connection failed'); + + // Verify that MemoryStore was NOT used as fallback + expect(mockMemoryStore).not.toHaveBeenCalled(); + }); + + it('should register error handler but let errors propagate to Express', () => { + cacheConfig.USE_REDIS = true; + const namespace = 'sessions'; + + // Create a mock session store with middleware methods + const mockSessionStore = { + get: jest.fn(), + set: jest.fn(), + destroy: jest.fn(), + }; + mockConnectRedis.mockReturnValue(mockSessionStore); + + const store = sessionCache(namespace); + + // Verify error handler was registered + expect(mockIoredisClient.on).toHaveBeenCalledWith('error', expect.any(Function)); + + // Get the error handler + const errorHandler = mockIoredisClient.on.mock.calls.find((call) => call[0] === 'error')[1]; + + // Simulate an error from Redis during a session operation + const redisError = new Error('Socket closed unexpectedly'); + + // The error handler should log but not swallow the error + const { logger } = require('@librechat/data-schemas'); + errorHandler(redisError); + + expect(logger.error).toHaveBeenCalledWith( + `Session store Redis error for namespace ${namespace}::`, + redisError, + ); + + // Now simulate what happens when session middleware tries to use the store + const callback = jest.fn(); + mockSessionStore.get.mockImplementation((sid, cb) => { + cb(new Error('Redis connection lost')); + }); + + // Call the store's get method (as Express session would) + store.get('test-session-id', callback); + + // The error should be passed to the callback, not swallowed + expect(callback).toHaveBeenCalledWith(new Error('Redis connection lost')); + }); + + it('should handle null ioredisClient gracefully', () => { + cacheConfig.USE_REDIS = true; + const namespace = 'sessions'; + + // Temporarily set ioredisClient to null (simulating connection not established) + const originalClient = require('./redisClients').ioredisClient; + require('./redisClients').ioredisClient = null; + + // ConnectRedis might accept null client but would fail on first use + // The important thing is it doesn't throw uncaught exceptions during construction + const store = sessionCache(namespace); + expect(store).toBeDefined(); + + // Restore original client + require('./redisClients').ioredisClient = originalClient; + }); }); describe('limiterCache', () => { @@ -274,8 +388,10 @@ describe('cacheFactory', () => { }); }); - it('should pass sendCommand function that calls ioredisClient.call', () => { + it('should pass sendCommand function that calls ioredisClient.call', async () => { cacheConfig.USE_REDIS = true; + mockIoredisClient.call.mockResolvedValue('test-value'); + limiterCache('rate-limit'); const sendCommandCall = mockRedisStore.mock.calls[0][0]; @@ -283,9 +399,29 @@ describe('cacheFactory', () => { // Test that sendCommand properly delegates to ioredisClient.call const args = ['GET', 'test-key']; - sendCommand(...args); + const result = await sendCommand(...args); expect(mockIoredisClient.call).toHaveBeenCalledWith(...args); + expect(result).toBe('test-value'); + }); + + it('should handle sendCommand errors properly', async () => { + cacheConfig.USE_REDIS = true; + + // Mock the call method to reject with an error + const testError = new Error('Redis error'); + mockIoredisClient.call.mockRejectedValue(testError); + + limiterCache('rate-limit'); + + const sendCommandCall = mockRedisStore.mock.calls[0][0]; + const sendCommand = sendCommandCall.sendCommand; + + // Test that sendCommand properly handles errors + const args = ['GET', 'test-key']; + + await expect(sendCommand(...args)).rejects.toThrow('Redis error'); + expect(mockIoredisClient.call).toHaveBeenCalledWith(...args); }); it('should handle undefined prefix', () => { diff --git a/api/cache/redisClients.js b/api/cache/redisClients.js index 46c2813e9..17889d24f 100644 --- a/api/cache/redisClients.js +++ b/api/cache/redisClients.js @@ -13,23 +13,82 @@ const ca = cacheConfig.REDIS_CA; /** @type {import('ioredis').Redis | import('ioredis').Cluster | null} */ let ioredisClient = null; if (cacheConfig.USE_REDIS) { + /** @type {import('ioredis').RedisOptions | import('ioredis').ClusterOptions} */ const redisOptions = { username: username, password: password, tls: ca ? { ca } : undefined, keyPrefix: `${cacheConfig.REDIS_KEY_PREFIX}${GLOBAL_PREFIX_SEPARATOR}`, maxListeners: cacheConfig.REDIS_MAX_LISTENERS, + retryStrategy: (times) => { + if ( + cacheConfig.REDIS_RETRY_MAX_ATTEMPTS > 0 && + times > cacheConfig.REDIS_RETRY_MAX_ATTEMPTS + ) { + logger.error( + `ioredis giving up after ${cacheConfig.REDIS_RETRY_MAX_ATTEMPTS} reconnection attempts`, + ); + return null; + } + const delay = Math.min(times * 50, cacheConfig.REDIS_RETRY_MAX_DELAY); + logger.info(`ioredis reconnecting... attempt ${times}, delay ${delay}ms`); + return delay; + }, + reconnectOnError: (err) => { + const targetError = 'READONLY'; + if (err.message.includes(targetError)) { + logger.warn('ioredis reconnecting due to READONLY error'); + return true; + } + return false; + }, + enableOfflineQueue: cacheConfig.REDIS_ENABLE_OFFLINE_QUEUE, + connectTimeout: cacheConfig.REDIS_CONNECT_TIMEOUT, + maxRetriesPerRequest: 3, }; ioredisClient = urls.length === 1 ? new IoRedis(cacheConfig.REDIS_URI, redisOptions) - : new IoRedis.Cluster(cacheConfig.REDIS_URI, { redisOptions }); + : new IoRedis.Cluster(cacheConfig.REDIS_URI, { + redisOptions, + clusterRetryStrategy: (times) => { + if ( + cacheConfig.REDIS_RETRY_MAX_ATTEMPTS > 0 && + times > cacheConfig.REDIS_RETRY_MAX_ATTEMPTS + ) { + logger.error( + `ioredis cluster giving up after ${cacheConfig.REDIS_RETRY_MAX_ATTEMPTS} reconnection attempts`, + ); + return null; + } + const delay = Math.min(times * 100, cacheConfig.REDIS_RETRY_MAX_DELAY); + logger.info(`ioredis cluster reconnecting... attempt ${times}, delay ${delay}ms`); + return delay; + }, + enableOfflineQueue: cacheConfig.REDIS_ENABLE_OFFLINE_QUEUE, + }); ioredisClient.on('error', (err) => { logger.error('ioredis client error:', err); }); + ioredisClient.on('connect', () => { + logger.info('ioredis client connected'); + }); + + ioredisClient.on('ready', () => { + logger.info('ioredis client ready'); + }); + + ioredisClient.on('reconnecting', (delay) => { + logger.info(`ioredis client reconnecting in ${delay}ms`); + }); + + ioredisClient.on('close', () => { + logger.warn('ioredis client connection closed'); + }); + /** Ping Interval to keep the Redis server connection alive (if enabled) */ let pingInterval = null; const clearPingInterval = () => { @@ -42,7 +101,9 @@ if (cacheConfig.USE_REDIS) { if (cacheConfig.REDIS_PING_INTERVAL > 0) { pingInterval = setInterval(() => { if (ioredisClient && ioredisClient.status === 'ready') { - ioredisClient.ping(); + ioredisClient.ping().catch((err) => { + logger.error('ioredis ping failed:', err); + }); } }, cacheConfig.REDIS_PING_INTERVAL * 1000); ioredisClient.on('close', clearPingInterval); @@ -56,8 +117,32 @@ if (cacheConfig.USE_REDIS) { /** * ** WARNING ** Keyv Redis client does not support Prefix like ioredis above. * The prefix feature will be handled by the Keyv-Redis store in cacheFactory.js + * @type {import('@keyv/redis').RedisClientOptions | import('@keyv/redis').RedisClusterOptions} */ - const redisOptions = { username, password, socket: { tls: ca != null, ca } }; + const redisOptions = { + username, + password, + socket: { + tls: ca != null, + ca, + connectTimeout: cacheConfig.REDIS_CONNECT_TIMEOUT, + reconnectStrategy: (retries) => { + if ( + cacheConfig.REDIS_RETRY_MAX_ATTEMPTS > 0 && + retries > cacheConfig.REDIS_RETRY_MAX_ATTEMPTS + ) { + logger.error( + `@keyv/redis client giving up after ${cacheConfig.REDIS_RETRY_MAX_ATTEMPTS} reconnection attempts`, + ); + return new Error('Max reconnection attempts reached'); + } + const delay = Math.min(retries * 100, cacheConfig.REDIS_RETRY_MAX_DELAY); + logger.info(`@keyv/redis reconnecting... attempt ${retries}, delay ${delay}ms`); + return delay; + }, + }, + disableOfflineQueue: !cacheConfig.REDIS_ENABLE_OFFLINE_QUEUE, + }; keyvRedisClient = urls.length === 1 @@ -73,6 +158,27 @@ if (cacheConfig.USE_REDIS) { logger.error('@keyv/redis client error:', err); }); + keyvRedisClient.on('connect', () => { + logger.info('@keyv/redis client connected'); + }); + + keyvRedisClient.on('ready', () => { + logger.info('@keyv/redis client ready'); + }); + + keyvRedisClient.on('reconnecting', () => { + logger.info('@keyv/redis client reconnecting...'); + }); + + keyvRedisClient.on('disconnect', () => { + logger.warn('@keyv/redis client disconnected'); + }); + + keyvRedisClient.connect().catch((err) => { + logger.error('@keyv/redis initial connection failed:', err); + throw err; + }); + /** Ping Interval to keep the Redis server connection alive (if enabled) */ let pingInterval = null; const clearPingInterval = () => { @@ -85,7 +191,9 @@ if (cacheConfig.USE_REDIS) { if (cacheConfig.REDIS_PING_INTERVAL > 0) { pingInterval = setInterval(() => { if (keyvRedisClient && keyvRedisClient.isReady) { - keyvRedisClient.ping(); + keyvRedisClient.ping().catch((err) => { + logger.error('@keyv/redis ping failed:', err); + }); } }, cacheConfig.REDIS_PING_INTERVAL * 1000); keyvRedisClient.on('disconnect', clearPingInterval); diff --git a/redis-config/README.md b/redis-config/README.md index 024d0b168..607075569 100644 --- a/redis-config/README.md +++ b/redis-config/README.md @@ -174,6 +174,15 @@ REDIS_KEY_PREFIX=librechat # Connection limits REDIS_MAX_LISTENERS=40 + +# Ping interval to keep connection alive (seconds, 0 to disable) +REDIS_PING_INTERVAL=0 + +# Reconnection configuration +REDIS_RETRY_MAX_DELAY=3000 # Max delay between reconnection attempts (ms) +REDIS_RETRY_MAX_ATTEMPTS=10 # Max reconnection attempts (0 = infinite) +REDIS_CONNECT_TIMEOUT=10000 # Connection timeout (ms) +REDIS_ENABLE_OFFLINE_QUEUE=true # Queue commands when disconnected ``` ## TLS/SSL Redis Setup