From 4a1d2b0d941a565ef8514f9abce2218ba5b0f2ff Mon Sep 17 00:00:00 2001 From: Andrei Blizorukov <55080535+ablizorukov@users.noreply.github.com> Date: Mon, 19 Jan 2026 22:32:57 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=93=8A=20fix:=20MeiliSearch=20Sync=20Thre?= =?UTF-8?q?shold=20&=20Document=20Count=20Accuracy=20(#11406)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🔧 fix: meilisearch incorrect count of total documents & performance improvement Temporary documents were counted & removed 2 redundant heavy calls to the database, use known information instead 🔧 fix: respect MEILI_SYNC_THRESHOLD value Do not sync with meili if threshold was not reached * refactor: reformat lint * fix: forces update if meili index settingsUpdated --- api/db/indexSync.js | 39 ++-- api/db/indexSync.spec.js | 465 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 487 insertions(+), 17 deletions(-) create mode 100644 api/db/indexSync.spec.js diff --git a/api/db/indexSync.js b/api/db/indexSync.js index b39f018b3a..8e8e999d92 100644 --- a/api/db/indexSync.js +++ b/api/db/indexSync.js @@ -13,6 +13,11 @@ const searchEnabled = isEnabled(process.env.SEARCH); const indexingDisabled = isEnabled(process.env.MEILI_NO_SYNC); let currentTimeout = null; +const defaultSyncThreshold = 1000; +const syncThreshold = process.env.MEILI_SYNC_THRESHOLD + ? parseInt(process.env.MEILI_SYNC_THRESHOLD, 10) + : defaultSyncThreshold; + class MeiliSearchClient { static instance = null; @@ -221,25 +226,25 @@ async function performSync(flowManager, flowId, flowType) { } // Check if we need to sync messages + logger.info('[indexSync] Requesting message sync progress...'); const messageProgress = await Message.getSyncProgress(); if (!messageProgress.isComplete || settingsUpdated) { logger.info( `[indexSync] Messages need syncing: ${messageProgress.totalProcessed}/${messageProgress.totalDocuments} indexed`, ); - // Check if we should do a full sync or incremental - const messageCount = await Message.countDocuments(); + const messageCount = messageProgress.totalDocuments; const messagesIndexed = messageProgress.totalProcessed; - const syncThreshold = parseInt(process.env.MEILI_SYNC_THRESHOLD || '1000', 10); + const unindexedMessages = messageCount - messagesIndexed; - if (messageCount - messagesIndexed > syncThreshold) { - logger.info('[indexSync] Starting full message sync due to large difference'); - await Message.syncWithMeili(); - messagesSync = true; - } else if (messageCount !== messagesIndexed) { - logger.warn('[indexSync] Messages out of sync, performing incremental sync'); + if (settingsUpdated || unindexedMessages > syncThreshold) { + logger.info(`[indexSync] Starting message sync (${unindexedMessages} unindexed)`); await Message.syncWithMeili(); messagesSync = true; + } else if (unindexedMessages > 0) { + logger.info( + `[indexSync] ${unindexedMessages} messages unindexed (below threshold: ${syncThreshold}, skipping)`, + ); } } else { logger.info( @@ -254,18 +259,18 @@ async function performSync(flowManager, flowId, flowType) { `[indexSync] Conversations need syncing: ${convoProgress.totalProcessed}/${convoProgress.totalDocuments} indexed`, ); - const convoCount = await Conversation.countDocuments(); + const convoCount = convoProgress.totalDocuments; const convosIndexed = convoProgress.totalProcessed; - const syncThreshold = parseInt(process.env.MEILI_SYNC_THRESHOLD || '1000', 10); - if (convoCount - convosIndexed > syncThreshold) { - logger.info('[indexSync] Starting full conversation sync due to large difference'); - await Conversation.syncWithMeili(); - convosSync = true; - } else if (convoCount !== convosIndexed) { - logger.warn('[indexSync] Convos out of sync, performing incremental sync'); + const unindexedConvos = convoCount - convosIndexed; + if (settingsUpdated || unindexedConvos > syncThreshold) { + logger.info(`[indexSync] Starting convos sync (${unindexedConvos} unindexed)`); await Conversation.syncWithMeili(); convosSync = true; + } else if (unindexedConvos > 0) { + logger.info( + `[indexSync] ${unindexedConvos} convos unindexed (below threshold: ${syncThreshold}, skipping)`, + ); } } else { logger.info( diff --git a/api/db/indexSync.spec.js b/api/db/indexSync.spec.js new file mode 100644 index 0000000000..c2e5901d6a --- /dev/null +++ b/api/db/indexSync.spec.js @@ -0,0 +1,465 @@ +/** + * Unit tests for performSync() function in indexSync.js + * + * Tests use real mongoose with mocked model methods, only mocking external calls. + */ + +const mongoose = require('mongoose'); + +// Mock only external dependencies (not internal classes/models) +const mockLogger = { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + debug: jest.fn(), +}; + +const mockMeiliHealth = jest.fn(); +const mockMeiliIndex = jest.fn(); +const mockBatchResetMeiliFlags = jest.fn(); +const mockIsEnabled = jest.fn(); +const mockGetLogStores = jest.fn(); + +// Create mock models that will be reused +const createMockModel = (collectionName) => ({ + collection: { name: collectionName }, + getSyncProgress: jest.fn(), + syncWithMeili: jest.fn(), + countDocuments: jest.fn(), +}); + +const originalMessageModel = mongoose.models.Message; +const originalConversationModel = mongoose.models.Conversation; + +// Mock external modules +jest.mock('@librechat/data-schemas', () => ({ + logger: mockLogger, +})); + +jest.mock('meilisearch', () => ({ + MeiliSearch: jest.fn(() => ({ + health: mockMeiliHealth, + index: mockMeiliIndex, + })), +})); + +jest.mock('./utils', () => ({ + batchResetMeiliFlags: mockBatchResetMeiliFlags, +})); + +jest.mock('@librechat/api', () => ({ + isEnabled: mockIsEnabled, + FlowStateManager: jest.fn(), +})); + +jest.mock('~/cache', () => ({ + getLogStores: mockGetLogStores, +})); + +// Set environment before module load +process.env.MEILI_HOST = 'http://localhost:7700'; +process.env.MEILI_MASTER_KEY = 'test-key'; +process.env.SEARCH = 'true'; +process.env.MEILI_SYNC_THRESHOLD = '1000'; // Set threshold before module loads + +describe('performSync() - syncThreshold logic', () => { + const ORIGINAL_ENV = process.env; + let Message; + let Conversation; + + beforeAll(() => { + Message = createMockModel('messages'); + Conversation = createMockModel('conversations'); + + mongoose.models.Message = Message; + mongoose.models.Conversation = Conversation; + }); + + beforeEach(() => { + // Reset all mocks + jest.clearAllMocks(); + // Reset modules to ensure fresh load of indexSync.js and its top-level consts (like syncThreshold) + jest.resetModules(); + + // Set up environment + process.env = { ...ORIGINAL_ENV }; + process.env.MEILI_HOST = 'http://localhost:7700'; + process.env.MEILI_MASTER_KEY = 'test-key'; + process.env.SEARCH = 'true'; + delete process.env.MEILI_NO_SYNC; + + // Re-ensure models are available in mongoose after resetModules + // We must require mongoose again to get the fresh instance that indexSync will use + const mongoose = require('mongoose'); + mongoose.models.Message = Message; + mongoose.models.Conversation = Conversation; + + // Mock isEnabled + mockIsEnabled.mockImplementation((val) => val === 'true' || val === true); + + // Mock MeiliSearch client responses + mockMeiliHealth.mockResolvedValue({ status: 'available' }); + mockMeiliIndex.mockReturnValue({ + getSettings: jest.fn().mockResolvedValue({ filterableAttributes: ['user'] }), + updateSettings: jest.fn().mockResolvedValue({}), + search: jest.fn().mockResolvedValue({ hits: [] }), + }); + + mockBatchResetMeiliFlags.mockResolvedValue(undefined); + }); + + afterEach(() => { + process.env = ORIGINAL_ENV; + }); + + afterAll(() => { + mongoose.models.Message = originalMessageModel; + mongoose.models.Conversation = originalConversationModel; + }); + + test('triggers sync when unindexed messages exceed syncThreshold', async () => { + // Arrange: Set threshold before module load + process.env.MEILI_SYNC_THRESHOLD = '1000'; + + // Arrange: 1050 unindexed messages > 1000 threshold + Message.getSyncProgress.mockResolvedValue({ + totalProcessed: 100, + totalDocuments: 1150, // 1050 unindexed + isComplete: false, + }); + + Conversation.getSyncProgress.mockResolvedValue({ + totalProcessed: 50, + totalDocuments: 50, + isComplete: true, + }); + + Message.syncWithMeili.mockResolvedValue(undefined); + + // Act + const indexSync = require('./indexSync'); + await indexSync(); + + // Assert: No countDocuments calls + expect(Message.countDocuments).not.toHaveBeenCalled(); + expect(Conversation.countDocuments).not.toHaveBeenCalled(); + + // Assert: Message sync triggered because 1050 > 1000 + expect(Message.syncWithMeili).toHaveBeenCalledTimes(1); + expect(mockLogger.info).toHaveBeenCalledWith( + '[indexSync] Messages need syncing: 100/1150 indexed', + ); + expect(mockLogger.info).toHaveBeenCalledWith( + '[indexSync] Starting message sync (1050 unindexed)', + ); + + // Assert: Conversation sync NOT triggered (already complete) + expect(Conversation.syncWithMeili).not.toHaveBeenCalled(); + }); + + test('skips sync when unindexed messages are below syncThreshold', async () => { + // Arrange: 50 unindexed messages < 1000 threshold + Message.getSyncProgress.mockResolvedValue({ + totalProcessed: 100, + totalDocuments: 150, // 50 unindexed + isComplete: false, + }); + + Conversation.getSyncProgress.mockResolvedValue({ + totalProcessed: 50, + totalDocuments: 50, + isComplete: true, + }); + + process.env.MEILI_SYNC_THRESHOLD = '1000'; + + // Act + const indexSync = require('./indexSync'); + await indexSync(); + + // Assert: No countDocuments calls + expect(Message.countDocuments).not.toHaveBeenCalled(); + expect(Conversation.countDocuments).not.toHaveBeenCalled(); + + // Assert: Message sync NOT triggered because 50 < 1000 + expect(Message.syncWithMeili).not.toHaveBeenCalled(); + expect(mockLogger.info).toHaveBeenCalledWith( + '[indexSync] Messages need syncing: 100/150 indexed', + ); + expect(mockLogger.info).toHaveBeenCalledWith( + '[indexSync] 50 messages unindexed (below threshold: 1000, skipping)', + ); + + // Assert: Conversation sync NOT triggered (already complete) + expect(Conversation.syncWithMeili).not.toHaveBeenCalled(); + }); + + test('respects syncThreshold at boundary (exactly at threshold)', async () => { + // Arrange: 1000 unindexed messages = 1000 threshold (NOT greater than) + Message.getSyncProgress.mockResolvedValue({ + totalProcessed: 100, + totalDocuments: 1100, // 1000 unindexed + isComplete: false, + }); + + Conversation.getSyncProgress.mockResolvedValue({ + totalProcessed: 0, + totalDocuments: 0, + isComplete: true, + }); + + process.env.MEILI_SYNC_THRESHOLD = '1000'; + + // Act + const indexSync = require('./indexSync'); + await indexSync(); + + // Assert: No countDocuments calls + expect(Message.countDocuments).not.toHaveBeenCalled(); + + // Assert: Message sync NOT triggered because 1000 is NOT > 1000 + expect(Message.syncWithMeili).not.toHaveBeenCalled(); + expect(mockLogger.info).toHaveBeenCalledWith( + '[indexSync] Messages need syncing: 100/1100 indexed', + ); + expect(mockLogger.info).toHaveBeenCalledWith( + '[indexSync] 1000 messages unindexed (below threshold: 1000, skipping)', + ); + }); + + test('triggers sync when unindexed is threshold + 1', async () => { + // Arrange: 1001 unindexed messages > 1000 threshold + Message.getSyncProgress.mockResolvedValue({ + totalProcessed: 100, + totalDocuments: 1101, // 1001 unindexed + isComplete: false, + }); + + Conversation.getSyncProgress.mockResolvedValue({ + totalProcessed: 0, + totalDocuments: 0, + isComplete: true, + }); + + Message.syncWithMeili.mockResolvedValue(undefined); + + process.env.MEILI_SYNC_THRESHOLD = '1000'; + + // Act + const indexSync = require('./indexSync'); + await indexSync(); + + // Assert: No countDocuments calls + expect(Message.countDocuments).not.toHaveBeenCalled(); + + // Assert: Message sync triggered because 1001 > 1000 + expect(Message.syncWithMeili).toHaveBeenCalledTimes(1); + expect(mockLogger.info).toHaveBeenCalledWith( + '[indexSync] Messages need syncing: 100/1101 indexed', + ); + expect(mockLogger.info).toHaveBeenCalledWith( + '[indexSync] Starting message sync (1001 unindexed)', + ); + }); + + test('uses totalDocuments from convoProgress for conversation sync decisions', async () => { + // Arrange: Messages complete, conversations need sync + Message.getSyncProgress.mockResolvedValue({ + totalProcessed: 100, + totalDocuments: 100, + isComplete: true, + }); + + Conversation.getSyncProgress.mockResolvedValue({ + totalProcessed: 50, + totalDocuments: 1100, // 1050 unindexed > 1000 threshold + isComplete: false, + }); + + Conversation.syncWithMeili.mockResolvedValue(undefined); + + process.env.MEILI_SYNC_THRESHOLD = '1000'; + + // Act + const indexSync = require('./indexSync'); + await indexSync(); + + // Assert: No countDocuments calls (the optimization) + expect(Message.countDocuments).not.toHaveBeenCalled(); + expect(Conversation.countDocuments).not.toHaveBeenCalled(); + + // Assert: Only conversation sync triggered + expect(Message.syncWithMeili).not.toHaveBeenCalled(); + expect(Conversation.syncWithMeili).toHaveBeenCalledTimes(1); + expect(mockLogger.info).toHaveBeenCalledWith( + '[indexSync] Conversations need syncing: 50/1100 indexed', + ); + expect(mockLogger.info).toHaveBeenCalledWith( + '[indexSync] Starting convos sync (1050 unindexed)', + ); + }); + + test('skips sync when collections are fully synced', async () => { + // Arrange: Everything already synced + Message.getSyncProgress.mockResolvedValue({ + totalProcessed: 100, + totalDocuments: 100, + isComplete: true, + }); + + Conversation.getSyncProgress.mockResolvedValue({ + totalProcessed: 50, + totalDocuments: 50, + isComplete: true, + }); + + // Act + const indexSync = require('./indexSync'); + await indexSync(); + + // Assert: No countDocuments calls + expect(Message.countDocuments).not.toHaveBeenCalled(); + expect(Conversation.countDocuments).not.toHaveBeenCalled(); + + // Assert: No sync triggered + expect(Message.syncWithMeili).not.toHaveBeenCalled(); + expect(Conversation.syncWithMeili).not.toHaveBeenCalled(); + + // Assert: Correct logs + expect(mockLogger.info).toHaveBeenCalledWith('[indexSync] Messages are fully synced: 100/100'); + expect(mockLogger.info).toHaveBeenCalledWith( + '[indexSync] Conversations are fully synced: 50/50', + ); + }); + + test('triggers message sync when settingsUpdated even if below syncThreshold', async () => { + // Arrange: Only 50 unindexed messages (< 1000 threshold), but settings were updated + Message.getSyncProgress.mockResolvedValue({ + totalProcessed: 100, + totalDocuments: 150, // 50 unindexed + isComplete: false, + }); + + Conversation.getSyncProgress.mockResolvedValue({ + totalProcessed: 50, + totalDocuments: 50, + isComplete: true, + }); + + Message.syncWithMeili.mockResolvedValue(undefined); + + // Mock settings update scenario + mockMeiliIndex.mockReturnValue({ + getSettings: jest.fn().mockResolvedValue({ filterableAttributes: [] }), // No user field + updateSettings: jest.fn().mockResolvedValue({}), + search: jest.fn().mockResolvedValue({ hits: [] }), + }); + + process.env.MEILI_SYNC_THRESHOLD = '1000'; + + // Act + const indexSync = require('./indexSync'); + await indexSync(); + + // Assert: Flags were reset due to settings update + expect(mockBatchResetMeiliFlags).toHaveBeenCalledWith(Message.collection); + expect(mockBatchResetMeiliFlags).toHaveBeenCalledWith(Conversation.collection); + + // Assert: Message sync triggered despite being below threshold (50 < 1000) + expect(Message.syncWithMeili).toHaveBeenCalledTimes(1); + expect(mockLogger.info).toHaveBeenCalledWith( + '[indexSync] Settings updated. Forcing full re-sync to reindex with new configuration...', + ); + expect(mockLogger.info).toHaveBeenCalledWith( + '[indexSync] Starting message sync (50 unindexed)', + ); + }); + + test('triggers conversation sync when settingsUpdated even if below syncThreshold', async () => { + // Arrange: Messages complete, conversations have 50 unindexed (< 1000 threshold), but settings were updated + Message.getSyncProgress.mockResolvedValue({ + totalProcessed: 100, + totalDocuments: 100, + isComplete: true, + }); + + Conversation.getSyncProgress.mockResolvedValue({ + totalProcessed: 50, + totalDocuments: 100, // 50 unindexed + isComplete: false, + }); + + Conversation.syncWithMeili.mockResolvedValue(undefined); + + // Mock settings update scenario + mockMeiliIndex.mockReturnValue({ + getSettings: jest.fn().mockResolvedValue({ filterableAttributes: [] }), // No user field + updateSettings: jest.fn().mockResolvedValue({}), + search: jest.fn().mockResolvedValue({ hits: [] }), + }); + + process.env.MEILI_SYNC_THRESHOLD = '1000'; + + // Act + const indexSync = require('./indexSync'); + await indexSync(); + + // Assert: Flags were reset due to settings update + expect(mockBatchResetMeiliFlags).toHaveBeenCalledWith(Message.collection); + expect(mockBatchResetMeiliFlags).toHaveBeenCalledWith(Conversation.collection); + + // Assert: Conversation sync triggered despite being below threshold (50 < 1000) + expect(Conversation.syncWithMeili).toHaveBeenCalledTimes(1); + expect(mockLogger.info).toHaveBeenCalledWith( + '[indexSync] Settings updated. Forcing full re-sync to reindex with new configuration...', + ); + expect(mockLogger.info).toHaveBeenCalledWith('[indexSync] Starting convos sync (50 unindexed)'); + }); + + test('triggers both message and conversation sync when settingsUpdated even if both below syncThreshold', async () => { + // Arrange: Set threshold before module load + process.env.MEILI_SYNC_THRESHOLD = '1000'; + + // Arrange: Both have documents below threshold (50 each), but settings were updated + Message.getSyncProgress.mockResolvedValue({ + totalProcessed: 100, + totalDocuments: 150, // 50 unindexed + isComplete: false, + }); + + Conversation.getSyncProgress.mockResolvedValue({ + totalProcessed: 50, + totalDocuments: 100, // 50 unindexed + isComplete: false, + }); + + Message.syncWithMeili.mockResolvedValue(undefined); + Conversation.syncWithMeili.mockResolvedValue(undefined); + + // Mock settings update scenario + mockMeiliIndex.mockReturnValue({ + getSettings: jest.fn().mockResolvedValue({ filterableAttributes: [] }), // No user field + updateSettings: jest.fn().mockResolvedValue({}), + search: jest.fn().mockResolvedValue({ hits: [] }), + }); + + // Act + const indexSync = require('./indexSync'); + await indexSync(); + + // Assert: Flags were reset due to settings update + expect(mockBatchResetMeiliFlags).toHaveBeenCalledWith(Message.collection); + expect(mockBatchResetMeiliFlags).toHaveBeenCalledWith(Conversation.collection); + + // Assert: Both syncs triggered despite both being below threshold + expect(Message.syncWithMeili).toHaveBeenCalledTimes(1); + expect(Conversation.syncWithMeili).toHaveBeenCalledTimes(1); + expect(mockLogger.info).toHaveBeenCalledWith( + '[indexSync] Settings updated. Forcing full re-sync to reindex with new configuration...', + ); + expect(mockLogger.info).toHaveBeenCalledWith( + '[indexSync] Starting message sync (50 unindexed)', + ); + expect(mockLogger.info).toHaveBeenCalledWith('[indexSync] Starting convos sync (50 unindexed)'); + }); +});