🫙 fix: Force MeiliSearch Full Sync on Empty Index State (#12202)

* fix: meili index sync with unindexed documents

- Updated `performSync` function to force a full sync when a fresh MeiliSearch index is detected, even if the number of unindexed messages or convos is below the sync threshold.
- Added logging to indicate when a fresh index is detected and a full sync is initiated.
- Introduced new tests to validate the behavior of the sync logic under various conditions, ensuring proper handling of fresh indexes and threshold scenarios.

This change improves the reliability of the synchronization process, ensuring that all documents are indexed correctly when starting with a fresh index.

* refactor: update sync logic for unindexed documents in MeiliSearch

- Renamed variables in `performSync` to improve clarity, changing `freshIndex` to `noneIndexed` for better understanding of the sync condition.
- Adjusted the logic to ensure a full sync is forced when no messages or conversations are marked as indexed, even if below the sync threshold.
- Updated related tests to reflect the new logging messages and conditions, enhancing the accuracy of the sync threshold logic.

This change improves the readability and reliability of the synchronization process, ensuring all documents are indexed correctly when starting with a fresh index.

* fix: enhance MeiliSearch index creation error handling

- Updated the `mongoMeili` function to improve logging and error handling during index creation in MeiliSearch.
- Added handling for `MeiliSearchTimeOutError` to log a warning when index creation times out.
- Enhanced logging to differentiate between successful index creation and specific failure reasons, including cases where the index already exists.
- Improved debug logging for index creation tasks to provide clearer insights into the process.

This change enhances the robustness of the index creation process and improves observability for troubleshooting.

* fix: update MeiliSearch index creation error handling

- Modified the `mongoMeili` function to check for any status other than 'succeeded' during index creation, enhancing error detection.
- Improved logging to provide clearer insights when an index creation task fails, particularly for cases where the index already exists.

This change strengthens the error handling mechanism for index creation in MeiliSearch, ensuring better observability and reliability.

api/db/indexSync.js

@@ -236,8 +236,12 @@ async function performSync(flowManager, flowId, flowType) {
       const messageCount = messageProgress.totalDocuments;
       const messagesIndexed = messageProgress.totalProcessed;
       const unindexedMessages = messageCount - messagesIndexed;
+      const noneIndexed = messagesIndexed === 0 && unindexedMessages > 0;
 
-      if (settingsUpdated || unindexedMessages > syncThreshold) {
+      if (settingsUpdated || noneIndexed || unindexedMessages > syncThreshold) {
+        if (noneIndexed && !settingsUpdated) {
+          logger.info('[indexSync] No messages marked as indexed, forcing full sync');
+        }
         logger.info(`[indexSync] Starting message sync (${unindexedMessages} unindexed)`);
         await Message.syncWithMeili();
         messagesSync = true;
@@ -261,9 +265,13 @@ async function performSync(flowManager, flowId, flowType) {
 
       const convoCount = convoProgress.totalDocuments;
       const convosIndexed = convoProgress.totalProcessed;
-
       const unindexedConvos = convoCount - convosIndexed;
-      if (settingsUpdated || unindexedConvos > syncThreshold) {
+      const noneConvosIndexed = convosIndexed === 0 && unindexedConvos > 0;
+
+      if (settingsUpdated || noneConvosIndexed || unindexedConvos > syncThreshold) {
+        if (noneConvosIndexed && !settingsUpdated) {
+          logger.info('[indexSync] No conversations marked as indexed, forcing full sync');
+        }
         logger.info(`[indexSync] Starting convos sync (${unindexedConvos} unindexed)`);
         await Conversation.syncWithMeili();
         convosSync = true;

api/db/indexSync.spec.js

@@ -462,4 +462,69 @@ describe('performSync() - syncThreshold logic', () => {
     );
     expect(mockLogger.info).toHaveBeenCalledWith('[indexSync] Starting convos sync (50 unindexed)');
   });
+
+  test('forces sync when zero documents indexed (reset scenario) even if below threshold', async () => {
+    Message.getSyncProgress.mockResolvedValue({
+      totalProcessed: 0,
+      totalDocuments: 680,
+      isComplete: false,
+    });
+
+    Conversation.getSyncProgress.mockResolvedValue({
+      totalProcessed: 0,
+      totalDocuments: 76,
+      isComplete: false,
+    });
+
+    Message.syncWithMeili.mockResolvedValue(undefined);
+    Conversation.syncWithMeili.mockResolvedValue(undefined);
+
+    const indexSync = require('./indexSync');
+    await indexSync();
+
+    expect(Message.syncWithMeili).toHaveBeenCalledTimes(1);
+    expect(Conversation.syncWithMeili).toHaveBeenCalledTimes(1);
+    expect(mockLogger.info).toHaveBeenCalledWith(
+      '[indexSync] No messages marked as indexed, forcing full sync',
+    );
+    expect(mockLogger.info).toHaveBeenCalledWith(
+      '[indexSync] Starting message sync (680 unindexed)',
+    );
+    expect(mockLogger.info).toHaveBeenCalledWith(
+      '[indexSync] No conversations marked as indexed, forcing full sync',
+    );
+    expect(mockLogger.info).toHaveBeenCalledWith('[indexSync] Starting convos sync (76 unindexed)');
+  });
+
+  test('does NOT force sync when some documents already indexed and below threshold', async () => {
+    Message.getSyncProgress.mockResolvedValue({
+      totalProcessed: 630,
+      totalDocuments: 680,
+      isComplete: false,
+    });
+
+    Conversation.getSyncProgress.mockResolvedValue({
+      totalProcessed: 70,
+      totalDocuments: 76,
+      isComplete: false,
+    });
+
+    const indexSync = require('./indexSync');
+    await indexSync();
+
+    expect(Message.syncWithMeili).not.toHaveBeenCalled();
+    expect(Conversation.syncWithMeili).not.toHaveBeenCalled();
+    expect(mockLogger.info).not.toHaveBeenCalledWith(
+      '[indexSync] No messages marked as indexed, forcing full sync',
+    );
+    expect(mockLogger.info).not.toHaveBeenCalledWith(
+      '[indexSync] No conversations marked as indexed, forcing full sync',
+    );
+    expect(mockLogger.info).toHaveBeenCalledWith(
+      '[indexSync] 50 messages unindexed (below threshold: 1000, skipping)',
+    );
+    expect(mockLogger.info).toHaveBeenCalledWith(
+      '[indexSync] 6 convos unindexed (below threshold: 1000, skipping)',
+    );
+  });
 });

packages/data-schemas/src/models/plugins/mongoMeili.ts

@@ -1,7 +1,7 @@
 import _ from 'lodash';
-import { MeiliSearch } from 'meilisearch';
 import { parseTextParts } from 'librechat-data-provider';
-import type { SearchResponse, SearchParams, Index } from 'meilisearch';
+import { MeiliSearch, MeiliSearchTimeOutError } from 'meilisearch';
+import type { SearchResponse, SearchParams, Index, MeiliSearchErrorInfo } from 'meilisearch';
 import type {
   CallbackWithoutResultAndOptionalError,
   FilterQuery,
@@ -581,7 +581,6 @@ export default function mongoMeili(schema: Schema, options: MongoMeiliOptions):
   /** Create index only if it doesn't exist */
   const index = client.index<MeiliIndexable>(indexName);
 
-  // Check if index exists and create if needed
   (async () => {
     try {
       await index.getRawInfo();
@@ -591,18 +590,34 @@ export default function mongoMeili(schema: Schema, options: MongoMeiliOptions):
       if (errorCode === 'index_not_found') {
         try {
           logger.info(`[mongoMeili] Creating new index: ${indexName}`);
-          await client.createIndex(indexName, { primaryKey });
+          const enqueued = await client.createIndex(indexName, { primaryKey });
-          logger.info(`[mongoMeili] Successfully created index: ${indexName}`);
+          const task = await client.waitForTask(enqueued.taskUid, {
+            timeOutMs: 10000,
+            intervalMs: 100,
+          });
+          logger.debug(`[mongoMeili] Index ${indexName} creation task:`, task);
+          if (task.status !== 'succeeded') {
+            const taskError = task.error as MeiliSearchErrorInfo | null;
+            if (taskError?.code === 'index_already_exists') {
+              logger.debug(`[mongoMeili] Index ${indexName} was created by another instance`);
+            } else {
+              logger.warn(`[mongoMeili] Index ${indexName} creation failed:`, taskError);
+            }
+          } else {
+            logger.info(`[mongoMeili] Successfully created index: ${indexName}`);
+          }
         } catch (createError) {
-          // Index might have been created by another instance
+          if (createError instanceof MeiliSearchTimeOutError) {
-          logger.debug(`[mongoMeili] Index ${indexName} may already exist:`, createError);
+            logger.warn(`[mongoMeili] Timed out waiting for index ${indexName} creation`);
+          } else {
+            logger.warn(`[mongoMeili] Error creating index ${indexName}:`, createError);
+          }
         }
       } else {
         logger.error(`[mongoMeili] Error checking index ${indexName}:`, error);
       }
     }
 
-    // Configure index settings to make 'user' field filterable
     try {
       await index.updateSettings({
         filterableAttributes: ['user'],