🕵️ refactor: Optimize Message Search Performance (#9818)

* 🕵️ feat: Enhance Index Sync and MeiliSearch filtering for User Field

- Implemented `ensureFilterableAttributes` function to configure MeiliSearch indexes for messages and conversations to filter by user.
- Updated sync logic to trigger a full re-sync if the user field is missing or index settings are modified.
- Adjusted search queries in Conversation and Message models to include user filtering.
- Ensured 'user' field is marked as filterable in MongoDB schema for both messages and conversations.

This update improves data integrity and search capabilities by ensuring user-related data is properly indexed and retrievable.

* fix: message processing in Search component to use linear list and not tree

* feat: Implement user filtering in MeiliSearch for shared links

* refactor: Optimize message search retrieval by batching database calls

* chore: Update MeiliSearch parameters type to use SearchParams for improved type safety

api/db/indexSync.js

@@ -1,10 +1,8 @@
 const mongoose = require('mongoose');
 const { MeiliSearch } = require('meilisearch');
 const { logger } = require('@librechat/data-schemas');
-const { FlowStateManager } = require('@librechat/api');
 const { CacheKeys } = require('librechat-data-provider');
-
-const { isEnabled } = require('~/server/utils');
+const { isEnabled, FlowStateManager } = require('@librechat/api');
 const { getLogStores } = require('~/cache');
 
 const Conversation = mongoose.models.Conversation;
@@ -31,6 +29,81 @@ class MeiliSearchClient {
   }
 }
 
+/**
+ * Ensures indexes have proper filterable attributes configured and checks if documents have user field
+ * @param {MeiliSearch} client - MeiliSearch client instance
+ * @returns {Promise<boolean>} - true if configuration was updated or re-sync is needed
+ */
+async function ensureFilterableAttributes(client) {
+  try {
+    // Check and update messages index
+    try {
+      const messagesIndex = client.index('messages');
+      const settings = await messagesIndex.getSettings();
+
+      if (!settings.filterableAttributes || !settings.filterableAttributes.includes('user')) {
+        logger.info('[indexSync] Configuring messages index to filter by user...');
+        await messagesIndex.updateSettings({
+          filterableAttributes: ['user'],
+        });
+        logger.info('[indexSync] Messages index configured for user filtering');
+        logger.info('[indexSync] Index configuration updated. Full re-sync will be triggered.');
+        return true;
+      }
+
+      // Check if existing documents have user field indexed
+      try {
+        const searchResult = await messagesIndex.search('', { limit: 1 });
+        if (searchResult.hits.length > 0 && !searchResult.hits[0].user) {
+          logger.info('[indexSync] Existing messages missing user field, re-sync needed');
+          return true;
+        }
+      } catch (searchError) {
+        logger.debug('[indexSync] Could not check message documents:', searchError.message);
+      }
+    } catch (error) {
+      if (error.code !== 'index_not_found') {
+        logger.warn('[indexSync] Could not check/update messages index settings:', error.message);
+      }
+    }
+
+    // Check and update conversations index
+    try {
+      const convosIndex = client.index('convos');
+      const settings = await convosIndex.getSettings();
+
+      if (!settings.filterableAttributes || !settings.filterableAttributes.includes('user')) {
+        logger.info('[indexSync] Configuring convos index to filter by user...');
+        await convosIndex.updateSettings({
+          filterableAttributes: ['user'],
+        });
+        logger.info('[indexSync] Convos index configured for user filtering');
+        logger.info('[indexSync] Index configuration updated. Full re-sync will be triggered.');
+        return true;
+      }
+
+      // Check if existing documents have user field indexed
+      try {
+        const searchResult = await convosIndex.search('', { limit: 1 });
+        if (searchResult.hits.length > 0 && !searchResult.hits[0].user) {
+          logger.info('[indexSync] Existing conversations missing user field, re-sync needed');
+          return true;
+        }
+      } catch (searchError) {
+        logger.debug('[indexSync] Could not check conversation documents:', searchError.message);
+      }
+    } catch (error) {
+      if (error.code !== 'index_not_found') {
+        logger.warn('[indexSync] Could not check/update convos index settings:', error.message);
+      }
+    }
+  } catch (error) {
+    logger.error('[indexSync] Error ensuring filterable attributes:', error);
+  }
+
+  return false;
+}
+
 /**
  * Performs the actual sync operations for messages and conversations
  */
@@ -47,12 +120,27 @@ async function performSync() {
     return { messagesSync: false, convosSync: false };
   }
 
+  /** Ensures indexes have proper filterable attributes configured */
+  const configUpdated = await ensureFilterableAttributes(client);
+
   let messagesSync = false;
   let convosSync = false;
 
+  // If configuration was just updated or documents are missing user field, force a full re-sync
+  if (configUpdated) {
+    logger.info('[indexSync] Forcing full re-sync to ensure user field is properly indexed...');
+
+    // Reset sync flags to force full re-sync
+    await Message.collection.updateMany({ _meiliIndex: true }, { $set: { _meiliIndex: false } });
+    await Conversation.collection.updateMany(
+      { _meiliIndex: true },
+      { $set: { _meiliIndex: false } },
+    );
+  }
+
   // Check if we need to sync messages
   const messageProgress = await Message.getSyncProgress();
-  if (!messageProgress.isComplete) {
+  if (!messageProgress.isComplete || configUpdated) {
     logger.info(
       `[indexSync] Messages need syncing: ${messageProgress.totalProcessed}/${messageProgress.totalDocuments} indexed`,
     );
@@ -79,7 +167,7 @@ async function performSync() {
 
   // Check if we need to sync conversations
   const convoProgress = await Conversation.getSyncProgress();
-  if (!convoProgress.isComplete) {
+  if (!convoProgress.isComplete || configUpdated) {
     logger.info(
       `[indexSync] Conversations need syncing: ${convoProgress.totalProcessed}/${convoProgress.totalDocuments} indexed`,
     );

api/models/Conversation.js

@@ -174,7 +174,7 @@ module.exports = {
 
     if (search) {
       try {
-        const meiliResults = await Conversation.meiliSearch(search);
+        const meiliResults = await Conversation.meiliSearch(search, { filter: `user = "${user}"` });
         const matchingIds = Array.isArray(meiliResults.hits)
           ? meiliResults.hits.map((result) => result.conversationId)
           : [];

api/server/routes/messages.js

@@ -3,8 +3,8 @@ const { logger } = require('@librechat/data-schemas');
 const { ContentTypes } = require('librechat-data-provider');
 const {
   saveConvo,
-  saveMessage,
   getMessage,
+  saveMessage,
   getMessages,
   updateMessage,
   deleteMessages,
@@ -58,34 +58,51 @@ router.get('/', async (req, res) => {
       const nextCursor = messages.length > pageSize ? messages.pop()[sortField] : null;
       response = { messages, nextCursor };
     } else if (search) {
-      const searchResults = await Message.meiliSearch(search, undefined, true);
+      const searchResults = await Message.meiliSearch(search, { filter: `user = "${user}"` }, true);
 
       const messages = searchResults.hits || [];
 
       const result = await getConvosQueried(req.user.id, messages, cursor);
 
-      const activeMessages = [];
+      const messageIds = [];
+      const cleanedMessages = [];
       for (let i = 0; i < messages.length; i++) {
         let message = messages[i];
         if (message.conversationId.includes('--')) {
           message.conversationId = cleanUpPrimaryKeyValue(message.conversationId);
         }
         if (result.convoMap[message.conversationId]) {
-          const convo = result.convoMap[message.conversationId];
-
-          const dbMessage = await getMessage({ user, messageId: message.messageId });
-          activeMessages.push({
-            ...message,
-            title: convo.title,
-            conversationId: message.conversationId,
-            model: convo.model,
-            isCreatedByUser: dbMessage?.isCreatedByUser,
-            endpoint: dbMessage?.endpoint,
-            iconURL: dbMessage?.iconURL,
-          });
+          messageIds.push(message.messageId);
+          cleanedMessages.push(message);
         }
       }
 
+      const dbMessages = await getMessages({
+        user,
+        messageId: { $in: messageIds },
+      });
+
+      const dbMessageMap = {};
+      for (const dbMessage of dbMessages) {
+        dbMessageMap[dbMessage.messageId] = dbMessage;
+      }
+
+      const activeMessages = [];
+      for (const message of cleanedMessages) {
+        const convo = result.convoMap[message.conversationId];
+        const dbMessage = dbMessageMap[message.messageId];
+
+        activeMessages.push({
+          ...message,
+          title: convo.title,
+          conversationId: message.conversationId,
+          model: convo.model,
+          isCreatedByUser: dbMessage?.isCreatedByUser,
+          endpoint: dbMessage?.endpoint,
+          iconURL: dbMessage?.iconURL,
+        });
+      }
+
       response = { messages: activeMessages, nextCursor: null };
     } else {
       response = { messages: [], nextCursor: null };

client/src/routes/Search.tsx

@@ -1,6 +1,5 @@
 import { useEffect, useMemo } from 'react';
 import { useRecoilValue } from 'recoil';
-import { buildTree } from 'librechat-data-provider';
 import { Spinner, useToastContext } from '@librechat/client';
 import MinimalMessagesWrapper from '~/components/Chat/Messages/MinimalMessages';
 import { useNavScrolling, useLocalize, useAuthContext } from '~/hooks';
@@ -43,9 +42,20 @@ export default function Search() {
   });
 
   const messages = useMemo(() => {
-    const msgs = searchMessages?.pages.flatMap((page) => page.messages) || [];
-    const dataTree = buildTree({ messages: msgs, fileMap });
-    return dataTree?.length === 0 ? null : (dataTree ?? null);
+    const msgs =
+      searchMessages?.pages.flatMap((page) =>
+        page.messages.map((message) => {
+          if (!message.files || !fileMap) {
+            return message;
+          }
+          return {
+            ...message,
+            files: message.files.map((file) => fileMap[file.file_id ?? ''] ?? file),
+          };
+        }),
+      ) || [];
+
+    return msgs.length === 0 ? null : msgs;
   }, [fileMap, searchMessages?.pages]);
 
   useEffect(() => {

packages/data-schemas/src/methods/share.test.ts

@@ -427,13 +427,14 @@ describe('Share Methods', () => {
       expect(privateResults.links[0].title).toBe('Private Share');
     });
 
-    test('should handle search with mocked meiliSearch', async () => {
+    test('should handle search with mocked meiliSearch and user filter', async () => {
       const userId = new mongoose.Types.ObjectId().toString();
 
       // Mock meiliSearch method
-      Conversation.meiliSearch = jest.fn().mockResolvedValue({
+      const meiliSearchMock = jest.fn().mockResolvedValue({
         hits: [{ conversationId: 'conv1' }],
       });
+      Conversation.meiliSearch = meiliSearchMock;
 
       await SharedLink.create([
         {
@@ -464,6 +465,9 @@ describe('Share Methods', () => {
 
       expect(result.links).toHaveLength(1);
       expect(result.links[0].title).toBe('Matching Share');
+
+      // Verify that meiliSearch was called with the correct user filter
+      expect(meiliSearchMock).toHaveBeenCalledWith('search term', { filter: `user = "${userId}"` });
     });
 
     test('should handle empty results', async () => {
@@ -475,6 +479,98 @@ describe('Share Methods', () => {
       expect(result.nextCursor).toBeUndefined();
     });
 
+    test('should only return shares from search results for the current user', async () => {
+      const userId1 = new mongoose.Types.ObjectId().toString();
+      const userId2 = new mongoose.Types.ObjectId().toString();
+
+      // Mock meiliSearch to simulate finding conversations from both users
+      const meiliSearchMock = jest.fn().mockImplementation((searchTerm, params) => {
+        // Simulate MeiliSearch filtering by user
+        const filter = params?.filter;
+        if (filter && filter.includes(userId1)) {
+          return Promise.resolve({
+            hits: [{ conversationId: 'conv1' }, { conversationId: 'conv3' }],
+          });
+        } else if (filter && filter.includes(userId2)) {
+          return Promise.resolve({ hits: [{ conversationId: 'conv2' }] });
+        }
+        // Without filter, would return all conversations (security issue)
+        return Promise.resolve({
+          hits: [
+            { conversationId: 'conv1' },
+            { conversationId: 'conv2' },
+            { conversationId: 'conv3' },
+          ],
+        });
+      });
+      Conversation.meiliSearch = meiliSearchMock;
+
+      // Create shares for different users
+      await SharedLink.create([
+        {
+          shareId: 'share1',
+          conversationId: 'conv1',
+          user: userId1,
+          title: 'User 1 Share',
+          isPublic: true,
+        },
+        {
+          shareId: 'share2',
+          conversationId: 'conv2',
+          user: userId2,
+          title: 'User 2 Share',
+          isPublic: true,
+        },
+        {
+          shareId: 'share3',
+          conversationId: 'conv3',
+          user: userId1,
+          title: 'Another User 1 Share',
+          isPublic: true,
+        },
+      ]);
+
+      // Search as userId1
+      const result1 = await shareMethods.getSharedLinks(
+        userId1,
+        undefined,
+        10,
+        true,
+        'createdAt',
+        'desc',
+        'search term',
+      );
+
+      // Should only get shares from conversations belonging to userId1
+      expect(result1.links).toHaveLength(2);
+      expect(result1.links.every((link) => link.title.includes('User 1'))).toBe(true);
+
+      // Verify correct filter was used
+      expect(meiliSearchMock).toHaveBeenCalledWith('search term', {
+        filter: `user = "${userId1}"`,
+      });
+
+      // Search as userId2
+      const result2 = await shareMethods.getSharedLinks(
+        userId2,
+        undefined,
+        10,
+        true,
+        'createdAt',
+        'desc',
+        'search term',
+      );
+
+      // Should only get shares from conversations belonging to userId2
+      expect(result2.links).toHaveLength(1);
+      expect(result2.links[0].title).toBe('User 2 Share');
+
+      // Verify correct filter was used for second user
+      expect(meiliSearchMock).toHaveBeenCalledWith('search term', {
+        filter: `user = "${userId2}"`,
+      });
+    });
+
     test('should only return shares for the specified user', async () => {
       const userId1 = new mongoose.Types.ObjectId().toString();
       const userId2 = new mongoose.Types.ObjectId().toString();

packages/data-schemas/src/methods/share.ts

@@ -150,7 +150,9 @@ export function createShareMethods(mongoose: typeof import('mongoose')) {
 
       if (search && search.trim()) {
         try {
-          const searchResults = await Conversation.meiliSearch(search);
+          const searchResults = await Conversation.meiliSearch(search, {
+            filter: `user = "${user}"`,
+          });
 
           if (!searchResults?.hits?.length) {
             return {

packages/data-schemas/src/models/plugins/mongoMeili.ts

@@ -1,6 +1,6 @@
 import _ from 'lodash';
 import { MeiliSearch } from 'meilisearch';
-import type { SearchResponse, Index } from 'meilisearch';
+import type { SearchResponse, SearchParams, Index } from 'meilisearch';
 import type {
   CallbackWithoutResultAndOptionalError,
   FilterQuery,
@@ -75,7 +75,7 @@ export interface SchemaWithMeiliMethods extends Model<DocumentWithMeiliIndex> {
   setMeiliIndexSettings(settings: Record<string, unknown>): Promise<unknown>;
   meiliSearch(
     q: string,
-    params?: Record<string, unknown>,
+    params?: SearchParams,
     populate?: boolean,
   ): Promise<SearchResponse<MeiliIndexable, Record<string, unknown>>>;
 }
@@ -386,7 +386,7 @@ const createMeiliMongooseModel = ({
     static async meiliSearch(
       this: SchemaWithMeiliMethods,
       q: string,
-      params: Record<string, unknown>,
+      params: SearchParams,
       populate: boolean,
     ): Promise<SearchResponse<MeiliIndexable, Record<string, unknown>>> {
       const data = await index.search(q, params);
@@ -644,6 +644,16 @@ export default function mongoMeili(schema: Schema, options: MongoMeiliOptions):
         logger.error(`[mongoMeili] Error checking index ${indexName}:`, error);
       }
     }
+
+    // Configure index settings to make 'user' field filterable
+    try {
+      await index.updateSettings({
+        filterableAttributes: ['user'],
+      });
+      logger.debug(`[mongoMeili] Updated index ${indexName} settings to make 'user' filterable`);
+    } catch (settingsError) {
+      logger.error(`[mongoMeili] Error updating index settings for ${indexName}:`, settingsError);
+    }
   })();
 
   // Collect attributes from the schema that should be indexed
@@ -654,6 +664,13 @@ export default function mongoMeili(schema: Schema, options: MongoMeiliOptions):
     }, []),
   ];
 
+  // CRITICAL: Always include 'user' field for proper filtering
+  // This ensures existing deployments can filter by user after migration
+  if (schema.obj.user && !attributesToIndex.includes('user')) {
+    attributesToIndex.push('user');
+    logger.debug(`[mongoMeili] Added 'user' field to ${indexName} index attributes`);
+  }
+
   schema.loadClass(createMeiliMongooseModel({ index, attributesToIndex, syncOptions }));
 
   // Register Mongoose hooks

packages/data-schemas/src/schema/convo.ts

@@ -19,6 +19,7 @@ const convoSchema: Schema<IConversation> = new Schema(
     user: {
       type: String,
       index: true,
+      meiliIndex: true,
     },
     messages: [{ type: Schema.Types.ObjectId, ref: 'Message' }],
     agentOptions: {

packages/data-schemas/src/schema/message.ts

@@ -21,6 +21,7 @@ const messageSchema: Schema<IMessage> = new Schema(
       index: true,
       required: true,
       default: null,
+      meiliIndex: true,
     },
     model: {
       type: String,