🕵️ refactor: Optimize Message Search Performance (#9818)

* 🕵️ feat: Enhance Index Sync and MeiliSearch filtering for User Field

- Implemented `ensureFilterableAttributes` function to configure MeiliSearch indexes for messages and conversations to filter by user.
- Updated sync logic to trigger a full re-sync if the user field is missing or index settings are modified.
- Adjusted search queries in Conversation and Message models to include user filtering.
- Ensured 'user' field is marked as filterable in MongoDB schema for both messages and conversations.

This update improves data integrity and search capabilities by ensuring user-related data is properly indexed and retrievable.

* fix: message processing in Search component to use linear list and not tree

* feat: Implement user filtering in MeiliSearch for shared links

* refactor: Optimize message search retrieval by batching database calls

* chore: Update MeiliSearch parameters type to use SearchParams for improved type safety

packages/data-schemas/src/methods/share.test.ts

@@ -427,13 +427,14 @@ describe('Share Methods', () => {
       expect(privateResults.links[0].title).toBe('Private Share');
     });
 
-    test('should handle search with mocked meiliSearch', async () => {
+    test('should handle search with mocked meiliSearch and user filter', async () => {
       const userId = new mongoose.Types.ObjectId().toString();
 
       // Mock meiliSearch method
-      Conversation.meiliSearch = jest.fn().mockResolvedValue({
+      const meiliSearchMock = jest.fn().mockResolvedValue({
         hits: [{ conversationId: 'conv1' }],
       });
+      Conversation.meiliSearch = meiliSearchMock;
 
       await SharedLink.create([
         {
@@ -464,6 +465,9 @@ describe('Share Methods', () => {
 
       expect(result.links).toHaveLength(1);
       expect(result.links[0].title).toBe('Matching Share');
+
+      // Verify that meiliSearch was called with the correct user filter
+      expect(meiliSearchMock).toHaveBeenCalledWith('search term', { filter: `user = "${userId}"` });
     });
 
     test('should handle empty results', async () => {
@@ -475,6 +479,98 @@ describe('Share Methods', () => {
       expect(result.nextCursor).toBeUndefined();
     });
 
+    test('should only return shares from search results for the current user', async () => {
+      const userId1 = new mongoose.Types.ObjectId().toString();
+      const userId2 = new mongoose.Types.ObjectId().toString();
+
+      // Mock meiliSearch to simulate finding conversations from both users
+      const meiliSearchMock = jest.fn().mockImplementation((searchTerm, params) => {
+        // Simulate MeiliSearch filtering by user
+        const filter = params?.filter;
+        if (filter && filter.includes(userId1)) {
+          return Promise.resolve({
+            hits: [{ conversationId: 'conv1' }, { conversationId: 'conv3' }],
+          });
+        } else if (filter && filter.includes(userId2)) {
+          return Promise.resolve({ hits: [{ conversationId: 'conv2' }] });
+        }
+        // Without filter, would return all conversations (security issue)
+        return Promise.resolve({
+          hits: [
+            { conversationId: 'conv1' },
+            { conversationId: 'conv2' },
+            { conversationId: 'conv3' },
+          ],
+        });
+      });
+      Conversation.meiliSearch = meiliSearchMock;
+
+      // Create shares for different users
+      await SharedLink.create([
+        {
+          shareId: 'share1',
+          conversationId: 'conv1',
+          user: userId1,
+          title: 'User 1 Share',
+          isPublic: true,
+        },
+        {
+          shareId: 'share2',
+          conversationId: 'conv2',
+          user: userId2,
+          title: 'User 2 Share',
+          isPublic: true,
+        },
+        {
+          shareId: 'share3',
+          conversationId: 'conv3',
+          user: userId1,
+          title: 'Another User 1 Share',
+          isPublic: true,
+        },
+      ]);
+
+      // Search as userId1
+      const result1 = await shareMethods.getSharedLinks(
+        userId1,
+        undefined,
+        10,
+        true,
+        'createdAt',
+        'desc',
+        'search term',
+      );
+
+      // Should only get shares from conversations belonging to userId1
+      expect(result1.links).toHaveLength(2);
+      expect(result1.links.every((link) => link.title.includes('User 1'))).toBe(true);
+
+      // Verify correct filter was used
+      expect(meiliSearchMock).toHaveBeenCalledWith('search term', {
+        filter: `user = "${userId1}"`,
+      });
+
+      // Search as userId2
+      const result2 = await shareMethods.getSharedLinks(
+        userId2,
+        undefined,
+        10,
+        true,
+        'createdAt',
+        'desc',
+        'search term',
+      );
+
+      // Should only get shares from conversations belonging to userId2
+      expect(result2.links).toHaveLength(1);
+      expect(result2.links[0].title).toBe('User 2 Share');
+
+      // Verify correct filter was used for second user
+      expect(meiliSearchMock).toHaveBeenCalledWith('search term', {
+        filter: `user = "${userId2}"`,
+      });
+    });
+
     test('should only return shares for the specified user', async () => {
       const userId1 = new mongoose.Types.ObjectId().toString();
       const userId2 = new mongoose.Types.ObjectId().toString();

packages/data-schemas/src/methods/share.ts

@@ -150,7 +150,9 @@ export function createShareMethods(mongoose: typeof import('mongoose')) {
 
       if (search && search.trim()) {
         try {
-          const searchResults = await Conversation.meiliSearch(search);
+          const searchResults = await Conversation.meiliSearch(search, {
+            filter: `user = "${user}"`,
+          });
 
           if (!searchResults?.hits?.length) {
             return {

packages/data-schemas/src/models/plugins/mongoMeili.ts

@@ -1,6 +1,6 @@
 import _ from 'lodash';
 import { MeiliSearch } from 'meilisearch';
-import type { SearchResponse, Index } from 'meilisearch';
+import type { SearchResponse, SearchParams, Index } from 'meilisearch';
 import type {
   CallbackWithoutResultAndOptionalError,
   FilterQuery,
@@ -75,7 +75,7 @@ export interface SchemaWithMeiliMethods extends Model<DocumentWithMeiliIndex> {
   setMeiliIndexSettings(settings: Record<string, unknown>): Promise<unknown>;
   meiliSearch(
     q: string,
-    params?: Record<string, unknown>,
+    params?: SearchParams,
     populate?: boolean,
   ): Promise<SearchResponse<MeiliIndexable, Record<string, unknown>>>;
 }
@@ -386,7 +386,7 @@ const createMeiliMongooseModel = ({
     static async meiliSearch(
       this: SchemaWithMeiliMethods,
       q: string,
-      params: Record<string, unknown>,
+      params: SearchParams,
       populate: boolean,
     ): Promise<SearchResponse<MeiliIndexable, Record<string, unknown>>> {
       const data = await index.search(q, params);
@@ -644,6 +644,16 @@ export default function mongoMeili(schema: Schema, options: MongoMeiliOptions):
         logger.error(`[mongoMeili] Error checking index ${indexName}:`, error);
       }
     }
+
+    // Configure index settings to make 'user' field filterable
+    try {
+      await index.updateSettings({
+        filterableAttributes: ['user'],
+      });
+      logger.debug(`[mongoMeili] Updated index ${indexName} settings to make 'user' filterable`);
+    } catch (settingsError) {
+      logger.error(`[mongoMeili] Error updating index settings for ${indexName}:`, settingsError);
+    }
   })();
 
   // Collect attributes from the schema that should be indexed
@@ -654,6 +664,13 @@ export default function mongoMeili(schema: Schema, options: MongoMeiliOptions):
     }, []),
   ];
 
+  // CRITICAL: Always include 'user' field for proper filtering
+  // This ensures existing deployments can filter by user after migration
+  if (schema.obj.user && !attributesToIndex.includes('user')) {
+    attributesToIndex.push('user');
+    logger.debug(`[mongoMeili] Added 'user' field to ${indexName} index attributes`);
+  }
+
   schema.loadClass(createMeiliMongooseModel({ index, attributesToIndex, syncOptions }));
 
   // Register Mongoose hooks

packages/data-schemas/src/schema/convo.ts

@@ -19,6 +19,7 @@ const convoSchema: Schema<IConversation> = new Schema(
     user: {
       type: String,
       index: true,
+      meiliIndex: true,
     },
     messages: [{ type: Schema.Types.ObjectId, ref: 'Message' }],
     agentOptions: {

packages/data-schemas/src/schema/message.ts

@@ -21,6 +21,7 @@ const messageSchema: Schema<IMessage> = new Schema(
       index: true,
       required: true,
       default: null,
+      meiliIndex: true,
     },
     model: {
       type: String,