⏳ refactor: Exclude Temporary Conversations and Messages from Meilisearch Indexing (#10872)

Temporary chat data should not show up when searching.

Now we check whether a TTL has been set on a conversation/message
before indexing it in meilisearch. If there is a TTL, we skip it.

packages/data-schemas/src/models/plugins/mongoMeili.spec.ts

@@ -0,0 +1,110 @@
+import { MongoMemoryServer } from 'mongodb-memory-server';
+import mongoose from 'mongoose';
+import { EModelEndpoint } from 'librechat-data-provider';
+import { createConversationModel } from '~/models/convo';
+import { createMessageModel } from '~/models/message';
+import { SchemaWithMeiliMethods } from '~/models/plugins/mongoMeili';
+
+const mockAddDocuments = jest.fn();
+const mockIndex = jest.fn().mockReturnValue({
+  getRawInfo: jest.fn(),
+  updateSettings: jest.fn(),
+  addDocuments: mockAddDocuments,
+  getDocuments: jest.fn().mockReturnValue({ results: [] }),
+});
+jest.mock('meilisearch', () => {
+  return {
+    MeiliSearch: jest.fn().mockImplementation(() => {
+      return {
+        index: mockIndex,
+      };
+    }),
+  };
+});
+
+describe('Meilisearch Mongoose plugin', () => {
+  const OLD_ENV = process.env;
+
+  let mongoServer: MongoMemoryServer;
+
+  beforeAll(async () => {
+    process.env = {
+      ...OLD_ENV,
+      // Set a fake meilisearch host/key so that we activate the meilisearch plugin
+      MEILI_HOST: 'foo',
+      MEILI_MASTER_KEY: 'bar',
+    };
+
+    mongoServer = await MongoMemoryServer.create();
+    const mongoUri = mongoServer.getUri();
+    await mongoose.connect(mongoUri);
+  });
+
+  beforeEach(() => {
+    mockAddDocuments.mockClear();
+  });
+
+  afterAll(async () => {
+    await mongoose.disconnect();
+    await mongoServer.stop();
+
+    process.env = OLD_ENV;
+  });
+
+  test('saving conversation indexes w/ meilisearch', async () => {
+    await createConversationModel(mongoose).create({
+      conversationId: new mongoose.Types.ObjectId(),
+      user: new mongoose.Types.ObjectId(),
+      title: 'Test Conversation',
+      endpoint: EModelEndpoint.openAI,
+    });
+    expect(mockAddDocuments).toHaveBeenCalled();
+  });
+
+  test('saving TTL conversation does NOT index w/ meilisearch', async () => {
+    await createConversationModel(mongoose).create({
+      conversationId: new mongoose.Types.ObjectId(),
+      user: new mongoose.Types.ObjectId(),
+      title: 'Test Conversation',
+      endpoint: EModelEndpoint.openAI,
+      expiredAt: new Date(),
+    });
+    expect(mockAddDocuments).not.toHaveBeenCalled();
+  });
+
+  test('saving messages indexes w/ meilisearch', async () => {
+    await createMessageModel(mongoose).create({
+      messageId: new mongoose.Types.ObjectId(),
+      conversationId: new mongoose.Types.ObjectId(),
+      user: new mongoose.Types.ObjectId(),
+      isCreatedByUser: true,
+    });
+    expect(mockAddDocuments).toHaveBeenCalled();
+  });
+
+  test('saving TTL messages does NOT index w/ meilisearch', async () => {
+    await createMessageModel(mongoose).create({
+      messageId: new mongoose.Types.ObjectId(),
+      conversationId: new mongoose.Types.ObjectId(),
+      user: new mongoose.Types.ObjectId(),
+      isCreatedByUser: true,
+      expiredAt: new Date(),
+    });
+    expect(mockAddDocuments).not.toHaveBeenCalled();
+  });
+
+  test('sync w/ meili does not include TTL documents', async () => {
+    const conversationModel = createConversationModel(mongoose) as SchemaWithMeiliMethods;
+    await conversationModel.create({
+      conversationId: new mongoose.Types.ObjectId(),
+      user: new mongoose.Types.ObjectId(),
+      title: 'Test Conversation',
+      endpoint: EModelEndpoint.openAI,
+      expiredAt: new Date(),
+    });
+
+    await conversationModel.syncWithMeili();
+
+    expect(mockAddDocuments).not.toHaveBeenCalled();
+  });
+});

packages/data-schemas/src/models/plugins/mongoMeili.ts

@@ -183,7 +183,9 @@ const createMeiliMongooseModel = ({
         );
 
         // Build query with resume capability
-        const query: FilterQuery<unknown> = {};
+        const query: FilterQuery<unknown> = {
+          expiredAt: { $exists: false }, // Do not sync TTL documents
+        };
         if (options?.resumeFromId) {
           query._id = { $gt: options.resumeFromId };
         }
@@ -430,6 +432,11 @@ const createMeiliMongooseModel = ({
       this: DocumentWithMeiliIndex,
       next: CallbackWithoutResultAndOptionalError,
     ): Promise<void> {
+      // If this conversation or message has a TTL, don't index it
+      if (!_.isNil(this.expiredAt)) {
+        return next();
+      }
+
       const object = this.preprocessObjectForIndex!();
       const maxRetries = 3;
       let retryCount = 0;