diff --git a/api/db/utils.js b/api/db/utils.js index 4a311d9832..32051be78d 100644 --- a/api/db/utils.js +++ b/api/db/utils.js @@ -26,7 +26,7 @@ async function batchResetMeiliFlags(collection) { try { while (hasMore) { const docs = await collection - .find({ expiredAt: null, _meiliIndex: true }, { projection: { _id: 1 } }) + .find({ expiredAt: null, _meiliIndex: { $ne: false } }, { projection: { _id: 1 } }) .limit(BATCH_SIZE) .toArray(); diff --git a/api/db/utils.spec.js b/api/db/utils.spec.js index 8b32b4aea8..adf4f6cd86 100644 --- a/api/db/utils.spec.js +++ b/api/db/utils.spec.js @@ -265,8 +265,8 @@ describe('batchResetMeiliFlags', () => { const result = await batchResetMeiliFlags(testCollection); - // Only one document has _meiliIndex: true - expect(result).toBe(1); + // both documents should be updated + expect(result).toBe(2); }); it('should handle mixed document states correctly', async () => { @@ -275,16 +275,18 @@ describe('batchResetMeiliFlags', () => { { _id: new mongoose.Types.ObjectId(), expiredAt: null, _meiliIndex: false }, { _id: new mongoose.Types.ObjectId(), expiredAt: new Date(), _meiliIndex: true }, { _id: new mongoose.Types.ObjectId(), expiredAt: null, _meiliIndex: true }, + { _id: new mongoose.Types.ObjectId(), expiredAt: null, _meiliIndex: null }, + { _id: new mongoose.Types.ObjectId(), expiredAt: null }, ]); const result = await batchResetMeiliFlags(testCollection); - expect(result).toBe(2); + expect(result).toBe(4); const flaggedDocs = await testCollection .find({ expiredAt: null, _meiliIndex: false }) .toArray(); - expect(flaggedDocs).toHaveLength(3); // 2 were updated, 1 was already false + expect(flaggedDocs).toHaveLength(5); // 4 were updated, 1 was already false }); }); diff --git a/packages/data-schemas/src/models/plugins/mongoMeili.spec.ts b/packages/data-schemas/src/models/plugins/mongoMeili.spec.ts index a289b88fe0..d988624d13 100644 --- a/packages/data-schemas/src/models/plugins/mongoMeili.spec.ts +++ b/packages/data-schemas/src/models/plugins/mongoMeili.spec.ts @@ -1015,4 +1015,239 @@ describe('Meilisearch Mongoose plugin', () => { ]); }); }); + + describe('Missing _meiliIndex property handling in sync process', () => { + test('syncWithMeili includes documents with missing _meiliIndex', async () => { + const conversationModel = createConversationModel(mongoose) as SchemaWithMeiliMethods; + await conversationModel.deleteMany({}); + mockAddDocumentsInBatches.mockClear(); + + // Insert documents with different _meiliIndex states + await conversationModel.collection.insertMany([ + { + conversationId: new mongoose.Types.ObjectId(), + user: new mongoose.Types.ObjectId(), + title: 'Missing _meiliIndex', + endpoint: EModelEndpoint.openAI, + expiredAt: null, + // _meiliIndex is not set (missing/undefined) + }, + { + conversationId: new mongoose.Types.ObjectId(), + user: new mongoose.Types.ObjectId(), + title: 'Explicit false', + endpoint: EModelEndpoint.openAI, + expiredAt: null, + _meiliIndex: false, + }, + { + conversationId: new mongoose.Types.ObjectId(), + user: new mongoose.Types.ObjectId(), + title: 'Already indexed', + endpoint: EModelEndpoint.openAI, + expiredAt: null, + _meiliIndex: true, + }, + ]); + + // Run sync + await conversationModel.syncWithMeili(); + + // Should have processed 2 documents (missing and false, but not true) + expect(mockAddDocumentsInBatches).toHaveBeenCalled(); + + // Check that both documents without _meiliIndex=true are now indexed + const indexedCount = await conversationModel.countDocuments({ + expiredAt: null, + _meiliIndex: true, + }); + expect(indexedCount).toBe(3); // All 3 should now be indexed + + // Verify documents with missing _meiliIndex were updated + const docsWithMissingIndex = await conversationModel.countDocuments({ + expiredAt: null, + title: 'Missing _meiliIndex', + _meiliIndex: true, + }); + expect(docsWithMissingIndex).toBe(1); + }); + + test('getSyncProgress counts documents with missing _meiliIndex as not indexed', async () => { + const messageModel = createMessageModel(mongoose) as SchemaWithMeiliMethods; + await messageModel.deleteMany({}); + + // Insert documents with different _meiliIndex states + await messageModel.collection.insertMany([ + { + messageId: new mongoose.Types.ObjectId(), + conversationId: new mongoose.Types.ObjectId(), + user: new mongoose.Types.ObjectId(), + isCreatedByUser: true, + expiredAt: null, + _meiliIndex: true, + }, + { + messageId: new mongoose.Types.ObjectId(), + conversationId: new mongoose.Types.ObjectId(), + user: new mongoose.Types.ObjectId(), + isCreatedByUser: true, + expiredAt: null, + _meiliIndex: false, + }, + { + messageId: new mongoose.Types.ObjectId(), + conversationId: new mongoose.Types.ObjectId(), + user: new mongoose.Types.ObjectId(), + isCreatedByUser: true, + expiredAt: null, + // _meiliIndex is missing + }, + ]); + + const progress = await messageModel.getSyncProgress(); + + // Total should be 3 + expect(progress.totalDocuments).toBe(3); + // Only 1 is indexed (with _meiliIndex: true) + expect(progress.totalProcessed).toBe(1); + // Not complete since 2 documents are not indexed + expect(progress.isComplete).toBe(false); + }); + + test('query with _meiliIndex: { $ne: true } includes missing values', async () => { + const conversationModel = createConversationModel(mongoose) as SchemaWithMeiliMethods; + await conversationModel.deleteMany({}); + + // Insert documents with different _meiliIndex states + await conversationModel.collection.insertMany([ + { + conversationId: new mongoose.Types.ObjectId(), + user: new mongoose.Types.ObjectId(), + title: 'Missing', + endpoint: EModelEndpoint.openAI, + expiredAt: null, + // _meiliIndex is missing + }, + { + conversationId: new mongoose.Types.ObjectId(), + user: new mongoose.Types.ObjectId(), + title: 'False', + endpoint: EModelEndpoint.openAI, + expiredAt: null, + _meiliIndex: false, + }, + { + conversationId: new mongoose.Types.ObjectId(), + user: new mongoose.Types.ObjectId(), + title: 'True', + endpoint: EModelEndpoint.openAI, + expiredAt: null, + _meiliIndex: true, + }, + ]); + + // Query for documents where _meiliIndex is not true (used in syncWithMeili) + const unindexedDocs = await conversationModel.find({ + expiredAt: null, + _meiliIndex: { $ne: true }, + }); + + // Should find 2 documents (missing and false, but not true) + expect(unindexedDocs.length).toBe(2); + const titles = unindexedDocs.map((doc) => doc.title).sort(); + expect(titles).toEqual(['False', 'Missing']); + }); + + test('syncWithMeili processes all documents where _meiliIndex is not true', async () => { + const messageModel = createMessageModel(mongoose) as SchemaWithMeiliMethods; + await messageModel.deleteMany({}); + mockAddDocumentsInBatches.mockClear(); + + // Create a mix of documents with missing and false _meiliIndex + await messageModel.collection.insertMany([ + { + messageId: new mongoose.Types.ObjectId(), + conversationId: new mongoose.Types.ObjectId(), + user: new mongoose.Types.ObjectId(), + isCreatedByUser: true, + expiredAt: null, + // _meiliIndex missing + }, + { + messageId: new mongoose.Types.ObjectId(), + conversationId: new mongoose.Types.ObjectId(), + user: new mongoose.Types.ObjectId(), + isCreatedByUser: true, + expiredAt: null, + _meiliIndex: false, + }, + { + messageId: new mongoose.Types.ObjectId(), + conversationId: new mongoose.Types.ObjectId(), + user: new mongoose.Types.ObjectId(), + isCreatedByUser: true, + expiredAt: null, + // _meiliIndex missing + }, + ]); + + // Count documents that should be synced (where _meiliIndex: { $ne: true }) + const toSyncCount = await messageModel.countDocuments({ + expiredAt: null, + _meiliIndex: { $ne: true }, + }); + expect(toSyncCount).toBe(3); // All 3 should be synced + + await messageModel.syncWithMeili(); + + // All should now be indexed + const indexedCount = await messageModel.countDocuments({ + expiredAt: null, + _meiliIndex: true, + }); + expect(indexedCount).toBe(3); + }); + + test('syncWithMeili treats missing _meiliIndex same as false', async () => { + const conversationModel = createConversationModel(mongoose) as SchemaWithMeiliMethods; + await conversationModel.deleteMany({}); + mockAddDocumentsInBatches.mockClear(); + + // Insert one document with missing _meiliIndex and one with false + await conversationModel.collection.insertMany([ + { + conversationId: new mongoose.Types.ObjectId(), + user: new mongoose.Types.ObjectId(), + title: 'Missing', + endpoint: EModelEndpoint.openAI, + expiredAt: null, + // _meiliIndex is missing + }, + { + conversationId: new mongoose.Types.ObjectId(), + user: new mongoose.Types.ObjectId(), + title: 'False', + endpoint: EModelEndpoint.openAI, + expiredAt: null, + _meiliIndex: false, + }, + ]); + + // Both should be picked up by the sync query + const toSync = await conversationModel.find({ + expiredAt: null, + _meiliIndex: { $ne: true }, + }); + expect(toSync.length).toBe(2); + + await conversationModel.syncWithMeili(); + + // Both should be indexed after sync + const afterSync = await conversationModel.find({ + expiredAt: null, + _meiliIndex: true, + }); + expect(afterSync.length).toBe(2); + }); + }); }); diff --git a/packages/data-schemas/src/models/plugins/mongoMeili.ts b/packages/data-schemas/src/models/plugins/mongoMeili.ts index 92fc5f328c..1cbe0d8761 100644 --- a/packages/data-schemas/src/models/plugins/mongoMeili.ts +++ b/packages/data-schemas/src/models/plugins/mongoMeili.ts @@ -162,8 +162,8 @@ const createMeiliMongooseModel = ({ /** * Synchronizes data between the MongoDB collection and the MeiliSearch index by - * incrementally indexing only documents where `expiredAt` is `null` and `_meiliIndex` is `false` - * (i.e., non-expired documents that have not yet been indexed). + * incrementally indexing only documents where `expiredAt` is `null` and `_meiliIndex` is not `true` + * (i.e., non-expired documents that have not yet been indexed, including those with missing or null `_meiliIndex`). * */ static async syncWithMeili(this: SchemaWithMeiliMethods): Promise { const startTime = Date.now(); @@ -196,7 +196,7 @@ const createMeiliMongooseModel = ({ while (hasMore) { const query: FilterQuery = { expiredAt: null, - _meiliIndex: false, + _meiliIndex: { $ne: true }, }; try {