Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-12-21 19:00:13 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: Batch indexing (#606)

Browse source
This commit is contained in:
HyunggyuJang 2023-07-08 11:20:57 +09:00 • committed by GitHub
parent cf3889d8e4
commit f0e2639269
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 15 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

23
api/models/plugins/mongoMeili.js
View file

@ -30,14 +30,17 @@ const createMeiliMongooseModel = function ({ index, indexName, client, attribute
// Push a mongoDB collection to Meili index // Push a mongoDB collection to Meili index
static async syncWithMeili() { static async syncWithMeili() {
await this.resetIndex(); await this.resetIndex();
// const docs = await this.find();
const docs = await this.find({ _meiliIndex: { $in: [null, false] } }); const docs = await this.find({ _meiliIndex: { $in: [null, false] } });
console.log('docs', docs.length); console.log('docs', docs.length);
await Promise.all( const objs = docs.map((doc) => doc.preprocessObjectForIndex());
docs.map(function (doc) { try {
return doc.addObjectToMeili(); await index.addDocuments(objs);
}) const ids = docs.map((doc) => doc._id);
); await this.collection.updateMany({ _id: { $in: ids } }, { $set: { _meiliIndex: true } });
} catch (error) {
console.log('Error adding document to Meili');
console.error(error);
}
} }
// Set one or more settings of the meili index // Set one or more settings of the meili index
@ -84,15 +87,19 @@ const createMeiliMongooseModel = function ({ index, indexName, client, attribute
return data; return data;
} }
// Push new document to Meili preprocessObjectForIndex() {
async addObjectToMeili() {
const object = _.pick(this.toJSON(), attributesToIndex); const object = _.pick(this.toJSON(), attributesToIndex);
// NOTE: MeiliSearch does not allow | in primary key, so we replace it with - for Bing convoIds // NOTE: MeiliSearch does not allow | in primary key, so we replace it with - for Bing convoIds
// object.conversationId = object.conversationId.replace(/\|/g, '-'); // object.conversationId = object.conversationId.replace(/\|/g, '-');
if (object.conversationId && object.conversationId.includes('|')) { if (object.conversationId && object.conversationId.includes('|')) {
object.conversationId = object.conversationId.replace(/\|/g, '--'); object.conversationId = object.conversationId.replace(/\|/g, '--');
} }
return object
}
// Push new document to Meili
async addObjectToMeili() {
const object = this.preprocessObjectForIndex()
try { try {
// console.log('Adding document to Meili', object); // console.log('Adding document to Meili', object);
await index.addDocuments([object]); await index.addDocuments([object]);