Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-12-17 00:40:14 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

feat: Batch indexing (#606)

Browse source
This commit is contained in:
HyunggyuJang 2023-07-08 11:20:57 +09:00 • committed by GitHub
parent cf3889d8e4
commit f0e2639269
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 15 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

23
api/models/plugins/mongoMeili.js
View file

@ -30,14 +30,17 @@ const createMeiliMongooseModel = function ({ index, indexName, client, attribute
// Push a mongoDB collection to Meili index
static async syncWithMeili() {
await this.resetIndex();
// const docs = await this.find();
const docs = await this.find({ _meiliIndex: { $in: [null, false] } });
console.log('docs', docs.length);
await Promise.all(
docs.map(function (doc) {
return doc.addObjectToMeili();
})
);
const objs = docs.map((doc) => doc.preprocessObjectForIndex());
try {
await index.addDocuments(objs);
const ids = docs.map((doc) => doc._id);
await this.collection.updateMany({ _id: { $in: ids } }, { $set: { _meiliIndex: true } });
} catch (error) {
console.log('Error adding document to Meili');
console.error(error);
}
}
// Set one or more settings of the meili index
@ -84,15 +87,19 @@ const createMeiliMongooseModel = function ({ index, indexName, client, attribute
return data;
}
// Push new document to Meili
async addObjectToMeili() {
preprocessObjectForIndex() {
const object = _.pick(this.toJSON(), attributesToIndex);
// NOTE: MeiliSearch does not allow | in primary key, so we replace it with - for Bing convoIds
// object.conversationId = object.conversationId.replace(/\|/g, '-');
if (object.conversationId && object.conversationId.includes('|')) {
object.conversationId = object.conversationId.replace(/\|/g, '--');
}
return object
}
// Push new document to Meili
async addObjectToMeili() {
const object = this.preprocessObjectForIndex()
try {
// console.log('Adding document to Meili', object);
await index.addDocuments([object]);