diff --git a/api/models/Conversation.js b/api/models/Conversation.js index 953e42528a..84299b2aa1 100644 --- a/api/models/Conversation.js +++ b/api/models/Conversation.js @@ -1,4 +1,6 @@ -const { Conversation } = require('./schema/'); +// const { Conversation } = require('./plugins'); +const Conversation = require('./schema/convoSchema'); +const { cleanUpPrimaryKeyValue } = require('../lib/utils/misc'); const { getMessages, deleteMessages } = require('./Message'); const getConvo = async (user, conversationId) => { @@ -87,7 +89,7 @@ module.exports = { promises.push( Conversation.findOne({ user, - conversationId: convo.conversationId + conversationId: cleanUpPrimaryKeyValue(convo.conversationId) }).exec() ) ); diff --git a/api/models/Message.js b/api/models/Message.js index b2422a4c4e..a886c02859 100644 --- a/api/models/Message.js +++ b/api/models/Message.js @@ -1,4 +1,4 @@ -const { Message } = require('./schema/'); +const Message = require('./schema/messageSchema'); module.exports = { Message, saveMessage: async ({ messageId, conversationId, parentMessageId, sender, text, isCreatedByUser=false, error }) => { diff --git a/api/models/plugins/mongoMeili.js b/api/models/plugins/mongoMeili.js new file mode 100644 index 0000000000..c2875b9dfa --- /dev/null +++ b/api/models/plugins/mongoMeili.js @@ -0,0 +1,198 @@ +const { MeiliSearch } = require('meilisearch'); +const { cleanUpPrimaryKeyValue } = require('../../lib/utils/misc'); +const _ = require('lodash'); + +const validateOptions = function (options) { + const requiredKeys = ['host', 'apiKey', 'indexName']; + requiredKeys.forEach((key) => { + if (!options[key]) throw new Error(`Missing mongoMeili Option: ${key}`); + }); +}; + +const createMeiliMongooseModel = function ({ index, indexName, client, attributesToIndex }) { + // console.log('attributesToIndex', attributesToIndex); + const primaryKey = attributesToIndex[0]; + // MeiliMongooseModel is of type Mongoose.Model + class MeiliMongooseModel { + // Clear Meili index + static async clearMeiliIndex() { + await index.delete(); + // await index.deleteAllDocuments(); + await this.collection.updateMany( + { _meiliIndex: true }, + { $set: { _meiliIndex: false } } + ); + } + + static async resetIndex() { + await this.clearMeiliIndex(); + await client.createIndex(indexName, { primaryKey }); + } + // Clear Meili index + // Push a mongoDB collection to Meili index + static async syncWithMeili() { + await this.resetIndex(); + // const docs = await this.find(); + const docs = await this.find({ _meiliIndex: { $in: [null, false] } }); + console.log('docs', docs.length); + await Promise.all( + docs.map(function (doc) { + return doc.addObjectToMeili(); + }) + ); + } + + // Set one or more settings of the meili index + static async setMeiliIndexSettings(settings) { + return await index.updateSettings(settings); + } + + // Search the index + static async meiliSearch(q, params, populate) { + const data = await index.search(q, params); + + // Populate hits with content from mongodb + if (populate) { + // Find objects into mongodb matching `objectID` from Meili search + const query = {}; + // query[primaryKey] = { $in: _.map(data.hits, primaryKey) }; + query[primaryKey] = _.map(data.hits, hit => cleanUpPrimaryKeyValue(hit[primaryKey])); + // console.log('query', query); + const hitsFromMongoose = await this.find( + query, + _.reduce( + this.schema.obj, + function (results, value, key) { + return { ...results, [key]: 1 }; + }, + { _id: 1 } + ), + ); + + // Add additional data from mongodb into Meili search hits + const populatedHits = data.hits.map(function (hit) { + const query = {}; + query[primaryKey] = hit[primaryKey]; + const originalHit = _.find(hitsFromMongoose, query); + + return { + ...(originalHit ? originalHit.toJSON() : {}), + ...hit + }; + }); + data.hits = populatedHits; + } + + return data; + } + + // Push new document to Meili + async addObjectToMeili() { + const object = _.pick(this.toJSON(), attributesToIndex); + // NOTE: MeiliSearch does not allow | in primary key, so we replace it with - for Bing convoIds + // object.conversationId = object.conversationId.replace(/\|/g, '-'); + if (object.conversationId && object.conversationId.includes('|')) { + object.conversationId = object.conversationId.replace(/\|/g, '--'); + } + + try { + // console.log('Adding document to Meili', object); + await index.addDocuments([object]); + } catch (error) { + console.log('Error adding document to Meili'); + console.error(error); + } + + await this.collection.updateMany({ _id: this._id }, { $set: { _meiliIndex: true } }); + } + + // Update an existing document in Meili + async updateObjectToMeili() { + const object = _.pick(this.toJSON(), attributesToIndex); + await index.updateDocuments([object]); + } + + // Delete a document from Meili + async deleteObjectFromMeili() { + await index.deleteDocument(this._id); + } + + // * schema.post('save') + postSaveHook() { + if (this._meiliIndex) { + this.updateObjectToMeili(); + } else { + this.addObjectToMeili(); + } + } + + // * schema.post('update') + postUpdateHook() { + if (this._meiliIndex) { + this.updateObjectToMeili(); + } + } + + // * schema.post('remove') + postRemoveHook() { + if (this._meiliIndex) { + this.deleteObjectFromMeili(); + } + } + } + + return MeiliMongooseModel; +}; + +module.exports = function mongoMeili(schema, options) { + // Vaidate Options for mongoMeili + validateOptions(options); + + // Add meiliIndex to schema + schema.add({ + _meiliIndex: { + type: Boolean, + required: false, + select: false, + default: false + } + }); + + const { host, apiKey, indexName, primaryKey } = options; + + // Setup MeiliSearch Client + const client = new MeiliSearch({ host, apiKey }); + + // Asynchronously create the index + client.createIndex(indexName, { primaryKey }); + + // Setup the index to search for this schema + const index = client.index(indexName); + + const attributesToIndex = [ + ..._.reduce( + schema.obj, + function (results, value, key) { + return value.meiliIndex ? [...results, key] : results; + // }, []), '_id']; + }, + [] + ) + ]; + + schema.loadClass(createMeiliMongooseModel({ index, indexName, client, attributesToIndex })); + + // Register hooks + schema.post('save', function (doc) { + doc.postSaveHook(); + }); + schema.post('update', function (doc) { + doc.postUpdateHook(); + }); + schema.post('remove', function (doc) { + doc.postRemoveHook(); + }); + schema.post('findOneAndUpdate', function (doc) { + doc.postSaveHook(); + }); +}; diff --git a/api/models/schema/convoSchema.js b/api/models/schema/convoSchema.js index 01f85e3b15..fe25aa024e 100644 --- a/api/models/schema/convoSchema.js +++ b/api/models/schema/convoSchema.js @@ -1,5 +1,6 @@ const mongoose = require('mongoose'); -module.exports = mongoose.Schema( +const mongoMeili = require('../plugins/mongoMeili'); +const convoSchema = mongoose.Schema( { conversationId: { type: String, @@ -50,4 +51,15 @@ module.exports = mongoose.Schema( messages: [{ type: mongoose.Schema.Types.ObjectId, ref: 'Message' }] }, { timestamps: true } -); \ No newline at end of file +); + +convoSchema.plugin(mongoMeili, { + host: process.env.MEILI_HOST, + apiKey: process.env.MEILI_KEY, + indexName: 'convos', // Will get created automatically if it doesn't exist already + primaryKey: 'conversationId' +}); + +const Conversation = mongoose.models.Conversation || mongoose.model('Conversation', convoSchema); + +module.exports = Conversation; diff --git a/api/models/schema/index.js b/api/models/schema/index.js deleted file mode 100644 index 53efd98616..0000000000 --- a/api/models/schema/index.js +++ /dev/null @@ -1,46 +0,0 @@ -const mongoose = require('mongoose'); -const convoSchema = require('./convoSchema'); -const messageSchema = require('./messageSchema'); -const { MeiliSearch } = require('meilisearch'); -const mongoMeili = require('../../lib/db/mongoMeili'); - -(async () => { - try { - const client = new MeiliSearch({ - host: process.env.MEILI_HOST, - apiKey: process.env.MEILI_KEY - }); - - const { status } = await client.health(); - console.log(`Meilisearch: ${status}`); - const result = status === 'available' && !!process.env.SEARCH; - - if (!result) { - throw new Error('Meilisearch not available'); - } - - convoSchema.plugin(mongoMeili, { - host: process.env.MEILI_HOST, - apiKey: process.env.MEILI_KEY, - indexName: 'convos', // Will get created automatically if it doesn't exist already - primaryKey: 'conversationId' - }); - - messageSchema.plugin(mongoMeili, { - host: process.env.MEILI_HOST, - apiKey: process.env.MEILI_KEY, - indexName: 'messages', - primaryKey: 'messageId' - }); - - } catch (error) { - console.log('Meilisearch error, search will be disabled'); - console.error(error); - } -})(); - -const Conversation = -mongoose.models.Conversation || mongoose.model('Conversation', convoSchema); -const Message = mongoose.models.Message || mongoose.model('Message', messageSchema); - -module.exports = { Conversation, Message }; diff --git a/api/models/schema/messageSchema.js b/api/models/schema/messageSchema.js index 3a2214d314..3ecb68a55a 100644 --- a/api/models/schema/messageSchema.js +++ b/api/models/schema/messageSchema.js @@ -1,5 +1,6 @@ const mongoose = require('mongoose'); -module.exports = mongoose.Schema({ +const mongoMeili = require('../plugins/mongoMeili'); +const messageSchema = mongoose.Schema({ messageId: { type: String, unique: true, @@ -51,4 +52,15 @@ module.exports = mongoose.Schema({ select: false, default: false } -}, { timestamps: true }); \ No newline at end of file +}, { timestamps: true }); + +messageSchema.plugin(mongoMeili, { + host: process.env.MEILI_HOST, + apiKey: process.env.MEILI_KEY, + indexName: 'messages', + primaryKey: 'messageId' +}); + +const Message = mongoose.models.Message || mongoose.model('Message', messageSchema); + +module.exports = Message; \ No newline at end of file