diff --git a/api/lib/db/indexSync.js b/api/lib/db/indexSync.js new file mode 100644 index 0000000000..cfa8650514 --- /dev/null +++ b/api/lib/db/indexSync.js @@ -0,0 +1,47 @@ +const mongoose = require('mongoose'); +const Conversation = mongoose.models.Conversation; +const Message = mongoose.models.Message; +const { MeiliSearch } = require('meilisearch'); + +// eslint-disable-next-line no-unused-vars +async function indexSync(req, res, next) { + try { + const client = new MeiliSearch({ + host: process.env.MEILI_HOST, + apiKey: process.env.MEILI_KEY + }); + + const { status } = await client.health(); + console.log(`Meilisearch: ${status}`); + const result = status === 'available' && !!process.env.SEARCH; + + if (!result) { + throw new Error('Meilisearch not available'); + } + + const messageCount = await Message.countDocuments(); + const convoCount = await Conversation.countDocuments(); + const messages = await client.index('messages').getStats(); + const convos = await client.index('convos').getStats(); + const messagesIndexed = messages.numberOfDocuments; + const convosIndexed = convos.numberOfDocuments; + + console.log(`There are ${messageCount} messages in the database, ${messagesIndexed} indexed`); + console.log(`There are ${convoCount} convos in the database, ${convosIndexed} indexed`); + + if (messageCount !== messagesIndexed) { + console.log('Messages out of sync, indexing'); + await Message.syncWithMeili(); + } + + if (convoCount !== convosIndexed) { + console.log('Convos out of sync, indexing'); + await Conversation.syncWithMeili(); + } + } catch (err) { + console.error(err); + // res.status(500).json({ error: 'Server error' }); + } +} + +module.exports = indexSync; diff --git a/api/lib/db/mongoMeili.js b/api/lib/db/mongoMeili.js deleted file mode 100644 index 63330b4624..0000000000 --- a/api/lib/db/mongoMeili.js +++ /dev/null @@ -1,190 +0,0 @@ -const { MeiliSearch } = require('meilisearch'); -const _ = require('lodash'); - -const validateOptions = function (options) { - const requiredKeys = ['host', 'apiKey', 'indexName']; - requiredKeys.forEach((key) => { - if (!options[key]) throw new Error(`Missing mongoMeili Option: ${key}`); - }); -}; - -const createMeiliMongooseModel = function ({ index, indexName, client, attributesToIndex }) { - // console.log('attributesToIndex', attributesToIndex); - const primaryKey = attributesToIndex[0]; - // MeiliMongooseModel is of type Mongoose.Model - class MeiliMongooseModel { - // Clear Meili index - static async clearMeiliIndex() { - await index.delete(); - // await index.deleteAllDocuments(); - await this.collection.updateMany( - { _meiliIndex: true }, - { $set: { _meiliIndex: false } } - ); - } - - static async resetIndex() { - await this.clearMeiliIndex(); - await client.createIndex(indexName, { primaryKey }); - } - // Clear Meili index - // Push a mongoDB collection to Meili index - static async syncWithMeili() { - await this.resetIndex(); - // const docs = await this.find(); - const docs = await this.find({ _meiliIndex: { $in: [null, false] } }); - console.log('docs', docs.length); - await Promise.all( - docs.map(function (doc) { - return doc.addObjectToMeili(); - }) - ); - } - - // Set one or more settings of the meili index - static async setMeiliIndexSettings(settings) { - return await index.updateSettings(settings); - } - - // Search the index - static async meiliSearch(q, params, populate) { - const data = await index.search(q, params); - - // Populate hits with content from mongodb - if (populate) { - // Find objects into mongodb matching `objectID` from Meili search - const query = {}; - query[primaryKey] = { $in: _.map(data.hits, primaryKey) }; - const hitsFromMongoose = await this.find( - query, - _.reduce( - this.schema.obj, - function (results, value, key) { - return { ...results, [key]: 1 }; - }, - { _id: 1 } - ), - ); - - // Add additional data from mongodb into Meili search hits - const populatedHits = data.hits.map(function (hit) { - const query = {}; - query[primaryKey] = hit[primaryKey]; - const originalHit = _.find(hitsFromMongoose, query); - - return { - ...(originalHit ? originalHit.toJSON() : {}), - ...hit - }; - }); - data.hits = populatedHits; - } - - return data; - } - - // Push new document to Meili - async addObjectToMeili() { - const object = _.pick(this.toJSON(), attributesToIndex); - - try { - // console.log('Adding document to Meili', object); - await index.addDocuments([object]); - } catch (error) { - console.log('Error adding document to Meili'); - console.error(error); - } - - await this.collection.updateMany({ _id: this._id }, { $set: { _meiliIndex: true } }); - } - - // Update an existing document in Meili - async updateObjectToMeili() { - const object = pick(this.toJSON(), attributesToIndex); - await index.updateDocuments([object]); - } - - // Delete a document from Meili - async deleteObjectFromMeili() { - await index.deleteDocument(this._id); - } - - // * schema.post('save') - postSaveHook() { - if (this._meiliIndex) { - this.updateObjectToMeili(); - } else { - this.addObjectToMeili(); - } - } - - // * schema.post('update') - postUpdateHook() { - if (this._meiliIndex) { - this.updateObjectToMeili(); - } - } - - // * schema.post('remove') - postRemoveHook() { - if (this._meiliIndex) { - this.deleteObjectFromMeili(); - } - } - } - - return MeiliMongooseModel; -}; - -module.exports = function mongoMeili(schema, options) { - // Vaidate Options for mongoMeili - validateOptions(options); - - // Add meiliIndex to schema - schema.add({ - _meiliIndex: { - type: Boolean, - required: false, - select: false, - default: false - } - }); - - const { host, apiKey, indexName, primaryKey } = options; - - // Setup MeiliSearch Client - const client = new MeiliSearch({ host, apiKey }); - - // Asynchronously create the index - client.createIndex(indexName, { primaryKey }); - - // Setup the index to search for this schema - const index = client.index(indexName); - - const attributesToIndex = [ - ..._.reduce( - schema.obj, - function (results, value, key) { - return value.meiliIndex ? [...results, key] : results; - // }, []), '_id']; - }, - [] - ) - ]; - - schema.loadClass(createMeiliMongooseModel({ index, indexName, client, attributesToIndex })); - - // Register hooks - schema.post('save', function (doc) { - doc.postSaveHook(); - }); - schema.post('update', function (doc) { - doc.postUpdateHook(); - }); - schema.post('remove', function (doc) { - doc.postRemoveHook(); - }); - schema.post('findOneAndUpdate', function (doc) { - doc.postSaveHook(); - }); -};