search: correctly register/export schema/models for mongoMeili

api/models/Conversation.js

@@ -1,4 +1,6 @@
-const { Conversation } = require('./schema/');
+// const { Conversation } = require('./plugins');
+const Conversation = require('./schema/convoSchema');
+const { cleanUpPrimaryKeyValue } = require('../lib/utils/misc');
 const { getMessages, deleteMessages } = require('./Message');
 
 const getConvo = async (user, conversationId) => {
@@ -87,7 +89,7 @@ module.exports = {
         promises.push(
           Conversation.findOne({
             user,
-            conversationId: convo.conversationId
+            conversationId: cleanUpPrimaryKeyValue(convo.conversationId)
           }).exec()
         )
       );

api/models/Message.js

@@ -1,4 +1,4 @@
-const { Message } = require('./schema/');
+const Message = require('./schema/messageSchema');
 module.exports = {
   Message,
   saveMessage: async ({ messageId, conversationId, parentMessageId, sender, text, isCreatedByUser=false, error }) => {

api/models/plugins/mongoMeili.js

@@ -0,0 +1,198 @@
+const { MeiliSearch } = require('meilisearch');
+const { cleanUpPrimaryKeyValue } = require('../../lib/utils/misc');
+const _ = require('lodash');
+
+const validateOptions = function (options) {
+  const requiredKeys = ['host', 'apiKey', 'indexName'];
+  requiredKeys.forEach((key) => {
+    if (!options[key]) throw new Error(`Missing mongoMeili Option: ${key}`);
+  });
+};
+
+const createMeiliMongooseModel = function ({ index, indexName, client, attributesToIndex }) {
+  // console.log('attributesToIndex', attributesToIndex);
+  const primaryKey = attributesToIndex[0];
+  // MeiliMongooseModel is of type Mongoose.Model
+  class MeiliMongooseModel {
+    // Clear Meili index
+    static async clearMeiliIndex() {
+      await index.delete();
+      // await index.deleteAllDocuments();
+      await this.collection.updateMany(
+        { _meiliIndex: true },
+        { $set: { _meiliIndex: false } }
+      );
+    }
+
+    static async resetIndex() {
+      await this.clearMeiliIndex();
+      await client.createIndex(indexName, { primaryKey });
+    }
+    // Clear Meili index
+    // Push a mongoDB collection to Meili index
+    static async syncWithMeili() {
+      await this.resetIndex();
+      // const docs = await this.find();
+      const docs = await this.find({ _meiliIndex: { $in: [null, false] } });
+      console.log('docs', docs.length);
+      await Promise.all(
+        docs.map(function (doc) {
+          return doc.addObjectToMeili();
+        })
+      );
+    }
+
+    // Set one or more settings of the meili index
+    static async setMeiliIndexSettings(settings) {
+      return await index.updateSettings(settings);
+    }
+
+    // Search the index
+    static async meiliSearch(q, params, populate) {
+      const data = await index.search(q, params);
+
+      // Populate hits with content from mongodb
+      if (populate) {
+        // Find objects into mongodb matching `objectID` from Meili search
+        const query = {};
+        // query[primaryKey] = { $in: _.map(data.hits, primaryKey) };
+        query[primaryKey] = _.map(data.hits, hit => cleanUpPrimaryKeyValue(hit[primaryKey]));
+        // console.log('query', query);
+        const hitsFromMongoose = await this.find(
+          query,
+          _.reduce(
+            this.schema.obj,
+            function (results, value, key) {
+              return { ...results, [key]: 1 };
+            },
+            { _id: 1 }
+          ),
+        );
+
+        // Add additional data from mongodb into Meili search hits
+        const populatedHits = data.hits.map(function (hit) {
+          const query = {};
+          query[primaryKey] = hit[primaryKey];
+          const originalHit = _.find(hitsFromMongoose, query);
+
+          return {
+            ...(originalHit ? originalHit.toJSON() : {}),
+            ...hit
+          };
+        });
+        data.hits = populatedHits;
+      }
+
+      return data;
+    }
+
+    // Push new document to Meili
+    async addObjectToMeili() {
+      const object = _.pick(this.toJSON(), attributesToIndex);
+      // NOTE: MeiliSearch does not allow | in primary key, so we replace it with - for Bing convoIds
+      // object.conversationId = object.conversationId.replace(/\|/g, '-');
+      if (object.conversationId && object.conversationId.includes('|')) {
+        object.conversationId = object.conversationId.replace(/\|/g, '--');
+      }
+
+      try {
+        // console.log('Adding document to Meili', object);
+        await index.addDocuments([object]);
+      } catch (error) {
+        console.log('Error adding document to Meili');
+        console.error(error);
+      }
+
+      await this.collection.updateMany({ _id: this._id }, { $set: { _meiliIndex: true } });
+    }
+
+    // Update an existing document in Meili
+    async updateObjectToMeili() {
+      const object = _.pick(this.toJSON(), attributesToIndex);
+      await index.updateDocuments([object]);
+    }
+
+    // Delete a document from Meili
+    async deleteObjectFromMeili() {
+      await index.deleteDocument(this._id);
+    }
+
+    // * schema.post('save')
+    postSaveHook() {
+      if (this._meiliIndex) {
+        this.updateObjectToMeili();
+      } else {
+        this.addObjectToMeili();
+      }
+    }
+
+    // * schema.post('update')
+    postUpdateHook() {
+      if (this._meiliIndex) {
+        this.updateObjectToMeili();
+      }
+    }
+
+    // * schema.post('remove')
+    postRemoveHook() {
+      if (this._meiliIndex) {
+        this.deleteObjectFromMeili();
+      }
+    }
+  }
+
+  return MeiliMongooseModel;
+};
+
+module.exports = function mongoMeili(schema, options) {
+  // Vaidate Options for mongoMeili
+  validateOptions(options);
+
+  // Add meiliIndex to schema
+  schema.add({
+    _meiliIndex: {
+      type: Boolean,
+      required: false,
+      select: false,
+      default: false
+    }
+  });
+
+  const { host, apiKey, indexName, primaryKey } = options;
+
+  // Setup MeiliSearch Client
+  const client = new MeiliSearch({ host, apiKey });
+
+  // Asynchronously create the index
+  client.createIndex(indexName, { primaryKey });
+
+  // Setup the index to search for this schema
+  const index = client.index(indexName);
+
+  const attributesToIndex = [
+    ..._.reduce(
+      schema.obj,
+      function (results, value, key) {
+        return value.meiliIndex ? [...results, key] : results;
+        // }, []), '_id'];
+      },
+      []
+    )
+  ];
+
+  schema.loadClass(createMeiliMongooseModel({ index, indexName, client, attributesToIndex }));
+
+  // Register hooks
+  schema.post('save', function (doc) {
+    doc.postSaveHook();
+  });
+  schema.post('update', function (doc) {
+    doc.postUpdateHook();
+  });
+  schema.post('remove', function (doc) {
+    doc.postRemoveHook();
+  });
+  schema.post('findOneAndUpdate', function (doc) {
+    doc.postSaveHook();
+  });
+};

api/models/schema/convoSchema.js

@@ -1,5 +1,6 @@
 const mongoose = require('mongoose');
-module.exports = mongoose.Schema(
+const mongoMeili = require('../plugins/mongoMeili');
+const convoSchema = mongoose.Schema(
   {
     conversationId: {
       type: String,
@@ -51,3 +52,14 @@ module.exports = mongoose.Schema(
   },
   { timestamps: true }
 );
+
+convoSchema.plugin(mongoMeili, {
+  host: process.env.MEILI_HOST,
+  apiKey: process.env.MEILI_KEY,
+  indexName: 'convos', // Will get created automatically if it doesn't exist already
+  primaryKey: 'conversationId'
+});
+
+const Conversation = mongoose.models.Conversation || mongoose.model('Conversation', convoSchema);
+
+module.exports = Conversation;

api/models/schema/index.js

@@ -1,46 +0,0 @@
-const mongoose = require('mongoose');
-const convoSchema = require('./convoSchema');
-const messageSchema = require('./messageSchema');
-const { MeiliSearch } = require('meilisearch');
-const mongoMeili = require('../../lib/db/mongoMeili');
-
-(async () => {
-  try {
-    const client = new MeiliSearch({
-      host: process.env.MEILI_HOST,
-      apiKey: process.env.MEILI_KEY
-    });
-
-    const { status } = await client.health();
-    console.log(`Meilisearch: ${status}`);
-    const result = status === 'available' && !!process.env.SEARCH;
-
-    if (!result) {
-      throw new Error('Meilisearch not available');
-    }
-
-    convoSchema.plugin(mongoMeili, {
-      host: process.env.MEILI_HOST,
-      apiKey: process.env.MEILI_KEY,
-      indexName: 'convos', // Will get created automatically if it doesn't exist already
-      primaryKey: 'conversationId'
-    });
-
-    messageSchema.plugin(mongoMeili, {
-      host: process.env.MEILI_HOST,
-      apiKey: process.env.MEILI_KEY,
-      indexName: 'messages',
-      primaryKey: 'messageId'
-    });
-
-  } catch (error) {
-    console.log('Meilisearch error, search will be disabled');
-    console.error(error);
-  }
-})();
-
-const Conversation =
-mongoose.models.Conversation || mongoose.model('Conversation', convoSchema);
-const Message = mongoose.models.Message || mongoose.model('Message', messageSchema);
-
-module.exports = { Conversation, Message };

api/models/schema/messageSchema.js

@@ -1,5 +1,6 @@
 const mongoose = require('mongoose');
-module.exports = mongoose.Schema({
+const mongoMeili = require('../plugins/mongoMeili');
+const messageSchema = mongoose.Schema({
   messageId: {
     type: String,
     unique: true,
@@ -52,3 +53,14 @@ module.exports = mongoose.Schema({
     default: false 
   }
 }, { timestamps: true });
+
+messageSchema.plugin(mongoMeili, {
+  host: process.env.MEILI_HOST,
+  apiKey: process.env.MEILI_KEY,
+  indexName: 'messages',
+  primaryKey: 'messageId'
+});
+
+const Message = mongoose.models.Message || mongoose.model('Message', messageSchema);
+
+module.exports = Message;