feat: reorganize api files, add mongoMeili

api/app/index.js

@@ -1,12 +1,12 @@
-const { askClient } = require('./chatgpt-client');
+const { askClient } = require('./clients/chatgpt-client');
-const { browserClient } = require('./chatgpt-browser');
+const { browserClient } = require('./clients/chatgpt-browser');
-const customClient = require('./chatgpt-custom');
+const { askBing } = require('./clients/bingai');
-const { askBing } = require('./bingai');
+const { askSydney } = require('./clients/sydney');
-const { askSydney } = require('./sydney');
+const customClient = require('./clients/chatgpt-custom');
 const titleConvo = require('./titleConvo');
-const getCitations = require('./getCitations');
+const getCitations = require('../lib/parse/getCitations');
-const citeText = require('./citeText');
+const citeText = require('../lib/parse/citeText');
-const detectCode = require('./detectCode');
+const detectCode = require('../lib/parse/detectCode');
 
 module.exports = {
   askClient,

api/app/search.js

@@ -1,24 +0,0 @@
-const mongoose = require('mongoose');
-const mongomeili = require('mongomeili');
-const { messageSchema, Message } = require('../models/Message');
-
-// Add the '{ meiliIndex: true }' property to index these attributes with MeiliSearch
-// const MovieSchema = new mongoose.Schema({
-//   title: { type: String, required: true, meiliIndex: true },
-//   director: { type: String, required: true, meiliIndex: true },
-//   year: { type: String, required: true, meiliIndex: true }
-// });
-
-// Specify your MeiliSearch credentials
-// messageSchema.plugin(mongomeili, {
-//   host: 'http://localhost:7700',
-//   apiKey: 'MASTER_KEY',
-//   indexName: 'messages' // Will get created automatically if it doesn't exist already
-// });
-
-(async () => {
-  await Message.syncWithMeili();
-  const result = await Message.meiliSearch({ query: 'quantum' });
-  
-  console.log(result);
-})();

api/lib/mongoMeili.js

@@ -0,0 +1,171 @@
+const { MeiliSearch } = require('meilisearch');
+const _ = require('lodash');
+
+const validateOptions = function (options) {
+  const requiredKeys = ['host', 'apiKey', 'indexName'];
+  requiredKeys.forEach(key => {
+    if (!options[key]) throw new Error(`Missing mongoMeili Option: ${key}`);
+  });
+};
+
+const createMeiliMongooseModel = function ({ index, indexName, client, attributesToIndex }) {
+
+  console.log('attributesToIndex', attributesToIndex);
+  // MeiliMongooseModel is of type Mongoose.Model
+  class MeiliMongooseModel {
+
+    // Clear Meili index
+    static async clearMeiliIndex() {
+      await index.delete();
+      // await index.deleteAllDocuments();
+      await this.collection.updateMany({ _meiliIndex: true }, { $set: { _meiliIndex: false } });
+    }
+  
+    static async resetIndex() {
+      await this.clearMeiliIndex();
+      await client.createIndex(indexName, { primaryKey: attributesToIndex[0] });
+    }
+    // Clear Meili index
+    // Push a mongoDB collection to Meili index
+    static async syncWithMeili() {
+      await this.resetIndex();
+      // const docs = await this.find();
+      const docs = await this.find({ _meiliIndex: { $in: [null, false] } });
+      console.log('docs', docs.length)
+      await Promise.all(docs.map(function(doc) {
+        return doc.addObjectToMeili();
+      }));
+    }
+  
+    // Set one or more settings of the meili index
+    static async setMeiliIndexSettings(settings) {
+      return await index.updateSettings(settings);
+    }
+  
+    // Search the index
+    static async meiliSearch({ query, params, populate }) {
+      const data = await index.search(query, params);
+  
+      // Populate hits with content from mongodb
+      if (populate) {
+        
+        // Find objects into mongodb matching `objectID` from Meili search
+        const hitsFromMongoose = await this.find(
+          {
+            _id: { $in: _.map(data.hits, '_id') },
+          },
+          _.reduce( this.schema.obj, function (results, value, key) { return { ...results, [key]: 1 } }, { _id: 1 } )
+        );
+  
+        // Add additional data from mongodb into Meili search hits
+        const populatedHits = data.hits.map(function(hit) {
+          const originalHit = _.find(hitsFromMongoose, {
+            _id: hit._id
+          });
+
+          return {
+            ...(originalHit ? originalHit.toJSON() : {}),
+            ...hit,
+          };
+        });
+        data.hits = populatedHits;
+      }
+  
+      return data;
+    }
+  
+    // Push new document to Meili
+    async addObjectToMeili() {
+      const object = _.pick(this.toJSON(), attributesToIndex);
+      // object.id = object._id.toString();
+      try {
+        // console.log('Adding document to Meili', object);
+        await index.addDocuments([object]);
+      } catch (error) {
+        console.log('Error adding document to Meili');
+        console.error(error);
+      }
+
+      await this.collection.updateMany(
+        { _id: this._id },
+        { $set: { _meiliIndex: true } }
+      );
+    }
+  
+    // Update an existing document in Meili
+    async updateObjectToMeili() {
+      const object = pick(this.toJSON(), attributesToIndex);
+      await index.updateDocuments([object]);
+    }
+  
+    // Delete a document from Meili
+    async deleteObjectFromMeili() {
+      await index.deleteDocument(this._id);
+    }
+  
+    // * schema.post('save')
+    postSaveHook() {
+      if (this._meiliIndex) {
+        this.updateObjectToMeili();
+      } else {
+        this.addObjectToMeili();
+      }
+    }
+  
+    // * schema.post('update')
+    postUpdateHook() {
+      if (this._meiliIndex) {
+        this.updateObjectToMeili();
+      }
+    }
+  
+    // * schema.post('remove')
+    postRemoveHook() {
+      if (this._meiliIndex) {
+        this.deleteObjectFromMeili();
+      }
+    }
+  }
+
+  return MeiliMongooseModel;
+}
+
+module.exports = function mongoMeili(schema, options) {
+
+  // Vaidate Options for mongoMeili
+  validateOptions(options);
+
+  // Add meiliIndex to schema
+  schema.add({
+    _meiliIndex: { 
+      type: Boolean, 
+      required: false, 
+      select: false, 
+      default: false 
+    }
+  });
+
+  const { host, apiKey, indexName } = options;
+
+  // Setup MeiliSearch Client
+  const client = new MeiliSearch({ host, apiKey });
+  
+  // Asynchronously create the index
+  client.createIndex(indexName, { primaryKey: 'messageId' });
+
+  // Setup the index to search for this schema
+  const index = client.index(indexName);
+
+  const attributesToIndex = [..._.reduce(schema.obj, function (results, value, key) {
+    return value.meiliIndex ? [...results, key] : results;
+  // }, []), '_id'];
+  }, [])];
+
+  schema.loadClass(createMeiliMongooseModel({ index, indexName, client, attributesToIndex }));
+
+  // Register hooks
+  schema.post('save', function (doc) { doc.postSaveHook() });
+  schema.post('update', function (doc) { doc.postUpdateHook() });
+  schema.post('remove', function (doc) { doc.postRemoveHook() });
+  schema.post('findOneAndUpdate', function(doc) { doc.postSaveHook() });
+};

api/lib/parse/detectCode.js

@@ -1,5 +1,5 @@
 const { ModelOperations } = require('@vscode/vscode-languagedetection');
-const languages = require('../utils/languages.js');
+const languages = require('./languages.js');
 const codeRegex = /(```[\s\S]*?```)/g;
 // const languageMatch = /```(\w+)/;
 const replaceRegex = /```\w+\n/g;

api/models/Message.js

@@ -1,13 +1,12 @@
 const mongoose = require('mongoose');
-// const mongomeili = require('mongomeili');
+const mongoMeili = require('../lib/mongoMeili');
-const MeiliSearch = require('meilisearch');
-const _ = require('lodash');
 
 const messageSchema = mongoose.Schema({
   messageId: {
     type: String,
     unique: true,
-    required: true
+    required: true,
+    meiliIndex: true
   },
   conversationId: {
     type: String,
@@ -55,11 +54,11 @@ const messageSchema = mongoose.Schema({
   }
 }, { timestamps: true });
 
-// messageSchema.plugin(mongomeili, {
+messageSchema.plugin(mongoMeili, {
-//   host: 'http://localhost:7700',
+  host: 'http://localhost:7700',
-//   apiKey: 'MASTER_KEY',
+  apiKey: 'MASTER_KEY',
-//   indexName: 'messages' // Will get created automatically if it doesn't exist already
+  indexName: 'messages' // Will get created automatically if it doesn't exist already
-// });
+});
 
 const Message = mongoose.models.Message || mongoose.model('Message', messageSchema);
 

api/server/routes/search.js

@@ -1,13 +1,22 @@
 const express = require('express');
 const router = express.Router();
 const { Message } = require('../../models/Message');
+// const { MeiliSearch } = require('meilisearch');
 
 router.get('/sync', async function (req, res) {
-  console.log(Message);
+  // await Message.setMeiliIndexSettings({ primaryKey: 'messageId' });
+  // res.send('updated settings');
+  // await Message.clearMeiliIndex();
+  // res.send('deleted index');
   await Message.syncWithMeili();
-  const result = await Message.meiliSearch({ query: 'computing' });
+  res.send('synced');
+});
+
+router.get('/', async function (req, res) {
+  const { q } = req.query;
+  const result = await Message.meiliSearch({ query: q });
   console.log(result);
   res.send(result);
-})
+});
 
 module.exports = router;