backend logic drafted, moving to frontend

api/lib/db/migrateDb.js

@@ -36,6 +36,7 @@ async function migrateDb() {
         if (message.sender.toLowerCase() === 'user') {
           message.isCreatedByUser = true;
         }
+        
         promises.push(message.save());
       });
       await Promise.all(promises);

api/lib/db/mongoMeili.js

@@ -3,22 +3,23 @@ const _ = require('lodash');
 
 const validateOptions = function (options) {
   const requiredKeys = ['host', 'apiKey', 'indexName'];
-  requiredKeys.forEach(key => {
+  requiredKeys.forEach((key) => {
     if (!options[key]) throw new Error(`Missing mongoMeili Option: ${key}`);
   });
 };
 
 const createMeiliMongooseModel = function ({ index, indexName, client, attributesToIndex }) {
-
   console.log('attributesToIndex', attributesToIndex);
   // MeiliMongooseModel is of type Mongoose.Model
   class MeiliMongooseModel {
-
     // Clear Meili index
     static async clearMeiliIndex() {
       await index.delete();
       // await index.deleteAllDocuments();
-      await this.collection.updateMany({ _meiliIndex: true }, { $set: { _meiliIndex: false } });
+      await this.collection.updateMany(
+        { _meiliIndex: true },
+        { $set: { _meiliIndex: false } }
+      );
     }
 
     static async resetIndex() {
@@ -31,10 +32,12 @@ const createMeiliMongooseModel = function ({ index, indexName, client, attribute
       await this.resetIndex();
       // const docs = await this.find();
       const docs = await this.find({ _meiliIndex: { $in: [null, false] } });
-      console.log('docs', docs.length)
-      await Promise.all(docs.map(function(doc) {
+      console.log('docs', docs.length);
+      await Promise.all(
+        docs.map(function (doc) {
           return doc.addObjectToMeili();
-      }));
+        })
+      );
     }
 
     // Set one or more settings of the meili index
@@ -43,18 +46,23 @@ const createMeiliMongooseModel = function ({ index, indexName, client, attribute
     }
 
     // Search the index
-    static async meiliSearch({ query, params, populate }) {
-      const data = await index.search(query, params);
+    static async meiliSearch(q, params, populate) {
+      const data = await index.search(q, params);
 
       // Populate hits with content from mongodb
       if (populate) {
-        
         // Find objects into mongodb matching `objectID` from Meili search
         const hitsFromMongoose = await this.find(
           {
-            _id: { $in: _.map(data.hits, '_id') },
+            _id: { $in: _.map(data.hits, '_id') }
           },
-          _.reduce( this.schema.obj, function (results, value, key) { return { ...results, [key]: 1 } }, { _id: 1 } )
+          _.reduce(
+            this.schema.obj,
+            function (results, value, key) {
+              return { ...results, [key]: 1 };
+            },
+            { _id: 1 }
+          )
         );
 
         // Add additional data from mongodb into Meili search hits
@@ -65,7 +73,7 @@ const createMeiliMongooseModel = function ({ index, indexName, client, attribute
 
           return {
             ...(originalHit ? originalHit.toJSON() : {}),
-            ...hit,
+            ...hit
           };
         });
         data.hits = populatedHits;
@@ -77,7 +85,13 @@ const createMeiliMongooseModel = function ({ index, indexName, client, attribute
     // Push new document to Meili
     async addObjectToMeili() {
       const object = _.pick(this.toJSON(), attributesToIndex);
-      // object.id = object._id.toString();
+      // const title = (await this.getTitle()) || 'New Chat'; // Get title value
+      // const objectWithTitle = {
+      //   ...this.toJSON(),
+      //   title
+      // };
+      // const object = _.pick(objectWithTitle, attributesToIndex); // Pick desired attributes
+
       try {
         // console.log('Adding document to Meili', object);
         await index.addDocuments([object]);
@@ -86,10 +100,7 @@ const createMeiliMongooseModel = function ({ index, indexName, client, attribute
         console.error(error);
       }
 
-      await this.collection.updateMany(
-        { _id: this._id },
-        { $set: { _meiliIndex: true } }
-      );
+      await this.collection.updateMany({ _id: this._id }, { $set: { _meiliIndex: true } });
     }
 
     // Update an existing document in Meili
@@ -128,10 +139,9 @@ const createMeiliMongooseModel = function ({ index, indexName, client, attribute
   }
 
   return MeiliMongooseModel;
-}
+};
 
 module.exports = function mongoMeili(schema, options) {
-
   // Vaidate Options for mongoMeili
   validateOptions(options);
 
@@ -145,27 +155,41 @@ module.exports = function mongoMeili(schema, options) {
     }
   });
 
-  const { host, apiKey, indexName } = options;
+  const { host, apiKey, indexName, primaryKey } = options;
 
   // Setup MeiliSearch Client
   const client = new MeiliSearch({ host, apiKey });
 
   // Asynchronously create the index
-  client.createIndex(indexName, { primaryKey: 'messageId' });
+  client.createIndex(indexName, { primaryKey });
 
   // Setup the index to search for this schema
   const index = client.index(indexName);
 
-  const attributesToIndex = [..._.reduce(schema.obj, function (results, value, key) {
+  const attributesToIndex = [
+    ..._.reduce(
+      schema.obj,
+      function (results, value, key) {
         return value.meiliIndex ? [...results, key] : results;
         // }, []), '_id'];
-  }, [])];
+      },
+      []
+    )
+  ];
 
   schema.loadClass(createMeiliMongooseModel({ index, indexName, client, attributesToIndex }));
 
   // Register hooks
-  schema.post('save', function (doc) { doc.postSaveHook() });
-  schema.post('update', function (doc) { doc.postUpdateHook() });
-  schema.post('remove', function (doc) { doc.postRemoveHook() });
-  schema.post('findOneAndUpdate', function(doc) { doc.postSaveHook() });
+  schema.post('save', function (doc) {
+    doc.postSaveHook();
+  });
+  schema.post('update', function (doc) {
+    doc.postUpdateHook();
+  });
+  schema.post('remove', function (doc) {
+    doc.postRemoveHook();
+  });
+  schema.post('findOneAndUpdate', function (doc) {
+    doc.postSaveHook();
+  });
 };

api/lib/utils/reduceHits.js

@@ -1,6 +1,6 @@
 const mergeSort = require('./mergeSort');
 
-function reduceHits(hits) {
+function reduceMessages(hits) {
   const counts = {};
 
   for (const hit of hits) {
@@ -23,4 +23,34 @@ function reduceHits(hits) {
   return mergeSort(result, (a, b) => b.count - a.count);
 }
 
-module.exports = reduceHits;
+function reduceHits(hits, titles = []) {
+  const counts = {};
+  const titleMap = {};
+  const convos = [...hits, ...titles];
+
+  for (const convo of convos) {
+    if (!counts[convo.conversationId]) {
+      counts[convo.conversationId] = 1;
+    } else {
+      counts[convo.conversationId]++;
+    }
+
+    if (convo.title) {
+      titleMap[convo.conversationId] = convo._formatted.title;
+    }
+  }
+
+  const result = [];
+
+  for (const [conversationId, count] of Object.entries(counts)) {
+    result.push({
+      conversationId,
+      count,
+      title: titleMap[conversationId] ? titleMap[conversationId] : null
+    });
+  }
+
+  return mergeSort(result, (a, b) => b.count - a.count);
+}
+
+module.exports = { reduceMessages, reduceHits };

api/models/Config.js

@@ -0,0 +1,84 @@
+const mongoose = require('mongoose');
+const major = [0, 0];
+const minor = [0, 0];
+const patch = [0, 5];
+
+const configSchema = mongoose.Schema(
+  {
+    tag: {
+      type: String,
+      required: true,
+      validate: {
+        validator: function (tag) {
+          const [part1, part2, part3] = tag.replace('v', '').split('.').map(Number);
+
+          // Check if all parts are numbers
+          if (isNaN(part1) || isNaN(part2) || isNaN(part3)) {
+            return false;
+          }
+
+          // Check if all parts are within their respective ranges
+          if (part1 < major[0] || part1 > major[1]) {
+            return false;
+          }
+          if (part2 < minor[0] || part2 > minor[1]) {
+            return false;
+          }
+          if (part3 < patch[0] || part3 > patch[1]) {
+            return false;
+          }
+          return true;
+        },
+        message: 'Invalid tag value'
+      }
+    },
+    searchEnabled: {
+      type: Boolean,
+      default: false
+    },
+    usersEnabled: {
+      type: Boolean,
+      default: false
+    },
+    startupCounts: {
+      type: Number,
+      default: 0
+    }
+  },
+  { timestamps: true }
+);
+
+// Instance method
+ConfigSchema.methods.incrementCount = function () {
+  this.startupCounts += 1;
+};
+
+// Static methods
+ConfigSchema.statics.findByTag = async function (tag) {
+  return await this.findOne({ tag });
+};
+
+ConfigSchema.statics.updateByTag = async function (tag, update) {
+  return await this.findOneAndUpdate({ tag }, update, { new: true });
+};
+
+const Config = mongoose.models.Config || mongoose.model('Config', configSchema);
+
+module.exports = {
+  getConfigs: async (filter) => {
+    try {
+      return await Config.find(filter).exec();
+    } catch (error) {
+      console.error(error);
+      return { config: 'Error getting configs' };
+    }
+  },
+  deleteConfigs: async (filter) => {
+    try {
+      return await Config.deleteMany(filter).exec();
+    } catch (error) {
+      console.error(error);
+      return { config: 'Error deleting configs' };
+    }
+  }
+};

api/models/Conversation.js

@@ -1,4 +1,5 @@
 const mongoose = require('mongoose');
+const mongoMeili = require('../lib/db/mongoMeili');
 const { getMessages, deleteMessages } = require('./Message');
 
 const convoSchema = mongoose.Schema(
@@ -6,7 +7,9 @@ const convoSchema = mongoose.Schema(
     conversationId: {
       type: String,
       unique: true,
-      required: true
+      required: true,
+      index: true,
+      meiliIndex: true
     },
     parentMessageId: {
       type: String,
@@ -14,7 +17,8 @@ const convoSchema = mongoose.Schema(
     },
     title: {
       type: String,
-      default: 'New Chat'
+      default: 'New Chat',
+      meiliIndex: true
     },
     jailbreakConversationId: {
       type: String,
@@ -51,6 +55,13 @@ const convoSchema = mongoose.Schema(
   { timestamps: true }
 );
 
+convoSchema.plugin(mongoMeili, {
+  host: process.env.MEILI_HOST,
+  apiKey: process.env.MEILI_KEY,
+  indexName: 'convos', // Will get created automatically if it doesn't exist already
+  primaryKey: 'conversationId',
+});
+
 const Conversation =
   mongoose.models.Conversation || mongoose.model('Conversation', convoSchema);
 

api/models/Message.js

@@ -1,11 +1,12 @@
 const mongoose = require('mongoose');
-const mongoMeili = require('../lib/mongoMeili');
+const mongoMeili = require('../lib/db/mongoMeili');
 
 const messageSchema = mongoose.Schema({
   messageId: {
     type: String,
     unique: true,
     required: true,
+    index: true,
     meiliIndex: true
   },
   conversationId: {
@@ -55,9 +56,10 @@ const messageSchema = mongoose.Schema({
 }, { timestamps: true });
 
 messageSchema.plugin(mongoMeili, {
-  host: 'http://localhost:7700',
-  apiKey: 'MASTER_KEY',
-  indexName: 'messages' // Will get created automatically if it doesn't exist already
+  host: process.env.MEILI_HOST,
+  apiKey: process.env.MEILI_KEY,
+  indexName: 'messages', // Will get created automatically if it doesn't exist already
+  primaryKey: 'messageId',
 });
 
 const Message = mongoose.models.Message || mongoose.model('Message', messageSchema);

api/package-lock.json

@@ -14,6 +14,7 @@
         "@waylaidwanderer/chatgpt-api": "^1.28.2",
         "axios": "^1.3.4",
         "cors": "^2.8.5",
+        "crypto": "^1.0.1",
         "dotenv": "^16.0.3",
         "express": "^4.18.2",
         "express-session": "^1.17.3",
@@ -2237,6 +2238,12 @@
         "node": ">= 8"
       }
     },
+    "node_modules/crypto": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/crypto/-/crypto-1.0.1.tgz",
+      "integrity": "sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig==",
+      "deprecated": "This package is no longer supported. It's now a built-in Node module. If you've depended on crypto, you should switch to the one that's built-in."
+    },
     "node_modules/debug": {
       "version": "4.3.4",
       "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
@@ -6886,6 +6893,11 @@
         "which": "^2.0.1"
       }
     },
+    "crypto": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/crypto/-/crypto-1.0.1.tgz",
+      "integrity": "sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig=="
+    },
     "debug": {
       "version": "4.3.4",
       "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",

api/server/routes/search.js

@@ -1,24 +1,25 @@
 const express = require('express');
 const router = express.Router();
 const { Message } = require('../../models/Message');
-const reduceHits = require('../../lib/utils/reduceHits');
+const { Conversation } = require('../../models/Conversation');
+const {reduceMessages, reduceHits} = require('../../lib/utils/reduceHits');
 // const { MeiliSearch } = require('meilisearch');
 
 router.get('/sync', async function (req, res) {
-  // await Message.setMeiliIndexSettings({ primaryKey: 'messageId' });
-  // res.send('updated settings');
-  // await Message.clearMeiliIndex();
-  // res.send('deleted index');
   await Message.syncWithMeili();
+  await Conversation.syncWithMeili();
   res.send('synced');
 });
 
 router.get('/', async function (req, res) {
   const { q } = req.query;
-  const result = await Message.meiliSearch({ query: q });
-  const sortedHits = reduceHits(result.hits);
-  console.log(sortedHits);
-  res.send(sortedHits);
+  const message = await Message.meiliSearch(q, { attributesToHighlight: ['text', 'sender'] });
+  const title = await Conversation.meiliSearch(q, { attributesToHighlight: ['title'] });
+  // console.log('titles', title);
+  // console.log(sortedHits);
+  const sortedHits = reduceHits(message.hits, title.hits);
+  // const sortedHits = reduceMessages(message.hits);
+  res.status(200).send({sortedHits});
 });
 
 router.get('/clear', async function (req, res) {