feat: utitlize lean queries, remove migration script, index createdAt timestamps (#698)

* feat(mongoDb): utitlize lean queries and index createdAt timestamps for cosmosDB support

* fix: remove unnecessary lean() method from deleteMany calls

* fix: remove unnecessary lean() method from deleteMany calls

* fix: remove lean() from queries that need hydration

* chore(migrateDb.js): remove unused migration script
fix(Preset.js): return lean documents when retrieving presets
refactor(index.js): remove migration script from server initialization
refactor(convos.js): remove toObject() when sending conversation object
refactor(presets.js): remove toObject() when sending presets object

api/lib/db/migrateDb.js

@@ -1,124 +0,0 @@
-const mongoose = require('mongoose');
-const { Conversation } = require('../../models/Conversation');
-const { getMessages } = require('../../models/');
-
-const migrateToStrictFollowParentMessageIdChain = async () => {
-  try {
-    const conversations = await Conversation.find({ endpoint: null, model: null }).exec();
-
-    if (!conversations || conversations.length === 0) {
-      return { noNeed: true };
-    }
-
-    console.log('Migration: To strict follow the parentMessageId chain.');
-
-    for (let convo of conversations) {
-      const messages = await getMessages({
-        conversationId: convo.conversationId,
-        messageId: { $exists: false },
-      });
-
-      let model;
-      let oldId;
-      const promises = [];
-      messages.forEach((message, i) => {
-        const msgObj = message.toObject();
-        const newId = msgObj.id;
-        if (i === 0) {
-          message.parentMessageId = '00000000-0000-0000-0000-000000000000';
-        } else {
-          message.parentMessageId = oldId;
-        }
-
-        oldId = newId;
-        message.messageId = newId;
-        if (message.sender.toLowerCase() !== 'user' && !model) {
-          model = message.sender.toLowerCase();
-        }
-
-        if (message.sender.toLowerCase() === 'user') {
-          message.isCreatedByUser = true;
-        }
-
-        promises.push(message.save());
-      });
-      await Promise.all(promises);
-
-      await Conversation.findOneAndUpdate(
-        { conversationId: convo.conversationId },
-        { model },
-        { new: true },
-      ).exec();
-    }
-
-    try {
-      await mongoose.connection.db.collection('messages').dropIndex('id_1');
-    } catch (error) {
-      console.log('[Migrate] Index doesn\'t exist or already dropped');
-    }
-  } catch (error) {
-    console.log(error);
-    return { message: '[Migrate] Error migrating conversations' };
-  }
-};
-
-const migrateToSupportBetterCustomization = async () => {
-  try {
-    const conversations = await Conversation.find({ endpoint: null }).exec();
-
-    if (!conversations || conversations.length === 0) {
-      return { noNeed: true };
-    }
-
-    console.log('Migration: To support better customization.');
-
-    const promises = [];
-    for (let convo of conversations) {
-      const originalModel = convo?.model;
-
-      if (originalModel === 'chatgpt') {
-        convo.endpoint = 'openAI';
-        convo.model = 'gpt-3.5-turbo';
-      } else if (originalModel === 'chatgptCustom') {
-        convo.endpoint = 'openAI';
-        convo.model = 'gpt-3.5-turbo';
-      } else if (originalModel === 'bingai') {
-        convo.endpoint = 'bingAI';
-        convo.model = null;
-        convo.jailbreak = false;
-      } else if (originalModel === 'sydney') {
-        convo.endpoint = 'bingAI';
-        convo.model = null;
-        convo.jailbreak = true;
-      } else if (originalModel === 'chatgptBrowser') {
-        convo.endpoint = 'chatGPTBrowser';
-        convo.model = 'text-davinci-002-render-sha';
-        convo.jailbreak = true;
-      } else {
-        convo.endpoint = 'openAI';
-        convo.model = 'gpt-3.5-turbo';
-      }
-
-      promises.push(convo.save());
-    }
-
-    await Promise.all(promises);
-  } catch (error) {
-    console.log(error);
-    return { message: '[Migrate] Error migrating conversations' };
-  }
-};
-
-async function migrateDb() {
-  let ret = [];
-  ret[0] = await migrateToStrictFollowParentMessageIdChain();
-  ret[1] = await migrateToSupportBetterCustomization();
-
-  const isMigrated = !!ret.find((element) => !element?.noNeed);
-
-  if (!isMigrated) {
-    console.log('[Migrate] Nothing to migrate');
-  }
-}
-
-module.exports = migrateDb;

api/models/Config.js

@@ -55,7 +55,7 @@ configSchema.methods.incrementCount = function () {
 
 // Static methods
 configSchema.statics.findByTag = async function (tag) {
-  return await this.findOne({ tag });
+  return await this.findOne({ tag }).lean();
 };
 
 configSchema.statics.updateByTag = async function (tag, update) {
@@ -67,7 +67,7 @@ const Config = mongoose.models.Config || mongoose.model('Config', configSchema);
 module.exports = {
   getConfigs: async (filter) => {
     try {
-      return await Config.find(filter).exec();
+      return await Config.find(filter).lean();
     } catch (error) {
       console.error(error);
       return { config: 'Error getting configs' };
@@ -75,7 +75,7 @@ module.exports = {
   },
   deleteConfigs: async (filter) => {
     try {
-      return await Config.deleteMany(filter).exec();
+      return await Config.deleteMany(filter);
     } catch (error) {
       console.error(error);
       return { config: 'Error deleting configs' };

api/models/Conversation.js

@@ -4,7 +4,7 @@ const { getMessages, deleteMessages } = require('./Message');
 
 const getConvo = async (user, conversationId) => {
   try {
-    return await Conversation.findOne({ user, conversationId }).exec();
+    return await Conversation.findOne({ user, conversationId }).lean();
   } catch (error) {
     console.log(error);
     return { message: 'Error getting single conversation' };
@@ -24,7 +24,7 @@ module.exports = {
       return await Conversation.findOneAndUpdate({ conversationId: conversationId, user }, update, {
         new: true,
         upsert: true,
-      }).exec();
+      });
     } catch (error) {
       console.log(error);
       return { message: 'Error saving conversation' };
@@ -35,10 +35,10 @@ module.exports = {
       const totalConvos = (await Conversation.countDocuments({ user })) || 1;
       const totalPages = Math.ceil(totalConvos / pageSize);
       const convos = await Conversation.find({ user })
-        .sort({ createdAt: -1, created: -1 })
+        .sort({ createdAt: -1 })
         .skip((pageNumber - 1) * pageSize)
         .limit(pageSize)
-        .exec();
+        .lean();
       return { conversations: convos, pages: totalPages, pageNumber, pageSize };
     } catch (error) {
       console.log(error);
@@ -62,7 +62,7 @@ module.exports = {
           Conversation.findOne({
             user,
             conversationId: convo.conversationId,
-          }).exec(),
+          }).lean(),
         ),
       );
 
@@ -121,7 +121,7 @@ module.exports = {
   deleteConvos: async (user, filter) => {
     let toRemove = await Conversation.find({ ...filter, user }).select('conversationId');
     const ids = toRemove.map((instance) => instance.conversationId);
-    let deleteCount = await Conversation.deleteMany({ ...filter, user }).exec();
+    let deleteCount = await Conversation.deleteMany({ ...filter, user });
     deleteCount.messages = await deleteMessages({ conversationId: { $in: ids } });
     return deleteCount;
   },

api/models/Message.js

@@ -78,12 +78,12 @@ module.exports = {
   },
   async deleteMessagesSince({ messageId, conversationId }) {
     try {
-      const message = await Message.findOne({ messageId }).exec();
+      const message = await Message.findOne({ messageId }).lean();
 
       if (message) {
-        return await Message.find({ conversationId })
+        return await Message.find({ conversationId }).deleteMany({
-          .deleteMany({ createdAt: { $gt: message.createdAt } })
+          createdAt: { $gt: message.createdAt },
-          .exec();
+        });
       }
     } catch (err) {
       console.error(`Error deleting messages: ${err}`);
@@ -93,7 +93,7 @@ module.exports = {
 
   async getMessages(filter) {
     try {
-      return await Message.find(filter).sort({ createdAt: 1 }).exec();
+      return await Message.find(filter).sort({ createdAt: 1 }).lean();
     } catch (err) {
       console.error(`Error getting messages: ${err}`);
       throw new Error('Failed to get messages.');
@@ -102,7 +102,7 @@ module.exports = {
 
   async deleteMessages(filter) {
     try {
-      return await Message.deleteMany(filter).exec();
+      return await Message.deleteMany(filter);
     } catch (err) {
       console.error(`Error deleting messages: ${err}`);
       throw new Error('Failed to delete messages.');

api/models/Preset.js

@@ -2,7 +2,7 @@ const Preset = require('./schema/presetSchema');
 
 const getPreset = async (user, presetId) => {
   try {
-    return await Preset.findOne({ user, presetId }).exec();
+    return await Preset.findOne({ user, presetId }).lean();
   } catch (error) {
     console.log(error);
     return { message: 'Error getting single preset' };
@@ -14,10 +14,10 @@ module.exports = {
   getPreset,
   getPresets: async (user, filter) => {
     try {
-      return await Preset.find({ ...filter, user }).exec();
+      return await Preset.find({ ...filter, user }).lean();
     } catch (error) {
       console.log(error);
-      return { message: 'Error retriving presets' };
+      return { message: 'Error retrieving presets' };
     }
   },
   savePreset: async (user, { presetId, newPresetId, ...preset }) => {
@@ -31,7 +31,7 @@ module.exports = {
         { presetId, user },
         { $set: update },
         { new: true, upsert: true },
-      ).exec();
+      );
     } catch (error) {
       console.log(error);
       return { message: 'Error saving preset' };
@@ -40,7 +40,7 @@ module.exports = {
   deletePresets: async (user, filter) => {
     // let toRemove = await Preset.find({ ...filter, user }).select('presetId');
     // const ids = toRemove.map((instance) => instance.presetId);
-    let deleteCount = await Preset.deleteMany({ ...filter, user }).exec();
+    let deleteCount = await Preset.deleteMany({ ...filter, user });
     return deleteCount;
   },
 };

api/models/Prompt.js

@@ -34,7 +34,7 @@ module.exports = {
   },
   getPrompts: async (filter) => {
     try {
-      return await Prompt.find(filter).exec();
+      return await Prompt.find(filter).lean();
     } catch (error) {
       console.error(error);
       return { prompt: 'Error getting prompts' };
@@ -42,7 +42,7 @@ module.exports = {
   },
   deletePrompts: async (filter) => {
     try {
-      return await Prompt.deleteMany(filter).exec();
+      return await Prompt.deleteMany(filter);
     } catch (error) {
       console.error(error);
       return { prompt: 'Error deleting prompts' };

api/models/schema/convoSchema.js

@@ -61,6 +61,8 @@ if (process.env.MEILI_HOST && process.env.MEILI_MASTER_KEY) {
   });
 }
 
+convoSchema.index({ createdAt: 1 });
+
 const Conversation = mongoose.models.Conversation || mongoose.model('Conversation', convoSchema);
 
 module.exports = Conversation;

api/models/schema/messageSchema.js

@@ -100,6 +100,8 @@ if (process.env.MEILI_HOST && process.env.MEILI_MASTER_KEY) {
   });
 }
 
+messageSchema.index({ createdAt: 1 });
+
 const Message = mongoose.models.Message || mongoose.model('Message', messageSchema);
 
 module.exports = Message;

api/server/index.js

@@ -1,7 +1,6 @@
 const express = require('express');
 const session = require('express-session');
 const connectDb = require('../lib/db/connectDb');
-const migrateDb = require('../lib/db/migrateDb');
 const indexSync = require('../lib/db/indexSync');
 const path = require('path');
 const cors = require('cors');
@@ -28,7 +27,6 @@ config.validate(); // Validate the config
 (async () => {
   await connectDb();
   console.log('Connected to MongoDB');
-  await migrateDb();
   await indexSync();
 
   const app = express();

api/server/routes/convos.js

@@ -14,7 +14,7 @@ router.get('/:conversationId', requireJwtAuth, async (req, res) => {
   const convo = await getConvo(req.user.id, conversationId);
 
   if (convo) {
-    res.status(200).send(convo.toObject());
+    res.status(200).send(convo);
   } else {
     res.status(404).end();
   }

api/server/routes/presets.js

@@ -6,7 +6,7 @@ const requireJwtAuth = require('../../middleware/requireJwtAuth');
 
 router.get('/', requireJwtAuth, async (req, res) => {
   const presets = (await getPresets(req.user.id)).map((preset) => {
-    return preset.toObject();
+    return preset;
   });
   res.status(200).send(presets);
 });
@@ -20,7 +20,7 @@ router.post('/', requireJwtAuth, async (req, res) => {
     await savePreset(req.user.id, update);
 
     const presets = (await getPresets(req.user.id)).map((preset) => {
-      return preset.toObject();
+      return preset;
     });
     res.status(201).send(presets);
   } catch (error) {
@@ -41,12 +41,8 @@ router.post('/delete', requireJwtAuth, async (req, res) => {
 
   try {
     await deletePresets(req.user.id, filter);
-
+    const presets = await getPresets(req.user.id);
-    const presets = (await getPresets(req.user.id)).map((preset) => preset.toObject());
-
-    // console.log('delete preset response', presets);
     res.status(201).send(presets);
-    // res.status(201).send(dbResponse);
   } catch (error) {
     console.error(error);
     res.status(500).send(error);

api/server/services/PluginService.js

@@ -3,7 +3,7 @@ const { encrypt, decrypt } = require('../../utils/');
 
 const getUserPluginAuthValue = async (user, authField) => {
   try {
-    const pluginAuth = await PluginAuth.findOne({ user, authField });
+    const pluginAuth = await PluginAuth.findOne({ user, authField }).lean();
     if (!pluginAuth) {
       return null;
     }
@@ -43,7 +43,7 @@ const getUserPluginAuthValue = async (user, authField) => {
 const updateUserPluginAuth = async (userId, authField, pluginKey, value) => {
   try {
     const encryptedValue = encrypt(value);
-    const pluginAuth = await PluginAuth.findOne({ userId, authField });
+    const pluginAuth = await PluginAuth.findOne({ userId, authField }).lean();
     if (pluginAuth) {
       const pluginAuth = await PluginAuth.updateOne(
         { userId, authField },

api/server/services/auth.service.js

@@ -54,7 +54,7 @@ const registerUser = async (user) => {
   const { email, password, name, username } = user;
 
   try {
-    const existingUser = await User.findOne({ email });
+    const existingUser = await User.findOne({ email }).lean();
 
     if (existingUser) {
       console.info(
@@ -104,7 +104,7 @@ const registerUser = async (user) => {
  * @returns
  */
 const requestPasswordReset = async (email) => {
-  const user = await User.findOne({ email });
+  const user = await User.findOne({ email }).lean();
   if (!user) {
     return new Error('Email does not exist');
   }