diff --git a/api/lib/db/migrateDb.js b/api/lib/db/migrateDb.js index 2bd1cb358f..315f7094bf 100644 --- a/api/lib/db/migrateDb.js +++ b/api/lib/db/migrateDb.js @@ -36,6 +36,7 @@ async function migrateDb() { if (message.sender.toLowerCase() === 'user') { message.isCreatedByUser = true; } + promises.push(message.save()); }); await Promise.all(promises); diff --git a/api/lib/mongoMeili.js b/api/lib/db/mongoMeili.js similarity index 66% rename from api/lib/mongoMeili.js rename to api/lib/db/mongoMeili.js index a7514d1918..5890f3ed20 100644 --- a/api/lib/mongoMeili.js +++ b/api/lib/db/mongoMeili.js @@ -3,24 +3,25 @@ const _ = require('lodash'); const validateOptions = function (options) { const requiredKeys = ['host', 'apiKey', 'indexName']; - requiredKeys.forEach(key => { + requiredKeys.forEach((key) => { if (!options[key]) throw new Error(`Missing mongoMeili Option: ${key}`); }); }; const createMeiliMongooseModel = function ({ index, indexName, client, attributesToIndex }) { - console.log('attributesToIndex', attributesToIndex); // MeiliMongooseModel is of type Mongoose.Model class MeiliMongooseModel { - // Clear Meili index static async clearMeiliIndex() { await index.delete(); // await index.deleteAllDocuments(); - await this.collection.updateMany({ _meiliIndex: true }, { $set: { _meiliIndex: false } }); + await this.collection.updateMany( + { _meiliIndex: true }, + { $set: { _meiliIndex: false } } + ); } - + static async resetIndex() { await this.clearMeiliIndex(); await client.createIndex(indexName, { primaryKey: attributesToIndex[0] }); @@ -31,53 +32,66 @@ const createMeiliMongooseModel = function ({ index, indexName, client, attribute await this.resetIndex(); // const docs = await this.find(); const docs = await this.find({ _meiliIndex: { $in: [null, false] } }); - console.log('docs', docs.length) - await Promise.all(docs.map(function(doc) { - return doc.addObjectToMeili(); - })); + console.log('docs', docs.length); + await Promise.all( + docs.map(function (doc) { + return doc.addObjectToMeili(); + }) + ); } - + // Set one or more settings of the meili index static async setMeiliIndexSettings(settings) { return await index.updateSettings(settings); } - + // Search the index - static async meiliSearch({ query, params, populate }) { - const data = await index.search(query, params); - + static async meiliSearch(q, params, populate) { + const data = await index.search(q, params); + // Populate hits with content from mongodb if (populate) { - // Find objects into mongodb matching `objectID` from Meili search const hitsFromMongoose = await this.find( { - _id: { $in: _.map(data.hits, '_id') }, + _id: { $in: _.map(data.hits, '_id') } }, - _.reduce( this.schema.obj, function (results, value, key) { return { ...results, [key]: 1 } }, { _id: 1 } ) + _.reduce( + this.schema.obj, + function (results, value, key) { + return { ...results, [key]: 1 }; + }, + { _id: 1 } + ) ); - + // Add additional data from mongodb into Meili search hits - const populatedHits = data.hits.map(function(hit) { + const populatedHits = data.hits.map(function (hit) { const originalHit = _.find(hitsFromMongoose, { _id: hit._id }); return { ...(originalHit ? originalHit.toJSON() : {}), - ...hit, + ...hit }; }); data.hits = populatedHits; } - + return data; } - + // Push new document to Meili async addObjectToMeili() { const object = _.pick(this.toJSON(), attributesToIndex); - // object.id = object._id.toString(); + // const title = (await this.getTitle()) || 'New Chat'; // Get title value + // const objectWithTitle = { + // ...this.toJSON(), + // title + // }; + // const object = _.pick(objectWithTitle, attributesToIndex); // Pick desired attributes + try { // console.log('Adding document to Meili', object); await index.addDocuments([object]); @@ -86,23 +100,20 @@ const createMeiliMongooseModel = function ({ index, indexName, client, attribute console.error(error); } - await this.collection.updateMany( - { _id: this._id }, - { $set: { _meiliIndex: true } } - ); + await this.collection.updateMany({ _id: this._id }, { $set: { _meiliIndex: true } }); } - + // Update an existing document in Meili async updateObjectToMeili() { const object = pick(this.toJSON(), attributesToIndex); await index.updateDocuments([object]); } - + // Delete a document from Meili async deleteObjectFromMeili() { await index.deleteDocument(this._id); } - + // * schema.post('save') postSaveHook() { if (this._meiliIndex) { @@ -111,14 +122,14 @@ const createMeiliMongooseModel = function ({ index, indexName, client, attribute this.addObjectToMeili(); } } - + // * schema.post('update') postUpdateHook() { if (this._meiliIndex) { this.updateObjectToMeili(); } } - + // * schema.post('remove') postRemoveHook() { if (this._meiliIndex) { @@ -128,44 +139,57 @@ const createMeiliMongooseModel = function ({ index, indexName, client, attribute } return MeiliMongooseModel; -} +}; module.exports = function mongoMeili(schema, options) { - // Vaidate Options for mongoMeili validateOptions(options); // Add meiliIndex to schema schema.add({ - _meiliIndex: { - type: Boolean, - required: false, - select: false, - default: false + _meiliIndex: { + type: Boolean, + required: false, + select: false, + default: false } }); - const { host, apiKey, indexName } = options; + const { host, apiKey, indexName, primaryKey } = options; // Setup MeiliSearch Client const client = new MeiliSearch({ host, apiKey }); - + // Asynchronously create the index - client.createIndex(indexName, { primaryKey: 'messageId' }); + client.createIndex(indexName, { primaryKey }); // Setup the index to search for this schema const index = client.index(indexName); - const attributesToIndex = [..._.reduce(schema.obj, function (results, value, key) { - return value.meiliIndex ? [...results, key] : results; - // }, []), '_id']; - }, [])]; + const attributesToIndex = [ + ..._.reduce( + schema.obj, + function (results, value, key) { + return value.meiliIndex ? [...results, key] : results; + // }, []), '_id']; + }, + [] + ) + ]; schema.loadClass(createMeiliMongooseModel({ index, indexName, client, attributesToIndex })); // Register hooks - schema.post('save', function (doc) { doc.postSaveHook() }); - schema.post('update', function (doc) { doc.postUpdateHook() }); - schema.post('remove', function (doc) { doc.postRemoveHook() }); - schema.post('findOneAndUpdate', function(doc) { doc.postSaveHook() }); -}; \ No newline at end of file + schema.post('save', function (doc) { + doc.postSaveHook(); + }); + schema.post('update', function (doc) { + doc.postUpdateHook(); + }); + schema.post('remove', function (doc) { + doc.postRemoveHook(); + }); + schema.post('findOneAndUpdate', function (doc) { + doc.postSaveHook(); + }); +}; diff --git a/api/lib/utils/reduceHits.js b/api/lib/utils/reduceHits.js index ef94af3f5c..2cc6b7940d 100644 --- a/api/lib/utils/reduceHits.js +++ b/api/lib/utils/reduceHits.js @@ -1,8 +1,8 @@ const mergeSort = require('./mergeSort'); -function reduceHits(hits) { +function reduceMessages(hits) { const counts = {}; - + for (const hit of hits) { if (!counts[hit.conversationId]) { counts[hit.conversationId] = 1; @@ -10,17 +10,47 @@ function reduceHits(hits) { counts[hit.conversationId]++; } } - + const result = []; - + for (const [conversationId, count] of Object.entries(counts)) { result.push({ conversationId, count }); } - + return mergeSort(result, (a, b) => b.count - a.count); } -module.exports = reduceHits; \ No newline at end of file +function reduceHits(hits, titles = []) { + const counts = {}; + const titleMap = {}; + const convos = [...hits, ...titles]; + + for (const convo of convos) { + if (!counts[convo.conversationId]) { + counts[convo.conversationId] = 1; + } else { + counts[convo.conversationId]++; + } + + if (convo.title) { + titleMap[convo.conversationId] = convo._formatted.title; + } + } + + const result = []; + + for (const [conversationId, count] of Object.entries(counts)) { + result.push({ + conversationId, + count, + title: titleMap[conversationId] ? titleMap[conversationId] : null + }); + } + + return mergeSort(result, (a, b) => b.count - a.count); +} + +module.exports = { reduceMessages, reduceHits }; diff --git a/api/models/Config.js b/api/models/Config.js new file mode 100644 index 0000000000..57192a461e --- /dev/null +++ b/api/models/Config.js @@ -0,0 +1,84 @@ +const mongoose = require('mongoose'); +const major = [0, 0]; +const minor = [0, 0]; +const patch = [0, 5]; + +const configSchema = mongoose.Schema( + { + tag: { + type: String, + required: true, + validate: { + validator: function (tag) { + const [part1, part2, part3] = tag.replace('v', '').split('.').map(Number); + + // Check if all parts are numbers + if (isNaN(part1) || isNaN(part2) || isNaN(part3)) { + return false; + } + + // Check if all parts are within their respective ranges + if (part1 < major[0] || part1 > major[1]) { + return false; + } + if (part2 < minor[0] || part2 > minor[1]) { + return false; + } + if (part3 < patch[0] || part3 > patch[1]) { + return false; + } + return true; + }, + message: 'Invalid tag value' + } + }, + searchEnabled: { + type: Boolean, + default: false + }, + usersEnabled: { + type: Boolean, + default: false + }, + startupCounts: { + type: Number, + default: 0 + } + }, + { timestamps: true } +); + +// Instance method +ConfigSchema.methods.incrementCount = function () { + this.startupCounts += 1; +}; + +// Static methods +ConfigSchema.statics.findByTag = async function (tag) { + return await this.findOne({ tag }); +}; + +ConfigSchema.statics.updateByTag = async function (tag, update) { + return await this.findOneAndUpdate({ tag }, update, { new: true }); +}; + +const Config = mongoose.models.Config || mongoose.model('Config', configSchema); + +module.exports = { + getConfigs: async (filter) => { + try { + return await Config.find(filter).exec(); + } catch (error) { + console.error(error); + return { config: 'Error getting configs' }; + } + }, + deleteConfigs: async (filter) => { + try { + return await Config.deleteMany(filter).exec(); + } catch (error) { + console.error(error); + return { config: 'Error deleting configs' }; + } + } +}; diff --git a/api/models/Conversation.js b/api/models/Conversation.js index ea48644318..81d3646d98 100644 --- a/api/models/Conversation.js +++ b/api/models/Conversation.js @@ -1,4 +1,5 @@ const mongoose = require('mongoose'); +const mongoMeili = require('../lib/db/mongoMeili'); const { getMessages, deleteMessages } = require('./Message'); const convoSchema = mongoose.Schema( @@ -6,7 +7,9 @@ const convoSchema = mongoose.Schema( conversationId: { type: String, unique: true, - required: true + required: true, + index: true, + meiliIndex: true }, parentMessageId: { type: String, @@ -14,7 +17,8 @@ const convoSchema = mongoose.Schema( }, title: { type: String, - default: 'New Chat' + default: 'New Chat', + meiliIndex: true }, jailbreakConversationId: { type: String, @@ -51,6 +55,13 @@ const convoSchema = mongoose.Schema( { timestamps: true } ); +convoSchema.plugin(mongoMeili, { + host: process.env.MEILI_HOST, + apiKey: process.env.MEILI_KEY, + indexName: 'convos', // Will get created automatically if it doesn't exist already + primaryKey: 'conversationId', +}); + const Conversation = mongoose.models.Conversation || mongoose.model('Conversation', convoSchema); diff --git a/api/models/Message.js b/api/models/Message.js index 46eb065ea6..70e82dfb73 100644 --- a/api/models/Message.js +++ b/api/models/Message.js @@ -1,11 +1,12 @@ const mongoose = require('mongoose'); -const mongoMeili = require('../lib/mongoMeili'); +const mongoMeili = require('../lib/db/mongoMeili'); const messageSchema = mongoose.Schema({ messageId: { type: String, unique: true, required: true, + index: true, meiliIndex: true }, conversationId: { @@ -55,9 +56,10 @@ const messageSchema = mongoose.Schema({ }, { timestamps: true }); messageSchema.plugin(mongoMeili, { - host: 'http://localhost:7700', - apiKey: 'MASTER_KEY', - indexName: 'messages' // Will get created automatically if it doesn't exist already + host: process.env.MEILI_HOST, + apiKey: process.env.MEILI_KEY, + indexName: 'messages', // Will get created automatically if it doesn't exist already + primaryKey: 'messageId', }); const Message = mongoose.models.Message || mongoose.model('Message', messageSchema); diff --git a/api/package-lock.json b/api/package-lock.json index ede925b9bf..22b16f0122 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -14,6 +14,7 @@ "@waylaidwanderer/chatgpt-api": "^1.28.2", "axios": "^1.3.4", "cors": "^2.8.5", + "crypto": "^1.0.1", "dotenv": "^16.0.3", "express": "^4.18.2", "express-session": "^1.17.3", @@ -2237,6 +2238,12 @@ "node": ">= 8" } }, + "node_modules/crypto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/crypto/-/crypto-1.0.1.tgz", + "integrity": "sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig==", + "deprecated": "This package is no longer supported. It's now a built-in Node module. If you've depended on crypto, you should switch to the one that's built-in." + }, "node_modules/debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", @@ -6886,6 +6893,11 @@ "which": "^2.0.1" } }, + "crypto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/crypto/-/crypto-1.0.1.tgz", + "integrity": "sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig==" + }, "debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", diff --git a/api/server/routes/search.js b/api/server/routes/search.js index 9336cf2370..4b92f18c70 100644 --- a/api/server/routes/search.js +++ b/api/server/routes/search.js @@ -1,24 +1,25 @@ const express = require('express'); const router = express.Router(); const { Message } = require('../../models/Message'); -const reduceHits = require('../../lib/utils/reduceHits'); +const { Conversation } = require('../../models/Conversation'); +const {reduceMessages, reduceHits} = require('../../lib/utils/reduceHits'); // const { MeiliSearch } = require('meilisearch'); router.get('/sync', async function (req, res) { - // await Message.setMeiliIndexSettings({ primaryKey: 'messageId' }); - // res.send('updated settings'); - // await Message.clearMeiliIndex(); - // res.send('deleted index'); await Message.syncWithMeili(); + await Conversation.syncWithMeili(); res.send('synced'); }); router.get('/', async function (req, res) { const { q } = req.query; - const result = await Message.meiliSearch({ query: q }); - const sortedHits = reduceHits(result.hits); - console.log(sortedHits); - res.send(sortedHits); + const message = await Message.meiliSearch(q, { attributesToHighlight: ['text', 'sender'] }); + const title = await Conversation.meiliSearch(q, { attributesToHighlight: ['title'] }); + // console.log('titles', title); + // console.log(sortedHits); + const sortedHits = reduceHits(message.hits, title.hits); + // const sortedHits = reduceMessages(message.hits); + res.status(200).send({sortedHits}); }); router.get('/clear', async function (req, res) {