diff --git a/api/app/bingai.js b/api/app/clients/bingai.js similarity index 100% rename from api/app/bingai.js rename to api/app/clients/bingai.js diff --git a/api/app/chatgpt-browser.js b/api/app/clients/chatgpt-browser.js similarity index 100% rename from api/app/chatgpt-browser.js rename to api/app/clients/chatgpt-browser.js diff --git a/api/app/chatgpt-client.js b/api/app/clients/chatgpt-client.js similarity index 100% rename from api/app/chatgpt-client.js rename to api/app/clients/chatgpt-client.js diff --git a/api/app/chatgpt-custom.js b/api/app/clients/chatgpt-custom.js similarity index 100% rename from api/app/chatgpt-custom.js rename to api/app/clients/chatgpt-custom.js diff --git a/api/app/sydney.js b/api/app/clients/sydney.js similarity index 100% rename from api/app/sydney.js rename to api/app/clients/sydney.js diff --git a/api/app/index.js b/api/app/index.js index 7b61f8de95..235b22b2ef 100644 --- a/api/app/index.js +++ b/api/app/index.js @@ -1,12 +1,12 @@ -const { askClient } = require('./chatgpt-client'); -const { browserClient } = require('./chatgpt-browser'); -const customClient = require('./chatgpt-custom'); -const { askBing } = require('./bingai'); -const { askSydney } = require('./sydney'); +const { askClient } = require('./clients/chatgpt-client'); +const { browserClient } = require('./clients/chatgpt-browser'); +const { askBing } = require('./clients/bingai'); +const { askSydney } = require('./clients/sydney'); +const customClient = require('./clients/chatgpt-custom'); const titleConvo = require('./titleConvo'); -const getCitations = require('./getCitations'); -const citeText = require('./citeText'); -const detectCode = require('./detectCode'); +const getCitations = require('../lib/parse/getCitations'); +const citeText = require('../lib/parse/citeText'); +const detectCode = require('../lib/parse/detectCode'); module.exports = { askClient, diff --git a/api/app/search.js b/api/app/search.js deleted file mode 100644 index f10c77a6e5..0000000000 --- a/api/app/search.js +++ /dev/null @@ -1,24 +0,0 @@ -const mongoose = require('mongoose'); -const mongomeili = require('mongomeili'); -const { messageSchema, Message } = require('../models/Message'); - -// Add the '{ meiliIndex: true }' property to index these attributes with MeiliSearch -// const MovieSchema = new mongoose.Schema({ -// title: { type: String, required: true, meiliIndex: true }, -// director: { type: String, required: true, meiliIndex: true }, -// year: { type: String, required: true, meiliIndex: true } -// }); - -// Specify your MeiliSearch credentials -// messageSchema.plugin(mongomeili, { -// host: 'http://localhost:7700', -// apiKey: 'MASTER_KEY', -// indexName: 'messages' // Will get created automatically if it doesn't exist already -// }); - -(async () => { - await Message.syncWithMeili(); - const result = await Message.meiliSearch({ query: 'quantum' }); - - console.log(result); -})(); diff --git a/api/lib/mongoMeili.js b/api/lib/mongoMeili.js new file mode 100644 index 0000000000..a7514d1918 --- /dev/null +++ b/api/lib/mongoMeili.js @@ -0,0 +1,171 @@ +const { MeiliSearch } = require('meilisearch'); +const _ = require('lodash'); + +const validateOptions = function (options) { + const requiredKeys = ['host', 'apiKey', 'indexName']; + requiredKeys.forEach(key => { + if (!options[key]) throw new Error(`Missing mongoMeili Option: ${key}`); + }); +}; + +const createMeiliMongooseModel = function ({ index, indexName, client, attributesToIndex }) { + + console.log('attributesToIndex', attributesToIndex); + // MeiliMongooseModel is of type Mongoose.Model + class MeiliMongooseModel { + + // Clear Meili index + static async clearMeiliIndex() { + await index.delete(); + // await index.deleteAllDocuments(); + await this.collection.updateMany({ _meiliIndex: true }, { $set: { _meiliIndex: false } }); + } + + static async resetIndex() { + await this.clearMeiliIndex(); + await client.createIndex(indexName, { primaryKey: attributesToIndex[0] }); + } + // Clear Meili index + // Push a mongoDB collection to Meili index + static async syncWithMeili() { + await this.resetIndex(); + // const docs = await this.find(); + const docs = await this.find({ _meiliIndex: { $in: [null, false] } }); + console.log('docs', docs.length) + await Promise.all(docs.map(function(doc) { + return doc.addObjectToMeili(); + })); + } + + // Set one or more settings of the meili index + static async setMeiliIndexSettings(settings) { + return await index.updateSettings(settings); + } + + // Search the index + static async meiliSearch({ query, params, populate }) { + const data = await index.search(query, params); + + // Populate hits with content from mongodb + if (populate) { + + // Find objects into mongodb matching `objectID` from Meili search + const hitsFromMongoose = await this.find( + { + _id: { $in: _.map(data.hits, '_id') }, + }, + _.reduce( this.schema.obj, function (results, value, key) { return { ...results, [key]: 1 } }, { _id: 1 } ) + ); + + // Add additional data from mongodb into Meili search hits + const populatedHits = data.hits.map(function(hit) { + const originalHit = _.find(hitsFromMongoose, { + _id: hit._id + }); + + return { + ...(originalHit ? originalHit.toJSON() : {}), + ...hit, + }; + }); + data.hits = populatedHits; + } + + return data; + } + + // Push new document to Meili + async addObjectToMeili() { + const object = _.pick(this.toJSON(), attributesToIndex); + // object.id = object._id.toString(); + try { + // console.log('Adding document to Meili', object); + await index.addDocuments([object]); + } catch (error) { + console.log('Error adding document to Meili'); + console.error(error); + } + + await this.collection.updateMany( + { _id: this._id }, + { $set: { _meiliIndex: true } } + ); + } + + // Update an existing document in Meili + async updateObjectToMeili() { + const object = pick(this.toJSON(), attributesToIndex); + await index.updateDocuments([object]); + } + + // Delete a document from Meili + async deleteObjectFromMeili() { + await index.deleteDocument(this._id); + } + + // * schema.post('save') + postSaveHook() { + if (this._meiliIndex) { + this.updateObjectToMeili(); + } else { + this.addObjectToMeili(); + } + } + + // * schema.post('update') + postUpdateHook() { + if (this._meiliIndex) { + this.updateObjectToMeili(); + } + } + + // * schema.post('remove') + postRemoveHook() { + if (this._meiliIndex) { + this.deleteObjectFromMeili(); + } + } + } + + return MeiliMongooseModel; +} + +module.exports = function mongoMeili(schema, options) { + + // Vaidate Options for mongoMeili + validateOptions(options); + + // Add meiliIndex to schema + schema.add({ + _meiliIndex: { + type: Boolean, + required: false, + select: false, + default: false + } + }); + + const { host, apiKey, indexName } = options; + + // Setup MeiliSearch Client + const client = new MeiliSearch({ host, apiKey }); + + // Asynchronously create the index + client.createIndex(indexName, { primaryKey: 'messageId' }); + + // Setup the index to search for this schema + const index = client.index(indexName); + + const attributesToIndex = [..._.reduce(schema.obj, function (results, value, key) { + return value.meiliIndex ? [...results, key] : results; + // }, []), '_id']; + }, [])]; + + schema.loadClass(createMeiliMongooseModel({ index, indexName, client, attributesToIndex })); + + // Register hooks + schema.post('save', function (doc) { doc.postSaveHook() }); + schema.post('update', function (doc) { doc.postUpdateHook() }); + schema.post('remove', function (doc) { doc.postRemoveHook() }); + schema.post('findOneAndUpdate', function(doc) { doc.postSaveHook() }); +}; \ No newline at end of file diff --git a/api/app/citeText.js b/api/lib/parse/citeText.js similarity index 100% rename from api/app/citeText.js rename to api/lib/parse/citeText.js diff --git a/api/app/detectCode.js b/api/lib/parse/detectCode.js similarity index 96% rename from api/app/detectCode.js rename to api/lib/parse/detectCode.js index 0d6fad0534..3456c21b34 100644 --- a/api/app/detectCode.js +++ b/api/lib/parse/detectCode.js @@ -1,5 +1,5 @@ const { ModelOperations } = require('@vscode/vscode-languagedetection'); -const languages = require('../utils/languages.js'); +const languages = require('./languages.js'); const codeRegex = /(```[\s\S]*?```)/g; // const languageMatch = /```(\w+)/; const replaceRegex = /```\w+\n/g; diff --git a/api/app/getCitations.js b/api/lib/parse/getCitations.js similarity index 100% rename from api/app/getCitations.js rename to api/lib/parse/getCitations.js diff --git a/api/utils/languages.js b/api/lib/parse/languages.js similarity index 100% rename from api/utils/languages.js rename to api/lib/parse/languages.js diff --git a/api/app/regexSplit.mjs b/api/lib/parse/regexSplit.mjs similarity index 100% rename from api/app/regexSplit.mjs rename to api/lib/parse/regexSplit.mjs diff --git a/api/models/Message.js b/api/models/Message.js index b69a39ad9c..46eb065ea6 100644 --- a/api/models/Message.js +++ b/api/models/Message.js @@ -1,13 +1,12 @@ const mongoose = require('mongoose'); -// const mongomeili = require('mongomeili'); -const MeiliSearch = require('meilisearch'); -const _ = require('lodash'); +const mongoMeili = require('../lib/mongoMeili'); const messageSchema = mongoose.Schema({ messageId: { type: String, unique: true, - required: true + required: true, + meiliIndex: true }, conversationId: { type: String, @@ -55,11 +54,11 @@ const messageSchema = mongoose.Schema({ } }, { timestamps: true }); -// messageSchema.plugin(mongomeili, { -// host: 'http://localhost:7700', -// apiKey: 'MASTER_KEY', -// indexName: 'messages' // Will get created automatically if it doesn't exist already -// }); +messageSchema.plugin(mongoMeili, { + host: 'http://localhost:7700', + apiKey: 'MASTER_KEY', + indexName: 'messages' // Will get created automatically if it doesn't exist already +}); const Message = mongoose.models.Message || mongoose.model('Message', messageSchema); diff --git a/api/server/routes/search.js b/api/server/routes/search.js index 83f3c265c7..65d85dae5a 100644 --- a/api/server/routes/search.js +++ b/api/server/routes/search.js @@ -1,13 +1,22 @@ const express = require('express'); const router = express.Router(); const { Message } = require('../../models/Message'); +// const { MeiliSearch } = require('meilisearch'); router.get('/sync', async function (req, res) { - console.log(Message); + // await Message.setMeiliIndexSettings({ primaryKey: 'messageId' }); + // res.send('updated settings'); + // await Message.clearMeiliIndex(); + // res.send('deleted index'); await Message.syncWithMeili(); - const result = await Message.meiliSearch({ query: 'computing' }); + res.send('synced'); +}); + +router.get('/', async function (req, res) { + const { q } = req.query; + const result = await Message.meiliSearch({ query: q }); console.log(result); res.send(result); -}) +}); module.exports = router;