diff --git a/api/.env.example b/api/.env.example index adf7588083..252b4b94de 100644 --- a/api/.env.example +++ b/api/.env.example @@ -18,12 +18,47 @@ MONGO_URI="mongodb://127.0.0.1:27017/chatgpt-clone" # API key configuration. # Leave blank if you don't want them. OPENAI_KEY= -CHATGPT_TOKEN= BING_TOKEN= -# User System +# ChatGPT Browser Client (free but use at your own risk) +# Access token from https://chat.openai.com/api/auth/session +# Exposes your access token to a 3rd party +CHATGPT_TOKEN= +# If you have access to other models on the official site, you can use them here. +# Defaults to 'text-davinci-002-render-sha' if left empty. +# options: gpt-4, text-davinci-002-render, text-davinci-002-render-paid, or text-davinci-002-render-sha +# You cannot use a model that your account does not have access to. You can check +# which ones you have access to by opening DevTools and going to the Network tab. +# Refresh the page and look at the response body for https://chat.openai.com/backend-api/models. +BROWSER_MODEL= +# ENABLING SEARCH MESSAGES/CONVOS +# Requires installation of free self-hosted Meilisearch or Paid Remote Plan (Remote not tested) +# The easiest setup for this is through docker-compose, which takes care of it for you. +# SEARCH=TRUE +SEARCH=TRUE + +# REQUIRED FOR SEARCH: MeiliSearch Host, mainly for api server to connect to the search server. +# must replace '0.0.0.0' with 'meilisearch' if serving meilisearch with docker-compose +# MEILI_HOST='http://0.0.0.0:7700' # <-- local/remote +MEILI_HOST='http://meilisearch:7700' # <-- docker-compose + +# REQUIRED FOR SEARCH: MeiliSearch HTTP Address, mainly for docker-compose to expose the search server. +# must replace '0.0.0.0' with 'meilisearch' if serving meilisearch with docker-compose +# MEILI_HTTP_ADDR='0.0.0.0:7700' # <-- local/remote +MEILI_HTTP_ADDR='meilisearch:7700' # <-- docker-compose + +# REQUIRED FOR SEARCH: In production env., needs a secure key, feel free to generate your own. +# This master key must be at least 16 bytes, composed of valid UTF-8 characters. +# Meilisearch will throw an error and refuse to launch if no master key is provided or if it is under 16 bytes, +# Meilisearch will suggest a secure autogenerated master key. +# Using docker, it seems recognized as production so use a secure key. +# MEILI_MASTER_KEY= # <-- no/insecure key for local/remote +MEILI_MASTER_KEY=JKMW-hGc7v_D1FkJVdbRSDNFLZcUv3S75yrxXP0SmcU # <-- ready made secure key for docker-compose + + +# User System # global enable/disable the sample user system. # this is not a ready to use user system. # dont't use it, unless you can write your own code. -ENABLE_USER_SYSTEM= +ENABLE_USER_SYSTEM=FALSE \ No newline at end of file diff --git a/api/.prettierrc b/api/.prettierrc new file mode 100644 index 0000000000..34e12e2f49 --- /dev/null +++ b/api/.prettierrc @@ -0,0 +1,22 @@ +{ + "arrowParens": "avoid", + "bracketSpacing": true, + "endOfLine": "lf", + "htmlWhitespaceSensitivity": "css", + "insertPragma": false, + "singleAttributePerLine": true, + "bracketSameLine": false, + "jsxBracketSameLine": false, + "jsxSingleQuote": false, + "printWidth": 110, + "proseWrap": "preserve", + "quoteProps": "as-needed", + "requirePragma": false, + "semi": true, + "singleQuote": true, + "tabWidth": 2, + "trailingComma": "none", + "useTabs": false, + "vueIndentScriptAndStyle": false, + "parser": "babel" +} \ No newline at end of file diff --git a/api/app/clients/chatgpt-browser.js b/api/app/clients/chatgpt-browser.js index 7776083b62..01de94cd65 100644 --- a/api/app/clients/chatgpt-browser.js +++ b/api/app/clients/chatgpt-browser.js @@ -1,5 +1,6 @@ require('dotenv').config(); const { KeyvFile } = require('keyv-file'); +const set = new Set(["gpt-4", "text-davinci-002-render", "text-davinci-002-render-paid", "text-davinci-002-render-sha"]); const clientOptions = { // Warning: This will expose your access token to a third party. Consider the risks before using this. @@ -10,6 +11,12 @@ const clientOptions = { proxy: process.env.PROXY || null, }; +// You can check which models you have access to by opening DevTools and going to the Network tab. +// Refresh the page and look at the response body for https://chat.openai.com/backend-api/models. +if (set.has(process.env.BROWSER_MODEL)) { + clientOptions.model = process.env.BROWSER_MODEL; +} + const browserClient = async ({ text, onProgress, convo, abortController }) => { const { ChatGPTBrowserClient } = await import('@waylaidwanderer/chatgpt-api'); diff --git a/api/app/titleConvo.js b/api/app/titleConvo.js index a089d6ca76..9575377b44 100644 --- a/api/app/titleConvo.js +++ b/api/app/titleConvo.js @@ -1,7 +1,7 @@ const { Configuration, OpenAIApi } = require('openai'); const _ = require('lodash'); -const proxyEnvToAxiosProxy = (proxyString) => { +const proxyEnvToAxiosProxy = proxyString => { if (!proxyString) return null; const regex = /^([^:]+):\/\/(?:([^:@]*):?([^:@]*)@)?([^:]+)(?::(\d+))?/; @@ -18,33 +18,37 @@ const proxyEnvToAxiosProxy = (proxyString) => { const titleConvo = async ({ model, text, response }) => { let title = 'New Chat'; + + const request = { + model: 'gpt-3.5-turbo', + messages: [ + { + role: 'system', + content: + 'You are a title-generator with one job: giving a conversation, detect the language and titling the conversation provided by a user in title case, using the same language.' + }, + { + role: 'user', + content: `In 5 words or less, summarize the conversation below with a title in title case using the language the user writes in. Don't refer to the participants of the conversation nor the language. Do not include punctuation or quotation marks. Your response should be in title case, exclusively containing the title. Conversation:\n\nUser: "${text}"\n\n${model}: "${JSON.stringify( + response?.text + )}"\n\nTitle: ` + } + ], + temperature: 0, + presence_penalty: 0, + frequency_penalty: 0 + }; + + // console.log('REQUEST', request); + try { const configuration = new Configuration({ apiKey: process.env.OPENAI_KEY }); const openai = new OpenAIApi(configuration); - const completion = await openai.createChatCompletion( - { - model: 'gpt-3.5-turbo', - messages: [ - { - role: 'system', - content: - 'You are a title-generator with one job: giving a conversation, detect the language and titling the conversation provided by a user in title case, using the same language.' - }, - { - role: 'user', - content: `In 5 words or less, summarize the conversation below with a title in title case using the language the user writes in. Don't refer to the participants of the conversation nor the language. Do not include punctuation or quotation marks. Your response should be in title case, exclusively containing the title. Conversation:\n\nUser: "${text}"\n\n${model}: "${JSON.stringify( - response?.text - )}"\n\nTitle: ` - } - ], - temperature: 0, - presence_penalty: 0, - frequency_penalty: 0, - }, - { proxy: proxyEnvToAxiosProxy(process.env.PROXY || null) } - ); + const completion = await openai.createChatCompletion(request, { + proxy: proxyEnvToAxiosProxy(process.env.PROXY || null) + }); //eslint-disable-next-line title = completion.data.choices[0].message.content.replace(/["\.]/g, ''); diff --git a/api/lib/db/indexSync.js b/api/lib/db/indexSync.js new file mode 100644 index 0000000000..e42f5319e6 --- /dev/null +++ b/api/lib/db/indexSync.js @@ -0,0 +1,70 @@ +const mongoose = require('mongoose'); +const Conversation = mongoose.models.Conversation; +const Message = mongoose.models.Message; +const { MeiliSearch } = require('meilisearch'); +let currentTimeout = null; + +// eslint-disable-next-line no-unused-vars +async function indexSync(req, res, next) { + try { + if (!process.env.MEILI_HOST || !process.env.MEILI_MASTER_KEY || !process.env.SEARCH) { + throw new Error('Meilisearch not configured, search will be disabled.'); + } + + const client = new MeiliSearch({ + host: process.env.MEILI_HOST, + apiKey: process.env.MEILI_MASTER_KEY + }); + + const { status } = await client.health(); + // console.log(`Meilisearch: ${status}`); + const result = status === 'available' && !!process.env.SEARCH; + + if (!result) { + throw new Error('Meilisearch not available'); + } + + const messageCount = await Message.countDocuments(); + const convoCount = await Conversation.countDocuments(); + const messages = await client.index('messages').getStats(); + const convos = await client.index('convos').getStats(); + const messagesIndexed = messages.numberOfDocuments; + const convosIndexed = convos.numberOfDocuments; + + console.log(`There are ${messageCount} messages in the database, ${messagesIndexed} indexed`); + console.log(`There are ${convoCount} convos in the database, ${convosIndexed} indexed`); + + if (messageCount !== messagesIndexed) { + console.log('Messages out of sync, indexing'); + await Message.syncWithMeili(); + } + + if (convoCount !== convosIndexed) { + console.log('Convos out of sync, indexing'); + await Conversation.syncWithMeili(); + } + } catch (err) { + // console.log('in index sync'); + if (err.message.includes('not found')) { + console.log('Creating indices...'); + currentTimeout = setTimeout(async () => { + try { + await Message.syncWithMeili(); + await Conversation.syncWithMeili(); + } catch (err) { + console.error('Trouble creating indices, try restarting the server.'); + } + }, 750); + } else { + console.error(err); + // res.status(500).json({ error: 'Server error' }); + } + } +} + +process.on('exit', () => { + console.log('Clearing sync timeouts before exiting...'); + clearTimeout(currentTimeout); +}); + +module.exports = indexSync; diff --git a/api/lib/utils/misc.js b/api/lib/utils/misc.js new file mode 100644 index 0000000000..3d971751fc --- /dev/null +++ b/api/lib/utils/misc.js @@ -0,0 +1,15 @@ +const cleanUpPrimaryKeyValue = (value) => { + // For Bing convoId handling + return value.replace(/--/g, '-'); +}; + +function replaceSup(text) { + if (!text.includes('')) return text; + const replacedText = text.replace(//g, '^').replace(/\s+<\/sup>/g, '^'); + return replacedText; +} + +module.exports = { + cleanUpPrimaryKeyValue, + replaceSup +}; diff --git a/api/models/Config.js b/api/models/Config.js index 57192a461e..bee45603ff 100644 --- a/api/models/Config.js +++ b/api/models/Config.js @@ -49,16 +49,16 @@ const configSchema = mongoose.Schema( ); // Instance method -ConfigSchema.methods.incrementCount = function () { +configSchema.methods.incrementCount = function () { this.startupCounts += 1; }; // Static methods -ConfigSchema.statics.findByTag = async function (tag) { +configSchema.statics.findByTag = async function (tag) { return await this.findOne({ tag }); }; -ConfigSchema.statics.updateByTag = async function (tag, update) { +configSchema.statics.updateByTag = async function (tag, update) { return await this.findOneAndUpdate({ tag }, update, { new: true }); }; diff --git a/api/models/Conversation.js b/api/models/Conversation.js index bc25b3f57d..732005b968 100644 --- a/api/models/Conversation.js +++ b/api/models/Conversation.js @@ -1,70 +1,8 @@ -const mongoose = require('mongoose'); -const mongoMeili = require('../lib/db/mongoMeili'); +// const { Conversation } = require('./plugins'); +const Conversation = require('./schema/convoSchema'); +const { cleanUpPrimaryKeyValue } = require('../lib/utils/misc'); const { getMessages, deleteMessages } = require('./Message'); -const convoSchema = mongoose.Schema( - { - conversationId: { - type: String, - unique: true, - required: true, - index: true, - meiliIndex: true - }, - parentMessageId: { - type: String, - required: true - }, - title: { - type: String, - default: 'New Chat', - meiliIndex: true - }, - jailbreakConversationId: { - type: String, - default: null - }, - conversationSignature: { - type: String, - default: null - }, - clientId: { - type: String - }, - invocationId: { - type: String - }, - chatGptLabel: { - type: String, - default: null - }, - promptPrefix: { - type: String, - default: null - }, - model: { - type: String, - required: true - }, - user: { - type: String - }, - suggestions: [{ type: String }], - messages: [{ type: mongoose.Schema.Types.ObjectId, ref: 'Message' }] - }, - { timestamps: true } -); - -// convoSchema.plugin(mongoMeili, { -// host: process.env.MEILI_HOST, -// apiKey: process.env.MEILI_KEY, -// indexName: 'convos', // Will get created automatically if it doesn't exist already -// primaryKey: 'conversationId' -// }); - -const Conversation = - mongoose.models.Conversation || mongoose.model('Conversation', convoSchema); - const getConvo = async (user, conversationId) => { try { return await Conversation.findOne({ user, conversationId }).exec(); @@ -143,15 +81,16 @@ module.exports = { } const cache = {}; + const convoMap = {}; const promises = []; // will handle a syncing solution soon const deletedConvoIds = []; - convoIds.forEach((convo) => + convoIds.forEach(convo => promises.push( Conversation.findOne({ user, - conversationId: convo.conversationId + conversationId: cleanUpPrimaryKeyValue(convo.conversationId) }).exec() ) ); @@ -166,6 +105,7 @@ module.exports = { cache[page] = []; } cache[page].push(convo); + convoMap[convo.conversationId] = convo; return true; } }); @@ -183,6 +123,7 @@ module.exports = { pageSize, // will handle a syncing solution soon filter: new Set(deletedConvoIds), + convoMap }; } catch (error) { console.log(error); @@ -208,6 +149,7 @@ module.exports = { }, deleteConvos: async (user, filter) => { let deleteCount = await Conversation.deleteMany({ ...filter, user }).exec(); + console.log('deleteCount', deleteCount); deleteCount.messages = await deleteMessages(filter); return deleteCount; } diff --git a/api/models/Message.js b/api/models/Message.js index cb6c531291..a886c02859 100644 --- a/api/models/Message.js +++ b/api/models/Message.js @@ -1,71 +1,5 @@ -const mongoose = require('mongoose'); -const mongoMeili = require('../lib/db/mongoMeili'); - -const messageSchema = mongoose.Schema({ - messageId: { - type: String, - unique: true, - required: true, - index: true, - meiliIndex: true - }, - conversationId: { - type: String, - required: true, - meiliIndex: true - }, - conversationSignature: { - type: String, - // required: true - }, - clientId: { - type: String, - }, - invocationId: { - type: String, - }, - parentMessageId: { - type: String, - // required: true - }, - sender: { - type: String, - required: true, - meiliIndex: true - }, - text: { - type: String, - required: true, - meiliIndex: true - }, - isCreatedByUser: { - type: Boolean, - required: true, - default: false - }, - error: { - type: Boolean, - default: false - }, - _meiliIndex: { - type: Boolean, - required: false, - select: false, - default: false - } -}, { timestamps: true }); - -// messageSchema.plugin(mongoMeili, { -// host: process.env.MEILI_HOST, -// apiKey: process.env.MEILI_KEY, -// indexName: 'messages', // Will get created automatically if it doesn't exist already -// primaryKey: 'messageId', -// }); - -const Message = mongoose.models.Message || mongoose.model('Message', messageSchema); - +const Message = require('./schema/messageSchema'); module.exports = { - messageSchema, Message, saveMessage: async ({ messageId, conversationId, parentMessageId, sender, text, isCreatedByUser=false, error }) => { try { diff --git a/api/lib/db/mongoMeili.js b/api/models/plugins/mongoMeili.js similarity index 79% rename from api/lib/db/mongoMeili.js rename to api/models/plugins/mongoMeili.js index 63330b4624..b428a9cd82 100644 --- a/api/lib/db/mongoMeili.js +++ b/api/models/plugins/mongoMeili.js @@ -1,4 +1,6 @@ +const mongoose = require('mongoose'); const { MeiliSearch } = require('meilisearch'); +const { cleanUpPrimaryKeyValue } = require('../../lib/utils/misc'); const _ = require('lodash'); const validateOptions = function (options) { @@ -54,7 +56,9 @@ const createMeiliMongooseModel = function ({ index, indexName, client, attribute if (populate) { // Find objects into mongodb matching `objectID` from Meili search const query = {}; - query[primaryKey] = { $in: _.map(data.hits, primaryKey) }; + // query[primaryKey] = { $in: _.map(data.hits, primaryKey) }; + query[primaryKey] = _.map(data.hits, hit => cleanUpPrimaryKeyValue(hit[primaryKey])); + // console.log('query', query); const hitsFromMongoose = await this.find( query, _.reduce( @@ -86,13 +90,18 @@ const createMeiliMongooseModel = function ({ index, indexName, client, attribute // Push new document to Meili async addObjectToMeili() { const object = _.pick(this.toJSON(), attributesToIndex); + // NOTE: MeiliSearch does not allow | in primary key, so we replace it with - for Bing convoIds + // object.conversationId = object.conversationId.replace(/\|/g, '-'); + if (object.conversationId && object.conversationId.includes('|')) { + object.conversationId = object.conversationId.replace(/\|/g, '--'); + } try { // console.log('Adding document to Meili', object); await index.addDocuments([object]); } catch (error) { - console.log('Error adding document to Meili'); - console.error(error); + // console.log('Error adding document to Meili'); + // console.error(error); } await this.collection.updateMany({ _id: this._id }, { $set: { _meiliIndex: true } }); @@ -100,7 +109,7 @@ const createMeiliMongooseModel = function ({ index, indexName, client, attribute // Update an existing document in Meili async updateObjectToMeili() { - const object = pick(this.toJSON(), attributesToIndex); + const object = _.pick(this.toJSON(), attributesToIndex); await index.updateDocuments([object]); } @@ -184,6 +193,18 @@ module.exports = function mongoMeili(schema, options) { schema.post('remove', function (doc) { doc.postRemoveHook(); }); + schema.post('deleteMany', function () { + // console.log('deleteMany hook', doc); + if (Object.prototype.hasOwnProperty.call(schema.obj, 'messages')) { + console.log('Syncing convos...'); + mongoose.model('Conversation').syncWithMeili(); + } + + if (Object.prototype.hasOwnProperty.call(schema.obj, 'messageId')) { + console.log('Syncing messages...'); + mongoose.model('Message').syncWithMeili(); + } + }); schema.post('findOneAndUpdate', function (doc) { doc.postSaveHook(); }); diff --git a/api/models/schema/convoSchema.js b/api/models/schema/convoSchema.js new file mode 100644 index 0000000000..a654b1425a --- /dev/null +++ b/api/models/schema/convoSchema.js @@ -0,0 +1,67 @@ +const mongoose = require('mongoose'); +const mongoMeili = require('../plugins/mongoMeili'); +const convoSchema = mongoose.Schema( + { + conversationId: { + type: String, + unique: true, + required: true, + index: true, + meiliIndex: true + }, + parentMessageId: { + type: String, + required: true + }, + title: { + type: String, + default: 'New Chat', + meiliIndex: true + }, + jailbreakConversationId: { + type: String, + default: null + }, + conversationSignature: { + type: String, + default: null + }, + clientId: { + type: String + }, + invocationId: { + type: String + }, + chatGptLabel: { + type: String, + default: null + }, + promptPrefix: { + type: String, + default: null + }, + model: { + type: String, + required: true + }, + user: { + type: String + }, + suggestions: [{ type: String }], + messages: [{ type: mongoose.Schema.Types.ObjectId, ref: 'Message' }] + }, + { timestamps: true } +); + +if (process.env.MEILI_HOST && process.env.MEILI_MASTER_KEY) { + convoSchema.plugin(mongoMeili, { + host: process.env.MEILI_HOST, + apiKey: process.env.MEILI_MASTER_KEY, + indexName: 'convos', // Will get created automatically if it doesn't exist already + primaryKey: 'conversationId' + }); +} + +const Conversation = mongoose.models.Conversation || mongoose.model('Conversation', convoSchema); + +module.exports = Conversation; diff --git a/api/models/schema/messageSchema.js b/api/models/schema/messageSchema.js new file mode 100644 index 0000000000..9d21dd5b92 --- /dev/null +++ b/api/models/schema/messageSchema.js @@ -0,0 +1,71 @@ +const mongoose = require('mongoose'); +const mongoMeili = require('../plugins/mongoMeili'); +const messageSchema = mongoose.Schema( + { + messageId: { + type: String, + unique: true, + required: true, + index: true, + meiliIndex: true + }, + conversationId: { + type: String, + required: true, + meiliIndex: true + }, + conversationSignature: { + type: String + // required: true + }, + clientId: { + type: String + }, + invocationId: { + type: String + }, + parentMessageId: { + type: String + // required: true + }, + sender: { + type: String, + required: true, + meiliIndex: true + }, + text: { + type: String, + required: true, + meiliIndex: true + }, + isCreatedByUser: { + type: Boolean, + required: true, + default: false + }, + error: { + type: Boolean, + default: false + }, + _meiliIndex: { + type: Boolean, + required: false, + select: false, + default: false + } + }, + { timestamps: true } +); + +if (process.env.MEILI_HOST && process.env.MEILI_MASTER_KEY) { + messageSchema.plugin(mongoMeili, { + host: process.env.MEILI_HOST, + apiKey: process.env.MEILI_MASTER_KEY, + indexName: 'messages', + primaryKey: 'messageId' + }); +} + +const Message = mongoose.models.Message || mongoose.model('Message', messageSchema); + +module.exports = Message; diff --git a/api/server/controllers/errorController.js b/api/server/controllers/errorController.js new file mode 100644 index 0000000000..e22b21c123 --- /dev/null +++ b/api/server/controllers/errorController.js @@ -0,0 +1,33 @@ +//handle duplicates +const handleDuplicateKeyError = (err, res) => { + const field = Object.keys(err.keyValue); + const code = 409; + const error = `An document with that ${field} already exists.`; + console.log('congrats you hit the duped keys error'); + res.status(code).send({ messages: error, fields: field }); +}; + +//handle validation errors +const handleValidationError = (err, res) => { + console.log('congrats you hit the validation middleware'); + let errors = Object.values(err.errors).map(el => el.message); + let fields = Object.values(err.errors).map(el => el.path); + let code = 400; + if (errors.length > 1) { + const formattedErrors = errors.join(' '); + res.status(code).send({ messages: formattedErrors, fields: fields }); + } else { + res.status(code).send({ messages: errors, fields: fields }); + } +}; + +// eslint-disable-next-line no-unused-vars +module.exports = (err, req, res, next) => { + try { + console.log('congrats you hit the error middleware'); + if (err.name === 'ValidationError') return (err = handleValidationError(err, res)); + if (err.code && err.code == 11000) return (err = handleDuplicateKeyError(err, res)); + } catch (err) { + res.status(500).send('An unknown error occurred.'); + } +}; diff --git a/api/server/index.js b/api/server/index.js index 93804a4390..d929bd5d8a 100644 --- a/api/server/index.js +++ b/api/server/index.js @@ -1,70 +1,96 @@ const express = require('express'); -const session = require('express-session') +const session = require('express-session'); const connectDb = require('../lib/db/connectDb'); const migrateDb = require('../lib/db/migrateDb'); +const indexSync = require('../lib/db/indexSync'); const path = require('path'); const cors = require('cors'); const routes = require('./routes'); -const app = express(); +const errorController = require('./controllers/errorController'); + const port = process.env.PORT || 3080; -const host = process.env.HOST || 'localhost' -const userSystemEnabled = process.env.ENABLE_USER_SYSTEM || false +const host = process.env.HOST || 'localhost'; +const userSystemEnabled = process.env.ENABLE_USER_SYSTEM || false; const projectPath = path.join(__dirname, '..', '..', 'client'); -connectDb().then(() => { + +(async () => { + await connectDb(); console.log('Connected to MongoDB'); - migrateDb(); -}); + await migrateDb(); + await indexSync(); -app.use(cors()); -app.use(express.json()); -app.use(express.static(path.join(projectPath, 'public'))); -app.set('trust proxy', 1) // trust first proxy -app.use(session({ - secret: 'chatgpt-clone-random-secrect', - resave: false, - saveUninitialized: true, -})) + const app = express(); + app.use(errorController); + app.use(cors()); + app.use(express.json()); + app.use(express.static(path.join(projectPath, 'public'))); + app.set('trust proxy', 1); // trust first proxy + app.use( + session({ + secret: 'chatgpt-clone-random-secrect', + resave: false, + saveUninitialized: true + }) + ); -/* chore: potential redirect error here, can only comment out this block; + /* chore: potential redirect error here, can only comment out this block; comment back in if using auth routes i guess */ -// app.get('/', routes.authenticatedOrRedirect, function (req, res) { -// console.log(path.join(projectPath, 'public', 'index.html')); -// res.sendFile(path.join(projectPath, 'public', 'index.html')); -// }); + // app.get('/', routes.authenticatedOrRedirect, function (req, res) { + // console.log(path.join(projectPath, 'public', 'index.html')); + // res.sendFile(path.join(projectPath, 'public', 'index.html')); + // }); -app.get('/api/me', function (req, res) { - if (userSystemEnabled) { - const user = req?.session?.user - - if (user) - res.send(JSON.stringify({username: user?.username, display: user?.display})); + app.get('/api/me', function (req, res) { + if (userSystemEnabled) { + const user = req?.session?.user; + + if (user) res.send(JSON.stringify({ username: user?.username, display: user?.display })); + else res.send(JSON.stringify(null)); + } else { + res.send(JSON.stringify({ username: 'anonymous_user', display: 'Anonymous User' })); + } + }); + + app.use('/api/search', routes.authenticatedOr401, routes.search); + app.use('/api/ask', routes.authenticatedOr401, routes.ask); + app.use('/api/messages', routes.authenticatedOr401, routes.messages); + app.use('/api/convos', routes.authenticatedOr401, routes.convos); + app.use('/api/customGpts', routes.authenticatedOr401, routes.customGpts); + app.use('/api/prompts', routes.authenticatedOr401, routes.prompts); + app.use('/auth', routes.auth); + + app.get('/api/models', function (req, res) { + const hasOpenAI = !!process.env.OPENAI_KEY; + const hasChatGpt = !!process.env.CHATGPT_TOKEN; + const hasBing = !!process.env.BING_TOKEN; + + res.send(JSON.stringify({ hasOpenAI, hasChatGpt, hasBing })); + }); + + app.listen(port, host, () => { + if (host == '0.0.0.0') + console.log( + `Server listening on all interface at port ${port}. Use http://localhost:${port} to access it` + ); else - res.send(JSON.stringify(null)); + console.log( + `Server listening at http://${host == '0.0.0.0' ? 'localhost' : host}:${port}` + ); + }); +})(); + +let messageCount = 0; +process.on('uncaughtException', (err) => { + if (!err.message.includes('fetch failed')) { + console.error('There was an uncaught error:', err.message); + } + + if (err.message.includes('fetch failed')) { + if (messageCount === 0) { + console.error('Meilisearch error, search will be disabled'); + messageCount++; + } } else { - res.send(JSON.stringify({username: 'anonymous_user', display: 'Anonymous User'})); + process.exit(1); } }); - -app.use('/api/search', routes.authenticatedOr401, routes.search); -app.use('/api/ask', routes.authenticatedOr401, routes.ask); -app.use('/api/messages', routes.authenticatedOr401, routes.messages); -app.use('/api/convos', routes.authenticatedOr401, routes.convos); -app.use('/api/customGpts', routes.authenticatedOr401, routes.customGpts); -app.use('/api/prompts', routes.authenticatedOr401, routes.prompts); -app.use('/auth', routes.auth); - - -app.get('/api/models', function (req, res) { - const hasOpenAI = !!process.env.OPENAI_KEY; - const hasChatGpt = !!process.env.CHATGPT_TOKEN; - const hasBing = !!process.env.BING_TOKEN; - - res.send(JSON.stringify({ hasOpenAI, hasChatGpt, hasBing })); -}); - -app.listen(port, host, () => { - if (host=='0.0.0.0') - console.log(`Server listening on all interface at port ${port}. Use http://localhost:${port} to access it`); - else - console.log(`Server listening at http://${host=='0.0.0.0'?'localhost':host}:${port}`); -}); \ No newline at end of file diff --git a/api/server/routes/convos.js b/api/server/routes/convos.js index 6f4b5cd736..7b2f9c72f9 100644 --- a/api/server/routes/convos.js +++ b/api/server/routes/convos.js @@ -19,25 +19,22 @@ router.get('/:conversationId', async (req, res) => { router.post('/gen_title', async (req, res) => { const { conversationId } = req.body.arg; - const convo = await getConvo(req?.session?.user?.username, conversationId) - const firstMessage = (await getMessages({ conversationId }))[0] - const secondMessage = (await getMessages({ conversationId }))[1] + const convo = await getConvo(req?.session?.user?.username, conversationId); + const firstMessage = (await getMessages({ conversationId }))[0]; + const secondMessage = (await getMessages({ conversationId }))[1]; const title = convo.jailbreakConversationId ? await getConvoTitle(req?.session?.user?.username, conversationId) : await titleConvo({ - model: convo?.model, - message: firstMessage?.text, - response: JSON.stringify(secondMessage?.text || '') - }); + model: convo?.model, + message: firstMessage?.text, + response: JSON.stringify(secondMessage?.text || '') + }); - await saveConvo( - req?.session?.user?.username, - { - conversationId, - title - } - ) + await saveConvo(req?.session?.user?.username, { + conversationId, + title + }); res.status(200).send(title); }); diff --git a/api/server/routes/search.js b/api/server/routes/search.js index 3ab8f2ad2f..5583efeabc 100644 --- a/api/server/routes/search.js +++ b/api/server/routes/search.js @@ -1,9 +1,10 @@ const express = require('express'); const router = express.Router(); +const { MeiliSearch } = require('meilisearch'); const { Message } = require('../../models/Message'); const { Conversation, getConvosQueried } = require('../../models/Conversation'); -const { reduceMessages, reduceHits } = require('../../lib/utils/reduceHits'); -// const { MeiliSearch } = require('meilisearch'); +const { reduceHits } = require('../../lib/utils/reduceHits'); +const { cleanUpPrimaryKeyValue } = require('../../lib/utils/misc'); const cache = new Map(); router.get('/sync', async function (req, res) { @@ -22,8 +23,8 @@ router.get('/', async function (req, res) { if (cache.has(key)) { console.log('cache hit', key); const cached = cache.get(key); - const { pages, pageSize } = cached; - res.status(200).send({ conversations: cached[pageNumber], pages, pageNumber, pageSize }); + const { pages, pageSize, messages } = cached; + res.status(200).send({ conversations: cached[pageNumber], pages, pageNumber, pageSize, messages }); return; } else { cache.clear(); @@ -49,11 +50,29 @@ router.get('/', async function (req, res) { }; }); const titles = (await Conversation.meiliSearch(q)).hits; + console.log('message hits:', messages.length, 'convo hits:', titles.length); const sortedHits = reduceHits(messages, titles); const result = await getConvosQueried(user, sortedHits, pageNumber); - cache.set(q, result.cache); - delete result.cache; - result.messages = messages.filter((message) => !result.filter.has(message.conversationId)); + + const activeMessages = []; + for (let i = 0; i < messages.length; i++) { + let message = messages[i]; + if (message.conversationId.includes('--')) { + message.conversationId = cleanUpPrimaryKeyValue(message.conversationId); + } + if (result.convoMap[message.conversationId] && !message.error) { + message = { ...message, title: result.convoMap[message.conversationId].title }; + activeMessages.push(message); + } + } + result.messages = activeMessages; + if (result.cache) { + result.cache.messages = activeMessages; + cache.set(key, result.cache); + delete result.cache; + } + delete result.convoMap; + // for debugging // console.log(result, messages.length); res.status(200).send(result); } catch (error) { @@ -78,4 +97,22 @@ router.get('/test', async function (req, res) { res.send(messages); }); +router.get('/enable', async function (req, res) { + let result = false; + try { + const client = new MeiliSearch({ + host: process.env.MEILI_HOST, + apiKey: process.env.MEILI_MASTER_KEY + }); + + const { status } = await client.health(); + // console.log(`Meilisearch: ${status}`); + result = status === 'available' && !!process.env.SEARCH; + return res.send(result); + } catch (error) { + // console.error(error); + return res.send(false); + } +}); + module.exports = router; diff --git a/client/.prettierrc b/client/.prettierrc new file mode 100644 index 0000000000..34e12e2f49 --- /dev/null +++ b/client/.prettierrc @@ -0,0 +1,22 @@ +{ + "arrowParens": "avoid", + "bracketSpacing": true, + "endOfLine": "lf", + "htmlWhitespaceSensitivity": "css", + "insertPragma": false, + "singleAttributePerLine": true, + "bracketSameLine": false, + "jsxBracketSameLine": false, + "jsxSingleQuote": false, + "printWidth": 110, + "proseWrap": "preserve", + "quoteProps": "as-needed", + "requirePragma": false, + "semi": true, + "singleQuote": true, + "tabWidth": 2, + "trailingComma": "none", + "useTabs": false, + "vueIndentScriptAndStyle": false, + "parser": "babel" +} \ No newline at end of file diff --git a/client/package-lock.json b/client/package-lock.json index 9859af60aa..56ccd4ec6c 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -34,8 +34,10 @@ "react-transition-group": "^4.4.5", "rehype-highlight": "^6.0.0", "rehype-katex": "^6.0.2", + "rehype-raw": "^6.1.1", "remark-gfm": "^3.0.1", "remark-math": "^5.1.1", + "remark-supersub": "^1.0.0", "swr": "^2.0.3", "tailwind-merge": "^1.9.1", "tailwindcss-animate": "^1.0.5", @@ -3185,6 +3187,11 @@ "integrity": "sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==", "dev": true }, + "node_modules/@types/parse5": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/@types/parse5/-/parse5-6.0.3.tgz", + "integrity": "sha512-SuT16Q1K51EAVPz1K29DJ/sXjhSQ0zjvsypYJ6tlwVsRV9jwW5Adq2ch8Dq8kDBCkYnELS7N7VNCSB5nC56t/g==" + }, "node_modules/@types/prop-types": { "version": "15.7.5", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.5.tgz", @@ -6812,6 +6819,45 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/hast-util-raw": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/hast-util-raw/-/hast-util-raw-7.2.3.tgz", + "integrity": "sha512-RujVQfVsOrxzPOPSzZFiwofMArbQke6DJjnFfceiEbFh7S05CbPt0cYN+A5YeD3pso0JQk6O1aHBnx9+Pm2uqg==", + "dependencies": { + "@types/hast": "^2.0.0", + "@types/parse5": "^6.0.0", + "hast-util-from-parse5": "^7.0.0", + "hast-util-to-parse5": "^7.0.0", + "html-void-elements": "^2.0.0", + "parse5": "^6.0.0", + "unist-util-position": "^4.0.0", + "unist-util-visit": "^4.0.0", + "vfile": "^5.0.0", + "web-namespaces": "^2.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-to-parse5": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/hast-util-to-parse5/-/hast-util-to-parse5-7.1.0.tgz", + "integrity": "sha512-YNRgAJkH2Jky5ySkIqFXTQiaqcAtJyVE+D5lkN6CdtOqrnkLfGYYrEcKuHOJZlp+MwjSwuD3fZuawI+sic/RBw==", + "dependencies": { + "@types/hast": "^2.0.0", + "comma-separated-tokens": "^2.0.0", + "property-information": "^6.0.0", + "space-separated-tokens": "^2.0.0", + "web-namespaces": "^2.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/hast-util-to-text": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/hast-util-to-text/-/hast-util-to-text-3.1.2.tgz", @@ -6926,6 +6972,15 @@ "integrity": "sha512-DV5Ln36z34NNTDgnz0EWGBLZENelNAtkiFA4kyNOG2tDI6Mz1uSWiq1wAKdyjnJwyDiDO7Fa2SO1CTxPXL8VxA==", "dev": true }, + "node_modules/html-void-elements": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/html-void-elements/-/html-void-elements-2.0.1.tgz", + "integrity": "sha512-0quDb7s97CfemeJAnW9wC0hw78MtW7NU3hqtCD75g2vFlDLt36llsYD7uB7SUzojLMP24N5IatXf7ylGXiGG9A==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/http-deceiver": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/http-deceiver/-/http-deceiver-1.2.7.tgz", @@ -10916,6 +10971,20 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/rehype-raw": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/rehype-raw/-/rehype-raw-6.1.1.tgz", + "integrity": "sha512-d6AKtisSRtDRX4aSPsJGTfnzrX2ZkHQLE5kiUuGOeEoLpbEulFF4hj0mLPbsa+7vmguDKOVVEQdHKDSwoaIDsQ==", + "dependencies": { + "@types/hast": "^2.0.0", + "hast-util-raw": "^7.2.0", + "unified": "^10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/remark-gfm": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/remark-gfm/-/remark-gfm-3.0.1.tgz", @@ -10975,6 +11044,14 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/remark-supersub": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/remark-supersub/-/remark-supersub-1.0.0.tgz", + "integrity": "sha512-3SYsphMqpAWbr8AZozdcypozinl/lly3e7BEwPG3YT5J9uZQaDcELBF6/sr/OZoAlFxy2nhNFWSrZBu/ZPRT3Q==", + "dependencies": { + "unist-util-visit": "^4.0.0" + } + }, "node_modules/require-from-string": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", @@ -15214,6 +15291,11 @@ "integrity": "sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==", "dev": true }, + "@types/parse5": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/@types/parse5/-/parse5-6.0.3.tgz", + "integrity": "sha512-SuT16Q1K51EAVPz1K29DJ/sXjhSQ0zjvsypYJ6tlwVsRV9jwW5Adq2ch8Dq8kDBCkYnELS7N7VNCSB5nC56t/g==" + }, "@types/prop-types": { "version": "15.7.5", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.5.tgz", @@ -18033,6 +18115,37 @@ "@types/hast": "^2.0.0" } }, + "hast-util-raw": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/hast-util-raw/-/hast-util-raw-7.2.3.tgz", + "integrity": "sha512-RujVQfVsOrxzPOPSzZFiwofMArbQke6DJjnFfceiEbFh7S05CbPt0cYN+A5YeD3pso0JQk6O1aHBnx9+Pm2uqg==", + "requires": { + "@types/hast": "^2.0.0", + "@types/parse5": "^6.0.0", + "hast-util-from-parse5": "^7.0.0", + "hast-util-to-parse5": "^7.0.0", + "html-void-elements": "^2.0.0", + "parse5": "^6.0.0", + "unist-util-position": "^4.0.0", + "unist-util-visit": "^4.0.0", + "vfile": "^5.0.0", + "web-namespaces": "^2.0.0", + "zwitch": "^2.0.0" + } + }, + "hast-util-to-parse5": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/hast-util-to-parse5/-/hast-util-to-parse5-7.1.0.tgz", + "integrity": "sha512-YNRgAJkH2Jky5ySkIqFXTQiaqcAtJyVE+D5lkN6CdtOqrnkLfGYYrEcKuHOJZlp+MwjSwuD3fZuawI+sic/RBw==", + "requires": { + "@types/hast": "^2.0.0", + "comma-separated-tokens": "^2.0.0", + "property-information": "^6.0.0", + "space-separated-tokens": "^2.0.0", + "web-namespaces": "^2.0.0", + "zwitch": "^2.0.0" + } + }, "hast-util-to-text": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/hast-util-to-text/-/hast-util-to-text-3.1.2.tgz", @@ -18134,6 +18247,11 @@ "integrity": "sha512-DV5Ln36z34NNTDgnz0EWGBLZENelNAtkiFA4kyNOG2tDI6Mz1uSWiq1wAKdyjnJwyDiDO7Fa2SO1CTxPXL8VxA==", "dev": true }, + "html-void-elements": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/html-void-elements/-/html-void-elements-2.0.1.tgz", + "integrity": "sha512-0quDb7s97CfemeJAnW9wC0hw78MtW7NU3hqtCD75g2vFlDLt36llsYD7uB7SUzojLMP24N5IatXf7ylGXiGG9A==" + }, "http-deceiver": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/http-deceiver/-/http-deceiver-1.2.7.tgz", @@ -20727,6 +20845,16 @@ "unified": "^10.0.0" } }, + "rehype-raw": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/rehype-raw/-/rehype-raw-6.1.1.tgz", + "integrity": "sha512-d6AKtisSRtDRX4aSPsJGTfnzrX2ZkHQLE5kiUuGOeEoLpbEulFF4hj0mLPbsa+7vmguDKOVVEQdHKDSwoaIDsQ==", + "requires": { + "@types/hast": "^2.0.0", + "hast-util-raw": "^7.2.0", + "unified": "^10.0.0" + } + }, "remark-gfm": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/remark-gfm/-/remark-gfm-3.0.1.tgz", @@ -20770,6 +20898,14 @@ "unified": "^10.0.0" } }, + "remark-supersub": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/remark-supersub/-/remark-supersub-1.0.0.tgz", + "integrity": "sha512-3SYsphMqpAWbr8AZozdcypozinl/lly3e7BEwPG3YT5J9uZQaDcELBF6/sr/OZoAlFxy2nhNFWSrZBu/ZPRT3Q==", + "requires": { + "unist-util-visit": "^4.0.0" + } + }, "require-from-string": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", diff --git a/client/package.json b/client/package.json index 43aafea595..1b8fb0458f 100644 --- a/client/package.json +++ b/client/package.json @@ -44,8 +44,10 @@ "react-transition-group": "^4.4.5", "rehype-highlight": "^6.0.0", "rehype-katex": "^6.0.2", + "rehype-raw": "^6.1.1", "remark-gfm": "^3.0.1", "remark-math": "^5.1.1", + "remark-supersub": "^1.0.0", "swr": "^2.0.3", "tailwind-merge": "^1.9.1", "tailwindcss-animate": "^1.0.5", diff --git a/client/src/App.jsx b/client/src/App.jsx index 6ca299aa55..6ee3be2a6b 100644 --- a/client/src/App.jsx +++ b/client/src/App.jsx @@ -6,42 +6,34 @@ import Nav from './components/Nav'; import MobileNav from './components/Nav/MobileNav'; import useDocumentTitle from '~/hooks/useDocumentTitle'; import { useSelector, useDispatch } from 'react-redux'; +import userAuth from './utils/userAuth'; import { setUser } from './store/userReducer'; +import { setSearchState } from './store/searchSlice'; import axios from 'axios'; const App = () => { const dispatch = useDispatch(); - + const { messages, messageTree } = useSelector((state) => state.messages); const { user } = useSelector((state) => state.user); const { title } = useSelector((state) => state.convo); - const [ navVisible, setNavVisible ]= useState(false) + const [navVisible, setNavVisible] = useState(false); useDocumentTitle(title); - useEffect(async () => { - try { - const response = await axios.get('/api/me', { - timeout: 1000, - withCredentials: true - }); - const user = response.data; - if (user) { - dispatch(setUser(user)); - } else { - console.log('Not login!'); - window.location.href = '/auth/login'; - } - } catch (error) { - console.error(error); - console.log('Not login!'); - window.location.href = '/auth/login'; - } - }, []) + useEffect(() => { + axios.get('/api/search/enable').then((res) => { console.log(res.data); dispatch(setSearchState(res.data))}); + userAuth() + .then((user) => dispatch(setUser(user))) + .catch((err) => console.log(err)); + }, []); if (user) return (
-