diff --git a/.gitignore b/.gitignore
index fa70c3c4c4..e201974879 100644
--- a/.gitignore
+++ b/.gitignore
@@ -47,7 +47,6 @@ bower_components/
.env
cache.json
api/data/
-.eslintrc.js
owner.yml
archive
.vscode/settings.json
diff --git a/api/app/bingai.js b/api/app/bingai.js
index d1175c0bfc..dcf150d1cf 100644
--- a/api/app/bingai.js
+++ b/api/app/bingai.js
@@ -1,7 +1,7 @@
require('dotenv').config();
const { KeyvFile } = require('keyv-file');
-const askBing = async ({ text, progressCallback, convo }) => {
+const askBing = async ({ text, onProgress, convo }) => {
const { BingAIClient } = (await import('@waylaidwanderer/chatgpt-api'));
const bingAIClient = new BingAIClient({
@@ -14,16 +14,15 @@ const askBing = async ({ text, progressCallback, convo }) => {
proxy: process.env.PROXY || null,
});
- let options = {
- onProgress: async (partialRes) => await progressCallback(partialRes),
- };
-
+ let options = { onProgress };
if (convo) {
options = { ...options, ...convo };
}
- const res = await bingAIClient.sendMessage(text, options
- );
+ if (options?.jailbreakConversationId == 'false')
+ options.jailbreakConversationId = false
+
+ const res = await bingAIClient.sendMessage(text, options);
return res;
diff --git a/api/app/chatgpt-browser.js b/api/app/chatgpt-browser.js
index 8a3c903641..a5bea22729 100644
--- a/api/app/chatgpt-browser.js
+++ b/api/app/chatgpt-browser.js
@@ -10,7 +10,7 @@ const clientOptions = {
proxy: process.env.PROXY || null,
};
-const browserClient = async ({ text, progressCallback, convo }) => {
+const browserClient = async ({ text, onProgress, convo }) => {
const { ChatGPTBrowserClient } = await import('@waylaidwanderer/chatgpt-api');
const store = {
@@ -18,10 +18,7 @@ const browserClient = async ({ text, progressCallback, convo }) => {
};
const client = new ChatGPTBrowserClient(clientOptions, store);
-
- let options = {
- onProgress: async (partialRes) => await progressCallback(partialRes)
- };
+ let options = { onProgress };
if (!!convo.parentMessageId && !!convo.conversationId) {
options = { ...options, ...convo };
diff --git a/api/app/chatgpt-client.js b/api/app/chatgpt-client.js
index afd31e0a81..350d1210ce 100644
--- a/api/app/chatgpt-client.js
+++ b/api/app/chatgpt-client.js
@@ -9,17 +9,14 @@ const clientOptions = {
debug: false
};
-const askClient = async ({ text, progressCallback, convo }) => {
+const askClient = async ({ text, onProgress, convo }) => {
const ChatGPTClient = (await import('@waylaidwanderer/chatgpt-api')).default;
const store = {
store: new KeyvFile({ filename: './data/cache.json' })
};
const client = new ChatGPTClient(process.env.OPENAI_KEY, clientOptions, store);
-
- let options = {
- onProgress: async (partialRes) => await progressCallback(partialRes)
- };
+ let options = { onProgress };
if (!!convo.parentMessageId && !!convo.conversationId) {
options = { ...options, ...convo };
diff --git a/api/app/chatgpt-custom.js b/api/app/chatgpt-custom.js
index 9eabb80ccd..e7a0ee0503 100644
--- a/api/app/chatgpt-custom.js
+++ b/api/app/chatgpt-custom.js
@@ -9,7 +9,7 @@ const clientOptions = {
debug: false
};
-const customClient = async ({ text, progressCallback, convo, promptPrefix, chatGptLabel }) => {
+const customClient = async ({ text, onProgress, convo, promptPrefix, chatGptLabel }) => {
const ChatGPTClient = (await import('@waylaidwanderer/chatgpt-api')).default;
const store = {
store: new KeyvFile({ filename: './data/cache.json' })
@@ -23,10 +23,7 @@ const customClient = async ({ text, progressCallback, convo, promptPrefix, chatG
const client = new ChatGPTClient(process.env.OPENAI_KEY, clientOptions, store);
- let options = {
- onProgress: async (partialRes) => await progressCallback(partialRes)
- };
-
+ let options = { onProgress };
if (!!convo.parentMessageId && !!convo.conversationId) {
options = { ...options, ...convo };
}
diff --git a/api/app/chatgpt.js b/api/app/chatgpt.js
deleted file mode 100644
index 18edcfca83..0000000000
--- a/api/app/chatgpt.js
+++ /dev/null
@@ -1,38 +0,0 @@
-require('dotenv').config();
-const Keyv = require('keyv');
-const { Configuration, OpenAIApi } = require('openai');
-const messageStore = new Keyv(process.env.MONGODB_URI, { namespace: 'chatgpt' });
-
-const ask = async (question, progressCallback, convo) => {
- const { ChatGPTAPI } = await import('chatgpt');
- const api = new ChatGPTAPI({ apiKey: process.env.OPENAI_KEY, messageStore });
- let options = {
- onProgress: async (partialRes) => {
- if (partialRes.text.length > 0) {
- await progressCallback(partialRes);
- }
- }
- };
-
- if (!!convo.parentMessageId && !!convo.conversationId) {
- options = { ...options, ...convo };
- }
-
- const res = await api.sendMessage(question, options);
- return res;
-};
-
-const titleConvo = async (message, response, model) => {
- const configuration = new Configuration({
- apiKey: process.env.OPENAI_KEY
- });
- const openai = new OpenAIApi(configuration);
- const completion = await openai.createCompletion({
- model: 'text-davinci-002',
- prompt: `Write a short title in title case, ideally in 5 words or less, and do not refer to the user or ${model}, that summarizes this conversation:\nUser:"${message}"\n${model}:"${response}"\nTitle: `
- });
-
- return completion.data.choices[0].text.replace(/\n/g, '');
-};
-
-module.exports = { ask, titleConvo };
diff --git a/api/app/sydney.js b/api/app/sydney.js
index fe47c74f57..3466f71c17 100644
--- a/api/app/sydney.js
+++ b/api/app/sydney.js
@@ -1,7 +1,7 @@
require('dotenv').config();
const { KeyvFile } = require('keyv-file');
-const askSydney = async ({ text, progressCallback, convo }) => {
+const askSydney = async ({ text, onProgress, convo }) => {
const { BingAIClient } = (await import('@waylaidwanderer/chatgpt-api'));
const sydneyClient = new BingAIClient({
@@ -15,10 +15,10 @@ const askSydney = async ({ text, progressCallback, convo }) => {
let options = {
jailbreakConversationId: true,
- onProgress: async (partialRes) => await progressCallback(partialRes),
+ onProgress,
};
- if (convo.parentMessageId) {
+ if (convo.jailbreakConversationId) {
options = { ...options, jailbreakConversationId: convo.jailbreakConversationId, parentMessageId: convo.parentMessageId };
}
diff --git a/api/app/titleConvo.js b/api/app/titleConvo.js
index e0461eabaf..e37fcb3b0c 100644
--- a/api/app/titleConvo.js
+++ b/api/app/titleConvo.js
@@ -1,24 +1,59 @@
const { Configuration, OpenAIApi } = require('openai');
+const _ = require('lodash');
-const titleConvo = async ({ message, response, model }) => {
- const configuration = new Configuration({
- apiKey: process.env.OPENAI_KEY
- });
- const openai = new OpenAIApi(configuration);
- const completion = await openai.createChatCompletion({
- model: 'gpt-3.5-turbo',
- messages: [
- {
- role: 'system',
- content:
- 'You are a title-generator with one job: giving a conversation, detect the language and titling the conversation provided by a user in title case, using the same language.'
- },
- { role: 'user', content: `In 5 words or less, summarize the conversation below with a title in title case using the language the user writes in. Don't refer to the participants of the conversation by name. Do not include punctuation or quotation marks. Your response should be in title case, exclusively containing the title. Conversation:\n\nUser: "${message}"\n\n${model}: "${response}"\n\nTitle: ` },
- ]
- });
+const proxyEnvToAxiosProxy = (proxyString) => {
+ if (!proxyString) return null;
- //eslint-disable-next-line
- return completion.data.choices[0].message.content.replace(/["\.]/g, '');
+ const regex = /^([^:]+):\/\/(?:([^:@]*):?([^:@]*)@)?([^:]+)(?::(\d+))?/;
+ const [, protocol, username, password, host, port] = proxyString.match(regex);
+ const proxyConfig = {
+ protocol,
+ host,
+ port: port ? parseInt(port) : undefined,
+ auth: username && password ? { username, password } : undefined
+ };
+
+ return proxyConfig;
};
-module.exports = titleConvo;
+const titleConvo = async ({ model, text, response }) => {
+ let title = 'New Chat';
+ try {
+ const configuration = new Configuration({
+ apiKey: process.env.OPENAI_KEY
+ });
+ const openai = new OpenAIApi(configuration);
+ const completion = await openai.createChatCompletion(
+ {
+ model: 'gpt-3.5-turbo',
+ messages: [
+ {
+ role: 'system',
+ content:
+ 'You are a title-generator with one job: giving a conversation, detect the language and titling the conversation provided by a user in title case, using the same language.'
+ },
+ {
+ role: 'user',
+ content: `In 5 words or less, summarize the conversation below with a title in title case using the language the user writes in. Don't refer to the participants of the conversation by name. Do not include punctuation or quotation marks. Your response should be in title case, exclusively containing the title. Conversation:\n\nUser: "${text}"\n\n${model}: "${JSON.stringify(
+ response?.text
+ )}"\n\nTitle: `
+ }
+ ]
+ },
+ { proxy: proxyEnvToAxiosProxy(process.env.PROXY || null) }
+ );
+
+ //eslint-disable-next-line
+ title = completion.data.choices[0].message.content.replace(/["\.]/g, '');
+ } catch (e) {
+ console.error(e);
+ console.log('There was an issue generating title, see error above');
+ }
+
+ console.log('CONVERSATION TITLE', title);
+ return title;
+};
+
+const throttledTitleConvo = _.throttle(titleConvo, 1000);
+
+module.exports = throttledTitleConvo;
diff --git a/api/models/Conversation.js b/api/models/Conversation.js
index 1ff778e957..45834379e8 100644
--- a/api/models/Conversation.js
+++ b/api/models/Conversation.js
@@ -1,48 +1,53 @@
const mongoose = require('mongoose');
+const crypto = require('crypto');
const { getMessages, deleteMessages } = require('./Message');
-const convoSchema = mongoose.Schema({
- conversationId: {
- type: String,
- unique: true,
- required: true
+const convoSchema = mongoose.Schema(
+ {
+ conversationId: {
+ type: String,
+ unique: true,
+ required: true
+ },
+ parentMessageId: {
+ type: String,
+ required: true
+ },
+ title: {
+ type: String,
+ default: 'New Chat'
+ },
+ jailbreakConversationId: {
+ type: String,
+ default: null
+ },
+ conversationSignature: {
+ type: String,
+ default: null
+ },
+ clientId: {
+ type: String
+ },
+ invocationId: {
+ type: String
+ },
+ chatGptLabel: {
+ type: String,
+ default: null
+ },
+ promptPrefix: {
+ type: String,
+ default: null
+ },
+ model: {
+ type: String,
+ required: true
+ },
+ suggestions: [{ type: String }],
+ messages: [{ type: mongoose.Schema.Types.ObjectId, ref: 'Message' }]
},
- parentMessageId: {
- type: String,
- required: true
- },
- title: {
- type: String,
- default: 'New conversation'
- },
- jailbreakConversationId: {
- type: String
- },
- conversationSignature: {
- type: String
- },
- clientId: {
- type: String
- },
- invocationId: {
- type: String
- },
- chatGptLabel: {
- type: String
- },
- promptPrefix: {
- type: String
- },
- model: {
- type: String
- },
- suggestions: [{ type: String }],
- messages: [{ type: mongoose.Schema.Types.ObjectId, ref: 'Message' }],
- created: {
- type: Date,
- default: Date.now
- }
-});
+ { timestamps: true }
+);
const Conversation =
mongoose.models.Conversation || mongoose.model('Conversation', convoSchema);
@@ -57,13 +62,24 @@ const getConvo = async (conversationId) => {
};
module.exports = {
- saveConvo: async ({ conversationId, title, ...convo }) => {
+ saveConvo: async ({ conversationId, newConversationId, title, ...convo }) => {
try {
const messages = await getMessages({ conversationId });
const update = { ...convo, messages };
if (title) {
update.title = title;
}
+ if (newConversationId) {
+ update.conversationId = newConversationId;
+ }
+ if (!update.jailbreakConversationId) {
+ update.jailbreakConversationId = null;
+ }
+ if (update.model !== 'chatgptCustom' && update.chatGptLabel && update.promptPrefix) {
+ console.log('Validation error: resetting chatgptCustom fields', update);
+ update.chatGptLabel = null;
+ update.promptPrefix = null;
+ }
return await Conversation.findOneAndUpdate(
{ conversationId },
@@ -85,20 +101,18 @@ module.exports = {
return { message: 'Error updating conversation' };
}
},
- // getConvos: async () => await Conversation.find({}).sort({ created: -1 }).exec(),
- getConvos: async (pageNumber = 1, pageSize = 12) => {
+ // getConvos: async () => await Conversation.find({}).sort({ createdAt: -1 }).exec(),
+ getConvosByPage: async (pageNumber = 1, pageSize = 12) => {
try {
- const skip = (pageNumber - 1) * pageSize;
- // const limit = pageNumber * pageSize;
-
- const conversations = await Conversation.find({})
- .sort({ created: -1 })
- .skip(skip)
- // .limit(limit)
+ const totalConvos = (await Conversation.countDocuments()) || 1;
+ const totalPages = Math.ceil(totalConvos / pageSize);
+ const convos = await Conversation.find()
+ .sort({ createdAt: -1, created: -1 })
+ .skip((pageNumber - 1) * pageSize)
.limit(pageSize)
.exec();
- return conversations;
+ return { conversations: convos, pages: totalPages, pageNumber, pageSize };
} catch (error) {
console.log(error);
return { message: 'Error getting conversations' };
@@ -118,5 +132,60 @@ module.exports = {
let deleteCount = await Conversation.deleteMany(filter).exec();
deleteCount.messages = await deleteMessages(filter);
return deleteCount;
+ },
+ migrateDb: async () => {
+ try {
+ const conversations = await Conversation.find({ model: null }).exec();
+
+ if (!conversations || conversations.length === 0)
+ return { message: '[Migrate] No conversations to migrate' };
+
+ for (let convo of conversations) {
+ const messages = await getMessages({
+ conversationId: convo.conversationId,
+ messageId: { $exists: false }
+ });
+
+ let model;
+ let oldId;
+ const promises = [];
+ messages.forEach((message, i) => {
+ const msgObj = message.toObject();
+ const newId = msgObj.id;
+ if (i === 0) {
+ message.parentMessageId = '00000000-0000-0000-0000-000000000000';
+ } else {
+ message.parentMessageId = oldId;
+ }
+
+ oldId = newId;
+ message.messageId = newId;
+ if (message.sender.toLowerCase() !== 'user' && !model) {
+ model = message.sender.toLowerCase();
+ }
+
+ if (message.sender.toLowerCase() === 'user') {
+ message.isCreatedByUser = true;
+ }
+ promises.push(message.save());
+ });
+ await Promise.all(promises);
+
+ await Conversation.findOneAndUpdate(
+ { conversationId: convo.conversationId },
+ { model },
+ { new: true }
+ ).exec();
+ }
+
+ try {
+ await mongoose.connection.db.collection('messages').dropIndex('id_1');
+ } catch (error) {
+ console.log("[Migrate] Index doesn't exist or already dropped");
+ }
+ } catch (error) {
+ console.log(error);
+ return { message: '[Migrate] Error migrating conversations' };
+ }
}
};
diff --git a/api/models/CustomGpt.js b/api/models/CustomGpt.js
index 33bb75b124..1f02bdc4db 100644
--- a/api/models/CustomGpt.js
+++ b/api/models/CustomGpt.js
@@ -12,11 +12,7 @@ const customGptSchema = mongoose.Schema({
type: String,
required: true
},
- created: {
- type: Date,
- default: Date.now
- }
-});
+}, { timestamps: true });
const CustomGpt = mongoose.models.CustomGpt || mongoose.model('CustomGpt', customGptSchema);
diff --git a/api/models/Message.js b/api/models/Message.js
index e6657c060a..90165acb86 100644
--- a/api/models/Message.js
+++ b/api/models/Message.js
@@ -1,7 +1,7 @@
const mongoose = require('mongoose');
const messageSchema = mongoose.Schema({
- id: {
+ messageId: {
type: String,
unique: true,
required: true
@@ -32,33 +32,50 @@ const messageSchema = mongoose.Schema({
type: String,
required: true
},
- created: {
- type: Date,
- default: Date.now
- }
-});
+ isCreatedByUser: {
+ type: Boolean,
+ required: true,
+ default: false
+ },
+ error: {
+ type: Boolean,
+ default: false
+ },
+}, { timestamps: true });
const Message = mongoose.models.Message || mongoose.model('Message', messageSchema);
module.exports = {
- saveMessage: async ({ id, conversationId, parentMessageId, sender, text }) => {
+ saveMessage: async ({ messageId, conversationId, parentMessageId, sender, text, isCreatedByUser=false, error }) => {
try {
- await Message.create({
- id,
+ await Message.findOneAndUpdate({ messageId }, {
conversationId,
parentMessageId,
sender,
- text
- });
- return { id, conversationId, parentMessageId, sender, text };
+ text,
+ isCreatedByUser,
+ error
+ }, { upsert: true, new: true });
+ return { messageId, conversationId, parentMessageId, sender, text, isCreatedByUser };
} catch (error) {
console.error(error);
return { message: 'Error saving message' };
}
},
+ deleteMessagesSince: async ({ messageId, conversationId }) => {
+ try {
+ const message = await Message.findOne({ messageId }).exec()
+
+ if (message)
+ return await Message.find({ conversationId }).deleteMany({ createdAt: { $gt: message.createdAt } }).exec();
+ } catch (error) {
+ console.error(error);
+ return { message: 'Error deleting messages' };
+ }
+ },
getMessages: async (filter) => {
try {
- return await Message.find(filter).exec()
+ return await Message.find(filter).sort({createdAt: 1}).exec()
} catch (error) {
console.error(error);
return { message: 'Error getting messages' };
diff --git a/api/models/Prompt.js b/api/models/Prompt.js
index 612278d038..f122d5af40 100644
--- a/api/models/Prompt.js
+++ b/api/models/Prompt.js
@@ -12,11 +12,7 @@ const promptSchema = mongoose.Schema({
category: {
type: String,
},
- created: {
- type: Date,
- default: Date.now
- }
-});
+}, { timestamps: true });
const Prompt = mongoose.models.Prompt || mongoose.model('Prompt', promptSchema);
diff --git a/api/models/index.js b/api/models/index.js
index 8af5a1aa9c..cd75b9c6a3 100644
--- a/api/models/index.js
+++ b/api/models/index.js
@@ -1,13 +1,15 @@
-const { saveMessage, deleteMessages } = require('./Message');
+const { saveMessage, deleteMessagesSince, deleteMessages } = require('./Message');
const { getCustomGpts, updateCustomGpt, updateByLabel, deleteCustomGpts } = require('./CustomGpt');
-const { getConvoTitle, getConvo, saveConvo } = require('./Conversation');
+const { getConvoTitle, getConvo, saveConvo, migrateDb } = require('./Conversation');
module.exports = {
saveMessage,
+ deleteMessagesSince,
deleteMessages,
getConvoTitle,
getConvo,
saveConvo,
+ migrateDb,
getCustomGpts,
updateCustomGpt,
updateByLabel,
diff --git a/api/package-lock.json b/api/package-lock.json
index 911fda953e..f934f00b8d 100644
--- a/api/package-lock.json
+++ b/api/package-lock.json
@@ -17,8 +17,10 @@
"express": "^4.18.2",
"keyv": "^4.5.2",
"keyv-file": "^0.2.0",
+ "lodash": "^4.17.21",
"mongoose": "^6.9.0",
- "openai": "^3.1.0"
+ "openai": "^3.1.0",
+ "sanitize-html": "^2.10.0"
},
"devDependencies": {
"nodemon": "^2.0.20",
@@ -2210,6 +2212,14 @@
}
}
},
+ "node_modules/deepmerge": {
+ "version": "4.3.0",
+ "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.0.tgz",
+ "integrity": "sha512-z2wJZXrmeHdvYJp/Ux55wIjqo81G5Bp4c+oELTW+7ar6SogWHajt5a9gO3s3IDaGSAXjDk0vlQKN3rms8ab3og==",
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
"node_modules/defaults": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.4.tgz",
@@ -2246,6 +2256,57 @@
"npm": "1.2.8000 || >= 1.4.16"
}
},
+ "node_modules/dom-serializer": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz",
+ "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==",
+ "dependencies": {
+ "domelementtype": "^2.3.0",
+ "domhandler": "^5.0.2",
+ "entities": "^4.2.0"
+ },
+ "funding": {
+ "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1"
+ }
+ },
+ "node_modules/domelementtype": {
+ "version": "2.3.0",
+ "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz",
+ "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==",
+ "funding": [
+ {
+ "type": "github",
+ "url": "https://github.com/sponsors/fb55"
+ }
+ ]
+ },
+ "node_modules/domhandler": {
+ "version": "5.0.3",
+ "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz",
+ "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==",
+ "dependencies": {
+ "domelementtype": "^2.3.0"
+ },
+ "engines": {
+ "node": ">= 4"
+ },
+ "funding": {
+ "url": "https://github.com/fb55/domhandler?sponsor=1"
+ }
+ },
+ "node_modules/domutils": {
+ "version": "3.0.1",
+ "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.0.1.tgz",
+ "integrity": "sha512-z08c1l761iKhDFtfXO04C7kTdPBLi41zwOZl00WS8b5eiaebNpY00HKbztwBq+e3vyqWNwWF3mP9YLUeqIrF+Q==",
+ "dependencies": {
+ "dom-serializer": "^2.0.0",
+ "domelementtype": "^2.3.0",
+ "domhandler": "^5.0.1"
+ },
+ "funding": {
+ "url": "https://github.com/fb55/domutils?sponsor=1"
+ }
+ },
"node_modules/dotenv": {
"version": "16.0.3",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.3.tgz",
@@ -2277,6 +2338,17 @@
"node": ">= 0.8"
}
},
+ "node_modules/entities": {
+ "version": "4.4.0",
+ "resolved": "https://registry.npmjs.org/entities/-/entities-4.4.0.tgz",
+ "integrity": "sha512-oYp7156SP8LkeGD0GF85ad1X9Ai79WtRsZ2gxJqtBuzH+98YUV6jkHEKlZkMbcrjJjIVJNIDP/3WL9wQkoPbWA==",
+ "engines": {
+ "node": ">=0.12"
+ },
+ "funding": {
+ "url": "https://github.com/fb55/entities?sponsor=1"
+ }
+ },
"node_modules/escape-html": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
@@ -2750,6 +2822,24 @@
"url": "https://github.com/sponsors/ljharb"
}
},
+ "node_modules/htmlparser2": {
+ "version": "8.0.1",
+ "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.1.tgz",
+ "integrity": "sha512-4lVbmc1diZC7GUJQtRQ5yBAeUCL1exyMwmForWkRLnwyzWBFxN633SALPMGYaWZvKe9j1pRZJpauvmxENSp/EA==",
+ "funding": [
+ "https://github.com/fb55/htmlparser2?sponsor=1",
+ {
+ "type": "github",
+ "url": "https://github.com/sponsors/fb55"
+ }
+ ],
+ "dependencies": {
+ "domelementtype": "^2.3.0",
+ "domhandler": "^5.0.2",
+ "domutils": "^3.0.1",
+ "entities": "^4.3.0"
+ }
+ },
"node_modules/http-errors": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz",
@@ -2957,6 +3047,14 @@
"node": ">=0.12.0"
}
},
+ "node_modules/is-plain-object": {
+ "version": "5.0.0",
+ "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz",
+ "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==",
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
"node_modules/is-stream": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz",
@@ -3272,6 +3370,17 @@
"resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz",
"integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA=="
},
+ "node_modules/nanoid": {
+ "version": "3.3.4",
+ "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.4.tgz",
+ "integrity": "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==",
+ "bin": {
+ "nanoid": "bin/nanoid.cjs"
+ },
+ "engines": {
+ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1"
+ }
+ },
"node_modules/negotiator": {
"version": "0.6.3",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
@@ -3468,6 +3577,11 @@
"p-defer": "^3.0.0"
}
},
+ "node_modules/parse-srcset": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/parse-srcset/-/parse-srcset-1.0.2.tgz",
+ "integrity": "sha512-/2qh0lav6CmI15FzA3i/2Bzk2zCgQhGMkvhOhKNcBVQ1ldgpbfiNTVslmooUmWJcADi1f1kIeynbDRVzNlfR6Q=="
+ },
"node_modules/parseurl": {
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
@@ -3576,6 +3690,29 @@
"resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-6.1.0.tgz",
"integrity": "sha512-KO0m2f1HkrPe9S0ldjx7za9BJjeHqBku5Ch8JyxETxT8dEFGz1PwgrHaOQupVYitpzbFSYm7nnljxD8dik2c+g=="
},
+ "node_modules/postcss": {
+ "version": "8.4.21",
+ "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.21.tgz",
+ "integrity": "sha512-tP7u/Sn/dVxK2NnruI4H9BG+x+Wxz6oeZ1cJ8P6G/PZY0IKk4k/63TDsQf2kQq3+qoJeLm2kIBUNlZe3zgb4Zg==",
+ "funding": [
+ {
+ "type": "opencollective",
+ "url": "https://opencollective.com/postcss/"
+ },
+ {
+ "type": "tidelift",
+ "url": "https://tidelift.com/funding/github/npm/postcss"
+ }
+ ],
+ "dependencies": {
+ "nanoid": "^3.3.4",
+ "picocolors": "^1.0.0",
+ "source-map-js": "^1.0.2"
+ },
+ "engines": {
+ "node": "^10 || ^12 || >=14"
+ }
+ },
"node_modules/process": {
"version": "0.11.10",
"resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz",
@@ -3795,6 +3932,30 @@
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
},
+ "node_modules/sanitize-html": {
+ "version": "2.10.0",
+ "resolved": "https://registry.npmjs.org/sanitize-html/-/sanitize-html-2.10.0.tgz",
+ "integrity": "sha512-JqdovUd81dG4k87vZt6uA6YhDfWkUGruUu/aPmXLxXi45gZExnt9Bnw/qeQU8oGf82vPyaE0vO4aH0PbobB9JQ==",
+ "dependencies": {
+ "deepmerge": "^4.2.2",
+ "escape-string-regexp": "^4.0.0",
+ "htmlparser2": "^8.0.0",
+ "is-plain-object": "^5.0.0",
+ "parse-srcset": "^1.0.2",
+ "postcss": "^8.3.11"
+ }
+ },
+ "node_modules/sanitize-html/node_modules/escape-string-regexp": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz",
+ "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==",
+ "engines": {
+ "node": ">=10"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
"node_modules/saslprep": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz",
@@ -3984,6 +4145,14 @@
"atomic-sleep": "^1.0.0"
}
},
+ "node_modules/source-map-js": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz",
+ "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==",
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
"node_modules/sparse-bitfield": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",
@@ -6274,6 +6443,11 @@
"ms": "2.1.2"
}
},
+ "deepmerge": {
+ "version": "4.3.0",
+ "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.0.tgz",
+ "integrity": "sha512-z2wJZXrmeHdvYJp/Ux55wIjqo81G5Bp4c+oELTW+7ar6SogWHajt5a9gO3s3IDaGSAXjDk0vlQKN3rms8ab3og=="
+ },
"defaults": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.4.tgz",
@@ -6297,6 +6471,39 @@
"resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz",
"integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg=="
},
+ "dom-serializer": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz",
+ "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==",
+ "requires": {
+ "domelementtype": "^2.3.0",
+ "domhandler": "^5.0.2",
+ "entities": "^4.2.0"
+ }
+ },
+ "domelementtype": {
+ "version": "2.3.0",
+ "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz",
+ "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw=="
+ },
+ "domhandler": {
+ "version": "5.0.3",
+ "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz",
+ "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==",
+ "requires": {
+ "domelementtype": "^2.3.0"
+ }
+ },
+ "domutils": {
+ "version": "3.0.1",
+ "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.0.1.tgz",
+ "integrity": "sha512-z08c1l761iKhDFtfXO04C7kTdPBLi41zwOZl00WS8b5eiaebNpY00HKbztwBq+e3vyqWNwWF3mP9YLUeqIrF+Q==",
+ "requires": {
+ "dom-serializer": "^2.0.0",
+ "domelementtype": "^2.3.0",
+ "domhandler": "^5.0.1"
+ }
+ },
"dotenv": {
"version": "16.0.3",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.3.tgz",
@@ -6322,6 +6529,11 @@
"resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
"integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w=="
},
+ "entities": {
+ "version": "4.4.0",
+ "resolved": "https://registry.npmjs.org/entities/-/entities-4.4.0.tgz",
+ "integrity": "sha512-oYp7156SP8LkeGD0GF85ad1X9Ai79WtRsZ2gxJqtBuzH+98YUV6jkHEKlZkMbcrjJjIVJNIDP/3WL9wQkoPbWA=="
+ },
"escape-html": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
@@ -6686,6 +6898,17 @@
"resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
"integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A=="
},
+ "htmlparser2": {
+ "version": "8.0.1",
+ "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.1.tgz",
+ "integrity": "sha512-4lVbmc1diZC7GUJQtRQ5yBAeUCL1exyMwmForWkRLnwyzWBFxN633SALPMGYaWZvKe9j1pRZJpauvmxENSp/EA==",
+ "requires": {
+ "domelementtype": "^2.3.0",
+ "domhandler": "^5.0.2",
+ "domutils": "^3.0.1",
+ "entities": "^4.3.0"
+ }
+ },
"http-errors": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz",
@@ -6825,6 +7048,11 @@
"integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
"dev": true
},
+ "is-plain-object": {
+ "version": "5.0.0",
+ "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz",
+ "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q=="
+ },
"is-stream": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz",
@@ -7073,6 +7301,11 @@
"resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz",
"integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA=="
},
+ "nanoid": {
+ "version": "3.3.4",
+ "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.4.tgz",
+ "integrity": "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw=="
+ },
"negotiator": {
"version": "0.6.3",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
@@ -7216,6 +7449,11 @@
"p-defer": "^3.0.0"
}
},
+ "parse-srcset": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/parse-srcset/-/parse-srcset-1.0.2.tgz",
+ "integrity": "sha512-/2qh0lav6CmI15FzA3i/2Bzk2zCgQhGMkvhOhKNcBVQ1ldgpbfiNTVslmooUmWJcADi1f1kIeynbDRVzNlfR6Q=="
+ },
"parseurl": {
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
@@ -7302,6 +7540,16 @@
"resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-6.1.0.tgz",
"integrity": "sha512-KO0m2f1HkrPe9S0ldjx7za9BJjeHqBku5Ch8JyxETxT8dEFGz1PwgrHaOQupVYitpzbFSYm7nnljxD8dik2c+g=="
},
+ "postcss": {
+ "version": "8.4.21",
+ "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.21.tgz",
+ "integrity": "sha512-tP7u/Sn/dVxK2NnruI4H9BG+x+Wxz6oeZ1cJ8P6G/PZY0IKk4k/63TDsQf2kQq3+qoJeLm2kIBUNlZe3zgb4Zg==",
+ "requires": {
+ "nanoid": "^3.3.4",
+ "picocolors": "^1.0.0",
+ "source-map-js": "^1.0.2"
+ }
+ },
"process": {
"version": "0.11.10",
"resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz",
@@ -7457,6 +7705,26 @@
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
},
+ "sanitize-html": {
+ "version": "2.10.0",
+ "resolved": "https://registry.npmjs.org/sanitize-html/-/sanitize-html-2.10.0.tgz",
+ "integrity": "sha512-JqdovUd81dG4k87vZt6uA6YhDfWkUGruUu/aPmXLxXi45gZExnt9Bnw/qeQU8oGf82vPyaE0vO4aH0PbobB9JQ==",
+ "requires": {
+ "deepmerge": "^4.2.2",
+ "escape-string-regexp": "^4.0.0",
+ "htmlparser2": "^8.0.0",
+ "is-plain-object": "^5.0.0",
+ "parse-srcset": "^1.0.2",
+ "postcss": "^8.3.11"
+ },
+ "dependencies": {
+ "escape-string-regexp": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz",
+ "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA=="
+ }
+ }
+ },
"saslprep": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz",
@@ -7614,6 +7882,11 @@
"atomic-sleep": "^1.0.0"
}
},
+ "source-map-js": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz",
+ "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw=="
+ },
"sparse-bitfield": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",
diff --git a/api/package.json b/api/package.json
index 28bd6769d3..561908fad3 100644
--- a/api/package.json
+++ b/api/package.json
@@ -27,8 +27,10 @@
"express": "^4.18.2",
"keyv": "^4.5.2",
"keyv-file": "^0.2.0",
+ "lodash": "^4.17.21",
"mongoose": "^6.9.0",
- "openai": "^3.1.0"
+ "openai": "^3.1.0",
+ "sanitize-html": "^2.10.0"
},
"devDependencies": {
"nodemon": "^2.0.20",
diff --git a/api/server/index.js b/api/server/index.js
index b380932531..bda88b14f4 100644
--- a/api/server/index.js
+++ b/api/server/index.js
@@ -1,5 +1,6 @@
const express = require('express');
const dbConnect = require('../models/dbConnect');
+const { migrateDb } = require('../models');
const path = require('path');
const cors = require('cors');
const routes = require('./routes');
@@ -7,7 +8,10 @@ const app = express();
const port = process.env.PORT || 3080;
const host = process.env.HOST || 'localhost'
const projectPath = path.join(__dirname, '..', '..', 'client');
-dbConnect().then(() => console.log('Connected to MongoDB'));
+dbConnect().then(() => {
+ console.log('Connected to MongoDB');
+ migrateDb();
+});
app.use(cors());
app.use(express.json());
diff --git a/api/server/routes/ask.js b/api/server/routes/ask.js
index 1f18082dd4..ad2a7178c2 100644
--- a/api/server/routes/ask.js
+++ b/api/server/routes/ask.js
@@ -3,37 +3,100 @@ const crypto = require('crypto');
const router = express.Router();
const askBing = require('./askBing');
const askSydney = require('./askSydney');
-const {
- titleConvo,
- askClient,
- browserClient,
- customClient,
- detectCode
-} = require('../../app/');
-const { getConvo, saveMessage, deleteMessages, saveConvo } = require('../../models');
-const { handleError, sendMessage } = require('./handlers');
+const { titleConvo, askClient, browserClient, customClient } = require('../../app/');
+const { getConvo, saveMessage, getConvoTitle, saveConvo } = require('../../models');
+const { handleError, sendMessage, createOnProgress, handleText } = require('./handlers');
+const { getMessages } = require('../../models/Message');
router.use('/bing', askBing);
router.use('/sydney', askSydney);
router.post('/', async (req, res) => {
- let { model, text, parentMessageId, conversationId, chatGptLabel, promptPrefix } = req.body;
+ let { model, text, parentMessageId, conversationId: oldConversationId, ...convo } = req.body;
if (text.length === 0) {
- return handleError(res, 'Prompt empty or too short');
+ return handleError(res, { text: 'Prompt empty or too short' });
}
+ const conversationId = oldConversationId || crypto.randomUUID();
+
const userMessageId = crypto.randomUUID();
- let userMessage = { id: userMessageId, sender: 'User', text };
+ const userParentMessageId = parentMessageId || '00000000-0000-0000-0000-000000000000';
+ let userMessage = {
+ messageId: userMessageId,
+ sender: 'User',
+ text,
+ parentMessageId: userParentMessageId,
+ conversationId,
+ isCreatedByUser: true
+ };
console.log('ask log', {
model,
...userMessage,
- parentMessageId,
- conversationId,
- chatGptLabel,
- promptPrefix
+ ...convo
});
+ await saveMessage(userMessage);
+ await saveConvo({ ...userMessage, model, ...convo });
+
+ return await ask({
+ userMessage,
+ model,
+ convo,
+ preSendRequest: true,
+ req,
+ res
+ });
+});
+
+router.post('/regenerate', async (req, res) => {
+ const { model } = req.body;
+
+ const oldUserMessage = await getMessages({ messageId: req.body });
+
+ if (oldUserMessage) {
+ const convo = await getConvo(userMessage?.conversationId);
+
+ const userMessageId = crypto.randomUUID();
+
+ let userMessage = {
+ ...userMessage,
+ messageId: userMessageId
+ };
+
+ console.log('ask log for regeneration', {
+ model,
+ ...userMessage,
+ ...convo
+ });
+
+ return await ask({
+ userMessage,
+ model,
+ convo,
+ preSendRequest: false,
+ req,
+ res
+ });
+ } else return handleError(res, { text: 'Parent message not found' });
+});
+
+const ask = async ({
+ userMessage,
+ overrideParentMessageId = null,
+ model,
+ convo,
+ preSendRequest = true,
+ req,
+ res
+}) => {
+ let {
+ text,
+ parentMessageId: userParentMessageId,
+ conversationId,
+ messageId: userMessageId
+ } = userMessage;
+
let client;
if (model === 'chatgpt') {
@@ -44,15 +107,6 @@ router.post('/', async (req, res) => {
client = browserClient;
}
- if (model === 'chatgptCustom' && !chatGptLabel && conversationId) {
- const convo = await getConvo({ conversationId });
- if (convo) {
- console.log('found convo for custom gpt', { convo })
- chatGptLabel = convo.chatGptLabel;
- promptPrefix = convo.promptPrefix;
- }
- }
-
res.writeHead(200, {
Connection: 'keep-alive',
'Content-Type': 'text/event-stream',
@@ -61,54 +115,27 @@ router.post('/', async (req, res) => {
'X-Accel-Buffering': 'no'
});
+ if (preSendRequest) sendMessage(res, { message: userMessage, created: true });
+
try {
- let i = 0;
- let tokens = '';
- const progressCallback = async (partial) => {
- if (i === 0 && typeof partial === 'object') {
- userMessage.parentMessageId = parentMessageId ? parentMessageId : partial.id;
- userMessage.conversationId = conversationId ? conversationId : partial.conversationId;
- await saveMessage(userMessage);
- sendMessage(res, { ...partial, initial: true });
- i++;
- }
-
- if (typeof partial === 'object') {
- sendMessage(res, { ...partial, message: true });
- } else {
- tokens += partial === text ? '' : partial;
- if (tokens.match(/^\n/)) {
- tokens = tokens.replace(/^\n/, '');
- }
-
- if (tokens.includes('[DONE]')) {
- tokens = tokens.replace('[DONE]', '');
- }
-
- // tokens = await detectCode(tokens);
- sendMessage(res, { text: tokens, message: true, initial: i === 0 ? true : false });
- i++;
- }
- };
-
+ const progressCallback = createOnProgress();
let gptResponse = await client({
text,
- progressCallback,
+ onProgress: progressCallback.call(null, model, { res, text }),
convo: {
- parentMessageId,
- conversationId
+ parentMessageId: userParentMessageId,
+ conversationId,
+ ...convo
},
- chatGptLabel,
- promptPrefix
+ ...convo
});
console.log('CLIENT RESPONSE', gptResponse);
if (!gptResponse.parentMessageId) {
gptResponse.text = gptResponse.response;
- gptResponse.id = gptResponse.messageId;
- gptResponse.parentMessageId = gptResponse.messageId;
- userMessage.parentMessageId = parentMessageId ? parentMessageId : gptResponse.messageId;
+ // gptResponse.id = gptResponse.messageId;
+ gptResponse.parentMessageId = overrideParentMessageId || userMessageId;
userMessage.conversationId = conversationId
? conversationId
: gptResponse.conversationId;
@@ -121,37 +148,65 @@ router.post('/', async (req, res) => {
gptResponse.text.toLowerCase().includes('no response') ||
gptResponse.text.toLowerCase().includes('no answer')
) {
- return handleError(res, 'Prompt empty or too short');
- }
-
- if (!parentMessageId) {
- gptResponse.title = await titleConvo({
- model,
- message: text,
- response: JSON.stringify(gptResponse.text)
+ await saveMessage({
+ messageId: crypto.randomUUID(),
+ sender: model,
+ conversationId,
+ parentMessageId: overrideParentMessageId || userMessageId,
+ error: true,
+ text: 'Prompt empty or too short'
});
- }
- gptResponse.sender = model === 'chatgptCustom' ? chatGptLabel : model;
- gptResponse.final = true;
- gptResponse.text = await detectCode(gptResponse.text);
-
- if (chatGptLabel?.length > 0 && model === 'chatgptCustom') {
- gptResponse.chatGptLabel = chatGptLabel;
+ return handleError(res, { text: 'Prompt empty or too short' });
}
- if (promptPrefix?.length > 0 && model === 'chatgptCustom') {
- gptResponse.promptPrefix = promptPrefix;
+ gptResponse.sender = model === 'chatgptCustom' ? convo.chatGptLabel : model;
+ gptResponse.model = model;
+ // gptResponse.final = true;
+ gptResponse.text = await handleText(gptResponse);
+
+ if (convo.chatGptLabel?.length > 0 && model === 'chatgptCustom') {
+ gptResponse.chatGptLabel = convo.chatGptLabel;
}
+ if (convo.promptPrefix?.length > 0 && model === 'chatgptCustom') {
+ gptResponse.promptPrefix = convo.promptPrefix;
+ }
+
+ // override the parentMessageId, for the regeneration.
+ gptResponse.parentMessageId = overrideParentMessageId || userMessageId;
+
await saveMessage(gptResponse);
await saveConvo(gptResponse);
- sendMessage(res, gptResponse);
+ sendMessage(res, {
+ title: await getConvoTitle(conversationId),
+ final: true,
+ requestMessage: userMessage,
+ responseMessage: gptResponse
+ });
res.end();
+
+ if (userParentMessageId == '00000000-0000-0000-0000-000000000000') {
+ const title = await titleConvo({ model, text, response: gptResponse });
+
+ await saveConvo({
+ conversationId,
+ title
+ });
+ }
} catch (error) {
console.log(error);
- await deleteMessages({ id: userMessageId });
- handleError(res, error.message);
+ // await deleteMessages({ messageId: userMessageId });
+ const errorMessage = {
+ messageId: crypto.randomUUID(),
+ sender: model,
+ conversationId,
+ parentMessageId: overrideParentMessageId || userMessageId,
+ error: true,
+ text: error.message
+ };
+ await saveMessage(errorMessage);
+ handleError(res, errorMessage);
}
-});
+};
module.exports = router;
diff --git a/api/server/routes/askBing.js b/api/server/routes/askBing.js
index 76c39f6804..cfbac77185 100644
--- a/api/server/routes/askBing.js
+++ b/api/server/routes/askBing.js
@@ -1,21 +1,72 @@
const express = require('express');
const crypto = require('crypto');
const router = express.Router();
-const { titleConvo, getCitations, citeText, askBing } = require('../../app/');
-const { saveMessage, deleteMessages, saveConvo } = require('../../models');
-const { handleError, sendMessage } = require('./handlers');
-const citationRegex = /\[\^\d+?\^]/g;
+const { titleConvo, askBing } = require('../../app/');
+const { saveMessage, getConvoTitle, saveConvo } = require('../../models');
+const { handleError, sendMessage, createOnProgress, handleText } = require('./handlers');
router.post('/', async (req, res) => {
- const { model, text, ...convo } = req.body;
+ const {
+ model,
+ text,
+ parentMessageId,
+ conversationId: oldConversationId,
+ ...convo
+ } = req.body;
if (text.length === 0) {
- return handleError(res, 'Prompt empty or too short');
+ return handleError(res, { text: 'Prompt empty or too short' });
}
- const userMessageId = crypto.randomUUID();
- let userMessage = { id: userMessageId, sender: 'User', text };
+ const conversationId = oldConversationId || crypto.randomUUID();
+ const isNewConversation = !oldConversationId;
- console.log('ask log', { model, ...userMessage, ...convo });
+ const userMessageId = crypto.randomUUID();
+ const userParentMessageId = parentMessageId || '00000000-0000-0000-0000-000000000000';
+ let userMessage = {
+ messageId: userMessageId,
+ sender: 'User',
+ text,
+ parentMessageId: userParentMessageId,
+ conversationId,
+ isCreatedByUser: true
+ };
+
+ console.log('ask log', {
+ model,
+ ...userMessage,
+ ...convo
+ });
+
+ await saveMessage(userMessage);
+ await saveConvo({ ...userMessage, model, ...convo });
+
+ return await ask({
+ isNewConversation,
+ userMessage,
+ model,
+ convo,
+ preSendRequest: true,
+ req,
+ res
+ });
+});
+
+const ask = async ({
+ isNewConversation,
+ overrideParentMessageId = null,
+ userMessage,
+ model,
+ convo,
+ preSendRequest = true,
+ req,
+ res
+}) => {
+ let {
+ text,
+ parentMessageId: userParentMessageId,
+ conversationId,
+ messageId: userMessageId
+ } = userMessage;
res.writeHead(200, {
Connection: 'keep-alive',
@@ -25,62 +76,91 @@ router.post('/', async (req, res) => {
'X-Accel-Buffering': 'no'
});
- try {
- let tokens = '';
- const progressCallback = async (partial) => {
- tokens += partial === text ? '' : partial;
- // tokens = appendCode(tokens);
- tokens = citeText(tokens, true);
- sendMessage(res, { text: tokens, message: true });
- };
+ if (preSendRequest) sendMessage(res, { message: userMessage, created: true });
+ try {
+ const progressCallback = createOnProgress();
let response = await askBing({
text,
- progressCallback,
- convo
+ onProgress: progressCallback.call(null, model, {
+ res,
+ text,
+ parentMessageId: overrideParentMessageId || userMessageId
+ }),
+ convo: {
+ ...convo,
+ parentMessageId: userParentMessageId,
+ conversationId
+ }
});
- console.log('BING RESPONSE');
+ console.log('BING RESPONSE', response);
// console.dir(response, { depth: null });
const hasCitations = response.response.match(citationRegex)?.length > 0;
userMessage.conversationSignature =
convo.conversationSignature || response.conversationSignature;
- userMessage.conversationId = convo.conversationId || response.conversationId;
+ userMessage.conversationId = response.conversationId || conversationId;
userMessage.invocationId = response.invocationId;
await saveMessage(userMessage);
- if (!convo.conversationSignature) {
- response.title = await titleConvo({
- model,
- message: text,
- response: JSON.stringify(response.response)
+ // Bing API will not use our conversationId at the first time,
+ // so change the placeholder conversationId to the real one.
+ // Attition: the api will also create new conversationId while using invalid userMessage.parentMessageId,
+ // but in this situation, don't change the conversationId, but create new convo.
+ if (conversationId != userMessage.conversationId && isNewConversation)
+ await saveConvo({
+ conversationId: conversationId,
+ newConversationId: userMessage.conversationId
});
- }
+ conversationId = userMessage.conversationId;
response.text = response.response;
delete response.response;
- response.id = response.details.messageId;
+ // response.id = response.details.messageId;
response.suggestions =
response.details.suggestedResponses &&
response.details.suggestedResponses.map((s) => s.text);
response.sender = model;
- response.final = true;
+ // response.final = true;
- const links = getCitations(response);
- response.text =
- citeText(response) +
- (links?.length > 0 && hasCitations ? `\n${links}` : '');
+ // override the parentMessageId, for the regeneration.
+ response.parentMessageId =
+ overrideParentMessageId || response.parentMessageId || userMessageId;
+ response.text = await handleText(response, true);
await saveMessage(response);
- await saveConvo(response);
- sendMessage(res, response);
+ await saveConvo({ ...response, model, chatGptLabel: null, promptPrefix: null, ...convo });
+ sendMessage(res, {
+ title: await getConvoTitle(conversationId),
+ final: true,
+ requestMessage: userMessage,
+ responseMessage: response
+ });
res.end();
+
+ if (userParentMessageId == '00000000-0000-0000-0000-000000000000') {
+ const title = await titleConvo({ model, text, response });
+
+ await saveConvo({
+ conversationId,
+ title
+ });
+ }
} catch (error) {
console.log(error);
- await deleteMessages({ id: userMessageId });
- handleError(res, error.message);
+ // await deleteMessages({ messageId: userMessageId });
+ const errorMessage = {
+ messageId: crypto.randomUUID(),
+ sender: model,
+ conversationId,
+ parentMessageId: overrideParentMessageId || userMessageId,
+ error: true,
+ text: error.message
+ };
+ await saveMessage(errorMessage);
+ handleError(res, errorMessage);
}
-});
+};
module.exports = router;
diff --git a/api/server/routes/askSydney.js b/api/server/routes/askSydney.js
index 219f9b2437..00f287fa7b 100644
--- a/api/server/routes/askSydney.js
+++ b/api/server/routes/askSydney.js
@@ -1,21 +1,72 @@
const express = require('express');
const crypto = require('crypto');
const router = express.Router();
-const { titleConvo, getCitations, citeText, askSydney } = require('../../app/');
-const { saveMessage, deleteMessages, saveConvo, getConvoTitle } = require('../../models');
-const { handleError, sendMessage } = require('./handlers');
-const citationRegex = /\[\^\d+?\^]/g;
+const { titleConvo, askSydney } = require('../../app/');
+const { saveMessage, saveConvo, getConvoTitle } = require('../../models');
+const { handleError, sendMessage, createOnProgress, handleText } = require('./handlers');
router.post('/', async (req, res) => {
- const { model, text, ...convo } = req.body;
+ const {
+ model,
+ text,
+ parentMessageId,
+ conversationId: oldConversationId,
+ ...convo
+ } = req.body;
if (text.length === 0) {
- return handleError(res, 'Prompt empty or too short');
+ return handleError(res, { text: 'Prompt empty or too short' });
}
- const userMessageId = crypto.randomUUID();
- let userMessage = { id: userMessageId, sender: 'User', text };
+ const conversationId = oldConversationId || crypto.randomUUID();
+ const isNewConversation = !oldConversationId;
- console.log('ask log', { model, ...userMessage, ...convo });
+ const userMessageId = crypto.randomUUID();
+ const userParentMessageId = parentMessageId || '00000000-0000-0000-0000-000000000000';
+ let userMessage = {
+ messageId: userMessageId,
+ sender: 'User',
+ text,
+ parentMessageId: userParentMessageId,
+ conversationId,
+ isCreatedByUser: true
+ };
+
+ console.log('ask log', {
+ model,
+ ...userMessage,
+ ...convo
+ });
+
+ await saveMessage(userMessage);
+ await saveConvo({ ...userMessage, model, ...convo });
+
+ return await ask({
+ isNewConversation,
+ userMessage,
+ model,
+ convo,
+ preSendRequest: true,
+ req,
+ res
+ });
+});
+
+const ask = async ({
+ isNewConversation,
+ overrideParentMessageId = null,
+ userMessage,
+ model,
+ convo,
+ preSendRequest = true,
+ req,
+ res
+}) => {
+ let {
+ text,
+ parentMessageId: userParentMessageId,
+ conversationId,
+ messageId: userMessageId
+ } = userMessage;
res.writeHead(200, {
Connection: 'keep-alive',
@@ -25,41 +76,38 @@ router.post('/', async (req, res) => {
'X-Accel-Buffering': 'no'
});
- try {
- let tokens = '';
- const progressCallback = async (partial) => {
- tokens += partial === text ? '' : partial;
- // tokens = appendCode(tokens);
- tokens = citeText(tokens, true);
- sendMessage(res, { text: tokens, message: true });
- };
+ if (preSendRequest) sendMessage(res, { message: userMessage, created: true });
+ try {
+ const progressCallback = createOnProgress();
let response = await askSydney({
text,
- progressCallback,
- convo
+ onProgress: progressCallback.call(null, model, {
+ res,
+ text,
+ parentMessageId: overrideParentMessageId || userMessageId
+ }),
+ convo: {
+ parentMessageId: userParentMessageId,
+ conversationId,
+ ...convo
+ }
});
- console.log('SYDNEY RESPONSE');
- console.log(response.response);
+ console.log('SYDNEY RESPONSE', response);
// console.dir(response, { depth: null });
- const hasCitations = response.response.match(citationRegex)?.length > 0;
+
+ userMessage.conversationSignature =
+ convo.conversationSignature || response.conversationSignature;
+ userMessage.conversationId = response.conversationId || conversationId;
+ userMessage.invocationId = response.invocationId;
+ // Unlike gpt and bing, Sydney will never accept our given userMessage.messageId, it will generate its own one.
+ await saveMessage(userMessage);
// Save sydney response
- response.id = response.messageId;
- // response.parentMessageId = convo.parentMessageId ? convo.parentMessageId : response.messageId;
- response.parentMessageId = response.messageId;
+ // response.id = response.messageId;
response.invocationId = convo.invocationId ? convo.invocationId + 1 : 1;
- response.title = convo.jailbreakConversationId
- ? await getConvoTitle(convo.conversationId)
- : await titleConvo({
- model,
- message: text,
- response: JSON.stringify(response.response)
- });
- response.conversationId = convo.conversationId
- ? convo.conversationId
- : crypto.randomUUID();
+ response.conversationId = conversationId ? conversationId : crypto.randomUUID();
response.conversationSignature = convo.conversationSignature
? convo.conversationSignature
: crypto.randomUUID();
@@ -69,28 +117,61 @@ router.post('/', async (req, res) => {
response.details.suggestedResponses &&
response.details.suggestedResponses.map((s) => s.text);
response.sender = model;
- response.final = true;
+ // response.final = true;
- const links = getCitations(response);
- response.text =
- citeText(response) +
- (links?.length > 0 && hasCitations ? `\n${links}` : '');
+ // override the parentMessageId, for the regeneration.
+ response.parentMessageId =
+ overrideParentMessageId || response.parentMessageId || userMessageId;
// Save user message
- userMessage.conversationId = response.conversationId;
- userMessage.parentMessageId = response.parentMessageId;
+ userMessage.conversationId = response.conversationId || conversationId;
await saveMessage(userMessage);
+ // Bing API will not use our conversationId at the first time,
+ // so change the placeholder conversationId to the real one.
+ // Attition: the api will also create new conversationId while using invalid userMessage.parentMessageId,
+ // but in this situation, don't change the conversationId, but create new convo.
+ if (conversationId != userMessage.conversationId && isNewConversation)
+ await saveConvo({
+ conversationId: conversationId,
+ newConversationId: userMessage.conversationId
+ });
+ conversationId = userMessage.conversationId;
+
+ response.text = await handleText(response, true);
// Save sydney response & convo, then send
await saveMessage(response);
- await saveConvo(response);
- sendMessage(res, response);
+ await saveConvo({ ...response, model, chatGptLabel: null, promptPrefix: null, ...convo });
+ sendMessage(res, {
+ title: await getConvoTitle(conversationId),
+ final: true,
+ requestMessage: userMessage,
+ responseMessage: response
+ });
res.end();
+
+ if (userParentMessageId == '00000000-0000-0000-0000-000000000000') {
+ const title = await titleConvo({ model, text, response });
+
+ await saveConvo({
+ conversationId,
+ title
+ });
+ }
} catch (error) {
console.log(error);
- await deleteMessages({ id: userMessageId });
- handleError(res, error.message);
+ // await deleteMessages({ messageId: userMessageId });
+ const errorMessage = {
+ messageId: crypto.randomUUID(),
+ sender: model,
+ conversationId,
+ parentMessageId: overrideParentMessageId || userMessageId,
+ error: true,
+ text: error.message
+ };
+ await saveMessage(errorMessage);
+ handleError(res, errorMessage);
}
-});
+};
module.exports = router;
diff --git a/api/server/routes/convos.js b/api/server/routes/convos.js
index 4b9320873f..99fb824708 100644
--- a/api/server/routes/convos.js
+++ b/api/server/routes/convos.js
@@ -1,10 +1,10 @@
const express = require('express');
const router = express.Router();
-const { getConvos, deleteConvos, updateConvo } = require('../../models/Conversation');
+const { getConvosByPage, deleteConvos, updateConvo } = require('../../models/Conversation');
router.get('/', async (req, res) => {
const pageNumber = req.query.pageNumber || 1;
- res.status(200).send(await getConvos(pageNumber));
+ res.status(200).send(await getConvosByPage(pageNumber));
});
router.post('/clear', async (req, res) => {
diff --git a/api/server/routes/handlers.js b/api/server/routes/handlers.js
index edd64e6184..3ba4e81243 100644
--- a/api/server/routes/handlers.js
+++ b/api/server/routes/handlers.js
@@ -1,5 +1,11 @@
-const handleError = (res, errorMessage) => {
- res.status(500).write(`event: error\ndata: ${errorMessage}`);
+const _ = require('lodash');
+const sanitizeHtml = require('sanitize-html');
+const citationRegex = /\[\^\d+?\^]/g;
+const { getCitations, citeText, detectCode } = require('../../app/');
+// const htmlTagRegex = /(<\/?\s*[a-zA-Z]*\s*(?:\s+[a-zA-Z]+\s*=\s*(?:"[^"]*"|'[^']*'))*\s*(?:\/?)>|<\s*[a-zA-Z]+\s*(?:\s+[a-zA-Z]+\s*=\s*(?:"[^"]*"|'[^']*'))*\s*(?:\/?>|<\/?>))/g;
+
+const handleError = (res, message) => {
+ res.write(`event: error\ndata: ${JSON.stringify(message)}\n\n`);
res.end();
};
@@ -10,4 +16,65 @@ const sendMessage = (res, message) => {
res.write(`event: message\ndata: ${JSON.stringify(message)}\n\n`);
};
-module.exports = { handleError, sendMessage };
+const createOnProgress = () => {
+ let i = 0;
+ let tokens = '';
+
+ const progressCallback = async (partial, { res, text, bing = false, ...rest }) => {
+ tokens += partial === text ? '' : partial;
+ tokens = tokens.replaceAll('[DONE]', '');
+
+ if (tokens.match(/^\n/)) {
+ tokens = tokens.replace(/^\n/, '');
+ }
+
+ // const htmlTags = tokens.match(htmlTagRegex);
+ // if (tokens.includes('```') && htmlTags && htmlTags.length > 0) {
+ // htmlTags.forEach((tag) => {
+ // const sanitizedTag = sanitizeHtml(tag);
+ // tokens = tokens.replaceAll(tag, sanitizedTag);
+ // });
+ // }
+
+ if (bing) {
+ tokens = citeText(tokens, true);
+ }
+
+ sendMessage(res, { text: tokens, message: true, initial: i === 0, ...rest });
+ i++;
+ };
+
+ const onProgress = (model, opts) => {
+ const bingModels = new Set(['bingai', 'sydney']);
+ return _.partialRight(progressCallback, { ...opts, bing: bingModels.has(model) });
+ };
+
+ return onProgress;
+};
+
+const handleText = async (response, bing = false) => {
+ let { text } = response;
+ text = await detectCode(text);
+ response.text = text;
+
+ if (bing) {
+ // const hasCitations = response.response.match(citationRegex)?.length > 0;
+ const links = getCitations(response);
+ if (response.text.match(citationRegex)?.length > 0) {
+ text = citeText(response);
+ }
+ text += links?.length > 0 ? `\n${links}` : '';
+ }
+
+ // const htmlTags = text.match(htmlTagRegex);
+ // if (text.includes('```') && htmlTags && htmlTags.length > 0) {
+ // htmlTags.forEach((tag) => {
+ // const sanitizedTag = sanitizeHtml(tag);
+ // text = text.replaceAll(tag, sanitizedTag);
+ // });
+ // }
+
+ return text;
+};
+
+module.exports = { handleError, sendMessage, createOnProgress, handleText };
diff --git a/client/package-lock.json b/client/package-lock.json
index fbfcb2b345..d5e3024751 100644
--- a/client/package-lock.json
+++ b/client/package-lock.json
@@ -11125,9 +11125,9 @@
}
},
"node_modules/webpack": {
- "version": "5.75.0",
- "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.75.0.tgz",
- "integrity": "sha512-piaIaoVJlqMsPtX/+3KTTO6jfvrSYgauFVdt8cr9LTHKmcq/AMd4mhzsiP7ZF/PGRNPGA8336jldh9l2Kt2ogQ==",
+ "version": "5.76.1",
+ "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.76.1.tgz",
+ "integrity": "sha512-4+YIK4Abzv8172/SGqObnUjaIHjLEuUasz9EwQj/9xmPPkYJy2Mh03Q/lJfSD3YLzbxy5FeTq5Uw0323Oh6SJQ==",
"dev": true,
"dependencies": {
"@types/eslint-scope": "^3.7.3",
@@ -19431,9 +19431,9 @@
}
},
"webpack": {
- "version": "5.75.0",
- "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.75.0.tgz",
- "integrity": "sha512-piaIaoVJlqMsPtX/+3KTTO6jfvrSYgauFVdt8cr9LTHKmcq/AMd4mhzsiP7ZF/PGRNPGA8336jldh9l2Kt2ogQ==",
+ "version": "5.76.1",
+ "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.76.1.tgz",
+ "integrity": "sha512-4+YIK4Abzv8172/SGqObnUjaIHjLEuUasz9EwQj/9xmPPkYJy2Mh03Q/lJfSD3YLzbxy5FeTq5Uw0323Oh6SJQ==",
"dev": true,
"requires": {
"@types/eslint-scope": "^3.7.3",
diff --git a/client/src/App.jsx b/client/src/App.jsx
index 9bab1f5939..5f1446fd0b 100644
--- a/client/src/App.jsx
+++ b/client/src/App.jsx
@@ -8,7 +8,7 @@ import useDocumentTitle from '~/hooks/useDocumentTitle';
import { useSelector } from 'react-redux';
const App = () => {
- const { messages } = useSelector((state) => state.messages);
+ const { messages, messageTree } = useSelector((state) => state.messages);
const { title } = useSelector((state) => state.convo);
const { conversationId } = useSelector((state) => state.convo);
const [ navVisible, setNavVisible ]= useState(false)
@@ -25,6 +25,7 @@ const App = () => {
) : (