Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-12-17 00:40:14 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

chore: delegate text handling to one place, html sanitization in progress

Browse source
This commit is contained in:
Danny Avila 2023-03-14 16:05:46 -04:00
parent 2e20b28c4d
commit d0ef0f84c8
4 changed files with 27 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

7
api/server/routes/ask.js
View file

@ -8,10 +8,10 @@ const {
askClient, askClient,
browserClient, browserClient,
customClient, customClient,
detectCode // detectCode
} = require('../../app/'); } = require('../../app/');
const { getConvo, saveMessage, getConvoTitle, saveConvo } = require('../../models'); const { getConvo, saveMessage, getConvoTitle, saveConvo } = require('../../models');
const { handleError, sendMessage, createOnProgress } = require('./handlers'); const { handleError, sendMessage, createOnProgress, handleText } = require('./handlers');
const { getMessages } = require('../../models/Message'); const { getMessages } = require('../../models/Message');
router.use('/bing', askBing); router.use('/bing', askBing);
@ -177,7 +177,8 @@ const ask = async ({
gptResponse.sender = model === 'chatgptCustom' ? convo.chatGptLabel : model; gptResponse.sender = model === 'chatgptCustom' ? convo.chatGptLabel : model;
// gptResponse.final = true; // gptResponse.final = true;
gptResponse.text = await detectCode(gptResponse.text); // gptResponse.text = await detectCode(gptResponse.text);
gptResponse.text = await handleText(gptResponse.text);
if (convo.chatGptLabel?.length > 0 && model === 'chatgptCustom') { if (convo.chatGptLabel?.length > 0 && model === 'chatgptCustom') {
gptResponse.chatGptLabel = convo.chatGptLabel; gptResponse.chatGptLabel = convo.chatGptLabel;

3
api/server/routes/askBing.js
View file

@ -3,7 +3,7 @@ const crypto = require('crypto');
const router = express.Router(); const router = express.Router();
const { titleConvo, getCitations, citeText, askBing } = require('../../app/'); const { titleConvo, getCitations, citeText, askBing } = require('../../app/');
const { saveMessage, getConvoTitle, saveConvo } = require('../../models'); const { saveMessage, getConvoTitle, saveConvo } = require('../../models');
const { handleError, sendMessage, createOnProgress } = require('./handlers'); const { handleError, sendMessage, createOnProgress, handleText } = require('./handlers');
const citationRegex = /\[\^\d+?\^]/g; const citationRegex = /\[\^\d+?\^]/g;
router.post('/', async (req, res) => { router.post('/', async (req, res) => {
@ -113,6 +113,7 @@ const ask = async ({
response.text = response.text =
citeText(response) + citeText(response) +
(links?.length > 0 && hasCitations ? `\n<small>${links}</small>` : ''); (links?.length > 0 && hasCitations ? `\n<small>${links}</small>` : '');
response.text = await handleText(response.text);
await saveMessage(response); await saveMessage(response);
await saveConvo({...response, model, ...convo}); await saveConvo({...response, model, ...convo});

3
api/server/routes/askSydney.js
View file

@ -3,7 +3,7 @@ const crypto = require('crypto');
const router = express.Router(); const router = express.Router();
const { titleConvo, getCitations, citeText, askSydney } = require('../../app/'); const { titleConvo, getCitations, citeText, askSydney } = require('../../app/');
const { saveMessage, saveConvo, getConvoTitle } = require('../../models'); const { saveMessage, saveConvo, getConvoTitle } = require('../../models');
const { handleError, sendMessage, createOnProgress } = require('./handlers'); const { handleError, sendMessage, createOnProgress, handleText } = require('./handlers');
const citationRegex = /\[\^\d+?\^]/g; const citationRegex = /\[\^\d+?\^]/g;
router.post('/', async (req, res) => { router.post('/', async (req, res) => {
@ -114,6 +114,7 @@ const ask = async ({
response.text = response.text =
citeText(response) + citeText(response) +
(links?.length > 0 && hasCitations ? `\n<small>${links}</small>` : ''); (links?.length > 0 && hasCitations ? `\n<small>${links}</small>` : '');
response.text = await handleText(response.text);
// Save user message // Save user message
userMessage.conversationId = response.conversationId || conversationId; userMessage.conversationId = response.conversationId || conversationId;

24
api/server/routes/handlers.js
View file

@ -1,4 +1,4 @@
const { citeText } = require('../../app/'); const { citeText, detectCode } = require('../../app/');
const _ = require('lodash'); const _ = require('lodash');
const sanitizeHtml = require('sanitize-html'); const sanitizeHtml = require('sanitize-html');
@ -20,11 +20,14 @@ const createOnProgress = () => {
const progressCallback = async (partial, { res, text, bing = false, ...rest }) => { const progressCallback = async (partial, { res, text, bing = false, ...rest }) => {
tokens += partial === text ? '' : partial; tokens += partial === text ? '' : partial;
tokens = tokens.trim();
tokens = tokens.replaceAll('[DONE]', ''); tokens = tokens.replaceAll('[DONE]', '');
if (tokens.includes('```')) {
tokens = sanitizeHtml(tokens); if (tokens.match(/^\n/)) {
tokens = tokens.replace(/^\n/, '');
} }
// if (tokens.includes('```')) {
// tokens = sanitizeHtml(tokens);
// }
if (bing) { if (bing) {
tokens = citeText(tokens, true); tokens = citeText(tokens, true);
@ -42,4 +45,15 @@ const createOnProgress = () => {
return onProgress; return onProgress;
}; };
module.exports = { handleError, sendMessage, createOnProgress }; const handleText = async (input) => {
let text = input;
text = await detectCode(text);
// if (text.includes('```')) {
// text = sanitizeHtml(text);
// text = text.replaceAll(') =&gt;', ') =>');
// }
return text;
};
module.exports = { handleError, sendMessage, createOnProgress, handleText };