Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2026-01-30 14:25:19 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: reorganize api files, add mongoMeili

Browse source
This commit is contained in:
Daniel Avila 2023-03-16 19:38:16 -04:00
parent 854f1c3572
commit 9995a159aa
15 changed files with 200 additions and 45 deletions
Download patch file Download diff file Expand all files Collapse all files

29
api/app/citeText.js
View file

@ -1,29 +0,0 @@
const citationRegex = /\[\^\d+?\^]/g;
const citeText = (res, noLinks = false) => {
let result = res.text || res;
const citations = Array.from(new Set(result.match(citationRegex)));
if (citations?.length === 0) return result;
if (noLinks) {
citations.forEach((citation) => {
const digit = citation.match(/\d+?/g)[0];
result = result.replaceAll(citation, `<sup>[${digit}](#) </sup>`);
});
return result;
}
let sources = res.details.sourceAttributions;
if (sources?.length === 0) return result;
sources = sources.map((source) => source.seeMoreUrl);
citations.forEach((citation) => {
const digit = citation.match(/\d+?/g)[0];
result = result.replaceAll(citation, `<sup>[${digit}](${sources[digit - 1]}) </sup>`);
});
return result;
};
module.exports = citeText;

0
api/app/bingai.js → api/app/clients/bingai.js
View file

0
api/app/chatgpt-browser.js → api/app/clients/chatgpt-browser.js
View file

0
api/app/chatgpt-client.js → api/app/clients/chatgpt-client.js
View file

0
api/app/chatgpt-custom.js → api/app/clients/chatgpt-custom.js
View file

0
api/app/sydney.js → api/app/clients/sydney.js
View file

52
api/app/detectCode.js
View file

@ -1,52 +0,0 @@
const { ModelOperations } = require('@vscode/vscode-languagedetection');
const languages = require('../utils/languages.js');
const codeRegex = /(```[\s\S]*?```)/g;
// const languageMatch = /```(\w+)/;
const replaceRegex = /```\w+\n/g;
const detectCode = async (input) => {
try {
let text = input;
if (!text.match(codeRegex)) {
return text;
}
const langMatches = text.match(replaceRegex);
if (langMatches?.length > 0) {
langMatches.forEach(match => {
let lang = match.split('```')[1].trim();
if (languages.has(lang)) {
return;
}
console.log('[detectCode.js] replacing', match, 'with', '```shell');
text = text.replace(match, '```shell\n');
});
return text;
}
const modelOperations = new ModelOperations();
const regexSplit = (await import('./regexSplit.mjs')).default;
const parts = regexSplit(text, codeRegex);
const output = parts.map(async (part) => {
if (part.match(codeRegex)) {
const code = part.slice(3, -3);
let lang = (await modelOperations.runModel(code))[0].languageId;
return part.replace(/^```/, `\`\`\`${languages.has(lang) ? lang : 'shell'}`);
} else {
return part;
}
});
return (await Promise.all(output)).join('');
} catch (e) {
console.log('Error in detectCode function\n', e);
return input;
}
};
module.exports = detectCode;

13
api/app/getCitations.js
View file

@ -1,13 +0,0 @@
// const regex = / \[\d+\..*?\]\(.*?\)/g;
const regex = / \[.*?]\(.*?\)/g;
const getCitations = (res) => {
const textBlocks = res.details.adaptiveCards[0].body;
if (!textBlocks) return '';
let links = textBlocks[textBlocks.length - 1]?.text.match(regex);
if (links?.length === 0 || !links) return '';
links = links.map((link) => link.trim());
return links.join('\n');
};
module.exports = getCitations;

16
api/app/index.js
View file

@ -1,12 +1,12 @@
const { askClient } = require('./chatgpt-client');
const { browserClient } = require('./chatgpt-browser');
const customClient = require('./chatgpt-custom');
const { askBing } = require('./bingai');
const { askSydney } = require('./sydney');
const { askClient } = require('./clients/chatgpt-client');
const { browserClient } = require('./clients/chatgpt-browser');
const { askBing } = require('./clients/bingai');
const { askSydney } = require('./clients/sydney');
const customClient = require('./clients/chatgpt-custom');
const titleConvo = require('./titleConvo');
const getCitations = require('./getCitations');
const citeText = require('./citeText');
const detectCode = require('./detectCode');
const getCitations = require('../lib/parse/getCitations');
const citeText = require('../lib/parse/citeText');
const detectCode = require('../lib/parse/detectCode');
module.exports = {
askClient,

46
api/app/regexSplit.mjs
View file

@ -1,46 +0,0 @@
const primaryRegex = /```([^`\n]*?)\n([\s\S]*?)\n```/g;
const secondaryRegex = /```([^`\n]*?)\n?([\s\S]*?)\n?```/g;
const unenclosedCodeTest = (text) => {
let workingText = text;
// if (workingText.startsWith('<') || (!workingText.startsWith('`') && workingText.match(/```/g)?.length === 1)) {
// workingText = `\`\`\`${workingText}`
// }
return workingText.trim();
};
export default function regexSplit(string) {
let matches = [...string.matchAll(primaryRegex)];
if (!matches[0]) {
matches = [...string.matchAll(secondaryRegex)];
}
const output = [matches[0].input.slice(0, matches[0].index)];
// console.log(matches);
for (let i = 0; i < matches.length; i++) {
const [fullMatch, language, code] = matches[i];
// const formattedCode = code.replace(/`+/g, '\\`');
output.push(`\`\`\`${language}\n${code}\n\`\`\``);
if (i < matches.length - 1) {
let nextText = string.slice(matches[i].index + fullMatch.length, matches[i + 1].index);
nextText = unenclosedCodeTest(nextText);
output.push(nextText);
} else {
const lastMatch = matches[matches.length - 1][0];
// console.log(lastMatch);
// console.log(matches[0].input.split(lastMatch));
let rest = matches[0].input.split(lastMatch)[1]
if (rest) {
rest = unenclosedCodeTest(rest);
output.push(rest);
}
}
}
return output;
}

24
api/app/search.js
View file

@ -1,24 +0,0 @@
const mongoose = require('mongoose');
const mongomeili = require('mongomeili');
const { messageSchema, Message } = require('../models/Message');
// Add the '{ meiliIndex: true }' property to index these attributes with MeiliSearch
// const MovieSchema = new mongoose.Schema({
// title: { type: String, required: true, meiliIndex: true },
// director: { type: String, required: true, meiliIndex: true },
// year: { type: String, required: true, meiliIndex: true }
// });
// Specify your MeiliSearch credentials
// messageSchema.plugin(mongomeili, {
// host: 'http://localhost:7700',
// apiKey: 'MASTER_KEY',
// indexName: 'messages' // Will get created automatically if it doesn't exist already
// });
(async () => {
await Message.syncWithMeili();
const result = await Message.meiliSearch({ query: 'quantum' });
console.log(result);
})();