Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-12-18 01:10:14 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

📂 feat: RAG Improvements (#2169)

Browse source 
* feat: new vector file processing strategy

* chore: remove unused client files

* chore: remove more unused client files

* chore: remove more unused client files and move used to new dir

* chore(DataIcon): add className

* WIP: Model Endpoint Settings Update, draft additional context settings

* feat: improve parsing for augmented prompt, add full context option

* chore: remove volume mounting from rag.yml as no longer necessary
This commit is contained in:
Danny Avila 2024-03-22 19:07:08 -04:00 • committed by GitHub
parent f427ad792a
commit 45a95acec2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
40 changed files with 715 additions and 2046 deletions
Download patch file Download diff file Expand all files Collapse all files

43
api/server/services/Files/process.js
View file

@ -1,6 +1,5 @@
const path = require('path');
const { v4 } = require('uuid');
const axios = require('axios');
const mime = require('mime/lite');
const {
isUUID,
@ -265,50 +264,22 @@ const uploadImageBuffer = async ({ req, context }) => {
*/
const processFileUpload = async ({ req, res, file, metadata }) => {
const isAssistantUpload = metadata.endpoint === EModelEndpoint.assistants;
const source = isAssistantUpload ? FileSources.openai : req.app.locals.fileStrategy;
const source = isAssistantUpload ? FileSources.openai : FileSources.vectordb;
const { handleFileUpload } = getStrategyFunctions(source);
const { file_id, temp_file_id } = metadata;
let embedded = false;
if (process.env.RAG_API_URL) {
try {
const jwtToken = req.headers.authorization.split(' ')[1];
const filepath = `./uploads/temp/${file.path.split('uploads/temp/')[1]}`;
const response = await axios.post(
`${process.env.RAG_API_URL}/embed`,
{
filename: file.originalname,
file_content_type: file.mimetype,
filepath,
file_id,
},
{
headers: {
Authorization: `Bearer ${jwtToken}`,
'Content-Type': 'application/json',
},
},
);
if (response.status === 200) {
embedded = true;
}
} catch (error) {
logger.error('Error embedding file', error);
throw new Error(error);
}
} else if (!isAssistantUpload) {
logger.error('RAG_API_URL not set, cannot support process file upload');
throw new Error('RAG_API_URL not set, cannot support process file upload');
}
/** @type {OpenAI | undefined} */
let openai;
if (source === FileSources.openai) {
({ openai } = await initializeClient({ req }));
}
const { id, bytes, filename, filepath } = await handleFileUpload({ req, file, file_id, openai });
const { id, bytes, filename, filepath, embedded } = await handleFileUpload({
req,
file,
file_id,
openai,
});
if (isAssistantUpload && !metadata.message_file) {
await openai.beta.assistants.files.create(metadata.assistant_id, {