📂 feat: RAG Improvements (#2169)

* feat: new vector file processing strategy

* chore: remove unused client files

* chore: remove more unused client files

* chore: remove more unused client files and move used to new dir

* chore(DataIcon): add className

* WIP: Model Endpoint Settings Update, draft additional context settings

* feat: improve parsing for augmented prompt, add full context option

* chore: remove volume mounting from rag.yml as no longer necessary

api/server/services/Files/VectorDB/crud.js

@@ -0,0 +1,96 @@
+const fs = require('fs');
+const axios = require('axios');
+const FormData = require('form-data');
+const { FileSources } = require('librechat-data-provider');
+const { logger } = require('~/config');
+
+/**
+ * Deletes a file from the vector database. This function takes a file object, constructs the full path, and
+ * verifies the path's validity before deleting the file. If the path is invalid, an error is thrown.
+ *
+ * @param {Express.Request} req - The request object from Express. It should have an `app.locals.paths` object with
+ *                       a `publicPath` property.
+ * @param {MongoFile} file - The file object to be deleted. It should have a `filepath` property that is
+ *                           a string representing the path of the file relative to the publicPath.
+ *
+ * @returns {Promise<void>}
+ *          A promise that resolves when the file has been successfully deleted, or throws an error if the
+ *          file path is invalid or if there is an error in deletion.
+ */
+const deleteVectors = async (req, file) => {
+  if (file.embedded && process.env.RAG_API_URL) {
+    const jwtToken = req.headers.authorization.split(' ')[1];
+    axios.delete(`${process.env.RAG_API_URL}/documents`, {
+      headers: {
+        Authorization: `Bearer ${jwtToken}`,
+        'Content-Type': 'application/json',
+        accept: 'application/json',
+      },
+      data: [file.file_id],
+    });
+  }
+};
+
+/**
+ * Uploads a file to the configured Vector database
+ *
+ * @param {Object} params - The params object.
+ * @param {Object} params.req - The request object from Express. It should have a `user` property with an `id`
+ *                       representing the user, and an `app.locals.paths` object with an `uploads` path.
+ * @param {Express.Multer.File} params.file - The file object, which is part of the request. The file object should
+ *                                     have a `path` property that points to the location of the uploaded file.
+ * @param {string} params.file_id - The file ID.
+ *
+ * @returns {Promise<{ filepath: string, bytes: number }>}
+ *          A promise that resolves to an object containing:
+ *            - filepath: The path where the file is saved.
+ *            - bytes: The size of the file in bytes.
+ */
+async function uploadVectors({ req, file, file_id }) {
+  if (!process.env.RAG_API_URL) {
+    throw new Error('RAG_API_URL not defined');
+  }
+
+  try {
+    const jwtToken = req.headers.authorization.split(' ')[1];
+    const formData = new FormData();
+    formData.append('file_id', file_id);
+    formData.append('file', fs.createReadStream(file.path));
+
+    const formHeaders = formData.getHeaders(); // Automatically sets the correct Content-Type
+
+    const response = await axios.post(`${process.env.RAG_API_URL}/embed`, formData, {
+      headers: {
+        Authorization: `Bearer ${jwtToken}`,
+        accept: 'application/json',
+        ...formHeaders,
+      },
+    });
+
+    const responseData = response.data;
+    logger.debug('Response from embedding file', responseData);
+
+    if (responseData.known_type === false) {
+      throw new Error(`File embedding failed. The filetype ${file.mimetype} is not supported`);
+    }
+
+    if (!responseData.status) {
+      throw new Error('File embedding failed.');
+    }
+
+    return {
+      bytes: file.size,
+      filename: file.originalname,
+      filepath: FileSources.vectordb,
+      embedded: Boolean(responseData.known_type),
+    };
+  } catch (error) {
+    logger.error('Error embedding file', error);
+    throw new Error(error.message || 'An error occurred during file upload.');
+  }
+}
+
+module.exports = {
+  deleteVectors,
+  uploadVectors,
+};