🚀 Feat: Streamline File Strategies & GPT-4-Vision Settings (#1535)

* chore: fix `endpoint` typescript issues and typo in console info message

* feat(api): files GET endpoint and save only file_id references to messages

* refactor(client): `useGetFiles` query hook, update file types, optimistic update of filesQuery on file upload

* refactor(buildTree): update to use params object and accept fileMap

* feat: map files to messages; refactor(ChatView): messages only available after files are fetched

* fix: fetch files only when authenticated

* feat(api): AppService
- rename app.locals.configs to app.locals.paths
- load custom config use fileStrategy from yaml config in app.locals

* refactor: separate Firebase and Local strategies, call based on config

* refactor: modularize file strategies and employ with use of DALL-E

* refactor(librechat.yaml): add fileStrategy field

* feat: add source to MongoFile schema, as well as BatchFile, and ExtendedFile types

* feat: employ file strategies for upload/delete files

* refactor(deleteFirebaseFile): add user id validation for firebase file deletion

* chore(deleteFirebaseFile): update jsdocs

* feat: employ strategies for vision requests

* fix(client): handle messages with deleted files

* fix(client): ensure `filesToDelete` always saves/sends `file.source`

* feat(openAI): configurable `resendImages` and `imageDetail`

* refactor(getTokenCountForMessage): recursive process only when array of Objects and only their values (not keys) aside from `image_url` types

* feat(OpenAIClient): calculateImageTokenCost

* chore: remove comment

* refactor(uploadAvatar): employ fileStrategy for avatars, from social logins or user upload

* docs: update docs on how to configure fileStrategy

* fix(ci): mock winston and winston related modules, update DALLE3.spec.js with changes made

* refactor(redis): change terminal message to reflect current development state

* fix(DALL-E-2): pass fileStrategy to dall-e

api/server/services/Files/Local/crud.js

@@ -0,0 +1,174 @@
+const fs = require('fs');
+const path = require('path');
+const axios = require('axios');
+const { logger } = require('~/config');
+const paths = require('~/config/paths');
+
+/**
+ * Saves a file to a specified output path with a new filename.
+ *
+ * @param {Express.Multer.File} file - The file object to be saved. Should contain properties like 'originalname' and 'path'.
+ * @param {string} outputPath - The path where the file should be saved.
+ * @param {string} outputFilename - The new filename for the saved file (without extension).
+ * @returns {Promise<string>} The full path of the saved file.
+ * @throws Will throw an error if the file saving process fails.
+ */
+async function saveFile(file, outputPath, outputFilename) {
+  try {
+    if (!fs.existsSync(outputPath)) {
+      fs.mkdirSync(outputPath, { recursive: true });
+    }
+
+    const fileExtension = path.extname(file.originalname);
+    const filenameWithExt = outputFilename + fileExtension;
+    const outputFilePath = path.join(outputPath, filenameWithExt);
+    fs.copyFileSync(file.path, outputFilePath);
+    fs.unlinkSync(file.path);
+
+    return outputFilePath;
+  } catch (error) {
+    logger.error('[saveFile] Error while saving the file:', error);
+    throw error;
+  }
+}
+
+/**
+ * Saves an uploaded image file to a specified directory based on the user's ID and a filename.
+ *
+ * @param {Express.Request} req - The Express request object, containing the user's information and app configuration.
+ * @param {Express.Multer.File} file - The uploaded file object.
+ * @param {string} filename - The new filename to assign to the saved image (without extension).
+ * @returns {Promise<void>}
+ * @throws Will throw an error if the image saving process fails.
+ */
+const saveLocalImage = async (req, file, filename) => {
+  const imagePath = req.app.locals.paths.imageOutput;
+  const outputPath = path.join(imagePath, req.user.id ?? '');
+  await saveFile(file, outputPath, filename);
+};
+
+/**
+ * Saves a file from a given URL to a local directory. The function fetches the file using the provided URL,
+ * determines the content type, and saves it to a specified local directory with the correct file extension.
+ * If the specified directory does not exist, it is created. The function returns the name of the saved file
+ * or null in case of an error.
+ *
+ * @param {Object} params - The parameters object.
+ * @param {string} params.userId - The user's unique identifier. This is used to create a user-specific path
+ *                                 in the local file system.
+ * @param {string} params.URL - The URL of the file to be downloaded and saved.
+ * @param {string} params.fileName - The desired file name for the saved file. This may be modified to include
+ *                                   the correct file extension based on the content type.
+ * @param {string} [params.basePath='images'] - Optional. The base directory where the file will be saved.
+ *                                              Defaults to 'images' if not specified.
+ *
+ * @returns {Promise<string|null>}
+ *          A promise that resolves to the file name if the file is successfully saved, or null if there is an error.
+ */
+async function saveFileFromURL({ userId, URL, fileName, basePath = 'images' }) {
+  try {
+    // Fetch the file from the URL
+    const response = await axios({
+      url: URL,
+      responseType: 'stream',
+    });
+
+    // Get the content type from the response headers
+    const contentType = response.headers['content-type'];
+    let extension = contentType.split('/').pop();
+
+    // Construct the outputPath based on the basePath and userId
+    const outputPath = path.join(paths.publicPath, basePath, userId.toString());
+
+    // Check if the output directory exists, if not, create it
+    if (!fs.existsSync(outputPath)) {
+      fs.mkdirSync(outputPath, { recursive: true });
+    }
+
+    // Replace or append the correct extension
+    const extRegExp = new RegExp(path.extname(fileName) + '$');
+    fileName = fileName.replace(extRegExp, `.${extension}`);
+    if (!path.extname(fileName)) {
+      fileName += `.${extension}`;
+    }
+
+    // Create a writable stream for the output path
+    const outputFilePath = path.join(outputPath, fileName);
+    const writer = fs.createWriteStream(outputFilePath);
+
+    // Pipe the response data to the output file
+    response.data.pipe(writer);
+
+    return new Promise((resolve, reject) => {
+      writer.on('finish', () => resolve(fileName));
+      writer.on('error', reject);
+    });
+  } catch (error) {
+    logger.error('[saveFileFromURL] Error while saving the file:', error);
+    return null;
+  }
+}
+
+/**
+ * Constructs a local file path for a given file name and base path. This function simply joins the base
+ * path and the file name to create a file path. It does not check for the existence of the file at the path.
+ *
+ * @param {Object} params - The parameters object.
+ * @param {string} params.fileName - The name of the file for which the path is to be constructed. This should
+ *                                   include the file extension.
+ * @param {string} [params.basePath='images'] - Optional. The base directory to be used for constructing the file path.
+ *                                              Defaults to 'images' if not specified.
+ *
+ * @returns {string}
+ *          The constructed local file path.
+ */
+async function getLocalFileURL({ fileName, basePath = 'images' }) {
+  return path.posix.join('/', basePath, fileName);
+}
+
+/**
+ * Validates if a given filepath is within a specified subdirectory under a base path. This function constructs
+ * the expected base path using the base, subfolder, and user id from the request, and then checks if the
+ * provided filepath starts with this constructed base path.
+ *
+ * @param {Express.Request} req - The request object from Express. It should contain a `user` property with an `id`.
+ * @param {string} base - The base directory path.
+ * @param {string} subfolder - The subdirectory under the base path.
+ * @param {string} filepath - The complete file path to be validated.
+ *
+ * @returns {boolean}
+ *          Returns true if the filepath is within the specified base and subfolder, false otherwise.
+ */
+const isValidPath = (req, base, subfolder, filepath) => {
+  const normalizedBase = path.resolve(base, subfolder, req.user.id);
+  const normalizedFilepath = path.resolve(filepath);
+  return normalizedFilepath.startsWith(normalizedBase);
+};
+
+/**
+ * Deletes a file from the filesystem. This function takes a file object, constructs the full path, and
+ * verifies the path's validity before deleting the file. If the path is invalid, an error is thrown.
+ *
+ * @param {Express.Request} req - The request object from Express. It should have an `app.locals.paths` object with
+ *                       a `publicPath` property.
+ * @param {MongoFile} file - The file object to be deleted. It should have a `filepath` property that is
+ *                           a string representing the path of the file relative to the publicPath.
+ *
+ * @returns {Promise<void>}
+ *          A promise that resolves when the file has been successfully deleted, or throws an error if the
+ *          file path is invalid or if there is an error in deletion.
+ */
+const deleteLocalFile = async (req, file) => {
+  const { publicPath } = req.app.locals.paths;
+  const parts = file.filepath.split(path.sep);
+  const subfolder = parts[1];
+  const filepath = path.join(publicPath, file.filepath);
+
+  if (!isValidPath(req, publicPath, subfolder, filepath)) {
+    throw new Error('Invalid file path');
+  }
+
+  await fs.promises.unlink(filepath);
+};
+
+module.exports = { saveFile, saveLocalImage, saveFileFromURL, getLocalFileURL, deleteLocalFile };

api/server/services/Files/Local/images.js

@@ -0,0 +1,140 @@
+const fs = require('fs');
+const path = require('path');
+const sharp = require('sharp');
+const { resizeImage } = require('../images/resize');
+const { updateFile } = require('~/models/File');
+
+/**
+ * Converts an image file to the WebP format. The function first resizes the image based on the specified
+ * resolution.
+ *
+ * If the original image is already in WebP format, it writes the resized image back. Otherwise,
+ * it converts the image to WebP format before saving.
+ *
+ * The original image is deleted after conversion.
+ *
+ * @param {Object} req - The request object from Express. It should have a `user` property with an `id`
+ *                       representing the user, and an `app.locals.paths` object with an `imageOutput` path.
+ * @param {Express.Multer.File} file - The file object, which is part of the request. The file object should
+ *                                     have a `path` property that points to the location of the uploaded file.
+ * @param {string} [resolution='high'] - Optional. The desired resolution for the image resizing. Default is 'high'.
+ *
+ * @returns {Promise<{ filepath: string, bytes: number, width: number, height: number}>}
+ *          A promise that resolves to an object containing:
+ *            - filepath: The path where the converted WebP image is saved.
+ *            - bytes: The size of the converted image in bytes.
+ *            - width: The width of the converted image.
+ *            - height: The height of the converted image.
+ */
+async function uploadLocalImage(req, file, resolution = 'high') {
+  const inputFilePath = file.path;
+  const { buffer: resizedBuffer, width, height } = await resizeImage(inputFilePath, resolution);
+  const extension = path.extname(inputFilePath);
+
+  const { imageOutput } = req.app.locals.paths;
+  const userPath = path.join(imageOutput, req.user.id);
+
+  if (!fs.existsSync(userPath)) {
+    fs.mkdirSync(userPath, { recursive: true });
+  }
+
+  const newPath = path.join(userPath, path.basename(inputFilePath));
+
+  if (extension.toLowerCase() === '.webp') {
+    const bytes = Buffer.byteLength(resizedBuffer);
+    await fs.promises.writeFile(newPath, resizedBuffer);
+    const filepath = path.posix.join('/', 'images', req.user.id, path.basename(newPath));
+    return { filepath, bytes, width, height };
+  }
+
+  const outputFilePath = newPath.replace(extension, '.webp');
+  const data = await sharp(resizedBuffer).toFormat('webp').toBuffer();
+  await fs.promises.writeFile(outputFilePath, data);
+  const bytes = Buffer.byteLength(data);
+  const filepath = path.posix.join('/', 'images', req.user.id, path.basename(outputFilePath));
+  await fs.promises.unlink(inputFilePath);
+  return { filepath, bytes, width, height };
+}
+
+/**
+ * Encodes an image file to base64.
+ * @param {string} imagePath - The path to the image file.
+ * @returns {Promise<string>} A promise that resolves with the base64 encoded image data.
+ */
+function encodeImage(imagePath) {
+  return new Promise((resolve, reject) => {
+    fs.readFile(imagePath, (err, data) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve(data.toString('base64'));
+      }
+    });
+  });
+}
+
+/**
+ * Local: Updates the file and encodes the image to base64,
+ * for image payload handling: tuple order of [filepath, base64].
+ * @param {Object} req - The request object.
+ * @param {MongoFile} file - The file object.
+ * @returns {Promise<[MongoFile, string]>} - A promise that resolves to an array of results from updateFile and encodeImage.
+ */
+async function prepareImagesLocal(req, file) {
+  const { publicPath, imageOutput } = req.app.locals.paths;
+  const userPath = path.join(imageOutput, req.user.id);
+
+  if (!fs.existsSync(userPath)) {
+    fs.mkdirSync(userPath, { recursive: true });
+  }
+  const filepath = path.join(publicPath, file.filepath);
+
+  const promises = [];
+  promises.push(updateFile({ file_id: file.file_id }));
+  promises.push(encodeImage(filepath));
+  return await Promise.all(promises);
+}
+
+/**
+ * Uploads a user's avatar to Firebase Storage and returns the URL.
+ * If the 'manual' flag is set to 'true', it also updates the user's avatar URL in the database.
+ *
+ * @param {object} params - The parameters object.
+ * @param {Buffer} params.buffer - The Buffer containing the avatar image in WebP format.
+ * @param {object} params.User - The User document (mongoose); TODO: remove direct use of Model, `User`
+ * @param {string} params.manual - A string flag indicating whether the update is manual ('true' or 'false').
+ * @returns {Promise<string>} - A promise that resolves with the URL of the uploaded avatar.
+ * @throws {Error} - Throws an error if Firebase is not initialized or if there is an error in uploading.
+ */
+async function processLocalAvatar({ buffer, User, manual }) {
+  const userDir = path.resolve(
+    __dirname,
+    '..',
+    '..',
+    '..',
+    '..',
+    '..',
+    'client',
+    'public',
+    'images',
+    User._id.toString(),
+  );
+  const fileName = `avatar-${new Date().getTime()}.png`;
+  const urlRoute = `/images/${User._id.toString()}/${fileName}`;
+  const avatarPath = path.join(userDir, fileName);
+
+  await fs.promises.mkdir(userDir, { recursive: true });
+  await fs.promises.writeFile(avatarPath, buffer);
+
+  const isManual = manual === 'true';
+  let url = `${urlRoute}?manual=${isManual}`;
+
+  if (isManual) {
+    User.avatar = url;
+    await User.save();
+  }
+
+  return url;
+}
+
+module.exports = { uploadLocalImage, encodeImage, prepareImagesLocal, processLocalAvatar };

api/server/services/Files/Local/index.js

@@ -0,0 +1,7 @@
+const images = require('./images');
+const crud = require('./crud');
+
+module.exports = {
+  ...crud,
+  ...images,
+};