🚀 Feat: Streamline File Strategies & GPT-4-Vision Settings (#1535)

* chore: fix `endpoint` typescript issues and typo in console info message

* feat(api): files GET endpoint and save only file_id references to messages

* refactor(client): `useGetFiles` query hook, update file types, optimistic update of filesQuery on file upload

* refactor(buildTree): update to use params object and accept fileMap

* feat: map files to messages; refactor(ChatView): messages only available after files are fetched

* fix: fetch files only when authenticated

* feat(api): AppService
- rename app.locals.configs to app.locals.paths
- load custom config use fileStrategy from yaml config in app.locals

* refactor: separate Firebase and Local strategies, call based on config

* refactor: modularize file strategies and employ with use of DALL-E

* refactor(librechat.yaml): add fileStrategy field

* feat: add source to MongoFile schema, as well as BatchFile, and ExtendedFile types

* feat: employ file strategies for upload/delete files

* refactor(deleteFirebaseFile): add user id validation for firebase file deletion

* chore(deleteFirebaseFile): update jsdocs

* feat: employ strategies for vision requests

* fix(client): handle messages with deleted files

* fix(client): ensure `filesToDelete` always saves/sends `file.source`

* feat(openAI): configurable `resendImages` and `imageDetail`

* refactor(getTokenCountForMessage): recursive process only when array of Objects and only their values (not keys) aside from `image_url` types

* feat(OpenAIClient): calculateImageTokenCost

* chore: remove comment

* refactor(uploadAvatar): employ fileStrategy for avatars, from social logins or user upload

* docs: update docs on how to configure fileStrategy

* fix(ci): mock winston and winston related modules, update DALLE3.spec.js with changes made

* refactor(redis): change terminal message to reflect current development state

* fix(DALL-E-2): pass fileStrategy to dall-e

api/server/services/Files/Firebase/crud.js

@@ -0,0 +1,174 @@
+const fetch = require('node-fetch');
+const { ref, uploadBytes, getDownloadURL, deleteObject } = require('firebase/storage');
+const { getFirebaseStorage } = require('./initialize');
+
+/**
+ * Deletes a file from Firebase Storage.
+ * @param {string} directory - The directory name
+ * @param {string} fileName - The name of the file to delete.
+ * @returns {Promise<void>} A promise that resolves when the file is deleted.
+ */
+async function deleteFile(basePath, fileName) {
+  const storage = getFirebaseStorage();
+  if (!storage) {
+    console.error('Firebase is not initialized. Cannot delete file from Firebase Storage.');
+    throw new Error('Firebase is not initialized');
+  }
+
+  const storageRef = ref(storage, `${basePath}/${fileName}`);
+
+  try {
+    await deleteObject(storageRef);
+    console.log('File deleted successfully from Firebase Storage');
+  } catch (error) {
+    console.error('Error deleting file from Firebase Storage:', error.message);
+    throw error;
+  }
+}
+
+/**
+ * Saves an file from a given URL to Firebase Storage. The function first initializes the Firebase Storage
+ * reference, then uploads the file to a specified basePath in the Firebase Storage. It handles initialization
+ * errors and upload errors, logging them to the console. If the upload is successful, the file name is returned.
+ *
+ * @param {Object} params - The parameters object.
+ * @param {string} params.userId - The user's unique identifier. This is used to create a user-specific basePath
+ *                                 in Firebase Storage.
+ * @param {string} params.URL - The URL of the file to be uploaded. The file at this URL will be fetched
+ *                              and uploaded to Firebase Storage.
+ * @param {string} params.fileName - The name that will be used to save the file in Firebase Storage. This
+ *                                   should include the file extension.
+ * @param {string} [params.basePath='images'] - Optional. The base basePath in Firebase Storage where the file will
+ *                                          be stored. Defaults to 'images' if not specified.
+ *
+ * @returns {Promise<string|null>}
+ *          A promise that resolves to the file name if the file is successfully uploaded, or null if there
+ *          is an error in initialization or upload.
+ */
+async function saveURLToFirebase({ userId, URL, fileName, basePath = 'images' }) {
+  const storage = getFirebaseStorage();
+  if (!storage) {
+    console.error('Firebase is not initialized. Cannot save file to Firebase Storage.');
+    return null;
+  }
+
+  const storageRef = ref(storage, `${basePath}/${userId.toString()}/${fileName}`);
+
+  try {
+    await uploadBytes(storageRef, await fetch(URL).then((response) => response.buffer()));
+    return fileName;
+  } catch (error) {
+    console.error('Error uploading file to Firebase Storage:', error.message);
+    return null;
+  }
+}
+
+/**
+ * Retrieves the download URL for a specified file from Firebase Storage. This function initializes the
+ * Firebase Storage and generates a reference to the file based on the provided basePath and file name. If
+ * Firebase Storage is not initialized or if there is an error in fetching the URL, the error is logged
+ * to the console.
+ *
+ * @param {Object} params - The parameters object.
+ * @param {string} params.fileName - The name of the file for which the URL is to be retrieved. This should
+ *                                   include the file extension.
+ * @param {string} [params.basePath='images'] - Optional. The base basePath in Firebase Storage where the file is
+ *                                          stored. Defaults to 'images' if not specified.
+ *
+ * @returns {Promise<string|null>}
+ *          A promise that resolves to the download URL of the file if successful, or null if there is an
+ *          error in initialization or fetching the URL.
+ */
+async function getFirebaseURL({ fileName, basePath = 'images' }) {
+  const storage = getFirebaseStorage();
+  if (!storage) {
+    console.error('Firebase is not initialized. Cannot get image URL from Firebase Storage.');
+    return null;
+  }
+
+  const storageRef = ref(storage, `${basePath}/${fileName}`);
+
+  try {
+    return await getDownloadURL(storageRef);
+  } catch (error) {
+    console.error('Error fetching file URL from Firebase Storage:', error.message);
+    return null;
+  }
+}
+
+/**
+ * Uploads a buffer to Firebase Storage.
+ *
+ * @param {Object} params - The parameters object.
+ * @param {string} params.userId - The user's unique identifier. This is used to create a user-specific basePath
+ *                                 in Firebase Storage.
+ * @param {string} params.fileName - The name of the file to be saved in Firebase Storage.
+ * @param {string} params.buffer - The buffer to be uploaded.
+ * @param {string} [params.basePath='images'] - Optional. The base basePath in Firebase Storage where the file will
+ *                                          be stored. Defaults to 'images' if not specified.
+ *
+ * @returns {Promise<string>} - A promise that resolves to the download URL of the uploaded file.
+ */
+async function saveBufferToFirebase({ userId, buffer, fileName, basePath = 'images' }) {
+  const storage = getFirebaseStorage();
+  if (!storage) {
+    throw new Error('Firebase is not initialized');
+  }
+
+  const storageRef = ref(storage, `${basePath}/${userId}/${fileName}`);
+  await uploadBytes(storageRef, buffer);
+
+  // Assuming you have a function to get the download URL
+  return await getFirebaseURL({ fileName, basePath: `${basePath}/${userId}` });
+}
+
+/**
+ * Extracts and decodes the file path from a Firebase Storage URL.
+ *
+ * @param {string} urlString - The Firebase Storage URL.
+ * @returns {string} The decoded file path.
+ */
+function extractFirebaseFilePath(urlString) {
+  try {
+    const url = new URL(urlString);
+    const pathRegex = /\/o\/(.+?)(\?|$)/;
+    const match = url.pathname.match(pathRegex);
+
+    if (match && match[1]) {
+      return decodeURIComponent(match[1]);
+    }
+
+    return '';
+  } catch (error) {
+    // If URL parsing fails, return an empty string
+    return '';
+  }
+}
+
+/**
+ * Deletes a file from Firebase storage. This function determines the filepath from the
+ * Firebase storage URL via regex for deletion. Validated by the user's ID.
+ *
+ * @param {Express.Request} req - The request object from Express.
+ * It should contain a `user` object with an `id` property.
+ * @param {MongoFile} file - The file object to be deleted.
+ *
+ * @returns {Promise<void>}
+ *          A promise that resolves when the file has been successfully deleted from Firebase storage.
+ *          Throws an error if there is an issue with deletion.
+ */
+const deleteFirebaseFile = async (req, file) => {
+  const fileName = extractFirebaseFilePath(file.filepath);
+  if (!fileName.includes(req.user.id)) {
+    throw new Error('Invalid file path');
+  }
+  await deleteFile('', fileName);
+};
+
+module.exports = {
+  deleteFile,
+  getFirebaseURL,
+  saveURLToFirebase,
+  deleteFirebaseFile,
+  saveBufferToFirebase,
+};

api/server/services/Files/Firebase/images.js

@@ -1,45 +1,105 @@
-const fetch = require('node-fetch');
-const { ref, uploadBytes, getDownloadURL } = require('firebase/storage');
-const { getFirebaseStorage } = require('./initialize');
+const fs = require('fs');
+const path = require('path');
+const sharp = require('sharp');
+const { resizeImage } = require('../images/resize');
+const { saveBufferToFirebase } = require('./crud');
+const { updateFile } = require('~/models/File');
+const { logger } = require('~/config');
 
-async function saveImageToFirebaseStorage(userId, imageUrl, imageName) {
-  const storage = getFirebaseStorage();
-  if (!storage) {
-    console.error('Firebase is not initialized. Cannot save image to Firebase Storage.');
-    return null;
+/**
+ * Converts an image file to the WebP format. The function first resizes the image based on the specified
+ * resolution.
+ *
+ *
+ * @param {Object} req - The request object from Express. It should have a `user` property with an `id`
+ *                       representing the user, and an `app.locals.paths` object with an `imageOutput` path.
+ * @param {Express.Multer.File} file - The file object, which is part of the request. The file object should
+ *                                     have a `path` property that points to the location of the uploaded file.
+ * @param {string} [resolution='high'] - Optional. The desired resolution for the image resizing. Default is 'high'.
+ *
+ * @returns {Promise<{ filepath: string, bytes: number, width: number, height: number}>}
+ *          A promise that resolves to an object containing:
+ *            - filepath: The path where the converted WebP image is saved.
+ *            - bytes: The size of the converted image in bytes.
+ *            - width: The width of the converted image.
+ *            - height: The height of the converted image.
+ */
+async function uploadImageToFirebase(req, file, resolution = 'high') {
+  const inputFilePath = file.path;
+  const { buffer: resizedBuffer, width, height } = await resizeImage(inputFilePath, resolution);
+  const extension = path.extname(inputFilePath);
+  const userId = req.user.id;
+
+  let webPBuffer;
+  let fileName = path.basename(inputFilePath);
+  if (extension.toLowerCase() === '.webp') {
+    webPBuffer = resizedBuffer;
+  } else {
+    webPBuffer = await sharp(resizedBuffer).toFormat('webp').toBuffer();
+    // Replace or append the correct extension
+    const extRegExp = new RegExp(path.extname(fileName) + '$');
+    fileName = fileName.replace(extRegExp, '.webp');
+    if (!path.extname(fileName)) {
+      fileName += '.webp';
+    }
   }
 
-  const storageRef = ref(storage, `images/${userId.toString()}/${imageName}`);
+  const downloadURL = await saveBufferToFirebase({ userId, buffer: webPBuffer, fileName });
 
+  await fs.promises.unlink(inputFilePath);
+
+  const bytes = Buffer.byteLength(webPBuffer);
+  return { filepath: downloadURL, bytes, width, height };
+}
+
+/**
+ * Local: Updates the file and returns the URL in expected order/format
+ * for image payload handling: tuple order of [filepath, URL].
+ * @param {Object} req - The request object.
+ * @param {MongoFile} file - The file object.
+ * @returns {Promise<[MongoFile, string]>} - A promise that resolves to an array of results from updateFile and encodeImage.
+ */
+async function prepareImageURL(req, file) {
+  const { filepath } = file;
+  const promises = [];
+  promises.push(updateFile({ file_id: file.file_id }));
+  promises.push(filepath);
+  return await Promise.all(promises);
+}
+
+/**
+ * Uploads a user's avatar to Firebase Storage and returns the URL.
+ * If the 'manual' flag is set to 'true', it also updates the user's avatar URL in the database.
+ *
+ * @param {object} params - The parameters object.
+ * @param {Buffer} params.buffer - The Buffer containing the avatar image in WebP format.
+ * @param {object} params.User - The User document (mongoose); TODO: remove direct use of Model, `User`
+ * @param {string} params.manual - A string flag indicating whether the update is manual ('true' or 'false').
+ * @returns {Promise<string>} - A promise that resolves with the URL of the uploaded avatar.
+ * @throws {Error} - Throws an error if Firebase is not initialized or if there is an error in uploading.
+ */
+async function processFirebaseAvatar({ buffer, User, manual }) {
   try {
-    // Upload image to Firebase Storage using the image URL
-    await uploadBytes(storageRef, await fetch(imageUrl).then((response) => response.buffer()));
-    return imageName;
+    const downloadURL = await saveBufferToFirebase({
+      userId: User._id.toString(),
+      buffer,
+      fileName: 'avatar.png',
+    });
+
+    const isManual = manual === 'true';
+
+    const url = `${downloadURL}?manual=${isManual}`;
+
+    if (isManual) {
+      User.avatar = url;
+      await User.save();
+    }
+
+    return url;
   } catch (error) {
-    console.error('Error uploading image to Firebase Storage:', error.message);
-    return null;
+    logger.error('Error uploading profile picture:', error);
+    throw error;
   }
 }
 
-async function getFirebaseStorageImageUrl(imageName) {
-  const storage = getFirebaseStorage();
-  if (!storage) {
-    console.error('Firebase is not initialized. Cannot get image URL from Firebase Storage.');
-    return null;
-  }
-
-  const storageRef = ref(storage, `images/${imageName}`);
-
-  try {
-    // Get the download URL for the image from Firebase Storage
-    return `![generated image](${await getDownloadURL(storageRef)})`;
-  } catch (error) {
-    console.error('Error fetching image URL from Firebase Storage:', error.message);
-    return null;
-  }
-}
-
-module.exports = {
-  saveImageToFirebaseStorage,
-  getFirebaseStorageImageUrl,
-};
+module.exports = { uploadImageToFirebase, prepareImageURL, processFirebaseAvatar };

api/server/services/Files/Firebase/index.js

@@ -1,7 +1,9 @@
+const crud = require('./crud');
 const images = require('./images');
 const initialize = require('./initialize');
 
 module.exports = {
+  ...crud,
   ...images,
   ...initialize,
 };