⬇️ feat: Assistant File Downloads (#2234)

* WIP: basic route for file downloads and file strategy for generating readablestream to pipe as res

* chore(DALLE3): add typing for OpenAI client

* chore: add `CONSOLE_JSON` notes to dotenv.md

* WIP: first pass OpenAI Assistants File Output handling

* feat: first pass assistants output file download from openai

* chore: yml vs. yaml variation to .gitignore for `librechat.yml`

* refactor(retrieveAndProcessFile): remove redundancies

* fix(syncMessages): explicit sort of apiMessages to fix message order on abort

* chore: add logs for warnings and errors, show toast on frontend

* chore: add logger where console was still being used

api/server/services/Files/Firebase/crud.js

@@ -2,9 +2,10 @@ const fs = require('fs');
 const path = require('path');
 const axios = require('axios');
 const fetch = require('node-fetch');
-const { ref, uploadBytes, getDownloadURL, deleteObject } = require('firebase/storage');
+const { ref, uploadBytes, getDownloadURL, getStream, deleteObject } = require('firebase/storage');
 const { getBufferMetadata } = require('~/server/utils');
 const { getFirebaseStorage } = require('./initialize');
+const { logger } = require('~/config');
 
 /**
  * Deletes a file from Firebase Storage.
@@ -15,7 +16,7 @@ const { getFirebaseStorage } = require('./initialize');
 async function deleteFile(basePath, fileName) {
   const storage = getFirebaseStorage();
   if (!storage) {
-    console.error('Firebase is not initialized. Cannot delete file from Firebase Storage.');
+    logger.error('Firebase is not initialized. Cannot delete file from Firebase Storage.');
     throw new Error('Firebase is not initialized');
   }
 
@@ -23,9 +24,9 @@ async function deleteFile(basePath, fileName) {
 
   try {
     await deleteObject(storageRef);
-    console.log('File deleted successfully from Firebase Storage');
+    logger.debug('File deleted successfully from Firebase Storage');
   } catch (error) {
-    console.error('Error deleting file from Firebase Storage:', error.message);
+    logger.error('Error deleting file from Firebase Storage:', error.message);
     throw error;
   }
 }
@@ -51,7 +52,7 @@ async function deleteFile(basePath, fileName) {
 async function saveURLToFirebase({ userId, URL, fileName, basePath = 'images' }) {
   const storage = getFirebaseStorage();
   if (!storage) {
-    console.error('Firebase is not initialized. Cannot save file to Firebase Storage.');
+    logger.error('Firebase is not initialized. Cannot save file to Firebase Storage.');
     return null;
   }
 
@@ -63,7 +64,7 @@ async function saveURLToFirebase({ userId, URL, fileName, basePath = 'images' })
     await uploadBytes(storageRef, buffer);
     return await getBufferMetadata(buffer);
   } catch (error) {
-    console.error('Error uploading file to Firebase Storage:', error.message);
+    logger.error('Error uploading file to Firebase Storage:', error.message);
     return null;
   }
 }
@@ -87,7 +88,7 @@ async function saveURLToFirebase({ userId, URL, fileName, basePath = 'images' })
 async function getFirebaseURL({ fileName, basePath = 'images' }) {
   const storage = getFirebaseStorage();
   if (!storage) {
-    console.error('Firebase is not initialized. Cannot get image URL from Firebase Storage.');
+    logger.error('Firebase is not initialized. Cannot get image URL from Firebase Storage.');
     return null;
   }
 
@@ -96,7 +97,7 @@ async function getFirebaseURL({ fileName, basePath = 'images' }) {
   try {
     return await getDownloadURL(storageRef);
   } catch (error) {
-    console.error('Error fetching file URL from Firebase Storage:', error.message);
+    logger.error('Error fetching file URL from Firebase Storage:', error.message);
     return null;
   }
 }
@@ -212,6 +213,26 @@ async function uploadFileToFirebase({ req, file, file_id }) {
   return { filepath: downloadURL, bytes };
 }
 
+/**
+ * Retrieves a readable stream for a file from Firebase storage.
+ *
+ * @param {string} filepath - The filepath.
+ * @returns {ReadableStream} A readable stream of the file.
+ */
+function getFirebaseFileStream(filepath) {
+  try {
+    const storage = getFirebaseStorage();
+    if (!storage) {
+      throw new Error('Firebase is not initialized');
+    }
+    const fileRef = ref(storage, filepath);
+    return getStream(fileRef);
+  } catch (error) {
+    logger.error('Error getting Firebase file stream:', error);
+    throw error;
+  }
+}
+
 module.exports = {
   deleteFile,
   getFirebaseURL,
@@ -219,4 +240,5 @@ module.exports = {
   deleteFirebaseFile,
   uploadFileToFirebase,
   saveBufferToFirebase,
+  getFirebaseFileStream,
 };

api/server/services/Files/Local/crud.js

@@ -255,6 +255,21 @@ async function uploadLocalFile({ req, file, file_id }) {
   return { filepath, bytes };
 }
 
+/**
+ * Retrieves a readable stream for a file from local storage.
+ *
+ * @param {string} filepath - The filepath.
+ * @returns {ReadableStream} A readable stream of the file.
+ */
+function getLocalFileStream(filepath) {
+  try {
+    return fs.createReadStream(filepath);
+  } catch (error) {
+    logger.error('Error getting local file stream:', error);
+    throw error;
+  }
+}
+
 module.exports = {
   saveLocalFile,
   saveLocalImage,
@@ -263,4 +278,5 @@ module.exports = {
   getLocalFileURL,
   deleteLocalFile,
   uploadLocalFile,
+  getLocalFileStream,
 };

api/server/services/Files/OpenAI/crud.js

@@ -60,4 +60,20 @@ async function deleteOpenAIFile(req, file, openai) {
   }
 }
 
-module.exports = { uploadOpenAIFile, deleteOpenAIFile };
+/**
+ * Retrieves a readable stream for a file from local storage.
+ *
+ * @param {string} file_id - The file_id.
+ * @param {OpenAI} openai - The initialized OpenAI client.
+ * @returns {Promise<ReadableStream>} A readable stream of the file.
+ */
+async function getOpenAIFileStream(file_id, openai) {
+  try {
+    return await openai.files.content(file_id);
+  } catch (error) {
+    logger.error('Error getting OpenAI file download stream:', error);
+    throw error;
+  }
+}
+
+module.exports = { uploadOpenAIFile, deleteOpenAIFile, getOpenAIFileStream };

api/server/services/Files/images/convert.js

@@ -3,6 +3,7 @@ const path = require('path');
 const sharp = require('sharp');
 const { resizeImageBuffer } = require('./resize');
 const { getStrategyFunctions } = require('../strategies');
+const { logger } = require('~/config');
 
 /**
  * Converts an image file or buffer to WebP format with specified resolution.
@@ -61,7 +62,7 @@ async function convertToWebP(req, file, resolution = 'high', basename = '') {
     const bytes = Buffer.byteLength(outputBuffer);
     return { filepath: savedFilePath, bytes, width, height };
   } catch (err) {
-    console.error(err);
+    logger.error(err);
     throw err;
   }
 }

api/server/services/Files/process.js

@@ -1,6 +1,6 @@
 const path = require('path');
+const mime = require('mime');
 const { v4 } = require('uuid');
-const mime = require('mime/lite');
 const {
   isUUID,
   megabyte,
@@ -13,13 +13,11 @@ const {
 const { convertToWebP, resizeAndConvert } = require('~/server/services/Files/images');
 const { initializeClient } = require('~/server/services/Endpoints/assistants');
 const { createFile, updateFileUsage, deleteFiles } = require('~/models/File');
-const { isEnabled, determineFileType } = require('~/server/utils');
 const { LB_QueueAsyncCall } = require('~/server/utils/queue');
 const { getStrategyFunctions } = require('./strategies');
+const { determineFileType } = require('~/server/utils');
 const { logger } = require('~/config');
 
-const { GPTS_DOWNLOAD_IMAGES = 'true' } = process.env;
-
 const processFiles = async (files) => {
   const promises = [];
   for (let file of files) {
@@ -293,9 +291,10 @@ const processFileUpload = async ({ req, res, file, metadata }) => {
       file_id: id ?? file_id,
       temp_file_id,
       bytes,
-      filepath: isAssistantUpload ? `${openai.baseURL}/files/${id}` : filepath,
       filename: filename ?? file.originalname,
+      filepath: isAssistantUpload ? `${openai.baseURL}/files/${id}` : filepath,
       context: isAssistantUpload ? FileContext.assistants : FileContext.message_attachment,
+      model: isAssistantUpload ? req.body.model : undefined,
       type: file.mimetype,
       embedded,
       source,
@@ -305,6 +304,77 @@ const processFileUpload = async ({ req, res, file, metadata }) => {
   res.status(200).json({ message: 'File uploaded and processed successfully', ...result });
 };
 
+/**
+ * @param {object} params - The params object.
+ * @param {OpenAI} params.openai - The OpenAI client instance.
+ * @param {string} params.file_id - The ID of the file to retrieve.
+ * @param {string} params.userId - The user ID.
+ * @param {string} params.filename - The name of the file.
+ * @param {boolean} [params.saveFile=false] - Whether to save the file metadata to the database.
+ * @param {boolean} [params.updateUsage=false] - Whether to update file usage in database.
+ */
+const processOpenAIFile = async ({
+  openai,
+  file_id,
+  userId,
+  filename,
+  saveFile = false,
+  updateUsage = false,
+}) => {
+  const _file = await openai.files.retrieve(file_id);
+  const filepath = `${openai.baseURL}/files/${userId}/${file_id}/${filename}`;
+  const file = {
+    ..._file,
+    file_id,
+    filepath,
+    usage: 1,
+    filename,
+    user: userId,
+    source: FileSources.openai,
+    model: openai.req.body.model,
+    type: mime.getType(filename),
+    context: FileContext.assistants_output,
+  };
+
+  if (saveFile) {
+    await createFile(file, true);
+  } else if (updateUsage) {
+    try {
+      await updateFileUsage({ file_id });
+    } catch (error) {
+      logger.error('Error updating file usage', error);
+    }
+  }
+
+  return file;
+};
+
+/**
+ * Process OpenAI image files, convert to webp, save and return file metadata.
+ * @param {object} params - The params object.
+ * @param {Express.Request} params.req - The Express request object.
+ * @param {Buffer} params.buffer - The image buffer.
+ * @param {string} params.file_id - The file ID.
+ * @param {string} params.filename - The filename.
+ * @param {string} params.fileExt - The file extension.
+ * @returns {Promise<MongoFile>} The file metadata.
+ */
+const processOpenAIImageOutput = async ({ req, buffer, file_id, filename, fileExt }) => {
+  const _file = await convertToWebP(req, buffer, 'high', `${file_id}${fileExt}`);
+  const file = {
+    ..._file,
+    file_id,
+    usage: 1,
+    filename,
+    user: req.user.id,
+    type: 'image/webp',
+    source: req.app.locals.fileStrategy,
+    context: FileContext.assistants_output,
+  };
+  createFile(file, true);
+  return file;
+};
+
 /**
  * Retrieves and processes an OpenAI file based on its type.
  *
@@ -328,107 +398,69 @@ async function retrieveAndProcessFile({
     return null;
   }
 
-  if (client.attachedFileIds?.has(file_id)) {
-    return {
-      file_id,
-      // filepath: TODO: local source filepath?,
-      source: FileSources.openai,
-    };
+  if (client.attachedFileIds?.has(file_id) || client.processedFileIds?.has(file_id)) {
+    return processOpenAIFile({ ...processArgs, updateUsage: true });
   }
 
   let basename = _basename;
-  const downloadImages = isEnabled(GPTS_DOWNLOAD_IMAGES);
+  const fileExt = path.extname(basename);
+  const processArgs = { openai, file_id, filename: basename, userId: client.req.user.id };
 
   /**
-   * @param {string} file_id - The ID of the file to retrieve.
-   * @param {boolean} [save] - Whether to save the file metadata to the database.
+   * @returns {Promise<Buffer>} The file data buffer.
    */
-  const retrieveFile = async (file_id, save = false) => {
-    const _file = await openai.files.retrieve(file_id);
-    const filepath = `/api/files/download/${file_id}`;
-    const file = {
-      ..._file,
-      type: mime.getType(_file.filename),
-      filepath,
-      usage: 1,
-      file_id,
-      context: _file.purpose ?? FileContext.message_attachment,
-      source: FileSources.openai,
-    };
-
-    if (save) {
-      await createFile(file, true);
-    } else {
-      try {
-        await updateFileUsage({ file_id });
-      } catch (error) {
-        logger.error('Error updating file usage', error);
-      }
-    }
-
-    return file;
-  };
-
-  // If image downloads are not enabled or no basename provided, return only the file metadata
-  if (!downloadImages || (!basename && !downloadImages)) {
-    return await retrieveFile(file_id, true);
-  }
-
-  let data;
-  try {
+  const getDataBuffer = async () => {
     const response = await openai.files.content(file_id);
-    data = await response.arrayBuffer();
-  } catch (error) {
-    logger.error('Error downloading file from OpenAI:', error);
-    return await retrieveFile(file_id);
-  }
-
-  if (!data) {
-    return await retrieveFile(file_id);
-  }
-  const dataBuffer = Buffer.from(data);
-
-  /**
-   * @param {Buffer} dataBuffer
-   * @param {string} fileExt
-   */
-  const processAsImage = async (dataBuffer, fileExt) => {
-    // Logic to process image files, convert to webp, etc.
-    const _file = await convertToWebP(client.req, dataBuffer, 'high', `${file_id}${fileExt}`);
-    const file = {
-      ..._file,
-      type: 'image/webp',
-      usage: 1,
-      file_id,
-      source: FileSources.openai,
-    };
-    createFile(file, true);
-    return file;
+    const arrayBuffer = await response.arrayBuffer();
+    return Buffer.from(arrayBuffer);
   };
 
-  /** @param {Buffer} dataBuffer */
-  const processOtherFileTypes = async (dataBuffer) => {
-    // Logic to handle other file types
-    logger.debug('[retrieveAndProcessFile] Non-image file type detected');
-    return { filepath: `/api/files/download/${file_id}`, bytes: dataBuffer.length };
-  };
+  // If no basename provided, return only the file metadata
+  if (!basename) {
+    return await processOpenAIFile({ ...processArgs, saveFile: true });
+  }
+
+  let dataBuffer;
+  if (unknownType || !fileExt || imageExtRegex.test(basename)) {
+    try {
+      dataBuffer = await getDataBuffer();
+    } catch (error) {
+      logger.error('Error downloading file from OpenAI:', error);
+      dataBuffer = null;
+    }
+  }
+
+  if (!dataBuffer) {
+    return await processOpenAIFile({ ...processArgs, saveFile: true });
+  }
 
   // If the filetype is unknown, inspect the file
-  if (unknownType || !path.extname(basename)) {
+  if (dataBuffer && (unknownType || !fileExt)) {
     const detectedExt = await determineFileType(dataBuffer);
-    if (detectedExt && imageExtRegex.test('.' + detectedExt)) {
-      return await processAsImage(dataBuffer, detectedExt);
-    } else {
-      return await processOtherFileTypes(dataBuffer);
-    }
-  }
+    const isImageOutput = detectedExt && imageExtRegex.test('.' + detectedExt);
 
-  // Existing logic for processing known image types
-  if (downloadImages && basename && path.extname(basename) && imageExtRegex.test(basename)) {
-    return await processAsImage(dataBuffer, path.extname(basename));
+    if (!isImageOutput) {
+      return await processOpenAIFile({ ...processArgs, saveFile: true });
+    }
+
+    return await processOpenAIImageOutput({
+      file_id,
+      req: client.req,
+      buffer: dataBuffer,
+      filename: basename,
+      fileExt: detectedExt,
+    });
+  } else if (dataBuffer && imageExtRegex.test(basename)) {
+    return await processOpenAIImageOutput({
+      file_id,
+      req: client.req,
+      buffer: dataBuffer,
+      filename: basename,
+      fileExt,
+    });
   } else {
-    logger.debug('[retrieveAndProcessFile] Not an image or invalid extension: ', basename);
-    return await processOtherFileTypes(dataBuffer);
+    logger.debug(`[retrieveAndProcessFile] Non-image file type detected: ${basename}`);
+    return await processOpenAIFile({ ...processArgs, saveFile: true });
   }
 }
 

api/server/services/Files/strategies.js

@@ -7,6 +7,7 @@ const {
   saveBufferToFirebase,
   uploadImageToFirebase,
   processFirebaseAvatar,
+  getFirebaseFileStream,
 } = require('./Firebase');
 const {
   getLocalFileURL,
@@ -16,8 +17,9 @@ const {
   uploadLocalImage,
   prepareImagesLocal,
   processLocalAvatar,
+  getLocalFileStream,
 } = require('./Local');
-const { uploadOpenAIFile, deleteOpenAIFile } = require('./OpenAI');
+const { uploadOpenAIFile, deleteOpenAIFile, getOpenAIFileStream } = require('./OpenAI');
 const { uploadVectors, deleteVectors } = require('./VectorDB');
 
 /**
@@ -35,6 +37,7 @@ const firebaseStrategy = () => ({
   prepareImagePayload: prepareImageURL,
   processAvatar: processFirebaseAvatar,
   handleImageUpload: uploadImageToFirebase,
+  getDownloadStream: getFirebaseFileStream,
 });
 
 /**
@@ -51,6 +54,7 @@ const localStrategy = () => ({
   processAvatar: processLocalAvatar,
   handleImageUpload: uploadLocalImage,
   prepareImagePayload: prepareImagesLocal,
+  getDownloadStream: getLocalFileStream,
 });
 
 /**
@@ -70,6 +74,8 @@ const vectorStrategy = () => ({
   handleImageUpload: null,
   /** @type {typeof prepareImagesLocal | null} */
   prepareImagePayload: null,
+  /** @type {typeof getLocalFileStream | null} */
+  getDownloadStream: null,
   handleFileUpload: uploadVectors,
   deleteFile: deleteVectors,
 });
@@ -94,6 +100,7 @@ const openAIStrategy = () => ({
   prepareImagePayload: null,
   deleteFile: deleteOpenAIFile,
   handleFileUpload: uploadOpenAIFile,
+  getDownloadStream: getOpenAIFileStream,
 });
 
 // Strategy Selector