⬇️ feat: Assistant File Downloads (#2234)

* WIP: basic route for file downloads and file strategy for generating readablestream to pipe as res

* chore(DALLE3): add typing for OpenAI client

* chore: add `CONSOLE_JSON` notes to dotenv.md

* WIP: first pass OpenAI Assistants File Output handling

* feat: first pass assistants output file download from openai

* chore: yml vs. yaml variation to .gitignore for `librechat.yml`

* refactor(retrieveAndProcessFile): remove redundancies

* fix(syncMessages): explicit sort of apiMessages to fix message order on abort

* chore: add logs for warnings and errors, show toast on frontend

* chore: add logger where console was still being used

api/app/clients/prompts/createContextHandlers.js

@@ -1,5 +1,6 @@
 const axios = require('axios');
 const { isEnabled } = require('~/server/utils');
+const { logger } = require('~/config');
 
 const footer = `Use the context as your learned knowledge to better answer the user.
 
@@ -55,7 +56,7 @@ function createContextHandlers(req, userMessageContent) {
         processedFiles.push(file);
         processedIds.add(file.file_id);
       } catch (error) {
-        console.error(`Error processing file ${file.filename}:`, error);
+        logger.error(`Error processing file ${file.filename}:`, error);
       }
     }
   };
@@ -144,8 +145,8 @@ function createContextHandlers(req, userMessageContent) {
 
       return prompt;
     } catch (error) {
-      console.error('Error creating context:', error);
-      throw error; // Re-throw the error to propagate it to the caller
+      logger.error('Error creating context:', error);
+      throw error;
     }
   };
 

api/app/clients/tools/structured/DALLE3.js

@@ -43,6 +43,7 @@ class DALLE3 extends Tool {
       config.httpAgent = new HttpsProxyAgent(process.env.PROXY);
     }
 
+    /** @type {OpenAI} */
     this.openai = new OpenAI(config);
     this.name = 'dalle';
     this.description = `Use DALLE to create images from text descriptions.

api/app/clients/tools/util/loadToolSuite.js

@@ -1,5 +1,6 @@
 const { getUserPluginAuthValue } = require('~/server/services/PluginService');
 const { availableTools } = require('../');
+const { logger } = require('~/config');
 
 /**
  * Loads a suite of tools with authentication values for a given user, supporting alternate authentication fields.
@@ -30,7 +31,7 @@ const loadToolSuite = async ({ pluginKey, tools, user, options = {} }) => {
           return value;
         }
       } catch (err) {
-        console.error(`Error fetching plugin auth value for ${field}: ${err.message}`);
+        logger.error(`Error fetching plugin auth value for ${field}: ${err.message}`);
       }
     }
     return null;
@@ -41,7 +42,7 @@ const loadToolSuite = async ({ pluginKey, tools, user, options = {} }) => {
     if (authValue !== null) {
       authValues[auth.authField] = authValue;
     } else {
-      console.warn(`No auth value found for ${auth.authField}`);
+      logger.warn(`[loadToolSuite] No auth value found for ${auth.authField}`);
     }
   }
 

api/models/Transaction.js

@@ -2,6 +2,7 @@ const mongoose = require('mongoose');
 const { isEnabled } = require('../server/utils/handleText');
 const transactionSchema = require('./schema/transaction');
 const { getMultiplier } = require('./tx');
+const { logger } = require('~/config');
 const Balance = require('./Balance');
 const cancelRate = 1.15;
 
@@ -64,7 +65,7 @@ async function getTransactions(filter) {
   try {
     return await Transaction.find(filter).lean();
   } catch (error) {
-    console.error('Error querying transactions:', error);
+    logger.error('Error querying transactions:', error);
     throw error;
   }
 }

api/models/schema/fileSchema.js

@@ -15,7 +15,9 @@ const mongoose = require('mongoose');
  * @property {'file'} object - Type of object, always 'file'
  * @property {string} type - Type of file
  * @property {number} usage - Number of uses of the file
+ * @property {string} [context] - Context of the file origin
  * @property {boolean} [embedded] - Whether or not the file is embedded in vector db
+ * @property {string} [model] - The model to identify the group region of the file (for Azure OpenAI hosting)
  * @property {string} [source] - The source of the file
  * @property {number} [width] - Optional width of the file
  * @property {number} [height] - Optional height of the file
@@ -82,6 +84,9 @@ const fileSchema = mongoose.Schema(
       type: String,
       default: FileSources.local,
     },
+    model: {
+      type: String,
+    },
     width: Number,
     height: Number,
     expiresAt: {

api/server/middleware/moderateText.js

@@ -1,5 +1,6 @@
 const axios = require('axios');
 const denyRequest = require('./denyRequest');
+const { logger } = require('~/config');
 
 async function moderateText(req, res, next) {
   if (process.env.OPENAI_MODERATION === 'true') {
@@ -28,7 +29,7 @@ async function moderateText(req, res, next) {
         return await denyRequest(req, res, errorMessage);
       }
     } catch (error) {
-      console.error('Error in moderateText:', error);
+      logger.error('Error in moderateText:', error);
       const errorMessage = 'error in moderation check';
       return await denyRequest(req, res, errorMessage);
     }

api/server/routes/assistants/chat.js

@@ -597,7 +597,7 @@ router.post('/', validateModel, buildEndpointOption, setHeaders, async (req, res
 
     /** @type {ResponseMessage} */
     const responseMessage = {
-      ...response.finalMessage,
+      ...(response.responseMessage ?? response.finalMessage),
       parentMessageId: userMessageId,
       conversationId,
       user: req.user.id,

api/server/routes/files/files.js

@@ -1,12 +1,13 @@
-const axios = require('axios');
 const fs = require('fs').promises;
 const express = require('express');
-const { isUUID } = require('librechat-data-provider');
+const { isUUID, FileSources } = require('librechat-data-provider');
 const {
   filterFile,
   processFileUpload,
   processDeleteRequest,
 } = require('~/server/services/Files/process');
+const { initializeClient } = require('~/server/services/Endpoints/assistants');
+const { getStrategyFunctions } = require('~/server/services/Files/strategies');
 const { getFiles } = require('~/models/File');
 const { logger } = require('~/config');
 
@@ -65,28 +66,63 @@ router.delete('/', async (req, res) => {
   }
 });
 
-router.get('/download/:fileId', async (req, res) => {
+router.get('/download/:userId/:filepath', async (req, res) => {
   try {
-    const { fileId } = req.params;
+    const { userId, filepath } = req.params;
 
-    const options = {
-      headers: {
-        // TODO: Client initialization for OpenAI API Authentication
-        Authorization: `Bearer ${process.env.OPENAI_API_KEY}`,
-      },
-      responseType: 'stream',
+    if (userId !== req.user.id) {
+      logger.warn(`${errorPrefix} forbidden: ${file_id}`);
+      return res.status(403).send('Forbidden');
+    }
+
+    const parts = filepath.split('/');
+    const file_id = parts[2];
+    const [file] = await getFiles({ file_id });
+    const errorPrefix = `File download requested by user ${userId}`;
+
+    if (!file) {
+      logger.warn(`${errorPrefix} not found: ${file_id}`);
+      return res.status(404).send('File not found');
+    }
+
+    if (!file.filepath.includes(userId)) {
+      logger.warn(`${errorPrefix} forbidden: ${file_id}`);
+      return res.status(403).send('Forbidden');
+    }
+
+    if (file.source === FileSources.openai && !file.model) {
+      logger.warn(`${errorPrefix} has no associated model: ${file_id}`);
+      return res.status(400).send('The model used when creating this file is not available');
+    }
+
+    const { getDownloadStream } = getStrategyFunctions(file.source);
+    if (!getDownloadStream) {
+      logger.warn(`${errorPrefix} has no stream method implemented: ${file.source}`);
+      return res.status(501).send('Not Implemented');
+    }
+
+    const setHeaders = () => {
+      res.setHeader('Content-Disposition', `attachment; filename="${file.filename}"`);
+      res.setHeader('Content-Type', 'application/octet-stream');
     };
 
-    const fileResponse = await axios.get(`https://api.openai.com/v1/files/${fileId}`, {
-      headers: options.headers,
-    });
-    const { filename } = fileResponse.data;
-
-    const response = await axios.get(`https://api.openai.com/v1/files/${fileId}/content`, options);
-    res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
-    response.data.pipe(res);
+    /** @type {{ body: import('stream').PassThrough } | undefined} */
+    let passThrough;
+    /** @type {ReadableStream | undefined} */
+    let fileStream;
+    if (file.source === FileSources.openai) {
+      req.body = { model: file.model };
+      const { openai } = await initializeClient({ req, res });
+      passThrough = await getDownloadStream(file_id, openai);
+      setHeaders();
+      passThrough.body.pipe(res);
+    } else {
+      fileStream = getDownloadStream(file_id);
+      setHeaders();
+      fileStream.pipe(res);
+    }
   } catch (error) {
-    console.error('Error downloading file:', error);
+    logger.error('Error downloading file:', error);
     res.status(500).send('Error downloading file');
   }
 });

api/server/services/Files/Firebase/crud.js

@@ -2,9 +2,10 @@ const fs = require('fs');
 const path = require('path');
 const axios = require('axios');
 const fetch = require('node-fetch');
-const { ref, uploadBytes, getDownloadURL, deleteObject } = require('firebase/storage');
+const { ref, uploadBytes, getDownloadURL, getStream, deleteObject } = require('firebase/storage');
 const { getBufferMetadata } = require('~/server/utils');
 const { getFirebaseStorage } = require('./initialize');
+const { logger } = require('~/config');
 
 /**
  * Deletes a file from Firebase Storage.
@@ -15,7 +16,7 @@ const { getFirebaseStorage } = require('./initialize');
 async function deleteFile(basePath, fileName) {
   const storage = getFirebaseStorage();
   if (!storage) {
-    console.error('Firebase is not initialized. Cannot delete file from Firebase Storage.');
+    logger.error('Firebase is not initialized. Cannot delete file from Firebase Storage.');
     throw new Error('Firebase is not initialized');
   }
 
@@ -23,9 +24,9 @@ async function deleteFile(basePath, fileName) {
 
   try {
     await deleteObject(storageRef);
-    console.log('File deleted successfully from Firebase Storage');
+    logger.debug('File deleted successfully from Firebase Storage');
   } catch (error) {
-    console.error('Error deleting file from Firebase Storage:', error.message);
+    logger.error('Error deleting file from Firebase Storage:', error.message);
     throw error;
   }
 }
@@ -51,7 +52,7 @@ async function deleteFile(basePath, fileName) {
 async function saveURLToFirebase({ userId, URL, fileName, basePath = 'images' }) {
   const storage = getFirebaseStorage();
   if (!storage) {
-    console.error('Firebase is not initialized. Cannot save file to Firebase Storage.');
+    logger.error('Firebase is not initialized. Cannot save file to Firebase Storage.');
     return null;
   }
 
@@ -63,7 +64,7 @@ async function saveURLToFirebase({ userId, URL, fileName, basePath = 'images' })
     await uploadBytes(storageRef, buffer);
     return await getBufferMetadata(buffer);
   } catch (error) {
-    console.error('Error uploading file to Firebase Storage:', error.message);
+    logger.error('Error uploading file to Firebase Storage:', error.message);
     return null;
   }
 }
@@ -87,7 +88,7 @@ async function saveURLToFirebase({ userId, URL, fileName, basePath = 'images' })
 async function getFirebaseURL({ fileName, basePath = 'images' }) {
   const storage = getFirebaseStorage();
   if (!storage) {
-    console.error('Firebase is not initialized. Cannot get image URL from Firebase Storage.');
+    logger.error('Firebase is not initialized. Cannot get image URL from Firebase Storage.');
     return null;
   }
 
@@ -96,7 +97,7 @@ async function getFirebaseURL({ fileName, basePath = 'images' }) {
   try {
     return await getDownloadURL(storageRef);
   } catch (error) {
-    console.error('Error fetching file URL from Firebase Storage:', error.message);
+    logger.error('Error fetching file URL from Firebase Storage:', error.message);
     return null;
   }
 }
@@ -212,6 +213,26 @@ async function uploadFileToFirebase({ req, file, file_id }) {
   return { filepath: downloadURL, bytes };
 }
 
+/**
+ * Retrieves a readable stream for a file from Firebase storage.
+ *
+ * @param {string} filepath - The filepath.
+ * @returns {ReadableStream} A readable stream of the file.
+ */
+function getFirebaseFileStream(filepath) {
+  try {
+    const storage = getFirebaseStorage();
+    if (!storage) {
+      throw new Error('Firebase is not initialized');
+    }
+    const fileRef = ref(storage, filepath);
+    return getStream(fileRef);
+  } catch (error) {
+    logger.error('Error getting Firebase file stream:', error);
+    throw error;
+  }
+}
+
 module.exports = {
   deleteFile,
   getFirebaseURL,
@@ -219,4 +240,5 @@ module.exports = {
   deleteFirebaseFile,
   uploadFileToFirebase,
   saveBufferToFirebase,
+  getFirebaseFileStream,
 };

api/server/services/Files/Local/crud.js

@@ -255,6 +255,21 @@ async function uploadLocalFile({ req, file, file_id }) {
   return { filepath, bytes };
 }
 
+/**
+ * Retrieves a readable stream for a file from local storage.
+ *
+ * @param {string} filepath - The filepath.
+ * @returns {ReadableStream} A readable stream of the file.
+ */
+function getLocalFileStream(filepath) {
+  try {
+    return fs.createReadStream(filepath);
+  } catch (error) {
+    logger.error('Error getting local file stream:', error);
+    throw error;
+  }
+}
+
 module.exports = {
   saveLocalFile,
   saveLocalImage,
@@ -263,4 +278,5 @@ module.exports = {
   getLocalFileURL,
   deleteLocalFile,
   uploadLocalFile,
+  getLocalFileStream,
 };

api/server/services/Files/OpenAI/crud.js

@@ -60,4 +60,20 @@ async function deleteOpenAIFile(req, file, openai) {
   }
 }
 
-module.exports = { uploadOpenAIFile, deleteOpenAIFile };
+/**
+ * Retrieves a readable stream for a file from local storage.
+ *
+ * @param {string} file_id - The file_id.
+ * @param {OpenAI} openai - The initialized OpenAI client.
+ * @returns {Promise<ReadableStream>} A readable stream of the file.
+ */
+async function getOpenAIFileStream(file_id, openai) {
+  try {
+    return await openai.files.content(file_id);
+  } catch (error) {
+    logger.error('Error getting OpenAI file download stream:', error);
+    throw error;
+  }
+}
+
+module.exports = { uploadOpenAIFile, deleteOpenAIFile, getOpenAIFileStream };

api/server/services/Files/images/convert.js

@@ -3,6 +3,7 @@ const path = require('path');
 const sharp = require('sharp');
 const { resizeImageBuffer } = require('./resize');
 const { getStrategyFunctions } = require('../strategies');
+const { logger } = require('~/config');
 
 /**
  * Converts an image file or buffer to WebP format with specified resolution.
@@ -61,7 +62,7 @@ async function convertToWebP(req, file, resolution = 'high', basename = '') {
     const bytes = Buffer.byteLength(outputBuffer);
     return { filepath: savedFilePath, bytes, width, height };
   } catch (err) {
-    console.error(err);
+    logger.error(err);
     throw err;
   }
 }

api/server/services/Files/process.js

@@ -1,6 +1,6 @@
 const path = require('path');
+const mime = require('mime');
 const { v4 } = require('uuid');
-const mime = require('mime/lite');
 const {
   isUUID,
   megabyte,
@@ -13,13 +13,11 @@ const {
 const { convertToWebP, resizeAndConvert } = require('~/server/services/Files/images');
 const { initializeClient } = require('~/server/services/Endpoints/assistants');
 const { createFile, updateFileUsage, deleteFiles } = require('~/models/File');
-const { isEnabled, determineFileType } = require('~/server/utils');
 const { LB_QueueAsyncCall } = require('~/server/utils/queue');
 const { getStrategyFunctions } = require('./strategies');
+const { determineFileType } = require('~/server/utils');
 const { logger } = require('~/config');
 
-const { GPTS_DOWNLOAD_IMAGES = 'true' } = process.env;
-
 const processFiles = async (files) => {
   const promises = [];
   for (let file of files) {
@@ -293,9 +291,10 @@ const processFileUpload = async ({ req, res, file, metadata }) => {
       file_id: id ?? file_id,
       temp_file_id,
       bytes,
-      filepath: isAssistantUpload ? `${openai.baseURL}/files/${id}` : filepath,
       filename: filename ?? file.originalname,
+      filepath: isAssistantUpload ? `${openai.baseURL}/files/${id}` : filepath,
       context: isAssistantUpload ? FileContext.assistants : FileContext.message_attachment,
+      model: isAssistantUpload ? req.body.model : undefined,
       type: file.mimetype,
       embedded,
       source,
@@ -305,6 +304,77 @@ const processFileUpload = async ({ req, res, file, metadata }) => {
   res.status(200).json({ message: 'File uploaded and processed successfully', ...result });
 };
 
+/**
+ * @param {object} params - The params object.
+ * @param {OpenAI} params.openai - The OpenAI client instance.
+ * @param {string} params.file_id - The ID of the file to retrieve.
+ * @param {string} params.userId - The user ID.
+ * @param {string} params.filename - The name of the file.
+ * @param {boolean} [params.saveFile=false] - Whether to save the file metadata to the database.
+ * @param {boolean} [params.updateUsage=false] - Whether to update file usage in database.
+ */
+const processOpenAIFile = async ({
+  openai,
+  file_id,
+  userId,
+  filename,
+  saveFile = false,
+  updateUsage = false,
+}) => {
+  const _file = await openai.files.retrieve(file_id);
+  const filepath = `${openai.baseURL}/files/${userId}/${file_id}/${filename}`;
+  const file = {
+    ..._file,
+    file_id,
+    filepath,
+    usage: 1,
+    filename,
+    user: userId,
+    source: FileSources.openai,
+    model: openai.req.body.model,
+    type: mime.getType(filename),
+    context: FileContext.assistants_output,
+  };
+
+  if (saveFile) {
+    await createFile(file, true);
+  } else if (updateUsage) {
+    try {
+      await updateFileUsage({ file_id });
+    } catch (error) {
+      logger.error('Error updating file usage', error);
+    }
+  }
+
+  return file;
+};
+
+/**
+ * Process OpenAI image files, convert to webp, save and return file metadata.
+ * @param {object} params - The params object.
+ * @param {Express.Request} params.req - The Express request object.
+ * @param {Buffer} params.buffer - The image buffer.
+ * @param {string} params.file_id - The file ID.
+ * @param {string} params.filename - The filename.
+ * @param {string} params.fileExt - The file extension.
+ * @returns {Promise<MongoFile>} The file metadata.
+ */
+const processOpenAIImageOutput = async ({ req, buffer, file_id, filename, fileExt }) => {
+  const _file = await convertToWebP(req, buffer, 'high', `${file_id}${fileExt}`);
+  const file = {
+    ..._file,
+    file_id,
+    usage: 1,
+    filename,
+    user: req.user.id,
+    type: 'image/webp',
+    source: req.app.locals.fileStrategy,
+    context: FileContext.assistants_output,
+  };
+  createFile(file, true);
+  return file;
+};
+
 /**
  * Retrieves and processes an OpenAI file based on its type.
  *
@@ -328,107 +398,69 @@ async function retrieveAndProcessFile({
     return null;
   }
 
-  if (client.attachedFileIds?.has(file_id)) {
-    return {
-      file_id,
-      // filepath: TODO: local source filepath?,
-      source: FileSources.openai,
-    };
+  if (client.attachedFileIds?.has(file_id) || client.processedFileIds?.has(file_id)) {
+    return processOpenAIFile({ ...processArgs, updateUsage: true });
   }
 
   let basename = _basename;
-  const downloadImages = isEnabled(GPTS_DOWNLOAD_IMAGES);
+  const fileExt = path.extname(basename);
+  const processArgs = { openai, file_id, filename: basename, userId: client.req.user.id };
 
   /**
-   * @param {string} file_id - The ID of the file to retrieve.
-   * @param {boolean} [save] - Whether to save the file metadata to the database.
+   * @returns {Promise<Buffer>} The file data buffer.
    */
-  const retrieveFile = async (file_id, save = false) => {
-    const _file = await openai.files.retrieve(file_id);
-    const filepath = `/api/files/download/${file_id}`;
-    const file = {
-      ..._file,
-      type: mime.getType(_file.filename),
-      filepath,
-      usage: 1,
-      file_id,
-      context: _file.purpose ?? FileContext.message_attachment,
-      source: FileSources.openai,
-    };
-
-    if (save) {
-      await createFile(file, true);
-    } else {
-      try {
-        await updateFileUsage({ file_id });
-      } catch (error) {
-        logger.error('Error updating file usage', error);
-      }
-    }
-
-    return file;
-  };
-
-  // If image downloads are not enabled or no basename provided, return only the file metadata
-  if (!downloadImages || (!basename && !downloadImages)) {
-    return await retrieveFile(file_id, true);
-  }
-
-  let data;
-  try {
+  const getDataBuffer = async () => {
     const response = await openai.files.content(file_id);
-    data = await response.arrayBuffer();
-  } catch (error) {
-    logger.error('Error downloading file from OpenAI:', error);
-    return await retrieveFile(file_id);
-  }
-
-  if (!data) {
-    return await retrieveFile(file_id);
-  }
-  const dataBuffer = Buffer.from(data);
-
-  /**
-   * @param {Buffer} dataBuffer
-   * @param {string} fileExt
-   */
-  const processAsImage = async (dataBuffer, fileExt) => {
-    // Logic to process image files, convert to webp, etc.
-    const _file = await convertToWebP(client.req, dataBuffer, 'high', `${file_id}${fileExt}`);
-    const file = {
-      ..._file,
-      type: 'image/webp',
-      usage: 1,
-      file_id,
-      source: FileSources.openai,
-    };
-    createFile(file, true);
-    return file;
+    const arrayBuffer = await response.arrayBuffer();
+    return Buffer.from(arrayBuffer);
   };
 
-  /** @param {Buffer} dataBuffer */
-  const processOtherFileTypes = async (dataBuffer) => {
-    // Logic to handle other file types
-    logger.debug('[retrieveAndProcessFile] Non-image file type detected');
-    return { filepath: `/api/files/download/${file_id}`, bytes: dataBuffer.length };
-  };
+  // If no basename provided, return only the file metadata
+  if (!basename) {
+    return await processOpenAIFile({ ...processArgs, saveFile: true });
+  }
+
+  let dataBuffer;
+  if (unknownType || !fileExt || imageExtRegex.test(basename)) {
+    try {
+      dataBuffer = await getDataBuffer();
+    } catch (error) {
+      logger.error('Error downloading file from OpenAI:', error);
+      dataBuffer = null;
+    }
+  }
+
+  if (!dataBuffer) {
+    return await processOpenAIFile({ ...processArgs, saveFile: true });
+  }
 
   // If the filetype is unknown, inspect the file
-  if (unknownType || !path.extname(basename)) {
+  if (dataBuffer && (unknownType || !fileExt)) {
     const detectedExt = await determineFileType(dataBuffer);
-    if (detectedExt && imageExtRegex.test('.' + detectedExt)) {
-      return await processAsImage(dataBuffer, detectedExt);
-    } else {
-      return await processOtherFileTypes(dataBuffer);
-    }
-  }
+    const isImageOutput = detectedExt && imageExtRegex.test('.' + detectedExt);
 
-  // Existing logic for processing known image types
-  if (downloadImages && basename && path.extname(basename) && imageExtRegex.test(basename)) {
-    return await processAsImage(dataBuffer, path.extname(basename));
+    if (!isImageOutput) {
+      return await processOpenAIFile({ ...processArgs, saveFile: true });
+    }
+
+    return await processOpenAIImageOutput({
+      file_id,
+      req: client.req,
+      buffer: dataBuffer,
+      filename: basename,
+      fileExt: detectedExt,
+    });
+  } else if (dataBuffer && imageExtRegex.test(basename)) {
+    return await processOpenAIImageOutput({
+      file_id,
+      req: client.req,
+      buffer: dataBuffer,
+      filename: basename,
+      fileExt,
+    });
   } else {
-    logger.debug('[retrieveAndProcessFile] Not an image or invalid extension: ', basename);
-    return await processOtherFileTypes(dataBuffer);
+    logger.debug(`[retrieveAndProcessFile] Non-image file type detected: ${basename}`);
+    return await processOpenAIFile({ ...processArgs, saveFile: true });
   }
 }
 

api/server/services/Files/strategies.js

@@ -7,6 +7,7 @@ const {
   saveBufferToFirebase,
   uploadImageToFirebase,
   processFirebaseAvatar,
+  getFirebaseFileStream,
 } = require('./Firebase');
 const {
   getLocalFileURL,
@@ -16,8 +17,9 @@ const {
   uploadLocalImage,
   prepareImagesLocal,
   processLocalAvatar,
+  getLocalFileStream,
 } = require('./Local');
-const { uploadOpenAIFile, deleteOpenAIFile } = require('./OpenAI');
+const { uploadOpenAIFile, deleteOpenAIFile, getOpenAIFileStream } = require('./OpenAI');
 const { uploadVectors, deleteVectors } = require('./VectorDB');
 
 /**
@@ -35,6 +37,7 @@ const firebaseStrategy = () => ({
   prepareImagePayload: prepareImageURL,
   processAvatar: processFirebaseAvatar,
   handleImageUpload: uploadImageToFirebase,
+  getDownloadStream: getFirebaseFileStream,
 });
 
 /**
@@ -51,6 +54,7 @@ const localStrategy = () => ({
   processAvatar: processLocalAvatar,
   handleImageUpload: uploadLocalImage,
   prepareImagePayload: prepareImagesLocal,
+  getDownloadStream: getLocalFileStream,
 });
 
 /**
@@ -70,6 +74,8 @@ const vectorStrategy = () => ({
   handleImageUpload: null,
   /** @type {typeof prepareImagesLocal | null} */
   prepareImagePayload: null,
+  /** @type {typeof getLocalFileStream | null} */
+  getDownloadStream: null,
   handleFileUpload: uploadVectors,
   deleteFile: deleteVectors,
 });
@@ -94,6 +100,7 @@ const openAIStrategy = () => ({
   prepareImagePayload: null,
   deleteFile: deleteOpenAIFile,
   handleFileUpload: uploadOpenAIFile,
+  getDownloadStream: getOpenAIFileStream,
 });
 
 // Strategy Selector

api/server/services/Runs/StreamRunManager.js

@@ -102,18 +102,20 @@ class StreamRunManager {
    * @returns {Promise<void>}
    */
   async addContentData(data) {
-    const { type, index } = data;
-    this.finalMessage.content[index] = { type, [type]: data[type] };
+    const { type, index, edited } = data;
+    /** @type {ContentPart} */
+    const contentPart = data[type];
+    this.finalMessage.content[index] = { type, [type]: contentPart };
 
-    if (type === ContentTypes.TEXT) {
-      this.text += data[type].value;
+    if (type === ContentTypes.TEXT && !edited) {
+      this.text += contentPart.value;
       return;
     }
 
     const contentData = {
       index,
       type,
-      [type]: data[type],
+      [type]: contentPart,
       thread_id: this.thread_id,
       messageId: this.finalMessage.messageId,
       conversationId: this.finalMessage.conversationId,
@@ -593,7 +595,7 @@ class StreamRunManager {
    */
   async handleMessageEvent(event) {
     if (event.event === AssistantStreamEvents.ThreadMessageCompleted) {
-      this.messageCompleted(event);
+      await this.messageCompleted(event);
     }
   }
 
@@ -613,6 +615,7 @@ class StreamRunManager {
     this.addContentData({
       [ContentTypes.TEXT]: { value: result.text },
       type: ContentTypes.TEXT,
+      edited: result.edited,
       index,
     });
     this.messages.push(message);

api/server/services/Threads/manage.js

@@ -2,10 +2,9 @@ const path = require('path');
 const { v4 } = require('uuid');
 const {
   Constants,
-  FilePurpose,
   ContentTypes,
-  imageExtRegex,
   EModelEndpoint,
+  AnnotationTypes,
   defaultOrderQuery,
 } = require('librechat-data-provider');
 const { retrieveAndProcessFile } = require('~/server/services/Files/process');
@@ -434,13 +433,15 @@ async function checkMessageGaps({ openai, latestMessageId, thread_id, run_id, co
   }
 
   let addedCurrentMessage = false;
-  const apiMessages = response.data.map((msg) => {
-    if (msg.id === currentMessage.id) {
-      addedCurrentMessage = true;
-      return currentMessage;
-    }
-    return msg;
-  });
+  const apiMessages = response.data
+    .map((msg) => {
+      if (msg.id === currentMessage.id) {
+        addedCurrentMessage = true;
+        return currentMessage;
+      }
+      return msg;
+    })
+    .sort((a, b) => new Date(a.created_at) - new Date(b.created_at));
 
   if (!addedCurrentMessage) {
     apiMessages.push(currentMessage);
@@ -496,6 +497,44 @@ const recordUsage = async ({
   );
 };
 
+/**
+ * Safely replaces the annotated text within the specified range denoted by start_index and end_index,
+ * after verifying that the text within that range matches the given annotation text.
+ * Proceeds with the replacement even if a mismatch is found, but logs a warning.
+ *
+ * @param {string} originalText The original text content.
+ * @param {number} start_index The starting index where replacement should begin.
+ * @param {number} end_index The ending index where replacement should end.
+ * @param {string} expectedText The text expected to be found in the specified range.
+ * @param {string} replacementText The text to insert in place of the existing content.
+ * @returns {string} The text with the replacement applied, regardless of text match.
+ */
+function replaceAnnotation(originalText, start_index, end_index, expectedText, replacementText) {
+  if (start_index < 0 || end_index > originalText.length || start_index > end_index) {
+    logger.warn(`Invalid range specified for annotation replacement.
+    Attempting replacement with \`replace\` method instead...
+    length: ${originalText.length}
+    start_index: ${start_index}
+    end_index: ${end_index}`);
+    return originalText.replace(originalText, replacementText);
+  }
+
+  const actualTextInRange = originalText.substring(start_index, end_index);
+
+  if (actualTextInRange !== expectedText) {
+    logger.warn(`The text within the specified range does not match the expected annotation text.
+    Attempting replacement with \`replace\` method instead...
+    Expected: ${expectedText}
+    Actual: ${actualTextInRange}`);
+
+    return originalText.replace(originalText, replacementText);
+  }
+
+  const beforeText = originalText.substring(0, start_index);
+  const afterText = originalText.substring(end_index);
+  return beforeText + replacementText + afterText;
+}
+
 /**
  * Sorts, processes, and flattens messages to a single string.
  *
@@ -509,89 +548,90 @@ async function processMessages({ openai, client, messages = [] }) {
   const sorted = messages.sort((a, b) => a.created_at - b.created_at);
 
   let text = '';
+  let edited = false;
   for (const message of sorted) {
     message.files = [];
     for (const content of message.content) {
-      const processImageFile =
-        content.type === 'image_file' && !client.processedFileIds.has(content.image_file?.file_id);
-      if (processImageFile) {
-        const { file_id } = content.image_file;
+      const type = content.type;
+      const contentType = content[type];
+      const currentFileId = contentType?.file_id;
 
+      if (type === ContentTypes.IMAGE_FILE && !client.processedFileIds.has(currentFileId)) {
         const file = await retrieveAndProcessFile({
           openai,
           client,
-          file_id,
-          basename: `${file_id}.png`,
+          file_id: currentFileId,
+          basename: `${currentFileId}.png`,
         });
-        client.processedFileIds.add(file_id);
+
+        client.processedFileIds.add(currentFileId);
         message.files.push(file);
         continue;
       }
 
-      text += (content.text?.value ?? '') + ' ';
-      logger.debug('[processMessages] Processing message:', { value: text });
+      let currentText = contentType?.value ?? '';
+
+      /** @type {{ annotations: Annotation[] }} */
+      const { annotations } = contentType ?? {};
 
       // Process annotations if they exist
-      if (!content.text?.annotations?.length) {
+      if (!annotations?.length) {
+        text += currentText + ' ';
         continue;
       }
 
-      logger.debug('[processMessages] Processing annotations:', content.text.annotations);
-      for (const annotation of content.text.annotations) {
-        logger.debug('Current annotation:', annotation);
+      logger.debug('[processMessages] Processing annotations:', annotations);
+      for (const annotation of annotations) {
         let file;
-        const processFilePath =
-          annotation.file_path && !client.processedFileIds.has(annotation.file_path?.file_id);
+        const type = annotation.type;
+        const annotationType = annotation[type];
+        const file_id = annotationType?.file_id;
+        const alreadyProcessed = client.processedFileIds.has(file_id);
 
-        if (processFilePath) {
-          const basename = imageExtRegex.test(annotation.text)
-            ? path.basename(annotation.text)
-            : null;
+        if (alreadyProcessed) {
+          const { file_id } = annotationType || {};
+          file = await retrieveAndProcessFile({ openai, client, file_id, unknownType: true });
+        } else if (type === AnnotationTypes.FILE_PATH) {
+          const basename = path.basename(annotation.text);
           file = await retrieveAndProcessFile({
             openai,
             client,
-            file_id: annotation.file_path.file_id,
+            file_id,
             basename,
           });
-          client.processedFileIds.add(annotation.file_path.file_id);
-        }
-
-        const processFileCitation =
-          annotation.file_citation &&
-          !client.processedFileIds.has(annotation.file_citation?.file_id);
-
-        if (processFileCitation) {
+        } else if (type === AnnotationTypes.FILE_CITATION) {
           file = await retrieveAndProcessFile({
             openai,
             client,
-            file_id: annotation.file_citation.file_id,
+            file_id,
             unknownType: true,
           });
-          client.processedFileIds.add(annotation.file_citation.file_id);
         }
 
-        if (!file && (annotation.file_path || annotation.file_citation)) {
-          const { file_id } = annotation.file_citation || annotation.file_path || {};
-          file = await retrieveAndProcessFile({ openai, client, file_id, unknownType: true });
-          client.processedFileIds.add(file_id);
+        if (file.filepath) {
+          currentText = replaceAnnotation(
+            currentText,
+            annotation.start_index,
+            annotation.end_index,
+            annotation.text,
+            file.filepath,
+          );
+          edited = true;
         }
 
+        text += currentText + ' ';
+
         if (!file) {
           continue;
         }
 
-        if (file.purpose && file.purpose === FilePurpose.Assistants) {
-          text = text.replace(annotation.text, file.filename);
-        } else if (file.filepath) {
-          text = text.replace(annotation.text, file.filepath);
-        }
-
+        client.processedFileIds.add(file_id);
         message.files.push(file);
       }
     }
   }
 
-  return { messages: sorted, text };
+  return { messages: sorted, text, edited };
 }
 
 module.exports = {

api/typedefs.js

@@ -436,7 +436,13 @@
 
 /**
  * @exports ThreadMessage
- * @typedef {import('openai').OpenAI.Beta.Threads.ThreadMessage} ThreadMessage
+ * @typedef {import('openai').OpenAI.Beta.Threads.Message} ThreadMessage
+ * @memberof typedefs
+ */
+
+/**
+ * @exports Annotation
+ * @typedef {import('openai').OpenAI.Beta.Threads.Messages.Annotation} Annotation
  * @memberof typedefs
  */