🚀 feat: Integrate Azure Blob Storage for file handling and image uploads (#6153)

* 🚀 feat: Integrate Azure Blob Storage for file handling and image uploads

* 🐼 refactor: Correct module import case for Azure in strategies.js

* 🚀 feat: Add Azure support in SourceIcon component

* 🚀 feat: Enhance Azure Blob Service initialization with Managed Identity support

* 🐼 refactor: Remove unused Azure dependencies from package.json and package-lock.json

* 🐼 refactor: Remove unused Azure dependencies from package.json and package-lock.json

* 🐼 refactor: Remove unused Azure dependencies from package.json and package-lock.json

* 🚀 feat: Add Azure SDK dependencies for identity and storage blob

* 🔧 fix: Reorganize imports in strategies.js for better clarity

* 🔧 fix: Correct comment formatting in strategies.js for consistency

* 🔧 fix: Improve comment formatting in strategies.js for consistency

api/server/services/AppService.js

@@ -7,6 +7,7 @@ const {
 } = require('librechat-data-provider');
 const { checkVariables, checkHealth, checkConfig, checkAzureVariables } = require('./start/checks');
 const { azureAssistantsDefaults, assistantsConfigSetup } = require('./start/assistants');
+const { initializeAzureBlobService } = require('./Files/Azure/initialize');
 const { initializeFirebase } = require('./Files/Firebase/initialize');
 const { initializeS3 } = require('./Files/S3/initialize');
 const loadCustomConfig = require('./Config/loadCustomConfig');
@@ -45,6 +46,8 @@ const AppService = async (app) => {
 
   if (fileStrategy === FileSources.firebase) {
     initializeFirebase();
+  } else if (fileStrategy === FileSources.azure) {
+    initializeAzureBlobService();
   } else if (fileStrategy === FileSources.s3) {
     initializeS3();
   }

api/server/services/Files/Azure/crud.js

@@ -0,0 +1,196 @@
+const fs = require('fs');
+const path = require('path');
+const axios = require('axios');
+const fetch = require('node-fetch');
+const { logger } = require('~/config');
+const { getAzureContainerClient } = require('./initialize');
+
+const defaultBasePath = 'images';
+
+/**
+ * Uploads a buffer to Azure Blob Storage.
+ *
+ * Files will be stored at the path: {basePath}/{userId}/{fileName} within the container.
+ *
+ * @param {Object} params
+ * @param {string} params.userId - The user's id.
+ * @param {Buffer} params.buffer - The buffer to upload.
+ * @param {string} params.fileName - The name of the file.
+ * @param {string} [params.basePath='images'] - The base folder within the container.
+ * @param {string} [params.containerName] - The Azure Blob container name.
+ * @returns {Promise<string>} The URL of the uploaded blob.
+ */
+async function saveBufferToAzure({
+  userId,
+  buffer,
+  fileName,
+  basePath = defaultBasePath,
+  containerName,
+}) {
+  try {
+    const containerClient = getAzureContainerClient(containerName);
+    // Create the container if it doesn't exist. This is done per operation.
+    await containerClient.createIfNotExists({
+      access: process.env.AZURE_STORAGE_PUBLIC_ACCESS ? 'blob' : undefined,
+    });
+    const blobPath = `${basePath}/${userId}/${fileName}`;
+    const blockBlobClient = containerClient.getBlockBlobClient(blobPath);
+    await blockBlobClient.uploadData(buffer);
+    return blockBlobClient.url;
+  } catch (error) {
+    logger.error('[saveBufferToAzure] Error uploading buffer:', error);
+    throw error;
+  }
+}
+
+/**
+ * Saves a file from a URL to Azure Blob Storage.
+ *
+ * @param {Object} params
+ * @param {string} params.userId - The user's id.
+ * @param {string} params.URL - The URL of the file.
+ * @param {string} params.fileName - The name of the file.
+ * @param {string} [params.basePath='images'] - The base folder within the container.
+ * @param {string} [params.containerName] - The Azure Blob container name.
+ * @returns {Promise<string>} The URL of the uploaded blob.
+ */
+async function saveURLToAzure({
+  userId,
+  URL,
+  fileName,
+  basePath = defaultBasePath,
+  containerName,
+}) {
+  try {
+    const response = await fetch(URL);
+    const buffer = await response.buffer();
+    return await saveBufferToAzure({ userId, buffer, fileName, basePath, containerName });
+  } catch (error) {
+    logger.error('[saveURLToAzure] Error uploading file from URL:', error);
+    throw error;
+  }
+}
+
+/**
+ * Retrieves a blob URL from Azure Blob Storage.
+ *
+ * @param {Object} params
+ * @param {string} params.fileName - The file name.
+ * @param {string} [params.basePath='images'] - The base folder used during upload.
+ * @param {string} [params.userId] - If files are stored in a user-specific directory.
+ * @param {string} [params.containerName] - The Azure Blob container name.
+ * @returns {Promise<string>} The blob's URL.
+ */
+async function getAzureURL({ fileName, basePath = defaultBasePath, userId, containerName }) {
+  try {
+    const containerClient = getAzureContainerClient(containerName);
+    const blobPath = userId ? `${basePath}/${userId}/${fileName}` : `${basePath}/${fileName}`;
+    const blockBlobClient = containerClient.getBlockBlobClient(blobPath);
+    return blockBlobClient.url;
+  } catch (error) {
+    logger.error('[getAzureURL] Error retrieving blob URL:', error);
+    throw error;
+  }
+}
+
+/**
+ * Deletes a blob from Azure Blob Storage.
+ *
+ * @param {Object} params
+ * @param {string} params.fileName - The name of the file.
+ * @param {string} [params.basePath='images'] - The base folder where the file is stored.
+ * @param {string} params.userId - The user's id.
+ * @param {string} [params.containerName] - The Azure Blob container name.
+ */
+async function deleteFileFromAzure({
+  fileName,
+  basePath = defaultBasePath,
+  userId,
+  containerName,
+}) {
+  try {
+    const containerClient = getAzureContainerClient(containerName);
+    const blobPath = `${basePath}/${userId}/${fileName}`;
+    const blockBlobClient = containerClient.getBlockBlobClient(blobPath);
+    await blockBlobClient.delete();
+    logger.debug('[deleteFileFromAzure] Blob deleted successfully from Azure Blob Storage');
+  } catch (error) {
+    logger.error('[deleteFileFromAzure] Error deleting blob:', error.message);
+    if (error.statusCode === 404) {
+      return;
+    }
+    throw error;
+  }
+}
+
+/**
+ * Uploads a file from the local file system to Azure Blob Storage.
+ *
+ * This function reads the file from disk and then uploads it to Azure Blob Storage
+ * at the path: {basePath}/{userId}/{fileName}.
+ *
+ * @param {Object} params
+ * @param {object} params.req - The Express request object.
+ * @param {Express.Multer.File} params.file - The file object.
+ * @param {string} params.file_id - The file id.
+ * @param {string} [params.basePath='images'] - The base folder within the container.
+ * @param {string} [params.containerName] - The Azure Blob container name.
+ * @returns {Promise<{ filepath: string, bytes: number }>} An object containing the blob URL and its byte size.
+ */
+async function uploadFileToAzure({
+  req,
+  file,
+  file_id,
+  basePath = defaultBasePath,
+  containerName,
+}) {
+  try {
+    const inputFilePath = file.path;
+    const inputBuffer = await fs.promises.readFile(inputFilePath);
+    const bytes = Buffer.byteLength(inputBuffer);
+    const userId = req.user.id;
+    const fileName = `${file_id}__${path.basename(inputFilePath)}`;
+    const fileURL = await saveBufferToAzure({
+      userId,
+      buffer: inputBuffer,
+      fileName,
+      basePath,
+      containerName,
+    });
+    await fs.promises.unlink(inputFilePath);
+    return { filepath: fileURL, bytes };
+  } catch (error) {
+    logger.error('[uploadFileToAzure] Error uploading file:', error);
+    throw error;
+  }
+}
+
+/**
+ * Retrieves a readable stream for a blob from Azure Blob Storage.
+ *
+ * @param {object} _req - The Express request object.
+ * @param {string} fileURL - The URL of the blob.
+ * @returns {Promise<ReadableStream>} A readable stream of the blob.
+ */
+async function getAzureFileStream(_req, fileURL) {
+  try {
+    const response = await axios({
+      method: 'get',
+      url: fileURL,
+      responseType: 'stream',
+    });
+    return response.data;
+  } catch (error) {
+    logger.error('[getAzureFileStream] Error getting blob stream:', error);
+    throw error;
+  }
+}
+
+module.exports = {
+  saveBufferToAzure,
+  saveURLToAzure,
+  getAzureURL,
+  deleteFileFromAzure,
+  uploadFileToAzure,
+  getAzureFileStream,
+};

api/server/services/Files/Azure/images.js

@@ -0,0 +1,124 @@
+const fs = require('fs');
+const path = require('path');
+const sharp = require('sharp');
+const { resizeImageBuffer } = require('../images/resize');
+const { updateUser } = require('~/models/userMethods');
+const { updateFile } = require('~/models/File');
+const { logger } = require('~/config');
+const { saveBufferToAzure } = require('./crud');
+
+/**
+ * Uploads an image file to Azure Blob Storage.
+ * It resizes and converts the image similar to your Firebase implementation.
+ *
+ * @param {Object} params
+ * @param {object} params.req - The Express request object.
+ * @param {Express.Multer.File} params.file - The file object.
+ * @param {string} params.file_id - The file id.
+ * @param {EModelEndpoint} params.endpoint - The endpoint parameters.
+ * @param {string} [params.resolution='high'] - The image resolution.
+ * @param {string} [params.basePath='images'] - The base folder within the container.
+ * @param {string} [params.containerName] - The Azure Blob container name.
+ * @returns {Promise<{ filepath: string, bytes: number, width: number, height: number }>}
+ */
+async function uploadImageToAzure({
+  req,
+  file,
+  file_id,
+  endpoint,
+  resolution = 'high',
+  basePath = 'images',
+  containerName,
+}) {
+  try {
+    const inputFilePath = file.path;
+    const inputBuffer = await fs.promises.readFile(inputFilePath);
+    const {
+      buffer: resizedBuffer,
+      width,
+      height,
+    } = await resizeImageBuffer(inputBuffer, resolution, endpoint);
+    const extension = path.extname(inputFilePath);
+    const userId = req.user.id;
+    let webPBuffer;
+    let fileName = `${file_id}__${path.basename(inputFilePath)}`;
+    const targetExtension = `.${req.app.locals.imageOutputType}`;
+
+    if (extension.toLowerCase() === targetExtension) {
+      webPBuffer = resizedBuffer;
+    } else {
+      webPBuffer = await sharp(resizedBuffer).toFormat(req.app.locals.imageOutputType).toBuffer();
+      const extRegExp = new RegExp(path.extname(fileName) + '$');
+      fileName = fileName.replace(extRegExp, targetExtension);
+      if (!path.extname(fileName)) {
+        fileName += targetExtension;
+      }
+    }
+    const downloadURL = await saveBufferToAzure({
+      userId,
+      buffer: webPBuffer,
+      fileName,
+      basePath,
+      containerName,
+    });
+    await fs.promises.unlink(inputFilePath);
+    const bytes = Buffer.byteLength(webPBuffer);
+    return { filepath: downloadURL, bytes, width, height };
+  } catch (error) {
+    logger.error('[uploadImageToAzure] Error uploading image:', error);
+    throw error;
+  }
+}
+
+/**
+ * Prepares the image URL and updates the file record.
+ *
+ * @param {object} req - The Express request object.
+ * @param {MongoFile} file - The file object.
+ * @returns {Promise<[MongoFile, string]>}
+ */
+async function prepareAzureImageURL(req, file) {
+  const { filepath } = file;
+  const promises = [];
+  promises.push(updateFile({ file_id: file.file_id }));
+  promises.push(filepath);
+  return await Promise.all(promises);
+}
+
+/**
+ * Uploads and processes a user's avatar to Azure Blob Storage.
+ *
+ * @param {Object} params
+ * @param {Buffer} params.buffer - The avatar image buffer.
+ * @param {string} params.userId - The user's id.
+ * @param {string} params.manual - Flag to indicate manual update.
+ * @param {string} [params.basePath='images'] - The base folder within the container.
+ * @param {string} [params.containerName] - The Azure Blob container name.
+ * @returns {Promise<string>} The URL of the avatar.
+ */
+async function processAzureAvatar({ buffer, userId, manual, basePath = 'images', containerName }) {
+  try {
+    const downloadURL = await saveBufferToAzure({
+      userId,
+      buffer,
+      fileName: 'avatar.png',
+      basePath,
+      containerName,
+    });
+    const isManual = manual === 'true';
+    const url = `${downloadURL}?manual=${isManual}`;
+    if (isManual) {
+      await updateUser(userId, { avatar: url });
+    }
+    return url;
+  } catch (error) {
+    logger.error('[processAzureAvatar] Error uploading profile picture to Azure:', error);
+    throw error;
+  }
+}
+
+module.exports = {
+  uploadImageToAzure,
+  prepareAzureImageURL,
+  processAzureAvatar,
+};

api/server/services/Files/Azure/index.js

@@ -0,0 +1,9 @@
+const crud = require('./crud');
+const images = require('./images');
+const initialize = require('./initialize');
+
+module.exports = {
+  ...crud,
+  ...images,
+  ...initialize,
+};

api/server/services/Files/Azure/initialize.js

@@ -0,0 +1,55 @@
+const { BlobServiceClient } = require('@azure/storage-blob');
+const { logger } = require('~/config');
+
+let blobServiceClient = null;
+let azureWarningLogged = false;
+
+/**
+ * Initializes the Azure Blob Service client.
+ * This function establishes a connection by checking if a connection string is provided.
+ * If available, the connection string is used; otherwise, Managed Identity (via DefaultAzureCredential) is utilized.
+ * Note: Container creation (and its public access settings) is handled later in the CRUD functions.
+ * @returns {BlobServiceClient|null} The initialized client, or null if the required configuration is missing.
+ */
+const initializeAzureBlobService = () => {
+  if (blobServiceClient) {
+    return blobServiceClient;
+  }
+  const connectionString = process.env.AZURE_STORAGE_CONNECTION_STRING;
+  if (connectionString) {
+    blobServiceClient = BlobServiceClient.fromConnectionString(connectionString);
+    logger.info('Azure Blob Service initialized using connection string');
+  } else {
+    const { DefaultAzureCredential } = require('@azure/identity');
+    const accountName = process.env.AZURE_STORAGE_ACCOUNT_NAME;
+    if (!accountName) {
+      if (!azureWarningLogged) {
+        logger.error(
+          '[initializeAzureBlobService] Azure Blob Service not initialized. Connection string missing and AZURE_STORAGE_ACCOUNT_NAME not provided.',
+        );
+        azureWarningLogged = true;
+      }
+      return null;
+    }
+    const url = `https://${accountName}.blob.core.windows.net`;
+    const credential = new DefaultAzureCredential();
+    blobServiceClient = new BlobServiceClient(url, credential);
+    logger.info('Azure Blob Service initialized using Managed Identity');
+  }
+  return blobServiceClient;
+};
+
+/**
+ * Retrieves the Azure ContainerClient for the given container name.
+ * @param {string} [containerName=process.env.AZURE_CONTAINER_NAME || 'files'] - The container name.
+ * @returns {ContainerClient|null} The Azure ContainerClient.
+ */
+const getAzureContainerClient = (containerName = process.env.AZURE_CONTAINER_NAME || 'files') => {
+  const serviceClient = initializeAzureBlobService();
+  return serviceClient ? serviceClient.getContainerClient(containerName) : null;
+};
+
+module.exports = {
+  initializeAzureBlobService,
+  getAzureContainerClient,
+};

api/server/services/Files/strategies.js

@@ -32,6 +32,17 @@ const {
   processS3Avatar,
   uploadFileToS3,
 } = require('./S3');
+const {
+  saveBufferToAzure,
+  saveURLToAzure,
+  getAzureURL,
+  deleteFileFromAzure,
+  uploadFileToAzure,
+  getAzureFileStream,
+  uploadImageToAzure,
+  prepareAzureImageURL,
+  processAzureAvatar,
+} = require('./Azure');
 const { uploadOpenAIFile, deleteOpenAIFile, getOpenAIFileStream } = require('./OpenAI');
 const { getCodeOutputDownloadStream, uploadCodeEnvFile } = require('./Code');
 const { uploadVectors, deleteVectors } = require('./VectorDB');
@@ -85,6 +96,22 @@ const s3Strategy = () => ({
   getDownloadStream: getS3FileStream,
 });
 
+/**
+ * Azure Blob Storage Strategy Functions
+ *
+ * */
+const azureStrategy = () => ({
+  handleFileUpload: uploadFileToAzure,
+  saveURL: saveURLToAzure,
+  getFileURL: getAzureURL,
+  deleteFile: deleteFileFromAzure,
+  saveBuffer: saveBufferToAzure,
+  prepareImagePayload: prepareAzureImageURL,
+  processAvatar: processAzureAvatar,
+  handleImageUpload: uploadImageToAzure,
+  getDownloadStream: getAzureFileStream,
+});
+
 /**
  * VectorDB Storage Strategy Functions
  *
@@ -184,7 +211,7 @@ const getStrategyFunctions = (fileSource) => {
   } else if (fileSource === FileSources.openai) {
     return openAIStrategy();
   } else if (fileSource === FileSources.azure) {
-    return openAIStrategy();
+    return azureStrategy();
   } else if (fileSource === FileSources.vectordb) {
     return vectorStrategy();
   } else if (fileSource === FileSources.s3) {