Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-12-16 16:30:15 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

🔧 fix: Azure Blob Integration and File Source References (#6575)

Browse source 
* 🔧 fix: Update file source references to include 'azure_blob' for correct service initialization

* 🔧 fix: Add Azure Blob Storage Emulator entries to .gitignore

* fix: Update file source references to include 'azure_blob' for correct service initialization

* fix: Refactor Azure Blob Storage functions to use environment variables for access control and container name, fix deletion improper logging and improper params

* fix: Add basePath determination for agent file uploads based on MIME type

* fix: Implement file streaming to Azure Blob Storage to optimize memory usage during uploads (non-images)

* fix: Update SourceIcon to include 'azure_blob' class and adjust model setting in useSelectorEffects for assistants

* chore: import order

---------

Co-authored-by: Danny Avila <danny@librechat.ai>
This commit is contained in:
Ruben Talstra 2025-03-31 19:44:20 +02:00 • committed by GitHub
parent d60f2ed50b
commit bc039cea29
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 93 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

99
api/server/services/Files/Azure/crud.js
View file

@ -1,11 +1,13 @@
const fs = require('fs');
const path = require('path');
const mime = require('mime');
const axios = require('axios');
const fetch = require('node-fetch');
const { logger } = require('~/config');
const { getAzureContainerClient } = require('./initialize');
const defaultBasePath = 'images';
const { AZURE_STORAGE_PUBLIC_ACCESS = 'true', AZURE_CONTAINER_NAME = 'files' } = process.env;
/**
* Uploads a buffer to Azure Blob Storage.
@ -29,10 +31,9 @@ async function saveBufferToAzure({
}) {
try {
const containerClient = getAzureContainerClient(containerName);
const access = AZURE_STORAGE_PUBLIC_ACCESS?.toLowerCase() === 'true' ? 'blob' : undefined;
// Create the container if it doesn't exist. This is done per operation.
await containerClient.createIfNotExists({
access: process.env.AZURE_STORAGE_PUBLIC_ACCESS ? 'blob' : undefined,
});
await containerClient.createIfNotExists({ access });
const blobPath = `${basePath}/${userId}/${fileName}`;
const blockBlobClient = containerClient.getBlockBlobClient(blobPath);
await blockBlobClient.uploadData(buffer);
@ -97,25 +98,21 @@ async function getAzureURL({ fileName, basePath = defaultBasePath, userId, conta
* Deletes a blob from Azure Blob Storage.
*
* @param {Object} params
* @param {string} params.fileName - The name of the file.
* @param {string} [params.basePath='images'] - The base folder where the file is stored.
* @param {string} params.userId - The user's id.
* @param {string} [params.containerName] - The Azure Blob container name.
* @param {ServerRequest} params.req - The Express request object.
* @param {MongoFile} params.file - The file object.
*/
async function deleteFileFromAzure({
fileName,
basePath = defaultBasePath,
userId,
containerName,
}) {
async function deleteFileFromAzure(req, file) {
try {
const containerClient = getAzureContainerClient(containerName);
const blobPath = `${basePath}/${userId}/${fileName}`;
const containerClient = getAzureContainerClient(AZURE_CONTAINER_NAME);
const blobPath = file.filepath.split(`${AZURE_CONTAINER_NAME}/`)[1];
if (!blobPath.includes(req.user.id)) {
throw new Error('User ID not found in blob path');
}
const blockBlobClient = containerClient.getBlockBlobClient(blobPath);
await blockBlobClient.delete();
logger.debug('[deleteFileFromAzure] Blob deleted successfully from Azure Blob Storage');
} catch (error) {
logger.error('[deleteFileFromAzure] Error deleting blob:', error.message);
logger.error('[deleteFileFromAzure] Error deleting blob:', error);
if (error.statusCode === 404) {
return;
}
@ -123,6 +120,65 @@ async function deleteFileFromAzure({
}
}
/**
* Streams a file from disk directly to Azure Blob Storage without loading
* the entire file into memory.
*
* @param {Object} params
* @param {string} params.userId - The user's id.
* @param {string} params.filePath - The local file path to upload.
* @param {string} params.fileName - The name of the file in Azure.
* @param {string} [params.basePath='images'] - The base folder within the container.
* @param {string} [params.containerName] - The Azure Blob container name.
* @returns {Promise<string>} The URL of the uploaded blob.
*/
async function streamFileToAzure({
userId,
filePath,
fileName,
basePath = defaultBasePath,
containerName,
}) {
try {
const containerClient = getAzureContainerClient(containerName);
const access = AZURE_STORAGE_PUBLIC_ACCESS?.toLowerCase() === 'true' ? 'blob' : undefined;
// Create the container if it doesn't exist
await containerClient.createIfNotExists({ access });
const blobPath = `${basePath}/${userId}/${fileName}`;
const blockBlobClient = containerClient.getBlockBlobClient(blobPath);
// Get file size for proper content length
const stats = await fs.promises.stat(filePath);
// Create read stream from the file
const fileStream = fs.createReadStream(filePath);
const blobContentType = mime.getType(fileName);
await blockBlobClient.uploadStream(
fileStream,
undefined, // Use default concurrency (5)
undefined, // Use default buffer size (8MB)
{
blobHTTPHeaders: {
blobContentType,
},
onProgress: (progress) => {
logger.debug(
`[streamFileToAzure] Upload progress: ${progress.loadedBytes} bytes of ${stats.size}`,
);
},
},
);
return blockBlobClient.url;
} catch (error) {
logger.error('[streamFileToAzure] Error streaming file:', error);
throw error;
}
}
/**
* Uploads a file from the local file system to Azure Blob Storage.
*
@ -146,18 +202,19 @@ async function uploadFileToAzure({
}) {
try {
const inputFilePath = file.path;
const inputBuffer = await fs.promises.readFile(inputFilePath);
const bytes = Buffer.byteLength(inputBuffer);
const stats = await fs.promises.stat(inputFilePath);
const bytes = stats.size;
const userId = req.user.id;
const fileName = `${file_id}__${path.basename(inputFilePath)}`;
const fileURL = await saveBufferToAzure({
const fileURL = await streamFileToAzure({
userId,
buffer: inputBuffer,
filePath: inputFilePath,
fileName,
basePath,
containerName,
});
await fs.promises.unlink(inputFilePath);
return { filepath: fileURL, bytes };
} catch (error) {
logger.error('[uploadFileToAzure] Error uploading file:', error);