const fs = require('fs'); const path = require('path'); const fetch = require('node-fetch'); const { FileSources } = require('librechat-data-provider'); const { PutObjectCommand, GetObjectCommand, HeadObjectCommand, DeleteObjectCommand, } = require('@aws-sdk/client-s3'); const { getSignedUrl } = require('@aws-sdk/s3-request-presigner'); const { initializeS3 } = require('./initialize'); const { logger } = require('~/config'); const bucketName = process.env.AWS_BUCKET_NAME; const defaultBasePath = 'images'; let s3UrlExpirySeconds = 7 * 24 * 60 * 60; let s3RefreshExpiryMs = null; if (process.env.S3_URL_EXPIRY_SECONDS !== undefined) { const parsed = parseInt(process.env.S3_URL_EXPIRY_SECONDS, 10); if (!isNaN(parsed) && parsed > 0) { s3UrlExpirySeconds = Math.min(parsed, 7 * 24 * 60 * 60); } else { logger.warn( `[S3] Invalid S3_URL_EXPIRY_SECONDS value: "${process.env.S3_URL_EXPIRY_SECONDS}". Using 7-day expiry.`, ); } } if (process.env.S3_REFRESH_EXPIRY_MS !== null && process.env.S3_REFRESH_EXPIRY_MS) { const parsed = parseInt(process.env.S3_REFRESH_EXPIRY_MS, 10); if (!isNaN(parsed) && parsed > 0) { s3RefreshExpiryMs = parsed; logger.info(`[S3] Using custom refresh expiry time: ${s3RefreshExpiryMs}ms`); } else { logger.warn( `[S3] Invalid S3_REFRESH_EXPIRY_MS value: "${process.env.S3_REFRESH_EXPIRY_MS}". Using default refresh logic.`, ); } } /** * Constructs the S3 key based on the base path, user ID, and file name. */ const getS3Key = (basePath, userId, fileName) => `${basePath}/${userId}/${fileName}`; /** * Uploads a buffer to S3 and returns a signed URL. * * @param {Object} params * @param {string} params.userId - The user's unique identifier. * @param {Buffer} params.buffer - The buffer containing file data. * @param {string} params.fileName - The file name to use in S3. * @param {string} [params.basePath='images'] - The base path in the bucket. * @returns {Promise} Signed URL of the uploaded file. */ async function saveBufferToS3({ userId, buffer, fileName, basePath = defaultBasePath }) { const key = getS3Key(basePath, userId, fileName); const params = { Bucket: bucketName, Key: key, Body: buffer }; try { const s3 = initializeS3(); await s3.send(new PutObjectCommand(params)); return await getS3URL({ userId, fileName, basePath }); } catch (error) { logger.error('[saveBufferToS3] Error uploading buffer to S3:', error.message); throw error; } } /** * Retrieves a URL for a file stored in S3. * Returns a signed URL with expiration time or a proxy URL based on config * * @param {Object} params * @param {string} params.userId - The user's unique identifier. * @param {string} params.fileName - The file name in S3. * @param {string} [params.basePath='images'] - The base path in the bucket. * @returns {Promise} A URL to access the S3 object */ async function getS3URL({ userId, fileName, basePath = defaultBasePath }) { const key = getS3Key(basePath, userId, fileName); const params = { Bucket: bucketName, Key: key }; try { const s3 = initializeS3(); return await getSignedUrl(s3, new GetObjectCommand(params), { expiresIn: s3UrlExpirySeconds }); } catch (error) { logger.error('[getS3URL] Error getting signed URL from S3:', error.message); throw error; } } /** * Saves a file from a given URL to S3. * * @param {Object} params * @param {string} params.userId - The user's unique identifier. * @param {string} params.URL - The source URL of the file. * @param {string} params.fileName - The file name to use in S3. * @param {string} [params.basePath='images'] - The base path in the bucket. * @returns {Promise} Signed URL of the uploaded file. */ async function saveURLToS3({ userId, URL, fileName, basePath = defaultBasePath }) { try { const response = await fetch(URL); const buffer = await response.buffer(); // Optionally you can call getBufferMetadata(buffer) if needed. return await saveBufferToS3({ userId, buffer, fileName, basePath }); } catch (error) { logger.error('[saveURLToS3] Error uploading file from URL to S3:', error.message); throw error; } } /** * Deletes a file from S3. * * @param {Object} params * @param {ServerRequest} params.req * @param {MongoFile} params.file - The file object to delete. * @returns {Promise} */ async function deleteFileFromS3(req, file) { const key = extractKeyFromS3Url(file.filepath); const params = { Bucket: bucketName, Key: key }; if (!key.includes(req.user.id)) { const message = `[deleteFileFromS3] User ID mismatch: ${req.user.id} vs ${key}`; logger.error(message); throw new Error(message); } try { const s3 = initializeS3(); try { const headCommand = new HeadObjectCommand(params); await s3.send(headCommand); logger.debug('[deleteFileFromS3] File exists, proceeding with deletion'); } catch (headErr) { if (headErr.name === 'NotFound') { logger.warn(`[deleteFileFromS3] File does not exist: ${key}`); return; } } const deleteResult = await s3.send(new DeleteObjectCommand(params)); logger.debug('[deleteFileFromS3] Delete command response:', JSON.stringify(deleteResult)); try { await s3.send(new HeadObjectCommand(params)); logger.error('[deleteFileFromS3] File still exists after deletion!'); } catch (verifyErr) { if (verifyErr.name === 'NotFound') { logger.debug(`[deleteFileFromS3] Verified file is deleted: ${key}`); } else { logger.error('[deleteFileFromS3] Error verifying deletion:', verifyErr); } } logger.debug('[deleteFileFromS3] S3 File deletion completed'); } catch (error) { logger.error(`[deleteFileFromS3] Error deleting file from S3: ${error.message}`); logger.error(error.stack); // If the file is not found, we can safely return. if (error.code === 'NoSuchKey') { return; } throw error; } } /** * Uploads a local file to S3 by streaming it directly without loading into memory. * * @param {Object} params * @param {import('express').Request} params.req - The Express request (must include user). * @param {Express.Multer.File} params.file - The file object from Multer. * @param {string} params.file_id - Unique file identifier. * @param {string} [params.basePath='images'] - The base path in the bucket. * @returns {Promise<{ filepath: string, bytes: number }>} */ async function uploadFileToS3({ req, file, file_id, basePath = defaultBasePath }) { try { const inputFilePath = file.path; const userId = req.user.id; const fileName = `${file_id}__${path.basename(inputFilePath)}`; const key = getS3Key(basePath, userId, fileName); const stats = await fs.promises.stat(inputFilePath); const bytes = stats.size; const fileStream = fs.createReadStream(inputFilePath); const s3 = initializeS3(); const uploadParams = { Bucket: bucketName, Key: key, Body: fileStream, }; await s3.send(new PutObjectCommand(uploadParams)); const fileURL = await getS3URL({ userId, fileName, basePath }); return { filepath: fileURL, bytes }; } catch (error) { logger.error('[uploadFileToS3] Error streaming file to S3:', error); try { if (file && file.path) { await fs.promises.unlink(file.path); } } catch (unlinkError) { logger.error( '[uploadFileToS3] Error deleting temporary file, likely already deleted:', unlinkError.message, ); } throw error; } } /** * Extracts the S3 key from a URL or returns the key if already properly formatted * * @param {string} fileUrlOrKey - The file URL or key * @returns {string} The S3 key */ function extractKeyFromS3Url(fileUrlOrKey) { if (!fileUrlOrKey) { throw new Error('Invalid input: URL or key is empty'); } try { const url = new URL(fileUrlOrKey); return url.pathname.substring(1); } catch (error) { const parts = fileUrlOrKey.split('/'); if (parts.length >= 3 && !fileUrlOrKey.startsWith('http') && !fileUrlOrKey.startsWith('/')) { return fileUrlOrKey; } return fileUrlOrKey.startsWith('/') ? fileUrlOrKey.substring(1) : fileUrlOrKey; } } /** * Retrieves a readable stream for a file stored in S3. * * @param {ServerRequest} req - Server request object. * @param {string} filePath - The S3 key of the file. * @returns {Promise} */ async function getS3FileStream(_req, filePath) { try { const Key = extractKeyFromS3Url(filePath); const params = { Bucket: bucketName, Key }; const s3 = initializeS3(); const data = await s3.send(new GetObjectCommand(params)); return data.Body; // Returns a Node.js ReadableStream. } catch (error) { logger.error('[getS3FileStream] Error retrieving S3 file stream:', error); throw error; } } /** * Determines if a signed S3 URL is close to expiration * * @param {string} signedUrl - The signed S3 URL * @param {number} bufferSeconds - Buffer time in seconds * @returns {boolean} True if the URL needs refreshing */ function needsRefresh(signedUrl, bufferSeconds) { try { // Parse the URL const url = new URL(signedUrl); // Check if it has the signature parameters that indicate it's a signed URL // X-Amz-Signature is the most reliable indicator for AWS signed URLs if (!url.searchParams.has('X-Amz-Signature')) { // Not a signed URL, so no expiration to check (or it's already a proxy URL) return false; } // Extract the expiration time from the URL const expiresParam = url.searchParams.get('X-Amz-Expires'); const dateParam = url.searchParams.get('X-Amz-Date'); if (!expiresParam || !dateParam) { // Missing expiration information, assume it needs refresh to be safe return true; } // Parse the AWS date format (YYYYMMDDTHHMMSSZ) const year = dateParam.substring(0, 4); const month = dateParam.substring(4, 6); const day = dateParam.substring(6, 8); const hour = dateParam.substring(9, 11); const minute = dateParam.substring(11, 13); const second = dateParam.substring(13, 15); const dateObj = new Date(`${year}-${month}-${day}T${hour}:${minute}:${second}Z`); const expiresAtDate = new Date(dateObj.getTime() + parseInt(expiresParam) * 1000); // Check if it's close to expiration const now = new Date(); // If S3_REFRESH_EXPIRY_MS is set, use it to determine if URL is expired if (s3RefreshExpiryMs !== null) { const urlCreationTime = dateObj.getTime(); const urlAge = now.getTime() - urlCreationTime; return urlAge >= s3RefreshExpiryMs; } // Otherwise use the default buffer-based logic const bufferTime = new Date(now.getTime() + bufferSeconds * 1000); return expiresAtDate <= bufferTime; } catch (error) { logger.error('Error checking URL expiration:', error); // If we can't determine, assume it needs refresh to be safe return true; } } /** * Generates a new URL for an expired S3 URL * @param {string} currentURL - The current file URL * @returns {Promise} */ async function getNewS3URL(currentURL) { try { const s3Key = extractKeyFromS3Url(currentURL); if (!s3Key) { return; } const keyParts = s3Key.split('/'); if (keyParts.length < 3) { return; } const basePath = keyParts[0]; const userId = keyParts[1]; const fileName = keyParts.slice(2).join('/'); return await getS3URL({ userId, fileName, basePath, }); } catch (error) { logger.error('Error getting new S3 URL:', error); } } /** * Refreshes S3 URLs for an array of files if they're expired or close to expiring * * @param {MongoFile[]} files - Array of file documents * @param {(files: MongoFile[]) => Promise} batchUpdateFiles - Function to update files in the database * @param {number} [bufferSeconds=3600] - Buffer time in seconds to check for expiration * @returns {Promise} The files with refreshed URLs if needed */ async function refreshS3FileUrls(files, batchUpdateFiles, bufferSeconds = 3600) { if (!files || !Array.isArray(files) || files.length === 0) { return files; } const filesToUpdate = []; for (let i = 0; i < files.length; i++) { const file = files[i]; if (!file?.file_id) { continue; } if (file.source !== FileSources.s3) { continue; } if (!file.filepath) { continue; } if (!needsRefresh(file.filepath, bufferSeconds)) { continue; } try { const newURL = await getNewS3URL(file.filepath); if (!newURL) { continue; } filesToUpdate.push({ file_id: file.file_id, filepath: newURL, }); files[i].filepath = newURL; } catch (error) { logger.error(`Error refreshing S3 URL for file ${file.file_id}:`, error); } } if (filesToUpdate.length > 0) { await batchUpdateFiles(filesToUpdate); } return files; } /** * Refreshes a single S3 URL if it's expired or close to expiring * * @param {{ filepath: string, source: string }} fileObj - Simple file object containing filepath and source * @param {number} [bufferSeconds=3600] - Buffer time in seconds to check for expiration * @returns {Promise} The refreshed URL or the original URL if no refresh needed */ async function refreshS3Url(fileObj, bufferSeconds = 3600) { if (!fileObj || fileObj.source !== FileSources.s3 || !fileObj.filepath) { return fileObj?.filepath || ''; } if (!needsRefresh(fileObj.filepath, bufferSeconds)) { return fileObj.filepath; } try { const s3Key = extractKeyFromS3Url(fileObj.filepath); if (!s3Key) { logger.warn(`Unable to extract S3 key from URL: ${fileObj.filepath}`); return fileObj.filepath; } const keyParts = s3Key.split('/'); if (keyParts.length < 3) { logger.warn(`Invalid S3 key format: ${s3Key}`); return fileObj.filepath; } const basePath = keyParts[0]; const userId = keyParts[1]; const fileName = keyParts.slice(2).join('/'); const newUrl = await getS3URL({ userId, fileName, basePath, }); logger.debug(`Refreshed S3 URL for key: ${s3Key}`); return newUrl; } catch (error) { logger.error(`Error refreshing S3 URL: ${error.message}`); return fileObj.filepath; } } module.exports = { saveBufferToS3, saveURLToS3, getS3URL, deleteFileFromS3, uploadFileToS3, getS3FileStream, refreshS3FileUrls, refreshS3Url, needsRefresh, getNewS3URL, };