mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-03-12 19:12:36 +01:00
🗂️ refactor: Migrate S3 Storage to TypeScript in packages/api (#11947)
* Migrate S3 storage module with unit and integration tests - Migrate S3 CRUD and image operations to packages/api/src/storage/s3/ - Add S3ImageService class with dependency injection - Add unit tests using aws-sdk-client-mock - Add integration tests with real s3 bucket (condition presence of AWS_TEST_BUCKET_NAME) * AI Review Findings Fixes * chore: tests and refactor S3 storage types - Added mock implementations for the 'sharp' library in various test files to improve image processing testing. - Updated type references in S3 storage files from MongoFile to TFile for consistency and type safety. - Refactored S3 CRUD operations to ensure proper handling of file types and improve code clarity. - Enhanced integration tests to validate S3 file operations and error handling more effectively. * chore: rename test file * Remove duplicate import of refreshS3Url * chore: imports order * fix: remove duplicate imports for S3 URL handling in UserController * fix: remove duplicate import of refreshS3FileUrls in files.js * test: Add mock implementations for 'sharp' and '@librechat/api' in UserController tests - Introduced mock functions for the 'sharp' library to facilitate image processing tests, including metadata retrieval and buffer conversion. - Enhanced mocking for '@librechat/api' to ensure consistent behavior in tests, particularly for the needsRefresh and getNewS3URL functions. --------- Co-authored-by: Danny Avila <danny@librechat.ai>
This commit is contained in:
Atef Bellaaj
Danny Avila
parent
428ef2eb15
commit
ca6ce8fceb
27 changed files with 2455 additions and 1697 deletions
|
|
@ -1,6 +1,8 @@
|
|||
const { logger, webSearchKeys } = require('@librechat/data-schemas');
|
||||
const { Tools, CacheKeys, Constants, FileSources } = require('librechat-data-provider');
|
||||
const {
|
||||
getNewS3URL,
|
||||
needsRefresh,
|
||||
MCPOAuthHandler,
|
||||
MCPTokenStorage,
|
||||
normalizeHttpError,
|
||||
|
|
@ -10,7 +12,6 @@ const { updateUserPluginAuth, deleteUserPluginAuth } = require('~/server/service
|
|||
const { verifyEmail, resendVerificationEmail } = require('~/server/services/AuthService');
|
||||
const { getMCPManager, getFlowStateManager, getMCPServersRegistry } = require('~/config');
|
||||
const { invalidateCachedTools } = require('~/server/services/Config/getCachedTools');
|
||||
const { needsRefresh, getNewS3URL } = require('~/server/services/Files/S3/crud');
|
||||
const { processDeleteRequest } = require('~/server/services/Files/process');
|
||||
const { getAppConfig } = require('~/server/services/Config');
|
||||
const { getLogStores } = require('~/cache');
|
||||
|
|
|
|||
|
|
@ -59,7 +59,16 @@ jest.mock('~/server/services/AuthService', () => ({
|
|||
resendVerificationEmail: jest.fn(),
|
||||
}));
|
||||
|
||||
jest.mock('~/server/services/Files/S3/crud', () => ({
|
||||
jest.mock('sharp', () =>
|
||||
jest.fn(() => ({
|
||||
metadata: jest.fn().mockResolvedValue({}),
|
||||
toFormat: jest.fn().mockReturnThis(),
|
||||
toBuffer: jest.fn().mockResolvedValue(Buffer.alloc(0)),
|
||||
})),
|
||||
);
|
||||
|
||||
jest.mock('@librechat/api', () => ({
|
||||
...jest.requireActual('@librechat/api'),
|
||||
needsRefresh: jest.fn(),
|
||||
getNewS3URL: jest.fn(),
|
||||
}));
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ const fs = require('fs').promises;
|
|||
const { nanoid } = require('nanoid');
|
||||
const { logger } = require('@librechat/data-schemas');
|
||||
const {
|
||||
refreshS3Url,
|
||||
agentCreateSchema,
|
||||
agentUpdateSchema,
|
||||
refreshListAvatars,
|
||||
|
|
@ -33,7 +34,6 @@ const {
|
|||
const { getStrategyFunctions } = require('~/server/services/Files/strategies');
|
||||
const { resizeAvatar } = require('~/server/services/Files/images/avatar');
|
||||
const { getFileStrategy } = require('~/server/utils/getFileStrategy');
|
||||
const { refreshS3Url } = require('~/server/services/Files/S3/crud');
|
||||
const { filterFile } = require('~/server/services/Files/process');
|
||||
const { getCachedTools } = require('~/server/services/Config');
|
||||
const { getLogStores } = require('~/cache');
|
||||
|
|
|
|||
|
|
@ -22,7 +22,16 @@ jest.mock('~/server/services/Files/images/avatar', () => ({
|
|||
resizeAvatar: jest.fn(),
|
||||
}));
|
||||
|
||||
jest.mock('~/server/services/Files/S3/crud', () => ({
|
||||
jest.mock('sharp', () =>
|
||||
jest.fn(() => ({
|
||||
metadata: jest.fn().mockResolvedValue({}),
|
||||
toFormat: jest.fn().mockReturnThis(),
|
||||
toBuffer: jest.fn().mockResolvedValue(Buffer.alloc(0)),
|
||||
})),
|
||||
);
|
||||
|
||||
jest.mock('@librechat/api', () => ({
|
||||
...jest.requireActual('@librechat/api'),
|
||||
refreshS3Url: jest.fn(),
|
||||
}));
|
||||
|
||||
|
|
@ -72,7 +81,7 @@ const {
|
|||
findPubliclyAccessibleResources,
|
||||
} = require('~/server/services/PermissionService');
|
||||
|
||||
const { refreshS3Url } = require('~/server/services/Files/S3/crud');
|
||||
const { refreshS3Url } = require('@librechat/api');
|
||||
|
||||
/**
|
||||
* @type {import('mongoose').Model<import('@librechat/data-schemas').IAgent>}
|
||||
|
|
|
|||
|
|
@ -39,7 +39,16 @@ jest.mock('~/server/services/Tools/credentials', () => ({
|
|||
loadAuthValues: jest.fn(),
|
||||
}));
|
||||
|
||||
jest.mock('~/server/services/Files/S3/crud', () => ({
|
||||
jest.mock('sharp', () =>
|
||||
jest.fn(() => ({
|
||||
metadata: jest.fn().mockResolvedValue({}),
|
||||
toFormat: jest.fn().mockReturnThis(),
|
||||
toBuffer: jest.fn().mockResolvedValue(Buffer.alloc(0)),
|
||||
})),
|
||||
);
|
||||
|
||||
jest.mock('@librechat/api', () => ({
|
||||
...jest.requireActual('@librechat/api'),
|
||||
refreshS3FileUrls: jest.fn(),
|
||||
}));
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ const fs = require('fs').promises;
|
|||
const express = require('express');
|
||||
const { EnvVar } = require('@librechat/agents');
|
||||
const { logger } = require('@librechat/data-schemas');
|
||||
const { refreshS3FileUrls } = require('@librechat/api');
|
||||
const {
|
||||
Time,
|
||||
isUUID,
|
||||
|
|
@ -25,7 +26,6 @@ const { getStrategyFunctions } = require('~/server/services/Files/strategies');
|
|||
const { getOpenAIClient } = require('~/server/controllers/assistants/helpers');
|
||||
const { checkPermission } = require('~/server/services/PermissionService');
|
||||
const { loadAuthValues } = require('~/server/services/Tools/credentials');
|
||||
const { refreshS3FileUrls } = require('~/server/services/Files/S3/crud');
|
||||
const { hasAccessToFilesViaAgent } = require('~/server/services/Files');
|
||||
const { cleanFileName } = require('~/server/utils/files');
|
||||
const { hasCapability } = require('~/server/middleware');
|
||||
|
|
|
|||
|
|
@ -32,7 +32,16 @@ jest.mock('~/server/services/Tools/credentials', () => ({
|
|||
loadAuthValues: jest.fn(),
|
||||
}));
|
||||
|
||||
jest.mock('~/server/services/Files/S3/crud', () => ({
|
||||
jest.mock('sharp', () =>
|
||||
jest.fn(() => ({
|
||||
metadata: jest.fn().mockResolvedValue({}),
|
||||
toFormat: jest.fn().mockReturnThis(),
|
||||
toBuffer: jest.fn().mockResolvedValue(Buffer.alloc(0)),
|
||||
})),
|
||||
);
|
||||
|
||||
jest.mock('@librechat/api', () => ({
|
||||
...jest.requireActual('@librechat/api'),
|
||||
refreshS3FileUrls: jest.fn(),
|
||||
}));
|
||||
|
||||
|
|
|
|||
|
|
@ -1,556 +0,0 @@
|
|||
const fs = require('fs');
|
||||
const fetch = require('node-fetch');
|
||||
const { logger } = require('@librechat/data-schemas');
|
||||
const { FileSources } = require('librechat-data-provider');
|
||||
const { getSignedUrl } = require('@aws-sdk/s3-request-presigner');
|
||||
const { initializeS3, deleteRagFile, isEnabled } = require('@librechat/api');
|
||||
const {
|
||||
PutObjectCommand,
|
||||
GetObjectCommand,
|
||||
HeadObjectCommand,
|
||||
DeleteObjectCommand,
|
||||
} = require('@aws-sdk/client-s3');
|
||||
|
||||
const bucketName = process.env.AWS_BUCKET_NAME;
|
||||
const defaultBasePath = 'images';
|
||||
const endpoint = process.env.AWS_ENDPOINT_URL;
|
||||
const forcePathStyle = isEnabled(process.env.AWS_FORCE_PATH_STYLE);
|
||||
|
||||
let s3UrlExpirySeconds = 2 * 60; // 2 minutes
|
||||
let s3RefreshExpiryMs = null;
|
||||
|
||||
if (process.env.S3_URL_EXPIRY_SECONDS !== undefined) {
|
||||
const parsed = parseInt(process.env.S3_URL_EXPIRY_SECONDS, 10);
|
||||
|
||||
if (!isNaN(parsed) && parsed > 0) {
|
||||
s3UrlExpirySeconds = Math.min(parsed, 7 * 24 * 60 * 60);
|
||||
} else {
|
||||
logger.warn(
|
||||
`[S3] Invalid S3_URL_EXPIRY_SECONDS value: "${process.env.S3_URL_EXPIRY_SECONDS}". Using 2-minute expiry.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (process.env.S3_REFRESH_EXPIRY_MS !== null && process.env.S3_REFRESH_EXPIRY_MS) {
|
||||
const parsed = parseInt(process.env.S3_REFRESH_EXPIRY_MS, 10);
|
||||
|
||||
if (!isNaN(parsed) && parsed > 0) {
|
||||
s3RefreshExpiryMs = parsed;
|
||||
logger.info(`[S3] Using custom refresh expiry time: ${s3RefreshExpiryMs}ms`);
|
||||
} else {
|
||||
logger.warn(
|
||||
`[S3] Invalid S3_REFRESH_EXPIRY_MS value: "${process.env.S3_REFRESH_EXPIRY_MS}". Using default refresh logic.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructs the S3 key based on the base path, user ID, and file name.
|
||||
*/
|
||||
const getS3Key = (basePath, userId, fileName) => `${basePath}/${userId}/${fileName}`;
|
||||
|
||||
/**
|
||||
* Uploads a buffer to S3 and returns a signed URL.
|
||||
*
|
||||
* @param {Object} params
|
||||
* @param {string} params.userId - The user's unique identifier.
|
||||
* @param {Buffer} params.buffer - The buffer containing file data.
|
||||
* @param {string} params.fileName - The file name to use in S3.
|
||||
* @param {string} [params.basePath='images'] - The base path in the bucket.
|
||||
* @returns {Promise<string>} Signed URL of the uploaded file.
|
||||
*/
|
||||
async function saveBufferToS3({ userId, buffer, fileName, basePath = defaultBasePath }) {
|
||||
const key = getS3Key(basePath, userId, fileName);
|
||||
const params = { Bucket: bucketName, Key: key, Body: buffer };
|
||||
|
||||
try {
|
||||
const s3 = initializeS3();
|
||||
await s3.send(new PutObjectCommand(params));
|
||||
return await getS3URL({ userId, fileName, basePath });
|
||||
} catch (error) {
|
||||
logger.error('[saveBufferToS3] Error uploading buffer to S3:', error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves a URL for a file stored in S3.
|
||||
* Returns a signed URL with expiration time or a proxy URL based on config
|
||||
*
|
||||
* @param {Object} params
|
||||
* @param {string} params.userId - The user's unique identifier.
|
||||
* @param {string} params.fileName - The file name in S3.
|
||||
* @param {string} [params.basePath='images'] - The base path in the bucket.
|
||||
* @param {string} [params.customFilename] - Custom filename for Content-Disposition header (overrides extracted filename).
|
||||
* @param {string} [params.contentType] - Custom content type for the response.
|
||||
* @returns {Promise<string>} A URL to access the S3 object
|
||||
*/
|
||||
async function getS3URL({
|
||||
userId,
|
||||
fileName,
|
||||
basePath = defaultBasePath,
|
||||
customFilename = null,
|
||||
contentType = null,
|
||||
}) {
|
||||
const key = getS3Key(basePath, userId, fileName);
|
||||
const params = { Bucket: bucketName, Key: key };
|
||||
|
||||
// Add response headers if specified
|
||||
if (customFilename) {
|
||||
params.ResponseContentDisposition = `attachment; filename="${customFilename}"`;
|
||||
}
|
||||
|
||||
if (contentType) {
|
||||
params.ResponseContentType = contentType;
|
||||
}
|
||||
|
||||
try {
|
||||
const s3 = initializeS3();
|
||||
return await getSignedUrl(s3, new GetObjectCommand(params), { expiresIn: s3UrlExpirySeconds });
|
||||
} catch (error) {
|
||||
logger.error('[getS3URL] Error getting signed URL from S3:', error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves a file from a given URL to S3.
|
||||
*
|
||||
* @param {Object} params
|
||||
* @param {string} params.userId - The user's unique identifier.
|
||||
* @param {string} params.URL - The source URL of the file.
|
||||
* @param {string} params.fileName - The file name to use in S3.
|
||||
* @param {string} [params.basePath='images'] - The base path in the bucket.
|
||||
* @returns {Promise<string>} Signed URL of the uploaded file.
|
||||
*/
|
||||
async function saveURLToS3({ userId, URL, fileName, basePath = defaultBasePath }) {
|
||||
try {
|
||||
const response = await fetch(URL);
|
||||
const buffer = await response.buffer();
|
||||
// Optionally you can call getBufferMetadata(buffer) if needed.
|
||||
return await saveBufferToS3({ userId, buffer, fileName, basePath });
|
||||
} catch (error) {
|
||||
logger.error('[saveURLToS3] Error uploading file from URL to S3:', error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes a file from S3.
|
||||
*
|
||||
* @param {Object} params
|
||||
* @param {ServerRequest} params.req
|
||||
* @param {MongoFile} params.file - The file object to delete.
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function deleteFileFromS3(req, file) {
|
||||
await deleteRagFile({ userId: req.user.id, file });
|
||||
|
||||
const key = extractKeyFromS3Url(file.filepath);
|
||||
const params = { Bucket: bucketName, Key: key };
|
||||
if (!key.includes(req.user.id)) {
|
||||
const message = `[deleteFileFromS3] User ID mismatch: ${req.user.id} vs ${key}`;
|
||||
logger.error(message);
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
try {
|
||||
const s3 = initializeS3();
|
||||
|
||||
try {
|
||||
const headCommand = new HeadObjectCommand(params);
|
||||
await s3.send(headCommand);
|
||||
logger.debug('[deleteFileFromS3] File exists, proceeding with deletion');
|
||||
} catch (headErr) {
|
||||
if (headErr.name === 'NotFound') {
|
||||
logger.warn(`[deleteFileFromS3] File does not exist: ${key}`);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const deleteResult = await s3.send(new DeleteObjectCommand(params));
|
||||
logger.debug('[deleteFileFromS3] Delete command response:', JSON.stringify(deleteResult));
|
||||
try {
|
||||
await s3.send(new HeadObjectCommand(params));
|
||||
logger.error('[deleteFileFromS3] File still exists after deletion!');
|
||||
} catch (verifyErr) {
|
||||
if (verifyErr.name === 'NotFound') {
|
||||
logger.debug(`[deleteFileFromS3] Verified file is deleted: ${key}`);
|
||||
} else {
|
||||
logger.error('[deleteFileFromS3] Error verifying deletion:', verifyErr);
|
||||
}
|
||||
}
|
||||
|
||||
logger.debug('[deleteFileFromS3] S3 File deletion completed');
|
||||
} catch (error) {
|
||||
logger.error(`[deleteFileFromS3] Error deleting file from S3: ${error.message}`);
|
||||
logger.error(error.stack);
|
||||
|
||||
// If the file is not found, we can safely return.
|
||||
if (error.code === 'NoSuchKey') {
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Uploads a local file to S3 by streaming it directly without loading into memory.
|
||||
*
|
||||
* @param {Object} params
|
||||
* @param {import('express').Request} params.req - The Express request (must include user).
|
||||
* @param {Express.Multer.File} params.file - The file object from Multer.
|
||||
* @param {string} params.file_id - Unique file identifier.
|
||||
* @param {string} [params.basePath='images'] - The base path in the bucket.
|
||||
* @returns {Promise<{ filepath: string, bytes: number }>}
|
||||
*/
|
||||
async function uploadFileToS3({ req, file, file_id, basePath = defaultBasePath }) {
|
||||
try {
|
||||
const inputFilePath = file.path;
|
||||
const userId = req.user.id;
|
||||
const fileName = `${file_id}__${file.originalname}`;
|
||||
const key = getS3Key(basePath, userId, fileName);
|
||||
|
||||
const stats = await fs.promises.stat(inputFilePath);
|
||||
const bytes = stats.size;
|
||||
const fileStream = fs.createReadStream(inputFilePath);
|
||||
|
||||
const s3 = initializeS3();
|
||||
const uploadParams = {
|
||||
Bucket: bucketName,
|
||||
Key: key,
|
||||
Body: fileStream,
|
||||
};
|
||||
|
||||
await s3.send(new PutObjectCommand(uploadParams));
|
||||
const fileURL = await getS3URL({ userId, fileName, basePath });
|
||||
return { filepath: fileURL, bytes };
|
||||
} catch (error) {
|
||||
logger.error('[uploadFileToS3] Error streaming file to S3:', error);
|
||||
try {
|
||||
if (file && file.path) {
|
||||
await fs.promises.unlink(file.path);
|
||||
}
|
||||
} catch (unlinkError) {
|
||||
logger.error(
|
||||
'[uploadFileToS3] Error deleting temporary file, likely already deleted:',
|
||||
unlinkError.message,
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the S3 key from a URL or returns the key if already properly formatted
|
||||
*
|
||||
* @param {string} fileUrlOrKey - The file URL or key
|
||||
* @returns {string} The S3 key
|
||||
*/
|
||||
function extractKeyFromS3Url(fileUrlOrKey) {
|
||||
if (!fileUrlOrKey) {
|
||||
throw new Error('Invalid input: URL or key is empty');
|
||||
}
|
||||
|
||||
try {
|
||||
const url = new URL(fileUrlOrKey);
|
||||
const hostname = url.hostname;
|
||||
const pathname = url.pathname.substring(1); // Remove leading slash
|
||||
|
||||
// Explicit path-style with custom endpoint: use endpoint pathname for precise key extraction.
|
||||
// Handles endpoints with a base path (e.g. https://example.com/storage/).
|
||||
if (endpoint && forcePathStyle) {
|
||||
const endpointUrl = new URL(endpoint);
|
||||
const startPos =
|
||||
endpointUrl.pathname.length +
|
||||
(endpointUrl.pathname.endsWith('/') ? 0 : 1) +
|
||||
bucketName.length +
|
||||
1;
|
||||
const key = url.pathname.substring(startPos);
|
||||
if (!key) {
|
||||
logger.warn(
|
||||
`[extractKeyFromS3Url] Extracted key is empty for endpoint path-style URL: ${fileUrlOrKey}`,
|
||||
);
|
||||
} else {
|
||||
logger.debug(`[extractKeyFromS3Url] fileUrlOrKey: ${fileUrlOrKey}, Extracted key: ${key}`);
|
||||
}
|
||||
return key;
|
||||
}
|
||||
|
||||
if (
|
||||
hostname === 's3.amazonaws.com' ||
|
||||
hostname.match(/^s3[-.][a-z0-9-]+\.amazonaws\.com$/) ||
|
||||
(bucketName && pathname.startsWith(`${bucketName}/`))
|
||||
) {
|
||||
// Path-style: https://s3.amazonaws.com/bucket-name/key or custom endpoint (MinIO, R2, etc.)
|
||||
// Strip the bucket name (first path segment)
|
||||
const firstSlashIndex = pathname.indexOf('/');
|
||||
if (firstSlashIndex > 0) {
|
||||
const key = pathname.substring(firstSlashIndex + 1);
|
||||
|
||||
if (key === '') {
|
||||
logger.warn(
|
||||
`[extractKeyFromS3Url] Extracted key is empty after removing bucket name from URL: ${fileUrlOrKey}`,
|
||||
);
|
||||
} else {
|
||||
logger.debug(
|
||||
`[extractKeyFromS3Url] fileUrlOrKey: ${fileUrlOrKey}, Extracted key: ${key}`,
|
||||
);
|
||||
}
|
||||
|
||||
return key;
|
||||
} else {
|
||||
logger.warn(
|
||||
`[extractKeyFromS3Url] Unable to extract key from path-style URL: ${fileUrlOrKey}`,
|
||||
);
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
// Virtual-hosted-style or other: https://bucket-name.s3.amazonaws.com/key
|
||||
// Just return the pathname without leading slash
|
||||
logger.debug(`[extractKeyFromS3Url] fileUrlOrKey: ${fileUrlOrKey}, Extracted key: ${pathname}`);
|
||||
return pathname;
|
||||
} catch (error) {
|
||||
if (fileUrlOrKey.startsWith('http://') || fileUrlOrKey.startsWith('https://')) {
|
||||
logger.error(
|
||||
`[extractKeyFromS3Url] Error parsing URL: ${fileUrlOrKey}, Error: ${error.message}`,
|
||||
);
|
||||
} else {
|
||||
logger.debug(`[extractKeyFromS3Url] Non-URL input, using fallback: ${fileUrlOrKey}`);
|
||||
}
|
||||
|
||||
const parts = fileUrlOrKey.split('/');
|
||||
|
||||
if (parts.length >= 3 && !fileUrlOrKey.startsWith('http') && !fileUrlOrKey.startsWith('/')) {
|
||||
return fileUrlOrKey;
|
||||
}
|
||||
|
||||
const key = fileUrlOrKey.startsWith('/') ? fileUrlOrKey.substring(1) : fileUrlOrKey;
|
||||
logger.debug(
|
||||
`[extractKeyFromS3Url] FALLBACK. fileUrlOrKey: ${fileUrlOrKey}, Extracted key: ${key}`,
|
||||
);
|
||||
return key;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves a readable stream for a file stored in S3.
|
||||
*
|
||||
* @param {ServerRequest} req - Server request object.
|
||||
* @param {string} filePath - The S3 key of the file.
|
||||
* @returns {Promise<NodeJS.ReadableStream>}
|
||||
*/
|
||||
async function getS3FileStream(_req, filePath) {
|
||||
try {
|
||||
const Key = extractKeyFromS3Url(filePath);
|
||||
const params = { Bucket: bucketName, Key };
|
||||
const s3 = initializeS3();
|
||||
const data = await s3.send(new GetObjectCommand(params));
|
||||
return data.Body; // Returns a Node.js ReadableStream.
|
||||
} catch (error) {
|
||||
logger.error('[getS3FileStream] Error retrieving S3 file stream:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines if a signed S3 URL is close to expiration
|
||||
*
|
||||
* @param {string} signedUrl - The signed S3 URL
|
||||
* @param {number} bufferSeconds - Buffer time in seconds
|
||||
* @returns {boolean} True if the URL needs refreshing
|
||||
*/
|
||||
function needsRefresh(signedUrl, bufferSeconds) {
|
||||
try {
|
||||
// Parse the URL
|
||||
const url = new URL(signedUrl);
|
||||
|
||||
// Check if it has the signature parameters that indicate it's a signed URL
|
||||
// X-Amz-Signature is the most reliable indicator for AWS signed URLs
|
||||
if (!url.searchParams.has('X-Amz-Signature')) {
|
||||
// Not a signed URL, so no expiration to check (or it's already a proxy URL)
|
||||
return false;
|
||||
}
|
||||
|
||||
// Extract the expiration time from the URL
|
||||
const expiresParam = url.searchParams.get('X-Amz-Expires');
|
||||
const dateParam = url.searchParams.get('X-Amz-Date');
|
||||
|
||||
if (!expiresParam || !dateParam) {
|
||||
// Missing expiration information, assume it needs refresh to be safe
|
||||
return true;
|
||||
}
|
||||
|
||||
// Parse the AWS date format (YYYYMMDDTHHMMSSZ)
|
||||
const year = dateParam.substring(0, 4);
|
||||
const month = dateParam.substring(4, 6);
|
||||
const day = dateParam.substring(6, 8);
|
||||
const hour = dateParam.substring(9, 11);
|
||||
const minute = dateParam.substring(11, 13);
|
||||
const second = dateParam.substring(13, 15);
|
||||
|
||||
const dateObj = new Date(`${year}-${month}-${day}T${hour}:${minute}:${second}Z`);
|
||||
const expiresAtDate = new Date(dateObj.getTime() + parseInt(expiresParam) * 1000);
|
||||
|
||||
// Check if it's close to expiration
|
||||
const now = new Date();
|
||||
|
||||
// If S3_REFRESH_EXPIRY_MS is set, use it to determine if URL is expired
|
||||
if (s3RefreshExpiryMs !== null) {
|
||||
const urlCreationTime = dateObj.getTime();
|
||||
const urlAge = now.getTime() - urlCreationTime;
|
||||
return urlAge >= s3RefreshExpiryMs;
|
||||
}
|
||||
|
||||
// Otherwise use the default buffer-based logic
|
||||
const bufferTime = new Date(now.getTime() + bufferSeconds * 1000);
|
||||
return expiresAtDate <= bufferTime;
|
||||
} catch (error) {
|
||||
logger.error('Error checking URL expiration:', error);
|
||||
// If we can't determine, assume it needs refresh to be safe
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates a new URL for an expired S3 URL
|
||||
* @param {string} currentURL - The current file URL
|
||||
* @returns {Promise<string | undefined>}
|
||||
*/
|
||||
async function getNewS3URL(currentURL) {
|
||||
try {
|
||||
const s3Key = extractKeyFromS3Url(currentURL);
|
||||
if (!s3Key) {
|
||||
return;
|
||||
}
|
||||
const keyParts = s3Key.split('/');
|
||||
if (keyParts.length < 3) {
|
||||
return;
|
||||
}
|
||||
|
||||
const basePath = keyParts[0];
|
||||
const userId = keyParts[1];
|
||||
const fileName = keyParts.slice(2).join('/');
|
||||
|
||||
return await getS3URL({
|
||||
userId,
|
||||
fileName,
|
||||
basePath,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Error getting new S3 URL:', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Refreshes S3 URLs for an array of files if they're expired or close to expiring
|
||||
*
|
||||
* @param {MongoFile[]} files - Array of file documents
|
||||
* @param {(files: MongoFile[]) => Promise<void>} batchUpdateFiles - Function to update files in the database
|
||||
* @param {number} [bufferSeconds=3600] - Buffer time in seconds to check for expiration
|
||||
* @returns {Promise<MongoFile[]>} The files with refreshed URLs if needed
|
||||
*/
|
||||
async function refreshS3FileUrls(files, batchUpdateFiles, bufferSeconds = 3600) {
|
||||
if (!files || !Array.isArray(files) || files.length === 0) {
|
||||
return files;
|
||||
}
|
||||
|
||||
const filesToUpdate = [];
|
||||
|
||||
for (let i = 0; i < files.length; i++) {
|
||||
const file = files[i];
|
||||
if (!file?.file_id) {
|
||||
continue;
|
||||
}
|
||||
if (file.source !== FileSources.s3) {
|
||||
continue;
|
||||
}
|
||||
if (!file.filepath) {
|
||||
continue;
|
||||
}
|
||||
if (!needsRefresh(file.filepath, bufferSeconds)) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const newURL = await getNewS3URL(file.filepath);
|
||||
if (!newURL) {
|
||||
continue;
|
||||
}
|
||||
filesToUpdate.push({
|
||||
file_id: file.file_id,
|
||||
filepath: newURL,
|
||||
});
|
||||
files[i].filepath = newURL;
|
||||
} catch (error) {
|
||||
logger.error(`Error refreshing S3 URL for file ${file.file_id}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
if (filesToUpdate.length > 0) {
|
||||
await batchUpdateFiles(filesToUpdate);
|
||||
}
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
/**
|
||||
* Refreshes a single S3 URL if it's expired or close to expiring
|
||||
*
|
||||
* @param {{ filepath: string, source: string }} fileObj - Simple file object containing filepath and source
|
||||
* @param {number} [bufferSeconds=3600] - Buffer time in seconds to check for expiration
|
||||
* @returns {Promise<string>} The refreshed URL or the original URL if no refresh needed
|
||||
*/
|
||||
async function refreshS3Url(fileObj, bufferSeconds = 3600) {
|
||||
if (!fileObj || fileObj.source !== FileSources.s3 || !fileObj.filepath) {
|
||||
return fileObj?.filepath || '';
|
||||
}
|
||||
|
||||
if (!needsRefresh(fileObj.filepath, bufferSeconds)) {
|
||||
return fileObj.filepath;
|
||||
}
|
||||
|
||||
try {
|
||||
const s3Key = extractKeyFromS3Url(fileObj.filepath);
|
||||
if (!s3Key) {
|
||||
logger.warn(`Unable to extract S3 key from URL: ${fileObj.filepath}`);
|
||||
return fileObj.filepath;
|
||||
}
|
||||
|
||||
const keyParts = s3Key.split('/');
|
||||
if (keyParts.length < 3) {
|
||||
logger.warn(`Invalid S3 key format: ${s3Key}`);
|
||||
return fileObj.filepath;
|
||||
}
|
||||
|
||||
const basePath = keyParts[0];
|
||||
const userId = keyParts[1];
|
||||
const fileName = keyParts.slice(2).join('/');
|
||||
|
||||
const newUrl = await getS3URL({
|
||||
userId,
|
||||
fileName,
|
||||
basePath,
|
||||
});
|
||||
|
||||
logger.debug(`Refreshed S3 URL for key: ${s3Key}`);
|
||||
return newUrl;
|
||||
} catch (error) {
|
||||
logger.error(`Error refreshing S3 URL: ${error.message}`);
|
||||
return fileObj.filepath;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
saveBufferToS3,
|
||||
saveURLToS3,
|
||||
getS3URL,
|
||||
deleteFileFromS3,
|
||||
uploadFileToS3,
|
||||
getS3FileStream,
|
||||
refreshS3FileUrls,
|
||||
refreshS3Url,
|
||||
needsRefresh,
|
||||
getNewS3URL,
|
||||
extractKeyFromS3Url,
|
||||
};
|
||||
|
|
@ -1,129 +0,0 @@
|
|||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const sharp = require('sharp');
|
||||
const { logger } = require('@librechat/data-schemas');
|
||||
const { resizeImageBuffer } = require('../images/resize');
|
||||
const { updateUser, updateFile } = require('~/models');
|
||||
const { saveBufferToS3 } = require('./crud');
|
||||
|
||||
const defaultBasePath = 'images';
|
||||
|
||||
/**
|
||||
* Resizes, converts, and uploads an image file to S3.
|
||||
*
|
||||
* @param {Object} params
|
||||
* @param {import('express').Request} params.req - Express request (expects `user` and `appConfig.imageOutputType`).
|
||||
* @param {Express.Multer.File} params.file - File object from Multer.
|
||||
* @param {string} params.file_id - Unique file identifier.
|
||||
* @param {any} params.endpoint - Endpoint identifier used in image processing.
|
||||
* @param {string} [params.resolution='high'] - Desired image resolution.
|
||||
* @param {string} [params.basePath='images'] - Base path in the bucket.
|
||||
* @returns {Promise<{ filepath: string, bytes: number, width: number, height: number }>}
|
||||
*/
|
||||
async function uploadImageToS3({
|
||||
req,
|
||||
file,
|
||||
file_id,
|
||||
endpoint,
|
||||
resolution = 'high',
|
||||
basePath = defaultBasePath,
|
||||
}) {
|
||||
try {
|
||||
const appConfig = req.config;
|
||||
const inputFilePath = file.path;
|
||||
const inputBuffer = await fs.promises.readFile(inputFilePath);
|
||||
const {
|
||||
buffer: resizedBuffer,
|
||||
width,
|
||||
height,
|
||||
} = await resizeImageBuffer(inputBuffer, resolution, endpoint);
|
||||
const extension = path.extname(inputFilePath);
|
||||
const userId = req.user.id;
|
||||
|
||||
let processedBuffer;
|
||||
let fileName = `${file_id}__${path.basename(inputFilePath)}`;
|
||||
const targetExtension = `.${appConfig.imageOutputType}`;
|
||||
|
||||
if (extension.toLowerCase() === targetExtension) {
|
||||
processedBuffer = resizedBuffer;
|
||||
} else {
|
||||
processedBuffer = await sharp(resizedBuffer).toFormat(appConfig.imageOutputType).toBuffer();
|
||||
fileName = fileName.replace(new RegExp(path.extname(fileName) + '$'), targetExtension);
|
||||
if (!path.extname(fileName)) {
|
||||
fileName += targetExtension;
|
||||
}
|
||||
}
|
||||
|
||||
const downloadURL = await saveBufferToS3({
|
||||
userId,
|
||||
buffer: processedBuffer,
|
||||
fileName,
|
||||
basePath,
|
||||
});
|
||||
await fs.promises.unlink(inputFilePath);
|
||||
const bytes = Buffer.byteLength(processedBuffer);
|
||||
return { filepath: downloadURL, bytes, width, height };
|
||||
} catch (error) {
|
||||
logger.error('[uploadImageToS3] Error uploading image to S3:', error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates a file record and returns its signed URL.
|
||||
*
|
||||
* @param {import('express').Request} req - Express request.
|
||||
* @param {Object} file - File metadata.
|
||||
* @returns {Promise<[Promise<any>, string]>}
|
||||
*/
|
||||
async function prepareImageURLS3(req, file) {
|
||||
try {
|
||||
const updatePromise = updateFile({ file_id: file.file_id });
|
||||
return Promise.all([updatePromise, file.filepath]);
|
||||
} catch (error) {
|
||||
logger.error('[prepareImageURLS3] Error preparing image URL:', error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Processes a user's avatar image by uploading it to S3 and updating the user's avatar URL if required.
|
||||
*
|
||||
* @param {Object} params
|
||||
* @param {Buffer} params.buffer - Avatar image buffer.
|
||||
* @param {string} params.userId - User's unique identifier.
|
||||
* @param {string} params.manual - 'true' or 'false' flag for manual update.
|
||||
* @param {string} [params.agentId] - Optional agent ID if this is an agent avatar.
|
||||
* @param {string} [params.basePath='images'] - Base path in the bucket.
|
||||
* @returns {Promise<string>} Signed URL of the uploaded avatar.
|
||||
*/
|
||||
async function processS3Avatar({ buffer, userId, manual, agentId, basePath = defaultBasePath }) {
|
||||
try {
|
||||
const metadata = await sharp(buffer).metadata();
|
||||
const extension = metadata.format === 'gif' ? 'gif' : 'png';
|
||||
const timestamp = new Date().getTime();
|
||||
|
||||
/** Unique filename with timestamp and optional agent ID */
|
||||
const fileName = agentId
|
||||
? `agent-${agentId}-avatar-${timestamp}.${extension}`
|
||||
: `avatar-${timestamp}.${extension}`;
|
||||
|
||||
const downloadURL = await saveBufferToS3({ userId, buffer, fileName, basePath });
|
||||
|
||||
// Only update user record if this is a user avatar (manual === 'true')
|
||||
if (manual === 'true' && !agentId) {
|
||||
await updateUser(userId, { avatar: downloadURL });
|
||||
}
|
||||
|
||||
return downloadURL;
|
||||
} catch (error) {
|
||||
logger.error('[processS3Avatar] Error processing S3 avatar:', error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
uploadImageToS3,
|
||||
prepareImageURLS3,
|
||||
processS3Avatar,
|
||||
};
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
const crud = require('./crud');
|
||||
const images = require('./images');
|
||||
|
||||
module.exports = {
|
||||
...crud,
|
||||
...images,
|
||||
};
|
||||
|
|
@ -1,6 +1,13 @@
|
|||
const { FileSources } = require('librechat-data-provider');
|
||||
const {
|
||||
getS3URL,
|
||||
saveURLToS3,
|
||||
parseDocument,
|
||||
uploadFileToS3,
|
||||
S3ImageService,
|
||||
saveBufferToS3,
|
||||
getS3FileStream,
|
||||
deleteFileFromS3,
|
||||
uploadMistralOCR,
|
||||
uploadAzureMistralOCR,
|
||||
uploadGoogleVertexMistralOCR,
|
||||
|
|
@ -27,17 +34,18 @@ const {
|
|||
processLocalAvatar,
|
||||
getLocalFileStream,
|
||||
} = require('./Local');
|
||||
const {
|
||||
getS3URL,
|
||||
saveURLToS3,
|
||||
saveBufferToS3,
|
||||
getS3FileStream,
|
||||
uploadImageToS3,
|
||||
prepareImageURLS3,
|
||||
deleteFileFromS3,
|
||||
processS3Avatar,
|
||||
uploadFileToS3,
|
||||
} = require('./S3');
|
||||
const { resizeImageBuffer } = require('./images/resize');
|
||||
const { updateUser, updateFile } = require('~/models');
|
||||
|
||||
const s3ImageService = new S3ImageService({
|
||||
resizeImageBuffer,
|
||||
updateUser,
|
||||
updateFile,
|
||||
});
|
||||
|
||||
const uploadImageToS3 = (params) => s3ImageService.uploadImageToS3(params);
|
||||
const prepareImageURLS3 = (_req, file) => s3ImageService.prepareImageURL(file);
|
||||
const processS3Avatar = (params) => s3ImageService.processAvatar(params);
|
||||
const {
|
||||
saveBufferToAzure,
|
||||
saveURLToAzure,
|
||||
|
|
|
|||
|
|
@ -1,72 +0,0 @@
|
|||
const { getS3URL } = require('../../../../../server/services/Files/S3/crud');
|
||||
|
||||
// Mock AWS SDK
|
||||
jest.mock('@aws-sdk/client-s3', () => ({
|
||||
S3Client: jest.fn(() => ({
|
||||
send: jest.fn(),
|
||||
})),
|
||||
GetObjectCommand: jest.fn(),
|
||||
}));
|
||||
|
||||
jest.mock('@aws-sdk/s3-request-presigner', () => ({
|
||||
getSignedUrl: jest.fn(),
|
||||
}));
|
||||
|
||||
jest.mock('../../../../../config', () => ({
|
||||
logger: {
|
||||
error: jest.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
const { getSignedUrl } = require('@aws-sdk/s3-request-presigner');
|
||||
const { GetObjectCommand } = require('@aws-sdk/client-s3');
|
||||
|
||||
describe('S3 crud.js - test only new parameter changes', () => {
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
process.env.AWS_BUCKET_NAME = 'test-bucket';
|
||||
});
|
||||
|
||||
// Test only the new customFilename parameter
|
||||
it('should include customFilename in response headers when provided', async () => {
|
||||
getSignedUrl.mockResolvedValue('https://test-presigned-url.com');
|
||||
|
||||
await getS3URL({
|
||||
userId: 'user123',
|
||||
fileName: 'test.pdf',
|
||||
customFilename: 'cleaned_filename.pdf',
|
||||
});
|
||||
|
||||
// Verify the new ResponseContentDisposition parameter is added to GetObjectCommand
|
||||
const commandArgs = GetObjectCommand.mock.calls[0][0];
|
||||
expect(commandArgs.ResponseContentDisposition).toBe(
|
||||
'attachment; filename="cleaned_filename.pdf"',
|
||||
);
|
||||
});
|
||||
|
||||
// Test only the new contentType parameter
|
||||
it('should include contentType in response headers when provided', async () => {
|
||||
getSignedUrl.mockResolvedValue('https://test-presigned-url.com');
|
||||
|
||||
await getS3URL({
|
||||
userId: 'user123',
|
||||
fileName: 'test.pdf',
|
||||
contentType: 'application/pdf',
|
||||
});
|
||||
|
||||
// Verify the new ResponseContentType parameter is added to GetObjectCommand
|
||||
const commandArgs = GetObjectCommand.mock.calls[0][0];
|
||||
expect(commandArgs.ResponseContentType).toBe('application/pdf');
|
||||
});
|
||||
|
||||
it('should work without new parameters (backward compatibility)', async () => {
|
||||
getSignedUrl.mockResolvedValue('https://test-presigned-url.com');
|
||||
|
||||
const result = await getS3URL({
|
||||
userId: 'user123',
|
||||
fileName: 'test.pdf',
|
||||
});
|
||||
|
||||
expect(result).toBe('https://test-presigned-url.com');
|
||||
});
|
||||
});
|
||||
|
|
@ -1,876 +0,0 @@
|
|||
const fs = require('fs');
|
||||
const fetch = require('node-fetch');
|
||||
const { Readable } = require('stream');
|
||||
const { FileSources } = require('librechat-data-provider');
|
||||
const {
|
||||
PutObjectCommand,
|
||||
GetObjectCommand,
|
||||
HeadObjectCommand,
|
||||
DeleteObjectCommand,
|
||||
} = require('@aws-sdk/client-s3');
|
||||
const { getSignedUrl } = require('@aws-sdk/s3-request-presigner');
|
||||
|
||||
// Mock dependencies
|
||||
jest.mock('fs');
|
||||
jest.mock('node-fetch');
|
||||
jest.mock('@aws-sdk/s3-request-presigner');
|
||||
jest.mock('@aws-sdk/client-s3');
|
||||
|
||||
jest.mock('@librechat/api', () => ({
|
||||
initializeS3: jest.fn(),
|
||||
deleteRagFile: jest.fn().mockResolvedValue(undefined),
|
||||
isEnabled: jest.fn((val) => val === 'true'),
|
||||
}));
|
||||
|
||||
jest.mock('@librechat/data-schemas', () => ({
|
||||
logger: {
|
||||
debug: jest.fn(),
|
||||
info: jest.fn(),
|
||||
warn: jest.fn(),
|
||||
error: jest.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
const { initializeS3, deleteRagFile } = require('@librechat/api');
|
||||
const { logger } = require('@librechat/data-schemas');
|
||||
|
||||
// Set env vars before requiring crud so module-level constants pick them up
|
||||
process.env.AWS_BUCKET_NAME = 'test-bucket';
|
||||
process.env.S3_URL_EXPIRY_SECONDS = '120';
|
||||
|
||||
const {
|
||||
saveBufferToS3,
|
||||
saveURLToS3,
|
||||
getS3URL,
|
||||
deleteFileFromS3,
|
||||
uploadFileToS3,
|
||||
getS3FileStream,
|
||||
refreshS3FileUrls,
|
||||
refreshS3Url,
|
||||
needsRefresh,
|
||||
getNewS3URL,
|
||||
extractKeyFromS3Url,
|
||||
} = require('~/server/services/Files/S3/crud');
|
||||
|
||||
describe('S3 CRUD Operations', () => {
|
||||
let mockS3Client;
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
|
||||
// Setup mock S3 client
|
||||
mockS3Client = {
|
||||
send: jest.fn(),
|
||||
};
|
||||
initializeS3.mockReturnValue(mockS3Client);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
delete process.env.S3_URL_EXPIRY_SECONDS;
|
||||
delete process.env.S3_REFRESH_EXPIRY_MS;
|
||||
delete process.env.AWS_BUCKET_NAME;
|
||||
});
|
||||
|
||||
describe('saveBufferToS3', () => {
|
||||
it('should upload a buffer to S3 and return a signed URL', async () => {
|
||||
const mockBuffer = Buffer.from('test data');
|
||||
const mockSignedUrl =
|
||||
'https://s3.amazonaws.com/test-bucket/images/user123/test.jpg?signature=abc';
|
||||
|
||||
mockS3Client.send.mockResolvedValue({});
|
||||
getSignedUrl.mockResolvedValue(mockSignedUrl);
|
||||
|
||||
const result = await saveBufferToS3({
|
||||
userId: 'user123',
|
||||
buffer: mockBuffer,
|
||||
fileName: 'test.jpg',
|
||||
basePath: 'images',
|
||||
});
|
||||
|
||||
expect(mockS3Client.send).toHaveBeenCalledWith(expect.any(PutObjectCommand));
|
||||
expect(result).toBe(mockSignedUrl);
|
||||
});
|
||||
|
||||
it('should use default basePath if not provided', async () => {
|
||||
const mockBuffer = Buffer.from('test data');
|
||||
const mockSignedUrl =
|
||||
'https://s3.amazonaws.com/test-bucket/images/user123/test.jpg?signature=abc';
|
||||
|
||||
mockS3Client.send.mockResolvedValue({});
|
||||
getSignedUrl.mockResolvedValue(mockSignedUrl);
|
||||
|
||||
await saveBufferToS3({
|
||||
userId: 'user123',
|
||||
buffer: mockBuffer,
|
||||
fileName: 'test.jpg',
|
||||
});
|
||||
|
||||
expect(getSignedUrl).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should handle S3 upload errors', async () => {
|
||||
const mockBuffer = Buffer.from('test data');
|
||||
const error = new Error('S3 upload failed');
|
||||
|
||||
mockS3Client.send.mockRejectedValue(error);
|
||||
|
||||
await expect(
|
||||
saveBufferToS3({
|
||||
userId: 'user123',
|
||||
buffer: mockBuffer,
|
||||
fileName: 'test.jpg',
|
||||
}),
|
||||
).rejects.toThrow('S3 upload failed');
|
||||
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
'[saveBufferToS3] Error uploading buffer to S3:',
|
||||
'S3 upload failed',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getS3URL', () => {
|
||||
it('should return a signed URL for a file', async () => {
|
||||
const mockSignedUrl =
|
||||
'https://s3.amazonaws.com/test-bucket/images/user123/file.pdf?signature=xyz';
|
||||
getSignedUrl.mockResolvedValue(mockSignedUrl);
|
||||
|
||||
const result = await getS3URL({
|
||||
userId: 'user123',
|
||||
fileName: 'file.pdf',
|
||||
basePath: 'documents',
|
||||
});
|
||||
|
||||
expect(result).toBe(mockSignedUrl);
|
||||
expect(getSignedUrl).toHaveBeenCalledWith(
|
||||
mockS3Client,
|
||||
expect.any(GetObjectCommand),
|
||||
expect.objectContaining({ expiresIn: 120 }),
|
||||
);
|
||||
});
|
||||
|
||||
it('should add custom filename to Content-Disposition header', async () => {
|
||||
const mockSignedUrl =
|
||||
'https://s3.amazonaws.com/test-bucket/images/user123/file.pdf?signature=xyz';
|
||||
getSignedUrl.mockResolvedValue(mockSignedUrl);
|
||||
|
||||
await getS3URL({
|
||||
userId: 'user123',
|
||||
fileName: 'file.pdf',
|
||||
customFilename: 'custom-name.pdf',
|
||||
});
|
||||
|
||||
expect(getSignedUrl).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should add custom content type', async () => {
|
||||
const mockSignedUrl =
|
||||
'https://s3.amazonaws.com/test-bucket/images/user123/file.pdf?signature=xyz';
|
||||
getSignedUrl.mockResolvedValue(mockSignedUrl);
|
||||
|
||||
await getS3URL({
|
||||
userId: 'user123',
|
||||
fileName: 'file.pdf',
|
||||
contentType: 'application/pdf',
|
||||
});
|
||||
|
||||
expect(getSignedUrl).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should handle errors when getting signed URL', async () => {
|
||||
const error = new Error('Failed to sign URL');
|
||||
getSignedUrl.mockRejectedValue(error);
|
||||
|
||||
await expect(
|
||||
getS3URL({
|
||||
userId: 'user123',
|
||||
fileName: 'file.pdf',
|
||||
}),
|
||||
).rejects.toThrow('Failed to sign URL');
|
||||
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
'[getS3URL] Error getting signed URL from S3:',
|
||||
'Failed to sign URL',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('saveURLToS3', () => {
|
||||
it('should fetch a file from URL and save to S3', async () => {
|
||||
const mockBuffer = Buffer.from('downloaded data');
|
||||
const mockResponse = {
|
||||
buffer: jest.fn().mockResolvedValue(mockBuffer),
|
||||
};
|
||||
const mockSignedUrl =
|
||||
'https://s3.amazonaws.com/test-bucket/images/user123/downloaded.jpg?signature=abc';
|
||||
|
||||
fetch.mockResolvedValue(mockResponse);
|
||||
mockS3Client.send.mockResolvedValue({});
|
||||
getSignedUrl.mockResolvedValue(mockSignedUrl);
|
||||
|
||||
const result = await saveURLToS3({
|
||||
userId: 'user123',
|
||||
URL: 'https://example.com/image.jpg',
|
||||
fileName: 'downloaded.jpg',
|
||||
});
|
||||
|
||||
expect(fetch).toHaveBeenCalledWith('https://example.com/image.jpg');
|
||||
expect(mockS3Client.send).toHaveBeenCalled();
|
||||
expect(result).toBe(mockSignedUrl);
|
||||
});
|
||||
|
||||
it('should handle fetch errors', async () => {
|
||||
const error = new Error('Network error');
|
||||
fetch.mockRejectedValue(error);
|
||||
|
||||
await expect(
|
||||
saveURLToS3({
|
||||
userId: 'user123',
|
||||
URL: 'https://example.com/image.jpg',
|
||||
fileName: 'downloaded.jpg',
|
||||
}),
|
||||
).rejects.toThrow('Network error');
|
||||
|
||||
expect(logger.error).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('deleteFileFromS3', () => {
|
||||
const mockReq = {
|
||||
user: { id: 'user123' },
|
||||
};
|
||||
|
||||
it('should delete a file from S3', async () => {
|
||||
const mockFile = {
|
||||
filepath: 'https://s3.amazonaws.com/test-bucket/images/user123/file.jpg',
|
||||
file_id: 'file123',
|
||||
};
|
||||
|
||||
// Mock HeadObject to verify file exists
|
||||
mockS3Client.send
|
||||
.mockResolvedValueOnce({}) // First HeadObject - exists
|
||||
.mockResolvedValueOnce({}) // DeleteObject
|
||||
.mockRejectedValueOnce({ name: 'NotFound' }); // Second HeadObject - deleted
|
||||
|
||||
await deleteFileFromS3(mockReq, mockFile);
|
||||
|
||||
expect(deleteRagFile).toHaveBeenCalledWith({ userId: 'user123', file: mockFile });
|
||||
expect(mockS3Client.send).toHaveBeenCalledWith(expect.any(HeadObjectCommand));
|
||||
expect(mockS3Client.send).toHaveBeenCalledWith(expect.any(DeleteObjectCommand));
|
||||
});
|
||||
|
||||
it('should handle file not found gracefully', async () => {
|
||||
const mockFile = {
|
||||
filepath: 'https://s3.amazonaws.com/test-bucket/images/user123/nonexistent.jpg',
|
||||
file_id: 'file123',
|
||||
};
|
||||
|
||||
mockS3Client.send.mockRejectedValue({ name: 'NotFound' });
|
||||
|
||||
await deleteFileFromS3(mockReq, mockFile);
|
||||
|
||||
expect(logger.warn).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should throw error if user ID does not match', async () => {
|
||||
const mockFile = {
|
||||
filepath: 'https://s3.amazonaws.com/test-bucket/images/different-user/file.jpg',
|
||||
file_id: 'file123',
|
||||
};
|
||||
|
||||
await expect(deleteFileFromS3(mockReq, mockFile)).rejects.toThrow('User ID mismatch');
|
||||
expect(logger.error).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should handle NoSuchKey error', async () => {
|
||||
const mockFile = {
|
||||
filepath: 'https://s3.amazonaws.com/test-bucket/images/user123/file.jpg',
|
||||
file_id: 'file123',
|
||||
};
|
||||
|
||||
mockS3Client.send
|
||||
.mockResolvedValueOnce({}) // HeadObject - exists
|
||||
.mockRejectedValueOnce({ code: 'NoSuchKey' }); // DeleteObject fails
|
||||
|
||||
await deleteFileFromS3(mockReq, mockFile);
|
||||
|
||||
expect(logger.debug).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('uploadFileToS3', () => {
|
||||
const mockReq = {
|
||||
user: { id: 'user123' },
|
||||
};
|
||||
|
||||
it('should upload a file from disk to S3', async () => {
|
||||
const mockFile = {
|
||||
path: '/tmp/upload.jpg',
|
||||
originalname: 'photo.jpg',
|
||||
};
|
||||
const mockStats = { size: 1024 };
|
||||
const mockSignedUrl =
|
||||
'https://s3.amazonaws.com/test-bucket/images/user123/file123__photo.jpg?signature=xyz';
|
||||
|
||||
fs.promises = { stat: jest.fn().mockResolvedValue(mockStats) };
|
||||
fs.createReadStream = jest.fn().mockReturnValue(new Readable());
|
||||
mockS3Client.send.mockResolvedValue({});
|
||||
getSignedUrl.mockResolvedValue(mockSignedUrl);
|
||||
|
||||
const result = await uploadFileToS3({
|
||||
req: mockReq,
|
||||
file: mockFile,
|
||||
file_id: 'file123',
|
||||
basePath: 'images',
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
filepath: mockSignedUrl,
|
||||
bytes: 1024,
|
||||
});
|
||||
expect(fs.createReadStream).toHaveBeenCalledWith('/tmp/upload.jpg');
|
||||
expect(mockS3Client.send).toHaveBeenCalledWith(expect.any(PutObjectCommand));
|
||||
});
|
||||
|
||||
it('should handle upload errors and clean up temp file', async () => {
|
||||
const mockFile = {
|
||||
path: '/tmp/upload.jpg',
|
||||
originalname: 'photo.jpg',
|
||||
};
|
||||
const error = new Error('Upload failed');
|
||||
|
||||
fs.promises = {
|
||||
stat: jest.fn().mockResolvedValue({ size: 1024 }),
|
||||
unlink: jest.fn().mockResolvedValue(),
|
||||
};
|
||||
fs.createReadStream = jest.fn().mockReturnValue(new Readable());
|
||||
mockS3Client.send.mockRejectedValue(error);
|
||||
|
||||
await expect(
|
||||
uploadFileToS3({
|
||||
req: mockReq,
|
||||
file: mockFile,
|
||||
file_id: 'file123',
|
||||
}),
|
||||
).rejects.toThrow('Upload failed');
|
||||
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
'[uploadFileToS3] Error streaming file to S3:',
|
||||
error,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getS3FileStream', () => {
|
||||
it('should return a readable stream for a file', async () => {
|
||||
const mockStream = new Readable();
|
||||
const mockResponse = { Body: mockStream };
|
||||
|
||||
mockS3Client.send.mockResolvedValue(mockResponse);
|
||||
|
||||
const result = await getS3FileStream(
|
||||
{},
|
||||
'https://s3.amazonaws.com/test-bucket/images/user123/file.pdf',
|
||||
);
|
||||
|
||||
expect(result).toBe(mockStream);
|
||||
expect(mockS3Client.send).toHaveBeenCalledWith(expect.any(GetObjectCommand));
|
||||
});
|
||||
|
||||
it('should handle errors when retrieving stream', async () => {
|
||||
const error = new Error('Stream error');
|
||||
mockS3Client.send.mockRejectedValue(error);
|
||||
|
||||
await expect(getS3FileStream({}, 'images/user123/file.pdf')).rejects.toThrow('Stream error');
|
||||
expect(logger.error).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('needsRefresh', () => {
|
||||
it('should return false for non-signed URLs', () => {
|
||||
const url = 'https://example.com/proxy/file.jpg';
|
||||
const result = needsRefresh(url, 3600);
|
||||
expect(result).toBe(false);
|
||||
});
|
||||
|
||||
it('should return true for expired signed URLs', () => {
|
||||
const now = new Date();
|
||||
const past = new Date(now.getTime() - 3600 * 1000); // 1 hour ago
|
||||
const dateStr = past
|
||||
.toISOString()
|
||||
.replace(/[-:]/g, '')
|
||||
.replace(/\.\d{3}/, '');
|
||||
|
||||
const url = `https://s3.amazonaws.com/bucket/key?X-Amz-Signature=abc&X-Amz-Date=${dateStr}&X-Amz-Expires=60`;
|
||||
const result = needsRefresh(url, 60);
|
||||
expect(result).toBe(true);
|
||||
});
|
||||
|
||||
it('should return false for URLs that are not close to expiration', () => {
|
||||
const now = new Date();
|
||||
const recent = new Date(now.getTime() - 10 * 1000); // 10 seconds ago
|
||||
const dateStr = recent
|
||||
.toISOString()
|
||||
.replace(/[-:]/g, '')
|
||||
.replace(/\.\d{3}/, '');
|
||||
|
||||
const url = `https://s3.amazonaws.com/bucket/key?X-Amz-Signature=abc&X-Amz-Date=${dateStr}&X-Amz-Expires=7200`;
|
||||
const result = needsRefresh(url, 60);
|
||||
expect(result).toBe(false);
|
||||
});
|
||||
|
||||
it('should use custom refresh expiry when S3_REFRESH_EXPIRY_MS is set', () => {
|
||||
process.env.S3_REFRESH_EXPIRY_MS = '30000'; // 30 seconds
|
||||
|
||||
const now = new Date();
|
||||
const recent = new Date(now.getTime() - 31 * 1000); // 31 seconds ago
|
||||
const dateStr = recent
|
||||
.toISOString()
|
||||
.replace(/[-:]/g, '')
|
||||
.replace(/\.\d{3}/, '');
|
||||
|
||||
const url = `https://s3.amazonaws.com/bucket/key?X-Amz-Signature=abc&X-Amz-Date=${dateStr}&X-Amz-Expires=7200`;
|
||||
|
||||
// Need to reload the module to pick up the env var change
|
||||
jest.resetModules();
|
||||
const { needsRefresh: needsRefreshReloaded } = require('~/server/services/Files/S3/crud');
|
||||
|
||||
const result = needsRefreshReloaded(url, 60);
|
||||
expect(result).toBe(true);
|
||||
});
|
||||
|
||||
it('should return true for malformed URLs', () => {
|
||||
const url = 'not-a-valid-url';
|
||||
const result = needsRefresh(url, 3600);
|
||||
expect(result).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getNewS3URL', () => {
|
||||
it('should generate a new URL from an existing S3 URL', async () => {
|
||||
const currentURL =
|
||||
'https://s3.amazonaws.com/test-bucket/images/user123/file.jpg?signature=old';
|
||||
const newURL = 'https://s3.amazonaws.com/test-bucket/images/user123/file.jpg?signature=new';
|
||||
|
||||
getSignedUrl.mockResolvedValue(newURL);
|
||||
|
||||
const result = await getNewS3URL(currentURL);
|
||||
|
||||
expect(result).toBe(newURL);
|
||||
expect(getSignedUrl).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should return undefined for invalid URLs', async () => {
|
||||
const result = await getNewS3URL('invalid-url');
|
||||
expect(result).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should handle errors gracefully', async () => {
|
||||
const currentURL = 'https://s3.amazonaws.com/test-bucket/images/user123/file.jpg';
|
||||
getSignedUrl.mockRejectedValue(new Error('Failed'));
|
||||
|
||||
const result = await getNewS3URL(currentURL);
|
||||
|
||||
expect(result).toBeUndefined();
|
||||
expect(logger.error).toHaveBeenCalledWith('Error getting new S3 URL:', expect.any(Error));
|
||||
});
|
||||
|
||||
it('should construct GetObjectCommand with correct key (no bucket name duplication)', async () => {
|
||||
const currentURL =
|
||||
'https://s3.amazonaws.com/my-bucket/images/user123/file.jpg?X-Amz-Signature=old';
|
||||
getSignedUrl.mockResolvedValue(
|
||||
'https://s3.amazonaws.com/test-bucket/images/user123/file.jpg?signature=new',
|
||||
);
|
||||
|
||||
await getNewS3URL(currentURL);
|
||||
|
||||
expect(GetObjectCommand).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ Key: 'images/user123/file.jpg' }),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('refreshS3FileUrls', () => {
|
||||
it('should refresh expired URLs for multiple files', async () => {
|
||||
const now = new Date();
|
||||
const past = new Date(now.getTime() - 3600 * 1000);
|
||||
const dateStr = past
|
||||
.toISOString()
|
||||
.replace(/[-:]/g, '')
|
||||
.replace(/\.\d{3}/, '');
|
||||
|
||||
const files = [
|
||||
{
|
||||
file_id: 'file1',
|
||||
source: FileSources.s3,
|
||||
filepath: `https://s3.amazonaws.com/bucket/images/user123/file1.jpg?X-Amz-Signature=abc&X-Amz-Date=${dateStr}&X-Amz-Expires=60`,
|
||||
},
|
||||
{
|
||||
file_id: 'file2',
|
||||
source: FileSources.s3,
|
||||
filepath: `https://s3.amazonaws.com/bucket/images/user123/file2.jpg?X-Amz-Signature=def&X-Amz-Date=${dateStr}&X-Amz-Expires=60`,
|
||||
},
|
||||
];
|
||||
|
||||
const newURL1 = 'https://s3.amazonaws.com/bucket/images/user123/file1.jpg?signature=new1';
|
||||
const newURL2 = 'https://s3.amazonaws.com/bucket/images/user123/file2.jpg?signature=new2';
|
||||
|
||||
getSignedUrl.mockResolvedValueOnce(newURL1).mockResolvedValueOnce(newURL2);
|
||||
|
||||
const mockBatchUpdate = jest.fn().mockResolvedValue();
|
||||
|
||||
const result = await refreshS3FileUrls(files, mockBatchUpdate, 60);
|
||||
|
||||
expect(result[0].filepath).toBe(newURL1);
|
||||
expect(result[1].filepath).toBe(newURL2);
|
||||
expect(mockBatchUpdate).toHaveBeenCalledWith([
|
||||
{ file_id: 'file1', filepath: newURL1 },
|
||||
{ file_id: 'file2', filepath: newURL2 },
|
||||
]);
|
||||
});
|
||||
|
||||
it('should skip non-S3 files', async () => {
|
||||
const files = [
|
||||
{
|
||||
file_id: 'file1',
|
||||
source: 'local',
|
||||
filepath: '/local/path/file.jpg',
|
||||
},
|
||||
];
|
||||
|
||||
const mockBatchUpdate = jest.fn();
|
||||
|
||||
const result = await refreshS3FileUrls(files, mockBatchUpdate);
|
||||
|
||||
expect(result).toEqual(files);
|
||||
expect(mockBatchUpdate).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should handle empty or invalid input', async () => {
|
||||
const mockBatchUpdate = jest.fn();
|
||||
|
||||
const result1 = await refreshS3FileUrls(null, mockBatchUpdate);
|
||||
expect(result1).toBe(null);
|
||||
|
||||
const result2 = await refreshS3FileUrls([], mockBatchUpdate);
|
||||
expect(result2).toEqual([]);
|
||||
|
||||
expect(mockBatchUpdate).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should handle errors for individual files gracefully', async () => {
|
||||
const now = new Date();
|
||||
const past = new Date(now.getTime() - 3600 * 1000);
|
||||
const dateStr = past
|
||||
.toISOString()
|
||||
.replace(/[-:]/g, '')
|
||||
.replace(/\.\d{3}/, '');
|
||||
|
||||
const files = [
|
||||
{
|
||||
file_id: 'file1',
|
||||
source: FileSources.s3,
|
||||
filepath: `https://s3.amazonaws.com/bucket/images/user123/file1.jpg?X-Amz-Signature=abc&X-Amz-Date=${dateStr}&X-Amz-Expires=60`,
|
||||
},
|
||||
];
|
||||
|
||||
getSignedUrl.mockRejectedValue(new Error('Failed to refresh'));
|
||||
const mockBatchUpdate = jest.fn();
|
||||
|
||||
await refreshS3FileUrls(files, mockBatchUpdate, 60);
|
||||
|
||||
expect(logger.error).toHaveBeenCalledWith('Error getting new S3 URL:', expect.any(Error));
|
||||
expect(mockBatchUpdate).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('refreshS3Url', () => {
|
||||
it('should refresh an expired S3 URL', async () => {
|
||||
const now = new Date();
|
||||
const past = new Date(now.getTime() - 3600 * 1000);
|
||||
const dateStr = past
|
||||
.toISOString()
|
||||
.replace(/[-:]/g, '')
|
||||
.replace(/\.\d{3}/, '');
|
||||
|
||||
const fileObj = {
|
||||
source: FileSources.s3,
|
||||
filepath: `https://s3.amazonaws.com/bucket/images/user123/file.jpg?X-Amz-Signature=abc&X-Amz-Date=${dateStr}&X-Amz-Expires=60`,
|
||||
};
|
||||
|
||||
const newURL = 'https://s3.amazonaws.com/bucket/images/user123/file.jpg?signature=new';
|
||||
getSignedUrl.mockResolvedValue(newURL);
|
||||
|
||||
const result = await refreshS3Url(fileObj, 60);
|
||||
|
||||
expect(result).toBe(newURL);
|
||||
});
|
||||
|
||||
it('should return original URL if not expired', async () => {
|
||||
const fileObj = {
|
||||
source: FileSources.s3,
|
||||
filepath: 'https://example.com/proxy/file.jpg',
|
||||
};
|
||||
|
||||
const result = await refreshS3Url(fileObj, 3600);
|
||||
|
||||
expect(result).toBe(fileObj.filepath);
|
||||
expect(getSignedUrl).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should return empty string for null input', async () => {
|
||||
const result = await refreshS3Url(null);
|
||||
expect(result).toBe('');
|
||||
});
|
||||
|
||||
it('should return original URL for non-S3 files', async () => {
|
||||
const fileObj = {
|
||||
source: 'local',
|
||||
filepath: '/local/path/file.jpg',
|
||||
};
|
||||
|
||||
const result = await refreshS3Url(fileObj);
|
||||
|
||||
expect(result).toBe(fileObj.filepath);
|
||||
});
|
||||
|
||||
it('should handle errors and return original URL', async () => {
|
||||
const now = new Date();
|
||||
const past = new Date(now.getTime() - 3600 * 1000);
|
||||
const dateStr = past
|
||||
.toISOString()
|
||||
.replace(/[-:]/g, '')
|
||||
.replace(/\.\d{3}/, '');
|
||||
|
||||
const fileObj = {
|
||||
source: FileSources.s3,
|
||||
filepath: `https://s3.amazonaws.com/bucket/images/user123/file.jpg?X-Amz-Signature=abc&X-Amz-Date=${dateStr}&X-Amz-Expires=60`,
|
||||
};
|
||||
|
||||
getSignedUrl.mockRejectedValue(new Error('Refresh failed'));
|
||||
|
||||
const result = await refreshS3Url(fileObj, 60);
|
||||
|
||||
expect(result).toBe(fileObj.filepath);
|
||||
expect(logger.error).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractKeyFromS3Url', () => {
|
||||
it('should extract key from a full S3 URL', () => {
|
||||
const url = 'https://s3.amazonaws.com/test-bucket/images/user123/file.jpg';
|
||||
const result = extractKeyFromS3Url(url);
|
||||
expect(result).toBe('images/user123/file.jpg');
|
||||
});
|
||||
|
||||
it('should extract key from a signed S3 URL with query parameters', () => {
|
||||
const url =
|
||||
'https://s3.amazonaws.com/test-bucket/documents/user456/report.pdf?X-Amz-Signature=abc123&X-Amz-Date=20260107';
|
||||
const result = extractKeyFromS3Url(url);
|
||||
expect(result).toBe('documents/user456/report.pdf');
|
||||
});
|
||||
|
||||
it('should extract key from S3 URL with different domain format', () => {
|
||||
const url = 'https://test-bucket.s3.amazonaws.com/uploads/user789/image.png';
|
||||
const result = extractKeyFromS3Url(url);
|
||||
expect(result).toBe('uploads/user789/image.png');
|
||||
});
|
||||
|
||||
it('should return key as-is if already properly formatted (3+ parts, no http)', () => {
|
||||
const key = 'images/user123/file.jpg';
|
||||
const result = extractKeyFromS3Url(key);
|
||||
expect(result).toBe('images/user123/file.jpg');
|
||||
});
|
||||
|
||||
it('should handle key with leading slash by removing it', () => {
|
||||
const key = '/images/user123/file.jpg';
|
||||
const result = extractKeyFromS3Url(key);
|
||||
expect(result).toBe('images/user123/file.jpg');
|
||||
});
|
||||
|
||||
it('should handle simple key without slashes', () => {
|
||||
const key = 'simple-file.txt';
|
||||
const result = extractKeyFromS3Url(key);
|
||||
expect(result).toBe('simple-file.txt');
|
||||
});
|
||||
|
||||
it('should handle key with only two parts', () => {
|
||||
const key = 'folder/file.txt';
|
||||
const result = extractKeyFromS3Url(key);
|
||||
expect(result).toBe('folder/file.txt');
|
||||
});
|
||||
|
||||
it('should throw error for empty input', () => {
|
||||
expect(() => extractKeyFromS3Url('')).toThrow('Invalid input: URL or key is empty');
|
||||
});
|
||||
|
||||
it('should throw error for null input', () => {
|
||||
expect(() => extractKeyFromS3Url(null)).toThrow('Invalid input: URL or key is empty');
|
||||
});
|
||||
|
||||
it('should throw error for undefined input', () => {
|
||||
expect(() => extractKeyFromS3Url(undefined)).toThrow('Invalid input: URL or key is empty');
|
||||
});
|
||||
|
||||
it('should handle URLs with encoded characters', () => {
|
||||
const url = 'https://s3.amazonaws.com/test-bucket/images/user123/my%20file%20name.jpg';
|
||||
const result = extractKeyFromS3Url(url);
|
||||
expect(result).toBe('images/user123/my%20file%20name.jpg');
|
||||
});
|
||||
|
||||
it('should handle deep nested paths', () => {
|
||||
const url = 'https://s3.amazonaws.com/bucket/a/b/c/d/e/f/file.jpg';
|
||||
const result = extractKeyFromS3Url(url);
|
||||
expect(result).toBe('a/b/c/d/e/f/file.jpg');
|
||||
});
|
||||
|
||||
it('should log debug message when extracting from URL', () => {
|
||||
const url = 'https://s3.amazonaws.com/bucket/images/user123/file.jpg';
|
||||
extractKeyFromS3Url(url);
|
||||
expect(logger.debug).toHaveBeenCalledWith(
|
||||
expect.stringContaining('[extractKeyFromS3Url] fileUrlOrKey:'),
|
||||
);
|
||||
});
|
||||
|
||||
it('should log fallback debug message for non-URL input', () => {
|
||||
const key = 'simple-file.txt';
|
||||
extractKeyFromS3Url(key);
|
||||
expect(logger.debug).toHaveBeenCalledWith(
|
||||
expect.stringContaining('[extractKeyFromS3Url] FALLBACK'),
|
||||
);
|
||||
});
|
||||
|
||||
it('should handle valid URLs that contain only a bucket', () => {
|
||||
const url = 'https://s3.amazonaws.com/test-bucket/';
|
||||
const result = extractKeyFromS3Url(url);
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining(
|
||||
'[extractKeyFromS3Url] Extracted key is empty after removing bucket name from URL: https://s3.amazonaws.com/test-bucket/',
|
||||
),
|
||||
);
|
||||
expect(result).toBe('');
|
||||
});
|
||||
|
||||
it('should handle invalid URLs that contain only a bucket', () => {
|
||||
const url = 'https://s3.amazonaws.com/test-bucket';
|
||||
const result = extractKeyFromS3Url(url);
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining(
|
||||
'[extractKeyFromS3Url] Unable to extract key from path-style URL: https://s3.amazonaws.com/test-bucket',
|
||||
),
|
||||
);
|
||||
expect(result).toBe('');
|
||||
});
|
||||
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html
|
||||
|
||||
// Path-style requests
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#path-style-access
|
||||
// https://s3.region-code.amazonaws.com/bucket-name/key-name
|
||||
it('should handle formatted according to Path-style regional endpoint', () => {
|
||||
const url = 'https://s3.us-west-2.amazonaws.com/amzn-s3-demo-bucket1/dogs/puppy.jpg';
|
||||
const result = extractKeyFromS3Url(url);
|
||||
expect(result).toBe('dogs/puppy.jpg');
|
||||
});
|
||||
|
||||
// virtual host style
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#virtual-hosted-style-access
|
||||
// https://bucket-name.s3.region-code.amazonaws.com/key-name
|
||||
it('should handle formatted according to Virtual-hosted–style Regional endpoint', () => {
|
||||
const url = 'https://amzn-s3-demo-bucket1.s3.us-west-2.amazonaws.com/dogs/puppy.png';
|
||||
const result = extractKeyFromS3Url(url);
|
||||
expect(result).toBe('dogs/puppy.png');
|
||||
});
|
||||
|
||||
// Legacy endpoints
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#VirtualHostingBackwardsCompatibility
|
||||
|
||||
// s3‐Region
|
||||
// https://bucket-name.s3-region-code.amazonaws.com
|
||||
it('should handle formatted according to s3‐Region', () => {
|
||||
const url = 'https://amzn-s3-demo-bucket1.s3-us-west-2.amazonaws.com/puppy.png';
|
||||
const result = extractKeyFromS3Url(url);
|
||||
expect(result).toBe('puppy.png');
|
||||
|
||||
const testcase2 = 'https://amzn-s3-demo-bucket1.s3-us-west-2.amazonaws.com/cats/kitten.png';
|
||||
const result2 = extractKeyFromS3Url(testcase2);
|
||||
expect(result2).toBe('cats/kitten.png');
|
||||
});
|
||||
|
||||
// Legacy global endpoint
|
||||
// bucket-name.s3.amazonaws.com
|
||||
it('should handle formatted according to Legacy global endpoint', () => {
|
||||
const url = 'https://amzn-s3-demo-bucket1.s3.amazonaws.com/dogs/puppy.png';
|
||||
const result = extractKeyFromS3Url(url);
|
||||
expect(result).toBe('dogs/puppy.png');
|
||||
});
|
||||
|
||||
it('should handle malformed URL and log error', () => {
|
||||
const malformedUrl = 'https://invalid url with spaces.com/key';
|
||||
const result = extractKeyFromS3Url(malformedUrl);
|
||||
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
expect.stringContaining('[extractKeyFromS3Url] Error parsing URL:'),
|
||||
);
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.stringContaining(malformedUrl));
|
||||
|
||||
expect(result).toBe(malformedUrl);
|
||||
});
|
||||
|
||||
it('should return empty string for regional path-style URL with only bucket (no key)', () => {
|
||||
const url = 'https://s3.us-west-2.amazonaws.com/my-bucket';
|
||||
const result = extractKeyFromS3Url(url);
|
||||
expect(result).toBe('');
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining('[extractKeyFromS3Url] Unable to extract key from path-style URL:'),
|
||||
);
|
||||
});
|
||||
|
||||
it('should not log error when given a plain S3 key (non-URL input)', () => {
|
||||
extractKeyFromS3Url('images/user123/file.jpg');
|
||||
expect(logger.error).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should strip bucket from custom endpoint URLs (MinIO, R2, etc.) using bucketName', () => {
|
||||
// bucketName is the module-level const 'test-bucket', set before require at top of file
|
||||
expect(
|
||||
extractKeyFromS3Url('https://minio.example.com/test-bucket/images/user123/file.jpg'),
|
||||
).toBe('images/user123/file.jpg');
|
||||
expect(
|
||||
extractKeyFromS3Url(
|
||||
'https://abc123.r2.cloudflarestorage.com/test-bucket/images/user123/avatar.png',
|
||||
),
|
||||
).toBe('images/user123/avatar.png');
|
||||
});
|
||||
|
||||
it('should use endpoint base path when AWS_ENDPOINT_URL and AWS_FORCE_PATH_STYLE are set', () => {
|
||||
process.env.AWS_BUCKET_NAME = 'test-bucket';
|
||||
process.env.AWS_ENDPOINT_URL = 'https://minio.example.com';
|
||||
process.env.AWS_FORCE_PATH_STYLE = 'true';
|
||||
jest.resetModules();
|
||||
const { extractKeyFromS3Url: fn } = require('~/server/services/Files/S3/crud');
|
||||
|
||||
expect(fn('https://minio.example.com/test-bucket/images/user123/file.jpg')).toBe(
|
||||
'images/user123/file.jpg',
|
||||
);
|
||||
|
||||
delete process.env.AWS_ENDPOINT_URL;
|
||||
delete process.env.AWS_FORCE_PATH_STYLE;
|
||||
});
|
||||
|
||||
it('should handle endpoint with a base path', () => {
|
||||
process.env.AWS_BUCKET_NAME = 'test-bucket';
|
||||
process.env.AWS_ENDPOINT_URL = 'https://example.com/storage/';
|
||||
process.env.AWS_FORCE_PATH_STYLE = 'true';
|
||||
jest.resetModules();
|
||||
const { extractKeyFromS3Url: fn } = require('~/server/services/Files/S3/crud');
|
||||
|
||||
expect(fn('https://example.com/storage/test-bucket/images/user123/file.jpg')).toBe(
|
||||
'images/user123/file.jpg',
|
||||
);
|
||||
|
||||
delete process.env.AWS_ENDPOINT_URL;
|
||||
delete process.env.AWS_FORCE_PATH_STYLE;
|
||||
});
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue