diff --git a/.env.example b/.env.example index 50229b1997..b7ec3a3dad 100644 --- a/.env.example +++ b/.env.example @@ -658,6 +658,9 @@ AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_REGION= AWS_BUCKET_NAME= +# Required for path-style S3-compatible providers (MinIO, Hetzner, Backblaze B2, etc.) +# that don't support virtual-hosted-style URLs (bucket.endpoint). Not needed for AWS S3. +# AWS_FORCE_PATH_STYLE=false #========================# # Azure Blob Storage # diff --git a/api/server/services/Files/S3/crud.js b/api/server/services/Files/S3/crud.js index efd2d7734b..c821c0696c 100644 --- a/api/server/services/Files/S3/crud.js +++ b/api/server/services/Files/S3/crud.js @@ -3,7 +3,7 @@ const fetch = require('node-fetch'); const { logger } = require('@librechat/data-schemas'); const { FileSources } = require('librechat-data-provider'); const { getSignedUrl } = require('@aws-sdk/s3-request-presigner'); -const { initializeS3, deleteRagFile } = require('@librechat/api'); +const { initializeS3, deleteRagFile, isEnabled } = require('@librechat/api'); const { PutObjectCommand, GetObjectCommand, @@ -13,6 +13,8 @@ const { const bucketName = process.env.AWS_BUCKET_NAME; const defaultBasePath = 'images'; +const endpoint = process.env.AWS_ENDPOINT_URL; +const forcePathStyle = isEnabled(process.env.AWS_FORCE_PATH_STYLE); let s3UrlExpirySeconds = 2 * 60; // 2 minutes let s3RefreshExpiryMs = null; @@ -255,6 +257,26 @@ function extractKeyFromS3Url(fileUrlOrKey) { const hostname = url.hostname; const pathname = url.pathname.substring(1); // Remove leading slash + // Explicit path-style with custom endpoint: use endpoint pathname for precise key extraction. + // Handles endpoints with a base path (e.g. https://example.com/storage/). + if (endpoint && forcePathStyle) { + const endpointUrl = new URL(endpoint); + const startPos = + endpointUrl.pathname.length + + (endpointUrl.pathname.endsWith('/') ? 0 : 1) + + bucketName.length + + 1; + const key = url.pathname.substring(startPos); + if (!key) { + logger.warn( + `[extractKeyFromS3Url] Extracted key is empty for endpoint path-style URL: ${fileUrlOrKey}`, + ); + } else { + logger.debug(`[extractKeyFromS3Url] fileUrlOrKey: ${fileUrlOrKey}, Extracted key: ${key}`); + } + return key; + } + if ( hostname === 's3.amazonaws.com' || hostname.match(/^s3[-.][a-z0-9-]+\.amazonaws\.com$/) || diff --git a/api/test/services/Files/S3/crud.test.js b/api/test/services/Files/S3/crud.test.js index c55bbbcf97..c7b46fba4c 100644 --- a/api/test/services/Files/S3/crud.test.js +++ b/api/test/services/Files/S3/crud.test.js @@ -19,6 +19,7 @@ jest.mock('@aws-sdk/client-s3'); jest.mock('@librechat/api', () => ({ initializeS3: jest.fn(), deleteRagFile: jest.fn().mockResolvedValue(undefined), + isEnabled: jest.fn((val) => val === 'true'), })); jest.mock('@librechat/data-schemas', () => ({ @@ -841,5 +842,35 @@ describe('S3 CRUD Operations', () => { ), ).toBe('images/user123/avatar.png'); }); + + it('should use endpoint base path when AWS_ENDPOINT_URL and AWS_FORCE_PATH_STYLE are set', () => { + process.env.AWS_BUCKET_NAME = 'test-bucket'; + process.env.AWS_ENDPOINT_URL = 'https://minio.example.com'; + process.env.AWS_FORCE_PATH_STYLE = 'true'; + jest.resetModules(); + const { extractKeyFromS3Url: fn } = require('~/server/services/Files/S3/crud'); + + expect(fn('https://minio.example.com/test-bucket/images/user123/file.jpg')).toBe( + 'images/user123/file.jpg', + ); + + delete process.env.AWS_ENDPOINT_URL; + delete process.env.AWS_FORCE_PATH_STYLE; + }); + + it('should handle endpoint with a base path', () => { + process.env.AWS_BUCKET_NAME = 'test-bucket'; + process.env.AWS_ENDPOINT_URL = 'https://example.com/storage/'; + process.env.AWS_FORCE_PATH_STYLE = 'true'; + jest.resetModules(); + const { extractKeyFromS3Url: fn } = require('~/server/services/Files/S3/crud'); + + expect(fn('https://example.com/storage/test-bucket/images/user123/file.jpg')).toBe( + 'images/user123/file.jpg', + ); + + delete process.env.AWS_ENDPOINT_URL; + delete process.env.AWS_FORCE_PATH_STYLE; + }); }); }); diff --git a/packages/api/src/cdn/__tests__/s3.test.ts b/packages/api/src/cdn/__tests__/s3.test.ts new file mode 100644 index 0000000000..048c652a45 --- /dev/null +++ b/packages/api/src/cdn/__tests__/s3.test.ts @@ -0,0 +1,123 @@ +import type { S3Client } from '@aws-sdk/client-s3'; + +const mockLogger = { info: jest.fn(), error: jest.fn() }; + +jest.mock('@aws-sdk/client-s3', () => ({ + S3Client: jest.fn(), +})); + +jest.mock('@librechat/data-schemas', () => ({ + logger: mockLogger, +})); + +describe('initializeS3', () => { + const REQUIRED_ENV = { + AWS_REGION: 'us-east-1', + AWS_BUCKET_NAME: 'test-bucket', + AWS_ACCESS_KEY_ID: 'test-key-id', + AWS_SECRET_ACCESS_KEY: 'test-secret', + }; + + beforeEach(() => { + jest.resetModules(); + jest.clearAllMocks(); + Object.assign(process.env, REQUIRED_ENV); + delete process.env.AWS_ENDPOINT_URL; + delete process.env.AWS_FORCE_PATH_STYLE; + }); + + afterEach(() => { + for (const key of Object.keys(REQUIRED_ENV)) { + delete process.env[key]; + } + delete process.env.AWS_ENDPOINT_URL; + delete process.env.AWS_FORCE_PATH_STYLE; + }); + + async function load() { + const { S3Client: MockS3Client } = jest.requireMock('@aws-sdk/client-s3') as { + S3Client: jest.MockedClass; + }; + const { initializeS3 } = await import('../s3'); + return { MockS3Client, initializeS3 }; + } + + it('should initialize with region and credentials', async () => { + const { MockS3Client, initializeS3 } = await load(); + initializeS3(); + expect(MockS3Client).toHaveBeenCalledWith( + expect.objectContaining({ + region: 'us-east-1', + credentials: { accessKeyId: 'test-key-id', secretAccessKey: 'test-secret' }, + }), + ); + }); + + it('should include endpoint when AWS_ENDPOINT_URL is set', async () => { + process.env.AWS_ENDPOINT_URL = 'https://fsn1.your-objectstorage.com'; + const { MockS3Client, initializeS3 } = await load(); + initializeS3(); + expect(MockS3Client).toHaveBeenCalledWith( + expect.objectContaining({ endpoint: 'https://fsn1.your-objectstorage.com' }), + ); + }); + + it('should not include endpoint when AWS_ENDPOINT_URL is not set', async () => { + const { MockS3Client, initializeS3 } = await load(); + initializeS3(); + const config = MockS3Client.mock.calls[0][0] as Record; + expect(config).not.toHaveProperty('endpoint'); + }); + + it('should set forcePathStyle when AWS_FORCE_PATH_STYLE is true', async () => { + process.env.AWS_FORCE_PATH_STYLE = 'true'; + const { MockS3Client, initializeS3 } = await load(); + initializeS3(); + expect(MockS3Client).toHaveBeenCalledWith(expect.objectContaining({ forcePathStyle: true })); + }); + + it('should not set forcePathStyle when AWS_FORCE_PATH_STYLE is false', async () => { + process.env.AWS_FORCE_PATH_STYLE = 'false'; + const { MockS3Client, initializeS3 } = await load(); + initializeS3(); + const config = MockS3Client.mock.calls[0][0] as Record; + expect(config).not.toHaveProperty('forcePathStyle'); + }); + + it('should not set forcePathStyle when AWS_FORCE_PATH_STYLE is not set', async () => { + const { MockS3Client, initializeS3 } = await load(); + initializeS3(); + const config = MockS3Client.mock.calls[0][0] as Record; + expect(config).not.toHaveProperty('forcePathStyle'); + }); + + it('should return null and log error when AWS_REGION is not set', async () => { + delete process.env.AWS_REGION; + const { initializeS3 } = await load(); + const result = initializeS3(); + expect(result).toBeNull(); + expect(mockLogger.error).toHaveBeenCalledWith( + '[initializeS3] AWS_REGION is not set. Cannot initialize S3.', + ); + }); + + it('should return the same instance on subsequent calls', async () => { + const { MockS3Client, initializeS3 } = await load(); + const first = initializeS3(); + const second = initializeS3(); + expect(first).toBe(second); + expect(MockS3Client).toHaveBeenCalledTimes(1); + }); + + it('should use default credentials chain when keys are not provided', async () => { + delete process.env.AWS_ACCESS_KEY_ID; + delete process.env.AWS_SECRET_ACCESS_KEY; + const { MockS3Client, initializeS3 } = await load(); + initializeS3(); + const config = MockS3Client.mock.calls[0][0] as Record; + expect(config).not.toHaveProperty('credentials'); + expect(mockLogger.info).toHaveBeenCalledWith( + '[initializeS3] S3 initialized using default credentials (IRSA).', + ); + }); +}); diff --git a/packages/api/src/cdn/s3.ts b/packages/api/src/cdn/s3.ts index 683a7887fa..f6f8527ce4 100644 --- a/packages/api/src/cdn/s3.ts +++ b/packages/api/src/cdn/s3.ts @@ -1,5 +1,6 @@ import { S3Client } from '@aws-sdk/client-s3'; import { logger } from '@librechat/data-schemas'; +import { isEnabled } from '~/utils/common'; let s3: S3Client | null = null; @@ -31,8 +32,8 @@ export const initializeS3 = (): S3Client | null => { const config = { region, - // Conditionally add the endpoint if it is provided ...(endpoint ? { endpoint } : {}), + ...(isEnabled(process.env.AWS_FORCE_PATH_STYLE) ? { forcePathStyle: true } : {}), }; if (accessKeyId && secretAccessKey) {