mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-02-23 10:54:11 +01:00
🪣 fix: S3 path-style URL support for MinIO, R2, and custom endpoints (#11894)
Some checks failed
Docker Dev Branch Images Build / build (Dockerfile, lc-dev, node) (push) Has been cancelled
Docker Dev Branch Images Build / build (Dockerfile.multi, lc-dev-api, api-build) (push) Has been cancelled
Docker Dev Images Build / build (Dockerfile, librechat-dev, node) (push) Has been cancelled
Docker Dev Images Build / build (Dockerfile.multi, librechat-dev-api, api-build) (push) Has been cancelled
Sync Locize Translations & Create Translation PR / Sync Translation Keys with Locize (push) Has been cancelled
Sync Locize Translations & Create Translation PR / Create Translation PR on Version Published (push) Has been cancelled
Some checks failed
Docker Dev Branch Images Build / build (Dockerfile, lc-dev, node) (push) Has been cancelled
Docker Dev Branch Images Build / build (Dockerfile.multi, lc-dev-api, api-build) (push) Has been cancelled
Docker Dev Images Build / build (Dockerfile, librechat-dev, node) (push) Has been cancelled
Docker Dev Images Build / build (Dockerfile.multi, librechat-dev-api, api-build) (push) Has been cancelled
Sync Locize Translations & Create Translation PR / Sync Translation Keys with Locize (push) Has been cancelled
Sync Locize Translations & Create Translation PR / Create Translation PR on Version Published (push) Has been cancelled
* 🪣 fix: S3 path-style URL support for MinIO, R2, and custom endpoints `extractKeyFromS3Url` now uses `AWS_BUCKET_NAME` to automatically detect and strip the bucket prefix from path-style URLs, fixing `NoSuchKey` errors on URL refresh for any S3-compatible provider using a custom endpoint (MinIO, Cloudflare R2, Hetzner, Backblaze B2, etc.). No additional configuration required — the bucket name is already a required env var for S3 to function. `initializeS3` now passes `forcePathStyle: true` to the S3Client constructor when `AWS_FORCE_PATH_STYLE=true` is set. Required for providers whose SSL certificates do not support virtual-hosted-style bucket subdomains (e.g. Hetzner Object Storage), which previously caused 401 / SignatureDoesNotMatch on upload. Additional fixes: - Suppress error log noise in `extractKeyFromS3Url` catch path: plain S3 keys no longer log as errors, only inputs that start with http(s):// do - Fix test env var ordering so module-level constants pick up `AWS_BUCKET_NAME` and `S3_URL_EXPIRY_SECONDS` correctly before the module is required - Add missing `deleteRagFile` mock and assertion in `deleteFileFromS3` tests - Add `AWS_BUCKET_NAME` cleanup to `afterEach` to prevent cross-test pollution - Add `initializeS3` unit tests covering endpoint, forcePathStyle, credentials, singleton, and IRSA code paths - Document `AWS_FORCE_PATH_STYLE` in `.env.example`, `dotenv.mdx`, and `s3.mdx` * 🪣 fix: Enhance S3 URL key extraction for custom endpoints Updated `extractKeyFromS3Url` to support precise key extraction when using custom endpoints with path-style URLs. The logic now accounts for the `AWS_ENDPOINT_URL` and `AWS_FORCE_PATH_STYLE` environment variables, ensuring correct key handling for various S3-compatible providers. Added unit tests to verify the new functionality, including scenarios for endpoints with base paths. This improves compatibility and reduces potential errors when interacting with S3-like services.
This commit is contained in:
parent
b7bfdfa8b2
commit
7692fa837e
5 changed files with 182 additions and 2 deletions
123
packages/api/src/cdn/__tests__/s3.test.ts
Normal file
123
packages/api/src/cdn/__tests__/s3.test.ts
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
import type { S3Client } from '@aws-sdk/client-s3';
|
||||
|
||||
const mockLogger = { info: jest.fn(), error: jest.fn() };
|
||||
|
||||
jest.mock('@aws-sdk/client-s3', () => ({
|
||||
S3Client: jest.fn(),
|
||||
}));
|
||||
|
||||
jest.mock('@librechat/data-schemas', () => ({
|
||||
logger: mockLogger,
|
||||
}));
|
||||
|
||||
describe('initializeS3', () => {
|
||||
const REQUIRED_ENV = {
|
||||
AWS_REGION: 'us-east-1',
|
||||
AWS_BUCKET_NAME: 'test-bucket',
|
||||
AWS_ACCESS_KEY_ID: 'test-key-id',
|
||||
AWS_SECRET_ACCESS_KEY: 'test-secret',
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
jest.resetModules();
|
||||
jest.clearAllMocks();
|
||||
Object.assign(process.env, REQUIRED_ENV);
|
||||
delete process.env.AWS_ENDPOINT_URL;
|
||||
delete process.env.AWS_FORCE_PATH_STYLE;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
for (const key of Object.keys(REQUIRED_ENV)) {
|
||||
delete process.env[key];
|
||||
}
|
||||
delete process.env.AWS_ENDPOINT_URL;
|
||||
delete process.env.AWS_FORCE_PATH_STYLE;
|
||||
});
|
||||
|
||||
async function load() {
|
||||
const { S3Client: MockS3Client } = jest.requireMock('@aws-sdk/client-s3') as {
|
||||
S3Client: jest.MockedClass<typeof S3Client>;
|
||||
};
|
||||
const { initializeS3 } = await import('../s3');
|
||||
return { MockS3Client, initializeS3 };
|
||||
}
|
||||
|
||||
it('should initialize with region and credentials', async () => {
|
||||
const { MockS3Client, initializeS3 } = await load();
|
||||
initializeS3();
|
||||
expect(MockS3Client).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
region: 'us-east-1',
|
||||
credentials: { accessKeyId: 'test-key-id', secretAccessKey: 'test-secret' },
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should include endpoint when AWS_ENDPOINT_URL is set', async () => {
|
||||
process.env.AWS_ENDPOINT_URL = 'https://fsn1.your-objectstorage.com';
|
||||
const { MockS3Client, initializeS3 } = await load();
|
||||
initializeS3();
|
||||
expect(MockS3Client).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ endpoint: 'https://fsn1.your-objectstorage.com' }),
|
||||
);
|
||||
});
|
||||
|
||||
it('should not include endpoint when AWS_ENDPOINT_URL is not set', async () => {
|
||||
const { MockS3Client, initializeS3 } = await load();
|
||||
initializeS3();
|
||||
const config = MockS3Client.mock.calls[0][0] as Record<string, unknown>;
|
||||
expect(config).not.toHaveProperty('endpoint');
|
||||
});
|
||||
|
||||
it('should set forcePathStyle when AWS_FORCE_PATH_STYLE is true', async () => {
|
||||
process.env.AWS_FORCE_PATH_STYLE = 'true';
|
||||
const { MockS3Client, initializeS3 } = await load();
|
||||
initializeS3();
|
||||
expect(MockS3Client).toHaveBeenCalledWith(expect.objectContaining({ forcePathStyle: true }));
|
||||
});
|
||||
|
||||
it('should not set forcePathStyle when AWS_FORCE_PATH_STYLE is false', async () => {
|
||||
process.env.AWS_FORCE_PATH_STYLE = 'false';
|
||||
const { MockS3Client, initializeS3 } = await load();
|
||||
initializeS3();
|
||||
const config = MockS3Client.mock.calls[0][0] as Record<string, unknown>;
|
||||
expect(config).not.toHaveProperty('forcePathStyle');
|
||||
});
|
||||
|
||||
it('should not set forcePathStyle when AWS_FORCE_PATH_STYLE is not set', async () => {
|
||||
const { MockS3Client, initializeS3 } = await load();
|
||||
initializeS3();
|
||||
const config = MockS3Client.mock.calls[0][0] as Record<string, unknown>;
|
||||
expect(config).not.toHaveProperty('forcePathStyle');
|
||||
});
|
||||
|
||||
it('should return null and log error when AWS_REGION is not set', async () => {
|
||||
delete process.env.AWS_REGION;
|
||||
const { initializeS3 } = await load();
|
||||
const result = initializeS3();
|
||||
expect(result).toBeNull();
|
||||
expect(mockLogger.error).toHaveBeenCalledWith(
|
||||
'[initializeS3] AWS_REGION is not set. Cannot initialize S3.',
|
||||
);
|
||||
});
|
||||
|
||||
it('should return the same instance on subsequent calls', async () => {
|
||||
const { MockS3Client, initializeS3 } = await load();
|
||||
const first = initializeS3();
|
||||
const second = initializeS3();
|
||||
expect(first).toBe(second);
|
||||
expect(MockS3Client).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('should use default credentials chain when keys are not provided', async () => {
|
||||
delete process.env.AWS_ACCESS_KEY_ID;
|
||||
delete process.env.AWS_SECRET_ACCESS_KEY;
|
||||
const { MockS3Client, initializeS3 } = await load();
|
||||
initializeS3();
|
||||
const config = MockS3Client.mock.calls[0][0] as Record<string, unknown>;
|
||||
expect(config).not.toHaveProperty('credentials');
|
||||
expect(mockLogger.info).toHaveBeenCalledWith(
|
||||
'[initializeS3] S3 initialized using default credentials (IRSA).',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
import { S3Client } from '@aws-sdk/client-s3';
|
||||
import { logger } from '@librechat/data-schemas';
|
||||
import { isEnabled } from '~/utils/common';
|
||||
|
||||
let s3: S3Client | null = null;
|
||||
|
||||
|
|
@ -31,8 +32,8 @@ export const initializeS3 = (): S3Client | null => {
|
|||
|
||||
const config = {
|
||||
region,
|
||||
// Conditionally add the endpoint if it is provided
|
||||
...(endpoint ? { endpoint } : {}),
|
||||
...(isEnabled(process.env.AWS_FORCE_PATH_STYLE) ? { forcePathStyle: true } : {}),
|
||||
};
|
||||
|
||||
if (accessKeyId && secretAccessKey) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue