mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-02-22 18:34:08 +01:00
🪣 fix: S3 path-style URL support for MinIO, R2, and custom endpoints (#11894)
Some checks are pending
Docker Dev Branch Images Build / build (Dockerfile, lc-dev, node) (push) Waiting to run
Docker Dev Branch Images Build / build (Dockerfile.multi, lc-dev-api, api-build) (push) Waiting to run
Docker Dev Images Build / build (Dockerfile, librechat-dev, node) (push) Waiting to run
Docker Dev Images Build / build (Dockerfile.multi, librechat-dev-api, api-build) (push) Waiting to run
Sync Locize Translations & Create Translation PR / Sync Translation Keys with Locize (push) Waiting to run
Sync Locize Translations & Create Translation PR / Create Translation PR on Version Published (push) Blocked by required conditions
Some checks are pending
Docker Dev Branch Images Build / build (Dockerfile, lc-dev, node) (push) Waiting to run
Docker Dev Branch Images Build / build (Dockerfile.multi, lc-dev-api, api-build) (push) Waiting to run
Docker Dev Images Build / build (Dockerfile, librechat-dev, node) (push) Waiting to run
Docker Dev Images Build / build (Dockerfile.multi, librechat-dev-api, api-build) (push) Waiting to run
Sync Locize Translations & Create Translation PR / Sync Translation Keys with Locize (push) Waiting to run
Sync Locize Translations & Create Translation PR / Create Translation PR on Version Published (push) Blocked by required conditions
* 🪣 fix: S3 path-style URL support for MinIO, R2, and custom endpoints `extractKeyFromS3Url` now uses `AWS_BUCKET_NAME` to automatically detect and strip the bucket prefix from path-style URLs, fixing `NoSuchKey` errors on URL refresh for any S3-compatible provider using a custom endpoint (MinIO, Cloudflare R2, Hetzner, Backblaze B2, etc.). No additional configuration required — the bucket name is already a required env var for S3 to function. `initializeS3` now passes `forcePathStyle: true` to the S3Client constructor when `AWS_FORCE_PATH_STYLE=true` is set. Required for providers whose SSL certificates do not support virtual-hosted-style bucket subdomains (e.g. Hetzner Object Storage), which previously caused 401 / SignatureDoesNotMatch on upload. Additional fixes: - Suppress error log noise in `extractKeyFromS3Url` catch path: plain S3 keys no longer log as errors, only inputs that start with http(s):// do - Fix test env var ordering so module-level constants pick up `AWS_BUCKET_NAME` and `S3_URL_EXPIRY_SECONDS` correctly before the module is required - Add missing `deleteRagFile` mock and assertion in `deleteFileFromS3` tests - Add `AWS_BUCKET_NAME` cleanup to `afterEach` to prevent cross-test pollution - Add `initializeS3` unit tests covering endpoint, forcePathStyle, credentials, singleton, and IRSA code paths - Document `AWS_FORCE_PATH_STYLE` in `.env.example`, `dotenv.mdx`, and `s3.mdx` * 🪣 fix: Enhance S3 URL key extraction for custom endpoints Updated `extractKeyFromS3Url` to support precise key extraction when using custom endpoints with path-style URLs. The logic now accounts for the `AWS_ENDPOINT_URL` and `AWS_FORCE_PATH_STYLE` environment variables, ensuring correct key handling for various S3-compatible providers. Added unit tests to verify the new functionality, including scenarios for endpoints with base paths. This improves compatibility and reduces potential errors when interacting with S3-like services.
This commit is contained in:
parent
b7bfdfa8b2
commit
7692fa837e
5 changed files with 182 additions and 2 deletions
|
|
@ -3,7 +3,7 @@ const fetch = require('node-fetch');
|
|||
const { logger } = require('@librechat/data-schemas');
|
||||
const { FileSources } = require('librechat-data-provider');
|
||||
const { getSignedUrl } = require('@aws-sdk/s3-request-presigner');
|
||||
const { initializeS3, deleteRagFile } = require('@librechat/api');
|
||||
const { initializeS3, deleteRagFile, isEnabled } = require('@librechat/api');
|
||||
const {
|
||||
PutObjectCommand,
|
||||
GetObjectCommand,
|
||||
|
|
@ -13,6 +13,8 @@ const {
|
|||
|
||||
const bucketName = process.env.AWS_BUCKET_NAME;
|
||||
const defaultBasePath = 'images';
|
||||
const endpoint = process.env.AWS_ENDPOINT_URL;
|
||||
const forcePathStyle = isEnabled(process.env.AWS_FORCE_PATH_STYLE);
|
||||
|
||||
let s3UrlExpirySeconds = 2 * 60; // 2 minutes
|
||||
let s3RefreshExpiryMs = null;
|
||||
|
|
@ -255,6 +257,26 @@ function extractKeyFromS3Url(fileUrlOrKey) {
|
|||
const hostname = url.hostname;
|
||||
const pathname = url.pathname.substring(1); // Remove leading slash
|
||||
|
||||
// Explicit path-style with custom endpoint: use endpoint pathname for precise key extraction.
|
||||
// Handles endpoints with a base path (e.g. https://example.com/storage/).
|
||||
if (endpoint && forcePathStyle) {
|
||||
const endpointUrl = new URL(endpoint);
|
||||
const startPos =
|
||||
endpointUrl.pathname.length +
|
||||
(endpointUrl.pathname.endsWith('/') ? 0 : 1) +
|
||||
bucketName.length +
|
||||
1;
|
||||
const key = url.pathname.substring(startPos);
|
||||
if (!key) {
|
||||
logger.warn(
|
||||
`[extractKeyFromS3Url] Extracted key is empty for endpoint path-style URL: ${fileUrlOrKey}`,
|
||||
);
|
||||
} else {
|
||||
logger.debug(`[extractKeyFromS3Url] fileUrlOrKey: ${fileUrlOrKey}, Extracted key: ${key}`);
|
||||
}
|
||||
return key;
|
||||
}
|
||||
|
||||
if (
|
||||
hostname === 's3.amazonaws.com' ||
|
||||
hostname.match(/^s3[-.][a-z0-9-]+\.amazonaws\.com$/) ||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue