From f95d5aaf4dc48b24b60304a6e60498130307fa82 Mon Sep 17 00:00:00 2001 From: heptapod <164861708+leondape@users.noreply.github.com> Date: Wed, 19 Mar 2025 14:51:56 +0100 Subject: [PATCH 1/4] =?UTF-8?q?=F0=9F=94=92feat:=20Enable=20OpenID=20Auto-?= =?UTF-8?q?Redirect=20(#6066)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * added feature for oidc auto redirection * Added Cooldown logic for OIDC auto redirect for failed login attempts * 🔧 feat: Implement custom logout redirect handling and enhance OpenID auto-redirect logic * 🔧 refactor: Update getLoginError to use TranslationKeys for improved type safety * 🔧 feat: Localize redirect message to OpenID provider in Login component --------- Co-authored-by: Ruben Talstra --- .env.example | 3 + api/server/routes/__tests__/config.spec.js | 1 + api/server/routes/config.js | 1 + api/server/routes/oauth.js | 4 +- client/src/common/types.ts | 2 +- client/src/components/Auth/Login.tsx | 64 +++++++++++++++++++++- client/src/hooks/AuthContext.tsx | 29 +++++++--- client/src/locales/en/translation.json | 1 + client/src/routes/Root.tsx | 7 +-- client/src/utils/getLoginError.ts | 6 +- packages/data-provider/src/config.ts | 1 + 11 files changed, 102 insertions(+), 17 deletions(-) diff --git a/.env.example b/.env.example index b86092d56d..2a86619db6 100644 --- a/.env.example +++ b/.env.example @@ -432,6 +432,9 @@ OPENID_NAME_CLAIM= OPENID_BUTTON_LABEL= OPENID_IMAGE_URL= +# Set to true to automatically redirect to the OpenID provider when a user visits the login page +# This will bypass the login form completely for users, only use this if OpenID is your only authentication method +OPENID_AUTO_REDIRECT=false # LDAP LDAP_URL= diff --git a/api/server/routes/__tests__/config.spec.js b/api/server/routes/__tests__/config.spec.js index 13af53f299..0bb80bb9ee 100644 --- a/api/server/routes/__tests__/config.spec.js +++ b/api/server/routes/__tests__/config.spec.js @@ -18,6 +18,7 @@ afterEach(() => { delete process.env.OPENID_ISSUER; delete process.env.OPENID_SESSION_SECRET; delete process.env.OPENID_BUTTON_LABEL; + delete process.env.OPENID_AUTO_REDIRECT; delete process.env.OPENID_AUTH_URL; delete process.env.GITHUB_CLIENT_ID; delete process.env.GITHUB_CLIENT_SECRET; diff --git a/api/server/routes/config.js b/api/server/routes/config.js index c36f4a9b8a..e8d2fe57ac 100644 --- a/api/server/routes/config.js +++ b/api/server/routes/config.js @@ -58,6 +58,7 @@ router.get('/', async function (req, res) { !!process.env.OPENID_SESSION_SECRET, openidLabel: process.env.OPENID_BUTTON_LABEL || 'Continue with OpenID', openidImageUrl: process.env.OPENID_IMAGE_URL, + openidAutoRedirect: isEnabled(process.env.OPENID_AUTO_REDIRECT), serverDomain: process.env.DOMAIN_SERVER || 'http://localhost:3080', emailLoginEnabled, registrationEnabled: !ldap?.enabled && isEnabled(process.env.ALLOW_REGISTRATION), diff --git a/api/server/routes/oauth.js b/api/server/routes/oauth.js index 046370798b..9006b25c5b 100644 --- a/api/server/routes/oauth.js +++ b/api/server/routes/oauth.js @@ -31,7 +31,9 @@ const oauthHandler = async (req, res) => { router.get('/error', (req, res) => { // A single error message is pushed by passport when authentication fails. logger.error('Error in OAuth authentication:', { message: req.session.messages.pop() }); - res.redirect(`${domains.client}/login`); + + // Redirect to login page with auth_failed parameter to prevent infinite redirect loops + res.redirect(`${domains.client}/login?redirect=false`); }); /** diff --git a/client/src/common/types.ts b/client/src/common/types.ts index 975f468930..118cefce16 100644 --- a/client/src/common/types.ts +++ b/client/src/common/types.ts @@ -401,7 +401,7 @@ export type TAuthContext = { isAuthenticated: boolean; error: string | undefined; login: (data: t.TLoginUser) => void; - logout: () => void; + logout: (redirect?: string) => void; setError: React.Dispatch>; roles?: Record; }; diff --git a/client/src/components/Auth/Login.tsx b/client/src/components/Auth/Login.tsx index a332553701..48cbfe1a91 100644 --- a/client/src/components/Auth/Login.tsx +++ b/client/src/components/Auth/Login.tsx @@ -1,16 +1,78 @@ -import { useOutletContext } from 'react-router-dom'; +import { useOutletContext, useSearchParams } from 'react-router-dom'; +import { useEffect, useState } from 'react'; import { useAuthContext } from '~/hooks/AuthContext'; import type { TLoginLayoutContext } from '~/common'; import { ErrorMessage } from '~/components/Auth/ErrorMessage'; import { getLoginError } from '~/utils'; import { useLocalize } from '~/hooks'; import LoginForm from './LoginForm'; +import SocialButton from '~/components/Auth/SocialButton'; +import { OpenIDIcon } from '~/components'; function Login() { const localize = useLocalize(); const { error, setError, login } = useAuthContext(); const { startupConfig } = useOutletContext(); + const [searchParams, setSearchParams] = useSearchParams(); + // Determine if auto-redirect should be disabled based on the URL parameter + const disableAutoRedirect = searchParams.get('redirect') === 'false'; + + // Persist the disable flag locally so that once detected, auto-redirect stays disabled. + const [isAutoRedirectDisabled, setIsAutoRedirectDisabled] = useState(disableAutoRedirect); + + // Once the disable flag is detected, update local state and remove the parameter from the URL. + useEffect(() => { + if (disableAutoRedirect) { + setIsAutoRedirectDisabled(true); + const newParams = new URLSearchParams(searchParams); + newParams.delete('redirect'); + setSearchParams(newParams, { replace: true }); + } + }, [disableAutoRedirect, searchParams, setSearchParams]); + + // Determine whether we should auto-redirect to OpenID. + const shouldAutoRedirect = + startupConfig?.openidLoginEnabled && + startupConfig?.openidAutoRedirect && + startupConfig?.serverDomain && + !isAutoRedirectDisabled; + + useEffect(() => { + if (shouldAutoRedirect) { + console.log('Auto-redirecting to OpenID provider...'); + window.location.href = `${startupConfig.serverDomain}/oauth/openid`; + } + }, [shouldAutoRedirect, startupConfig]); + + // Render fallback UI if auto-redirect is active. + if (shouldAutoRedirect) { + return ( +
+

+ {localize('com_ui_redirecting_to_provider', { 0: startupConfig.openidLabel })} +

+
+ + startupConfig.openidImageUrl ? ( + OpenID Logo + ) : ( + + ) + } + label={startupConfig.openidLabel} + id="openid" + /> +
+
+ ); + } + return ( <> {error != null && {localize(getLoginError(error))}} diff --git a/client/src/hooks/AuthContext.tsx b/client/src/hooks/AuthContext.tsx index 2828a1bc5b..e21d19ebf1 100644 --- a/client/src/hooks/AuthContext.tsx +++ b/client/src/hooks/AuthContext.tsx @@ -6,6 +6,7 @@ import { useContext, useCallback, createContext, + useRef, } from 'react'; import { useNavigate } from 'react-router-dom'; import { useRecoilState } from 'recoil'; @@ -35,6 +36,8 @@ const AuthContextProvider = ({ const [token, setToken] = useState(undefined); const [error, setError] = useState(undefined); const [isAuthenticated, setIsAuthenticated] = useState(false); + const logoutRedirectRef = useRef(undefined); + const { data: userRole = null } = useGetRole(SystemRoles.USER, { enabled: !!(isAuthenticated && (user?.role ?? '')), }); @@ -52,16 +55,17 @@ const AuthContextProvider = ({ //@ts-ignore - ok for token to be undefined initially setTokenHeader(token); setIsAuthenticated(isAuthenticated); - if (redirect == null) { + // Use a custom redirect if set + const finalRedirect = logoutRedirectRef.current || redirect; + // Clear the stored redirect + logoutRedirectRef.current = undefined; + if (finalRedirect == null) { return; } - if (redirect.startsWith('http://') || redirect.startsWith('https://')) { - // For external links, use window.location - window.location.href = redirect; - // Or if you want to open in a new tab: - // window.open(redirect, '_blank'); + if (finalRedirect.startsWith('http://') || finalRedirect.startsWith('https://')) { + window.location.href = finalRedirect; } else { - navigate(redirect, { replace: true }); + navigate(finalRedirect, { replace: true }); } }, [navigate, setUser], @@ -106,7 +110,16 @@ const AuthContextProvider = ({ }); const refreshToken = useRefreshTokenMutation(); - const logout = useCallback(() => logoutUser.mutate(undefined), [logoutUser]); + const logout = useCallback( + (redirect?: string) => { + if (redirect) { + logoutRedirectRef.current = redirect; + } + logoutUser.mutate(undefined); + }, + [logoutUser], + ); + const userQuery = useGetUserQuery({ enabled: !!(token ?? '') }); const login = (data: t.TLoginUser) => { diff --git a/client/src/locales/en/translation.json b/client/src/locales/en/translation.json index 86d2ce6b41..58dd833c27 100644 --- a/client/src/locales/en/translation.json +++ b/client/src/locales/en/translation.json @@ -104,6 +104,7 @@ "com_auth_google_login": "Continue with Google", "com_auth_here": "HERE", "com_auth_login": "Login", + "com_ui_redirecting_to_provider": "Redirecting to {{0}}, please wait...", "com_auth_login_with_new_password": "You may now login with your new password.", "com_auth_name_max_length": "Name must be less than 80 characters", "com_auth_name_min_length": "Name must be at least 3 characters", diff --git a/client/src/routes/Root.tsx b/client/src/routes/Root.tsx index a7d999ae45..da02b7c4c2 100644 --- a/client/src/routes/Root.tsx +++ b/client/src/routes/Root.tsx @@ -1,5 +1,5 @@ import React, { useState, useEffect } from 'react'; -import { Outlet, useNavigate } from 'react-router-dom'; +import { Outlet } from 'react-router-dom'; import type { ContextType } from '~/common'; import { AgentsMapContext, @@ -15,7 +15,6 @@ import { Nav, MobileNav } from '~/components/Nav'; import { Banner } from '~/components/Banners'; export default function Root() { - const navigate = useNavigate(); const [showTerms, setShowTerms] = useState(false); const [bannerHeight, setBannerHeight] = useState(0); const [navVisible, setNavVisible] = useState(() => { @@ -44,10 +43,10 @@ export default function Root() { setShowTerms(false); }; + // Pass the desired redirect parameter to logout const handleDeclineTerms = () => { setShowTerms(false); - logout(); - navigate('/login'); + logout('/login?redirect=false'); }; if (!isAuthenticated) { diff --git a/client/src/utils/getLoginError.ts b/client/src/utils/getLoginError.ts index 27fafed0cf..492948d6e6 100644 --- a/client/src/utils/getLoginError.ts +++ b/client/src/utils/getLoginError.ts @@ -1,5 +1,7 @@ -const getLoginError = (errorText: string) => { - const defaultError = 'com_auth_error_login'; +import { TranslationKeys } from '~/hooks'; + +const getLoginError = (errorText: string): TranslationKeys => { + const defaultError: TranslationKeys = 'com_auth_error_login'; if (!errorText) { return defaultError; diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 96c8071727..679d600eb9 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -514,6 +514,7 @@ export type TStartupConfig = { appleLoginEnabled: boolean; openidLabel: string; openidImageUrl: string; + openidAutoRedirect: boolean; /** LDAP Auth Configuration */ ldap?: { /** LDAP enabled */ From 0a4a16d1f7139bbb40ebd888fbe65f13d3584d73 Mon Sep 17 00:00:00 2001 From: Ruben Talstra Date: Wed, 19 Mar 2025 15:45:52 +0100 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=9A=80=20feat:=20Integrate=20`Azure?= =?UTF-8?q?=20Blob=20Storage`=20for=20file=20handling=20and=20image=20uplo?= =?UTF-8?q?ads=20(#6153)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🚀 feat: Integrate Azure Blob Storage for file handling and image uploads * 🐼 refactor: Correct module import case for Azure in strategies.js * 🚀 feat: Add Azure support in SourceIcon component * 🚀 feat: Enhance Azure Blob Service initialization with Managed Identity support * 🐼 refactor: Remove unused Azure dependencies from package.json and package-lock.json * 🐼 refactor: Remove unused Azure dependencies from package.json and package-lock.json * 🐼 refactor: Remove unused Azure dependencies from package.json and package-lock.json * 🚀 feat: Add Azure SDK dependencies for identity and storage blob * 🔧 fix: Reorganize imports in strategies.js for better clarity * 🔧 fix: Correct comment formatting in strategies.js for consistency * 🔧 fix: Improve comment formatting in strategies.js for consistency --- .env.example | 8 + api/package.json | 2 + api/server/services/AppService.js | 3 + api/server/services/Files/Azure/crud.js | 196 ++++++++ api/server/services/Files/Azure/images.js | 124 ++++++ api/server/services/Files/Azure/index.js | 9 + api/server/services/Files/Azure/initialize.js | 55 +++ api/server/services/Files/strategies.js | 29 +- package-lock.json | 417 ++++++++++++++++-- 9 files changed, 801 insertions(+), 42 deletions(-) create mode 100644 api/server/services/Files/Azure/crud.js create mode 100644 api/server/services/Files/Azure/images.js create mode 100644 api/server/services/Files/Azure/index.js create mode 100644 api/server/services/Files/Azure/initialize.js diff --git a/.env.example b/.env.example index 2a86619db6..d9a4d52d92 100644 --- a/.env.example +++ b/.env.example @@ -485,6 +485,14 @@ AWS_SECRET_ACCESS_KEY= AWS_REGION= AWS_BUCKET_NAME= +#========================# +# Azure Blob Storage # +#========================# + +AZURE_STORAGE_CONNECTION_STRING= +AZURE_STORAGE_PUBLIC_ACCESS=false +AZURE_CONTAINER_NAME=files + #========================# # Shared Links # #========================# diff --git a/api/package.json b/api/package.json index 36edce6baa..a8ece630af 100644 --- a/api/package.json +++ b/api/package.json @@ -37,6 +37,8 @@ "@anthropic-ai/sdk": "^0.37.0", "@aws-sdk/client-s3": "^3.758.0", "@aws-sdk/s3-request-presigner": "^3.758.0", + "@azure/identity": "^4.7.0", + "@azure/storage-blob": "^12.26.0", "@azure/search-documents": "^12.0.0", "@google/generative-ai": "^0.23.0", "@googleapis/youtube": "^20.0.0", diff --git a/api/server/services/AppService.js b/api/server/services/AppService.js index 925ffe93de..3fdae6ac10 100644 --- a/api/server/services/AppService.js +++ b/api/server/services/AppService.js @@ -7,6 +7,7 @@ const { } = require('librechat-data-provider'); const { checkVariables, checkHealth, checkConfig, checkAzureVariables } = require('./start/checks'); const { azureAssistantsDefaults, assistantsConfigSetup } = require('./start/assistants'); +const { initializeAzureBlobService } = require('./Files/Azure/initialize'); const { initializeFirebase } = require('./Files/Firebase/initialize'); const { initializeS3 } = require('./Files/S3/initialize'); const loadCustomConfig = require('./Config/loadCustomConfig'); @@ -45,6 +46,8 @@ const AppService = async (app) => { if (fileStrategy === FileSources.firebase) { initializeFirebase(); + } else if (fileStrategy === FileSources.azure) { + initializeAzureBlobService(); } else if (fileStrategy === FileSources.s3) { initializeS3(); } diff --git a/api/server/services/Files/Azure/crud.js b/api/server/services/Files/Azure/crud.js new file mode 100644 index 0000000000..638da34b27 --- /dev/null +++ b/api/server/services/Files/Azure/crud.js @@ -0,0 +1,196 @@ +const fs = require('fs'); +const path = require('path'); +const axios = require('axios'); +const fetch = require('node-fetch'); +const { logger } = require('~/config'); +const { getAzureContainerClient } = require('./initialize'); + +const defaultBasePath = 'images'; + +/** + * Uploads a buffer to Azure Blob Storage. + * + * Files will be stored at the path: {basePath}/{userId}/{fileName} within the container. + * + * @param {Object} params + * @param {string} params.userId - The user's id. + * @param {Buffer} params.buffer - The buffer to upload. + * @param {string} params.fileName - The name of the file. + * @param {string} [params.basePath='images'] - The base folder within the container. + * @param {string} [params.containerName] - The Azure Blob container name. + * @returns {Promise} The URL of the uploaded blob. + */ +async function saveBufferToAzure({ + userId, + buffer, + fileName, + basePath = defaultBasePath, + containerName, +}) { + try { + const containerClient = getAzureContainerClient(containerName); + // Create the container if it doesn't exist. This is done per operation. + await containerClient.createIfNotExists({ + access: process.env.AZURE_STORAGE_PUBLIC_ACCESS ? 'blob' : undefined, + }); + const blobPath = `${basePath}/${userId}/${fileName}`; + const blockBlobClient = containerClient.getBlockBlobClient(blobPath); + await blockBlobClient.uploadData(buffer); + return blockBlobClient.url; + } catch (error) { + logger.error('[saveBufferToAzure] Error uploading buffer:', error); + throw error; + } +} + +/** + * Saves a file from a URL to Azure Blob Storage. + * + * @param {Object} params + * @param {string} params.userId - The user's id. + * @param {string} params.URL - The URL of the file. + * @param {string} params.fileName - The name of the file. + * @param {string} [params.basePath='images'] - The base folder within the container. + * @param {string} [params.containerName] - The Azure Blob container name. + * @returns {Promise} The URL of the uploaded blob. + */ +async function saveURLToAzure({ + userId, + URL, + fileName, + basePath = defaultBasePath, + containerName, +}) { + try { + const response = await fetch(URL); + const buffer = await response.buffer(); + return await saveBufferToAzure({ userId, buffer, fileName, basePath, containerName }); + } catch (error) { + logger.error('[saveURLToAzure] Error uploading file from URL:', error); + throw error; + } +} + +/** + * Retrieves a blob URL from Azure Blob Storage. + * + * @param {Object} params + * @param {string} params.fileName - The file name. + * @param {string} [params.basePath='images'] - The base folder used during upload. + * @param {string} [params.userId] - If files are stored in a user-specific directory. + * @param {string} [params.containerName] - The Azure Blob container name. + * @returns {Promise} The blob's URL. + */ +async function getAzureURL({ fileName, basePath = defaultBasePath, userId, containerName }) { + try { + const containerClient = getAzureContainerClient(containerName); + const blobPath = userId ? `${basePath}/${userId}/${fileName}` : `${basePath}/${fileName}`; + const blockBlobClient = containerClient.getBlockBlobClient(blobPath); + return blockBlobClient.url; + } catch (error) { + logger.error('[getAzureURL] Error retrieving blob URL:', error); + throw error; + } +} + +/** + * Deletes a blob from Azure Blob Storage. + * + * @param {Object} params + * @param {string} params.fileName - The name of the file. + * @param {string} [params.basePath='images'] - The base folder where the file is stored. + * @param {string} params.userId - The user's id. + * @param {string} [params.containerName] - The Azure Blob container name. + */ +async function deleteFileFromAzure({ + fileName, + basePath = defaultBasePath, + userId, + containerName, +}) { + try { + const containerClient = getAzureContainerClient(containerName); + const blobPath = `${basePath}/${userId}/${fileName}`; + const blockBlobClient = containerClient.getBlockBlobClient(blobPath); + await blockBlobClient.delete(); + logger.debug('[deleteFileFromAzure] Blob deleted successfully from Azure Blob Storage'); + } catch (error) { + logger.error('[deleteFileFromAzure] Error deleting blob:', error.message); + if (error.statusCode === 404) { + return; + } + throw error; + } +} + +/** + * Uploads a file from the local file system to Azure Blob Storage. + * + * This function reads the file from disk and then uploads it to Azure Blob Storage + * at the path: {basePath}/{userId}/{fileName}. + * + * @param {Object} params + * @param {object} params.req - The Express request object. + * @param {Express.Multer.File} params.file - The file object. + * @param {string} params.file_id - The file id. + * @param {string} [params.basePath='images'] - The base folder within the container. + * @param {string} [params.containerName] - The Azure Blob container name. + * @returns {Promise<{ filepath: string, bytes: number }>} An object containing the blob URL and its byte size. + */ +async function uploadFileToAzure({ + req, + file, + file_id, + basePath = defaultBasePath, + containerName, +}) { + try { + const inputFilePath = file.path; + const inputBuffer = await fs.promises.readFile(inputFilePath); + const bytes = Buffer.byteLength(inputBuffer); + const userId = req.user.id; + const fileName = `${file_id}__${path.basename(inputFilePath)}`; + const fileURL = await saveBufferToAzure({ + userId, + buffer: inputBuffer, + fileName, + basePath, + containerName, + }); + await fs.promises.unlink(inputFilePath); + return { filepath: fileURL, bytes }; + } catch (error) { + logger.error('[uploadFileToAzure] Error uploading file:', error); + throw error; + } +} + +/** + * Retrieves a readable stream for a blob from Azure Blob Storage. + * + * @param {object} _req - The Express request object. + * @param {string} fileURL - The URL of the blob. + * @returns {Promise} A readable stream of the blob. + */ +async function getAzureFileStream(_req, fileURL) { + try { + const response = await axios({ + method: 'get', + url: fileURL, + responseType: 'stream', + }); + return response.data; + } catch (error) { + logger.error('[getAzureFileStream] Error getting blob stream:', error); + throw error; + } +} + +module.exports = { + saveBufferToAzure, + saveURLToAzure, + getAzureURL, + deleteFileFromAzure, + uploadFileToAzure, + getAzureFileStream, +}; diff --git a/api/server/services/Files/Azure/images.js b/api/server/services/Files/Azure/images.js new file mode 100644 index 0000000000..a83b700af3 --- /dev/null +++ b/api/server/services/Files/Azure/images.js @@ -0,0 +1,124 @@ +const fs = require('fs'); +const path = require('path'); +const sharp = require('sharp'); +const { resizeImageBuffer } = require('../images/resize'); +const { updateUser } = require('~/models/userMethods'); +const { updateFile } = require('~/models/File'); +const { logger } = require('~/config'); +const { saveBufferToAzure } = require('./crud'); + +/** + * Uploads an image file to Azure Blob Storage. + * It resizes and converts the image similar to your Firebase implementation. + * + * @param {Object} params + * @param {object} params.req - The Express request object. + * @param {Express.Multer.File} params.file - The file object. + * @param {string} params.file_id - The file id. + * @param {EModelEndpoint} params.endpoint - The endpoint parameters. + * @param {string} [params.resolution='high'] - The image resolution. + * @param {string} [params.basePath='images'] - The base folder within the container. + * @param {string} [params.containerName] - The Azure Blob container name. + * @returns {Promise<{ filepath: string, bytes: number, width: number, height: number }>} + */ +async function uploadImageToAzure({ + req, + file, + file_id, + endpoint, + resolution = 'high', + basePath = 'images', + containerName, +}) { + try { + const inputFilePath = file.path; + const inputBuffer = await fs.promises.readFile(inputFilePath); + const { + buffer: resizedBuffer, + width, + height, + } = await resizeImageBuffer(inputBuffer, resolution, endpoint); + const extension = path.extname(inputFilePath); + const userId = req.user.id; + let webPBuffer; + let fileName = `${file_id}__${path.basename(inputFilePath)}`; + const targetExtension = `.${req.app.locals.imageOutputType}`; + + if (extension.toLowerCase() === targetExtension) { + webPBuffer = resizedBuffer; + } else { + webPBuffer = await sharp(resizedBuffer).toFormat(req.app.locals.imageOutputType).toBuffer(); + const extRegExp = new RegExp(path.extname(fileName) + '$'); + fileName = fileName.replace(extRegExp, targetExtension); + if (!path.extname(fileName)) { + fileName += targetExtension; + } + } + const downloadURL = await saveBufferToAzure({ + userId, + buffer: webPBuffer, + fileName, + basePath, + containerName, + }); + await fs.promises.unlink(inputFilePath); + const bytes = Buffer.byteLength(webPBuffer); + return { filepath: downloadURL, bytes, width, height }; + } catch (error) { + logger.error('[uploadImageToAzure] Error uploading image:', error); + throw error; + } +} + +/** + * Prepares the image URL and updates the file record. + * + * @param {object} req - The Express request object. + * @param {MongoFile} file - The file object. + * @returns {Promise<[MongoFile, string]>} + */ +async function prepareAzureImageURL(req, file) { + const { filepath } = file; + const promises = []; + promises.push(updateFile({ file_id: file.file_id })); + promises.push(filepath); + return await Promise.all(promises); +} + +/** + * Uploads and processes a user's avatar to Azure Blob Storage. + * + * @param {Object} params + * @param {Buffer} params.buffer - The avatar image buffer. + * @param {string} params.userId - The user's id. + * @param {string} params.manual - Flag to indicate manual update. + * @param {string} [params.basePath='images'] - The base folder within the container. + * @param {string} [params.containerName] - The Azure Blob container name. + * @returns {Promise} The URL of the avatar. + */ +async function processAzureAvatar({ buffer, userId, manual, basePath = 'images', containerName }) { + try { + const downloadURL = await saveBufferToAzure({ + userId, + buffer, + fileName: 'avatar.png', + basePath, + containerName, + }); + const isManual = manual === 'true'; + const url = `${downloadURL}?manual=${isManual}`; + if (isManual) { + await updateUser(userId, { avatar: url }); + } + return url; + } catch (error) { + logger.error('[processAzureAvatar] Error uploading profile picture to Azure:', error); + throw error; + } +} + +module.exports = { + uploadImageToAzure, + prepareAzureImageURL, + processAzureAvatar, +}; diff --git a/api/server/services/Files/Azure/index.js b/api/server/services/Files/Azure/index.js new file mode 100644 index 0000000000..27ad97a852 --- /dev/null +++ b/api/server/services/Files/Azure/index.js @@ -0,0 +1,9 @@ +const crud = require('./crud'); +const images = require('./images'); +const initialize = require('./initialize'); + +module.exports = { + ...crud, + ...images, + ...initialize, +}; diff --git a/api/server/services/Files/Azure/initialize.js b/api/server/services/Files/Azure/initialize.js new file mode 100644 index 0000000000..56df24d04a --- /dev/null +++ b/api/server/services/Files/Azure/initialize.js @@ -0,0 +1,55 @@ +const { BlobServiceClient } = require('@azure/storage-blob'); +const { logger } = require('~/config'); + +let blobServiceClient = null; +let azureWarningLogged = false; + +/** + * Initializes the Azure Blob Service client. + * This function establishes a connection by checking if a connection string is provided. + * If available, the connection string is used; otherwise, Managed Identity (via DefaultAzureCredential) is utilized. + * Note: Container creation (and its public access settings) is handled later in the CRUD functions. + * @returns {BlobServiceClient|null} The initialized client, or null if the required configuration is missing. + */ +const initializeAzureBlobService = () => { + if (blobServiceClient) { + return blobServiceClient; + } + const connectionString = process.env.AZURE_STORAGE_CONNECTION_STRING; + if (connectionString) { + blobServiceClient = BlobServiceClient.fromConnectionString(connectionString); + logger.info('Azure Blob Service initialized using connection string'); + } else { + const { DefaultAzureCredential } = require('@azure/identity'); + const accountName = process.env.AZURE_STORAGE_ACCOUNT_NAME; + if (!accountName) { + if (!azureWarningLogged) { + logger.error( + '[initializeAzureBlobService] Azure Blob Service not initialized. Connection string missing and AZURE_STORAGE_ACCOUNT_NAME not provided.', + ); + azureWarningLogged = true; + } + return null; + } + const url = `https://${accountName}.blob.core.windows.net`; + const credential = new DefaultAzureCredential(); + blobServiceClient = new BlobServiceClient(url, credential); + logger.info('Azure Blob Service initialized using Managed Identity'); + } + return blobServiceClient; +}; + +/** + * Retrieves the Azure ContainerClient for the given container name. + * @param {string} [containerName=process.env.AZURE_CONTAINER_NAME || 'files'] - The container name. + * @returns {ContainerClient|null} The Azure ContainerClient. + */ +const getAzureContainerClient = (containerName = process.env.AZURE_CONTAINER_NAME || 'files') => { + const serviceClient = initializeAzureBlobService(); + return serviceClient ? serviceClient.getContainerClient(containerName) : null; +}; + +module.exports = { + initializeAzureBlobService, + getAzureContainerClient, +}; diff --git a/api/server/services/Files/strategies.js b/api/server/services/Files/strategies.js index 7fcf10af03..d05ea03728 100644 --- a/api/server/services/Files/strategies.js +++ b/api/server/services/Files/strategies.js @@ -32,6 +32,17 @@ const { processS3Avatar, uploadFileToS3, } = require('./S3'); +const { + saveBufferToAzure, + saveURLToAzure, + getAzureURL, + deleteFileFromAzure, + uploadFileToAzure, + getAzureFileStream, + uploadImageToAzure, + prepareAzureImageURL, + processAzureAvatar, +} = require('./Azure'); const { uploadOpenAIFile, deleteOpenAIFile, getOpenAIFileStream } = require('./OpenAI'); const { getCodeOutputDownloadStream, uploadCodeEnvFile } = require('./Code'); const { uploadVectors, deleteVectors } = require('./VectorDB'); @@ -85,6 +96,22 @@ const s3Strategy = () => ({ getDownloadStream: getS3FileStream, }); +/** + * Azure Blob Storage Strategy Functions + * + * */ +const azureStrategy = () => ({ + handleFileUpload: uploadFileToAzure, + saveURL: saveURLToAzure, + getFileURL: getAzureURL, + deleteFile: deleteFileFromAzure, + saveBuffer: saveBufferToAzure, + prepareImagePayload: prepareAzureImageURL, + processAvatar: processAzureAvatar, + handleImageUpload: uploadImageToAzure, + getDownloadStream: getAzureFileStream, +}); + /** * VectorDB Storage Strategy Functions * @@ -184,7 +211,7 @@ const getStrategyFunctions = (fileSource) => { } else if (fileSource === FileSources.openai) { return openAIStrategy(); } else if (fileSource === FileSources.azure) { - return openAIStrategy(); + return azureStrategy(); } else if (fileSource === FileSources.vectordb) { return vectorStrategy(); } else if (fileSource === FileSources.s3) { diff --git a/package-lock.json b/package-lock.json index 8223d0a1d9..fa44a4ec22 100644 --- a/package-lock.json +++ b/package-lock.json @@ -53,7 +53,9 @@ "@anthropic-ai/sdk": "^0.37.0", "@aws-sdk/client-s3": "^3.758.0", "@aws-sdk/s3-request-presigner": "^3.758.0", + "@azure/identity": "^4.7.0", "@azure/search-documents": "^12.0.0", + "@azure/storage-blob": "^12.26.0", "@google/generative-ai": "^0.23.0", "@googleapis/youtube": "^20.0.0", "@keyv/mongo": "^2.1.8", @@ -3612,13 +3614,6 @@ } } }, - "client/node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "dev": true, - "license": "0BSD" - }, "client/node_modules/update-browserslist-db": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.2.tgz", @@ -11912,41 +11907,44 @@ } }, "node_modules/@azure/abort-controller": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.0.0.tgz", - "integrity": "sha512-RP/mR/WJchR+g+nQFJGOec+nzeN/VvjlwbinccoqfhTsTHbb8X5+mLDp48kHT0ueyum0BNSwGm0kX0UZuIqTGg==", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", + "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", + "license": "MIT", "dependencies": { - "tslib": "^2.2.0" + "tslib": "^2.6.2" }, "engines": { "node": ">=18.0.0" } }, "node_modules/@azure/core-auth": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.6.0.tgz", - "integrity": "sha512-3X9wzaaGgRaBCwhLQZDtFp5uLIXCPrGbwJNWPPugvL4xbIGgScv77YzzxToKGLAKvG9amDoofMoP+9hsH1vs1w==", + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.9.0.tgz", + "integrity": "sha512-FPwHpZywuyasDSLMqJ6fhbOK3TqUdviZNF8OqRGA4W5Ewib2lEEZ+pBsYcBa88B2NGO/SEnYPGhyBqNlE8ilSw==", + "license": "MIT", "dependencies": { "@azure/abort-controller": "^2.0.0", - "@azure/core-util": "^1.1.0", - "tslib": "^2.2.0" + "@azure/core-util": "^1.11.0", + "tslib": "^2.6.2" }, "engines": { "node": ">=18.0.0" } }, "node_modules/@azure/core-client": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.8.0.tgz", - "integrity": "sha512-+gHS3gEzPlhyQBMoqVPOTeNH031R5DM/xpCvz72y38C09rg4Hui/1sJS/ujoisDZbbSHyuRLVWdFlwL0pIFwbg==", + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.9.3.tgz", + "integrity": "sha512-/wGw8fJ4mdpJ1Cum7s1S+VQyXt1ihwKLzfabS1O/RDADnmzVc01dHn44qD0BvGH6KlZNzOMW95tEpKqhkCChPA==", + "license": "MIT", "dependencies": { "@azure/abort-controller": "^2.0.0", "@azure/core-auth": "^1.4.0", "@azure/core-rest-pipeline": "^1.9.1", "@azure/core-tracing": "^1.0.0", - "@azure/core-util": "^1.0.0", + "@azure/core-util": "^1.6.1", "@azure/logger": "^1.0.0", - "tslib": "^2.2.0" + "tslib": "^2.6.2" }, "engines": { "node": ">=18.0.0" @@ -11976,6 +11974,21 @@ "node": ">=12.0.0" } }, + "node_modules/@azure/core-lro": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/@azure/core-lro/-/core-lro-2.7.2.tgz", + "integrity": "sha512-0YIpccoX8m/k00O7mDDMdJpbr6mf1yWo2dfmxt5A8XVZVVMz2SSKaEbMCeJRvgQ0IaSlqhjT47p4hVIRRy90xw==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-util": "^1.2.0", + "@azure/logger": "^1.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@azure/core-paging": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/@azure/core-paging/-/core-paging-1.5.0.tgz", @@ -11988,40 +12001,146 @@ } }, "node_modules/@azure/core-rest-pipeline": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.14.0.tgz", - "integrity": "sha512-Tp4M6NsjCmn9L5p7HsW98eSOS7A0ibl3e5ntZglozT0XuD/0y6i36iW829ZbBq0qihlGgfaeFpkLjZ418KDm1Q==", + "version": "1.19.1", + "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.19.1.tgz", + "integrity": "sha512-zHeoI3NCs53lLBbWNzQycjnYKsA1CVKlnzSNuSFcUDwBp8HHVObePxrM7HaX+Ha5Ks639H7chNC9HOaIhNS03w==", + "license": "MIT", "dependencies": { "@azure/abort-controller": "^2.0.0", - "@azure/core-auth": "^1.4.0", + "@azure/core-auth": "^1.8.0", "@azure/core-tracing": "^1.0.1", - "@azure/core-util": "^1.3.0", + "@azure/core-util": "^1.11.0", "@azure/logger": "^1.0.0", - "http-proxy-agent": "^5.0.0", - "https-proxy-agent": "^5.0.0", - "tslib": "^2.2.0" + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.0", + "tslib": "^2.6.2" }, "engines": { "node": ">=18.0.0" } }, - "node_modules/@azure/core-tracing": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.0.1.tgz", - "integrity": "sha512-I5CGMoLtX+pI17ZdiFJZgxMJApsK6jjfm85hpgp3oazCdq5Wxgh4wMr7ge/TTWW1B5WBuvIOI1fMU/FrOAMKrw==", + "node_modules/@azure/core-rest-pipeline/node_modules/agent-base": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz", + "integrity": "sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/@azure/core-rest-pipeline/node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", "dependencies": { - "tslib": "^2.2.0" + "agent-base": "^7.1.0", + "debug": "^4.3.4" }, "engines": { - "node": ">=12.0.0" + "node": ">= 14" + } + }, + "node_modules/@azure/core-rest-pipeline/node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@azure/core-tracing": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.2.0.tgz", + "integrity": "sha512-UKTiEJPkWcESPYJz3X5uKRYyOcJD+4nYph+KpfdPRnQJVrZfk0KJgdnaAWKfhsBBtAf/D58Az4AvCJEmWgIBAg==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" } }, "node_modules/@azure/core-util": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.7.0.tgz", - "integrity": "sha512-Zq2i3QO6k9DA8vnm29mYM4G8IE9u1mhF1GUabVEqPNX8Lj833gdxQ2NAFxt2BZsfAL+e9cT8SyVN7dFVJ/Hf0g==", + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.11.0.tgz", + "integrity": "sha512-DxOSLua+NdpWoSqULhjDyAZTXFdP/LKkqtYuxxz1SCN289zk3OG8UOpnCQAz/tygyACBtWp/BoO72ptK7msY8g==", + "license": "MIT", "dependencies": { "@azure/abort-controller": "^2.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-xml": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/@azure/core-xml/-/core-xml-1.4.5.tgz", + "integrity": "sha512-gT4H8mTaSXRz7eGTuQyq1aIJnJqeXzpOe9Ay7Z3FrCouer14CbV3VzjnJrNrQfbBpGBLO9oy8BmrY75A0p53cA==", + "license": "MIT", + "dependencies": { + "fast-xml-parser": "^5.0.7", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-xml/node_modules/fast-xml-parser": { + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.0.9.tgz", + "integrity": "sha512-2mBwCiuW3ycKQQ6SOesSB8WeF+fIGb6I/GG5vU5/XEptwFFhp9PE8b9O7fbs2dpq9fXn4ULR3UsfydNUCntf5A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "strnum": "^2.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, + "node_modules/@azure/core-xml/node_modules/strnum": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.0.5.tgz", + "integrity": "sha512-YAT3K/sgpCUxhxNMrrdhtod3jckkpYwH6JAuwmUdXZsmzH1wUyzTMrrK2wYCEEqlKwrWDd35NeuUkbBy/1iK+Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, + "node_modules/@azure/identity": { + "version": "4.8.0", + "resolved": "https://registry.npmjs.org/@azure/identity/-/identity-4.8.0.tgz", + "integrity": "sha512-l9ALUGHtFB/JfsqmA+9iYAp2a+cCwdNO/cyIr2y7nJLJsz1aae6qVP8XxT7Kbudg0IQRSIMXj0+iivFdbD1xPA==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-auth": "^1.9.0", + "@azure/core-client": "^1.9.2", + "@azure/core-rest-pipeline": "^1.17.0", + "@azure/core-tracing": "^1.0.0", + "@azure/core-util": "^1.11.0", + "@azure/logger": "^1.0.0", + "@azure/msal-browser": "^4.2.0", + "@azure/msal-node": "^3.2.3", + "events": "^3.0.0", + "jws": "^4.0.0", + "open": "^10.1.0", + "stoppable": "^1.1.0", "tslib": "^2.2.0" }, "engines": { @@ -12039,6 +12158,50 @@ "node": ">=14.0.0" } }, + "node_modules/@azure/msal-browser": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/@azure/msal-browser/-/msal-browser-4.7.0.tgz", + "integrity": "sha512-H4AIPhIQVe1qW4+BJaitqod6UGQiXE3juj7q2ZBsOPjuZicQaqcbnBp2gCroF/icS0+TJ9rGuyCBJbjlAqVOGA==", + "license": "MIT", + "dependencies": { + "@azure/msal-common": "15.2.1" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/@azure/msal-common": { + "version": "15.2.1", + "resolved": "https://registry.npmjs.org/@azure/msal-common/-/msal-common-15.2.1.tgz", + "integrity": "sha512-eZHtYE5OHDN0o2NahCENkczQ6ffGc0MoUSAI3hpwGpZBHJXaEQMMZPWtIx86da2L9w7uT+Tr/xgJbGwIkvTZTQ==", + "license": "MIT", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/@azure/msal-node": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@azure/msal-node/-/msal-node-3.3.0.tgz", + "integrity": "sha512-ulsT3EHF1RQ29X55cxBLgKsIKWni9JdbUqG7sipGVP4uhWcBpmm/vhKOMH340+27Acm9+kHGnN/5XmQ5LrIDgA==", + "license": "MIT", + "dependencies": { + "@azure/msal-common": "15.2.1", + "jsonwebtoken": "^9.0.0", + "uuid": "^8.3.0" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/@azure/msal-node/node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/@azure/search-documents": { "version": "12.0.0", "resolved": "https://registry.npmjs.org/@azure/search-documents/-/search-documents-12.0.0.tgz", @@ -12058,6 +12221,30 @@ "node": ">=18.0.0" } }, + "node_modules/@azure/storage-blob": { + "version": "12.27.0", + "resolved": "https://registry.npmjs.org/@azure/storage-blob/-/storage-blob-12.27.0.tgz", + "integrity": "sha512-IQjj9RIzAKatmNca3D6bT0qJ+Pkox1WZGOg2esJF2YLHb45pQKOwGPIAV+w3rfgkj7zV3RMxpn/c6iftzSOZJQ==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.4.0", + "@azure/core-client": "^1.6.2", + "@azure/core-http-compat": "^2.0.0", + "@azure/core-lro": "^2.2.0", + "@azure/core-paging": "^1.1.1", + "@azure/core-rest-pipeline": "^1.10.1", + "@azure/core-tracing": "^1.1.2", + "@azure/core-util": "^1.6.1", + "@azure/core-xml": "^1.4.3", + "@azure/logger": "^1.0.0", + "events": "^3.0.0", + "tslib": "^2.2.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.26.2", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.26.2.tgz", @@ -23563,6 +23750,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", + "devOptional": true, "engines": { "node": ">= 10" } @@ -24553,6 +24741,7 @@ "version": "6.0.2", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "devOptional": true, "dependencies": { "debug": "4" }, @@ -25638,6 +25827,21 @@ "integrity": "sha512-HpGFw18DgFWlncDfjTa2rcQ4W88O1mC8e8yZ2AvQY5KDaktSTwo+KRf6nHK6FRI5FyRyb/5T6+TSxfP7QyGsmQ==", "dev": true }, + "node_modules/bundle-name": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-4.1.0.tgz", + "integrity": "sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q==", + "license": "MIT", + "dependencies": { + "run-applescript": "^7.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/busboy": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", @@ -26957,6 +27161,34 @@ "node": ">=0.10.0" } }, + "node_modules/default-browser": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/default-browser/-/default-browser-5.2.1.tgz", + "integrity": "sha512-WY/3TUME0x3KPYdRRxEJJvXRHV4PyPoUsxtZa78lwItwRQRHhd2U9xOscaT/YTf8uCXIAjeJOFBVEh/7FtD8Xg==", + "license": "MIT", + "dependencies": { + "bundle-name": "^4.1.0", + "default-browser-id": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/default-browser-id": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/default-browser-id/-/default-browser-id-5.0.0.tgz", + "integrity": "sha512-A6p/pu/6fyBcA1TRz/GqWYPViplrftcW2gZC9q79ngNCKAeR/X3gcEdXQHl4KNXV+3wgIJ1CPkJQ3IHM6lcsyA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/define-data-property": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", @@ -26974,6 +27206,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/define-lazy-prop": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-3.0.0.tgz", + "integrity": "sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/define-properties": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", @@ -30213,6 +30457,7 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", + "devOptional": true, "dependencies": { "@tootallnate/once": "2", "agent-base": "6", @@ -30232,6 +30477,7 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "devOptional": true, "dependencies": { "agent-base": "6", "debug": "4" @@ -30865,6 +31111,21 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/is-docker": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-3.0.0.tgz", + "integrity": "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ==", + "license": "MIT", + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -30946,6 +31207,24 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/is-inside-container": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-inside-container/-/is-inside-container-1.0.0.tgz", + "integrity": "sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA==", + "license": "MIT", + "dependencies": { + "is-docker": "^3.0.0" + }, + "bin": { + "is-inside-container": "cli.js" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-map": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/is-map/-/is-map-2.0.3.tgz", @@ -31231,6 +31510,21 @@ "url": "https://github.com/sponsors/mesqueeb" } }, + "node_modules/is-wsl": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-3.1.0.tgz", + "integrity": "sha512-UcVfVfaK4Sc4m7X3dUSoHoozQGBEFeDC+zVo06t98xe8CzHSZZBekNXH+tu0NalHolcJ/QAGqS46Hef7QXBIMw==", + "license": "MIT", + "dependencies": { + "is-inside-container": "^1.0.0" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", @@ -35610,6 +35904,24 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/open": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/open/-/open-10.1.0.tgz", + "integrity": "sha512-mnkeQ1qP5Ue2wd+aivTD3NHd/lZ96Lu0jgf0pwktLPtx6cTZiH7tyeGRRHs0zX0rbrahXPnXlUnbeXyaBBuIaw==", + "license": "MIT", + "dependencies": { + "default-browser": "^5.2.1", + "define-lazy-prop": "^3.0.0", + "is-inside-container": "^1.0.0", + "is-wsl": "^3.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/openai": { "version": "4.80.1", "resolved": "https://registry.npmjs.org/openai/-/openai-4.80.1.tgz", @@ -39756,6 +40068,18 @@ "node": ">= 18" } }, + "node_modules/run-applescript": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/run-applescript/-/run-applescript-7.0.0.tgz", + "integrity": "sha512-9by4Ij99JUr/MCFBUkDKLWK3G9HVXmabKz9U5MlIAIuvuzkiOicRYs8XJLxX+xahD+mLiiCYDqF9dKAgtzKP1A==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", @@ -40524,6 +40848,16 @@ "node": ">= 0.4" } }, + "node_modules/stoppable": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/stoppable/-/stoppable-1.1.0.tgz", + "integrity": "sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw==", + "license": "MIT", + "engines": { + "node": ">=4", + "npm": ">=6" + } + }, "node_modules/stream-browserify": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-3.0.0.tgz", @@ -41692,9 +42026,10 @@ } }, "node_modules/tslib": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", - "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==" + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" }, "node_modules/tty-browserify": { "version": "0.0.1", From a7e7813a0947955e2f8f8c8e3637d9dc2b3f55fa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 19 Mar 2025 12:48:57 -0400 Subject: [PATCH 3/4] =?UTF-8?q?=E2=9A=A1=20build(deps-dev):=20bump=20@babe?= =?UTF-8?q?l/helpers=20from=207.26.9=20to=207.26.10=20(#6413)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [@babel/helpers](https://github.com/babel/babel/tree/HEAD/packages/babel-helpers) from 7.26.9 to 7.26.10. - [Release notes](https://github.com/babel/babel/releases) - [Changelog](https://github.com/babel/babel/blob/main/CHANGELOG.md) - [Commits](https://github.com/babel/babel/commits/v7.26.10/packages/babel-helpers) --- updated-dependencies: - dependency-name: "@babel/helpers" dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/package-lock.json b/package-lock.json index fa44a4ec22..f78f2e3686 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2010,21 +2010,6 @@ "node": ">=6.9.0" } }, - "client/node_modules/@babel/helpers": { - "version": "7.26.9", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.26.9.tgz", - "integrity": "sha512-Mz/4+y8udxBKdmzt/UjPACs4G3j5SshJJEFFKxlCGPydG4JAHXxjWjAwjd09tf6oINvl1VfMJo+nB7H2YKQ0dA==", - "dev": true, - "license": "MIT", - "peer": true, - "dependencies": { - "@babel/template": "^7.26.9", - "@babel/types": "^7.26.9" - }, - "engines": { - "node": ">=6.9.0" - } - }, "client/node_modules/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": { "version": "7.25.9", "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression/-/plugin-bugfix-safari-id-destructuring-collision-in-function-expression-7.25.9.tgz", @@ -12649,12 +12634,13 @@ } }, "node_modules/@babel/helpers": { - "version": "7.25.6", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.25.6.tgz", - "integrity": "sha512-Xg0tn4HcfTijTwfDwYlvVCl43V6h4KyVVX2aEm4qdO/PC6L2YvzLHFdmxhoeSA3eslcE6+ZVXHgWwopXYLNq4Q==", + "version": "7.26.10", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.26.10.tgz", + "integrity": "sha512-UPYc3SauzZ3JGgj87GgZ89JVdC5dj0AoetR5Bw6wj4niittNyFh6+eOGonYvJ1ao6B8lEa3Q3klS7ADZ53bc5g==", + "license": "MIT", "dependencies": { - "@babel/template": "^7.25.0", - "@babel/types": "^7.25.6" + "@babel/template": "^7.26.9", + "@babel/types": "^7.26.10" }, "engines": { "node": ">=6.9.0" @@ -14209,9 +14195,9 @@ } }, "node_modules/@babel/types": { - "version": "7.26.9", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.9.tgz", - "integrity": "sha512-Y3IR1cRnOxOCDvMmNiym7XpXQ93iGDDPHx+Zj+NM+rg0fBaShfQLkg+hKPaZCEvg5N/LeCo4+Rj/i3FuJsIQaw==", + "version": "7.26.10", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.10.tgz", + "integrity": "sha512-emqcG3vHrpxUKTrxcblR36dcrcoRDvKmnL/dCL6ZsHaShW80qxCAcNhzQZrpeM765VzEos+xOi4s+r4IXzTwdQ==", "license": "MIT", "dependencies": { "@babel/helper-string-parser": "^7.25.9", From 692fba51d84c4319a682bf96756ac3087821b47a Mon Sep 17 00:00:00 2001 From: Ruben Talstra Date: Thu, 20 Mar 2025 14:00:59 +0100 Subject: [PATCH 4/4] =?UTF-8?q?=F0=9F=9A=80=20feat:=20Add=20support=20for?= =?UTF-8?q?=20custom=20AWS=20endpoint=20in=20S3=20initialization=20(#6431)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 1 + api/server/services/Files/S3/initialize.js | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index d9a4d52d92..54f9c4a96c 100644 --- a/.env.example +++ b/.env.example @@ -480,6 +480,7 @@ FIREBASE_APP_ID= # S3 AWS Bucket # #========================# +AWS_ENDPOINT_URL= AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_REGION= diff --git a/api/server/services/Files/S3/initialize.js b/api/server/services/Files/S3/initialize.js index d85945f708..2daec25235 100644 --- a/api/server/services/Files/S3/initialize.js +++ b/api/server/services/Files/S3/initialize.js @@ -9,6 +9,8 @@ let s3 = null; * If AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are provided, they will be used. * Otherwise, the AWS SDK's default credentials chain (including IRSA) is used. * + * If AWS_ENDPOINT_URL is provided, it will be used as the endpoint. + * * @returns {S3Client|null} An instance of S3Client if the region is provided; otherwise, null. */ const initializeS3 = () => { @@ -22,18 +24,26 @@ const initializeS3 = () => { return null; } + // Read the custom endpoint if provided. + const endpoint = process.env.AWS_ENDPOINT_URL; const accessKeyId = process.env.AWS_ACCESS_KEY_ID; const secretAccessKey = process.env.AWS_SECRET_ACCESS_KEY; + const config = { + region, + // Conditionally add the endpoint if it is provided + ...(endpoint ? { endpoint } : {}), + }; + if (accessKeyId && secretAccessKey) { s3 = new S3Client({ - region, + ...config, credentials: { accessKeyId, secretAccessKey }, }); logger.info('[initializeS3] S3 initialized with provided credentials.'); } else { // When using IRSA, credentials are automatically provided via the IAM Role attached to the ServiceAccount. - s3 = new S3Client({ region }); + s3 = new S3Client(config); logger.info('[initializeS3] S3 initialized using default credentials (IRSA).'); }