📂 refactor: Cleanup File Filtering Logic, Improve Validation (#10414)

* feat: add filterFilesByEndpointConfig to filter disabled file processing by provider

* chore: explicit define of endpointFileConfig for better debugging

* refactor: move `normalizeEndpointName` to data-provider as used app-wide

* chore: remove overrideEndpoint from useFileHandling

* refactor: improve endpoint file config selection

* refactor: update filterFilesByEndpointConfig to accept structured parameters and improve endpoint file config handling

* refactor: replace defaultFileConfig with getEndpointFileConfig for improved file configuration handling across components

* test: add comprehensive unit tests for getEndpointFileConfig to validate endpoint configuration handling

* refactor: streamline agent endpoint assignment and improve file filtering logic

* feat: add error handling for disabled file uploads in endpoint configuration

* refactor: update encodeAndFormat functions to accept structured parameters for provider and endpoint

* refactor: streamline requestFiles handling in initializeAgent function

* fix: getEndpointFileConfig partial config merging scenarios

* refactor: enhance mergeWithDefault function to support document-supported providers with comprehensive MIME types

* refactor: user-configured default file config in getEndpointFileConfig

* fix: prevent file handling when endpoint is disabled and file is dragged to chat

* refactor: move `getEndpointField` to `data-provider` and update usage across components and hooks

* fix: prioritize endpointType based on agent.endpoint in file filtering logic

* fix: prioritize agent.endpoint in file filtering logic and remove unnecessary endpointType defaulting

api/server/controllers/agents/client.js

@@ -210,7 +210,10 @@ class AgentClient extends BaseClient {
     const { files, image_urls } = await encodeAndFormat(
       this.options.req,
       attachments,
-      this.options.agent.provider,
+      {
+        provider: this.options.agent.provider,
+        endpoint: this.options.endpoint,
+      },
       VisionModes.agents,
     );
     message.image_urls = image_urls.length ? image_urls : undefined;

api/server/routes/files/multer.js

@@ -3,7 +3,11 @@ const path = require('path');
 const crypto = require('crypto');
 const multer = require('multer');
 const { sanitizeFilename } = require('@librechat/api');
-const { fileConfig: defaultFileConfig, mergeFileConfig } = require('librechat-data-provider');
+const {
+  mergeFileConfig,
+  getEndpointFileConfig,
+  fileConfig: defaultFileConfig,
+} = require('librechat-data-provider');
 const { getAppConfig } = require('~/server/services/Config');
 
 const storage = multer.diskStorage({
@@ -53,12 +57,14 @@ const createFileFilter = (customFileConfig) => {
     }
 
     const endpoint = req.body.endpoint;
-    const supportedTypes =
-      customFileConfig?.endpoints?.[endpoint]?.supportedMimeTypes ??
-      customFileConfig?.endpoints?.default.supportedMimeTypes ??
-      defaultFileConfig?.endpoints?.[endpoint]?.supportedMimeTypes;
+    const endpointType = req.body.endpointType;
+    const endpointFileConfig = getEndpointFileConfig({
+      fileConfig: customFileConfig,
+      endpoint,
+      endpointType,
+    });
 
-    if (!defaultFileConfig.checkType(file.mimetype, supportedTypes)) {
+    if (!defaultFileConfig.checkType(file.mimetype, endpointFileConfig.supportedMimeTypes)) {
       return cb(new Error('Unsupported file type: ' + file.mimetype), false);
     }
 

api/server/services/Config/getEndpointsConfig.js

@@ -109,7 +109,7 @@ async function getEndpointsConfig(req) {
  * @returns {Promise<boolean>}
  */
 const checkCapability = async (req, capability) => {
-  const isAgents = isAgentsEndpoint(req.body?.original_endpoint || req.body?.endpoint);
+  const isAgents = isAgentsEndpoint(req.body?.endpointType || req.body?.endpoint);
   const endpointsConfig = await getEndpointsConfig(req);
   const capabilities =
     isAgents || endpointsConfig?.[EModelEndpoint.agents]?.capabilities != null

api/server/services/Config/loadConfigModels.js

@@ -1,5 +1,9 @@
-const { isUserProvided, normalizeEndpointName } = require('@librechat/api');
-const { EModelEndpoint, extractEnvVariable } = require('librechat-data-provider');
+const { isUserProvided } = require('@librechat/api');
+const {
+  EModelEndpoint,
+  extractEnvVariable,
+  normalizeEndpointName,
+} = require('librechat-data-provider');
 const { fetchModels } = require('~/server/services/ModelService');
 const { getAppConfig } = require('./app');
 

api/server/services/Endpoints/agents/agent.js

@@ -3,12 +3,14 @@ const {
   primeResources,
   getModelMaxTokens,
   extractLibreChatParams,
+  filterFilesByEndpointConfig,
   optionalChainWithEmptyCheck,
 } = require('@librechat/api');
 const {
   ErrorTypes,
   EModelEndpoint,
   EToolResources,
+  paramEndpoints,
   isAgentsEndpoint,
   replaceSpecialVars,
   providerEndpointMap,
@@ -71,6 +73,9 @@ const initializeAgent = async ({
 
   const { resendFiles, maxContextTokens, modelOptions } = extractLibreChatParams(_modelOptions);
 
+  const provider = agent.provider;
+  agent.endpoint = provider;
+
   if (isInitialAgent && conversationId != null && resendFiles) {
     const fileIds = (await getConvoFiles(conversationId)) ?? [];
     /** @type {Set<EToolResources>} */
@@ -88,6 +93,19 @@ const initializeAgent = async ({
     currentFiles = await processFiles(requestFiles);
   }
 
+  if (currentFiles && currentFiles.length) {
+    let endpointType;
+    if (!paramEndpoints.has(agent.endpoint)) {
+      endpointType = EModelEndpoint.custom;
+    }
+
+    currentFiles = filterFilesByEndpointConfig(req, {
+      files: currentFiles,
+      endpoint: agent.endpoint,
+      endpointType,
+    });
+  }
+
   const { attachments, tool_resources } = await primeResources({
     req,
     getFiles,
@@ -98,7 +116,6 @@ const initializeAgent = async ({
     requestFileSet: new Set(requestFiles?.map((file) => file.file_id)),
   });
 
-  const provider = agent.provider;
   const {
     tools: structuredTools,
     toolContextMap,
@@ -113,7 +130,6 @@ const initializeAgent = async ({
     tool_resources,
   })) ?? {};
 
-  agent.endpoint = provider;
   const { getOptions, overrideProvider } = getProviderConfig({ provider, appConfig });
   if (overrideProvider !== agent.provider) {
     agent.provider = overrideProvider;

api/server/services/Files/images/encode.js

@@ -84,11 +84,15 @@ const blobStorageSources = new Set([FileSources.azure_blob, FileSources.s3]);
  * Encodes and formats the given files.
  * @param {ServerRequest} req - The request object.
  * @param {Array<MongoFile>} files - The array of files to encode and format.
- * @param {EModelEndpoint} [endpoint] - Optional: The endpoint for the image.
+ * @param {object} params - Object containing provider/endpoint information
+ * @param {Providers | EModelEndpoint | string} [params.provider] - The provider for the image
+ * @param {string} [params.endpoint] - Optional: The endpoint for the image
  * @param {string} [mode] - Optional: The endpoint mode for the image.
  * @returns {Promise<{ files: MongoFile[]; image_urls: MessageContentImageUrl[] }>} - A promise that resolves to the result object containing the encoded images and file details.
  */
-async function encodeAndFormat(req, files, endpoint, mode) {
+async function encodeAndFormat(req, files, params, mode) {
+  const { provider, endpoint } = params;
+  const effectiveEndpoint = endpoint ?? provider;
   const promises = [];
   /** @type {Record<FileSources, Pick<ReturnType<typeof getStrategyFunctions>, 'prepareImagePayload' | 'getDownloadStream'>>} */
   const encodingMethods = {};
@@ -134,7 +138,7 @@ async function encodeAndFormat(req, files, endpoint, mode) {
       } catch (error) {
         logger.error('Error processing image from blob storage:', error);
       }
-    } else if (source !== FileSources.local && base64Only.has(endpoint)) {
+    } else if (source !== FileSources.local && base64Only.has(effectiveEndpoint)) {
       const [_file, imageURL] = await preparePayload(req, file);
       promises.push([_file, await fetchImageToBase64(imageURL)]);
       continue;
@@ -184,15 +188,19 @@ async function encodeAndFormat(req, files, endpoint, mode) {
       continue;
     }
 
-    if (endpoint && endpoint === EModelEndpoint.google && mode === VisionModes.generative) {
+    if (
+      effectiveEndpoint &&
+      effectiveEndpoint === EModelEndpoint.google &&
+      mode === VisionModes.generative
+    ) {
       delete imagePart.image_url;
       imagePart.inlineData = {
         mimeType: file.type,
         data: imageContent,
       };
-    } else if (endpoint && endpoint === EModelEndpoint.google) {
+    } else if (effectiveEndpoint && effectiveEndpoint === EModelEndpoint.google) {
       imagePart.image_url = imagePart.image_url.url;
-    } else if (endpoint && endpoint === EModelEndpoint.anthropic) {
+    } else if (effectiveEndpoint && effectiveEndpoint === EModelEndpoint.anthropic) {
       imagePart.type = 'image';
       imagePart.source = {
         type: 'base64',

api/server/services/Files/process.js

@@ -15,6 +15,7 @@ const {
   checkOpenAIStorage,
   removeNullishValues,
   isAssistantsEndpoint,
+  getEndpointFileConfig,
 } = require('librechat-data-provider');
 const { EnvVar } = require('@librechat/agents');
 const { logger } = require('@librechat/data-schemas');
@@ -994,7 +995,7 @@ async function saveBase64Image(
  */
 function filterFile({ req, image, isAvatar }) {
   const { file } = req;
-  const { endpoint, file_id, width, height } = req.body;
+  const { endpoint, endpointType, file_id, width, height } = req.body;
 
   if (!file_id && !isAvatar) {
     throw new Error('No file_id provided');
@@ -1016,9 +1017,13 @@ function filterFile({ req, image, isAvatar }) {
   const appConfig = req.config;
   const fileConfig = mergeFileConfig(appConfig.fileConfig);
 
-  const { fileSizeLimit: sizeLimit, supportedMimeTypes } =
-    fileConfig.endpoints[endpoint] ?? fileConfig.endpoints.default;
-  const fileSizeLimit = isAvatar === true ? fileConfig.avatarSizeLimit : sizeLimit;
+  const endpointFileConfig = getEndpointFileConfig({
+    endpoint,
+    fileConfig,
+    endpointType,
+  });
+  const fileSizeLimit =
+    isAvatar === true ? fileConfig.avatarSizeLimit : endpointFileConfig.fileSizeLimit;
 
   if (file.size > fileSizeLimit) {
     throw new Error(
@@ -1028,7 +1033,10 @@ function filterFile({ req, image, isAvatar }) {
     );
   }
 
-  const isSupportedMimeType = fileConfig.checkType(file.mimetype, supportedMimeTypes);
+  const isSupportedMimeType = fileConfig.checkType(
+    file.mimetype,
+    endpointFileConfig.supportedMimeTypes,
+  );
 
   if (!isSupportedMimeType) {
     throw new Error('Unsupported file type');