⚙️ fix: File Config Handling (#4664)

* chore: typing

* refactor: create file filter from custom fileConfig, if provided

* refactor: use logger utility to avoid overly verbose axios error logs when using RAG_API

* fix(useFileHandling): use memoization/callbacks to make sure the appropriate fileConfig is used; refactor: move endpoint to first field applied to formdata

* chore: update librechat-data-provider version to 0.7.54

* chore: revert type change

api/server/routes/files/multer.js

@@ -30,21 +30,41 @@ const importFileFilter = (req, file, cb) => {
   }
 };
 
-const fileFilter = (req, file, cb) => {
-  if (!file) {
-    return cb(new Error('No file provided'), false);
-  }
+/**
+ *
+ * @param {import('librechat-data-provider').FileConfig | undefined} customFileConfig
+ */
+const createFileFilter = (customFileConfig) => {
+  /**
+   * @param {ServerRequest} req
+   * @param {Express.Multer.File}
+   * @param {import('multer').FileFilterCallback} cb
+   */
+  const fileFilter = (req, file, cb) => {
+    if (!file) {
+      return cb(new Error('No file provided'), false);
+    }
 
-  if (!defaultFileConfig.checkType(file.mimetype)) {
-    return cb(new Error('Unsupported file type: ' + file.mimetype), false);
-  }
+    const endpoint = req.body.endpoint;
+    const supportedTypes =
+      customFileConfig?.endpoints?.[endpoint]?.supportedMimeTypes ??
+      customFileConfig?.endpoints?.default.supportedMimeTypes ??
+      defaultFileConfig?.endpoints?.[endpoint]?.supportedMimeTypes;
 
-  cb(null, true);
+    if (!defaultFileConfig.checkType(file.mimetype, supportedTypes)) {
+      return cb(new Error('Unsupported file type: ' + file.mimetype), false);
+    }
+
+    cb(null, true);
+  };
+
+  return fileFilter;
 };
 
 const createMulterInstance = async () => {
   const customConfig = await getCustomConfig();
   const fileConfig = mergeFileConfig(customConfig?.fileConfig);
+  const fileFilter = createFileFilter(fileConfig);
   return multer({
     storage,
     fileFilter,

api/server/services/Files/VectorDB/crud.js

@@ -2,6 +2,7 @@ const fs = require('fs');
 const axios = require('axios');
 const FormData = require('form-data');
 const { FileSources } = require('librechat-data-provider');
+const { logAxiosError } = require('~/utils');
 const { logger } = require('~/config');
 
 /**
@@ -32,7 +33,10 @@ const deleteVectors = async (req, file) => {
       data: [file.file_id],
     });
   } catch (error) {
-    logger.error('Error deleting vectors', error);
+    logAxiosError({
+      error,
+      message: 'Error deleting vectors',
+    });
     throw new Error(error.message || 'An error occurred during file deletion.');
   }
 };
@@ -91,7 +95,10 @@ async function uploadVectors({ req, file, file_id }) {
       embedded: Boolean(responseData.known_type),
     };
   } catch (error) {
-    logger.error('Error embedding file', error);
+    logAxiosError({
+      error,
+      message: 'Error uploading vectors',
+    });
     throw new Error(error.message || 'An error occurred during file upload.');
   }
 }

client/src/hooks/Files/useFileHandling.ts

@@ -1,7 +1,7 @@
 import { v4 } from 'uuid';
 import debounce from 'lodash/debounce';
 import { useQueryClient } from '@tanstack/react-query';
-import { useState, useEffect, useCallback, useRef } from 'react';
+import { useState, useEffect, useCallback, useRef, useMemo } from 'react';
 import {
   megabyte,
   QueryKeys,
@@ -47,14 +47,24 @@ const useFileHandling = (params?: UseFileHandling) => {
   const agent_id = params?.additionalMetadata?.agent_id ?? '';
   const assistant_id = params?.additionalMetadata?.assistant_id ?? '';
 
-  const { data: fileConfig = defaultFileConfig } = useGetFileConfig({
+  const { data: fileConfig = null } = useGetFileConfig({
     select: (data) => mergeFileConfig(data),
   });
-  const endpoint =
-    params?.overrideEndpoint ?? conversation?.endpointType ?? conversation?.endpoint ?? 'default';
 
-  const { fileLimit, fileSizeLimit, totalSizeLimit, supportedMimeTypes } =
-    fileConfig.endpoints[endpoint] ?? fileConfig.endpoints.default;
+  const endpoint = useMemo(
+    () =>
+      params?.overrideEndpoint ?? conversation?.endpointType ?? conversation?.endpoint ?? 'default',
+    [params?.overrideEndpoint, conversation?.endpointType, conversation?.endpoint],
+  );
+
+  const { fileLimit, fileSizeLimit, totalSizeLimit, supportedMimeTypes } = useMemo(
+    () =>
+      fileConfig?.endpoints[endpoint] ??
+      fileConfig?.endpoints.default ??
+      defaultFileConfig.endpoints[endpoint] ??
+      defaultFileConfig.endpoints.default,
+    [fileConfig, endpoint],
+  );
 
   const displayToast = useCallback(() => {
     if (errors.length > 1) {
@@ -146,6 +156,7 @@ const useFileHandling = (params?: UseFileHandling) => {
     startUploadTimer(extendedFile.file_id, filename, extendedFile.size);
 
     const formData = new FormData();
+    formData.append('endpoint', endpoint);
     formData.append('file', extendedFile.file as File, encodeURIComponent(filename));
     formData.append('file_id', extendedFile.file_id);
 
@@ -167,8 +178,6 @@ const useFileHandling = (params?: UseFileHandling) => {
       }
     }
 
-    formData.append('endpoint', endpoint);
-
     if (!isAssistantsEndpoint(endpoint)) {
       uploadFile.mutate(formData);
       return;
@@ -203,81 +212,86 @@ const useFileHandling = (params?: UseFileHandling) => {
     uploadFile.mutate(formData);
   };
 
-  const validateFiles = (fileList: File[]) => {
-    const existingFiles = Array.from(files.values());
-    const incomingTotalSize = fileList.reduce((total, file) => total + file.size, 0);
-    if (incomingTotalSize === 0) {
-      setError('com_error_files_empty');
-      return false;
-    }
-    const currentTotalSize = existingFiles.reduce((total, file) => total + file.size, 0);
-
-    if (fileList.length + files.size > fileLimit) {
-      setError(`You can only upload up to ${fileLimit} files at a time.`);
-      return false;
-    }
-
-    for (let i = 0; i < fileList.length; i++) {
-      let originalFile = fileList[i];
-      let fileType = originalFile.type;
-      const extension = originalFile.name.split('.').pop() ?? '';
-      const knownCodeType = codeTypeMapping[extension];
-
-      // Infer MIME type for Known Code files when the type is empty or a mismatch
-      if (knownCodeType && (!fileType || fileType !== knownCodeType)) {
-        fileType = knownCodeType;
+  const validateFiles = useCallback(
+    (fileList: File[]) => {
+      const existingFiles = Array.from(files.values());
+      const incomingTotalSize = fileList.reduce((total, file) => total + file.size, 0);
+      if (incomingTotalSize === 0) {
+        setError('com_error_files_empty');
+        return false;
       }
+      const currentTotalSize = existingFiles.reduce((total, file) => total + file.size, 0);
 
-      // Check if the file type is still empty after the extension check
-      if (!fileType) {
-        setError('Unable to determine file type for: ' + originalFile.name);
+      if (fileList.length + files.size > fileLimit) {
+        setError(`You can only upload up to ${fileLimit} files at a time.`);
         return false;
       }
 
-      // Replace empty type with inferred type
-      if (originalFile.type !== fileType) {
-        const newFile = new File([originalFile], originalFile.name, { type: fileType });
-        originalFile = newFile;
-        fileList[i] = newFile;
+      for (let i = 0; i < fileList.length; i++) {
+        let originalFile = fileList[i];
+        let fileType = originalFile.type;
+        const extension = originalFile.name.split('.').pop() ?? '';
+        const knownCodeType = codeTypeMapping[extension];
+
+        // Infer MIME type for Known Code files when the type is empty or a mismatch
+        if (knownCodeType && (!fileType || fileType !== knownCodeType)) {
+          fileType = knownCodeType;
+        }
+
+        // Check if the file type is still empty after the extension check
+        if (!fileType) {
+          setError('Unable to determine file type for: ' + originalFile.name);
+          return false;
+        }
+
+        // Replace empty type with inferred type
+        if (originalFile.type !== fileType) {
+          const newFile = new File([originalFile], originalFile.name, { type: fileType });
+          originalFile = newFile;
+          fileList[i] = newFile;
+        }
+
+        if (!checkType(originalFile.type, supportedMimeTypes)) {
+          console.log(originalFile);
+          setError('Currently, unsupported file type: ' + originalFile.type);
+          return false;
+        }
+
+        if (originalFile.size >= fileSizeLimit) {
+          setError(`File size exceeds ${fileSizeLimit / megabyte} MB.`);
+          return false;
+        }
       }
 
-      if (!checkType(originalFile.type, supportedMimeTypes)) {
-        console.log(originalFile);
-        setError('Currently, unsupported file type: ' + originalFile.type);
+      if (currentTotalSize + incomingTotalSize > totalSizeLimit) {
+        setError(`The total size of the files cannot exceed ${totalSizeLimit / megabyte} MB.`);
         return false;
       }
 
-      if (originalFile.size >= fileSizeLimit) {
-        setError(`File size exceeds ${fileSizeLimit / megabyte} MB.`);
+      const combinedFilesInfo = [
+        ...existingFiles.map(
+          (file) =>
+            `${file.file?.name ?? file.filename}-${file.size}-${
+              file.type?.split('/')[0] ?? 'file'
+            }`,
+        ),
+        ...fileList.map(
+          (file: File | undefined) =>
+            `${file?.name}-${file?.size}-${file?.type.split('/')[0] ?? 'file'}`,
+        ),
+      ];
+
+      const uniqueFilesSet = new Set(combinedFilesInfo);
+
+      if (uniqueFilesSet.size !== combinedFilesInfo.length) {
+        setError('com_error_files_dupe');
         return false;
       }
-    }
 
-    if (currentTotalSize + incomingTotalSize > totalSizeLimit) {
-      setError(`The total size of the files cannot exceed ${totalSizeLimit / megabyte} MB.`);
-      return false;
-    }
-
-    const combinedFilesInfo = [
-      ...existingFiles.map(
-        (file) =>
-          `${file.file?.name ?? file.filename}-${file.size}-${file.type?.split('/')[0] ?? 'file'}`,
-      ),
-      ...fileList.map(
-        (file: File | undefined) =>
-          `${file?.name}-${file?.size}-${file?.type.split('/')[0] ?? 'file'}`,
-      ),
-    ];
-
-    const uniqueFilesSet = new Set(combinedFilesInfo);
-
-    if (uniqueFilesSet.size !== combinedFilesInfo.length) {
-      setError('com_error_files_dupe');
-      return false;
-    }
-
-    return true;
-  };
+      return true;
+    },
+    [files, fileLimit, fileSizeLimit, totalSizeLimit, supportedMimeTypes],
+  );
 
   const loadImage = (extendedFile: ExtendedFile, preview: string) => {
     const img = new Image();

package-lock.json

@@ -36283,7 +36283,7 @@
     },
     "packages/data-provider": {
       "name": "librechat-data-provider",
-      "version": "0.7.53",
+      "version": "0.7.54",
       "license": "ISC",
       "dependencies": {
         "@types/js-yaml": "^4.0.9",

packages/data-provider/package.json

@@ -1,6 +1,6 @@
 {
   "name": "librechat-data-provider",
-  "version": "0.7.53",
+  "version": "0.7.54",
   "description": "data services for librechat apps",
   "main": "dist/index.js",
   "module": "dist/index.es.js",