From 49ee88b6e8d20b71df7496d5c39c41df05090571 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Thu, 7 Nov 2024 11:11:20 -0500 Subject: [PATCH] =?UTF-8?q?=E2=9A=99=EF=B8=8F=20fix:=20File=20Config=20Han?= =?UTF-8?q?dling=20(#4664)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: typing * refactor: create file filter from custom fileConfig, if provided * refactor: use logger utility to avoid overly verbose axios error logs when using RAG_API * fix(useFileHandling): use memoization/callbacks to make sure the appropriate fileConfig is used; refactor: move endpoint to first field applied to formdata * chore: update librechat-data-provider version to 0.7.54 * chore: revert type change --- api/server/routes/files/multer.js | 36 +++-- api/server/services/Files/VectorDB/crud.js | 11 +- client/src/hooks/Files/useFileHandling.ts | 154 +++++++++++---------- package-lock.json | 2 +- packages/data-provider/package.json | 2 +- 5 files changed, 123 insertions(+), 82 deletions(-) diff --git a/api/server/routes/files/multer.js b/api/server/routes/files/multer.js index 4f0d38f22f..e37ae49fc1 100644 --- a/api/server/routes/files/multer.js +++ b/api/server/routes/files/multer.js @@ -30,21 +30,41 @@ const importFileFilter = (req, file, cb) => { } }; -const fileFilter = (req, file, cb) => { - if (!file) { - return cb(new Error('No file provided'), false); - } +/** + * + * @param {import('librechat-data-provider').FileConfig | undefined} customFileConfig + */ +const createFileFilter = (customFileConfig) => { + /** + * @param {ServerRequest} req + * @param {Express.Multer.File} + * @param {import('multer').FileFilterCallback} cb + */ + const fileFilter = (req, file, cb) => { + if (!file) { + return cb(new Error('No file provided'), false); + } - if (!defaultFileConfig.checkType(file.mimetype)) { - return cb(new Error('Unsupported file type: ' + file.mimetype), false); - } + const endpoint = req.body.endpoint; + const supportedTypes = + customFileConfig?.endpoints?.[endpoint]?.supportedMimeTypes ?? + customFileConfig?.endpoints?.default.supportedMimeTypes ?? + defaultFileConfig?.endpoints?.[endpoint]?.supportedMimeTypes; - cb(null, true); + if (!defaultFileConfig.checkType(file.mimetype, supportedTypes)) { + return cb(new Error('Unsupported file type: ' + file.mimetype), false); + } + + cb(null, true); + }; + + return fileFilter; }; const createMulterInstance = async () => { const customConfig = await getCustomConfig(); const fileConfig = mergeFileConfig(customConfig?.fileConfig); + const fileFilter = createFileFilter(fileConfig); return multer({ storage, fileFilter, diff --git a/api/server/services/Files/VectorDB/crud.js b/api/server/services/Files/VectorDB/crud.js index 3d16c93e15..a4d48064d7 100644 --- a/api/server/services/Files/VectorDB/crud.js +++ b/api/server/services/Files/VectorDB/crud.js @@ -2,6 +2,7 @@ const fs = require('fs'); const axios = require('axios'); const FormData = require('form-data'); const { FileSources } = require('librechat-data-provider'); +const { logAxiosError } = require('~/utils'); const { logger } = require('~/config'); /** @@ -32,7 +33,10 @@ const deleteVectors = async (req, file) => { data: [file.file_id], }); } catch (error) { - logger.error('Error deleting vectors', error); + logAxiosError({ + error, + message: 'Error deleting vectors', + }); throw new Error(error.message || 'An error occurred during file deletion.'); } }; @@ -91,7 +95,10 @@ async function uploadVectors({ req, file, file_id }) { embedded: Boolean(responseData.known_type), }; } catch (error) { - logger.error('Error embedding file', error); + logAxiosError({ + error, + message: 'Error uploading vectors', + }); throw new Error(error.message || 'An error occurred during file upload.'); } } diff --git a/client/src/hooks/Files/useFileHandling.ts b/client/src/hooks/Files/useFileHandling.ts index 37dc37f896..251e043901 100644 --- a/client/src/hooks/Files/useFileHandling.ts +++ b/client/src/hooks/Files/useFileHandling.ts @@ -1,7 +1,7 @@ import { v4 } from 'uuid'; import debounce from 'lodash/debounce'; import { useQueryClient } from '@tanstack/react-query'; -import { useState, useEffect, useCallback, useRef } from 'react'; +import { useState, useEffect, useCallback, useRef, useMemo } from 'react'; import { megabyte, QueryKeys, @@ -47,14 +47,24 @@ const useFileHandling = (params?: UseFileHandling) => { const agent_id = params?.additionalMetadata?.agent_id ?? ''; const assistant_id = params?.additionalMetadata?.assistant_id ?? ''; - const { data: fileConfig = defaultFileConfig } = useGetFileConfig({ + const { data: fileConfig = null } = useGetFileConfig({ select: (data) => mergeFileConfig(data), }); - const endpoint = - params?.overrideEndpoint ?? conversation?.endpointType ?? conversation?.endpoint ?? 'default'; - const { fileLimit, fileSizeLimit, totalSizeLimit, supportedMimeTypes } = - fileConfig.endpoints[endpoint] ?? fileConfig.endpoints.default; + const endpoint = useMemo( + () => + params?.overrideEndpoint ?? conversation?.endpointType ?? conversation?.endpoint ?? 'default', + [params?.overrideEndpoint, conversation?.endpointType, conversation?.endpoint], + ); + + const { fileLimit, fileSizeLimit, totalSizeLimit, supportedMimeTypes } = useMemo( + () => + fileConfig?.endpoints[endpoint] ?? + fileConfig?.endpoints.default ?? + defaultFileConfig.endpoints[endpoint] ?? + defaultFileConfig.endpoints.default, + [fileConfig, endpoint], + ); const displayToast = useCallback(() => { if (errors.length > 1) { @@ -146,6 +156,7 @@ const useFileHandling = (params?: UseFileHandling) => { startUploadTimer(extendedFile.file_id, filename, extendedFile.size); const formData = new FormData(); + formData.append('endpoint', endpoint); formData.append('file', extendedFile.file as File, encodeURIComponent(filename)); formData.append('file_id', extendedFile.file_id); @@ -167,8 +178,6 @@ const useFileHandling = (params?: UseFileHandling) => { } } - formData.append('endpoint', endpoint); - if (!isAssistantsEndpoint(endpoint)) { uploadFile.mutate(formData); return; @@ -203,81 +212,86 @@ const useFileHandling = (params?: UseFileHandling) => { uploadFile.mutate(formData); }; - const validateFiles = (fileList: File[]) => { - const existingFiles = Array.from(files.values()); - const incomingTotalSize = fileList.reduce((total, file) => total + file.size, 0); - if (incomingTotalSize === 0) { - setError('com_error_files_empty'); - return false; - } - const currentTotalSize = existingFiles.reduce((total, file) => total + file.size, 0); - - if (fileList.length + files.size > fileLimit) { - setError(`You can only upload up to ${fileLimit} files at a time.`); - return false; - } - - for (let i = 0; i < fileList.length; i++) { - let originalFile = fileList[i]; - let fileType = originalFile.type; - const extension = originalFile.name.split('.').pop() ?? ''; - const knownCodeType = codeTypeMapping[extension]; - - // Infer MIME type for Known Code files when the type is empty or a mismatch - if (knownCodeType && (!fileType || fileType !== knownCodeType)) { - fileType = knownCodeType; + const validateFiles = useCallback( + (fileList: File[]) => { + const existingFiles = Array.from(files.values()); + const incomingTotalSize = fileList.reduce((total, file) => total + file.size, 0); + if (incomingTotalSize === 0) { + setError('com_error_files_empty'); + return false; } + const currentTotalSize = existingFiles.reduce((total, file) => total + file.size, 0); - // Check if the file type is still empty after the extension check - if (!fileType) { - setError('Unable to determine file type for: ' + originalFile.name); + if (fileList.length + files.size > fileLimit) { + setError(`You can only upload up to ${fileLimit} files at a time.`); return false; } - // Replace empty type with inferred type - if (originalFile.type !== fileType) { - const newFile = new File([originalFile], originalFile.name, { type: fileType }); - originalFile = newFile; - fileList[i] = newFile; + for (let i = 0; i < fileList.length; i++) { + let originalFile = fileList[i]; + let fileType = originalFile.type; + const extension = originalFile.name.split('.').pop() ?? ''; + const knownCodeType = codeTypeMapping[extension]; + + // Infer MIME type for Known Code files when the type is empty or a mismatch + if (knownCodeType && (!fileType || fileType !== knownCodeType)) { + fileType = knownCodeType; + } + + // Check if the file type is still empty after the extension check + if (!fileType) { + setError('Unable to determine file type for: ' + originalFile.name); + return false; + } + + // Replace empty type with inferred type + if (originalFile.type !== fileType) { + const newFile = new File([originalFile], originalFile.name, { type: fileType }); + originalFile = newFile; + fileList[i] = newFile; + } + + if (!checkType(originalFile.type, supportedMimeTypes)) { + console.log(originalFile); + setError('Currently, unsupported file type: ' + originalFile.type); + return false; + } + + if (originalFile.size >= fileSizeLimit) { + setError(`File size exceeds ${fileSizeLimit / megabyte} MB.`); + return false; + } } - if (!checkType(originalFile.type, supportedMimeTypes)) { - console.log(originalFile); - setError('Currently, unsupported file type: ' + originalFile.type); + if (currentTotalSize + incomingTotalSize > totalSizeLimit) { + setError(`The total size of the files cannot exceed ${totalSizeLimit / megabyte} MB.`); return false; } - if (originalFile.size >= fileSizeLimit) { - setError(`File size exceeds ${fileSizeLimit / megabyte} MB.`); + const combinedFilesInfo = [ + ...existingFiles.map( + (file) => + `${file.file?.name ?? file.filename}-${file.size}-${ + file.type?.split('/')[0] ?? 'file' + }`, + ), + ...fileList.map( + (file: File | undefined) => + `${file?.name}-${file?.size}-${file?.type.split('/')[0] ?? 'file'}`, + ), + ]; + + const uniqueFilesSet = new Set(combinedFilesInfo); + + if (uniqueFilesSet.size !== combinedFilesInfo.length) { + setError('com_error_files_dupe'); return false; } - } - if (currentTotalSize + incomingTotalSize > totalSizeLimit) { - setError(`The total size of the files cannot exceed ${totalSizeLimit / megabyte} MB.`); - return false; - } - - const combinedFilesInfo = [ - ...existingFiles.map( - (file) => - `${file.file?.name ?? file.filename}-${file.size}-${file.type?.split('/')[0] ?? 'file'}`, - ), - ...fileList.map( - (file: File | undefined) => - `${file?.name}-${file?.size}-${file?.type.split('/')[0] ?? 'file'}`, - ), - ]; - - const uniqueFilesSet = new Set(combinedFilesInfo); - - if (uniqueFilesSet.size !== combinedFilesInfo.length) { - setError('com_error_files_dupe'); - return false; - } - - return true; - }; + return true; + }, + [files, fileLimit, fileSizeLimit, totalSizeLimit, supportedMimeTypes], + ); const loadImage = (extendedFile: ExtendedFile, preview: string) => { const img = new Image(); diff --git a/package-lock.json b/package-lock.json index 74536ed1bb..ffe6d4cb38 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36283,7 +36283,7 @@ }, "packages/data-provider": { "name": "librechat-data-provider", - "version": "0.7.53", + "version": "0.7.54", "license": "ISC", "dependencies": { "@types/js-yaml": "^4.0.9", diff --git a/packages/data-provider/package.json b/packages/data-provider/package.json index bfc6776dae..483640a766 100644 --- a/packages/data-provider/package.json +++ b/packages/data-provider/package.json @@ -1,6 +1,6 @@ { "name": "librechat-data-provider", - "version": "0.7.53", + "version": "0.7.54", "description": "data services for librechat apps", "main": "dist/index.js", "module": "dist/index.es.js",