Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-12-17 08:50:15 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

⚙️ fix: File Config Handling (#4664)

Browse source 
* chore: typing

* refactor: create file filter from custom fileConfig, if provided

* refactor: use logger utility to avoid overly verbose axios error logs when using RAG_API

* fix(useFileHandling): use memoization/callbacks to make sure the appropriate fileConfig is used; refactor: move endpoint to first field applied to formdata

* chore: update librechat-data-provider version to 0.7.54

* chore: revert type change
This commit is contained in:
Danny Avila 2024-11-07 11:11:20 -05:00 • committed by GitHub
parent d60a0af878
commit 49ee88b6e8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 123 additions and 82 deletions
Download patch file Download diff file Expand all files Collapse all files

154
client/src/hooks/Files/useFileHandling.ts
View file

@ -1,7 +1,7 @@
import { v4 } from 'uuid';
import debounce from 'lodash/debounce';
import { useQueryClient } from '@tanstack/react-query';
import { useState, useEffect, useCallback, useRef } from 'react';
import { useState, useEffect, useCallback, useRef, useMemo } from 'react';
import {
megabyte,
QueryKeys,
@ -47,14 +47,24 @@ const useFileHandling = (params?: UseFileHandling) => {
const agent_id = params?.additionalMetadata?.agent_id ?? '';
const assistant_id = params?.additionalMetadata?.assistant_id ?? '';
const { data: fileConfig = defaultFileConfig } = useGetFileConfig({
const { data: fileConfig = null } = useGetFileConfig({
select: (data) => mergeFileConfig(data),
});
const endpoint =
params?.overrideEndpoint ?? conversation?.endpointType ?? conversation?.endpoint ?? 'default';
const { fileLimit, fileSizeLimit, totalSizeLimit, supportedMimeTypes } =
fileConfig.endpoints[endpoint] ?? fileConfig.endpoints.default;
const endpoint = useMemo(
() =>
params?.overrideEndpoint ?? conversation?.endpointType ?? conversation?.endpoint ?? 'default',
[params?.overrideEndpoint, conversation?.endpointType, conversation?.endpoint],
);
const { fileLimit, fileSizeLimit, totalSizeLimit, supportedMimeTypes } = useMemo(
() =>
fileConfig?.endpoints[endpoint] ??
fileConfig?.endpoints.default ??
defaultFileConfig.endpoints[endpoint] ??
defaultFileConfig.endpoints.default,
[fileConfig, endpoint],
);
const displayToast = useCallback(() => {
if (errors.length > 1) {
@ -146,6 +156,7 @@ const useFileHandling = (params?: UseFileHandling) => {
startUploadTimer(extendedFile.file_id, filename, extendedFile.size);
const formData = new FormData();
formData.append('endpoint', endpoint);
formData.append('file', extendedFile.file as File, encodeURIComponent(filename));
formData.append('file_id', extendedFile.file_id);
@ -167,8 +178,6 @@ const useFileHandling = (params?: UseFileHandling) => {
}
}
formData.append('endpoint', endpoint);
if (!isAssistantsEndpoint(endpoint)) {
uploadFile.mutate(formData);
return;
@ -203,81 +212,86 @@ const useFileHandling = (params?: UseFileHandling) => {
uploadFile.mutate(formData);
};
const validateFiles = (fileList: File[]) => {
const existingFiles = Array.from(files.values());
const incomingTotalSize = fileList.reduce((total, file) => total + file.size, 0);
if (incomingTotalSize === 0) {
setError('com_error_files_empty');
return false;
}
const currentTotalSize = existingFiles.reduce((total, file) => total + file.size, 0);
if (fileList.length + files.size > fileLimit) {
setError(`You can only upload up to ${fileLimit} files at a time.`);
return false;
}
for (let i = 0; i < fileList.length; i++) {
let originalFile = fileList[i];
let fileType = originalFile.type;
const extension = originalFile.name.split('.').pop() ?? '';
const knownCodeType = codeTypeMapping[extension];
// Infer MIME type for Known Code files when the type is empty or a mismatch
if (knownCodeType && (!fileType || fileType !== knownCodeType)) {
fileType = knownCodeType;
const validateFiles = useCallback(
(fileList: File[]) => {
const existingFiles = Array.from(files.values());
const incomingTotalSize = fileList.reduce((total, file) => total + file.size, 0);
if (incomingTotalSize === 0) {
setError('com_error_files_empty');
return false;
}
const currentTotalSize = existingFiles.reduce((total, file) => total + file.size, 0);
// Check if the file type is still empty after the extension check
if (!fileType) {
setError('Unable to determine file type for: ' + originalFile.name);
if (fileList.length + files.size > fileLimit) {
setError(`You can only upload up to ${fileLimit} files at a time.`);
return false;
}
// Replace empty type with inferred type
if (originalFile.type !== fileType) {
const newFile = new File([originalFile], originalFile.name, { type: fileType });
originalFile = newFile;
fileList[i] = newFile;
for (let i = 0; i < fileList.length; i++) {
let originalFile = fileList[i];
let fileType = originalFile.type;
const extension = originalFile.name.split('.').pop() ?? '';
const knownCodeType = codeTypeMapping[extension];
// Infer MIME type for Known Code files when the type is empty or a mismatch
if (knownCodeType && (!fileType || fileType !== knownCodeType)) {
fileType = knownCodeType;
}
// Check if the file type is still empty after the extension check
if (!fileType) {
setError('Unable to determine file type for: ' + originalFile.name);
return false;
}
// Replace empty type with inferred type
if (originalFile.type !== fileType) {
const newFile = new File([originalFile], originalFile.name, { type: fileType });
originalFile = newFile;
fileList[i] = newFile;
}
if (!checkType(originalFile.type, supportedMimeTypes)) {
console.log(originalFile);
setError('Currently, unsupported file type: ' + originalFile.type);
return false;
}
if (originalFile.size >= fileSizeLimit) {
setError(`File size exceeds ${fileSizeLimit / megabyte} MB.`);
return false;
}
}
if (!checkType(originalFile.type, supportedMimeTypes)) {
console.log(originalFile);
setError('Currently, unsupported file type: ' + originalFile.type);
if (currentTotalSize + incomingTotalSize > totalSizeLimit) {
setError(`The total size of the files cannot exceed ${totalSizeLimit / megabyte} MB.`);
return false;
}
if (originalFile.size >= fileSizeLimit) {
setError(`File size exceeds ${fileSizeLimit / megabyte} MB.`);
const combinedFilesInfo = [
...existingFiles.map(
(file) =>
`${file.file?.name ?? file.filename}-${file.size}-${
file.type?.split('/')[0] ?? 'file'
}`,
),
...fileList.map(
(file: File | undefined) =>
`${file?.name}-${file?.size}-${file?.type.split('/')[0] ?? 'file'}`,
),
];
const uniqueFilesSet = new Set(combinedFilesInfo);
if (uniqueFilesSet.size !== combinedFilesInfo.length) {
setError('com_error_files_dupe');
return false;
}
}
if (currentTotalSize + incomingTotalSize > totalSizeLimit) {
setError(`The total size of the files cannot exceed ${totalSizeLimit / megabyte} MB.`);
return false;
}
const combinedFilesInfo = [
...existingFiles.map(
(file) =>
`${file.file?.name ?? file.filename}-${file.size}-${file.type?.split('/')[0] ?? 'file'}`,
),
...fileList.map(
(file: File | undefined) =>
`${file?.name}-${file?.size}-${file?.type.split('/')[0] ?? 'file'}`,
),
];
const uniqueFilesSet = new Set(combinedFilesInfo);
if (uniqueFilesSet.size !== combinedFilesInfo.length) {
setError('com_error_files_dupe');
return false;
}
return true;
};
return true;
},
[files, fileLimit, fileSizeLimit, totalSizeLimit, supportedMimeTypes],
);
const loadImage = (extendedFile: ExtendedFile, preview: string) => {
const img = new Image();