mirror of
https://github.com/danny-avila/LibreChat.git
synced 2025-12-20 02:10:15 +01:00
feat: Vision Support + New UI (#1203)
* feat: add timer duration to showToast, show toast for preset selection * refactor: replace old /chat/ route with /c/. e2e tests will fail here * refactor: move typedefs to root of /api/ and add a few to assistant types in TS * refactor: reorganize data-provider imports, fix dependency cycle, strategize new plan to separate react dependent packages * feat: add dataService for uploading images * feat(data-provider): add mutation keys * feat: file resizing and upload * WIP: initial API image handling * fix: catch JSON.parse of localStorage tools * chore: experimental: use module-alias for absolute imports * refactor: change temp_file_id strategy * fix: updating files state by using Map and defining react query callbacks in a way that keeps them during component unmount, initial delete handling * feat: properly handle file deletion * refactor: unexpose complete filepath and resize from server for higher fidelity * fix: make sure resized height, width is saved, catch bad requests * refactor: use absolute imports * fix: prevent setOptions from being called more than once for OpenAIClient, made note to fix for PluginsClient * refactor: import supportsFiles and models vars from schemas * fix: correctly replace temp file id * refactor(BaseClient): use absolute imports, pass message 'opts' to buildMessages method, count tokens for nested objects/arrays * feat: add validateVisionModel to determine if model has vision capabilities * chore(checkBalance): update jsdoc * feat: formatVisionMessage: change message content format dependent on role and image_urls passed * refactor: add usage to File schema, make create and updateFile, correctly set and remove TTL * feat: working vision support TODO: file size, type, amount validations, making sure they are styled right, and making sure you can add images from the clipboard/dragging * feat: clipboard support for uploading images * feat: handle files on drop to screen, refactor top level view code to Presentation component so the useDragHelpers hook has ChatContext * fix(Images): replace uploaded images in place * feat: add filepath validation to protect sensitive files * fix: ensure correct file_ids are push and not the Map key values * fix(ToastContext): type issue * feat: add basic file validation * fix(useDragHelpers): correct context issue with `files` dependency * refactor: consolidate setErrors logic to setError * feat: add dialog Image overlay on image click * fix: close endpoints menu on click * chore: set detail to auto, make note for configuration * fix: react warning (button desc. of button) * refactor: optimize filepath handling, pass file_ids to images for easier re-use * refactor: optimize image file handling, allow re-using files in regen, pass more file metadata in messages * feat: lazy loading images including use of upload preview * fix: SetKeyDialog closing, stopPropagation on Dialog content click * style(EndpointMenuItem): tighten up the style, fix dark theme showing in lightmode, make menu more ux friendly * style: change maxheight of all settings textareas to 138px from 300px * style: better styling for textarea and enclosing buttons * refactor(PresetItems): swap back edit and delete icons * feat: make textarea placeholder dynamic to endpoint * style: show user hover buttons only on hover when message is streaming * fix: ordered list not going past 9, fix css * feat: add User/AI labels; style: hide loading spinner * feat: add back custom footer, change original footer text * feat: dynamic landing icons based on endpoint * chore: comment out assistants route * fix: autoScroll to newest on /c/ view * fix: Export Conversation on new UI * style: match message style of official more closely * ci: fix api jest unit tests, comment out e2e tests for now as they will fail until addressed * feat: more file validation and use blob in preview field, not filepath, to fix temp deletion * feat: filefilter for multer * feat: better AI labels based on custom name, model, and endpoint instead of `ChatGPT`
This commit is contained in:
Danny Avila
GitHub
parent
345f4b2e85
commit
317cdd3f77
113 changed files with 2680 additions and 675 deletions
|
|
@ -1,56 +1,255 @@
|
|||
import { v4 } from 'uuid';
|
||||
import debounce from 'lodash/debounce';
|
||||
import { useState, useEffect, useCallback } from 'react';
|
||||
import type { ExtendedFile } from '~/common';
|
||||
import { useToastContext } from '~/Providers/ToastContext';
|
||||
import { useChatContext } from '~/Providers/ChatContext';
|
||||
import { useUploadImageMutation } from '~/data-provider';
|
||||
import { NotificationSeverity } from '~/common';
|
||||
|
||||
const sizeMB = 20;
|
||||
const maxSize = 25;
|
||||
const fileLimit = 10;
|
||||
const sizeLimit = sizeMB * 1024 * 1024; // 20 MB
|
||||
const totalSizeLimit = maxSize * 1024 * 1024; // 25 MB
|
||||
const supportedTypes = ['image/jpeg', 'image/jpg', 'image/png', 'image/webp'];
|
||||
|
||||
const useFileHandling = () => {
|
||||
const { files, setFiles } = useChatContext();
|
||||
const { showToast } = useToastContext();
|
||||
const [errors, setErrors] = useState<string[]>([]);
|
||||
const setError = (error: string) => setErrors((prevErrors) => [...prevErrors, error]);
|
||||
const { files, setFiles, setFilesLoading } = useChatContext();
|
||||
|
||||
const displayToast = useCallback(() => {
|
||||
if (errors.length > 1) {
|
||||
const errorList = Array.from(new Set(errors))
|
||||
.map((e, i) => `${i > 0 ? '• ' : ''}${e}\n`)
|
||||
.join('');
|
||||
showToast({
|
||||
message: errorList,
|
||||
severity: NotificationSeverity.ERROR,
|
||||
duration: 5000,
|
||||
});
|
||||
} else if (errors.length === 1) {
|
||||
showToast({
|
||||
message: errors[0],
|
||||
severity: NotificationSeverity.ERROR,
|
||||
duration: 5000,
|
||||
});
|
||||
}
|
||||
|
||||
setErrors([]);
|
||||
}, [errors, showToast]);
|
||||
|
||||
const debouncedDisplayToast = debounce(displayToast, 250);
|
||||
|
||||
useEffect(() => {
|
||||
if (errors.length > 0) {
|
||||
debouncedDisplayToast();
|
||||
}
|
||||
|
||||
return () => debouncedDisplayToast.cancel();
|
||||
}, [errors, debouncedDisplayToast]);
|
||||
|
||||
const addFile = (newFile: ExtendedFile) => {
|
||||
setFiles((currentFiles) => [...currentFiles, newFile]);
|
||||
setFiles((currentFiles) => {
|
||||
const updatedFiles = new Map(currentFiles);
|
||||
updatedFiles.set(newFile.file_id, newFile);
|
||||
return updatedFiles;
|
||||
});
|
||||
};
|
||||
|
||||
const replaceFile = (newFile: ExtendedFile) => {
|
||||
setFiles((currentFiles) =>
|
||||
currentFiles.map((f) => (f.preview === newFile.preview ? newFile : f)),
|
||||
);
|
||||
setFiles((currentFiles) => {
|
||||
const updatedFiles = new Map(currentFiles);
|
||||
updatedFiles.set(newFile.file_id, newFile);
|
||||
return updatedFiles;
|
||||
});
|
||||
};
|
||||
|
||||
const handleFiles = (files: FileList | File[]) => {
|
||||
Array.from(files).forEach((originalFile) => {
|
||||
if (!originalFile.type.startsWith('image/')) {
|
||||
// TODO: showToast('Only image files are supported');
|
||||
// TODO: handle other file types
|
||||
return;
|
||||
const updateFileById = (fileId: string, updates: Partial<ExtendedFile>) => {
|
||||
setFiles((currentFiles) => {
|
||||
if (!currentFiles.has(fileId)) {
|
||||
console.warn(`File with id ${fileId} not found.`);
|
||||
return currentFiles;
|
||||
}
|
||||
const preview = URL.createObjectURL(originalFile);
|
||||
const extendedFile: ExtendedFile = {
|
||||
file: originalFile,
|
||||
preview,
|
||||
progress: 0,
|
||||
};
|
||||
addFile(extendedFile);
|
||||
|
||||
// async processing
|
||||
if (originalFile.type.startsWith('image/')) {
|
||||
const updatedFiles = new Map(currentFiles);
|
||||
const currentFile = updatedFiles.get(fileId);
|
||||
if (!currentFile) {
|
||||
console.warn(`File with id ${fileId} not found.`);
|
||||
return currentFiles;
|
||||
}
|
||||
updatedFiles.set(fileId, { ...currentFile, ...updates });
|
||||
|
||||
return updatedFiles;
|
||||
});
|
||||
};
|
||||
|
||||
const deleteFileById = (fileId: string) => {
|
||||
setFiles((currentFiles) => {
|
||||
const updatedFiles = new Map(currentFiles);
|
||||
if (updatedFiles.has(fileId)) {
|
||||
updatedFiles.delete(fileId);
|
||||
} else {
|
||||
console.warn(`File with id ${fileId} not found.`);
|
||||
}
|
||||
return updatedFiles;
|
||||
});
|
||||
};
|
||||
|
||||
const uploadImage = useUploadImageMutation({
|
||||
onSuccess: (data) => {
|
||||
console.log('upload success', data);
|
||||
updateFileById(data.temp_file_id, {
|
||||
progress: 0.9,
|
||||
filepath: data.filepath,
|
||||
});
|
||||
|
||||
setTimeout(() => {
|
||||
const file = files.get(data.temp_file_id);
|
||||
updateFileById(data.temp_file_id, {
|
||||
progress: 1,
|
||||
file_id: data.file_id,
|
||||
temp_file_id: data.temp_file_id,
|
||||
filepath: data.filepath,
|
||||
// filepath: file?.preview,
|
||||
preview: file?.preview,
|
||||
type: data.type,
|
||||
height: data.height,
|
||||
width: data.width,
|
||||
filename: data.filename,
|
||||
});
|
||||
}, 300);
|
||||
},
|
||||
onError: (error, body) => {
|
||||
console.log('upload error', error);
|
||||
deleteFileById(body.file_id);
|
||||
setError('An error occurred while uploading the file.');
|
||||
},
|
||||
});
|
||||
|
||||
const uploadFile = async (extendedFile: ExtendedFile) => {
|
||||
const formData = new FormData();
|
||||
formData.append('file', extendedFile.file);
|
||||
formData.append('file_id', extendedFile.file_id);
|
||||
if (extendedFile.width) {
|
||||
formData.append('width', extendedFile.width?.toString());
|
||||
}
|
||||
if (extendedFile.height) {
|
||||
formData.append('height', extendedFile.height?.toString());
|
||||
}
|
||||
|
||||
uploadImage.mutate({ formData, file_id: extendedFile.file_id });
|
||||
};
|
||||
|
||||
const validateFiles = (fileList: File[]) => {
|
||||
const existingFiles = Array.from(files.values());
|
||||
const incomingTotalSize = fileList.reduce((total, file) => total + file.size, 0);
|
||||
const currentTotalSize = existingFiles.reduce((total, file) => total + file.size, 0);
|
||||
|
||||
if (fileList.length + files.size > fileLimit) {
|
||||
setError(`You can only upload up to ${fileLimit} files at a time.`);
|
||||
return false;
|
||||
}
|
||||
|
||||
for (let i = 0; i < fileList.length; i++) {
|
||||
const originalFile = fileList[i];
|
||||
if (!supportedTypes.includes(originalFile.type)) {
|
||||
setError('Currently, only JPEG, JPG, PNG, and WEBP files are supported.');
|
||||
return false;
|
||||
}
|
||||
|
||||
if (originalFile.size >= sizeLimit) {
|
||||
setError(`File size exceeds ${sizeMB} MB.`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (currentTotalSize + incomingTotalSize > totalSizeLimit) {
|
||||
setError(`The total size of the files cannot exceed ${maxSize} MB.`);
|
||||
return false;
|
||||
}
|
||||
|
||||
const combinedFilesInfo = [
|
||||
...existingFiles.map(
|
||||
(file) => `${file.file.name}-${file.size}-${file.type?.split('/')[0] ?? 'file'}`,
|
||||
),
|
||||
...fileList.map((file) => `${file.name}-${file.size}-${file.type?.split('/')[0] ?? 'file'}`),
|
||||
];
|
||||
|
||||
const uniqueFilesSet = new Set(combinedFilesInfo);
|
||||
|
||||
if (uniqueFilesSet.size !== combinedFilesInfo.length) {
|
||||
setError('Duplicate file detected.');
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
const handleFiles = async (_files: FileList | File[]) => {
|
||||
const fileList = Array.from(_files);
|
||||
/* Validate files */
|
||||
let filesAreValid: boolean;
|
||||
try {
|
||||
filesAreValid = validateFiles(fileList);
|
||||
} catch (error) {
|
||||
console.error('file validation error', error);
|
||||
setError('An error occurred while validating the file.');
|
||||
return;
|
||||
}
|
||||
if (!filesAreValid) {
|
||||
setFilesLoading(false);
|
||||
return;
|
||||
}
|
||||
|
||||
/* Process files */
|
||||
fileList.forEach((originalFile) => {
|
||||
const file_id = v4();
|
||||
try {
|
||||
const preview = URL.createObjectURL(originalFile);
|
||||
let extendedFile: ExtendedFile = {
|
||||
file_id,
|
||||
file: originalFile,
|
||||
preview,
|
||||
progress: 0.2,
|
||||
size: originalFile.size,
|
||||
};
|
||||
|
||||
addFile(extendedFile);
|
||||
|
||||
// async processing
|
||||
const img = new Image();
|
||||
img.onload = () => {
|
||||
img.onload = async () => {
|
||||
extendedFile.width = img.width;
|
||||
extendedFile.height = img.height;
|
||||
extendedFile.progress = 1; // Update loading status
|
||||
extendedFile = {
|
||||
...extendedFile,
|
||||
progress: 0.6,
|
||||
};
|
||||
replaceFile(extendedFile);
|
||||
URL.revokeObjectURL(preview); // Clean up the object URL
|
||||
|
||||
await uploadFile(extendedFile);
|
||||
// This gets cleaned up in the Image component, after receiving the server image
|
||||
// URL.revokeObjectURL(preview);
|
||||
};
|
||||
img.src = preview;
|
||||
} else {
|
||||
// TODO: non-image files
|
||||
// extendedFile.progress = false;
|
||||
// replaceFile(extendedFile);
|
||||
} catch (error) {
|
||||
deleteFileById(file_id);
|
||||
console.log('file handling error', error);
|
||||
setError('An error occurred while processing the file.');
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
const handleFileChange = (event: React.ChangeEvent<HTMLInputElement>) => {
|
||||
event.stopPropagation();
|
||||
if (event.target.files) {
|
||||
setFilesLoading(true);
|
||||
handleFiles(event.target.files);
|
||||
// reset the input
|
||||
event.target.value = '';
|
||||
}
|
||||
};
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue