feat: Vision Support + New UI (#1203)

* feat: add timer duration to showToast, show toast for preset selection

* refactor: replace old /chat/ route with /c/. e2e tests will fail here

* refactor: move typedefs to root of /api/ and add a few to assistant types in TS

* refactor: reorganize data-provider imports, fix dependency cycle, strategize new plan to separate react dependent packages

* feat: add dataService for uploading images

* feat(data-provider): add mutation keys

* feat: file resizing and upload

* WIP: initial API image handling

* fix: catch JSON.parse of localStorage tools

* chore: experimental: use module-alias for absolute imports

* refactor: change temp_file_id strategy

* fix: updating files state by using Map and defining react query callbacks in a way that keeps them during component unmount, initial delete handling

* feat: properly handle file deletion

* refactor: unexpose complete filepath and resize from server for higher fidelity

* fix: make sure resized height, width is saved, catch bad requests

* refactor: use absolute imports

* fix: prevent setOptions from being called more than once for OpenAIClient, made note to fix for PluginsClient

* refactor: import supportsFiles and models vars from schemas

* fix: correctly replace temp file id

* refactor(BaseClient): use absolute imports, pass message 'opts' to buildMessages method, count tokens for nested objects/arrays

* feat: add validateVisionModel to determine if model has vision capabilities

* chore(checkBalance): update jsdoc

* feat: formatVisionMessage: change message content format dependent on role and image_urls passed

* refactor: add usage to File schema, make create and updateFile, correctly set and remove TTL

* feat: working vision support
TODO: file size, type, amount validations, making sure they are styled right, and making sure you can add images from the clipboard/dragging

* feat: clipboard support for uploading images

* feat: handle files on drop to screen, refactor top level view code to Presentation component so the useDragHelpers hook  has ChatContext

* fix(Images): replace uploaded images in place

* feat: add filepath validation to protect sensitive files

* fix: ensure correct file_ids are push and not the Map key values

* fix(ToastContext): type issue

* feat: add basic file validation

* fix(useDragHelpers): correct context issue with `files` dependency

* refactor: consolidate setErrors logic to setError

* feat: add dialog Image overlay on image click

* fix: close endpoints menu on click

* chore: set detail to auto, make note for configuration

* fix: react warning (button desc. of button)

* refactor: optimize filepath handling, pass file_ids to images for easier re-use

* refactor: optimize image file handling, allow re-using files in regen, pass more file metadata in messages

* feat: lazy loading images including use of upload preview

* fix: SetKeyDialog closing, stopPropagation on Dialog content click

* style(EndpointMenuItem): tighten up the style, fix dark theme showing in lightmode, make menu more ux friendly

* style: change maxheight of all settings textareas to 138px from 300px

* style: better styling for textarea and enclosing buttons

* refactor(PresetItems): swap back edit and delete icons

* feat: make textarea placeholder dynamic to endpoint

* style: show user hover buttons only on hover when message is streaming

* fix: ordered list not going past 9, fix css

* feat: add User/AI labels; style: hide loading spinner

* feat: add back custom footer, change original footer text

* feat: dynamic landing icons based on endpoint

* chore: comment out assistants route

* fix: autoScroll to newest on /c/ view

* fix: Export Conversation on new UI

* style: match message style of official more closely

* ci: fix api jest unit tests, comment out e2e tests for now as they will fail until addressed

* feat: more file validation and use blob in preview field, not filepath, to fix temp deletion

* feat: filefilter for multer

* feat: better AI labels based on custom name, model, and endpoint instead of  `ChatGPT`

client/src/components/Chat/Input/ChatForm.tsx

@@ -10,8 +10,16 @@ import store from '~/store';
 
 export default function ChatForm({ index = 0 }) {
   const [text, setText] = useRecoilState(store.textByIndex(index));
-  const { ask, files, setFiles, conversation, isSubmitting, handleStopGenerating } =
-    useChatContext();
+  const {
+    ask,
+    files,
+    setFiles,
+    conversation,
+    isSubmitting,
+    handleStopGenerating,
+    filesLoading,
+    setFilesLoading,
+  } = useChatContext();
 
   const submitMessage = () => {
     ask({ text });
@@ -29,7 +37,7 @@ export default function ChatForm({ index = 0 }) {
       <div className="relative flex h-full flex-1 items-stretch md:flex-col">
         <div className="flex w-full items-center">
           <div className="[&:has(textarea:focus)]:border-token-border-xheavy border-token-border-heavy shadow-xs dark:shadow-xs relative flex w-full flex-grow flex-col overflow-hidden rounded-2xl border border-black/10 bg-white shadow-[0_0_0_2px_rgba(255,255,255,0.95)] dark:border-gray-600 dark:bg-gray-800 dark:text-white dark:shadow-[0_0_0_2px_rgba(52,53,65,0.95)] [&:has(textarea:focus)]:shadow-[0_2px_6px_rgba(0,0,0,.05)]">
-            <Images files={files} setFiles={setFiles} />
+            <Images files={files} setFiles={setFiles} setFilesLoading={setFilesLoading} />
             <Textarea
               value={text}
               onChange={(e: ChangeEvent<HTMLTextAreaElement>) => setText(e.target.value)}
@@ -38,7 +46,11 @@ export default function ChatForm({ index = 0 }) {
               endpoint={conversation?.endpoint}
             />
             <AttachFile endpoint={conversation?.endpoint ?? ''} />
-            {isSubmitting ? <StopButton stop={handleStopGenerating} /> : <SendButton text={text} />}
+            {isSubmitting ? (
+              <StopButton stop={handleStopGenerating} />
+            ) : (
+              <SendButton text={text} disabled={filesLoading} />
+            )}
           </div>
         </div>
       </div>

client/src/components/Chat/Input/Files/AttachFile.tsx

@@ -1,8 +1,7 @@
-import type { EModelEndpoint } from 'librechat-data-provider';
+import { EModelEndpoint, supportsFiles } from 'librechat-data-provider';
 import { AttachmentIcon } from '~/components/svg';
 import { FileUpload } from '~/components/ui';
 import { useFileHandling } from '~/hooks';
-import { supportsFiles } from '~/common';
 
 export default function AttachFile({ endpoint }: { endpoint: EModelEndpoint | '' }) {
   const { handleFileChange } = useFileHandling();
@@ -11,9 +10,14 @@ export default function AttachFile({ endpoint }: { endpoint: EModelEndpoint | ''
   }
 
   return (
-    <div className="absolute bottom-1 left-0 md:left-1">
+    <div className="absolute bottom-2 left-2 md:bottom-3 md:left-4">
       <FileUpload handleFileChange={handleFileChange} className="flex">
-        <button className="btn relative p-0 text-black dark:text-white" aria-label="Attach files">
+        <button
+          type="button"
+          className="btn relative p-0 text-black dark:text-white"
+          aria-label="Attach files"
+          style={{ padding: 0 }}
+        >
           <div className="flex w-full items-center justify-center gap-2">
             <AttachmentIcon />
           </div>

client/src/components/Chat/Input/Files/Image.tsx

@@ -87,6 +87,7 @@ const Image = ({
         </div>
       </div>
       <button
+        type="button"
         className="absolute right-1 top-1 -translate-y-1/2 translate-x-1/2 rounded-full border border-white bg-gray-500 p-0.5 text-white transition-colors hover:bg-black hover:opacity-100 group-hover:opacity-100 md:opacity-0"
         onClick={onDelete}
       >

client/src/components/Chat/Input/Files/Images.tsx

@@ -1,25 +1,100 @@
-import Image from './Image';
+import debounce from 'lodash/debounce';
+import { useState, useEffect, useCallback } from 'react';
+import type { BatchFile } from 'librechat-data-provider';
+import { useDeleteFilesMutation } from '~/data-provider';
 import { ExtendedFile } from '~/common';
+import Image from './Image';
 
 export default function Images({
-  files,
+  files: _files,
   setFiles,
+  setFilesLoading,
 }: {
-  files: ExtendedFile[];
-  setFiles: React.Dispatch<React.SetStateAction<ExtendedFile[]>>;
+  files: Map<string, ExtendedFile>;
+  setFiles: React.Dispatch<React.SetStateAction<Map<string, ExtendedFile>>>;
+  setFilesLoading: React.Dispatch<React.SetStateAction<boolean>>;
 }) {
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  const [_batch, setFileDeleteBatch] = useState<BatchFile[]>([]);
+  const files = Array.from(_files.values());
+
+  useEffect(() => {
+    if (!files) {
+      return;
+    }
+
+    if (files.length === 0) {
+      return;
+    }
+
+    if (files.some((file) => file.progress < 1)) {
+      return;
+    }
+
+    if (files.every((file) => file.progress === 1)) {
+      setFilesLoading(false);
+    }
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [files]);
+
+  const deleteFiles = useDeleteFilesMutation({
+    onSuccess: () => {
+      console.log('Files deleted');
+    },
+    onError: (error) => {
+      console.log('Error deleting files:', error);
+    },
+  });
+
+  const executeBatchDelete = useCallback(
+    (filesToDelete: BatchFile[]) => {
+      console.log('Deleting files:', filesToDelete);
+      deleteFiles.mutate({ files: filesToDelete });
+      setFileDeleteBatch([]);
+    },
+    [deleteFiles],
+  );
+
+  // eslint-disable-next-line react-hooks/exhaustive-deps
+  const debouncedDelete = useCallback(debounce(executeBatchDelete, 1000), []);
+
+  useEffect(() => {
+    // Cleanup function for debouncedDelete when component unmounts or before re-render
+    return () => debouncedDelete.cancel();
+  }, [debouncedDelete]);
+
   if (files.length === 0) {
     return null;
   }
 
+  const deleteFile = (_file: ExtendedFile) => {
+    const { file_id, progress, temp_file_id = '', filepath = '' } = _file;
+    if (progress < 1) {
+      return;
+    }
+    const file = {
+      file_id,
+      filepath,
+    };
+
+    setFiles((currentFiles) => {
+      const updatedFiles = new Map(currentFiles);
+      updatedFiles.delete(file_id);
+      updatedFiles.delete(temp_file_id);
+      return updatedFiles;
+    });
+
+    setFileDeleteBatch((prevBatch) => {
+      const newBatch = [...prevBatch, file];
+      debouncedDelete(newBatch);
+      return newBatch;
+    });
+  };
+
   return (
     <div className="mx-2 mt-2 flex flex-wrap gap-2 px-2.5 md:pl-0 md:pr-4">
       {files.map((file: ExtendedFile, index: number) => {
-        const handleDelete = () => {
-          setFiles((currentFiles) =>
-            currentFiles.filter((_file) => file.preview !== _file.preview),
-          );
-        };
+        const handleDelete = () => deleteFile(file);
         return (
           <Image key={index} url={file.preview} onDelete={handleDelete} progress={file.progress} />
         );

client/src/components/Chat/Input/SendButton.tsx

@@ -1,10 +1,14 @@
 import { SendIcon } from '~/components/svg';
+import { cn } from '~/utils';
 
-export default function SendButton({ text }) {
+export default function SendButton({ text, disabled }) {
   return (
     <button
-      disabled={!text}
-      className="enabled:bg-brand-purple absolute bottom-2.5 right-1.5 rounded-lg rounded-md border border-black p-0.5 p-1 text-white transition-colors enabled:bg-black disabled:bg-black disabled:text-gray-400 disabled:opacity-10 dark:border-white dark:bg-white dark:disabled:bg-white md:bottom-3 md:right-3 md:p-[2px]"
+      disabled={!text || disabled}
+      className={cn(
+        'enabled:bg-brand-purple absolute rounded-lg rounded-md border border-black p-0.5 p-1 text-white transition-colors enabled:bg-black disabled:bg-black disabled:text-gray-400 disabled:opacity-10 dark:border-white dark:bg-white dark:disabled:bg-white ',
+        'bottom-1.5 right-1.5 md:bottom-2.5 md:right-3 md:p-[2px]',
+      )}
       data-testid="send-button"
       type="submit"
     >

client/src/components/Chat/Input/Textarea.tsx

@@ -1,31 +1,27 @@
 import TextareaAutosize from 'react-textarea-autosize';
-import { supportsFiles } from '~/common';
-import { useTextarea } from '~/hooks';
+import { supportsFiles } from 'librechat-data-provider';
 import { cn, removeFocusOutlines } from '~/utils';
+import { useTextarea } from '~/hooks';
 
 export default function Textarea({ value, onChange, setText, submitMessage, endpoint }) {
   const {
     inputRef,
-    handleKeyDown,
+    handlePaste,
     handleKeyUp,
+    handleKeyDown,
     handleCompositionStart,
     handleCompositionEnd,
     onHeightChange,
     placeholder,
   } = useTextarea({ setText, submitMessage });
 
-  const className = supportsFiles[endpoint]
-    ? // ? 'm-0 w-full resize-none border-0 bg-transparent py-3.5 pr-10 focus:ring-0 focus-visible:ring-0 dark:bg-transparent placeholder-black/50 dark:placeholder-white/50 pl-10 md:py-3.5 md:pr-12 md:pl-[55px]'
-  // : 'm-0 w-full resize-none border-0 bg-transparent py-[10px] pr-10 focus:ring-0 focus-visible:ring-0 dark:bg-transparent md:py-4 md:pr-12 gizmo:md:py-3.5 gizmo:placeholder-black/50 gizmo:dark:placeholder-white/50 pl-3 md:pl-4';
-    'm-0 w-full resize-none border-0 bg-transparent py-3.5 pr-10 focus:ring-0 focus-visible:ring-0 dark:bg-transparent placeholder-black/50 dark:placeholder-white/50 pl-10 md:py-3.5 md:pr-12 md:pl-[55px]'
-    : 'm-0 w-full resize-none border-0 bg-transparent py-[10px] pr-10 focus:ring-0 focus-visible:ring-0 dark:bg-transparent md:py-3.5 md:pr-12 placeholder-black/50 dark:placeholder-white/50 pl-3 md:pl-4';
-
   return (
     <TextareaAutosize
       ref={inputRef}
       autoFocus
       value={value}
       onChange={onChange}
+      onPaste={handlePaste}
       onKeyUp={handleKeyUp}
       onKeyDown={handleKeyDown}
       onCompositionStart={handleCompositionStart}
@@ -34,12 +30,15 @@ export default function Textarea({ value, onChange, setText, submitMessage, endp
       id="prompt-textarea"
       tabIndex={0}
       data-testid="text-input"
-      // style={{ maxHeight: '200px', height: '52px', overflowY: 'hidden' }}
+      style={{ height: 44, overflowY: 'hidden' }}
       rows={1}
       placeholder={placeholder}
-      // className="m-0 w-full resize-none border-0 bg-transparent py-[10px] pr-10 focus:ring-0 focus-visible:ring-0 dark:bg-transparent md:py-4 md:pr-12 gizmo:md:py-3.5 gizmo:placeholder-black/50 gizmo:dark:placeholder-white/50 pl-12 gizmo:pl-10 md:pl-[46px] gizmo:md:pl-[55px]"
-      // className="gizmo:md:py-3.5 gizmo:placeholder-black/50 gizmo:dark:placeholder-white/50 gizmo:pl-10 gizmo:md:pl-[55px] m-0 h-auto max-h-52 w-full resize-none overflow-y-hidden border-0 bg-transparent py-[10px] pl-12 pr-10 focus:ring-0 focus-visible:ring-0 dark:bg-transparent md:py-4 md:pl-[46px] md:pr-12"
-      className={cn(className, removeFocusOutlines, 'max-h-52')}
+      className={cn(
+        supportsFiles[endpoint] ? ' pl-10 md:pl-[55px]' : 'pl-3 md:pl-4',
+        'm-0 w-full resize-none border-0 bg-transparent py-[10px] pr-10 placeholder-black/50 focus:ring-0 focus-visible:ring-0 dark:bg-transparent dark:placeholder-white/50 md:py-3.5 md:pr-12 ',
+        removeFocusOutlines,
+        'max-h-52',
+      )}
     />
   );
 }