🗃️ feat: General File Support for OpenAI, Azure, Custom, Anthropic and Google (RAG) (#2143)

* refactor: re-purpose `resendImages` as `resendFiles`

* refactor: re-purpose `resendImages` as `resendFiles`

* feat: upload general files

* feat: embed file during upload

* feat: delete file embeddings on file deletion

* chore(fileConfig): add epub+zip type

* feat(encodeAndFormat): handle non-image files

* feat(createContextHandlers): build context prompt from file attachments and successful RAG

* fix: prevent non-temp files as well as embedded files to be deleted on new conversation

* fix: remove temp_file_id on usage, prevent non-temp files as well as embedded files to be deleted on new conversation

* fix: prevent non-temp files as well as embedded files to be deleted on new conversation

* feat(OpenAI/Anthropic/Google): basic RAG support

* fix: delete `resendFiles` only when true (Default)

* refactor(RAG): update endpoints and pass JWT

* fix(resendFiles): default values

* fix(context/processFile): query unique ids only

* feat: rag-api.yaml

* feat: file upload improved ux for longer uploads

* chore: await embed call and catch embedding errors

* refactor: store augmentedPrompt in Client

* refactor(processFileUpload): throw error if not assistant file upload

* fix(useFileHandling): handle markdown empty mimetype issue

* chore: necessary compose file changes

client/src/hooks/Files/useDelayedUploadToast.ts

@@ -0,0 +1,34 @@
+import { useState } from 'react';
+import { useToastContext } from '~/Providers/ToastContext';
+import useLocalize from '~/hooks/useLocalize';
+
+export const useDelayedUploadToast = () => {
+  const localize = useLocalize();
+  const { showToast } = useToastContext();
+  const [uploadTimers, setUploadTimers] = useState({});
+
+  const startUploadTimer = (fileId: string, fileName: string) => {
+    const timer = setTimeout(() => {
+      const message = localize('com_ui_upload_delay', fileName);
+      showToast({
+        message,
+        status: 'warning',
+        duration: 7000,
+      });
+    }, 3000); // 3 seconds delay
+
+    setUploadTimers((prev) => ({ ...prev, [fileId]: timer }));
+  };
+
+  const clearUploadTimer = (fileId: string) => {
+    if (uploadTimers[fileId]) {
+      clearTimeout(uploadTimers[fileId]);
+      setUploadTimers((prev) => {
+        const { [fileId]: _, ...rest } = prev as Record<string, unknown>;
+        return rest;
+      });
+    }
+  };
+
+  return { startUploadTimer, clearUploadTimer };
+};

client/src/hooks/Files/useFileDeletion.ts

@@ -48,6 +48,7 @@ const useFileDeletion = ({
         temp_file_id = '',
         filepath = '',
         source = FileSources.local,
+        embedded,
         attached,
       } = _file as TFile & { attached?: boolean };
 
@@ -58,6 +59,7 @@ const useFileDeletion = ({
       }
       const file: BatchFile = {
         file_id,
+        embedded,
         filepath,
         source,
       };
@@ -89,12 +91,13 @@ const useFileDeletion = ({
   const deleteFiles = useCallback(
     ({ files, setFiles }: { files: ExtendedFile[] | TFile[]; setFiles?: FileMapSetter }) => {
       const batchFiles = files.map((_file) => {
-        const { file_id, filepath = '', source = FileSources.local } = _file;
+        const { file_id, embedded, filepath = '', source = FileSources.local } = _file;
 
         return {
+          source,
           file_id,
           filepath,
-          source,
+          embedded,
         };
       });
 

client/src/hooks/Files/useFileHandling.ts

@@ -9,6 +9,7 @@ import {
 } from 'librechat-data-provider';
 import type { ExtendedFile, FileSetter } from '~/common';
 import { useUploadFileMutation, useGetFileConfig } from '~/data-provider';
+import { useDelayedUploadToast } from './useDelayedUploadToast';
 import { useToastContext } from '~/Providers/ToastContext';
 import { useChatContext } from '~/Providers/ChatContext';
 import useUpdateFiles from './useUpdateFiles';
@@ -24,6 +25,7 @@ type UseFileHandling = {
 const useFileHandling = (params?: UseFileHandling) => {
   const { showToast } = useToastContext();
   const [errors, setErrors] = useState<string[]>([]);
+  const { startUploadTimer, clearUploadTimer } = useDelayedUploadToast();
   const { files, setFiles, setFilesLoading, conversation } = useChatContext();
   const setError = (error: string) => setErrors((prevErrors) => [...prevErrors, error]);
   const { addFile, replaceFile, updateFileById, deleteFileById } = useUpdateFiles(
@@ -72,6 +74,7 @@ const useFileHandling = (params?: UseFileHandling) => {
 
   const uploadFile = useUploadFileMutation({
     onSuccess: (data) => {
+      clearUploadTimer(data.temp_file_id);
       console.log('upload success', data);
       updateFileById(
         data.temp_file_id,
@@ -95,6 +98,7 @@ const useFileHandling = (params?: UseFileHandling) => {
             width: data.width,
             filename: data.filename,
             source: data.source,
+            embedded: data.embedded,
           },
           params?.additionalMetadata?.assistant_id ? true : false,
         );
@@ -103,6 +107,7 @@ const useFileHandling = (params?: UseFileHandling) => {
     onError: (error, body) => {
       console.log('upload error', error);
       const file_id = body.get('file_id');
+      clearUploadTimer(file_id as string);
       deleteFileById(file_id as string);
       setError(
         (error as { response: { data: { message?: string } } })?.response?.data?.message ??
@@ -117,6 +122,8 @@ const useFileHandling = (params?: UseFileHandling) => {
       return;
     }
 
+    startUploadTimer(extendedFile.file_id, extendedFile.file?.name || 'File');
+
     const formData = new FormData();
     formData.append('file', extendedFile.file as File);
     formData.append('file_id', extendedFile.file_id);
@@ -159,7 +166,27 @@ const useFileHandling = (params?: UseFileHandling) => {
     }
 
     for (let i = 0; i < fileList.length; i++) {
-      const originalFile = fileList[i];
+      let originalFile = fileList[i];
+      let fileType = originalFile.type;
+
+      // Infer MIME type for Markdown files when the type is empty
+      if (!fileType && originalFile.name.endsWith('.md')) {
+        fileType = 'text/markdown';
+      }
+
+      // Check if the file type is still empty after the extension check
+      if (!fileType) {
+        setError('Unable to determine file type for: ' + originalFile.name);
+        return false;
+      }
+
+      // Replace empty type with inferred type
+      if (originalFile.type !== fileType) {
+        const newFile = new File([originalFile], originalFile.name, { type: fileType });
+        originalFile = newFile;
+        fileList[i] = newFile;
+      }
+
       if (!checkType(originalFile.type, supportedMimeTypes)) {
         console.log(originalFile);
         setError('Currently, unsupported file type: ' + originalFile.type);

client/src/hooks/Messages/useProgress.ts

@@ -1,6 +1,7 @@
 import { useState, useEffect } from 'react';
 
-export default function useProgress(initialProgress = 0.01) {
+export default function useProgress(initialProgress = 0.01, increment = 0.007) {
+  const [incrementValue] = useState(increment);
   const [progress, setProgress] = useState(initialProgress);
 
   useEffect(() => {
@@ -20,7 +21,7 @@ export default function useProgress(initialProgress = 0.01) {
             clearInterval(timer);
             return 1;
           }
-          return Math.min(prevProgress + 0.007, 0.95);
+          return Math.min(prevProgress + incrementValue, 0.95);
         });
       }, 200);
     }
@@ -29,7 +30,7 @@ export default function useProgress(initialProgress = 0.01) {
       clearInterval(timer);
       clearTimeout(timeout);
     };
-  }, [progress, initialProgress]);
+  }, [progress, initialProgress, incrementValue]);
 
   return progress;
 }

client/src/hooks/useNewConvo.ts

@@ -165,9 +165,10 @@ const useNewConvo = (index = 0) => {
 
       if (conversation.conversationId === 'new' && !modelsData) {
         const filesToDelete = Array.from(files.values())
-          .filter((file) => file.filepath && file.source)
+          .filter((file) => file.filepath && file.source && !file.embedded && file.temp_file_id)
           .map((file) => ({
             file_id: file.file_id,
+            embedded: !!file.embedded,
             filepath: file.filepath as string,
             source: file.source as FileSources, // Ensure that the source is of type FileSources
           }));