🔍 feat: Mistral OCR API / Upload Files as Text (#6274)

* refactor: move `loadAuthValues` to `~/services/Tools/credentials`

* feat: add createAxiosInstance function to configure axios with proxy support

* WIP: First pass mistral ocr

* refactor: replace getConvoFiles with getToolFiles for improved file retrieval logic

* refactor: improve document formatting in encodeAndFormat function

* refactor: remove unused resendFiles parameter from buildOptions function (this option comes from the agent config)

* fix: update getFiles call to include files with `text` property as well

* refactor: move file handling to `initializeAgentOptions`

* refactor: enhance addImageURLs method to handle OCR text and improve message formatting

* refactor: update message formatting to handle OCR text in various content types

* refactor: remove unused resendFiles property from compactAgentsSchema

* fix: add error handling for Mistral OCR document upload and logging

* refactor: integrate OCR capability into file upload options and configuration

* refactor: skip processing for text source files in delete request, as they are directly tied to database

* feat: add metadata field to ExtendedFile type and update PanelColumns and PanelTable components for localization and metadata handling

* fix: source icon styling

* wip: first pass, frontend file context agent resources

* refactor: add hover card with contextual information for File Context (OCR) in FileContext component

* feat: enhance file processing by integrating file retrieval for OCR resources in agent initialization

* feat: implement OCR config; fix: agent resource deletion for ocr files

* feat: enhance agent initialization by adding OCR capability check in resource priming

* ci: fix `~/config` module mock

* ci: add OCR property expectation in AppService tests

* refactor: simplify OCR config loading by removing environment variable extraction, to be done when OCR is actually performed

* ci: add unit test to ensure environment variable references are not parsed in OCR config

* refactor: disable base64 image inclusion in OCR request

* refactor: enhance OCR configuration handling by validating environment variables and providing defaults

* refactor: use file stream from disk for mistral ocr api

client/src/components/Chat/Input/Files/AttachFileMenu.tsx

@@ -1,7 +1,7 @@
 import * as Ariakit from '@ariakit/react';
 import React, { useRef, useState, useMemo } from 'react';
-import { FileSearch, ImageUpIcon, TerminalSquareIcon } from 'lucide-react';
 import { EToolResources, EModelEndpoint } from 'librechat-data-provider';
+import { FileSearch, ImageUpIcon, TerminalSquareIcon, FileType2Icon } from 'lucide-react';
 import { FileUpload, TooltipAnchor, DropdownPopup } from '~/components/ui';
 import { useGetEndpointsQuery } from '~/data-provider';
 import { AttachmentIcon } from '~/components/svg';
@@ -49,6 +49,17 @@ const AttachFile = ({ isRTL, disabled, handleFileChange }: AttachFileProps) => {
       },
     ];
 
+    if (capabilities.includes(EToolResources.ocr)) {
+      items.push({
+        label: localize('com_ui_upload_ocr_text'),
+        onClick: () => {
+          setToolResource(EToolResources.ocr);
+          handleUploadClick();
+        },
+        icon: <FileType2Icon className="icon-md" />,
+      });
+    }
+
     if (capabilities.includes(EToolResources.file_search)) {
       items.push({
         label: localize('com_ui_upload_file_search'),

client/src/components/Chat/Input/Files/DragDropModal.tsx

@@ -1,6 +1,6 @@
 import React, { useMemo } from 'react';
 import { EModelEndpoint, EToolResources } from 'librechat-data-provider';
-import { FileSearch, ImageUpIcon, TerminalSquareIcon } from 'lucide-react';
+import { FileSearch, ImageUpIcon, FileType2Icon, TerminalSquareIcon } from 'lucide-react';
 import OGDialogTemplate from '~/components/ui/OGDialogTemplate';
 import { useGetEndpointsQuery } from '~/data-provider';
 import useLocalize from '~/hooks/useLocalize';
@@ -50,6 +50,12 @@ const DragDropModal = ({ onOptionSelect, setShowModal, files, isVisible }: DragD
           value: EToolResources.execute_code,
           icon: <TerminalSquareIcon className="icon-md" />,
         });
+      } else if (capability === EToolResources.ocr) {
+        _options.push({
+          label: localize('com_ui_upload_ocr_text'),
+          value: EToolResources.ocr,
+          icon: <FileType2Icon className="icon-md" />,
+        });
       }
     }
 

client/src/components/Chat/Input/Files/FilePreview.tsx

@@ -19,7 +19,7 @@ const FilePreview = ({
   };
   className?: string;
 }) => {
-  const radius = 55; // Radius of the SVG circle
+  const radius = 55;
   const circumference = 2 * Math.PI * radius;
   const progress = useProgress(
     file?.['progress'] ?? 1,
@@ -27,16 +27,15 @@ const FilePreview = ({
     (file as ExtendedFile | undefined)?.size ?? 1,
   );
 
-  // Calculate the offset based on the loading progress
   const offset = circumference - progress * circumference;
   const circleCSSProperties = {
     transition: 'stroke-dashoffset 0.5s linear',
   };
 
   return (
-    <div className={cn('size-10 shrink-0 overflow-hidden rounded-xl', className)}>
+    <div className={cn('relative size-10 shrink-0 overflow-hidden rounded-xl', className)}>
       <FileIcon file={file} fileType={fileType} />
-      <SourceIcon source={file?.source} />
+      <SourceIcon source={file?.source} isCodeFile={!!file?.['metadata']?.fileIdentifier} />
       {progress < 1 && (
         <ProgressCircle
           circumference={circumference}

client/src/components/Chat/Input/Files/SourceIcon.tsx

@@ -1,3 +1,4 @@
+import { Terminal, Type, Database } from 'lucide-react';
 import { EModelEndpoint, FileSources } from 'librechat-data-provider';
 import { MinimalIcon } from '~/components/Endpoints';
 import { cn } from '~/utils';
@@ -6,9 +7,13 @@ const sourceToEndpoint = {
   [FileSources.openai]: EModelEndpoint.openAI,
   [FileSources.azure]: EModelEndpoint.azureOpenAI,
 };
+
 const sourceToClassname = {
   [FileSources.openai]: 'bg-white/75 dark:bg-black/65',
   [FileSources.azure]: 'azure-bg-color opacity-85',
+  [FileSources.execute_code]: 'bg-black text-white opacity-85',
+  [FileSources.text]: 'bg-blue-100 dark:bg-blue-900 opacity-85 text-white',
+  [FileSources.vectordb]: 'bg-yellow-100 dark:bg-yellow-900 opacity-85 text-white',
 };
 
 const defaultClassName =
@@ -16,13 +21,41 @@ const defaultClassName =
 
 export default function SourceIcon({
   source,
+  isCodeFile,
   className = defaultClassName,
 }: {
   source?: FileSources;
+  isCodeFile?: boolean;
   className?: string;
 }) {
-  if (source === FileSources.local || source === FileSources.firebase) {
-    return null;
+  if (isCodeFile === true) {
+    return (
+      <div className={cn(className, sourceToClassname[FileSources.execute_code] ?? '')}>
+        <span className="flex items-center justify-center">
+          <Terminal className="h-3 w-3" />
+        </span>
+      </div>
+    );
+  }
+
+  if (source === FileSources.text) {
+    return (
+      <div className={cn(className, sourceToClassname[source] ?? '')}>
+        <span className="flex items-center justify-center">
+          <Type className="h-3 w-3" />
+        </span>
+      </div>
+    );
+  }
+
+  if (source === FileSources.vectordb) {
+    return (
+      <div className={cn(className, sourceToClassname[source] ?? '')}>
+        <span className="flex items-center justify-center">
+          <Database className="h-3 w-3" />
+        </span>
+      </div>
+    );
   }
 
   const endpoint = sourceToEndpoint[source ?? ''];
@@ -31,7 +64,7 @@ export default function SourceIcon({
     return null;
   }
   return (
-    <button type="button" className={cn(className, sourceToClassname[source ?? ''] ?? '')}>
+    <div className={cn(className, sourceToClassname[source ?? ''] ?? '')}>
       <span className="flex items-center justify-center">
         <MinimalIcon
           endpoint={endpoint}
@@ -40,6 +73,6 @@ export default function SourceIcon({
           iconClassName="h-3 w-3"
         />
       </span>
-    </button>
+    </div>
   );
 }