🔍 feat: Mistral OCR API / Upload Files as Text (#6274)

* refactor: move `loadAuthValues` to `~/services/Tools/credentials`

* feat: add createAxiosInstance function to configure axios with proxy support

* WIP: First pass mistral ocr

* refactor: replace getConvoFiles with getToolFiles for improved file retrieval logic

* refactor: improve document formatting in encodeAndFormat function

* refactor: remove unused resendFiles parameter from buildOptions function (this option comes from the agent config)

* fix: update getFiles call to include files with `text` property as well

* refactor: move file handling to `initializeAgentOptions`

* refactor: enhance addImageURLs method to handle OCR text and improve message formatting

* refactor: update message formatting to handle OCR text in various content types

* refactor: remove unused resendFiles property from compactAgentsSchema

* fix: add error handling for Mistral OCR document upload and logging

* refactor: integrate OCR capability into file upload options and configuration

* refactor: skip processing for text source files in delete request, as they are directly tied to database

* feat: add metadata field to ExtendedFile type and update PanelColumns and PanelTable components for localization and metadata handling

* fix: source icon styling

* wip: first pass, frontend file context agent resources

* refactor: add hover card with contextual information for File Context (OCR) in FileContext component

* feat: enhance file processing by integrating file retrieval for OCR resources in agent initialization

* feat: implement OCR config; fix: agent resource deletion for ocr files

* feat: enhance agent initialization by adding OCR capability check in resource priming

* ci: fix `~/config` module mock

* ci: add OCR property expectation in AppService tests

* refactor: simplify OCR config loading by removing environment variable extraction, to be done when OCR is actually performed

* ci: add unit test to ensure environment variable references are not parsed in OCR config

* refactor: disable base64 image inclusion in OCR request

* refactor: enhance OCR configuration handling by validating environment variables and providing defaults

* refactor: use file stream from disk for mistral ocr api
This commit is contained in:
Danny Avila 2025-03-10 17:23:46 -04:00 committed by GitHub
parent 9db00edfc4
commit ded3cd8876
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
48 changed files with 1621 additions and 131 deletions

View file

@ -0,0 +1,128 @@
import { useState, useRef } from 'react';
import {
EModelEndpoint,
EToolResources,
mergeFileConfig,
fileConfig as defaultFileConfig,
} from 'librechat-data-provider';
import type { ExtendedFile } from '~/common';
import { useFileHandling, useLocalize, useLazyEffect } from '~/hooks';
import FileRow from '~/components/Chat/Input/Files/FileRow';
import { useGetFileConfig } from '~/data-provider';
import { HoverCard, HoverCardContent, HoverCardPortal, HoverCardTrigger } from '~/components/ui';
import { AttachmentIcon, CircleHelpIcon } from '~/components/svg';
import { useChatContext } from '~/Providers';
import { ESide } from '~/common';
export default function FileContext({
agent_id,
files: _files,
}: {
agent_id: string;
files?: [string, ExtendedFile][];
}) {
const localize = useLocalize();
const { setFilesLoading } = useChatContext();
const fileInputRef = useRef<HTMLInputElement>(null);
const [files, setFiles] = useState<Map<string, ExtendedFile>>(new Map());
const { data: fileConfig = defaultFileConfig } = useGetFileConfig({
select: (data) => mergeFileConfig(data),
});
const { handleFileChange } = useFileHandling({
overrideEndpoint: EModelEndpoint.agents,
additionalMetadata: { agent_id, tool_resource: EToolResources.ocr },
fileSetter: setFiles,
});
useLazyEffect(
() => {
if (_files) {
setFiles(new Map(_files));
}
},
[_files],
750,
);
const endpointFileConfig = fileConfig.endpoints[EModelEndpoint.agents];
const isUploadDisabled = endpointFileConfig.disabled ?? false;
if (isUploadDisabled) {
return null;
}
const handleButtonClick = () => {
// necessary to reset the input
if (fileInputRef.current) {
fileInputRef.current.value = '';
}
fileInputRef.current?.click();
};
return (
<div className="w-full">
<HoverCard openDelay={50}>
<div className="mb-2 flex items-center gap-2">
<HoverCardTrigger asChild>
<span className="flex items-center gap-2">
<label className="text-token-text-primary block font-medium">
{localize('com_agents_file_context')}
</label>
<CircleHelpIcon className="h-4 w-4 text-text-tertiary" />
</span>
</HoverCardTrigger>
<HoverCardPortal>
<HoverCardContent side={ESide.Top} className="w-80">
<div className="space-y-2">
<p className="text-sm text-text-secondary">
{localize('com_agents_file_context_info')}
</p>
</div>
</HoverCardContent>
</HoverCardPortal>
</div>
</HoverCard>
<div className="flex flex-col gap-3">
{/* File Context (OCR) Files */}
<FileRow
files={files}
setFiles={setFiles}
setFilesLoading={setFilesLoading}
agent_id={agent_id}
tool_resource={EToolResources.ocr}
Wrapper={({ children }) => <div className="flex flex-wrap gap-2">{children}</div>}
/>
<div>
<button
type="button"
disabled={!agent_id}
className="btn btn-neutral border-token-border-light relative h-9 w-full rounded-lg font-medium"
onClick={handleButtonClick}
>
<div className="flex w-full items-center justify-center gap-1">
<AttachmentIcon className="text-token-text-primary h-4 w-4" />
<input
multiple={true}
type="file"
style={{ display: 'none' }}
tabIndex={-1}
ref={fileInputRef}
disabled={!agent_id}
onChange={handleFileChange}
/>
{localize('com_ui_upload_file_context')}
</div>
</button>
</div>
{/* Disabled Message */}
{agent_id ? null : (
<div className="text-xs text-text-secondary">
{localize('com_agents_file_context_disabled')}
</div>
)}
</div>
</div>
);
}