Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2026-02-13 04:54:24 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

📁 feat: Send Attachments Directly to Provider (OpenAI) (#9098)

Browse source 
* refactor: change references from direct upload to direct attach to better reflect functionality

since we are just using base64 encoding strategy now rather than Files/File API for sending our attachments directly to the provider, the upload nomenclature no longer makes sense. direct_attach better describes the different methods of sending attachments to providers anyways even if we later introduce direct upload support

* feat: add upload to provider option for openai (and agent) ui

* chore: move anthropic pdf validator over to packages/api

* feat: simple pdf validation according to openai docs

* feat: add provider agnostic validatePdf logic to start handling multiple endpoints

* feat: add handling for openai specific documentPart formatting

* refactor: move require statement to proper place at top of file

* chore: add in openAI endpoint for the rest of the document handling logic

* feat: add direct attach support for azureOpenAI endpoint and agents

* feat: add pdf validation for azureOpenAI endpoint

* refactor: unify all the endpoint checks with isDocumentSupportedEndpoint

* refactor: consolidate Upload to Provider vs Upload image logic for clarity

* refactor: remove anthropic from anthropic_multimodal fileType since we support multiple providers now
This commit is contained in:
Dustin Healy 2025-08-17 02:14:25 -07:00 • committed by Dustin Healy
parent 89843262b2
commit b5aadf1302
10 changed files with 122 additions and 64 deletions
Download patch file Download diff file Expand all files Collapse all files

2
client/src/common/agents-types.ts
View file

@ -21,7 +21,7 @@ export type TAgentCapabilities = {
[AgentCapabilities.execute_code]: boolean;
[AgentCapabilities.end_after_tools]?: boolean;
[AgentCapabilities.hide_sequential_outputs]?: boolean;
[AgentCapabilities.direct_upload]?: boolean;
[AgentCapabilities.direct_attach]?: boolean;
};
export type AgentForm = {

39
client/src/components/Chat/Input/Files/AttachFileMenu.tsx
View file

@ -8,7 +8,12 @@ import {
FileType2Icon,
FileImageIcon,
} from 'lucide-react';
import { EToolResources, EModelEndpoint, defaultAgentCapabilities } from 'librechat-data-provider';
import {
EToolResources,
EModelEndpoint,
defaultAgentCapabilities,
isDocumentSupportedEndpoint,
} from 'librechat-data-provider';
import {
FileUpload,
TooltipAnchor,
@ -72,7 +77,7 @@ const AttachFileMenu = ({
* */
const capabilities = useAgentCapabilities(agentsConfig?.capabilities ?? defaultAgentCapabilities);
const handleUploadClick = (fileType?: 'image' | 'document' | 'anthropic_multimodal') => {
const handleUploadClick = (fileType?: 'image' | 'document' | 'multimodal') => {
if (!inputRef.current) {
return;
}
@ -81,7 +86,7 @@ const AttachFileMenu = ({
inputRef.current.accept = 'image/*';
} else if (fileType === 'document') {
inputRef.current.accept = '.pdf,application/pdf';
} else if (fileType === 'anthropic_multimodal') {
} else if (fileType === 'multimodal') {
inputRef.current.accept = 'image/*,.pdf,application/pdf';
} else {
inputRef.current.accept = '';
@ -92,15 +97,22 @@ const AttachFileMenu = ({
const dropdownItems = useMemo(() => {
const createMenuItems = (
onAction: (fileType?: 'image' | 'document' | 'anthropic_multimodal') => void,
onAction: (fileType?: 'image' | 'document' | 'multimodal') => void,
) => {
const items: MenuItemProps[] = [];
// this is temporary until i add direct upload support for the other providers and can make a more robust solution
const isAnthropicAgent = agent?.provider === 'anthropic';
const shouldShowDirectUpload = endpoint === EModelEndpoint.anthropic || isAnthropicAgent;
const shouldShowDirectAttach = isDocumentSupportedEndpoint(agent?.provider ?? endpoint);
if (!shouldShowDirectUpload) {
if (shouldShowDirectAttach) {
items.push({
label: localize('com_ui_upload_provider'),
onClick: () => {
setToolResource(EToolResources.direct_attach);
onAction('multimodal');
},
icon: <FileImageIcon className="icon-md" />,
});
} else {
items.push({
label: localize('com_ui_upload_image_input'),
onClick: () => {
@ -111,17 +123,6 @@ const AttachFileMenu = ({
});
}
if (shouldShowDirectUpload) {
items.push({
label: localize('com_ui_upload_provider'),
onClick: () => {
setToolResource(EToolResources.direct_upload);
onAction('anthropic_multimodal');
},
icon: <FileImageIcon className="icon-md" />,
});
}
if (capabilities.ocrEnabled) {
items.push({
label: localize('com_ui_upload_ocr_text'),

8
client/src/hooks/Agents/useAgentCapabilities.ts
View file

@ -9,7 +9,7 @@ interface AgentCapabilitiesResult {
fileSearchEnabled: boolean;
webSearchEnabled: boolean;
codeEnabled: boolean;
directUploadEnabled: boolean;
directAttachEnabled: boolean;
}
export default function useAgentCapabilities(
@ -50,8 +50,8 @@ export default function useAgentCapabilities(
[capabilities],
);
const directUploadEnabled = useMemo(
() => capabilities?.includes(AgentCapabilities.direct_upload) ?? false,
const directAttachEnabled = useMemo(
() => capabilities?.includes(AgentCapabilities.direct_attach) ?? false,
[capabilities],
);
@ -63,6 +63,6 @@ export default function useAgentCapabilities(
artifactsEnabled,
webSearchEnabled,
fileSearchEnabled,
directUploadEnabled,
directAttachEnabled,
};
}