✨ refactor: Integrate Capabilities into Agent File Uploads and Tool Handling (#5048)

* refactor: support drag/drop files for agents, handle undefined tool_resource edge cases

* refactor: consolidate endpoints config logic to dedicated getter

* refactor: Enhance agent tools loading logic to respect capabilities and filter tools accordingly

* refactor: Integrate endpoint capabilities into file upload dropdown for dynamic resource handling

* refactor: Implement capability checks for agent file upload operations

* fix: non-image tool_resource check

client/src/common/types.ts

@@ -464,6 +464,7 @@ export interface ExtendedFile {
   source?: FileSources;
   attached?: boolean;
   embedded?: boolean;
+  tool_resource?: string;
 }
 
 export type ContextType = { navVisible: boolean; setNavVisible: (visible: boolean) => void };

client/src/components/Chat/Input/Files/AttachFileMenu.tsx

@@ -1,7 +1,8 @@
 import * as Ariakit from '@ariakit/react';
-import React, { useRef, useState } from 'react';
+import React, { useRef, useState, useMemo } from 'react';
 import { FileSearch, ImageUpIcon, TerminalSquareIcon } from 'lucide-react';
-import { EToolResources } from 'librechat-data-provider';
+import { EToolResources, EModelEndpoint } from 'librechat-data-provider';
+import { useGetEndpointsQuery } from 'librechat-data-provider/react-query';
 import { FileUpload, TooltipAnchor, DropdownPopup } from '~/components/ui';
 import { AttachmentIcon } from '~/components/svg';
 import { useLocalize } from '~/hooks';
@@ -19,6 +20,12 @@ const AttachFile = ({ isRTL, disabled, setToolResource, handleFileChange }: Atta
   const isUploadDisabled = disabled ?? false;
   const inputRef = useRef<HTMLInputElement>(null);
   const [isPopoverActive, setIsPopoverActive] = useState(false);
+  const { data: endpointsConfig } = useGetEndpointsQuery();
+
+  const capabilities = useMemo(
+    () => endpointsConfig?.[EModelEndpoint.agents]?.capabilities ?? [],
+    [endpointsConfig],
+  );
 
   const handleUploadClick = (isImage?: boolean) => {
     if (!inputRef.current) {
@@ -30,32 +37,42 @@ const AttachFile = ({ isRTL, disabled, setToolResource, handleFileChange }: Atta
     inputRef.current.accept = '';
   };
 
-  const dropdownItems = [
-    {
-      label: localize('com_ui_upload_image_input'),
-      onClick: () => {
-        setToolResource?.(undefined);
-        handleUploadClick(true);
+  const dropdownItems = useMemo(() => {
+    const items = [
+      {
+        label: localize('com_ui_upload_image_input'),
+        onClick: () => {
+          setToolResource?.(undefined);
+          handleUploadClick(true);
+        },
+        icon: <ImageUpIcon className="icon-md" />,
       },
-      icon: <ImageUpIcon className="icon-md" />,
-    },
-    {
-      label: localize('com_ui_upload_file_search'),
-      onClick: () => {
-        setToolResource?.(EToolResources.file_search);
-        handleUploadClick();
-      },
-      icon: <FileSearch className="icon-md" />,
-    },
-    {
-      label: localize('com_ui_upload_code_files'),
-      onClick: () => {
-        setToolResource?.(EToolResources.execute_code);
-        handleUploadClick();
-      },
-      icon: <TerminalSquareIcon className="icon-md" />,
-    },
-  ];
+    ];
+
+    if (capabilities.includes(EToolResources.file_search)) {
+      items.push({
+        label: localize('com_ui_upload_file_search'),
+        onClick: () => {
+          setToolResource?.(EToolResources.file_search);
+          handleUploadClick();
+        },
+        icon: <FileSearch className="icon-md" />,
+      });
+    }
+
+    if (capabilities.includes(EToolResources.execute_code)) {
+      items.push({
+        label: localize('com_ui_upload_code_files'),
+        onClick: () => {
+          setToolResource?.(EToolResources.execute_code);
+          handleUploadClick();
+        },
+        icon: <TerminalSquareIcon className="icon-md" />,
+      });
+    }
+
+    return items;
+  }, [capabilities, localize, setToolResource]);
 
   const menuTrigger = (
     <TooltipAnchor

client/src/components/Chat/Input/Files/DragDropModal.tsx

@@ -0,0 +1,90 @@
+import React, { useMemo } from 'react';
+import { EModelEndpoint, EToolResources } from 'librechat-data-provider';
+import { useGetEndpointsQuery } from 'librechat-data-provider/react-query';
+import { FileSearch, ImageUpIcon, TerminalSquareIcon } from 'lucide-react';
+import OGDialogTemplate from '~/components/ui/OGDialogTemplate';
+import useLocalize from '~/hooks/useLocalize';
+import { OGDialog } from '~/components/ui';
+
+interface DragDropModalProps {
+  onOptionSelect: (option: string | undefined) => void;
+  files: File[];
+  isVisible: boolean;
+  setShowModal: (showModal: boolean) => void;
+}
+
+interface FileOption {
+  label: string;
+  value?: EToolResources;
+  icon: React.JSX.Element;
+  condition?: boolean;
+}
+
+const DragDropModal = ({ onOptionSelect, setShowModal, files, isVisible }: DragDropModalProps) => {
+  const localize = useLocalize();
+  const { data: endpointsConfig } = useGetEndpointsQuery();
+  const capabilities = useMemo(
+    () => endpointsConfig?.[EModelEndpoint.agents]?.capabilities ?? [],
+    [endpointsConfig],
+  );
+
+  const options = useMemo(() => {
+    const _options: FileOption[] = [
+      {
+        label: localize('com_ui_upload_image_input'),
+        value: undefined,
+        icon: <ImageUpIcon className="icon-md" />,
+        condition: files.every((file) => file.type.startsWith('image/')),
+      },
+    ];
+    for (const capability of capabilities) {
+      if (capability === EToolResources.file_search) {
+        _options.push({
+          label: localize('com_ui_upload_file_search'),
+          value: EToolResources.file_search,
+          icon: <FileSearch className="icon-md" />,
+        });
+      } else if (capability === EToolResources.execute_code) {
+        _options.push({
+          label: localize('com_ui_upload_code_files'),
+          value: EToolResources.execute_code,
+          icon: <TerminalSquareIcon className="icon-md" />,
+        });
+      }
+    }
+
+    return _options;
+  }, [capabilities, files, localize]);
+
+  if (!isVisible) {
+    return null;
+  }
+
+  return (
+    <OGDialog open={isVisible} onOpenChange={setShowModal}>
+      <OGDialogTemplate
+        title={localize('com_ui_upload_type')}
+        className="w-11/12 sm:w-[440px] md:w-[400px] lg:w-[360px]"
+        main={
+          <div className="flex flex-col gap-2">
+            {options.map(
+              (option, index) =>
+                option.condition !== false && (
+                  <button
+                    key={index}
+                    onClick={() => onOptionSelect(option.value)}
+                    className="flex items-center gap-2 rounded-lg p-2 hover:bg-surface-active-alt"
+                  >
+                    {option.icon}
+                    <span>{option.label}</span>
+                  </button>
+                ),
+            )}
+          </div>
+        }
+      />
+    </OGDialog>
+  );
+};
+
+export default DragDropModal;

client/src/components/Chat/Input/Files/DragDropWrapper.tsx

@@ -0,0 +1,29 @@
+import { useDragHelpers } from '~/hooks';
+import DragDropOverlay from '~/components/Chat/Input/Files/DragDropOverlay';
+import DragDropModal from '~/components/Chat/Input/Files/DragDropModal';
+import { cn } from '~/utils';
+
+interface DragDropWrapperProps {
+  children: React.ReactNode;
+  className?: string;
+}
+
+export default function DragDropWrapper({ children, className }: DragDropWrapperProps) {
+  const { isOver, canDrop, drop, showModal, setShowModal, draggedFiles, handleOptionSelect } =
+    useDragHelpers();
+
+  const isActive = canDrop && isOver;
+
+  return (
+    <div ref={drop} className={cn('relative flex h-full w-full', className)}>
+      {children}
+      {isActive && <DragDropOverlay />}
+      <DragDropModal
+        files={draggedFiles}
+        isVisible={showModal}
+        setShowModal={setShowModal}
+        onOptionSelect={handleOptionSelect}
+      />
+    </div>
+  );
+}

client/src/components/Chat/Presentation.tsx

@@ -3,11 +3,11 @@ import { useEffect, useMemo } from 'react';
 import { useGetStartupConfig } from 'librechat-data-provider/react-query';
 import { FileSources, LocalStorageKeys, getConfigDefaults } from 'librechat-data-provider';
 import type { ExtendedFile } from '~/common';
-import { useDragHelpers, useSetFilesToDelete } from '~/hooks';
-import DragDropOverlay from './Input/Files/DragDropOverlay';
+import DragDropWrapper from '~/components/Chat/Input/Files/DragDropWrapper';
 import { useDeleteFilesMutation } from '~/data-provider';
 import Artifacts from '~/components/Artifacts/Artifacts';
 import { SidePanel } from '~/components/SidePanel';
+import { useSetFilesToDelete } from '~/hooks';
 import store from '~/store';
 
 const defaultInterface = getConfigDefaults().interface;
@@ -33,7 +33,6 @@ export default function Presentation({
   );
 
   const setFilesToDelete = useSetFilesToDelete();
-  const { isOver, canDrop, drop } = useDragHelpers();
 
   const { mutateAsync } = useDeleteFilesMutation({
     onSuccess: () => {
@@ -66,8 +65,6 @@ export default function Presentation({
     mutateAsync({ files });
   }, [mutateAsync]);
 
-  const isActive = canDrop && isOver;
-
   const defaultLayout = useMemo(() => {
     const resizableLayout = localStorage.getItem('react-resizable-panels:layout');
     return typeof resizableLayout === 'string' ? JSON.parse(resizableLayout) : undefined;
@@ -79,20 +76,16 @@ export default function Presentation({
   const fullCollapse = useMemo(() => localStorage.getItem('fullPanelCollapse') === 'true', []);
 
   const layout = () => (
-    <div className="transition-width relative flex h-full w-full flex-1 flex-col items-stretch overflow-hidden bg-white pt-0 dark:bg-gray-800">
+    <div className="transition-width relative flex h-full w-full flex-1 flex-col items-stretch overflow-hidden bg-presentation pt-0">
       <div className="flex h-full flex-col" role="presentation">
         {children}
-        {isActive && <DragDropOverlay />}
       </div>
     </div>
   );
 
   if (useSidePanel && !hideSidePanel && interfaceConfig.sidePanel === true) {
     return (
-      <div
-        ref={drop}
-        className="relative flex w-full grow overflow-hidden bg-white dark:bg-gray-800"
-      >
+      <DragDropWrapper className="relative flex w-full grow overflow-hidden bg-presentation">
         <SidePanel
           defaultLayout={defaultLayout}
           defaultCollapsed={defaultCollapsed}
@@ -107,17 +100,16 @@ export default function Presentation({
         >
           <main className="flex h-full flex-col" role="main">
             {children}
-            {isActive && <DragDropOverlay />}
           </main>
         </SidePanel>
-      </div>
+      </DragDropWrapper>
     );
   }
 
   return (
-    <div ref={drop} className="relative flex w-full grow overflow-hidden bg-white dark:bg-gray-800">
+    <DragDropWrapper className="relative flex w-full grow overflow-hidden bg-presentation">
       {layout()}
       {panel != null && panel}
-    </div>
+    </DragDropWrapper>
   );
 }

client/src/hooks/Files/useDragHelpers.ts

@@ -1,42 +1,76 @@
+import { useState, useMemo } from 'react';
 import { useDrop } from 'react-dnd';
+import { useRecoilValue } from 'recoil';
 import { NativeTypes } from 'react-dnd-html5-backend';
+import { useQueryClient } from '@tanstack/react-query';
+import {
+  isAgentsEndpoint,
+  EModelEndpoint,
+  AgentCapabilities,
+  QueryKeys,
+} from 'librechat-data-provider';
+import type * as t from 'librechat-data-provider';
 import type { DropTargetMonitor } from 'react-dnd';
 import useFileHandling from './useFileHandling';
+import store from '~/store';
 
 export default function useDragHelpers() {
-  const { files, handleFiles } = useFileHandling();
+  const queryClient = useQueryClient();
+  const { handleFiles } = useFileHandling();
+  const [showModal, setShowModal] = useState(false);
+  const [draggedFiles, setDraggedFiles] = useState<File[]>([]);
+  const conversation = useRecoilValue(store.conversationByIndex(0)) || undefined;
+
+  const handleOptionSelect = (toolResource: string | undefined) => {
+    handleFiles(draggedFiles, toolResource);
+    setShowModal(false);
+    setDraggedFiles([]);
+  };
+
+  const isAgents = useMemo(
+    () => isAgentsEndpoint(conversation?.endpoint),
+    [conversation?.endpoint],
+  );
+
   const [{ canDrop, isOver }, drop] = useDrop(
     () => ({
       accept: [NativeTypes.FILE],
       drop(item: { files: File[] }) {
         console.log('drop', item.files);
-        handleFiles(item.files);
-      },
-      canDrop() {
-        // console.log('canDrop', item.files, item.items);
-        return true;
-      },
-      // hover() {
-      //   // console.log('hover', item.files, item.items);
-      // },
-      collect: (monitor: DropTargetMonitor) => {
-        // const item = monitor.getItem() as File[];
-        // if (item) {
-        //   console.log('collect', item.files, item.items);
-        // }
+        if (!isAgents) {
+          handleFiles(item.files);
+          return;
+        }
 
-        return {
-          isOver: monitor.isOver(),
-          canDrop: monitor.canDrop(),
-        };
+        const endpointsConfig = queryClient.getQueryData<t.TEndpointsConfig>([QueryKeys.endpoints]);
+        const agentsConfig = endpointsConfig?.[EModelEndpoint.agents];
+        const codeEnabled =
+          agentsConfig?.capabilities?.includes(AgentCapabilities.execute_code) === true;
+        const fileSearchEnabled =
+          agentsConfig?.capabilities?.includes(AgentCapabilities.file_search) === true;
+        if (!codeEnabled && !fileSearchEnabled) {
+          handleFiles(item.files);
+          return;
+        }
+        setDraggedFiles(item.files);
+        setShowModal(true);
       },
+      canDrop: () => true,
+      collect: (monitor: DropTargetMonitor) => ({
+        isOver: monitor.isOver(),
+        canDrop: monitor.canDrop(),
+      }),
     }),
-    [files],
+    [],
   );
 
   return {
     canDrop,
     isOver,
     drop,
+    showModal,
+    setShowModal,
+    draggedFiles,
+    handleOptionSelect,
   };
 }

client/src/hooks/Files/useFileHandling.ts

@@ -187,8 +187,9 @@ const useFileHandling = (params?: UseFileHandling) => {
       if (!agent_id) {
         formData.append('message_file', 'true');
       }
-      if (toolResource != null) {
-        formData.append('tool_resource', toolResource);
+      const tool_resource = extendedFile.tool_resource ?? toolResource;
+      if (tool_resource != null) {
+        formData.append('tool_resource', tool_resource);
       }
       if (conversation?.agent_id != null && formData.get('agent_id') == null) {
         formData.append('agent_id', conversation.agent_id);
@@ -327,7 +328,7 @@ const useFileHandling = (params?: UseFileHandling) => {
     img.src = preview;
   };
 
-  const handleFiles = async (_files: FileList | File[]) => {
+  const handleFiles = async (_files: FileList | File[], _toolResource?: string) => {
     abortControllerRef.current = new AbortController();
     const fileList = Array.from(_files);
     /* Validate files */
@@ -358,9 +359,22 @@ const useFileHandling = (params?: UseFileHandling) => {
           size: originalFile.size,
         };
 
+        if (_toolResource != null && _toolResource !== '') {
+          extendedFile.tool_resource = _toolResource;
+        }
+
+        const isImage = originalFile.type.split('/')[0] === 'image';
+        const tool_resource =
+          extendedFile.tool_resource ?? params?.additionalMetadata?.tool_resource ?? toolResource;
+        if (isAgentsEndpoint(endpoint) && !isImage && tool_resource == null) {
+          /** Note: this needs to be removed when we can support files to providers */
+          setError('com_error_files_unsupported_capability');
+          continue;
+        }
+
         addFile(extendedFile);
 
-        if (originalFile.type.split('/')[0] === 'image') {
+        if (isImage) {
           loadImage(extendedFile, preview);
           continue;
         }

client/src/localization/languages/Eng.ts

@@ -42,6 +42,7 @@ export default {
   com_error_files_dupe: 'Duplicate file detected.',
   com_error_files_validation: 'An error occurred while validating the file.',
   com_error_files_process: 'An error occurred while processing the file.',
+  com_error_files_unsupported_capability: 'No capabilities enabled that support this file type.',
   com_error_files_upload: 'An error occurred while uploading the file.',
   com_error_files_upload_canceled:
     'The file upload request was canceled. Note: the file upload may still be processing and will need to be manually deleted.',
@@ -203,6 +204,7 @@ export default {
   com_ui_next: 'Next',
   com_ui_stop: 'Stop',
   com_ui_upload_files: 'Upload files',
+  com_ui_upload_type: 'Select Upload Type',
   com_ui_upload_image_input: 'Upload Image',
   com_ui_upload_file_search: 'Upload for File Search',
   com_ui_upload_code_files: 'Upload for Code Interpreter',

client/src/style.css

@@ -39,6 +39,7 @@
   --font-size-xl: 1.25rem;
 }
 html {
+  --presentation: var(--white);
   --text-primary: var(--gray-800);
   --text-secondary: var(--gray-600);
   --text-secondary-alt: var(--gray-500);
@@ -92,6 +93,7 @@ html {
   --switch-unchecked: 0 0% 58%;
 }
 .dark {
+  --presentation: var(--gray-800);
   --text-primary: var(--gray-100);
   --text-secondary: var(--gray-300);
   --text-secondary-alt: var(--gray-400);