📂 feat: RAG Improvements (#2169)

* feat: new vector file processing strategy

* chore: remove unused client files

* chore: remove more unused client files

* chore: remove more unused client files and move used to new dir

* chore(DataIcon): add className

* WIP: Model Endpoint Settings Update, draft additional context settings

* feat: improve parsing for augmented prompt, add full context option

* chore: remove volume mounting from rag.yml as no longer necessary

client/src/components/Chat/Input/Files/FileUpload.tsx

@@ -0,0 +1,88 @@
+import React, { useState } from 'react';
+import { FileUp } from 'lucide-react';
+import { cn } from '~/utils/';
+import { useLocalize } from '~/hooks';
+
+type FileUploadProps = {
+  onFileSelected: (jsonData: Record<string, unknown>) => void;
+  className?: string;
+  containerClassName?: string;
+  successText?: string;
+  invalidText?: string;
+  validator?: ((data: Record<string, unknown>) => boolean) | null;
+  text?: string;
+  id?: string;
+};
+
+const FileUpload: React.FC<FileUploadProps> = ({
+  onFileSelected,
+  className = '',
+  containerClassName = '',
+  successText = null,
+  invalidText = null,
+  validator = null,
+  text = null,
+  id = '1',
+}) => {
+  const [statusColor, setStatusColor] = useState<string>('text-gray-600');
+  const [status, setStatus] = useState<null | string>(null);
+  const localize = useLocalize();
+
+  const handleFileChange = (event: React.ChangeEvent<HTMLInputElement>): void => {
+    const file = event.target.files?.[0];
+    if (!file) {
+      return;
+    }
+
+    const reader = new FileReader();
+    reader.onload = (e) => {
+      const jsonData = JSON.parse(e.target?.result as string);
+      if (validator && !validator(jsonData)) {
+        setStatus('invalid');
+        setStatusColor('text-red-600');
+        return;
+      }
+
+      if (validator) {
+        setStatus('success');
+        setStatusColor('text-green-500 dark:text-green-500');
+      }
+
+      onFileSelected(jsonData);
+    };
+    reader.readAsText(file);
+  };
+
+  let statusText: string;
+  if (!status) {
+    statusText = text ?? localize('com_endpoint_import');
+  } else if (status === 'success') {
+    statusText = successText ?? localize('com_ui_upload_success');
+  } else {
+    statusText = invalidText ?? localize('com_ui_upload_invalid');
+  }
+
+  return (
+    <label
+      htmlFor={`file-upload-${id}`}
+      className={cn(
+        'mr-1 flex h-auto cursor-pointer items-center rounded bg-transparent px-2 py-1 text-xs font-medium font-normal transition-colors hover:bg-gray-100 hover:text-green-600 dark:bg-transparent dark:text-gray-300 dark:hover:bg-gray-700 dark:hover:text-green-500',
+        statusColor,
+        containerClassName,
+      )}
+    >
+      <FileUp className="mr-1 flex w-[22px] items-center stroke-1" />
+      <span className="flex text-xs ">{statusText}</span>
+      <input
+        id={`file-upload-${id}`}
+        value=""
+        type="file"
+        className={cn('hidden ', className)}
+        accept=".json"
+        onChange={handleFileChange}
+      />
+    </label>
+  );
+};
+
+export default FileUpload;

client/src/components/Chat/Input/HeaderOptions.tsx

@@ -4,7 +4,7 @@ import { Root, Anchor } from '@radix-ui/react-popover';
 import { useState, useEffect, useMemo } from 'react';
 import { tPresetUpdateSchema, EModelEndpoint } from 'librechat-data-provider';
 import type { TPreset } from 'librechat-data-provider';
-import { EndpointSettings, SaveAsPresetDialog } from '~/components/Endpoints';
+import { EndpointSettings, SaveAsPresetDialog, AlternativeSettings } from '~/components/Endpoints';
 import { ModelSelect } from '~/components/Input/ModelSelect';
 import { PluginStoreDialog } from '~/components';
 import OptionsPopover from './OptionsPopover';
@@ -15,7 +15,7 @@ import { Button } from '~/components/ui';
 import { cn, cardStyle } from '~/utils/';
 import store from '~/store';
 
-export default function OptionsBar() {
+export default function HeaderOptions() {
   const [saveAsDialogShow, setSaveAsDialogShow] = useState<boolean>(false);
   const [showPluginStoreDialog, setShowPluginStoreDialog] = useRecoilState(
     store.showPluginStoreDialog,
@@ -102,6 +102,7 @@ export default function OptionsBar() {
                   setOption={setOption}
                   isMultiChat={true}
                 />
+                <AlternativeSettings conversation={conversation} setOption={setOption} />
               </div>
             </OptionsPopover>
             <SaveAsPresetDialog

client/src/components/Chat/Input/OptionsBar.tsx

@@ -1,165 +0,0 @@
-import { useRecoilState } from 'recoil';
-import { Settings2 } from 'lucide-react';
-import { useState, useEffect, useMemo } from 'react';
-import { tPresetUpdateSchema, EModelEndpoint } from 'librechat-data-provider';
-import type { TPreset } from 'librechat-data-provider';
-import { PluginStoreDialog } from '~/components';
-import {
-  EndpointSettings,
-  SaveAsPresetDialog,
-  EndpointOptionsPopover,
-} from '~/components/Endpoints';
-import { ModelSelect } from '~/components/Input/ModelSelect';
-import GenerationButtons from './GenerationButtons';
-import PopoverButtons from './PopoverButtons';
-import { useSetIndexOptions } from '~/hooks';
-import { useChatContext } from '~/Providers';
-import { Button } from '~/components/ui';
-import { cn, cardStyle } from '~/utils/';
-import store from '~/store';
-
-export default function OptionsBar({ messagesTree }) {
-  const [opacityClass, setOpacityClass] = useState('full-opacity');
-  const [saveAsDialogShow, setSaveAsDialogShow] = useState<boolean>(false);
-  const [showPluginStoreDialog, setShowPluginStoreDialog] = useRecoilState(
-    store.showPluginStoreDialog,
-  );
-
-  const { showPopover, conversation, latestMessage, setShowPopover, setShowBingToneSetting } =
-    useChatContext();
-  const { setOption } = useSetIndexOptions();
-
-  const { endpoint, conversationId, jailbreak } = conversation ?? {};
-
-  const altConditions: { [key: string]: boolean } = {
-    bingAI: !!(latestMessage && conversation?.jailbreak && endpoint === 'bingAI'),
-  };
-
-  const altSettings: { [key: string]: () => void } = {
-    bingAI: () => setShowBingToneSetting((prev) => !prev),
-  };
-
-  const noSettings = useMemo<{ [key: string]: boolean }>(
-    () => ({
-      [EModelEndpoint.chatGPTBrowser]: true,
-      [EModelEndpoint.bingAI]: jailbreak ? false : conversationId !== 'new',
-    }),
-    [jailbreak, conversationId],
-  );
-
-  useEffect(() => {
-    if (showPopover) {
-      return;
-    } else if (messagesTree && messagesTree.length >= 1) {
-      setOpacityClass('show');
-    } else {
-      setOpacityClass('full-opacity');
-    }
-  }, [messagesTree, showPopover]);
-
-  useEffect(() => {
-    if (endpoint && noSettings[endpoint]) {
-      setShowPopover(false);
-    }
-    // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [endpoint, noSettings]);
-
-  const saveAsPreset = () => {
-    setSaveAsDialogShow(true);
-  };
-
-  if (!endpoint) {
-    return null;
-  }
-
-  const triggerAdvancedMode = altConditions[endpoint]
-    ? altSettings[endpoint]
-    : () => setShowPopover((prev) => !prev);
-  return (
-    <div className="absolute left-0 right-0 mx-auto mb-2 last:mb-2 md:mx-4 md:last:mb-6 lg:mx-auto lg:max-w-2xl xl:max-w-3xl">
-      <GenerationButtons
-        endpoint={endpoint}
-        showPopover={showPopover}
-        opacityClass={opacityClass}
-      />
-      <span className="flex w-full flex-col items-center justify-center gap-0 md:order-none md:m-auto md:gap-2">
-        <div
-          className={cn(
-            'options-bar z-[61] flex w-full flex-wrap items-center justify-center gap-2',
-            showPopover ? '' : opacityClass,
-          )}
-          onMouseEnter={() => {
-            if (showPopover) {
-              return;
-            }
-            setOpacityClass('full-opacity');
-          }}
-          onMouseLeave={() => {
-            if (showPopover) {
-              return;
-            }
-            if (!messagesTree || messagesTree.length === 0) {
-              return;
-            }
-            setOpacityClass('show');
-          }}
-          onFocus={() => {
-            if (showPopover) {
-              return;
-            }
-            setOpacityClass('full-opacity');
-          }}
-          onBlur={() => {
-            if (showPopover) {
-              return;
-            }
-            if (!messagesTree || messagesTree.length === 0) {
-              return;
-            }
-            setOpacityClass('show');
-          }}
-        >
-          <ModelSelect conversation={conversation} setOption={setOption} isMultiChat={true} />
-          {!noSettings[endpoint] && (
-            <Button
-              id="advanced-mode-button"
-              customId="advanced-mode-button"
-              type="button"
-              className={cn(
-                cardStyle,
-                'min-w-4 z-50 flex h-[40px] flex-none items-center justify-center px-3 focus:ring-0 focus:ring-offset-0',
-              )}
-              onClick={triggerAdvancedMode}
-            >
-              <Settings2 id="advanced-settings" className="w-4 text-gray-600 dark:text-white" />
-            </Button>
-          )}
-        </div>
-        <EndpointOptionsPopover
-          visible={showPopover}
-          saveAsPreset={saveAsPreset}
-          closePopover={() => setShowPopover(false)}
-          PopoverButtons={<PopoverButtons />}
-        >
-          <div className="px-4 py-4">
-            <EndpointSettings
-              conversation={conversation}
-              setOption={setOption}
-              isMultiChat={true}
-            />
-          </div>
-        </EndpointOptionsPopover>
-        <SaveAsPresetDialog
-          open={saveAsDialogShow}
-          onOpenChange={setSaveAsDialogShow}
-          preset={
-            tPresetUpdateSchema.parse({
-              ...conversation,
-            }) as TPreset
-          }
-        />
-        <PluginStoreDialog isOpen={showPluginStoreDialog} setIsOpen={setShowPluginStoreDialog} />
-      </span>
-    </div>
-  );
-}

client/src/components/Chat/Input/OptionsPopover.tsx

@@ -63,7 +63,7 @@ export default function OptionsPopover({
             <div className="flex w-full items-center bg-gray-50 px-2 py-2 dark:bg-gray-700">
               <Button
                 type="button"
-                className="h-auto justify-start rounded-md bg-transparent px-2 py-1 text-xs font-medium font-normal text-black hover:bg-gray-100 hover:text-black dark:bg-transparent dark:text-white dark:hover:bg-gray-600"
+                className="h-auto w-[150px] justify-start rounded-md border-2 border-gray-300/50 bg-transparent px-2 py-1 text-xs font-medium font-normal text-black hover:bg-gray-100 hover:text-black focus:ring-1 focus:ring-green-500/90 dark:border-gray-500/50 dark:bg-transparent dark:text-white dark:hover:bg-gray-600 dark:focus:ring-green-500"
                 onClick={saveAsPreset}
               >
                 <Save className="mr-1 w-[14px]" />

client/src/components/Chat/Input/PopoverButtons.tsx

@@ -1,24 +1,33 @@
-import { EModelEndpoint } from 'librechat-data-provider';
+import { useRecoilState } from 'recoil';
+import { EModelEndpoint, SettingsViews } from 'librechat-data-provider';
 import type { ReactNode } from 'react';
-import { MessagesSquared, GPTIcon } from '~/components/svg';
+import { MessagesSquared, GPTIcon, AssistantIcon, DataIcon } from '~/components/svg';
 import { useChatContext } from '~/Providers';
 import { Button } from '~/components/ui';
 import { useLocalize } from '~/hooks';
 import { cn } from '~/utils/';
+import store from '~/store';
 
 type TPopoverButton = {
   label: string;
   buttonClass: string;
   handler: () => void;
+  type?: 'alternative';
   icon: ReactNode;
 };
 
 export default function PopoverButtons({
   buttonClass,
   iconClass = '',
+  endpoint: _overrideEndpoint,
+  endpointType: overrideEndpointType,
+  model: overrideModel,
 }: {
   buttonClass?: string;
   iconClass?: string;
+  endpoint?: EModelEndpoint | string;
+  endpointType?: EModelEndpoint | string;
+  model?: string | null;
 }) {
   const {
     conversation,
@@ -28,9 +37,13 @@ export default function PopoverButtons({
     setShowAgentSettings,
   } = useChatContext();
   const localize = useLocalize();
+  const [settingsView, setSettingsView] = useRecoilState(store.currentSettingsView);
+
+  const { model: _model, endpoint: _endpoint, endpointType } = conversation ?? {};
+  const overrideEndpoint = overrideEndpointType ?? _overrideEndpoint;
+  const endpoint = overrideEndpoint ?? endpointType ?? _endpoint;
+  const model = overrideModel ?? _model;
 
-  const { model, endpoint: _endpoint, endpointType } = conversation ?? {};
-  const endpoint = endpointType ?? _endpoint;
   const isGenerativeModel = model?.toLowerCase()?.includes('gemini');
   const isChatModel = !isGenerativeModel && model?.toLowerCase()?.includes('chat');
   const isTextModel = !isGenerativeModel && !isChatModel && /code|text/.test(model ?? '');
@@ -38,10 +51,12 @@ export default function PopoverButtons({
   const { showExamples } = optionSettings;
   const showExamplesButton = !isGenerativeModel && !isTextModel && isChatModel;
 
-  const triggerExamples = () =>
+  const triggerExamples = () => {
+    setSettingsView(SettingsViews.default);
     setOptionSettings((prev) => ({ ...prev, showExamples: !prev.showExamples }));
+  };
 
-  const buttons: { [key: string]: TPopoverButton[] } = {
+  const endpointSpecificbuttons: { [key: string]: TPopoverButton[] } = {
     [EModelEndpoint.google]: [
       {
         label: localize(showExamples ? 'com_hide_examples' : 'com_show_examples'),
@@ -56,14 +71,16 @@ export default function PopoverButtons({
           showAgentSettings ? 'com_show_completion_settings' : 'com_show_agent_settings',
         ),
         buttonClass: '',
-        handler: () => setShowAgentSettings((prev) => !prev),
+        handler: () => {
+          setSettingsView(SettingsViews.default);
+          setShowAgentSettings((prev) => !prev);
+        },
         icon: <GPTIcon className={cn('mr-1 w-[14px]', iconClass)} size={24} />,
       },
     ],
   };
 
-  const endpointButtons = buttons[endpoint ?? ''];
-  if (!endpointButtons) {
+  if (!endpoint) {
     return null;
   }
 
@@ -71,23 +88,71 @@ export default function PopoverButtons({
     return null;
   }
 
+  const additionalButtons: { [key: string]: TPopoverButton[] } = {
+    [SettingsViews.default]: [
+      {
+        label: 'Context Settings',
+        buttonClass: '',
+        type: 'alternative',
+        handler: () => setSettingsView(SettingsViews.advanced),
+        icon: <DataIcon className={cn('mr-1 h-6 w-[14px]', iconClass)} />,
+      },
+    ],
+    [SettingsViews.advanced]: [
+      {
+        label: 'Model Settings',
+        buttonClass: '',
+        type: 'alternative',
+        handler: () => setSettingsView(SettingsViews.default),
+        icon: <AssistantIcon className={cn('mr-1 h-6 w-[14px]', iconClass)} />,
+      },
+    ],
+  };
+
+  const endpointButtons = endpointSpecificbuttons[endpoint] ?? [];
+
+  const disabled = true;
+
   return (
-    <div>
-      {endpointButtons.map((button, index) => (
-        <Button
-          key={`button-${index}`}
-          type="button"
-          className={cn(
-            button.buttonClass,
-            'ml-1 h-auto justify-start bg-transparent px-2 py-1 text-xs font-medium font-normal text-black hover:bg-gray-100 hover:text-black focus:ring-0 focus:ring-offset-0 dark:bg-transparent dark:text-white dark:hover:bg-gray-600 dark:hover:text-white dark:focus:outline-none dark:focus:ring-offset-0',
-            buttonClass ?? '',
-          )}
-          onClick={button.handler}
-        >
-          {button.icon}
-          {button.label}
-        </Button>
-      ))}
+    <div className="flex w-full justify-between">
+      <div className="flex items-center justify-start">
+        {endpointButtons.map((button, index) => (
+          <Button
+            key={`button-${index}`}
+            type="button"
+            className={cn(
+              button.buttonClass,
+              'border-2 border-gray-300/50 focus:ring-1 focus:ring-green-500/90 dark:border-gray-500/50 dark:focus:ring-green-500',
+              'ml-1 h-full bg-transparent px-2 py-1 text-xs font-medium font-normal text-black hover:bg-gray-100 hover:text-black dark:bg-transparent dark:text-white dark:hover:bg-gray-600 dark:hover:text-white',
+              buttonClass ?? '',
+            )}
+            onClick={button.handler}
+          >
+            {button.icon}
+            {button.label}
+          </Button>
+        ))}
+      </div>
+      {disabled ? null : (
+        <div className="flex w-[150px] items-center justify-end">
+          {additionalButtons[settingsView].map((button, index) => (
+            <Button
+              key={`button-${index}`}
+              type="button"
+              className={cn(
+                button.buttonClass,
+                'flex justify-center border-2 border-gray-300/50 focus:ring-1 focus:ring-green-500/90 dark:border-gray-500/50 dark:focus:ring-green-500',
+                'h-full w-full bg-transparent px-2 py-1 text-xs font-medium font-normal text-black hover:bg-gray-100 hover:text-black dark:bg-transparent dark:text-white dark:hover:bg-gray-600 dark:hover:text-white',
+                buttonClass ?? '',
+              )}
+              onClick={button.handler}
+            >
+              {button.icon}
+              {button.label}
+            </Button>
+          ))}
+        </div>
+      )}
     </div>
   );
 }

client/src/components/Chat/Menus/Presets/EditPresetDialog.tsx

@@ -30,7 +30,7 @@ const EditPresetDialog = ({
     select: mapEndpoints,
   });
 
-  const { endpoint } = preset || {};
+  const { endpoint, endpointType, model } = preset || {};
   if (!endpoint) {
     return null;
   }
@@ -81,7 +81,7 @@ const EditPresetDialog = ({
                   />
                 </div>
               </div>
-              <div className="col-span-2 flex items-start justify-start gap-4 sm:col-span-1">
+              <div className="col-span-2 flex items-start justify-between gap-4 sm:col-span-4">
                 <div className="flex w-full flex-col">
                   <Label
                     htmlFor="endpoint"
@@ -92,6 +92,9 @@ const EditPresetDialog = ({
                   <PopoverButtons
                     buttonClass="ml-0 w-full border border-gray-100 dark:border-gray-600 dark:bg-gray-700 dark:hover:bg-gray-600 p-2 h-[40px] justify-center mt-0"
                     iconClass="hidden lg:block w-4 "
+                    endpoint={endpoint}
+                    endpointType={endpointType}
+                    model={model}
                   />
                 </div>
               </div>

client/src/components/Chat/Menus/Presets/PresetItems.tsx

@@ -5,7 +5,7 @@ import { Flipper, Flipped } from 'react-flip-toolkit';
 import { useGetEndpointsQuery } from 'librechat-data-provider/react-query';
 import type { FC } from 'react';
 import type { TPreset } from 'librechat-data-provider';
-import FileUpload from '~/components/Input/EndpointMenu/FileUpload';
+import FileUpload from '~/components/Chat/Input/Files/FileUpload';
 import { PinIcon, EditIcon, TrashIcon } from '~/components/svg';
 import DialogTemplate from '~/components/ui/DialogTemplate';
 import { getPresetTitle, getEndpointField } from '~/utils';