🪙 feat: Configure Max Context and Output Tokens (#2648)

* chore: make frequent 'error' log into 'debug' log * feat: add maxContextTokens as a conversation field * refactor(settings): increase popover height * feat: add DynamicInputNumber and maxContextTokens to all endpoints that support it (frontend), fix schema * feat: maxContextTokens handling (backend) * style: revert popover height * feat: max tokens * fix: Ollama Vision firebase compatibility * fix: Ollama Vision, use message_file_map to determine multimodal request * refactor: bring back MobileNav and improve title styling
2025-12-21 10:50:14 +01:00 · 2024-05-09 13:27:13 -04:00 · 2024-05-09 13:27:13 -04:00 · 6ba7f60eec
commit 6ba7f60eec
parent 5293b73b6d
26 changed files with 420 additions and 22 deletions
--- a/client/src/components/Chat/Input/OptionsPopover.tsx
+++ b/client/src/components/Chat/Input/OptionsPopover.tsx
@ -45,7 +45,7 @@ export default function OptionsPopover({

  const localize = useLocalize();
  const cardStyle =
-    'shadow-xl rounded-md min-w-[75px] font-normal bg-white border-black/10 border dark:bg-gray-700 text-black dark:text-white ';
+    'shadow-xl rounded-md min-w-[75px] font-normal bg-white border-black/10 border dark:bg-gray-700 text-black dark:text-white';

  if (!visible) {
    return null;
--- a/client/src/components/Endpoints/Settings/Anthropic.tsx
+++ b/client/src/components/Endpoints/Settings/Anthropic.tsx
@ -1,7 +1,6 @@
 import React from 'react';
 import TextareaAutosize from 'react-textarea-autosize';
 import type { TModelSelectProps } from '~/common';
-import { ESide } from '~/common';
 import {
  Input,
  Label,
@ -12,9 +11,11 @@ import {
  SelectDropDown,
  HoverCardTrigger,
 } from '~/components/ui';
+import { cn, defaultTextProps, optionText, removeFocusOutlines } from '~/utils';
+import { DynamicInputNumber } from '~/components/SidePanel/Parameters';
 import OptionHover from './OptionHover';
-import { cn, defaultTextProps, optionText, removeFocusOutlines } from '~/utils/';
 import { useLocalize } from '~/hooks';
+import { ESide } from '~/common';

 export default function Settings({ conversation, setOption, models, readonly }: TModelSelectProps) {
  const localize = useLocalize();
@ -83,6 +84,28 @@ export default function Settings({ conversation, setOption, models, readonly }:
        </div>
      </div>
      <div className="col-span-5 flex flex-col items-center justify-start gap-6 px-3 sm:col-span-2">
+        <DynamicInputNumber
+          columnSpan={2}
+          settingKey="maxContextTokens"
+          setOption={setOption}
+          label="com_endpoint_context_tokens"
+          labelCode={true}
+          description="com_endpoint_context_info"
+          descriptionCode={true}
+          placeholder="com_nav_theme_system"
+          placeholderCode={true}
+          descriptionSide="right"
+          conversation={conversation}
+          readonly={readonly}
+          range={{
+            min: 10,
+            max: 2000000,
+            step: 1000,
+          }}
+          className="mt-1 w-full justify-between"
+          inputClassName="w-1/3"
+          showDefault={false}
+        />
        <HoverCard openDelay={300}>
          <HoverCardTrigger className="grid w-full items-center gap-2">
            <div className="flex justify-between">
--- a/client/src/components/Endpoints/Settings/Google.tsx
+++ b/client/src/components/Endpoints/Settings/Google.tsx
@ -11,8 +11,9 @@ import {
  SelectDropDown,
  HoverCardTrigger,
 } from '~/components/ui';
-import OptionHover from './OptionHover';
 import { cn, defaultTextProps, optionText, removeFocusOutlines } from '~/utils';
+import { DynamicInputNumber } from '~/components/SidePanel/Parameters';
+import OptionHover from './OptionHover';
 import { useLocalize } from '~/hooks';
 import { ESide } from '~/common';

@ -103,6 +104,28 @@ export default function Settings({ conversation, setOption, models, readonly }:
        </div>
      </div>
      <div className="col-span-5 flex flex-col items-center justify-start gap-6 px-3 sm:col-span-2">
+        <DynamicInputNumber
+          columnSpan={2}
+          settingKey="maxContextTokens"
+          setOption={setOption}
+          label="com_endpoint_context_tokens"
+          labelCode={true}
+          description="com_endpoint_context_info"
+          descriptionCode={true}
+          placeholder="com_nav_theme_system"
+          placeholderCode={true}
+          descriptionSide="right"
+          conversation={conversation}
+          readonly={readonly}
+          range={{
+            min: 10,
+            max: 2000000,
+            step: 1000,
+          }}
+          className="mt-1 w-full justify-between"
+          inputClassName="w-1/3"
+          showDefault={false}
+        />
        <HoverCard openDelay={300}>
          <HoverCardTrigger className="grid w-full items-center gap-2">
            <div className="flex justify-between">
--- a/client/src/components/Endpoints/Settings/OpenAI.tsx
+++ b/client/src/components/Endpoints/Settings/OpenAI.tsx
@ -18,7 +18,7 @@ import {
  HoverCardTrigger,
 } from '~/components/ui';
 import { cn, defaultTextProps, optionText, removeFocusOutlines } from '~/utils';
-import { DynamicTags } from '~/components/SidePanel/Parameters';
+import { DynamicTags, DynamicInputNumber } from '~/components/SidePanel/Parameters';
 import { useLocalize, useDebouncedInput } from '~/hooks';
 import type { TModelSelectProps } from '~/common';
 import OptionHover from './OptionHover';
@ -154,6 +154,50 @@ export default function Settings({ conversation, setOption, models, readonly }:
        </div>
      </div>
      <div className="col-span-5 flex flex-col items-center justify-start gap-6 px-3 sm:col-span-2">
+        <DynamicInputNumber
+          columnSpan={2}
+          settingKey="maxContextTokens"
+          setOption={setOption}
+          label="com_endpoint_context_tokens"
+          labelCode={true}
+          description="com_endpoint_context_info"
+          descriptionCode={true}
+          placeholder="com_nav_theme_system"
+          placeholderCode={true}
+          descriptionSide="right"
+          conversation={conversation}
+          readonly={readonly}
+          range={{
+            min: 10,
+            max: 2000000,
+            step: 1000,
+          }}
+          className="mt-1 w-full justify-between"
+          inputClassName="w-1/3"
+          showDefault={false}
+        />
+        <DynamicInputNumber
+          columnSpan={2}
+          settingKey="max_tokens"
+          setOption={setOption}
+          label="com_endpoint_max_output_tokens"
+          labelCode={true}
+          description="com_endpoint_openai_max_tokens"
+          descriptionCode={true}
+          placeholder="com_nav_theme_system"
+          placeholderCode={true}
+          descriptionSide="top"
+          conversation={conversation}
+          readonly={readonly}
+          range={{
+            min: 10,
+            max: 2000000,
+            step: 1000,
+          }}
+          className="mt-1 w-full justify-between"
+          inputClassName="w-1/3"
+          showDefault={false}
+        />
        <HoverCard openDelay={300}>
          <HoverCardTrigger className="grid w-full items-center gap-2">
            <div className="flex justify-between">
--- a/client/src/components/Endpoints/Settings/Plugins.tsx
+++ b/client/src/components/Endpoints/Settings/Plugins.tsx
@ -14,9 +14,16 @@ import {
  HoverCardTrigger,
  MultiSelectDropDown,
 } from '~/components/ui';
-import { cn, defaultTextProps, optionText, removeFocusOutlines } from '~/utils';
+import {
+  cn,
+  defaultTextProps,
+  optionText,
+  removeFocusOutlines,
+  processPlugins,
+  selectPlugins,
+} from '~/utils';
+import { DynamicInputNumber } from '~/components/SidePanel/Parameters';
 import { useLocalize, useDebouncedInput } from '~/hooks';
-import { processPlugins, selectPlugins } from '~/utils';
 import OptionHover from './OptionHover';
 import { ESide } from '~/common';
 import store from '~/store';
@ -170,6 +177,28 @@ export default function Settings({
          containerClassName="flex w-full resize-none border border-transparent"
          labelClassName="dark:text-white"
        />
+        <DynamicInputNumber
+          columnSpan={2}
+          settingKey="maxContextTokens"
+          setOption={setOption}
+          label="com_endpoint_context_tokens"
+          labelCode={true}
+          description="com_endpoint_context_info"
+          descriptionCode={true}
+          placeholder="com_nav_theme_system"
+          placeholderCode={true}
+          descriptionSide="right"
+          conversation={conversation}
+          readonly={readonly}
+          range={{
+            min: 10,
+            max: 2000000,
+            step: 1000,
+          }}
+          className="mt-1 w-full justify-between"
+          inputClassName="w-1/3"
+          showDefault={false}
+        />
        <HoverCard openDelay={300}>
          <HoverCardTrigger className="grid w-full items-center gap-2">
            <div className="flex justify-between">
--- a/client/src/components/Nav/MobileNav.tsx
+++ b/client/src/components/Nav/MobileNav.tsx
@ -0,0 +1,73 @@
+import React from 'react';
+import { useRecoilValue } from 'recoil';
+import type { Dispatch, SetStateAction } from 'react';
+import { useLocalize, useNewConvo } from '~/hooks';
+import store from '~/store';
+
+export default function MobileNav({
+  setNavVisible,
+}: {
+  setNavVisible: Dispatch<SetStateAction<boolean>>;
+}) {
+  const localize = useLocalize();
+  const { newConversation } = useNewConvo(0);
+  const conversation = useRecoilValue(store.conversationByIndex(0));
+  const { title = 'New Chat' } = conversation || {};
+
+  return (
+    <div className="border-token-border-medium bg-token-main-surface-primary sticky top-0 z-10 flex min-h-[40px] items-center justify-center border-b bg-white pl-1 dark:bg-gray-800 dark:text-white md:hidden md:hidden">
+      <button
+        type="button"
+        data-testid="mobile-header-new-chat-button"
+        className="inline-flex h-10 w-10 items-center justify-center rounded-md hover:text-gray-800 focus:outline-none focus:ring-2 focus:ring-inset focus:ring-white active:opacity-50 dark:hover:text-white"
+        onClick={() =>
+          setNavVisible((prev) => {
+            localStorage.setItem('navVisible', JSON.stringify(!prev));
+            return !prev;
+          })
+        }
+      >
+        <span className="sr-only">{localize('com_nav_open_sidebar')}</span>
+        <svg
+          width="24"
+          height="24"
+          viewBox="0 0 24 24"
+          fill="none"
+          xmlns="http://www.w3.org/2000/svg"
+          className="icon-md"
+        >
+          <path
+            fillRule="evenodd"
+            clipRule="evenodd"
+            d="M3 8C3 7.44772 3.44772 7 4 7H20C20.5523 7 21 7.44772 21 8C21 8.55228 20.5523 9 20 9H4C3.44772 9 3 8.55228 3 8ZM3 16C3 15.4477 3.44772 15 4 15H14C14.5523 15 15 15.4477 15 16C15 16.5523 14.5523 17 14 17H4C3.44772 17 3 16.5523 3 16Z"
+            fill="currentColor"
+          />
+        </svg>
+      </button>
+      <h1 className="flex-1 overflow-hidden text-ellipsis whitespace-nowrap text-center text-sm font-normal">
+        {title || localize('com_ui_new_chat')}
+      </h1>
+      <button
+        type="button"
+        className="inline-flex h-10 w-10 items-center justify-center rounded-md hover:text-gray-800 focus:outline-none focus:ring-2 focus:ring-inset focus:ring-white active:opacity-50 dark:hover:text-white"
+        onClick={() => newConversation()}
+      >
+        <svg
+          width="24"
+          height="24"
+          viewBox="0 0 24 24"
+          fill="none"
+          xmlns="http://www.w3.org/2000/svg"
+          className="icon-md"
+        >
+          <path
+            fillRule="evenodd"
+            clipRule="evenodd"
+            d="M16.7929 2.79289C18.0118 1.57394 19.9882 1.57394 21.2071 2.79289C22.4261 4.01184 22.4261 5.98815 21.2071 7.20711L12.7071 15.7071C12.5196 15.8946 12.2652 16 12 16H9C8.44772 16 8 15.5523 8 15V12C8 11.7348 8.10536 11.4804 8.29289 11.2929L16.7929 2.79289ZM19.7929 4.20711C19.355 3.7692 18.645 3.7692 18.2071 4.2071L10 12.4142V14H11.5858L19.7929 5.79289C20.2308 5.35499 20.2308 4.64501 19.7929 4.20711ZM6 5C5.44772 5 5 5.44771 5 6V18C5 18.5523 5.44772 19 6 19H18C18.5523 19 19 18.5523 19 18V14C19 13.4477 19.4477 13 20 13C20.5523 13 21 13.4477 21 14V18C21 19.6569 19.6569 21 18 21H6C4.34315 21 3 19.6569 3 18V6C3 4.34314 4.34315 3 6 3H10C10.5523 3 11 3.44771 11 4C11 4.55228 10.5523 5 10 5H6Z"
+            fill="currentColor"
+          />
+        </svg>
+      </button>
+    </div>
+  );
+}
--- a/client/src/components/Nav/index.ts
+++ b/client/src/components/Nav/index.ts
@ -2,6 +2,7 @@ export * from './ExportConversation';
 export * from './SettingsTabs/';
 export { default as ClearConvos } from './ClearConvos';
 export { default as Logout } from './Logout';
+export { default as MobileNav } from './MobileNav';
 export { default as Nav } from './Nav';
 export { default as NavLink } from './NavLink';
 export { default as NavLinks } from './NavLinks';
--- a/client/src/components/SidePanel/Parameters/DynamicInputNumber.tsx
+++ b/client/src/components/SidePanel/Parameters/DynamicInputNumber.tsx
@ -0,0 +1,108 @@
+import { OptionTypes } from 'librechat-data-provider';
+import type { DynamicSettingProps } from 'librechat-data-provider';
+import type { ValueType } from '@rc-component/mini-decimal';
+import { Label, HoverCard, InputNumber, HoverCardTrigger } from '~/components/ui';
+import { useLocalize, useDebouncedInput, useParameterEffects } from '~/hooks';
+import { cn, defaultTextProps, optionText } from '~/utils';
+import { ESide } from '~/common';
+import { useChatContext } from '~/Providers';
+import OptionHover from './OptionHover';
+
+function DynamicInputNumber({
+  label,
+  settingKey,
+  defaultValue,
+  description,
+  columnSpan,
+  setOption,
+  optionType,
+  readonly = false,
+  showDefault = true,
+  labelCode,
+  descriptionCode,
+  placeholderCode,
+  placeholder,
+  conversation,
+  range,
+  className = '',
+  inputClassName = '',
+}: DynamicSettingProps) {
+  const localize = useLocalize();
+  const { preset } = useChatContext();
+
+  const [setInputValue, inputValue] = useDebouncedInput<ValueType | null>({
+    optionKey: optionType !== OptionTypes.Custom ? settingKey : undefined,
+    initialValue:
+      optionType !== OptionTypes.Custom
+        ? (conversation?.[settingKey] as number)
+        : (defaultValue as number),
+    setter: () => ({}),
+    setOption,
+  });
+
+  useParameterEffects({
+    preset,
+    settingKey,
+    defaultValue: typeof defaultValue === 'undefined' ? '' : defaultValue,
+    conversation,
+    inputValue,
+    setInputValue,
+  });
+
+  return (
+    <div
+      className={cn(
+        'flex flex-col items-center justify-start gap-6',
+        columnSpan ? `col-span-${columnSpan}` : 'col-span-full',
+        className,
+      )}
+    >
+      <HoverCard openDelay={300}>
+        <HoverCardTrigger className="grid w-full items-center gap-2">
+          <div className="flex justify-between">
+            <Label
+              htmlFor={`${settingKey}-dynamic-setting`}
+              className="text-left text-sm font-medium"
+            >
+              {labelCode ? localize(label ?? '') || label : label ?? settingKey}{' '}
+              {showDefault && (
+                <small className="opacity-40">
+                  ({localize('com_endpoint_default')}: {defaultValue})
+                </small>
+              )}
+            </Label>
+            <InputNumber
+              id={`${settingKey}-dynamic-setting-input-number`}
+              disabled={readonly}
+              value={inputValue}
+              onChange={setInputValue}
+              min={range?.min}
+              max={range?.max}
+              step={range?.step}
+              placeholder={
+                placeholderCode ? localize(placeholder ?? '') || placeholder : placeholder
+              }
+              controls={false}
+              className={cn(
+                defaultTextProps,
+                cn(
+                  optionText,
+                  'reset-rc-number-input reset-rc-number-input-text-right h-auto w-12 border-0 group-hover/temp:border-gray-200',
+                ),
+                inputClassName,
+              )}
+            />
+          </div>
+        </HoverCardTrigger>
+        {description && (
+          <OptionHover
+            description={descriptionCode ? localize(description) || description : description}
+            side={ESide.Left}
+          />
+        )}
+      </HoverCard>
+    </div>
+  );
+}
+
+export default DynamicInputNumber;
--- a/client/src/components/SidePanel/Parameters/index.ts
+++ b/client/src/components/SidePanel/Parameters/index.ts
@ -1,3 +1,4 @@
+export { default as DynamicInputNumber } from './DynamicInputNumber';
 export { default as DynamicDropdown } from './DynamicDropdown';
 export { default as DynamicCheckbox } from './DynamicCheckbox';
 export { default as DynamicTextarea } from './DynamicTextarea';