From 6ba7f60eec0db4b782c3d264bed56251da034f42 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Thu, 9 May 2024 13:27:13 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=AA=99=20feat:=20Configure=20Max=20Contex?= =?UTF-8?q?t=20and=20Output=20Tokens=20(#2648)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: make frequent 'error' log into 'debug' log * feat: add maxContextTokens as a conversation field * refactor(settings): increase popover height * feat: add DynamicInputNumber and maxContextTokens to all endpoints that support it (frontend), fix schema * feat: maxContextTokens handling (backend) * style: revert popover height * feat: max tokens * fix: Ollama Vision firebase compatibility * fix: Ollama Vision, use message_file_map to determine multimodal request * refactor: bring back MobileNav and improve title styling --- api/app/clients/AnthropicClient.js | 5 +- api/app/clients/GoogleClient.js | 5 +- api/app/clients/OpenAIClient.js | 13 ++- api/app/clients/specs/FakeClient.js | 3 +- api/models/plugins/mongoMeili.js | 2 +- api/models/schema/defaults.js | 6 + .../Endpoints/anthropic/buildOptions.js | 12 +- .../services/Endpoints/custom/buildOptions.js | 14 ++- .../Endpoints/gptPlugins/buildOptions.js | 2 + .../services/Endpoints/openAI/buildOptions.js | 14 ++- api/server/services/Files/images/encode.js | 2 +- .../components/Chat/Input/OptionsPopover.tsx | 2 +- .../Endpoints/Settings/Anthropic.tsx | 27 ++++- .../components/Endpoints/Settings/Google.tsx | 25 +++- .../components/Endpoints/Settings/OpenAI.tsx | 46 +++++++- .../components/Endpoints/Settings/Plugins.tsx | 33 +++++- client/src/components/Nav/MobileNav.tsx | 73 ++++++++++++ client/src/components/Nav/index.ts | 1 + .../Parameters/DynamicInputNumber.tsx | 108 ++++++++++++++++++ .../components/SidePanel/Parameters/index.ts | 1 + client/src/localization/languages/Eng.ts | 6 + client/src/routes/Root.tsx | 3 +- packages/data-provider/package.json | 2 +- packages/data-provider/src/generate.ts | 8 ++ packages/data-provider/src/schemas.ts | 28 +++++ packages/data-provider/src/types.ts | 1 + 26 files changed, 420 insertions(+), 22 deletions(-) create mode 100644 client/src/components/Nav/MobileNav.tsx create mode 100644 client/src/components/SidePanel/Parameters/DynamicInputNumber.tsx diff --git a/api/app/clients/AnthropicClient.js b/api/app/clients/AnthropicClient.js index 48e1f22b64..def067637c 100644 --- a/api/app/clients/AnthropicClient.js +++ b/api/app/clients/AnthropicClient.js @@ -75,7 +75,9 @@ class AnthropicClient extends BaseClient { this.options.attachments?.then((attachments) => this.checkVisionRequest(attachments)); this.maxContextTokens = - getModelMaxTokens(this.modelOptions.model, EModelEndpoint.anthropic) ?? 100000; + this.options.maxContextTokens ?? + getModelMaxTokens(this.modelOptions.model, EModelEndpoint.anthropic) ?? + 100000; this.maxResponseTokens = this.modelOptions.maxOutputTokens || 1500; this.maxPromptTokens = this.options.maxPromptTokens || this.maxContextTokens - this.maxResponseTokens; @@ -652,6 +654,7 @@ class AnthropicClient extends BaseClient { getSaveOptions() { return { + maxContextTokens: this.options.maxContextTokens, promptPrefix: this.options.promptPrefix, modelLabel: this.options.modelLabel, resendFiles: this.options.resendFiles, diff --git a/api/app/clients/GoogleClient.js b/api/app/clients/GoogleClient.js index 3637cf58c2..e3a33d3e7c 100644 --- a/api/app/clients/GoogleClient.js +++ b/api/app/clients/GoogleClient.js @@ -138,7 +138,10 @@ class GoogleClient extends BaseClient { !isGenerativeModel && !isChatModel && /code|text/.test(this.modelOptions.model); const { isTextModel } = this; - this.maxContextTokens = getModelMaxTokens(this.modelOptions.model, EModelEndpoint.google); + this.maxContextTokens = + this.options.maxContextTokens ?? + getModelMaxTokens(this.modelOptions.model, EModelEndpoint.google); + // The max prompt tokens is determined by the max context tokens minus the max response tokens. // Earlier messages will be dropped until the prompt is within the limit. this.maxResponseTokens = this.modelOptions.maxOutputTokens || settings.maxOutputTokens.default; diff --git a/api/app/clients/OpenAIClient.js b/api/app/clients/OpenAIClient.js index 97815d819e..c232e27817 100644 --- a/api/app/clients/OpenAIClient.js +++ b/api/app/clients/OpenAIClient.js @@ -161,11 +161,13 @@ class OpenAIClient extends BaseClient { model.startsWith('text-chat') || model.startsWith('text-davinci-002-render'); this.maxContextTokens = + this.options.maxContextTokens ?? getModelMaxTokens( model, this.options.endpointType ?? this.options.endpoint, this.options.endpointTokenConfig, - ) ?? 4095; // 1 less than maximum + ) ?? + 4095; // 1 less than maximum if (this.shouldSummarize) { this.maxContextTokens = Math.floor(this.maxContextTokens / 2); @@ -407,6 +409,7 @@ class OpenAIClient extends BaseClient { getSaveOptions() { return { + maxContextTokens: this.options.maxContextTokens, chatGptLabel: this.options.chatGptLabel, promptPrefix: this.options.promptPrefix, resendFiles: this.options.resendFiles, @@ -435,7 +438,11 @@ class OpenAIClient extends BaseClient { * @returns {Promise} */ async addImageURLs(message, attachments) { - const { files, image_urls } = await encodeAndFormat(this.options.req, attachments); + const { files, image_urls } = await encodeAndFormat( + this.options.req, + attachments, + this.options.endpoint, + ); message.image_urls = image_urls.length ? image_urls : undefined; return files; } @@ -1158,7 +1165,7 @@ ${convo} }); } - if (this.options.attachments && this.options.endpoint?.toLowerCase() === 'ollama') { + if (this.message_file_map && this.options.endpoint?.toLowerCase() === 'ollama') { const ollamaClient = new OllamaClient({ baseURL }); return await ollamaClient.chatCompletion({ payload: modelOptions, diff --git a/api/app/clients/specs/FakeClient.js b/api/app/clients/specs/FakeClient.js index a5915adcf2..7f4b75e1db 100644 --- a/api/app/clients/specs/FakeClient.js +++ b/api/app/clients/specs/FakeClient.js @@ -40,7 +40,8 @@ class FakeClient extends BaseClient { }; } - this.maxContextTokens = getModelMaxTokens(this.modelOptions.model) ?? 4097; + this.maxContextTokens = + this.options.maxContextTokens ?? getModelMaxTokens(this.modelOptions.model) ?? 4097; } buildMessages() {} getTokenCount(str) { diff --git a/api/models/plugins/mongoMeili.js b/api/models/plugins/mongoMeili.js index 79dd30b11c..261b5c50c7 100644 --- a/api/models/plugins/mongoMeili.js +++ b/api/models/plugins/mongoMeili.js @@ -348,7 +348,7 @@ module.exports = function mongoMeili(schema, options) { try { meiliDoc = await client.index('convos').getDocument(doc.conversationId); } catch (error) { - logger.error( + logger.debug( '[MeiliMongooseModel.findOneAndUpdate] Convo not found in MeiliSearch and will index ' + doc.conversationId, error, diff --git a/api/models/schema/defaults.js b/api/models/schema/defaults.js index 7e786d1c49..16ceb6aa20 100644 --- a/api/models/schema/defaults.js +++ b/api/models/schema/defaults.js @@ -104,6 +104,12 @@ const conversationPreset = { type: String, }, tools: { type: [{ type: String }], default: undefined }, + maxContextTokens: { + type: Number, + }, + max_tokens: { + type: Number, + }, }; const agentOptions = { diff --git a/api/server/services/Endpoints/anthropic/buildOptions.js b/api/server/services/Endpoints/anthropic/buildOptions.js index 77588c293d..677eabc6ae 100644 --- a/api/server/services/Endpoints/anthropic/buildOptions.js +++ b/api/server/services/Endpoints/anthropic/buildOptions.js @@ -1,5 +1,14 @@ const buildOptions = (endpoint, parsedBody) => { - const { modelLabel, promptPrefix, resendFiles, iconURL, greeting, spec, ...rest } = parsedBody; + const { + modelLabel, + promptPrefix, + maxContextTokens, + resendFiles, + iconURL, + greeting, + spec, + ...rest + } = parsedBody; const endpointOption = { endpoint, modelLabel, @@ -8,6 +17,7 @@ const buildOptions = (endpoint, parsedBody) => { iconURL, greeting, spec, + maxContextTokens, modelOptions: { ...rest, }, diff --git a/api/server/services/Endpoints/custom/buildOptions.js b/api/server/services/Endpoints/custom/buildOptions.js index 8722e82362..83f8d387dd 100644 --- a/api/server/services/Endpoints/custom/buildOptions.js +++ b/api/server/services/Endpoints/custom/buildOptions.js @@ -1,6 +1,15 @@ const buildOptions = (endpoint, parsedBody, endpointType) => { - const { chatGptLabel, promptPrefix, resendFiles, imageDetail, iconURL, greeting, spec, ...rest } = - parsedBody; + const { + chatGptLabel, + promptPrefix, + maxContextTokens, + resendFiles, + imageDetail, + iconURL, + greeting, + spec, + ...rest + } = parsedBody; const endpointOption = { endpoint, endpointType, @@ -11,6 +20,7 @@ const buildOptions = (endpoint, parsedBody, endpointType) => { iconURL, greeting, spec, + maxContextTokens, modelOptions: { ...rest, }, diff --git a/api/server/services/Endpoints/gptPlugins/buildOptions.js b/api/server/services/Endpoints/gptPlugins/buildOptions.js index f5875ec38f..ec098e9e3b 100644 --- a/api/server/services/Endpoints/gptPlugins/buildOptions.js +++ b/api/server/services/Endpoints/gptPlugins/buildOptions.js @@ -7,6 +7,7 @@ const buildOptions = (endpoint, parsedBody) => { iconURL, greeting, spec, + maxContextTokens, ...modelOptions } = parsedBody; const endpointOption = { @@ -21,6 +22,7 @@ const buildOptions = (endpoint, parsedBody) => { iconURL, greeting, spec, + maxContextTokens, modelOptions, }; diff --git a/api/server/services/Endpoints/openAI/buildOptions.js b/api/server/services/Endpoints/openAI/buildOptions.js index f90886ed71..1a6ebea4b6 100644 --- a/api/server/services/Endpoints/openAI/buildOptions.js +++ b/api/server/services/Endpoints/openAI/buildOptions.js @@ -1,6 +1,15 @@ const buildOptions = (endpoint, parsedBody) => { - const { chatGptLabel, promptPrefix, resendFiles, imageDetail, iconURL, greeting, spec, ...rest } = - parsedBody; + const { + chatGptLabel, + promptPrefix, + maxContextTokens, + resendFiles, + imageDetail, + iconURL, + greeting, + spec, + ...rest + } = parsedBody; const endpointOption = { endpoint, chatGptLabel, @@ -10,6 +19,7 @@ const buildOptions = (endpoint, parsedBody) => { iconURL, greeting, spec, + maxContextTokens, modelOptions: { ...rest, }, diff --git a/api/server/services/Files/images/encode.js b/api/server/services/Files/images/encode.js index a9d27140f5..4edb0bd56c 100644 --- a/api/server/services/Files/images/encode.js +++ b/api/server/services/Files/images/encode.js @@ -23,7 +23,7 @@ async function fetchImageToBase64(url) { } } -const base64Only = new Set([EModelEndpoint.google, EModelEndpoint.anthropic]); +const base64Only = new Set([EModelEndpoint.google, EModelEndpoint.anthropic, 'Ollama', 'ollama']); /** * Encodes and formats the given files. diff --git a/client/src/components/Chat/Input/OptionsPopover.tsx b/client/src/components/Chat/Input/OptionsPopover.tsx index b123105e53..3b6d895cb6 100644 --- a/client/src/components/Chat/Input/OptionsPopover.tsx +++ b/client/src/components/Chat/Input/OptionsPopover.tsx @@ -45,7 +45,7 @@ export default function OptionsPopover({ const localize = useLocalize(); const cardStyle = - 'shadow-xl rounded-md min-w-[75px] font-normal bg-white border-black/10 border dark:bg-gray-700 text-black dark:text-white '; + 'shadow-xl rounded-md min-w-[75px] font-normal bg-white border-black/10 border dark:bg-gray-700 text-black dark:text-white'; if (!visible) { return null; diff --git a/client/src/components/Endpoints/Settings/Anthropic.tsx b/client/src/components/Endpoints/Settings/Anthropic.tsx index 55ef42a23f..92f511d2ce 100644 --- a/client/src/components/Endpoints/Settings/Anthropic.tsx +++ b/client/src/components/Endpoints/Settings/Anthropic.tsx @@ -1,7 +1,6 @@ import React from 'react'; import TextareaAutosize from 'react-textarea-autosize'; import type { TModelSelectProps } from '~/common'; -import { ESide } from '~/common'; import { Input, Label, @@ -12,9 +11,11 @@ import { SelectDropDown, HoverCardTrigger, } from '~/components/ui'; +import { cn, defaultTextProps, optionText, removeFocusOutlines } from '~/utils'; +import { DynamicInputNumber } from '~/components/SidePanel/Parameters'; import OptionHover from './OptionHover'; -import { cn, defaultTextProps, optionText, removeFocusOutlines } from '~/utils/'; import { useLocalize } from '~/hooks'; +import { ESide } from '~/common'; export default function Settings({ conversation, setOption, models, readonly }: TModelSelectProps) { const localize = useLocalize(); @@ -83,6 +84,28 @@ export default function Settings({ conversation, setOption, models, readonly }:
+
diff --git a/client/src/components/Endpoints/Settings/Google.tsx b/client/src/components/Endpoints/Settings/Google.tsx index 23fad05845..98d7d007d1 100644 --- a/client/src/components/Endpoints/Settings/Google.tsx +++ b/client/src/components/Endpoints/Settings/Google.tsx @@ -11,8 +11,9 @@ import { SelectDropDown, HoverCardTrigger, } from '~/components/ui'; -import OptionHover from './OptionHover'; import { cn, defaultTextProps, optionText, removeFocusOutlines } from '~/utils'; +import { DynamicInputNumber } from '~/components/SidePanel/Parameters'; +import OptionHover from './OptionHover'; import { useLocalize } from '~/hooks'; import { ESide } from '~/common'; @@ -103,6 +104,28 @@ export default function Settings({ conversation, setOption, models, readonly }:
+
diff --git a/client/src/components/Endpoints/Settings/OpenAI.tsx b/client/src/components/Endpoints/Settings/OpenAI.tsx index ce04a7f417..8563cb359f 100644 --- a/client/src/components/Endpoints/Settings/OpenAI.tsx +++ b/client/src/components/Endpoints/Settings/OpenAI.tsx @@ -18,7 +18,7 @@ import { HoverCardTrigger, } from '~/components/ui'; import { cn, defaultTextProps, optionText, removeFocusOutlines } from '~/utils'; -import { DynamicTags } from '~/components/SidePanel/Parameters'; +import { DynamicTags, DynamicInputNumber } from '~/components/SidePanel/Parameters'; import { useLocalize, useDebouncedInput } from '~/hooks'; import type { TModelSelectProps } from '~/common'; import OptionHover from './OptionHover'; @@ -154,6 +154,50 @@ export default function Settings({ conversation, setOption, models, readonly }:
+ +
diff --git a/client/src/components/Endpoints/Settings/Plugins.tsx b/client/src/components/Endpoints/Settings/Plugins.tsx index 64cd114650..9085810e1d 100644 --- a/client/src/components/Endpoints/Settings/Plugins.tsx +++ b/client/src/components/Endpoints/Settings/Plugins.tsx @@ -14,9 +14,16 @@ import { HoverCardTrigger, MultiSelectDropDown, } from '~/components/ui'; -import { cn, defaultTextProps, optionText, removeFocusOutlines } from '~/utils'; +import { + cn, + defaultTextProps, + optionText, + removeFocusOutlines, + processPlugins, + selectPlugins, +} from '~/utils'; +import { DynamicInputNumber } from '~/components/SidePanel/Parameters'; import { useLocalize, useDebouncedInput } from '~/hooks'; -import { processPlugins, selectPlugins } from '~/utils'; import OptionHover from './OptionHover'; import { ESide } from '~/common'; import store from '~/store'; @@ -170,6 +177,28 @@ export default function Settings({ containerClassName="flex w-full resize-none border border-transparent" labelClassName="dark:text-white" /> +
diff --git a/client/src/components/Nav/MobileNav.tsx b/client/src/components/Nav/MobileNav.tsx new file mode 100644 index 0000000000..35defea508 --- /dev/null +++ b/client/src/components/Nav/MobileNav.tsx @@ -0,0 +1,73 @@ +import React from 'react'; +import { useRecoilValue } from 'recoil'; +import type { Dispatch, SetStateAction } from 'react'; +import { useLocalize, useNewConvo } from '~/hooks'; +import store from '~/store'; + +export default function MobileNav({ + setNavVisible, +}: { + setNavVisible: Dispatch>; +}) { + const localize = useLocalize(); + const { newConversation } = useNewConvo(0); + const conversation = useRecoilValue(store.conversationByIndex(0)); + const { title = 'New Chat' } = conversation || {}; + + return ( +
+ +

+ {title || localize('com_ui_new_chat')} +

+ +
+ ); +} diff --git a/client/src/components/Nav/index.ts b/client/src/components/Nav/index.ts index 625e2918b5..4a3de60076 100644 --- a/client/src/components/Nav/index.ts +++ b/client/src/components/Nav/index.ts @@ -2,6 +2,7 @@ export * from './ExportConversation'; export * from './SettingsTabs/'; export { default as ClearConvos } from './ClearConvos'; export { default as Logout } from './Logout'; +export { default as MobileNav } from './MobileNav'; export { default as Nav } from './Nav'; export { default as NavLink } from './NavLink'; export { default as NavLinks } from './NavLinks'; diff --git a/client/src/components/SidePanel/Parameters/DynamicInputNumber.tsx b/client/src/components/SidePanel/Parameters/DynamicInputNumber.tsx new file mode 100644 index 0000000000..3807f2ed6f --- /dev/null +++ b/client/src/components/SidePanel/Parameters/DynamicInputNumber.tsx @@ -0,0 +1,108 @@ +import { OptionTypes } from 'librechat-data-provider'; +import type { DynamicSettingProps } from 'librechat-data-provider'; +import type { ValueType } from '@rc-component/mini-decimal'; +import { Label, HoverCard, InputNumber, HoverCardTrigger } from '~/components/ui'; +import { useLocalize, useDebouncedInput, useParameterEffects } from '~/hooks'; +import { cn, defaultTextProps, optionText } from '~/utils'; +import { ESide } from '~/common'; +import { useChatContext } from '~/Providers'; +import OptionHover from './OptionHover'; + +function DynamicInputNumber({ + label, + settingKey, + defaultValue, + description, + columnSpan, + setOption, + optionType, + readonly = false, + showDefault = true, + labelCode, + descriptionCode, + placeholderCode, + placeholder, + conversation, + range, + className = '', + inputClassName = '', +}: DynamicSettingProps) { + const localize = useLocalize(); + const { preset } = useChatContext(); + + const [setInputValue, inputValue] = useDebouncedInput({ + optionKey: optionType !== OptionTypes.Custom ? settingKey : undefined, + initialValue: + optionType !== OptionTypes.Custom + ? (conversation?.[settingKey] as number) + : (defaultValue as number), + setter: () => ({}), + setOption, + }); + + useParameterEffects({ + preset, + settingKey, + defaultValue: typeof defaultValue === 'undefined' ? '' : defaultValue, + conversation, + inputValue, + setInputValue, + }); + + return ( +
+ + +
+ + +
+ + {description && ( + + )} + +
+ ); +} + +export default DynamicInputNumber; diff --git a/client/src/components/SidePanel/Parameters/index.ts b/client/src/components/SidePanel/Parameters/index.ts index 63b7b55eea..e5c59ced58 100644 --- a/client/src/components/SidePanel/Parameters/index.ts +++ b/client/src/components/SidePanel/Parameters/index.ts @@ -1,3 +1,4 @@ +export { default as DynamicInputNumber } from './DynamicInputNumber'; export { default as DynamicDropdown } from './DynamicDropdown'; export { default as DynamicCheckbox } from './DynamicCheckbox'; export { default as DynamicTextarea } from './DynamicTextarea'; diff --git a/client/src/localization/languages/Eng.ts b/client/src/localization/languages/Eng.ts index f43d2b04ec..b6511a992f 100644 --- a/client/src/localization/languages/Eng.ts +++ b/client/src/localization/languages/Eng.ts @@ -280,6 +280,9 @@ export default { com_endpoint_tone_style: 'Tone Style', com_endpoint_token_count: 'Token count', com_endpoint_output: 'Output', + com_endpoint_context_tokens: 'Max Context Tokens', + com_endpoint_context_info: `The maximum number of tokens that can be used for context. Use this for control of how many tokens are sent per request. + If unspecified, will use system defaults based on known models' context size. Setting higher values may result in errors and/or higher token cost.`, com_endpoint_google_temp: 'Higher values = more random, while lower values = more focused and deterministic. We recommend altering this or Top P but not both.', com_endpoint_google_topp: @@ -305,6 +308,9 @@ export default { com_endpoint_max_output_tokens: 'Max Output Tokens', com_endpoint_stop: 'Stop Sequences', com_endpoint_stop_placeholder: 'Separate values by pressing `Enter`', + com_endpoint_openai_max_tokens: `Optional \`max_tokens\` field, representing the maximum number of tokens that can be generated in the chat completion. + + The total length of input tokens and generated tokens is limited by the models context length. You may experience errors if this number exceeds the max context tokens.`, com_endpoint_openai_temp: 'Higher values = more random, while lower values = more focused and deterministic. We recommend altering this or Top P but not both.', com_endpoint_openai_max: diff --git a/client/src/routes/Root.tsx b/client/src/routes/Root.tsx index 4afe07fc88..529de11726 100644 --- a/client/src/routes/Root.tsx +++ b/client/src/routes/Root.tsx @@ -5,7 +5,7 @@ import { useGetSearchEnabledQuery } from 'librechat-data-provider/react-query'; import type { ContextType } from '~/common'; import { useAuthContext, useAssistantsMap, useFileMap } from '~/hooks'; import { AssistantsMapContext, FileMapContext } from '~/Providers'; -import { Nav } from '~/components/Nav'; +import { Nav, MobileNav } from '~/components/Nav'; import store from '~/store'; export default function Root() { @@ -45,6 +45,7 @@ export default function Root() {
diff --git a/packages/data-provider/package.json b/packages/data-provider/package.json index 8a1be2f304..661f2a7888 100644 --- a/packages/data-provider/package.json +++ b/packages/data-provider/package.json @@ -1,6 +1,6 @@ { "name": "librechat-data-provider", - "version": "0.6.0", + "version": "0.6.1", "description": "data services for librechat apps", "main": "dist/index.js", "module": "dist/index.es.js", diff --git a/packages/data-provider/src/generate.ts b/packages/data-provider/src/generate.ts index 6bf7b70f92..fcb088473e 100644 --- a/packages/data-provider/src/generate.ts +++ b/packages/data-provider/src/generate.ts @@ -72,6 +72,8 @@ export type DynamicSettingProps = Partial & { setOption: TSetOption; conversation: TConversation | TPreset | null; defaultValue?: number | boolean | string | string[]; + className?: string; + inputClassName?: string; }; const requiredSettingFields = ['key', 'type', 'component']; @@ -508,6 +510,7 @@ export const generateOpenAISchema = (customOpenAI: OpenAISettings) => { frequency_penalty: true, resendFiles: true, imageDetail: true, + maxContextTokens: true, }) .transform((obj) => ({ ...obj, @@ -521,6 +524,7 @@ export const generateOpenAISchema = (customOpenAI: OpenAISettings) => { resendFiles: typeof obj.resendFiles === 'boolean' ? obj.resendFiles : defaults.resendFiles.default, imageDetail: obj.imageDetail ?? defaults.imageDetail.default, + maxContextTokens: obj.maxContextTokens ?? undefined, })) .catch(() => ({ model: defaults.model.default, @@ -532,6 +536,7 @@ export const generateOpenAISchema = (customOpenAI: OpenAISettings) => { frequency_penalty: defaults.frequency_penalty.default, resendFiles: defaults.resendFiles.default, imageDetail: defaults.imageDetail.default, + maxContextTokens: undefined, })); }; @@ -547,6 +552,7 @@ export const generateGoogleSchema = (customGoogle: GoogleSettings) => { maxOutputTokens: true, topP: true, topK: true, + maxContextTokens: true, }) .transform((obj) => { const isGemini = obj?.model?.toLowerCase()?.includes('gemini'); @@ -571,6 +577,7 @@ export const generateGoogleSchema = (customGoogle: GoogleSettings) => { maxOutputTokens, topP: obj.topP ?? defaults.topP.default, topK: obj.topK ?? defaults.topK.default, + maxContextTokens: obj.maxContextTokens ?? undefined, }; }) .catch(() => ({ @@ -582,5 +589,6 @@ export const generateGoogleSchema = (customGoogle: GoogleSettings) => { maxOutputTokens: defaults.maxOutputTokens.default, topP: defaults.topP.default, topK: defaults.topK.default, + maxContextTokens: undefined, })); }; diff --git a/packages/data-provider/src/schemas.ts b/packages/data-provider/src/schemas.ts index 7ca011404c..6f13098984 100644 --- a/packages/data-provider/src/schemas.ts +++ b/packages/data-provider/src/schemas.ts @@ -105,6 +105,12 @@ export const openAISettings = { resendFiles: { default: true, }, + maxContextTokens: { + default: undefined, + }, + max_tokens: { + default: undefined, + }, imageDetail: { default: ImageDetail.auto, }, @@ -309,6 +315,8 @@ export const tConversationSchema = z.object({ maxOutputTokens: z.number().optional(), agentOptions: tAgentOptionsSchema.nullable().optional(), file_ids: z.array(z.string()).optional(), + maxContextTokens: z.number().optional(), + max_tokens: z.number().optional(), /** @deprecated */ resendImages: z.boolean().optional(), /* vision */ @@ -382,6 +390,8 @@ export const openAISchema = tConversationSchema iconURL: true, greeting: true, spec: true, + maxContextTokens: true, + max_tokens: true, }) .transform((obj) => ({ ...obj, @@ -399,6 +409,8 @@ export const openAISchema = tConversationSchema iconURL: obj.iconURL ?? undefined, greeting: obj.greeting ?? undefined, spec: obj.spec ?? undefined, + maxContextTokens: obj.maxContextTokens ?? undefined, + max_tokens: obj.max_tokens ?? undefined, })) .catch(() => ({ model: openAISettings.model.default, @@ -414,6 +426,8 @@ export const openAISchema = tConversationSchema iconURL: undefined, greeting: undefined, spec: undefined, + maxContextTokens: undefined, + max_tokens: undefined, })); export const googleSchema = tConversationSchema @@ -429,6 +443,7 @@ export const googleSchema = tConversationSchema iconURL: true, greeting: true, spec: true, + maxContextTokens: true, }) .transform((obj) => { const isGemini = obj?.model?.toLowerCase()?.includes('gemini'); @@ -456,6 +471,7 @@ export const googleSchema = tConversationSchema iconURL: obj.iconURL ?? undefined, greeting: obj.greeting ?? undefined, spec: obj.spec ?? undefined, + maxContextTokens: obj.maxContextTokens ?? undefined, }; }) .catch(() => ({ @@ -470,6 +486,7 @@ export const googleSchema = tConversationSchema iconURL: undefined, greeting: undefined, spec: undefined, + maxContextTokens: undefined, })); export const bingAISchema = tConversationSchema @@ -520,6 +537,7 @@ export const anthropicSchema = tConversationSchema iconURL: true, greeting: true, spec: true, + maxContextTokens: true, }) .transform((obj) => ({ ...obj, @@ -534,6 +552,7 @@ export const anthropicSchema = tConversationSchema iconURL: obj.iconURL ?? undefined, greeting: obj.greeting ?? undefined, spec: obj.spec ?? undefined, + maxContextTokens: obj.maxContextTokens ?? undefined, })) .catch(() => ({ model: 'claude-1', @@ -547,6 +566,7 @@ export const anthropicSchema = tConversationSchema iconURL: undefined, greeting: undefined, spec: undefined, + maxContextTokens: undefined, })); export const chatGPTBrowserSchema = tConversationSchema @@ -576,6 +596,7 @@ export const gptPluginsSchema = tConversationSchema iconURL: true, greeting: true, spec: true, + maxContextTokens: true, }) .transform((obj) => ({ ...obj, @@ -596,6 +617,7 @@ export const gptPluginsSchema = tConversationSchema iconURL: obj.iconURL ?? undefined, greeting: obj.greeting ?? undefined, spec: obj.spec ?? undefined, + maxContextTokens: obj.maxContextTokens ?? undefined, })) .catch(() => ({ model: 'gpt-3.5-turbo', @@ -615,6 +637,7 @@ export const gptPluginsSchema = tConversationSchema iconURL: undefined, greeting: undefined, spec: undefined, + maxContextTokens: undefined, })); export function removeNullishValues(obj: T): T { @@ -688,6 +711,8 @@ export const compactOpenAISchema = tConversationSchema iconURL: true, greeting: true, spec: true, + maxContextTokens: true, + max_tokens: true, }) .transform((obj: Partial) => { const newObj: Partial = { ...obj }; @@ -727,6 +752,7 @@ export const compactGoogleSchema = tConversationSchema iconURL: true, greeting: true, spec: true, + maxContextTokens: true, }) .transform((obj) => { const newObj: Partial = { ...obj }; @@ -760,6 +786,7 @@ export const compactAnthropicSchema = tConversationSchema iconURL: true, greeting: true, spec: true, + maxContextTokens: true, }) .transform((obj) => { const newObj: Partial = { ...obj }; @@ -807,6 +834,7 @@ export const compactPluginsSchema = tConversationSchema iconURL: true, greeting: true, spec: true, + maxContextTokens: true, }) .transform((obj) => { const newObj: Partial = { ...obj }; diff --git a/packages/data-provider/src/types.ts b/packages/data-provider/src/types.ts index 901730336d..895b763fe6 100644 --- a/packages/data-provider/src/types.ts +++ b/packages/data-provider/src/types.ts @@ -17,6 +17,7 @@ export type TEndpointOption = { endpointType?: EModelEndpoint; modelDisplayLabel?: string; resendFiles?: boolean; + maxContextTokens?: number; imageDetail?: ImageDetail; model?: string | null; promptPrefix?: string;