From 422d1a2c91fdaa9b461ae0d36bd81746b9ba352f Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 20 Jul 2024 08:53:16 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=A4=96=20feat:=20new=20Anthropic=20Defaul?= =?UTF-8?q?t=20Settings=20/=20Increased=20Output=20Tokens=20for=203.5-Sonn?= =?UTF-8?q?et=20(#3407)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: bump data-provider * feat: Add anthropicSettings to endpointSettings The commit adds the `anthropicSettings` object to the `endpointSettings` in the `schemas.ts` file. This allows for the configuration of settings specific to the `anthropic` model endpoint. * chore: adjust maxoutputtokens localization * feat: Update AnthropicClient to use anthropicSettings for default model options and increased output beta header * ci: new anthropic tests --- api/app/clients/AnthropicClient.js | 42 ++++-- api/app/clients/specs/AnthropicClient.test.js | 96 +++++++++++++- .../Endpoints/Settings/Anthropic.tsx | 64 +++++---- client/src/localization/languages/Eng.ts | 4 +- package-lock.json | 2 +- packages/data-provider/package.json | 2 +- packages/data-provider/src/schemas.ts | 121 ++++++++++++++---- 7 files changed, 262 insertions(+), 69 deletions(-) diff --git a/api/app/clients/AnthropicClient.js b/api/app/clients/AnthropicClient.js index 16b21ea2e3..3bc33af398 100644 --- a/api/app/clients/AnthropicClient.js +++ b/api/app/clients/AnthropicClient.js @@ -4,6 +4,7 @@ const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = requ const { Constants, EModelEndpoint, + anthropicSettings, getResponseSender, validateVisionModel, } = require('librechat-data-provider'); @@ -31,6 +32,8 @@ function delayBeforeRetry(attempts, baseDelay = 1000) { return new Promise((resolve) => setTimeout(resolve, baseDelay * attempts)); } +const { legacy } = anthropicSettings; + class AnthropicClient extends BaseClient { constructor(apiKey, options = {}) { super(apiKey, options); @@ -63,15 +66,20 @@ class AnthropicClient extends BaseClient { const modelOptions = this.options.modelOptions || {}; this.modelOptions = { ...modelOptions, - // set some good defaults (check for undefined in some cases because they may be 0) - model: modelOptions.model || 'claude-1', - temperature: typeof modelOptions.temperature === 'undefined' ? 1 : modelOptions.temperature, // 0 - 1, 1 is default - topP: typeof modelOptions.topP === 'undefined' ? 0.7 : modelOptions.topP, // 0 - 1, default: 0.7 - topK: typeof modelOptions.topK === 'undefined' ? 40 : modelOptions.topK, // 1-40, default: 40 - stop: modelOptions.stop, // no stop method for now + model: modelOptions.model || anthropicSettings.model.default, }; this.isClaude3 = this.modelOptions.model.includes('claude-3'); + this.isLegacyOutput = !this.modelOptions.model.includes('claude-3-5-sonnet'); + + if ( + this.isLegacyOutput && + this.modelOptions.maxOutputTokens && + this.modelOptions.maxOutputTokens > legacy.maxOutputTokens.default + ) { + this.modelOptions.maxOutputTokens = legacy.maxOutputTokens.default; + } + this.useMessages = this.isClaude3 || !!this.options.attachments; this.defaultVisionModel = this.options.visionModel ?? 'claude-3-sonnet-20240229'; @@ -121,10 +129,11 @@ class AnthropicClient extends BaseClient { /** * Get the initialized Anthropic client. + * @param {Partial} requestOptions - The options for the client. * @returns {Anthropic} The Anthropic client instance. */ - getClient() { - /** @type {Anthropic.default.RequestOptions} */ + getClient(requestOptions) { + /** @type {Anthropic.ClientOptions} */ const options = { fetch: this.fetch, apiKey: this.apiKey, @@ -138,6 +147,12 @@ class AnthropicClient extends BaseClient { options.baseURL = this.options.reverseProxyUrl; } + if (requestOptions?.model && requestOptions.model.includes('claude-3-5-sonnet')) { + options.defaultHeaders = { + 'anthropic-beta': 'max-tokens-3-5-sonnet-2024-07-15', + }; + } + return new Anthropic(options); } @@ -558,8 +573,6 @@ class AnthropicClient extends BaseClient { } logger.debug('modelOptions', { modelOptions }); - - const client = this.getClient(); const metadata = { user_id: this.user, }; @@ -587,7 +600,7 @@ class AnthropicClient extends BaseClient { if (this.useMessages) { requestOptions.messages = payload; - requestOptions.max_tokens = maxOutputTokens || 1500; + requestOptions.max_tokens = maxOutputTokens || legacy.maxOutputTokens.default; } else { requestOptions.prompt = payload; requestOptions.max_tokens_to_sample = maxOutputTokens || 1500; @@ -614,6 +627,7 @@ class AnthropicClient extends BaseClient { while (attempts < maxRetries) { let response; try { + const client = this.getClient(requestOptions); response = await this.createResponse(client, requestOptions); signal.addEventListener('abort', () => { @@ -742,7 +756,11 @@ class AnthropicClient extends BaseClient { }; try { - const response = await this.createResponse(this.getClient(), requestOptions, true); + const response = await this.createResponse( + this.getClient(requestOptions), + requestOptions, + true, + ); let promptTokens = response?.usage?.input_tokens; let completionTokens = response?.usage?.output_tokens; if (!promptTokens) { diff --git a/api/app/clients/specs/AnthropicClient.test.js b/api/app/clients/specs/AnthropicClient.test.js index 52324914b9..f559159bf0 100644 --- a/api/app/clients/specs/AnthropicClient.test.js +++ b/api/app/clients/specs/AnthropicClient.test.js @@ -1,4 +1,6 @@ -const AnthropicClient = require('../AnthropicClient'); +const { anthropicSettings } = require('librechat-data-provider'); +const AnthropicClient = require('~/app/clients/AnthropicClient'); + const HUMAN_PROMPT = '\n\nHuman:'; const AI_PROMPT = '\n\nAssistant:'; @@ -22,7 +24,7 @@ describe('AnthropicClient', () => { const options = { modelOptions: { model, - temperature: 0.7, + temperature: anthropicSettings.temperature.default, }, }; client = new AnthropicClient('test-api-key'); @@ -33,7 +35,42 @@ describe('AnthropicClient', () => { it('should set the options correctly', () => { expect(client.apiKey).toBe('test-api-key'); expect(client.modelOptions.model).toBe(model); - expect(client.modelOptions.temperature).toBe(0.7); + expect(client.modelOptions.temperature).toBe(anthropicSettings.temperature.default); + }); + + it('should set legacy maxOutputTokens for non-Claude-3 models', () => { + const client = new AnthropicClient('test-api-key'); + client.setOptions({ + modelOptions: { + model: 'claude-2', + maxOutputTokens: anthropicSettings.maxOutputTokens.default, + }, + }); + expect(client.modelOptions.maxOutputTokens).toBe( + anthropicSettings.legacy.maxOutputTokens.default, + ); + }); + it('should not set maxOutputTokens if not provided', () => { + const client = new AnthropicClient('test-api-key'); + client.setOptions({ + modelOptions: { + model: 'claude-3', + }, + }); + expect(client.modelOptions.maxOutputTokens).toBeUndefined(); + }); + + it('should not set legacy maxOutputTokens for Claude-3 models', () => { + const client = new AnthropicClient('test-api-key'); + client.setOptions({ + modelOptions: { + model: 'claude-3-opus-20240229', + maxOutputTokens: anthropicSettings.legacy.maxOutputTokens.default, + }, + }); + expect(client.modelOptions.maxOutputTokens).toBe( + anthropicSettings.legacy.maxOutputTokens.default, + ); }); }); @@ -136,4 +173,57 @@ describe('AnthropicClient', () => { expect(prompt).toContain('You are Claude-2'); }); }); + + describe('getClient', () => { + it('should set legacy maxOutputTokens for non-Claude-3 models', () => { + const client = new AnthropicClient('test-api-key'); + client.setOptions({ + modelOptions: { + model: 'claude-2', + maxOutputTokens: anthropicSettings.legacy.maxOutputTokens.default, + }, + }); + expect(client.modelOptions.maxOutputTokens).toBe( + anthropicSettings.legacy.maxOutputTokens.default, + ); + }); + + it('should not set legacy maxOutputTokens for Claude-3 models', () => { + const client = new AnthropicClient('test-api-key'); + client.setOptions({ + modelOptions: { + model: 'claude-3-opus-20240229', + maxOutputTokens: anthropicSettings.legacy.maxOutputTokens.default, + }, + }); + expect(client.modelOptions.maxOutputTokens).toBe( + anthropicSettings.legacy.maxOutputTokens.default, + ); + }); + + it('should add beta header for claude-3-5-sonnet model', () => { + const client = new AnthropicClient('test-api-key'); + const modelOptions = { + model: 'claude-3-5-sonnet-20240307', + }; + client.setOptions({ modelOptions }); + const anthropicClient = client.getClient(modelOptions); + expect(anthropicClient._options.defaultHeaders).toBeDefined(); + expect(anthropicClient._options.defaultHeaders).toHaveProperty('anthropic-beta'); + expect(anthropicClient._options.defaultHeaders['anthropic-beta']).toBe( + 'max-tokens-3-5-sonnet-2024-07-15', + ); + }); + + it('should not add beta header for other models', () => { + const client = new AnthropicClient('test-api-key'); + client.setOptions({ + modelOptions: { + model: 'claude-2', + }, + }); + const anthropicClient = client.getClient(); + expect(anthropicClient.defaultHeaders).not.toHaveProperty('anthropic-beta'); + }); + }); }); diff --git a/client/src/components/Endpoints/Settings/Anthropic.tsx b/client/src/components/Endpoints/Settings/Anthropic.tsx index 4980d2dedd..2d45ca93da 100644 --- a/client/src/components/Endpoints/Settings/Anthropic.tsx +++ b/client/src/components/Endpoints/Settings/Anthropic.tsx @@ -1,5 +1,5 @@ -import React from 'react'; import TextareaAutosize from 'react-textarea-autosize'; +import { anthropicSettings } from 'librechat-data-provider'; import type { TModelSelectProps, OnInputNumberChange } from '~/common'; import { Input, @@ -41,15 +41,31 @@ export default function Settings({ conversation, setOption, models, readonly }: return null; } - const setModel = setOption('model'); const setModelLabel = setOption('modelLabel'); const setPromptPrefix = setOption('promptPrefix'); const setTemperature = setOption('temperature'); const setTopP = setOption('topP'); const setTopK = setOption('topK'); - const setMaxOutputTokens = setOption('maxOutputTokens'); const setResendFiles = setOption('resendFiles'); + const setModel = (newModel: string) => { + const modelSetter = setOption('model'); + const maxOutputSetter = setOption('maxOutputTokens'); + if (maxOutputTokens) { + maxOutputSetter(anthropicSettings.maxOutputTokens.set(maxOutputTokens, newModel)); + } + modelSetter(newModel); + }; + + const setMaxOutputTokens = (value: number) => { + const setter = setOption('maxOutputTokens'); + if (model) { + setter(anthropicSettings.maxOutputTokens.set(value, model)); + } else { + setter(value); + } + }; + return (
@@ -139,14 +155,16 @@ export default function Settings({ conversation, setOption, models, readonly }:
setTemperature(Number(value))} - max={1} + max={anthropicSettings.temperature.max} min={0} step={0.01} controls={false} @@ -161,10 +179,10 @@ export default function Settings({ conversation, setOption, models, readonly }:
setTemperature(value[0])} - doubleClickHandler={() => setTemperature(1)} - max={1} + doubleClickHandler={() => setTemperature(anthropicSettings.temperature.default)} + max={anthropicSettings.temperature.max} min={0} step={0.01} className="flex h-4 w-full" @@ -178,7 +196,7 @@ export default function Settings({ conversation, setOption, models, readonly }: setTopP(Number(value))} - max={1} + max={anthropicSettings.topP.max} min={0} step={0.01} controls={false} @@ -203,8 +221,8 @@ export default function Settings({ conversation, setOption, models, readonly }: disabled={readonly} value={[topP ?? 0.7]} onValueChange={(value) => setTopP(value[0])} - doubleClickHandler={() => setTopP(1)} - max={1} + doubleClickHandler={() => setTopP(anthropicSettings.topP.default)} + max={anthropicSettings.topP.max} min={0} step={0.01} className="flex h-4 w-full" @@ -219,7 +237,7 @@ export default function Settings({ conversation, setOption, models, readonly }: setTopK(Number(value))} - max={40} + max={anthropicSettings.topK.max} min={1} step={0.01} controls={false} @@ -244,8 +262,8 @@ export default function Settings({ conversation, setOption, models, readonly }: disabled={readonly} value={[topK ?? 5]} onValueChange={(value) => setTopK(value[0])} - doubleClickHandler={() => setTopK(0)} - max={40} + doubleClickHandler={() => setTopK(anthropicSettings.topK.default)} + max={anthropicSettings.topK.max} min={1} step={0.01} className="flex h-4 w-full" @@ -258,16 +276,14 @@ export default function Settings({ conversation, setOption, models, readonly }:
setMaxOutputTokens(Number(value))} - max={4000} + max={anthropicSettings.maxOutputTokens.max} min={1} step={1} controls={false} @@ -282,10 +298,12 @@ export default function Settings({ conversation, setOption, models, readonly }:
setMaxOutputTokens(value[0])} - doubleClickHandler={() => setMaxOutputTokens(0)} - max={4000} + doubleClickHandler={() => + setMaxOutputTokens(anthropicSettings.maxOutputTokens.default) + } + max={anthropicSettings.maxOutputTokens.max} min={1} step={1} className="flex h-4 w-full" diff --git a/client/src/localization/languages/Eng.ts b/client/src/localization/languages/Eng.ts index e7d468782e..024dbfacd5 100644 --- a/client/src/localization/languages/Eng.ts +++ b/client/src/localization/languages/Eng.ts @@ -391,7 +391,7 @@ export default { com_endpoint_google_topk: 'Top-k changes how the model selects tokens for output. A top-k of 1 means the selected token is the most probable among all tokens in the model\'s vocabulary (also called greedy decoding), while a top-k of 3 means that the next token is selected from among the 3 most probable tokens (using temperature).', com_endpoint_google_maxoutputtokens: - ' Maximum number of tokens that can be generated in the response. Specify a lower value for shorter responses and a higher value for longer responses.', + 'Maximum number of tokens that can be generated in the response. Specify a lower value for shorter responses and a higher value for longer responses. Note: models may stop before reaching this maximum.', com_endpoint_google_custom_name_placeholder: 'Set a custom name for Google', com_endpoint_prompt_prefix_placeholder: 'Set custom instructions or context. Ignored if empty.', com_endpoint_instructions_assistants_placeholder: @@ -439,7 +439,7 @@ export default { com_endpoint_anthropic_topk: 'Top-k changes how the model selects tokens for output. A top-k of 1 means the selected token is the most probable among all tokens in the model\'s vocabulary (also called greedy decoding), while a top-k of 3 means that the next token is selected from among the 3 most probable tokens (using temperature).', com_endpoint_anthropic_maxoutputtokens: - 'Maximum number of tokens that can be generated in the response. Specify a lower value for shorter responses and a higher value for longer responses.', + 'Maximum number of tokens that can be generated in the response. Specify a lower value for shorter responses and a higher value for longer responses. Note: models may stop before reaching this maximum.', com_endpoint_anthropic_custom_name_placeholder: 'Set a custom name for Anthropic', com_endpoint_frequency_penalty: 'Frequency Penalty', com_endpoint_presence_penalty: 'Presence Penalty', diff --git a/package-lock.json b/package-lock.json index 9f2fae80f6..0e8c3a3601 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29437,7 +29437,7 @@ }, "packages/data-provider": { "name": "librechat-data-provider", - "version": "0.7.2", + "version": "0.7.4", "license": "ISC", "dependencies": { "@types/js-yaml": "^4.0.9", diff --git a/packages/data-provider/package.json b/packages/data-provider/package.json index 8c9a7d4429..7d438bd556 100644 --- a/packages/data-provider/package.json +++ b/packages/data-provider/package.json @@ -1,6 +1,6 @@ { "name": "librechat-data-provider", - "version": "0.7.3", + "version": "0.7.4", "description": "data services for librechat apps", "main": "dist/index.js", "module": "dist/index.es.js", diff --git a/packages/data-provider/src/schemas.ts b/packages/data-provider/src/schemas.ts index 049e84a87f..975283ee77 100644 --- a/packages/data-provider/src/schemas.ts +++ b/packages/data-provider/src/schemas.ts @@ -156,9 +156,70 @@ export const googleSettings = { }, }; +const ANTHROPIC_MAX_OUTPUT = 8192; +const LEGACY_ANTHROPIC_MAX_OUTPUT = 4096; +export const anthropicSettings = { + model: { + default: 'claude-3-5-sonnet-20240620', + }, + temperature: { + min: 0, + max: 1, + step: 0.01, + default: 1, + }, + maxOutputTokens: { + min: 1, + max: ANTHROPIC_MAX_OUTPUT, + step: 1, + default: ANTHROPIC_MAX_OUTPUT, + reset: (modelName: string) => { + if (modelName.includes('claude-3-5-sonnet')) { + return ANTHROPIC_MAX_OUTPUT; + } + + return 4096; + }, + set: (value: number, modelName: string) => { + if (!modelName.includes('claude-3-5-sonnet') && value > LEGACY_ANTHROPIC_MAX_OUTPUT) { + return LEGACY_ANTHROPIC_MAX_OUTPUT; + } + + return value; + }, + }, + topP: { + min: 0, + max: 1, + step: 0.01, + default: 0.7, + }, + topK: { + min: 1, + max: 40, + step: 1, + default: 5, + }, + resendFiles: { + default: true, + }, + maxContextTokens: { + default: undefined, + }, + legacy: { + maxOutputTokens: { + min: 1, + max: LEGACY_ANTHROPIC_MAX_OUTPUT, + step: 1, + default: LEGACY_ANTHROPIC_MAX_OUTPUT, + }, + }, +}; + export const endpointSettings = { [EModelEndpoint.openAI]: openAISettings, [EModelEndpoint.google]: googleSettings, + [EModelEndpoint.anthropic]: anthropicSettings, }; const google = endpointSettings[EModelEndpoint.google]; @@ -576,34 +637,40 @@ export const anthropicSchema = tConversationSchema spec: true, maxContextTokens: true, }) - .transform((obj) => ({ - ...obj, - model: obj.model ?? 'claude-1', - modelLabel: obj.modelLabel ?? null, - promptPrefix: obj.promptPrefix ?? null, - temperature: obj.temperature ?? 1, - maxOutputTokens: obj.maxOutputTokens ?? 4000, - topP: obj.topP ?? 0.7, - topK: obj.topK ?? 5, - resendFiles: typeof obj.resendFiles === 'boolean' ? obj.resendFiles : true, - iconURL: obj.iconURL ?? undefined, - greeting: obj.greeting ?? undefined, - spec: obj.spec ?? undefined, - maxContextTokens: obj.maxContextTokens ?? undefined, - })) + .transform((obj) => { + const model = obj.model ?? anthropicSettings.model.default; + return { + ...obj, + model, + modelLabel: obj.modelLabel ?? null, + promptPrefix: obj.promptPrefix ?? null, + temperature: obj.temperature ?? anthropicSettings.temperature.default, + maxOutputTokens: obj.maxOutputTokens ?? anthropicSettings.maxOutputTokens.reset(model), + topP: obj.topP ?? anthropicSettings.topP.default, + topK: obj.topK ?? anthropicSettings.topK.default, + resendFiles: + typeof obj.resendFiles === 'boolean' + ? obj.resendFiles + : anthropicSettings.resendFiles.default, + iconURL: obj.iconURL ?? undefined, + greeting: obj.greeting ?? undefined, + spec: obj.spec ?? undefined, + maxContextTokens: obj.maxContextTokens ?? anthropicSettings.maxContextTokens.default, + }; + }) .catch(() => ({ - model: 'claude-1', + model: anthropicSettings.model.default, modelLabel: null, promptPrefix: null, - temperature: 1, - maxOutputTokens: 4000, - topP: 0.7, - topK: 5, - resendFiles: true, + temperature: anthropicSettings.temperature.default, + maxOutputTokens: anthropicSettings.maxOutputTokens.default, + topP: anthropicSettings.topP.default, + topK: anthropicSettings.topK.default, + resendFiles: anthropicSettings.resendFiles.default, iconURL: undefined, greeting: undefined, spec: undefined, - maxContextTokens: undefined, + maxContextTokens: anthropicSettings.maxContextTokens.default, })); export const chatGPTBrowserSchema = tConversationSchema @@ -835,19 +902,19 @@ export const compactAnthropicSchema = tConversationSchema }) .transform((obj) => { const newObj: Partial = { ...obj }; - if (newObj.temperature === 1) { + if (newObj.temperature === anthropicSettings.temperature.default) { delete newObj.temperature; } - if (newObj.maxOutputTokens === 4000) { + if (newObj.maxOutputTokens === anthropicSettings.legacy.maxOutputTokens.default) { delete newObj.maxOutputTokens; } - if (newObj.topP === 0.7) { + if (newObj.topP === anthropicSettings.topP.default) { delete newObj.topP; } - if (newObj.topK === 5) { + if (newObj.topK === anthropicSettings.topK.default) { delete newObj.topK; } - if (newObj.resendFiles === true) { + if (newObj.resendFiles === anthropicSettings.resendFiles.default) { delete newObj.resendFiles; }