Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-12-21 19:00:13 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

🤖 feat: new Anthropic Default Settings / Increased Output Tokens for 3.5-Sonnet (#3407)

Browse source 
* chore: bump data-provider

* feat: Add anthropicSettings to endpointSettings

The commit adds the `anthropicSettings` object to the `endpointSettings` in the `schemas.ts` file. This allows for the configuration of settings specific to the `anthropic` model endpoint.

* chore: adjust maxoutputtokens localization

* feat: Update AnthropicClient to use anthropicSettings for default model options and increased output beta header

* ci: new anthropic tests
This commit is contained in:
Danny Avila 2024-07-20 08:53:16 -04:00 • committed by GitHub
parent 2ad097647c
commit 422d1a2c91
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 262 additions and 69 deletions
Download patch file Download diff file Expand all files Collapse all files

42
api/app/clients/AnthropicClient.js
View file

@ -4,6 +4,7 @@ const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = requ
const {
Constants,
EModelEndpoint,
anthropicSettings,
getResponseSender,
validateVisionModel,
} = require('librechat-data-provider');
@ -31,6 +32,8 @@ function delayBeforeRetry(attempts, baseDelay = 1000) {
return new Promise((resolve) => setTimeout(resolve, baseDelay * attempts));
}
const { legacy } = anthropicSettings;
class AnthropicClient extends BaseClient {
constructor(apiKey, options = {}) {
super(apiKey, options);
@ -63,15 +66,20 @@ class AnthropicClient extends BaseClient {
const modelOptions = this.options.modelOptions || {};
this.modelOptions = {
...modelOptions,
// set some good defaults (check for undefined in some cases because they may be 0)
model: modelOptions.model || 'claude-1',
temperature: typeof modelOptions.temperature === 'undefined' ? 1 : modelOptions.temperature, // 0 - 1, 1 is default
topP: typeof modelOptions.topP === 'undefined' ? 0.7 : modelOptions.topP, // 0 - 1, default: 0.7
topK: typeof modelOptions.topK === 'undefined' ? 40 : modelOptions.topK, // 1-40, default: 40
stop: modelOptions.stop, // no stop method for now
model: modelOptions.model || anthropicSettings.model.default,
};
this.isClaude3 = this.modelOptions.model.includes('claude-3');
this.isLegacyOutput = !this.modelOptions.model.includes('claude-3-5-sonnet');
if (
this.isLegacyOutput &&
this.modelOptions.maxOutputTokens &&
this.modelOptions.maxOutputTokens > legacy.maxOutputTokens.default
) {
this.modelOptions.maxOutputTokens = legacy.maxOutputTokens.default;
}
this.useMessages = this.isClaude3 || !!this.options.attachments;
this.defaultVisionModel = this.options.visionModel ?? 'claude-3-sonnet-20240229';
@ -121,10 +129,11 @@ class AnthropicClient extends BaseClient {
/**
* Get the initialized Anthropic client.
* @param {Partial<Anthropic.ClientOptions>} requestOptions - The options for the client.
* @returns {Anthropic} The Anthropic client instance.
*/
getClient() {
/** @type {Anthropic.default.RequestOptions} */
getClient(requestOptions) {
/** @type {Anthropic.ClientOptions} */
const options = {
fetch: this.fetch,
apiKey: this.apiKey,
@ -138,6 +147,12 @@ class AnthropicClient extends BaseClient {
options.baseURL = this.options.reverseProxyUrl;
}
if (requestOptions?.model && requestOptions.model.includes('claude-3-5-sonnet')) {
options.defaultHeaders = {
'anthropic-beta': 'max-tokens-3-5-sonnet-2024-07-15',
};
}
return new Anthropic(options);
}
@ -558,8 +573,6 @@ class AnthropicClient extends BaseClient {
}
logger.debug('modelOptions', { modelOptions });
const client = this.getClient();
const metadata = {
user_id: this.user,
};
@ -587,7 +600,7 @@ class AnthropicClient extends BaseClient {
if (this.useMessages) {
requestOptions.messages = payload;
requestOptions.max_tokens = maxOutputTokens || 1500;
requestOptions.max_tokens = maxOutputTokens || legacy.maxOutputTokens.default;
} else {
requestOptions.prompt = payload;
requestOptions.max_tokens_to_sample = maxOutputTokens || 1500;
@ -614,6 +627,7 @@ class AnthropicClient extends BaseClient {
while (attempts < maxRetries) {
let response;
try {
const client = this.getClient(requestOptions);
response = await this.createResponse(client, requestOptions);
signal.addEventListener('abort', () => {
@ -742,7 +756,11 @@ class AnthropicClient extends BaseClient {
};
try {
const response = await this.createResponse(this.getClient(), requestOptions, true);
const response = await this.createResponse(
this.getClient(requestOptions),
requestOptions,
true,
);
let promptTokens = response?.usage?.input_tokens;
let completionTokens = response?.usage?.output_tokens;
if (!promptTokens) {