Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2026-01-08 19:48:51 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

🤖 feat(Anthropic): Claude 3 & Vision Support (#1984)

Browse source 
* chore: bump anthropic SDK

* chore: update anthropic config settings (fileSupport, default models)

* feat: anthropic multi modal formatting

* refactor: update vision models and use endpoint specific max long side resizing

* feat(anthropic): multimodal messages, retry logic, and messages payload

* chore: add more safety to trimming content due to whitespace error for assistant messages

* feat(anthropic): token accounting and resending multiple images in progress

* chore: bump data-provider

* feat(anthropic): resendImages feature

* chore: optimize Edit/Ask controllers, switch model back to req model

* fix: false positive of invalid model

* refactor(validateVisionModel): use object as arg, pass in additional/available models

* refactor(validateModel): use helper function, `getModelsConfig`

* feat: add modelsConfig to endpointOption so it gets passed to all clients, use for properly validating vision models

* refactor: initialize default vision model and make sure it's available before assigning it

* refactor(useSSE): avoid resetting model if user selected a new model between request and response

* feat: show rate in transaction logging

* fix: return tokenCountMap regardless of payload shape
This commit is contained in:
Danny Avila 2024-03-06 00:04:52 -05:00 • committed by GitHub
parent b023c5683d
commit 8263ddda3f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
28 changed files with 599 additions and 115 deletions
Download patch file Download diff file Expand all files Collapse all files

22
packages/data-provider/src/config.ts
View file

@ -241,6 +241,8 @@ export const defaultModels = {
'code-bison-32k',
],
[EModelEndpoint.anthropic]: [
'claude-3-opus-20240229',
'claude-3-sonnet-20240229',
'claude-2.1',
'claude-2',
'claude-1.2',
@ -301,21 +303,31 @@ export const modularEndpoints = new Set<EModelEndpoint | string>([
export const supportsBalanceCheck = {
[EModelEndpoint.openAI]: true,
[EModelEndpoint.anthropic]: true,
[EModelEndpoint.azureOpenAI]: true,
[EModelEndpoint.gptPlugins]: true,
[EModelEndpoint.custom]: true,
};
export const visionModels = ['gpt-4-vision', 'llava-13b', 'gemini-pro-vision'];
export const visionModels = ['gpt-4-vision', 'llava-13b', 'gemini-pro-vision', 'claude-3'];
export function validateVisionModel(
model: string | undefined,
additionalModels: string[] | undefined = [],
) {
export function validateVisionModel({
model,
additionalModels = [],
availableModels,
}: {
model: string;
additionalModels?: string[];
availableModels?: string[];
}) {
if (!model) {
return false;
}
if (availableModels && !availableModels.includes(model)) {
return false;
}
return visionModels.concat(additionalModels).some((visionModel) => model.includes(visionModel));
}

1
packages/data-provider/src/file-config.ts
View file

@ -8,6 +8,7 @@ export const supportsFiles = {
[EModelEndpoint.google]: true,
[EModelEndpoint.assistants]: true,
[EModelEndpoint.azureOpenAI]: true,
[EModelEndpoint.anthropic]: true,
[EModelEndpoint.custom]: true,
};

7
packages/data-provider/src/schemas.ts
View file

@ -391,6 +391,7 @@ export const anthropicSchema = tConversationSchema
maxOutputTokens: true,
topP: true,
topK: true,
resendImages: true,
})
.transform((obj) => ({
...obj,
@ -401,6 +402,7 @@ export const anthropicSchema = tConversationSchema
maxOutputTokens: obj.maxOutputTokens ?? 4000,
topP: obj.topP ?? 0.7,
topK: obj.topK ?? 5,
resendImages: obj.resendImages ?? false,
}))
.catch(() => ({
model: 'claude-1',
@ -410,6 +412,7 @@ export const anthropicSchema = tConversationSchema
maxOutputTokens: 4000,
topP: 0.7,
topK: 5,
resendImages: false,
}));
export const chatGPTBrowserSchema = tConversationSchema
@ -568,6 +571,7 @@ export const compactAnthropicSchema = tConversationSchema
maxOutputTokens: true,
topP: true,
topK: true,
resendImages: true,
})
.transform((obj) => {
const newObj: Partial<TConversation> = { ...obj };
@ -583,6 +587,9 @@ export const compactAnthropicSchema = tConversationSchema
if (newObj.topK === 5) {
delete newObj.topK;
}
if (newObj.resendImages !== true) {
delete newObj.resendImages;
}
return removeNullishValues(newObj);
})