Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-12-22 03:10:15 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

🤖 feat(Anthropic): Claude 3 & Vision Support (#1984)

Browse source 
* chore: bump anthropic SDK

* chore: update anthropic config settings (fileSupport, default models)

* feat: anthropic multi modal formatting

* refactor: update vision models and use endpoint specific max long side resizing

* feat(anthropic): multimodal messages, retry logic, and messages payload

* chore: add more safety to trimming content due to whitespace error for assistant messages

* feat(anthropic): token accounting and resending multiple images in progress

* chore: bump data-provider

* feat(anthropic): resendImages feature

* chore: optimize Edit/Ask controllers, switch model back to req model

* fix: false positive of invalid model

* refactor(validateVisionModel): use object as arg, pass in additional/available models

* refactor(validateModel): use helper function, `getModelsConfig`

* feat: add modelsConfig to endpointOption so it gets passed to all clients, use for properly validating vision models

* refactor: initialize default vision model and make sure it's available before assigning it

* refactor(useSSE): avoid resetting model if user selected a new model between request and response

* feat: show rate in transaction logging

* fix: return tokenCountMap regardless of payload shape
This commit is contained in:
Danny Avila 2024-03-06 00:04:52 -05:00 • committed by GitHub
parent b023c5683d
commit 8263ddda3f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
28 changed files with 599 additions and 115 deletions
Download patch file Download diff file Expand all files Collapse all files

12
api/server/services/Files/images/encode.js
View file

@ -23,6 +23,8 @@ async function fetchImageToBase64(url) {
}
}
const base64Only = new Set([EModelEndpoint.google, EModelEndpoint.anthropic]);
/**
* Encodes and formats the given files.
* @param {Express.Request} req - The request object.
@ -50,7 +52,7 @@ async function encodeAndFormat(req, files, endpoint) {
encodingMethods[source] = prepareImagePayload;
/* Google doesn't support passing URLs to payload */
if (source !== FileSources.local && endpoint === EModelEndpoint.google) {
if (source !== FileSources.local && base64Only.has(endpoint)) {
const [_file, imageURL] = await prepareImagePayload(req, file);
promises.push([_file, await fetchImageToBase64(imageURL)]);
continue;
@ -81,6 +83,14 @@ async function encodeAndFormat(req, files, endpoint) {
if (endpoint && endpoint === EModelEndpoint.google) {
imagePart.image_url = imagePart.image_url.url;
} else if (endpoint && endpoint === EModelEndpoint.anthropic) {
imagePart.type = 'image';
imagePart.source = {
type: 'base64',
media_type: file.type,
data: imageContent,
};
delete imagePart.image_url;
}
result.image_urls.push(imagePart);