mirror of
https://github.com/danny-avila/LibreChat.git
synced 2025-12-19 18:00:15 +01:00
8263ddda3f
* chore: bump anthropic SDK * chore: update anthropic config settings (fileSupport, default models) * feat: anthropic multi modal formatting * refactor: update vision models and use endpoint specific max long side resizing * feat(anthropic): multimodal messages, retry logic, and messages payload * chore: add more safety to trimming content due to whitespace error for assistant messages * feat(anthropic): token accounting and resending multiple images in progress * chore: bump data-provider * feat(anthropic): resendImages feature * chore: optimize Edit/Ask controllers, switch model back to req model * fix: false positive of invalid model * refactor(validateVisionModel): use object as arg, pass in additional/available models * refactor(validateModel): use helper function, `getModelsConfig` * feat: add modelsConfig to endpointOption so it gets passed to all clients, use for properly validating vision models * refactor: initialize default vision model and make sure it's available before assigning it * refactor(useSSE): avoid resetting model if user selected a new model between request and response * feat: show rate in transaction logging * fix: return tokenCountMap regardless of payload shape
82 lines
3.5 KiB
JavaScript
82 lines
3.5 KiB
JavaScript
const sharp = require('sharp');
|
|
const { EModelEndpoint } = require('librechat-data-provider');
|
|
|
|
/**
|
|
* Resizes an image from a given buffer based on the specified resolution.
|
|
*
|
|
* @param {Buffer} inputBuffer - The buffer of the image to be resized.
|
|
* @param {'low' | 'high'} resolution - The resolution to resize the image to.
|
|
* 'low' for a maximum of 512x512 resolution,
|
|
* 'high' for a maximum of 768x2000 resolution.
|
|
* @param {EModelEndpoint} endpoint - Identifier for specific endpoint handling
|
|
* @returns {Promise<{buffer: Buffer, width: number, height: number}>} An object containing the resized image buffer and its dimensions.
|
|
* @throws Will throw an error if the resolution parameter is invalid.
|
|
*/
|
|
async function resizeImageBuffer(inputBuffer, resolution, endpoint) {
|
|
const maxLowRes = 512;
|
|
const maxShortSideHighRes = 768;
|
|
const maxLongSideHighRes = endpoint === EModelEndpoint.anthropic ? 1568 : 2000;
|
|
|
|
let newWidth, newHeight;
|
|
let resizeOptions = { fit: 'inside', withoutEnlargement: true };
|
|
|
|
if (resolution === 'low') {
|
|
resizeOptions.width = maxLowRes;
|
|
resizeOptions.height = maxLowRes;
|
|
} else if (resolution === 'high') {
|
|
const metadata = await sharp(inputBuffer).metadata();
|
|
const isWidthShorter = metadata.width < metadata.height;
|
|
|
|
if (isWidthShorter) {
|
|
// Width is the shorter side
|
|
newWidth = Math.min(metadata.width, maxShortSideHighRes);
|
|
// Calculate new height to maintain aspect ratio
|
|
newHeight = Math.round((metadata.height / metadata.width) * newWidth);
|
|
// Ensure the long side does not exceed the maximum allowed
|
|
if (newHeight > maxLongSideHighRes) {
|
|
newHeight = maxLongSideHighRes;
|
|
newWidth = Math.round((metadata.width / metadata.height) * newHeight);
|
|
}
|
|
} else {
|
|
// Height is the shorter side
|
|
newHeight = Math.min(metadata.height, maxShortSideHighRes);
|
|
// Calculate new width to maintain aspect ratio
|
|
newWidth = Math.round((metadata.width / metadata.height) * newHeight);
|
|
// Ensure the long side does not exceed the maximum allowed
|
|
if (newWidth > maxLongSideHighRes) {
|
|
newWidth = maxLongSideHighRes;
|
|
newHeight = Math.round((metadata.height / metadata.width) * newWidth);
|
|
}
|
|
}
|
|
|
|
resizeOptions.width = newWidth;
|
|
resizeOptions.height = newHeight;
|
|
} else {
|
|
throw new Error('Invalid resolution parameter');
|
|
}
|
|
|
|
const resizedBuffer = await sharp(inputBuffer).rotate().resize(resizeOptions).toBuffer();
|
|
|
|
const resizedMetadata = await sharp(resizedBuffer).metadata();
|
|
return { buffer: resizedBuffer, width: resizedMetadata.width, height: resizedMetadata.height };
|
|
}
|
|
|
|
/**
|
|
* Resizes an image buffer to webp format as well as reduces 150 px width.
|
|
*
|
|
* @param {Buffer} inputBuffer - The buffer of the image to be resized.
|
|
* @returns {Promise<{ buffer: Buffer, width: number, height: number, bytes: number }>} An object containing the resized image buffer, its size and dimensions.
|
|
* @throws Will throw an error if the resolution parameter is invalid.
|
|
*/
|
|
async function resizeAndConvert(inputBuffer) {
|
|
const resizedBuffer = await sharp(inputBuffer).resize({ width: 150 }).toFormat('webp').toBuffer();
|
|
const resizedMetadata = await sharp(resizedBuffer).metadata();
|
|
return {
|
|
buffer: resizedBuffer,
|
|
width: resizedMetadata.width,
|
|
height: resizedMetadata.height,
|
|
bytes: Buffer.byteLength(resizedBuffer),
|
|
};
|
|
}
|
|
|
|
module.exports = { resizeImageBuffer, resizeAndConvert };
|