Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-12-17 08:50:15 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

🤖 feat: Gemini 1.5 Support (+Vertex AI) (#2383)

Browse source 
* WIP: gemini-1.5 support

* feat: extended vertex ai support

* fix: handle possibly undefined modelName

* fix: gpt-4-turbo-preview invalid vision model

* feat: specify `fileConfig.imageOutputType` and make PNG default image conversion type

* feat: better truncation for errors including base64 strings

* fix: gemini inlineData formatting

* feat: RAG augmented prompt for gemini-1.5

* feat: gemini-1.5 rates and token window

* chore: adjust tokens, update docs, update vision Models

* chore: add back `ChatGoogleVertexAI` for chat models via vertex ai

* refactor: ask/edit controllers to not use `unfinished` field for google endpoint

* chore: remove comment

* chore(ci): fix AppService test

* chore: remove comment

* refactor(GoogleSearch): use `GOOGLE_SEARCH_API_KEY` instead, issue warning for old variable

* chore: bump data-provider to 0.5.4

* chore: update docs

* fix: condition for gemini-1.5 using generative ai lib

* chore: update docs

* ci: add additional AppService test for `imageOutputType`

* refactor: optimize new config value `imageOutputType`

* chore: bump CONFIG_VERSION

* fix(assistants): avatar upload
This commit is contained in:
Danny Avila 2024-04-16 08:32:40 -04:00 • committed by GitHub
parent fce7246ac1
commit 9d854dac07
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
37 changed files with 1030 additions and 258 deletions
Download patch file Download diff file Expand all files Collapse all files

25
api/server/services/Files/process.js
View file

@ -12,7 +12,7 @@ const {
hostImageIdSuffix,
hostImageNamePrefix,
} = require('librechat-data-provider');
const { convertToWebP, resizeAndConvert } = require('~/server/services/Files/images');
const { convertImage, resizeAndConvert } = require('~/server/services/Files/images');
const { initializeClient } = require('~/server/services/Endpoints/assistants');
const { createFile, updateFileUsage, deleteFiles } = require('~/models/File');
const { LB_QueueAsyncCall } = require('~/server/utils/queue');
@ -207,7 +207,7 @@ const processImageFile = async ({ req, res, file, metadata }) => {
filename: file.originalname,
context: FileContext.message_attachment,
source,
type: 'image/webp',
type: `image/${req.app.locals.imageOutputType}`,
width,
height,
},
@ -223,9 +223,9 @@ const processImageFile = async ({ req, res, file, metadata }) => {
* @param {Object} params - The parameters object.
* @param {Express.Request} params.req - The Express request object.
* @param {FileContext} params.context - The context of the file (e.g., 'avatar', 'image_generation', etc.)
* @param {boolean} [params.resize=true] - Whether to resize and convert the image to WebP. Default is `true`.
* @param {boolean} [params.resize=true] - Whether to resize and convert the image to target format. Default is `true`.
* @param {{ buffer: Buffer, width: number, height: number, bytes: number, filename: string, type: string, file_id: string }} [params.metadata] - Required metadata for the file if resize is false.
* @returns {Promise<{ filepath: string, filename: string, source: string, type: 'image/webp'}>}
* @returns {Promise<{ filepath: string, filename: string, source: string, type: string}>}
*/
const uploadImageBuffer = async ({ req, context, metadata = {}, resize = true }) => {
const source = req.app.locals.fileStrategy;
@ -233,9 +233,14 @@ const uploadImageBuffer = async ({ req, context, metadata = {}, resize = true })
let { buffer, width, height, bytes, filename, file_id, type } = metadata;
if (resize) {
file_id = v4();
type = 'image/webp';
({ buffer, width, height, bytes } = await resizeAndConvert(req.file.buffer));
filename = path.basename(req.file.originalname, path.extname(req.file.originalname)) + '.webp';
type = `image/${req.app.locals.imageOutputType}`;
({ buffer, width, height, bytes } = await resizeAndConvert({
inputBuffer: buffer,
desiredFormat: req.app.locals.imageOutputType,
}));
filename = `${path.basename(req.file.originalname, path.extname(req.file.originalname))}.${
req.app.locals.imageOutputType
}`;
}
const filepath = await saveBuffer({ userId: req.user.id, fileName: filename, buffer });
@ -363,7 +368,7 @@ const processOpenAIFile = async ({
};
/**
* Process OpenAI image files, convert to webp, save and return file metadata.
* Process OpenAI image files, convert to target format, save and return file metadata.
* @param {object} params - The params object.
* @param {Express.Request} params.req - The Express request object.
* @param {Buffer} params.buffer - The image buffer.
@ -375,12 +380,12 @@ const processOpenAIFile = async ({
const processOpenAIImageOutput = async ({ req, buffer, file_id, filename, fileExt }) => {
const currentDate = new Date();
const formattedDate = currentDate.toISOString();
const _file = await convertToWebP(req, buffer, 'high', `${file_id}${fileExt}`);
const _file = await convertImage(req, buffer, 'high', `${file_id}${fileExt}`);
const file = {
..._file,
usage: 1,
user: req.user.id,
type: 'image/webp',
type: `image/${req.app.locals.imageOutputType}`,
createdAt: formattedDate,
updatedAt: formattedDate,
source: req.app.locals.fileStrategy,