Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-12-20 02:10:15 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

📸 feat: Gemini vision, Improved Logs and Multi-modal Handling (#1368)

Browse source 
* feat: add GOOGLE_MODELS env var

* feat: add gemini vision support

* refactor(GoogleClient): adjust clientOptions handling depending on model

* fix(logger): fix redact logic and redact errors only

* fix(GoogleClient): do not allow non-multiModal messages when gemini-pro-vision is selected

* refactor(OpenAIClient): use `isVisionModel` client property to avoid calling validateVisionModel multiple times

* refactor: better debug logging by correctly traversing, redacting sensitive info, and logging condensed versions of long values

* refactor(GoogleClient): allow response errors to be thrown/caught above client handling so user receives meaningful error message
debug orderedMessages, parentMessageId, and buildMessages result

* refactor(AskController): use model from client.modelOptions.model when saving intermediate messages, which requires for the progress callback to be initialized after the client is initialized

* feat(useSSE): revert to previous model if the model was auto-switched by backend due to message attachments

* docs: update with google updates, notes about Gemini Pro Vision

* fix: redis should not be initialized without USE_REDIS and increase max listeners to 20
This commit is contained in:
Danny Avila 2023-12-16 20:45:27 -05:00 • committed by GitHub
parent 676f133545
commit 0c326797dd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
21 changed files with 356 additions and 210 deletions
Download patch file Download diff file Expand all files Collapse all files

31
api/server/services/Files/images/encode.js
View file

@ -1,7 +1,13 @@
const fs = require('fs');
const path = require('path');
const { EModelEndpoint } = require('librechat-data-provider');
const { updateFile } = require('~/models');
/**
* Encodes an image file to base64.
* @param {string} imagePath - The path to the image file.
* @returns {Promise<string>} A promise that resolves with the base64 encoded image data.
*/
function encodeImage(imagePath) {
return new Promise((resolve, reject) => {
fs.readFile(imagePath, (err, data) => {
@ -14,6 +20,12 @@ function encodeImage(imagePath) {
});
}
/**
* Updates the file and encodes the image.
* @param {Object} req - The request object.
* @param {Object} file - The file object.
* @returns {Promise<[MongoFile, string]>} - A promise that resolves to an array of results from updateFile and encodeImage.
*/
async function updateAndEncode(req, file) {
const { publicPath, imageOutput } = req.app.locals.config;
const userPath = path.join(imageOutput, req.user.id);
@ -29,7 +41,14 @@ async function updateAndEncode(req, file) {
return await Promise.all(promises);
}
async function encodeAndFormat(req, files) {
/**
* Encodes and formats the given files.
* @param {Express.Request} req - The request object.
* @param {Array<MongoFile>} files - The array of files to encode and format.
* @param {EModelEndpoint} [endpoint] - Optional: The endpoint for the image.
* @returns {Promise<Object>} - A promise that resolves to the result object containing the encoded images and file details.
*/
async function encodeAndFormat(req, files, endpoint) {
const promises = [];
for (let file of files) {
promises.push(updateAndEncode(req, file));
@ -46,13 +65,19 @@ async function encodeAndFormat(req, files) {
};
for (const [file, base64] of encodedImages) {
result.image_urls.push({
const imagePart = {
type: 'image_url',
image_url: {
url: `data:image/webp;base64,${base64}`,
detail,
},
});
};
if (endpoint && endpoint === EModelEndpoint.google) {
imagePart.image_url = imagePart.image_url.url;
}
result.image_urls.push(imagePart);
result.files.push({
file_id: file.file_id,