🤖 feat: Gemini 1.5 Support (+Vertex AI) (#2383)

* WIP: gemini-1.5 support

* feat: extended vertex ai support

* fix: handle possibly undefined modelName

* fix: gpt-4-turbo-preview invalid vision model

* feat: specify `fileConfig.imageOutputType` and make PNG default image conversion type

* feat: better truncation for errors including base64 strings

* fix: gemini inlineData formatting

* feat: RAG augmented prompt for gemini-1.5

* feat: gemini-1.5 rates and token window

* chore: adjust tokens, update docs, update vision Models

* chore: add back `ChatGoogleVertexAI` for chat models via vertex ai

* refactor: ask/edit controllers to not use `unfinished` field for google endpoint

* chore: remove comment

* chore(ci): fix AppService test

* chore: remove comment

* refactor(GoogleSearch): use `GOOGLE_SEARCH_API_KEY` instead, issue warning for old variable

* chore: bump data-provider to 0.5.4

* chore: update docs

* fix: condition for gemini-1.5 using generative ai lib

* chore: update docs

* ci: add additional AppService test for `imageOutputType`

* refactor: optimize new config value `imageOutputType`

* chore: bump CONFIG_VERSION

* fix(assistants): avatar upload

api/server/services/Files/images/avatar.js

@@ -6,10 +6,11 @@ const { logger } = require('~/config');
 
 /**
  * Uploads an avatar image for a user. This function can handle various types of input (URL, Buffer, or File object),
- * processes the image to a square format, converts it to WebP format, and returns the resized buffer.
+ * processes the image to a square format, converts it to target format, and returns the resized buffer.
  *
  * @param {Object} params - The parameters object.
  * @param {string} params.userId - The unique identifier of the user for whom the avatar is being uploaded.
+ * @param {string} options.desiredFormat - The desired output format of the image.
  * @param {(string|Buffer|File)} params.input - The input representing the avatar image. Can be a URL (string),
  *                                               a Buffer, or a File object.
  *
@@ -19,7 +20,7 @@ const { logger } = require('~/config');
  * @throws {Error} Throws an error if the user ID is undefined, the input type is invalid, the image fetching fails,
  *                 or any other error occurs during the processing.
  */
-async function resizeAvatar({ userId, input }) {
+async function resizeAvatar({ userId, input, desiredFormat }) {
   try {
     if (userId === undefined) {
       throw new Error('User ID is undefined');
@@ -53,7 +54,10 @@ async function resizeAvatar({ userId, input }) {
       })
       .toBuffer();
 
-    const { buffer } = await resizeAndConvert(squaredBuffer);
+    const { buffer } = await resizeAndConvert({
+      inputBuffer: squaredBuffer,
+      desiredFormat,
+    });
     return buffer;
   } catch (error) {
     logger.error('Error uploading the avatar:', error);

api/server/services/Files/images/convert.js

@@ -6,7 +6,7 @@ const { getStrategyFunctions } = require('../strategies');
 const { logger } = require('~/config');
 
 /**
- * Converts an image file or buffer to WebP format with specified resolution.
+ * Converts an image file or buffer to target output type with specified resolution.
  *
  * @param {Express.Request} req - The request object, containing user and app configuration data.
  * @param {Buffer | Express.Multer.File} file - The file object, containing either a path or a buffer.
@@ -15,7 +15,7 @@ const { logger } = require('~/config');
  * @returns {Promise<{filepath: string, bytes: number, width: number, height: number}>} An object containing the path, size, and dimensions of the converted image.
  * @throws Throws an error if there is an issue during the conversion process.
  */
-async function convertToWebP(req, file, resolution = 'high', basename = '') {
+async function convertImage(req, file, resolution = 'high', basename = '') {
   try {
     let inputBuffer;
     let outputBuffer;
@@ -38,13 +38,13 @@ async function convertToWebP(req, file, resolution = 'high', basename = '') {
       height,
     } = await resizeImageBuffer(inputBuffer, resolution);
 
-    // Check if the file is already in WebP format
-    // If it isn't, convert it:
-    if (extension === '.webp') {
+    // Check if the file is already in target format; if it isn't, convert it:
+    const targetExtension = `.${req.app.locals.imageOutputType}`;
+    if (extension === targetExtension) {
       outputBuffer = resizedBuffer;
     } else {
-      outputBuffer = await sharp(resizedBuffer).toFormat('webp').toBuffer();
-      extension = '.webp';
+      outputBuffer = await sharp(resizedBuffer).toFormat(req.app.locals.imageOutputType).toBuffer();
+      extension = targetExtension;
     }
 
     // Generate a new filename for the output file
@@ -67,4 +67,4 @@ async function convertToWebP(req, file, resolution = 'high', basename = '') {
   }
 }
 
-module.exports = { convertToWebP };
+module.exports = { convertImage };

api/server/services/Files/images/encode.js

@@ -1,5 +1,5 @@
 const axios = require('axios');
-const { EModelEndpoint, FileSources } = require('librechat-data-provider');
+const { EModelEndpoint, FileSources, VisionModes } = require('librechat-data-provider');
 const { getStrategyFunctions } = require('../strategies');
 const { logger } = require('~/config');
 
@@ -30,11 +30,20 @@ const base64Only = new Set([EModelEndpoint.google, EModelEndpoint.anthropic]);
  * @param {Express.Request} req - The request object.
  * @param {Array<MongoFile>} files - The array of files to encode and format.
  * @param {EModelEndpoint} [endpoint] - Optional: The endpoint for the image.
+ * @param {string} [mode] - Optional: The endpoint mode for the image.
  * @returns {Promise<Object>} - A promise that resolves to the result object containing the encoded images and file details.
  */
-async function encodeAndFormat(req, files, endpoint) {
+async function encodeAndFormat(req, files, endpoint, mode) {
   const promises = [];
   const encodingMethods = {};
+  const result = {
+    files: [],
+    image_urls: [],
+  };
+
+  if (!files || !files.length) {
+    return result;
+  }
 
   for (let file of files) {
     const source = file.source ?? FileSources.local;
@@ -69,11 +78,6 @@ async function encodeAndFormat(req, files, endpoint) {
   /** @type {Array<[MongoFile, string]>} */
   const formattedImages = await Promise.all(promises);
 
-  const result = {
-    files: [],
-    image_urls: [],
-  };
-
   for (const [file, imageContent] of formattedImages) {
     const fileMetadata = {
       type: file.type,
@@ -98,12 +102,18 @@ async function encodeAndFormat(req, files, endpoint) {
       image_url: {
         url: imageContent.startsWith('http')
           ? imageContent
-          : `data:image/webp;base64,${imageContent}`,
+          : `data:${file.type};base64,${imageContent}`,
         detail,
       },
     };
 
-    if (endpoint && endpoint === EModelEndpoint.google) {
+    if (endpoint && endpoint === EModelEndpoint.google && mode === VisionModes.generative) {
+      delete imagePart.image_url;
+      imagePart.inlineData = {
+        mimeType: file.type,
+        data: imageContent,
+      };
+    } else if (endpoint && endpoint === EModelEndpoint.google) {
       imagePart.image_url = imagePart.image_url.url;
     } else if (endpoint && endpoint === EModelEndpoint.anthropic) {
       imagePart.type = 'image';

api/server/services/Files/images/resize.js

@@ -62,14 +62,20 @@ async function resizeImageBuffer(inputBuffer, resolution, endpoint) {
 }
 
 /**
- * Resizes an image buffer to webp format as well as reduces by specified or default 150 px width.
+ * Resizes an image buffer to a specified format and width.
  *
- * @param {Buffer} inputBuffer - The buffer of the image to be resized.
- * @returns {Promise<{ buffer: Buffer, width: number, height: number, bytes: number }>} An object containing the resized image buffer, its size and dimensions.
- * @throws Will throw an error if the resolution parameter is invalid.
+ * @param {Object} options - The options for resizing and converting the image.
+ * @param {Buffer} options.inputBuffer - The buffer of the image to be resized.
+ * @param {string} options.desiredFormat - The desired output format of the image.
+ * @param {number} [options.width=150] - The desired width of the image. Defaults to 150 pixels.
+ * @returns {Promise<{ buffer: Buffer, width: number, height: number, bytes: number }>} An object containing the resized image buffer, its size, and dimensions.
+ * @throws Will throw an error if the resolution or format parameters are invalid.
  */
-async function resizeAndConvert(inputBuffer, width = 150) {
-  const resizedBuffer = await sharp(inputBuffer).resize({ width }).toFormat('webp').toBuffer();
+async function resizeAndConvert({ inputBuffer, desiredFormat, width = 150 }) {
+  const resizedBuffer = await sharp(inputBuffer)
+    .resize({ width })
+    .toFormat(desiredFormat)
+    .toBuffer();
   const resizedMetadata = await sharp(resizedBuffer).metadata();
   return {
     buffer: resizedBuffer,