feat: Google Gemini ❇️ (#1355)

* refactor: add gemini-pro to google Models list; use defaultModels for central model listing

* refactor(SetKeyDialog): create useMultipleKeys hook to use for Azure, export `isJson` from utils, use EModelEndpoint

* refactor(useUserKey): change variable names to make keyName setting more clear

* refactor(FileUpload): allow passing container className string

* feat(GoogleClient): Gemini support

* refactor(GoogleClient): alternate stream speed for Gemini models

* feat(Gemini): styling/settings configuration for Gemini

* refactor(GoogleClient): substract max response tokens from max context tokens if context is above 32k (I/O max is combined between the two)

* refactor(tokens): correct google max token counts and subtract max response tokens when input/output count are combined towards max context count

* feat(google/initializeClient): handle both local and user_provided credentials and write tests

* fix(GoogleClient): catch if credentials are undefined, handle if serviceKey is string or object correctly, handle no examples passed, throw error if not a Generative Language model and no service account JSON key is provided, throw error if it is a Generative m
odel, but not google API key was provided

* refactor(loadAsyncEndpoints/google): activate Google endpoint if either the service key JSON file is provided in /api/data, or a GOOGLE_KEY is defined.

* docs: updated Google configuration

* fix(ci): Mock import of Service Account Key JSON file (auth.json)

* Update apis_and_tokens.md

* feat: increase max output tokens slider for gemini pro

* refactor(GoogleSettings): handle max and default maxOutputTokens on model change

* chore: add sensitive redact regex

* docs: add warning about data privacy

* Update apis_and_tokens.md

api/app/clients/GoogleClient.js

@@ -1,10 +1,16 @@
 const { google } = require('googleapis');
 const { Agent, ProxyAgent } = require('undici');
 const { GoogleVertexAI } = require('langchain/llms/googlevertexai');
+const { ChatGoogleGenerativeAI } = require('@langchain/google-genai');
 const { ChatGoogleVertexAI } = require('langchain/chat_models/googlevertexai');
 const { AIMessage, HumanMessage, SystemMessage } = require('langchain/schema');
 const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken');
-const { getResponseSender, EModelEndpoint, endpointSettings } = require('librechat-data-provider');
+const {
+  getResponseSender,
+  EModelEndpoint,
+  endpointSettings,
+  AuthKeys,
+} = require('librechat-data-provider');
 const { getModelMaxTokens } = require('~/utils');
 const { formatMessage } = require('./prompts');
 const BaseClient = require('./BaseClient');
@@ -21,11 +27,24 @@ const settings = endpointSettings[EModelEndpoint.google];
 class GoogleClient extends BaseClient {
   constructor(credentials, options = {}) {
     super('apiKey', options);
-    this.credentials = credentials;
+    let creds = {};
-    this.client_email = credentials.client_email;
+
-    this.project_id = credentials.project_id;
+    if (typeof credentials === 'string') {
-    this.private_key = credentials.private_key;
+      creds = JSON.parse(credentials);
+    } else if (credentials) {
+      creds = credentials;
+    }
+
+    const serviceKey = creds[AuthKeys.GOOGLE_SERVICE_KEY] ?? {};
+    this.serviceKey =
+      serviceKey && typeof serviceKey === 'string' ? JSON.parse(serviceKey) : serviceKey ?? {};
+    this.client_email = this.serviceKey.client_email;
+    this.private_key = this.serviceKey.private_key;
+    this.project_id = this.serviceKey.project_id;
     this.access_token = null;
+
+    this.apiKey = creds[AuthKeys.GOOGLE_API_KEY];
+
     if (options.skipSetOptions) {
       return;
     }
@@ -85,7 +104,7 @@ class GoogleClient extends BaseClient {
       this.options = options;
     }
 
-    this.options.examples = this.options.examples
+    this.options.examples = (this.options.examples ?? [])
       .filter((ex) => ex)
       .filter((obj) => obj.input.content !== '' && obj.output.content !== '');
 
@@ -103,15 +122,24 @@ class GoogleClient extends BaseClient {
       // stop: modelOptions.stop // no stop method for now
     };
 
-    this.isChatModel = this.modelOptions.model.includes('chat');
+    // TODO: as of 12/14/23, only gemini models are "Generative AI" models provided by Google
+    this.isGenerativeModel = this.modelOptions.model.includes('gemini');
+    const { isGenerativeModel } = this;
+    this.isChatModel = !isGenerativeModel && this.modelOptions.model.includes('chat');
     const { isChatModel } = this;
-    this.isTextModel = !isChatModel && /code|text/.test(this.modelOptions.model);
+    this.isTextModel =
+      !isGenerativeModel && !isChatModel && /code|text/.test(this.modelOptions.model);
     const { isTextModel } = this;
 
     this.maxContextTokens = getModelMaxTokens(this.modelOptions.model, EModelEndpoint.google);
     // The max prompt tokens is determined by the max context tokens minus the max response tokens.
     // Earlier messages will be dropped until the prompt is within the limit.
     this.maxResponseTokens = this.modelOptions.maxOutputTokens || settings.maxOutputTokens.default;
+
+    if (this.maxContextTokens > 32000) {
+      this.maxContextTokens = this.maxContextTokens - this.maxResponseTokens;
+    }
+
     this.maxPromptTokens =
       this.options.maxPromptTokens || this.maxContextTokens - this.maxResponseTokens;
 
@@ -134,7 +162,7 @@ class GoogleClient extends BaseClient {
     this.userLabel = this.options.userLabel || 'User';
     this.modelLabel = this.options.modelLabel || 'Assistant';
 
-    if (isChatModel) {
+    if (isChatModel || isGenerativeModel) {
       // Use these faux tokens to help the AI understand the context since we are building the chat log ourselves.
       // Trying to use "<|im_start|>" causes the AI to still generate "<" or "<|" at the end sometimes for some reason,
       // without tripping the stop sequences, so I'm using "||>" instead.
@@ -189,6 +217,16 @@ class GoogleClient extends BaseClient {
   }
 
   buildMessages(messages = [], parentMessageId) {
+    if (!this.isGenerativeModel && !this.project_id) {
+      throw new Error(
+        '[GoogleClient] a Service Account JSON Key is required for PaLM 2 and Codey models (Vertex AI)',
+      );
+    } else if (this.isGenerativeModel && (!this.apiKey || this.apiKey === 'user_provided')) {
+      throw new Error(
+        '[GoogleClient] an API Key is required for Gemini models (Generative Language API)',
+      );
+    }
+
     if (this.isTextModel) {
       return this.buildMessagesPrompt(messages, parentMessageId);
     }
@@ -398,6 +436,16 @@ class GoogleClient extends BaseClient {
     return res.data;
   }
 
+  createLLM(clientOptions) {
+    if (this.isGenerativeModel) {
+      return new ChatGoogleGenerativeAI({ ...clientOptions, apiKey: this.apiKey });
+    }
+
+    return this.isTextModel
+      ? new GoogleVertexAI(clientOptions)
+      : new ChatGoogleVertexAI(clientOptions);
+  }
+
   async getCompletion(_payload, options = {}) {
     const { onProgress, abortController } = options;
     const { parameters, instances } = _payload;
@@ -408,7 +456,7 @@ class GoogleClient extends BaseClient {
     let clientOptions = {
       authOptions: {
         credentials: {
-          ...this.credentials,
+          ...this.serviceKey,
         },
         projectId: this.project_id,
       },
@@ -436,9 +484,7 @@ class GoogleClient extends BaseClient {
       clientOptions.examples = examples;
     }
 
-    const model = this.isTextModel
+    const model = this.createLLM(clientOptions);
-      ? new GoogleVertexAI(clientOptions)
-      : new ChatGoogleVertexAI(clientOptions);
 
     let reply = '';
     const messages = this.isTextModel
@@ -457,7 +503,9 @@ class GoogleClient extends BaseClient {
     });
 
     for await (const chunk of stream) {
-      await this.generateTextStream(chunk?.content ?? chunk, onProgress, { delay: 7 });
+      await this.generateTextStream(chunk?.content ?? chunk, onProgress, {
+        delay: this.isGenerativeModel ? 12 : 8,
+      });
       reply += chunk?.content ?? chunk;
     }
 

api/config/parsers.js

@@ -3,7 +3,7 @@ const winston = require('winston');
 const traverse = require('traverse');
 const { klona } = require('klona/full');
 
-const sensitiveKeys = [/^sk-\w+$/];
+const sensitiveKeys = [/^sk-\w+$/, /Bearer \w+/];
 
 /**
  * Determines if a given key string is sensitive.

api/jest.config.js

@@ -10,5 +10,6 @@ module.exports = {
   ],
   moduleNameMapper: {
     '~/(.*)': '<rootDir>/$1',
+    '~/data/auth.json': '<rootDir>/__mocks__/auth.mock.json',
   },
 };

api/package.json

@@ -31,6 +31,7 @@
     "@azure/search-documents": "^12.0.0",
     "@keyv/mongo": "^2.1.8",
     "@keyv/redis": "^2.8.0",
+    "@langchain/google-genai": "^0.0.2",
     "axios": "^1.3.4",
     "bcryptjs": "^2.4.3",
     "cheerio": "^1.0.0-rc.12",

api/server/services/Config/loadAsyncEndpoints.js

@@ -8,9 +8,9 @@ const { openAIApiKey, azureOpenAIApiKey, useAzurePlugins, userProvidedOpenAI, go
  */
 async function loadAsyncEndpoints() {
   let i = 0;
-  let key, googleUserProvides;
+  let serviceKey, googleUserProvides;
   try {
-    key = require('~/data/auth.json');
+    serviceKey = require('~/data/auth.json');
   } catch (e) {
     if (i === 0) {
       i++;
@@ -33,7 +33,7 @@ async function loadAsyncEndpoints() {
   }
   const plugins = transformToolsToMap(tools);
 
-  const google = key || googleUserProvides ? { userProvide: googleUserProvides } : false;
+  const google = serviceKey || googleKey ? { userProvide: googleUserProvides } : false;
 
   const gptPlugins =
     openAIApiKey || azureOpenAIApiKey

api/server/services/Config/loadDefaultModels.js

@@ -1,10 +1,10 @@
+const { EModelEndpoint, defaultModels } = require('librechat-data-provider');
+const { useAzurePlugins } = require('~/server/services/Config/EndpointService').config;
 const {
   getOpenAIModels,
   getChatGPTBrowserModels,
   getAnthropicModels,
 } = require('~/server/services/ModelService');
-const { EModelEndpoint } = require('librechat-data-provider');
-const { useAzurePlugins } = require('~/server/services/Config/EndpointService').config;
 
 const fitlerAssistantModels = (str) => {
   return /gpt-4|gpt-3\\.5/i.test(str) && !/vision|instruct/i.test(str);
@@ -21,18 +21,7 @@ async function loadDefaultModels() {
     [EModelEndpoint.openAI]: openAI,
     [EModelEndpoint.azureOpenAI]: azureOpenAI,
     [EModelEndpoint.assistant]: openAI.filter(fitlerAssistantModels),
-    [EModelEndpoint.google]: [
+    [EModelEndpoint.google]: defaultModels[EModelEndpoint.google],
-      'chat-bison',
-      'chat-bison-32k',
-      'codechat-bison',
-      'codechat-bison-32k',
-      'text-bison',
-      'text-bison-32k',
-      'text-unicorn',
-      'code-gecko',
-      'code-bison',
-      'code-bison-32k',
-    ],
     [EModelEndpoint.bingAI]: ['BingAI', 'Sydney'],
     [EModelEndpoint.chatGPTBrowser]: chatGPTBrowser,
     [EModelEndpoint.gptPlugins]: gptPlugins,

api/server/services/Endpoints/google/initializeClient.js

@@ -1,5 +1,5 @@
 const { GoogleClient } = require('~/app');
-const { EModelEndpoint } = require('librechat-data-provider');
+const { EModelEndpoint, AuthKeys } = require('librechat-data-provider');
 const { getUserKey, checkUserKeyExpiry } = require('~/server/services/UserService');
 
 const initializeClient = async ({ req, res, endpointOption }) => {
@@ -11,14 +11,26 @@ const initializeClient = async ({ req, res, endpointOption }) => {
   if (expiresAt && isUserProvided) {
     checkUserKeyExpiry(
       expiresAt,
-      'Your Google key has expired. Please provide your JSON credentials again.',
+      'Your Google Credentials have expired. Please provide your Service Account JSON Key or Generative Language API Key again.',
     );
     userKey = await getUserKey({ userId: req.user.id, name: EModelEndpoint.google });
   }
 
-  const apiKey = isUserProvided ? userKey : require('~/data/auth.json');
+  let serviceKey = {};
+  try {
+    serviceKey = require('~/data/auth.json');
+  } catch (e) {
+    // Do nothing
+  }
 
-  const client = new GoogleClient(apiKey, {
+  const credentials = isUserProvided
+    ? userKey
+    : {
+      [AuthKeys.GOOGLE_SERVICE_KEY]: serviceKey,
+      [AuthKeys.GOOGLE_API_KEY]: GOOGLE_KEY,
+    };
+
+  const client = new GoogleClient(credentials, {
     req,
     res,
     reverseProxyUrl: GOOGLE_REVERSE_PROXY ?? null,
@@ -28,7 +40,7 @@ const initializeClient = async ({ req, res, endpointOption }) => {
 
   return {
     client,
-    apiKey,
+    credentials,
   };
 };
 

api/server/services/Endpoints/google/initializeClient.spec.js

@@ -0,0 +1,84 @@
+const initializeClient = require('./initializeClient');
+const { GoogleClient } = require('~/app');
+const { checkUserKeyExpiry, getUserKey } = require('../../UserService');
+
+jest.mock('../../UserService', () => ({
+  checkUserKeyExpiry: jest.fn().mockImplementation((expiresAt, errorMessage) => {
+    if (new Date(expiresAt) < new Date()) {
+      throw new Error(errorMessage);
+    }
+  }),
+  getUserKey: jest.fn().mockImplementation(() => ({})),
+}));
+
+describe('google/initializeClient', () => {
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  test('should initialize GoogleClient with user-provided credentials', async () => {
+    process.env.GOOGLE_KEY = 'user_provided';
+    process.env.GOOGLE_REVERSE_PROXY = 'http://reverse.proxy';
+    process.env.PROXY = 'http://proxy';
+
+    const expiresAt = new Date(Date.now() + 60000).toISOString();
+
+    const req = {
+      body: { key: expiresAt },
+      user: { id: '123' },
+    };
+    const res = {};
+    const endpointOption = { modelOptions: { model: 'default-model' } };
+
+    const { client, credentials } = await initializeClient({ req, res, endpointOption });
+
+    expect(getUserKey).toHaveBeenCalledWith({ userId: '123', name: 'google' });
+    expect(client).toBeInstanceOf(GoogleClient);
+    expect(client.options.reverseProxyUrl).toBe('http://reverse.proxy');
+    expect(client.options.proxy).toBe('http://proxy');
+    expect(credentials).toEqual({});
+  });
+
+  test('should initialize GoogleClient with service key credentials', async () => {
+    process.env.GOOGLE_KEY = 'service_key';
+    process.env.GOOGLE_REVERSE_PROXY = 'http://reverse.proxy';
+    process.env.PROXY = 'http://proxy';
+
+    const req = {
+      body: { key: null },
+      user: { id: '123' },
+    };
+    const res = {};
+    const endpointOption = { modelOptions: { model: 'default-model' } };
+
+    const { client, credentials } = await initializeClient({ req, res, endpointOption });
+
+    expect(client).toBeInstanceOf(GoogleClient);
+    expect(client.options.reverseProxyUrl).toBe('http://reverse.proxy');
+    expect(client.options.proxy).toBe('http://proxy');
+    expect(credentials).toEqual({
+      GOOGLE_SERVICE_KEY: {},
+      GOOGLE_API_KEY: 'service_key',
+    });
+  });
+
+  test('should handle expired user-provided key', async () => {
+    process.env.GOOGLE_KEY = 'user_provided';
+
+    const expiresAt = new Date(Date.now() - 10000).toISOString(); // Expired
+    const req = {
+      body: { key: expiresAt },
+      user: { id: '123' },
+    };
+    const res = {};
+    const endpointOption = { modelOptions: { model: 'default-model' } };
+
+    checkUserKeyExpiry.mockImplementation((expiresAt, errorMessage) => {
+      throw new Error(errorMessage);
+    });
+
+    await expect(initializeClient({ req, res, endpointOption })).rejects.toThrow(
+      /Your Google Credentials have expired/,
+    );
+  });
+});

api/server/services/ModelService.js

@@ -1,6 +1,7 @@
 const Keyv = require('keyv');
 const axios = require('axios');
 const HttpsProxyAgent = require('https-proxy-agent');
+const { EModelEndpoint, defaultModels } = require('librechat-data-provider');
 const { isEnabled } = require('~/server/utils');
 const keyvRedis = require('~/cache/keyvRedis');
 const { extractBaseURL } = require('~/utils');
@@ -117,15 +118,7 @@ const getChatGPTBrowserModels = () => {
 };
 
 const getAnthropicModels = () => {
-  let models = [
+  let models = defaultModels[EModelEndpoint.anthropic];
-    'claude-2.1',
-    'claude-2',
-    'claude-1.2',
-    'claude-1',
-    'claude-1-100k',
-    'claude-instant-1',
-    'claude-instant-1-100k',
-  ];
   if (ANTHROPIC_MODELS) {
     models = String(ANTHROPIC_MODELS).split(',');
   }

api/test/__mocks__/auth.mock.json

@@ -0,0 +1,13 @@
+{
+  "type": "service_account",
+  "project_id": "",
+  "private_key_id": "",
+  "private_key": "",
+  "client_email": "",
+  "client_id": "",
+  "auth_uri": "",
+  "token_uri": "",
+  "auth_provider_x509_cert_url": "",
+  "client_x509_cert_url": "",
+  "universe_domain": ""
+}

api/utils/tokens.js

@@ -56,11 +56,12 @@ const maxTokensMap = {
     'gpt-4-1106': 127995, // -5 from max
   },
   [EModelEndpoint.google]: {
-    /* Max I/O is 32k combined, so -1000 to leave room for response */
+    /* Max I/O is combined so we subtract the amount from max response tokens for actual total */
-    'text-bison-32k': 31000,
+    gemini: 32750, // -10 from max
-    'chat-bison-32k': 31000,
+    'text-bison-32k': 32758, // -10 from max
-    'code-bison-32k': 31000,
+    'chat-bison-32k': 32758, // -10 from max
-    'codechat-bison-32k': 31000,
+    'code-bison-32k': 32758, // -10 from max
+    'codechat-bison-32k': 32758,
     /* Codey, -5 from max: 6144 */
     'code-': 6139,
     'codechat-': 6139,

api/utils/tokens.spec.js

@@ -114,6 +114,9 @@ describe('getModelMaxTokens', () => {
   });
 
   test('should return correct tokens for partial match - Google models', () => {
+    expect(getModelMaxTokens('gemini-pro', EModelEndpoint.google)).toBe(
+      maxTokensMap[EModelEndpoint.google]['gemini'],
+    );
     expect(getModelMaxTokens('code-', EModelEndpoint.google)).toBe(
       maxTokensMap[EModelEndpoint.google]['code-'],
     );

client/src/components/Chat/Input/HeaderOptions.tsx

@@ -93,7 +93,7 @@ export default function OptionsBar() {
               visible={showPopover}
               saveAsPreset={saveAsPreset}
               closePopover={() => setShowPopover(false)}
-              PopoverButtons={<PopoverButtons endpoint={endpoint} />}
+              PopoverButtons={<PopoverButtons />}
             >
               <div className="px-4 py-4">
                 <EndpointSettings

client/src/components/Chat/Input/PopoverButtons.tsx

@@ -13,18 +13,28 @@ type TPopoverButton = {
 };
 
 export default function PopoverButtons({
-  endpoint,
   buttonClass,
   iconClass = '',
 }: {
-  endpoint: EModelEndpoint;
   buttonClass?: string;
   iconClass?: string;
 }) {
-  const { optionSettings, setOptionSettings, showAgentSettings, setShowAgentSettings } =
+  const {
-    useChatContext();
+    conversation,
+    optionSettings,
+    setOptionSettings,
+    showAgentSettings,
+    setShowAgentSettings,
+  } = useChatContext();
+
+  const { model, endpoint } = conversation ?? {};
+  const isGenerativeModel = model?.toLowerCase()?.includes('gemini');
+  const isChatModel = !isGenerativeModel && model?.toLowerCase()?.includes('chat');
+  const isTextModel = !isGenerativeModel && !isChatModel && /code|text/.test(model ?? '');
+
+  const { showExamples } = optionSettings;
+  const showExamplesButton = !isGenerativeModel && !isTextModel && isChatModel;
 
-  const { showExamples, isCodeChat } = optionSettings;
   const triggerExamples = () =>
     setOptionSettings((prev) => ({ ...prev, showExamples: !prev.showExamples }));
 
@@ -32,7 +42,7 @@ export default function PopoverButtons({
     [EModelEndpoint.google]: [
       {
         label: (showExamples ? 'Hide' : 'Show') + ' Examples',
-        buttonClass: isCodeChat ? 'disabled' : '',
+        buttonClass: isGenerativeModel || isTextModel ? 'disabled' : '',
         handler: triggerExamples,
         icon: <MessagesSquared className={cn('mr-1 w-[14px]', iconClass)} />,
       },
@@ -47,11 +57,15 @@ export default function PopoverButtons({
     ],
   };
 
-  const endpointButtons = buttons[endpoint];
+  const endpointButtons = buttons[endpoint ?? ''];
   if (!endpointButtons) {
     return null;
   }
 
+  if (endpoint === EModelEndpoint.google && !showExamplesButton) {
+    return null;
+  }
+
   return (
     <div>
       {endpointButtons.map((button, index) => (

client/src/components/Endpoints/Icon.tsx

@@ -6,6 +6,7 @@ import {
   AzureMinimalIcon,
   PaLMIcon,
   CodeyIcon,
+  GeminiIcon,
 } from '~/components/svg';
 import { useAuthContext } from '~/hooks/AuthContext';
 import { IconProps } from '~/common';
@@ -59,12 +60,18 @@ const Icon: React.FC<IconProps> = (props) => {
         name: 'Plugins',
       },
       [EModelEndpoint.google]: {
-        icon: model?.includes('code') ? (
+        icon: model?.toLowerCase()?.includes('code') ? (
           <CodeyIcon size={size * 0.75} />
+        ) : model?.toLowerCase()?.includes('gemini') ? (
+          <GeminiIcon size={size * 0.7} />
         ) : (
           <PaLMIcon size={size * 0.7} />
         ),
-        name: model?.includes('code') ? 'Codey' : 'PaLM2',
+        name: model?.toLowerCase()?.includes('code')
+          ? 'Codey'
+          : model?.toLowerCase()?.includes('gemini')
+            ? 'Gemini'
+            : 'PaLM2',
       },
       [EModelEndpoint.anthropic]: {
         icon: <AnthropicIcon size={size * 0.5555555555555556} />,

client/src/components/Endpoints/Settings/Google.tsx

@@ -1,4 +1,4 @@
-import React from 'react';
+import { useEffect } from 'react';
 import TextareaAutosize from 'react-textarea-autosize';
 import { EModelEndpoint, endpointSettings } from 'librechat-data-provider';
 import type { TModelSelectProps } from '~/common';
@@ -18,11 +18,32 @@ import { useLocalize } from '~/hooks';
 
 export default function Settings({ conversation, setOption, models, readonly }: TModelSelectProps) {
   const localize = useLocalize();
+  const google = endpointSettings[EModelEndpoint.google];
+  const { model, modelLabel, promptPrefix, temperature, topP, topK, maxOutputTokens } =
+    conversation ?? {};
+
+  const isGeminiPro = model?.toLowerCase()?.includes('gemini-pro');
+
+  const maxOutputTokensMax = isGeminiPro
+    ? google.maxOutputTokens.maxGeminiPro
+    : google.maxOutputTokens.max;
+  const maxOutputTokensDefault = isGeminiPro
+    ? google.maxOutputTokens.defaultGeminiPro
+    : google.maxOutputTokens.default;
+
+  useEffect(
+    () => {
+      if (model) {
+        setOption('maxOutputTokens')(Math.min(Number(maxOutputTokens) ?? 0, maxOutputTokensMax));
+      }
+    },
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+    [model],
+  );
+
   if (!conversation) {
     return null;
   }
-  const { model, modelLabel, promptPrefix, temperature, topP, topK, maxOutputTokens } =
-    conversation;
 
   const setModel = setOption('model');
   const setModelLabel = setOption('modelLabel');
@@ -32,8 +53,9 @@ export default function Settings({ conversation, setOption, models, readonly }:
   const setTopK = setOption('topK');
   const setMaxOutputTokens = setOption('maxOutputTokens');
 
-  const isTextModel = !model?.includes('chat') && /code|text/.test(model ?? '');
+  const isGenerativeModel = model?.toLowerCase()?.includes('gemini');
-  const google = endpointSettings[EModelEndpoint.google];
+  const isChatModel = !isGenerativeModel && model?.toLowerCase()?.includes('chat');
+  const isTextModel = !isGenerativeModel && !isChatModel && /code|text/.test(model ?? '');
 
   return (
     <div className="grid grid-cols-5 gap-6">
@@ -216,15 +238,15 @@ export default function Settings({ conversation, setOption, models, readonly }:
               <Label htmlFor="max-tokens-int" className="text-left text-sm font-medium">
                 {localize('com_endpoint_max_output_tokens')}{' '}
                 <small className="opacity-40">
-                  ({localize('com_endpoint_default_with_num', google.maxOutputTokens.default + '')})
+                  ({localize('com_endpoint_default_with_num', maxOutputTokensDefault + '')})
                 </small>
               </Label>
               <InputNumber
                 id="max-tokens-int"
                 disabled={readonly}
                 value={maxOutputTokens}
-                onChange={(value) => setMaxOutputTokens(value ?? google.maxOutputTokens.default)}
+                onChange={(value) => setMaxOutputTokens(value ?? maxOutputTokensDefault)}
-                max={google.maxOutputTokens.max}
+                max={maxOutputTokensMax}
                 min={google.maxOutputTokens.min}
                 step={google.maxOutputTokens.step}
                 controls={false}
@@ -239,10 +261,10 @@ export default function Settings({ conversation, setOption, models, readonly }:
             </div>
             <Slider
               disabled={readonly}
-              value={[maxOutputTokens ?? google.maxOutputTokens.default]}
+              value={[maxOutputTokens ?? maxOutputTokensDefault]}
               onValueChange={(value) => setMaxOutputTokens(value[0])}
-              doubleClickHandler={() => setMaxOutputTokens(google.maxOutputTokens.default)}
+              doubleClickHandler={() => setMaxOutputTokens(maxOutputTokensDefault)}
-              max={google.maxOutputTokens.max}
+              max={maxOutputTokensMax}
               min={google.maxOutputTokens.min}
               step={google.maxOutputTokens.step}
               className="flex h-4 w-full"

client/src/components/Endpoints/Settings/MultiView/GoogleSettings.tsx

@@ -12,9 +12,12 @@ export default function GoogleView({ conversation, models, isPreset = false }) {
     return null;
   }
 
-  const { examples } = conversation;
+  const { examples, model } = conversation;
-  const { showExamples, isCodeChat } = optionSettings;
+  const isGenerativeModel = model?.toLowerCase()?.includes('gemini');
-  return showExamples && !isCodeChat ? (
+  const isChatModel = !isGenerativeModel && model?.toLowerCase()?.includes('chat');
+  const isTextModel = !isGenerativeModel && !isChatModel && /code|text/.test(model ?? '');
+  const { showExamples } = optionSettings;
+  return showExamples && isChatModel && !isTextModel ? (
     <Examples
       examples={examples ?? [{ input: { content: '' }, output: { content: '' } }]}
       setExample={setExample}

client/src/components/Input/EndpointMenu/FileUpload.tsx

@@ -6,6 +6,7 @@ import { useLocalize } from '~/hooks';
 type FileUploadProps = {
   onFileSelected: (jsonData: Record<string, unknown>) => void;
   className?: string;
+  containerClassName?: string;
   successText?: string;
   invalidText?: string;
   validator?: ((data: Record<string, unknown>) => boolean) | null;
@@ -16,6 +17,7 @@ type FileUploadProps = {
 const FileUpload: React.FC<FileUploadProps> = ({
   onFileSelected,
   className = '',
+  containerClassName = '',
   successText = null,
   invalidText = null,
   validator = null,
@@ -66,6 +68,7 @@ const FileUpload: React.FC<FileUploadProps> = ({
       className={cn(
         'mr-1 flex h-auto cursor-pointer items-center rounded bg-transparent px-2 py-1 text-xs font-medium font-normal transition-colors hover:bg-slate-200 hover:text-green-700 dark:bg-transparent dark:text-gray-300 dark:hover:bg-gray-800 dark:hover:text-green-500',
         statusColor,
+        containerClassName,
       )}
     >
       <FileUp className="mr-1 flex w-[22px] items-center stroke-1" />

client/src/components/Input/SetKeyDialog/GoogleConfig.tsx

@@ -1,8 +1,11 @@
 import React from 'react';
 import { object, string } from 'zod';
+import { AuthKeys } from 'librechat-data-provider';
 import type { TConfigProps } from '~/common';
-import FileUpload from '../EndpointMenu/FileUpload';
+import FileUpload from '~/components/Input/EndpointMenu/FileUpload';
-import { useLocalize } from '~/hooks';
+import { useLocalize, useMultipleKeys } from '~/hooks';
+import InputWithLabel from './InputWithLabel';
+import { Label } from '~/components/ui';
 
 const CredentialsSchema = object({
   client_email: string().email().min(3),
@@ -15,20 +18,43 @@ const validateCredentials = (credentials: Record<string, unknown>) => {
   return result.success;
 };
 
-const GoogleConfig = ({ setUserKey }: Pick<TConfigProps, 'setUserKey'>) => {
+const GoogleConfig = ({ userKey, setUserKey }: Pick<TConfigProps, 'userKey' | 'setUserKey'>) => {
   const localize = useLocalize();
+  const { getMultiKey, setMultiKey } = useMultipleKeys(setUserKey);
+
   return (
-    <FileUpload
+    <>
-      id="googleKey"
+      <div className="flex flex-row">
-      className="w-full"
+        <Label htmlFor={AuthKeys.GOOGLE_SERVICE_KEY} className="text-left text-sm font-medium">
-      text={localize('com_endpoint_config_key_import_json_key')}
+          {localize('com_endpoint_config_google_service_key')}
-      successText={localize('com_endpoint_config_key_import_json_key_success')}
+        </Label>
-      invalidText={localize('com_endpoint_config_key_import_json_key_invalid')}
+        <div className="mx-1 text-left text-sm text-gray-700 dark:text-gray-400">
-      validator={validateCredentials}
+          {localize('com_endpoint_config_google_cloud_platform')}
-      onFileSelected={(data) => {
+        </div>
-        setUserKey(JSON.stringify(data));
+        <br />
-      }}
+      </div>
-    />
+      <FileUpload
+        id={AuthKeys.GOOGLE_SERVICE_KEY}
+        className="w-full"
+        containerClassName="dark:bg-gray-700 h-10 max-h-10 w-full resize-none py-2 dark:ring-1 dark:ring-gray-400"
+        text={localize('com_endpoint_config_key_import_json_key')}
+        successText={localize('com_endpoint_config_key_import_json_key_success')}
+        invalidText={localize('com_endpoint_config_key_import_json_key_invalid')}
+        validator={validateCredentials}
+        onFileSelected={(data) => {
+          setMultiKey(AuthKeys.GOOGLE_SERVICE_KEY, JSON.stringify(data), userKey);
+        }}
+      />
+      <InputWithLabel
+        id={AuthKeys.GOOGLE_API_KEY}
+        value={getMultiKey(AuthKeys.GOOGLE_API_KEY, userKey) ?? ''}
+        onChange={(e: { target: { value: string } }) =>
+          setMultiKey(AuthKeys.GOOGLE_API_KEY, e.target.value ?? '', userKey)
+        }
+        label={localize('com_endpoint_config_google_api_key')}
+        subLabel={localize('com_endpoint_config_google_gemini_api')}
+      />
+    </>
   );
 };
 

client/src/components/Input/SetKeyDialog/HelpText.tsx

@@ -55,28 +55,45 @@ function HelpText({ endpoint }: { endpoint: string }) {
       </small>
     ),
     [EModelEndpoint.google]: (
-      <small className="break-all text-gray-600">
+      <>
-        {localize('com_endpoint_config_key_google_need_to')}{' '}
+        <small className="break-all text-gray-600">
-        <a
+          {localize('com_endpoint_config_google_service_key')}
-          target="_blank"
+          {': '}
-          href="https://console.cloud.google.com/vertex-ai"
+          {localize('com_endpoint_config_key_google_need_to')}{' '}
-          rel="noreferrer"
+          <a
-          className="text-blue-600 underline"
+            target="_blank"
-        >
+            href="https://console.cloud.google.com/vertex-ai"
-          {localize('com_endpoint_config_key_google_vertex_ai')}
+            rel="noreferrer"
-        </a>{' '}
+            className="text-blue-600 underline"
-        {localize('com_endpoint_config_key_google_vertex_api')}{' '}
+          >
-        <a
+            {localize('com_endpoint_config_key_google_vertex_ai')}
-          target="_blank"
+          </a>{' '}
-          href="https://console.cloud.google.com/projectselector/iam-admin/serviceaccounts/create?walkthrough_id=iam--create-service-account#step_index=1"
+          {localize('com_endpoint_config_key_google_vertex_api')}{' '}
-          rel="noreferrer"
+          <a
-          className="text-blue-600 underline"
+            target="_blank"
-        >
+            href="https://console.cloud.google.com/projectselector/iam-admin/serviceaccounts/create?walkthrough_id=iam--create-service-account#step_index=1"
-          {localize('com_endpoint_config_key_google_service_account')}
+            rel="noreferrer"
-        </a>
+            className="text-blue-600 underline"
-        {'. '}
+          >
-        {localize('com_endpoint_config_key_google_vertex_api_role')}
+            {localize('com_endpoint_config_key_google_service_account')}
-      </small>
+          </a>
+          {'. '}
+          {localize('com_endpoint_config_key_google_vertex_api_role')}
+        </small>
+        <small className="break-all text-gray-600">
+          {localize('com_endpoint_config_google_api_key')}
+          {': '}
+          {localize('com_endpoint_config_google_api_info')}{' '}
+          <a
+            target="_blank"
+            href="https://makersuite.google.com/app/apikey"
+            rel="noreferrer"
+            className="text-blue-600 underline"
+          >
+            {localize('com_endpoint_config_click_here')}
+          </a>{' '}
+        </small>
+      </>
     ),
   };
 

client/src/components/Input/SetKeyDialog/InputWithLabel.tsx

@@ -1,23 +1,29 @@
 import React, { ChangeEvent, FC } from 'react';
-import { Input, Label } from '~/components';
 import { cn, defaultTextPropsLabel, removeFocusOutlines } from '~/utils/';
+import { Input, Label } from '~/components/ui';
 import { useLocalize } from '~/hooks';
 
 interface InputWithLabelProps {
   value: string;
   onChange: (event: ChangeEvent<HTMLInputElement>) => void;
   label: string;
+  subLabel?: string;
   id: string;
 }
 
-const InputWithLabel: FC<InputWithLabelProps> = ({ value, onChange, label, id }) => {
+const InputWithLabel: FC<InputWithLabelProps> = ({ value, onChange, label, subLabel, id }) => {
   const localize = useLocalize();
   return (
     <>
-      <Label htmlFor={id} className="text-left text-sm font-medium">
+      <div className="flex flex-row">
-        {label}
+        <Label htmlFor={id} className="text-left text-sm font-medium">
+          {label}
+        </Label>
+        {subLabel && (
+          <div className="mx-1 text-left text-sm text-gray-700 dark:text-gray-400">{subLabel}</div>
+        )}
         <br />
-      </Label>
+      </div>
 
       <Input
         id={id}

client/src/components/Input/SetKeyDialog/OpenAIConfig.tsx

@@ -3,20 +3,14 @@ import React, { useEffect, useState } from 'react';
 // TODO: Temporarily remove checkbox until Plugins solution for Azure is figured out
 // import * as Checkbox from '@radix-ui/react-checkbox';
 // import { CheckIcon } from '@radix-ui/react-icons';
+import { useMultipleKeys } from '~/hooks/Input';
 import InputWithLabel from './InputWithLabel';
 import type { TConfigProps } from '~/common';
-
+import { isJson } from '~/utils/json';
-function isJson(str: string) {
-  try {
-    JSON.parse(str);
-  } catch (e) {
-    return false;
-  }
-  return true;
-}
 
 const OpenAIConfig = ({ userKey, setUserKey, endpoint }: TConfigProps) => {
   const [showPanel, setShowPanel] = useState(endpoint === 'azureOpenAI');
+  const { getMultiKey: getAzure, setMultiKey: setAzure } = useMultipleKeys(setUserKey);
 
   useEffect(() => {
     if (isJson(userKey)) {
@@ -31,24 +25,6 @@ const OpenAIConfig = ({ userKey, setUserKey, endpoint }: TConfigProps) => {
     }
   }, [showPanel]);
 
-  function getAzure(name: string) {
-    if (isJson(userKey)) {
-      const newKey = JSON.parse(userKey);
-      return newKey[name];
-    } else {
-      return '';
-    }
-  }
-
-  function setAzure(name: string, value: number | string | boolean) {
-    let newKey = {};
-    if (isJson(userKey)) {
-      newKey = JSON.parse(userKey);
-    }
-    newKey[name] = value;
-
-    setUserKey(JSON.stringify(newKey));
-  }
   return (
     <>
       {!showPanel ? (
@@ -64,36 +40,36 @@ const OpenAIConfig = ({ userKey, setUserKey, endpoint }: TConfigProps) => {
         <>
           <InputWithLabel
             id={'instanceNameLabel'}
-            value={getAzure('azureOpenAIApiInstanceName') ?? ''}
+            value={getAzure('azureOpenAIApiInstanceName', userKey) ?? ''}
             onChange={(e: { target: { value: string } }) =>
-              setAzure('azureOpenAIApiInstanceName', e.target.value ?? '')
+              setAzure('azureOpenAIApiInstanceName', e.target.value ?? '', userKey)
             }
             label={'Azure OpenAI Instance Name'}
           />
 
           <InputWithLabel
             id={'deploymentNameLabel'}
-            value={getAzure('azureOpenAIApiDeploymentName') ?? ''}
+            value={getAzure('azureOpenAIApiDeploymentName', userKey) ?? ''}
             onChange={(e: { target: { value: string } }) =>
-              setAzure('azureOpenAIApiDeploymentName', e.target.value ?? '')
+              setAzure('azureOpenAIApiDeploymentName', e.target.value ?? '', userKey)
             }
             label={'Azure OpenAI Deployment Name'}
           />
 
           <InputWithLabel
             id={'versionLabel'}
-            value={getAzure('azureOpenAIApiVersion') ?? ''}
+            value={getAzure('azureOpenAIApiVersion', userKey) ?? ''}
             onChange={(e: { target: { value: string } }) =>
-              setAzure('azureOpenAIApiVersion', e.target.value ?? '')
+              setAzure('azureOpenAIApiVersion', e.target.value ?? '', userKey)
             }
             label={'Azure OpenAI API Version'}
           />
 
           <InputWithLabel
             id={'apiKeyLabel'}
-            value={getAzure('azureOpenAIApiKey') ?? ''}
+            value={getAzure('azureOpenAIApiKey', userKey) ?? ''}
             onChange={(e: { target: { value: string } }) =>
-              setAzure('azureOpenAIApiKey', e.target.value ?? '')
+              setAzure('azureOpenAIApiKey', e.target.value ?? '', userKey)
             }
             label={'Azure OpenAI API Key'}
           />

client/src/components/Messages/Content/Error.tsx

@@ -1,17 +1,8 @@
 import React from 'react';
 import type { TOpenAIMessage } from 'librechat-data-provider';
-import { formatJSON, extractJson } from '~/utils/json';
+import { formatJSON, extractJson, isJson } from '~/utils/json';
 import CodeBlock from './CodeBlock';
 
-const isJson = (str: string) => {
-  try {
-    JSON.parse(str);
-  } catch (e) {
-    return false;
-  }
-  return true;
-};
-
 type TConcurrent = {
   limit: number;
 };

client/src/components/svg/GeminiIcon.tsx

@@ -0,0 +1,36 @@
+export default function GeminiIcon({
+  size = 25,
+  className = '',
+}: {
+  size?: number;
+  className?: string;
+}) {
+  return (
+    <svg
+      width={size}
+      height={size}
+      className={className}
+      viewBox="0 0 18 18"
+      preserveAspectRatio="xMidYMid meet"
+      focusable="false"
+    >
+      <path
+        fill="url(#_4rif_paint0_radial_897_42)"
+        d="M9 18c0-1.245-.24-2.415-.72-3.51a8.934 8.934 0 00-1.912-2.857A8.934 8.934 0 003.51 9.72 8.646 8.646 0 000 9a8.886 8.886 0 003.51-.697 9.247 9.247 0 002.857-1.936A8.934 8.934 0 008.28 3.51C8.76 2.415 9 1.245 9 0c0 1.245.232 2.415.697 3.51a9.247 9.247 0 001.936 2.857 9.247 9.247 0 002.857 1.936A8.886 8.886 0 0018 9c-1.245 0-2.415.24-3.51.72a8.934 8.934 0 00-2.857 1.912 9.247 9.247 0 00-1.935 2.858A8.886 8.886 0 009 18z"
+      />
+      <defs>
+        <radialGradient
+          id="_4rif_paint0_radial_897_42"
+          cx="0"
+          cy="0"
+          r="1"
+          gradientUnits="userSpaceOnUse"
+          gradientTransform="rotate(135 9 3.728) scale(25.4558 12.7279)"
+        >
+          <stop offset=".325" stopColor="#FFDDB7"></stop>
+          <stop offset=".706" stopColor="#076EFF"></stop>
+        </radialGradient>
+      </defs>
+    </svg>
+  );
+}

client/src/components/svg/index.ts

@@ -36,6 +36,7 @@ export { default as BingAIMinimalIcon } from './BingAIMinimalIcon';
 export { default as PaLMinimalIcon } from './PaLMinimalIcon';
 export { default as PaLMIcon } from './PaLMIcon';
 export { default as CodeyIcon } from './CodeyIcon';
+export { default as GeminiIcon } from './GeminiIcon';
 export { default as GoogleMinimalIcon } from './GoogleMinimalIcon';
 export { default as AnthropicMinimalIcon } from './AnthropicMinimalIcon';
 export { default as SendMessageIcon } from './SendMessageIcon';

client/src/hooks/Input/index.ts

@@ -2,3 +2,4 @@ export { default as useUserKey } from './useUserKey';
 export { default as useDebounce } from './useDebounce';
 export { default as useTextarea } from './useTextarea';
 export { default as useRequiresKey } from './useRequiresKey';
+export { default as useMultipleKeys } from './useMultipleKeys';

client/src/hooks/Input/useMultipleKeys.ts

@@ -0,0 +1,24 @@
+import { isJson } from '~/utils/json';
+
+export default function useMultipleKeys(setUserKey: React.Dispatch<React.SetStateAction<string>>) {
+  function getMultiKey(name: string, userKey: string) {
+    if (isJson(userKey)) {
+      const newKey = JSON.parse(userKey);
+      return newKey[name];
+    } else {
+      return '';
+    }
+  }
+
+  function setMultiKey(name: string, value: number | string | boolean, userKey: string) {
+    let newKey = {};
+    if (isJson(userKey)) {
+      newKey = JSON.parse(userKey);
+    }
+    newKey[name] = value;
+
+    setUserKey(JSON.stringify(newKey));
+  }
+
+  return { getMultiKey, setMultiKey };
+}

client/src/hooks/Input/useUserKey.ts

@@ -1,4 +1,5 @@
 import { useMemo, useCallback } from 'react';
+import { EModelEndpoint } from 'librechat-data-provider';
 import {
   useUserKeyQuery,
   useGetEndpointsQuery,
@@ -10,16 +11,16 @@ const useUserKey = (endpoint: string) => {
   const config = endpointsConfig?.[endpoint];
 
   const { azure } = config ?? {};
-  let keyEndpoint = endpoint;
+  let keyName = endpoint;
 
   if (azure) {
-    keyEndpoint = 'azureOpenAI';
+    keyName = EModelEndpoint.azureOpenAI;
-  } else if (keyEndpoint === 'gptPlugins') {
+  } else if (keyName === EModelEndpoint.gptPlugins) {
-    keyEndpoint = 'openAI';
+    keyName = EModelEndpoint.openAI;
   }
 
   const updateKey = useUpdateUserKeysMutation();
-  const checkUserKey = useUserKeyQuery(keyEndpoint);
+  const checkUserKey = useUserKeyQuery(keyName);
   const getExpiry = useCallback(() => {
     if (checkUserKey.data) {
       return checkUserKey.data.expiresAt;
@@ -40,15 +41,15 @@ const useUserKey = (endpoint: string) => {
   }, [getExpiry]);
 
   const saveUserKey = useCallback(
-    (value: string, expiresAt: number) => {
+    (userKey: string, expiresAt: number) => {
       const dateStr = new Date(expiresAt).toISOString();
       updateKey.mutate({
-        name: keyEndpoint,
+        name: keyName,
-        value,
+        value: userKey,
         expiresAt: dateStr,
       });
     },
-    [updateKey, keyEndpoint],
+    [updateKey, keyName],
   );
 
   return useMemo(

client/src/localization/languages/Eng.tsx

@@ -227,6 +227,12 @@ export default {
   com_endpoint_config_key_name_placeholder: 'Set API key first',
   com_endpoint_config_key_encryption: 'Your key will be encrypted and deleted at',
   com_endpoint_config_key_expiry: 'the expiry time',
+  com_endpoint_config_click_here: 'Click Here',
+  com_endpoint_config_google_service_key: 'Google Service Account Key',
+  com_endpoint_config_google_cloud_platform: '(from Google Cloud Platform)',
+  com_endpoint_config_google_api_key: 'Google API Key',
+  com_endpoint_config_google_gemini_api: '(Gemini API)',
+  com_endpoint_config_google_api_info: 'To get your Generative Language API key (for Gemini),',
   com_endpoint_config_key_import_json_key: 'Import Service Account JSON Key.',
   com_endpoint_config_key_import_json_key_success: 'Successfully Imported Service Account JSON Key',
   com_endpoint_config_key_import_json_key_invalid:

client/src/mobile.css

@@ -121,3 +121,12 @@
 .hide-scrollbar::-webkit-scrollbar {
   display: none; /* For WebKit browsers */
 }
+
+.gemini-gradient {
+  /* Adjust the colors and positioning as necessary to match the image */
+  background-image: radial-gradient(circle at center, #0000ff, #87cefa, #ffffff);
+  /* More styling for demonstration purposes */
+  border-radius: 50%;
+  height: 100px;
+  width: 100px;
+}

client/src/store/models.ts

@@ -1,10 +1,13 @@
 import { atom } from 'recoil';
-import { TModelsConfig, EModelEndpoint, openAIModels } from 'librechat-data-provider';
+import { EModelEndpoint, defaultModels } from 'librechat-data-provider';
+import type { TModelsConfig } from 'librechat-data-provider';
 
 const fitlerAssistantModels = (str: string) => {
   return /gpt-4|gpt-3\\.5/i.test(str) && !/vision|instruct/i.test(str);
 };
 
+const openAIModels = defaultModels[EModelEndpoint.openAI];
+
 const modelsConfig = atom<TModelsConfig>({
   key: 'models',
   default: {
@@ -14,14 +17,8 @@ const modelsConfig = atom<TModelsConfig>({
     [EModelEndpoint.azureOpenAI]: openAIModels,
     [EModelEndpoint.bingAI]: ['BingAI', 'Sydney'],
     [EModelEndpoint.chatGPTBrowser]: ['text-davinci-002-render-sha'],
-    [EModelEndpoint.google]: ['chat-bison', 'text-bison', 'codechat-bison'],
+    [EModelEndpoint.google]: defaultModels[EModelEndpoint.google],
-    [EModelEndpoint.anthropic]: [
+    [EModelEndpoint.anthropic]: defaultModels[EModelEndpoint.anthropic],
-      'claude-1',
-      'claude-1-100k',
-      'claude-instant-1',
-      'claude-instant-1-100k',
-      'claude-2',
-    ],
   },
 });
 

client/src/utils/json.ts

@@ -1,3 +1,12 @@
+export function isJson(str: string) {
+  try {
+    JSON.parse(str);
+  } catch (e) {
+    return false;
+  }
+  return true;
+}
+
 export function formatJSON(json: string) {
   try {
     return JSON.stringify(JSON.parse(json), null, 2);

docs/deployment/digitalocean.md

@@ -303,14 +303,14 @@ cd LibreChat/
 ``` 
 
 ### **2. Create a global environment file.**
-The default values are enough to get you started and running the app.
+The default values are enough to get you started and running the app, allowing you to provide your credentials from the web app.
 
 ```bash
 # Copies the example file as your global env file
 cp .env.example .env
 ```
 
-However, it's highly recommended you use environment variables for any sensitive credentials until we remove use of localStorage for passing credentials from the frontend
+However, if you'd like to provide any credentials for all users of your instance to consume, you can add them to the .env file as follows:
 
 ```bash
 nano .env

docs/features/logging_system.md

@@ -1,4 +1,4 @@
-### Logging
+### General
 
 LibreChat has central logging built into its backend (api).
 
@@ -13,6 +13,8 @@ Errors logs are also saved in the same location: `./api/logs/error-%DATE%.log`.
 > Note: Logs are rotated on a 14-day basis, so you will generate 1 error log file, 1 debug log file, and 1 meiliSync log file per 14 days.
 > Errors will also be present in debug log files as well, but provide stack traces and more detail in the error log files.
 
+### Setup
+
 Keep debug logs enabled with the following environment variable. Even if you never set this variable, debug logs will be generated, but you have the option to disable them by setting it to `FALSE`.
 
 ```bash

docs/install/apis_and_tokens.md

@@ -2,6 +2,20 @@
 
 This doc explains how to setup various tokens and APIs for the project. You will need some of these tokens and APIs to run the app and use its features. You must set up at least one of these tokens or APIs to run the app.
 
+### Docker notes
+
+**If you use docker, you should rebuild the docker image each time you update your credentials**
+
+Rebuild command:
+```bash
+npm run update:docker
+
+# OR, if you don't have npm
+docker-compose build --no-cache
+```
+
+Alternatively, you can create a new file named `docker-compose.override.yml` in the same directory as your main `docker-compose.yml` file for LibreChat, where you can set your .env variables as needed under `environment`. See the [docker docs](https://docs.docker.com/compose/multiple-compose-files/extends/#understanding-multiple-compose-files) for more info, and you can also view an example of an override file for LibreChat in the ["Manage Your Database" section](https://docs.librechat.ai/features/manage_your_database.html)
+
 ## OpenAI API key
 
 To get your OpenAI API key, you need to:
@@ -46,16 +60,57 @@ To get your Bing Access Token, you have a few options:
 - Go to [https://console.anthropic.com/account/keys](https://console.anthropic.com/account/keys) and get your api key
 - add it to `ANTHROPIC_API_KEY=` in the `.env` file
 
-## Google LLMs
+## Google
+
+For the Google Endpoint, you can either use the **Generative Language API** (for Gemini models), or the **Vertex AI API** (for PaLM2 & Codey models, Gemini support coming soon).
+
+The Generative Language API uses an API key, which you can get from **Google AI Studio**.
+
+For Vertex AI, you need a Service Account JSON key file, with appropriate access configured.
+
+Instructions for both are given below.
+
+Setting `GOOGLE_KEY=user_provided` in your .env file will configure both values to be provided from the client (or frontend) like so:
+
+![image](https://github.com/danny-avila/LibreChat/assets/110412045/728cbc04-4180-45a8-848c-ae5de2b02996)
+
+### Generative Language API (Gemini)
+
+**60 Gemini requests/minute are currently free until early next year when it enters general availability.**
+
+⚠️ Google will be using that free input/output to help improve the model, with data de-identified from your Google Account and API key.
+⚠️ During this period, your messages “may be accessible to trained reviewers.”
+
+To use Gemini models, you'll need an API key. If you don't already have one, create a key in Google AI Studio.
+
+<p><a class="button button-primary" href="https://makersuite.google.com/app/apikey" target="_blank" rel="noopener noreferrer">Get an API key here</a></p>
+
+Once you have your key, you can either provide it from the frontend by setting the following:
+
+```bash
+GOOGLE_KEY=user_provided
+```
+
+Or, provide the key in your .env file, which allows all users of your instance to use it.
+
+```bash
+GOOGLE_KEY=mY_SeCreT_w9347w8_kEY
+```
+
+> Notes:
+> - As of 12/15/23, Gemini Pro Vision is not yet supported but is planned.
+> - PaLM2 and Codey models cannot be accessed through the Generative Language API.
+
+### Vertex AI (PaLM 2 & Codey)
 
 To setup Google LLMs (via Google Cloud Vertex AI), first, signup for Google Cloud: https://cloud.google.com/
 
-You can usually get $300 starting credit, which makes this option free for 90 days. 
+You can usually get **$300 starting credit**, which makes this option free for 90 days.
 
-### Once signed up, Enable the Vertex AI API on Google Cloud:
+### 1. Once signed up, Enable the Vertex AI API on Google Cloud:
   - Go to [Vertex AI page on Google Cloud console](https://console.cloud.google.com/vertex-ai)
   - Click on "Enable API" if prompted
-### Create a Service Account with Vertex AI role:
+### 2. Create a Service Account with Vertex AI role:
   - **[Click here to create a Service Account](https://console.cloud.google.com/projectselector/iam-admin/serviceaccounts/create?walkthrough_id=iam--create-service-account#step_index=1)**
   - **Select or create a project**
   - ### Enter a service account ID (required), name and description are optional
@@ -63,7 +118,7 @@ You can usually get $300 starting credit, which makes this option free for 90 da
   - ### Click on "Create and Continue" to give at least the "Vertex AI User" role
       - ![image](https://github.com/danny-avila/LibreChat/assets/110412045/22d3a080-e71e-446e-8485-bcc5bf558dbb)
   - **Click on "Continue/Done"**
-### Create a JSON key to Save in Project Directory:
+### 3. Create a JSON key to Save in your Project Directory:
   - **Go back to [the Service Accounts page](https://console.cloud.google.com/projectselector/iam-admin/serviceaccounts)**
   - **Select your service account**
   - ### Click on "Keys"
@@ -75,6 +130,16 @@ You can usually get $300 starting credit, which makes this option free for 90 da
   - **Save it within the project directory, in `/api/data/`**
        - ![image](https://github.com/danny-avila/LibreChat/assets/110412045/f5b8bcb5-1b20-4751-81a1-d3757a4b3f2f)
 
+**Saving your JSON key file in the project directory which allows all users of your LibreChat instance to use it.**
+
+Alternatively, Once you have your JSON key file, you can also provide it from the frontend on a user-basis by setting the following:
+
+```bash
+GOOGLE_KEY=user_provided
+```
+
+> Notes:
+> - As of 12/15/23, Gemini and Gemini Pro Vision are not yet supported through Vertex AI but are planned.
 
 ## Azure OpenAI
 

package-lock.json

@@ -45,6 +45,7 @@
         "@azure/search-documents": "^12.0.0",
         "@keyv/mongo": "^2.1.8",
         "@keyv/redis": "^2.8.0",
+        "@langchain/google-genai": "^0.0.2",
         "axios": "^1.3.4",
         "bcryptjs": "^2.4.3",
         "cheerio": "^1.0.0-rc.12",
@@ -4832,6 +4833,14 @@
       "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.1.2.tgz",
       "integrity": "sha512-ou3elfqG/hZsbmF4bxeJhPHIf3G2pm0ujc39hYEZrfVqt7Vk/Zji6CXc3W0pmYM8BW1g40U+akTl9DKZhFhInQ=="
     },
+    "node_modules/@google/generative-ai": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/@google/generative-ai/-/generative-ai-0.1.1.tgz",
+      "integrity": "sha512-cbzKa8mT9YkTrT4XUuENIuvlqiJjwDgcD2Ks4L99Az9dWLgdXn8xnETEAZLOpqzoGx+1PuATZqlUnVRAeLbMgA==",
+      "engines": {
+        "node": ">=18.0.0"
+      }
+    },
     "node_modules/@headlessui/react": {
       "version": "1.7.17",
       "resolved": "https://registry.npmjs.org/@headlessui/react/-/react-1.7.17.tgz",
@@ -5465,6 +5474,72 @@
         "node": ">= 14"
       }
     },
+    "node_modules/@langchain/core": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/@langchain/core/-/core-0.1.1.tgz",
+      "integrity": "sha512-p+BzmILzGAWjHxzNy1ZygfsNWxZcipWH3zvY0WC33ly+nmBp4n+MCVd25hKQ4YYWOqoOGGWSzFJukd5PxaXMDQ==",
+      "dependencies": {
+        "ansi-styles": "^5.0.0",
+        "camelcase": "6",
+        "decamelize": "1.2.0",
+        "js-tiktoken": "^1.0.8",
+        "langsmith": "~0.0.48",
+        "ml-distance": "^4.0.0",
+        "p-queue": "^6.6.2",
+        "p-retry": "4",
+        "uuid": "^9.0.0",
+        "zod": "^3.22.3"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/@langchain/core/node_modules/ansi-styles": {
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz",
+      "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==",
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+      }
+    },
+    "node_modules/@langchain/core/node_modules/camelcase": {
+      "version": "6.3.0",
+      "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz",
+      "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==",
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/@langchain/core/node_modules/uuid": {
+      "version": "9.0.1",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz",
+      "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==",
+      "funding": [
+        "https://github.com/sponsors/broofa",
+        "https://github.com/sponsors/ctavan"
+      ],
+      "bin": {
+        "uuid": "dist/bin/uuid"
+      }
+    },
+    "node_modules/@langchain/google-genai": {
+      "version": "0.0.2",
+      "resolved": "https://registry.npmjs.org/@langchain/google-genai/-/google-genai-0.0.2.tgz",
+      "integrity": "sha512-Q6zgVeZ0IzD976LGhwl86RwyTn6zpdwltVTYGEEag3AyT3zDzALPiyEfORFxublQjIVeIoTiyDJ9MT9nXb1xwg==",
+      "dependencies": {
+        "@google/generative-ai": "^0.1.0",
+        "@langchain/core": "~0.1.0"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/@librechat/backend": {
       "resolved": "api",
       "link": true
@@ -15805,9 +15880,9 @@
       }
     },
     "node_modules/js-tiktoken": {
-      "version": "1.0.7",
+      "version": "1.0.8",
-      "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.7.tgz",
+      "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.8.tgz",
-      "integrity": "sha512-biba8u/clw7iesNEWLOLwrNGoBP2lA+hTaBLs/D45pJdUPFXyxD6nhcDVtADChghv4GgyAiMKYMiRx7x6h7Biw==",
+      "integrity": "sha512-r7XK3E9/I+SOrbAGqb39pyO/rHAS1diAOSRAvaaLfHgXjkUSK9AiSd+r84Vn2f/GvXJYRAxKj8NHrUvqlaH5qg==",
       "dependencies": {
         "base64-js": "^1.5.1"
       }
@@ -25370,7 +25445,7 @@
     },
     "packages/data-provider": {
       "name": "librechat-data-provider",
-      "version": "0.3.0",
+      "version": "0.3.1",
       "license": "ISC",
       "dependencies": {
         "axios": "^1.3.4",

packages/data-provider/src/schemas.ts

@@ -22,6 +22,47 @@ export const defaultEndpoints: EModelEndpoint[] = [
   EModelEndpoint.anthropic,
 ];
 
+export const defaultModels = {
+  [EModelEndpoint.google]: [
+    'gemini-pro',
+    'chat-bison',
+    'chat-bison-32k',
+    'codechat-bison',
+    'codechat-bison-32k',
+    'text-bison',
+    'text-bison-32k',
+    'text-unicorn',
+    'code-gecko',
+    'code-bison',
+    'code-bison-32k',
+  ],
+  [EModelEndpoint.anthropic]: [
+    'claude-2.1',
+    'claude-2',
+    'claude-1.2',
+    'claude-1',
+    'claude-1-100k',
+    'claude-instant-1',
+    'claude-instant-1-100k',
+  ],
+  [EModelEndpoint.openAI]: [
+    'gpt-3.5-turbo-16k-0613',
+    'gpt-3.5-turbo-16k',
+    'gpt-4-1106-preview',
+    'gpt-3.5-turbo',
+    'gpt-3.5-turbo-1106',
+    'gpt-4-vision-preview',
+    'gpt-4',
+    'gpt-3.5-turbo-instruct-0914',
+    'gpt-3.5-turbo-0613',
+    'gpt-3.5-turbo-0301',
+    'gpt-3.5-turbo-instruct',
+    'gpt-4-0613',
+    'text-davinci-003',
+    'gpt-4-0314',
+  ],
+};
+
 export const alternateName = {
   [EModelEndpoint.openAI]: 'OpenAI',
   [EModelEndpoint.assistant]: 'Assistants',
@@ -33,6 +74,11 @@ export const alternateName = {
   [EModelEndpoint.anthropic]: 'Anthropic',
 };
 
+export enum AuthKeys {
+  GOOGLE_SERVICE_KEY = 'GOOGLE_SERVICE_KEY',
+  GOOGLE_API_KEY = 'GOOGLE_API_KEY',
+}
+
 export const endpointSettings = {
   [EModelEndpoint.google]: {
     model: {
@@ -43,6 +89,8 @@ export const endpointSettings = {
       max: 2048,
       step: 1,
       default: 1024,
+      maxGeminiPro: 8192,
+      defaultGeminiPro: 8192,
     },
     temperature: {
       min: 0,
@@ -90,23 +138,6 @@ export const supportsFiles = {
   [EModelEndpoint.assistant]: true,
 };
 
-export const openAIModels = [
-  'gpt-3.5-turbo-16k-0613',
-  'gpt-3.5-turbo-16k',
-  'gpt-4-1106-preview',
-  'gpt-3.5-turbo',
-  'gpt-3.5-turbo-1106',
-  'gpt-4-vision-preview',
-  'gpt-4',
-  'gpt-3.5-turbo-instruct-0914',
-  'gpt-3.5-turbo-0613',
-  'gpt-3.5-turbo-0301',
-  'gpt-3.5-turbo-instruct',
-  'gpt-4-0613',
-  'text-davinci-003',
-  'gpt-4-0314',
-];
-
 export const visionModels = ['gpt-4-vision', 'llava-13b'];
 
 export const eModelEndpointSchema = z.nativeEnum(EModelEndpoint);
@@ -309,17 +340,31 @@ export const googleSchema = tConversationSchema
     topP: true,
     topK: true,
   })
-  .transform((obj) => ({
+  .transform((obj) => {
-    ...obj,
+    const isGeminiPro = obj?.model?.toLowerCase()?.includes('gemini-pro');
-    model: obj.model ?? google.model.default,
+
-    modelLabel: obj.modelLabel ?? null,
+    const maxOutputTokensMax = isGeminiPro
-    promptPrefix: obj.promptPrefix ?? null,
+      ? google.maxOutputTokens.maxGeminiPro
-    examples: obj.examples ?? [{ input: { content: '' }, output: { content: '' } }],
+      : google.maxOutputTokens.max;
-    temperature: obj.temperature ?? google.temperature.default,
+    const maxOutputTokensDefault = isGeminiPro
-    maxOutputTokens: obj.maxOutputTokens ?? google.maxOutputTokens.default,
+      ? google.maxOutputTokens.defaultGeminiPro
-    topP: obj.topP ?? google.topP.default,
+      : google.maxOutputTokens.default;
-    topK: obj.topK ?? google.topK.default,
+
-  }))
+    let maxOutputTokens = obj.maxOutputTokens ?? maxOutputTokensDefault;
+    maxOutputTokens = Math.min(maxOutputTokens, maxOutputTokensMax);
+
+    return {
+      ...obj,
+      model: obj.model ?? google.model.default,
+      modelLabel: obj.modelLabel ?? null,
+      promptPrefix: obj.promptPrefix ?? null,
+      examples: obj.examples ?? [{ input: { content: '' }, output: { content: '' } }],
+      temperature: obj.temperature ?? google.temperature.default,
+      maxOutputTokens,
+      topP: obj.topP ?? google.topP.default,
+      topK: obj.topK ?? google.topK.default,
+    };
+  })
   .catch(() => ({
     model: google.model.default,
     modelLabel: null,
@@ -579,6 +624,8 @@ export const getResponseSender = (endpointOption: TEndpointOption): string => {
   if (endpoint === EModelEndpoint.google) {
     if (modelLabel) {
       return modelLabel;
+    } else if (model && model.includes('gemini')) {
+      return 'Gemini';
     } else if (model && model.includes('code')) {
       return 'Codey';
     }