🔑 fix: Gemini Custom Endpoint Auth. for OAI-Compatible API (#10806)

* 🔧 fix: Gemini as Custom Endpoint Auth. Error for OAI-compatible API

* refactor: Google Compatibility in OpenAI Config

- Added a test to ensure `googleSearch` is filtered out when `web_search` is only present in `modelOptions`, not in `addParams` or `defaultParams`.
- Updated `transformToOpenAIConfig` to preserve `googleSearch` tools if `web_search` is explicitly enabled via `addParams` or `defaultParams`.
- Refactored the filtering logic for Google-specific tools to accommodate the new behavior.

packages/api/src/endpoints/google/llm.ts

@@ -121,9 +121,12 @@ export function getSafetySettings(
 export function getGoogleConfig(
   credentials: string | t.GoogleCredentials | undefined,
   options: t.GoogleConfigOptions = {},
+  acceptRawApiKey = false,
 ) {
   let creds: t.GoogleCredentials = {};
-  if (typeof credentials === 'string') {
+  if (acceptRawApiKey && typeof credentials === 'string') {
+    creds[AuthKeys.GOOGLE_API_KEY] = credentials;
+  } else if (typeof credentials === 'string') {
     try {
       creds = JSON.parse(credentials);
     } catch (err: unknown) {

packages/api/src/endpoints/openai/config.google.spec.ts

@@ -69,6 +69,26 @@ describe('getOpenAIConfig - Google Compatibility', () => {
         expect(result.tools).toEqual([]);
       });
 
+      it('should filter out googleSearch when web_search is only in modelOptions (not explicitly in addParams/defaultParams)', () => {
+        const apiKey = JSON.stringify({ GOOGLE_API_KEY: 'test-google-key' });
+        const endpoint = 'Gemini (Custom)';
+        const options = {
+          modelOptions: {
+            model: 'gemini-2.0-flash-exp',
+            web_search: true,
+          },
+          customParams: {
+            defaultParamsEndpoint: 'google',
+          },
+          reverseProxyUrl: 'https://generativelanguage.googleapis.com/v1beta/openai',
+        };
+
+        const result = getOpenAIConfig(apiKey, options, endpoint);
+
+        /** googleSearch should be filtered out since web_search was not explicitly added via addParams or defaultParams */
+        expect(result.tools).toEqual([]);
+      });
+
       it('should handle web_search with mixed Google and OpenAI params in addParams', () => {
         const apiKey = JSON.stringify({ GOOGLE_API_KEY: 'test-google-key' });
         const endpoint = 'Gemini (Custom)';

packages/api/src/endpoints/openai/config.ts

@@ -77,23 +77,29 @@ export function getOpenAIConfig(
       headers = Object.assign(headers ?? {}, transformed.configOptions?.defaultHeaders);
     }
   } else if (isGoogle) {
-    const googleResult = getGoogleConfig(apiKey, {
+    const googleResult = getGoogleConfig(
-      modelOptions,
+      apiKey,
-      reverseProxyUrl: baseURL ?? undefined,
+      {
-      authHeader: true,
+        modelOptions,
-      addParams,
+        reverseProxyUrl: baseURL ?? undefined,
-      dropParams,
+        authHeader: true,
-      defaultParams,
+        addParams,
-    });
+        dropParams,
+        defaultParams,
+      },
+      true,
+    );
     /** Transform handles addParams/dropParams - it knows about OpenAI params */
     const transformed = transformToOpenAIConfig({
       addParams,
       dropParams,
+      defaultParams,
+      tools: googleResult.tools,
       llmConfig: googleResult.llmConfig,
       fromEndpoint: EModelEndpoint.google,
     });
     llmConfig = transformed.llmConfig;
-    tools = googleResult.tools;
+    tools = transformed.tools;
   } else {
     const openaiResult = getOpenAILLMConfig({
       azure,

packages/api/src/endpoints/openai/transform.ts

@@ -1,28 +1,48 @@
 import { EModelEndpoint } from 'librechat-data-provider';
+import type { GoogleAIToolType } from '@langchain/google-common';
 import type { ClientOptions } from '@librechat/agents';
 import type * as t from '~/types';
 import { knownOpenAIParams } from './llm';
 
 const anthropicExcludeParams = new Set(['anthropicApiUrl']);
-const googleExcludeParams = new Set(['safetySettings', 'location', 'baseUrl', 'customHeaders']);
+const googleExcludeParams = new Set([
+  'safetySettings',
+  'location',
+  'baseUrl',
+  'customHeaders',
+  'thinkingConfig',
+  'thinkingBudget',
+  'includeThoughts',
+]);
+
+/** Google-specific tool types that have no OpenAI-compatible equivalent */
+const googleToolsToFilter = new Set(['googleSearch']);
+
+export type ConfigTools = Array<Record<string, unknown>> | Array<GoogleAIToolType>;
 
 /**
  * Transforms a Non-OpenAI LLM config to an OpenAI-conformant config.
  * Non-OpenAI parameters are moved to modelKwargs.
  * Also extracts configuration options that belong in configOptions.
  * Handles addParams and dropParams for parameter customization.
+ * Filters out provider-specific tools that have no OpenAI equivalent.
  */
 export function transformToOpenAIConfig({
+  tools,
   addParams,
   dropParams,
+  defaultParams,
   llmConfig,
   fromEndpoint,
 }: {
+  tools?: ConfigTools;
   addParams?: Record<string, unknown>;
   dropParams?: string[];
+  defaultParams?: Record<string, unknown>;
   llmConfig: ClientOptions;
   fromEndpoint: string;
 }): {
+  tools: ConfigTools;
   llmConfig: t.OAIClientOptions;
   configOptions: Partial<t.OpenAIConfiguration>;
 } {
@@ -58,18 +78,9 @@ export function transformToOpenAIConfig({
       hasModelKwargs = true;
       continue;
     } else if (isGoogle && key === 'authOptions') {
-      // Handle Google authOptions
       modelKwargs = Object.assign({}, modelKwargs, value as Record<string, unknown>);
       hasModelKwargs = true;
       continue;
-    } else if (
-      isGoogle &&
-      (key === 'thinkingConfig' || key === 'thinkingBudget' || key === 'includeThoughts')
-    ) {
-      // Handle Google thinking configuration
-      modelKwargs = Object.assign({}, modelKwargs, { [key]: value });
-      hasModelKwargs = true;
-      continue;
     }
 
     if (knownOpenAIParams.has(key)) {
@@ -121,7 +132,34 @@ export function transformToOpenAIConfig({
     }
   }
 
+  /**
+   * Filter out provider-specific tools that have no OpenAI equivalent.
+   * Exception: If web_search was explicitly enabled via addParams or defaultParams,
+   * preserve googleSearch tools (pass through in Google-native format).
+   */
+  const webSearchExplicitlyEnabled =
+    addParams?.web_search === true || defaultParams?.web_search === true;
+
+  const filterGoogleTool = (tool: unknown): boolean => {
+    if (!isGoogle) {
+      return true;
+    }
+    if (typeof tool !== 'object' || tool === null) {
+      return false;
+    }
+    const toolKeys = Object.keys(tool as Record<string, unknown>);
+    const isGoogleSpecificTool = toolKeys.some((key) => googleToolsToFilter.has(key));
+    /** Preserve googleSearch if web_search was explicitly enabled */
+    if (isGoogleSpecificTool && webSearchExplicitlyEnabled) {
+      return true;
+    }
+    return !isGoogleSpecificTool;
+  };
+
+  const filteredTools = Array.isArray(tools) ? tools.filter(filterGoogleTool) : [];
+
   return {
+    tools: filteredTools,
     llmConfig: openAIConfig as t.OAIClientOptions,
     configOptions,
   };