diff --git a/api/server/controllers/agents/request.js b/api/server/controllers/agents/request.js index 110d2fdd57..faf3905349 100644 --- a/api/server/controllers/agents/request.js +++ b/api/server/controllers/agents/request.js @@ -1,6 +1,10 @@ -const { sendEvent } = require('@librechat/api'); const { logger } = require('@librechat/data-schemas'); const { Constants } = require('librechat-data-provider'); +const { + sendEvent, + sanitizeFileForTransmit, + sanitizeMessageForTransmit, +} = require('@librechat/api'); const { handleAbortError, createAbortController, @@ -224,13 +228,13 @@ const AgentController = async (req, res, next, initializeClient, addTitle) => { conversation.title = conversation && !conversation.title ? null : conversation?.title || 'New Chat'; - // Process files if needed + // Process files if needed (sanitize to remove large text fields before transmission) if (req.body.files && client.options?.attachments) { userMessage.files = []; const messageFiles = new Set(req.body.files.map((file) => file.file_id)); - for (let attachment of client.options.attachments) { + for (const attachment of client.options.attachments) { if (messageFiles.has(attachment.file_id)) { - userMessage.files.push({ ...attachment }); + userMessage.files.push(sanitizeFileForTransmit(attachment)); } } delete userMessage.image_urls; @@ -245,7 +249,7 @@ const AgentController = async (req, res, next, initializeClient, addTitle) => { final: true, conversation, title: conversation.title, - requestMessage: userMessage, + requestMessage: sanitizeMessageForTransmit(userMessage), responseMessage: finalResponse, }); res.end(); @@ -273,7 +277,7 @@ const AgentController = async (req, res, next, initializeClient, addTitle) => { final: true, conversation, title: conversation.title, - requestMessage: userMessage, + requestMessage: sanitizeMessageForTransmit(userMessage), responseMessage: finalResponse, error: { message: 'Request was aborted during completion' }, }); diff --git a/api/server/middleware/abortMiddleware.js b/api/server/middleware/abortMiddleware.js index 9ceff1bc02..1f762ca808 100644 --- a/api/server/middleware/abortMiddleware.js +++ b/api/server/middleware/abortMiddleware.js @@ -1,5 +1,5 @@ const { logger } = require('@librechat/data-schemas'); -const { countTokens, isEnabled, sendEvent } = require('@librechat/api'); +const { countTokens, isEnabled, sendEvent, sanitizeMessageForTransmit } = require('@librechat/api'); const { isAssistantsEndpoint, ErrorTypes, Constants } = require('librechat-data-provider'); const { truncateText, smartTruncateText } = require('~/app/clients/prompts'); const clearPendingReq = require('~/cache/clearPendingReq'); @@ -290,7 +290,7 @@ const createAbortController = (req, res, getAbortData, getReqData) => { title: conversation && !conversation.title ? null : conversation?.title || 'New Chat', final: true, conversation, - requestMessage: userMessage, + requestMessage: sanitizeMessageForTransmit(userMessage), responseMessage: responseMessage, }; }; diff --git a/api/server/middleware/error.js b/api/server/middleware/error.js index 270663a46c..fef7e60ef7 100644 --- a/api/server/middleware/error.js +++ b/api/server/middleware/error.js @@ -1,7 +1,7 @@ const crypto = require('crypto'); const { logger } = require('@librechat/data-schemas'); const { parseConvo } = require('librechat-data-provider'); -const { sendEvent, handleError } = require('@librechat/api'); +const { sendEvent, handleError, sanitizeMessageForTransmit } = require('@librechat/api'); const { saveMessage, getMessages } = require('~/models/Message'); const { getConvo } = require('~/models/Conversation'); @@ -71,7 +71,7 @@ const sendError = async (req, res, options, callback) => { return sendEvent(res, { final: true, - requestMessage: query?.[0] ? query[0] : requestMessage, + requestMessage: sanitizeMessageForTransmit(query?.[0] ?? requestMessage), responseMessage: errorMessage, conversation: convo, }); diff --git a/client/src/components/Chat/Input/Files/DragDropModal.tsx b/client/src/components/Chat/Input/Files/DragDropModal.tsx index 015a590d55..eb5f86d3b9 100644 --- a/client/src/components/Chat/Input/Files/DragDropModal.tsx +++ b/client/src/components/Chat/Input/Files/DragDropModal.tsx @@ -2,6 +2,7 @@ import React, { useMemo } from 'react'; import { useRecoilValue } from 'recoil'; import { OGDialog, OGDialogTemplate } from '@librechat/client'; import { + inferMimeType, EToolResources, EModelEndpoint, defaultAgentCapabilities, @@ -56,18 +57,26 @@ const DragDropModal = ({ onOptionSelect, setShowModal, files, isVisible }: DragD const _options: FileOption[] = []; const currentProvider = provider || endpoint; + /** Helper to get inferred MIME type for a file */ + const getFileType = (file: File) => inferMimeType(file.name, file.type); + // Check if provider supports document upload if (isDocumentSupportedProvider(endpointType) || isDocumentSupportedProvider(currentProvider)) { const isGoogleProvider = currentProvider === EModelEndpoint.google; const validFileTypes = isGoogleProvider - ? files.every( - (file) => - file.type?.startsWith('image/') || - file.type?.startsWith('video/') || - file.type?.startsWith('audio/') || - file.type === 'application/pdf', - ) - : files.every((file) => file.type?.startsWith('image/') || file.type === 'application/pdf'); + ? files.every((file) => { + const type = getFileType(file); + return ( + type?.startsWith('image/') || + type?.startsWith('video/') || + type?.startsWith('audio/') || + type === 'application/pdf' + ); + }) + : files.every((file) => { + const type = getFileType(file); + return type?.startsWith('image/') || type === 'application/pdf'; + }); _options.push({ label: localize('com_ui_upload_provider'), @@ -81,7 +90,7 @@ const DragDropModal = ({ onOptionSelect, setShowModal, files, isVisible }: DragD label: localize('com_ui_upload_image_input'), value: undefined, icon: , - condition: files.every((file) => file.type?.startsWith('image/')), + condition: files.every((file) => getFileType(file)?.startsWith('image/')), }); } if (capabilities.fileSearchEnabled && fileSearchAllowedByAgent) { diff --git a/client/src/components/Chat/Input/Files/__tests__/DragDropModal.spec.tsx b/client/src/components/Chat/Input/Files/__tests__/DragDropModal.spec.tsx index 2adad63b9a..44e632fa12 100644 --- a/client/src/components/Chat/Input/Files/__tests__/DragDropModal.spec.tsx +++ b/client/src/components/Chat/Input/Files/__tests__/DragDropModal.spec.tsx @@ -1,4 +1,8 @@ -import { EModelEndpoint, isDocumentSupportedProvider } from 'librechat-data-provider'; +import { + EModelEndpoint, + isDocumentSupportedProvider, + inferMimeType, +} from 'librechat-data-provider'; describe('DragDropModal - Provider Detection', () => { describe('endpointType priority over currentProvider', () => { @@ -118,4 +122,59 @@ describe('DragDropModal - Provider Detection', () => { ).toBe(true); }); }); + + describe('HEIC/HEIF file type inference', () => { + it('should infer image/heic for .heic files when browser returns empty type', () => { + const fileName = 'photo.heic'; + const browserType = ''; + + const inferredType = inferMimeType(fileName, browserType); + expect(inferredType).toBe('image/heic'); + }); + + it('should infer image/heif for .heif files when browser returns empty type', () => { + const fileName = 'photo.heif'; + const browserType = ''; + + const inferredType = inferMimeType(fileName, browserType); + expect(inferredType).toBe('image/heif'); + }); + + it('should handle uppercase .HEIC extension', () => { + const fileName = 'IMG_1234.HEIC'; + const browserType = ''; + + const inferredType = inferMimeType(fileName, browserType); + expect(inferredType).toBe('image/heic'); + }); + + it('should preserve browser-provided type when available', () => { + const fileName = 'photo.jpg'; + const browserType = 'image/jpeg'; + + const inferredType = inferMimeType(fileName, browserType); + expect(inferredType).toBe('image/jpeg'); + }); + + it('should not override browser type even if extension differs', () => { + const fileName = 'renamed.heic'; + const browserType = 'image/png'; + + const inferredType = inferMimeType(fileName, browserType); + expect(inferredType).toBe('image/png'); + }); + + it('should correctly identify HEIC as image type for upload options', () => { + const heicType = inferMimeType('photo.heic', ''); + expect(heicType.startsWith('image/')).toBe(true); + }); + + it('should return empty string for unknown extension with no browser type', () => { + const fileName = 'file.xyz'; + const browserType = ''; + + const inferredType = inferMimeType(fileName, browserType); + expect(inferredType).toBe(''); + }); + }); }); diff --git a/client/src/utils/files.ts b/client/src/utils/files.ts index 93c810e7ce..b4d362d456 100644 --- a/client/src/utils/files.ts +++ b/client/src/utils/files.ts @@ -9,9 +9,9 @@ import { import { megabyte, QueryKeys, + inferMimeType, excelMimeTypes, EToolResources, - codeTypeMapping, fileConfig as defaultFileConfig, } from 'librechat-data-provider'; import type { TFile, EndpointFileConfig, FileConfig } from 'librechat-data-provider'; @@ -257,14 +257,7 @@ export const validateFiles = ({ for (let i = 0; i < fileList.length; i++) { let originalFile = fileList[i]; - let fileType = originalFile.type; - const extension = originalFile.name.split('.').pop() ?? ''; - const knownCodeType = codeTypeMapping[extension]; - - // Infer MIME type for Known Code files when the type is empty or a mismatch - if (knownCodeType && (!fileType || fileType !== knownCodeType)) { - fileType = knownCodeType; - } + const fileType = inferMimeType(originalFile.name, originalFile.type); // Check if the file type is still empty after the extension check if (!fileType) { diff --git a/packages/api/src/endpoints/google/llm.ts b/packages/api/src/endpoints/google/llm.ts index 7934a03c55..b486ae6517 100644 --- a/packages/api/src/endpoints/google/llm.ts +++ b/packages/api/src/endpoints/google/llm.ts @@ -121,9 +121,12 @@ export function getSafetySettings( export function getGoogleConfig( credentials: string | t.GoogleCredentials | undefined, options: t.GoogleConfigOptions = {}, + acceptRawApiKey = false, ) { let creds: t.GoogleCredentials = {}; - if (typeof credentials === 'string') { + if (acceptRawApiKey && typeof credentials === 'string') { + creds[AuthKeys.GOOGLE_API_KEY] = credentials; + } else if (typeof credentials === 'string') { try { creds = JSON.parse(credentials); } catch (err: unknown) { diff --git a/packages/api/src/endpoints/openai/config.google.spec.ts b/packages/api/src/endpoints/openai/config.google.spec.ts index 73b133b478..8533672277 100644 --- a/packages/api/src/endpoints/openai/config.google.spec.ts +++ b/packages/api/src/endpoints/openai/config.google.spec.ts @@ -69,6 +69,26 @@ describe('getOpenAIConfig - Google Compatibility', () => { expect(result.tools).toEqual([]); }); + it('should filter out googleSearch when web_search is only in modelOptions (not explicitly in addParams/defaultParams)', () => { + const apiKey = JSON.stringify({ GOOGLE_API_KEY: 'test-google-key' }); + const endpoint = 'Gemini (Custom)'; + const options = { + modelOptions: { + model: 'gemini-2.0-flash-exp', + web_search: true, + }, + customParams: { + defaultParamsEndpoint: 'google', + }, + reverseProxyUrl: 'https://generativelanguage.googleapis.com/v1beta/openai', + }; + + const result = getOpenAIConfig(apiKey, options, endpoint); + + /** googleSearch should be filtered out since web_search was not explicitly added via addParams or defaultParams */ + expect(result.tools).toEqual([]); + }); + it('should handle web_search with mixed Google and OpenAI params in addParams', () => { const apiKey = JSON.stringify({ GOOGLE_API_KEY: 'test-google-key' }); const endpoint = 'Gemini (Custom)'; diff --git a/packages/api/src/endpoints/openai/config.ts b/packages/api/src/endpoints/openai/config.ts index c84d3b07c3..2540bbb815 100644 --- a/packages/api/src/endpoints/openai/config.ts +++ b/packages/api/src/endpoints/openai/config.ts @@ -77,23 +77,29 @@ export function getOpenAIConfig( headers = Object.assign(headers ?? {}, transformed.configOptions?.defaultHeaders); } } else if (isGoogle) { - const googleResult = getGoogleConfig(apiKey, { - modelOptions, - reverseProxyUrl: baseURL ?? undefined, - authHeader: true, - addParams, - dropParams, - defaultParams, - }); + const googleResult = getGoogleConfig( + apiKey, + { + modelOptions, + reverseProxyUrl: baseURL ?? undefined, + authHeader: true, + addParams, + dropParams, + defaultParams, + }, + true, + ); /** Transform handles addParams/dropParams - it knows about OpenAI params */ const transformed = transformToOpenAIConfig({ addParams, dropParams, + defaultParams, + tools: googleResult.tools, llmConfig: googleResult.llmConfig, fromEndpoint: EModelEndpoint.google, }); llmConfig = transformed.llmConfig; - tools = googleResult.tools; + tools = transformed.tools; } else { const openaiResult = getOpenAILLMConfig({ azure, diff --git a/packages/api/src/endpoints/openai/transform.ts b/packages/api/src/endpoints/openai/transform.ts index 27cce5d3eb..c65e2cd6f5 100644 --- a/packages/api/src/endpoints/openai/transform.ts +++ b/packages/api/src/endpoints/openai/transform.ts @@ -1,28 +1,48 @@ import { EModelEndpoint } from 'librechat-data-provider'; +import type { GoogleAIToolType } from '@langchain/google-common'; import type { ClientOptions } from '@librechat/agents'; import type * as t from '~/types'; import { knownOpenAIParams } from './llm'; const anthropicExcludeParams = new Set(['anthropicApiUrl']); -const googleExcludeParams = new Set(['safetySettings', 'location', 'baseUrl', 'customHeaders']); +const googleExcludeParams = new Set([ + 'safetySettings', + 'location', + 'baseUrl', + 'customHeaders', + 'thinkingConfig', + 'thinkingBudget', + 'includeThoughts', +]); + +/** Google-specific tool types that have no OpenAI-compatible equivalent */ +const googleToolsToFilter = new Set(['googleSearch']); + +export type ConfigTools = Array> | Array; /** * Transforms a Non-OpenAI LLM config to an OpenAI-conformant config. * Non-OpenAI parameters are moved to modelKwargs. * Also extracts configuration options that belong in configOptions. * Handles addParams and dropParams for parameter customization. + * Filters out provider-specific tools that have no OpenAI equivalent. */ export function transformToOpenAIConfig({ + tools, addParams, dropParams, + defaultParams, llmConfig, fromEndpoint, }: { + tools?: ConfigTools; addParams?: Record; dropParams?: string[]; + defaultParams?: Record; llmConfig: ClientOptions; fromEndpoint: string; }): { + tools: ConfigTools; llmConfig: t.OAIClientOptions; configOptions: Partial; } { @@ -58,18 +78,9 @@ export function transformToOpenAIConfig({ hasModelKwargs = true; continue; } else if (isGoogle && key === 'authOptions') { - // Handle Google authOptions modelKwargs = Object.assign({}, modelKwargs, value as Record); hasModelKwargs = true; continue; - } else if ( - isGoogle && - (key === 'thinkingConfig' || key === 'thinkingBudget' || key === 'includeThoughts') - ) { - // Handle Google thinking configuration - modelKwargs = Object.assign({}, modelKwargs, { [key]: value }); - hasModelKwargs = true; - continue; } if (knownOpenAIParams.has(key)) { @@ -121,7 +132,34 @@ export function transformToOpenAIConfig({ } } + /** + * Filter out provider-specific tools that have no OpenAI equivalent. + * Exception: If web_search was explicitly enabled via addParams or defaultParams, + * preserve googleSearch tools (pass through in Google-native format). + */ + const webSearchExplicitlyEnabled = + addParams?.web_search === true || defaultParams?.web_search === true; + + const filterGoogleTool = (tool: unknown): boolean => { + if (!isGoogle) { + return true; + } + if (typeof tool !== 'object' || tool === null) { + return false; + } + const toolKeys = Object.keys(tool as Record); + const isGoogleSpecificTool = toolKeys.some((key) => googleToolsToFilter.has(key)); + /** Preserve googleSearch if web_search was explicitly enabled */ + if (isGoogleSpecificTool && webSearchExplicitlyEnabled) { + return true; + } + return !isGoogleSpecificTool; + }; + + const filteredTools = Array.isArray(tools) ? tools.filter(filterGoogleTool) : []; + return { + tools: filteredTools, llmConfig: openAIConfig as t.OAIClientOptions, configOptions, }; diff --git a/packages/api/src/utils/index.ts b/packages/api/src/utils/index.ts index 76f11289cb..2aa02bcdd3 100644 --- a/packages/api/src/utils/index.ts +++ b/packages/api/src/utils/index.ts @@ -21,3 +21,4 @@ export { default as Tokenizer, countTokens } from './tokenizer'; export * from './yaml'; export * from './http'; export * from './tokens'; +export * from './message'; diff --git a/packages/api/src/utils/message.spec.ts b/packages/api/src/utils/message.spec.ts new file mode 100644 index 0000000000..144ebc1a92 --- /dev/null +++ b/packages/api/src/utils/message.spec.ts @@ -0,0 +1,122 @@ +import { sanitizeFileForTransmit, sanitizeMessageForTransmit } from './message'; + +describe('sanitizeFileForTransmit', () => { + it('should remove text field from file', () => { + const file = { + file_id: 'test-123', + filename: 'test.txt', + text: 'This is a very long text content that should be stripped', + bytes: 1000, + }; + + const result = sanitizeFileForTransmit(file); + + expect(result.file_id).toBe('test-123'); + expect(result.filename).toBe('test.txt'); + expect(result.bytes).toBe(1000); + expect(result).not.toHaveProperty('text'); + }); + + it('should remove _id and __v fields', () => { + const file = { + file_id: 'test-123', + _id: 'mongo-id', + __v: 0, + filename: 'test.txt', + }; + + const result = sanitizeFileForTransmit(file); + + expect(result.file_id).toBe('test-123'); + expect(result).not.toHaveProperty('_id'); + expect(result).not.toHaveProperty('__v'); + }); + + it('should not modify original file object', () => { + const file = { + file_id: 'test-123', + text: 'original text', + }; + + sanitizeFileForTransmit(file); + + expect(file.text).toBe('original text'); + }); +}); + +describe('sanitizeMessageForTransmit', () => { + it('should remove fileContext from message', () => { + const message = { + messageId: 'msg-123', + text: 'Hello world', + fileContext: 'This is a very long context that should be stripped', + }; + + const result = sanitizeMessageForTransmit(message); + + expect(result.messageId).toBe('msg-123'); + expect(result.text).toBe('Hello world'); + expect(result).not.toHaveProperty('fileContext'); + }); + + it('should sanitize files array', () => { + const message = { + messageId: 'msg-123', + files: [ + { file_id: 'file-1', text: 'long text 1', filename: 'a.txt' }, + { file_id: 'file-2', text: 'long text 2', filename: 'b.txt' }, + ], + }; + + const result = sanitizeMessageForTransmit(message); + + expect(result.files).toHaveLength(2); + expect(result.files?.[0].file_id).toBe('file-1'); + expect(result.files?.[0].filename).toBe('a.txt'); + expect(result.files?.[0]).not.toHaveProperty('text'); + expect(result.files?.[1]).not.toHaveProperty('text'); + }); + + it('should handle null/undefined message', () => { + expect(sanitizeMessageForTransmit(null as unknown as object)).toBeNull(); + expect(sanitizeMessageForTransmit(undefined as unknown as object)).toBeUndefined(); + }); + + it('should handle message without files', () => { + const message = { + messageId: 'msg-123', + text: 'Hello', + }; + + const result = sanitizeMessageForTransmit(message); + + expect(result.messageId).toBe('msg-123'); + expect(result.text).toBe('Hello'); + }); + + it('should create new array reference for empty files array (immutability)', () => { + const message = { + messageId: 'msg-123', + files: [] as { file_id: string }[], + }; + + const result = sanitizeMessageForTransmit(message); + + expect(result.files).toEqual([]); + // New array reference ensures full immutability even for empty arrays + expect(result.files).not.toBe(message.files); + }); + + it('should not modify original message object', () => { + const message = { + messageId: 'msg-123', + fileContext: 'original context', + files: [{ file_id: 'file-1', text: 'original text' }], + }; + + sanitizeMessageForTransmit(message); + + expect(message.fileContext).toBe('original context'); + expect(message.files[0].text).toBe('original text'); + }); +}); diff --git a/packages/api/src/utils/message.ts b/packages/api/src/utils/message.ts new file mode 100644 index 0000000000..312826b6ba --- /dev/null +++ b/packages/api/src/utils/message.ts @@ -0,0 +1,68 @@ +import type { TFile, TMessage } from 'librechat-data-provider'; + +/** Fields to strip from files before client transmission */ +const FILE_STRIP_FIELDS = ['text', '_id', '__v'] as const; + +/** Fields to strip from messages before client transmission */ +const MESSAGE_STRIP_FIELDS = ['fileContext'] as const; + +/** + * Strips large/unnecessary fields from a file object before transmitting to client. + * Use this within existing loops when building file arrays to avoid extra iterations. + * + * @param file - The file object to sanitize + * @returns A new file object without the stripped fields + * + * @example + * // Use in existing file processing loop: + * for (const attachment of client.options.attachments) { + * if (messageFiles.has(attachment.file_id)) { + * userMessage.files.push(sanitizeFileForTransmit(attachment)); + * } + * } + */ +export function sanitizeFileForTransmit>( + file: T, +): Omit { + const sanitized = { ...file }; + for (const field of FILE_STRIP_FIELDS) { + delete sanitized[field as keyof typeof sanitized]; + } + return sanitized; +} + +/** + * Sanitizes a message object before transmitting to client. + * Removes large fields like `fileContext` and strips `text` from embedded files. + * + * @param message - The message object to sanitize + * @returns A new message object safe for client transmission + * + * @example + * sendEvent(res, { + * final: true, + * requestMessage: sanitizeMessageForTransmit(userMessage), + * responseMessage: response, + * }); + */ +export function sanitizeMessageForTransmit>( + message: T, +): Omit { + if (!message) { + return message as Omit; + } + + const sanitized = { ...message }; + + // Remove message-level fields + for (const field of MESSAGE_STRIP_FIELDS) { + delete sanitized[field as keyof typeof sanitized]; + } + + // Always create a new array when files exist to maintain full immutability + if (Array.isArray(sanitized.files)) { + sanitized.files = sanitized.files.map((file) => sanitizeFileForTransmit(file)); + } + + return sanitized; +} diff --git a/packages/data-provider/src/file-config.ts b/packages/data-provider/src/file-config.ts index 1ef41f0af5..e5c84e69d3 100644 --- a/packages/data-provider/src/file-config.ts +++ b/packages/data-provider/src/file-config.ts @@ -200,6 +200,27 @@ export const codeTypeMapping: { [key: string]: string } = { tsv: 'text/tab-separated-values', }; +/** Maps image extensions to MIME types for formats browsers may not recognize */ +export const imageTypeMapping: { [key: string]: string } = { + heic: 'image/heic', + heif: 'image/heif', +}; + +/** + * Infers the MIME type from a file's extension when the browser doesn't recognize it + * @param fileName - The name of the file including extension + * @param currentType - The current MIME type reported by the browser (may be empty) + * @returns The inferred MIME type if browser didn't provide one, otherwise the original type + */ +export function inferMimeType(fileName: string, currentType: string): string { + if (currentType) { + return currentType; + } + + const extension = fileName.split('.').pop()?.toLowerCase() ?? ''; + return codeTypeMapping[extension] || imageTypeMapping[extension] || currentType; +} + export const retrievalMimeTypes = [ /^(text\/(x-c|x-c\+\+|x-h|html|x-java|markdown|x-php|x-python|x-script\.python|x-ruby|x-tex|plain|vtt|xml))$/, /^(application\/(json|pdf|vnd\.openxmlformats-officedocument\.(wordprocessingml\.document|presentationml\.presentation)))$/,