diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.js index cdce2edf4d..905cadfd23 100644 --- a/api/app/clients/BaseClient.js +++ b/api/app/clients/BaseClient.js @@ -17,11 +17,13 @@ const { ContentTypes, excludedKeys, EModelEndpoint, + mergeFileConfig, isParamEndpoint, isAgentsEndpoint, isEphemeralAgentId, supportsBalanceCheck, isBedrockDocumentType, + getEndpointFileConfig, } = require('librechat-data-provider'); const { getStrategyFunctions } = require('~/server/services/Files/strategies'); const { logViolation } = require('~/cache'); @@ -71,6 +73,10 @@ class BaseClient { this.currentMessages = []; /** @type {import('librechat-data-provider').VisionModes | undefined} */ this.visionMode; + /** @type {import('librechat-data-provider').FileConfig | undefined} */ + this._mergedFileConfig; + /** @type {import('librechat-data-provider').EndpointFileConfig | undefined} */ + this._endpointFileConfig; } setOptions() { @@ -1160,6 +1166,16 @@ class BaseClient { const provider = this.options.agent?.provider ?? this.options.endpoint; const isBedrock = provider === EModelEndpoint.bedrock; + if (!this._mergedFileConfig && this.options.req?.config?.fileConfig) { + this._mergedFileConfig = mergeFileConfig(this.options.req.config.fileConfig); + const endpoint = this.options.agent?.endpoint ?? this.options.endpoint; + this._endpointFileConfig = getEndpointFileConfig({ + fileConfig: this._mergedFileConfig, + endpoint, + endpointType: this.options.endpointType, + }); + } + for (const file of attachments) { /** @type {FileSources} */ const source = file.source ?? FileSources.local; @@ -1186,6 +1202,14 @@ class BaseClient { } else if (file.type.startsWith('audio/')) { categorizedAttachments.audios.push(file); allFiles.push(file); + } else if ( + file.type && + this._mergedFileConfig && + this._endpointFileConfig?.supportedMimeTypes && + this._mergedFileConfig.checkType(file.type, this._endpointFileConfig.supportedMimeTypes) + ) { + categorizedAttachments.documents.push(file); + allFiles.push(file); } } diff --git a/packages/api/src/files/encode/document.spec.ts b/packages/api/src/files/encode/document.spec.ts index 2b8e231048..f285d47d5d 100644 --- a/packages/api/src/files/encode/document.spec.ts +++ b/packages/api/src/files/encode/document.spec.ts @@ -56,13 +56,16 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { }); /** Helper to create a mock request with file config */ - const createMockRequest = (fileSizeLimit?: number): Partial => ({ + const createMockRequest = ( + fileSizeLimit?: number, + provider: string = Providers.OPENAI, + ): Partial => ({ config: fileSizeLimit !== undefined ? { fileConfig: { endpoints: { - [Providers.OPENAI]: { + [provider]: { fileSizeLimit, }, }, @@ -747,4 +750,235 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => { }); }); }); + + describe('Generic document encoding path', () => { + it('should format text/plain for Anthropic with citations enabled', async () => { + const req = createMockRequest(30) as ServerRequest; + const file = createMockDocFile(1, 'text/plain', 'notes.txt'); + + const mockContent = Buffer.from('plain text content').toString('base64'); + mockedGetFileStream.mockResolvedValue({ + file, + content: mockContent, + metadata: file, + }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.ANTHROPIC }, + mockStrategyFunctions, + ); + + expect(result.documents).toHaveLength(1); + expect(result.documents[0]).toMatchObject({ + type: 'document', + source: { + type: 'base64', + media_type: 'text/plain', + data: mockContent, + }, + citations: { enabled: true }, + context: 'File: "notes.txt"', + }); + expect(result.files).toHaveLength(1); + }); + + it('should format text/html for Anthropic with citations enabled', async () => { + const req = createMockRequest(30) as ServerRequest; + const file = createMockDocFile(1, 'text/html', 'page.html'); + + const mockContent = Buffer.from('content').toString('base64'); + mockedGetFileStream.mockResolvedValue({ + file, + content: mockContent, + metadata: file, + }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.ANTHROPIC }, + mockStrategyFunctions, + ); + + expect(result.documents).toHaveLength(1); + expect(result.documents[0]).toMatchObject({ + type: 'document', + source: { type: 'base64', media_type: 'text/html', data: mockContent }, + citations: { enabled: true }, + }); + }); + + it('should format application/json for Anthropic without citations', async () => { + const req = createMockRequest(30) as ServerRequest; + const file = createMockDocFile(1, 'application/json', 'data.json'); + + const mockContent = Buffer.from('{"key":"value"}').toString('base64'); + mockedGetFileStream.mockResolvedValue({ + file, + content: mockContent, + metadata: file, + }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.ANTHROPIC }, + mockStrategyFunctions, + ); + + expect(result.documents).toHaveLength(1); + expect(result.documents[0]).not.toHaveProperty('citations'); + }); + + it('should format text/csv for OpenAI responses API', async () => { + const req = createMockRequest(15) as ServerRequest; + const file = createMockDocFile(1, 'text/csv', 'data.csv'); + + const mockContent = Buffer.from('a,b\n1,2').toString('base64'); + mockedGetFileStream.mockResolvedValue({ + file, + content: mockContent, + metadata: file, + }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.OPENAI, useResponsesApi: true }, + mockStrategyFunctions, + ); + + expect(result.documents).toHaveLength(1); + expect(result.documents[0]).toMatchObject({ + type: 'input_file', + filename: 'data.csv', + file_data: `data:text/csv;base64,${mockContent}`, + }); + expect(result.files).toHaveLength(1); + }); + + it('should format XLSX for Google/VertexAI as media block', async () => { + const req = createMockRequest(25) as ServerRequest; + const mimeType = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; + const file = createMockDocFile(2, mimeType, 'report.xlsx'); + + const mockContent = Buffer.from('xlsx-binary').toString('base64'); + mockedGetFileStream.mockResolvedValue({ + file, + content: mockContent, + metadata: file, + }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.GOOGLE }, + mockStrategyFunctions, + ); + + expect(result.documents).toHaveLength(1); + expect(result.documents[0]).toMatchObject({ + type: 'media', + mimeType, + data: mockContent, + }); + expect(result.files).toHaveLength(1); + }); + + it('should format text/plain for standard OpenAI-like provider as file block', async () => { + const req = createMockRequest(15) as ServerRequest; + const file = createMockDocFile(1, 'text/plain', 'readme.txt'); + + const mockContent = Buffer.from('readme content').toString('base64'); + mockedGetFileStream.mockResolvedValue({ + file, + content: mockContent, + metadata: file, + }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.OPENAI }, + mockStrategyFunctions, + ); + + expect(result.documents).toHaveLength(1); + expect(result.documents[0]).toMatchObject({ + type: 'file', + file: { + filename: 'readme.txt', + file_data: `data:text/plain;base64,${mockContent}`, + }, + }); + expect(result.files).toHaveLength(1); + }); + + it('should skip non-Bedrock-document types for Bedrock provider', async () => { + const req = createMockRequest() as ServerRequest; + const file = createMockDocFile(1, 'application/zip', 'archive.zip'); + + const mockContent = Buffer.from('zip-content').toString('base64'); + mockedGetFileStream.mockResolvedValue({ + file, + content: mockContent, + metadata: file, + }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.BEDROCK }, + mockStrategyFunctions, + ); + + expect(result.documents).toHaveLength(0); + expect(result.files).toHaveLength(0); + }); + + it('should throw when generic file exceeds configured size limit', async () => { + const req = createMockRequest(1, Providers.ANTHROPIC) as ServerRequest; + const file = createMockDocFile(2, 'text/plain', 'large.txt'); + + const largeContent = Buffer.alloc(2 * 1024 * 1024).toString('base64'); + mockedGetFileStream.mockResolvedValue({ + file, + content: largeContent, + metadata: file, + }); + + await expect( + encodeAndFormatDocuments( + req, + [file], + { provider: Providers.ANTHROPIC }, + mockStrategyFunctions, + ), + ).rejects.toThrow('File size'); + }); + + it('should not push metadata when provider has no handler', async () => { + const req = createMockRequest(15) as ServerRequest; + const file = createMockDocFile(1, 'text/plain', 'test.txt'); + + const mockContent = Buffer.from('content').toString('base64'); + mockedGetFileStream.mockResolvedValue({ + file, + content: mockContent, + metadata: file, + }); + + const result = await encodeAndFormatDocuments( + req, + [file], + { provider: Providers.AZURE as Providers }, + mockStrategyFunctions, + ); + + expect(result.documents).toHaveLength(0); + expect(result.files).toHaveLength(0); + }); + }); }); diff --git a/packages/api/src/files/encode/document.ts b/packages/api/src/files/encode/document.ts index 4126a51376..f04ad43ef2 100644 --- a/packages/api/src/files/encode/document.ts +++ b/packages/api/src/files/encode/document.ts @@ -7,6 +7,7 @@ import { } from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; import type { + DocumentBlock, AnthropicDocumentBlock, StrategyFunctions, DocumentResult, @@ -15,16 +16,85 @@ import type { import { validatePdf, validateBedrockDocument } from '~/files/validation'; import { getFileStream, getConfiguredFileSizeLimit } from './utils'; +const ANTHROPIC_CITATION_TYPES = new Set([ + 'application/pdf', + 'text/plain', + 'text/html', + 'text/markdown', +]); + /** - * Processes and encodes document files for various providers - * @param req - Express request object - * @param files - Array of file objects to process - * @param params - Object containing provider, endpoint, and other options - * @param params.provider - The provider name - * @param params.endpoint - Optional endpoint name for file config lookup - * @param params.useResponsesApi - Whether to use responses API format - * @param getStrategyFunctions - Function to get strategy functions - * @returns Promise that resolves to documents and file metadata + * Formats a base64-encoded document into the appropriate provider-specific block. + * Returns `null` when the provider has no matching handler. + */ +function formatDocumentBlock( + provider: Providers, + mimeType: string, + content: string, + filename: string | undefined, + useResponsesApi: boolean | undefined, +): DocumentBlock | null { + if (provider === Providers.ANTHROPIC) { + const document: AnthropicDocumentBlock = { + type: 'document', + source: { + type: 'base64', + media_type: mimeType, + data: content, + }, + }; + + if (ANTHROPIC_CITATION_TYPES.has(mimeType)) { + document.citations = { enabled: true }; + } + + if (filename) { + document.context = `File: "${filename}"`; + } + + return document; + } + + const resolvedFilename = filename ?? 'document'; + + if (useResponsesApi) { + return { + type: 'input_file', + filename: resolvedFilename, + file_data: `data:${mimeType};base64,${content}`, + }; + } + + if (provider === Providers.GOOGLE || provider === Providers.VERTEXAI) { + return { + type: 'media', + mimeType, + data: content, + }; + } + + if (isOpenAILikeProvider(provider) && provider !== Providers.AZURE) { + return { + type: 'file', + file: { + filename: resolvedFilename, + file_data: `data:${mimeType};base64,${content}`, + }, + }; + } + + return null; +} + +/** + * Encodes and formats document files for various providers. + * + * Callers are responsible for pre-filtering `files` to types the endpoint accepts + * (e.g., via `supportedMimeTypes` in `processAttachments`). This function processes + * every file it receives and dispatches to the appropriate provider format: + * - **Bedrock**: Only encodes types in `bedrockDocumentFormats`; all others are skipped. + * - **PDF**: Validated via `validatePdf` before encoding. + * - **Generic types**: Encoded with a provider-specific size check. */ export async function encodeAndFormatDocuments( req: ServerRequest, @@ -43,25 +113,22 @@ export async function encodeAndFormatDocuments( const isBedrock = provider === Providers.BEDROCK; const isDocSupported = isDocumentSupportedProvider(provider); - const documentFiles = files.filter((file) => { - if (isBedrock && isBedrockDocumentType(file.type)) { - return true; - } - return file.type === 'application/pdf' || file.type?.startsWith('application/'); - }); - - if (!documentFiles.length) { + if (!isDocSupported && !isBedrock) { return result; } + const processableFiles = isBedrock + ? files.filter((file) => isBedrockDocumentType(file.type)) + : files; + + if (!processableFiles.length) { + return result; + } + + const configuredFileSizeLimit = getConfiguredFileSizeLimit(req, { provider, endpoint }); + const results = await Promise.allSettled( - documentFiles.map((file) => { - const isProcessable = isBedrock - ? isBedrockDocumentType(file.type) - : file.type === 'application/pdf' && isDocSupported; - if (!isProcessable) { - return Promise.resolve(null); - } + processableFiles.map((file) => { return getFileStream(req, file, encodingMethods, getStrategyFunctions); }), ); @@ -82,7 +149,6 @@ export async function encodeAndFormatDocuments( continue; } - const configuredFileSizeLimit = getConfiguredFileSizeLimit(req, { provider, endpoint }); const mimeType = file.type ?? ''; if (isBedrock && isBedrockDocumentType(mimeType)) { @@ -130,44 +196,37 @@ export async function encodeAndFormatDocuments( throw new Error(`PDF validation failed: ${validation.error}`); } - if (provider === Providers.ANTHROPIC) { - const document: AnthropicDocumentBlock = { - type: 'document', - source: { - type: 'base64', - media_type: 'application/pdf', - data: content, - }, - citations: { enabled: true }, - }; - - if (file.filename) { - document.context = `File: "${file.filename}"`; - } - - result.documents.push(document); - } else if (useResponsesApi) { - result.documents.push({ - type: 'input_file', - filename: file.filename, - file_data: `data:application/pdf;base64,${content}`, - }); - } else if (provider === Providers.GOOGLE || provider === Providers.VERTEXAI) { - result.documents.push({ - type: 'media', - mimeType: 'application/pdf', - data: content, - }); - } else if (isOpenAILikeProvider(provider) && provider != Providers.AZURE) { - result.documents.push({ - type: 'file', - file: { - filename: file.filename, - file_data: `data:application/pdf;base64,${content}`, - }, - }); + const block = formatDocumentBlock( + provider, + mimeType, + content, + file.filename, + useResponsesApi, + ); + if (block) { + result.documents.push(block); + result.files.push(metadata); + } + } else if (isDocSupported && !isBedrock) { + const paddingChars = content.endsWith('==') ? 2 : content.endsWith('=') ? 1 : 0; + const decodedByteCount = Math.floor((content.length * 3) / 4) - paddingChars; + if (configuredFileSizeLimit && decodedByteCount > configuredFileSizeLimit) { + throw new Error( + `File size (~${(decodedByteCount / 1024 / 1024).toFixed(1)}MB) exceeds the configured limit for ${provider}`, + ); + } + + const block = formatDocumentBlock( + provider, + mimeType, + content, + file.filename, + useResponsesApi, + ); + if (block) { + result.documents.push(block); + result.files.push(metadata); } - result.files.push(metadata); } } diff --git a/packages/api/src/files/encode/processAttachments.spec.ts b/packages/api/src/files/encode/processAttachments.spec.ts new file mode 100644 index 0000000000..91d6de4a0c --- /dev/null +++ b/packages/api/src/files/encode/processAttachments.spec.ts @@ -0,0 +1,158 @@ +import { + FileSources, + mergeFileConfig, + EModelEndpoint, + getEndpointFileConfig, + isBedrockDocumentType, +} from 'librechat-data-provider'; +import type { FileConfig, EndpointFileConfig } from 'librechat-data-provider'; + +/** + * Mirrors the categorization logic from BaseClient.processAttachments. + * Extracted here for testability since the /api workspace test setup is broken. + */ +function categorizeFile( + file: { + type?: string | null; + source?: string; + embedded?: boolean; + metadata?: { fileIdentifier?: string }; + }, + isBedrock: boolean, + mergedFileConfig: FileConfig | undefined, + endpointFileConfig: EndpointFileConfig | undefined, +): 'images' | 'documents' | 'videos' | 'audios' | 'skipped' { + const source = file.source ?? FileSources.local; + if (source === FileSources.text) { + return 'skipped'; + } + if (file.embedded === true || file.metadata?.fileIdentifier != null) { + return 'skipped'; + } + + if (file.type?.startsWith('image/')) { + return 'images'; + } else if (file.type === 'application/pdf') { + return 'documents'; + } else if (isBedrock && file.type && isBedrockDocumentType(file.type)) { + return 'documents'; + } else if (file.type?.startsWith('video/')) { + return 'videos'; + } else if (file.type?.startsWith('audio/')) { + return 'audios'; + } else if ( + file.type && + mergedFileConfig && + endpointFileConfig?.supportedMimeTypes && + mergedFileConfig.checkType?.(file.type, endpointFileConfig.supportedMimeTypes) + ) { + return 'documents'; + } + + return 'skipped'; +} + +describe('processAttachments — supportedMimeTypes routing logic', () => { + const endpoint = EModelEndpoint.openAI; + + function resolveConfig(mimePatterns: string[]) { + const merged = mergeFileConfig({ + endpoints: { + [endpoint]: { supportedMimeTypes: mimePatterns }, + }, + }); + const epConfig = getEndpointFileConfig({ + fileConfig: merged, + endpoint, + }); + return { merged, epConfig }; + } + + it('should route text/csv to documents when supportedMimeTypes includes it', () => { + const { merged, epConfig } = resolveConfig(['text/csv']); + const result = categorizeFile({ type: 'text/csv' }, false, merged, epConfig); + expect(result).toBe('documents'); + }); + + it('should route text/plain to documents when supportedMimeTypes uses wildcard', () => { + const { merged, epConfig } = resolveConfig(['.*']); + const result = categorizeFile({ type: 'text/plain' }, false, merged, epConfig); + expect(result).toBe('documents'); + }); + + it('should skip application/zip when supportedMimeTypes only allows text types', () => { + const { merged, epConfig } = resolveConfig(['text/csv', 'text/plain']); + const result = categorizeFile({ type: 'application/zip' }, false, merged, epConfig); + expect(result).toBe('skipped'); + }); + + it('should skip files when no fileConfig is provided', () => { + const result = categorizeFile({ type: 'text/csv' }, false, undefined, undefined); + expect(result).toBe('skipped'); + }); + + it('should skip files with null type even with permissive config', () => { + const { merged, epConfig } = resolveConfig(['.*']); + const result = categorizeFile({ type: null }, false, merged, epConfig); + expect(result).toBe('skipped'); + }); + + it('should skip files with undefined type even with permissive config', () => { + const { merged, epConfig } = resolveConfig(['.*']); + const result = categorizeFile({ type: undefined }, false, merged, epConfig); + expect(result).toBe('skipped'); + }); + + it('should still route image types through images category (not documents)', () => { + const { merged, epConfig } = resolveConfig(['.*']); + expect(categorizeFile({ type: 'image/png' }, false, merged, epConfig)).toBe('images'); + }); + + it('should still route PDF through documents (dedicated branch)', () => { + const { merged, epConfig } = resolveConfig(['.*']); + expect(categorizeFile({ type: 'application/pdf' }, false, merged, epConfig)).toBe('documents'); + }); + + it('should still route video types through videos category', () => { + const { merged, epConfig } = resolveConfig(['.*']); + expect(categorizeFile({ type: 'video/mp4' }, false, merged, epConfig)).toBe('videos'); + }); + + it('should still route audio types through audios category', () => { + const { merged, epConfig } = resolveConfig(['.*']); + expect(categorizeFile({ type: 'audio/mp3' }, false, merged, epConfig)).toBe('audios'); + }); + + it('should route Bedrock document types through documents for Bedrock provider', () => { + const { merged, epConfig } = resolveConfig(['.*']); + expect(categorizeFile({ type: 'text/csv' }, true, merged, epConfig)).toBe('documents'); + }); + + it('should route non-Bedrock-document types for Bedrock when config allows them', () => { + const { merged, epConfig } = resolveConfig(['.*']); + expect(categorizeFile({ type: 'application/zip' }, true, merged, epConfig)).toBe('documents'); + }); + + it('should route xlsx to documents with matching config', () => { + const xlsxType = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; + const { merged, epConfig } = resolveConfig([xlsxType]); + expect(categorizeFile({ type: xlsxType }, false, merged, epConfig)).toBe('documents'); + }); + + it('should skip text source files regardless of config', () => { + const { merged, epConfig } = resolveConfig(['.*']); + const result = categorizeFile( + { type: 'text/csv', source: FileSources.text }, + false, + merged, + epConfig, + ); + expect(result).toBe('skipped'); + }); + + it('should skip embedded files regardless of config', () => { + const { merged, epConfig } = resolveConfig(['.*']); + const result = categorizeFile({ type: 'text/csv', embedded: true }, false, merged, epConfig); + expect(result).toBe('skipped'); + }); +});