diff --git a/api/server/services/Files/images/encode.js b/api/server/services/Files/images/encode.js index ea9dc9f521..e719e58af3 100644 --- a/api/server/services/Files/images/encode.js +++ b/api/server/services/Files/images/encode.js @@ -1,12 +1,14 @@ const axios = require('axios'); -const { logAxiosError } = require('@librechat/api'); const { logger } = require('@librechat/data-schemas'); +const { logAxiosError, validateImage } = require('@librechat/api'); const { FileSources, VisionModes, ImageDetail, ContentTypes, EModelEndpoint, + mergeFileConfig, + getEndpointFileConfig, } = require('librechat-data-provider'); const { getStrategyFunctions } = require('~/server/services/Files/strategies'); @@ -152,6 +154,17 @@ async function encodeAndFormat(req, files, params, mode) { const formattedImages = await Promise.all(promises); promises.length = 0; + /** Extract configured file size limit from fileConfig for this endpoint */ + let configuredFileSizeLimit; + if (req.config?.fileConfig) { + const fileConfig = mergeFileConfig(req.config.fileConfig); + const endpointConfig = getEndpointFileConfig({ + fileConfig, + endpoint: effectiveEndpoint, + }); + configuredFileSizeLimit = endpointConfig?.fileSizeLimit; + } + for (const [file, imageContent] of formattedImages) { const fileMetadata = { type: file.type, @@ -172,6 +185,26 @@ async function encodeAndFormat(req, files, params, mode) { continue; } + /** Validate image buffer against size limits */ + if (file.height && file.width) { + const imageBuffer = imageContent.startsWith('http') + ? null + : Buffer.from(imageContent, 'base64'); + + if (imageBuffer) { + const validation = await validateImage( + imageBuffer, + imageBuffer.length, + effectiveEndpoint, + configuredFileSizeLimit, + ); + + if (!validation.isValid) { + throw new Error(`Image validation failed for ${file.filename}: ${validation.error}`); + } + } + } + const imagePart = { type: ContentTypes.IMAGE_URL, image_url: { diff --git a/packages/api/src/files/filter.spec.ts b/packages/api/src/files/filter.spec.ts index efcd3b4f89..e7858f486f 100644 --- a/packages/api/src/files/filter.spec.ts +++ b/packages/api/src/files/filter.spec.ts @@ -689,4 +689,625 @@ describe('filterFilesByEndpointConfig', () => { expect(result).toEqual([]); }); }); + + describe('file size filtering', () => { + it('should filter out files exceeding fileSizeLimit', () => { + const req = { + config: { + fileConfig: { + endpoints: { + [Providers.OPENAI]: { + disabled: false, + fileSizeLimit: 5 /** 5 MB in config (gets converted to bytes) */, + }, + }, + }, + }, + } as unknown as ServerRequest; + + const smallFile = { + ...createMockFile('small.pdf'), + bytes: 1024 * 1024 * 3 /** 3 MB */, + } as IMongoFile; + + const largeFile = { + ...createMockFile('large.pdf'), + bytes: 1024 * 1024 * 10 /** 10 MB */, + } as IMongoFile; + + const files = [smallFile, largeFile]; + + const result = filterFilesByEndpointConfig(req, { + files, + endpoint: Providers.OPENAI, + }); + + /** Only small file should pass */ + expect(result).toEqual([smallFile]); + }); + + it('should keep all files when no fileSizeLimit is set', () => { + const req = { + config: { + fileConfig: { + endpoints: { + [Providers.OPENAI]: { + disabled: false, + }, + }, + }, + }, + } as unknown as ServerRequest; + + const files = [ + { ...createMockFile('file1.pdf'), bytes: 1024 * 1024 * 100 } as IMongoFile, + { ...createMockFile('file2.pdf'), bytes: 1024 * 1024 * 200 } as IMongoFile, + ]; + + const result = filterFilesByEndpointConfig(req, { + files, + endpoint: Providers.OPENAI, + }); + + expect(result).toEqual(files); + }); + + it('should filter all files if all exceed fileSizeLimit', () => { + const req = { + config: { + fileConfig: { + endpoints: { + [Providers.OPENAI]: { + disabled: false, + fileSizeLimit: 1 /** 1 MB */, + }, + }, + }, + }, + } as unknown as ServerRequest; + + const files = [ + { ...createMockFile('file1.pdf'), bytes: 1024 * 1024 * 5 } as IMongoFile, + { ...createMockFile('file2.pdf'), bytes: 1024 * 1024 * 10 } as IMongoFile, + ]; + + const result = filterFilesByEndpointConfig(req, { + files, + endpoint: Providers.OPENAI, + }); + + expect(result).toEqual([]); + }); + + it('should handle fileSizeLimit of 0 as unlimited', () => { + const req = { + config: { + fileConfig: { + endpoints: { + [Providers.OPENAI]: { + disabled: false, + fileSizeLimit: 0, + totalSizeLimit: 0 /** Also set total limit to 0 for unlimited */, + }, + }, + }, + }, + } as unknown as ServerRequest; + + const files = [{ ...createMockFile('huge.pdf'), bytes: 1024 * 1024 * 1000 } as IMongoFile]; + + const result = filterFilesByEndpointConfig(req, { + files, + endpoint: Providers.OPENAI, + }); + + /** 0 means no limit, so file should pass */ + expect(result).toEqual(files); + }); + }); + + describe('MIME type filtering', () => { + it('should filter out files with unsupported MIME types', () => { + const req = { + config: { + fileConfig: { + endpoints: { + [Providers.OPENAI]: { + disabled: false, + supportedMimeTypes: ['^application/pdf$', '^image/png$'], + }, + }, + }, + }, + } as unknown as ServerRequest; + + const pdfFile = { + ...createMockFile('doc.pdf'), + type: 'application/pdf', + } as IMongoFile; + + const pngFile = { + ...createMockFile('image.png'), + type: 'image/png', + } as IMongoFile; + + const videoFile = { + ...createMockFile('video.mp4'), + type: 'video/mp4', + } as IMongoFile; + + const files = [pdfFile, pngFile, videoFile]; + + const result = filterFilesByEndpointConfig(req, { + files, + endpoint: Providers.OPENAI, + }); + + /** Only PDF and PNG should pass */ + expect(result).toEqual([pdfFile, pngFile]); + }); + + it('should keep all files when supportedMimeTypes is not set', () => { + const req = { + config: { + fileConfig: { + endpoints: { + [Providers.OPENAI]: { + disabled: false, + }, + }, + }, + }, + } as unknown as ServerRequest; + + const files = [ + { ...createMockFile('doc.pdf'), type: 'application/pdf' } as IMongoFile, + { ...createMockFile('video.mp4'), type: 'video/mp4' } as IMongoFile, + { ...createMockFile('audio.mp3'), type: 'audio/mp3' } as IMongoFile, + ]; + + const result = filterFilesByEndpointConfig(req, { + files, + endpoint: Providers.OPENAI, + }); + + expect(result).toEqual(files); + }); + + it('should handle regex patterns for MIME type matching', () => { + const req = { + config: { + fileConfig: { + endpoints: { + [Providers.OPENAI]: { + disabled: false, + supportedMimeTypes: ['^image/.*$'] /** All image types */, + }, + }, + }, + }, + } as unknown as ServerRequest; + + const jpegFile = { + ...createMockFile('photo.jpg'), + type: 'image/jpeg', + } as IMongoFile; + + const pngFile = { + ...createMockFile('graphic.png'), + type: 'image/png', + } as IMongoFile; + + const gifFile = { + ...createMockFile('animation.gif'), + type: 'image/gif', + } as IMongoFile; + + const pdfFile = { + ...createMockFile('doc.pdf'), + type: 'application/pdf', + } as IMongoFile; + + const files = [jpegFile, pngFile, gifFile, pdfFile]; + + const result = filterFilesByEndpointConfig(req, { + files, + endpoint: Providers.OPENAI, + }); + + /** Only image files should pass */ + expect(result).toEqual([jpegFile, pngFile, gifFile]); + }); + + it('should filter all files if none match supported MIME types', () => { + const req = { + config: { + fileConfig: { + endpoints: { + [Providers.OPENAI]: { + disabled: false, + supportedMimeTypes: ['^application/pdf$'], + }, + }, + }, + }, + } as unknown as ServerRequest; + + const files = [ + { ...createMockFile('video.mp4'), type: 'video/mp4' } as IMongoFile, + { ...createMockFile('audio.mp3'), type: 'audio/mp3' } as IMongoFile, + ]; + + const result = filterFilesByEndpointConfig(req, { + files, + endpoint: Providers.OPENAI, + }); + + expect(result).toEqual([]); + }); + + it('should handle empty supportedMimeTypes array', () => { + const req = { + config: { + fileConfig: { + endpoints: { + [Providers.OPENAI]: { + disabled: false, + supportedMimeTypes: [], + }, + }, + }, + }, + } as unknown as ServerRequest; + + const files = [{ ...createMockFile('doc.pdf'), type: 'application/pdf' } as IMongoFile]; + + const result = filterFilesByEndpointConfig(req, { + files, + endpoint: Providers.OPENAI, + }); + + /** Empty array means allow all */ + expect(result).toEqual(files); + }); + }); + + describe('total size limit filtering', () => { + it('should filter files when total size exceeds totalSizeLimit', () => { + const req = { + config: { + fileConfig: { + endpoints: { + [Providers.OPENAI]: { + disabled: false, + totalSizeLimit: 10 /** 10 MB total */, + }, + }, + }, + }, + } as unknown as ServerRequest; + + const file1 = { + ...createMockFile('file1.pdf'), + bytes: 1024 * 1024 * 4 /** 4 MB */, + } as IMongoFile; + + const file2 = { + ...createMockFile('file2.pdf'), + bytes: 1024 * 1024 * 4 /** 4 MB */, + } as IMongoFile; + + const file3 = { + ...createMockFile('file3.pdf'), + bytes: 1024 * 1024 * 4 /** 4 MB */, + } as IMongoFile; + + const files = [file1, file2, file3]; + + const result = filterFilesByEndpointConfig(req, { + files, + endpoint: Providers.OPENAI, + }); + + /** Only first two files should pass (8 MB total) */ + expect(result).toEqual([file1, file2]); + }); + + it('should keep all files when totalSizeLimit is not exceeded', () => { + const req = { + config: { + fileConfig: { + endpoints: { + [Providers.OPENAI]: { + disabled: false, + totalSizeLimit: 20 /** 20 MB total */, + }, + }, + }, + }, + } as unknown as ServerRequest; + + const files = [ + { ...createMockFile('file1.pdf'), bytes: 1024 * 1024 * 5 } as IMongoFile, + { ...createMockFile('file2.pdf'), bytes: 1024 * 1024 * 5 } as IMongoFile, + ]; + + const result = filterFilesByEndpointConfig(req, { + files, + endpoint: Providers.OPENAI, + }); + + expect(result).toEqual(files); + }); + + it('should handle totalSizeLimit of 0 as unlimited', () => { + const req = { + config: { + fileConfig: { + endpoints: { + [Providers.OPENAI]: { + disabled: false, + totalSizeLimit: 0, + }, + }, + }, + }, + } as unknown as ServerRequest; + + const files = [ + { ...createMockFile('file1.pdf'), bytes: 1024 * 1024 * 100 } as IMongoFile, + { ...createMockFile('file2.pdf'), bytes: 1024 * 1024 * 100 } as IMongoFile, + ]; + + const result = filterFilesByEndpointConfig(req, { + files, + endpoint: Providers.OPENAI, + }); + + /** 0 means no limit */ + expect(result).toEqual(files); + }); + + it('should skip files that exceed totalSizeLimit and continue with remaining files', () => { + const req = { + config: { + fileConfig: { + endpoints: { + [Providers.OPENAI]: { + disabled: false, + totalSizeLimit: 5 /** 5 MB total */, + }, + }, + }, + }, + } as unknown as ServerRequest; + + const files = [ + { ...createMockFile('large.pdf'), bytes: 1024 * 1024 * 10 } as IMongoFile, + { ...createMockFile('small.pdf'), bytes: 1024 * 1024 * 1 } as IMongoFile, + ]; + + const result = filterFilesByEndpointConfig(req, { + files, + endpoint: Providers.OPENAI, + }); + + /** First file exceeds total limit, so it's skipped. Small file fits and is included. */ + expect(result).toEqual([files[1]]); + }); + + it('should keep files in order until total size limit is reached', () => { + const req = { + config: { + fileConfig: { + endpoints: { + [Providers.OPENAI]: { + disabled: false, + totalSizeLimit: 7 /** 7 MB total */, + }, + }, + }, + }, + } as unknown as ServerRequest; + + const file1 = { + ...createMockFile('file1.pdf'), + bytes: 1024 * 1024 * 2 /** 2 MB */, + } as IMongoFile; + + const file2 = { + ...createMockFile('file2.pdf'), + bytes: 1024 * 1024 * 3 /** 3 MB */, + } as IMongoFile; + + const file3 = { + ...createMockFile('file3.pdf'), + bytes: 1024 * 1024 * 2 /** 2 MB */, + } as IMongoFile; + + const file4 = { + ...createMockFile('file4.pdf'), + bytes: 1024 * 1024 * 1 /** 1 MB */, + } as IMongoFile; + + const files = [file1, file2, file3, file4]; + + const result = filterFilesByEndpointConfig(req, { + files, + endpoint: Providers.OPENAI, + }); + + /** file1 (2MB) + file2 (3MB) = 5MB ✓, + file3 (2MB) = 7MB ✓, + file4 would exceed */ + expect(result).toEqual([file1, file2, file3]); + }); + }); + + describe('combined filtering scenarios', () => { + it('should apply size and MIME type filters together', () => { + const req = { + config: { + fileConfig: { + endpoints: { + [Providers.OPENAI]: { + disabled: false, + fileSizeLimit: 5 /** 5 MB per file */, + supportedMimeTypes: ['^application/pdf$', '^image/.*$'], + }, + }, + }, + }, + } as unknown as ServerRequest; + + const validPdf = { + ...createMockFile('valid.pdf'), + type: 'application/pdf', + bytes: 1024 * 1024 * 3 /** 3 MB */, + } as IMongoFile; + + const validImage = { + ...createMockFile('valid.png'), + type: 'image/png', + bytes: 1024 * 1024 * 2 /** 2 MB */, + } as IMongoFile; + + const tooLargePdf = { + ...createMockFile('large.pdf'), + type: 'application/pdf', + bytes: 1024 * 1024 * 10 /** 10 MB - exceeds size limit */, + } as IMongoFile; + + const wrongTypeVideo = { + ...createMockFile('video.mp4'), + type: 'video/mp4', + bytes: 1024 * 1024 * 2 /** 2 MB - wrong MIME type */, + } as IMongoFile; + + const files = [validPdf, validImage, tooLargePdf, wrongTypeVideo]; + + const result = filterFilesByEndpointConfig(req, { + files, + endpoint: Providers.OPENAI, + }); + + /** Only validPdf and validImage should pass both filters */ + expect(result).toEqual([validPdf, validImage]); + }); + + it('should apply all three filters together', () => { + const req = { + config: { + fileConfig: { + endpoints: { + [Providers.OPENAI]: { + disabled: false, + fileSizeLimit: 5 /** 5 MB per file */, + totalSizeLimit: 8 /** 8 MB total */, + supportedMimeTypes: ['^application/pdf$'], + }, + }, + }, + }, + } as unknown as ServerRequest; + + const pdf1 = { + ...createMockFile('pdf1.pdf'), + type: 'application/pdf', + bytes: 1024 * 1024 * 3 /** 3 MB */, + } as IMongoFile; + + const pdf2 = { + ...createMockFile('pdf2.pdf'), + type: 'application/pdf', + bytes: 1024 * 1024 * 4 /** 4 MB */, + } as IMongoFile; + + const pdf3 = { + ...createMockFile('pdf3.pdf'), + type: 'application/pdf', + bytes: 1024 * 1024 * 2 /** 2 MB */, + } as IMongoFile; + + const largePdf = { + ...createMockFile('large.pdf'), + type: 'application/pdf', + bytes: 1024 * 1024 * 10 /** 10 MB - exceeds individual size limit */, + } as IMongoFile; + + const wrongType = { + ...createMockFile('image.png'), + type: 'image/png', + bytes: 1024 * 1024 * 1 /** Wrong type */, + } as IMongoFile; + + const files = [pdf1, pdf2, pdf3, largePdf, wrongType]; + + const result = filterFilesByEndpointConfig(req, { + files, + endpoint: Providers.OPENAI, + }); + + /** + * largePdf filtered by size (10MB > 5MB limit) + * wrongType filtered by MIME type + * Remaining: pdf1 (3MB) + pdf2 (4MB) = 7MB ✓ + * pdf3 (2MB) would make total 9MB > 8MB limit, so filtered by total + */ + expect(result).toEqual([pdf1, pdf2]); + }); + + it('should handle mixed validation with some files passing all checks', () => { + const req = { + config: { + fileConfig: { + endpoints: { + [Providers.ANTHROPIC]: { + disabled: false, + fileSizeLimit: 10, + totalSizeLimit: 20, + supportedMimeTypes: ['^application/.*$', '^text/.*$'], + }, + }, + }, + }, + } as unknown as ServerRequest; + + const file1 = { + ...createMockFile('doc.pdf'), + type: 'application/pdf', + bytes: 1024 * 1024 * 5, + } as IMongoFile; + + const file2 = { + ...createMockFile('text.txt'), + type: 'text/plain', + bytes: 1024 * 1024 * 8, + } as IMongoFile; + + const file3 = { + ...createMockFile('data.json'), + type: 'application/json', + bytes: 1024 * 1024 * 6, + } as IMongoFile; + + const file4 = { + ...createMockFile('video.mp4'), + type: 'video/mp4', + bytes: 1024 * 1024 * 3 /** Wrong MIME type */, + } as IMongoFile; + + const files = [file1, file2, file3, file4]; + + const result = filterFilesByEndpointConfig(req, { + files, + endpoint: Providers.ANTHROPIC, + }); + + /** + * file4 filtered by MIME type + * file1 (5MB) + file2 (8MB) = 13MB ✓ + * file3 (6MB) would make 19MB < 20MB ✓ + */ + expect(result).toEqual([file1, file2, file3]); + }); + }); }); diff --git a/packages/api/src/files/filter.ts b/packages/api/src/files/filter.ts index eaf2a6fe90..4597dc987e 100644 --- a/packages/api/src/files/filter.ts +++ b/packages/api/src/files/filter.ts @@ -1,15 +1,32 @@ -import { getEndpointFileConfig, mergeFileConfig } from 'librechat-data-provider'; +import { getEndpointFileConfig, mergeFileConfig, fileConfig } from 'librechat-data-provider'; import type { IMongoFile } from '@librechat/data-schemas'; import type { ServerRequest } from '~/types'; /** - * Filters out files if the endpoint/provider has file uploads disabled + * Checks if a MIME type is supported by the endpoint configuration + * @param mimeType - The MIME type to check + * @param supportedMimeTypes - Array of RegExp patterns to match against + * @returns True if the MIME type matches any pattern + */ +function isMimeTypeSupported(mimeType: string, supportedMimeTypes?: RegExp[]): boolean { + if (!supportedMimeTypes || supportedMimeTypes.length === 0) { + return true; + } + return fileConfig.checkType(mimeType, supportedMimeTypes); +} + +/** + * Filters out files based on endpoint configuration including: + * - Disabled status + * - File size limits + * - MIME type restrictions + * - Total size limits * @param req - The server request object containing config * @param params - Object containing files, endpoint, and endpointType * @param params.files - Array of processed file documents from MongoDB * @param params.endpoint - The endpoint name to check configuration for * @param params.endpointType - The endpoint type to check configuration for - * @returns Filtered array of files (empty if disabled) + * @returns Filtered array of files */ export function filterFilesByEndpointConfig( req: ServerRequest, @@ -25,9 +42,9 @@ export function filterFilesByEndpointConfig( return []; } - const fileConfig = mergeFileConfig(req.config?.fileConfig); + const mergedFileConfig = mergeFileConfig(req.config?.fileConfig); const endpointFileConfig = getEndpointFileConfig({ - fileConfig, + fileConfig: mergedFileConfig, endpoint, endpointType, }); @@ -40,5 +57,40 @@ export function filterFilesByEndpointConfig( return []; } - return files; + const { fileSizeLimit, supportedMimeTypes, totalSizeLimit } = endpointFileConfig; + + /** Filter files based on individual file size and MIME type */ + let filteredFiles = files; + + /** Filter by individual file size limit */ + if (fileSizeLimit !== undefined && fileSizeLimit > 0) { + filteredFiles = filteredFiles.filter((file) => { + return file.bytes <= fileSizeLimit; + }); + } + + /** Filter by MIME type */ + if (supportedMimeTypes && supportedMimeTypes.length > 0) { + filteredFiles = filteredFiles.filter((file) => { + return isMimeTypeSupported(file.type, supportedMimeTypes); + }); + } + + /** Filter by total size limit - keep files until total exceeds limit */ + if (totalSizeLimit !== undefined && totalSizeLimit > 0) { + let totalSize = 0; + const withinTotalLimit: IMongoFile[] = []; + + for (let i = 0; i < filteredFiles.length; i++) { + const file = filteredFiles[i]; + if (totalSize + file.bytes <= totalSizeLimit) { + withinTotalLimit.push(file); + totalSize += file.bytes; + } + } + + filteredFiles = withinTotalLimit; + } + + return filteredFiles; } diff --git a/packages/api/src/files/validation.ts b/packages/api/src/files/validation.ts index 7c1eccd8ed..4b36ac0bff 100644 --- a/packages/api/src/files/validation.ts +++ b/packages/api/src/files/validation.ts @@ -16,6 +16,11 @@ export interface AudioValidationResult { error?: string; } +export interface ImageValidationResult { + isValid: boolean; + error?: string; +} + export async function validatePdf( pdfBuffer: Buffer, fileSize: number, @@ -229,3 +234,53 @@ export async function validateAudio( return { isValid: true }; } + +/** + * Validates image files for different providers + * @param imageBuffer - The image file as a buffer + * @param fileSize - The file size in bytes + * @param provider - The provider to validate for + * @param configuredFileSizeLimit - Optional configured file size limit from fileConfig (in bytes) + * @returns Promise that resolves to validation result + */ +export async function validateImage( + imageBuffer: Buffer, + fileSize: number, + provider: Providers | string, + configuredFileSizeLimit?: number, +): Promise { + if (provider === Providers.GOOGLE || provider === Providers.VERTEXAI) { + const providerLimit = mbToBytes(20); + const effectiveLimit = configuredFileSizeLimit ?? providerLimit; + + if (fileSize > effectiveLimit) { + const limitMB = Math.round(effectiveLimit / (1024 * 1024)); + return { + isValid: false, + error: `Image file size (${Math.round(fileSize / (1024 * 1024))}MB) exceeds the ${limitMB}MB limit`, + }; + } + } + + if (provider === Providers.ANTHROPIC) { + const providerLimit = mbToBytes(5); + const effectiveLimit = configuredFileSizeLimit ?? providerLimit; + + if (fileSize > effectiveLimit) { + const limitMB = Math.round(effectiveLimit / (1024 * 1024)); + return { + isValid: false, + error: `Image file size (${Math.round(fileSize / (1024 * 1024))}MB) exceeds the ${limitMB}MB limit`, + }; + } + } + + if (!imageBuffer || imageBuffer.length < 10) { + return { + isValid: false, + error: 'Invalid image file: too small or corrupted', + }; + } + + return { isValid: true }; +}