mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-02-20 17:34:10 +01:00
📎 feat: Upload as Text Support for Plaintext, STT, RAG, and Token Limits (#8868)
* 🪶 feat: Add Support for Uploading Plaintext Files feat: delineate between OCR and text handling in fileConfig field of config file - also adds support for passing in mimetypes as just plain file extensions feat: add showLabel bool to support future synthetic component DynamicDropdownInput feat: add new combination dropdown-input component in params panel to support file type token limits refactor: move hovercard to side to align with other hovercards chore: clean up autogenerated comments feat: add delineation to file upload path between text and ocr configured filetypes feat: add token limit checks during file upload refactor: move textParsing out of ocrEnabled logic refactor: clean up types for filetype config refactor: finish decoupling DynamicDropdownInput from fileTokenLimits fix: move image token cost function into file to fix circular dependency causing unittest to fail and remove unused var for linter chore: remove out of scope code following review refactor: make fileTokenLimit conform to existing styles chore: remove unused localization string chore: undo changes to DynamicInput and other strays feat: add fileTokenLimit to all provider config panels fix: move textParsing back into ocr tool_resource block for now so that it doesn't interfere with other upload types * 📤 feat: Add RAG API Endpoint Support for Text Parsing (#8849) * feat: implement RAG API integration for text parsing with fallback to native parsing * chore: remove TODO now that placeholder and fllback are implemented * ✈️ refactor: Migrate Text Parsing to TS (#8892) * refactor: move generateShortLivedToken to packages/api * refactor: move textParsing logic into packages/api * refactor: reduce nesting and dry code with createTextFile * fix: add proper source handling * fix: mock new parseText and parseTextNative functions in jest file * ci: add test coverage for textParser * 💬 feat: Add Audio File Support to Upload as Text (#8893) * feat: add STT support for Upload as Text * refactor: move processAudioFile to packages/api * refactor: move textParsing from utils to files * fix: remove audio/mp3 from unsupported mimetypes test since it is now supported * ✂️ feat: Configurable File Token Limits and Truncation (#8911) * feat: add configurable fileTokenLimit default value * fix: add stt to fileConfig merge logic * fix: add fileTokenLimit to mergeFileConfig logic so configurable value is actually respected from yaml * feat: add token limiting to parsed text files * fix: add extraction logic and update tests so fileTokenLimit isnt sent to LLM providers * fix: address comments * refactor: rename textTokenLimiter.ts to text.ts * chore: update form-data package to address CVE-2025-7783 and update package-lock * feat: use default supported mime types for ocr on frontend file validation * fix: should be using logger.debug not console.debug * fix: mock existsSync in text.spec.ts * fix: mock logger rather than every one of its function calls * fix: reorganize imports and streamline file upload processing logic * refactor: update createTextFile function to use destructured parameters and improve readability * chore: update file validation to use EToolResources for improved type safety * chore: update import path for types in audio processing module * fix: update file configuration access and replace console.debug with logger.debug for improved logging --------- Co-authored-by: Dustin Healy <dustinhealy1@gmail.com> Co-authored-by: Dustin Healy <54083382+dustinhealy@users.noreply.github.com>
This commit is contained in:
Danny Avila
GitHub
parent
74bc0440f0
commit
48f6f8f2f8
41 changed files with 847 additions and 75 deletions
255
packages/api/src/files/text.spec.ts
Normal file
255
packages/api/src/files/text.spec.ts
Normal file
|
|
@ -0,0 +1,255 @@
|
|||
import { FileSources } from 'librechat-data-provider';
|
||||
import { Readable } from 'stream';
|
||||
|
||||
jest.mock('@librechat/data-schemas', () => ({
|
||||
logger: {
|
||||
debug: jest.fn(),
|
||||
warn: jest.fn(),
|
||||
error: jest.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
import { parseTextNative, parseText } from './text';
|
||||
|
||||
jest.mock('fs', () => ({
|
||||
readFileSync: jest.fn(),
|
||||
createReadStream: jest.fn(),
|
||||
}));
|
||||
|
||||
jest.mock('../crypto/jwt', () => ({
|
||||
generateShortLivedToken: jest.fn(),
|
||||
}));
|
||||
|
||||
jest.mock('axios', () => ({
|
||||
get: jest.fn(),
|
||||
post: jest.fn(),
|
||||
interceptors: {
|
||||
request: { use: jest.fn(), eject: jest.fn() },
|
||||
response: { use: jest.fn(), eject: jest.fn() },
|
||||
},
|
||||
}));
|
||||
|
||||
jest.mock('form-data', () => {
|
||||
return jest.fn().mockImplementation(() => ({
|
||||
append: jest.fn(),
|
||||
getHeaders: jest.fn().mockReturnValue({ 'content-type': 'multipart/form-data' }),
|
||||
}));
|
||||
});
|
||||
|
||||
import fs, { ReadStream } from 'fs';
|
||||
import axios from 'axios';
|
||||
import FormData from 'form-data';
|
||||
import { generateShortLivedToken } from '../crypto/jwt';
|
||||
|
||||
const mockedFs = fs as jest.Mocked<typeof fs>;
|
||||
const mockedAxios = axios as jest.Mocked<typeof axios>;
|
||||
const mockedFormData = FormData as jest.MockedClass<typeof FormData>;
|
||||
const mockedGenerateShortLivedToken = generateShortLivedToken as jest.MockedFunction<
|
||||
typeof generateShortLivedToken
|
||||
>;
|
||||
|
||||
describe('text', () => {
|
||||
const mockFile: Express.Multer.File = {
|
||||
fieldname: 'file',
|
||||
originalname: 'test.txt',
|
||||
encoding: '7bit',
|
||||
mimetype: 'text/plain',
|
||||
size: 100,
|
||||
destination: '/tmp',
|
||||
filename: 'test.txt',
|
||||
path: '/tmp/test.txt',
|
||||
buffer: Buffer.from('test content'),
|
||||
stream: new Readable(),
|
||||
};
|
||||
|
||||
const mockReq = {
|
||||
user: { id: 'user123' },
|
||||
};
|
||||
|
||||
const mockFileId = 'file123';
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
delete process.env.RAG_API_URL;
|
||||
});
|
||||
|
||||
describe('parseTextNative', () => {
|
||||
it('should successfully parse a text file', () => {
|
||||
const mockText = 'Hello, world!';
|
||||
mockedFs.readFileSync.mockReturnValue(mockText);
|
||||
|
||||
const result = parseTextNative(mockFile);
|
||||
|
||||
expect(mockedFs.readFileSync).toHaveBeenCalledWith('/tmp/test.txt', 'utf8');
|
||||
expect(result).toEqual({
|
||||
text: mockText,
|
||||
bytes: Buffer.byteLength(mockText, 'utf8'),
|
||||
source: FileSources.text,
|
||||
});
|
||||
});
|
||||
|
||||
it('should throw an error when file cannot be read', () => {
|
||||
const mockError = new Error('File not found');
|
||||
mockedFs.readFileSync.mockImplementation(() => {
|
||||
throw mockError;
|
||||
});
|
||||
|
||||
expect(() => parseTextNative(mockFile)).toThrow(
|
||||
'Failed to read file as text: Error: File not found',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseText', () => {
|
||||
beforeEach(() => {
|
||||
mockedGenerateShortLivedToken.mockReturnValue('mock-jwt-token');
|
||||
|
||||
const mockFormDataInstance = {
|
||||
append: jest.fn(),
|
||||
getHeaders: jest.fn().mockReturnValue({ 'content-type': 'multipart/form-data' }),
|
||||
};
|
||||
mockedFormData.mockImplementation(() => mockFormDataInstance as unknown as FormData);
|
||||
|
||||
mockedFs.createReadStream.mockReturnValue({} as unknown as ReadStream);
|
||||
});
|
||||
|
||||
it('should fall back to native parsing when RAG_API_URL is not defined', async () => {
|
||||
const mockText = 'Native parsing result';
|
||||
mockedFs.readFileSync.mockReturnValue(mockText);
|
||||
|
||||
const result = await parseText({
|
||||
req: mockReq,
|
||||
file: mockFile,
|
||||
file_id: mockFileId,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
text: mockText,
|
||||
bytes: Buffer.byteLength(mockText, 'utf8'),
|
||||
source: FileSources.text,
|
||||
});
|
||||
expect(mockedAxios.get).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should fall back to native parsing when health check fails', async () => {
|
||||
process.env.RAG_API_URL = 'http://rag-api.test';
|
||||
const mockText = 'Native parsing result';
|
||||
mockedFs.readFileSync.mockReturnValue(mockText);
|
||||
|
||||
mockedAxios.get.mockRejectedValue(new Error('Health check failed'));
|
||||
|
||||
const result = await parseText({
|
||||
req: mockReq,
|
||||
file: mockFile,
|
||||
file_id: mockFileId,
|
||||
});
|
||||
|
||||
expect(mockedAxios.get).toHaveBeenCalledWith('http://rag-api.test/health', {
|
||||
timeout: 5000,
|
||||
});
|
||||
expect(result).toEqual({
|
||||
text: mockText,
|
||||
bytes: Buffer.byteLength(mockText, 'utf8'),
|
||||
source: FileSources.text,
|
||||
});
|
||||
});
|
||||
|
||||
it('should fall back to native parsing when health check returns non-OK status', async () => {
|
||||
process.env.RAG_API_URL = 'http://rag-api.test';
|
||||
const mockText = 'Native parsing result';
|
||||
mockedFs.readFileSync.mockReturnValue(mockText);
|
||||
|
||||
mockedAxios.get.mockResolvedValue({
|
||||
status: 500,
|
||||
statusText: 'Internal Server Error',
|
||||
});
|
||||
|
||||
const result = await parseText({
|
||||
req: mockReq,
|
||||
file: mockFile,
|
||||
file_id: mockFileId,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
text: mockText,
|
||||
bytes: Buffer.byteLength(mockText, 'utf8'),
|
||||
source: FileSources.text,
|
||||
});
|
||||
});
|
||||
|
||||
it('should accept empty text as valid RAG API response', async () => {
|
||||
process.env.RAG_API_URL = 'http://rag-api.test';
|
||||
|
||||
mockedAxios.get.mockResolvedValue({
|
||||
status: 200,
|
||||
statusText: 'OK',
|
||||
});
|
||||
|
||||
mockedAxios.post.mockResolvedValue({
|
||||
data: {
|
||||
text: '',
|
||||
},
|
||||
});
|
||||
|
||||
const result = await parseText({
|
||||
req: mockReq,
|
||||
file: mockFile,
|
||||
file_id: mockFileId,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
text: '',
|
||||
bytes: 0,
|
||||
source: FileSources.text,
|
||||
});
|
||||
});
|
||||
|
||||
it('should fall back to native parsing when RAG API response lacks text property', async () => {
|
||||
process.env.RAG_API_URL = 'http://rag-api.test';
|
||||
const mockText = 'Native parsing result';
|
||||
mockedFs.readFileSync.mockReturnValue(mockText);
|
||||
|
||||
mockedAxios.get.mockResolvedValue({
|
||||
status: 200,
|
||||
statusText: 'OK',
|
||||
});
|
||||
|
||||
mockedAxios.post.mockResolvedValue({
|
||||
data: {},
|
||||
});
|
||||
|
||||
const result = await parseText({
|
||||
req: mockReq,
|
||||
file: mockFile,
|
||||
file_id: mockFileId,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
text: mockText,
|
||||
bytes: Buffer.byteLength(mockText, 'utf8'),
|
||||
source: FileSources.text,
|
||||
});
|
||||
});
|
||||
|
||||
it('should fall back to native parsing when user is undefined', async () => {
|
||||
process.env.RAG_API_URL = 'http://rag-api.test';
|
||||
const mockText = 'Native parsing result';
|
||||
mockedFs.readFileSync.mockReturnValue(mockText);
|
||||
|
||||
const result = await parseText({
|
||||
req: { user: undefined },
|
||||
file: mockFile,
|
||||
file_id: mockFileId,
|
||||
});
|
||||
|
||||
expect(mockedGenerateShortLivedToken).not.toHaveBeenCalled();
|
||||
expect(mockedAxios.get).not.toHaveBeenCalled();
|
||||
expect(mockedAxios.post).not.toHaveBeenCalled();
|
||||
expect(result).toEqual({
|
||||
text: mockText,
|
||||
bytes: Buffer.byteLength(mockText, 'utf8'),
|
||||
source: FileSources.text,
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue