mirror of
https://github.com/danny-avila/LibreChat.git
synced 2025-12-17 08:50:15 +01:00
📎 feat: Upload as Text Support for Plaintext, STT, RAG, and Token Limits (#8868)
* 🪶 feat: Add Support for Uploading Plaintext Files feat: delineate between OCR and text handling in fileConfig field of config file - also adds support for passing in mimetypes as just plain file extensions feat: add showLabel bool to support future synthetic component DynamicDropdownInput feat: add new combination dropdown-input component in params panel to support file type token limits refactor: move hovercard to side to align with other hovercards chore: clean up autogenerated comments feat: add delineation to file upload path between text and ocr configured filetypes feat: add token limit checks during file upload refactor: move textParsing out of ocrEnabled logic refactor: clean up types for filetype config refactor: finish decoupling DynamicDropdownInput from fileTokenLimits fix: move image token cost function into file to fix circular dependency causing unittest to fail and remove unused var for linter chore: remove out of scope code following review refactor: make fileTokenLimit conform to existing styles chore: remove unused localization string chore: undo changes to DynamicInput and other strays feat: add fileTokenLimit to all provider config panels fix: move textParsing back into ocr tool_resource block for now so that it doesn't interfere with other upload types * 📤 feat: Add RAG API Endpoint Support for Text Parsing (#8849) * feat: implement RAG API integration for text parsing with fallback to native parsing * chore: remove TODO now that placeholder and fllback are implemented * ✈️ refactor: Migrate Text Parsing to TS (#8892) * refactor: move generateShortLivedToken to packages/api * refactor: move textParsing logic into packages/api * refactor: reduce nesting and dry code with createTextFile * fix: add proper source handling * fix: mock new parseText and parseTextNative functions in jest file * ci: add test coverage for textParser * 💬 feat: Add Audio File Support to Upload as Text (#8893) * feat: add STT support for Upload as Text * refactor: move processAudioFile to packages/api * refactor: move textParsing from utils to files * fix: remove audio/mp3 from unsupported mimetypes test since it is now supported * ✂️ feat: Configurable File Token Limits and Truncation (#8911) * feat: add configurable fileTokenLimit default value * fix: add stt to fileConfig merge logic * fix: add fileTokenLimit to mergeFileConfig logic so configurable value is actually respected from yaml * feat: add token limiting to parsed text files * fix: add extraction logic and update tests so fileTokenLimit isnt sent to LLM providers * fix: address comments * refactor: rename textTokenLimiter.ts to text.ts * chore: update form-data package to address CVE-2025-7783 and update package-lock * feat: use default supported mime types for ocr on frontend file validation * fix: should be using logger.debug not console.debug * fix: mock existsSync in text.spec.ts * fix: mock logger rather than every one of its function calls * fix: reorganize imports and streamline file upload processing logic * refactor: update createTextFile function to use destructured parameters and improve readability * chore: update file validation to use EToolResources for improved type safety * chore: update import path for types in audio processing module * fix: update file configuration access and replace console.debug with logger.debug for improved logging --------- Co-authored-by: Dustin Healy <dustinhealy1@gmail.com> Co-authored-by: Dustin Healy <54083382+dustinhealy@users.noreply.github.com>
This commit is contained in:
Danny Avila
GitHub
parent
74bc0440f0
commit
48f6f8f2f8
41 changed files with 847 additions and 75 deletions
|
|
@ -17,7 +17,8 @@ const {
|
|||
isAssistantsEndpoint,
|
||||
} = require('librechat-data-provider');
|
||||
const { EnvVar } = require('@librechat/agents');
|
||||
const { sanitizeFilename } = require('@librechat/api');
|
||||
const { logger } = require('@librechat/data-schemas');
|
||||
const { sanitizeFilename, parseText, processAudioFile } = require('@librechat/api');
|
||||
const {
|
||||
convertImage,
|
||||
resizeAndConvert,
|
||||
|
|
@ -33,7 +34,7 @@ const { checkCapability } = require('~/server/services/Config');
|
|||
const { LB_QueueAsyncCall } = require('~/server/utils/queue');
|
||||
const { getStrategyFunctions } = require('./strategies');
|
||||
const { determineFileType } = require('~/server/utils');
|
||||
const { logger } = require('~/config');
|
||||
const { STTService } = require('./Audio/STTService');
|
||||
|
||||
/**
|
||||
* Creates a modular file upload wrapper that ensures filename sanitization
|
||||
|
|
@ -552,51 +553,84 @@ const processAgentFileUpload = async ({ req, res, metadata }) => {
|
|||
}
|
||||
// Note: File search processing continues to dual storage logic below
|
||||
} else if (tool_resource === EToolResources.ocr) {
|
||||
const isOCREnabled = await checkCapability(req, AgentCapabilities.ocr);
|
||||
if (!isOCREnabled) {
|
||||
throw new Error('OCR capability is not enabled for Agents');
|
||||
}
|
||||
|
||||
const { handleFileUpload: uploadOCR } = getStrategyFunctions(
|
||||
appConfig?.ocr?.strategy ?? FileSources.mistral_ocr,
|
||||
);
|
||||
const { file_id, temp_file_id = null } = metadata;
|
||||
|
||||
const {
|
||||
text,
|
||||
bytes,
|
||||
// TODO: OCR images support?
|
||||
images: _i,
|
||||
filename,
|
||||
filepath: ocrFileURL,
|
||||
} = await uploadOCR({ req, appConfig, file, loadAuthValues });
|
||||
|
||||
const fileInfo = removeNullishValues({
|
||||
text,
|
||||
bytes,
|
||||
file_id,
|
||||
temp_file_id,
|
||||
user: req.user.id,
|
||||
type: 'text/plain',
|
||||
filepath: ocrFileURL,
|
||||
source: FileSources.text,
|
||||
filename: filename ?? file.originalname,
|
||||
model: messageAttachment ? undefined : req.body.model,
|
||||
context: messageAttachment ? FileContext.message_attachment : FileContext.agents,
|
||||
});
|
||||
|
||||
if (!messageAttachment && tool_resource) {
|
||||
await addAgentResourceFile({
|
||||
req,
|
||||
/**
|
||||
* @param {object} params
|
||||
* @param {string} params.text
|
||||
* @param {number} params.bytes
|
||||
* @param {string} params.filepath
|
||||
* @param {string} params.type
|
||||
* @return {Promise<void>}
|
||||
*/
|
||||
const createTextFile = async ({ text, bytes, filepath, type = 'text/plain' }) => {
|
||||
const fileInfo = removeNullishValues({
|
||||
text,
|
||||
bytes,
|
||||
file_id,
|
||||
agent_id,
|
||||
tool_resource,
|
||||
temp_file_id,
|
||||
user: req.user.id,
|
||||
type,
|
||||
filepath: filepath ?? file.path,
|
||||
source: FileSources.text,
|
||||
filename: file.originalname,
|
||||
model: messageAttachment ? undefined : req.body.model,
|
||||
context: messageAttachment ? FileContext.message_attachment : FileContext.agents,
|
||||
});
|
||||
|
||||
if (!messageAttachment && tool_resource) {
|
||||
await addAgentResourceFile({
|
||||
req,
|
||||
file_id,
|
||||
agent_id,
|
||||
tool_resource,
|
||||
});
|
||||
}
|
||||
const result = await createFile(fileInfo, true);
|
||||
return res
|
||||
.status(200)
|
||||
.json({ message: 'Agent file uploaded and processed successfully', ...result });
|
||||
};
|
||||
|
||||
const fileConfig = mergeFileConfig(appConfig.fileConfig);
|
||||
|
||||
const shouldUseOCR = fileConfig.checkType(
|
||||
file.mimetype,
|
||||
fileConfig.ocr?.supportedMimeTypes || [],
|
||||
);
|
||||
|
||||
if (shouldUseOCR && !(await checkCapability(req, AgentCapabilities.ocr))) {
|
||||
throw new Error('OCR capability is not enabled for Agents');
|
||||
} else if (shouldUseOCR) {
|
||||
const { handleFileUpload: uploadOCR } = getStrategyFunctions(
|
||||
appConfig?.ocr?.strategy ?? FileSources.mistral_ocr,
|
||||
);
|
||||
const { text, bytes, filepath: ocrFileURL } = await uploadOCR({ req, file, loadAuthValues });
|
||||
return await createTextFile({ text, bytes, filepath: ocrFileURL });
|
||||
}
|
||||
const result = await createFile(fileInfo, true);
|
||||
return res
|
||||
.status(200)
|
||||
.json({ message: 'Agent file uploaded and processed successfully', ...result });
|
||||
|
||||
const shouldUseSTT = fileConfig.checkType(
|
||||
file.mimetype,
|
||||
fileConfig.stt?.supportedMimeTypes || [],
|
||||
);
|
||||
|
||||
if (shouldUseSTT) {
|
||||
const sttService = await STTService.getInstance();
|
||||
const { text, bytes } = await processAudioFile({ file, sttService });
|
||||
return await createTextFile({ text, bytes });
|
||||
}
|
||||
|
||||
const shouldUseText = fileConfig.checkType(
|
||||
file.mimetype,
|
||||
fileConfig.text?.supportedMimeTypes || [],
|
||||
);
|
||||
|
||||
if (!shouldUseText) {
|
||||
throw new Error(`File type ${file.mimetype} is not supported for OCR or text parsing`);
|
||||
}
|
||||
|
||||
const { text, bytes } = await parseText({ req, file, file_id });
|
||||
return await createTextFile({ text, bytes, type: file.mimetype });
|
||||
}
|
||||
|
||||
// Dual storage pattern for RAG files: Storage + Vector DB
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue