diff --git a/api/server/services/Endpoints/agents/initialize.js b/api/server/services/Endpoints/agents/initialize.js index 69767e191c..1fd326709a 100644 --- a/api/server/services/Endpoints/agents/initialize.js +++ b/api/server/services/Endpoints/agents/initialize.js @@ -23,6 +23,7 @@ const { } = require('~/server/controllers/agents/callbacks'); const { loadAgentTools, loadToolsForExecution } = require('~/server/services/ToolService'); const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions'); +const { provisionToCodeEnv, provisionToVectorDB } = require('~/server/services/Files/provision'); const { getModelsConfig } = require('~/server/controllers/ModelController'); const { checkPermission } = require('~/server/services/PermissionService'); const AgentClient = require('~/server/controllers/agents/client'); @@ -216,6 +217,8 @@ const initializeClient = async ({ req, res, signal, endpointOption }) => { getToolFilesByIds: db.getToolFilesByIds, getCodeGeneratedFiles: db.getCodeGeneratedFiles, filterFilesByAgentAccess, + provisionToCodeEnv, + provisionToVectorDB, }, ); @@ -297,6 +300,8 @@ const initializeClient = async ({ req, res, signal, endpointOption }) => { getToolFilesByIds: db.getToolFilesByIds, getCodeGeneratedFiles: db.getCodeGeneratedFiles, filterFilesByAgentAccess, + provisionToCodeEnv, + provisionToVectorDB, }, ); diff --git a/api/server/services/Files/process.js b/api/server/services/Files/process.js index f7d7731975..d0927dca2a 100644 --- a/api/server/services/Files/process.js +++ b/api/server/services/Files/process.js @@ -466,6 +466,22 @@ const processFileUpload = async ({ req, res, metadata }) => { * @param {FileMetadata} params.metadata - Additional metadata for the file. * @returns {Promise} */ +/** + * Resolves the file interaction mode from the merged file config. + * Checks endpoint-level config first, then global config. + * Returns 'deferred' as the default when nothing is configured. + * + * @param {object} req - The Express request object + * @param {object} appConfig - The application config + * @returns {string} - The resolved interaction mode: 'text' | 'provider' | 'deferred' | 'legacy' + */ +const resolveInteractionMode = (req, appConfig) => { + const fileConfig = mergeFileConfig(appConfig?.fileConfig); + const endpoint = req.body?.endpoint; + const endpointConfig = getEndpointFileConfig({ fileConfig, endpoint }); + return endpointConfig?.defaultFileInteraction ?? fileConfig?.defaultFileInteraction ?? 'deferred'; +}; + const processAgentFileUpload = async ({ req, res, metadata }) => { const { file } = req; const appConfig = req.config; @@ -473,11 +489,19 @@ const processAgentFileUpload = async ({ req, res, metadata }) => { let messageAttachment = !!metadata.message_file; + let effectiveToolResource = tool_resource; if (agent_id && !tool_resource && !messageAttachment) { - throw new Error('No tool resource provided for agent file upload'); + const interactionMode = resolveInteractionMode(req, appConfig); + if (interactionMode === 'legacy') { + throw new Error('No tool resource provided for agent file upload'); + } + // In unified mode: 'text' routes to context processing, 'deferred'/'provider' fall through to standard storage + if (interactionMode === 'text') { + effectiveToolResource = EToolResources.context; + } } - if (tool_resource === EToolResources.file_search && file.mimetype.startsWith('image')) { + if (effectiveToolResource === EToolResources.file_search && file.mimetype.startsWith('image')) { throw new Error('Image uploads are not supported for file search tool resources'); } @@ -489,7 +513,7 @@ const processAgentFileUpload = async ({ req, res, metadata }) => { let fileInfoMetadata; const entity_id = messageAttachment === true ? undefined : agent_id; const basePath = mime.getType(file.originalname)?.startsWith('image') ? 'images' : 'uploads'; - if (tool_resource === EToolResources.execute_code) { + if (effectiveToolResource === EToolResources.execute_code) { const isCodeEnabled = await checkCapability(req, AgentCapabilities.execute_code); if (!isCodeEnabled) { throw new Error('Code execution is not enabled for Agents'); @@ -505,13 +529,13 @@ const processAgentFileUpload = async ({ req, res, metadata }) => { entity_id, }); fileInfoMetadata = { fileIdentifier }; - } else if (tool_resource === EToolResources.file_search) { + } else if (effectiveToolResource === EToolResources.file_search) { const isFileSearchEnabled = await checkCapability(req, AgentCapabilities.file_search); if (!isFileSearchEnabled) { throw new Error('File search is not enabled for Agents'); } // Note: File search processing continues to dual storage logic below - } else if (tool_resource === EToolResources.context) { + } else if (effectiveToolResource === EToolResources.context) { const { file_id, temp_file_id = null } = metadata; /** @@ -543,11 +567,11 @@ const processAgentFileUpload = async ({ req, res, metadata }) => { context: messageAttachment ? FileContext.message_attachment : FileContext.agents, }); - if (!messageAttachment && tool_resource) { + if (!messageAttachment && effectiveToolResource) { await db.addAgentResourceFile({ file_id, agent_id, - tool_resource, + tool_resource: effectiveToolResource, updatingUserId: req?.user?.id, }); } @@ -636,7 +660,7 @@ const processAgentFileUpload = async ({ req, res, metadata }) => { const isImageFile = file.mimetype.startsWith('image'); const source = getFileStrategy(appConfig, { isImage: isImageFile }); - if (tool_resource === EToolResources.file_search) { + if (effectiveToolResource === EToolResources.file_search) { // FIRST: Upload to Storage for permanent backup (S3/local/etc.) const { handleFileUpload } = getStrategyFunctions(source); const sanitizedUploadFn = createSanitizedUploadWrapper(handleFileUpload); @@ -676,18 +700,18 @@ const processAgentFileUpload = async ({ req, res, metadata }) => { let { bytes, filename, filepath: _filepath, height, width } = storageResult; // For RAG files, use embedding result; for others, use storage result let embedded = storageResult.embedded; - if (tool_resource === EToolResources.file_search) { + if (effectiveToolResource === EToolResources.file_search) { embedded = embeddingResult?.embedded; filename = embeddingResult?.filename || filename; } let filepath = _filepath; - if (!messageAttachment && tool_resource) { + if (!messageAttachment && effectiveToolResource) { await db.addAgentResourceFile({ file_id, agent_id, - tool_resource, + tool_resource: effectiveToolResource, updatingUserId: req?.user?.id, }); } diff --git a/api/server/services/Files/provision.js b/api/server/services/Files/provision.js new file mode 100644 index 0000000000..e70c9abc03 --- /dev/null +++ b/api/server/services/Files/provision.js @@ -0,0 +1,135 @@ +const fs = require('fs'); +const { EnvVar } = require('@librechat/agents'); +const { logger } = require('@librechat/data-schemas'); +const { FileSources } = require('librechat-data-provider'); +const { loadAuthValues } = require('~/server/services/Tools/credentials'); +const { getStrategyFunctions } = require('./strategies'); +const { updateFile } = require('~/models'); + +/** + * Provisions a file to the code execution environment. + * Gets a read stream from our storage and uploads to the code env. + * + * @param {object} params + * @param {object} params.req - Express request object (needs req.user.id) + * @param {import('librechat-data-provider').TFile} params.file - The file record from DB + * @param {string} [params.entity_id] - Optional entity ID (agent_id) + * @returns {Promise} The fileIdentifier from the code env + */ +async function provisionToCodeEnv({ req, file, entity_id = '' }) { + const { getDownloadStream } = getStrategyFunctions(file.source); + if (!getDownloadStream) { + throw new Error( + `Cannot provision file "${file.filename}" to code env: storage source "${file.source}" does not support download streams`, + ); + } + + const { handleFileUpload: uploadCodeEnvFile } = getStrategyFunctions(FileSources.execute_code); + const result = await loadAuthValues({ userId: req.user.id, authFields: [EnvVar.CODE_API_KEY] }); + const stream = await getDownloadStream(req, file.filepath); + + const fileIdentifier = await uploadCodeEnvFile({ + req, + stream, + filename: file.filename, + apiKey: result[EnvVar.CODE_API_KEY], + entity_id, + }); + + const updatedMetadata = { + ...file.metadata, + fileIdentifier, + }; + + await updateFile({ + file_id: file.file_id, + metadata: updatedMetadata, + }); + + logger.debug( + `[provisionToCodeEnv] Provisioned file "${file.filename}" (${file.file_id}) to code env`, + ); + + return fileIdentifier; +} + +/** + * Provisions a file to the vector DB for file_search/RAG. + * Gets the file from our storage and uploads vectors/embeddings. + * + * @param {object} params + * @param {object} params.req - Express request object + * @param {import('librechat-data-provider').TFile} params.file - The file record from DB + * @param {string} [params.entity_id] - Optional entity ID (agent_id) + * @returns {Promise<{ embedded: boolean }>} Embedding result + */ +async function provisionToVectorDB({ req, file, entity_id }) { + if (!process.env.RAG_API_URL) { + logger.warn('[provisionToVectorDB] RAG_API_URL not defined, skipping vector provisioning'); + return { embedded: false }; + } + + const { getDownloadStream } = getStrategyFunctions(file.source); + if (!getDownloadStream) { + throw new Error( + `Cannot provision file "${file.filename}" to vector DB: storage source "${file.source}" does not support download streams`, + ); + } + + // The uploadVectors function expects a file-like object with a `path` property for fs.createReadStream. + // Since we're provisioning from storage (not a multer upload), we need to stream to a temp file first. + const os = require('os'); + const path = require('path'); + const tmpPath = path.join(os.tmpdir(), `provision-${file.file_id}-${file.filename}`); + + try { + const stream = await getDownloadStream(req, file.filepath); + await new Promise((resolve, reject) => { + const writeStream = fs.createWriteStream(tmpPath); + stream.pipe(writeStream); + writeStream.on('finish', resolve); + writeStream.on('error', reject); + stream.on('error', reject); + }); + + const { uploadVectors } = require('./VectorDB/crud'); + const tempFile = { + path: tmpPath, + originalname: file.filename, + mimetype: file.type, + size: file.bytes, + }; + + const embeddingResult = await uploadVectors({ + req, + file: tempFile, + file_id: file.file_id, + entity_id, + }); + + const embedded = embeddingResult?.embedded ?? false; + + await updateFile({ + file_id: file.file_id, + embedded, + }); + + logger.debug( + `[provisionToVectorDB] Provisioned file "${file.filename}" (${file.file_id}) to vector DB, embedded=${embedded}`, + ); + + return { embedded }; + } finally { + // Clean up temp file + try { + fs.unlinkSync(tmpPath); + } catch { + // Ignore cleanup errors + } + } +} + +module.exports = { + provisionToCodeEnv, + provisionToVectorDB, +}; diff --git a/packages/api/src/agents/initialize.ts b/packages/api/src/agents/initialize.ts index 81bc89cac4..0d07cc59dc 100644 --- a/packages/api/src/agents/initialize.ts +++ b/packages/api/src/agents/initialize.ts @@ -31,7 +31,11 @@ import { filterFilesByEndpointConfig } from '~/files'; import { generateArtifactsPrompt } from '~/prompts'; import { getProviderConfig } from '~/endpoints'; import { primeResources } from './resources'; -import type { TFilterFilesByAgentAccess } from './resources'; +import type { + TFilterFilesByAgentAccess, + TProvisionToCodeEnv, + TProvisionToVectorDB, +} from './resources'; /** * Fraction of context budget reserved as headroom when no explicit maxContextTokens is set. @@ -143,6 +147,10 @@ export interface InitializeAgentDbMethods extends EndpointDbMethods { parentMessageId?: string; files?: Array<{ file_id: string }>; }> | null>; + /** Optional: provision a file to the code execution environment */ + provisionToCodeEnv?: TProvisionToCodeEnv; + /** Optional: provision a file to the vector DB for file_search */ + provisionToVectorDB?: TProvisionToVectorDB; } /** @@ -205,6 +213,14 @@ export async function initializeAgent( const provider = agent.provider; agent.endpoint = provider; + /** Build the set of tool resources the agent has enabled */ + const toolResourceSet = new Set(); + for (const tool of agent.tools ?? []) { + if (EToolResources[tool as keyof typeof EToolResources]) { + toolResourceSet.add(EToolResources[tool as keyof typeof EToolResources]); + } + } + /** * Load conversation files for ALL agents, not just the initial agent. * This enables handoff agents to access files that were uploaded earlier @@ -213,12 +229,6 @@ export async function initializeAgent( */ if (conversationId != null && resendFiles) { const fileIds = (await db.getConvoFiles(conversationId)) ?? []; - const toolResourceSet = new Set(); - for (const tool of agent.tools ?? []) { - if (EToolResources[tool as keyof typeof EToolResources]) { - toolResourceSet.add(EToolResources[tool as keyof typeof EToolResources]); - } - } const toolFiles = (await db.getToolFilesByIds(fileIds, toolResourceSet)) as IMongoFile[]; @@ -293,6 +303,9 @@ export async function initializeAgent( : undefined, tool_resources: agent.tool_resources, requestFileSet: new Set(requestFiles?.map((file) => file.file_id)), + enabledToolResources: toolResourceSet, + provisionToCodeEnv: db.provisionToCodeEnv, + provisionToVectorDB: db.provisionToVectorDB, }); const { diff --git a/packages/api/src/agents/resources.ts b/packages/api/src/agents/resources.ts index e147c743cf..581d2d8f07 100644 --- a/packages/api/src/agents/resources.ts +++ b/packages/api/src/agents/resources.ts @@ -5,6 +5,26 @@ import type { IMongoFile, AppConfig, IUser } from '@librechat/data-schemas'; import type { FilterQuery, QueryOptions, ProjectionType } from 'mongoose'; import type { Request as ServerRequest } from 'express'; +/** + * Function type for provisioning a file to the code execution environment. + * @returns The fileIdentifier from the code env + */ +export type TProvisionToCodeEnv = (params: { + req: ServerRequest & { user?: IUser }; + file: TFile; + entity_id?: string; +}) => Promise; + +/** + * Function type for provisioning a file to the vector DB for file_search. + * @returns Object with embedded status + */ +export type TProvisionToVectorDB = (params: { + req: ServerRequest & { user?: IUser }; + file: TFile; + entity_id?: string; +}) => Promise<{ embedded: boolean }>; + /** * Function type for retrieving files from the database * @param filter - MongoDB filter query for files @@ -100,6 +120,7 @@ const categorizeFileForToolResources = ({ requestFileSet: Set; processedResourceFiles: Set; }): void => { + // No early returns — a file can belong to multiple tool resources simultaneously if (file.metadata?.fileIdentifier) { addFileToResource({ file, @@ -107,7 +128,6 @@ const categorizeFileForToolResources = ({ tool_resources, processedResourceFiles, }); - return; } if (file.embedded === true) { @@ -117,7 +137,6 @@ const categorizeFileForToolResources = ({ tool_resources, processedResourceFiles, }); - return; } if ( @@ -163,6 +182,9 @@ export const primeResources = async ({ attachments: _attachments, tool_resources: _tool_resources, agentId, + enabledToolResources, + provisionToCodeEnv, + provisionToVectorDB, }: { req: ServerRequest & { user?: IUser }; appConfig?: AppConfig; @@ -172,6 +194,12 @@ export const primeResources = async ({ getFiles: TGetFiles; filterFiles?: TFilterFilesByAgentAccess; agentId?: string; + /** Set of tool resource types the agent has enabled (e.g., execute_code, file_search) */ + enabledToolResources?: Set; + /** Optional callback to provision a file to the code execution environment */ + provisionToCodeEnv?: TProvisionToCodeEnv; + /** Optional callback to provision a file to the vector DB for file_search */ + provisionToVectorDB?: TProvisionToVectorDB; }): Promise<{ attachments: Array | undefined; tool_resources: AgentToolResources | undefined; @@ -309,6 +337,87 @@ export const primeResources = async ({ } } + /** + * Lazy provisioning: for deferred files that haven't been provisioned to the + * agent's enabled tool resources, provision them now (at chat-request start). + * This handles files uploaded via the unified upload flow (no tool_resource chosen at upload time). + */ + if (enabledToolResources && enabledToolResources.size > 0 && attachments.length > 0) { + const needsCodeEnv = + enabledToolResources.has(EToolResources.execute_code) && provisionToCodeEnv != null; + const needsVectorDB = + enabledToolResources.has(EToolResources.file_search) && provisionToVectorDB != null; + + if (needsCodeEnv || needsVectorDB) { + for (const file of attachments) { + if (!file?.file_id) { + continue; + } + + // Skip images for file_search (not supported) + const isImage = file.type?.startsWith('image') ?? false; + + // Provision to code env if needed and not already provisioned + if ( + needsCodeEnv && + !file.metadata?.fileIdentifier && + !processedResourceFiles.has(`${EToolResources.execute_code}:${file.file_id}`) + ) { + try { + const fileIdentifier = await provisionToCodeEnv({ + req: req as ServerRequest & { user?: IUser }, + file, + entity_id: agentId, + }); + // Update the file object in-place so categorization picks it up + file.metadata = { ...file.metadata, fileIdentifier }; + addFileToResource({ + file, + resourceType: EToolResources.execute_code, + tool_resources, + processedResourceFiles, + }); + } catch (error) { + logger.error( + `[primeResources] Failed to provision file "${file.filename}" to code env`, + error, + ); + } + } + + // Provision to vector DB if needed and not already provisioned + if ( + needsVectorDB && + !isImage && + file.embedded !== true && + !processedResourceFiles.has(`${EToolResources.file_search}:${file.file_id}`) + ) { + try { + const result = await provisionToVectorDB({ + req: req as ServerRequest & { user?: IUser }, + file, + entity_id: agentId, + }); + if (result.embedded) { + file.embedded = true; + addFileToResource({ + file, + resourceType: EToolResources.file_search, + tool_resources, + processedResourceFiles, + }); + } + } catch (error) { + logger.error( + `[primeResources] Failed to provision file "${file.filename}" to vector DB`, + error, + ); + } + } + } + } + } + return { attachments: attachments.length > 0 ? attachments : [], tool_resources }; } catch (error) { logger.error('Error priming resources', error); diff --git a/packages/data-provider/src/file-config.ts b/packages/data-provider/src/file-config.ts index 32a1a28cc9..d98cd5cc04 100644 --- a/packages/data-provider/src/file-config.ts +++ b/packages/data-provider/src/file-config.ts @@ -459,12 +459,16 @@ const supportedMimeTypesSchema = z }, ); +export const FileInteractionMode = z.enum(['text', 'provider', 'deferred', 'legacy']); +export type TFileInteractionMode = z.infer; + export const endpointFileConfigSchema = z.object({ disabled: z.boolean().optional(), fileLimit: z.number().min(0).optional(), fileSizeLimit: z.number().min(0).optional(), totalSizeLimit: z.number().min(0).optional(), supportedMimeTypes: supportedMimeTypesSchema.optional(), + defaultFileInteraction: FileInteractionMode.optional(), }); export const fileConfigSchema = z.object({ @@ -496,6 +500,7 @@ export const fileConfigSchema = z.object({ supportedMimeTypes: supportedMimeTypesSchema.optional(), }) .optional(), + defaultFileInteraction: FileInteractionMode.optional(), }); export type TFileConfig = z.infer; @@ -541,6 +546,8 @@ function mergeWithDefault( fileSizeLimit: endpointConfig.fileSizeLimit ?? defaultConfig.fileSizeLimit, totalSizeLimit: endpointConfig.totalSizeLimit ?? defaultConfig.totalSizeLimit, supportedMimeTypes: endpointConfig.supportedMimeTypes ?? defaultMimeTypes, + defaultFileInteraction: + endpointConfig.defaultFileInteraction ?? defaultConfig.defaultFileInteraction, }; } @@ -669,6 +676,10 @@ export function mergeFileConfig(dynamic: z.infer | unde return mergedConfig; } + if (dynamic.defaultFileInteraction !== undefined) { + mergedConfig.defaultFileInteraction = dynamic.defaultFileInteraction; + } + if (dynamic.serverFileSizeLimit !== undefined) { mergedConfig.serverFileSizeLimit = mbToBytes(dynamic.serverFileSizeLimit); } @@ -758,6 +769,10 @@ export function mergeFileConfig(dynamic: z.infer | unde dynamicEndpoint.supportedMimeTypes as unknown as string[], ); } + + if (dynamicEndpoint.defaultFileInteraction !== undefined) { + mergedEndpoint.defaultFileInteraction = dynamicEndpoint.defaultFileInteraction; + } } return mergedConfig; diff --git a/packages/data-provider/src/types/files.ts b/packages/data-provider/src/types/files.ts index 1eb8c200d6..4d97011df1 100644 --- a/packages/data-provider/src/types/files.ts +++ b/packages/data-provider/src/types/files.ts @@ -42,6 +42,7 @@ export type EndpointFileConfig = { fileSizeLimit?: number; totalSizeLimit?: number; supportedMimeTypes?: RegExp[]; + defaultFileInteraction?: 'text' | 'provider' | 'deferred' | 'legacy'; }; export type FileConfig = { @@ -67,6 +68,7 @@ export type FileConfig = { supportedMimeTypes?: RegExp[]; }; checkType?: (fileType: string, supportedTypes: RegExp[]) => boolean; + defaultFileInteraction?: 'text' | 'provider' | 'deferred' | 'legacy'; }; export type FileConfigInput = { @@ -91,6 +93,7 @@ export type FileConfigInput = { supportedMimeTypes?: string[]; }; checkType?: (fileType: string, supportedTypes: RegExp[]) => boolean; + defaultFileInteraction?: 'text' | 'provider' | 'deferred' | 'legacy'; }; export type TFile = {