diff --git a/api/server/services/Endpoints/agents/initialize.js b/api/server/services/Endpoints/agents/initialize.js index 69767e191c..4f9dfc79d0 100644 --- a/api/server/services/Endpoints/agents/initialize.js +++ b/api/server/services/Endpoints/agents/initialize.js @@ -1,5 +1,5 @@ const { logger } = require('@librechat/data-schemas'); -const { createContentAggregator } = require('@librechat/agents'); +const { Constants, createContentAggregator } = require('@librechat/agents'); const { initializeAgent, validateAgentModel, @@ -23,6 +23,12 @@ const { } = require('~/server/controllers/agents/callbacks'); const { loadAgentTools, loadToolsForExecution } = require('~/server/services/ToolService'); const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions'); +const { + loadCodeApiKey, + provisionToCodeEnv, + provisionToVectorDB, + checkSessionsAlive, +} = require('~/server/services/Files/provision'); const { getModelsConfig } = require('~/server/controllers/ModelController'); const { checkPermission } = require('~/server/services/PermissionService'); const AgentClient = require('~/server/controllers/agents/client'); @@ -142,6 +148,70 @@ const initializeClient = async ({ req, res, signal, endpointOption }) => { return result; }, toolEndCallback, + provisionFiles: async (toolNames, agentId) => { + const ctx = agentToolContexts.get(agentId); + if (!ctx?.provisionState) { + return; + } + + const { provisionState } = ctx; + const needsCode = + toolNames.includes(Constants.EXECUTE_CODE) || + toolNames.includes(Constants.PROGRAMMATIC_TOOL_CALLING); + const needsSearch = toolNames.includes('file_search'); + + if (!needsCode && !needsSearch) { + return; + } + + /** @type {import('@librechat/api').TFileUpdate[]} */ + const pendingUpdates = []; + + if (needsCode && provisionState.codeEnvFiles.length > 0 && provisionState.codeApiKey) { + const results = await Promise.allSettled( + provisionState.codeEnvFiles.map(async (file) => { + const { fileIdentifier, fileUpdate } = await provisionToCodeEnv({ + req, + file, + entity_id: agentId, + apiKey: provisionState.codeApiKey, + }); + file.metadata = { ...file.metadata, fileIdentifier }; + pendingUpdates.push(fileUpdate); + }), + ); + for (const result of results) { + if (result.status === 'rejected') { + logger.error('[provisionFiles] Code env provisioning failed', result.reason); + } + } + provisionState.codeEnvFiles = []; + } + + if (needsSearch && provisionState.vectorDBFiles.length > 0) { + const results = await Promise.allSettled( + provisionState.vectorDBFiles.map(async (file) => { + const result = await provisionToVectorDB({ req, file, entity_id: agentId }); + if (result.embedded) { + file.embedded = true; + if (result.fileUpdate) { + pendingUpdates.push(result.fileUpdate); + } + } + }), + ); + for (const result of results) { + if (result.status === 'rejected') { + logger.error('[provisionFiles] Vector DB provisioning failed', result.reason); + } + } + provisionState.vectorDBFiles = []; + } + + if (pendingUpdates.length > 0) { + await Promise.allSettled(pendingUpdates.map((update) => db.updateFile(update))); + } + }, }; const summarizationOptions = @@ -216,6 +286,11 @@ const initializeClient = async ({ req, res, signal, endpointOption }) => { getToolFilesByIds: db.getToolFilesByIds, getCodeGeneratedFiles: db.getCodeGeneratedFiles, filterFilesByAgentAccess, + provisionToCodeEnv, + provisionToVectorDB, + checkSessionsAlive, + loadCodeApiKey, + updateFile: db.updateFile, }, ); @@ -228,6 +303,7 @@ const initializeClient = async ({ req, res, signal, endpointOption }) => { userMCPAuthMap: primaryConfig.userMCPAuthMap, tool_resources: primaryConfig.tool_resources, actionsEnabled: primaryConfig.actionsEnabled, + provisionState: primaryConfig.provisionState, }); const agent_ids = primaryConfig.agent_ids; @@ -297,6 +373,11 @@ const initializeClient = async ({ req, res, signal, endpointOption }) => { getToolFilesByIds: db.getToolFilesByIds, getCodeGeneratedFiles: db.getCodeGeneratedFiles, filterFilesByAgentAccess, + provisionToCodeEnv, + provisionToVectorDB, + checkSessionsAlive, + loadCodeApiKey, + updateFile: db.updateFile, }, ); @@ -313,6 +394,7 @@ const initializeClient = async ({ req, res, signal, endpointOption }) => { userMCPAuthMap: config.userMCPAuthMap, tool_resources: config.tool_resources, actionsEnabled: config.actionsEnabled, + provisionState: config.provisionState, }); agentConfigs.set(agentId, config); @@ -396,6 +478,7 @@ const initializeClient = async ({ req, res, signal, endpointOption }) => { userMCPAuthMap: config.userMCPAuthMap, tool_resources: config.tool_resources, actionsEnabled: config.actionsEnabled, + provisionState: config.provisionState, }); } diff --git a/api/server/services/Files/process.js b/api/server/services/Files/process.js index f7d7731975..d0927dca2a 100644 --- a/api/server/services/Files/process.js +++ b/api/server/services/Files/process.js @@ -466,6 +466,22 @@ const processFileUpload = async ({ req, res, metadata }) => { * @param {FileMetadata} params.metadata - Additional metadata for the file. * @returns {Promise} */ +/** + * Resolves the file interaction mode from the merged file config. + * Checks endpoint-level config first, then global config. + * Returns 'deferred' as the default when nothing is configured. + * + * @param {object} req - The Express request object + * @param {object} appConfig - The application config + * @returns {string} - The resolved interaction mode: 'text' | 'provider' | 'deferred' | 'legacy' + */ +const resolveInteractionMode = (req, appConfig) => { + const fileConfig = mergeFileConfig(appConfig?.fileConfig); + const endpoint = req.body?.endpoint; + const endpointConfig = getEndpointFileConfig({ fileConfig, endpoint }); + return endpointConfig?.defaultFileInteraction ?? fileConfig?.defaultFileInteraction ?? 'deferred'; +}; + const processAgentFileUpload = async ({ req, res, metadata }) => { const { file } = req; const appConfig = req.config; @@ -473,11 +489,19 @@ const processAgentFileUpload = async ({ req, res, metadata }) => { let messageAttachment = !!metadata.message_file; + let effectiveToolResource = tool_resource; if (agent_id && !tool_resource && !messageAttachment) { - throw new Error('No tool resource provided for agent file upload'); + const interactionMode = resolveInteractionMode(req, appConfig); + if (interactionMode === 'legacy') { + throw new Error('No tool resource provided for agent file upload'); + } + // In unified mode: 'text' routes to context processing, 'deferred'/'provider' fall through to standard storage + if (interactionMode === 'text') { + effectiveToolResource = EToolResources.context; + } } - if (tool_resource === EToolResources.file_search && file.mimetype.startsWith('image')) { + if (effectiveToolResource === EToolResources.file_search && file.mimetype.startsWith('image')) { throw new Error('Image uploads are not supported for file search tool resources'); } @@ -489,7 +513,7 @@ const processAgentFileUpload = async ({ req, res, metadata }) => { let fileInfoMetadata; const entity_id = messageAttachment === true ? undefined : agent_id; const basePath = mime.getType(file.originalname)?.startsWith('image') ? 'images' : 'uploads'; - if (tool_resource === EToolResources.execute_code) { + if (effectiveToolResource === EToolResources.execute_code) { const isCodeEnabled = await checkCapability(req, AgentCapabilities.execute_code); if (!isCodeEnabled) { throw new Error('Code execution is not enabled for Agents'); @@ -505,13 +529,13 @@ const processAgentFileUpload = async ({ req, res, metadata }) => { entity_id, }); fileInfoMetadata = { fileIdentifier }; - } else if (tool_resource === EToolResources.file_search) { + } else if (effectiveToolResource === EToolResources.file_search) { const isFileSearchEnabled = await checkCapability(req, AgentCapabilities.file_search); if (!isFileSearchEnabled) { throw new Error('File search is not enabled for Agents'); } // Note: File search processing continues to dual storage logic below - } else if (tool_resource === EToolResources.context) { + } else if (effectiveToolResource === EToolResources.context) { const { file_id, temp_file_id = null } = metadata; /** @@ -543,11 +567,11 @@ const processAgentFileUpload = async ({ req, res, metadata }) => { context: messageAttachment ? FileContext.message_attachment : FileContext.agents, }); - if (!messageAttachment && tool_resource) { + if (!messageAttachment && effectiveToolResource) { await db.addAgentResourceFile({ file_id, agent_id, - tool_resource, + tool_resource: effectiveToolResource, updatingUserId: req?.user?.id, }); } @@ -636,7 +660,7 @@ const processAgentFileUpload = async ({ req, res, metadata }) => { const isImageFile = file.mimetype.startsWith('image'); const source = getFileStrategy(appConfig, { isImage: isImageFile }); - if (tool_resource === EToolResources.file_search) { + if (effectiveToolResource === EToolResources.file_search) { // FIRST: Upload to Storage for permanent backup (S3/local/etc.) const { handleFileUpload } = getStrategyFunctions(source); const sanitizedUploadFn = createSanitizedUploadWrapper(handleFileUpload); @@ -676,18 +700,18 @@ const processAgentFileUpload = async ({ req, res, metadata }) => { let { bytes, filename, filepath: _filepath, height, width } = storageResult; // For RAG files, use embedding result; for others, use storage result let embedded = storageResult.embedded; - if (tool_resource === EToolResources.file_search) { + if (effectiveToolResource === EToolResources.file_search) { embedded = embeddingResult?.embedded; filename = embeddingResult?.filename || filename; } let filepath = _filepath; - if (!messageAttachment && tool_resource) { + if (!messageAttachment && effectiveToolResource) { await db.addAgentResourceFile({ file_id, agent_id, - tool_resource, + tool_resource: effectiveToolResource, updatingUserId: req?.user?.id, }); } diff --git a/api/server/services/Files/provision.js b/api/server/services/Files/provision.js new file mode 100644 index 0000000000..1db7afd341 --- /dev/null +++ b/api/server/services/Files/provision.js @@ -0,0 +1,275 @@ +const fs = require('fs'); +const path = require('path'); +const os = require('os'); +const { EnvVar, getCodeBaseURL } = require('@librechat/agents'); +const { + logAxiosError, + createAxiosInstance, + codeServerHttpAgent, + codeServerHttpsAgent, +} = require('@librechat/api'); +const { logger } = require('@librechat/data-schemas'); +const { FileSources } = require('librechat-data-provider'); +const { loadAuthValues } = require('~/server/services/Tools/credentials'); +const { getStrategyFunctions } = require('./strategies'); + +const axios = createAxiosInstance(); + +/** + * Loads the CODE_API_KEY for a user. Call once per request and pass the result + * to provisionToCodeEnv / checkSessionsAlive to avoid redundant lookups. + * + * @param {string} userId + * @returns {Promise} The CODE_API_KEY + */ +async function loadCodeApiKey(userId) { + const result = await loadAuthValues({ userId, authFields: [EnvVar.CODE_API_KEY] }); + return result[EnvVar.CODE_API_KEY]; +} + +/** + * Provisions a file to the code execution environment. + * Gets a read stream from our storage and uploads to the code env. + * + * @param {object} params + * @param {object} params.req - Express request object (needs req.user.id) + * @param {import('librechat-data-provider').TFile} params.file - The file record from DB + * @param {string} [params.entity_id] - Optional entity ID (agent_id) + * @param {string} [params.apiKey] - Pre-loaded CODE_API_KEY (avoids redundant loadAuthValues) + * @returns {Promise<{ fileIdentifier: string, fileUpdate: object }>} Result with deferred DB update + */ +async function provisionToCodeEnv({ req, file, entity_id = '', apiKey }) { + const { getDownloadStream } = getStrategyFunctions(file.source); + if (!getDownloadStream) { + throw new Error( + `Cannot provision file "${file.filename}" to code env: storage source "${file.source}" does not support download streams`, + ); + } + + const resolvedApiKey = apiKey ?? (await loadCodeApiKey(req.user.id)); + const { handleFileUpload: uploadCodeEnvFile } = getStrategyFunctions(FileSources.execute_code); + const stream = await getDownloadStream(req, file.filepath); + + const fileIdentifier = await uploadCodeEnvFile({ + req, + stream, + filename: file.filename, + apiKey: resolvedApiKey, + entity_id, + }); + + logger.debug( + `[provisionToCodeEnv] Provisioned file "${file.filename}" (${file.file_id}) to code env`, + ); + + return { + fileIdentifier, + fileUpdate: { file_id: file.file_id, metadata: { ...file.metadata, fileIdentifier } }, + }; +} + +/** + * Provisions a file to the vector DB for file_search/RAG. + * Gets the file from our storage and uploads vectors/embeddings. + * + * @param {object} params + * @param {object} params.req - Express request object + * @param {import('librechat-data-provider').TFile} params.file - The file record from DB + * @param {string} [params.entity_id] - Optional entity ID (agent_id) + * @param {import('stream').Readable} [params.existingStream] - Pre-fetched download stream (avoids duplicate storage fetch) + * @returns {Promise<{ embedded: boolean, fileUpdate: object | null }>} Result with deferred DB update + */ +async function provisionToVectorDB({ req, file, entity_id, existingStream }) { + if (!process.env.RAG_API_URL) { + logger.warn('[provisionToVectorDB] RAG_API_URL not defined, skipping vector provisioning'); + return { embedded: false, fileUpdate: null }; + } + + const tmpPath = path.join(os.tmpdir(), `provision-${file.file_id}${path.extname(file.filename)}`); + + try { + let stream = existingStream; + if (!stream) { + const { getDownloadStream } = getStrategyFunctions(file.source); + if (!getDownloadStream) { + throw new Error( + `Cannot provision file "${file.filename}" to vector DB: storage source "${file.source}" does not support download streams`, + ); + } + stream = await getDownloadStream(req, file.filepath); + } + + // uploadVectors expects a file-like object with a `path` property for fs.createReadStream. + // Since we're provisioning from storage (not a multer upload), we stream to a temp file first. + await new Promise((resolve, reject) => { + const writeStream = fs.createWriteStream(tmpPath); + stream.pipe(writeStream); + writeStream.on('finish', resolve); + writeStream.on('error', reject); + stream.on('error', reject); + }); + + const { uploadVectors } = require('./VectorDB/crud'); + const tempFile = { + path: tmpPath, + originalname: file.filename, + mimetype: file.type, + size: file.bytes, + }; + + const embeddingResult = await uploadVectors({ + req, + file: tempFile, + file_id: file.file_id, + entity_id, + }); + + const embedded = embeddingResult?.embedded ?? false; + + logger.debug( + `[provisionToVectorDB] Provisioned file "${file.filename}" (${file.file_id}) to vector DB, embedded=${embedded}`, + ); + + return { + embedded, + fileUpdate: embedded ? { file_id: file.file_id, embedded } : null, + }; + } finally { + try { + fs.unlinkSync(tmpPath); + } catch { + // Ignore cleanup errors + } + } +} + +/** + * Check if a single code env file is still alive by querying its session. + * + * @param {object} params + * @param {import('librechat-data-provider').TFile} params.file - File with metadata.fileIdentifier + * @param {string} params.apiKey - CODE_API_KEY + * @returns {Promise} true if the file is still accessible in the code env + */ +async function checkCodeEnvFileAlive({ file, apiKey }) { + if (!file.metadata?.fileIdentifier) { + return false; + } + + try { + const baseURL = getCodeBaseURL(); + const [filePath, queryString] = file.metadata.fileIdentifier.split('?'); + const session_id = filePath.split('/')[0]; + + let queryParams = {}; + if (queryString) { + queryParams = Object.fromEntries(new URLSearchParams(queryString).entries()); + } + + const response = await axios({ + method: 'get', + url: `${baseURL}/files/${session_id}`, + params: { detail: 'summary', ...queryParams }, + headers: { + 'User-Agent': 'LibreChat/1.0', + 'X-API-Key': apiKey, + }, + httpAgent: codeServerHttpAgent, + httpsAgent: codeServerHttpsAgent, + timeout: 5000, + }); + + const found = response.data?.some((f) => f.name?.startsWith(filePath)); + return !!found; + } catch (error) { + logAxiosError({ + message: `[checkCodeEnvFileAlive] Error checking file "${file.filename}": ${error.message}`, + error, + }); + return false; + } +} + +/** + * Batch-check code env file liveness by session_id. + * Groups files by session, makes one API call per session. + * + * @param {object} params + * @param {import('librechat-data-provider').TFile[]} params.files - Files with metadata.fileIdentifier + * @param {string} params.apiKey - Pre-loaded CODE_API_KEY + * @param {number} [params.staleSafeWindowMs=21600000] - Skip check if file updated within this window (default 6h) + * @returns {Promise>} Set of file_ids that are confirmed alive + */ +async function checkSessionsAlive({ files, apiKey, staleSafeWindowMs = 6 * 60 * 60 * 1000 }) { + const aliveFileIds = new Set(); + const now = Date.now(); + + // Group files by session_id, skip recently-updated files (fast pre-filter) + /** @type {Map>} */ + const sessionGroups = new Map(); + + for (const file of files) { + if (!file.metadata?.fileIdentifier) { + continue; + } + + const updatedAt = file.updatedAt ? new Date(file.updatedAt).getTime() : 0; + if (now - updatedAt < staleSafeWindowMs) { + aliveFileIds.add(file.file_id); + continue; + } + + const [filePath] = file.metadata.fileIdentifier.split('?'); + const session_id = filePath.split('/')[0]; + + if (!sessionGroups.has(session_id)) { + sessionGroups.set(session_id, []); + } + sessionGroups.get(session_id).push({ file_id: file.file_id, filePath }); + } + + // One API call per session (in parallel) + const baseURL = getCodeBaseURL(); + const sessionChecks = Array.from(sessionGroups.entries()).map( + async ([session_id, fileEntries]) => { + try { + const response = await axios({ + method: 'get', + url: `${baseURL}/files/${session_id}`, + params: { detail: 'summary' }, + headers: { + 'User-Agent': 'LibreChat/1.0', + 'X-API-Key': apiKey, + }, + httpAgent: codeServerHttpAgent, + httpsAgent: codeServerHttpsAgent, + timeout: 5000, + }); + + const remoteFiles = response.data ?? []; + for (const { file_id, filePath } of fileEntries) { + if (remoteFiles.some((f) => f.name?.startsWith(filePath))) { + aliveFileIds.add(file_id); + } + } + } catch (error) { + logAxiosError({ + message: `[checkSessionsAlive] Error checking session "${session_id}": ${error.message}`, + error, + }); + // All files in this session treated as expired + } + }, + ); + + await Promise.allSettled(sessionChecks); + return aliveFileIds; +} + +module.exports = { + loadCodeApiKey, + provisionToCodeEnv, + provisionToVectorDB, + checkCodeEnvFileAlive, + checkSessionsAlive, +}; diff --git a/client/src/components/Chat/Input/Files/AttachFileMenu.tsx b/client/src/components/Chat/Input/Files/AttachFileMenu.tsx index 181d219c08..d1f4e7105c 100644 --- a/client/src/components/Chat/Input/Files/AttachFileMenu.tsx +++ b/client/src/components/Chat/Input/Files/AttachFileMenu.tsx @@ -110,6 +110,10 @@ const AttachFileMenu = ({ ephemeralAgent, ); + const isUnifiedMode = + endpointFileConfig?.defaultFileInteraction != null && + endpointFileConfig.defaultFileInteraction !== 'legacy'; + const handleUploadClick = (fileType?: FileUploadType) => { if (!inputRef.current) { return; @@ -132,6 +136,12 @@ const AttachFileMenu = ({ inputRef.current.accept = ''; }; + /** Unified mode: single click triggers file upload with no tool_resource */ + const handleUnifiedUpload = () => { + setToolResource(undefined); + handleUploadClick(); + }; + const dropdownItems = useMemo(() => { const createMenuItems = (onAction: (fileType?: FileUploadType) => void) => { const items: MenuItemProps[] = []; @@ -286,6 +296,47 @@ const AttachFileMenu = ({ } }; + if (isUnifiedMode) { + return ( + <> + { + handleFileChange(e, toolResource); + }} + > + +
+ +
+ + } + id="attach-file-button" + description={localize('com_sidepanel_attach_files')} + disabled={isUploadDisabled} + /> + + + + ); + } + return ( <> ; /** Callback to process tool artifacts (code output files, file citations, etc.) */ toolEndCallback?: ToolEndCallback; + /** Called once per batch before tool execution to lazily provision files to tool environments */ + provisionFiles?: (toolNames: string[], agentId?: string) => Promise; } /** @@ -51,7 +53,7 @@ export interface ToolExecuteOptions { * executes them in parallel, and resolves with the results. */ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHandler { - const { loadTools, toolEndCallback } = options; + const { loadTools, toolEndCallback, provisionFiles } = options; return { handle: async (_event: string, data: ToolExecuteBatchRequest) => { @@ -61,6 +63,11 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand await runOutsideTracing(async () => { try { const toolNames = [...new Set(toolCalls.map((tc: ToolCallRequest) => tc.name))]; + + if (provisionFiles) { + await provisionFiles(toolNames, agentId); + } + const { loadedTools, configurable: toolConfigurable } = await loadTools( toolNames, agentId, diff --git a/packages/api/src/agents/initialize.ts b/packages/api/src/agents/initialize.ts index 81bc89cac4..e01f57098a 100644 --- a/packages/api/src/agents/initialize.ts +++ b/packages/api/src/agents/initialize.ts @@ -31,7 +31,15 @@ import { filterFilesByEndpointConfig } from '~/files'; import { generateArtifactsPrompt } from '~/prompts'; import { getProviderConfig } from '~/endpoints'; import { primeResources } from './resources'; -import type { TFilterFilesByAgentAccess } from './resources'; +import type { ProvisionState } from './resources'; +import type { + TFileUpdate, + TFilterFilesByAgentAccess, + TProvisionToCodeEnv, + TProvisionToVectorDB, + TCheckSessionsAlive, + TLoadCodeApiKey, +} from './resources'; /** * Fraction of context budget reserved as headroom when no explicit maxContextTokens is set. @@ -66,6 +74,10 @@ export type InitializedAgent = Agent & { actionsEnabled?: boolean; /** Maximum characters allowed in a single tool result before truncation. */ maxToolResultChars?: number; + /** Warnings from lazy file provisioning (e.g., failed uploads) */ + provisionWarnings?: string[]; + /** State for deferred file provisioning — actual uploads happen at tool invocation time */ + provisionState?: ProvisionState; }; /** @@ -143,6 +155,16 @@ export interface InitializeAgentDbMethods extends EndpointDbMethods { parentMessageId?: string; files?: Array<{ file_id: string }>; }> | null>; + /** Optional: provision a file to the code execution environment */ + provisionToCodeEnv?: TProvisionToCodeEnv; + /** Optional: provision a file to the vector DB for file_search */ + provisionToVectorDB?: TProvisionToVectorDB; + /** Optional: batch-check code env file liveness */ + checkSessionsAlive?: TCheckSessionsAlive; + /** Optional: load CODE_API_KEY once per request */ + loadCodeApiKey?: TLoadCodeApiKey; + /** Optional: persist file metadata updates after provisioning */ + updateFile?: (data: TFileUpdate) => Promise; } /** @@ -205,6 +227,14 @@ export async function initializeAgent( const provider = agent.provider; agent.endpoint = provider; + /** Build the set of tool resources the agent has enabled */ + const toolResourceSet = new Set(); + for (const tool of agent.tools ?? []) { + if (EToolResources[tool as keyof typeof EToolResources]) { + toolResourceSet.add(EToolResources[tool as keyof typeof EToolResources]); + } + } + /** * Load conversation files for ALL agents, not just the initial agent. * This enables handoff agents to access files that were uploaded earlier @@ -213,12 +243,6 @@ export async function initializeAgent( */ if (conversationId != null && resendFiles) { const fileIds = (await db.getConvoFiles(conversationId)) ?? []; - const toolResourceSet = new Set(); - for (const tool of agent.tools ?? []) { - if (EToolResources[tool as keyof typeof EToolResources]) { - toolResourceSet.add(EToolResources[tool as keyof typeof EToolResources]); - } - } const toolFiles = (await db.getToolFilesByIds(fileIds, toolResourceSet)) as IMongoFile[]; @@ -282,7 +306,12 @@ export async function initializeAgent( }); } - const { attachments: primedAttachments, tool_resources } = await primeResources({ + const { + attachments: primedAttachments, + tool_resources, + provisionState, + warnings: provisionWarnings, + } = await primeResources({ req: req as never, getFiles: db.getFiles as never, filterFiles: db.filterFilesByAgentAccess, @@ -293,6 +322,9 @@ export async function initializeAgent( : undefined, tool_resources: agent.tool_resources, requestFileSet: new Set(requestFiles?.map((file) => file.file_id)), + enabledToolResources: toolResourceSet, + checkSessionsAlive: db.checkSessionsAlive, + loadCodeApiKey: db.loadCodeApiKey, }); const { @@ -450,6 +482,9 @@ export async function initializeAgent( useLegacyContent: !!options.useLegacyContent, tools: (tools ?? []) as GenericTool[] & string[], maxToolResultChars: maxToolResultCharsResolved, + provisionState, + provisionWarnings: + provisionWarnings != null && provisionWarnings.length > 0 ? provisionWarnings : undefined, maxContextTokens: maxContextTokens != null && maxContextTokens > 0 ? maxContextTokens diff --git a/packages/api/src/agents/resources.test.ts b/packages/api/src/agents/resources.test.ts index 641fb9284c..d1017319cb 100644 --- a/packages/api/src/agents/resources.test.ts +++ b/packages/api/src/agents/resources.test.ts @@ -108,7 +108,7 @@ describe('primeResources', () => { }); expect(mockGetFiles).not.toHaveBeenCalled(); - expect(result.attachments).toBeUndefined(); + expect(result.attachments).toEqual([]); expect(result.tool_resources).toEqual(tool_resources); }); }); @@ -1334,7 +1334,7 @@ describe('primeResources', () => { role: 'USER', agentId: 'agent_shared', }); - expect(result.attachments).toBeUndefined(); + expect(result.attachments).toEqual([]); }); it('should skip filtering when filterFiles is not provided', async () => { @@ -1502,8 +1502,8 @@ describe('primeResources', () => { expect(mockGetFiles).not.toHaveBeenCalled(); // When appConfig agents endpoint is missing, context is disabled - // and no attachments are provided, the function returns undefined - expect(result.attachments).toBeUndefined(); + // and no attachments are provided, the function returns an empty array + expect(result.attachments).toEqual([]); }); it('should handle undefined tool_resources', async () => { @@ -1517,7 +1517,7 @@ describe('primeResources', () => { }); expect(result.tool_resources).toEqual({}); - expect(result.attachments).toBeUndefined(); + expect(result.attachments).toEqual([]); }); it('should handle empty requestFileSet', async () => { diff --git a/packages/api/src/agents/resources.ts b/packages/api/src/agents/resources.ts index e147c743cf..0983bd51ca 100644 --- a/packages/api/src/agents/resources.ts +++ b/packages/api/src/agents/resources.ts @@ -5,6 +5,61 @@ import type { IMongoFile, AppConfig, IUser } from '@librechat/data-schemas'; import type { FilterQuery, QueryOptions, ProjectionType } from 'mongoose'; import type { Request as ServerRequest } from 'express'; +/** Deferred DB update from provisioning (batched after all files are provisioned) */ +export type TFileUpdate = { + file_id: string; + metadata?: Record; + embedded?: boolean; +}; + +/** + * Function type for provisioning a file to the code execution environment. + * @returns The fileIdentifier and a deferred DB update object + */ +export type TProvisionToCodeEnv = (params: { + req: ServerRequest & { user?: IUser }; + file: TFile; + entity_id?: string; + apiKey?: string; +}) => Promise<{ fileIdentifier: string; fileUpdate: TFileUpdate }>; + +/** + * Function type for provisioning a file to the vector DB for file_search. + * @returns Object with embedded status and a deferred DB update object + */ +export type TProvisionToVectorDB = (params: { + req: ServerRequest & { user?: IUser }; + file: TFile; + entity_id?: string; + existingStream?: unknown; +}) => Promise<{ embedded: boolean; fileUpdate: TFileUpdate | null }>; + +/** + * Function type for batch-checking code env file liveness. + * Groups files by session, makes one API call per session. + * @returns Set of file_ids that are confirmed alive + */ +export type TCheckSessionsAlive = (params: { + files: TFile[]; + apiKey: string; + staleSafeWindowMs?: number; +}) => Promise>; + +/** Loads CODE_API_KEY for a user. Call once per request. */ +export type TLoadCodeApiKey = (userId: string) => Promise; + +/** State computed during primeResources for lazy provisioning at tool invocation time */ +export type ProvisionState = { + /** Files that need uploading to the code execution environment */ + codeEnvFiles: TFile[]; + /** Files that need embedding into the vector DB for file_search */ + vectorDBFiles: TFile[]; + /** Pre-loaded CODE_API_KEY to avoid redundant credential fetches */ + codeApiKey?: string; + /** Set of file_ids confirmed alive in code env (from staleness check) */ + aliveFileIds: Set; +}; + /** * Function type for retrieving files from the database * @param filter - MongoDB filter query for files @@ -39,7 +94,7 @@ export type TFilterFilesByAgentAccess = (params: { * @param params.tool_resources - The agent's tool resources object to update * @param params.processedResourceFiles - Set tracking processed files per resource type */ -const addFileToResource = ({ +export const addFileToResource = ({ file, resourceType, tool_resources, @@ -100,6 +155,7 @@ const categorizeFileForToolResources = ({ requestFileSet: Set; processedResourceFiles: Set; }): void => { + // No early returns — a file can belong to multiple tool resources simultaneously if (file.metadata?.fileIdentifier) { addFileToResource({ file, @@ -107,7 +163,6 @@ const categorizeFileForToolResources = ({ tool_resources, processedResourceFiles, }); - return; } if (file.embedded === true) { @@ -117,7 +172,6 @@ const categorizeFileForToolResources = ({ tool_resources, processedResourceFiles, }); - return; } if ( @@ -163,6 +217,9 @@ export const primeResources = async ({ attachments: _attachments, tool_resources: _tool_resources, agentId, + enabledToolResources, + checkSessionsAlive, + loadCodeApiKey, }: { req: ServerRequest & { user?: IUser }; appConfig?: AppConfig; @@ -172,9 +229,17 @@ export const primeResources = async ({ getFiles: TGetFiles; filterFiles?: TFilterFilesByAgentAccess; agentId?: string; + /** Set of tool resource types the agent has enabled (e.g., execute_code, file_search) */ + enabledToolResources?: Set; + /** Optional callback to batch-check code env file liveness by session */ + checkSessionsAlive?: TCheckSessionsAlive; + /** Optional callback to load CODE_API_KEY once per request */ + loadCodeApiKey?: TLoadCodeApiKey; }): Promise<{ - attachments: Array | undefined; + attachments: Array; tool_resources: AgentToolResources | undefined; + provisionState?: ProvisionState; + warnings: string[]; }> => { try { /** @@ -282,7 +347,7 @@ export const primeResources = async ({ } if (!_attachments) { - return { attachments: attachments.length > 0 ? attachments : undefined, tool_resources }; + return { attachments, tool_resources, warnings: [] }; } const files = await _attachments; @@ -309,7 +374,97 @@ export const primeResources = async ({ } } - return { attachments: attachments.length > 0 ? attachments : [], tool_resources }; + /** + * Lazy provisioning: instead of provisioning files now, compute which files + * need provisioning and return that state. Actual provisioning happens at + * tool invocation time via the ON_TOOL_EXECUTE handler. + */ + const warnings: string[] = []; + let provisionState: ProvisionState | undefined; + + if (enabledToolResources && enabledToolResources.size > 0 && attachments.length > 0) { + const needsCodeEnv = enabledToolResources.has(EToolResources.execute_code); + const needsVectorDB = enabledToolResources.has(EToolResources.file_search); + + if (needsCodeEnv || needsVectorDB) { + let codeApiKey: string | undefined; + if (needsCodeEnv && loadCodeApiKey && req.user?.id) { + try { + codeApiKey = await loadCodeApiKey(req.user.id); + } catch (error) { + logger.error('[primeResources] Failed to load CODE_API_KEY', error); + warnings.push('Code execution file provisioning unavailable'); + } + } + + // Batch staleness check: identify which code env files are still alive + let aliveFileIds: Set = new Set(); + if (needsCodeEnv && codeApiKey && checkSessionsAlive) { + const filesWithIdentifiers = attachments.filter( + (f) => f?.metadata?.fileIdentifier && f.file_id, + ); + if (filesWithIdentifiers.length > 0) { + aliveFileIds = await checkSessionsAlive({ + files: filesWithIdentifiers as TFile[], + apiKey: codeApiKey, + }); + } + } + + // Compute which files need provisioning (don't actually provision yet) + const codeEnvFiles: TFile[] = []; + const vectorDBFiles: TFile[] = []; + + for (const file of attachments) { + if (!file?.file_id) { + continue; + } + + if ( + needsCodeEnv && + codeApiKey && + !processedResourceFiles.has(`${EToolResources.execute_code}:${file.file_id}`) + ) { + const hasFileIdentifier = !!file.metadata?.fileIdentifier; + const isStale = hasFileIdentifier && !aliveFileIds.has(file.file_id); + + if (!hasFileIdentifier || isStale) { + if (isStale) { + logger.info( + `[primeResources] Code env file expired for "${file.filename}" (${file.file_id}), will re-provision on tool use`, + ); + file.metadata = { ...file.metadata, fileIdentifier: undefined }; + } + codeEnvFiles.push(file); + } else { + // File is alive, categorize it now + addFileToResource({ + file, + resourceType: EToolResources.execute_code, + tool_resources, + processedResourceFiles, + }); + } + } + + const isImage = file.type?.startsWith('image') ?? false; + if ( + needsVectorDB && + !isImage && + file.embedded !== true && + !processedResourceFiles.has(`${EToolResources.file_search}:${file.file_id}`) + ) { + vectorDBFiles.push(file); + } + } + + if (codeEnvFiles.length > 0 || vectorDBFiles.length > 0) { + provisionState = { codeEnvFiles, vectorDBFiles, codeApiKey, aliveFileIds }; + } + } + } + + return { attachments, tool_resources, provisionState, warnings }; } catch (error) { logger.error('Error priming resources', error); @@ -329,6 +484,8 @@ export const primeResources = async ({ return { attachments: safeAttachments, tool_resources: _tool_resources, + provisionState: undefined, + warnings: [], }; } }; diff --git a/packages/data-provider/src/file-config.ts b/packages/data-provider/src/file-config.ts index 7ec184755d..f3ade42020 100644 --- a/packages/data-provider/src/file-config.ts +++ b/packages/data-provider/src/file-config.ts @@ -444,12 +444,16 @@ export const fileConfig = { const supportedMimeTypesSchema = z.array(z.string()).optional(); +export const FileInteractionMode = z.enum(['text', 'provider', 'deferred', 'legacy']); +export type TFileInteractionMode = z.infer; + export const endpointFileConfigSchema = z.object({ disabled: z.boolean().optional(), fileLimit: z.number().min(0).optional(), fileSizeLimit: z.number().min(0).optional(), totalSizeLimit: z.number().min(0).optional(), supportedMimeTypes: supportedMimeTypesSchema.optional(), + defaultFileInteraction: FileInteractionMode.optional(), }); export const fileConfigSchema = z.object({ @@ -481,6 +485,7 @@ export const fileConfigSchema = z.object({ supportedMimeTypes: supportedMimeTypesSchema.optional(), }) .optional(), + defaultFileInteraction: FileInteractionMode.optional(), }); export type TFileConfig = z.infer; @@ -526,6 +531,8 @@ function mergeWithDefault( fileSizeLimit: endpointConfig.fileSizeLimit ?? defaultConfig.fileSizeLimit, totalSizeLimit: endpointConfig.totalSizeLimit ?? defaultConfig.totalSizeLimit, supportedMimeTypes: endpointConfig.supportedMimeTypes ?? defaultMimeTypes, + defaultFileInteraction: + endpointConfig.defaultFileInteraction ?? defaultConfig.defaultFileInteraction, }; } @@ -654,6 +661,10 @@ export function mergeFileConfig(dynamic: z.infer | unde return mergedConfig; } + if (dynamic.defaultFileInteraction !== undefined) { + mergedConfig.defaultFileInteraction = dynamic.defaultFileInteraction; + } + if (dynamic.serverFileSizeLimit !== undefined) { mergedConfig.serverFileSizeLimit = mbToBytes(dynamic.serverFileSizeLimit); } @@ -745,6 +756,10 @@ export function mergeFileConfig(dynamic: z.infer | unde dynamicEndpoint.supportedMimeTypes as unknown as string[], ); } + + if (dynamicEndpoint.defaultFileInteraction !== undefined) { + mergedEndpoint.defaultFileInteraction = dynamicEndpoint.defaultFileInteraction; + } } return mergedConfig; diff --git a/packages/data-provider/src/types/files.ts b/packages/data-provider/src/types/files.ts index 1eb8c200d6..4d97011df1 100644 --- a/packages/data-provider/src/types/files.ts +++ b/packages/data-provider/src/types/files.ts @@ -42,6 +42,7 @@ export type EndpointFileConfig = { fileSizeLimit?: number; totalSizeLimit?: number; supportedMimeTypes?: RegExp[]; + defaultFileInteraction?: 'text' | 'provider' | 'deferred' | 'legacy'; }; export type FileConfig = { @@ -67,6 +68,7 @@ export type FileConfig = { supportedMimeTypes?: RegExp[]; }; checkType?: (fileType: string, supportedTypes: RegExp[]) => boolean; + defaultFileInteraction?: 'text' | 'provider' | 'deferred' | 'legacy'; }; export type FileConfigInput = { @@ -91,6 +93,7 @@ export type FileConfigInput = { supportedMimeTypes?: string[]; }; checkType?: (fileType: string, supportedTypes: RegExp[]) => boolean; + defaultFileInteraction?: 'text' | 'provider' | 'deferred' | 'legacy'; }; export type TFile = {