🔧 feat: Unified file experience — schema, deferred upload, lazy provisioning

Introduces the foundation for a unified file upload experience where users upload files once without choosing a tool_resource upfront. Files are stored in the configured storage strategy and lazily provisioned to tool environments (execute_code, file_search) at chat-request time based on agent capabilities. Phase 1 - Schema + Server-Side Unified Upload: - Add FileInteractionMode enum (text/provider/deferred/legacy) to fileConfigSchema - Add defaultFileInteraction field to EndpointFileConfig and FileConfig types - Update mergeFileConfig/mergeWithDefault to propagate the new field - Modify processAgentFileUpload to support uploads without tool_resource using effectiveToolResource resolved from config (default: deferred) Phase 2 - Lazy Provisioning + Multi-Resource Support: - Create provision.js with provisionToCodeEnv and provisionToVectorDB - Extend primeResources with lazy provisioning step that provisions deferred files to enabled tool environments at chat-request start - Remove early returns in categorizeFileForToolResources so files can exist in multiple tool_resources simultaneously - Wire provisioning callbacks through initializeAgent dependency injection
2026-04-07 00:15:23 +02:00 · 2026-03-21 18:07:14 -04:00 · 2026-03-21 18:07:14 -04:00 · bdf4ab6043
commit bdf4ab6043
parent 04e65bb21a
7 changed files with 324 additions and 20 deletions
--- a/api/server/services/Endpoints/agents/initialize.js
+++ b/api/server/services/Endpoints/agents/initialize.js
@ -23,6 +23,7 @@ const {
 } = require('~/server/controllers/agents/callbacks');
 const { loadAgentTools, loadToolsForExecution } = require('~/server/services/ToolService');
 const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions');
+const { provisionToCodeEnv, provisionToVectorDB } = require('~/server/services/Files/provision');
 const { getModelsConfig } = require('~/server/controllers/ModelController');
 const { checkPermission } = require('~/server/services/PermissionService');
 const AgentClient = require('~/server/controllers/agents/client');
@ -216,6 +217,8 @@ const initializeClient = async ({ req, res, signal, endpointOption }) => {
      getToolFilesByIds: db.getToolFilesByIds,
      getCodeGeneratedFiles: db.getCodeGeneratedFiles,
      filterFilesByAgentAccess,
+      provisionToCodeEnv,
+      provisionToVectorDB,
    },
  );

@ -297,6 +300,8 @@ const initializeClient = async ({ req, res, signal, endpointOption }) => {
        getToolFilesByIds: db.getToolFilesByIds,
        getCodeGeneratedFiles: db.getCodeGeneratedFiles,
        filterFilesByAgentAccess,
+        provisionToCodeEnv,
+        provisionToVectorDB,
      },
    );

--- a/api/server/services/Files/process.js
+++ b/api/server/services/Files/process.js
@ -466,6 +466,22 @@ const processFileUpload = async ({ req, res, metadata }) => {
 * @param {FileMetadata} params.metadata - Additional metadata for the file.
 * @returns {Promise<void>}
 */
+/**
+ * Resolves the file interaction mode from the merged file config.
+ * Checks endpoint-level config first, then global config.
+ * Returns 'deferred' as the default when nothing is configured.
+ *
+ * @param {object} req - The Express request object
+ * @param {object} appConfig - The application config
+ * @returns {string} - The resolved interaction mode: 'text' | 'provider' | 'deferred' | 'legacy'
+ */
+const resolveInteractionMode = (req, appConfig) => {
+  const fileConfig = mergeFileConfig(appConfig?.fileConfig);
+  const endpoint = req.body?.endpoint;
+  const endpointConfig = getEndpointFileConfig({ fileConfig, endpoint });
+  return endpointConfig?.defaultFileInteraction ?? fileConfig?.defaultFileInteraction ?? 'deferred';
+};
+
 const processAgentFileUpload = async ({ req, res, metadata }) => {
  const { file } = req;
  const appConfig = req.config;
@ -473,11 +489,19 @@ const processAgentFileUpload = async ({ req, res, metadata }) => {

  let messageAttachment = !!metadata.message_file;

+  let effectiveToolResource = tool_resource;
  if (agent_id && !tool_resource && !messageAttachment) {
-    throw new Error('No tool resource provided for agent file upload');
+    const interactionMode = resolveInteractionMode(req, appConfig);
+    if (interactionMode === 'legacy') {
+      throw new Error('No tool resource provided for agent file upload');
+    }
+    // In unified mode: 'text' routes to context processing, 'deferred'/'provider' fall through to standard storage
+    if (interactionMode === 'text') {
+      effectiveToolResource = EToolResources.context;
+    }
  }

-  if (tool_resource === EToolResources.file_search && file.mimetype.startsWith('image')) {
+  if (effectiveToolResource === EToolResources.file_search && file.mimetype.startsWith('image')) {
    throw new Error('Image uploads are not supported for file search tool resources');
  }

@ -489,7 +513,7 @@ const processAgentFileUpload = async ({ req, res, metadata }) => {
  let fileInfoMetadata;
  const entity_id = messageAttachment === true ? undefined : agent_id;
  const basePath = mime.getType(file.originalname)?.startsWith('image') ? 'images' : 'uploads';
-  if (tool_resource === EToolResources.execute_code) {
+  if (effectiveToolResource === EToolResources.execute_code) {
    const isCodeEnabled = await checkCapability(req, AgentCapabilities.execute_code);
    if (!isCodeEnabled) {
      throw new Error('Code execution is not enabled for Agents');
@ -505,13 +529,13 @@ const processAgentFileUpload = async ({ req, res, metadata }) => {
      entity_id,
    });
    fileInfoMetadata = { fileIdentifier };
-  } else if (tool_resource === EToolResources.file_search) {
+  } else if (effectiveToolResource === EToolResources.file_search) {
    const isFileSearchEnabled = await checkCapability(req, AgentCapabilities.file_search);
    if (!isFileSearchEnabled) {
      throw new Error('File search is not enabled for Agents');
    }
    // Note: File search processing continues to dual storage logic below
-  } else if (tool_resource === EToolResources.context) {
+  } else if (effectiveToolResource === EToolResources.context) {
    const { file_id, temp_file_id = null } = metadata;

    /**
@ -543,11 +567,11 @@ const processAgentFileUpload = async ({ req, res, metadata }) => {
        context: messageAttachment ? FileContext.message_attachment : FileContext.agents,
      });

-      if (!messageAttachment && tool_resource) {
+      if (!messageAttachment && effectiveToolResource) {
        await db.addAgentResourceFile({
          file_id,
          agent_id,
-          tool_resource,
+          tool_resource: effectiveToolResource,
          updatingUserId: req?.user?.id,
        });
      }
@ -636,7 +660,7 @@ const processAgentFileUpload = async ({ req, res, metadata }) => {
  const isImageFile = file.mimetype.startsWith('image');
  const source = getFileStrategy(appConfig, { isImage: isImageFile });

-  if (tool_resource === EToolResources.file_search) {
+  if (effectiveToolResource === EToolResources.file_search) {
    // FIRST: Upload to Storage for permanent backup (S3/local/etc.)
    const { handleFileUpload } = getStrategyFunctions(source);
    const sanitizedUploadFn = createSanitizedUploadWrapper(handleFileUpload);
@ -676,18 +700,18 @@ const processAgentFileUpload = async ({ req, res, metadata }) => {
  let { bytes, filename, filepath: _filepath, height, width } = storageResult;
  // For RAG files, use embedding result; for others, use storage result
  let embedded = storageResult.embedded;
-  if (tool_resource === EToolResources.file_search) {
+  if (effectiveToolResource === EToolResources.file_search) {
    embedded = embeddingResult?.embedded;
    filename = embeddingResult?.filename || filename;
  }

  let filepath = _filepath;

-  if (!messageAttachment && tool_resource) {
+  if (!messageAttachment && effectiveToolResource) {
    await db.addAgentResourceFile({
      file_id,
      agent_id,
-      tool_resource,
+      tool_resource: effectiveToolResource,
      updatingUserId: req?.user?.id,
    });
  }
--- a/api/server/services/Files/provision.js
+++ b/api/server/services/Files/provision.js
@ -0,0 +1,135 @@
+const fs = require('fs');
+const { EnvVar } = require('@librechat/agents');
+const { logger } = require('@librechat/data-schemas');
+const { FileSources } = require('librechat-data-provider');
+const { loadAuthValues } = require('~/server/services/Tools/credentials');
+const { getStrategyFunctions } = require('./strategies');
+const { updateFile } = require('~/models');
+
+/**
+ * Provisions a file to the code execution environment.
+ * Gets a read stream from our storage and uploads to the code env.
+ *
+ * @param {object} params
+ * @param {object} params.req - Express request object (needs req.user.id)
+ * @param {import('librechat-data-provider').TFile} params.file - The file record from DB
+ * @param {string} [params.entity_id] - Optional entity ID (agent_id)
+ * @returns {Promise<string>} The fileIdentifier from the code env
+ */
+async function provisionToCodeEnv({ req, file, entity_id = '' }) {
+  const { getDownloadStream } = getStrategyFunctions(file.source);
+  if (!getDownloadStream) {
+    throw new Error(
+      `Cannot provision file "${file.filename}" to code env: storage source "${file.source}" does not support download streams`,
+    );
+  }
+
+  const { handleFileUpload: uploadCodeEnvFile } = getStrategyFunctions(FileSources.execute_code);
+  const result = await loadAuthValues({ userId: req.user.id, authFields: [EnvVar.CODE_API_KEY] });
+  const stream = await getDownloadStream(req, file.filepath);
+
+  const fileIdentifier = await uploadCodeEnvFile({
+    req,
+    stream,
+    filename: file.filename,
+    apiKey: result[EnvVar.CODE_API_KEY],
+    entity_id,
+  });
+
+  const updatedMetadata = {
+    ...file.metadata,
+    fileIdentifier,
+  };
+
+  await updateFile({
+    file_id: file.file_id,
+    metadata: updatedMetadata,
+  });
+
+  logger.debug(
+    `[provisionToCodeEnv] Provisioned file "${file.filename}" (${file.file_id}) to code env`,
+  );
+
+  return fileIdentifier;
+}
+
+/**
+ * Provisions a file to the vector DB for file_search/RAG.
+ * Gets the file from our storage and uploads vectors/embeddings.
+ *
+ * @param {object} params
+ * @param {object} params.req - Express request object
+ * @param {import('librechat-data-provider').TFile} params.file - The file record from DB
+ * @param {string} [params.entity_id] - Optional entity ID (agent_id)
+ * @returns {Promise<{ embedded: boolean }>} Embedding result
+ */
+async function provisionToVectorDB({ req, file, entity_id }) {
+  if (!process.env.RAG_API_URL) {
+    logger.warn('[provisionToVectorDB] RAG_API_URL not defined, skipping vector provisioning');
+    return { embedded: false };
+  }
+
+  const { getDownloadStream } = getStrategyFunctions(file.source);
+  if (!getDownloadStream) {
+    throw new Error(
+      `Cannot provision file "${file.filename}" to vector DB: storage source "${file.source}" does not support download streams`,
+    );
+  }
+
+  // The uploadVectors function expects a file-like object with a `path` property for fs.createReadStream.
+  // Since we're provisioning from storage (not a multer upload), we need to stream to a temp file first.
+  const os = require('os');
+  const path = require('path');
+  const tmpPath = path.join(os.tmpdir(), `provision-${file.file_id}-${file.filename}`);
+
+  try {
+    const stream = await getDownloadStream(req, file.filepath);
+    await new Promise((resolve, reject) => {
+      const writeStream = fs.createWriteStream(tmpPath);
+      stream.pipe(writeStream);
+      writeStream.on('finish', resolve);
+      writeStream.on('error', reject);
+      stream.on('error', reject);
+    });
+
+    const { uploadVectors } = require('./VectorDB/crud');
+    const tempFile = {
+      path: tmpPath,
+      originalname: file.filename,
+      mimetype: file.type,
+      size: file.bytes,
+    };
+
+    const embeddingResult = await uploadVectors({
+      req,
+      file: tempFile,
+      file_id: file.file_id,
+      entity_id,
+    });
+
+    const embedded = embeddingResult?.embedded ?? false;
+
+    await updateFile({
+      file_id: file.file_id,
+      embedded,
+    });
+
+    logger.debug(
+      `[provisionToVectorDB] Provisioned file "${file.filename}" (${file.file_id}) to vector DB, embedded=${embedded}`,
+    );
+
+    return { embedded };
+  } finally {
+    // Clean up temp file
+    try {
+      fs.unlinkSync(tmpPath);
+    } catch {
+      // Ignore cleanup errors
+    }
+  }
+}
+
+module.exports = {
+  provisionToCodeEnv,
+  provisionToVectorDB,
+};
--- a/packages/api/src/agents/initialize.ts
+++ b/packages/api/src/agents/initialize.ts
@ -31,7 +31,11 @@ import { filterFilesByEndpointConfig } from '~/files';
 import { generateArtifactsPrompt } from '~/prompts';
 import { getProviderConfig } from '~/endpoints';
 import { primeResources } from './resources';
-import type { TFilterFilesByAgentAccess } from './resources';
+import type {
+  TFilterFilesByAgentAccess,
+  TProvisionToCodeEnv,
+  TProvisionToVectorDB,
+} from './resources';

 /**
 * Fraction of context budget reserved as headroom when no explicit maxContextTokens is set.
@ -143,6 +147,10 @@ export interface InitializeAgentDbMethods extends EndpointDbMethods {
    parentMessageId?: string;
    files?: Array<{ file_id: string }>;
  }> | null>;
+  /** Optional: provision a file to the code execution environment */
+  provisionToCodeEnv?: TProvisionToCodeEnv;
+  /** Optional: provision a file to the vector DB for file_search */
+  provisionToVectorDB?: TProvisionToVectorDB;
 }

 /**
@ -205,6 +213,14 @@ export async function initializeAgent(
  const provider = agent.provider;
  agent.endpoint = provider;

+  /** Build the set of tool resources the agent has enabled */
+  const toolResourceSet = new Set<EToolResources>();
+  for (const tool of agent.tools ?? []) {
+    if (EToolResources[tool as keyof typeof EToolResources]) {
+      toolResourceSet.add(EToolResources[tool as keyof typeof EToolResources]);
+    }
+  }
+
  /**
   * Load conversation files for ALL agents, not just the initial agent.
   * This enables handoff agents to access files that were uploaded earlier
@ -213,12 +229,6 @@ export async function initializeAgent(
   */
  if (conversationId != null && resendFiles) {
    const fileIds = (await db.getConvoFiles(conversationId)) ?? [];
-    const toolResourceSet = new Set<EToolResources>();
-    for (const tool of agent.tools ?? []) {
-      if (EToolResources[tool as keyof typeof EToolResources]) {
-        toolResourceSet.add(EToolResources[tool as keyof typeof EToolResources]);
-      }
-    }

    const toolFiles = (await db.getToolFilesByIds(fileIds, toolResourceSet)) as IMongoFile[];

@ -293,6 +303,9 @@ export async function initializeAgent(
      : undefined,
    tool_resources: agent.tool_resources,
    requestFileSet: new Set(requestFiles?.map((file) => file.file_id)),
+    enabledToolResources: toolResourceSet,
+    provisionToCodeEnv: db.provisionToCodeEnv,
+    provisionToVectorDB: db.provisionToVectorDB,
  });

  const {
--- a/packages/api/src/agents/resources.ts
+++ b/packages/api/src/agents/resources.ts
@ -5,6 +5,26 @@ import type { IMongoFile, AppConfig, IUser } from '@librechat/data-schemas';
 import type { FilterQuery, QueryOptions, ProjectionType } from 'mongoose';
 import type { Request as ServerRequest } from 'express';

+/**
+ * Function type for provisioning a file to the code execution environment.
+ * @returns The fileIdentifier from the code env
+ */
+export type TProvisionToCodeEnv = (params: {
+  req: ServerRequest & { user?: IUser };
+  file: TFile;
+  entity_id?: string;
+}) => Promise<string>;
+
+/**
+ * Function type for provisioning a file to the vector DB for file_search.
+ * @returns Object with embedded status
+ */
+export type TProvisionToVectorDB = (params: {
+  req: ServerRequest & { user?: IUser };
+  file: TFile;
+  entity_id?: string;
+}) => Promise<{ embedded: boolean }>;
+
 /**
 * Function type for retrieving files from the database
 * @param filter - MongoDB filter query for files
@ -100,6 +120,7 @@ const categorizeFileForToolResources = ({
  requestFileSet: Set<string>;
  processedResourceFiles: Set<string>;
 }): void => {
+  // No early returns — a file can belong to multiple tool resources simultaneously
  if (file.metadata?.fileIdentifier) {
    addFileToResource({
      file,
@ -107,7 +128,6 @@ const categorizeFileForToolResources = ({
      tool_resources,
      processedResourceFiles,
    });
-    return;
  }

  if (file.embedded === true) {
@ -117,7 +137,6 @@ const categorizeFileForToolResources = ({
      tool_resources,
      processedResourceFiles,
    });
-    return;
  }

  if (
@ -163,6 +182,9 @@ export const primeResources = async ({
  attachments: _attachments,
  tool_resources: _tool_resources,
  agentId,
+  enabledToolResources,
+  provisionToCodeEnv,
+  provisionToVectorDB,
 }: {
  req: ServerRequest & { user?: IUser };
  appConfig?: AppConfig;
@ -172,6 +194,12 @@ export const primeResources = async ({
  getFiles: TGetFiles;
  filterFiles?: TFilterFilesByAgentAccess;
  agentId?: string;
+  /** Set of tool resource types the agent has enabled (e.g., execute_code, file_search) */
+  enabledToolResources?: Set<EToolResources>;
+  /** Optional callback to provision a file to the code execution environment */
+  provisionToCodeEnv?: TProvisionToCodeEnv;
+  /** Optional callback to provision a file to the vector DB for file_search */
+  provisionToVectorDB?: TProvisionToVectorDB;
 }): Promise<{
  attachments: Array<TFile | undefined> | undefined;
  tool_resources: AgentToolResources | undefined;
@ -309,6 +337,87 @@ export const primeResources = async ({
      }
    }

+    /**
+     * Lazy provisioning: for deferred files that haven't been provisioned to the
+     * agent's enabled tool resources, provision them now (at chat-request start).
+     * This handles files uploaded via the unified upload flow (no tool_resource chosen at upload time).
+     */
+    if (enabledToolResources && enabledToolResources.size > 0 && attachments.length > 0) {
+      const needsCodeEnv =
+        enabledToolResources.has(EToolResources.execute_code) && provisionToCodeEnv != null;
+      const needsVectorDB =
+        enabledToolResources.has(EToolResources.file_search) && provisionToVectorDB != null;
+
+      if (needsCodeEnv || needsVectorDB) {
+        for (const file of attachments) {
+          if (!file?.file_id) {
+            continue;
+          }
+
+          // Skip images for file_search (not supported)
+          const isImage = file.type?.startsWith('image') ?? false;
+
+          // Provision to code env if needed and not already provisioned
+          if (
+            needsCodeEnv &&
+            !file.metadata?.fileIdentifier &&
+            !processedResourceFiles.has(`${EToolResources.execute_code}:${file.file_id}`)
+          ) {
+            try {
+              const fileIdentifier = await provisionToCodeEnv({
+                req: req as ServerRequest & { user?: IUser },
+                file,
+                entity_id: agentId,
+              });
+              // Update the file object in-place so categorization picks it up
+              file.metadata = { ...file.metadata, fileIdentifier };
+              addFileToResource({
+                file,
+                resourceType: EToolResources.execute_code,
+                tool_resources,
+                processedResourceFiles,
+              });
+            } catch (error) {
+              logger.error(
+                `[primeResources] Failed to provision file "${file.filename}" to code env`,
+                error,
+              );
+            }
+          }
+
+          // Provision to vector DB if needed and not already provisioned
+          if (
+            needsVectorDB &&
+            !isImage &&
+            file.embedded !== true &&
+            !processedResourceFiles.has(`${EToolResources.file_search}:${file.file_id}`)
+          ) {
+            try {
+              const result = await provisionToVectorDB({
+                req: req as ServerRequest & { user?: IUser },
+                file,
+                entity_id: agentId,
+              });
+              if (result.embedded) {
+                file.embedded = true;
+                addFileToResource({
+                  file,
+                  resourceType: EToolResources.file_search,
+                  tool_resources,
+                  processedResourceFiles,
+                });
+              }
+            } catch (error) {
+              logger.error(
+                `[primeResources] Failed to provision file "${file.filename}" to vector DB`,
+                error,
+              );
+            }
+          }
+        }
+      }
+    }
+
    return { attachments: attachments.length > 0 ? attachments : [], tool_resources };
  } catch (error) {
    logger.error('Error priming resources', error);
--- a/packages/data-provider/src/file-config.ts
+++ b/packages/data-provider/src/file-config.ts
@ -459,12 +459,16 @@ const supportedMimeTypesSchema = z
    },
  );

+export const FileInteractionMode = z.enum(['text', 'provider', 'deferred', 'legacy']);
+export type TFileInteractionMode = z.infer<typeof FileInteractionMode>;
+
 export const endpointFileConfigSchema = z.object({
  disabled: z.boolean().optional(),
  fileLimit: z.number().min(0).optional(),
  fileSizeLimit: z.number().min(0).optional(),
  totalSizeLimit: z.number().min(0).optional(),
  supportedMimeTypes: supportedMimeTypesSchema.optional(),
+  defaultFileInteraction: FileInteractionMode.optional(),
 });

 export const fileConfigSchema = z.object({
@ -496,6 +500,7 @@ export const fileConfigSchema = z.object({
      supportedMimeTypes: supportedMimeTypesSchema.optional(),
    })
    .optional(),
+  defaultFileInteraction: FileInteractionMode.optional(),
 });

 export type TFileConfig = z.infer<typeof fileConfigSchema>;
@ -541,6 +546,8 @@ function mergeWithDefault(
    fileSizeLimit: endpointConfig.fileSizeLimit ?? defaultConfig.fileSizeLimit,
    totalSizeLimit: endpointConfig.totalSizeLimit ?? defaultConfig.totalSizeLimit,
    supportedMimeTypes: endpointConfig.supportedMimeTypes ?? defaultMimeTypes,
+    defaultFileInteraction:
+      endpointConfig.defaultFileInteraction ?? defaultConfig.defaultFileInteraction,
  };
 }

@ -669,6 +676,10 @@ export function mergeFileConfig(dynamic: z.infer<typeof fileConfigSchema> | unde
    return mergedConfig;
  }

+  if (dynamic.defaultFileInteraction !== undefined) {
+    mergedConfig.defaultFileInteraction = dynamic.defaultFileInteraction;
+  }
+
  if (dynamic.serverFileSizeLimit !== undefined) {
    mergedConfig.serverFileSizeLimit = mbToBytes(dynamic.serverFileSizeLimit);
  }
@ -758,6 +769,10 @@ export function mergeFileConfig(dynamic: z.infer<typeof fileConfigSchema> | unde
        dynamicEndpoint.supportedMimeTypes as unknown as string[],
      );
    }
+
+    if (dynamicEndpoint.defaultFileInteraction !== undefined) {
+      mergedEndpoint.defaultFileInteraction = dynamicEndpoint.defaultFileInteraction;
+    }
  }

  return mergedConfig;
--- a/packages/data-provider/src/types/files.ts
+++ b/packages/data-provider/src/types/files.ts
@ -42,6 +42,7 @@ export type EndpointFileConfig = {
  fileSizeLimit?: number;
  totalSizeLimit?: number;
  supportedMimeTypes?: RegExp[];
+  defaultFileInteraction?: 'text' | 'provider' | 'deferred' | 'legacy';
 };

 export type FileConfig = {
@ -67,6 +68,7 @@ export type FileConfig = {
    supportedMimeTypes?: RegExp[];
  };
  checkType?: (fileType: string, supportedTypes: RegExp[]) => boolean;
+  defaultFileInteraction?: 'text' | 'provider' | 'deferred' | 'legacy';
 };

 export type FileConfigInput = {
@ -91,6 +93,7 @@ export type FileConfigInput = {
    supportedMimeTypes?: string[];
  };
  checkType?: (fileType: string, supportedTypes: RegExp[]) => boolean;
+  defaultFileInteraction?: 'text' | 'provider' | 'deferred' | 'legacy';
 };

 export type TFile = {