🚧 chore: merge latest dev build (#4288)

* fix: agent initialization, add `collectedUsage` handling

* style: improve side panel styling

* refactor(loadAgent): Optimize order agent project ID retrieval

* feat: code execution

* fix: typing issues

* feat: ExecuteCode content part

* refactor: use local state for default collapsed state of analysis content parts

* fix: code parsing in ExecuteCode component

* chore: bump agents package, export loadAuthValues

* refactor: Update handleTools.js to use EnvVar for code execution tool authentication

* WIP

* feat: download code outputs

* fix(useEventHandlers): type issues

* feat: backend handling for code outputs

* Refactor: Remove console.log statement in Part.tsx

* refactor: add attachments to TMessage/messageSchema

* WIP: prelim handling for code outputs

* feat: attachments rendering

* refactor: improve attachments rendering

* fix: attachments, nullish edge case, handle attachments from event stream, bump agents package

* fix filename download

* fix: tool assignment for 'run code' on agent creation

* fix: image handling by adding attachments

* refactor: prevent agent creation without provider/model

* refactor: remove unnecessary space in agent creation success message

* refactor: select first model if selecting provider from empty on form

* fix: Agent avatar bug

* fix: `defaultAgentFormValues` causing boolean typing issue and typeerror

* fix: capabilities counting as tools, causing duplication of them

* fix: formatted messages edge case where consecutive content text type parts with the latter having tool_call_ids would cause consecutive AI messages to be created. furthermore, content could not be an array for tool_use messages (anthropic limitation)

* chore: bump @librechat/agents dependency to version 1.6.9

* feat: bedrock agents

* feat: new Agents icon

* feat: agent titling

* feat: agent landing

* refactor: allow sharing agent globally only if user is admin or author

* feat: initial AgentPanelSkeleton

* feat: AgentPanelSkeleton

* feat: collaborative agents

* chore: add potential authorName as part of schema

* chore: Remove unnecessary console.log statement

* WIP: agent model parameters

* chore: ToolsDialog typing and tool related localization chnages

* refactor: update tool instance type (latest langchain class), and rename google tool to 'google' proper

* chore: add back tools

* feat: Agent knowledge files upload

* refactor: better verbiage for disabled knowledge

* chore: debug logs for file deletions

* chore: debug logs for file deletions

* feat: upload/delete agent knowledge/file-search files

* feat: file search UI for agents

* feat: first pass, file search tool

* chore: update default agent capabilities and info

api/server/controllers/agents/callbacks.js

@@ -1,8 +1,13 @@
+const { Tools } = require('librechat-data-provider');
 const { GraphEvents, ToolEndHandler, ChatModelStreamHandler } = require('@librechat/agents');
+const { processCodeOutput } = require('~/server/services/Files/Code/process');
+const { logger } = require('~/config');
 
 /** @typedef {import('@librechat/agents').Graph} Graph */
 /** @typedef {import('@librechat/agents').EventHandler} EventHandler */
 /** @typedef {import('@librechat/agents').ModelEndData} ModelEndData */
+/** @typedef {import('@librechat/agents').ToolEndData} ToolEndData */
+/** @typedef {import('@librechat/agents').ToolEndCallback} ToolEndCallback */
 /** @typedef {import('@librechat/agents').ChatModelStreamHandler} ChatModelStreamHandler */
 /** @typedef {import('@librechat/agents').ContentAggregatorResult['aggregateContent']} ContentAggregator */
 /** @typedef {import('@librechat/agents').GraphEvents} GraphEvents */
@@ -58,11 +63,12 @@ class ModelEndHandler {
  * @param {Object} options - The options object.
  * @param {ServerResponse} options.res - The options object.
  * @param {ContentAggregator} options.aggregateContent - The options object.
+ * @param {ToolEndCallback} options.toolEndCallback - Callback to use when tool ends.
  * @param {Array<UsageMetadata>} options.collectedUsage - The list of collected usage metadata.
  * @returns {Record<string, t.EventHandler>} The default handlers.
  * @throws {Error} If the request is not found.
  */
-function getDefaultHandlers({ res, aggregateContent, collectedUsage }) {
+function getDefaultHandlers({ res, aggregateContent, toolEndCallback, collectedUsage }) {
   if (!res || !aggregateContent) {
     throw new Error(
       `[getDefaultHandlers] Missing required options: res: ${!res}, aggregateContent: ${!aggregateContent}`,
@@ -70,7 +76,7 @@ function getDefaultHandlers({ res, aggregateContent, collectedUsage }) {
   }
   const handlers = {
     [GraphEvents.CHAT_MODEL_END]: new ModelEndHandler(collectedUsage),
-    [GraphEvents.TOOL_END]: new ToolEndHandler(),
+    [GraphEvents.TOOL_END]: new ToolEndHandler(toolEndCallback),
     [GraphEvents.CHAT_MODEL_STREAM]: new ChatModelStreamHandler(),
     [GraphEvents.ON_RUN_STEP]: {
       /**
@@ -121,7 +127,67 @@ function getDefaultHandlers({ res, aggregateContent, collectedUsage }) {
   return handlers;
 }
 
+/**
+ *
+ * @param {Object} params
+ * @param {ServerRequest} params.req
+ * @param {ServerResponse} params.res
+ * @param {Promise<MongoFile | { filename: string; filepath: string; expires: number;} | null>[]} params.artifactPromises
+ * @returns {ToolEndCallback} The tool end callback.
+ */
+function createToolEndCallback({ req, res, artifactPromises }) {
+  /**
+   * @type {ToolEndCallback}
+   */
+  return async (data, metadata) => {
+    const output = data?.output;
+    if (!output) {
+      return;
+    }
+
+    if (output.name !== Tools.execute_code) {
+      return;
+    }
+
+    const { tool_call_id, artifact } = output;
+    if (!artifact.files) {
+      return;
+    }
+
+    for (const file of artifact.files) {
+      const { id, name } = file;
+      artifactPromises.push(
+        (async () => {
+          const fileMetadata = await processCodeOutput({
+            req,
+            id,
+            name,
+            toolCallId: tool_call_id,
+            messageId: metadata.run_id,
+            sessionId: artifact.session_id,
+            conversationId: metadata.thread_id,
+          });
+          if (!res.headersSent) {
+            return fileMetadata;
+          }
+
+          if (!fileMetadata) {
+            return null;
+          }
+
+          res.write(`event: attachment\ndata: ${JSON.stringify(fileMetadata)}\n\n`);
+          return fileMetadata;
+        })().catch((error) => {
+          logger.error('Error processing code output:', error);
+          return null;
+        }),
+      );
+    }
+  };
+}
+
 module.exports = {
   sendEvent,
   getDefaultHandlers,
+  createToolEndCallback,
 };

api/server/controllers/agents/client.js

@@ -10,7 +10,9 @@
 const { Callback, createMetadataAggregator } = require('@librechat/agents');
 const {
   Constants,
+  openAISchema,
   EModelEndpoint,
+  anthropicSchema,
   bedrockOutputParser,
   providerEndpointMap,
   removeNullishValues,
@@ -35,11 +37,10 @@ const { logger } = require('~/config');
 
 /** @typedef {import('@librechat/agents').MessageContentComplex} MessageContentComplex */
 
-// const providerSchemas = {
-// [EModelEndpoint.bedrock]: true,
-// };
-
 const providerParsers = {
+  [EModelEndpoint.openAI]: openAISchema,
+  [EModelEndpoint.azureOpenAI]: openAISchema,
+  [EModelEndpoint.anthropic]: anthropicSchema,
   [EModelEndpoint.bedrock]: bedrockOutputParser,
 };
 
@@ -57,10 +58,11 @@ class AgentClient extends BaseClient {
     this.run;
 
     const {
-      maxContextTokens,
-      modelOptions = {},
       contentParts,
       collectedUsage,
+      artifactPromises,
+      maxContextTokens,
+      modelOptions = {},
       ...clientOptions
     } = options;
 
@@ -70,6 +72,8 @@ class AgentClient extends BaseClient {
     this.contentParts = contentParts;
     /** @type {Array<UsageMetadata>} */
     this.collectedUsage = collectedUsage;
+    /** @type {ArtifactPromises} */
+    this.artifactPromises = artifactPromises;
     this.options = Object.assign({ endpoint: options.endpoint }, clientOptions);
   }
 
@@ -477,7 +481,6 @@ class AgentClient extends BaseClient {
           provider: providerEndpointMap[this.options.agent.provider],
           thread_id: this.conversationId,
         },
-        run_id: this.responseMessageId,
         signal: abortController.signal,
         streamMode: 'values',
         version: 'v2',

api/server/controllers/agents/run.js

@@ -45,10 +45,9 @@ async function createRun({
   );
 
   const graphConfig = {
-    runId,
-    llmConfig,
     tools,
     toolMap,
+    llmConfig,
     instructions: agent.instructions,
     additional_instructions: agent.additional_instructions,
   };
@@ -59,6 +58,7 @@ async function createRun({
   }
 
   return Run.create({
+    runId,
     graphConfig,
     customHandlers,
   });

api/server/controllers/agents/v1.js

@@ -1,5 +1,5 @@
 const { nanoid } = require('nanoid');
-const { FileContext, Constants } = require('librechat-data-provider');
+const { FileContext, Constants, Tools, SystemRoles } = require('librechat-data-provider');
 const {
   getAgent,
   createAgent,
@@ -14,6 +14,11 @@ const { updateAgentProjects } = require('~/models/Agent');
 const { deleteFileByFilter } = require('~/models/File');
 const { logger } = require('~/config');
 
+const systemTools = {
+  [Tools.execute_code]: true,
+  [Tools.file_search]: true,
+};
+
 /**
  * Creates an Agent.
  * @route POST /Agents
@@ -27,9 +32,17 @@ const createAgentHandler = async (req, res) => {
     const { tools = [], provider, name, description, instructions, model, ...agentData } = req.body;
     const { id: userId } = req.user;
 
-    agentData.tools = tools
-      .map((tool) => (typeof tool === 'string' ? req.app.locals.availableTools[tool] : tool))
-      .filter(Boolean);
+    agentData.tools = [];
+
+    for (const tool of tools) {
+      if (req.app.locals.availableTools[tool]) {
+        agentData.tools.push(tool);
+      }
+
+      if (systemTools[tool]) {
+        agentData.tools.push(tool);
+      }
+    }
 
     Object.assign(agentData, {
       author: userId,
@@ -80,10 +93,24 @@ const getAgentHandler = async (req, res) => {
       return res.status(404).json({ error: 'Agent not found' });
     }
 
+    agent.author = agent.author.toString();
+    agent.isCollaborative = !!agent.isCollaborative;
+
     if (agent.author !== author) {
       delete agent.author;
     }
 
+    if (!agent.isCollaborative && agent.author !== author && req.user.role !== SystemRoles.ADMIN) {
+      return res.status(200).json({
+        id: agent.id,
+        name: agent.name,
+        avatar: agent.avatar,
+        author: agent.author,
+        projectIds: agent.projectIds,
+        isCollaborative: agent.isCollaborative,
+      });
+    }
+
     return res.status(200).json(agent);
   } catch (error) {
     logger.error('[/Agents/:id] Error retrieving agent', error);
@@ -106,12 +133,29 @@ const updateAgentHandler = async (req, res) => {
     const { projectIds, removeProjectIds, ...updateData } = req.body;
 
     let updatedAgent;
+    const query = { id, author: req.user.id };
+    if (req.user.role === SystemRoles.ADMIN) {
+      delete query.author;
+    }
     if (Object.keys(updateData).length > 0) {
-      updatedAgent = await updateAgent({ id, author: req.user.id }, updateData);
+      updatedAgent = await updateAgent(query, updateData);
     }
 
     if (projectIds || removeProjectIds) {
-      updatedAgent = await updateAgentProjects(id, projectIds, removeProjectIds);
+      updatedAgent = await updateAgentProjects({
+        user: req.user,
+        agentId: id,
+        projectIds,
+        removeProjectIds,
+      });
+    }
+
+    if (updatedAgent.author) {
+      updatedAgent.author = updatedAgent.author.toString();
+    }
+
+    if (updatedAgent.author !== req.user.id) {
+      delete updatedAgent.author;
     }
 
     return res.json(updatedAgent);

api/server/controllers/assistants/v2.js

@@ -149,7 +149,6 @@ const updateAssistant = async ({ req, openai, assistant_id, updateData }) => {
  * @param {string} params.assistant_id
  * @param {string} params.tool_resource
  * @param {string} params.file_id
- * @param {AssistantUpdateParams} params.updateData
  * @returns {Promise<Assistant>} The updated assistant.
  */
 const addResourceFileId = async ({ req, openai, assistant_id, tool_resource, file_id }) => {

api/server/routes/agents/chat.js

@@ -10,6 +10,7 @@ const {
 } = require('~/server/middleware');
 const { initializeClient } = require('~/server/services/Endpoints/agents');
 const AgentController = require('~/server/controllers/agents/request');
+const addTitle = require('~/server/services/Endpoints/agents/title');
 
 router.post('/abort', handleAbort());
 
@@ -28,7 +29,7 @@ router.post(
   buildEndpointOption,
   setHeaders,
   async (req, res, next) => {
-    await AgentController(req, res, next, initializeClient);
+    await AgentController(req, res, next, initializeClient, addTitle);
   },
 );
 

api/server/routes/agents/v1.js

@@ -2,6 +2,7 @@ const multer = require('multer');
 const express = require('express');
 const { PermissionTypes, Permissions } = require('librechat-data-provider');
 const { requireJwtAuth, generateCheckAccess } = require('~/server/middleware');
+const { getAvailableTools } = require('~/server/controllers/PluginController');
 const v1 = require('~/server/controllers/agents/v1');
 const actions = require('./actions');
 
@@ -36,9 +37,7 @@ router.use('/actions', actions);
  * @route GET /agents/tools
  * @returns {TPlugin[]} 200 - application/json
  */
-router.use('/tools', (req, res) => {
-  res.json([]);
-});
+router.use('/tools', getAvailableTools);
 
 /**
  * Creates an agent.

api/server/routes/bedrock/chat.js

@@ -10,7 +10,7 @@ const {
 } = require('~/server/middleware');
 const { initializeClient } = require('~/server/services/Endpoints/bedrock');
 const AgentController = require('~/server/controllers/agents/request');
-const addTitle = require('~/server/services/Endpoints/bedrock/title');
+const addTitle = require('~/server/services/Endpoints/agents/title');
 
 router.post('/abort', handleAbort());
 

api/server/routes/files/files.js

@@ -1,18 +1,22 @@
 const fs = require('fs').promises;
 const express = require('express');
+const { EnvVar } = require('@librechat/agents');
 const {
   isUUID,
-  checkOpenAIStorage,
   FileSources,
   EModelEndpoint,
+  isAgentsEndpoint,
+  checkOpenAIStorage,
 } = require('librechat-data-provider');
 const {
   filterFile,
   processFileUpload,
   processDeleteRequest,
+  processAgentFileUpload,
 } = require('~/server/services/Files/process');
 const { getStrategyFunctions } = require('~/server/services/Files/strategies');
 const { getOpenAIClient } = require('~/server/controllers/assistants/helpers');
+const { loadAuthValues } = require('~/app/clients/tools/util');
 const { getFiles } = require('~/models/File');
 const { logger } = require('~/config');
 
@@ -64,6 +68,11 @@ router.delete('/', async (req, res) => {
 
     await processDeleteRequest({ req, files });
 
+    logger.debug(
+      `[/files] Files deleted successfully: ${files.map(
+        (f, i) => `${f.file_id}${i < files.length - 1 ? ', ' : ''}`,
+      )}`,
+    );
     res.status(200).json({ message: 'Files deleted successfully' });
   } catch (error) {
     logger.error('[/files] Error deleting files:', error);
@@ -71,6 +80,36 @@ router.delete('/', async (req, res) => {
   }
 });
 
+router.get('/code/download/:sessionId/:fileId', async (req, res) => {
+  try {
+    const { sessionId, fileId } = req.params;
+    const logPrefix = `Session ID: ${sessionId} | File ID: ${fileId} | Code output download requested by user `;
+    logger.debug(logPrefix);
+
+    if (!sessionId || !fileId) {
+      return res.status(400).send('Bad request');
+    }
+
+    const { getDownloadStream } = getStrategyFunctions(FileSources.execute_code);
+    if (!getDownloadStream) {
+      logger.warn(
+        `${logPrefix} has no stream method implemented for ${FileSources.execute_code} source`,
+      );
+      return res.status(501).send('Not Implemented');
+    }
+
+    const result = await loadAuthValues({ userId: req.user.id, authFields: [EnvVar.CODE_API_KEY] });
+
+    /** @type {AxiosResponse<ReadableStream> | undefined} */
+    const response = await getDownloadStream(`${sessionId}/${fileId}`, result[EnvVar.CODE_API_KEY]);
+    res.set(response.headers);
+    response.data.pipe(res);
+  } catch (error) {
+    logger.error('Error downloading file:', error);
+    res.status(500).send('Error downloading file');
+  }
+});
+
 router.get('/download/:userId/:file_id', async (req, res) => {
   try {
     const { userId, file_id } = req.params;
@@ -154,6 +193,10 @@ router.post('/', async (req, res) => {
     metadata.temp_file_id = metadata.file_id;
     metadata.file_id = req.file_id;
 
+    if (isAgentsEndpoint(metadata.endpoint)) {
+      return await processAgentFileUpload({ req, res, file, metadata });
+    }
+
     await processFileUpload({ req, res, file, metadata });
   } catch (error) {
     let message = 'Error processing file';

api/server/services/Config/EndpointService.js

@@ -49,6 +49,10 @@ module.exports = {
       process.env.BEDROCK_AWS_SECRET_ACCESS_KEY ?? process.env.BEDROCK_AWS_DEFAULT_REGION,
     ),
     /* key will be part of separate config */
-    [EModelEndpoint.agents]: generateConfig(process.env.I_AM_A_TEAPOT),
+    [EModelEndpoint.agents]: generateConfig(
+      process.env.EXPERIMENTAL_AGENTS,
+      undefined,
+      EModelEndpoint.agents,
+    ),
   },
 };

api/server/services/Endpoints/agents/build.js

@@ -1,13 +1,12 @@
-const { getAgent } = require('~/models/Agent');
+const { loadAgent } = require('~/models/Agent');
 const { logger } = require('~/config');
 
 const buildOptions = (req, endpoint, parsedBody) => {
   const { agent_id, instructions, spec, ...model_parameters } = parsedBody;
 
-  const agentPromise = getAgent({
-    id: agent_id,
-    // TODO: better author handling
-    author: req.user.id,
+  const agentPromise = loadAgent({
+    req,
+    agent_id,
   }).catch((error) => {
     logger.error(`[/agents/:${agent_id}] Error retrieving agent during build options step`, error);
     return undefined;

api/server/services/Endpoints/agents/initialize.js

@@ -14,14 +14,16 @@ const { tool } = require('@langchain/core/tools');
 const { createContentAggregator } = require('@librechat/agents');
 const {
   EModelEndpoint,
-  providerEndpointMap,
   getResponseSender,
+  providerEndpointMap,
 } = require('librechat-data-provider');
-const { getDefaultHandlers } = require('~/server/controllers/agents/callbacks');
-// for testing purposes
-// const createTavilySearchTool = require('~/app/clients/tools/structured/TavilySearch');
+const {
+  getDefaultHandlers,
+  createToolEndCallback,
+} = require('~/server/controllers/agents/callbacks');
 const initAnthropic = require('~/server/services/Endpoints/anthropic/initializeClient');
 const initOpenAI = require('~/server/services/Endpoints/openAI/initializeClient');
+const getBedrockOptions = require('~/server/services/Endpoints/bedrock/options');
 const { loadAgentTools } = require('~/server/services/ToolService');
 const AgentClient = require('~/server/controllers/agents/client');
 const { getModelMaxTokens } = require('~/utils');
@@ -50,6 +52,7 @@ const providerConfigMap = {
   [EModelEndpoint.openAI]: initOpenAI,
   [EModelEndpoint.azureOpenAI]: initOpenAI,
   [EModelEndpoint.anthropic]: initAnthropic,
+  [EModelEndpoint.bedrock]: getBedrockOptions,
 };
 
 const initializeClient = async ({ req, res, endpointOption }) => {
@@ -58,34 +61,33 @@ const initializeClient = async ({ req, res, endpointOption }) => {
   }
 
   // TODO: use endpointOption to determine options/modelOptions
+  /** @type {Array<UsageMetadata>} */
+  const collectedUsage = [];
+  /** @type {ArtifactPromises} */
+  const artifactPromises = [];
   const { contentParts, aggregateContent } = createContentAggregator();
-  const eventHandlers = getDefaultHandlers({ res, aggregateContent });
-
-  // const tools = [createTavilySearchTool()];
-  // const tools = [_getWeather];
-  // const tool_calls = [{ name: 'getPeople_action_swapi---dev' }];
-  // const tool_calls = [{ name: 'dalle' }];
-  // const tool_calls = [{ name: 'getItmOptions_action_YWlhcGkzLn' }];
-  // const tool_calls = [{ name: 'tavily_search_results_json' }];
-  // const tool_calls = [
-  //   { name: 'searchListings_action_emlsbG93NT' },
-  //   { name: 'searchAddress_action_emlsbG93NT' },
-  //   { name: 'searchMLS_action_emlsbG93NT' },
-  //   { name: 'searchCoordinates_action_emlsbG93NT' },
-  //   { name: 'searchUrl_action_emlsbG93NT' },
-  //   { name: 'getPropertyDetails_action_emlsbG93NT' },
-  // ];
+  const toolEndCallback = createToolEndCallback({ req, res, artifactPromises });
+  const eventHandlers = getDefaultHandlers({
+    res,
+    aggregateContent,
+    toolEndCallback,
+    collectedUsage,
+  });
 
   if (!endpointOption.agent) {
     throw new Error('No agent promise provided');
   }
 
-  /** @type {Agent} */
+  /** @type {Agent | null} */
   const agent = await endpointOption.agent;
+  if (!agent) {
+    throw new Error('Agent not found');
+  }
   const { tools, toolMap } = await loadAgentTools({
     req,
     tools: agent.tools,
     agent_id: agent.id,
+    tool_resources: agent.tool_resources,
     // openAIApiKey: process.env.OPENAI_API_KEY,
   });
 
@@ -121,8 +123,11 @@ const initializeClient = async ({ req, res, endpointOption }) => {
     contentParts,
     modelOptions,
     eventHandlers,
+    collectedUsage,
+    artifactPromises,
     endpoint: EModelEndpoint.agents,
     configOptions: options.configOptions,
+    attachments: endpointOption.attachments,
     maxContextTokens:
       agent.max_context_tokens ??
       getModelMaxTokens(modelOptions.model, providerEndpointMap[agent.provider]),

api/server/services/Endpoints/agents/title.js

@@ -33,7 +33,7 @@ const addTitle = async (req, { text, response, client }) => {
       conversationId: response.conversationId,
       title,
     },
-    { context: 'api/server/services/Endpoints/bedrock/title.js' },
+    { context: 'api/server/services/Endpoints/agents/title.js' },
   );
 };
 

api/server/services/Files/Code/crud.js

@@ -0,0 +1,34 @@
+// downloadStream.js
+
+const axios = require('axios');
+const { getCodeBaseURL } = require('@librechat/agents');
+
+const baseURL = getCodeBaseURL();
+
+/**
+ * Retrieves a download stream for a specified file.
+ * @param {string} fileIdentifier - The identifier for the file (e.g., "sessionId/fileId").
+ * @param {string} apiKey - The API key for authentication.
+ * @returns {Promise<AxiosResponse>} A promise that resolves to a readable stream of the file content.
+ * @throws {Error} If there's an error during the download process.
+ */
+async function getCodeOutputDownloadStream(fileIdentifier, apiKey) {
+  try {
+    const response = await axios({
+      method: 'get',
+      url: `${baseURL}/download/${fileIdentifier}`,
+      responseType: 'stream',
+      headers: {
+        'User-Agent': 'LibreChat/1.0',
+        'X-API-Key': apiKey,
+      },
+      timeout: 15000,
+    });
+
+    return response;
+  } catch (error) {
+    throw new Error(`Error downloading file: ${error.message}`);
+  }
+}
+
+module.exports = { getCodeOutputDownloadStream };

api/server/services/Files/Code/index.js

@@ -0,0 +1,5 @@
+const crud = require('./crud');
+
+module.exports = {
+  ...crud,
+};

api/server/services/Files/Code/process.js

@@ -0,0 +1,87 @@
+const path = require('path');
+const { v4 } = require('uuid');
+const axios = require('axios');
+const { getCodeBaseURL, EnvVar } = require('@librechat/agents');
+const { FileContext, imageExtRegex } = require('librechat-data-provider');
+const { convertImage } = require('~/server/services/Files/images/convert');
+const { loadAuthValues } = require('~/app/clients/tools/util');
+const { createFile } = require('~/models/File');
+const { logger } = require('~/config');
+
+/**
+ * Process OpenAI image files, convert to target format, save and return file metadata.
+ * @param {ServerRequest} params.req - The Express request object.
+ * @param {string} params.id - The file ID.
+ * @param {string} params.name - The filename.
+ * @param {string} params.toolCallId - The tool call ID that generated the file.
+ * @param {string} params.sessionId - The code execution session ID.
+ * @param {string} params.conversationId - The current conversation ID.
+ * @param {string} params.messageId - The current message ID.
+ * @returns {Promise<MongoFile & { messageId: string, toolCallId: string } | { filename: string; filepath: string; expiresAt: number; conversationId: string; toolCallId: string; messageId: string } | undefined>} The file metadata or undefined if an error occurs.
+ */
+const processCodeOutput = async ({
+  req,
+  id,
+  name,
+  toolCallId,
+  conversationId,
+  messageId,
+  sessionId,
+}) => {
+  const currentDate = new Date();
+  const baseURL = getCodeBaseURL();
+  const fileExt = path.extname(name);
+  if (!fileExt || !imageExtRegex.test(name)) {
+    return {
+      filename: name,
+      filepath: `/api/files/code/download/${sessionId}/${id}`,
+      /** Note: expires 24 hours after creation */
+      expiresAt: currentDate.getTime() + 86400000,
+      conversationId,
+      toolCallId,
+      messageId,
+    };
+  }
+
+  try {
+    const formattedDate = currentDate.toISOString();
+    const result = await loadAuthValues({ userId: req.user.id, authFields: [EnvVar.CODE_API_KEY] });
+    const response = await axios({
+      method: 'get',
+      url: `${baseURL}/download/${sessionId}/${id}`,
+      responseType: 'arraybuffer',
+      headers: {
+        'User-Agent': 'LibreChat/1.0',
+        'X-API-Key': result[EnvVar.CODE_API_KEY],
+      },
+      timeout: 15000,
+    });
+
+    const buffer = Buffer.from(response.data, 'binary');
+
+    const file_id = v4();
+    const _file = await convertImage(req, buffer, 'high', `${file_id}${fileExt}`);
+    const file = {
+      ..._file,
+      file_id,
+      usage: 1,
+      filename: name,
+      conversationId,
+      user: req.user.id,
+      type: `image/${req.app.locals.imageOutputType}`,
+      createdAt: formattedDate,
+      updatedAt: formattedDate,
+      source: req.app.locals.fileStrategy,
+      context: FileContext.execute_code,
+    };
+    createFile(file, true);
+    /** Note: `messageId` & `toolCallId` are not part of file DB schema; message object records associated file ID */
+    return Object.assign(file, { messageId, toolCallId });
+  } catch (error) {
+    logger.error('Error downloading file:', error);
+  }
+};
+
+module.exports = {
+  processCodeOutput,
+};

api/server/services/Files/process.js

@@ -8,6 +8,7 @@ const {
   FileSources,
   imageExtRegex,
   EModelEndpoint,
+  EToolResources,
   mergeFileConfig,
   hostImageIdSuffix,
   checkOpenAIStorage,
@@ -16,6 +17,7 @@ const {
 } = require('librechat-data-provider');
 const { addResourceFileId, deleteResourceFileId } = require('~/server/controllers/assistants/v2');
 const { convertImage, resizeAndConvert } = require('~/server/services/Files/images');
+const { addAgentResourceFile, removeAgentResourceFile } = require('~/models/Agent');
 const { getOpenAIClient } = require('~/server/controllers/assistants/helpers');
 const { createFile, updateFileUsage, deleteFiles } = require('~/models/File');
 const { LB_QueueAsyncCall } = require('~/server/utils/queue');
@@ -124,6 +126,17 @@ const processDeleteRequest = async ({ req, files }) => {
   for (const file of files) {
     const source = file.source ?? FileSources.local;
 
+    if (req.body.agent_id && req.body.tool_resource) {
+      promises.push(
+        removeAgentResourceFile({
+          req,
+          file_id: file.file_id,
+          agent_id: req.body.agent_id,
+          tool_resource: req.body.tool_resource,
+        }),
+      );
+    }
+
     if (checkOpenAIStorage(source) && !client[source]) {
       await initializeClients();
     }
@@ -398,6 +411,95 @@ const processFileUpload = async ({ req, res, file, metadata }) => {
   res.status(200).json({ message: 'File uploaded and processed successfully', ...result });
 };
 
+/**
+ * Applies the current strategy for file uploads.
+ * Saves file metadata to the database with an expiry TTL.
+ * Files must be deleted from the server filesystem manually.
+ *
+ * @param {Object} params - The parameters object.
+ * @param {Express.Request} params.req - The Express request object.
+ * @param {Express.Response} params.res - The Express response object.
+ * @param {Express.Multer.File} params.file - The uploaded file.
+ * @param {FileMetadata} params.metadata - Additional metadata for the file.
+ * @returns {Promise<void>}
+ */
+const processAgentFileUpload = async ({ req, res, file, metadata }) => {
+  const { agent_id, tool_resource } = metadata;
+  if (agent_id && !tool_resource) {
+    throw new Error('No tool resource provided for agent file upload');
+  }
+
+  if (tool_resource === EToolResources.file_search && file.mimetype.startsWith('image')) {
+    throw new Error('Image uploads are not supported for file search tool resources');
+  }
+
+  let messageAttachment = !!metadata.message_file;
+  if (!messageAttachment && !agent_id) {
+    throw new Error('No agent ID provided for agent file upload');
+  }
+
+  const source =
+    tool_resource === EToolResources.file_search
+      ? FileSources.vectordb
+      : req.app.locals.fileStrategy;
+  const { handleFileUpload } = getStrategyFunctions(source);
+  const { file_id, temp_file_id } = metadata;
+
+  const {
+    bytes,
+    filename,
+    filepath: _filepath,
+    embedded,
+    height,
+    width,
+  } = await handleFileUpload({
+    req,
+    file,
+    file_id,
+  });
+
+  let filepath = _filepath;
+
+  if (!messageAttachment && tool_resource) {
+    await addAgentResourceFile({
+      req,
+      agent_id,
+      file_id,
+      tool_resource: tool_resource,
+    });
+  }
+
+  if (file.mimetype.startsWith('image')) {
+    const result = await processImageFile({
+      req,
+      file,
+      metadata: { file_id: v4() },
+      returnFile: true,
+    });
+    filepath = result.filepath;
+  }
+
+  const result = await createFile(
+    {
+      user: req.user.id,
+      file_id,
+      temp_file_id,
+      bytes,
+      filepath,
+      filename: filename ?? file.originalname,
+      context: messageAttachment ? FileContext.message_attachment : FileContext.agents,
+      model: messageAttachment ? undefined : req.body.model,
+      type: file.mimetype,
+      embedded,
+      source,
+      height,
+      width,
+    },
+    true,
+  );
+  res.status(200).json({ message: 'Agent file uploaded and processed successfully', ...result });
+};
+
 /**
  * @param {object} params - The params object.
  * @param {OpenAI} params.openai - The OpenAI client instance.
@@ -654,5 +756,6 @@ module.exports = {
   uploadImageBuffer,
   processFileUpload,
   processDeleteRequest,
+  processAgentFileUpload,
   retrieveAndProcessFile,
 };

api/server/services/Files/strategies.js

@@ -21,6 +21,7 @@ const {
 } = require('./Local');
 const { uploadOpenAIFile, deleteOpenAIFile, getOpenAIFileStream } = require('./OpenAI');
 const { uploadVectors, deleteVectors } = require('./VectorDB');
+const { getCodeOutputDownloadStream } = require('./Code');
 
 /**
  * Firebase Storage Strategy Functions
@@ -103,6 +104,31 @@ const openAIStrategy = () => ({
   getDownloadStream: getOpenAIFileStream,
 });
 
+/**
+ * Code Output Strategy Functions
+ *
+ * Note: null values mean that the strategy is not supported.
+ * */
+const codeOutputStrategy = () => ({
+  /** @type {typeof saveFileFromURL | null} */
+  saveURL: null,
+  /** @type {typeof getLocalFileURL | null} */
+  getFileURL: null,
+  /** @type {typeof saveLocalBuffer | null} */
+  saveBuffer: null,
+  /** @type {typeof processLocalAvatar | null} */
+  processAvatar: null,
+  /** @type {typeof uploadLocalImage | null} */
+  handleImageUpload: null,
+  /** @type {typeof prepareImagesLocal | null} */
+  prepareImagePayload: null,
+  /** @type {typeof deleteLocalFile | null} */
+  deleteFile: null,
+  /** @type {typeof uploadVectors | null} */
+  handleFileUpload: null,
+  getDownloadStream: getCodeOutputDownloadStream,
+});
+
 // Strategy Selector
 const getStrategyFunctions = (fileSource) => {
   if (fileSource === FileSources.firebase) {
@@ -115,6 +141,8 @@ const getStrategyFunctions = (fileSource) => {
     return openAIStrategy();
   } else if (fileSource === FileSources.vectordb) {
     return vectorStrategy();
+  } else if (fileSource === FileSources.execute_code) {
+    return codeOutputStrategy();
   } else {
     throw new Error('Invalid file source');
   }

api/server/services/ToolService.js

@@ -1,9 +1,8 @@
 const fs = require('fs');
 const path = require('path');
-const { StructuredTool } = require('langchain/tools');
-const { tool: toolFn } = require('@langchain/core/tools');
 const { zodToJsonSchema } = require('zod-to-json-schema');
 const { Calculator } = require('langchain/tools/calculator');
+const { tool: toolFn, Tool } = require('@langchain/core/tools');
 const {
   Tools,
   ContentTypes,
@@ -70,7 +69,7 @@ function loadAndFormatTools({ directory, adminFilter = [], adminIncluded = [] })
       continue;
     }
 
-    if (!ToolClass || !(ToolClass.prototype instanceof StructuredTool)) {
+    if (!ToolClass || !(ToolClass.prototype instanceof Tool)) {
       continue;
     }
 
@@ -378,11 +377,12 @@ async function processRequiredActions(client, requiredActions) {
  * @param {Object} params - Run params containing user and request information.
  * @param {ServerRequest} params.req - The request object.
  * @param {string} params.agent_id - The agent ID.
- * @param {string[]} params.tools - The agent's available tools.
+ * @param {Agent['tools']} params.tools - The agent's available tools.
+ * @param {Agent['tool_resources']} params.tool_resources - The agent's available tool resources.
  * @param {string | undefined} [params.openAIApiKey] - The OpenAI API key.
  * @returns {Promise<{ tools?: StructuredTool[]; toolMap?: Record<string, StructuredTool>}>} The combined toolMap.
  */
-async function loadAgentTools({ req, agent_id, tools, openAIApiKey }) {
+async function loadAgentTools({ req, agent_id, tools, tool_resources, openAIApiKey }) {
   if (!tools || tools.length === 0) {
     return {};
   }
@@ -394,6 +394,7 @@ async function loadAgentTools({ req, agent_id, tools, openAIApiKey }) {
     options: {
       req,
       openAIApiKey,
+      tool_resources,
       returnMetadata: true,
       processFileURL,
       uploadImageBuffer,
@@ -405,6 +406,10 @@ async function loadAgentTools({ req, agent_id, tools, openAIApiKey }) {
   const agentTools = [];
   for (let i = 0; i < loadedTools.length; i++) {
     const tool = loadedTools[i];
+    if (tool.name && (tool.name === Tools.execute_code || tool.name === Tools.file_search)) {
+      agentTools.push(tool);
+      continue;
+    }
 
     const toolInstance = toolFn(
       async (...args) => {

api/server/utils/handleText.js

@@ -1,6 +1,8 @@
 const {
   Capabilities,
   EModelEndpoint,
+  isAgentsEndpoint,
+  AgentCapabilities,
   isAssistantsEndpoint,
   defaultRetrievalModels,
   defaultAssistantsVersion,
@@ -160,8 +162,8 @@ const isUserProvided = (value) => value === 'user_provided';
 /**
  * Generate the configuration for a given key and base URL.
  * @param {string} key
- * @param {string} baseURL
- * @param {string} endpoint
+ * @param {string} [baseURL]
+ * @param {string} [endpoint]
  * @returns {boolean | { userProvide: boolean, userProvideURL?: boolean }}
  */
 function generateConfig(key, baseURL, endpoint) {
@@ -177,7 +179,7 @@ function generateConfig(key, baseURL, endpoint) {
   }
 
   const assistants = isAssistantsEndpoint(endpoint);
-
+  const agents = isAgentsEndpoint(endpoint);
   if (assistants) {
     config.retrievalModels = defaultRetrievalModels;
     config.capabilities = [
@@ -189,6 +191,18 @@ function generateConfig(key, baseURL, endpoint) {
     ];
   }
 
+  if (agents) {
+    config.capabilities = [
+      AgentCapabilities.file_search,
+      AgentCapabilities.actions,
+      AgentCapabilities.tools,
+    ];
+
+    if (key === 'EXPERIMENTAL_RUN_CODE') {
+      config.capabilities.push(AgentCapabilities.execute_code);
+    }
+  }
+
   if (assistants && endpoint === EModelEndpoint.azureAssistants) {
     config.version = defaultAssistantsVersion.azureAssistants;
   } else if (assistants) {