🚧 chore: merge latest dev build (#4288)

* fix: agent initialization, add `collectedUsage` handling

* style: improve side panel styling

* refactor(loadAgent): Optimize order agent project ID retrieval

* feat: code execution

* fix: typing issues

* feat: ExecuteCode content part

* refactor: use local state for default collapsed state of analysis content parts

* fix: code parsing in ExecuteCode component

* chore: bump agents package, export loadAuthValues

* refactor: Update handleTools.js to use EnvVar for code execution tool authentication

* WIP

* feat: download code outputs

* fix(useEventHandlers): type issues

* feat: backend handling for code outputs

* Refactor: Remove console.log statement in Part.tsx

* refactor: add attachments to TMessage/messageSchema

* WIP: prelim handling for code outputs

* feat: attachments rendering

* refactor: improve attachments rendering

* fix: attachments, nullish edge case, handle attachments from event stream, bump agents package

* fix filename download

* fix: tool assignment for 'run code' on agent creation

* fix: image handling by adding attachments

* refactor: prevent agent creation without provider/model

* refactor: remove unnecessary space in agent creation success message

* refactor: select first model if selecting provider from empty on form

* fix: Agent avatar bug

* fix: `defaultAgentFormValues` causing boolean typing issue and typeerror

* fix: capabilities counting as tools, causing duplication of them

* fix: formatted messages edge case where consecutive content text type parts with the latter having tool_call_ids would cause consecutive AI messages to be created. furthermore, content could not be an array for tool_use messages (anthropic limitation)

* chore: bump @librechat/agents dependency to version 1.6.9

* feat: bedrock agents

* feat: new Agents icon

* feat: agent titling

* feat: agent landing

* refactor: allow sharing agent globally only if user is admin or author

* feat: initial AgentPanelSkeleton

* feat: AgentPanelSkeleton

* feat: collaborative agents

* chore: add potential authorName as part of schema

* chore: Remove unnecessary console.log statement

* WIP: agent model parameters

* chore: ToolsDialog typing and tool related localization chnages

* refactor: update tool instance type (latest langchain class), and rename google tool to 'google' proper

* chore: add back tools

* feat: Agent knowledge files upload

* refactor: better verbiage for disabled knowledge

* chore: debug logs for file deletions

* chore: debug logs for file deletions

* feat: upload/delete agent knowledge/file-search files

* feat: file search UI for agents

* feat: first pass, file search tool

* chore: update default agent capabilities and info

api/server/services/Config/EndpointService.js

@@ -49,6 +49,10 @@ module.exports = {
       process.env.BEDROCK_AWS_SECRET_ACCESS_KEY ?? process.env.BEDROCK_AWS_DEFAULT_REGION,
     ),
     /* key will be part of separate config */
-    [EModelEndpoint.agents]: generateConfig(process.env.I_AM_A_TEAPOT),
+    [EModelEndpoint.agents]: generateConfig(
+      process.env.EXPERIMENTAL_AGENTS,
+      undefined,
+      EModelEndpoint.agents,
+    ),
   },
 };

api/server/services/Endpoints/agents/build.js

@@ -1,13 +1,12 @@
-const { getAgent } = require('~/models/Agent');
+const { loadAgent } = require('~/models/Agent');
 const { logger } = require('~/config');
 
 const buildOptions = (req, endpoint, parsedBody) => {
   const { agent_id, instructions, spec, ...model_parameters } = parsedBody;
 
-  const agentPromise = getAgent({
-    id: agent_id,
-    // TODO: better author handling
-    author: req.user.id,
+  const agentPromise = loadAgent({
+    req,
+    agent_id,
   }).catch((error) => {
     logger.error(`[/agents/:${agent_id}] Error retrieving agent during build options step`, error);
     return undefined;

api/server/services/Endpoints/agents/initialize.js

@@ -14,14 +14,16 @@ const { tool } = require('@langchain/core/tools');
 const { createContentAggregator } = require('@librechat/agents');
 const {
   EModelEndpoint,
-  providerEndpointMap,
   getResponseSender,
+  providerEndpointMap,
 } = require('librechat-data-provider');
-const { getDefaultHandlers } = require('~/server/controllers/agents/callbacks');
-// for testing purposes
-// const createTavilySearchTool = require('~/app/clients/tools/structured/TavilySearch');
+const {
+  getDefaultHandlers,
+  createToolEndCallback,
+} = require('~/server/controllers/agents/callbacks');
 const initAnthropic = require('~/server/services/Endpoints/anthropic/initializeClient');
 const initOpenAI = require('~/server/services/Endpoints/openAI/initializeClient');
+const getBedrockOptions = require('~/server/services/Endpoints/bedrock/options');
 const { loadAgentTools } = require('~/server/services/ToolService');
 const AgentClient = require('~/server/controllers/agents/client');
 const { getModelMaxTokens } = require('~/utils');
@@ -50,6 +52,7 @@ const providerConfigMap = {
   [EModelEndpoint.openAI]: initOpenAI,
   [EModelEndpoint.azureOpenAI]: initOpenAI,
   [EModelEndpoint.anthropic]: initAnthropic,
+  [EModelEndpoint.bedrock]: getBedrockOptions,
 };
 
 const initializeClient = async ({ req, res, endpointOption }) => {
@@ -58,34 +61,33 @@ const initializeClient = async ({ req, res, endpointOption }) => {
   }
 
   // TODO: use endpointOption to determine options/modelOptions
+  /** @type {Array<UsageMetadata>} */
+  const collectedUsage = [];
+  /** @type {ArtifactPromises} */
+  const artifactPromises = [];
   const { contentParts, aggregateContent } = createContentAggregator();
-  const eventHandlers = getDefaultHandlers({ res, aggregateContent });
-
-  // const tools = [createTavilySearchTool()];
-  // const tools = [_getWeather];
-  // const tool_calls = [{ name: 'getPeople_action_swapi---dev' }];
-  // const tool_calls = [{ name: 'dalle' }];
-  // const tool_calls = [{ name: 'getItmOptions_action_YWlhcGkzLn' }];
-  // const tool_calls = [{ name: 'tavily_search_results_json' }];
-  // const tool_calls = [
-  //   { name: 'searchListings_action_emlsbG93NT' },
-  //   { name: 'searchAddress_action_emlsbG93NT' },
-  //   { name: 'searchMLS_action_emlsbG93NT' },
-  //   { name: 'searchCoordinates_action_emlsbG93NT' },
-  //   { name: 'searchUrl_action_emlsbG93NT' },
-  //   { name: 'getPropertyDetails_action_emlsbG93NT' },
-  // ];
+  const toolEndCallback = createToolEndCallback({ req, res, artifactPromises });
+  const eventHandlers = getDefaultHandlers({
+    res,
+    aggregateContent,
+    toolEndCallback,
+    collectedUsage,
+  });
 
   if (!endpointOption.agent) {
     throw new Error('No agent promise provided');
   }
 
-  /** @type {Agent} */
+  /** @type {Agent | null} */
   const agent = await endpointOption.agent;
+  if (!agent) {
+    throw new Error('Agent not found');
+  }
   const { tools, toolMap } = await loadAgentTools({
     req,
     tools: agent.tools,
     agent_id: agent.id,
+    tool_resources: agent.tool_resources,
     // openAIApiKey: process.env.OPENAI_API_KEY,
   });
 
@@ -121,8 +123,11 @@ const initializeClient = async ({ req, res, endpointOption }) => {
     contentParts,
     modelOptions,
     eventHandlers,
+    collectedUsage,
+    artifactPromises,
     endpoint: EModelEndpoint.agents,
     configOptions: options.configOptions,
+    attachments: endpointOption.attachments,
     maxContextTokens:
       agent.max_context_tokens ??
       getModelMaxTokens(modelOptions.model, providerEndpointMap[agent.provider]),

api/server/services/Endpoints/agents/title.js

@@ -33,7 +33,7 @@ const addTitle = async (req, { text, response, client }) => {
       conversationId: response.conversationId,
       title,
     },
-    { context: 'api/server/services/Endpoints/bedrock/title.js' },
+    { context: 'api/server/services/Endpoints/agents/title.js' },
   );
 };
 

api/server/services/Files/Code/crud.js

@@ -0,0 +1,34 @@
+// downloadStream.js
+
+const axios = require('axios');
+const { getCodeBaseURL } = require('@librechat/agents');
+
+const baseURL = getCodeBaseURL();
+
+/**
+ * Retrieves a download stream for a specified file.
+ * @param {string} fileIdentifier - The identifier for the file (e.g., "sessionId/fileId").
+ * @param {string} apiKey - The API key for authentication.
+ * @returns {Promise<AxiosResponse>} A promise that resolves to a readable stream of the file content.
+ * @throws {Error} If there's an error during the download process.
+ */
+async function getCodeOutputDownloadStream(fileIdentifier, apiKey) {
+  try {
+    const response = await axios({
+      method: 'get',
+      url: `${baseURL}/download/${fileIdentifier}`,
+      responseType: 'stream',
+      headers: {
+        'User-Agent': 'LibreChat/1.0',
+        'X-API-Key': apiKey,
+      },
+      timeout: 15000,
+    });
+
+    return response;
+  } catch (error) {
+    throw new Error(`Error downloading file: ${error.message}`);
+  }
+}
+
+module.exports = { getCodeOutputDownloadStream };

api/server/services/Files/Code/index.js

@@ -0,0 +1,5 @@
+const crud = require('./crud');
+
+module.exports = {
+  ...crud,
+};

api/server/services/Files/Code/process.js

@@ -0,0 +1,87 @@
+const path = require('path');
+const { v4 } = require('uuid');
+const axios = require('axios');
+const { getCodeBaseURL, EnvVar } = require('@librechat/agents');
+const { FileContext, imageExtRegex } = require('librechat-data-provider');
+const { convertImage } = require('~/server/services/Files/images/convert');
+const { loadAuthValues } = require('~/app/clients/tools/util');
+const { createFile } = require('~/models/File');
+const { logger } = require('~/config');
+
+/**
+ * Process OpenAI image files, convert to target format, save and return file metadata.
+ * @param {ServerRequest} params.req - The Express request object.
+ * @param {string} params.id - The file ID.
+ * @param {string} params.name - The filename.
+ * @param {string} params.toolCallId - The tool call ID that generated the file.
+ * @param {string} params.sessionId - The code execution session ID.
+ * @param {string} params.conversationId - The current conversation ID.
+ * @param {string} params.messageId - The current message ID.
+ * @returns {Promise<MongoFile & { messageId: string, toolCallId: string } | { filename: string; filepath: string; expiresAt: number; conversationId: string; toolCallId: string; messageId: string } | undefined>} The file metadata or undefined if an error occurs.
+ */
+const processCodeOutput = async ({
+  req,
+  id,
+  name,
+  toolCallId,
+  conversationId,
+  messageId,
+  sessionId,
+}) => {
+  const currentDate = new Date();
+  const baseURL = getCodeBaseURL();
+  const fileExt = path.extname(name);
+  if (!fileExt || !imageExtRegex.test(name)) {
+    return {
+      filename: name,
+      filepath: `/api/files/code/download/${sessionId}/${id}`,
+      /** Note: expires 24 hours after creation */
+      expiresAt: currentDate.getTime() + 86400000,
+      conversationId,
+      toolCallId,
+      messageId,
+    };
+  }
+
+  try {
+    const formattedDate = currentDate.toISOString();
+    const result = await loadAuthValues({ userId: req.user.id, authFields: [EnvVar.CODE_API_KEY] });
+    const response = await axios({
+      method: 'get',
+      url: `${baseURL}/download/${sessionId}/${id}`,
+      responseType: 'arraybuffer',
+      headers: {
+        'User-Agent': 'LibreChat/1.0',
+        'X-API-Key': result[EnvVar.CODE_API_KEY],
+      },
+      timeout: 15000,
+    });
+
+    const buffer = Buffer.from(response.data, 'binary');
+
+    const file_id = v4();
+    const _file = await convertImage(req, buffer, 'high', `${file_id}${fileExt}`);
+    const file = {
+      ..._file,
+      file_id,
+      usage: 1,
+      filename: name,
+      conversationId,
+      user: req.user.id,
+      type: `image/${req.app.locals.imageOutputType}`,
+      createdAt: formattedDate,
+      updatedAt: formattedDate,
+      source: req.app.locals.fileStrategy,
+      context: FileContext.execute_code,
+    };
+    createFile(file, true);
+    /** Note: `messageId` & `toolCallId` are not part of file DB schema; message object records associated file ID */
+    return Object.assign(file, { messageId, toolCallId });
+  } catch (error) {
+    logger.error('Error downloading file:', error);
+  }
+};
+
+module.exports = {
+  processCodeOutput,
+};

api/server/services/Files/process.js

@@ -8,6 +8,7 @@ const {
   FileSources,
   imageExtRegex,
   EModelEndpoint,
+  EToolResources,
   mergeFileConfig,
   hostImageIdSuffix,
   checkOpenAIStorage,
@@ -16,6 +17,7 @@ const {
 } = require('librechat-data-provider');
 const { addResourceFileId, deleteResourceFileId } = require('~/server/controllers/assistants/v2');
 const { convertImage, resizeAndConvert } = require('~/server/services/Files/images');
+const { addAgentResourceFile, removeAgentResourceFile } = require('~/models/Agent');
 const { getOpenAIClient } = require('~/server/controllers/assistants/helpers');
 const { createFile, updateFileUsage, deleteFiles } = require('~/models/File');
 const { LB_QueueAsyncCall } = require('~/server/utils/queue');
@@ -124,6 +126,17 @@ const processDeleteRequest = async ({ req, files }) => {
   for (const file of files) {
     const source = file.source ?? FileSources.local;
 
+    if (req.body.agent_id && req.body.tool_resource) {
+      promises.push(
+        removeAgentResourceFile({
+          req,
+          file_id: file.file_id,
+          agent_id: req.body.agent_id,
+          tool_resource: req.body.tool_resource,
+        }),
+      );
+    }
+
     if (checkOpenAIStorage(source) && !client[source]) {
       await initializeClients();
     }
@@ -398,6 +411,95 @@ const processFileUpload = async ({ req, res, file, metadata }) => {
   res.status(200).json({ message: 'File uploaded and processed successfully', ...result });
 };
 
+/**
+ * Applies the current strategy for file uploads.
+ * Saves file metadata to the database with an expiry TTL.
+ * Files must be deleted from the server filesystem manually.
+ *
+ * @param {Object} params - The parameters object.
+ * @param {Express.Request} params.req - The Express request object.
+ * @param {Express.Response} params.res - The Express response object.
+ * @param {Express.Multer.File} params.file - The uploaded file.
+ * @param {FileMetadata} params.metadata - Additional metadata for the file.
+ * @returns {Promise<void>}
+ */
+const processAgentFileUpload = async ({ req, res, file, metadata }) => {
+  const { agent_id, tool_resource } = metadata;
+  if (agent_id && !tool_resource) {
+    throw new Error('No tool resource provided for agent file upload');
+  }
+
+  if (tool_resource === EToolResources.file_search && file.mimetype.startsWith('image')) {
+    throw new Error('Image uploads are not supported for file search tool resources');
+  }
+
+  let messageAttachment = !!metadata.message_file;
+  if (!messageAttachment && !agent_id) {
+    throw new Error('No agent ID provided for agent file upload');
+  }
+
+  const source =
+    tool_resource === EToolResources.file_search
+      ? FileSources.vectordb
+      : req.app.locals.fileStrategy;
+  const { handleFileUpload } = getStrategyFunctions(source);
+  const { file_id, temp_file_id } = metadata;
+
+  const {
+    bytes,
+    filename,
+    filepath: _filepath,
+    embedded,
+    height,
+    width,
+  } = await handleFileUpload({
+    req,
+    file,
+    file_id,
+  });
+
+  let filepath = _filepath;
+
+  if (!messageAttachment && tool_resource) {
+    await addAgentResourceFile({
+      req,
+      agent_id,
+      file_id,
+      tool_resource: tool_resource,
+    });
+  }
+
+  if (file.mimetype.startsWith('image')) {
+    const result = await processImageFile({
+      req,
+      file,
+      metadata: { file_id: v4() },
+      returnFile: true,
+    });
+    filepath = result.filepath;
+  }
+
+  const result = await createFile(
+    {
+      user: req.user.id,
+      file_id,
+      temp_file_id,
+      bytes,
+      filepath,
+      filename: filename ?? file.originalname,
+      context: messageAttachment ? FileContext.message_attachment : FileContext.agents,
+      model: messageAttachment ? undefined : req.body.model,
+      type: file.mimetype,
+      embedded,
+      source,
+      height,
+      width,
+    },
+    true,
+  );
+  res.status(200).json({ message: 'Agent file uploaded and processed successfully', ...result });
+};
+
 /**
  * @param {object} params - The params object.
  * @param {OpenAI} params.openai - The OpenAI client instance.
@@ -654,5 +756,6 @@ module.exports = {
   uploadImageBuffer,
   processFileUpload,
   processDeleteRequest,
+  processAgentFileUpload,
   retrieveAndProcessFile,
 };

api/server/services/Files/strategies.js

@@ -21,6 +21,7 @@ const {
 } = require('./Local');
 const { uploadOpenAIFile, deleteOpenAIFile, getOpenAIFileStream } = require('./OpenAI');
 const { uploadVectors, deleteVectors } = require('./VectorDB');
+const { getCodeOutputDownloadStream } = require('./Code');
 
 /**
  * Firebase Storage Strategy Functions
@@ -103,6 +104,31 @@ const openAIStrategy = () => ({
   getDownloadStream: getOpenAIFileStream,
 });
 
+/**
+ * Code Output Strategy Functions
+ *
+ * Note: null values mean that the strategy is not supported.
+ * */
+const codeOutputStrategy = () => ({
+  /** @type {typeof saveFileFromURL | null} */
+  saveURL: null,
+  /** @type {typeof getLocalFileURL | null} */
+  getFileURL: null,
+  /** @type {typeof saveLocalBuffer | null} */
+  saveBuffer: null,
+  /** @type {typeof processLocalAvatar | null} */
+  processAvatar: null,
+  /** @type {typeof uploadLocalImage | null} */
+  handleImageUpload: null,
+  /** @type {typeof prepareImagesLocal | null} */
+  prepareImagePayload: null,
+  /** @type {typeof deleteLocalFile | null} */
+  deleteFile: null,
+  /** @type {typeof uploadVectors | null} */
+  handleFileUpload: null,
+  getDownloadStream: getCodeOutputDownloadStream,
+});
+
 // Strategy Selector
 const getStrategyFunctions = (fileSource) => {
   if (fileSource === FileSources.firebase) {
@@ -115,6 +141,8 @@ const getStrategyFunctions = (fileSource) => {
     return openAIStrategy();
   } else if (fileSource === FileSources.vectordb) {
     return vectorStrategy();
+  } else if (fileSource === FileSources.execute_code) {
+    return codeOutputStrategy();
   } else {
     throw new Error('Invalid file source');
   }

api/server/services/ToolService.js

@@ -1,9 +1,8 @@
 const fs = require('fs');
 const path = require('path');
-const { StructuredTool } = require('langchain/tools');
-const { tool: toolFn } = require('@langchain/core/tools');
 const { zodToJsonSchema } = require('zod-to-json-schema');
 const { Calculator } = require('langchain/tools/calculator');
+const { tool: toolFn, Tool } = require('@langchain/core/tools');
 const {
   Tools,
   ContentTypes,
@@ -70,7 +69,7 @@ function loadAndFormatTools({ directory, adminFilter = [], adminIncluded = [] })
       continue;
     }
 
-    if (!ToolClass || !(ToolClass.prototype instanceof StructuredTool)) {
+    if (!ToolClass || !(ToolClass.prototype instanceof Tool)) {
       continue;
     }
 
@@ -378,11 +377,12 @@ async function processRequiredActions(client, requiredActions) {
  * @param {Object} params - Run params containing user and request information.
  * @param {ServerRequest} params.req - The request object.
  * @param {string} params.agent_id - The agent ID.
- * @param {string[]} params.tools - The agent's available tools.
+ * @param {Agent['tools']} params.tools - The agent's available tools.
+ * @param {Agent['tool_resources']} params.tool_resources - The agent's available tool resources.
  * @param {string | undefined} [params.openAIApiKey] - The OpenAI API key.
  * @returns {Promise<{ tools?: StructuredTool[]; toolMap?: Record<string, StructuredTool>}>} The combined toolMap.
  */
-async function loadAgentTools({ req, agent_id, tools, openAIApiKey }) {
+async function loadAgentTools({ req, agent_id, tools, tool_resources, openAIApiKey }) {
   if (!tools || tools.length === 0) {
     return {};
   }
@@ -394,6 +394,7 @@ async function loadAgentTools({ req, agent_id, tools, openAIApiKey }) {
     options: {
       req,
       openAIApiKey,
+      tool_resources,
       returnMetadata: true,
       processFileURL,
       uploadImageBuffer,
@@ -405,6 +406,10 @@ async function loadAgentTools({ req, agent_id, tools, openAIApiKey }) {
   const agentTools = [];
   for (let i = 0; i < loadedTools.length; i++) {
     const tool = loadedTools[i];
+    if (tool.name && (tool.name === Tools.execute_code || tool.name === Tools.file_search)) {
+      agentTools.push(tool);
+      continue;
+    }
 
     const toolInstance = toolFn(
       async (...args) => {