🔧 feat: Initial MCP Support (Tools) (#5015)

* 📝 chore: Add comment to clarify purpose of check_updates.sh script

* feat: mcp package

* feat: add librechat-mcp package and update dependencies

* feat: refactor MCPConnectionSingleton to handle transport initialization and connection management

* feat: change private methods to public in MCPConnectionSingleton for improved accessibility

* feat: filesystem demo

* chore: everything demo and move everything under mcp workspace

* chore: move ts-node to mcp workspace

* feat: mcp examples

* feat: working sse MCP example

* refactor: rename MCPConnectionSingleton to MCPConnection for clarity

* refactor: replace MCPConnectionSingleton with MCPConnection for consistency

* refactor: manager/connections

* refactor: update MCPConnection to use type definitions from mcp types

* refactor: update MCPManager to use winston logger and enhance server initialization

* refactor: share logger between connections and manager

* refactor: add schema definitions and update MCPManager to accept logger parameter

* feat: map available MCP tools

* feat: load manifest tools

* feat: add MCP tools delimiter constant and update plugin key generation

* feat: call MCP tools

* feat: update librechat-data-provider version to 0.7.63 and enhance StdioOptionsSchema with additional properties

* refactor: simplify typing

* chore: update types/packages

* feat: MCP Tool Content parsing

* chore: update dependencies and improve package configurations

* feat: add 'mcp' directory to package and update configurations

* refactor: return CONTENT_AND_ARTIFACT format for MCP callTool

* chore: bump @librechat/agents

* WIP: MCP artifacts

* chore: bump @librechat/agents to v1.8.7

* fix: ensure filename has extension when saving base64 image

* fix: move base64 buffer conversion before filename extension check

* chore: update backend review workflow to install MCP package

* fix: use correct `mime` method

* fix: enhance file metadata with message and tool call IDs in image saving process

* fix: refactor ToolCall component to handle MCP tool calls and improve domain extraction

* fix: update ToolItem component for default isInstalled value and improve localization in ToolSelectDialog

* fix: update ToolItem component to use consistent text color for tool description

* style: add theming to ToolSelectDialog

* fix: improve domain extraction logic in ToolCall component

* refactor: conversation item theming, fix rename UI bug, optimize props, add missing types

* feat: enhance MCP options schema with base options (iconPath to start) and make transport type optional, infer based on other option fields

* fix: improve reconnection logic with parallel init and exponential backoff and enhance transport debug logging

* refactor: improve logging format

* refactor: improve logging of available tools by displaying tool names

* refactor: improve reconnection/connection logic

* feat: add MCP package build process to Dockerfile

* feat: add fallback icon for tools without an image in ToolItem component

* feat: Assistants Support for MCP Tools

* fix(build): configure rollup to use output.dir for dynamic imports

* chore: update @librechat/agents to version 1.8.8 and add @langchain/anthropic dependency

* fix: update CONFIG_VERSION to 1.2.0

api/server/controllers/PluginController.js

@@ -1,6 +1,8 @@
 const { promises: fs } = require('fs');
 const { CacheKeys, AuthType } = require('librechat-data-provider');
 const { addOpenAPISpecs } = require('~/app/clients/tools/util/addOpenAPISpecs');
+const { getCustomConfig } = require('~/server/services/Config');
+const { getMCPManager } = require('~/config');
 const { getLogStores } = require('~/cache');
 
 /**
@@ -107,6 +109,12 @@ const getAvailableTools = async (req, res) => {
     const pluginManifest = await fs.readFile(req.app.locals.paths.pluginManifest, 'utf8');
 
     const jsonData = JSON.parse(pluginManifest);
+    const customConfig = await getCustomConfig();
+    if (customConfig?.mcpServers != null) {
+      const mcpManager = await getMCPManager();
+      await mcpManager.loadManifestTools(jsonData);
+    }
+
     /** @type {TPlugin[]} */
     const uniquePlugins = filterUniquePlugins(jsonData);
 

api/server/controllers/agents/callbacks.js

@@ -1,4 +1,4 @@
-const { Tools, StepTypes, imageGenTools } = require('librechat-data-provider');
+const { Tools, StepTypes, imageGenTools, FileContext } = require('librechat-data-provider');
 const {
   EnvVar,
   GraphEvents,
@@ -6,6 +6,7 @@ const {
   ChatModelStreamHandler,
 } = require('@librechat/agents');
 const { processCodeOutput } = require('~/server/services/Files/Code/process');
+const { saveBase64Image } = require('~/server/services/Files/process');
 const { loadAuthValues } = require('~/app/clients/tools/util');
 const { logger } = require('~/config');
 
@@ -191,7 +192,11 @@ function createToolEndCallback({ req, res, artifactPromises }) {
       return;
     }
 
-    if (imageGenTools.has(output.name) && output.artifact) {
+    if (!output.artifact) {
+      return;
+    }
+
+    if (imageGenTools.has(output.name)) {
       artifactPromises.push(
         (async () => {
           const fileMetadata = Object.assign(output.artifact, {
@@ -217,10 +222,53 @@ function createToolEndCallback({ req, res, artifactPromises }) {
       return;
     }
 
-    if (output.name !== Tools.execute_code) {
+    if (output.artifact.content) {
+      /** @type {FormattedContent[]} */
+      const content = output.artifact.content;
+      for (const part of content) {
+        if (part.type !== 'image_url') {
+          continue;
+        }
+        const { url } = part.image_url;
+        artifactPromises.push(
+          (async () => {
+            const filename = `${output.tool_call_id}-image-${new Date().getTime()}`;
+            const file = await saveBase64Image(url, {
+              req,
+              filename,
+              endpoint: metadata.provider,
+              context: FileContext.image_generation,
+            });
+            const fileMetadata = Object.assign(file, {
+              messageId: metadata.run_id,
+              toolCallId: output.tool_call_id,
+              conversationId: metadata.thread_id,
+            });
+            if (!res.headersSent) {
+              return fileMetadata;
+            }
+
+            if (!fileMetadata) {
+              return null;
+            }
+
+            res.write(`event: attachment\ndata: ${JSON.stringify(fileMetadata)}\n\n`);
+            return fileMetadata;
+          })().catch((error) => {
+            logger.error('Error processing artifact content:', error);
+            return null;
+          }),
+        );
+      }
       return;
     }
 
+    {
+      if (output.name !== Tools.execute_code) {
+        return;
+      }
+    }
+
     if (!output.artifact.files) {
       return;
     }

api/server/services/AppService.js

@@ -9,6 +9,7 @@ const { azureConfigSetup } = require('./start/azureOpenAI');
 const { loadAndFormatTools } = require('./ToolService');
 const { agentsConfigSetup } = require('./start/agents');
 const { initializeRoles } = require('~/models/Role');
+const { getMCPManager } = require('~/config');
 const paths = require('~/config/paths');
 
 /**
@@ -39,11 +40,17 @@ const AppService = async (app) => {
 
   /** @type {Record<string, FunctionTool} */
   const availableTools = loadAndFormatTools({
-    directory: paths.structuredTools,
     adminFilter: filteredTools,
     adminIncluded: includedTools,
+    directory: paths.structuredTools,
   });
 
+  if (config.mcpServers != null) {
+    const mcpManager = await getMCPManager();
+    await mcpManager.initializeMCP(config.mcpServers);
+    await mcpManager.mapAvailableTools(availableTools);
+  }
+
   const socialLogins =
     config?.registration?.socialLogins ?? configDefaults?.registration?.socialLogins;
   const interfaceConfig = await loadDefaultInterface(config, configDefaults);

api/server/services/Endpoints/agents/initialize.js

@@ -80,8 +80,7 @@ const initializeAgentOptions = async ({
 }) => {
   const { tools, toolContextMap } = await loadAgentTools({
     req,
-    tools: agent.tools,
-    agent_id: agent.id,
+    agent,
     tool_resources,
   });
 

api/server/services/Files/images/resize.js

@@ -58,7 +58,12 @@ async function resizeImageBuffer(inputBuffer, resolution, endpoint) {
   const resizedBuffer = await sharp(inputBuffer).rotate().resize(resizeOptions).toBuffer();
 
   const resizedMetadata = await sharp(resizedBuffer).metadata();
-  return { buffer: resizedBuffer, width: resizedMetadata.width, height: resizedMetadata.height };
+  return {
+    buffer: resizedBuffer,
+    bytes: resizedMetadata.size,
+    width: resizedMetadata.width,
+    height: resizedMetadata.height,
+  };
 }
 
 /**

api/server/services/Files/process.js

@@ -18,8 +18,12 @@ const {
   isAssistantsEndpoint,
 } = require('librechat-data-provider');
 const { EnvVar } = require('@librechat/agents');
+const {
+  convertImage,
+  resizeAndConvert,
+  resizeImageBuffer,
+} = require('~/server/services/Files/images');
 const { addResourceFileId, deleteResourceFileId } = require('~/server/controllers/assistants/v2');
-const { convertImage, resizeAndConvert } = require('~/server/services/Files/images');
 const { addAgentResourceFile, removeAgentResourceFiles } = require('~/models/Agent');
 const { getOpenAIClient } = require('~/server/controllers/assistants/helpers');
 const { createFile, updateFileUsage, deleteFiles } = require('~/models/File');
@@ -736,6 +740,73 @@ async function retrieveAndProcessFile({
   }
 }
 
+/**
+ * Converts a base64 string to a buffer.
+ * @param {string} base64String
+ * @returns {Buffer<ArrayBufferLike>}
+ */
+function base64ToBuffer(base64String) {
+  try {
+    const typeMatch = base64String.match(/^data:([A-Za-z-+/]+);base64,/);
+    const type = typeMatch ? typeMatch[1] : '';
+
+    const base64Data = base64String.replace(/^data:([A-Za-z-+/]+);base64,/, '');
+
+    if (!base64Data) {
+      throw new Error('Invalid base64 string');
+    }
+
+    return {
+      buffer: Buffer.from(base64Data, 'base64'),
+      type,
+    };
+  } catch (error) {
+    throw new Error(`Failed to convert base64 to buffer: ${error.message}`);
+  }
+}
+
+async function saveBase64Image(
+  url,
+  { req, file_id: _file_id, filename: _filename, endpoint, context, resolution = 'high' },
+) {
+  const file_id = _file_id ?? v4();
+
+  let filename = _filename;
+  const { buffer: inputBuffer, type } = base64ToBuffer(url);
+  if (!path.extname(_filename)) {
+    const extension = mime.getExtension(type);
+    if (extension) {
+      filename += `.${extension}`;
+    } else {
+      throw new Error(`Could not determine file extension from MIME type: ${type}`);
+    }
+  }
+
+  const image = await resizeImageBuffer(inputBuffer, resolution, endpoint);
+  const source = req.app.locals.fileStrategy;
+  const { saveBuffer } = getStrategyFunctions(source);
+  const filepath = await saveBuffer({
+    userId: req.user.id,
+    fileName: filename,
+    buffer: image.buffer,
+  });
+  return await createFile(
+    {
+      type,
+      source,
+      context,
+      file_id,
+      filepath,
+      filename,
+      user: req.user.id,
+      bytes: image.bytes,
+      width: image.width,
+      height: image.height,
+    },
+    true,
+  );
+}
+
 /**
  * Filters a file based on its size and the endpoint origin.
  *
@@ -810,6 +881,7 @@ module.exports = {
   filterFile,
   processFiles,
   processFileURL,
+  saveBase64Image,
   processImageFile,
   uploadImageBuffer,
   processFileUpload,

api/server/services/MCP.js

@@ -0,0 +1,57 @@
+const { tool } = require('@langchain/core/tools');
+const { Constants: AgentConstants } = require('@librechat/agents');
+const {
+  Constants,
+  convertJsonSchemaToZod,
+  isAssistantsEndpoint,
+} = require('librechat-data-provider');
+const { logger, getMCPManager } = require('~/config');
+
+/**
+ * Creates a general tool for an entire action set.
+ *
+ * @param {Object} params - The parameters for loading action sets.
+ * @param {ServerRequest} params.req - The name of the tool.
+ * @param {string} params.toolKey - The toolKey for the tool.
+ * @param {import('@librechat/agents').Providers | EModelEndpoint} params.provider - The provider for the tool.
+ * @param {string} params.model - The model for the tool.
+ * @returns { Promise<typeof tool | { _call: (toolInput: Object | string) => unknown}> } An object with `_call` method to execute the tool input.
+ */
+async function createMCPTool({ req, toolKey, provider }) {
+  const toolDefinition = req.app.locals.availableTools[toolKey]?.function;
+  if (!toolDefinition) {
+    logger.error(`Tool ${toolKey} not found in available tools`);
+    return null;
+  }
+  /** @type {LCTool} */
+  const { description, parameters } = toolDefinition;
+  const schema = convertJsonSchemaToZod(parameters);
+  const [toolName, serverName] = toolKey.split(Constants.mcp_delimiter);
+  /** @type {(toolInput: Object | string) => Promise<unknown>} */
+  const _call = async (toolInput) => {
+    try {
+      const mcpManager = await getMCPManager();
+      const result = await mcpManager.callTool(serverName, toolName, provider, toolInput);
+      if (isAssistantsEndpoint(provider) && Array.isArray(result)) {
+        return result[0];
+      }
+      return result;
+    } catch (error) {
+      logger.error(`${toolName} MCP server tool call failed`, error);
+      return `${toolName} MCP server tool call failed.`;
+    }
+  };
+
+  const toolInstance = tool(_call, {
+    schema,
+    name: toolKey,
+    description: description || '',
+    responseFormat: AgentConstants.CONTENT_AND_ARTIFACT,
+  });
+  toolInstance.mcp = true;
+  return toolInstance;
+}
+
+module.exports = {
+  createMCPTool,
+};

api/server/services/ToolService.js

@@ -176,6 +176,7 @@ async function processRequiredActions(client, requiredActions) {
     model: client.req.body.model ?? 'gpt-4o-mini',
     tools,
     functions: true,
+    endpoint: client.req.body.endpoint,
     options: {
       processFileURL,
       req: client.req,
@@ -374,22 +375,19 @@ async function processRequiredActions(client, requiredActions) {
  * Processes the runtime tool calls and returns the tool classes.
  * @param {Object} params - Run params containing user and request information.
  * @param {ServerRequest} params.req - The request object.
- * @param {string} params.agent_id - The agent ID.
- * @param {Agent['tools']} params.tools - The agent's available tools.
- * @param {Agent['tool_resources']} params.tool_resources - The agent's available tool resources.
+ * @param {Agent} params.agent - The agent to load tools for.
  * @param {string | undefined} [params.openAIApiKey] - The OpenAI API key.
  * @returns {Promise<{ tools?: StructuredTool[] }>} The agent tools.
  */
-async function loadAgentTools({ req, agent_id, tools, tool_resources, openAIApiKey }) {
-  if (!tools || tools.length === 0) {
+async function loadAgentTools({ req, agent, tool_resources, openAIApiKey }) {
+  if (!agent.tools || agent.tools.length === 0) {
     return {};
   }
   const { loadedTools, toolContextMap } = await loadTools({
-    user: req.user.id,
-    // model: req.body.model ?? 'gpt-4o-mini',
-    tools,
+    agent,
     functions: true,
-    isAgent: agent_id != null,
+    user: req.user.id,
+    tools: agent.tools,
     options: {
       req,
       openAIApiKey,
@@ -409,6 +407,11 @@ async function loadAgentTools({ req, agent_id, tools, tool_resources, openAIApiK
       continue;
     }
 
+    if (tool.mcp === true) {
+      agentTools.push(tool);
+      continue;
+    }
+
     const toolDefinition = {
       name: tool.name,
       schema: tool.schema,
@@ -434,10 +437,10 @@ async function loadAgentTools({ req, agent_id, tools, tool_resources, openAIApiK
   let actionSets = [];
   const ActionToolMap = {};
 
-  for (const toolName of tools) {
+  for (const toolName of agent.tools) {
     if (!ToolMap[toolName]) {
       if (!actionSets.length) {
-        actionSets = (await loadActionSets({ agent_id })) ?? [];
+        actionSets = (await loadActionSets({ agent_id: agent.id })) ?? [];
       }
 
       let actionSet = null;
@@ -473,7 +476,7 @@ async function loadAgentTools({ req, agent_id, tools, tool_resources, openAIApiK
             });
             if (!tool) {
               logger.warn(
-                `Invalid action: user: ${req.user.id} | agent_id: ${agent_id} | toolName: ${toolName}`,
+                `Invalid action: user: ${req.user.id} | agent_id: ${agent.id} | toolName: ${toolName}`,
               );
               throw new Error(`{"type":"${ErrorTypes.INVALID_ACTION}"}`);
             }
@@ -485,7 +488,7 @@ async function loadAgentTools({ req, agent_id, tools, tool_resources, openAIApiK
     }
   }
 
-  if (tools.length > 0 && agentTools.length === 0) {
+  if (agent.tools.length > 0 && agentTools.length === 0) {
     throw new Error('No tools found for the specified tool calls.');
   }