2025-08-23 03:27:05 -04:00
|
|
|
const { logger } = require('@librechat/data-schemas');
|
|
|
|
|
const { CacheKeys, Constants } = require('librechat-data-provider');
|
🩹 fix: MCP Server Recovery from Startup Inspection Failures (#12145)
* feat: MCP server reinitialization recovery mechanism
- Added functionality to store a stub configuration for MCP servers that fail inspection at startup, allowing for recovery via reinitialization.
- Introduced `reinspectServer` method in `MCPServersRegistry` to handle reinspection of previously failed servers.
- Enhanced `MCPServersInitializer` to log and manage server initialization failures, ensuring proper handling of inspection failures.
- Added integration tests to verify the recovery process for unreachable MCP servers, ensuring that stub configurations are stored and can be reinitialized successfully.
- Updated type definitions to include `inspectionFailed` flag in server configurations for better state management.
* fix: MCP server handling for inspection failures
- Updated `reinitMCPServer` to return a structured response when the server is unreachable, providing clearer feedback on the failure.
- Modified `ConnectionsRepository` to prevent connections to servers marked as inspection failed, improving error handling.
- Adjusted `MCPServersRegistry` methods to ensure proper management of server states, including throwing errors for non-failed servers during reinspection.
- Enhanced integration tests to validate the behavior of the system when dealing with unreachable MCP servers and inspection failures, ensuring robust recovery mechanisms.
* fix: Clear all cached server configurations in MCPServersRegistry
- Added a comment to clarify the necessity of clearing all cached server configurations when updating a server's configuration, as the cache is keyed by userId without a reverse index for enumeration.
* fix: Update integration test for file_tools_server inspection handling
- Modified the test to verify that the `file_tools_server` is stored as a stub when inspection fails, ensuring it can be reinitialized correctly.
- Adjusted expectations to confirm that the `inspectionFailed` flag is set to true for the stub configuration, enhancing the robustness of the recovery mechanism.
* test: Add unit tests for reinspecting servers in MCPServersRegistry
- Introduced tests for the `reinspectServer` method to validate error handling when called on a healthy server and when the server does not exist.
- Ensured that appropriate exceptions are thrown for both scenarios, enhancing the robustness of server state management.
* test: Add integration test for concurrent reinspectServer calls
- Introduced a new test to validate that multiple concurrent calls to reinspectServer do not crash or corrupt the server state.
- Ensured that at least one call succeeds and any failures are due to the server not being in a failed state, enhancing the reliability of the reinitialization process.
* test: Enhance integration test for concurrent MCP server reinitialization
- Added a new test to validate that concurrent calls to reinitialize the MCP server do not crash or corrupt the server state.
- Ensured that at least one call succeeds and that failures are handled gracefully, improving the reliability of the reinitialization process.
- Reset MCPManager instance after each test to maintain a clean state for subsequent tests.
2026-03-08 21:49:04 -04:00
|
|
|
const { getMCPManager, getMCPServersRegistry, getFlowStateManager } = require('~/config');
|
2025-08-23 03:27:05 -04:00
|
|
|
const { findToken, createToken, updateToken, deleteTokens } = require('~/models');
|
2025-09-21 07:56:40 -04:00
|
|
|
const { updateMCPServerTools } = require('~/server/services/Config');
|
2025-08-23 03:27:05 -04:00
|
|
|
const { getLogStores } = require('~/cache');
|
|
|
|
|
|
|
|
|
|
/**
|
2026-02-01 19:37:04 -05:00
|
|
|
* Reinitializes an MCP server connection and discovers available tools.
|
|
|
|
|
* When OAuth is required, uses discovery mode to list tools without full authentication
|
|
|
|
|
* (per MCP spec, tool listing should be possible without auth).
|
2025-08-23 03:27:05 -04:00
|
|
|
* @param {Object} params
|
2025-09-24 22:48:38 -04:00
|
|
|
* @param {IUser} params.user - The user from the request object.
|
2025-08-23 03:27:05 -04:00
|
|
|
* @param {string} params.serverName - The name of the MCP server
|
|
|
|
|
* @param {boolean} params.returnOnOAuth - Whether to initiate OAuth and return, or wait for OAuth flow to finish
|
|
|
|
|
* @param {AbortSignal} [params.signal] - The abort signal to handle cancellation.
|
|
|
|
|
* @param {boolean} [params.forceNew]
|
|
|
|
|
* @param {number} [params.connectionTimeout]
|
|
|
|
|
* @param {FlowStateManager<any>} [params.flowManager]
|
2026-02-01 19:37:04 -05:00
|
|
|
* @param {(authURL: string) => Promise<void>} [params.oauthStart]
|
2025-08-23 03:27:05 -04:00
|
|
|
* @param {Record<string, Record<string, string>>} [params.userMCPAuthMap]
|
|
|
|
|
*/
|
|
|
|
|
async function reinitMCPServer({
|
2025-09-24 22:48:38 -04:00
|
|
|
user,
|
2025-08-23 03:27:05 -04:00
|
|
|
signal,
|
|
|
|
|
forceNew,
|
|
|
|
|
serverName,
|
|
|
|
|
userMCPAuthMap,
|
|
|
|
|
connectionTimeout,
|
|
|
|
|
returnOnOAuth = true,
|
|
|
|
|
oauthStart: _oauthStart,
|
|
|
|
|
flowManager: _flowManager,
|
|
|
|
|
}) {
|
|
|
|
|
/** @type {MCPConnection | null} */
|
2025-09-26 06:24:36 -06:00
|
|
|
let connection = null;
|
2025-08-23 03:27:05 -04:00
|
|
|
/** @type {LCAvailableTools | null} */
|
|
|
|
|
let availableTools = null;
|
|
|
|
|
/** @type {ReturnType<MCPConnection['fetchTools']> | null} */
|
|
|
|
|
let tools = null;
|
|
|
|
|
let oauthRequired = false;
|
|
|
|
|
let oauthUrl = null;
|
2026-02-01 19:37:04 -05:00
|
|
|
|
2025-08-23 03:27:05 -04:00
|
|
|
try {
|
🩹 fix: MCP Server Recovery from Startup Inspection Failures (#12145)
* feat: MCP server reinitialization recovery mechanism
- Added functionality to store a stub configuration for MCP servers that fail inspection at startup, allowing for recovery via reinitialization.
- Introduced `reinspectServer` method in `MCPServersRegistry` to handle reinspection of previously failed servers.
- Enhanced `MCPServersInitializer` to log and manage server initialization failures, ensuring proper handling of inspection failures.
- Added integration tests to verify the recovery process for unreachable MCP servers, ensuring that stub configurations are stored and can be reinitialized successfully.
- Updated type definitions to include `inspectionFailed` flag in server configurations for better state management.
* fix: MCP server handling for inspection failures
- Updated `reinitMCPServer` to return a structured response when the server is unreachable, providing clearer feedback on the failure.
- Modified `ConnectionsRepository` to prevent connections to servers marked as inspection failed, improving error handling.
- Adjusted `MCPServersRegistry` methods to ensure proper management of server states, including throwing errors for non-failed servers during reinspection.
- Enhanced integration tests to validate the behavior of the system when dealing with unreachable MCP servers and inspection failures, ensuring robust recovery mechanisms.
* fix: Clear all cached server configurations in MCPServersRegistry
- Added a comment to clarify the necessity of clearing all cached server configurations when updating a server's configuration, as the cache is keyed by userId without a reverse index for enumeration.
* fix: Update integration test for file_tools_server inspection handling
- Modified the test to verify that the `file_tools_server` is stored as a stub when inspection fails, ensuring it can be reinitialized correctly.
- Adjusted expectations to confirm that the `inspectionFailed` flag is set to true for the stub configuration, enhancing the robustness of the recovery mechanism.
* test: Add unit tests for reinspecting servers in MCPServersRegistry
- Introduced tests for the `reinspectServer` method to validate error handling when called on a healthy server and when the server does not exist.
- Ensured that appropriate exceptions are thrown for both scenarios, enhancing the robustness of server state management.
* test: Add integration test for concurrent reinspectServer calls
- Introduced a new test to validate that multiple concurrent calls to reinspectServer do not crash or corrupt the server state.
- Ensured that at least one call succeeds and any failures are due to the server not being in a failed state, enhancing the reliability of the reinitialization process.
* test: Enhance integration test for concurrent MCP server reinitialization
- Added a new test to validate that concurrent calls to reinitialize the MCP server do not crash or corrupt the server state.
- Ensured that at least one call succeeds and that failures are handled gracefully, improving the reliability of the reinitialization process.
- Reset MCPManager instance after each test to maintain a clean state for subsequent tests.
2026-03-08 21:49:04 -04:00
|
|
|
const registry = getMCPServersRegistry();
|
|
|
|
|
const serverConfig = await registry.getServerConfig(serverName, user?.id);
|
|
|
|
|
if (serverConfig?.inspectionFailed) {
|
|
|
|
|
logger.info(
|
|
|
|
|
`[MCP Reinitialize] Server ${serverName} had failed inspection, attempting reinspection`,
|
|
|
|
|
);
|
|
|
|
|
try {
|
|
|
|
|
const storageLocation = serverConfig.dbId ? 'DB' : 'CACHE';
|
|
|
|
|
await registry.reinspectServer(serverName, storageLocation, user?.id);
|
|
|
|
|
logger.info(`[MCP Reinitialize] Reinspection succeeded for server: ${serverName}`);
|
|
|
|
|
} catch (reinspectError) {
|
|
|
|
|
logger.error(
|
|
|
|
|
`[MCP Reinitialize] Reinspection failed for server ${serverName}:`,
|
|
|
|
|
reinspectError,
|
|
|
|
|
);
|
|
|
|
|
return {
|
|
|
|
|
availableTools: null,
|
|
|
|
|
success: false,
|
|
|
|
|
message: `MCP server '${serverName}' is still unreachable`,
|
|
|
|
|
oauthRequired: false,
|
|
|
|
|
serverName,
|
|
|
|
|
oauthUrl: null,
|
|
|
|
|
tools: null,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2025-08-23 03:27:05 -04:00
|
|
|
const customUserVars = userMCPAuthMap?.[`${Constants.mcp_prefix}${serverName}`];
|
|
|
|
|
const flowManager = _flowManager ?? getFlowStateManager(getLogStores(CacheKeys.FLOWS));
|
|
|
|
|
const mcpManager = getMCPManager();
|
2026-02-01 19:37:04 -05:00
|
|
|
const tokenMethods = { findToken, updateToken, createToken, deleteTokens };
|
2025-08-23 03:27:05 -04:00
|
|
|
|
|
|
|
|
const oauthStart =
|
|
|
|
|
_oauthStart ??
|
|
|
|
|
(async (authURL) => {
|
2025-09-19 06:50:02 -04:00
|
|
|
logger.info(`[MCP Reinitialize] OAuth URL received for ${serverName}`);
|
2025-08-23 03:27:05 -04:00
|
|
|
oauthUrl = authURL;
|
|
|
|
|
oauthRequired = true;
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
try {
|
2025-09-26 06:24:36 -06:00
|
|
|
connection = await mcpManager.getConnection({
|
2025-09-24 22:48:38 -04:00
|
|
|
user,
|
2025-08-23 03:27:05 -04:00
|
|
|
signal,
|
|
|
|
|
forceNew,
|
|
|
|
|
oauthStart,
|
|
|
|
|
serverName,
|
|
|
|
|
flowManager,
|
2026-02-01 19:37:04 -05:00
|
|
|
tokenMethods,
|
2025-08-23 03:27:05 -04:00
|
|
|
returnOnOAuth,
|
|
|
|
|
customUserVars,
|
|
|
|
|
connectionTimeout,
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
logger.info(`[MCP Reinitialize] Successfully established connection for ${serverName}`);
|
|
|
|
|
} catch (err) {
|
2025-09-26 06:24:36 -06:00
|
|
|
logger.info(`[MCP Reinitialize] getConnection threw error: ${err.message}`);
|
2025-08-23 03:27:05 -04:00
|
|
|
logger.info(
|
|
|
|
|
`[MCP Reinitialize] OAuth state - oauthRequired: ${oauthRequired}, oauthUrl: ${oauthUrl ? 'present' : 'null'}`,
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
const isOAuthError =
|
|
|
|
|
err.message?.includes('OAuth') ||
|
|
|
|
|
err.message?.includes('authentication') ||
|
|
|
|
|
err.message?.includes('401');
|
|
|
|
|
|
|
|
|
|
const isOAuthFlowInitiated = err.message === 'OAuth flow initiated - return early';
|
|
|
|
|
|
|
|
|
|
if (isOAuthError || oauthRequired || isOAuthFlowInitiated) {
|
|
|
|
|
logger.info(
|
2026-02-01 19:37:04 -05:00
|
|
|
`[MCP Reinitialize] OAuth required for ${serverName}, attempting tool discovery without auth`,
|
2025-08-23 03:27:05 -04:00
|
|
|
);
|
|
|
|
|
oauthRequired = true;
|
2026-02-01 19:37:04 -05:00
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
const discoveryResult = await mcpManager.discoverServerTools({
|
|
|
|
|
user,
|
|
|
|
|
signal,
|
|
|
|
|
serverName,
|
|
|
|
|
flowManager,
|
|
|
|
|
tokenMethods,
|
|
|
|
|
oauthStart,
|
|
|
|
|
customUserVars,
|
|
|
|
|
connectionTimeout,
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
if (discoveryResult.tools && discoveryResult.tools.length > 0) {
|
|
|
|
|
tools = discoveryResult.tools;
|
|
|
|
|
logger.info(
|
|
|
|
|
`[MCP Reinitialize] Discovered ${tools.length} tools for ${serverName} without full auth`,
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
} catch (discoveryErr) {
|
|
|
|
|
logger.debug(
|
|
|
|
|
`[MCP Reinitialize] Tool discovery failed for ${serverName}: ${discoveryErr?.message ?? String(discoveryErr)}`,
|
|
|
|
|
);
|
|
|
|
|
}
|
2025-08-23 03:27:05 -04:00
|
|
|
} else {
|
|
|
|
|
logger.error(
|
|
|
|
|
`[MCP Reinitialize] Error initializing MCP server ${serverName} for user:`,
|
|
|
|
|
err,
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2025-09-26 06:24:36 -06:00
|
|
|
if (connection && !oauthRequired) {
|
|
|
|
|
tools = await connection.fetchTools();
|
2026-02-01 19:37:04 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (tools && tools.length > 0) {
|
2025-09-21 07:56:40 -04:00
|
|
|
availableTools = await updateMCPServerTools({
|
2025-10-30 22:09:56 +01:00
|
|
|
userId: user.id,
|
2025-08-23 03:27:05 -04:00
|
|
|
serverName,
|
|
|
|
|
tools,
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
logger.debug(
|
|
|
|
|
`[MCP Reinitialize] Sending response for ${serverName} - oauthRequired: ${oauthRequired}, oauthUrl: ${oauthUrl ? 'present' : 'null'}`,
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
const getResponseMessage = () => {
|
2026-02-01 19:37:04 -05:00
|
|
|
if (oauthRequired && tools && tools.length > 0) {
|
|
|
|
|
return `MCP server '${serverName}' tools discovered, OAuth required for execution`;
|
|
|
|
|
}
|
2025-08-23 03:27:05 -04:00
|
|
|
if (oauthRequired) {
|
|
|
|
|
return `MCP server '${serverName}' ready for OAuth authentication`;
|
|
|
|
|
}
|
2025-09-26 06:24:36 -06:00
|
|
|
if (connection) {
|
2025-08-23 03:27:05 -04:00
|
|
|
return `MCP server '${serverName}' reinitialized successfully`;
|
|
|
|
|
}
|
|
|
|
|
return `Failed to reinitialize MCP server '${serverName}'`;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
const result = {
|
|
|
|
|
availableTools,
|
2026-02-01 19:37:04 -05:00
|
|
|
success: Boolean(
|
|
|
|
|
(connection && !oauthRequired) ||
|
|
|
|
|
(oauthRequired && oauthUrl) ||
|
|
|
|
|
(tools && tools.length > 0),
|
|
|
|
|
),
|
2025-08-23 03:27:05 -04:00
|
|
|
message: getResponseMessage(),
|
|
|
|
|
oauthRequired,
|
|
|
|
|
serverName,
|
|
|
|
|
oauthUrl,
|
|
|
|
|
tools,
|
|
|
|
|
};
|
2026-02-01 19:37:04 -05:00
|
|
|
|
⚠️ fix: OAuth Error and Token Expiry Detection and Reporting Improvements (#10922)
* fix: create new flows on invalid_grant errors
* chore: fix failing test
* chore: keep isOAuthError test function in sync with implementation
* test: add tests for OAuth error detection on invalid grant errors
* test: add tests for creating new flows when token expires
* test: add test for flow clean up prior to creation
* refactor: consolidate token expiration handling in FlowStateManager
- Removed the old token expiration checks and replaced them with a new method, `isTokenExpired`, to streamline the logic.
- Introduced `normalizeExpirationTimestamp` to handle timestamp normalization for both seconds and milliseconds.
- Updated tests to ensure proper functionality of flow management with token expiration scenarios.
* fix: conditionally setup cleanup handlers in FlowStateManager
- Updated the FlowStateManager constructor to only call setupCleanupHandlers if the ci parameter is not set, improving flexibility in flow management.
* chore: enhance OAuth token refresh logging
- Introduced a new method, `processRefreshResponse`, to streamline the processing of token refresh responses from the OAuth server.
- Improved logging to provide detailed information about token refresh operations, including whether new tokens were received and if the refresh token was rotated.
- Updated existing token handling logic to utilize the new method, ensuring consistency and clarity in token management.
* chore: enhance logging for MCP server reinitialization
- Updated the logging in the reinitMCPServer function to provide more detailed information about the response, including success status, OAuth requirements, presence of the OAuth URL, and the count of tools involved. This improves the clarity and usefulness of logs for debugging purposes.
---------
Co-authored-by: Danny Avila <danny@librechat.ai>
2025-12-12 10:51:28 -08:00
|
|
|
logger.debug(`[MCP Reinitialize] Response for ${serverName}:`, {
|
|
|
|
|
success: result.success,
|
|
|
|
|
oauthRequired: result.oauthRequired,
|
|
|
|
|
oauthUrl: result.oauthUrl ? 'present' : null,
|
|
|
|
|
toolsCount: tools?.length ?? 0,
|
|
|
|
|
});
|
2026-02-01 19:37:04 -05:00
|
|
|
|
2025-08-23 03:27:05 -04:00
|
|
|
return result;
|
|
|
|
|
} catch (error) {
|
|
|
|
|
logger.error(
|
|
|
|
|
'[MCP Reinitialize] Error loading MCP Tools, servers may still be initializing:',
|
|
|
|
|
error,
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
module.exports = {
|
|
|
|
|
reinitMCPServer,
|
|
|
|
|
};
|