From d16f93b5f7954492fdf8df645d35984f9f31f815 Mon Sep 17 00:00:00 2001 From: Samuel Path Date: Fri, 29 Aug 2025 19:07:19 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=A8=20feat:=20MCP=20UI=20basic=20integ?= =?UTF-8?q?ration=20(#9299)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/package.json | 1 + client/src/common/types.ts | 7 + .../Chat/Messages/Content/ToolCallInfo.tsx | 38 ++++ .../Chat/Messages/Content/UIResourceGrid.tsx | 145 ++++++++++++++++ client/src/locales/en/translation.json | 1 + package-lock.json | 163 ++++++++++++++++-- packages/api/src/mcp/parsers.ts | 16 +- packages/api/src/mcp/types/index.ts | 8 + 8 files changed, 365 insertions(+), 14 deletions(-) create mode 100644 client/src/components/Chat/Messages/Content/UIResourceGrid.tsx diff --git a/client/package.json b/client/package.json index 3a9a57032..aad64ed55 100644 --- a/client/package.json +++ b/client/package.json @@ -37,6 +37,7 @@ "@headlessui/react": "^2.1.2", "@librechat/client": "*", "@marsidev/react-turnstile": "^1.1.0", + "@mcp-ui/client": "^5.7.0", "@radix-ui/react-accordion": "^1.1.2", "@radix-ui/react-alert-dialog": "^1.0.2", "@radix-ui/react-checkbox": "^1.0.3", diff --git a/client/src/common/types.ts b/client/src/common/types.ts index 9df024276..daa56f71c 100644 --- a/client/src/common/types.ts +++ b/client/src/common/types.ts @@ -630,3 +630,10 @@ declare global { google_tag_manager?: unknown; } } + +export type UIResource = { + uri: string; + mimeType: string; + text: string; + [key: string]: unknown; +}; diff --git a/client/src/components/Chat/Messages/Content/ToolCallInfo.tsx b/client/src/components/Chat/Messages/Content/ToolCallInfo.tsx index fd0fe8e1d..7a7930bba 100644 --- a/client/src/components/Chat/Messages/Content/ToolCallInfo.tsx +++ b/client/src/components/Chat/Messages/Content/ToolCallInfo.tsx @@ -1,5 +1,8 @@ import React from 'react'; import { useLocalize } from '~/hooks'; +import { UIResourceRenderer } from '@mcp-ui/client'; +import UIResourceGrid from './UIResourceGrid'; +import type { UIResource } from '~/common'; function OptimizedCodeBlock({ text, maxHeight = 320 }: { text: string; maxHeight?: number }) { return ( @@ -51,6 +54,21 @@ export default function ToolCallInfo({ : localize('com_assistants_attempt_info'); } + // Extract ui_resources from the output to display them in the UI + let uiResources: UIResource[] = []; + if (output?.includes('ui_resources')) { + const parsedOutput = JSON.parse(output); + const uiResourcesItem = parsedOutput.find( + (contentItem) => contentItem.metadata === 'ui_resources', + ); + if (uiResourcesItem?.text) { + uiResources = JSON.parse(atob(uiResourcesItem.text)) as UIResource[]; + } + output = JSON.stringify( + parsedOutput.filter((contentItem) => contentItem.metadata !== 'ui_resources'), + ); + } + return (
@@ -66,6 +84,26 @@ export default function ToolCallInfo({
+ {uiResources.length > 0 && ( +
+ {localize('com_ui_ui_resources')} +
+ )} +
+ {uiResources.length > 1 && } + + {uiResources.length === 1 && ( + { + console.log('Action:', result); + }} + htmlProps={{ + autoResizeIframe: { width: true, height: true }, + }} + /> + )} +
)}
diff --git a/client/src/components/Chat/Messages/Content/UIResourceGrid.tsx b/client/src/components/Chat/Messages/Content/UIResourceGrid.tsx new file mode 100644 index 000000000..6d5f77cfa --- /dev/null +++ b/client/src/components/Chat/Messages/Content/UIResourceGrid.tsx @@ -0,0 +1,145 @@ +import { UIResourceRenderer } from '@mcp-ui/client'; +import type { UIResource } from '~/common'; +import React, { useState } from 'react'; + +interface UIResourceGridProps { + uiResources: UIResource[]; +} + +const UIResourceGrid: React.FC = React.memo(({ uiResources }) => { + const [showLeftArrow, setShowLeftArrow] = useState(false); + const [showRightArrow, setShowRightArrow] = useState(true); + const [isContainerHovered, setIsContainerHovered] = useState(false); + const scrollContainerRef = React.useRef(null); + + const handleScroll = React.useCallback(() => { + if (!scrollContainerRef.current) return; + + const { scrollLeft, scrollWidth, clientWidth } = scrollContainerRef.current; + setShowLeftArrow(scrollLeft > 0); + setShowRightArrow(scrollLeft < scrollWidth - clientWidth - 10); + }, []); + + const scroll = React.useCallback((direction: 'left' | 'right') => { + if (!scrollContainerRef.current) return; + + const viewportWidth = scrollContainerRef.current.clientWidth; + const scrollAmount = Math.floor(viewportWidth * 0.9); + const currentScroll = scrollContainerRef.current.scrollLeft; + const newScroll = + direction === 'left' ? currentScroll - scrollAmount : currentScroll + scrollAmount; + + scrollContainerRef.current.scrollTo({ + left: newScroll, + behavior: 'smooth', + }); + }, []); + + React.useEffect(() => { + const container = scrollContainerRef.current; + if (container) { + container.addEventListener('scroll', handleScroll); + handleScroll(); + return () => container.removeEventListener('scroll', handleScroll); + } + }, [handleScroll]); + + if (uiResources.length === 0) { + return null; + } + + return ( +
setIsContainerHovered(true)} + onMouseLeave={() => setIsContainerHovered(false)} + > +
+ +
+ + {showLeftArrow && ( + + )} + +
+ {uiResources.map((uiResource, index) => { + const height = 360; + const width = 230; + + return ( +
+
+ { + console.log('Action:', result); + }} + htmlProps={{ + autoResizeIframe: { width: true, height: true }, + }} + /> +
+
+ ); + })} +
+ + {showRightArrow && ( + + )} +
+ ); +}); + +export default UIResourceGrid; diff --git a/client/src/locales/en/translation.json b/client/src/locales/en/translation.json index 8a15ebe34..70ed6dc2b 100644 --- a/client/src/locales/en/translation.json +++ b/client/src/locales/en/translation.json @@ -1198,6 +1198,7 @@ "com_ui_travel": "Travel", "com_ui_trust_app": "I trust this application", "com_ui_try_adjusting_search": "Try adjusting your search terms", + "com_ui_ui_resources": "UI Resources", "com_ui_unarchive": "Unarchive", "com_ui_unarchive_error": "Failed to unarchive conversation", "com_ui_unknown": "Unknown", diff --git a/package-lock.json b/package-lock.json index 1b42c31c5..d71bd3484 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2633,6 +2633,7 @@ "@headlessui/react": "^2.1.2", "@librechat/client": "*", "@marsidev/react-turnstile": "^1.1.0", + "@mcp-ui/client": "^5.7.0", "@radix-ui/react-accordion": "^1.1.2", "@radix-ui/react-alert-dialog": "^1.0.2", "@radix-ui/react-checkbox": "^1.0.3", @@ -22546,6 +22547,21 @@ "react-dom": "^17.0.2 || ^18.0.0 || ^19.0" } }, + "node_modules/@mcp-ui/client": { + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/@mcp-ui/client/-/client-5.7.0.tgz", + "integrity": "sha512-+HbPw3VS46WUSWmyJ34ZVnygb81QByA3luR6y0JDbyDZxjYtHw1FcIN7v9WbbE8PrfI0WcuWCSiNOO6sOGbwpQ==", + "license": "Apache-2.0", + "dependencies": { + "@modelcontextprotocol/sdk": "*", + "@quilted/threads": "^3.1.3", + "@r2wc/react-to-web-component": "^2.0.4", + "@remote-dom/core": "^1.8.0", + "@remote-dom/react": "^1.2.2", + "react": "^18.3.1", + "react-dom": "^18.3.1" + } + }, "node_modules/@microsoft/eslint-formatter-sarif": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/@microsoft/eslint-formatter-sarif/-/eslint-formatter-sarif-3.1.0.tgz", @@ -23229,6 +23245,16 @@ "node": ">=18" } }, + "node_modules/@preact/signals-core": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@preact/signals-core/-/signals-core-1.11.0.tgz", + "integrity": "sha512-jglbibeWHuFRzEWVFY/TT7wB1PppJxmcSfUHcK+2J9vBRtiooMfw6tAPttojNYrrpdGViqAYCbPpmWYlMm+eMQ==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/preact" + } + }, "node_modules/@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -23283,6 +23309,57 @@ "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" }, + "node_modules/@quilted/events": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@quilted/events/-/events-2.1.3.tgz", + "integrity": "sha512-4fHaSLND8rmZ+tce9/4FNmG5UWTRpFtM54kOekf3tLON4ZLLnYzjjldELD35efd7+lT5+E3cdkacqc56d+kCrQ==", + "license": "MIT", + "dependencies": { + "@preact/signals-core": "^1.8.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@quilted/threads": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@quilted/threads/-/threads-3.3.1.tgz", + "integrity": "sha512-0ASnjTH+hOu1Qwzi9NnsVcsbMhWVx8pEE8SXIHknqcc/1rXAU0QlKw9ARq0W43FAdzyVeuXeXtZN27ZC0iALKg==", + "license": "MIT", + "dependencies": { + "@quilted/events": "^2.1.3" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "@preact/signals-core": "^1.8.0" + }, + "peerDependenciesMeta": { + "@preact/signals-core": { + "optional": true + } + } + }, + "node_modules/@r2wc/core": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@r2wc/core/-/core-1.2.0.tgz", + "integrity": "sha512-vAfiuS5KywtV54SRzc4maEHcpdgeUyJzln+ATpNCOkO+ArIuOkTXd92b5YauVAd0A8B2rV/y9OeVW19vb73bUQ==", + "license": "MIT" + }, + "node_modules/@r2wc/react-to-web-component": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@r2wc/react-to-web-component/-/react-to-web-component-2.0.4.tgz", + "integrity": "sha512-g1dtTTEGETNUimYldTW+2hxY3mmJZjzPEca0vqCutUht2GHmpK9mT5r/urmEI7uSbOkn6HaymosgVy26lvU1JQ==", + "license": "MIT", + "dependencies": { + "@r2wc/core": "^1.0.0" + }, + "peerDependencies": { + "react": "^18.0.0 || ^19.0.0", + "react-dom": "^18.0.0 || ^19.0.0" + } + }, "node_modules/@radix-ui/number": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@radix-ui/number/-/number-1.1.1.tgz", @@ -26886,6 +26963,61 @@ "node": ">=14.0.0" } }, + "node_modules/@remote-dom/core": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@remote-dom/core/-/core-1.9.0.tgz", + "integrity": "sha512-h8OO2NRns2paXO/q5hkfXrwlZKq7oKj9XedGosi7J8OP3+aW7N2Gv4MBBVVQGCfOiZPkOj5m3sQH7FdyUWl7PQ==", + "license": "MIT", + "dependencies": { + "@remote-dom/polyfill": "^1.4.4", + "htm": "^3.1.1" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "@preact/signals-core": "^1.3.0" + }, + "peerDependenciesMeta": { + "@preact/signals-core": { + "optional": true + }, + "preact": { + "optional": true + } + } + }, + "node_modules/@remote-dom/polyfill": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/@remote-dom/polyfill/-/polyfill-1.4.5.tgz", + "integrity": "sha512-V1qkKIl/wXyDO0I+tQDH06cBBNyyViZF3IYorkTTBf58dorqOP5Ta51vCCWeekPgdSOPuEKvHhvu6kAaKqVgww==", + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@remote-dom/react": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@remote-dom/react/-/react-1.2.2.tgz", + "integrity": "sha512-PkvioODONTr1M0StGDYsR4Ssf5M0Rd4+IlWVvVoK3Zrw8nr7+5mJkgNofaj/z7i8Aep78L28PCW8/WduUt4unA==", + "license": "MIT", + "dependencies": { + "@remote-dom/core": "^1.7.0", + "@types/react": "^18.0.0", + "htm": "^3.1.1" + }, + "engines": { + "node": ">=18.0.0" + }, + "peerDependencies": { + "react": "^17.0.0 || ^18.0.0" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + } + } + }, "node_modules/@rollup/plugin-alias": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/@rollup/plugin-alias/-/plugin-alias-5.1.0.tgz", @@ -36609,6 +36741,12 @@ "integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==", "dev": true }, + "node_modules/htm": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/htm/-/htm-3.1.1.tgz", + "integrity": "sha512-983Vyg8NwUE7JkZ6NmOqpCZ+sh1bKv2iYTlUkzlWmA5JD2acKoxd4KVxbMmxX/85mtfdnDmTFoNKcg5DGAvxNQ==", + "license": "Apache-2.0" + }, "node_modules/html-encoding-sniffer": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz", @@ -45451,9 +45589,10 @@ } }, "node_modules/react": { - "version": "18.2.0", - "resolved": "https://registry.npmjs.org/react/-/react-18.2.0.tgz", - "integrity": "sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ==", + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", + "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", + "license": "MIT", "dependencies": { "loose-envify": "^1.1.0" }, @@ -45522,15 +45661,16 @@ } }, "node_modules/react-dom": { - "version": "18.2.0", - "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.2.0.tgz", - "integrity": "sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g==", + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", + "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", + "license": "MIT", "dependencies": { "loose-envify": "^1.1.0", - "scheduler": "^0.23.0" + "scheduler": "^0.23.2" }, "peerDependencies": { - "react": "^18.2.0" + "react": "^18.3.1" } }, "node_modules/react-flip-toolkit": { @@ -47452,9 +47592,10 @@ } }, "node_modules/scheduler": { - "version": "0.23.0", - "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.0.tgz", - "integrity": "sha512-CtuThmgHNg7zIZWAXi3AsyIzA3n4xx7aNyjwC2VJldO2LMVDhFK+63xGqq6CsJH4rTAt6/M+N4GhZiDYPx9eUw==", + "version": "0.23.2", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz", + "integrity": "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==", + "license": "MIT", "dependencies": { "loose-envify": "^1.1.0" } diff --git a/packages/api/src/mcp/parsers.ts b/packages/api/src/mcp/parsers.ts index b0722fc01..3f6cc51e5 100644 --- a/packages/api/src/mcp/parsers.ts +++ b/packages/api/src/mcp/parsers.ts @@ -111,6 +111,7 @@ export function formatToolContent( const formattedContent: t.FormattedContent[] = []; const imageUrls: t.FormattedContent[] = []; let currentTextBlock = ''; + let uiResources: t.UIResource[] = []; type ContentHandler = undefined | ((item: t.ToolContentPart) => void); @@ -142,9 +143,14 @@ export function formatToolContent( }, resource: (item) => { + if (item.resource.uri.startsWith('ui://')) { + uiResources.push(item.resource as t.UIResource); + return; + } + const resourceText = []; if (item.resource.text != null && item.resource.text) { - resourceText.push(item.resource.text); + resourceText.push(`Resource Text: ${item.resource.text}`); } if (item.resource.uri.length) { resourceText.push(`Resource URI: ${item.resource.uri}`); @@ -153,10 +159,10 @@ export function formatToolContent( resourceText.push(`Resource: ${item.resource.name}`); } if (item.resource.description) { - resourceText.push(`Description: ${item.resource.description}`); + resourceText.push(`Resource Description: ${item.resource.description}`); } if (item.resource.mimeType != null && item.resource.mimeType) { - resourceText.push(`Type: ${item.resource.mimeType}`); + resourceText.push(`Resource MIME Type: ${item.resource.mimeType}`); } currentTextBlock += (currentTextBlock ? '\n\n' : '') + resourceText.join('\n'); }, @@ -176,6 +182,10 @@ export function formatToolContent( formattedContent.push({ type: 'text', text: currentTextBlock }); } + if (uiResources.length) { + formattedContent.push({ type: 'text', metadata: 'ui_resources', text: btoa(JSON.stringify(uiResources))}); + } + const artifacts = imageUrls.length ? { content: imageUrls } : undefined; if (CONTENT_ARRAY_PROVIDERS.has(provider)) { return [formattedContent, artifacts]; diff --git a/packages/api/src/mcp/types/index.ts b/packages/api/src/mcp/types/index.ts index 6230ac15e..0e5672ea6 100644 --- a/packages/api/src/mcp/types/index.ts +++ b/packages/api/src/mcp/types/index.ts @@ -75,6 +75,7 @@ export type FormattedContent = | { type: 'text'; text: string; + metadata?: string; } | { type: 'image'; @@ -103,6 +104,13 @@ export type FormattedContentResult = [ undefined | { content: FormattedContent[] }, ]; +export type UIResource = { + uri: string; + mimeType: string; + text: string; + [key: string]: unknown; +} + export type ImageFormatter = (item: ImageContent) => FormattedContent; export type FormattedToolResponse = [