🎨 feat: MCP UI basic integration (#9299)

2025-12-16 16:30:15 +01:00 · 2025-08-29 19:07:19 +02:00 · 2025-08-29 19:07:19 +02:00 · d16f93b5f7
commit d16f93b5f7
parent 20b29bbfa6
8 changed files with 365 additions and 14 deletions
--- a/client/package.json
+++ b/client/package.json
@ -37,6 +37,7 @@
    "@headlessui/react": "^2.1.2",
    "@librechat/client": "*",
    "@marsidev/react-turnstile": "^1.1.0",
+    "@mcp-ui/client": "^5.7.0",
    "@radix-ui/react-accordion": "^1.1.2",
    "@radix-ui/react-alert-dialog": "^1.0.2",
    "@radix-ui/react-checkbox": "^1.0.3",
--- a/client/src/common/types.ts
+++ b/client/src/common/types.ts
@ -630,3 +630,10 @@ declare global {
    google_tag_manager?: unknown;
  }
 }
+
+export type UIResource = {
+  uri: string;
+  mimeType: string;
+  text: string;
+  [key: string]: unknown;
+};
--- a/client/src/components/Chat/Messages/Content/ToolCallInfo.tsx
+++ b/client/src/components/Chat/Messages/Content/ToolCallInfo.tsx
@ -1,5 +1,8 @@
 import React from 'react';
 import { useLocalize } from '~/hooks';
+import { UIResourceRenderer } from '@mcp-ui/client';
+import UIResourceGrid from './UIResourceGrid';
+import type { UIResource } from '~/common';

 function OptimizedCodeBlock({ text, maxHeight = 320 }: { text: string; maxHeight?: number }) {
  return (
@ -51,6 +54,21 @@ export default function ToolCallInfo({
        : localize('com_assistants_attempt_info');
  }

+  // Extract ui_resources from the output to display them in the UI
+  let uiResources: UIResource[] = [];
+  if (output?.includes('ui_resources')) {
+    const parsedOutput = JSON.parse(output);
+    const uiResourcesItem = parsedOutput.find(
+      (contentItem) => contentItem.metadata === 'ui_resources',
+    );
+    if (uiResourcesItem?.text) {
+      uiResources = JSON.parse(atob(uiResourcesItem.text)) as UIResource[];
+    }
+    output = JSON.stringify(
+      parsedOutput.filter((contentItem) => contentItem.metadata !== 'ui_resources'),
+    );
+  }
+
  return (
    <div className="w-full p-2">
      <div style={{ opacity: 1 }}>
@ -66,6 +84,26 @@ export default function ToolCallInfo({
            <div>
              <OptimizedCodeBlock text={formatText(output)} maxHeight={250} />
            </div>
+            {uiResources.length > 0 && (
+              <div className="my-2 text-sm font-medium text-text-primary">
+                {localize('com_ui_ui_resources')}
+              </div>
+            )}
+            <div>
+              {uiResources.length > 1 && <UIResourceGrid uiResources={uiResources} />}
+
+              {uiResources.length === 1 && (
+                <UIResourceRenderer
+                  resource={uiResources[0]}
+                  onUIAction={async (result) => {
+                    console.log('Action:', result);
+                  }}
+                  htmlProps={{
+                    autoResizeIframe: { width: true, height: true },
+                  }}
+                />
+              )}
+            </div>
          </>
        )}
      </div>
--- a/client/src/components/Chat/Messages/Content/UIResourceGrid.tsx
+++ b/client/src/components/Chat/Messages/Content/UIResourceGrid.tsx
@ -0,0 +1,145 @@
+import { UIResourceRenderer } from '@mcp-ui/client';
+import type { UIResource } from '~/common';
+import React, { useState } from 'react';
+
+interface UIResourceGridProps {
+  uiResources: UIResource[];
+}
+
+const UIResourceGrid: React.FC<UIResourceGridProps> = React.memo(({ uiResources }) => {
+  const [showLeftArrow, setShowLeftArrow] = useState(false);
+  const [showRightArrow, setShowRightArrow] = useState(true);
+  const [isContainerHovered, setIsContainerHovered] = useState(false);
+  const scrollContainerRef = React.useRef<HTMLDivElement>(null);
+
+  const handleScroll = React.useCallback(() => {
+    if (!scrollContainerRef.current) return;
+
+    const { scrollLeft, scrollWidth, clientWidth } = scrollContainerRef.current;
+    setShowLeftArrow(scrollLeft > 0);
+    setShowRightArrow(scrollLeft < scrollWidth - clientWidth - 10);
+  }, []);
+
+  const scroll = React.useCallback((direction: 'left' | 'right') => {
+    if (!scrollContainerRef.current) return;
+
+    const viewportWidth = scrollContainerRef.current.clientWidth;
+    const scrollAmount = Math.floor(viewportWidth * 0.9);
+    const currentScroll = scrollContainerRef.current.scrollLeft;
+    const newScroll =
+      direction === 'left' ? currentScroll - scrollAmount : currentScroll + scrollAmount;
+
+    scrollContainerRef.current.scrollTo({
+      left: newScroll,
+      behavior: 'smooth',
+    });
+  }, []);
+
+  React.useEffect(() => {
+    const container = scrollContainerRef.current;
+    if (container) {
+      container.addEventListener('scroll', handleScroll);
+      handleScroll();
+      return () => container.removeEventListener('scroll', handleScroll);
+    }
+  }, [handleScroll]);
+
+  if (uiResources.length === 0) {
+    return null;
+  }
+
+  return (
+    <div
+      className="relative mb-4 pt-3"
+      onMouseEnter={() => setIsContainerHovered(true)}
+      onMouseLeave={() => setIsContainerHovered(false)}
+    >
+      <div
+        className={`pointer-events-none absolute left-0 top-0 z-10 h-full w-24 bg-gradient-to-r from-surface-primary to-transparent transition-opacity duration-500 ease-in-out ${
+          showLeftArrow ? 'opacity-100' : 'opacity-0'
+        }`}
+      />
+
+      <div
+        className={`pointer-events-none absolute right-0 top-0 z-10 h-full w-24 bg-gradient-to-l from-surface-primary to-transparent transition-opacity duration-500 ease-in-out ${
+          showRightArrow ? 'opacity-100' : 'opacity-0'
+        }`}
+      />
+
+      {showLeftArrow && (
+        <button
+          type="button"
+          onClick={() => scroll('left')}
+          className={`absolute left-2 top-1/2 z-20 -translate-y-1/2 rounded-xl bg-white p-2 text-gray-800 shadow-lg transition-all duration-200 hover:scale-110 hover:bg-gray-100 hover:shadow-xl active:scale-95 dark:bg-gray-200 dark:text-gray-800 dark:hover:bg-gray-300 ${
+            isContainerHovered ? 'opacity-100' : 'pointer-events-none opacity-0'
+          }`}
+          aria-label="Scroll left"
+        >
+          <svg className="h-5 w-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+            <path
+              strokeLinecap="round"
+              strokeLinejoin="round"
+              strokeWidth={2}
+              d="M15 19l-7-7 7-7"
+            />
+          </svg>
+        </button>
+      )}
+
+      <div
+        ref={scrollContainerRef}
+        className="hide-scrollbar flex gap-4 overflow-x-auto scroll-smooth"
+      >
+        {uiResources.map((uiResource, index) => {
+          const height = 360;
+          const width = 230;
+
+          return (
+            <div
+              key={index}
+              className="flex-shrink-0 transform-gpu transition-all duration-300 ease-out animate-in fade-in-0 slide-in-from-bottom-5"
+              style={{
+                width: `${width}px`,
+                minHeight: `${height}px`,
+                animationDelay: `${index * 100}ms`,
+              }}
+            >
+              <div className="flex h-full flex-col">
+                <UIResourceRenderer
+                  resource={{
+                    uri: uiResource.uri,
+                    mimeType: uiResource.mimeType,
+                    text: uiResource.text,
+                  }}
+                  onUIAction={async (result) => {
+                    console.log('Action:', result);
+                  }}
+                  htmlProps={{
+                    autoResizeIframe: { width: true, height: true },
+                  }}
+                />
+              </div>
+            </div>
+          );
+        })}
+      </div>
+
+      {showRightArrow && (
+        <button
+          type="button"
+          onClick={() => scroll('right')}
+          className={`absolute right-2 top-1/2 z-20 -translate-y-1/2 rounded-xl bg-white p-2 text-gray-800 shadow-lg transition-all duration-200 hover:scale-110 hover:bg-gray-100 hover:shadow-xl active:scale-95 dark:bg-gray-200 dark:text-gray-800 dark:hover:bg-gray-300 ${
+            isContainerHovered ? 'opacity-100' : 'pointer-events-none opacity-0'
+          }`}
+          aria-label="Scroll right"
+        >
+          <svg className="h-5 w-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+            <path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M9 5l7 7-7 7" />
+          </svg>
+        </button>
+      )}
+    </div>
+  );
+});
+
+export default UIResourceGrid;
--- a/client/src/locales/en/translation.json
+++ b/client/src/locales/en/translation.json
@ -1198,6 +1198,7 @@
  "com_ui_travel": "Travel",
  "com_ui_trust_app": "I trust this application",
  "com_ui_try_adjusting_search": "Try adjusting your search terms",
+  "com_ui_ui_resources": "UI Resources",
  "com_ui_unarchive": "Unarchive",
  "com_ui_unarchive_error": "Failed to unarchive conversation",
  "com_ui_unknown": "Unknown",
--- a/package-lock.json
+++ b/package-lock.json
@ -2633,6 +2633,7 @@
        "@headlessui/react": "^2.1.2",
        "@librechat/client": "*",
        "@marsidev/react-turnstile": "^1.1.0",
+        "@mcp-ui/client": "^5.7.0",
        "@radix-ui/react-accordion": "^1.1.2",
        "@radix-ui/react-alert-dialog": "^1.0.2",
        "@radix-ui/react-checkbox": "^1.0.3",
@ -22546,6 +22547,21 @@
        "react-dom": "^17.0.2 || ^18.0.0 || ^19.0"
      }
    },
+    "node_modules/@mcp-ui/client": {
+      "version": "5.7.0",
+      "resolved": "https://registry.npmjs.org/@mcp-ui/client/-/client-5.7.0.tgz",
+      "integrity": "sha512-+HbPw3VS46WUSWmyJ34ZVnygb81QByA3luR6y0JDbyDZxjYtHw1FcIN7v9WbbE8PrfI0WcuWCSiNOO6sOGbwpQ==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@modelcontextprotocol/sdk": "*",
+        "@quilted/threads": "^3.1.3",
+        "@r2wc/react-to-web-component": "^2.0.4",
+        "@remote-dom/core": "^1.8.0",
+        "@remote-dom/react": "^1.2.2",
+        "react": "^18.3.1",
+        "react-dom": "^18.3.1"
+      }
+    },
    "node_modules/@microsoft/eslint-formatter-sarif": {
      "version": "3.1.0",
      "resolved": "https://registry.npmjs.org/@microsoft/eslint-formatter-sarif/-/eslint-formatter-sarif-3.1.0.tgz",
@ -23229,6 +23245,16 @@
        "node": ">=18"
      }
    },
+    "node_modules/@preact/signals-core": {
+      "version": "1.11.0",
+      "resolved": "https://registry.npmjs.org/@preact/signals-core/-/signals-core-1.11.0.tgz",
+      "integrity": "sha512-jglbibeWHuFRzEWVFY/TT7wB1PppJxmcSfUHcK+2J9vBRtiooMfw6tAPttojNYrrpdGViqAYCbPpmWYlMm+eMQ==",
+      "license": "MIT",
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/preact"
+      }
+    },
    "node_modules/@protobufjs/aspromise": {
      "version": "1.1.2",
      "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
@ -23283,6 +23309,57 @@
      "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz",
      "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw=="
    },
+    "node_modules/@quilted/events": {
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/@quilted/events/-/events-2.1.3.tgz",
+      "integrity": "sha512-4fHaSLND8rmZ+tce9/4FNmG5UWTRpFtM54kOekf3tLON4ZLLnYzjjldELD35efd7+lT5+E3cdkacqc56d+kCrQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@preact/signals-core": "^1.8.0"
+      },
+      "engines": {
+        "node": ">=14.0.0"
+      }
+    },
+    "node_modules/@quilted/threads": {
+      "version": "3.3.1",
+      "resolved": "https://registry.npmjs.org/@quilted/threads/-/threads-3.3.1.tgz",
+      "integrity": "sha512-0ASnjTH+hOu1Qwzi9NnsVcsbMhWVx8pEE8SXIHknqcc/1rXAU0QlKw9ARq0W43FAdzyVeuXeXtZN27ZC0iALKg==",
+      "license": "MIT",
+      "dependencies": {
+        "@quilted/events": "^2.1.3"
+      },
+      "engines": {
+        "node": ">=14.0.0"
+      },
+      "peerDependencies": {
+        "@preact/signals-core": "^1.8.0"
+      },
+      "peerDependenciesMeta": {
+        "@preact/signals-core": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@r2wc/core": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@r2wc/core/-/core-1.2.0.tgz",
+      "integrity": "sha512-vAfiuS5KywtV54SRzc4maEHcpdgeUyJzln+ATpNCOkO+ArIuOkTXd92b5YauVAd0A8B2rV/y9OeVW19vb73bUQ==",
+      "license": "MIT"
+    },
+    "node_modules/@r2wc/react-to-web-component": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/@r2wc/react-to-web-component/-/react-to-web-component-2.0.4.tgz",
+      "integrity": "sha512-g1dtTTEGETNUimYldTW+2hxY3mmJZjzPEca0vqCutUht2GHmpK9mT5r/urmEI7uSbOkn6HaymosgVy26lvU1JQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@r2wc/core": "^1.0.0"
+      },
+      "peerDependencies": {
+        "react": "^18.0.0 || ^19.0.0",
+        "react-dom": "^18.0.0 || ^19.0.0"
+      }
+    },
    "node_modules/@radix-ui/number": {
      "version": "1.1.1",
      "resolved": "https://registry.npmjs.org/@radix-ui/number/-/number-1.1.1.tgz",
@ -26886,6 +26963,61 @@
        "node": ">=14.0.0"
      }
    },
+    "node_modules/@remote-dom/core": {
+      "version": "1.9.0",
+      "resolved": "https://registry.npmjs.org/@remote-dom/core/-/core-1.9.0.tgz",
+      "integrity": "sha512-h8OO2NRns2paXO/q5hkfXrwlZKq7oKj9XedGosi7J8OP3+aW7N2Gv4MBBVVQGCfOiZPkOj5m3sQH7FdyUWl7PQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@remote-dom/polyfill": "^1.4.4",
+        "htm": "^3.1.1"
+      },
+      "engines": {
+        "node": ">=14.0.0"
+      },
+      "peerDependencies": {
+        "@preact/signals-core": "^1.3.0"
+      },
+      "peerDependenciesMeta": {
+        "@preact/signals-core": {
+          "optional": true
+        },
+        "preact": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@remote-dom/polyfill": {
+      "version": "1.4.5",
+      "resolved": "https://registry.npmjs.org/@remote-dom/polyfill/-/polyfill-1.4.5.tgz",
+      "integrity": "sha512-V1qkKIl/wXyDO0I+tQDH06cBBNyyViZF3IYorkTTBf58dorqOP5Ta51vCCWeekPgdSOPuEKvHhvu6kAaKqVgww==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=14.0.0"
+      }
+    },
+    "node_modules/@remote-dom/react": {
+      "version": "1.2.2",
+      "resolved": "https://registry.npmjs.org/@remote-dom/react/-/react-1.2.2.tgz",
+      "integrity": "sha512-PkvioODONTr1M0StGDYsR4Ssf5M0Rd4+IlWVvVoK3Zrw8nr7+5mJkgNofaj/z7i8Aep78L28PCW8/WduUt4unA==",
+      "license": "MIT",
+      "dependencies": {
+        "@remote-dom/core": "^1.7.0",
+        "@types/react": "^18.0.0",
+        "htm": "^3.1.1"
+      },
+      "engines": {
+        "node": ">=18.0.0"
+      },
+      "peerDependencies": {
+        "react": "^17.0.0 || ^18.0.0"
+      },
+      "peerDependenciesMeta": {
+        "react": {
+          "optional": true
+        }
+      }
+    },
    "node_modules/@rollup/plugin-alias": {
      "version": "5.1.0",
      "resolved": "https://registry.npmjs.org/@rollup/plugin-alias/-/plugin-alias-5.1.0.tgz",
@ -36609,6 +36741,12 @@
      "integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==",
      "dev": true
    },
+    "node_modules/htm": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/htm/-/htm-3.1.1.tgz",
+      "integrity": "sha512-983Vyg8NwUE7JkZ6NmOqpCZ+sh1bKv2iYTlUkzlWmA5JD2acKoxd4KVxbMmxX/85mtfdnDmTFoNKcg5DGAvxNQ==",
+      "license": "Apache-2.0"
+    },
    "node_modules/html-encoding-sniffer": {
      "version": "3.0.0",
      "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz",
@ -45451,9 +45589,10 @@
      }
    },
    "node_modules/react": {
-      "version": "18.2.0",
-      "resolved": "https://registry.npmjs.org/react/-/react-18.2.0.tgz",
-      "integrity": "sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ==",
+      "version": "18.3.1",
+      "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz",
+      "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==",
+      "license": "MIT",
      "dependencies": {
        "loose-envify": "^1.1.0"
      },
@ -45522,15 +45661,16 @@
      }
    },
    "node_modules/react-dom": {
-      "version": "18.2.0",
-      "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.2.0.tgz",
-      "integrity": "sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g==",
+      "version": "18.3.1",
+      "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz",
+      "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==",
+      "license": "MIT",
      "dependencies": {
        "loose-envify": "^1.1.0",
-        "scheduler": "^0.23.0"
+        "scheduler": "^0.23.2"
      },
      "peerDependencies": {
-        "react": "^18.2.0"
+        "react": "^18.3.1"
      }
    },
    "node_modules/react-flip-toolkit": {
@ -47452,9 +47592,10 @@
      }
    },
    "node_modules/scheduler": {
-      "version": "0.23.0",
-      "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.0.tgz",
-      "integrity": "sha512-CtuThmgHNg7zIZWAXi3AsyIzA3n4xx7aNyjwC2VJldO2LMVDhFK+63xGqq6CsJH4rTAt6/M+N4GhZiDYPx9eUw==",
+      "version": "0.23.2",
+      "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz",
+      "integrity": "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==",
+      "license": "MIT",
      "dependencies": {
        "loose-envify": "^1.1.0"
      }
--- a/packages/api/src/mcp/parsers.ts
+++ b/packages/api/src/mcp/parsers.ts
@ -111,6 +111,7 @@ export function formatToolContent(
  const formattedContent: t.FormattedContent[] = [];
  const imageUrls: t.FormattedContent[] = [];
  let currentTextBlock = '';
+  let uiResources: t.UIResource[] = [];

  type ContentHandler = undefined | ((item: t.ToolContentPart) => void);

@ -142,9 +143,14 @@ export function formatToolContent(
    },

    resource: (item) => {
+      if (item.resource.uri.startsWith('ui://')) {
+        uiResources.push(item.resource as t.UIResource);
+        return;
+      }
+
      const resourceText = [];
      if (item.resource.text != null && item.resource.text) {
-        resourceText.push(item.resource.text);
+        resourceText.push(`Resource Text: ${item.resource.text}`);
      }
      if (item.resource.uri.length) {
        resourceText.push(`Resource URI: ${item.resource.uri}`);
@ -153,10 +159,10 @@ export function formatToolContent(
        resourceText.push(`Resource: ${item.resource.name}`);
      }
      if (item.resource.description) {
-        resourceText.push(`Description: ${item.resource.description}`);
+        resourceText.push(`Resource Description: ${item.resource.description}`);
      }
      if (item.resource.mimeType != null && item.resource.mimeType) {
-        resourceText.push(`Type: ${item.resource.mimeType}`);
+        resourceText.push(`Resource MIME Type: ${item.resource.mimeType}`);
      }
      currentTextBlock += (currentTextBlock ? '\n\n' : '') + resourceText.join('\n');
    },
@ -176,6 +182,10 @@ export function formatToolContent(
    formattedContent.push({ type: 'text', text: currentTextBlock });
  }

+  if (uiResources.length) {
+    formattedContent.push({ type: 'text', metadata: 'ui_resources', text: btoa(JSON.stringify(uiResources))});
+  }
+
  const artifacts = imageUrls.length ? { content: imageUrls } : undefined;
  if (CONTENT_ARRAY_PROVIDERS.has(provider)) {
    return [formattedContent, artifacts];
--- a/packages/api/src/mcp/types/index.ts
+++ b/packages/api/src/mcp/types/index.ts
@ -75,6 +75,7 @@ export type FormattedContent =
  | {
      type: 'text';
      text: string;
+      metadata?: string;
    }
  | {
      type: 'image';
@ -103,6 +104,13 @@ export type FormattedContentResult = [
  undefined | { content: FormattedContent[] },
 ];

+export type UIResource = {
+  uri: string;
+  mimeType: string;
+  text: string;
+  [key: string]: unknown;
+}
+
 export type ImageFormatter = (item: ImageContent) => FormattedContent;

 export type FormattedToolResponse = [