From 9ae81a85a5cc0a889b5cb3988d916160e30fce8b Mon Sep 17 00:00:00 2001 From: Pierre-Luc Godin Date: Thu, 16 Oct 2025 14:51:01 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=92=BB=20feat:=20deeper=20MCP=20UI=20inte?= =?UTF-8?q?gration=20in=20the=20chat=20UI=20using=20plugins?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --------- Co-authored-by: Samuel Path Co-authored-by: Pierre-Luc Godin --- api/app/clients/tools/util/handleTools.js | 9 + .../agents/__tests__/callbacks.spec.js | 40 +- .../Chat/Messages/Content/Markdown.tsx | 8 + .../Messages/Content/UIResourceCarousel.tsx | 7 +- .../Content/__tests__/Markdown.mcpui.test.tsx | 92 ++++ .../__tests__/UIResourceCarousel.test.tsx | 60 ++- .../MCPUIResource/MCPUIResource.tsx | 74 +++ .../MCPUIResource/MCPUIResourceCarousel.tsx | 47 ++ .../__tests__/MCPUIResource.test.tsx | 259 ++++++++++ .../__tests__/MCPUIResourceCarousel.test.tsx | 454 ++++++++++++++++++ .../MCPUIResource/__tests__/plugin.test.ts | 249 ++++++++++ client/src/components/MCPUIResource/index.ts | 4 + client/src/components/MCPUIResource/plugin.ts | 84 ++++ client/src/components/MCPUIResource/types.ts | 12 + client/src/hooks/index.ts | 2 +- client/src/hooks/useResourcePermissions.ts | 2 + client/src/locales/en/translation.json | 2 + client/src/utils/index.ts | 56 +++ .../api/src/mcp/__tests__/parsers.test.ts | 8 +- packages/api/src/mcp/parsers.ts | 16 +- packages/api/src/mcp/types/index.ts | 6 +- packages/api/src/mcp/utils.ts | 2 +- 22 files changed, 1440 insertions(+), 53 deletions(-) create mode 100644 client/src/components/Chat/Messages/Content/__tests__/Markdown.mcpui.test.tsx create mode 100644 client/src/components/MCPUIResource/MCPUIResource.tsx create mode 100644 client/src/components/MCPUIResource/MCPUIResourceCarousel.tsx create mode 100644 client/src/components/MCPUIResource/__tests__/MCPUIResource.test.tsx create mode 100644 client/src/components/MCPUIResource/__tests__/MCPUIResourceCarousel.test.tsx create mode 100644 client/src/components/MCPUIResource/__tests__/plugin.test.ts create mode 100644 client/src/components/MCPUIResource/index.ts create mode 100644 client/src/components/MCPUIResource/plugin.ts create mode 100644 client/src/components/MCPUIResource/types.ts diff --git a/api/app/clients/tools/util/handleTools.js b/api/app/clients/tools/util/handleTools.js index c9abb0c3e3..18094a0bdf 100644 --- a/api/app/clients/tools/util/handleTools.js +++ b/api/app/clients/tools/util/handleTools.js @@ -335,6 +335,15 @@ Current Date & Time: ${replaceSpecialVars({ text: '{{iso_datetime}}' })} }; continue; } else if (tool && mcpToolPattern.test(tool)) { + toolContextMap[tool] = `# MCP Tool \`${tool}\`: + When this tool returns UI resources (URIs starting with "ui://"): + - You should usually introduce what you're showing before the marker + - For a single resource: \\ui0 + - For multiple resources shown separately: \\ui0 \\ui1 + - For multiple resources in a carousel: \\ui0,1,2 + - The UI will be rendered inline where you place the marker + - Format: \\ui{index} or \\ui{index1,index2,index3} where indices are 0-based, in the order they appear in the tool response`; + const [toolName, serverName] = tool.split(Constants.mcp_delimiter); if (toolName === Constants.mcp_server) { /** Placeholder used for UI purposes */ diff --git a/api/server/controllers/agents/__tests__/callbacks.spec.js b/api/server/controllers/agents/__tests__/callbacks.spec.js index 25f00bab8d..7922c31efa 100644 --- a/api/server/controllers/agents/__tests__/callbacks.spec.js +++ b/api/server/controllers/agents/__tests__/callbacks.spec.js @@ -73,10 +73,10 @@ describe('createToolEndCallback', () => { tool_call_id: 'tool123', artifact: { [Tools.ui_resources]: { - data: { - 0: { type: 'button', label: 'Click me' }, - 1: { type: 'input', placeholder: 'Enter text' }, - }, + data: [ + { type: 'button', label: 'Click me' }, + { type: 'input', placeholder: 'Enter text' }, + ], }, }, }; @@ -100,10 +100,10 @@ describe('createToolEndCallback', () => { messageId: 'run456', toolCallId: 'tool123', conversationId: 'thread789', - [Tools.ui_resources]: { - 0: { type: 'button', label: 'Click me' }, - 1: { type: 'input', placeholder: 'Enter text' }, - }, + [Tools.ui_resources]: [ + { type: 'button', label: 'Click me' }, + { type: 'input', placeholder: 'Enter text' }, + ], }); }); @@ -115,9 +115,7 @@ describe('createToolEndCallback', () => { tool_call_id: 'tool123', artifact: { [Tools.ui_resources]: { - data: { - 0: { type: 'carousel', items: [] }, - }, + data: [{ type: 'carousel', items: [] }], }, }, }; @@ -136,9 +134,7 @@ describe('createToolEndCallback', () => { messageId: 'run456', toolCallId: 'tool123', conversationId: 'thread789', - [Tools.ui_resources]: { - 0: { type: 'carousel', items: [] }, - }, + [Tools.ui_resources]: [{ type: 'carousel', items: [] }], }); }); @@ -155,9 +151,7 @@ describe('createToolEndCallback', () => { tool_call_id: 'tool123', artifact: { [Tools.ui_resources]: { - data: { - 0: { type: 'test' }, - }, + data: [{ type: 'test' }], }, }, }; @@ -184,9 +178,7 @@ describe('createToolEndCallback', () => { tool_call_id: 'tool123', artifact: { [Tools.ui_resources]: { - data: { - 0: { type: 'chart', data: [] }, - }, + data: [{ type: 'chart', data: [] }], }, [Tools.web_search]: { results: ['result1', 'result2'], @@ -209,9 +201,7 @@ describe('createToolEndCallback', () => { // Check ui_resources attachment const uiResourceAttachment = results.find((r) => r?.type === Tools.ui_resources); expect(uiResourceAttachment).toBeTruthy(); - expect(uiResourceAttachment[Tools.ui_resources]).toEqual({ - 0: { type: 'chart', data: [] }, - }); + expect(uiResourceAttachment[Tools.ui_resources]).toEqual([{ type: 'chart', data: [] }]); // Check web_search attachment const webSearchAttachment = results.find((r) => r?.type === Tools.web_search); @@ -250,7 +240,7 @@ describe('createToolEndCallback', () => { tool_call_id: 'tool123', artifact: { [Tools.ui_resources]: { - data: {}, + data: [], }, }, }; @@ -268,7 +258,7 @@ describe('createToolEndCallback', () => { messageId: 'run456', toolCallId: 'tool123', conversationId: 'thread789', - [Tools.ui_resources]: {}, + [Tools.ui_resources]: [], }); }); diff --git a/client/src/components/Chat/Messages/Content/Markdown.tsx b/client/src/components/Chat/Messages/Content/Markdown.tsx index 42136384d4..a763885d2f 100644 --- a/client/src/components/Chat/Messages/Content/Markdown.tsx +++ b/client/src/components/Chat/Messages/Content/Markdown.tsx @@ -9,6 +9,11 @@ import rehypeHighlight from 'rehype-highlight'; import remarkDirective from 'remark-directive'; import type { Pluggable } from 'unified'; import { Citation, CompositeCitation, HighlightedText } from '~/components/Web/Citation'; +import { + mcpUIResourcePlugin, + MCPUIResource, + MCPUIResourceCarousel, +} from '~/components/MCPUIResource'; import { Artifact, artifactPlugin } from '~/components/Artifacts/Artifact'; import { ArtifactProvider, CodeBlockProvider } from '~/Providers'; import MarkdownErrorBoundary from './MarkdownErrorBoundary'; @@ -55,6 +60,7 @@ const Markdown = memo(({ content = '', isLatestMessage }: TContentProps) => { artifactPlugin, [remarkMath, { singleDollarTextMath: false }], unicodeCitation, + mcpUIResourcePlugin, ]; if (isInitializing) { @@ -86,6 +92,8 @@ const Markdown = memo(({ content = '', isLatestMessage }: TContentProps) => { citation: Citation, 'highlighted-text': HighlightedText, 'composite-citation': CompositeCitation, + 'mcp-ui-resource': MCPUIResource, + 'mcp-ui-carousel': MCPUIResourceCarousel, } as { [nodeType: string]: React.ElementType; } diff --git a/client/src/components/Chat/Messages/Content/UIResourceCarousel.tsx b/client/src/components/Chat/Messages/Content/UIResourceCarousel.tsx index dfffa08935..1f64b47fd8 100644 --- a/client/src/components/Chat/Messages/Content/UIResourceCarousel.tsx +++ b/client/src/components/Chat/Messages/Content/UIResourceCarousel.tsx @@ -1,6 +1,8 @@ import React, { useState } from 'react'; import { UIResourceRenderer } from '@mcp-ui/client'; import type { UIResource } from 'librechat-data-provider'; +import useSubmitMessage from '~/hooks/Messages/useSubmitMessage'; +import { handleUIAction } from '~/utils'; interface UIResourceCarouselProps { uiResources: UIResource[]; @@ -11,6 +13,7 @@ const UIResourceCarousel: React.FC = React.memo(({ uiRe const [showRightArrow, setShowRightArrow] = useState(true); const [isContainerHovered, setIsContainerHovered] = useState(false); const scrollContainerRef = React.useRef(null); + const { submitMessage } = useSubmitMessage(); const handleScroll = React.useCallback(() => { if (!scrollContainerRef.current) return; @@ -111,9 +114,7 @@ const UIResourceCarousel: React.FC = React.memo(({ uiRe mimeType: uiResource.mimeType, text: uiResource.text, }} - onUIAction={async (result) => { - console.log('Action:', result); - }} + onUIAction={async (result) => handleUIAction(result, submitMessage)} htmlProps={{ autoResizeIframe: { width: true, height: true }, }} diff --git a/client/src/components/Chat/Messages/Content/__tests__/Markdown.mcpui.test.tsx b/client/src/components/Chat/Messages/Content/__tests__/Markdown.mcpui.test.tsx new file mode 100644 index 0000000000..108fdf30e0 --- /dev/null +++ b/client/src/components/Chat/Messages/Content/__tests__/Markdown.mcpui.test.tsx @@ -0,0 +1,92 @@ +import React from 'react'; +import { render, screen } from '@testing-library/react'; +import Markdown from '../Markdown'; +import { RecoilRoot } from 'recoil'; +import { UI_RESOURCE_MARKER } from '~/components/MCPUIResource/plugin'; +import { useMessageContext, useChatContext } from '~/Providers'; +import { useGetMessagesByConvoId } from '~/data-provider'; +import { useLocalize } from '~/hooks'; +import useSubmitMessage from '~/hooks/Messages/useSubmitMessage'; + +// Mocks for hooks used by MCPUIResource when rendered inside Markdown. +// Keep Provider components intact while mocking only the hooks we use. +jest.mock('~/Providers', () => ({ + ...jest.requireActual('~/Providers'), + useMessageContext: jest.fn(), + useChatContext: jest.fn(), +})); +jest.mock('~/data-provider'); +jest.mock('~/hooks'); +jest.mock('~/hooks/Messages/useSubmitMessage'); + +// Mock @mcp-ui/client to render identifiable elements for assertions +jest.mock('@mcp-ui/client', () => ({ + UIResourceRenderer: ({ resource }: any) => ( +
+ ), +})); + +const mockUseMessageContext = useMessageContext as jest.MockedFunction; +const mockUseChatContext = useChatContext as jest.MockedFunction; +const mockUseGetMessagesByConvoId = useGetMessagesByConvoId as jest.MockedFunction< + typeof useGetMessagesByConvoId +>; +const mockUseLocalize = useLocalize as jest.MockedFunction; +const mockUseSubmitMessage = useSubmitMessage as jest.MockedFunction; + +describe('Markdown with MCP UI markers (two attachments ui0 ui1)', () => { + beforeEach(() => { + jest.clearAllMocks(); + + mockUseMessageContext.mockReturnValue({ messageId: 'msg-weather' } as any); + mockUseChatContext.mockReturnValue({ conversation: { conversationId: 'conv1' } } as any); + mockUseLocalize.mockReturnValue(((key: string) => key) as any); + mockUseSubmitMessage.mockReturnValue({ submitMessage: jest.fn() } as any); + }); + + it('renders two UIResourceRenderer components for markers ui0 and ui1 across separate attachments', () => { + // Two tool responses, each produced one ui_resources attachment + const paris = { + uri: 'ui://weather/paris', + mimeType: 'text/html', + text: '
Paris Weather
', + }; + const nyc = { + uri: 'ui://weather/nyc', + mimeType: 'text/html', + text: '
NYC Weather
', + }; + + const messages = [ + { + messageId: 'msg-weather', + attachments: [ + { type: 'ui_resources', ui_resources: [paris] }, + { type: 'ui_resources', ui_resources: [nyc] }, + ], + }, + ]; + + mockUseGetMessagesByConvoId.mockReturnValue({ data: messages } as any); + + const content = [ + 'Here are the current weather conditions for both Paris and New York:', + '', + '- Paris: Slight rain, 53°F, humidity 76%, wind 9 mph.', + '- New York: Clear sky, 63°F, humidity 91%, wind 6 mph.', + '', + `Browse these weather cards for more details ${UI_RESOURCE_MARKER}0 ${UI_RESOURCE_MARKER}1`, + ].join('\n'); + + render( + + + , + ); + + const renderers = screen.getAllByTestId('ui-resource-renderer'); + expect(renderers).toHaveLength(2); + expect(renderers[0]).toHaveAttribute('data-resource-uri', 'ui://weather/paris'); + expect(renderers[1]).toHaveAttribute('data-resource-uri', 'ui://weather/nyc'); + }); +}); diff --git a/client/src/components/Chat/Messages/Content/__tests__/UIResourceCarousel.test.tsx b/client/src/components/Chat/Messages/Content/__tests__/UIResourceCarousel.test.tsx index 2878c6444f..7ec7a2601b 100644 --- a/client/src/components/Chat/Messages/Content/__tests__/UIResourceCarousel.test.tsx +++ b/client/src/components/Chat/Messages/Content/__tests__/UIResourceCarousel.test.tsx @@ -1,8 +1,8 @@ -import React from 'react'; import '@testing-library/jest-dom'; -import { render, screen, fireEvent, waitFor } from '@testing-library/react'; +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; import type { UIResource } from 'librechat-data-provider'; import UIResourceCarousel from '~/components/Chat/Messages/Content/UIResourceCarousel'; +import { handleUIAction } from '~/utils'; // Mock the UIResourceRenderer component jest.mock('@mcp-ui/client', () => ({ @@ -13,6 +13,20 @@ jest.mock('@mcp-ui/client', () => ({ ), })); +// Mock useSubmitMessage hook +const mockSubmitMessage = jest.fn(); +jest.mock('~/hooks/Messages/useSubmitMessage', () => ({ + __esModule: true, + default: () => ({ + submitMessage: mockSubmitMessage, + }), +})); + +// Mock handleUIAction utility +jest.mock('~/utils', () => ({ + handleUIAction: jest.fn(), +})); + // Mock scrollTo const mockScrollTo = jest.fn(); Object.defineProperty(HTMLElement.prototype, 'scrollTo', { @@ -29,8 +43,12 @@ describe('UIResourceCarousel', () => { { uri: 'resource5', mimeType: 'text/html', text: 'Resource 5' }, ]; + const mockHandleUIAction = handleUIAction as jest.MockedFunction; + beforeEach(() => { jest.clearAllMocks(); + mockSubmitMessage.mockClear(); + mockHandleUIAction.mockClear(); // Reset scroll properties Object.defineProperty(HTMLElement.prototype, 'scrollLeft', { configurable: true, @@ -141,18 +159,48 @@ describe('UIResourceCarousel', () => { }); }); - it('handles UIResource actions', async () => { - const consoleSpy = jest.spyOn(console, 'log').mockImplementation(); + it('handles UIResource actions using handleUIAction', async () => { render(); const renderer = screen.getByTestId('ui-resource-renderer'); fireEvent.click(renderer); await waitFor(() => { - expect(consoleSpy).toHaveBeenCalledWith('Action:', { action: 'test' }); + expect(mockHandleUIAction).toHaveBeenCalledWith({ action: 'test' }, mockSubmitMessage); + }); + }); + + it('calls handleUIAction with correct parameters for multiple resources', async () => { + render(); + + const renderers = screen.getAllByTestId('ui-resource-renderer'); + + // Click the second renderer + fireEvent.click(renderers[1]); + + await waitFor(() => { + expect(mockHandleUIAction).toHaveBeenCalledWith({ action: 'test' }, mockSubmitMessage); + expect(mockHandleUIAction).toHaveBeenCalledTimes(1); }); - consoleSpy.mockRestore(); + // Click the third renderer + fireEvent.click(renderers[2]); + + await waitFor(() => { + expect(mockHandleUIAction).toHaveBeenCalledTimes(2); + }); + }); + + it('passes correct submitMessage function to handleUIAction', async () => { + render(); + + const renderer = screen.getByTestId('ui-resource-renderer'); + fireEvent.click(renderer); + + await waitFor(() => { + expect(mockHandleUIAction).toHaveBeenCalledWith({ action: 'test' }, mockSubmitMessage); + expect(mockHandleUIAction).toHaveBeenCalledTimes(1); + }); }); it('applies correct dimensions to resource containers', () => { diff --git a/client/src/components/MCPUIResource/MCPUIResource.tsx b/client/src/components/MCPUIResource/MCPUIResource.tsx new file mode 100644 index 0000000000..7f4e44f837 --- /dev/null +++ b/client/src/components/MCPUIResource/MCPUIResource.tsx @@ -0,0 +1,74 @@ +import React, { useMemo } from 'react'; +import { UIResourceRenderer } from '@mcp-ui/client'; +import type { UIResource } from '~/common'; +import { handleUIAction } from '~/utils'; +import useSubmitMessage from '~/hooks/Messages/useSubmitMessage'; +import { useMessageContext, useChatContext } from '~/Providers'; +import { useGetMessagesByConvoId } from '~/data-provider'; +import { useLocalize } from '~/hooks'; + +interface MCPUIResourceProps { + node: { + properties: { + resourceIndex: number; + }; + }; +} + +/** + * Component that renders an MCP UI resource based on message context and index + */ +export function MCPUIResource(props: MCPUIResourceProps) { + const { resourceIndex } = props.node.properties; + const localize = useLocalize(); + const { submitMessage } = useSubmitMessage(); + const { messageId } = useMessageContext(); + const { conversation } = useChatContext(); + const { data: messages } = useGetMessagesByConvoId(conversation?.conversationId ?? '', { + enabled: !!conversation?.conversationId, + }); + + const uiResource = useMemo(() => { + const targetMessage = messages?.find((m) => m.messageId === messageId); + + if (!targetMessage?.attachments) { + return null; + } + + // Flatten all UI resources across attachments so indices are global + const allResources: UIResource[] = targetMessage.attachments + .filter((a) => a.type === 'ui_resources' && a['ui_resources']) + .flatMap((a) => a['ui_resources'] as UIResource[]); + + return allResources[resourceIndex] ?? null; + }, [messages, messageId, resourceIndex]); + + if (!uiResource) { + return ( + + {localize('com_ui_ui_resource_not_found', { 0: resourceIndex.toString() })} + + ); + } + + try { + return ( + + handleUIAction(result, submitMessage)} + htmlProps={{ + autoResizeIframe: { width: true, height: true }, + }} + /> + + ); + } catch (error) { + console.error('Error rendering UI resource:', error); + return ( + + {localize('com_ui_ui_resource_error', { 0: uiResource.name })} + + ); + } +} diff --git a/client/src/components/MCPUIResource/MCPUIResourceCarousel.tsx b/client/src/components/MCPUIResource/MCPUIResourceCarousel.tsx new file mode 100644 index 0000000000..00359d24b2 --- /dev/null +++ b/client/src/components/MCPUIResource/MCPUIResourceCarousel.tsx @@ -0,0 +1,47 @@ +import React, { useMemo } from 'react'; +import { useGetMessagesByConvoId } from '~/data-provider'; +import { useMessageContext, useChatContext } from '~/Providers'; +import UIResourceCarousel from '../Chat/Messages/Content/UIResourceCarousel'; +import type { UIResource } from '~/common'; + +interface MCPUIResourceCarouselProps { + node: { + properties: { + resourceIndices: number[]; + }; + }; +} + +export function MCPUIResourceCarousel(props: MCPUIResourceCarouselProps) { + const { messageId } = useMessageContext(); + const { conversation } = useChatContext(); + const { data: messages } = useGetMessagesByConvoId(conversation?.conversationId ?? '', { + enabled: !!conversation?.conversationId, + }); + + const uiResources = useMemo(() => { + const { resourceIndices } = props.node.properties; + + const targetMessage = messages?.find((m) => m.messageId === messageId); + + if (!targetMessage?.attachments) { + return []; + } + + const allResources: UIResource[] = targetMessage.attachments + .filter((a) => a.type === 'ui_resources' && a['ui_resources']) + .flatMap((a) => a['ui_resources'] as UIResource[]); + + const selectedResources: UIResource[] = resourceIndices + .map((i) => allResources[i]) + .filter(Boolean) as UIResource[]; + + return selectedResources; + }, [props.node.properties, messages, messageId]); + + if (uiResources.length === 0) { + return null; + } + + return ; +} diff --git a/client/src/components/MCPUIResource/__tests__/MCPUIResource.test.tsx b/client/src/components/MCPUIResource/__tests__/MCPUIResource.test.tsx new file mode 100644 index 0000000000..24b38aa5be --- /dev/null +++ b/client/src/components/MCPUIResource/__tests__/MCPUIResource.test.tsx @@ -0,0 +1,259 @@ +import React from 'react'; +import { render, screen } from '@testing-library/react'; +import { MCPUIResource } from '../MCPUIResource'; +import { useMessageContext, useChatContext } from '~/Providers'; +import { useGetMessagesByConvoId } from '~/data-provider'; +import { useLocalize } from '~/hooks'; +import useSubmitMessage from '~/hooks/Messages/useSubmitMessage'; +import { handleUIAction } from '~/utils'; + +// Mock dependencies +jest.mock('~/Providers'); +jest.mock('~/data-provider'); +jest.mock('~/hooks'); +jest.mock('~/hooks/Messages/useSubmitMessage'); +jest.mock('~/utils'); + +jest.mock('@mcp-ui/client', () => ({ + UIResourceRenderer: ({ resource, onUIAction }: any) => ( +
onUIAction({ action: 'test' })} + /> + ), +})); + +const mockUseMessageContext = useMessageContext as jest.MockedFunction; +const mockUseChatContext = useChatContext as jest.MockedFunction; +const mockUseGetMessagesByConvoId = useGetMessagesByConvoId as jest.MockedFunction< + typeof useGetMessagesByConvoId +>; +const mockUseLocalize = useLocalize as jest.MockedFunction; +const mockUseSubmitMessage = useSubmitMessage as jest.MockedFunction; +const mockHandleUIAction = handleUIAction as jest.MockedFunction; + +describe('MCPUIResource', () => { + const mockLocalize = (key: string, values?: any) => { + const translations: Record = { + com_ui_ui_resource_not_found: `UI resource at index ${values?.[0]} not found`, + com_ui_ui_resource_error: `Error rendering UI resource: ${values?.[0]}`, + }; + return translations[key] || key; + }; + + const mockSubmitMessageFn = jest.fn(); + + beforeEach(() => { + jest.clearAllMocks(); + mockUseMessageContext.mockReturnValue({ messageId: 'msg123' } as any); + mockUseChatContext.mockReturnValue({ + conversation: { conversationId: 'conv123' }, + } as any); + mockUseLocalize.mockReturnValue(mockLocalize as any); + mockUseSubmitMessage.mockReturnValue({ submitMessage: mockSubmitMessageFn } as any); + }); + + describe('resource fetching', () => { + it('should fetch and render UI resource from message attachments', () => { + const mockMessages = [ + { + messageId: 'msg123', + attachments: [ + { + type: 'ui_resources', + ui_resources: [ + { uri: 'ui://test/resource', mimeType: 'text/html', text: '

Test Resource

' }, + ], + }, + ], + }, + ]; + + mockUseGetMessagesByConvoId.mockReturnValue({ + data: mockMessages, + } as any); + + render(); + + const renderer = screen.getByTestId('ui-resource-renderer'); + expect(renderer).toBeInTheDocument(); + expect(renderer).toHaveAttribute('data-resource-uri', 'ui://test/resource'); + }); + + it('should show not found message when resource index is out of bounds', () => { + const mockMessages = [ + { + messageId: 'msg123', + attachments: [ + { + type: 'ui_resources', + ui_resources: [ + { uri: 'ui://test/resource', mimeType: 'text/html', text: '

Test Resource

' }, + ], + }, + ], + }, + ]; + + mockUseGetMessagesByConvoId.mockReturnValue({ + data: mockMessages, + } as any); + + render(); + + expect(screen.getByText('UI resource at index 5 not found')).toBeInTheDocument(); + expect(screen.queryByTestId('ui-resource-renderer')).not.toBeInTheDocument(); + }); + + it('should show not found when target message is not found', () => { + const mockMessages = [ + { + messageId: 'different-msg', + attachments: [ + { + type: 'ui_resources', + ui_resources: [ + { uri: 'ui://test/resource', mimeType: 'text/html', text: '

Test Resource

' }, + ], + }, + ], + }, + ]; + + mockUseGetMessagesByConvoId.mockReturnValue({ + data: mockMessages, + } as any); + + render(); + + expect(screen.getByText('UI resource at index 0 not found')).toBeInTheDocument(); + }); + + it('should show not found when no ui_resources attachments', () => { + const mockMessages = [ + { + messageId: 'msg123', + attachments: [ + { + type: 'web_search', + web_search: { results: [] }, + }, + ], + }, + ]; + + mockUseGetMessagesByConvoId.mockReturnValue({ + data: mockMessages, + } as any); + + render(); + + expect(screen.getByText('UI resource at index 0 not found')).toBeInTheDocument(); + }); + }); + + describe('UI action handling', () => { + it('should handle UI actions with handleUIAction', async () => { + const mockMessages = [ + { + messageId: 'msg123', + attachments: [ + { + type: 'ui_resources', + ui_resources: [ + { + uri: 'ui://test/resource', + mimeType: 'text/html', + text: '

Interactive Resource

', + }, + ], + }, + ], + }, + ]; + + mockUseGetMessagesByConvoId.mockReturnValue({ + data: mockMessages, + } as any); + + render(); + + const renderer = screen.getByTestId('ui-resource-renderer'); + renderer.click(); + + expect(mockHandleUIAction).toHaveBeenCalledWith({ action: 'test' }, mockSubmitMessageFn); + }); + }); + + describe('edge cases', () => { + it('should handle empty messages array', () => { + mockUseGetMessagesByConvoId.mockReturnValue({ + data: [], + } as any); + + render(); + + expect(screen.getByText('UI resource at index 0 not found')).toBeInTheDocument(); + }); + + it('should handle null messages data', () => { + mockUseGetMessagesByConvoId.mockReturnValue({ + data: null, + } as any); + + render(); + + expect(screen.getByText('UI resource at index 0 not found')).toBeInTheDocument(); + }); + + it('should handle missing conversation', () => { + mockUseChatContext.mockReturnValue({ + conversation: null, + } as any); + + mockUseGetMessagesByConvoId.mockReturnValue({ + data: null, + } as any); + + render(); + + expect(screen.getByText('UI resource at index 0 not found')).toBeInTheDocument(); + }); + + it('should handle multiple attachments of ui_resources type', () => { + const mockMessages = [ + { + messageId: 'msg123', + attachments: [ + { + type: 'ui_resources', + ui_resources: [ + { uri: 'ui://test/resource1', mimeType: 'text/html', text: '

Resource 1

' }, + ], + }, + { + type: 'ui_resources', + ui_resources: [ + { uri: 'ui://test/resource2', mimeType: 'text/html', text: '

Resource 2

' }, + { uri: 'ui://test/resource3', mimeType: 'text/html', text: '

Resource 3

' }, + ], + }, + ], + }, + ]; + + mockUseGetMessagesByConvoId.mockReturnValue({ + data: mockMessages, + } as any); + + // With global indexing across attachments, index 1 should pick the second overall resource + // Flattened order: [resource1, resource2, resource3] + render(); + + const renderer = screen.getByTestId('ui-resource-renderer'); + expect(renderer).toBeInTheDocument(); + expect(renderer).toHaveAttribute('data-resource-uri', 'ui://test/resource2'); + }); + }); +}); diff --git a/client/src/components/MCPUIResource/__tests__/MCPUIResourceCarousel.test.tsx b/client/src/components/MCPUIResource/__tests__/MCPUIResourceCarousel.test.tsx new file mode 100644 index 0000000000..74594e577d --- /dev/null +++ b/client/src/components/MCPUIResource/__tests__/MCPUIResourceCarousel.test.tsx @@ -0,0 +1,454 @@ +import React from 'react'; +import { render, screen } from '@testing-library/react'; +import { MCPUIResourceCarousel } from '../MCPUIResourceCarousel'; +import { useMessageContext, useChatContext } from '~/Providers'; +import { useGetMessagesByConvoId } from '~/data-provider'; + +// Mock dependencies +jest.mock('~/Providers'); +jest.mock('~/data-provider'); + +jest.mock('../../Chat/Messages/Content/UIResourceCarousel', () => ({ + __esModule: true, + default: ({ uiResources }: any) => ( +
+ {uiResources.map((resource: any, index: number) => ( +
+ ))} +
+ ), +})); + +const mockUseMessageContext = useMessageContext as jest.MockedFunction; +const mockUseChatContext = useChatContext as jest.MockedFunction; +const mockUseGetMessagesByConvoId = useGetMessagesByConvoId as jest.MockedFunction< + typeof useGetMessagesByConvoId +>; + +describe('MCPUIResourceCarousel', () => { + beforeEach(() => { + jest.clearAllMocks(); + mockUseMessageContext.mockReturnValue({ messageId: 'msg123' } as any); + mockUseChatContext.mockReturnValue({ + conversation: { conversationId: 'conv123' }, + } as any); + }); + + describe('multiple resource fetching', () => { + it('should fetch multiple resources by indices', () => { + const mockMessages = [ + { + messageId: 'msg123', + attachments: [ + { + type: 'ui_resources', + ui_resources: [ + { + uri: 'ui://test/resource0', + mimeType: 'text/html', + text: '

Resource 0 content

', + }, + { + uri: 'ui://test/resource1', + mimeType: 'text/html', + text: '

Resource 1 content

', + }, + { + uri: 'ui://test/resource2', + mimeType: 'text/html', + text: '

Resource 2 content

', + }, + { + uri: 'ui://test/resource3', + mimeType: 'text/html', + text: '

Resource 3 content

', + }, + ], + }, + ], + }, + ]; + + mockUseGetMessagesByConvoId.mockReturnValue({ + data: mockMessages, + } as any); + + render(); + + const carousel = screen.getByTestId('ui-resource-carousel'); + expect(carousel).toBeInTheDocument(); + expect(carousel).toHaveAttribute('data-resource-count', '3'); + + expect(screen.getByTestId('resource-0')).toHaveAttribute( + 'data-resource-uri', + 'ui://test/resource0', + ); + expect(screen.getByTestId('resource-1')).toHaveAttribute( + 'data-resource-uri', + 'ui://test/resource2', + ); + expect(screen.getByTestId('resource-2')).toHaveAttribute( + 'data-resource-uri', + 'ui://test/resource3', + ); + expect(screen.queryByTestId('resource-3')).not.toBeInTheDocument(); + }); + + it('should preserve resource order based on indices', () => { + const mockMessages = [ + { + messageId: 'msg123', + attachments: [ + { + type: 'ui_resources', + ui_resources: [ + { + uri: 'ui://test/resource0', + mimeType: 'text/html', + text: '

Resource 0 content

', + }, + { + uri: 'ui://test/resource1', + mimeType: 'text/html', + text: '

Resource 1 content

', + }, + { + uri: 'ui://test/resource2', + mimeType: 'text/html', + text: '

Resource 2 content

', + }, + ], + }, + ], + }, + ]; + + mockUseGetMessagesByConvoId.mockReturnValue({ + data: mockMessages, + } as any); + + render(); + + const resources = screen.getAllByTestId(/resource-\d/); + expect(resources[0]).toHaveAttribute('data-resource-uri', 'ui://test/resource2'); + expect(resources[1]).toHaveAttribute('data-resource-uri', 'ui://test/resource0'); + expect(resources[2]).toHaveAttribute('data-resource-uri', 'ui://test/resource1'); + }); + }); + + describe('partial matches', () => { + it('should only include valid resource indices', () => { + const mockMessages = [ + { + messageId: 'msg123', + attachments: [ + { + type: 'ui_resources', + ui_resources: [ + { + uri: 'ui://test/resource0', + mimeType: 'text/html', + text: '

Resource 0 content

', + }, + { + uri: 'ui://test/resource1', + mimeType: 'text/html', + text: '

Resource 1 content

', + }, + ], + }, + ], + }, + ]; + + mockUseGetMessagesByConvoId.mockReturnValue({ + data: mockMessages, + } as any); + + // Request indices 0, 1, 2, 3 but only 0 and 1 exist + render(); + + const carousel = screen.getByTestId('ui-resource-carousel'); + expect(carousel).toHaveAttribute('data-resource-count', '2'); + + expect(screen.getByTestId('resource-0')).toHaveAttribute( + 'data-resource-uri', + 'ui://test/resource0', + ); + expect(screen.getByTestId('resource-1')).toHaveAttribute( + 'data-resource-uri', + 'ui://test/resource1', + ); + }); + + it('should handle all invalid indices', () => { + const mockMessages = [ + { + messageId: 'msg123', + attachments: [ + { + type: 'ui_resources', + ui_resources: [ + { + uri: 'ui://test/resource0', + mimeType: 'text/html', + text: '

Resource 0 content

', + }, + { + uri: 'ui://test/resource1', + mimeType: 'text/html', + text: '

Resource 1 content

', + }, + ], + }, + ], + }, + ]; + + mockUseGetMessagesByConvoId.mockReturnValue({ + data: mockMessages, + } as any); + + // Request indices that don't exist + render(); + + expect(screen.queryByTestId('ui-resource-carousel')).not.toBeInTheDocument(); + }); + }); + + describe('error handling', () => { + it('should return null when no attachments', () => { + const mockMessages = [ + { + messageId: 'msg123', + attachments: undefined, + }, + ]; + + mockUseGetMessagesByConvoId.mockReturnValue({ + data: mockMessages, + } as any); + + const { container } = render( + , + ); + + expect(container.firstChild).toBeNull(); + expect(screen.queryByTestId('ui-resource-carousel')).not.toBeInTheDocument(); + }); + + it('should return null when message not found', () => { + const mockMessages = [ + { + messageId: 'different-msg', + attachments: [ + { + type: 'ui_resources', + ui_resources: [ + { + uri: 'ui://test/resource', + mimeType: 'text/html', + text: '

Resource content

', + }, + ], + }, + ], + }, + ]; + + mockUseGetMessagesByConvoId.mockReturnValue({ + data: mockMessages, + } as any); + + const { container } = render( + , + ); + + expect(container.firstChild).toBeNull(); + }); + + it('should return null when no ui_resources attachments', () => { + const mockMessages = [ + { + messageId: 'msg123', + attachments: [ + { + type: 'web_search', + web_search: { results: [] }, + }, + ], + }, + ]; + + mockUseGetMessagesByConvoId.mockReturnValue({ + data: mockMessages, + } as any); + + const { container } = render( + , + ); + + expect(container.firstChild).toBeNull(); + }); + }); + + describe('edge cases', () => { + it('should handle empty resourceIndices array', () => { + const mockMessages = [ + { + messageId: 'msg123', + attachments: [ + { + type: 'ui_resources', + ui_resources: [ + { + uri: 'ui://test/resource', + mimeType: 'text/html', + text: '

Resource content

', + }, + ], + }, + ], + }, + ]; + + mockUseGetMessagesByConvoId.mockReturnValue({ + data: mockMessages, + } as any); + + const { container } = render( + , + ); + + expect(container.firstChild).toBeNull(); + }); + + it('should handle duplicate indices', () => { + const mockMessages = [ + { + messageId: 'msg123', + attachments: [ + { + type: 'ui_resources', + ui_resources: [ + { + uri: 'ui://test/resource0', + mimeType: 'text/html', + text: '

Resource 0 content

', + }, + { + uri: 'ui://test/resource1', + mimeType: 'text/html', + text: '

Resource 1 content

', + }, + ], + }, + ], + }, + ]; + + mockUseGetMessagesByConvoId.mockReturnValue({ + data: mockMessages, + } as any); + + // Request same index multiple times + render(); + + // Should render each resource multiple times + const carousel = screen.getByTestId('ui-resource-carousel'); + expect(carousel).toHaveAttribute('data-resource-count', '5'); + + const resources = screen.getAllByTestId(/resource-\d/); + expect(resources).toHaveLength(5); + expect(resources[0]).toHaveAttribute('data-resource-uri', 'ui://test/resource0'); + expect(resources[1]).toHaveAttribute('data-resource-uri', 'ui://test/resource0'); + expect(resources[2]).toHaveAttribute('data-resource-uri', 'ui://test/resource1'); + expect(resources[3]).toHaveAttribute('data-resource-uri', 'ui://test/resource1'); + expect(resources[4]).toHaveAttribute('data-resource-uri', 'ui://test/resource0'); + }); + + it('should handle multiple ui_resources attachments', () => { + const mockMessages = [ + { + messageId: 'msg123', + attachments: [ + { + type: 'ui_resources', + ui_resources: [ + { + uri: 'ui://test/resource0', + mimeType: 'text/html', + text: '

Resource 0 content

', + }, + { + uri: 'ui://test/resource1', + mimeType: 'text/html', + text: '

Resource 1 content

', + }, + ], + }, + { + type: 'ui_resources', + ui_resources: [ + { uri: 'ui://test/resource2', mimeType: 'text/html', text: '

Resource 2

' }, + { uri: 'ui://test/resource3', mimeType: 'text/html', text: '

Resource 3

' }, + ], + }, + ], + }, + ]; + + mockUseGetMessagesByConvoId.mockReturnValue({ + data: mockMessages, + } as any); + + // Resources from both attachments should be accessible + render(); + + const carousel = screen.getByTestId('ui-resource-carousel'); + expect(carousel).toHaveAttribute('data-resource-count', '3'); + + // Note: indices 2 and 3 are from the second attachment and become accessible in the flattened array + expect(screen.getByTestId('resource-0')).toHaveAttribute( + 'data-resource-uri', + 'ui://test/resource0', + ); + expect(screen.getByTestId('resource-1')).toHaveAttribute( + 'data-resource-uri', + 'ui://test/resource2', + ); + expect(screen.getByTestId('resource-2')).toHaveAttribute( + 'data-resource-uri', + 'ui://test/resource3', + ); + }); + + it('should handle null messages data', () => { + mockUseGetMessagesByConvoId.mockReturnValue({ + data: null, + } as any); + + const { container } = render( + , + ); + + expect(container.firstChild).toBeNull(); + }); + + it('should handle missing conversation', () => { + mockUseChatContext.mockReturnValue({ + conversation: null, + } as any); + + mockUseGetMessagesByConvoId.mockReturnValue({ + data: null, + } as any); + + const { container } = render( + , + ); + + expect(container.firstChild).toBeNull(); + }); + }); +}); diff --git a/client/src/components/MCPUIResource/__tests__/plugin.test.ts b/client/src/components/MCPUIResource/__tests__/plugin.test.ts new file mode 100644 index 0000000000..9b00ff889c --- /dev/null +++ b/client/src/components/MCPUIResource/__tests__/plugin.test.ts @@ -0,0 +1,249 @@ +import { mcpUIResourcePlugin, UI_RESOURCE_MARKER } from '../plugin'; +import type { Node } from 'unist'; +import type { UIResourceNode } from '../types'; + +describe('mcpUIResourcePlugin', () => { + const createTextNode = (value: string): UIResourceNode => ({ + type: 'text', + value, + }); + + const createTree = (nodes: UIResourceNode[]): Node => + ({ + type: 'root', + children: nodes, + }) as Node; + + const processTree = (tree: Node) => { + const plugin = mcpUIResourcePlugin(); + plugin(tree); + return tree; + }; + + describe('single resource markers', () => { + it('should replace single UI resource marker with mcp-ui-resource node', () => { + const tree = createTree([createTextNode(`Here is a resource ${UI_RESOURCE_MARKER}0`)]); + processTree(tree); + + const children = (tree as any).children; + expect(children).toHaveLength(2); + expect(children[0]).toEqual({ type: 'text', value: 'Here is a resource ' }); + expect(children[1]).toEqual({ + type: 'mcp-ui-resource', + data: { + hName: 'mcp-ui-resource', + hProperties: { + resourceIndex: 0, + }, + }, + }); + }); + + it('should handle multiple single resource markers', () => { + const tree = createTree([ + createTextNode(`First ${UI_RESOURCE_MARKER}0 and second ${UI_RESOURCE_MARKER}1`), + ]); + processTree(tree); + + const children = (tree as any).children; + expect(children).toHaveLength(4); + expect(children[0]).toEqual({ type: 'text', value: 'First ' }); + expect(children[1].type).toBe('mcp-ui-resource'); + expect(children[1].data.hProperties.resourceIndex).toBe(0); + expect(children[2]).toEqual({ type: 'text', value: ' and second ' }); + expect(children[3].type).toBe('mcp-ui-resource'); + expect(children[3].data.hProperties.resourceIndex).toBe(1); + }); + + it('should handle large index numbers', () => { + const tree = createTree([createTextNode(`Resource ${UI_RESOURCE_MARKER}42`)]); + processTree(tree); + + const children = (tree as any).children; + expect(children[1].data.hProperties.resourceIndex).toBe(42); + }); + }); + + describe('carousel markers', () => { + it('should replace carousel marker with mcp-ui-carousel node', () => { + const tree = createTree([createTextNode(`Carousel ${UI_RESOURCE_MARKER}0,1,2`)]); + processTree(tree); + + const children = (tree as any).children; + expect(children).toHaveLength(2); + expect(children[0]).toEqual({ type: 'text', value: 'Carousel ' }); + expect(children[1]).toEqual({ + type: 'mcp-ui-carousel', + data: { + hName: 'mcp-ui-carousel', + hProperties: { + resourceIndices: [0, 1, 2], + }, + }, + }); + }); + + it('should handle large index numbers in carousel', () => { + const tree = createTree([createTextNode(`${UI_RESOURCE_MARKER}100,200,300`)]); + processTree(tree); + + const children = (tree as any).children; + expect(children[0].data.hProperties.resourceIndices).toEqual([100, 200, 300]); + }); + }); + + describe('mixed content', () => { + it('should handle text before and after markers', () => { + const tree = createTree([ + createTextNode(`Before ${UI_RESOURCE_MARKER}0 middle ${UI_RESOURCE_MARKER}1,2 after`), + ]); + processTree(tree); + + const children = (tree as any).children; + expect(children).toHaveLength(5); + expect(children[0].value).toBe('Before '); + expect(children[1].type).toBe('mcp-ui-resource'); + expect(children[2].value).toBe(' middle '); + expect(children[3].type).toBe('mcp-ui-carousel'); + expect(children[4].value).toBe(' after'); + }); + + it('should handle marker at start of text', () => { + const tree = createTree([createTextNode(`${UI_RESOURCE_MARKER}0 after`)]); + processTree(tree); + + const children = (tree as any).children; + expect(children).toHaveLength(2); + expect(children[0].type).toBe('mcp-ui-resource'); + expect(children[1].value).toBe(' after'); + }); + + it('should handle marker at end of text', () => { + const tree = createTree([createTextNode(`Before ${UI_RESOURCE_MARKER}0`)]); + processTree(tree); + + const children = (tree as any).children; + expect(children).toHaveLength(2); + expect(children[0].value).toBe('Before '); + expect(children[1].type).toBe('mcp-ui-resource'); + }); + + it('should handle consecutive markers', () => { + const tree = createTree([createTextNode(`${UI_RESOURCE_MARKER}0${UI_RESOURCE_MARKER}1`)]); + processTree(tree); + + const children = (tree as any).children; + expect(children).toHaveLength(2); + expect(children[0].type).toBe('mcp-ui-resource'); + expect(children[0].data.hProperties.resourceIndex).toBe(0); + expect(children[1].type).toBe('mcp-ui-resource'); + expect(children[1].data.hProperties.resourceIndex).toBe(1); + }); + }); + + describe('edge cases', () => { + it('should handle empty text nodes', () => { + const tree = createTree([createTextNode('')]); + processTree(tree); + + const children = (tree as any).children; + expect(children).toHaveLength(1); + expect(children[0]).toEqual({ type: 'text', value: '' }); + }); + + it('should handle text without markers', () => { + const tree = createTree([createTextNode('No markers here')]); + processTree(tree); + + const children = (tree as any).children; + expect(children).toHaveLength(1); + expect(children[0]).toEqual({ type: 'text', value: 'No markers here' }); + }); + + it('should handle non-text nodes', () => { + const tree = createTree([{ type: 'paragraph', children: [] }]); + processTree(tree); + + const children = (tree as any).children; + expect(children).toHaveLength(1); + expect(children[0].type).toBe('paragraph'); + }); + + it('should handle nested structures', () => { + const tree = { + type: 'root', + children: [ + { + type: 'paragraph', + children: [createTextNode(`Text with ${UI_RESOURCE_MARKER}0`)], + }, + ], + } as Node; + + processTree(tree); + + const paragraph = (tree as any).children[0]; + const textNodes = paragraph.children; + expect(textNodes).toHaveLength(2); + expect(textNodes[0].value).toBe('Text with '); + expect(textNodes[1].type).toBe('mcp-ui-resource'); + }); + + it('should not process nodes without value property', () => { + const tree = createTree([ + { + type: 'text', + // no value property + } as UIResourceNode, + ]); + processTree(tree); + + const children = (tree as any).children; + expect(children).toHaveLength(1); + expect(children[0].type).toBe('text'); + }); + }); + + describe('pattern validation', () => { + it('should not match invalid patterns', () => { + const invalidPatterns = [ + `${UI_RESOURCE_MARKER}`, + `${UI_RESOURCE_MARKER}abc`, + `${UI_RESOURCE_MARKER}-1`, + `${UI_RESOURCE_MARKER},1`, + `ui0`, // missing marker + ]; + + invalidPatterns.forEach((pattern) => { + const tree = createTree([createTextNode(pattern)]); + processTree(tree); + + const children = (tree as any).children; + + expect(children).toHaveLength(1); + expect(children[0].type).toBe('text'); + expect(children[0].value).toBe(pattern); + }); + }); + + it('should handle partial matches correctly', () => { + // Test that ui1.2 matches ui1 and leaves .2 + const tree1 = createTree([createTextNode(`${UI_RESOURCE_MARKER}1.2`)]); + processTree(tree1); + const children1 = (tree1 as any).children; + expect(children1).toHaveLength(2); + expect(children1[0].type).toBe('mcp-ui-resource'); + expect(children1[0].data.hProperties.resourceIndex).toBe(1); + expect(children1[1].value).toBe('.2'); + + // Test that ui1, matches as single resource followed by comma + const tree2 = createTree([createTextNode(`${UI_RESOURCE_MARKER}1,`)]); + processTree(tree2); + const children2 = (tree2 as any).children; + expect(children2).toHaveLength(2); + expect(children2[0].type).toBe('mcp-ui-resource'); + expect(children2[0].data.hProperties.resourceIndex).toBe(1); + expect(children2[1].value).toBe(','); + }); + }); +}); diff --git a/client/src/components/MCPUIResource/index.ts b/client/src/components/MCPUIResource/index.ts new file mode 100644 index 0000000000..9c42e5f461 --- /dev/null +++ b/client/src/components/MCPUIResource/index.ts @@ -0,0 +1,4 @@ +export { mcpUIResourcePlugin } from './plugin'; +export { MCPUIResource } from './MCPUIResource'; +export { MCPUIResourceCarousel } from './MCPUIResourceCarousel'; +export * from './types'; diff --git a/client/src/components/MCPUIResource/plugin.ts b/client/src/components/MCPUIResource/plugin.ts new file mode 100644 index 0000000000..28d3d7f41a --- /dev/null +++ b/client/src/components/MCPUIResource/plugin.ts @@ -0,0 +1,84 @@ +import { visit } from 'unist-util-visit'; +import type { Node } from 'unist'; +import type { UIResourceNode } from './types'; + +export const UI_RESOURCE_MARKER = '\\ui'; +export const UI_RESOURCE_PATTERN = new RegExp(`\\${UI_RESOURCE_MARKER}(\\d+(?:,\\d+)*)`, 'g'); + +/** + * Process text nodes and replace UI resource markers with components + */ +function processTree(tree: Node) { + visit(tree, 'text', (node, index, parent) => { + const textNode = node as UIResourceNode; + const parentNode = parent as UIResourceNode; + + if (typeof textNode.value !== 'string') return; + + const originalValue = textNode.value; + const segments: Array = []; + + let currentPosition = 0; + UI_RESOURCE_PATTERN.lastIndex = 0; + + let match: RegExpExecArray | null; + while ((match = UI_RESOURCE_PATTERN.exec(originalValue)) !== null) { + const matchIndex = match.index; + const matchText = match[0]; + const indicesString = match[1]; + const indices = indicesString.split(',').map(Number); + + if (matchIndex > currentPosition) { + const textBeforeMatch = originalValue.substring(currentPosition, matchIndex); + if (textBeforeMatch) { + segments.push({ type: 'text', value: textBeforeMatch }); + } + } + + if (indices.length === 1) { + segments.push({ + type: 'mcp-ui-resource', + data: { + hName: 'mcp-ui-resource', + hProperties: { + resourceIndex: indices[0], + }, + }, + }); + } else { + segments.push({ + type: 'mcp-ui-carousel', + data: { + hName: 'mcp-ui-carousel', + hProperties: { + resourceIndices: indices, + }, + }, + }); + } + + currentPosition = matchIndex + matchText.length; + } + + if (currentPosition < originalValue.length) { + const remainingText = originalValue.substring(currentPosition); + if (remainingText) { + segments.push({ type: 'text', value: remainingText }); + } + } + + if (segments.length > 0 && index !== undefined) { + parentNode.children?.splice(index, 1, ...segments); + return index + segments.length; + } + }); +} + +/** + * Remark plugin for processing MCP UI resource markers + */ +export function mcpUIResourcePlugin() { + return (tree: Node) => { + processTree(tree); + }; +} diff --git a/client/src/components/MCPUIResource/types.ts b/client/src/components/MCPUIResource/types.ts new file mode 100644 index 0000000000..f7dff19612 --- /dev/null +++ b/client/src/components/MCPUIResource/types.ts @@ -0,0 +1,12 @@ +export interface UIResourceNode { + type: string; + value?: string; + data?: { + hName: string; + hProperties: { + resourceIndex?: number; + resourceIndices?: number[]; + }; + }; + children?: UIResourceNode[]; +} diff --git a/client/src/hooks/index.ts b/client/src/hooks/index.ts index a33d1d0b92..964f07902c 100644 --- a/client/src/hooks/index.ts +++ b/client/src/hooks/index.ts @@ -33,4 +33,4 @@ export { default as useDocumentTitle } from './useDocumentTitle'; export { default as useSpeechToText } from './Input/useSpeechToText'; export { default as useTextToSpeech } from './Input/useTextToSpeech'; export { default as useGenerationsByLatest } from './useGenerationsByLatest'; -export { useResourcePermissions } from './useResourcePermissions'; +export { default as useResourcePermissions } from './useResourcePermissions'; diff --git a/client/src/hooks/useResourcePermissions.ts b/client/src/hooks/useResourcePermissions.ts index 12b30bb9ae..a58b735a87 100644 --- a/client/src/hooks/useResourcePermissions.ts +++ b/client/src/hooks/useResourcePermissions.ts @@ -24,3 +24,5 @@ export const useResourcePermissions = (resourceType: ResourceType, resourceId: s permissionBits: data?.permissionBits || 0, }; }; + +export default useResourcePermissions; diff --git a/client/src/locales/en/translation.json b/client/src/locales/en/translation.json index d4e0756608..3cd9dfdf00 100644 --- a/client/src/locales/en/translation.json +++ b/client/src/locales/en/translation.json @@ -1319,6 +1319,8 @@ "com_ui_trust_app": "I trust this application", "com_ui_try_adjusting_search": "Try adjusting your search terms", "com_ui_ui_resources": "UI Resources", + "com_ui_ui_resource_error": "UI Resource Error ({{0}})", + "com_ui_ui_resource_not_found": "UI Resource not found (index: {{0}})", "com_ui_unarchive": "Unarchive", "com_ui_unarchive_error": "Failed to unarchive conversation", "com_ui_unavailable": "Unavailable", diff --git a/client/src/utils/index.ts b/client/src/utils/index.ts index 7eea034ceb..d8159a49c1 100644 --- a/client/src/utils/index.ts +++ b/client/src/utils/index.ts @@ -124,3 +124,59 @@ export const normalizeLayout = (layout: number[]) => { return normalizedLayout; }; + +export const handleUIAction = async (result: any, submitMessage: any) => { + const supportedTypes = ['intent', 'tool', 'prompt']; + + const { type, payload } = result; + + if (!supportedTypes.includes(type)) { + return; + } + + let messageText = ''; + + if (type === 'intent') { + const { intent, params } = payload; + messageText = `The user clicked a button in an embedded UI Resource, and we got a message of type \`intent\`. +The intent is \`${intent}\` and the params are: + +\`\`\`json +${JSON.stringify(params, null, 2)} +\`\`\` + +Execute the intent that is mentioned in the message using the tools available to you. + `; + } else if (type === 'tool') { + const { toolName, params } = payload; + messageText = `The user clicked a button in an embedded UI Resource, and we got a message of type \`tool\`. +The tool name is \`${toolName}\` and the params are: + +\`\`\`json +${JSON.stringify(params, null, 2)} +\`\`\` + +Execute the tool that is mentioned in the message using the tools available to you. + `; + } else if (type === 'prompt') { + const { prompt } = payload; + messageText = `The user clicked a button in an embedded UI Resource, and we got a message of type \`prompt\`. +The prompt is: + +\`\`\` +${prompt} +\`\`\` + +Execute the intention of the prompt that is mentioned in the message using the tools available to you. + `; + } + + console.log('About to submit message:', messageText); + + try { + await submitMessage({ text: messageText }); + console.log('Message submitted successfully'); + } catch (error) { + console.error('Error submitting message:', error); + } +}; diff --git a/packages/api/src/mcp/__tests__/parsers.test.ts b/packages/api/src/mcp/__tests__/parsers.test.ts index a26373aef6..5e90e05c8a 100644 --- a/packages/api/src/mcp/__tests__/parsers.test.ts +++ b/packages/api/src/mcp/__tests__/parsers.test.ts @@ -182,7 +182,6 @@ describe('formatToolContent', () => { { type: 'text', text: - 'Resource Text: {"items": []}\n' + 'Resource URI: ui://carousel\n' + 'Resource: carousel\n' + 'Resource Description: A carousel component\n' + @@ -288,7 +287,6 @@ describe('formatToolContent', () => { type: 'text', text: 'Some text\n\n' + - 'Resource Text: {"label": "Click me"}\n' + 'Resource URI: ui://button\n' + 'Resource MIME Type: application/json\n\n' + 'Resource URI: file://data.csv\n' + @@ -332,10 +330,7 @@ describe('formatToolContent', () => { }, { type: 'text', - text: - 'Resource Text: {"type": "line"}\n' + - 'Resource URI: ui://graph\n' + - 'Resource MIME Type: application/json', + text: 'Resource URI: ui://graph\n' + 'Resource MIME Type: application/json', }, ]); expect(artifacts).toEqual({ @@ -414,7 +409,6 @@ describe('formatToolContent', () => { type: 'text', text: 'Middle section\n\n' + - 'Resource Text: {"type": "bar"}\n' + 'Resource URI: ui://chart\n' + 'Resource MIME Type: application/json\n\n' + 'Resource URI: https://api.example.com/data\n' + diff --git a/packages/api/src/mcp/parsers.ts b/packages/api/src/mcp/parsers.ts index 2f9ce8c06d..f616b997ae 100644 --- a/packages/api/src/mcp/parsers.ts +++ b/packages/api/src/mcp/parsers.ts @@ -138,12 +138,14 @@ export function formatToolContent( }, resource: (item) => { - if (item.resource.uri.startsWith('ui://')) { + const isUiResource = item.resource.uri.startsWith('ui://'); + + if (isUiResource) { uiResources.push(item.resource as UIResource); } const resourceText = []; - if (item.resource.text != null && item.resource.text) { + if (!isUiResource && item.resource.text != null && item.resource.text) { resourceText.push(`Resource Text: ${item.resource.text}`); } if (item.resource.uri.length) { @@ -177,10 +179,14 @@ export function formatToolContent( } let artifacts: t.Artifacts = undefined; - if (imageUrls.length || uiResources.length) { + if (imageUrls.length) { + artifacts = { content: imageUrls }; + } + + if (uiResources.length) { artifacts = { - ...(imageUrls.length && { content: imageUrls }), - ...(uiResources.length && { [Tools.ui_resources]: { data: uiResources } }), + ...artifacts, + [Tools.ui_resources]: { data: uiResources }, }; } diff --git a/packages/api/src/mcp/types/index.ts b/packages/api/src/mcp/types/index.ts index 346d6338e5..9f7ff92383 100644 --- a/packages/api/src/mcp/types/index.ts +++ b/packages/api/src/mcp/types/index.ts @@ -83,11 +83,7 @@ export type Provider = export type FormattedContent = | { type: 'text'; - metadata?: { - type: string; - data: UIResource[]; - }; - text?: string; + text: string; } | { type: 'image'; diff --git a/packages/api/src/mcp/utils.ts b/packages/api/src/mcp/utils.ts index fbd8ed7c95..fddebb9db3 100644 --- a/packages/api/src/mcp/utils.ts +++ b/packages/api/src/mcp/utils.ts @@ -12,7 +12,7 @@ export function normalizeServerName(serverName: string): string { } /** Replace non-matching characters with underscores. - This preserves the general structure while ensuring compatibility. + This preserves the general structure while ensuring compatibility. Trims leading/trailing underscores */ const normalized = serverName.replace(/[^a-zA-Z0-9_.-]/g, '_').replace(/^_+|_+$/g, '');