diff --git a/api/server/controllers/agents/__tests__/callbacks.spec.js b/api/server/controllers/agents/__tests__/callbacks.spec.js index 25f00bab8d..7922c31efa 100644 --- a/api/server/controllers/agents/__tests__/callbacks.spec.js +++ b/api/server/controllers/agents/__tests__/callbacks.spec.js @@ -73,10 +73,10 @@ describe('createToolEndCallback', () => { tool_call_id: 'tool123', artifact: { [Tools.ui_resources]: { - data: { - 0: { type: 'button', label: 'Click me' }, - 1: { type: 'input', placeholder: 'Enter text' }, - }, + data: [ + { type: 'button', label: 'Click me' }, + { type: 'input', placeholder: 'Enter text' }, + ], }, }, }; @@ -100,10 +100,10 @@ describe('createToolEndCallback', () => { messageId: 'run456', toolCallId: 'tool123', conversationId: 'thread789', - [Tools.ui_resources]: { - 0: { type: 'button', label: 'Click me' }, - 1: { type: 'input', placeholder: 'Enter text' }, - }, + [Tools.ui_resources]: [ + { type: 'button', label: 'Click me' }, + { type: 'input', placeholder: 'Enter text' }, + ], }); }); @@ -115,9 +115,7 @@ describe('createToolEndCallback', () => { tool_call_id: 'tool123', artifact: { [Tools.ui_resources]: { - data: { - 0: { type: 'carousel', items: [] }, - }, + data: [{ type: 'carousel', items: [] }], }, }, }; @@ -136,9 +134,7 @@ describe('createToolEndCallback', () => { messageId: 'run456', toolCallId: 'tool123', conversationId: 'thread789', - [Tools.ui_resources]: { - 0: { type: 'carousel', items: [] }, - }, + [Tools.ui_resources]: [{ type: 'carousel', items: [] }], }); }); @@ -155,9 +151,7 @@ describe('createToolEndCallback', () => { tool_call_id: 'tool123', artifact: { [Tools.ui_resources]: { - data: { - 0: { type: 'test' }, - }, + data: [{ type: 'test' }], }, }, }; @@ -184,9 +178,7 @@ describe('createToolEndCallback', () => { tool_call_id: 'tool123', artifact: { [Tools.ui_resources]: { - data: { - 0: { type: 'chart', data: [] }, - }, + data: [{ type: 'chart', data: [] }], }, [Tools.web_search]: { results: ['result1', 'result2'], @@ -209,9 +201,7 @@ describe('createToolEndCallback', () => { // Check ui_resources attachment const uiResourceAttachment = results.find((r) => r?.type === Tools.ui_resources); expect(uiResourceAttachment).toBeTruthy(); - expect(uiResourceAttachment[Tools.ui_resources]).toEqual({ - 0: { type: 'chart', data: [] }, - }); + expect(uiResourceAttachment[Tools.ui_resources]).toEqual([{ type: 'chart', data: [] }]); // Check web_search attachment const webSearchAttachment = results.find((r) => r?.type === Tools.web_search); @@ -250,7 +240,7 @@ describe('createToolEndCallback', () => { tool_call_id: 'tool123', artifact: { [Tools.ui_resources]: { - data: {}, + data: [], }, }, }; @@ -268,7 +258,7 @@ describe('createToolEndCallback', () => { messageId: 'run456', toolCallId: 'tool123', conversationId: 'thread789', - [Tools.ui_resources]: {}, + [Tools.ui_resources]: [], }); }); diff --git a/client/src/Providers/MessagesViewContext.tsx b/client/src/Providers/MessagesViewContext.tsx index 630ad277f1..137fffbcd0 100644 --- a/client/src/Providers/MessagesViewContext.tsx +++ b/client/src/Providers/MessagesViewContext.tsx @@ -28,6 +28,10 @@ interface MessagesViewContextValue { const MessagesViewContext = createContext(undefined); +// Export the context so it can be provided by other providers (e.g., ShareMessagesProvider) +export { MessagesViewContext }; +export type { MessagesViewContextValue }; + export function MessagesViewProvider({ children }: { children: React.ReactNode }) { const chatContext = useChatContext(); const addedChatContext = useAddedChatContext(); diff --git a/client/src/components/Chat/Messages/Content/Markdown.tsx b/client/src/components/Chat/Messages/Content/Markdown.tsx index 42136384d4..a763885d2f 100644 --- a/client/src/components/Chat/Messages/Content/Markdown.tsx +++ b/client/src/components/Chat/Messages/Content/Markdown.tsx @@ -9,6 +9,11 @@ import rehypeHighlight from 'rehype-highlight'; import remarkDirective from 'remark-directive'; import type { Pluggable } from 'unified'; import { Citation, CompositeCitation, HighlightedText } from '~/components/Web/Citation'; +import { + mcpUIResourcePlugin, + MCPUIResource, + MCPUIResourceCarousel, +} from '~/components/MCPUIResource'; import { Artifact, artifactPlugin } from '~/components/Artifacts/Artifact'; import { ArtifactProvider, CodeBlockProvider } from '~/Providers'; import MarkdownErrorBoundary from './MarkdownErrorBoundary'; @@ -55,6 +60,7 @@ const Markdown = memo(({ content = '', isLatestMessage }: TContentProps) => { artifactPlugin, [remarkMath, { singleDollarTextMath: false }], unicodeCitation, + mcpUIResourcePlugin, ]; if (isInitializing) { @@ -86,6 +92,8 @@ const Markdown = memo(({ content = '', isLatestMessage }: TContentProps) => { citation: Citation, 'highlighted-text': HighlightedText, 'composite-citation': CompositeCitation, + 'mcp-ui-resource': MCPUIResource, + 'mcp-ui-carousel': MCPUIResourceCarousel, } as { [nodeType: string]: React.ElementType; } diff --git a/client/src/components/Chat/Messages/Content/UIResourceCarousel.tsx b/client/src/components/Chat/Messages/Content/UIResourceCarousel.tsx index dfffa08935..15c420755b 100644 --- a/client/src/components/Chat/Messages/Content/UIResourceCarousel.tsx +++ b/client/src/components/Chat/Messages/Content/UIResourceCarousel.tsx @@ -1,6 +1,8 @@ import React, { useState } from 'react'; import { UIResourceRenderer } from '@mcp-ui/client'; import type { UIResource } from 'librechat-data-provider'; +import { useMessagesOperations } from '~/Providers'; +import { handleUIAction } from '~/utils'; interface UIResourceCarouselProps { uiResources: UIResource[]; @@ -11,6 +13,7 @@ const UIResourceCarousel: React.FC = React.memo(({ uiRe const [showRightArrow, setShowRightArrow] = useState(true); const [isContainerHovered, setIsContainerHovered] = useState(false); const scrollContainerRef = React.useRef(null); + const { ask } = useMessagesOperations(); const handleScroll = React.useCallback(() => { if (!scrollContainerRef.current) return; @@ -111,9 +114,7 @@ const UIResourceCarousel: React.FC = React.memo(({ uiRe mimeType: uiResource.mimeType, text: uiResource.text, }} - onUIAction={async (result) => { - console.log('Action:', result); - }} + onUIAction={async (result) => handleUIAction(result, ask)} htmlProps={{ autoResizeIframe: { width: true, height: true }, }} diff --git a/client/src/components/Chat/Messages/Content/__tests__/Markdown.mcpui.test.tsx b/client/src/components/Chat/Messages/Content/__tests__/Markdown.mcpui.test.tsx new file mode 100644 index 0000000000..6df66c9e15 --- /dev/null +++ b/client/src/components/Chat/Messages/Content/__tests__/Markdown.mcpui.test.tsx @@ -0,0 +1,106 @@ +import React from 'react'; +import { render, screen } from '@testing-library/react'; +import Markdown from '../Markdown'; +import { RecoilRoot } from 'recoil'; +import { UI_RESOURCE_MARKER } from '~/components/MCPUIResource/plugin'; +import { useMessageContext, useMessagesConversation, useMessagesOperations } from '~/Providers'; +import { useGetMessagesByConvoId } from '~/data-provider'; +import { useLocalize } from '~/hooks'; + +// Mocks for hooks used by MCPUIResource when rendered inside Markdown. +// Keep Provider components intact while mocking only the hooks we use. +jest.mock('~/Providers', () => ({ + ...jest.requireActual('~/Providers'), + useMessageContext: jest.fn(), + useMessagesConversation: jest.fn(), + useMessagesOperations: jest.fn(), +})); +jest.mock('~/data-provider'); +jest.mock('~/hooks'); + +// Mock @mcp-ui/client to render identifiable elements for assertions +jest.mock('@mcp-ui/client', () => ({ + UIResourceRenderer: ({ resource }: any) => ( +
+ ), +})); + +const mockUseMessageContext = useMessageContext as jest.MockedFunction; +const mockUseMessagesConversation = useMessagesConversation as jest.MockedFunction< + typeof useMessagesConversation +>; +const mockUseMessagesOperations = useMessagesOperations as jest.MockedFunction< + typeof useMessagesOperations +>; +const mockUseGetMessagesByConvoId = useGetMessagesByConvoId as jest.MockedFunction< + typeof useGetMessagesByConvoId +>; +const mockUseLocalize = useLocalize as jest.MockedFunction; + +describe('Markdown with MCP UI markers (resource IDs)', () => { + let currentTestMessages: any[] = []; + + beforeEach(() => { + jest.clearAllMocks(); + currentTestMessages = []; + + mockUseMessageContext.mockReturnValue({ messageId: 'msg-weather' } as any); + mockUseMessagesConversation.mockReturnValue({ + conversation: { conversationId: 'conv1' }, + conversationId: 'conv1', + } as any); + mockUseMessagesOperations.mockReturnValue({ + ask: jest.fn(), + getMessages: () => currentTestMessages, + } as any); + mockUseLocalize.mockReturnValue(((key: string) => key) as any); + }); + + it('renders two UIResourceRenderer components for markers with resource IDs across separate attachments', () => { + // Two tool responses, each produced one ui_resources attachment + const paris = { + resourceId: 'abc123', + uri: 'ui://weather/paris', + mimeType: 'text/html', + text: '
Paris Weather
', + }; + const nyc = { + resourceId: 'def456', + uri: 'ui://weather/nyc', + mimeType: 'text/html', + text: '
NYC Weather
', + }; + + currentTestMessages = [ + { + messageId: 'msg-weather', + attachments: [ + { type: 'ui_resources', ui_resources: [paris] }, + { type: 'ui_resources', ui_resources: [nyc] }, + ], + }, + ]; + + mockUseGetMessagesByConvoId.mockReturnValue({ data: currentTestMessages } as any); + + const content = [ + 'Here are the current weather conditions for both Paris and New York:', + '', + '- Paris: Slight rain, 53°F, humidity 76%, wind 9 mph.', + '- New York: Clear sky, 63°F, humidity 91%, wind 6 mph.', + '', + `Browse these weather cards for more details ${UI_RESOURCE_MARKER}{abc123} ${UI_RESOURCE_MARKER}{def456}`, + ].join('\n'); + + render( + + + , + ); + + const renderers = screen.getAllByTestId('ui-resource-renderer'); + expect(renderers).toHaveLength(2); + expect(renderers[0]).toHaveAttribute('data-resource-uri', 'ui://weather/paris'); + expect(renderers[1]).toHaveAttribute('data-resource-uri', 'ui://weather/nyc'); + }); +}); diff --git a/client/src/components/Chat/Messages/Content/__tests__/UIResourceCarousel.test.tsx b/client/src/components/Chat/Messages/Content/__tests__/UIResourceCarousel.test.tsx index 2878c6444f..6d208c2cf2 100644 --- a/client/src/components/Chat/Messages/Content/__tests__/UIResourceCarousel.test.tsx +++ b/client/src/components/Chat/Messages/Content/__tests__/UIResourceCarousel.test.tsx @@ -1,8 +1,8 @@ -import React from 'react'; import '@testing-library/jest-dom'; -import { render, screen, fireEvent, waitFor } from '@testing-library/react'; +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; import type { UIResource } from 'librechat-data-provider'; import UIResourceCarousel from '~/components/Chat/Messages/Content/UIResourceCarousel'; +import { handleUIAction } from '~/utils'; // Mock the UIResourceRenderer component jest.mock('@mcp-ui/client', () => ({ @@ -13,6 +13,19 @@ jest.mock('@mcp-ui/client', () => ({ ), })); +// Mock useMessagesOperations hook +const mockAsk = jest.fn(); +jest.mock('~/Providers', () => ({ + useMessagesOperations: () => ({ + ask: mockAsk, + }), +})); + +// Mock handleUIAction utility +jest.mock('~/utils', () => ({ + handleUIAction: jest.fn(), +})); + // Mock scrollTo const mockScrollTo = jest.fn(); Object.defineProperty(HTMLElement.prototype, 'scrollTo', { @@ -29,8 +42,12 @@ describe('UIResourceCarousel', () => { { uri: 'resource5', mimeType: 'text/html', text: 'Resource 5' }, ]; + const mockHandleUIAction = handleUIAction as jest.MockedFunction; + beforeEach(() => { jest.clearAllMocks(); + mockAsk.mockClear(); + mockHandleUIAction.mockClear(); // Reset scroll properties Object.defineProperty(HTMLElement.prototype, 'scrollLeft', { configurable: true, @@ -141,18 +158,48 @@ describe('UIResourceCarousel', () => { }); }); - it('handles UIResource actions', async () => { - const consoleSpy = jest.spyOn(console, 'log').mockImplementation(); + it('handles UIResource actions using handleUIAction', async () => { render(); const renderer = screen.getByTestId('ui-resource-renderer'); fireEvent.click(renderer); await waitFor(() => { - expect(consoleSpy).toHaveBeenCalledWith('Action:', { action: 'test' }); + expect(mockHandleUIAction).toHaveBeenCalledWith({ action: 'test' }, mockAsk); + }); + }); + + it('calls handleUIAction with correct parameters for multiple resources', async () => { + render(); + + const renderers = screen.getAllByTestId('ui-resource-renderer'); + + // Click the second renderer + fireEvent.click(renderers[1]); + + await waitFor(() => { + expect(mockHandleUIAction).toHaveBeenCalledWith({ action: 'test' }, mockAsk); + expect(mockHandleUIAction).toHaveBeenCalledTimes(1); }); - consoleSpy.mockRestore(); + // Click the third renderer + fireEvent.click(renderers[2]); + + await waitFor(() => { + expect(mockHandleUIAction).toHaveBeenCalledTimes(2); + }); + }); + + it('passes correct ask function to handleUIAction', async () => { + render(); + + const renderer = screen.getByTestId('ui-resource-renderer'); + fireEvent.click(renderer); + + await waitFor(() => { + expect(mockHandleUIAction).toHaveBeenCalledWith({ action: 'test' }, mockAsk); + expect(mockHandleUIAction).toHaveBeenCalledTimes(1); + }); }); it('applies correct dimensions to resource containers', () => { diff --git a/client/src/components/MCPUIResource/MCPUIResource.tsx b/client/src/components/MCPUIResource/MCPUIResource.tsx new file mode 100644 index 0000000000..87249a1ece --- /dev/null +++ b/client/src/components/MCPUIResource/MCPUIResource.tsx @@ -0,0 +1,63 @@ +import React from 'react'; +import { UIResourceRenderer } from '@mcp-ui/client'; +import { handleUIAction } from '~/utils'; +import { useConversationUIResources } from '~/hooks/Messages/useConversationUIResources'; +import { useMessagesConversation, useMessagesOperations } from '~/Providers'; +import { useLocalize } from '~/hooks'; + +interface MCPUIResourceProps { + node: { + properties: { + resourceId: string; + }; + }; +} + +/** + * Component that renders an MCP UI resource based on its resource ID. + * Works in both main app and share view. + */ +export function MCPUIResource(props: MCPUIResourceProps) { + const { resourceId } = props.node.properties; + const localize = useLocalize(); + const { ask } = useMessagesOperations(); + const { conversation } = useMessagesConversation(); + + const conversationResourceMap = useConversationUIResources( + conversation?.conversationId ?? undefined, + ); + + const uiResource = conversationResourceMap.get(resourceId ?? ''); + + if (!uiResource) { + return ( + + {localize('com_ui_ui_resource_not_found', { + 0: resourceId ?? '', + })} + + ); + } + + try { + return ( + + handleUIAction(result, ask)} + htmlProps={{ + autoResizeIframe: { width: true, height: true }, + sandboxPermissions: 'allow-popups', + }} + /> + + ); + } catch (error) { + console.error('Error rendering UI resource:', error); + return ( + + {localize('com_ui_ui_resource_error', { 0: uiResource.name })} + + ); + } +} diff --git a/client/src/components/MCPUIResource/MCPUIResourceCarousel.tsx b/client/src/components/MCPUIResource/MCPUIResourceCarousel.tsx new file mode 100644 index 0000000000..cf32318491 --- /dev/null +++ b/client/src/components/MCPUIResource/MCPUIResourceCarousel.tsx @@ -0,0 +1,37 @@ +import React, { useMemo } from 'react'; +import { useConversationUIResources } from '~/hooks/Messages/useConversationUIResources'; +import { useMessagesConversation } from '~/Providers'; +import UIResourceCarousel from '../Chat/Messages/Content/UIResourceCarousel'; +import type { UIResource } from 'librechat-data-provider'; + +interface MCPUIResourceCarouselProps { + node: { + properties: { + resourceIds?: string[]; + }; + }; +} + +/** + * Component that renders multiple MCP UI resources in a carousel. + * Works in both main app and share view. + */ +export function MCPUIResourceCarousel(props: MCPUIResourceCarouselProps) { + const { conversation } = useMessagesConversation(); + + const conversationResourceMap = useConversationUIResources( + conversation?.conversationId ?? undefined, + ); + + const uiResources = useMemo(() => { + const { resourceIds = [] } = props.node.properties; + + return resourceIds.map((id) => conversationResourceMap.get(id)).filter(Boolean) as UIResource[]; + }, [props.node.properties, conversationResourceMap]); + + if (uiResources.length === 0) { + return null; + } + + return ; +} diff --git a/client/src/components/MCPUIResource/__tests__/MCPUIResource.test.tsx b/client/src/components/MCPUIResource/__tests__/MCPUIResource.test.tsx new file mode 100644 index 0000000000..53896bb6fe --- /dev/null +++ b/client/src/components/MCPUIResource/__tests__/MCPUIResource.test.tsx @@ -0,0 +1,276 @@ +import React from 'react'; +import { render, screen } from '@testing-library/react'; +import { RecoilRoot } from 'recoil'; +import { MCPUIResource } from '../MCPUIResource'; +import { useMessageContext, useMessagesConversation, useMessagesOperations } from '~/Providers'; +import { useLocalize } from '~/hooks'; +import { handleUIAction } from '~/utils'; + +// Mock dependencies +jest.mock('~/Providers'); +jest.mock('~/hooks'); +jest.mock('~/utils'); + +jest.mock('@mcp-ui/client', () => ({ + UIResourceRenderer: ({ resource, onUIAction }: any) => ( +
onUIAction({ action: 'test' })} + /> + ), +})); + +const mockUseMessageContext = useMessageContext as jest.MockedFunction; +const mockUseMessagesConversation = useMessagesConversation as jest.MockedFunction< + typeof useMessagesConversation +>; +const mockUseMessagesOperations = useMessagesOperations as jest.MockedFunction< + typeof useMessagesOperations +>; +const mockUseLocalize = useLocalize as jest.MockedFunction; +const mockHandleUIAction = handleUIAction as jest.MockedFunction; + +describe('MCPUIResource', () => { + const mockLocalize = (key: string, values?: any) => { + const translations: Record = { + com_ui_ui_resource_not_found: `UI resource ${values?.[0]} not found`, + com_ui_ui_resource_error: `Error rendering UI resource: ${values?.[0]}`, + }; + return translations[key] || key; + }; + + const mockAskFn = jest.fn(); + + const renderWithRecoil = (ui: React.ReactNode) => render({ui}); + + // Store the current test's messages so getMessages can return them + let currentTestMessages: any[] = []; + + beforeEach(() => { + jest.clearAllMocks(); + currentTestMessages = []; + mockUseMessageContext.mockReturnValue({ messageId: 'msg123' } as any); + mockUseMessagesConversation.mockReturnValue({ + conversation: { conversationId: 'conv123' }, + conversationId: 'conv123', + } as any); + mockUseMessagesOperations.mockReturnValue({ + ask: mockAskFn, + getMessages: () => currentTestMessages, + regenerate: jest.fn(), + handleContinue: jest.fn(), + setMessages: jest.fn(), + } as any); + mockUseLocalize.mockReturnValue(mockLocalize as any); + }); + + describe('resource fetching', () => { + it('should fetch and render UI resource from message attachments', () => { + currentTestMessages = [ + { + messageId: 'msg123', + attachments: [ + { + type: 'ui_resources', + ui_resources: [ + { + resourceId: 'resource-1', + uri: 'ui://test/resource', + mimeType: 'text/html', + text: '

Test Resource

', + }, + ], + }, + ], + }, + ]; + + renderWithRecoil(); + + const renderer = screen.getByTestId('ui-resource-renderer'); + expect(renderer).toBeInTheDocument(); + expect(renderer).toHaveAttribute('data-resource-uri', 'ui://test/resource'); + }); + + it('should show not found message when resourceId does not exist', () => { + currentTestMessages = [ + { + messageId: 'msg123', + attachments: [ + { + type: 'ui_resources', + ui_resources: [ + { + resourceId: 'resource-1', + uri: 'ui://test/resource', + mimeType: 'text/html', + text: '

Test Resource

', + }, + ], + }, + ], + }, + ]; + + renderWithRecoil(); + + expect(screen.getByText('UI resource nonexistent-id not found')).toBeInTheDocument(); + expect(screen.queryByTestId('ui-resource-renderer')).not.toBeInTheDocument(); + }); + + it('should show not found when no ui_resources attachments', () => { + currentTestMessages = [ + { + messageId: 'msg123', + attachments: [ + { + type: 'web_search', + web_search: { results: [] }, + }, + ], + }, + ]; + + renderWithRecoil(); + + expect(screen.getByText('UI resource resource-1 not found')).toBeInTheDocument(); + }); + + it('should resolve resources by resourceId across conversation messages', () => { + mockUseMessageContext.mockReturnValue({ messageId: 'msg-current' } as any); + currentTestMessages = [ + { + messageId: 'msg-previous', + attachments: [ + { + type: 'ui_resources', + ui_resources: [ + { + resourceId: 'abc123', + uri: 'ui://test/resource-id', + mimeType: 'text/html', + text: '

Resource via ID

', + }, + ], + }, + ], + }, + { + messageId: 'msg-current', + attachments: [], + }, + ]; + + renderWithRecoil(); + + const renderer = screen.getByTestId('ui-resource-renderer'); + expect(renderer).toBeInTheDocument(); + expect(renderer).toHaveAttribute('data-resource-uri', 'ui://test/resource-id'); + }); + }); + + describe('UI action handling', () => { + it('should handle UI actions with handleUIAction', async () => { + currentTestMessages = [ + { + messageId: 'msg123', + attachments: [ + { + type: 'ui_resources', + ui_resources: [ + { + resourceId: 'resource-1', + uri: 'ui://test/resource', + mimeType: 'text/html', + text: '

Interactive Resource

', + }, + ], + }, + ], + }, + ]; + + renderWithRecoil(); + + const renderer = screen.getByTestId('ui-resource-renderer'); + renderer.click(); + + expect(mockHandleUIAction).toHaveBeenCalledWith({ action: 'test' }, mockAskFn); + }); + }); + + describe('edge cases', () => { + it('should handle empty messages array', () => { + currentTestMessages = []; + + renderWithRecoil(); + + expect(screen.getByText('UI resource resource-1 not found')).toBeInTheDocument(); + }); + + it('should handle null messages data', () => { + currentTestMessages = []; + + renderWithRecoil(); + + expect(screen.getByText('UI resource resource-1 not found')).toBeInTheDocument(); + }); + + it('should handle missing conversation', () => { + currentTestMessages = []; + mockUseMessagesConversation.mockReturnValue({ + conversation: null, + conversationId: null, + } as any); + + renderWithRecoil(); + + expect(screen.getByText('UI resource resource-1 not found')).toBeInTheDocument(); + }); + + it('should handle multiple attachments of ui_resources type', () => { + currentTestMessages = [ + { + messageId: 'msg123', + attachments: [ + { + type: 'ui_resources', + ui_resources: [ + { + resourceId: 'resource-1', + uri: 'ui://test/resource1', + mimeType: 'text/html', + text: '

Resource 1

', + }, + ], + }, + { + type: 'ui_resources', + ui_resources: [ + { + resourceId: 'resource-2', + uri: 'ui://test/resource2', + mimeType: 'text/html', + text: '

Resource 2

', + }, + { + resourceId: 'resource-3', + uri: 'ui://test/resource3', + mimeType: 'text/html', + text: '

Resource 3

', + }, + ], + }, + ], + }, + ]; + + renderWithRecoil(); + + const renderer = screen.getByTestId('ui-resource-renderer'); + expect(renderer).toBeInTheDocument(); + expect(renderer).toHaveAttribute('data-resource-uri', 'ui://test/resource2'); + }); + }); +}); diff --git a/client/src/components/MCPUIResource/__tests__/MCPUIResourceCarousel.test.tsx b/client/src/components/MCPUIResource/__tests__/MCPUIResourceCarousel.test.tsx new file mode 100644 index 0000000000..a9f7962ab0 --- /dev/null +++ b/client/src/components/MCPUIResource/__tests__/MCPUIResourceCarousel.test.tsx @@ -0,0 +1,263 @@ +import React from 'react'; +import { render, screen } from '@testing-library/react'; +import { RecoilRoot } from 'recoil'; +import { MCPUIResourceCarousel } from '../MCPUIResourceCarousel'; +import { useMessageContext, useMessagesConversation, useMessagesOperations } from '~/Providers'; + +// Mock dependencies +jest.mock('~/Providers'); + +jest.mock('../../Chat/Messages/Content/UIResourceCarousel', () => ({ + __esModule: true, + default: ({ uiResources }: any) => ( +
+ {uiResources.map((resource: any, index: number) => ( +
+ ))} +
+ ), +})); + +const mockUseMessageContext = useMessageContext as jest.MockedFunction; +const mockUseMessagesConversation = useMessagesConversation as jest.MockedFunction< + typeof useMessagesConversation +>; +const mockUseMessagesOperations = useMessagesOperations as jest.MockedFunction< + typeof useMessagesOperations +>; + +describe('MCPUIResourceCarousel', () => { + // Store the current test's messages so getMessages can return them + let currentTestMessages: any[] = []; + + beforeEach(() => { + jest.clearAllMocks(); + currentTestMessages = []; + mockUseMessageContext.mockReturnValue({ messageId: 'msg123' } as any); + mockUseMessagesConversation.mockReturnValue({ + conversation: { conversationId: 'conv123' }, + conversationId: 'conv123', + } as any); + mockUseMessagesOperations.mockReturnValue({ + getMessages: () => currentTestMessages, + ask: jest.fn(), + regenerate: jest.fn(), + handleContinue: jest.fn(), + setMessages: jest.fn(), + } as any); + }); + + const renderWithRecoil = (ui: React.ReactNode) => render({ui}); + + describe('multiple resource fetching', () => { + it('should fetch resources by resourceIds across conversation messages', () => { + mockUseMessageContext.mockReturnValue({ messageId: 'msg-current' } as any); + currentTestMessages = [ + { + messageId: 'msg-origin', + attachments: [ + { + type: 'ui_resources', + ui_resources: [ + { + resourceId: 'id-1', + uri: 'ui://test/resource-id1', + mimeType: 'text/html', + text: '

Resource via ID 1

', + }, + { + resourceId: 'id-2', + uri: 'ui://test/resource-id2', + mimeType: 'text/html', + text: '

Resource via ID 2

', + }, + ], + }, + ], + }, + { + messageId: 'msg-current', + attachments: [], + }, + ]; + + renderWithRecoil( + , + ); + + const carousel = screen.getByTestId('ui-resource-carousel'); + expect(carousel).toHaveAttribute('data-resource-count', '2'); + + expect(screen.getByTestId('resource-0')).toHaveAttribute( + 'data-resource-uri', + 'ui://test/resource-id2', + ); + expect(screen.getByTestId('resource-1')).toHaveAttribute( + 'data-resource-uri', + 'ui://test/resource-id1', + ); + }); + }); + + describe('error handling', () => { + it('should return null when no attachments', () => { + currentTestMessages = [ + { + messageId: 'msg123', + attachments: undefined, + }, + ]; + + const { container } = renderWithRecoil( + , + ); + + expect(container.firstChild).toBeNull(); + expect(screen.queryByTestId('ui-resource-carousel')).not.toBeInTheDocument(); + }); + + it('should return null when resources not found', () => { + currentTestMessages = [ + { + messageId: 'msg123', + attachments: [ + { + type: 'ui_resources', + ui_resources: [ + { + resourceId: 'existing-id', + uri: 'ui://test/resource', + mimeType: 'text/html', + text: '

Resource content

', + }, + ], + }, + ], + }, + ]; + + const { container } = renderWithRecoil( + , + ); + + expect(container.firstChild).toBeNull(); + }); + + it('should return null when no ui_resources attachments', () => { + currentTestMessages = [ + { + messageId: 'msg123', + attachments: [ + { + type: 'web_search', + web_search: { results: [] }, + }, + ], + }, + ]; + + const { container } = renderWithRecoil( + , + ); + + expect(container.firstChild).toBeNull(); + }); + }); + + describe('edge cases', () => { + it('should handle empty resourceIds array', () => { + currentTestMessages = [ + { + messageId: 'msg123', + attachments: [ + { + type: 'ui_resources', + ui_resources: [ + { + resourceId: 'test-id', + uri: 'ui://test/resource', + mimeType: 'text/html', + text: '

Resource content

', + }, + ], + }, + ], + }, + ]; + + const { container } = renderWithRecoil( + , + ); + + expect(container.firstChild).toBeNull(); + }); + + it('should handle duplicate resource IDs', () => { + currentTestMessages = [ + { + messageId: 'msg123', + attachments: [ + { + type: 'ui_resources', + ui_resources: [ + { + resourceId: 'id-a', + uri: 'ui://test/resource-a', + mimeType: 'text/html', + text: '

Resource A content

', + }, + { + resourceId: 'id-b', + uri: 'ui://test/resource-b', + mimeType: 'text/html', + text: '

Resource B content

', + }, + ], + }, + ], + }, + ]; + + renderWithRecoil( + , + ); + + const carousel = screen.getByTestId('ui-resource-carousel'); + expect(carousel).toHaveAttribute('data-resource-count', '5'); + + const resources = screen.getAllByTestId(/resource-\d/); + expect(resources).toHaveLength(5); + expect(resources[0]).toHaveAttribute('data-resource-uri', 'ui://test/resource-a'); + expect(resources[1]).toHaveAttribute('data-resource-uri', 'ui://test/resource-a'); + expect(resources[2]).toHaveAttribute('data-resource-uri', 'ui://test/resource-b'); + expect(resources[3]).toHaveAttribute('data-resource-uri', 'ui://test/resource-b'); + expect(resources[4]).toHaveAttribute('data-resource-uri', 'ui://test/resource-a'); + }); + + it('should handle null messages data', () => { + currentTestMessages = []; + + const { container } = renderWithRecoil( + , + ); + + expect(container.firstChild).toBeNull(); + }); + + it('should handle missing conversation', () => { + mockUseMessagesConversation.mockReturnValue({ + conversation: null, + conversationId: null, + } as any); + currentTestMessages = []; + + const { container } = renderWithRecoil( + , + ); + + expect(container.firstChild).toBeNull(); + }); + }); +}); diff --git a/client/src/components/MCPUIResource/__tests__/plugin.test.ts b/client/src/components/MCPUIResource/__tests__/plugin.test.ts new file mode 100644 index 0000000000..a478554918 --- /dev/null +++ b/client/src/components/MCPUIResource/__tests__/plugin.test.ts @@ -0,0 +1,301 @@ +import { mcpUIResourcePlugin, UI_RESOURCE_MARKER } from '../plugin'; +import type { Node } from 'unist'; +import type { UIResourceNode } from '../types'; + +describe('mcpUIResourcePlugin', () => { + const createTextNode = (value: string): UIResourceNode => ({ + type: 'text', + value, + }); + + const createTree = (nodes: UIResourceNode[]): Node => + ({ + type: 'root', + children: nodes, + }) as Node; + + const processTree = (tree: Node) => { + const plugin = mcpUIResourcePlugin(); + plugin(tree); + return tree; + }; + + describe('single resource markers', () => { + it('should replace single UI resource marker with mcp-ui-resource node', () => { + const tree = createTree([createTextNode(`Here is a resource ${UI_RESOURCE_MARKER}{abc123}`)]); + processTree(tree); + + const children = (tree as any).children; + expect(children).toHaveLength(2); + expect(children[0]).toEqual({ type: 'text', value: 'Here is a resource ' }); + expect(children[1].type).toBe('mcp-ui-resource'); + expect(children[1].data.hProperties).toMatchObject({ + resourceId: 'abc123', + }); + }); + + it('should handle multiple single resource markers', () => { + const tree = createTree([ + createTextNode(`First ${UI_RESOURCE_MARKER}{id1} and second ${UI_RESOURCE_MARKER}{id2}`), + ]); + processTree(tree); + + const children = (tree as any).children; + expect(children).toHaveLength(4); + expect(children[0]).toEqual({ type: 'text', value: 'First ' }); + expect(children[1].type).toBe('mcp-ui-resource'); + expect(children[1].data.hProperties).toMatchObject({ resourceId: 'id1' }); + expect(children[2]).toEqual({ type: 'text', value: ' and second ' }); + expect(children[3].type).toBe('mcp-ui-resource'); + expect(children[3].data.hProperties).toMatchObject({ resourceId: 'id2' }); + }); + + it('should handle hex IDs', () => { + const tree = createTree([createTextNode(`Resource ${UI_RESOURCE_MARKER}{a3f2b8c1d4}`)]); + processTree(tree); + + const children = (tree as any).children; + expect(children[1].data.hProperties).toMatchObject({ resourceId: 'a3f2b8c1d4' }); + }); + }); + + describe('carousel markers', () => { + it('should replace carousel marker with mcp-ui-carousel node', () => { + const tree = createTree([createTextNode(`Carousel ${UI_RESOURCE_MARKER}{id1,id2,id3}`)]); + processTree(tree); + + const children = (tree as any).children; + expect(children).toHaveLength(2); + expect(children[0]).toEqual({ type: 'text', value: 'Carousel ' }); + expect(children[1]).toEqual({ + type: 'mcp-ui-carousel', + data: { + hName: 'mcp-ui-carousel', + hProperties: { + resourceIds: ['id1', 'id2', 'id3'], + }, + }, + }); + }); + + it('should handle multiple IDs in carousel', () => { + const tree = createTree([createTextNode(`${UI_RESOURCE_MARKER}{alpha,beta,gamma}`)]); + processTree(tree); + + const children = (tree as any).children; + expect(children[0].data.hProperties.resourceIds).toEqual(['alpha', 'beta', 'gamma']); + }); + }); + + describe('id-based markers', () => { + it('should replace single ID marker with mcp-ui-resource node', () => { + const tree = createTree([createTextNode(`Check this ${UI_RESOURCE_MARKER}{abc123}`)]); + processTree(tree); + + const children = (tree as any).children; + expect(children).toHaveLength(2); + expect(children[0]).toEqual({ type: 'text', value: 'Check this ' }); + expect(children[1].type).toBe('mcp-ui-resource'); + expect(children[1].data.hProperties).toEqual({ + resourceId: 'abc123', + }); + }); + + it('should replace carousel ID marker with mcp-ui-carousel node', () => { + const tree = createTree([createTextNode(`${UI_RESOURCE_MARKER}{one,two,three}`)]); + processTree(tree); + + const children = (tree as any).children; + expect(children).toHaveLength(1); + expect(children[0]).toEqual({ + type: 'mcp-ui-carousel', + data: { + hName: 'mcp-ui-carousel', + hProperties: { + resourceIds: ['one', 'two', 'three'], + }, + }, + }); + }); + + it('should ignore empty IDs', () => { + const tree = createTree([createTextNode(`${UI_RESOURCE_MARKER}{}`)]); + processTree(tree); + + const children = (tree as any).children; + expect(children).toHaveLength(1); + expect(children[0]).toEqual({ type: 'text', value: `${UI_RESOURCE_MARKER}{}` }); + }); + }); + + describe('mixed content', () => { + it('should handle text before and after markers', () => { + const tree = createTree([ + createTextNode( + `Before ${UI_RESOURCE_MARKER}{id1} middle ${UI_RESOURCE_MARKER}{id2,id3} after`, + ), + ]); + processTree(tree); + + const children = (tree as any).children; + expect(children).toHaveLength(5); + expect(children[0].value).toBe('Before '); + expect(children[1].type).toBe('mcp-ui-resource'); + expect(children[2].value).toBe(' middle '); + expect(children[3].type).toBe('mcp-ui-carousel'); + expect(children[4].value).toBe(' after'); + }); + + it('should handle marker at start of text', () => { + const tree = createTree([createTextNode(`${UI_RESOURCE_MARKER}{id1} after`)]); + processTree(tree); + + const children = (tree as any).children; + expect(children).toHaveLength(2); + expect(children[0].type).toBe('mcp-ui-resource'); + expect(children[1].value).toBe(' after'); + }); + + it('should handle marker at end of text', () => { + const tree = createTree([createTextNode(`Before ${UI_RESOURCE_MARKER}{id1}`)]); + processTree(tree); + + const children = (tree as any).children; + expect(children).toHaveLength(2); + expect(children[0].value).toBe('Before '); + expect(children[1].type).toBe('mcp-ui-resource'); + }); + + it('should handle consecutive markers', () => { + const tree = createTree([ + createTextNode(`${UI_RESOURCE_MARKER}{id1}${UI_RESOURCE_MARKER}{id2}`), + ]); + processTree(tree); + + const children = (tree as any).children; + expect(children).toHaveLength(2); + expect(children[0].type).toBe('mcp-ui-resource'); + expect(children[0].data.hProperties).toEqual({ resourceId: 'id1' }); + expect(children[1].type).toBe('mcp-ui-resource'); + expect(children[1].data.hProperties).toEqual({ resourceId: 'id2' }); + }); + }); + + describe('edge cases', () => { + it('should handle empty text nodes', () => { + const tree = createTree([createTextNode('')]); + processTree(tree); + + const children = (tree as any).children; + expect(children).toHaveLength(1); + expect(children[0]).toEqual({ type: 'text', value: '' }); + }); + + it('should handle text without markers', () => { + const tree = createTree([createTextNode('No markers here')]); + processTree(tree); + + const children = (tree as any).children; + expect(children).toHaveLength(1); + expect(children[0]).toEqual({ type: 'text', value: 'No markers here' }); + }); + + it('should handle non-text nodes', () => { + const tree = createTree([{ type: 'paragraph', children: [] }]); + processTree(tree); + + const children = (tree as any).children; + expect(children).toHaveLength(1); + expect(children[0].type).toBe('paragraph'); + }); + + it('should handle nested structures', () => { + const tree = { + type: 'root', + children: [ + { + type: 'paragraph', + children: [createTextNode(`Text with ${UI_RESOURCE_MARKER}{id1}`)], + }, + ], + } as Node; + + processTree(tree); + + const paragraph = (tree as any).children[0]; + const textNodes = paragraph.children; + expect(textNodes).toHaveLength(2); + expect(textNodes[0].value).toBe('Text with '); + expect(textNodes[1].type).toBe('mcp-ui-resource'); + }); + + it('should not process nodes without value property', () => { + const tree = createTree([ + { + type: 'text', + // no value property + } as UIResourceNode, + ]); + processTree(tree); + + const children = (tree as any).children; + expect(children).toHaveLength(1); + expect(children[0].type).toBe('text'); + }); + }); + + describe('pattern validation', () => { + it('should not match marker alone', () => { + const tree = createTree([createTextNode(`${UI_RESOURCE_MARKER}`)]); + processTree(tree); + const children = (tree as any).children; + expect(children).toHaveLength(1); + expect(children[0].type).toBe('text'); + }); + + it('should not match marker without braces', () => { + const tree = createTree([createTextNode(`${UI_RESOURCE_MARKER}abc`)]); + processTree(tree); + const children = (tree as any).children; + expect(children).toHaveLength(1); + expect(children[0].type).toBe('text'); + }); + + it('should not match marker with leading comma', () => { + const tree = createTree([createTextNode(`${UI_RESOURCE_MARKER}{,id}`)]); + processTree(tree); + const children = (tree as any).children; + expect(children).toHaveLength(1); + expect(children[0].type).toBe('text'); + }); + + it('should not match marker without backslash', () => { + const tree = createTree([createTextNode('ui{id}')]); + processTree(tree); + const children = (tree as any).children; + expect(children).toHaveLength(1); + expect(children[0].type).toBe('text'); + }); + + it('should handle valid hex ID patterns', () => { + const validPatterns = [ + { input: `${UI_RESOURCE_MARKER}{abc123}`, id: 'abc123' }, + { input: `${UI_RESOURCE_MARKER}{a3f2b8c1d4}`, id: 'a3f2b8c1d4' }, + { input: `${UI_RESOURCE_MARKER}{1234567890}`, id: '1234567890' }, + { input: `${UI_RESOURCE_MARKER}{abcdef0123}`, id: 'abcdef0123' }, + { input: `${UI_RESOURCE_MARKER}{deadbeef}`, id: 'deadbeef' }, + { input: `${UI_RESOURCE_MARKER}{a1b2c3}`, id: 'a1b2c3' }, + ]; + + validPatterns.forEach(({ input, id }) => { + const tree = createTree([createTextNode(input)]); + processTree(tree); + + const children = (tree as any).children; + expect(children).toHaveLength(1); + expect(children[0].type).toBe('mcp-ui-resource'); + expect(children[0].data.hProperties).toEqual({ resourceId: id }); + }); + }); + }); +}); diff --git a/client/src/components/MCPUIResource/index.ts b/client/src/components/MCPUIResource/index.ts new file mode 100644 index 0000000000..9c42e5f461 --- /dev/null +++ b/client/src/components/MCPUIResource/index.ts @@ -0,0 +1,4 @@ +export { mcpUIResourcePlugin } from './plugin'; +export { MCPUIResource } from './MCPUIResource'; +export { MCPUIResourceCarousel } from './MCPUIResourceCarousel'; +export * from './types'; diff --git a/client/src/components/MCPUIResource/plugin.ts b/client/src/components/MCPUIResource/plugin.ts new file mode 100644 index 0000000000..436e7d862c --- /dev/null +++ b/client/src/components/MCPUIResource/plugin.ts @@ -0,0 +1,91 @@ +import { visit } from 'unist-util-visit'; +import type { Node } from 'unist'; +import type { UIResourceNode } from './types'; + +export const UI_RESOURCE_MARKER = '\\ui'; +// Pattern matches: \ui{id1} or \ui{id1,id2,id3} and captures everything between the braces +export const UI_RESOURCE_PATTERN = /\\ui\{([\w]+(?:,[\w]+)*)\}/g; + +/** + * Process text nodes and replace UI resource markers with components + */ +function processTree(tree: Node) { + visit(tree, 'text', (node, index, parent) => { + const textNode = node as UIResourceNode; + const parentNode = parent as UIResourceNode; + + if (typeof textNode.value !== 'string') return; + + const originalValue = textNode.value; + const segments: Array = []; + + let currentPosition = 0; + UI_RESOURCE_PATTERN.lastIndex = 0; + + let match: RegExpExecArray | null; + while ((match = UI_RESOURCE_PATTERN.exec(originalValue)) !== null) { + const matchIndex = match.index; + const matchText = match[0]; + const idGroup = match[1]; + const idValues = idGroup + .split(',') + .map((value) => value.trim()) + .filter(Boolean); + + if (matchIndex > currentPosition) { + const textBeforeMatch = originalValue.substring(currentPosition, matchIndex); + if (textBeforeMatch) { + segments.push({ type: 'text', value: textBeforeMatch }); + } + } + + if (idValues.length === 1) { + segments.push({ + type: 'mcp-ui-resource', + data: { + hName: 'mcp-ui-resource', + hProperties: { + resourceId: idValues[0], + }, + }, + }); + } else if (idValues.length > 1) { + segments.push({ + type: 'mcp-ui-carousel', + data: { + hName: 'mcp-ui-carousel', + hProperties: { + resourceIds: idValues, + }, + }, + }); + } else { + // Unable to parse marker; keep original text + segments.push({ type: 'text', value: matchText }); + } + + currentPosition = matchIndex + matchText.length; + } + + if (currentPosition < originalValue.length) { + const remainingText = originalValue.substring(currentPosition); + if (remainingText) { + segments.push({ type: 'text', value: remainingText }); + } + } + + if (segments.length > 0 && index !== undefined) { + parentNode.children?.splice(index, 1, ...segments); + return index + segments.length; + } + }); +} + +/** + * Remark plugin for processing MCP UI resource markers + */ +export function mcpUIResourcePlugin() { + return (tree: Node) => { + processTree(tree); + }; +} diff --git a/client/src/components/MCPUIResource/types.ts b/client/src/components/MCPUIResource/types.ts new file mode 100644 index 0000000000..6c559597df --- /dev/null +++ b/client/src/components/MCPUIResource/types.ts @@ -0,0 +1,12 @@ +export interface UIResourceNode { + type: string; + value?: string; + data?: { + hName: string; + hProperties: { + resourceId?: string; + resourceIds?: string[]; + }; + }; + children?: UIResourceNode[]; +} diff --git a/client/src/components/Share/ShareMessagesProvider.tsx b/client/src/components/Share/ShareMessagesProvider.tsx new file mode 100644 index 0000000000..8fec3573c1 --- /dev/null +++ b/client/src/components/Share/ShareMessagesProvider.tsx @@ -0,0 +1,44 @@ +import React, { useMemo } from 'react'; +import type { TMessage } from 'librechat-data-provider'; +import { MessagesViewContext } from '~/Providers/MessagesViewContext'; +import type { MessagesViewContextValue } from '~/Providers/MessagesViewContext'; + +interface ShareMessagesProviderProps { + messages: TMessage[]; + children: React.ReactNode; +} + +/** + * Minimal MessagesViewContext provider for share view. + * Provides conversation data needed by message components. + * Uses the same MessagesViewContext as the main app for compatibility with existing hooks. + * + * Note: conversationId is set to undefined because share view is read-only and doesn't + * need to check Recoil state for in-flight messages during streaming. + */ +export function ShareMessagesProvider({ messages, children }: ShareMessagesProviderProps) { + const contextValue = useMemo( + () => ({ + conversation: { conversationId: 'shared-conversation' }, + conversationId: undefined, + // These are required by the context but not used in share view + ask: () => Promise.resolve(), + regenerate: () => {}, + handleContinue: () => {}, + latestMessage: messages[messages.length - 1] ?? null, + isSubmitting: false, + isSubmittingFamily: false, + abortScroll: false, + setAbortScroll: () => {}, + index: 0, + setLatestMessage: () => {}, + getMessages: () => messages, + setMessages: () => {}, + }), + [messages], + ); + + return ( + {children} + ); +} diff --git a/client/src/components/Share/ShareView.tsx b/client/src/components/Share/ShareView.tsx index f58064acc8..9f75569c6d 100644 --- a/client/src/components/Share/ShareView.tsx +++ b/client/src/components/Share/ShareView.tsx @@ -21,6 +21,7 @@ import { ShareArtifactsContainer } from './ShareArtifacts'; import { useLocalize, useDocumentTitle } from '~/hooks'; import { useGetStartupConfig } from '~/data-provider'; import { ShareContext } from '~/Providers'; +import { ShareMessagesProvider } from './ShareMessagesProvider'; import MessagesView from './MessagesView'; import Footer from '../Chat/Footer'; import { cn } from '~/utils'; @@ -108,7 +109,9 @@ function SharedView() { onLangChange={handleLangChange} settingsLabel={localize('com_nav_settings')} /> - + + + ); } else { diff --git a/client/src/hooks/Messages/useConversationUIResources.ts b/client/src/hooks/Messages/useConversationUIResources.ts new file mode 100644 index 0000000000..2333f64e5f --- /dev/null +++ b/client/src/hooks/Messages/useConversationUIResources.ts @@ -0,0 +1,55 @@ +import { useMemo } from 'react'; +import { useRecoilValue } from 'recoil'; +import { Tools } from 'librechat-data-provider'; +import type { TAttachment, UIResource } from 'librechat-data-provider'; +import { useMessagesOperations } from '~/Providers'; +import store from '~/store'; + +/** + * Hook to collect all UI resources in a conversation, indexed by resource ID. + * This enables cross-turn resource references in the conversation. + * Works in both main app (using React Query cache) and share view (using context messages). + * + * @param conversationId - The ID of the conversation to collect resources from + * @returns A Map of resource IDs to UIResource objects + */ +export function useConversationUIResources( + conversationId: string | undefined, +): Map { + const { getMessages } = useMessagesOperations(); + + const conversationAttachmentsMap = useRecoilValue( + store.conversationAttachmentsSelector(conversationId), + ); + + return useMemo(() => { + const map = new Map(); + + const collectResources = (attachments?: TAttachment[]) => { + attachments + ?.filter((attachment) => attachment?.type === Tools.ui_resources) + .forEach((attachment) => { + const resources = attachment?.[Tools.ui_resources]; + if (Array.isArray(resources)) { + resources.forEach((resource) => { + if (resource?.resourceId) { + map.set(resource.resourceId, resource); + } + }); + } + }); + }; + + // Collect from messages (works in both main app and share view) + getMessages()?.forEach((message) => { + collectResources(message.attachments); + }); + + // Collect from in-flight messages (Recoil state during streaming - only when we have a conversationId) + if (conversationId) { + Object.values(conversationAttachmentsMap).forEach(collectResources); + } + + return map; + }, [conversationId, getMessages, conversationAttachmentsMap]); +} diff --git a/client/src/hooks/index.ts b/client/src/hooks/index.ts index 4df82be15a..f8e23a95e1 100644 --- a/client/src/hooks/index.ts +++ b/client/src/hooks/index.ts @@ -33,5 +33,5 @@ export { default as useDocumentTitle } from './useDocumentTitle'; export { default as useSpeechToText } from './Input/useSpeechToText'; export { default as useTextToSpeech } from './Input/useTextToSpeech'; export { default as useGenerationsByLatest } from './useGenerationsByLatest'; -export { useResourcePermissions } from './useResourcePermissions'; export { default as useLocalizedConfig } from './useLocalizedConfig'; +export { default as useResourcePermissions } from './useResourcePermissions'; diff --git a/client/src/hooks/useResourcePermissions.ts b/client/src/hooks/useResourcePermissions.ts index 12b30bb9ae..a58b735a87 100644 --- a/client/src/hooks/useResourcePermissions.ts +++ b/client/src/hooks/useResourcePermissions.ts @@ -24,3 +24,5 @@ export const useResourcePermissions = (resourceType: ResourceType, resourceId: s permissionBits: data?.permissionBits || 0, }; }; + +export default useResourcePermissions; diff --git a/client/src/locales/en/translation.json b/client/src/locales/en/translation.json index caf8c41de2..905fda8302 100644 --- a/client/src/locales/en/translation.json +++ b/client/src/locales/en/translation.json @@ -1328,6 +1328,8 @@ "com_ui_trust_app": "I trust this application", "com_ui_try_adjusting_search": "Try adjusting your search terms", "com_ui_ui_resources": "UI Resources", + "com_ui_ui_resource_error": "UI Resource Error ({{0}})", + "com_ui_ui_resource_not_found": "UI Resource not found (index: {{0}})", "com_ui_unarchive": "Unarchive", "com_ui_unarchive_error": "Failed to unarchive conversation", "com_ui_unavailable": "Unavailable", diff --git a/client/src/store/misc.ts b/client/src/store/misc.ts index 284382403e..5b649cbe02 100644 --- a/client/src/store/misc.ts +++ b/client/src/store/misc.ts @@ -1,4 +1,4 @@ -import { atom } from 'recoil'; +import { atom, selectorFamily } from 'recoil'; import { TAttachment } from 'librechat-data-provider'; import { atomWithLocalStorage } from './utils'; import { BadgeItem } from '~/common'; @@ -10,6 +10,43 @@ const messageAttachmentsMap = atom>({ default: {}, }); +/** + * Selector to get attachments for a specific conversation. + */ +const conversationAttachmentsSelector = selectorFamily< + Record, + string | undefined +>({ + key: 'conversationAttachments', + get: + (conversationId) => + ({ get }) => { + if (!conversationId) { + return {}; + } + + const attachmentsMap = get(messageAttachmentsMap); + const result: Record = {}; + + // Filter to only include attachments for this conversation + Object.entries(attachmentsMap).forEach(([messageId, attachments]) => { + if (!attachments) { + return; + } + + const relevantAttachments = attachments.filter( + (attachment) => attachment.conversationId === conversationId, + ); + + if (relevantAttachments.length > 0) { + result[messageId] = relevantAttachments; + } + }); + + return result; + }, +}); + const queriesEnabled = atom({ key: 'queriesEnabled', default: true, @@ -30,6 +67,7 @@ const chatBadges = atomWithLocalStorage[]>('chatBadges', [ export default { hideBannerHint, messageAttachmentsMap, + conversationAttachmentsSelector, queriesEnabled, isEditingBadges, chatBadges, diff --git a/client/src/utils/index.ts b/client/src/utils/index.ts index 7eea034ceb..41ed7f9c37 100644 --- a/client/src/utils/index.ts +++ b/client/src/utils/index.ts @@ -124,3 +124,59 @@ export const normalizeLayout = (layout: number[]) => { return normalizedLayout; }; + +export const handleUIAction = async (result: any, ask: any) => { + const supportedTypes = ['intent', 'tool', 'prompt']; + + const { type, payload } = result; + + if (!supportedTypes.includes(type)) { + return; + } + + let messageText = ''; + + if (type === 'intent') { + const { intent, params } = payload; + messageText = `The user clicked a button in an embedded UI Resource, and we got a message of type \`intent\`. +The intent is \`${intent}\` and the params are: + +\`\`\`json +${JSON.stringify(params, null, 2)} +\`\`\` + +Execute the intent that is mentioned in the message using the tools available to you. + `; + } else if (type === 'tool') { + const { toolName, params } = payload; + messageText = `The user clicked a button in an embedded UI Resource, and we got a message of type \`tool\`. +The tool name is \`${toolName}\` and the params are: + +\`\`\`json +${JSON.stringify(params, null, 2)} +\`\`\` + +Execute the tool that is mentioned in the message using the tools available to you. + `; + } else if (type === 'prompt') { + const { prompt } = payload; + messageText = `The user clicked a button in an embedded UI Resource, and we got a message of type \`prompt\`. +The prompt is: + +\`\`\` +${prompt} +\`\`\` + +Execute the intention of the prompt that is mentioned in the message using the tools available to you. + `; + } + + console.log('About to submit message:', messageText); + + try { + await ask({ text: messageText }); + console.log('Message submitted successfully'); + } catch (error) { + console.error('Error submitting message:', error); + } +}; diff --git a/packages/api/src/mcp/__tests__/parsers.test.ts b/packages/api/src/mcp/__tests__/parsers.test.ts index 721ba21d74..dd9a09a0fb 100644 --- a/packages/api/src/mcp/__tests__/parsers.test.ts +++ b/packages/api/src/mcp/__tests__/parsers.test.ts @@ -176,26 +176,22 @@ describe('formatToolContent', () => { }; const [content, artifacts] = formatToolContent(result, 'openai'); - expect(content).toEqual([ - { - type: 'text', - text: - 'Resource Text: {"items": []}\n' + - 'Resource URI: ui://carousel\n' + - 'Resource MIME Type: application/json', - }, - ]); - expect(artifacts).toEqual({ - ui_resources: { - data: [ - { - uri: 'ui://carousel', - mimeType: 'application/json', - text: '{"items": []}', - }, - ], - }, + expect(Array.isArray(content)).toBe(true); + const textContent = Array.isArray(content) ? content[0] : { text: '' }; + expect(textContent).toMatchObject({ type: 'text' }); + expect(textContent.text).toContain('UI Resource ID:'); + expect(textContent.text).toContain('UI Resource Marker: \\ui{'); + expect(textContent.text).toContain('Resource URI: ui://carousel'); + expect(textContent.text).toContain('Resource MIME Type: application/json'); + + const uiResourceArtifact = artifacts?.ui_resources?.data?.[0]; + expect(uiResourceArtifact).toBeTruthy(); + expect(uiResourceArtifact).toMatchObject({ + uri: 'ui://carousel', + mimeType: 'application/json', + text: '{"items": []}', }); + expect(uiResourceArtifact?.resourceId).toEqual(expect.any(String)); }); it('should handle regular resources', () => { @@ -271,28 +267,22 @@ describe('formatToolContent', () => { }; const [content, artifacts] = formatToolContent(result, 'openai'); - expect(content).toEqual([ - { - type: 'text', - text: - 'Some text\n\n' + - 'Resource Text: {"label": "Click me"}\n' + - 'Resource URI: ui://button\n' + - 'Resource MIME Type: application/json\n\n' + - 'Resource URI: file://data.csv', - }, - ]); - expect(artifacts).toEqual({ - ui_resources: { - data: [ - { - uri: 'ui://button', - mimeType: 'application/json', - text: '{"label": "Click me"}', - }, - ], - }, + expect(Array.isArray(content)).toBe(true); + const textEntry = Array.isArray(content) ? content[0] : { text: '' }; + expect(textEntry).toMatchObject({ type: 'text' }); + expect(textEntry.text).toContain('Some text'); + expect(textEntry.text).toContain('UI Resource Marker: \\ui{'); + expect(textEntry.text).toContain('Resource URI: ui://button'); + expect(textEntry.text).toContain('Resource MIME Type: application/json'); + expect(textEntry.text).toContain('Resource URI: file://data.csv'); + + const uiResource = artifacts?.ui_resources?.data?.[0]; + expect(uiResource).toMatchObject({ + uri: 'ui://button', + mimeType: 'application/json', + text: '{"label": "Click me"}', }); + expect(uiResource?.resourceId).toEqual(expect.any(String)); }); it('should handle both images and UI resources in artifacts', () => { @@ -312,19 +302,14 @@ describe('formatToolContent', () => { }; const [content, artifacts] = formatToolContent(result, 'openai'); - expect(content).toEqual([ - { - type: 'text', - text: 'Content with multimedia', - }, - { - type: 'text', - text: - 'Resource Text: {"type": "line"}\n' + - 'Resource URI: ui://graph\n' + - 'Resource MIME Type: application/json', - }, - ]); + expect(Array.isArray(content)).toBe(true); + if (Array.isArray(content)) { + expect(content[0]).toMatchObject({ type: 'text', text: 'Content with multimedia' }); + expect(content[1].type).toBe('text'); + expect(content[1].text).toContain('UI Resource Marker: \\ui{'); + expect(content[1].text).toContain('Resource URI: ui://graph'); + expect(content[1].text).toContain('Resource MIME Type: application/json'); + } expect(artifacts).toEqual({ content: [ { @@ -338,6 +323,7 @@ describe('formatToolContent', () => { uri: 'ui://graph', mimeType: 'application/json', text: '{"type": "line"}', + resourceId: expect.any(String), }, ], }, @@ -393,20 +379,21 @@ describe('formatToolContent', () => { }; const [content, artifacts] = formatToolContent(result, 'anthropic'); - expect(content).toEqual([ - { type: 'text', text: 'Introduction' }, - { - type: 'text', - text: - 'Middle section\n\n' + - 'Resource Text: {"type": "bar"}\n' + - 'Resource URI: ui://chart\n' + - 'Resource MIME Type: application/json\n\n' + - 'Resource URI: https://api.example.com/data', - }, - { type: 'text', text: 'Conclusion' }, - ]); - expect(artifacts).toEqual({ + expect(Array.isArray(content)).toBe(true); + if (Array.isArray(content)) { + expect(content[0]).toEqual({ type: 'text', text: 'Introduction' }); + expect(content[1].type).toBe('text'); + expect(content[1].text).toContain('Middle section'); + expect(content[1].text).toContain('UI Resource ID:'); + expect(content[1].text).toContain('UI Resource Marker: \\ui{'); + expect(content[1].text).toContain('Resource URI: ui://chart'); + expect(content[1].text).toContain('Resource MIME Type: application/json'); + expect(content[1].text).toContain('Resource URI: https://api.example.com/data'); + expect(content[2].type).toBe('text'); + expect(content[2].text).toContain('Conclusion'); + expect(content[2].text).toContain('UI Resource Markers Available:'); + } + expect(artifacts).toMatchObject({ content: [ { type: 'image_url', @@ -423,6 +410,7 @@ describe('formatToolContent', () => { uri: 'ui://chart', mimeType: 'application/json', text: '{"type": "bar"}', + resourceId: expect.any(String), }, ], }, diff --git a/packages/api/src/mcp/parsers.ts b/packages/api/src/mcp/parsers.ts index 1b8a26181b..65f8e44a3d 100644 --- a/packages/api/src/mcp/parsers.ts +++ b/packages/api/src/mcp/parsers.ts @@ -1,7 +1,12 @@ +import crypto from 'node:crypto'; import { Tools } from 'librechat-data-provider'; import type { UIResource } from 'librechat-data-provider'; import type * as t from './types'; +function generateResourceId(text: string): string { + return crypto.createHash('sha256').update(text).digest('hex').substring(0, 10); +} + const RECOGNIZED_PROVIDERS = new Set([ 'google', 'anthropic', @@ -132,21 +137,34 @@ export function formatToolContent( }, resource: (item) => { - if (item.resource.uri.startsWith('ui://')) { - uiResources.push(item.resource as UIResource); - } + const isUiResource = item.resource.uri.startsWith('ui://'); + const resourceText: string[] = []; - const resourceText = []; - if ('text' in item.resource && item.resource.text != null && item.resource.text) { + if (isUiResource) { + const resourceTextValue = 'text' in item.resource ? item.resource.text : undefined; + const contentToHash = resourceTextValue || item.resource.uri || ''; + const resourceId = generateResourceId(contentToHash); + const uiResource: UIResource = { + ...item.resource, + resourceId, + }; + uiResources.push(uiResource); + resourceText.push(`UI Resource ID: ${resourceId}`); + resourceText.push(`UI Resource Marker: \\ui{${resourceId}}`); + } else if ('text' in item.resource && item.resource.text != null && item.resource.text) { resourceText.push(`Resource Text: ${item.resource.text}`); } + if (item.resource.uri.length) { resourceText.push(`Resource URI: ${item.resource.uri}`); } if (item.resource.mimeType != null && item.resource.mimeType) { resourceText.push(`Resource MIME Type: ${item.resource.mimeType}`); } - currentTextBlock += (currentTextBlock ? '\n\n' : '') + resourceText.join('\n'); + + if (resourceText.length) { + currentTextBlock += (currentTextBlock ? '\n\n' : '') + resourceText.join('\n'); + } }, }; @@ -160,15 +178,34 @@ export function formatToolContent( } } + if (uiResources.length > 0) { + const uiInstructions = ` + +UI Resource Markers Available: +- Each resource above includes a stable ID and a marker hint like \`\\ui{abc123}\` +- You should usually introduce what you're showing before placing the marker +- For a single resource: \\ui{resource-id} +- For multiple resources shown separately: \\ui{resource-id-a} \\ui{resource-id-b} +- For multiple resources in a carousel: \\ui{resource-id-a,resource-id-b,resource-id-c} +- The UI will be rendered inline where you place the marker +- Format: \\ui{resource-id} or \\ui{id1,id2,id3} using the IDs provided above`; + + currentTextBlock += uiInstructions; + } + if (CONTENT_ARRAY_PROVIDERS.has(provider) && currentTextBlock) { formattedContent.push({ type: 'text', text: currentTextBlock }); } let artifacts: t.Artifacts = undefined; - if (imageUrls.length || uiResources.length) { + if (imageUrls.length) { + artifacts = { content: imageUrls }; + } + + if (uiResources.length) { artifacts = { - ...(imageUrls.length && { content: imageUrls }), - ...(uiResources.length && { [Tools.ui_resources]: { data: uiResources } }), + ...artifacts, + [Tools.ui_resources]: { data: uiResources }, }; } diff --git a/packages/api/src/mcp/types/index.ts b/packages/api/src/mcp/types/index.ts index 594c628ff4..63a0153cad 100644 --- a/packages/api/src/mcp/types/index.ts +++ b/packages/api/src/mcp/types/index.ts @@ -90,11 +90,7 @@ export type Provider = export type FormattedContent = | { type: 'text'; - metadata?: { - type: string; - data: UIResource[]; - }; - text?: string; + text: string; } | { type: 'image'; diff --git a/packages/api/src/mcp/utils.ts b/packages/api/src/mcp/utils.ts index fbd8ed7c95..fddebb9db3 100644 --- a/packages/api/src/mcp/utils.ts +++ b/packages/api/src/mcp/utils.ts @@ -12,7 +12,7 @@ export function normalizeServerName(serverName: string): string { } /** Replace non-matching characters with underscores. - This preserves the general structure while ensuring compatibility. + This preserves the general structure while ensuring compatibility. Trims leading/trailing underscores */ const normalized = serverName.replace(/[^a-zA-Z0-9_.-]/g, '_').replace(/^_+|_+$/g, ''); diff --git a/packages/data-provider/src/schemas.ts b/packages/data-provider/src/schemas.ts index ff60fc9881..7dabc549db 100644 --- a/packages/data-provider/src/schemas.ts +++ b/packages/data-provider/src/schemas.ts @@ -583,9 +583,8 @@ export type MemoryArtifact = { }; export type UIResource = { - type?: string; - data?: unknown; - uri?: string; + resourceId: string; + uri: string; mimeType?: string; text?: string; [key: string]: unknown;