mirror of
https://github.com/danny-avila/LibreChat.git
synced 2025-12-21 02:40:14 +01:00
💻 feat: Deeper MCP UI integration in the Chat UI (#9669)
* 💻 feat: deeper MCP UI integration in the chat UI using plugins --------- Co-authored-by: Samuel Path <samuel.path@shopify.com> Co-authored-by: Pierre-Luc Godin <pierreluc.godin@shopify.com> * 💻 refactor: Migrate MCP UI resources from index-based to ID-based referencing - Replace index-based resource markers with stable resource IDs - Update plugin to parse \ui{resourceId} format instead of \ui0 - Refactor components to use useMessagesOperations instead of useSubmitMessage - Add ShareMessagesProvider for UI resources in share view - Add useConversationUIResources hook for cross-turn resource lookups - Update parsers to generate resource IDs from content hashes - Update all tests to use resource IDs instead of indices - Add sandbox permissions for iframe popups - Remove deprecated MCP tool context instructions --------- Co-authored-by: Pierre-Luc Godin <pierreluc.godin@shopify.com>
This commit is contained in:
parent
4a0fbb07bc
commit
304bba853c
27 changed files with 1545 additions and 122 deletions
|
|
@ -9,6 +9,11 @@ import rehypeHighlight from 'rehype-highlight';
|
|||
import remarkDirective from 'remark-directive';
|
||||
import type { Pluggable } from 'unified';
|
||||
import { Citation, CompositeCitation, HighlightedText } from '~/components/Web/Citation';
|
||||
import {
|
||||
mcpUIResourcePlugin,
|
||||
MCPUIResource,
|
||||
MCPUIResourceCarousel,
|
||||
} from '~/components/MCPUIResource';
|
||||
import { Artifact, artifactPlugin } from '~/components/Artifacts/Artifact';
|
||||
import { ArtifactProvider, CodeBlockProvider } from '~/Providers';
|
||||
import MarkdownErrorBoundary from './MarkdownErrorBoundary';
|
||||
|
|
@ -55,6 +60,7 @@ const Markdown = memo(({ content = '', isLatestMessage }: TContentProps) => {
|
|||
artifactPlugin,
|
||||
[remarkMath, { singleDollarTextMath: false }],
|
||||
unicodeCitation,
|
||||
mcpUIResourcePlugin,
|
||||
];
|
||||
|
||||
if (isInitializing) {
|
||||
|
|
@ -86,6 +92,8 @@ const Markdown = memo(({ content = '', isLatestMessage }: TContentProps) => {
|
|||
citation: Citation,
|
||||
'highlighted-text': HighlightedText,
|
||||
'composite-citation': CompositeCitation,
|
||||
'mcp-ui-resource': MCPUIResource,
|
||||
'mcp-ui-carousel': MCPUIResourceCarousel,
|
||||
} as {
|
||||
[nodeType: string]: React.ElementType;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import React, { useState } from 'react';
|
||||
import { UIResourceRenderer } from '@mcp-ui/client';
|
||||
import type { UIResource } from 'librechat-data-provider';
|
||||
import { useMessagesOperations } from '~/Providers';
|
||||
import { handleUIAction } from '~/utils';
|
||||
|
||||
interface UIResourceCarouselProps {
|
||||
uiResources: UIResource[];
|
||||
|
|
@ -11,6 +13,7 @@ const UIResourceCarousel: React.FC<UIResourceCarouselProps> = React.memo(({ uiRe
|
|||
const [showRightArrow, setShowRightArrow] = useState(true);
|
||||
const [isContainerHovered, setIsContainerHovered] = useState(false);
|
||||
const scrollContainerRef = React.useRef<HTMLDivElement>(null);
|
||||
const { ask } = useMessagesOperations();
|
||||
|
||||
const handleScroll = React.useCallback(() => {
|
||||
if (!scrollContainerRef.current) return;
|
||||
|
|
@ -111,9 +114,7 @@ const UIResourceCarousel: React.FC<UIResourceCarouselProps> = React.memo(({ uiRe
|
|||
mimeType: uiResource.mimeType,
|
||||
text: uiResource.text,
|
||||
}}
|
||||
onUIAction={async (result) => {
|
||||
console.log('Action:', result);
|
||||
}}
|
||||
onUIAction={async (result) => handleUIAction(result, ask)}
|
||||
htmlProps={{
|
||||
autoResizeIframe: { width: true, height: true },
|
||||
}}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,106 @@
|
|||
import React from 'react';
|
||||
import { render, screen } from '@testing-library/react';
|
||||
import Markdown from '../Markdown';
|
||||
import { RecoilRoot } from 'recoil';
|
||||
import { UI_RESOURCE_MARKER } from '~/components/MCPUIResource/plugin';
|
||||
import { useMessageContext, useMessagesConversation, useMessagesOperations } from '~/Providers';
|
||||
import { useGetMessagesByConvoId } from '~/data-provider';
|
||||
import { useLocalize } from '~/hooks';
|
||||
|
||||
// Mocks for hooks used by MCPUIResource when rendered inside Markdown.
|
||||
// Keep Provider components intact while mocking only the hooks we use.
|
||||
jest.mock('~/Providers', () => ({
|
||||
...jest.requireActual('~/Providers'),
|
||||
useMessageContext: jest.fn(),
|
||||
useMessagesConversation: jest.fn(),
|
||||
useMessagesOperations: jest.fn(),
|
||||
}));
|
||||
jest.mock('~/data-provider');
|
||||
jest.mock('~/hooks');
|
||||
|
||||
// Mock @mcp-ui/client to render identifiable elements for assertions
|
||||
jest.mock('@mcp-ui/client', () => ({
|
||||
UIResourceRenderer: ({ resource }: any) => (
|
||||
<div data-testid="ui-resource-renderer" data-resource-uri={resource?.uri} />
|
||||
),
|
||||
}));
|
||||
|
||||
const mockUseMessageContext = useMessageContext as jest.MockedFunction<typeof useMessageContext>;
|
||||
const mockUseMessagesConversation = useMessagesConversation as jest.MockedFunction<
|
||||
typeof useMessagesConversation
|
||||
>;
|
||||
const mockUseMessagesOperations = useMessagesOperations as jest.MockedFunction<
|
||||
typeof useMessagesOperations
|
||||
>;
|
||||
const mockUseGetMessagesByConvoId = useGetMessagesByConvoId as jest.MockedFunction<
|
||||
typeof useGetMessagesByConvoId
|
||||
>;
|
||||
const mockUseLocalize = useLocalize as jest.MockedFunction<typeof useLocalize>;
|
||||
|
||||
describe('Markdown with MCP UI markers (resource IDs)', () => {
|
||||
let currentTestMessages: any[] = [];
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
currentTestMessages = [];
|
||||
|
||||
mockUseMessageContext.mockReturnValue({ messageId: 'msg-weather' } as any);
|
||||
mockUseMessagesConversation.mockReturnValue({
|
||||
conversation: { conversationId: 'conv1' },
|
||||
conversationId: 'conv1',
|
||||
} as any);
|
||||
mockUseMessagesOperations.mockReturnValue({
|
||||
ask: jest.fn(),
|
||||
getMessages: () => currentTestMessages,
|
||||
} as any);
|
||||
mockUseLocalize.mockReturnValue(((key: string) => key) as any);
|
||||
});
|
||||
|
||||
it('renders two UIResourceRenderer components for markers with resource IDs across separate attachments', () => {
|
||||
// Two tool responses, each produced one ui_resources attachment
|
||||
const paris = {
|
||||
resourceId: 'abc123',
|
||||
uri: 'ui://weather/paris',
|
||||
mimeType: 'text/html',
|
||||
text: '<div>Paris Weather</div>',
|
||||
};
|
||||
const nyc = {
|
||||
resourceId: 'def456',
|
||||
uri: 'ui://weather/nyc',
|
||||
mimeType: 'text/html',
|
||||
text: '<div>NYC Weather</div>',
|
||||
};
|
||||
|
||||
currentTestMessages = [
|
||||
{
|
||||
messageId: 'msg-weather',
|
||||
attachments: [
|
||||
{ type: 'ui_resources', ui_resources: [paris] },
|
||||
{ type: 'ui_resources', ui_resources: [nyc] },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
mockUseGetMessagesByConvoId.mockReturnValue({ data: currentTestMessages } as any);
|
||||
|
||||
const content = [
|
||||
'Here are the current weather conditions for both Paris and New York:',
|
||||
'',
|
||||
'- Paris: Slight rain, 53°F, humidity 76%, wind 9 mph.',
|
||||
'- New York: Clear sky, 63°F, humidity 91%, wind 6 mph.',
|
||||
'',
|
||||
`Browse these weather cards for more details ${UI_RESOURCE_MARKER}{abc123} ${UI_RESOURCE_MARKER}{def456}`,
|
||||
].join('\n');
|
||||
|
||||
render(
|
||||
<RecoilRoot>
|
||||
<Markdown content={content} isLatestMessage={false} />
|
||||
</RecoilRoot>,
|
||||
);
|
||||
|
||||
const renderers = screen.getAllByTestId('ui-resource-renderer');
|
||||
expect(renderers).toHaveLength(2);
|
||||
expect(renderers[0]).toHaveAttribute('data-resource-uri', 'ui://weather/paris');
|
||||
expect(renderers[1]).toHaveAttribute('data-resource-uri', 'ui://weather/nyc');
|
||||
});
|
||||
});
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
import React from 'react';
|
||||
import '@testing-library/jest-dom';
|
||||
import { render, screen, fireEvent, waitFor } from '@testing-library/react';
|
||||
import { fireEvent, render, screen, waitFor } from '@testing-library/react';
|
||||
import type { UIResource } from 'librechat-data-provider';
|
||||
import UIResourceCarousel from '~/components/Chat/Messages/Content/UIResourceCarousel';
|
||||
import { handleUIAction } from '~/utils';
|
||||
|
||||
// Mock the UIResourceRenderer component
|
||||
jest.mock('@mcp-ui/client', () => ({
|
||||
|
|
@ -13,6 +13,19 @@ jest.mock('@mcp-ui/client', () => ({
|
|||
),
|
||||
}));
|
||||
|
||||
// Mock useMessagesOperations hook
|
||||
const mockAsk = jest.fn();
|
||||
jest.mock('~/Providers', () => ({
|
||||
useMessagesOperations: () => ({
|
||||
ask: mockAsk,
|
||||
}),
|
||||
}));
|
||||
|
||||
// Mock handleUIAction utility
|
||||
jest.mock('~/utils', () => ({
|
||||
handleUIAction: jest.fn(),
|
||||
}));
|
||||
|
||||
// Mock scrollTo
|
||||
const mockScrollTo = jest.fn();
|
||||
Object.defineProperty(HTMLElement.prototype, 'scrollTo', {
|
||||
|
|
@ -29,8 +42,12 @@ describe('UIResourceCarousel', () => {
|
|||
{ uri: 'resource5', mimeType: 'text/html', text: 'Resource 5' },
|
||||
];
|
||||
|
||||
const mockHandleUIAction = handleUIAction as jest.MockedFunction<typeof handleUIAction>;
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
mockAsk.mockClear();
|
||||
mockHandleUIAction.mockClear();
|
||||
// Reset scroll properties
|
||||
Object.defineProperty(HTMLElement.prototype, 'scrollLeft', {
|
||||
configurable: true,
|
||||
|
|
@ -141,18 +158,48 @@ describe('UIResourceCarousel', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('handles UIResource actions', async () => {
|
||||
const consoleSpy = jest.spyOn(console, 'log').mockImplementation();
|
||||
it('handles UIResource actions using handleUIAction', async () => {
|
||||
render(<UIResourceCarousel uiResources={mockUIResources.slice(0, 1)} />);
|
||||
|
||||
const renderer = screen.getByTestId('ui-resource-renderer');
|
||||
fireEvent.click(renderer);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(consoleSpy).toHaveBeenCalledWith('Action:', { action: 'test' });
|
||||
expect(mockHandleUIAction).toHaveBeenCalledWith({ action: 'test' }, mockAsk);
|
||||
});
|
||||
});
|
||||
|
||||
it('calls handleUIAction with correct parameters for multiple resources', async () => {
|
||||
render(<UIResourceCarousel uiResources={mockUIResources.slice(0, 3)} />);
|
||||
|
||||
const renderers = screen.getAllByTestId('ui-resource-renderer');
|
||||
|
||||
// Click the second renderer
|
||||
fireEvent.click(renderers[1]);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(mockHandleUIAction).toHaveBeenCalledWith({ action: 'test' }, mockAsk);
|
||||
expect(mockHandleUIAction).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
consoleSpy.mockRestore();
|
||||
// Click the third renderer
|
||||
fireEvent.click(renderers[2]);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(mockHandleUIAction).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
|
||||
it('passes correct ask function to handleUIAction', async () => {
|
||||
render(<UIResourceCarousel uiResources={mockUIResources.slice(0, 1)} />);
|
||||
|
||||
const renderer = screen.getByTestId('ui-resource-renderer');
|
||||
fireEvent.click(renderer);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(mockHandleUIAction).toHaveBeenCalledWith({ action: 'test' }, mockAsk);
|
||||
expect(mockHandleUIAction).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
it('applies correct dimensions to resource containers', () => {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue