mirror of
https://github.com/danny-avila/LibreChat.git
synced 2025-12-20 10:20:15 +01:00
💻 feat: Deeper MCP UI integration in the Chat UI (#9669)
* 💻 feat: deeper MCP UI integration in the chat UI using plugins --------- Co-authored-by: Samuel Path <samuel.path@shopify.com> Co-authored-by: Pierre-Luc Godin <pierreluc.godin@shopify.com> * 💻 refactor: Migrate MCP UI resources from index-based to ID-based referencing - Replace index-based resource markers with stable resource IDs - Update plugin to parse \ui{resourceId} format instead of \ui0 - Refactor components to use useMessagesOperations instead of useSubmitMessage - Add ShareMessagesProvider for UI resources in share view - Add useConversationUIResources hook for cross-turn resource lookups - Update parsers to generate resource IDs from content hashes - Update all tests to use resource IDs instead of indices - Add sandbox permissions for iframe popups - Remove deprecated MCP tool context instructions --------- Co-authored-by: Pierre-Luc Godin <pierreluc.godin@shopify.com>
This commit is contained in:
Samuel Path
Danny Avila
parent
4a0fbb07bc
commit
304bba853c
27 changed files with 1545 additions and 122 deletions
|
|
@ -176,26 +176,22 @@ describe('formatToolContent', () => {
|
|||
};
|
||||
|
||||
const [content, artifacts] = formatToolContent(result, 'openai');
|
||||
expect(content).toEqual([
|
||||
{
|
||||
type: 'text',
|
||||
text:
|
||||
'Resource Text: {"items": []}\n' +
|
||||
'Resource URI: ui://carousel\n' +
|
||||
'Resource MIME Type: application/json',
|
||||
},
|
||||
]);
|
||||
expect(artifacts).toEqual({
|
||||
ui_resources: {
|
||||
data: [
|
||||
{
|
||||
uri: 'ui://carousel',
|
||||
mimeType: 'application/json',
|
||||
text: '{"items": []}',
|
||||
},
|
||||
],
|
||||
},
|
||||
expect(Array.isArray(content)).toBe(true);
|
||||
const textContent = Array.isArray(content) ? content[0] : { text: '' };
|
||||
expect(textContent).toMatchObject({ type: 'text' });
|
||||
expect(textContent.text).toContain('UI Resource ID:');
|
||||
expect(textContent.text).toContain('UI Resource Marker: \\ui{');
|
||||
expect(textContent.text).toContain('Resource URI: ui://carousel');
|
||||
expect(textContent.text).toContain('Resource MIME Type: application/json');
|
||||
|
||||
const uiResourceArtifact = artifacts?.ui_resources?.data?.[0];
|
||||
expect(uiResourceArtifact).toBeTruthy();
|
||||
expect(uiResourceArtifact).toMatchObject({
|
||||
uri: 'ui://carousel',
|
||||
mimeType: 'application/json',
|
||||
text: '{"items": []}',
|
||||
});
|
||||
expect(uiResourceArtifact?.resourceId).toEqual(expect.any(String));
|
||||
});
|
||||
|
||||
it('should handle regular resources', () => {
|
||||
|
|
@ -271,28 +267,22 @@ describe('formatToolContent', () => {
|
|||
};
|
||||
|
||||
const [content, artifacts] = formatToolContent(result, 'openai');
|
||||
expect(content).toEqual([
|
||||
{
|
||||
type: 'text',
|
||||
text:
|
||||
'Some text\n\n' +
|
||||
'Resource Text: {"label": "Click me"}\n' +
|
||||
'Resource URI: ui://button\n' +
|
||||
'Resource MIME Type: application/json\n\n' +
|
||||
'Resource URI: file://data.csv',
|
||||
},
|
||||
]);
|
||||
expect(artifacts).toEqual({
|
||||
ui_resources: {
|
||||
data: [
|
||||
{
|
||||
uri: 'ui://button',
|
||||
mimeType: 'application/json',
|
||||
text: '{"label": "Click me"}',
|
||||
},
|
||||
],
|
||||
},
|
||||
expect(Array.isArray(content)).toBe(true);
|
||||
const textEntry = Array.isArray(content) ? content[0] : { text: '' };
|
||||
expect(textEntry).toMatchObject({ type: 'text' });
|
||||
expect(textEntry.text).toContain('Some text');
|
||||
expect(textEntry.text).toContain('UI Resource Marker: \\ui{');
|
||||
expect(textEntry.text).toContain('Resource URI: ui://button');
|
||||
expect(textEntry.text).toContain('Resource MIME Type: application/json');
|
||||
expect(textEntry.text).toContain('Resource URI: file://data.csv');
|
||||
|
||||
const uiResource = artifacts?.ui_resources?.data?.[0];
|
||||
expect(uiResource).toMatchObject({
|
||||
uri: 'ui://button',
|
||||
mimeType: 'application/json',
|
||||
text: '{"label": "Click me"}',
|
||||
});
|
||||
expect(uiResource?.resourceId).toEqual(expect.any(String));
|
||||
});
|
||||
|
||||
it('should handle both images and UI resources in artifacts', () => {
|
||||
|
|
@ -312,19 +302,14 @@ describe('formatToolContent', () => {
|
|||
};
|
||||
|
||||
const [content, artifacts] = formatToolContent(result, 'openai');
|
||||
expect(content).toEqual([
|
||||
{
|
||||
type: 'text',
|
||||
text: 'Content with multimedia',
|
||||
},
|
||||
{
|
||||
type: 'text',
|
||||
text:
|
||||
'Resource Text: {"type": "line"}\n' +
|
||||
'Resource URI: ui://graph\n' +
|
||||
'Resource MIME Type: application/json',
|
||||
},
|
||||
]);
|
||||
expect(Array.isArray(content)).toBe(true);
|
||||
if (Array.isArray(content)) {
|
||||
expect(content[0]).toMatchObject({ type: 'text', text: 'Content with multimedia' });
|
||||
expect(content[1].type).toBe('text');
|
||||
expect(content[1].text).toContain('UI Resource Marker: \\ui{');
|
||||
expect(content[1].text).toContain('Resource URI: ui://graph');
|
||||
expect(content[1].text).toContain('Resource MIME Type: application/json');
|
||||
}
|
||||
expect(artifacts).toEqual({
|
||||
content: [
|
||||
{
|
||||
|
|
@ -338,6 +323,7 @@ describe('formatToolContent', () => {
|
|||
uri: 'ui://graph',
|
||||
mimeType: 'application/json',
|
||||
text: '{"type": "line"}',
|
||||
resourceId: expect.any(String),
|
||||
},
|
||||
],
|
||||
},
|
||||
|
|
@ -393,20 +379,21 @@ describe('formatToolContent', () => {
|
|||
};
|
||||
|
||||
const [content, artifacts] = formatToolContent(result, 'anthropic');
|
||||
expect(content).toEqual([
|
||||
{ type: 'text', text: 'Introduction' },
|
||||
{
|
||||
type: 'text',
|
||||
text:
|
||||
'Middle section\n\n' +
|
||||
'Resource Text: {"type": "bar"}\n' +
|
||||
'Resource URI: ui://chart\n' +
|
||||
'Resource MIME Type: application/json\n\n' +
|
||||
'Resource URI: https://api.example.com/data',
|
||||
},
|
||||
{ type: 'text', text: 'Conclusion' },
|
||||
]);
|
||||
expect(artifacts).toEqual({
|
||||
expect(Array.isArray(content)).toBe(true);
|
||||
if (Array.isArray(content)) {
|
||||
expect(content[0]).toEqual({ type: 'text', text: 'Introduction' });
|
||||
expect(content[1].type).toBe('text');
|
||||
expect(content[1].text).toContain('Middle section');
|
||||
expect(content[1].text).toContain('UI Resource ID:');
|
||||
expect(content[1].text).toContain('UI Resource Marker: \\ui{');
|
||||
expect(content[1].text).toContain('Resource URI: ui://chart');
|
||||
expect(content[1].text).toContain('Resource MIME Type: application/json');
|
||||
expect(content[1].text).toContain('Resource URI: https://api.example.com/data');
|
||||
expect(content[2].type).toBe('text');
|
||||
expect(content[2].text).toContain('Conclusion');
|
||||
expect(content[2].text).toContain('UI Resource Markers Available:');
|
||||
}
|
||||
expect(artifacts).toMatchObject({
|
||||
content: [
|
||||
{
|
||||
type: 'image_url',
|
||||
|
|
@ -423,6 +410,7 @@ describe('formatToolContent', () => {
|
|||
uri: 'ui://chart',
|
||||
mimeType: 'application/json',
|
||||
text: '{"type": "bar"}',
|
||||
resourceId: expect.any(String),
|
||||
},
|
||||
],
|
||||
},
|
||||
|
|
|
|||
|
|
@ -1,7 +1,12 @@
|
|||
import crypto from 'node:crypto';
|
||||
import { Tools } from 'librechat-data-provider';
|
||||
import type { UIResource } from 'librechat-data-provider';
|
||||
import type * as t from './types';
|
||||
|
||||
function generateResourceId(text: string): string {
|
||||
return crypto.createHash('sha256').update(text).digest('hex').substring(0, 10);
|
||||
}
|
||||
|
||||
const RECOGNIZED_PROVIDERS = new Set([
|
||||
'google',
|
||||
'anthropic',
|
||||
|
|
@ -132,21 +137,34 @@ export function formatToolContent(
|
|||
},
|
||||
|
||||
resource: (item) => {
|
||||
if (item.resource.uri.startsWith('ui://')) {
|
||||
uiResources.push(item.resource as UIResource);
|
||||
}
|
||||
const isUiResource = item.resource.uri.startsWith('ui://');
|
||||
const resourceText: string[] = [];
|
||||
|
||||
const resourceText = [];
|
||||
if ('text' in item.resource && item.resource.text != null && item.resource.text) {
|
||||
if (isUiResource) {
|
||||
const resourceTextValue = 'text' in item.resource ? item.resource.text : undefined;
|
||||
const contentToHash = resourceTextValue || item.resource.uri || '';
|
||||
const resourceId = generateResourceId(contentToHash);
|
||||
const uiResource: UIResource = {
|
||||
...item.resource,
|
||||
resourceId,
|
||||
};
|
||||
uiResources.push(uiResource);
|
||||
resourceText.push(`UI Resource ID: ${resourceId}`);
|
||||
resourceText.push(`UI Resource Marker: \\ui{${resourceId}}`);
|
||||
} else if ('text' in item.resource && item.resource.text != null && item.resource.text) {
|
||||
resourceText.push(`Resource Text: ${item.resource.text}`);
|
||||
}
|
||||
|
||||
if (item.resource.uri.length) {
|
||||
resourceText.push(`Resource URI: ${item.resource.uri}`);
|
||||
}
|
||||
if (item.resource.mimeType != null && item.resource.mimeType) {
|
||||
resourceText.push(`Resource MIME Type: ${item.resource.mimeType}`);
|
||||
}
|
||||
currentTextBlock += (currentTextBlock ? '\n\n' : '') + resourceText.join('\n');
|
||||
|
||||
if (resourceText.length) {
|
||||
currentTextBlock += (currentTextBlock ? '\n\n' : '') + resourceText.join('\n');
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
|
|
@ -160,15 +178,34 @@ export function formatToolContent(
|
|||
}
|
||||
}
|
||||
|
||||
if (uiResources.length > 0) {
|
||||
const uiInstructions = `
|
||||
|
||||
UI Resource Markers Available:
|
||||
- Each resource above includes a stable ID and a marker hint like \`\\ui{abc123}\`
|
||||
- You should usually introduce what you're showing before placing the marker
|
||||
- For a single resource: \\ui{resource-id}
|
||||
- For multiple resources shown separately: \\ui{resource-id-a} \\ui{resource-id-b}
|
||||
- For multiple resources in a carousel: \\ui{resource-id-a,resource-id-b,resource-id-c}
|
||||
- The UI will be rendered inline where you place the marker
|
||||
- Format: \\ui{resource-id} or \\ui{id1,id2,id3} using the IDs provided above`;
|
||||
|
||||
currentTextBlock += uiInstructions;
|
||||
}
|
||||
|
||||
if (CONTENT_ARRAY_PROVIDERS.has(provider) && currentTextBlock) {
|
||||
formattedContent.push({ type: 'text', text: currentTextBlock });
|
||||
}
|
||||
|
||||
let artifacts: t.Artifacts = undefined;
|
||||
if (imageUrls.length || uiResources.length) {
|
||||
if (imageUrls.length) {
|
||||
artifacts = { content: imageUrls };
|
||||
}
|
||||
|
||||
if (uiResources.length) {
|
||||
artifacts = {
|
||||
...(imageUrls.length && { content: imageUrls }),
|
||||
...(uiResources.length && { [Tools.ui_resources]: { data: uiResources } }),
|
||||
...artifacts,
|
||||
[Tools.ui_resources]: { data: uiResources },
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -90,11 +90,7 @@ export type Provider =
|
|||
export type FormattedContent =
|
||||
| {
|
||||
type: 'text';
|
||||
metadata?: {
|
||||
type: string;
|
||||
data: UIResource[];
|
||||
};
|
||||
text?: string;
|
||||
text: string;
|
||||
}
|
||||
| {
|
||||
type: 'image';
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ export function normalizeServerName(serverName: string): string {
|
|||
}
|
||||
|
||||
/** Replace non-matching characters with underscores.
|
||||
This preserves the general structure while ensuring compatibility.
|
||||
This preserves the general structure while ensuring compatibility.
|
||||
Trims leading/trailing underscores
|
||||
*/
|
||||
const normalized = serverName.replace(/[^a-zA-Z0-9_.-]/g, '_').replace(/^_+|_+$/g, '');
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue