mirror of
https://github.com/danny-avila/LibreChat.git
synced 2025-09-22 08:12:00 +02:00
✂️ refactor: Artifacts and Tool Callbacks to Pass UI Resources (#9581)
* ✂️ refactor: use artifacts and callbacks to pass UI resources
* chore: imports
* refactor: Update UIResource type imports and definitions across components and tests
* refactor: Update ToolCallInfo test data structure and enhance TAttachment type definition
---------
Co-authored-by: Samuel Path <samuel.path@shopify.com>
This commit is contained in:
Danny Avila
GitHub
parent
916742ab9d
commit
180046a3c5
14 changed files with 1072 additions and 199 deletions
|
|
@ -161,7 +161,7 @@ describe('formatToolContent', () => {
|
|||
});
|
||||
|
||||
describe('resource handling', () => {
|
||||
it('should handle UI resources', () => {
|
||||
it('should handle UI resources in artifacts', () => {
|
||||
const result: t.MCPToolCallResponse = {
|
||||
content: [
|
||||
{
|
||||
|
|
@ -181,22 +181,27 @@ describe('formatToolContent', () => {
|
|||
expect(content).toEqual([
|
||||
{
|
||||
type: 'text',
|
||||
text: '',
|
||||
metadata: {
|
||||
type: 'ui_resources',
|
||||
data: [
|
||||
{
|
||||
uri: 'ui://carousel',
|
||||
mimeType: 'application/json',
|
||||
text: '{"items": []}',
|
||||
name: 'carousel',
|
||||
description: 'A carousel component',
|
||||
},
|
||||
],
|
||||
},
|
||||
text:
|
||||
'Resource Text: {"items": []}\n' +
|
||||
'Resource URI: ui://carousel\n' +
|
||||
'Resource: carousel\n' +
|
||||
'Resource Description: A carousel component\n' +
|
||||
'Resource MIME Type: application/json',
|
||||
},
|
||||
]);
|
||||
expect(artifacts).toBeUndefined();
|
||||
expect(artifacts).toEqual({
|
||||
ui_resources: {
|
||||
data: [
|
||||
{
|
||||
uri: 'ui://carousel',
|
||||
mimeType: 'application/json',
|
||||
text: '{"items": []}',
|
||||
name: 'carousel',
|
||||
description: 'A carousel component',
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle regular resources', () => {
|
||||
|
|
@ -281,24 +286,75 @@ describe('formatToolContent', () => {
|
|||
expect(content).toEqual([
|
||||
{
|
||||
type: 'text',
|
||||
text: 'Some text\n\n' + 'Resource URI: file://data.csv\n' + 'Resource: Data file',
|
||||
text:
|
||||
'Some text\n\n' +
|
||||
'Resource Text: {"label": "Click me"}\n' +
|
||||
'Resource URI: ui://button\n' +
|
||||
'Resource MIME Type: application/json\n\n' +
|
||||
'Resource URI: file://data.csv\n' +
|
||||
'Resource: Data file',
|
||||
},
|
||||
]);
|
||||
expect(artifacts).toEqual({
|
||||
ui_resources: {
|
||||
data: [
|
||||
{
|
||||
uri: 'ui://button',
|
||||
mimeType: 'application/json',
|
||||
text: '{"label": "Click me"}',
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle both images and UI resources in artifacts', () => {
|
||||
const result: t.MCPToolCallResponse = {
|
||||
content: [
|
||||
{ type: 'text', text: 'Content with multimedia' },
|
||||
{ type: 'image', data: 'base64imagedata', mimeType: 'image/png' },
|
||||
{
|
||||
type: 'resource',
|
||||
resource: {
|
||||
uri: 'ui://graph',
|
||||
mimeType: 'application/json',
|
||||
text: '{"type": "line"}',
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const [content, artifacts] = formatToolContent(result, 'openai');
|
||||
expect(content).toEqual([
|
||||
{
|
||||
type: 'text',
|
||||
text: 'Content with multimedia',
|
||||
},
|
||||
{
|
||||
type: 'text',
|
||||
text: '',
|
||||
metadata: {
|
||||
type: 'ui_resources',
|
||||
data: [
|
||||
{
|
||||
uri: 'ui://button',
|
||||
mimeType: 'application/json',
|
||||
text: '{"label": "Click me"}',
|
||||
},
|
||||
],
|
||||
},
|
||||
text:
|
||||
'Resource Text: {"type": "line"}\n' +
|
||||
'Resource URI: ui://graph\n' +
|
||||
'Resource MIME Type: application/json',
|
||||
},
|
||||
]);
|
||||
expect(artifacts).toBeUndefined();
|
||||
expect(artifacts).toEqual({
|
||||
content: [
|
||||
{
|
||||
type: 'image_url',
|
||||
image_url: { url: 'data:image/png;base64,base64imagedata' },
|
||||
},
|
||||
],
|
||||
ui_resources: {
|
||||
data: [
|
||||
{
|
||||
uri: 'ui://graph',
|
||||
mimeType: 'application/json',
|
||||
text: '{"type": "line"}',
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -358,25 +414,14 @@ describe('formatToolContent', () => {
|
|||
type: 'text',
|
||||
text:
|
||||
'Middle section\n\n' +
|
||||
'Resource Text: {"type": "bar"}\n' +
|
||||
'Resource URI: ui://chart\n' +
|
||||
'Resource MIME Type: application/json\n\n' +
|
||||
'Resource URI: https://api.example.com/data\n' +
|
||||
'Resource: API Data\n' +
|
||||
'Resource Description: External data source',
|
||||
},
|
||||
{ type: 'text', text: 'Conclusion' },
|
||||
{
|
||||
type: 'text',
|
||||
text: '',
|
||||
metadata: {
|
||||
type: 'ui_resources',
|
||||
data: [
|
||||
{
|
||||
uri: 'ui://chart',
|
||||
mimeType: 'application/json',
|
||||
text: '{"type": "bar"}',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
]);
|
||||
expect(artifacts).toEqual({
|
||||
content: [
|
||||
|
|
@ -389,6 +434,15 @@ describe('formatToolContent', () => {
|
|||
image_url: { url: 'https://example.com/image2.jpg' },
|
||||
},
|
||||
],
|
||||
ui_resources: {
|
||||
data: [
|
||||
{
|
||||
uri: 'ui://chart',
|
||||
mimeType: 'application/json',
|
||||
text: '{"type": "bar"}',
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,7 @@
|
|||
import { Tools } from 'librechat-data-provider';
|
||||
import type { UIResource } from 'librechat-data-provider';
|
||||
import type * as t from './types';
|
||||
|
||||
const RECOGNIZED_PROVIDERS = new Set([
|
||||
'google',
|
||||
'anthropic',
|
||||
|
|
@ -111,7 +114,7 @@ export function formatToolContent(
|
|||
const formattedContent: t.FormattedContent[] = [];
|
||||
const imageUrls: t.FormattedContent[] = [];
|
||||
let currentTextBlock = '';
|
||||
const uiResources: t.UIResource[] = [];
|
||||
const uiResources: UIResource[] = [];
|
||||
|
||||
type ContentHandler = undefined | ((item: t.ToolContentPart) => void);
|
||||
|
||||
|
|
@ -144,8 +147,7 @@ export function formatToolContent(
|
|||
|
||||
resource: (item) => {
|
||||
if (item.resource.uri.startsWith('ui://')) {
|
||||
uiResources.push(item.resource as t.UIResource);
|
||||
return;
|
||||
uiResources.push(item.resource as UIResource);
|
||||
}
|
||||
|
||||
const resourceText = [];
|
||||
|
|
@ -182,18 +184,14 @@ export function formatToolContent(
|
|||
formattedContent.push({ type: 'text', text: currentTextBlock });
|
||||
}
|
||||
|
||||
if (uiResources.length) {
|
||||
formattedContent.push({
|
||||
type: 'text',
|
||||
metadata: {
|
||||
type: 'ui_resources',
|
||||
data: uiResources,
|
||||
},
|
||||
text: '',
|
||||
});
|
||||
let artifacts: t.Artifacts = undefined;
|
||||
if (imageUrls.length || uiResources.length) {
|
||||
artifacts = {
|
||||
...(imageUrls.length && { content: imageUrls }),
|
||||
...(uiResources.length && { [Tools.ui_resources]: { data: uiResources } }),
|
||||
};
|
||||
}
|
||||
|
||||
const artifacts = imageUrls.length ? { content: imageUrls } : undefined;
|
||||
if (CONTENT_ARRAY_PROVIDERS.has(provider)) {
|
||||
return [formattedContent, artifacts];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,8 +6,9 @@ import {
|
|||
StdioOptionsSchema,
|
||||
WebSocketOptionsSchema,
|
||||
StreamableHTTPOptionsSchema,
|
||||
Tools,
|
||||
} from 'librechat-data-provider';
|
||||
import type { TPlugin, TUser } from 'librechat-data-provider';
|
||||
import type { UIResource, TPlugin, TUser } from 'librechat-data-provider';
|
||||
import type * as t from '@modelcontextprotocol/sdk/types.js';
|
||||
import type { TokenMethods } from '@librechat/data-schemas';
|
||||
import type { FlowStateManager } from '~/flow/manager';
|
||||
|
|
@ -86,7 +87,7 @@ export type FormattedContent =
|
|||
metadata?: {
|
||||
type: string;
|
||||
data: UIResource[];
|
||||
}
|
||||
};
|
||||
text?: string;
|
||||
}
|
||||
| {
|
||||
|
|
@ -111,18 +112,36 @@ export type FormattedContent =
|
|||
};
|
||||
};
|
||||
|
||||
export type FormattedContentResult = [
|
||||
string | FormattedContent[],
|
||||
undefined | { content: FormattedContent[] },
|
||||
];
|
||||
|
||||
export type UIResource = {
|
||||
uri: string;
|
||||
mimeType: string;
|
||||
text: string;
|
||||
export type FileSearchSource = {
|
||||
fileId: string;
|
||||
relevance: number;
|
||||
fileName?: string;
|
||||
metadata?: {
|
||||
storageType?: string;
|
||||
[key: string]: unknown;
|
||||
};
|
||||
[key: string]: unknown;
|
||||
};
|
||||
|
||||
export type Artifacts =
|
||||
| {
|
||||
content?: FormattedContent[];
|
||||
[Tools.ui_resources]?: {
|
||||
data: UIResource[];
|
||||
};
|
||||
[Tools.file_search]?: {
|
||||
sources: FileSearchSource[];
|
||||
fileCitations?: boolean;
|
||||
};
|
||||
[Tools.web_search]?: import('librechat-data-provider').SearchResultData;
|
||||
files?: Array<{ id: string; name: string }>;
|
||||
session_id?: string;
|
||||
file_ids?: string[];
|
||||
}
|
||||
| undefined;
|
||||
|
||||
export type FormattedContentResult = [string | FormattedContent[], undefined | Artifacts];
|
||||
|
||||
export type ImageFormatter = (item: ImageContent) => FormattedContent;
|
||||
|
||||
export type FormattedToolResponse = [
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue