🚧 chore: merge latest dev build (#4288)

* fix: agent initialization, add `collectedUsage` handling

* style: improve side panel styling

* refactor(loadAgent): Optimize order agent project ID retrieval

* feat: code execution

* fix: typing issues

* feat: ExecuteCode content part

* refactor: use local state for default collapsed state of analysis content parts

* fix: code parsing in ExecuteCode component

* chore: bump agents package, export loadAuthValues

* refactor: Update handleTools.js to use EnvVar for code execution tool authentication

* WIP

* feat: download code outputs

* fix(useEventHandlers): type issues

* feat: backend handling for code outputs

* Refactor: Remove console.log statement in Part.tsx

* refactor: add attachments to TMessage/messageSchema

* WIP: prelim handling for code outputs

* feat: attachments rendering

* refactor: improve attachments rendering

* fix: attachments, nullish edge case, handle attachments from event stream, bump agents package

* fix filename download

* fix: tool assignment for 'run code' on agent creation

* fix: image handling by adding attachments

* refactor: prevent agent creation without provider/model

* refactor: remove unnecessary space in agent creation success message

* refactor: select first model if selecting provider from empty on form

* fix: Agent avatar bug

* fix: `defaultAgentFormValues` causing boolean typing issue and typeerror

* fix: capabilities counting as tools, causing duplication of them

* fix: formatted messages edge case where consecutive content text type parts with the latter having tool_call_ids would cause consecutive AI messages to be created. furthermore, content could not be an array for tool_use messages (anthropic limitation)

* chore: bump @librechat/agents dependency to version 1.6.9

* feat: bedrock agents

* feat: new Agents icon

* feat: agent titling

* feat: agent landing

* refactor: allow sharing agent globally only if user is admin or author

* feat: initial AgentPanelSkeleton

* feat: AgentPanelSkeleton

* feat: collaborative agents

* chore: add potential authorName as part of schema

* chore: Remove unnecessary console.log statement

* WIP: agent model parameters

* chore: ToolsDialog typing and tool related localization chnages

* refactor: update tool instance type (latest langchain class), and rename google tool to 'google' proper

* chore: add back tools

* feat: Agent knowledge files upload

* refactor: better verbiage for disabled knowledge

* chore: debug logs for file deletions

* chore: debug logs for file deletions

* feat: upload/delete agent knowledge/file-search files

* feat: file search UI for agents

* feat: first pass, file search tool

* chore: update default agent capabilities and info
This commit is contained in:
Danny Avila 2024-09-30 17:17:57 -04:00 committed by GitHub
parent f33e75e2ee
commit ad74350036
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
123 changed files with 3611 additions and 1541 deletions

View file

@ -42,6 +42,8 @@ class BaseClient {
this.conversationId;
/** @type {string} */
this.responseMessageId;
/** @type {TAttachment[]} */
this.attachments;
/** The key for the usage object's input tokens
* @type {string} */
this.inputTokensKey = 'prompt_tokens';
@ -629,6 +631,10 @@ class BaseClient {
await this.userMessagePromise;
}
if (this.artifactPromises) {
responseMessage.attachments = (await Promise.all(this.artifactPromises)).filter((a) => a);
}
this.responsePromise = this.saveMessageToDatabase(responseMessage, saveOptions, user);
const messageCache = getLogStores(CacheKeys.MESSAGES);
messageCache.set(

View file

@ -0,0 +1,285 @@
const { ToolMessage } = require('@langchain/core/messages');
const { ContentTypes } = require('librechat-data-provider');
const { HumanMessage, AIMessage, SystemMessage } = require('langchain/schema');
const { formatAgentMessages } = require('./formatMessages');
describe('formatAgentMessages', () => {
it('should format simple user and AI messages', () => {
const payload = [
{ role: 'user', content: 'Hello' },
{ role: 'assistant', content: 'Hi there!' },
];
const result = formatAgentMessages(payload);
expect(result).toHaveLength(2);
expect(result[0]).toBeInstanceOf(HumanMessage);
expect(result[1]).toBeInstanceOf(AIMessage);
});
it('should handle system messages', () => {
const payload = [{ role: 'system', content: 'You are a helpful assistant.' }];
const result = formatAgentMessages(payload);
expect(result).toHaveLength(1);
expect(result[0]).toBeInstanceOf(SystemMessage);
});
it('should format messages with content arrays', () => {
const payload = [
{
role: 'user',
content: [{ type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Hello' }],
},
];
const result = formatAgentMessages(payload);
expect(result).toHaveLength(1);
expect(result[0]).toBeInstanceOf(HumanMessage);
});
it('should handle tool calls and create ToolMessages', () => {
const payload = [
{
role: 'assistant',
content: [
{
type: ContentTypes.TEXT,
[ContentTypes.TEXT]: 'Let me check that for you.',
tool_call_ids: ['123'],
},
{
type: ContentTypes.TOOL_CALL,
tool_call: {
id: '123',
name: 'search',
args: '{"query":"weather"}',
output: 'The weather is sunny.',
},
},
],
},
];
const result = formatAgentMessages(payload);
expect(result).toHaveLength(2);
expect(result[0]).toBeInstanceOf(AIMessage);
expect(result[1]).toBeInstanceOf(ToolMessage);
expect(result[0].tool_calls).toHaveLength(1);
expect(result[1].tool_call_id).toBe('123');
});
it('should handle multiple content parts in assistant messages', () => {
const payload = [
{
role: 'assistant',
content: [
{ type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Part 1' },
{ type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Part 2' },
],
},
];
const result = formatAgentMessages(payload);
expect(result).toHaveLength(1);
expect(result[0]).toBeInstanceOf(AIMessage);
expect(result[0].content).toHaveLength(2);
});
it('should throw an error for invalid tool call structure', () => {
const payload = [
{
role: 'assistant',
content: [
{
type: ContentTypes.TOOL_CALL,
tool_call: {
id: '123',
name: 'search',
args: '{"query":"weather"}',
output: 'The weather is sunny.',
},
},
],
},
];
expect(() => formatAgentMessages(payload)).toThrow('Invalid tool call structure');
});
it('should handle tool calls with non-JSON args', () => {
const payload = [
{
role: 'assistant',
content: [
{ type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Checking...', tool_call_ids: ['123'] },
{
type: ContentTypes.TOOL_CALL,
tool_call: {
id: '123',
name: 'search',
args: 'non-json-string',
output: 'Result',
},
},
],
},
];
const result = formatAgentMessages(payload);
expect(result).toHaveLength(2);
expect(result[0].tool_calls[0].args).toBe('non-json-string');
});
it('should handle complex tool calls with multiple steps', () => {
const payload = [
{
role: 'assistant',
content: [
{
type: ContentTypes.TEXT,
[ContentTypes.TEXT]: 'I\'ll search for that information.',
tool_call_ids: ['search_1'],
},
{
type: ContentTypes.TOOL_CALL,
tool_call: {
id: 'search_1',
name: 'search',
args: '{"query":"weather in New York"}',
output: 'The weather in New York is currently sunny with a temperature of 75°F.',
},
},
{
type: ContentTypes.TEXT,
[ContentTypes.TEXT]: 'Now, I\'ll convert the temperature.',
tool_call_ids: ['convert_1'],
},
{
type: ContentTypes.TOOL_CALL,
tool_call: {
id: 'convert_1',
name: 'convert_temperature',
args: '{"temperature": 75, "from": "F", "to": "C"}',
output: '23.89°C',
},
},
{ type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Here\'s your answer.' },
],
},
];
const result = formatAgentMessages(payload);
expect(result).toHaveLength(5);
expect(result[0]).toBeInstanceOf(AIMessage);
expect(result[1]).toBeInstanceOf(ToolMessage);
expect(result[2]).toBeInstanceOf(AIMessage);
expect(result[3]).toBeInstanceOf(ToolMessage);
expect(result[4]).toBeInstanceOf(AIMessage);
// Check first AIMessage
expect(result[0].content).toBe('I\'ll search for that information.');
expect(result[0].tool_calls).toHaveLength(1);
expect(result[0].tool_calls[0]).toEqual({
id: 'search_1',
name: 'search',
args: { query: 'weather in New York' },
});
// Check first ToolMessage
expect(result[1].tool_call_id).toBe('search_1');
expect(result[1].name).toBe('search');
expect(result[1].content).toBe(
'The weather in New York is currently sunny with a temperature of 75°F.',
);
// Check second AIMessage
expect(result[2].content).toBe('Now, I\'ll convert the temperature.');
expect(result[2].tool_calls).toHaveLength(1);
expect(result[2].tool_calls[0]).toEqual({
id: 'convert_1',
name: 'convert_temperature',
args: { temperature: 75, from: 'F', to: 'C' },
});
// Check second ToolMessage
expect(result[3].tool_call_id).toBe('convert_1');
expect(result[3].name).toBe('convert_temperature');
expect(result[3].content).toBe('23.89°C');
// Check final AIMessage
expect(result[4].content).toStrictEqual([
{ [ContentTypes.TEXT]: 'Here\'s your answer.', type: ContentTypes.TEXT },
]);
});
it.skip('should not produce two consecutive assistant messages and format content correctly', () => {
const payload = [
{ role: 'user', content: 'Hello' },
{
role: 'assistant',
content: [{ type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Hi there!' }],
},
{
role: 'assistant',
content: [{ type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'How can I help you?' }],
},
{ role: 'user', content: 'What\'s the weather?' },
{
role: 'assistant',
content: [
{
type: ContentTypes.TEXT,
[ContentTypes.TEXT]: 'Let me check that for you.',
tool_call_ids: ['weather_1'],
},
{
type: ContentTypes.TOOL_CALL,
tool_call: {
id: 'weather_1',
name: 'check_weather',
args: '{"location":"New York"}',
output: 'Sunny, 75°F',
},
},
],
},
{
role: 'assistant',
content: [
{ type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Here\'s the weather information.' },
],
},
];
const result = formatAgentMessages(payload);
// Check correct message count and types
expect(result).toHaveLength(6);
expect(result[0]).toBeInstanceOf(HumanMessage);
expect(result[1]).toBeInstanceOf(AIMessage);
expect(result[2]).toBeInstanceOf(HumanMessage);
expect(result[3]).toBeInstanceOf(AIMessage);
expect(result[4]).toBeInstanceOf(ToolMessage);
expect(result[5]).toBeInstanceOf(AIMessage);
// Check content of messages
expect(result[0].content).toStrictEqual([
{ [ContentTypes.TEXT]: 'Hello', type: ContentTypes.TEXT },
]);
expect(result[1].content).toStrictEqual([
{ [ContentTypes.TEXT]: 'Hi there!', type: ContentTypes.TEXT },
{ [ContentTypes.TEXT]: 'How can I help you?', type: ContentTypes.TEXT },
]);
expect(result[2].content).toStrictEqual([
{ [ContentTypes.TEXT]: 'What\'s the weather?', type: ContentTypes.TEXT },
]);
expect(result[3].content).toBe('Let me check that for you.');
expect(result[4].content).toBe('Sunny, 75°F');
expect(result[5].content).toStrictEqual([
{ [ContentTypes.TEXT]: 'Here\'s the weather information.', type: ContentTypes.TEXT },
]);
// Check that there are no consecutive AIMessages
const messageTypes = result.map((message) => message.constructor);
for (let i = 0; i < messageTypes.length - 1; i++) {
expect(messageTypes[i] === AIMessage && messageTypes[i + 1] === AIMessage).toBe(false);
}
// Additional check to ensure the consecutive assistant messages were combined
expect(result[1].content).toHaveLength(2);
});
});

View file

@ -155,10 +155,22 @@ const formatAgentMessages = (payload) => {
for (const part of message.content) {
if (part.type === ContentTypes.TEXT && part.tool_call_ids) {
// If there's pending content, add it as an AIMessage
/*
If there's pending content, it needs to be aggregated as a single string to prepare for tool calls.
For Anthropic models, the "tool_calls" field on a message is only respected if content is a string.
*/
if (currentContent.length > 0) {
messages.push(new AIMessage({ content: currentContent }));
let content = currentContent.reduce((acc, curr) => {
if (curr.type === ContentTypes.TEXT) {
return `${acc}${curr[ContentTypes.TEXT]}\n`;
}
return acc;
}, '');
content = `${content}\n${part[ContentTypes.TEXT] ?? ''}`.trim();
lastAIMessage = new AIMessage({ content });
messages.push(lastAIMessage);
currentContent = [];
continue;
}
// Create a new AIMessage with this text and prepare for tool calls

View file

@ -25,7 +25,6 @@ module.exports = {
// Basic Tools
CodeBrew,
AzureAiSearch,
GoogleSearchAPI,
WolframAlphaAPI,
OpenAICreateImage,
StableDiffusionAPI,
@ -37,8 +36,9 @@ module.exports = {
CodeSherpa,
StructuredSD,
StructuredACS,
GoogleSearchAPI,
CodeSherpaTools,
TraversaalSearch,
StructuredWolfram,
TavilySearchResults,
TraversaalSearch,
};

View file

@ -1,9 +1,9 @@
const { z } = require('zod');
const { StructuredTool } = require('langchain/tools');
const { Tool } = require('@langchain/core/tools');
const { SearchClient, AzureKeyCredential } = require('@azure/search-documents');
const { logger } = require('~/config');
class AzureAISearch extends StructuredTool {
class AzureAISearch extends Tool {
// Constants for default values
static DEFAULT_API_VERSION = '2023-11-01';
static DEFAULT_QUERY_TYPE = 'simple';

View file

@ -2,7 +2,7 @@ const { z } = require('zod');
const path = require('path');
const OpenAI = require('openai');
const { v4: uuidv4 } = require('uuid');
const { Tool } = require('langchain/tools');
const { Tool } = require('@langchain/core/tools');
const { HttpsProxyAgent } = require('https-proxy-agent');
const { FileContext } = require('librechat-data-provider');
const { getImageBasename } = require('~/server/services/Files/images');

View file

@ -4,11 +4,12 @@ const { getEnvironmentVariable } = require('@langchain/core/utils/env');
class GoogleSearchResults extends Tool {
static lc_name() {
return 'GoogleSearchResults';
return 'google';
}
constructor(fields = {}) {
super(fields);
this.name = 'google';
this.envVarApiKey = 'GOOGLE_SEARCH_API_KEY';
this.envVarSearchEngineId = 'GOOGLE_CSE_ID';
this.override = fields.override ?? false;

View file

@ -5,12 +5,12 @@ const path = require('path');
const axios = require('axios');
const sharp = require('sharp');
const { v4: uuidv4 } = require('uuid');
const { StructuredTool } = require('langchain/tools');
const { Tool } = require('@langchain/core/tools');
const { FileContext } = require('librechat-data-provider');
const paths = require('~/config/paths');
const { logger } = require('~/config');
class StableDiffusionAPI extends StructuredTool {
class StableDiffusionAPI extends Tool {
constructor(fields) {
super();
/** @type {string} User ID */

View file

@ -1,10 +1,10 @@
/* eslint-disable no-useless-escape */
const axios = require('axios');
const { z } = require('zod');
const { StructuredTool } = require('langchain/tools');
const { Tool } = require('@langchain/core/tools');
const { logger } = require('~/config');
class WolframAlphaAPI extends StructuredTool {
class WolframAlphaAPI extends Tool {
constructor(fields) {
super();
/* Used to initialize the Tool without necessary variables. */

View file

@ -0,0 +1,104 @@
const { z } = require('zod');
const axios = require('axios');
const { tool } = require('@langchain/core/tools');
const { Tools, EToolResources } = require('librechat-data-provider');
const { getFiles } = require('~/models/File');
const { logger } = require('~/config');
/**
*
* @param {Object} options
* @param {ServerRequest} options.req
* @param {Agent['tool_resources']} options.tool_resources
* @returns
*/
const createFileSearchTool = async (options) => {
const { req, tool_resources } = options;
const file_ids = tool_resources?.[EToolResources.file_search]?.file_ids ?? [];
const files = (await getFiles({ file_id: { $in: file_ids } })).map((file) => ({
file_id: file.file_id,
filename: file.filename,
}));
const fileList = files.map((file) => `- ${file.filename}`).join('\n');
const toolDescription = `Performs a semantic search based on a natural language query across the following files:\n${fileList}`;
const FileSearch = tool(
async ({ query }) => {
if (files.length === 0) {
return 'No files to search. Instruct the user to add files for the search.';
}
const jwtToken = req.headers.authorization.split(' ')[1];
if (!jwtToken) {
return 'There was an error authenticating the file search request.';
}
const queryPromises = files.map((file) =>
axios
.post(
`${process.env.RAG_API_URL}/query`,
{
file_id: file.file_id,
query,
k: 5,
},
{
headers: {
Authorization: `Bearer ${jwtToken}`,
'Content-Type': 'application/json',
},
},
)
.catch((error) => {
logger.error(
`Error encountered in \`file_search\` while querying file_id ${file._id}:`,
error,
);
return null;
}),
);
const results = await Promise.all(queryPromises);
const validResults = results.filter((result) => result !== null);
if (validResults.length === 0) {
return 'No results found or errors occurred while searching the files.';
}
const formattedResults = validResults
.flatMap((result) =>
result.data.map(([docInfo, relevanceScore]) => ({
filename: docInfo.metadata.source.split('/').pop(),
content: docInfo.page_content,
relevanceScore,
})),
)
.sort((a, b) => b.relevanceScore - a.relevanceScore);
const formattedString = formattedResults
.map(
(result) =>
`File: ${result.filename}\nRelevance: ${result.relevanceScore.toFixed(4)}\nContent: ${
result.content
}\n`,
)
.join('\n---\n');
return formattedString;
},
{
name: Tools.file_search,
description: toolDescription,
schema: z.object({
query: z
.string()
.describe(
'A natural language query to search for relevant information in the files. Be specific and use keywords related to the information you\'re looking for. The query will be used for semantic similarity matching against the file contents.',
),
}),
},
);
return FileSearch;
};
module.exports = createFileSearchTool;

View file

@ -1,8 +1,10 @@
const { Tools } = require('librechat-data-provider');
const { ZapierToolKit } = require('langchain/agents');
const { Calculator } = require('langchain/tools/calculator');
const { WebBrowser } = require('langchain/tools/webbrowser');
const { SerpAPI, ZapierNLAWrapper } = require('langchain/tools');
const { OpenAIEmbeddings } = require('langchain/embeddings/openai');
const { createCodeExecutionTool, EnvVar } = require('@librechat/agents');
const { getUserPluginAuthValue } = require('~/server/services/PluginService');
const {
availableTools,
@ -24,6 +26,7 @@ const {
StructuredWolfram,
TavilySearchResults,
} = require('../');
const createFileSearchTool = require('./createFileSearchTool');
const { loadToolSuite } = require('./loadToolSuite');
const { loadSpecs } = require('./loadSpecs');
const { logger } = require('~/config');
@ -97,6 +100,45 @@ const validateTools = async (user, tools = []) => {
}
};
const loadAuthValues = async ({ userId, authFields }) => {
let authValues = {};
/**
* Finds the first non-empty value for the given authentication field, supporting alternate fields.
* @param {string[]} fields Array of strings representing the authentication fields. Supports alternate fields delimited by "||".
* @returns {Promise<{ authField: string, authValue: string} | null>} An object containing the authentication field and value, or null if not found.
*/
const findAuthValue = async (fields) => {
for (const field of fields) {
let value = process.env[field];
if (value) {
return { authField: field, authValue: value };
}
try {
value = await getUserPluginAuthValue(userId, field);
} catch (err) {
if (field === fields[fields.length - 1] && !value) {
throw err;
}
}
if (value) {
return { authField: field, authValue: value };
}
}
return null;
};
for (let authField of authFields) {
const fields = authField.split('||');
const result = await findAuthValue(fields);
if (result) {
authValues[result.authField] = result.authValue;
}
}
return authValues;
};
/**
* Initializes a tool with authentication values for the given user, supporting alternate authentication fields.
* Authentication fields can have alternates separated by "||", and the first defined variable will be used.
@ -109,41 +151,7 @@ const validateTools = async (user, tools = []) => {
*/
const loadToolWithAuth = (userId, authFields, ToolConstructor, options = {}) => {
return async function () {
let authValues = {};
/**
* Finds the first non-empty value for the given authentication field, supporting alternate fields.
* @param {string[]} fields Array of strings representing the authentication fields. Supports alternate fields delimited by "||".
* @returns {Promise<{ authField: string, authValue: string} | null>} An object containing the authentication field and value, or null if not found.
*/
const findAuthValue = async (fields) => {
for (const field of fields) {
let value = process.env[field];
if (value) {
return { authField: field, authValue: value };
}
try {
value = await getUserPluginAuthValue(userId, field);
} catch (err) {
if (field === fields[fields.length - 1] && !value) {
throw err;
}
}
if (value) {
return { authField: field, authValue: value };
}
}
return null;
};
for (let authField of authFields) {
const fields = authField.split('||');
const result = await findAuthValue(fields);
if (result) {
authValues[result.authField] = result.authValue;
}
}
const authValues = await loadAuthValues({ userId, authFields });
return new ToolConstructor({ ...options, ...authValues, userId });
};
};
@ -264,6 +272,22 @@ const loadTools = async ({
const remainingTools = [];
for (const tool of tools) {
if (tool === Tools.execute_code) {
const authValues = await loadAuthValues({
userId: user.id,
authFields: [EnvVar.CODE_API_KEY],
});
requestedTools[tool] = () =>
createCodeExecutionTool({
user_id: user.id,
...authValues,
});
continue;
} else if (tool === Tools.file_search) {
requestedTools[tool] = () => createFileSearchTool(options);
continue;
}
if (customConstructors[tool]) {
requestedTools[tool] = customConstructors[tool];
continue;
@ -331,6 +355,7 @@ const loadTools = async ({
module.exports = {
loadToolWithAuth,
loadAuthValues,
validateTools,
loadTools,
};

View file

@ -1,8 +1,9 @@
const { validateTools, loadTools } = require('./handleTools');
const { validateTools, loadTools, loadAuthValues } = require('./handleTools');
const handleOpenAIErrors = require('./handleOpenAIErrors');
module.exports = {
handleOpenAIErrors,
loadAuthValues,
validateTools,
loadTools,
};