mirror of
https://github.com/danny-avila/LibreChat.git
synced 2025-12-17 17:00:15 +01:00
🎉 feat: Code Interpreter API and Agents Release (#4860)
* feat: Code Interpreter API & File Search Agent Uploads chore: add back code files wip: first pass, abstract key dialog refactor: influence checkbox on key changes refactor: update localization keys for 'execute code' to 'run code' wip: run code button refactor: add throwError parameter to loadAuthValues and getUserPluginAuthValue functions feat: first pass, API tool calling fix: handle missing toolId in callTool function and return 404 for non-existent tools feat: show code outputs fix: improve error handling in callTool function and log errors fix: handle potential null value for filepath in attachment destructuring fix: normalize language before rendering and prevent null return fix: add loading indicator in RunCode component while executing code feat: add support for conditional code execution in Markdown components feat: attachments refactor: remove bash fix: pass abort signal to graph/run refactor: debounce and rate limit tool call refactor: increase debounce delay for execute function feat: set code output attachments feat: image attachments refactor: apply message context refactor: pass `partIndex` feat: toolCall schema/model/methods feat: block indexing feat: get tool calls chore: imports chore: typing chore: condense type imports feat: get tool calls fix: block indexing chore: typing refactor: update tool calls mapping to support multiple results fix: add unique key to nav link for rendering wip: first pass, tool call results refactor: update query cache from successful tool call mutation style: improve result switcher styling chore: note on using \`.toObject()\` feat: add agent_id field to conversation schema chore: typing refactor: rename agentMap to agentsMap for consistency feat: Agent Name as chat input placeholder chore: bump agents 📦 chore: update @langchain dependencies to latest versions to match agents package 📦 chore: update @librechat/agents dependency to version 1.8.0 fix: Aborting agent stream removes sender; fix(bedrock): completion removes preset name label refactor: remove direct file parameter to use req.file, add `processAgentFileUpload` for image uploads feat: upload menu feat: prime message_file resources feat: implement conversation access validation in chat route refactor: remove file parameter from processFileUpload and use req.file instead feat: add savedMessageIds set to track saved message IDs in BaseClient, to prevent unnecessary double-write to db feat: prevent duplicate message saves by checking savedMessageIds in AgentController refactor: skip legacy RAG API handling for agents feat: add files field to convoSchema refactor: update request type annotations from Express.Request to ServerRequest in file processing functions feat: track conversation files fix: resendFiles, addPreviousAttachments handling feat: add ID validation for session_id and file_id in download route feat: entity_id for code file uploads/downloads fix: code file edge cases feat: delete related tool calls feat: add stream rate handling for LLM configuration feat: enhance system content with attached file information fix: improve error logging in resource priming function * WIP: PoC, sequential agents WIP: PoC Sequential Agents, first pass content data + bump agents package fix: package-lock WIP: PoC, o1 support, refactor bufferString feat: convertJsonSchemaToZod fix: form issues and schema defining erroneous model fix: max length issue on agent form instructions, limit conversation messages to sequential agents feat: add abort signal support to createRun function and AgentClient feat: PoC, hide prior sequential agent steps fix: update parameter naming from config to metadata in event handlers for clarity, add model to usage data refactor: use only last contentData, track model for usage data chore: bump agents package fix: content parts issue refactor: filter contentParts to include tool calls and relevant indices feat: show function calls refactor: filter context messages to exclude tool calls when no tools are available to the agent fix: ensure tool call content is not undefined in formatMessages feat: add agent_id field to conversationPreset schema feat: hide sequential agents feat: increase upload toast duration to 10 seconds * refactor: tool context handling & update Code API Key Dialog feat: toolContextMap chore: skipSpecs -> useSpecs ci: fix handleTools tests feat: API Key Dialog * feat: Agent Permissions Admin Controls feat: replace label with button for prompt permission toggle feat: update agent permissions feat: enable experimental agents and streamline capability configuration feat: implement access control for agents and enhance endpoint menu items feat: add welcome message for agent selection in localization feat: add agents permission to access control and update version to 0.7.57 * fix: update types in useAssistantListMap and useMentions hooks for better null handling * feat: mention agents * fix: agent tool resource race conditions when deleting agent tool resource files * feat: add error handling for code execution with user feedback * refactor: rename AdminControls to AdminSettings for clarity * style: add gap to button in AdminSettings for improved layout * refactor: separate agent query hooks and check access to enable fetching * fix: remove unused provider from agent initialization options, creates issue with custom endpoints * refactor: remove redundant/deprecated modelOptions from AgentClient processes * chore: update @librechat/agents to version 1.8.5 in package.json and package-lock.json * fix: minor styling issues + agent panel uniformity * fix: agent edge cases when set endpoint is no longer defined * refactor: remove unused cleanup function call from AppService * fix: update link in ApiKeyDialog to point to pricing page * fix: improve type handling and layout calculations in SidePanel component * fix: add missing localization string for agent selection in SidePanel * chore: form styling and localizations for upload filesearch/code interpreter * fix: model selection placeholder logic in AgentConfig component * style: agent capabilities * fix: add localization for provider selection and improve dropdown styling in ModelPanel * refactor: use gpt-4o-mini > gpt-3.5-turbo * fix: agents configuration for loadDefaultInterface and update related tests * feat: DALLE Agents support
This commit is contained in:
Danny Avila
GitHub
parent
affcebd48c
commit
1a815f5e19
189 changed files with 5056 additions and 1815 deletions
|
|
@ -15,8 +15,8 @@ const {
|
|||
StructuredWolfram,
|
||||
TavilySearchResults,
|
||||
} = require('../');
|
||||
const { primeFiles } = require('~/server/services/Files/Code/process');
|
||||
const createFileSearchTool = require('./createFileSearchTool');
|
||||
const { primeFiles: primeCodeFiles } = require('~/server/services/Files/Code/process');
|
||||
const { createFileSearchTool, primeFiles: primeSearchFiles } = require('./fileSearch');
|
||||
const { loadSpecs } = require('./loadSpecs');
|
||||
const { logger } = require('~/config');
|
||||
|
||||
|
|
@ -83,7 +83,7 @@ const validateTools = async (user, tools = []) => {
|
|||
}
|
||||
};
|
||||
|
||||
const loadAuthValues = async ({ userId, authFields }) => {
|
||||
const loadAuthValues = async ({ userId, authFields, throwError = true }) => {
|
||||
let authValues = {};
|
||||
|
||||
/**
|
||||
|
|
@ -98,7 +98,7 @@ const loadAuthValues = async ({ userId, authFields }) => {
|
|||
return { authField: field, authValue: value };
|
||||
}
|
||||
try {
|
||||
value = await getUserPluginAuthValue(userId, field);
|
||||
value = await getUserPluginAuthValue(userId, field, throwError);
|
||||
} catch (err) {
|
||||
if (field === fields[fields.length - 1] && !value) {
|
||||
throw err;
|
||||
|
|
@ -122,15 +122,18 @@ const loadAuthValues = async ({ userId, authFields }) => {
|
|||
return authValues;
|
||||
};
|
||||
|
||||
/** @typedef {typeof import('@langchain/core/tools').Tool} ToolConstructor */
|
||||
/** @typedef {import('@langchain/core/tools').Tool} Tool */
|
||||
|
||||
/**
|
||||
* Initializes a tool with authentication values for the given user, supporting alternate authentication fields.
|
||||
* Authentication fields can have alternates separated by "||", and the first defined variable will be used.
|
||||
*
|
||||
* @param {string} userId The user ID for which the tool is being loaded.
|
||||
* @param {Array<string>} authFields Array of strings representing the authentication fields. Supports alternate fields delimited by "||".
|
||||
* @param {typeof import('langchain/tools').Tool} ToolConstructor The constructor function for the tool to be initialized.
|
||||
* @param {ToolConstructor} ToolConstructor The constructor function for the tool to be initialized.
|
||||
* @param {Object} options Optional parameters to be passed to the tool constructor alongside authentication values.
|
||||
* @returns {Function} An Async function that, when called, asynchronously initializes and returns an instance of the tool with authentication.
|
||||
* @returns {() => Promise<Tool>} An Async function that, when called, asynchronously initializes and returns an instance of the tool with authentication.
|
||||
*/
|
||||
const loadToolWithAuth = (userId, authFields, ToolConstructor, options = {}) => {
|
||||
return async function () {
|
||||
|
|
@ -142,11 +145,12 @@ const loadToolWithAuth = (userId, authFields, ToolConstructor, options = {}) =>
|
|||
const loadTools = async ({
|
||||
user,
|
||||
model,
|
||||
functions = true,
|
||||
returnMap = false,
|
||||
isAgent,
|
||||
useSpecs,
|
||||
tools = [],
|
||||
options = {},
|
||||
skipSpecs = false,
|
||||
functions = true,
|
||||
returnMap = false,
|
||||
}) => {
|
||||
const toolConstructors = {
|
||||
calculator: Calculator,
|
||||
|
|
@ -174,11 +178,12 @@ const loadTools = async ({
|
|||
|
||||
const requestedTools = {};
|
||||
|
||||
if (functions) {
|
||||
if (functions === true) {
|
||||
toolConstructors.dalle = DALLE3;
|
||||
}
|
||||
|
||||
const imageGenOptions = {
|
||||
isAgent,
|
||||
req: options.req,
|
||||
fileStrategy: options.fileStrategy,
|
||||
processFileURL: options.processFileURL,
|
||||
|
|
@ -189,7 +194,6 @@ const loadTools = async ({
|
|||
const toolOptions = {
|
||||
serpapi: { location: 'Austin,Texas,United States', hl: 'en', gl: 'us' },
|
||||
dalle: imageGenOptions,
|
||||
'dall-e': imageGenOptions,
|
||||
'stable-diffusion': imageGenOptions,
|
||||
};
|
||||
|
||||
|
|
@ -203,24 +207,38 @@ const loadTools = async ({
|
|||
toolAuthFields[tool.pluginKey] = tool.authConfig.map((auth) => auth.authField);
|
||||
});
|
||||
|
||||
const toolContextMap = {};
|
||||
const remainingTools = [];
|
||||
|
||||
for (const tool of tools) {
|
||||
if (tool === Tools.execute_code) {
|
||||
const authValues = await loadAuthValues({
|
||||
userId: user,
|
||||
authFields: [EnvVar.CODE_API_KEY],
|
||||
});
|
||||
const files = await primeFiles(options, authValues[EnvVar.CODE_API_KEY]);
|
||||
requestedTools[tool] = () =>
|
||||
createCodeExecutionTool({
|
||||
requestedTools[tool] = async () => {
|
||||
const authValues = await loadAuthValues({
|
||||
userId: user,
|
||||
authFields: [EnvVar.CODE_API_KEY],
|
||||
});
|
||||
const codeApiKey = authValues[EnvVar.CODE_API_KEY];
|
||||
const { files, toolContext } = await primeCodeFiles(options, codeApiKey);
|
||||
if (toolContext) {
|
||||
toolContextMap[tool] = toolContext;
|
||||
}
|
||||
const CodeExecutionTool = createCodeExecutionTool({
|
||||
user_id: user,
|
||||
files,
|
||||
...authValues,
|
||||
});
|
||||
CodeExecutionTool.apiKey = codeApiKey;
|
||||
return CodeExecutionTool;
|
||||
};
|
||||
continue;
|
||||
} else if (tool === Tools.file_search) {
|
||||
requestedTools[tool] = () => createFileSearchTool(options);
|
||||
requestedTools[tool] = async () => {
|
||||
const { files, toolContext } = await primeSearchFiles(options);
|
||||
if (toolContext) {
|
||||
toolContextMap[tool] = toolContext;
|
||||
}
|
||||
return createFileSearchTool({ req: options.req, files });
|
||||
};
|
||||
continue;
|
||||
}
|
||||
|
||||
|
|
@ -241,13 +259,13 @@ const loadTools = async ({
|
|||
continue;
|
||||
}
|
||||
|
||||
if (functions) {
|
||||
if (functions === true) {
|
||||
remainingTools.push(tool);
|
||||
}
|
||||
}
|
||||
|
||||
let specs = null;
|
||||
if (functions && remainingTools.length > 0 && skipSpecs !== true) {
|
||||
if (useSpecs === true && functions === true && remainingTools.length > 0) {
|
||||
specs = await loadSpecs({
|
||||
llm: model,
|
||||
user,
|
||||
|
|
@ -270,23 +288,21 @@ const loadTools = async ({
|
|||
return requestedTools;
|
||||
}
|
||||
|
||||
// load tools
|
||||
let result = [];
|
||||
const toolPromises = [];
|
||||
for (const tool of tools) {
|
||||
const validTool = requestedTools[tool];
|
||||
if (!validTool) {
|
||||
continue;
|
||||
}
|
||||
const plugin = await validTool();
|
||||
|
||||
if (Array.isArray(plugin)) {
|
||||
result = [...result, ...plugin];
|
||||
} else if (plugin) {
|
||||
result.push(plugin);
|
||||
if (validTool) {
|
||||
toolPromises.push(
|
||||
validTool().catch((error) => {
|
||||
logger.error(`Error loading tool ${tool}:`, error);
|
||||
return null;
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
const loadedTools = (await Promise.all(toolPromises)).flatMap((plugin) => plugin || []);
|
||||
return { loadedTools, toolContextMap };
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue