Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-12-17 17:00:15 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

✨ refactor: Integrate Capabilities into Agent File Uploads and Tool Handling (#5048)

Browse source 
* refactor: support drag/drop files for agents, handle undefined tool_resource edge cases

* refactor: consolidate endpoints config logic to dedicated getter

* refactor: Enhance agent tools loading logic to respect capabilities and filter tools accordingly

* refactor: Integrate endpoint capabilities into file upload dropdown for dynamic resource handling

* refactor: Implement capability checks for agent file upload operations

* fix: non-image tool_resource check
This commit is contained in:
Danny Avila 2024-12-19 13:04:48 -05:00 • committed by GitHub
parent d68c874db4
commit 3fbbcb1cfe
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
17 changed files with 449 additions and 189 deletions
Download patch file Download diff file Expand all files Collapse all files

66
api/server/controllers/EndpointController.js
View file

@ -1,69 +1,7 @@
const { CacheKeys, EModelEndpoint, orderEndpointsConfig } = require('librechat-data-provider');
const { loadDefaultEndpointsConfig, loadConfigEndpoints } = require('~/server/services/Config');
const { getLogStores } = require('~/cache');
const { getEndpointsConfig } = require('~/server/services/Config');
async function endpointController(req, res) {
const cache = getLogStores(CacheKeys.CONFIG_STORE);
const cachedEndpointsConfig = await cache.get(CacheKeys.ENDPOINT_CONFIG);
if (cachedEndpointsConfig) {
res.send(cachedEndpointsConfig);
return;
}
const defaultEndpointsConfig = await loadDefaultEndpointsConfig(req);
const customConfigEndpoints = await loadConfigEndpoints(req);
/** @type {TEndpointsConfig} */
const mergedConfig = { ...defaultEndpointsConfig, ...customConfigEndpoints };
if (mergedConfig[EModelEndpoint.assistants] && req.app.locals?.[EModelEndpoint.assistants]) {
const { disableBuilder, retrievalModels, capabilities, version, ..._rest } =
req.app.locals[EModelEndpoint.assistants];
mergedConfig[EModelEndpoint.assistants] = {
...mergedConfig[EModelEndpoint.assistants],
version,
retrievalModels,
disableBuilder,
capabilities,
};
}
if (mergedConfig[EModelEndpoint.agents] && req.app.locals?.[EModelEndpoint.agents]) {
const { disableBuilder, capabilities, ..._rest } = req.app.locals[EModelEndpoint.agents];
mergedConfig[EModelEndpoint.agents] = {
...mergedConfig[EModelEndpoint.agents],
disableBuilder,
capabilities,
};
}
if (
mergedConfig[EModelEndpoint.azureAssistants] &&
req.app.locals?.[EModelEndpoint.azureAssistants]
) {
const { disableBuilder, retrievalModels, capabilities, version, ..._rest } =
req.app.locals[EModelEndpoint.azureAssistants];
mergedConfig[EModelEndpoint.azureAssistants] = {
...mergedConfig[EModelEndpoint.azureAssistants],
version,
retrievalModels,
disableBuilder,
capabilities,
};
}
if (mergedConfig[EModelEndpoint.bedrock] && req.app.locals?.[EModelEndpoint.bedrock]) {
const { availableRegions } = req.app.locals[EModelEndpoint.bedrock];
mergedConfig[EModelEndpoint.bedrock] = {
...mergedConfig[EModelEndpoint.bedrock],
availableRegions,
};
}
const endpointsConfig = orderEndpointsConfig(mergedConfig);
await cache.set(CacheKeys.ENDPOINT_CONFIG, endpointsConfig);
const endpointsConfig = await getEndpointsConfig(req);
res.send(JSON.stringify(endpointsConfig));
}

10
api/server/controllers/assistants/helpers.js
View file

@ -1,5 +1,4 @@
const {
CacheKeys,
SystemRoles,
EModelEndpoint,
defaultOrderQuery,
@ -9,7 +8,7 @@ const {
initializeClient: initAzureClient,
} = require('~/server/services/Endpoints/azureAssistants');
const { initializeClient } = require('~/server/services/Endpoints/assistants');
const { getLogStores } = require('~/cache');
const { getEndpointsConfig } = require('~/server/services/Config');
/**
* @param {Express.Request} req
@ -23,11 +22,8 @@ const getCurrentVersion = async (req, endpoint) => {
version = `v${req.body.version}`;
}
if (!version && endpoint) {
const cache = getLogStores(CacheKeys.CONFIG_STORE);
const cachedEndpointsConfig = await cache.get(CacheKeys.ENDPOINT_CONFIG);
version = `v${
cachedEndpointsConfig?.[endpoint]?.version ?? defaultAssistantsVersion[endpoint]
}`;
const endpointsConfig = await getEndpointsConfig(req);
version = `v${endpointsConfig?.[endpoint]?.version ?? defaultAssistantsVersion[endpoint]}`;
}
if (!version?.startsWith('v') && version.length !== 2) {
throw new Error(`[${req.baseUrl}] Invalid version: ${version}`);

75
api/server/services/Config/getEndpointsConfig.js Normal file
View file

@ -0,0 +1,75 @@
const { CacheKeys, EModelEndpoint, orderEndpointsConfig } = require('librechat-data-provider');
const loadDefaultEndpointsConfig = require('./loadDefaultEConfig');
const loadConfigEndpoints = require('./loadConfigEndpoints');
const getLogStores = require('~/cache/getLogStores');
/**
*
* @param {ServerRequest} req
* @returns {Promise<TEndpointsConfig>}
*/
async function getEndpointsConfig(req) {
const cache = getLogStores(CacheKeys.CONFIG_STORE);
const cachedEndpointsConfig = await cache.get(CacheKeys.ENDPOINT_CONFIG);
if (cachedEndpointsConfig) {
return cachedEndpointsConfig;
}
const defaultEndpointsConfig = await loadDefaultEndpointsConfig(req);
const customConfigEndpoints = await loadConfigEndpoints(req);
/** @type {TEndpointsConfig} */
const mergedConfig = { ...defaultEndpointsConfig, ...customConfigEndpoints };
if (mergedConfig[EModelEndpoint.assistants] && req.app.locals?.[EModelEndpoint.assistants]) {
const { disableBuilder, retrievalModels, capabilities, version, ..._rest } =
req.app.locals[EModelEndpoint.assistants];
mergedConfig[EModelEndpoint.assistants] = {
...mergedConfig[EModelEndpoint.assistants],
version,
retrievalModels,
disableBuilder,
capabilities,
};
}
if (mergedConfig[EModelEndpoint.agents] && req.app.locals?.[EModelEndpoint.agents]) {
const { disableBuilder, capabilities, ..._rest } = req.app.locals[EModelEndpoint.agents];
mergedConfig[EModelEndpoint.agents] = {
...mergedConfig[EModelEndpoint.agents],
disableBuilder,
capabilities,
};
}
if (
mergedConfig[EModelEndpoint.azureAssistants] &&
req.app.locals?.[EModelEndpoint.azureAssistants]
) {
const { disableBuilder, retrievalModels, capabilities, version, ..._rest } =
req.app.locals[EModelEndpoint.azureAssistants];
mergedConfig[EModelEndpoint.azureAssistants] = {
...mergedConfig[EModelEndpoint.azureAssistants],
version,
retrievalModels,
disableBuilder,
capabilities,
};
}
if (mergedConfig[EModelEndpoint.bedrock] && req.app.locals?.[EModelEndpoint.bedrock]) {
const { availableRegions } = req.app.locals[EModelEndpoint.bedrock];
mergedConfig[EModelEndpoint.bedrock] = {
...mergedConfig[EModelEndpoint.bedrock],
availableRegions,
};
}
const endpointsConfig = orderEndpointsConfig(mergedConfig);
await cache.set(CacheKeys.ENDPOINT_CONFIG, endpointsConfig);
return endpointsConfig;
}
module.exports = { getEndpointsConfig };

6
api/server/services/Config/index.js
View file

@ -3,10 +3,9 @@ const getCustomConfig = require('./getCustomConfig');
const loadCustomConfig = require('./loadCustomConfig');
const loadConfigModels = require('./loadConfigModels');
const loadDefaultModels = require('./loadDefaultModels');
const getEndpointsConfig = require('./getEndpointsConfig');
const loadOverrideConfig = require('./loadOverrideConfig');
const loadAsyncEndpoints = require('./loadAsyncEndpoints');
const loadConfigEndpoints = require('./loadConfigEndpoints');
const loadDefaultEndpointsConfig = require('./loadDefaultEConfig');
module.exports = {
config,
@ -16,6 +15,5 @@ module.exports = {
loadOverrideConfig,
loadAsyncEndpoints,
...getCustomConfig,
loadConfigEndpoints,
loadDefaultEndpointsConfig,
...getEndpointsConfig,
};

31
api/server/services/Files/process.js
View file

@ -12,6 +12,7 @@ const {
EToolResources,
mergeFileConfig,
hostImageIdSuffix,
AgentCapabilities,
checkOpenAIStorage,
removeNullishValues,
hostImageNamePrefix,
@ -27,6 +28,7 @@ const { addResourceFileId, deleteResourceFileId } = require('~/server/controller
const { addAgentResourceFile, removeAgentResourceFiles } = require('~/models/Agent');
const { getOpenAIClient } = require('~/server/controllers/assistants/helpers');
const { createFile, updateFileUsage, deleteFiles } = require('~/models/File');
const { getEndpointsConfig } = require('~/server/services/Config');
const { loadAuthValues } = require('~/app/clients/tools/util');
const { LB_QueueAsyncCall } = require('~/server/utils/queue');
const { getStrategyFunctions } = require('./strategies');
@ -451,6 +453,17 @@ const processFileUpload = async ({ req, res, metadata }) => {
res.status(200).json({ message: 'File uploaded and processed successfully', ...result });
};
/**
* @param {ServerRequest} req
* @param {AgentCapabilities} capability
* @returns {Promise<boolean>}
*/
const checkCapability = async (req, capability) => {
const endpointsConfig = await getEndpointsConfig(req);
const capabilities = endpointsConfig?.[EModelEndpoint.agents]?.capabilities ?? [];
return capabilities.includes(capability);
};
/**
* Applies the current strategy for file uploads.
* Saves file metadata to the database with an expiry TTL.
@ -478,9 +491,20 @@ const processAgentFileUpload = async ({ req, res, metadata }) => {
throw new Error('No agent ID provided for agent file upload');
}
const isImage = file.mimetype.startsWith('image');
if (!isImage && !tool_resource) {
/** Note: this needs to be removed when we can support files to providers */
throw new Error('No tool resource provided for non-image agent file upload');
}
let fileInfoMetadata;
const entity_id = messageAttachment === true ? undefined : agent_id;
if (tool_resource === EToolResources.execute_code) {
const isCodeEnabled = await checkCapability(req, AgentCapabilities.execute_code);
if (!isCodeEnabled) {
throw new Error('Code execution is not enabled for Agents');
}
const { handleFileUpload: uploadCodeEnvFile } = getStrategyFunctions(FileSources.execute_code);
const result = await loadAuthValues({ userId: req.user.id, authFields: [EnvVar.CODE_API_KEY] });
const stream = fs.createReadStream(file.path);
@ -492,6 +516,11 @@ const processAgentFileUpload = async ({ req, res, metadata }) => {
entity_id,
});
fileInfoMetadata = { fileIdentifier };
} else if (tool_resource === EToolResources.file_search) {
const isFileSearchEnabled = await checkCapability(req, AgentCapabilities.file_search);
if (!isFileSearchEnabled) {
throw new Error('File search is not enabled for Agents');
}
}
const source =
@ -527,7 +556,7 @@ const processAgentFileUpload = async ({ req, res, metadata }) => {
});
}
if (file.mimetype.startsWith('image')) {
if (isImage) {
const result = await processImageFile({
req,
file,

133
api/server/services/ToolService.js
View file

@ -8,13 +8,16 @@ const {
ErrorTypes,
ContentTypes,
imageGenTools,
EModelEndpoint,
actionDelimiter,
ImageVisionTool,
openapiToFunction,
AgentCapabilities,
validateAndParseOpenAPISpec,
} = require('librechat-data-provider');
const { processFileURL, uploadImageBuffer } = require('~/server/services/Files/process');
const { loadActionSets, createActionTool, domainParser } = require('./ActionService');
const { getEndpointsConfig } = require('~/server/services/Config');
const { recordUsage } = require('~/server/services/Threads');
const { loadTools } = require('~/app/clients/tools/util');
const { redactMessage } = require('~/config/parsers');
@ -383,11 +386,37 @@ async function loadAgentTools({ req, agent, tool_resources, openAIApiKey }) {
if (!agent.tools || agent.tools.length === 0) {
return {};
}
const endpointsConfig = await getEndpointsConfig(req);
const capabilities = endpointsConfig?.[EModelEndpoint.agents]?.capabilities ?? [];
const areToolsEnabled = capabilities.includes(AgentCapabilities.tools);
if (!areToolsEnabled) {
logger.debug('Tools are not enabled for this agent.');
return {};
}
const isFileSearchEnabled = capabilities.includes(AgentCapabilities.file_search);
const isCodeEnabled = capabilities.includes(AgentCapabilities.execute_code);
const areActionsEnabled = capabilities.includes(AgentCapabilities.actions);
const _agentTools = agent.tools?.filter((tool) => {
if (tool === Tools.file_search && !isFileSearchEnabled) {
return false;
} else if (tool === Tools.execute_code && !isCodeEnabled) {
return false;
}
return true;
});
if (!_agentTools || _agentTools.length === 0) {
return {};
}
const { loadedTools, toolContextMap } = await loadTools({
agent,
functions: true,
user: req.user.id,
tools: agent.tools,
tools: _agentTools,
options: {
req,
openAIApiKey,
@ -434,62 +463,74 @@ async function loadAgentTools({ req, agent, tool_resources, openAIApiKey }) {
return map;
}, {});
if (!areActionsEnabled) {
return {
tools: agentTools,
toolContextMap,
};
}
let actionSets = [];
const ActionToolMap = {};
for (const toolName of agent.tools) {
if (!ToolMap[toolName]) {
if (!actionSets.length) {
actionSets = (await loadActionSets({ agent_id: agent.id })) ?? [];
}
for (const toolName of _agentTools) {
if (ToolMap[toolName]) {
continue;
}
let actionSet = null;
let currentDomain = '';
for (let action of actionSets) {
const domain = await domainParser(req, action.metadata.domain, true);
if (toolName.includes(domain)) {
currentDomain = domain;
actionSet = action;
break;
}
}
if (!actionSets.length) {
actionSets = (await loadActionSets({ agent_id: agent.id })) ?? [];
}
if (actionSet) {
const validationResult = validateAndParseOpenAPISpec(actionSet.metadata.raw_spec);
if (validationResult.spec) {
const { requestBuilders, functionSignatures, zodSchemas } = openapiToFunction(
validationResult.spec,
true,
let actionSet = null;
let currentDomain = '';
for (let action of actionSets) {
const domain = await domainParser(req, action.metadata.domain, true);
if (toolName.includes(domain)) {
currentDomain = domain;
actionSet = action;
break;
}
}
if (!actionSet) {
continue;
}
const validationResult = validateAndParseOpenAPISpec(actionSet.metadata.raw_spec);
if (validationResult.spec) {
const { requestBuilders, functionSignatures, zodSchemas } = openapiToFunction(
validationResult.spec,
true,
);
const functionName = toolName.replace(`${actionDelimiter}${currentDomain}`, '');
const functionSig = functionSignatures.find((sig) => sig.name === functionName);
const requestBuilder = requestBuilders[functionName];
const zodSchema = zodSchemas[functionName];
if (requestBuilder) {
const tool = await createActionTool({
action: actionSet,
requestBuilder,
zodSchema,
name: toolName,
description: functionSig.description,
});
if (!tool) {
logger.warn(
`Invalid action: user: ${req.user.id} | agent_id: ${agent.id} | toolName: ${toolName}`,
);
const functionName = toolName.replace(`${actionDelimiter}${currentDomain}`, '');
const functionSig = functionSignatures.find((sig) => sig.name === functionName);
const requestBuilder = requestBuilders[functionName];
const zodSchema = zodSchemas[functionName];
if (requestBuilder) {
const tool = await createActionTool({
action: actionSet,
requestBuilder,
zodSchema,
name: toolName,
description: functionSig.description,
});
if (!tool) {
logger.warn(
`Invalid action: user: ${req.user.id} | agent_id: ${agent.id} | toolName: ${toolName}`,
);
throw new Error(`{"type":"${ErrorTypes.INVALID_ACTION}"}`);
}
agentTools.push(tool);
ActionToolMap[toolName] = tool;
}
throw new Error(`{"type":"${ErrorTypes.INVALID_ACTION}"}`);
}
agentTools.push(tool);
ActionToolMap[toolName] = tool;
}
}
}
if (agent.tools.length > 0 && agentTools.length === 0) {
throw new Error('No tools found for the specified tool calls.');
if (_agentTools.length > 0 && agentTools.length === 0) {
logger.warn(`No tools found for the specified tool calls: ${_agentTools.join(', ')}`);
return {};
}
return {