mirror of
https://github.com/danny-avila/LibreChat.git
synced 2025-12-18 01:10:14 +01:00
🎥 feat: YouTube Tool (#5582)
* adding youtube tool * refactor: use short `url` param instead of `videoUrl` * refactor: move API key retrieval to a separate credentials module * refactor: remove unnecessary `isEdited` message property * refactor: remove unnecessary `isEdited` message property pt. 2 * refactor: YouTube Tool with new `tool()` generator, handle tools already created by new `tool` generator * fix: only reset request data for multi-convo messages * refactor: enhance YouTube tool by adding transcript parsing and returning structured JSON responses * refactor: update transcript parsing to handle raw response and clean up text output * feat: support toolkits and refactor YouTube tool as a toolkit for better LLM usage * refactor: remove unused OpenAPI specs and streamline tools transformation in loadAsyncEndpoints * refactor: implement manifestToolMap for better tool management and streamline authentication handling * feat: support toolkits for assistants * refactor: rename loadedTools to toolDefinitions for clarity in PluginController and assistant controllers * feat: complete support of toolkits for assistants --------- Co-authored-by: Danilo Pejakovic <danilo.pejakovic@leoninestudios.com>
This commit is contained in:
Danny Avila
GitHub
parent
33f6093775
commit
352565c9a6
29 changed files with 456 additions and 102 deletions
|
|
@ -1,7 +1,7 @@
|
|||
const { promises: fs } = require('fs');
|
||||
const { CacheKeys, AuthType } = require('librechat-data-provider');
|
||||
const { addOpenAPISpecs } = require('~/app/clients/tools/util/addOpenAPISpecs');
|
||||
const { getCustomConfig } = require('~/server/services/Config');
|
||||
const { availableTools } = require('~/app/clients/tools');
|
||||
const { getMCPManager } = require('~/config');
|
||||
const { getLogStores } = require('~/cache');
|
||||
|
||||
|
|
@ -59,10 +59,9 @@ const getAvailablePluginsController = async (req, res) => {
|
|||
|
||||
/** @type {{ filteredTools: string[], includedTools: string[] }} */
|
||||
const { filteredTools = [], includedTools = [] } = req.app.locals;
|
||||
const pluginManifest = await fs.readFile(req.app.locals.paths.pluginManifest, 'utf8');
|
||||
const jsonData = JSON.parse(pluginManifest);
|
||||
const pluginManifest = availableTools;
|
||||
|
||||
const uniquePlugins = filterUniquePlugins(jsonData);
|
||||
const uniquePlugins = filterUniquePlugins(pluginManifest);
|
||||
let authenticatedPlugins = [];
|
||||
for (const plugin of uniquePlugins) {
|
||||
authenticatedPlugins.push(
|
||||
|
|
@ -106,17 +105,15 @@ const getAvailableTools = async (req, res) => {
|
|||
return;
|
||||
}
|
||||
|
||||
const pluginManifest = await fs.readFile(req.app.locals.paths.pluginManifest, 'utf8');
|
||||
|
||||
const jsonData = JSON.parse(pluginManifest);
|
||||
const pluginManifest = availableTools;
|
||||
const customConfig = await getCustomConfig();
|
||||
if (customConfig?.mcpServers != null) {
|
||||
const mcpManager = await getMCPManager();
|
||||
await mcpManager.loadManifestTools(jsonData);
|
||||
await mcpManager.loadManifestTools(pluginManifest);
|
||||
}
|
||||
|
||||
/** @type {TPlugin[]} */
|
||||
const uniquePlugins = filterUniquePlugins(jsonData);
|
||||
const uniquePlugins = filterUniquePlugins(pluginManifest);
|
||||
|
||||
const authenticatedPlugins = uniquePlugins.map((plugin) => {
|
||||
if (checkPluginAuth(plugin)) {
|
||||
|
|
@ -126,8 +123,12 @@ const getAvailableTools = async (req, res) => {
|
|||
}
|
||||
});
|
||||
|
||||
const toolDefinitions = req.app.locals.availableTools;
|
||||
const tools = authenticatedPlugins.filter(
|
||||
(plugin) => req.app.locals.availableTools[plugin.pluginKey] !== undefined,
|
||||
(plugin) =>
|
||||
toolDefinitions[plugin.pluginKey] !== undefined ||
|
||||
(plugin.toolkit === true &&
|
||||
Object.keys(toolDefinitions).some((key) => key.startsWith(`${plugin.pluginKey}_`))),
|
||||
);
|
||||
|
||||
await cache.set(CacheKeys.TOOLS, tools);
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ const { getStrategyFunctions } = require('~/server/services/Files/strategies');
|
|||
const { deleteAssistantActions } = require('~/server/services/ActionService');
|
||||
const { updateAssistantDoc, getAssistants } = require('~/models/Assistant');
|
||||
const { getOpenAIClient, fetchAssistants } = require('./helpers');
|
||||
const { manifestToolMap } = require('~/app/clients/tools');
|
||||
const { deleteFileByFilter } = require('~/models/File');
|
||||
const { logger } = require('~/config');
|
||||
|
||||
|
|
@ -35,9 +36,21 @@ const createAssistant = async (req, res) => {
|
|||
return tool;
|
||||
}
|
||||
|
||||
return req.app.locals.availableTools[tool];
|
||||
const toolDefinitions = req.app.locals.availableTools;
|
||||
const toolDef = toolDefinitions[tool];
|
||||
if (!toolDef && manifestToolMap[tool] && manifestToolMap[tool].toolkit === true) {
|
||||
return (
|
||||
Object.entries(toolDefinitions)
|
||||
.filter(([key]) => key.startsWith(`${tool}_`))
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
.map(([_, val]) => val)
|
||||
);
|
||||
}
|
||||
|
||||
return toolDef;
|
||||
})
|
||||
.filter((tool) => tool);
|
||||
.filter((tool) => tool)
|
||||
.flat();
|
||||
|
||||
let azureModelIdentifier = null;
|
||||
if (openai.locals?.azureOptions) {
|
||||
|
|
@ -128,9 +141,21 @@ const patchAssistant = async (req, res) => {
|
|||
return tool;
|
||||
}
|
||||
|
||||
return req.app.locals.availableTools[tool];
|
||||
const toolDefinitions = req.app.locals.availableTools;
|
||||
const toolDef = toolDefinitions[tool];
|
||||
if (!toolDef && manifestToolMap[tool] && manifestToolMap[tool].toolkit === true) {
|
||||
return (
|
||||
Object.entries(toolDefinitions)
|
||||
.filter(([key]) => key.startsWith(`${tool}_`))
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
.map(([_, val]) => val)
|
||||
);
|
||||
}
|
||||
|
||||
return toolDef;
|
||||
})
|
||||
.filter((tool) => tool);
|
||||
.filter((tool) => tool)
|
||||
.flat();
|
||||
|
||||
if (openai.locals?.azureOptions && updateData.model) {
|
||||
updateData.model = openai.locals.azureOptions.azureOpenAIApiDeploymentName;
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ const { ToolCallTypes } = require('librechat-data-provider');
|
|||
const validateAuthor = require('~/server/middleware/assistants/validateAuthor');
|
||||
const { validateAndUpdateTool } = require('~/server/services/ActionService');
|
||||
const { updateAssistantDoc } = require('~/models/Assistant');
|
||||
const { manifestToolMap } = require('~/app/clients/tools');
|
||||
const { getOpenAIClient } = require('./helpers');
|
||||
const { logger } = require('~/config');
|
||||
|
||||
|
|
@ -32,9 +33,21 @@ const createAssistant = async (req, res) => {
|
|||
return tool;
|
||||
}
|
||||
|
||||
return req.app.locals.availableTools[tool];
|
||||
const toolDefinitions = req.app.locals.availableTools;
|
||||
const toolDef = toolDefinitions[tool];
|
||||
if (!toolDef && manifestToolMap[tool] && manifestToolMap[tool].toolkit === true) {
|
||||
return (
|
||||
Object.entries(toolDefinitions)
|
||||
.filter(([key]) => key.startsWith(`${tool}_`))
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
.map(([_, val]) => val)
|
||||
);
|
||||
}
|
||||
|
||||
return toolDef;
|
||||
})
|
||||
.filter((tool) => tool);
|
||||
.filter((tool) => tool)
|
||||
.flat();
|
||||
|
||||
let azureModelIdentifier = null;
|
||||
if (openai.locals?.azureOptions) {
|
||||
|
|
@ -112,9 +125,30 @@ const updateAssistant = async ({ req, openai, assistant_id, updateData }) => {
|
|||
|
||||
let hasFileSearch = false;
|
||||
for (const tool of updateData.tools ?? []) {
|
||||
let actualTool = typeof tool === 'string' ? req.app.locals.availableTools[tool] : tool;
|
||||
const toolDefinitions = req.app.locals.availableTools;
|
||||
let actualTool = typeof tool === 'string' ? toolDefinitions[tool] : tool;
|
||||
|
||||
if (!actualTool) {
|
||||
if (!actualTool && manifestToolMap[tool] && manifestToolMap[tool].toolkit === true) {
|
||||
actualTool = Object.entries(toolDefinitions)
|
||||
.filter(([key]) => key.startsWith(`${tool}_`))
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
.map(([_, val]) => val);
|
||||
} else if (!actualTool) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (Array.isArray(actualTool)) {
|
||||
for (const subTool of actualTool) {
|
||||
if (!subTool.function) {
|
||||
tools.push(subTool);
|
||||
continue;
|
||||
}
|
||||
|
||||
const updatedTool = await validateAndUpdateTool({ req, tool: subTool, assistant_id });
|
||||
if (updatedTool) {
|
||||
tools.push(updatedTool);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -75,8 +75,9 @@ const createAbortController = (req, res, getAbortData, getReqData) => {
|
|||
|
||||
const abortKey = userMessage?.conversationId ?? req.user.id;
|
||||
const prevRequest = abortControllers.get(abortKey);
|
||||
const { overrideUserMessageId } = req?.body ?? {};
|
||||
|
||||
if (prevRequest && prevRequest?.abortController) {
|
||||
if (overrideUserMessageId != null && prevRequest && prevRequest?.abortController) {
|
||||
const data = prevRequest.abortController.getAbortData();
|
||||
getReqData({ userMessage: data?.userMessage });
|
||||
const addedAbortKey = `${abortKey}:${responseMessageId}`;
|
||||
|
|
|
|||
|
|
@ -1,6 +1,4 @@
|
|||
const { EModelEndpoint } = require('librechat-data-provider');
|
||||
const { addOpenAPISpecs } = require('~/app/clients/tools/util/addOpenAPISpecs');
|
||||
const { availableTools } = require('~/app/clients/tools');
|
||||
const { isUserProvided } = require('~/server/utils');
|
||||
const { config } = require('./EndpointService');
|
||||
|
||||
|
|
@ -28,22 +26,12 @@ async function loadAsyncEndpoints(req) {
|
|||
}
|
||||
}
|
||||
|
||||
const tools = await addOpenAPISpecs(availableTools);
|
||||
function transformToolsToMap(tools) {
|
||||
return tools.reduce((map, obj) => {
|
||||
map[obj.pluginKey] = obj.name;
|
||||
return map;
|
||||
}, {});
|
||||
}
|
||||
const plugins = transformToolsToMap(tools);
|
||||
|
||||
const google = serviceKey || googleKey ? { userProvide: googleUserProvides } : false;
|
||||
|
||||
const useAzure = req.app.locals[EModelEndpoint.azureOpenAI]?.plugins;
|
||||
const gptPlugins =
|
||||
useAzure || openAIApiKey || azureOpenAIApiKey
|
||||
? {
|
||||
plugins,
|
||||
availableAgents: ['classic', 'functions'],
|
||||
userProvide: useAzure ? false : userProvidedOpenAI,
|
||||
userProvideURL: useAzure
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { zodToJsonSchema } = require('zod-to-json-schema');
|
||||
const { tool: toolFn, Tool } = require('@langchain/core/tools');
|
||||
const { tool: toolFn, Tool, DynamicStructuredTool } = require('@langchain/core/tools');
|
||||
const { Calculator } = require('@langchain/community/tools/calculator');
|
||||
const {
|
||||
Tools,
|
||||
|
|
@ -16,6 +16,7 @@ const {
|
|||
validateAndParseOpenAPISpec,
|
||||
} = require('librechat-data-provider');
|
||||
const { processFileURL, uploadImageBuffer } = require('~/server/services/Files/process');
|
||||
const { createYouTubeTools, manifestToolMap, toolkits } = require('~/app/clients/tools');
|
||||
const { loadActionSets, createActionTool, domainParser } = require('./ActionService');
|
||||
const { getEndpointsConfig } = require('~/server/services/Config');
|
||||
const { recordUsage } = require('~/server/services/Threads');
|
||||
|
|
@ -97,7 +98,7 @@ function loadAndFormatTools({ directory, adminFilter = [], adminIncluded = [] })
|
|||
}
|
||||
|
||||
/** Basic Tools; schema: { input: string } */
|
||||
const basicToolInstances = [new Calculator()];
|
||||
const basicToolInstances = [new Calculator(), ...createYouTubeTools({ override: true })];
|
||||
for (const toolInstance of basicToolInstances) {
|
||||
const formattedTool = formatToOpenAIAssistantTool(toolInstance);
|
||||
tools.push(formattedTool);
|
||||
|
|
@ -173,7 +174,26 @@ async function processRequiredActions(client, requiredActions) {
|
|||
`[required actions] user: ${client.req.user.id} | thread_id: ${requiredActions[0].thread_id} | run_id: ${requiredActions[0].run_id}`,
|
||||
requiredActions,
|
||||
);
|
||||
const tools = requiredActions.map((action) => action.tool);
|
||||
const toolDefinitions = client.req.app.locals.availableTools;
|
||||
const seenToolkits = new Set();
|
||||
const tools = requiredActions
|
||||
.map((action) => {
|
||||
const toolName = action.tool;
|
||||
const toolDef = toolDefinitions[toolName];
|
||||
if (toolDef && !manifestToolMap[toolName]) {
|
||||
for (const toolkit of toolkits) {
|
||||
if (seenToolkits.has(toolkit.pluginKey)) {
|
||||
return;
|
||||
} else if (toolName.startsWith(`${toolkit.pluginKey}_`)) {
|
||||
seenToolkits.add(toolkit.pluginKey);
|
||||
return toolkit.pluginKey;
|
||||
}
|
||||
}
|
||||
}
|
||||
return toolName;
|
||||
})
|
||||
.filter((toolName) => !!toolName);
|
||||
|
||||
const { loadedTools } = await loadTools({
|
||||
user: client.req.user.id,
|
||||
model: client.req.body.model ?? 'gpt-4o-mini',
|
||||
|
|
@ -441,6 +461,11 @@ async function loadAgentTools({ req, agent, tool_resources, openAIApiKey }) {
|
|||
continue;
|
||||
}
|
||||
|
||||
if (tool instanceof DynamicStructuredTool) {
|
||||
agentTools.push(tool);
|
||||
continue;
|
||||
}
|
||||
|
||||
const toolDefinition = {
|
||||
name: tool.name,
|
||||
schema: tool.schema,
|
||||
|
|
|
|||
|
|
@ -29,7 +29,6 @@
|
|||
"endpoint": "openAI",
|
||||
"error": false,
|
||||
"isCreatedByUser": true,
|
||||
"isEdited": false,
|
||||
"model": null,
|
||||
"parentMessageId": "00000000-0000-0000-0000-000000000000",
|
||||
"sender": "user",
|
||||
|
|
@ -47,7 +46,6 @@
|
|||
"endpoint": "openAI",
|
||||
"error": false,
|
||||
"isCreatedByUser": false,
|
||||
"isEdited": false,
|
||||
"model": null,
|
||||
"parentMessageId": "b123942f-ca1a-4b16-9e1f-ea4af5171168",
|
||||
"sender": "GPT-3.5",
|
||||
|
|
@ -65,7 +63,6 @@
|
|||
"endpoint": "openAI",
|
||||
"error": false,
|
||||
"isCreatedByUser": true,
|
||||
"isEdited": false,
|
||||
"model": null,
|
||||
"parentMessageId": "549a4f45-cf93-4e3b-ae62-1abf02afbfc8",
|
||||
"sender": "user",
|
||||
|
|
@ -83,7 +80,6 @@
|
|||
"endpoint": "openAI",
|
||||
"error": false,
|
||||
"isCreatedByUser": false,
|
||||
"isEdited": false,
|
||||
"model": null,
|
||||
"parentMessageId": "880e5357-3e0c-4218-b351-fd3fc184adef",
|
||||
"sender": "GPT-3.5",
|
||||
|
|
@ -101,7 +97,6 @@
|
|||
"endpoint": "openAI",
|
||||
"error": false,
|
||||
"isCreatedByUser": true,
|
||||
"isEdited": false,
|
||||
"model": null,
|
||||
"parentMessageId": "e9796d11-3bdf-4e25-9f0e-4802bbbb8c6d",
|
||||
"sender": "user",
|
||||
|
|
@ -119,7 +114,6 @@
|
|||
"endpoint": "openAI",
|
||||
"error": false,
|
||||
"isCreatedByUser": false,
|
||||
"isEdited": false,
|
||||
"model": null,
|
||||
"parentMessageId": "04408c06-62dc-4961-8ef5-4336b68e7a0a",
|
||||
"sender": "GPT-3.5",
|
||||
|
|
|
|||
|
|
@ -27,7 +27,6 @@
|
|||
"endpoint": "azureOpenAI",
|
||||
"error": false,
|
||||
"isCreatedByUser": true,
|
||||
"isEdited": false,
|
||||
"model": null,
|
||||
"parentMessageId": "00000000-0000-0000-0000-000000000000",
|
||||
"sender": "User",
|
||||
|
|
@ -42,7 +41,6 @@
|
|||
"createdAt": "2024-05-28T18:08:55.390Z",
|
||||
"error": false,
|
||||
"isCreatedByUser": false,
|
||||
"isEdited": false,
|
||||
"model": "gpt-4o",
|
||||
"parentMessageId": "115a6247-8fb0-4937-a536-12956669098d",
|
||||
"sender": "GPT-4",
|
||||
|
|
@ -59,7 +57,6 @@
|
|||
"createdAt": "2024-05-28T18:09:27.444Z",
|
||||
"error": false,
|
||||
"isCreatedByUser": false,
|
||||
"isEdited": false,
|
||||
"model": "gpt-4o",
|
||||
"parentMessageId": "115a6247-8fb0-4937-a536-12956669098d",
|
||||
"sender": "GPT-4",
|
||||
|
|
@ -74,7 +71,6 @@
|
|||
"endpoint": "azureOpenAI",
|
||||
"error": false,
|
||||
"isCreatedByUser": true,
|
||||
"isEdited": false,
|
||||
"model": null,
|
||||
"parentMessageId": "00000000-0000-0000-0000-000000000000",
|
||||
"sender": "User",
|
||||
|
|
@ -89,7 +85,6 @@
|
|||
"createdAt": "2024-05-28T18:14:08.403Z",
|
||||
"error": false,
|
||||
"isCreatedByUser": false,
|
||||
"isEdited": true,
|
||||
"model": "gpt-4o",
|
||||
"parentMessageId": "599e1908-8c52-4a73-ba6b-f6dffbd79ba0",
|
||||
"sender": "GPT-4",
|
||||
|
|
|
|||
|
|
@ -29,7 +29,6 @@
|
|||
"endpoint": "openAI",
|
||||
"error": false,
|
||||
"isCreatedByUser": true,
|
||||
"isEdited": false,
|
||||
"model": null,
|
||||
"parentMessageId": "00000000-0000-0000-0000-000000000000",
|
||||
"sender": "User",
|
||||
|
|
@ -47,7 +46,6 @@
|
|||
"createdAt": "2024-05-01T16:35:12.604Z",
|
||||
"error": false,
|
||||
"isCreatedByUser": false,
|
||||
"isEdited": false,
|
||||
"model": "gpt-4-turbo",
|
||||
"parentMessageId": "9501f99d-9bbb-40cb-bbb2-16d79aeceb72",
|
||||
"sender": "Software Engineer",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue