🎥 feat: YouTube Tool (#5582)

* adding youtube tool

* refactor: use short `url` param instead of `videoUrl`

* refactor: move API key retrieval to a separate credentials module

* refactor: remove unnecessary `isEdited` message property

* refactor: remove unnecessary `isEdited` message property pt. 2

* refactor: YouTube Tool with new `tool()` generator, handle tools already created by new `tool` generator

* fix: only reset request data for multi-convo messages

* refactor: enhance YouTube tool by adding transcript parsing and returning structured JSON responses

* refactor: update transcript parsing to handle raw response and clean up text output

* feat: support toolkits and refactor YouTube tool as a toolkit for better LLM usage

* refactor: remove unused OpenAPI specs and streamline tools transformation in loadAsyncEndpoints

* refactor: implement manifestToolMap for better tool management and streamline authentication handling

* feat: support toolkits for assistants

* refactor: rename loadedTools to toolDefinitions for clarity in PluginController and assistant controllers

* feat: complete support of toolkits for assistants

---------

Co-authored-by: Danilo Pejakovic <danilo.pejakovic@leoninestudios.com>

.env.example

@@ -255,6 +255,10 @@ AZURE_AI_SEARCH_SEARCH_OPTION_SELECT=
 GOOGLE_SEARCH_API_KEY=
 GOOGLE_CSE_ID=
 
+# YOUTUBE
+#-----------------
+YOUTUBE_API_KEY=
+
 # SerpAPI
 #-----------------
 SERPAPI_API_KEY=

api/app/clients/PluginsClient.js

@@ -280,7 +280,6 @@ class PluginsClient extends OpenAIClient {
     logger.debug('[PluginsClient] sendMessage', { userMessageText: message, opts });
     const {
       user,
-      isEdited,
       conversationId,
       responseMessageId,
       saveOptions,
@@ -359,7 +358,6 @@ class PluginsClient extends OpenAIClient {
       conversationId,
       parentMessageId: userMessage.messageId,
       isCreatedByUser: false,
-      isEdited,
       model: this.modelOptions.model,
       sender: this.sender,
       promptTokens,

api/app/clients/prompts/formatMessages.spec.js

@@ -60,7 +60,6 @@ describe('formatMessage', () => {
         error: false,
         finish_reason: null,
         isCreatedByUser: true,
-        isEdited: false,
         model: null,
         parentMessageId: Constants.NO_PARENT,
         sender: 'User',

api/app/clients/tools/index.js

@@ -2,23 +2,40 @@ const availableTools = require('./manifest.json');
 
 // Structured Tools
 const DALLE3 = require('./structured/DALLE3');
+const OpenWeather = require('./structured/OpenWeather');
+const createYouTubeTools = require('./structured/YouTube');
 const StructuredWolfram = require('./structured/Wolfram');
 const StructuredACS = require('./structured/AzureAISearch');
 const StructuredSD = require('./structured/StableDiffusion');
 const GoogleSearchAPI = require('./structured/GoogleSearch');
 const TraversaalSearch = require('./structured/TraversaalSearch');
 const TavilySearchResults = require('./structured/TavilySearchResults');
-const OpenWeather = require('./structured/OpenWeather');
+
+/** @type {Record<string, TPlugin | undefined>} */
+const manifestToolMap = {};
+
+/** @type {Array<TPlugin>} */
+const toolkits = [];
+
+availableTools.forEach((tool) => {
+  manifestToolMap[tool.pluginKey] = tool;
+  if (tool.toolkit === true) {
+    toolkits.push(tool);
+  }
+});
 
 module.exports = {
+  toolkits,
   availableTools,
+  manifestToolMap,
   // Structured Tools
   DALLE3,
+  OpenWeather,
   StructuredSD,
   StructuredACS,
   GoogleSearchAPI,
   TraversaalSearch,
   StructuredWolfram,
+  createYouTubeTools,
   TavilySearchResults,
-  OpenWeather,
 };

api/app/clients/tools/manifest.json

@@ -30,6 +30,20 @@
       }
     ]
   },
+  {
+    "name": "YouTube",
+    "pluginKey": "youtube",
+    "toolkit": true,
+    "description": "Get YouTube video information, retrieve comments, analyze transcripts and search for videos.",
+    "icon": "https://www.youtube.com/s/desktop/7449ebf7/img/favicon_144x144.png",
+    "authConfig": [
+      {
+        "authField": "YOUTUBE_API_KEY",
+        "label": "YouTube API Key",
+        "description": "Your YouTube Data API v3 key."
+      }
+    ]
+  },
   {
     "name": "Wolfram",
     "pluginKey": "wolfram",

api/app/clients/tools/structured/TavilySearch.js

@@ -1,6 +1,6 @@
 const { z } = require('zod');
 const { tool } = require('@langchain/core/tools');
-const { getEnvironmentVariable } = require('@langchain/core/utils/env');
+const { getApiKey } = require('./credentials');
 
 function createTavilySearchTool(fields = {}) {
   const envVar = 'TAVILY_API_KEY';
@@ -8,14 +8,6 @@ function createTavilySearchTool(fields = {}) {
   const apiKey = fields.apiKey ?? getApiKey(envVar, override);
   const kwargs = fields?.kwargs ?? {};
 
-  function getApiKey(envVar, override) {
-    const key = getEnvironmentVariable(envVar);
-    if (!key && !override) {
-      throw new Error(`Missing ${envVar} environment variable.`);
-    }
-    return key;
-  }
-
   return tool(
     async (input) => {
       const { query, ...rest } = input;

api/app/clients/tools/structured/YouTube.js

@@ -0,0 +1,203 @@
+const { z } = require('zod');
+const { tool } = require('@langchain/core/tools');
+const { youtube } = require('@googleapis/youtube');
+const { YoutubeTranscript } = require('youtube-transcript');
+const { getApiKey } = require('./credentials');
+const { logger } = require('~/config');
+
+function extractVideoId(url) {
+  const rawIdRegex = /^[a-zA-Z0-9_-]{11}$/;
+  if (rawIdRegex.test(url)) {
+    return url;
+  }
+
+  const regex = new RegExp(
+    '(?:youtu\\.be/|youtube(?:\\.com)?/(?:' +
+      '(?:watch\\?v=)|(?:embed/)|(?:shorts/)|(?:live/)|(?:v/)|(?:/))?)' +
+      '([a-zA-Z0-9_-]{11})(?:\\S+)?$',
+  );
+  const match = url.match(regex);
+  return match ? match[1] : null;
+}
+
+function parseTranscript(transcriptResponse) {
+  if (!Array.isArray(transcriptResponse)) {
+    return '';
+  }
+
+  return transcriptResponse
+    .map((entry) => entry.text.trim())
+    .filter((text) => text)
+    .join(' ')
+    .replaceAll(''', '\'');
+}
+
+function createYouTubeTools(fields = {}) {
+  const envVar = 'YOUTUBE_API_KEY';
+  const override = fields.override ?? false;
+  const apiKey = fields.apiKey ?? fields[envVar] ?? getApiKey(envVar, override);
+
+  const youtubeClient = youtube({
+    version: 'v3',
+    auth: apiKey,
+  });
+
+  const searchTool = tool(
+    async ({ query, maxResults = 5 }) => {
+      const response = await youtubeClient.search.list({
+        part: 'snippet',
+        q: query,
+        type: 'video',
+        maxResults: maxResults || 5,
+      });
+      const result = response.data.items.map((item) => ({
+        title: item.snippet.title,
+        description: item.snippet.description,
+        url: `https://www.youtube.com/watch?v=${item.id.videoId}`,
+      }));
+      return JSON.stringify(result, null, 2);
+    },
+    {
+      name: 'youtube_search',
+      description: `Search for YouTube videos by keyword or phrase.
+- Required: query (search terms to find videos)
+- Optional: maxResults (number of videos to return, 1-50, default: 5)
+- Returns: List of videos with titles, descriptions, and URLs
+- Use for: Finding specific videos, exploring content, research
+Example: query="cooking pasta tutorials" maxResults=3`,
+      schema: z.object({
+        query: z.string().describe('Search query terms'),
+        maxResults: z.number().int().min(1).max(50).optional().describe('Number of results (1-50)'),
+      }),
+    },
+  );
+
+  const infoTool = tool(
+    async ({ url }) => {
+      const videoId = extractVideoId(url);
+      if (!videoId) {
+        throw new Error('Invalid YouTube URL or video ID');
+      }
+
+      const response = await youtubeClient.videos.list({
+        part: 'snippet,statistics',
+        id: videoId,
+      });
+
+      if (!response.data.items?.length) {
+        throw new Error('Video not found');
+      }
+      const video = response.data.items[0];
+
+      const result = {
+        title: video.snippet.title,
+        description: video.snippet.description,
+        views: video.statistics.viewCount,
+        likes: video.statistics.likeCount,
+        comments: video.statistics.commentCount,
+      };
+      return JSON.stringify(result, null, 2);
+    },
+    {
+      name: 'youtube_info',
+      description: `Get detailed metadata and statistics for a specific YouTube video.
+- Required: url (full YouTube URL or video ID)
+- Returns: Video title, description, view count, like count, comment count
+- Use for: Getting video metrics and basic metadata
+- DO NOT USE FOR VIDEO SUMMARIES, USE TRANSCRIPTS FOR COMPREHENSIVE ANALYSIS
+- Accepts both full URLs and video IDs
+Example: url="https://youtube.com/watch?v=abc123" or url="abc123"`,
+      schema: z.object({
+        url: z.string().describe('YouTube video URL or ID'),
+      }),
+    },
+  );
+
+  const commentsTool = tool(
+    async ({ url, maxResults = 10 }) => {
+      const videoId = extractVideoId(url);
+      if (!videoId) {
+        throw new Error('Invalid YouTube URL or video ID');
+      }
+
+      const response = await youtubeClient.commentThreads.list({
+        part: 'snippet',
+        videoId,
+        maxResults: maxResults || 10,
+      });
+
+      const result = response.data.items.map((item) => ({
+        author: item.snippet.topLevelComment.snippet.authorDisplayName,
+        text: item.snippet.topLevelComment.snippet.textDisplay,
+        likes: item.snippet.topLevelComment.snippet.likeCount,
+      }));
+      return JSON.stringify(result, null, 2);
+    },
+    {
+      name: 'youtube_comments',
+      description: `Retrieve top-level comments from a YouTube video.
+- Required: url (full YouTube URL or video ID)
+- Optional: maxResults (number of comments, 1-50, default: 10)
+- Returns: Comment text, author names, like counts
+- Use for: Sentiment analysis, audience feedback, engagement review
+Example: url="abc123" maxResults=20`,
+      schema: z.object({
+        url: z.string().describe('YouTube video URL or ID'),
+        maxResults: z
+          .number()
+          .int()
+          .min(1)
+          .max(50)
+          .optional()
+          .describe('Number of comments to retrieve'),
+      }),
+    },
+  );
+
+  const transcriptTool = tool(
+    async ({ url }) => {
+      const videoId = extractVideoId(url);
+      if (!videoId) {
+        throw new Error('Invalid YouTube URL or video ID');
+      }
+
+      try {
+        try {
+          const transcript = await YoutubeTranscript.fetchTranscript(videoId, { lang: 'en' });
+          return parseTranscript(transcript);
+        } catch (e) {
+          logger.error(e);
+        }
+
+        try {
+          const transcript = await YoutubeTranscript.fetchTranscript(videoId, { lang: 'de' });
+          return parseTranscript(transcript);
+        } catch (e) {
+          logger.error(e);
+        }
+
+        const transcript = await YoutubeTranscript.fetchTranscript(videoId);
+        return parseTranscript(transcript);
+      } catch (error) {
+        throw new Error(`Failed to fetch transcript: ${error.message}`);
+      }
+    },
+    {
+      name: 'youtube_transcript',
+      description: `Fetch and parse the transcript/captions of a YouTube video.
+- Required: url (full YouTube URL or video ID)
+- Returns: Full video transcript as plain text
+- Use for: Content analysis, summarization, translation reference
+- This is the "Go-to" tool for analyzing actual video content
+- Attempts to fetch English first, then German, then any available language
+Example: url="https://youtube.com/watch?v=abc123"`,
+      schema: z.object({
+        url: z.string().describe('YouTube video URL or ID'),
+      }),
+    },
+  );
+
+  return [searchTool, infoTool, commentsTool, transcriptTool];
+}
+
+module.exports = createYouTubeTools;

api/app/clients/tools/structured/credentials.js

@@ -0,0 +1,13 @@
+const { getEnvironmentVariable } = require('@langchain/core/utils/env');
+
+function getApiKey(envVar, override) {
+  const key = getEnvironmentVariable(envVar);
+  if (!key && !override) {
+    throw new Error(`Missing ${envVar} environment variable.`);
+  }
+  return key;
+}
+
+module.exports = {
+  getApiKey,
+};

api/app/clients/tools/util/handleTools.js

@@ -5,16 +5,18 @@ const { createCodeExecutionTool, EnvVar } = require('@librechat/agents');
 const { getUserPluginAuthValue } = require('~/server/services/PluginService');
 const {
   availableTools,
+  manifestToolMap,
   // Basic Tools
   GoogleSearchAPI,
   // Structured Tools
   DALLE3,
+  OpenWeather,
   StructuredSD,
   StructuredACS,
   TraversaalSearch,
   StructuredWolfram,
+  createYouTubeTools,
   TavilySearchResults,
-  OpenWeather,
 } = require('../');
 const { primeFiles: primeCodeFiles } = require('~/server/services/Files/Code/process');
 const { createFileSearchTool, primeFiles: primeSearchFiles } = require('./fileSearch');
@@ -146,6 +148,14 @@ const loadToolWithAuth = (userId, authFields, ToolConstructor, options = {}) =>
   };
 };
 
+/**
+ * @param {string} toolKey
+ * @returns {Array<string>}
+ */
+const getAuthFields = (toolKey) => {
+  return manifestToolMap[toolKey]?.authConfig.map((auth) => auth.authField) ?? [];
+};
+
 /**
  *
  * @param {object} object
@@ -174,19 +184,21 @@ const loadTools = async ({
   const toolConstructors = {
     calculator: Calculator,
     google: GoogleSearchAPI,
+    open_weather: OpenWeather,
     wolfram: StructuredWolfram,
     'stable-diffusion': StructuredSD,
     'azure-ai-search': StructuredACS,
     traversaal_search: TraversaalSearch,
     tavily_search_results_json: TavilySearchResults,
-    open_weather: OpenWeather,
   };
 
   const customConstructors = {
     serpapi: async () => {
-      let apiKey = process.env.SERPAPI_API_KEY;
+      const authFields = getAuthFields('serpapi');
+      let envVar = authFields[0] ?? '';
+      let apiKey = process.env[envVar];
       if (!apiKey) {
-        apiKey = await getUserPluginAuthValue(user, 'SERPAPI_API_KEY');
+        apiKey = await getUserPluginAuthValue(user, envVar);
       }
       return new SerpAPI(apiKey, {
         location: 'Austin,Texas,United States',
@@ -194,6 +206,11 @@ const loadTools = async ({
         gl: 'us',
       });
     },
+    youtube: async () => {
+      const authFields = getAuthFields('youtube');
+      const authValues = await loadAuthValues({ userId: user, authFields });
+      return createYouTubeTools(authValues);
+    },
   };
 
   const requestedTools = {};
@@ -218,16 +235,6 @@ const loadTools = async ({
     'stable-diffusion': imageGenOptions,
   };
 
-  const toolAuthFields = {};
-
-  availableTools.forEach((tool) => {
-    if (customConstructors[tool.pluginKey]) {
-      return;
-    }
-
-    toolAuthFields[tool.pluginKey] = tool.authConfig.map((auth) => auth.authField);
-  });
-
   const toolContextMap = {};
   const remainingTools = [];
   const appTools = options.req?.app?.locals?.availableTools ?? {};
@@ -282,7 +289,7 @@ const loadTools = async ({
       const options = toolOptions[tool] || {};
       const toolInstance = loadToolWithAuth(
         user,
-        toolAuthFields[tool],
+        getAuthFields(tool),
         toolConstructors[tool],
         options,
       );

api/models/Message.js

@@ -23,7 +23,6 @@ const idSchema = z.string().uuid();
  * @param {string} [params.error] - Any error associated with the message.
  * @param {boolean} [params.unfinished] - Indicates if the message is unfinished.
  * @param {Object[]} [params.files] - An array of files associated with the message.
- * @param {boolean} [params.isEdited] - Indicates if the message was edited.
  * @param {string} [params.finish_reason] - Reason for finishing the message.
  * @param {number} [params.tokenCount] - The number of tokens in the message.
  * @param {string} [params.plugin] - Plugin associated with the message.
@@ -77,7 +76,7 @@ async function saveMessage(req, params, metadata) {
  * @returns {Promise<Object>} The result of the bulk write operation.
  * @throws {Error} If there is an error in saving messages in bulk.
  */
-async function bulkSaveMessages(messages, overrideTimestamp=false) {
+async function bulkSaveMessages(messages, overrideTimestamp = false) {
   try {
     const bulkOps = messages.map((message) => ({
       updateOne: {
@@ -182,7 +181,6 @@ async function updateMessageText(req, { messageId, text }) {
 async function updateMessage(req, message, metadata) {
   try {
     const { messageId, ...update } = message;
-    update.isEdited = true;
     const updatedMessage = await Message.findOneAndUpdate(
       { messageId, user: req.user.id },
       update,
@@ -203,7 +201,6 @@ async function updateMessage(req, message, metadata) {
       text: updatedMessage.text,
       isCreatedByUser: updatedMessage.isCreatedByUser,
       tokenCount: updatedMessage.tokenCount,
-      isEdited: true,
     };
   } catch (err) {
     logger.error('Error updating message:', err);

api/models/Message.spec.js

@@ -100,7 +100,6 @@ describe('Message Operations', () => {
         expect.objectContaining({
           messageId: 'msg123',
           text: 'Hello, world!',
-          isEdited: true,
         }),
       );
     });

api/models/schema/messageSchema.js

@@ -62,10 +62,6 @@ const messageSchema = mongoose.Schema(
       required: true,
       default: false,
     },
-    isEdited: {
-      type: Boolean,
-      default: false,
-    },
     unfinished: {
       type: Boolean,
       default: false,

api/package.json

@@ -37,6 +37,7 @@
     "@anthropic-ai/sdk": "^0.32.1",
     "@azure/search-documents": "^12.0.0",
     "@google/generative-ai": "^0.21.0",
+    "@googleapis/youtube": "^20.0.0",
     "@keyv/mongo": "^2.1.8",
     "@keyv/redis": "^2.8.1",
     "@langchain/community": "^0.3.14",
@@ -105,6 +106,7 @@
     "ua-parser-js": "^1.0.36",
     "winston": "^3.11.0",
     "winston-daily-rotate-file": "^4.7.1",
+    "youtube-transcript": "^1.2.1",
     "zod": "^3.22.4"
   },
   "devDependencies": {

api/server/controllers/PluginController.js

@@ -1,7 +1,7 @@
-const { promises: fs } = require('fs');
 const { CacheKeys, AuthType } = require('librechat-data-provider');
 const { addOpenAPISpecs } = require('~/app/clients/tools/util/addOpenAPISpecs');
 const { getCustomConfig } = require('~/server/services/Config');
+const { availableTools } = require('~/app/clients/tools');
 const { getMCPManager } = require('~/config');
 const { getLogStores } = require('~/cache');
 
@@ -59,10 +59,9 @@ const getAvailablePluginsController = async (req, res) => {
 
     /** @type {{ filteredTools: string[], includedTools: string[] }} */
     const { filteredTools = [], includedTools = [] } = req.app.locals;
-    const pluginManifest = await fs.readFile(req.app.locals.paths.pluginManifest, 'utf8');
-    const jsonData = JSON.parse(pluginManifest);
+    const pluginManifest = availableTools;
 
-    const uniquePlugins = filterUniquePlugins(jsonData);
+    const uniquePlugins = filterUniquePlugins(pluginManifest);
     let authenticatedPlugins = [];
     for (const plugin of uniquePlugins) {
       authenticatedPlugins.push(
@@ -106,17 +105,15 @@ const getAvailableTools = async (req, res) => {
       return;
     }
 
-    const pluginManifest = await fs.readFile(req.app.locals.paths.pluginManifest, 'utf8');
-
-    const jsonData = JSON.parse(pluginManifest);
+    const pluginManifest = availableTools;
     const customConfig = await getCustomConfig();
     if (customConfig?.mcpServers != null) {
       const mcpManager = await getMCPManager();
-      await mcpManager.loadManifestTools(jsonData);
+      await mcpManager.loadManifestTools(pluginManifest);
     }
 
     /** @type {TPlugin[]} */
-    const uniquePlugins = filterUniquePlugins(jsonData);
+    const uniquePlugins = filterUniquePlugins(pluginManifest);
 
     const authenticatedPlugins = uniquePlugins.map((plugin) => {
       if (checkPluginAuth(plugin)) {
@@ -126,8 +123,12 @@ const getAvailableTools = async (req, res) => {
       }
     });
 
+    const toolDefinitions = req.app.locals.availableTools;
     const tools = authenticatedPlugins.filter(
-      (plugin) => req.app.locals.availableTools[plugin.pluginKey] !== undefined,
+      (plugin) =>
+        toolDefinitions[plugin.pluginKey] !== undefined ||
+        (plugin.toolkit === true &&
+          Object.keys(toolDefinitions).some((key) => key.startsWith(`${plugin.pluginKey}_`))),
     );
 
     await cache.set(CacheKeys.TOOLS, tools);

api/server/controllers/assistants/v1.js

@@ -6,6 +6,7 @@ const { getStrategyFunctions } = require('~/server/services/Files/strategies');
 const { deleteAssistantActions } = require('~/server/services/ActionService');
 const { updateAssistantDoc, getAssistants } = require('~/models/Assistant');
 const { getOpenAIClient, fetchAssistants } = require('./helpers');
+const { manifestToolMap } = require('~/app/clients/tools');
 const { deleteFileByFilter } = require('~/models/File');
 const { logger } = require('~/config');
 
@@ -35,9 +36,21 @@ const createAssistant = async (req, res) => {
           return tool;
         }
 
-        return req.app.locals.availableTools[tool];
+        const toolDefinitions = req.app.locals.availableTools;
+        const toolDef = toolDefinitions[tool];
+        if (!toolDef && manifestToolMap[tool] && manifestToolMap[tool].toolkit === true) {
+          return (
+            Object.entries(toolDefinitions)
+              .filter(([key]) => key.startsWith(`${tool}_`))
+              // eslint-disable-next-line no-unused-vars
+              .map(([_, val]) => val)
+          );
+        }
+
+        return toolDef;
       })
-      .filter((tool) => tool);
+      .filter((tool) => tool)
+      .flat();
 
     let azureModelIdentifier = null;
     if (openai.locals?.azureOptions) {
@@ -128,9 +141,21 @@ const patchAssistant = async (req, res) => {
           return tool;
         }
 
-        return req.app.locals.availableTools[tool];
+        const toolDefinitions = req.app.locals.availableTools;
+        const toolDef = toolDefinitions[tool];
+        if (!toolDef && manifestToolMap[tool] && manifestToolMap[tool].toolkit === true) {
+          return (
+            Object.entries(toolDefinitions)
+              .filter(([key]) => key.startsWith(`${tool}_`))
+              // eslint-disable-next-line no-unused-vars
+              .map(([_, val]) => val)
+          );
+        }
+
+        return toolDef;
       })
-      .filter((tool) => tool);
+      .filter((tool) => tool)
+      .flat();
 
     if (openai.locals?.azureOptions && updateData.model) {
       updateData.model = openai.locals.azureOptions.azureOpenAIApiDeploymentName;

api/server/controllers/assistants/v2.js

@@ -2,6 +2,7 @@ const { ToolCallTypes } = require('librechat-data-provider');
 const validateAuthor = require('~/server/middleware/assistants/validateAuthor');
 const { validateAndUpdateTool } = require('~/server/services/ActionService');
 const { updateAssistantDoc } = require('~/models/Assistant');
+const { manifestToolMap } = require('~/app/clients/tools');
 const { getOpenAIClient } = require('./helpers');
 const { logger } = require('~/config');
 
@@ -32,9 +33,21 @@ const createAssistant = async (req, res) => {
           return tool;
         }
 
-        return req.app.locals.availableTools[tool];
+        const toolDefinitions = req.app.locals.availableTools;
+        const toolDef = toolDefinitions[tool];
+        if (!toolDef && manifestToolMap[tool] && manifestToolMap[tool].toolkit === true) {
+          return (
+            Object.entries(toolDefinitions)
+              .filter(([key]) => key.startsWith(`${tool}_`))
+              // eslint-disable-next-line no-unused-vars
+              .map(([_, val]) => val)
+          );
+        }
+
+        return toolDef;
       })
-      .filter((tool) => tool);
+      .filter((tool) => tool)
+      .flat();
 
     let azureModelIdentifier = null;
     if (openai.locals?.azureOptions) {
@@ -112,9 +125,30 @@ const updateAssistant = async ({ req, openai, assistant_id, updateData }) => {
 
   let hasFileSearch = false;
   for (const tool of updateData.tools ?? []) {
-    let actualTool = typeof tool === 'string' ? req.app.locals.availableTools[tool] : tool;
+    const toolDefinitions = req.app.locals.availableTools;
+    let actualTool = typeof tool === 'string' ? toolDefinitions[tool] : tool;
 
-    if (!actualTool) {
+    if (!actualTool && manifestToolMap[tool] && manifestToolMap[tool].toolkit === true) {
+      actualTool = Object.entries(toolDefinitions)
+        .filter(([key]) => key.startsWith(`${tool}_`))
+        // eslint-disable-next-line no-unused-vars
+        .map(([_, val]) => val);
+    } else if (!actualTool) {
+      continue;
+    }
+
+    if (Array.isArray(actualTool)) {
+      for (const subTool of actualTool) {
+        if (!subTool.function) {
+          tools.push(subTool);
+          continue;
+        }
+
+        const updatedTool = await validateAndUpdateTool({ req, tool: subTool, assistant_id });
+        if (updatedTool) {
+          tools.push(updatedTool);
+        }
+      }
       continue;
     }
 

api/server/middleware/abortMiddleware.js

@@ -75,8 +75,9 @@ const createAbortController = (req, res, getAbortData, getReqData) => {
 
     const abortKey = userMessage?.conversationId ?? req.user.id;
     const prevRequest = abortControllers.get(abortKey);
+    const { overrideUserMessageId } = req?.body ?? {};
 
-    if (prevRequest && prevRequest?.abortController) {
+    if (overrideUserMessageId != null && prevRequest && prevRequest?.abortController) {
       const data = prevRequest.abortController.getAbortData();
       getReqData({ userMessage: data?.userMessage });
       const addedAbortKey = `${abortKey}:${responseMessageId}`;

api/server/services/Config/loadAsyncEndpoints.js

@@ -1,6 +1,4 @@
 const { EModelEndpoint } = require('librechat-data-provider');
-const { addOpenAPISpecs } = require('~/app/clients/tools/util/addOpenAPISpecs');
-const { availableTools } = require('~/app/clients/tools');
 const { isUserProvided } = require('~/server/utils');
 const { config } = require('./EndpointService');
 
@@ -28,22 +26,12 @@ async function loadAsyncEndpoints(req) {
     }
   }
 
-  const tools = await addOpenAPISpecs(availableTools);
-  function transformToolsToMap(tools) {
-    return tools.reduce((map, obj) => {
-      map[obj.pluginKey] = obj.name;
-      return map;
-    }, {});
-  }
-  const plugins = transformToolsToMap(tools);
-
   const google = serviceKey || googleKey ? { userProvide: googleUserProvides } : false;
 
   const useAzure = req.app.locals[EModelEndpoint.azureOpenAI]?.plugins;
   const gptPlugins =
     useAzure || openAIApiKey || azureOpenAIApiKey
       ? {
-        plugins,
         availableAgents: ['classic', 'functions'],
         userProvide: useAzure ? false : userProvidedOpenAI,
         userProvideURL: useAzure

api/server/services/ToolService.js

@@ -1,7 +1,7 @@
 const fs = require('fs');
 const path = require('path');
 const { zodToJsonSchema } = require('zod-to-json-schema');
-const { tool: toolFn, Tool } = require('@langchain/core/tools');
+const { tool: toolFn, Tool, DynamicStructuredTool } = require('@langchain/core/tools');
 const { Calculator } = require('@langchain/community/tools/calculator');
 const {
   Tools,
@@ -16,6 +16,7 @@ const {
   validateAndParseOpenAPISpec,
 } = require('librechat-data-provider');
 const { processFileURL, uploadImageBuffer } = require('~/server/services/Files/process');
+const { createYouTubeTools, manifestToolMap, toolkits } = require('~/app/clients/tools');
 const { loadActionSets, createActionTool, domainParser } = require('./ActionService');
 const { getEndpointsConfig } = require('~/server/services/Config');
 const { recordUsage } = require('~/server/services/Threads');
@@ -97,7 +98,7 @@ function loadAndFormatTools({ directory, adminFilter = [], adminIncluded = [] })
   }
 
   /** Basic Tools; schema: { input: string } */
-  const basicToolInstances = [new Calculator()];
+  const basicToolInstances = [new Calculator(), ...createYouTubeTools({ override: true })];
   for (const toolInstance of basicToolInstances) {
     const formattedTool = formatToOpenAIAssistantTool(toolInstance);
     tools.push(formattedTool);
@@ -173,7 +174,26 @@ async function processRequiredActions(client, requiredActions) {
     `[required actions] user: ${client.req.user.id} | thread_id: ${requiredActions[0].thread_id} | run_id: ${requiredActions[0].run_id}`,
     requiredActions,
   );
-  const tools = requiredActions.map((action) => action.tool);
+  const toolDefinitions = client.req.app.locals.availableTools;
+  const seenToolkits = new Set();
+  const tools = requiredActions
+    .map((action) => {
+      const toolName = action.tool;
+      const toolDef = toolDefinitions[toolName];
+      if (toolDef && !manifestToolMap[toolName]) {
+        for (const toolkit of toolkits) {
+          if (seenToolkits.has(toolkit.pluginKey)) {
+            return;
+          } else if (toolName.startsWith(`${toolkit.pluginKey}_`)) {
+            seenToolkits.add(toolkit.pluginKey);
+            return toolkit.pluginKey;
+          }
+        }
+      }
+      return toolName;
+    })
+    .filter((toolName) => !!toolName);
+
   const { loadedTools } = await loadTools({
     user: client.req.user.id,
     model: client.req.body.model ?? 'gpt-4o-mini',
@@ -441,6 +461,11 @@ async function loadAgentTools({ req, agent, tool_resources, openAIApiKey }) {
       continue;
     }
 
+    if (tool instanceof DynamicStructuredTool) {
+      agentTools.push(tool);
+      continue;
+    }
+
     const toolDefinition = {
       name: tool.name,
       schema: tool.schema,

api/server/utils/import/__data__/librechat-export.json

@@ -29,7 +29,6 @@
       "endpoint": "openAI",
       "error": false,
       "isCreatedByUser": true,
-      "isEdited": false,
       "model": null,
       "parentMessageId": "00000000-0000-0000-0000-000000000000",
       "sender": "user",
@@ -47,7 +46,6 @@
           "endpoint": "openAI",
           "error": false,
           "isCreatedByUser": false,
-          "isEdited": false,
           "model": null,
           "parentMessageId": "b123942f-ca1a-4b16-9e1f-ea4af5171168",
           "sender": "GPT-3.5",
@@ -65,7 +63,6 @@
               "endpoint": "openAI",
               "error": false,
               "isCreatedByUser": true,
-              "isEdited": false,
               "model": null,
               "parentMessageId": "549a4f45-cf93-4e3b-ae62-1abf02afbfc8",
               "sender": "user",
@@ -83,7 +80,6 @@
                   "endpoint": "openAI",
                   "error": false,
                   "isCreatedByUser": false,
-                  "isEdited": false,
                   "model": null,
                   "parentMessageId": "880e5357-3e0c-4218-b351-fd3fc184adef",
                   "sender": "GPT-3.5",
@@ -101,7 +97,6 @@
                       "endpoint": "openAI",
                       "error": false,
                       "isCreatedByUser": true,
-                      "isEdited": false,
                       "model": null,
                       "parentMessageId": "e9796d11-3bdf-4e25-9f0e-4802bbbb8c6d",
                       "sender": "user",
@@ -119,7 +114,6 @@
                           "endpoint": "openAI",
                           "error": false,
                           "isCreatedByUser": false,
-                          "isEdited": false,
                           "model": null,
                           "parentMessageId": "04408c06-62dc-4961-8ef5-4336b68e7a0a",
                           "sender": "GPT-3.5",

api/server/utils/import/__data__/librechat-opts-nonr-branches.json

@@ -27,7 +27,6 @@
       "endpoint": "azureOpenAI",
       "error": false,
       "isCreatedByUser": true,
-      "isEdited": false,
       "model": null,
       "parentMessageId": "00000000-0000-0000-0000-000000000000",
       "sender": "User",
@@ -42,7 +41,6 @@
       "createdAt": "2024-05-28T18:08:55.390Z",
       "error": false,
       "isCreatedByUser": false,
-      "isEdited": false,
       "model": "gpt-4o",
       "parentMessageId": "115a6247-8fb0-4937-a536-12956669098d",
       "sender": "GPT-4",
@@ -59,7 +57,6 @@
       "createdAt": "2024-05-28T18:09:27.444Z",
       "error": false,
       "isCreatedByUser": false,
-      "isEdited": false,
       "model": "gpt-4o",
       "parentMessageId": "115a6247-8fb0-4937-a536-12956669098d",
       "sender": "GPT-4",
@@ -74,7 +71,6 @@
       "endpoint": "azureOpenAI",
       "error": false,
       "isCreatedByUser": true,
-      "isEdited": false,
       "model": null,
       "parentMessageId": "00000000-0000-0000-0000-000000000000",
       "sender": "User",
@@ -89,7 +85,6 @@
       "createdAt": "2024-05-28T18:14:08.403Z",
       "error": false,
       "isCreatedByUser": false,
-      "isEdited": true,
       "model": "gpt-4o",
       "parentMessageId": "599e1908-8c52-4a73-ba6b-f6dffbd79ba0",
       "sender": "GPT-4",

api/server/utils/import/__data__/librechat-tree.json

@@ -29,7 +29,6 @@
       "endpoint": "openAI",
       "error": false,
       "isCreatedByUser": true,
-      "isEdited": false,
       "model": null,
       "parentMessageId": "00000000-0000-0000-0000-000000000000",
       "sender": "User",
@@ -47,7 +46,6 @@
           "createdAt": "2024-05-01T16:35:12.604Z",
           "error": false,
           "isCreatedByUser": false,
-          "isEdited": false,
           "model": "gpt-4-turbo",
           "parentMessageId": "9501f99d-9bbb-40cb-bbb2-16d79aeceb72",
           "sender": "Software Engineer",

client/src/components/Chat/Messages/Content/EditMessage.tsx

@@ -114,7 +114,6 @@ const EditMessage = ({
             ? {
               ...msg,
               text: data.text,
-              isEdited: true,
             }
             : msg,
         ),

client/src/components/Chat/Messages/Content/Parts/EditTextPart.tsx

@@ -119,7 +119,6 @@ const EditTextPart = ({
           ? {
             ...msg,
             content: updatedContent,
-            isEdited: true,
           }
           : msg,
       ),

client/src/components/SidePanel/Builder/AssistantPanel.tsx

@@ -1,25 +1,27 @@
 import { useState, useMemo } from 'react';
-import { useQueryClient } from '@tanstack/react-query';
 import { useForm, FormProvider, Controller, useWatch } from 'react-hook-form';
 import { useGetModelsQuery } from 'librechat-data-provider/react-query';
 import {
   Tools,
-  QueryKeys,
   Capabilities,
   actionDelimiter,
   ImageVisionTool,
   defaultAssistantFormValues,
 } from 'librechat-data-provider';
-import type { FunctionTool, TConfig, TPlugin } from 'librechat-data-provider';
+import type { FunctionTool, TConfig } from 'librechat-data-provider';
 import type { AssistantForm, AssistantPanelProps } from '~/common';
-import { useCreateAssistantMutation, useUpdateAssistantMutation } from '~/data-provider';
+import {
+  useCreateAssistantMutation,
+  useUpdateAssistantMutation,
+  useAvailableAgentToolsQuery,
+} from '~/data-provider';
 import { cn, cardStyle, defaultTextProps, removeFocusOutlines } from '~/utils';
 import AssistantConversationStarters from './AssistantConversationStarters';
 import { useAssistantsMapContext, useToastContext } from '~/Providers';
 import { useSelectAssistant, useLocalize } from '~/hooks';
 import { ToolSelectDialog } from '~/components/Tools';
-import CapabilitiesForm from './CapabilitiesForm';
 import AppendDateCheckbox from './AppendDateCheckbox';
+import CapabilitiesForm from './CapabilitiesForm';
 import { SelectDropDown } from '~/components/ui';
 import AssistantAvatar from './AssistantAvatar';
 import AssistantSelect from './AssistantSelect';
@@ -49,11 +51,10 @@ export default function AssistantPanel({
   assistantsConfig,
   version,
 }: AssistantPanelProps & { assistantsConfig?: TConfig | null }) {
-  const queryClient = useQueryClient();
   const modelsQuery = useGetModelsQuery();
   const assistantMap = useAssistantsMapContext();
 
-  const allTools = queryClient.getQueryData<TPlugin[]>([QueryKeys.tools]) ?? [];
+  const { data: allTools = [] } = useAvailableAgentToolsQuery();
   const { onSelect: onSelectAssistant } = useSelectAssistant(endpoint);
   const { showToast } = useToastContext();
   const localize = useLocalize();
@@ -227,6 +228,7 @@ export default function AssistantPanel({
                 value={field.value}
                 endpoint={endpoint}
                 documentsMap={documentsMap}
+                allTools={allTools}
                 setCurrentAssistantId={setCurrentAssistantId}
                 selectedAssistant={current_assistant_id ?? null}
                 createMutation={create}

client/src/components/SidePanel/Builder/AssistantSelect.tsx

@@ -1,5 +1,5 @@
 import { Plus } from 'lucide-react';
-import { useCallback, useEffect, useRef } from 'react';
+import { useMemo, useCallback, useEffect, useRef } from 'react';
 import {
   Tools,
   FileSources,
@@ -12,6 +12,7 @@ import {
 import type { UseFormReset } from 'react-hook-form';
 import type { UseMutationResult } from '@tanstack/react-query';
 import type {
+  TPlugin,
   Assistant,
   AssistantDocument,
   AssistantsEndpoint,
@@ -48,6 +49,7 @@ export default function AssistantSelect({
   selectedAssistant,
   setCurrentAssistantId,
   createMutation,
+  allTools,
 }: {
   reset: UseFormReset<AssistantForm>;
   value: TAssistantOption;
@@ -56,6 +58,7 @@ export default function AssistantSelect({
   documentsMap: Map<string, AssistantDocument> | null;
   setCurrentAssistantId: React.Dispatch<React.SetStateAction<string | undefined>>;
   createMutation: UseMutationResult<Assistant, Error, AssistantCreateParams>;
+  allTools?: TPlugin[];
 }) {
   const localize = useLocalize();
   const fileMap = useFileMapContext();
@@ -65,6 +68,11 @@ export default function AssistantSelect({
     {} as LastSelectedModels,
   );
 
+  const toolkits = useMemo(
+    () => new Set(allTools?.filter((tool) => tool.toolkit === true).map((tool) => tool.pluginKey)),
+    [allTools],
+  );
+
   const query = useListAssistantsQuery(endpoint, undefined, {
     select: (res) =>
       res.data.map((_assistant) => {
@@ -153,7 +161,7 @@ export default function AssistantSelect({
       const update = {
         ...assistant,
         label: assistant.name ?? '',
-        value: assistant.id ?? '',
+        value: assistant.id || '',
       };
 
       const actions: Actions = {
@@ -164,7 +172,7 @@ export default function AssistantSelect({
 
       (assistant.tools ?? [])
         .filter((tool) => tool.type !== 'function' || isImageVisionTool(tool))
-        .map((tool) => tool.function?.name || tool.type)
+        .map((tool) => (tool.function?.name ?? '') || tool.type)
         .forEach((tool) => {
           if (tool === Tools.file_search) {
             actions[Capabilities.retrieval] = true;
@@ -172,9 +180,22 @@ export default function AssistantSelect({
           actions[tool] = true;
         });
 
+      const seenToolkits = new Set<string>();
       const functions = (assistant.tools ?? [])
         .filter((tool) => tool.type === 'function' && !isImageVisionTool(tool))
-        .map((tool) => tool.function?.name ?? '');
+        .map((tool) => tool.function?.name ?? '')
+        .filter((fnName) => {
+          const fnPrefix = fnName.split('_')[0];
+          const seenToolkit = toolkits.has(fnPrefix);
+          if (seenToolkit) {
+            seenToolkits.add(fnPrefix);
+          }
+          return !seenToolkit;
+        });
+
+      if (seenToolkits.size > 0) {
+        functions.push(...Array.from(seenToolkits));
+      }
 
       const formValues: Partial<AssistantForm & Actions> = {
         functions,
@@ -210,7 +231,15 @@ export default function AssistantSelect({
       reset(formValues);
       setCurrentAssistantId(assistant.id);
     },
-    [query.data, reset, setCurrentAssistantId, createMutation, endpoint, lastSelectedModels],
+    [
+      query.data,
+      reset,
+      setCurrentAssistantId,
+      createMutation,
+      endpoint,
+      lastSelectedModels,
+      toolkits,
+    ],
   );
 
   useEffect(() => {

package-lock.json

@@ -46,6 +46,7 @@
         "@anthropic-ai/sdk": "^0.32.1",
         "@azure/search-documents": "^12.0.0",
         "@google/generative-ai": "^0.21.0",
+        "@googleapis/youtube": "^20.0.0",
         "@keyv/mongo": "^2.1.8",
         "@keyv/redis": "^2.8.1",
         "@langchain/community": "^0.3.14",
@@ -114,6 +115,7 @@
         "ua-parser-js": "^1.0.36",
         "winston": "^3.11.0",
         "winston-daily-rotate-file": "^4.7.1",
+        "youtube-transcript": "^1.2.1",
         "zod": "^3.22.4"
       },
       "devDependencies": {
@@ -8391,6 +8393,18 @@
         "node": ">=18.0.0"
       }
     },
+    "node_modules/@googleapis/youtube": {
+      "version": "20.0.0",
+      "resolved": "https://registry.npmjs.org/@googleapis/youtube/-/youtube-20.0.0.tgz",
+      "integrity": "sha512-wdt1J0JoKYhvpoS2XIRHX0g/9ul/B0fQeeJAhuuBIdYINuuLt6/oZYZZCBmkuhtkA3IllXgqgAXOjLtLRAnR2g==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "googleapis-common": "^7.0.0"
+      },
+      "engines": {
+        "node": ">=12.0.0"
+      }
+    },
     "node_modules/@grpc/grpc-js": {
       "version": "1.9.15",
       "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.9.15.tgz",
@@ -35501,6 +35515,15 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/youtube-transcript": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/youtube-transcript/-/youtube-transcript-1.2.1.tgz",
+      "integrity": "sha512-TvEGkBaajKw+B6y91ziLuBLsa5cawgowou+Bk0ciGpjELDfAzSzTGXaZmeSSkUeknCPpEr/WGApOHDwV7V+Y9Q==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=18.0.0"
+      }
+    },
     "node_modules/zod": {
       "version": "3.23.8",
       "resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz",
@@ -35528,7 +35551,7 @@
     },
     "packages/data-provider": {
       "name": "librechat-data-provider",
-      "version": "0.7.697",
+      "version": "0.7.698",
       "license": "ISC",
       "dependencies": {
         "axios": "^1.7.7",

packages/data-provider/package.json

@@ -1,6 +1,6 @@
 {
   "name": "librechat-data-provider",
-  "version": "0.7.697",
+  "version": "0.7.698",
   "description": "data services for librechat apps",
   "main": "dist/index.js",
   "module": "dist/index.es.js",

packages/data-provider/src/schemas.ts

@@ -386,6 +386,7 @@ export const tPluginSchema = z.object({
   authConfig: z.array(tPluginAuthConfigSchema).optional(),
   authenticated: z.boolean().optional(),
   isButton: z.boolean().optional(),
+  toolkit: z.boolean().optional(),
 });
 
 export type TPlugin = z.infer<typeof tPluginSchema>;
@@ -462,7 +463,6 @@ export const tMessageSchema = z.object({
   sender: z.string().optional(),
   text: z.string(),
   generation: z.string().nullable().optional(),
-  isEdited: z.boolean().optional(),
   isCreatedByUser: z.boolean(),
   error: z.boolean().optional(),
   clientTimestamp: z.string().optional(),