🔎 feat: Native Web Search with Citation References (#7516)

* WIP: search tool integration

* WIP: Add web search capabilities and API key management to agent actions

* WIP: web search capability to agent configuration and selection

* WIP: Add web search capability to backend agent configuration

* WIP: add web search option to default agent form values

* WIP: add attachments for web search

* feat: add plugin for processing web search citations

* WIP: first pass, Citation UI

* chore: remove console.log

* feat: Add AnimatedTabs component for tabbed UI functionality

* refactor: AnimatedTabs component with CSS animations and stable ID generation

* WIP example content

* feat: SearchContext for managing search results apart from MessageContext

* feat: Enhance AnimatedTabs with underline animation and state management

* WIP: first pass, Implement dynamic tab functionality in Sources component with search results integration

* fix: Update class names for improved styling in Sources and AnimatedTabs components

* feat: Improve styling and layout in Sources component with enhanced button and item designs

* feat: Refactor Sources component to integrate OGDialog for source display and improve layout

* style: Update background color in SourceItem and SourcesGroup components for improved visibility

* refactor: Sources component to enhance SourceItem structure and improve favicon handling

* style: Adjust font size of domain text in SourceItem for better readability

* feat: Add localization for citation source and details in CompositeCitation component

* style: add theming to Citation components

* feat: Enhance SourceItem component with dialog support and improved hovercard functionality

* feat: Add localization for sources tab and image alt text in Sources component

* style: Replace divs with spans for better semantic structure in CompositeCitation and Citation components

* refactor: Sources component to use useMemo for tab generation and improve performance

* chore: bump @librechat/agents to v2.4.318

* chore: update search result types

* fix: search results retrieval in ContentParts component, re-render attachments when expected

* feat: update sources style/types to use latest search result structure

* style: enhance Dialog (expanded) SourceItem component with link wrapping and improved styling

* style: update ImageItem component styling for improved title visibility

* refactor: remove SourceItemBase component and adjust SourceItem layout for improved styling

* chore: linting twcss order

* fix: prevent FileAttachment from rendering search attachments

* fix: append underscore to responseMessageId for unique identification to prevent mapping of previous latest message's attachments

* chore: remove unused parameter 'useSpecs' from loadTools function

* chore: twcss order

* WIP: WebSearch Tool UI

* refactor: add limit parameter to StackedFavicons for customizable source display

* refactor: optimize search results memoization by making more granular and separate conerns

* refactor: integrated StackedFavicons to WebSearch mid-run

* chore: bump @librechat/agents to expose handleToolCallChunks

* chore: use typedefs from dedicated file instead of defining them in AgentClient module

* WIP: first pass, search progress results

* refactor: move createOnSearchResults function to a dedicated search module

* chore: bump @librechat/agents to v2.4.320

* WIP: first pass, search results processed UX

* refactor: consolidate context variables in createOnSearchResults function

* chore: bump @librechat/agents to v2.4.321

* feat: add guidelines for web search tool response formatting in loadTools function

* feat: add isLast prop to Part component and update WebSearch logic for improved state handling

* style: update Hovercard styles for improved UI consistency

* feat: export FaviconImage component for improved accessibility in other modules

* refactor: export getCleanDomain function and use FaviconImage in Citation component for improved source representation

* refactor: implement SourceHovercard component for consistency and DRY compliance

* fix: replace <p> with <span> for snippet and title in SourceItem and SourceHovercard for consistency

* style: `not-prose`

* style: remove 'not-prose' class for consistency in SourceItem, Citation, and SourceHovercard components, adjust style classes

* refactor: `imageUrl` on hover and prevent duplicate sources

* refactor: enhance SourcesGroup dialog layout and improve source item presentation

* refactor: reorganize Web Components, save in same directory

* feat: add 'news' refType to refTypeMap for citation sources

* style: adjust Hovercard width for improved layout

* refactor: update tool usage guidelines for improved clarity and execution

* chore: linting

* feat: add Web Search badge with initial permissions and local storage logic

* feat: add webSearch support to interface and permissions schemas

* feat: implement Web Search API key management and localization updates

* feat: refactor Web Search API key handling and integrate new search API key form

* fix: remove unnecessary visibility state from FileAttachment component

* feat: update WebSearch component to use Globe icon and localized search label

* feat: enhance ApiKeyDialog with dropdown for reranker selection and update translations

* feat: implement dropdown menus for engine, scraper, and reranker selection in ApiKeyDialog

* chore: linting and add unknown instead of `any` type

* feat: refactor ApiKeyDialog and useAuthSearchTool for improved API key management

* refactor: update ocrSchema to use template literals for default apiKey and baseURL

* feat: add web search configuration and utility functions for environment variable extraction

* fix: ensure filepath is defined before checking its prefix in useAttachmentHandler

* feat: enhance web search functionality with improved configuration and environment variable extraction for authFields

* fix: update auth type in TPluginAction and TUpdateUserPlugins to use Partial<Record<string, string>>

* feat: implement web search authentication verification and enhance webSearchAuth structure

* feat: enhance ephemeral agent handling with new web search capability and type definition

* feat: enhance isEphemeralAgent function to include web search selection

* feat: refactor verifyWebSearchAuth to improve key handling and authentication checks

* feat: implement loadWebSearchAuth function for improved web search authentication handling

* feat: enhance web search authentication with new configuration options and refactor related types

* refactor: rename search engine to search provider and update related localization keys

* feat: update verifyWebSearchAuth to handle multiple authentication types and improve error handling

* feat: update ApiKeyDialog to accept authTypes prop and remove isUserProvided check

* feat: add tests for extractWebSearchEnvVars and loadWebSearchAuth functions

* feat: enhance loadWebSearchAuth to support specific service checks for providers, scrapers, and rerankers

* fix: update web search configuration key and adjust auth result handling in loadTools function

* feat: add new progress key for repeated web searching and update localization

* chore: bump @librechat/agents to 2.4.322

* feat: enhance loadTools function to include ISO time and improve search tool logging

* feat: update StackedFavicons to handle negative start index and improve citation attribution styling and text

* chore: update .gitignore to categorize AI-related files

* fix: mobile responsiveness of sources/citations hovercards

* feat: enhance source display with improved line clamping for better readability

* chore: bump @librechat/agents to v2.4.33

* feat: add handling for image sources in references mapping

* chore: bump librechat-data-provider version to 0.7.84

* chore: bump @librechat/agents version to 2.4.34

* fix: update auth handling to support multiple auth types in tools and allow key configuration in agent panel

* chore: remove redundant agent attribution text from search form

* fix: web search auth uninstall

* refactor: convert CheckboxButton to a forwardRef component and update setValue callback signature

* feat: add triggerRef prop to ApiKeyDialog components for improved dialog control

* feat: integrate triggerRef in CodeInterpreter and WebSearch components for enhanced dialog management

* feat: enhance ApiKeyDialog with additional links for Firecrawl and Jina API key guidance

* feat: implement web search configuration handling in ApiKeyDialog and add tests for dropdown visibility

* fix: update webSearchConfig reference in config route for correct payload assignment

* feat: update ApiKeyDialog to conditionally render sections based on authTypes and modify loadWebSearchAuth to correctly categorize authentication types

* feat: refactor ApiKeyDialog and related tests to use SearchCategories and RerankerTypes enums and remove nested ternaries

* refactor: move ThinkingButton rendering to improve layout consistency in ContentParts

* feat: integrate search context into Markdown component to conditionally include unicodeCitation plugin

* chore: bump @librechat/agents to v2.4.35

* chore: remove unused 18n key

* ci: add WEB_SEARCH permission testing and update AppService tests for new webSearch configuration

* ci: add more comprehensive tests for loadWebSearchAuth to validate authentication handling and authTypes structure

* chore: remove debugging console log from web.spec.ts to clean up test output

api/server/controllers/UserController.js

@@ -1,4 +1,10 @@
-const { FileSources } = require('librechat-data-provider');
+const {
+  Tools,
+  Constants,
+  FileSources,
+  webSearchKeys,
+  extractWebSearchEnvVars,
+} = require('librechat-data-provider');
 const {
   Balance,
   getFiles,
@@ -83,7 +89,6 @@ const deleteUserFiles = async (req) => {
 const updateUserPluginsController = async (req, res) => {
   const { user } = req;
   const { pluginKey, action, auth, isEntityTool } = req.body;
-  let authService;
   try {
     if (!isEntityTool) {
       const userPluginsService = await updateUserPluginsService(user, pluginKey, action);
@@ -95,32 +100,55 @@ const updateUserPluginsController = async (req, res) => {
       }
     }
 
-    if (auth) {
-      const keys = Object.keys(auth);
-      const values = Object.values(auth);
-      if (action === 'install' && keys.length > 0) {
-        for (let i = 0; i < keys.length; i++) {
-          authService = await updateUserPluginAuth(user.id, keys[i], pluginKey, values[i]);
-          if (authService instanceof Error) {
-            logger.error('[authService]', authService);
-            const { status, message } = authService;
-            res.status(status).send({ message });
-          }
+    if (auth == null) {
+      return res.status(200).send();
+    }
+
+    let keys = Object.keys(auth);
+    if (keys.length === 0 && pluginKey !== Tools.web_search) {
+      return res.status(200).send();
+    }
+    const values = Object.values(auth);
+
+    /** @type {number} */
+    let status = 200;
+    /** @type {string} */
+    let message;
+    /** @type {IPluginAuth | Error} */
+    let authService;
+
+    if (pluginKey === Tools.web_search) {
+      /** @type  {TCustomConfig['webSearch']} */
+      const webSearchConfig = req.app.locals?.webSearch;
+      keys = extractWebSearchEnvVars({
+        keys: action === 'install' ? keys : webSearchKeys,
+        config: webSearchConfig,
+      });
+    }
+
+    if (action === 'install') {
+      for (let i = 0; i < keys.length; i++) {
+        authService = await updateUserPluginAuth(user.id, keys[i], pluginKey, values[i]);
+        if (authService instanceof Error) {
+          logger.error('[authService]', authService);
+          ({ status, message } = authService);
         }
       }
-      if (action === 'uninstall' && keys.length > 0) {
-        for (let i = 0; i < keys.length; i++) {
-          authService = await deleteUserPluginAuth(user.id, keys[i]);
-          if (authService instanceof Error) {
-            logger.error('[authService]', authService);
-            const { status, message } = authService;
-            res.status(status).send({ message });
-          }
+    } else if (action === 'uninstall') {
+      for (let i = 0; i < keys.length; i++) {
+        authService = await deleteUserPluginAuth(user.id, keys[i]);
+        if (authService instanceof Error) {
+          logger.error('[authService]', authService);
+          ({ status, message } = authService);
         }
       }
     }
 
-    res.status(200).send();
+    if (status === 200) {
+      return res.status(status).send();
+    }
+
+    res.status(status).send({ message });
   } catch (err) {
     logger.error('[updateUserPluginsController]', err);
     return res.status(500).json({ message: 'Something went wrong.' });

api/server/controllers/agents/callbacks.js

@@ -237,6 +237,30 @@ function createToolEndCallback({ req, res, artifactPromises }) {
       return;
     }
 
+    if (output.artifact[Tools.web_search]) {
+      artifactPromises.push(
+        (async () => {
+          const name = `${output.name}_${output.tool_call_id}_${nanoid()}`;
+          const attachment = {
+            name,
+            type: Tools.web_search,
+            messageId: metadata.run_id,
+            toolCallId: output.tool_call_id,
+            conversationId: metadata.thread_id,
+            [Tools.web_search]: { ...output.artifact[Tools.web_search] },
+          };
+          if (!res.headersSent) {
+            return attachment;
+          }
+          res.write(`event: attachment\ndata: ${JSON.stringify(attachment)}\n\n`);
+          return attachment;
+        })().catch((error) => {
+          logger.error('Error processing artifact content:', error);
+          return null;
+        }),
+      );
+    }
+
     if (output.artifact.content) {
       /** @type {FormattedContent[]} */
       const content = output.artifact.content;

api/server/controllers/agents/client.js

@@ -39,9 +39,6 @@ const BaseClient = require('~/app/clients/BaseClient');
 const { logger, sendEvent } = require('~/config');
 const { createRun } = require('./run');
 
-/** @typedef {import('@librechat/agents').MessageContentComplex} MessageContentComplex */
-/** @typedef {import('@langchain/core/runnables').RunnableConfig} RunnableConfig */
-
 /**
  * @param {ServerRequest} req
  * @param {Agent} agent
@@ -543,7 +540,7 @@ class AgentClient extends BaseClient {
   }
 
   async chatCompletion({ payload, abortController = null }) {
-    /** @type {Partial<RunnableConfig> & { version: 'v1' | 'v2'; run_id?: string; streamMode: string }} */
+    /** @type {Partial<GraphRunnableConfig>} */
     let config;
     /** @type {ReturnType<createRun>} */
     let run;

api/server/controllers/tools.js

@@ -6,6 +6,7 @@ const {
   Permissions,
   ToolCallTypes,
   PermissionTypes,
+  loadWebSearchAuth,
 } = require('librechat-data-provider');
 const { processFileURL, uploadImageBuffer } = require('~/server/services/Files/process');
 const { processCodeOutput } = require('~/server/services/Files/Code/process');
@@ -24,6 +25,36 @@ const toolAccessPermType = {
   [Tools.execute_code]: PermissionTypes.RUN_CODE,
 };
 
+/**
+ * Verifies web search authentication, ensuring each category has at least
+ * one fully authenticated service.
+ *
+ * @param {ServerRequest} req - The request object
+ * @param {ServerResponse} res - The response object
+ * @returns {Promise<void>} A promise that resolves when the function has completed
+ */
+const verifyWebSearchAuth = async (req, res) => {
+  try {
+    const userId = req.user.id;
+    /** @type {TCustomConfig['webSearch']} */
+    const webSearchConfig = req.app.locals?.webSearch || {};
+    const result = await loadWebSearchAuth({
+      userId,
+      loadAuthValues,
+      webSearchConfig,
+      throwError: false,
+    });
+
+    return res.status(200).json({
+      authenticated: result.authenticated,
+      authTypes: result.authTypes,
+    });
+  } catch (error) {
+    console.error('Error in verifyWebSearchAuth:', error);
+    return res.status(500).json({ message: error.message });
+  }
+};
+
 /**
  * @param {ServerRequest} req - The request object, containing information about the HTTP request.
  * @param {ServerResponse} res - The response object, used to send back the desired HTTP response.
@@ -32,6 +63,9 @@ const toolAccessPermType = {
 const verifyToolAuth = async (req, res) => {
   try {
     const { toolId } = req.params;
+    if (toolId === Tools.web_search) {
+      return await verifyWebSearchAuth(req, res);
+    }
     const authFields = fieldsMap[toolId];
     if (!authFields) {
       res.status(404).json({ message: 'Tool not found' });