Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-12-23 20:00:15 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

🔎 feat: Native Web Search with Citation References (#7516)

Browse source 
* WIP: search tool integration

* WIP: Add web search capabilities and API key management to agent actions

* WIP: web search capability to agent configuration and selection

* WIP: Add web search capability to backend agent configuration

* WIP: add web search option to default agent form values

* WIP: add attachments for web search

* feat: add plugin for processing web search citations

* WIP: first pass, Citation UI

* chore: remove console.log

* feat: Add AnimatedTabs component for tabbed UI functionality

* refactor: AnimatedTabs component with CSS animations and stable ID generation

* WIP example content

* feat: SearchContext for managing search results apart from MessageContext

* feat: Enhance AnimatedTabs with underline animation and state management

* WIP: first pass, Implement dynamic tab functionality in Sources component with search results integration

* fix: Update class names for improved styling in Sources and AnimatedTabs components

* feat: Improve styling and layout in Sources component with enhanced button and item designs

* feat: Refactor Sources component to integrate OGDialog for source display and improve layout

* style: Update background color in SourceItem and SourcesGroup components for improved visibility

* refactor: Sources component to enhance SourceItem structure and improve favicon handling

* style: Adjust font size of domain text in SourceItem for better readability

* feat: Add localization for citation source and details in CompositeCitation component

* style: add theming to Citation components

* feat: Enhance SourceItem component with dialog support and improved hovercard functionality

* feat: Add localization for sources tab and image alt text in Sources component

* style: Replace divs with spans for better semantic structure in CompositeCitation and Citation components

* refactor: Sources component to use useMemo for tab generation and improve performance

* chore: bump @librechat/agents to v2.4.318

* chore: update search result types

* fix: search results retrieval in ContentParts component, re-render attachments when expected

* feat: update sources style/types to use latest search result structure

* style: enhance Dialog (expanded) SourceItem component with link wrapping and improved styling

* style: update ImageItem component styling for improved title visibility

* refactor: remove SourceItemBase component and adjust SourceItem layout for improved styling

* chore: linting twcss order

* fix: prevent FileAttachment from rendering search attachments

* fix: append underscore to responseMessageId for unique identification to prevent mapping of previous latest message's attachments

* chore: remove unused parameter 'useSpecs' from loadTools function

* chore: twcss order

* WIP: WebSearch Tool UI

* refactor: add limit parameter to StackedFavicons for customizable source display

* refactor: optimize search results memoization by making more granular and separate conerns

* refactor: integrated StackedFavicons to WebSearch mid-run

* chore: bump @librechat/agents to expose handleToolCallChunks

* chore: use typedefs from dedicated file instead of defining them in AgentClient module

* WIP: first pass, search progress results

* refactor: move createOnSearchResults function to a dedicated search module

* chore: bump @librechat/agents to v2.4.320

* WIP: first pass, search results processed UX

* refactor: consolidate context variables in createOnSearchResults function

* chore: bump @librechat/agents to v2.4.321

* feat: add guidelines for web search tool response formatting in loadTools function

* feat: add isLast prop to Part component and update WebSearch logic for improved state handling

* style: update Hovercard styles for improved UI consistency

* feat: export FaviconImage component for improved accessibility in other modules

* refactor: export getCleanDomain function and use FaviconImage in Citation component for improved source representation

* refactor: implement SourceHovercard component for consistency and DRY compliance

* fix: replace <p> with <span> for snippet and title in SourceItem and SourceHovercard for consistency

* style: `not-prose`

* style: remove 'not-prose' class for consistency in SourceItem, Citation, and SourceHovercard components, adjust style classes

* refactor: `imageUrl` on hover and prevent duplicate sources

* refactor: enhance SourcesGroup dialog layout and improve source item presentation

* refactor: reorganize Web Components, save in same directory

* feat: add 'news' refType to refTypeMap for citation sources

* style: adjust Hovercard width for improved layout

* refactor: update tool usage guidelines for improved clarity and execution

* chore: linting

* feat: add Web Search badge with initial permissions and local storage logic

* feat: add webSearch support to interface and permissions schemas

* feat: implement Web Search API key management and localization updates

* feat: refactor Web Search API key handling and integrate new search API key form

* fix: remove unnecessary visibility state from FileAttachment component

* feat: update WebSearch component to use Globe icon and localized search label

* feat: enhance ApiKeyDialog with dropdown for reranker selection and update translations

* feat: implement dropdown menus for engine, scraper, and reranker selection in ApiKeyDialog

* chore: linting and add unknown instead of `any` type

* feat: refactor ApiKeyDialog and useAuthSearchTool for improved API key management

* refactor: update ocrSchema to use template literals for default apiKey and baseURL

* feat: add web search configuration and utility functions for environment variable extraction

* fix: ensure filepath is defined before checking its prefix in useAttachmentHandler

* feat: enhance web search functionality with improved configuration and environment variable extraction for authFields

* fix: update auth type in TPluginAction and TUpdateUserPlugins to use Partial<Record<string, string>>

* feat: implement web search authentication verification and enhance webSearchAuth structure

* feat: enhance ephemeral agent handling with new web search capability and type definition

* feat: enhance isEphemeralAgent function to include web search selection

* feat: refactor verifyWebSearchAuth to improve key handling and authentication checks

* feat: implement loadWebSearchAuth function for improved web search authentication handling

* feat: enhance web search authentication with new configuration options and refactor related types

* refactor: rename search engine to search provider and update related localization keys

* feat: update verifyWebSearchAuth to handle multiple authentication types and improve error handling

* feat: update ApiKeyDialog to accept authTypes prop and remove isUserProvided check

* feat: add tests for extractWebSearchEnvVars and loadWebSearchAuth functions

* feat: enhance loadWebSearchAuth to support specific service checks for providers, scrapers, and rerankers

* fix: update web search configuration key and adjust auth result handling in loadTools function

* feat: add new progress key for repeated web searching and update localization

* chore: bump @librechat/agents to 2.4.322

* feat: enhance loadTools function to include ISO time and improve search tool logging

* feat: update StackedFavicons to handle negative start index and improve citation attribution styling and text

* chore: update .gitignore to categorize AI-related files

* fix: mobile responsiveness of sources/citations hovercards

* feat: enhance source display with improved line clamping for better readability

* chore: bump @librechat/agents to v2.4.33

* feat: add handling for image sources in references mapping

* chore: bump librechat-data-provider version to 0.7.84

* chore: bump @librechat/agents version to 2.4.34

* fix: update auth handling to support multiple auth types in tools and allow key configuration in agent panel

* chore: remove redundant agent attribution text from search form

* fix: web search auth uninstall

* refactor: convert CheckboxButton to a forwardRef component and update setValue callback signature

* feat: add triggerRef prop to ApiKeyDialog components for improved dialog control

* feat: integrate triggerRef in CodeInterpreter and WebSearch components for enhanced dialog management

* feat: enhance ApiKeyDialog with additional links for Firecrawl and Jina API key guidance

* feat: implement web search configuration handling in ApiKeyDialog and add tests for dropdown visibility

* fix: update webSearchConfig reference in config route for correct payload assignment

* feat: update ApiKeyDialog to conditionally render sections based on authTypes and modify loadWebSearchAuth to correctly categorize authentication types

* feat: refactor ApiKeyDialog and related tests to use SearchCategories and RerankerTypes enums and remove nested ternaries

* refactor: move ThinkingButton rendering to improve layout consistency in ContentParts

* feat: integrate search context into Markdown component to conditionally include unicodeCitation plugin

* chore: bump @librechat/agents to v2.4.35

* chore: remove unused 18n key

* ci: add WEB_SEARCH permission testing and update AppService tests for new webSearch configuration

* ci: add more comprehensive tests for loadWebSearchAuth to validate authentication handling and authTypes structure

* chore: remove debugging console log from web.spec.ts to clean up test output
This commit is contained in:
Danny Avila 2025-05-23 00:14:04 -04:00
parent bf80cf30b3
commit 0dbbf7de04
No known key found for this signature in database
GPG key ID: BF31EEB2C5CA0956
73 changed files with 6366 additions and 2003 deletions
Download patch file Download diff file Expand all files Collapse all files

107
client/src/components/Chat/Messages/Content/ContentParts.tsx
View file

@ -2,10 +2,12 @@ import { memo, useMemo, useState } from 'react';
import { useRecoilValue, useRecoilState } from 'recoil';
import { ContentTypes } from 'librechat-data-provider';
import type { TMessageContentParts, TAttachment, Agents } from 'librechat-data-provider';
import { useSearchResultsByTurn } from '~/hooks/Messages/useSearchResultsByTurn';
import { ThinkingButton } from '~/components/Artifacts/Thinking';
import useLocalize from '~/hooks/useLocalize';
import { mapAttachments } from '~/utils/map';
import { MessageContext } from '~/Providers';
import { MessageContext, SearchContext } from '~/Providers';
import Sources from '~/components/Web/Sources';
import { EditTextPart } from './Parts';
import store from '~/store';
import Part from './Part';
@ -42,13 +44,15 @@ const ContentParts = memo(
setSiblingIdx,
}: ContentPartsProps) => {
const localize = useLocalize();
const messageAttachmentsMap = useRecoilValue(store.messageAttachmentsMap);
const [showThinking, setShowThinking] = useRecoilState<boolean>(store.showThinking);
const [isExpanded, setIsExpanded] = useState(showThinking);
const messageAttachmentsMap = useRecoilValue(store.messageAttachmentsMap);
const attachmentMap = useMemo(
() => mapAttachments(attachments ?? messageAttachmentsMap[messageId] ?? []),
const messageAttachments = useMemo(
() => attachments ?? messageAttachmentsMap[messageId] ?? [],
[attachments, messageAttachmentsMap, messageId],
);
const searchResults = useSearchResultsByTurn(messageAttachments);
const attachmentMap = useMemo(() => mapAttachments(messageAttachments), [messageAttachments]);
const hasReasoningParts = useMemo(() => {
const hasThinkPart = content?.some((part) => part?.type === ContentTypes.THINK) ?? false;
@ -98,53 +102,56 @@ const ContentParts = memo(
return (
<>
{hasReasoningParts && (
<div className="mb-5">
<ThinkingButton
isExpanded={isExpanded}
onClick={() =>
setIsExpanded((prev) => {
const val = !prev;
setShowThinking(val);
return val;
})
}
label={
isSubmitting && isLast ? localize('com_ui_thinking') : localize('com_ui_thoughts')
}
/>
</div>
)}
{content
.filter((part) => part)
.map((part, idx) => {
const toolCallId =
(part?.[ContentTypes.TOOL_CALL] as Agents.ToolCall | undefined)?.id ?? '';
const attachments = attachmentMap[toolCallId];
<SearchContext.Provider value={{ searchResults }}>
<Sources />
{hasReasoningParts && (
<div className="mb-5">
<ThinkingButton
isExpanded={isExpanded}
onClick={() =>
setIsExpanded((prev) => {
const val = !prev;
setShowThinking(val);
return val;
})
}
label={
isSubmitting && isLast ? localize('com_ui_thinking') : localize('com_ui_thoughts')
}
/>
</div>
)}
{content
.filter((part) => part)
.map((part, idx) => {
const toolCallId =
(part?.[ContentTypes.TOOL_CALL] as Agents.ToolCall | undefined)?.id ?? '';
const attachments = attachmentMap[toolCallId];
return (
<MessageContext.Provider
key={`provider-${messageId}-${idx}`}
value={{
messageId,
conversationId,
partIndex: idx,
isExpanded,
nextType: content[idx + 1]?.type,
}}
>
<Part
part={part}
attachments={attachments}
isSubmitting={isSubmitting}
key={`part-${messageId}-${idx}`}
isCreatedByUser={isCreatedByUser}
isLast={idx === content.length - 1}
showCursor={idx === content.length - 1 && isLast}
/>
</MessageContext.Provider>
);
})}
return (
<MessageContext.Provider
key={`provider-${messageId}-${idx}`}
value={{
messageId,
isExpanded,
conversationId,
partIndex: idx,
nextType: content[idx + 1]?.type,
}}
>
<Part
part={part}
attachments={attachments}
isSubmitting={isSubmitting}
key={`part-${messageId}-${idx}`}
isCreatedByUser={isCreatedByUser}
isLast={idx === content.length - 1}
showCursor={idx === content.length - 1 && isLast}
/>
</MessageContext.Provider>
);
})}
</SearchContext.Provider>
</>
);
},

23
client/src/components/Chat/Messages/Content/Markdown.tsx
View file

@ -12,13 +12,16 @@ import type { Pluggable } from 'unified';
import {
useToastContext,
ArtifactProvider,
useSearchContext,
CodeBlockProvider,
useCodeBlockContext,
} from '~/Providers';
import { Citation, CompositeCitation, HighlightedText } from '~/components/Web/Citation';
import { Artifact, artifactPlugin } from '~/components/Artifacts/Artifact';
import { langSubset, preprocessLaTeX, handleDoubleClick } from '~/utils';
import CodeBlock from '~/components/Messages/Content/CodeBlock';
import useHasAccess from '~/hooks/Roles/useHasAccess';
import { unicodeCitation } from '~/components/Web';
import { useFileDownload } from '~/data-provider';
import useLocalize from '~/hooks/useLocalize';
import store from '~/store';
@ -172,6 +175,7 @@ type TContentProps = {
};
const Markdown = memo(({ content = '', isLatestMessage }: TContentProps) => {
const { searchResults } = useSearchContext();
const LaTeXParsing = useRecoilValue<boolean>(store.LaTeXParsing);
const isInitializing = content === '';
@ -197,16 +201,22 @@ const Markdown = memo(({ content = '', isLatestMessage }: TContentProps) => {
[],
);
const remarkPlugins: Pluggable[] = useMemo(
() => [
const searchTurns = useMemo(() => Object.keys(searchResults ?? {}).length, [searchResults]);
const remarkPlugins: Pluggable[] = useMemo(() => {
const plugins: Pluggable[] = [
supersub,
remarkGfm,
remarkDirective,
artifactPlugin,
[remarkMath, { singleDollarTextMath: true }],
],
[],
);
];
if (searchTurns > 0) {
plugins.push(unicodeCitation);
}
return plugins;
}, [searchTurns]);
if (isInitializing) {
return (
@ -232,6 +242,9 @@ const Markdown = memo(({ content = '', isLatestMessage }: TContentProps) => {
a,
p,
artifact: Artifact,
citation: Citation,
'highlighted-text': HighlightedText,
'composite-citation': CompositeCitation,
} as {
[nodeType: string]: React.ElementType;
}

11
client/src/components/Chat/Messages/Content/Part.tsx
View file

@ -12,6 +12,7 @@ import { ErrorMessage } from './MessageContent';
import RetrievalCall from './RetrievalCall';
import CodeAnalyze from './CodeAnalyze';
import Container from './Container';
import WebSearch from './WebSearch';
import ToolCall from './ToolCall';
import ImageGen from './ImageGen';
import Image from './Image';
@ -107,6 +108,16 @@ const Part = memo(
attachments={attachments}
/>
);
} else if (isToolCall && toolCall.name === Tools.web_search) {
return (
<WebSearch
output={toolCall.output ?? ''}
initialProgress={toolCall.progress ?? 0.1}
isSubmitting={isSubmitting}
attachments={attachments}
isLast={isLast}
/>
);
} else if (isToolCall) {
return (
<ToolCall

9
client/src/components/Chat/Messages/Content/Parts/Attachment.tsx
View file

@ -1,5 +1,5 @@
import { memo, useState, useEffect } from 'react';
import { imageExtRegex } from 'librechat-data-provider';
import { imageExtRegex, Tools } from 'librechat-data-provider';
import type { TAttachment, TFile, TAttachmentMetadata } from 'librechat-data-provider';
import FileContainer from '~/components/Chat/Input/Files/FileContainer';
import Image from '~/components/Chat/Messages/Content/Image';
@ -7,12 +7,12 @@ import { useAttachmentLink } from './LogLink';
import { cn } from '~/utils';
const FileAttachment = memo(({ attachment }: { attachment: Partial<TAttachment> }) => {
const [isVisible, setIsVisible] = useState(false);
const { handleDownload } = useAttachmentLink({
href: attachment.filepath ?? '',
filename: attachment.filename ?? '',
});
const extension = attachment.filename?.split('.').pop();
const [isVisible, setIsVisible] = useState(false);
useEffect(() => {
const timer = setTimeout(() => setIsVisible(true), 50);
@ -84,6 +84,9 @@ export default function Attachment({ attachment }: { attachment?: TAttachment })
if (!attachment) {
return null;
}
if (attachment.type === Tools.web_search) {
return null;
}
const { width, height, filepath = null } = attachment as TFile & TAttachmentMetadata;
const isImage =
@ -115,7 +118,7 @@ export function AttachmentGroup({ attachments }: { attachments?: TAttachment[] }
if (isImage) {
imageAttachments.push(attachment);
} else {
} else if (attachment.type !== Tools.web_search) {
fileAttachments.push(attachment);
}
});

91
client/src/components/Chat/Messages/Content/WebSearch.tsx Normal file
View file

@ -0,0 +1,91 @@
import { useMemo } from 'react';
import type { TAttachment } from 'librechat-data-provider';
import { StackedFavicons } from '~/components/Web/Sources';
import { useSearchContext } from '~/Providers';
import ProgressText from './ProgressText';
import { useLocalize } from '~/hooks';
type ProgressKeys =
| 'com_ui_web_searching'
| 'com_ui_web_searching_again'
| 'com_ui_web_search_processing'
| 'com_ui_web_search_reading';
export default function WebSearch({
initialProgress: progress = 0.1,
isSubmitting,
isLast,
output,
}: {
isLast?: boolean;
isSubmitting: boolean;
output?: string | null;
initialProgress: number;
attachments?: TAttachment[];
}) {
const localize = useLocalize();
const { searchResults } = useSearchContext();
const error = typeof output === 'string' && output.toLowerCase().includes('error processing');
const cancelled = (!isSubmitting && progress < 1) || error === true;
const complete = !isLast && progress === 1;
const finalizing = isSubmitting && isLast && progress === 1;
const processedSources = useMemo(() => {
if (complete && !finalizing) {
return [];
}
if (!searchResults) return [];
const values = Object.values(searchResults);
const result = values[values.length - 1];
if (!result) return [];
if (finalizing) {
return [...(result.organic || []), ...(result.topStories || [])];
}
return [...(result.organic || []), ...(result.topStories || [])].filter(
(source) => source.processed === true,
);
}, [searchResults, complete, finalizing]);
const turns = useMemo(() => {
if (!searchResults) return 0;
return Object.values(searchResults).length;
}, [searchResults]);
const clampedProgress = useMemo(() => {
return Math.min(progress, 0.99);
}, [progress]);
const showSources = processedSources.length > 0;
const progressText = useMemo(() => {
let text: ProgressKeys = turns > 1 ? 'com_ui_web_searching_again' : 'com_ui_web_searching';
if (showSources) {
text = 'com_ui_web_search_processing';
}
if (finalizing) {
text = 'com_ui_web_search_reading';
}
return localize(text);
}, [turns, localize, showSources, finalizing]);
if (complete || cancelled) {
return null;
}
return (
<>
<div className="relative my-2.5 flex size-5 shrink-0 items-center gap-2.5">
{showSources && (
<div className="mr-2">
<StackedFavicons sources={processedSources} start={-5} />
</div>
)}
<ProgressText
finishedText=""
hasInput={false}
error={cancelled}
isExpanded={false}
progress={clampedProgress}
inProgressText={progressText}
/>
</div>
</>
);
}