From 24625f56933601a94639e0345ed73b3b299ed20e Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Thu, 5 Feb 2026 14:10:19 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=A7=A9=20refactor:=20Tool=20Context=20Bui?= =?UTF-8?q?lders=20for=20Web=20Search=20&=20Image=20Gen=20(#11644)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: Web Search + Image Gen Tool Context - Added `buildWebSearchContext` function to create a structured context for web search tools, including citation format instructions. - Updated `loadTools` and `loadToolDefinitionsWrapper` functions to utilize the new web search context, improving tool initialization and response handling. - Introduced logic to handle image editing tools with `buildImageToolContext`, enhancing the overall tool management capabilities. - Refactored imports in `ToolService.js` to include the new context builders for better organization and maintainability. * fix: Trim critical output escape sequence instructions in web toolkit - Updated the critical output escape sequence instructions in the web toolkit to include a `.trim()` method, ensuring that unnecessary whitespace is removed from the output. This change enhances the consistency and reliability of the generated output. --- api/app/clients/tools/util/handleTools.js | 21 ++----------- api/server/services/ToolService.js | 36 +++++++++++++++++++++++ packages/api/src/tools/toolkits/index.ts | 1 + packages/api/src/tools/toolkits/web.ts | 23 +++++++++++++++ 4 files changed, 62 insertions(+), 19 deletions(-) create mode 100644 packages/api/src/tools/toolkits/web.ts diff --git a/api/app/clients/tools/util/handleTools.js b/api/app/clients/tools/util/handleTools.js index da4c687b4d..65c88ce83f 100644 --- a/api/app/clients/tools/util/handleTools.js +++ b/api/app/clients/tools/util/handleTools.js @@ -11,6 +11,7 @@ const { mcpToolPattern, loadWebSearchAuth, buildImageToolContext, + buildWebSearchContext, } = require('@librechat/api'); const { getMCPServersRegistry } = require('~/config'); const { @@ -19,7 +20,6 @@ const { Permissions, EToolResources, PermissionTypes, - replaceSpecialVars, } = require('librechat-data-provider'); const { availableTools, @@ -325,24 +325,7 @@ const loadTools = async ({ }); const { onSearchResults, onGetHighlights } = options?.[Tools.web_search] ?? {}; requestedTools[tool] = async () => { - toolContextMap[tool] = `# \`${tool}\`: -Current Date & Time: ${replaceSpecialVars({ text: '{{iso_datetime}}' })} - -**Execute immediately without preface.** After search, provide a brief summary addressing the query directly, then structure your response with clear Markdown formatting (## headers, lists, tables). Cite sources properly, tailor tone to query type, and provide comprehensive details. - -**CITATION FORMAT - UNICODE ESCAPE SEQUENCES ONLY:** -Use these EXACT escape sequences (copy verbatim): \\ue202 (before each anchor), \\ue200 (group start), \\ue201 (group end), \\ue203 (highlight start), \\ue204 (highlight end) - -Anchor pattern: \\ue202turn{N}{type}{index} where N=turn number, type=search|news|image|ref, index=0,1,2... - -**Examples (copy these exactly):** -- Single: "Statement.\\ue202turn0search0" -- Multiple: "Statement.\\ue202turn0search0\\ue202turn0news1" -- Group: "Statement. \\ue200\\ue202turn0search0\\ue202turn0news1\\ue201" -- Highlight: "\\ue203Cited text.\\ue204\\ue202turn0search0" -- Image: "See photo\\ue202turn0image0." - -**CRITICAL:** Output escape sequences EXACTLY as shown. Do NOT substitute with † or other symbols. Place anchors AFTER punctuation. Cite every non-obvious fact/quote. NEVER use markdown links, [1], footnotes, or HTML tags.`.trim(); + toolContextMap[tool] = buildWebSearchContext(); return createSearchTool({ ...result.authResult, onSearchResults, diff --git a/api/server/services/ToolService.js b/api/server/services/ToolService.js index 4f12f862c2..8b3dca83f2 100644 --- a/api/server/services/ToolService.js +++ b/api/server/services/ToolService.js @@ -17,6 +17,8 @@ const { loadToolDefinitions, GenerationJobManager, isActionDomainAllowed, + buildWebSearchContext, + buildImageToolContext, buildToolClassification, } = require('@librechat/api'); const { @@ -28,6 +30,7 @@ const { ContentTypes, imageGenTools, EModelEndpoint, + EToolResources, actionDelimiter, ImageVisionTool, openapiToFunction, @@ -682,9 +685,14 @@ async function loadToolDefinitionsWrapper({ req, res, agent, streamId = null, to /** @type {Record} */ const toolContextMap = {}; + const hasWebSearch = filteredTools.includes(Tools.web_search); const hasFileSearch = filteredTools.includes(Tools.file_search); const hasExecuteCode = filteredTools.includes(Tools.execute_code); + if (hasWebSearch) { + toolContextMap[Tools.web_search] = buildWebSearchContext(); + } + if (hasExecuteCode && tool_resources) { try { const authValues = await loadAuthValues({ @@ -722,6 +730,34 @@ async function loadToolDefinitionsWrapper({ req, res, agent, streamId = null, to } } + const imageFiles = tool_resources?.[EToolResources.image_edit]?.files ?? []; + if (imageFiles.length > 0) { + const hasOaiImageGen = filteredTools.includes('image_gen_oai'); + const hasGeminiImageGen = filteredTools.includes('gemini_image_gen'); + + if (hasOaiImageGen) { + const toolContext = buildImageToolContext({ + imageFiles, + toolName: `${EToolResources.image_edit}_oai`, + contextDescription: 'image editing', + }); + if (toolContext) { + toolContextMap.image_edit_oai = toolContext; + } + } + + if (hasGeminiImageGen) { + const toolContext = buildImageToolContext({ + imageFiles, + toolName: 'gemini_image_gen', + contextDescription: 'image context', + }); + if (toolContext) { + toolContextMap.gemini_image_gen = toolContext; + } + } + } + return { toolRegistry, userMCPAuthMap, diff --git a/packages/api/src/tools/toolkits/index.ts b/packages/api/src/tools/toolkits/index.ts index ce9e0584c4..efc2467b45 100644 --- a/packages/api/src/tools/toolkits/index.ts +++ b/packages/api/src/tools/toolkits/index.ts @@ -1,3 +1,4 @@ export * from './gemini'; export * from './imageContext'; export * from './oai'; +export * from './web'; diff --git a/packages/api/src/tools/toolkits/web.ts b/packages/api/src/tools/toolkits/web.ts new file mode 100644 index 0000000000..2c71aa41d2 --- /dev/null +++ b/packages/api/src/tools/toolkits/web.ts @@ -0,0 +1,23 @@ +import { Tools, replaceSpecialVars } from 'librechat-data-provider'; + +/** Builds the web search tool context with citation format instructions. */ +export function buildWebSearchContext(): string { + return `# \`${Tools.web_search}\`: +Current Date & Time: ${replaceSpecialVars({ text: '{{iso_datetime}}' })} + +**Execute immediately without preface.** After search, provide a brief summary addressing the query directly, then structure your response with clear Markdown formatting (## headers, lists, tables). Cite sources properly, tailor tone to query type, and provide comprehensive details. + +**CITATION FORMAT - UNICODE ESCAPE SEQUENCES ONLY:** +Use these EXACT escape sequences (copy verbatim): \\ue202 (before each anchor), \\ue200 (group start), \\ue201 (group end), \\ue203 (highlight start), \\ue204 (highlight end) + +Anchor pattern: \\ue202turn{N}{type}{index} where N=turn number, type=search|news|image|ref, index=0,1,2... + +**Examples (copy these exactly):** +- Single: "Statement.\\ue202turn0search0" +- Multiple: "Statement.\\ue202turn0search0\\ue202turn0news1" +- Group: "Statement. \\ue200\\ue202turn0search0\\ue202turn0news1\\ue201" +- Highlight: "\\ue203Cited text.\\ue204\\ue202turn0search0" +- Image: "See photo\\ue202turn0image0." + +**CRITICAL:** Output escape sequences EXACTLY as shown. Do NOT substitute with † or other symbols. Place anchors AFTER punctuation. Cite every non-obvious fact/quote. NEVER use markdown links, [1], footnotes, or HTML tags.`.trim(); +}