From 07511b3db8d4de690b7cdbb5052c6bc47b8b5a16 Mon Sep 17 00:00:00 2001 From: Sean McGrath Date: Tue, 26 Nov 2024 04:10:05 +1300 Subject: [PATCH 01/26] =?UTF-8?q?=F0=9F=8E=A8=20style:=20remove=20break-al?= =?UTF-8?q?l=20class=20in=20modelSpec=20menu=20(#4787)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/src/components/Chat/Menus/Models/ModelSpec.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/components/Chat/Menus/Models/ModelSpec.tsx b/client/src/components/Chat/Menus/Models/ModelSpec.tsx index 272c6af2b1..3902bb4c27 100644 --- a/client/src/components/Chat/Menus/Models/ModelSpec.tsx +++ b/client/src/components/Chat/Menus/Models/ModelSpec.tsx @@ -71,7 +71,7 @@ const MenuItem: FC = ({
{showIconInMenu && } -
+
{title}
{description}
From e0a5f879b623e851078635bdcc6d6f3d6db2ef0b Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 25 Nov 2024 13:33:06 -0500 Subject: [PATCH 02/26] =?UTF-8?q?=F0=9F=94=91=20fix:=20Azure=20Serverless?= =?UTF-8?q?=20Support=20for=20API=20Key=20Header=20&=20Version=20=20(#4791?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: azure validation/extraction types * fix: typing, add optional chaining for modelGroup and groupMap properties; expect azureOpenAIApiVersion in serverless tests * fix: add support for azureOpenAIApiVersion and api-key in serverless mode across clients * chore: update CONFIG_VERSION to 1.1.8, data-provider bump --- api/app/clients/ChatGPTClient.js | 10 +++ api/app/clients/OpenAIClient.js | 16 +++++ .../Endpoints/azureAssistants/initialize.js | 6 ++ .../Endpoints/gptPlugins/initialize.js | 6 ++ .../services/Endpoints/openAI/initialize.js | 6 ++ api/server/services/Endpoints/openAI/llm.js | 5 ++ package-lock.json | 2 +- packages/data-provider/package.json | 2 +- packages/data-provider/specs/azure.spec.ts | 13 ++-- packages/data-provider/src/azure.ts | 70 ++++++++++--------- packages/data-provider/src/config.ts | 6 +- 11 files changed, 100 insertions(+), 42 deletions(-) diff --git a/api/app/clients/ChatGPTClient.js b/api/app/clients/ChatGPTClient.js index 22f7cf3138..6a7ba7b989 100644 --- a/api/app/clients/ChatGPTClient.js +++ b/api/app/clients/ChatGPTClient.js @@ -227,6 +227,16 @@ class ChatGPTClient extends BaseClient { this.azure = !serverless && azureOptions; this.azureEndpoint = !serverless && genAzureChatCompletion(this.azure, modelOptions.model, this); + if (serverless === true) { + this.options.defaultQuery = azureOptions.azureOpenAIApiVersion + ? { 'api-version': azureOptions.azureOpenAIApiVersion } + : undefined; + this.options.headers['api-key'] = this.apiKey; + } + } + + if (this.options.defaultQuery) { + opts.defaultQuery = this.options.defaultQuery; } if (this.options.headers) { diff --git a/api/app/clients/OpenAIClient.js b/api/app/clients/OpenAIClient.js index d06ddd9177..6a8377be6f 100644 --- a/api/app/clients/OpenAIClient.js +++ b/api/app/clients/OpenAIClient.js @@ -838,6 +838,12 @@ class OpenAIClient extends BaseClient { this.options.dropParams = azureConfig.groupMap[groupName].dropParams; this.options.forcePrompt = azureConfig.groupMap[groupName].forcePrompt; this.azure = !serverless && azureOptions; + if (serverless === true) { + this.options.defaultQuery = azureOptions.azureOpenAIApiVersion + ? { 'api-version': azureOptions.azureOpenAIApiVersion } + : undefined; + this.options.headers['api-key'] = this.apiKey; + } } const titleChatCompletion = async () => { @@ -1169,6 +1175,10 @@ ${convo} opts.defaultHeaders = { ...opts.defaultHeaders, ...this.options.headers }; } + if (this.options.defaultQuery) { + opts.defaultQuery = this.options.defaultQuery; + } + if (this.options.proxy) { opts.httpAgent = new HttpsProxyAgent(this.options.proxy); } @@ -1207,6 +1217,12 @@ ${convo} this.azure = !serverless && azureOptions; this.azureEndpoint = !serverless && genAzureChatCompletion(this.azure, modelOptions.model, this); + if (serverless === true) { + this.options.defaultQuery = azureOptions.azureOpenAIApiVersion + ? { 'api-version': azureOptions.azureOpenAIApiVersion } + : undefined; + this.options.headers['api-key'] = this.apiKey; + } } if (this.azure || this.options.azure) { diff --git a/api/server/services/Endpoints/azureAssistants/initialize.js b/api/server/services/Endpoints/azureAssistants/initialize.js index 69a55c74bb..fc8024af07 100644 --- a/api/server/services/Endpoints/azureAssistants/initialize.js +++ b/api/server/services/Endpoints/azureAssistants/initialize.js @@ -135,6 +135,12 @@ const initializeClient = async ({ req, res, version, endpointOption, initAppClie clientOptions.reverseProxyUrl = baseURL ?? clientOptions.reverseProxyUrl; clientOptions.headers = opts.defaultHeaders; clientOptions.azure = !serverless && azureOptions; + if (serverless === true) { + clientOptions.defaultQuery = azureOptions.azureOpenAIApiVersion + ? { 'api-version': azureOptions.azureOpenAIApiVersion } + : undefined; + clientOptions.headers['api-key'] = apiKey; + } } } diff --git a/api/server/services/Endpoints/gptPlugins/initialize.js b/api/server/services/Endpoints/gptPlugins/initialize.js index 7e79d42564..7bfb43f004 100644 --- a/api/server/services/Endpoints/gptPlugins/initialize.js +++ b/api/server/services/Endpoints/gptPlugins/initialize.js @@ -96,6 +96,12 @@ const initializeClient = async ({ req, res, endpointOption }) => { apiKey = azureOptions.azureOpenAIApiKey; clientOptions.azure = !serverless && azureOptions; + if (serverless === true) { + clientOptions.defaultQuery = azureOptions.azureOpenAIApiVersion + ? { 'api-version': azureOptions.azureOpenAIApiVersion } + : undefined; + clientOptions.headers['api-key'] = apiKey; + } } else if (useAzure || (apiKey && apiKey.includes('{"azure') && !clientOptions.azure)) { clientOptions.azure = userProvidesKey ? JSON.parse(userValues.apiKey) : getAzureCredentials(); apiKey = clientOptions.azure.azureOpenAIApiKey; diff --git a/api/server/services/Endpoints/openAI/initialize.js b/api/server/services/Endpoints/openAI/initialize.js index 215b943730..a84be42b91 100644 --- a/api/server/services/Endpoints/openAI/initialize.js +++ b/api/server/services/Endpoints/openAI/initialize.js @@ -97,6 +97,12 @@ const initializeClient = async ({ apiKey = azureOptions.azureOpenAIApiKey; clientOptions.azure = !serverless && azureOptions; + if (serverless === true) { + clientOptions.defaultQuery = azureOptions.azureOpenAIApiVersion + ? { 'api-version': azureOptions.azureOpenAIApiVersion } + : undefined; + clientOptions.headers['api-key'] = apiKey; + } } else if (isAzureOpenAI) { clientOptions.azure = userProvidesKey ? JSON.parse(userValues.apiKey) : getAzureCredentials(); apiKey = clientOptions.azure.azureOpenAIApiKey; diff --git a/api/server/services/Endpoints/openAI/llm.js b/api/server/services/Endpoints/openAI/llm.js index bd51679e1b..e372c9d794 100644 --- a/api/server/services/Endpoints/openAI/llm.js +++ b/api/server/services/Endpoints/openAI/llm.js @@ -29,6 +29,7 @@ function getLLMConfig(apiKey, options = {}) { modelOptions = {}, reverseProxyUrl, useOpenRouter, + defaultQuery, headers, proxy, azure, @@ -74,6 +75,10 @@ function getLLMConfig(apiKey, options = {}) { } } + if (defaultQuery) { + configOptions.baseOptions.defaultQuery = defaultQuery; + } + if (proxy) { const proxyAgent = new HttpsProxyAgent(proxy); Object.assign(configOptions, { diff --git a/package-lock.json b/package-lock.json index 56937b6fff..d292fff47e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36137,7 +36137,7 @@ }, "packages/data-provider": { "name": "librechat-data-provider", - "version": "0.7.55", + "version": "0.7.56", "license": "ISC", "dependencies": { "@types/js-yaml": "^4.0.9", diff --git a/packages/data-provider/package.json b/packages/data-provider/package.json index ec47712975..95fde7c0b0 100644 --- a/packages/data-provider/package.json +++ b/packages/data-provider/package.json @@ -1,6 +1,6 @@ { "name": "librechat-data-provider", - "version": "0.7.55", + "version": "0.7.56", "description": "data services for librechat apps", "main": "dist/index.js", "module": "dist/index.es.js", diff --git a/packages/data-provider/specs/azure.spec.ts b/packages/data-provider/specs/azure.spec.ts index c73ca99e8e..5628d3f24b 100644 --- a/packages/data-provider/specs/azure.spec.ts +++ b/packages/data-provider/specs/azure.spec.ts @@ -94,8 +94,8 @@ describe('validateAzureGroups', () => { expect(isValid).toBe(true); const modelGroup = modelGroupMap['gpt-5-turbo']; expect(modelGroup).toBeDefined(); - expect(modelGroup.group).toBe('japan-east'); - expect(groupMap[modelGroup.group]).toBeDefined(); + expect(modelGroup?.group).toBe('japan-east'); + expect(groupMap[modelGroup?.group ?? '']).toBeDefined(); expect(modelNames).toContain('gpt-5-turbo'); const { azureOptions } = mapModelToAzureConfig({ modelName: 'gpt-5-turbo', @@ -323,6 +323,7 @@ describe('validateAzureGroups for Serverless Configurations', () => { expect(azureOptions).toEqual({ azureOpenAIApiKey: 'def456', + azureOpenAIApiVersion: '', }); expect(baseURL).toEqual('https://new-serverless.example.com/v1/completions'); expect(serverless).toBe(true); @@ -381,10 +382,10 @@ describe('validateAzureGroups with modelGroupMap and groupMap', () => { const { isValid, modelGroupMap, groupMap } = validateAzureGroups(validConfigs); expect(isValid).toBe(true); expect(modelGroupMap['gpt-4-turbo']).toBeDefined(); - expect(modelGroupMap['gpt-4-turbo'].group).toBe('us-east'); + expect(modelGroupMap['gpt-4-turbo']?.group).toBe('us-east'); expect(groupMap['us-east']).toBeDefined(); - expect(groupMap['us-east'].apiKey).toBe('prod-1234'); - expect(groupMap['us-east'].models['gpt-4-turbo']).toBeDefined(); + expect(groupMap['us-east']?.apiKey).toBe('prod-1234'); + expect(groupMap['us-east']?.models['gpt-4-turbo']).toBeDefined(); const { azureOptions, baseURL, headers } = mapModelToAzureConfig({ modelName: 'gpt-4-turbo', modelGroupMap, @@ -765,6 +766,7 @@ describe('validateAzureGroups with modelGroupMap and groupMap', () => { ); expect(azureOptions7).toEqual({ azureOpenAIApiKey: 'mistral-key', + azureOpenAIApiVersion: '', }); const { @@ -782,6 +784,7 @@ describe('validateAzureGroups with modelGroupMap and groupMap', () => { ); expect(azureOptions8).toEqual({ azureOpenAIApiKey: 'llama-key', + azureOpenAIApiVersion: '', }); }); }); diff --git a/packages/data-provider/src/azure.ts b/packages/data-provider/src/azure.ts index 79382eef7b..f5948820be 100644 --- a/packages/data-provider/src/azure.ts +++ b/packages/data-provider/src/azure.ts @@ -63,13 +63,13 @@ export function validateAzureGroups(configs: TAzureGroups): TAzureConfigValidati const { group: groupName, apiKey, - instanceName, - deploymentName, - version, - baseURL, + instanceName = '', + deploymentName = '', + version = '', + baseURL = '', additionalHeaders, models, - serverless, + serverless = false, ...rest } = group; @@ -120,9 +120,11 @@ export function validateAzureGroups(configs: TAzureGroups): TAzureConfigValidati continue; } + const groupDeploymentName = group.deploymentName ?? ''; + const groupVersion = group.version ?? ''; if (typeof model === 'boolean') { // For boolean models, check if group-level deploymentName and version are present. - if (!group.deploymentName || !group.version) { + if (!groupDeploymentName || !groupVersion) { errors.push( `Model "${modelName}" in group "${groupName}" is missing a deploymentName or version.`, ); @@ -133,11 +135,10 @@ export function validateAzureGroups(configs: TAzureGroups): TAzureConfigValidati group: groupName, }; } else { + const modelDeploymentName = model.deploymentName ?? ''; + const modelVersion = model.version ?? ''; // For object models, check if deploymentName and version are required but missing. - if ( - (!model.deploymentName && !group.deploymentName) || - (!model.version && !group.version) - ) { + if ((!modelDeploymentName && !groupDeploymentName) || (!modelVersion && !groupVersion)) { errors.push( `Model "${modelName}" in group "${groupName}" is missing a required deploymentName or version.`, ); @@ -146,8 +147,8 @@ export function validateAzureGroups(configs: TAzureGroups): TAzureConfigValidati modelGroupMap[modelName] = { group: groupName, - // deploymentName: model.deploymentName || group.deploymentName, - // version: model.version || group.version, + // deploymentName: modelDeploymentName || groupDeploymentName, + // version: modelVersion || groupVersion, }; } } @@ -190,26 +191,28 @@ export function mapModelToAzureConfig({ ); } - const instanceName = groupConfig.instanceName; + const instanceName = groupConfig.instanceName ?? ''; - if (!instanceName && !groupConfig.serverless) { + if (!instanceName && groupConfig.serverless !== true) { throw new Error( `Group "${modelConfig.group}" is missing an instanceName for non-serverless configuration.`, ); } - if (groupConfig.serverless && !groupConfig.baseURL) { + const baseURL = groupConfig.baseURL ?? ''; + if (groupConfig.serverless === true && !baseURL) { throw new Error( `Group "${modelConfig.group}" is missing the required base URL for serverless configuration.`, ); } - if (groupConfig.serverless) { + if (groupConfig.serverless === true) { const result: MappedAzureConfig = { azureOptions: { + azureOpenAIApiVersion: extractEnvVariable(groupConfig.version ?? ''), azureOpenAIApiKey: extractEnvVariable(groupConfig.apiKey), }, - baseURL: extractEnvVariable(groupConfig.baseURL as string), + baseURL: extractEnvVariable(baseURL), serverless: true, }; @@ -232,11 +235,11 @@ export function mapModelToAzureConfig({ } const modelDetails = groupConfig.models[modelName]; - const { deploymentName, version } = + const { deploymentName = '', version = '' } = typeof modelDetails === 'object' ? { - deploymentName: modelDetails.deploymentName || groupConfig.deploymentName, - version: modelDetails.version || groupConfig.version, + deploymentName: modelDetails.deploymentName ?? groupConfig.deploymentName, + version: modelDetails.version ?? groupConfig.version, } : { deploymentName: groupConfig.deploymentName, @@ -264,8 +267,8 @@ export function mapModelToAzureConfig({ const result: MappedAzureConfig = { azureOptions }; - if (groupConfig.baseURL) { - result.baseURL = extractEnvVariable(groupConfig.baseURL); + if (baseURL) { + result.baseURL = extractEnvVariable(baseURL); } if (groupConfig.additionalHeaders) { @@ -287,15 +290,17 @@ export function mapGroupToAzureConfig({ throw new Error(`Group named "${groupName}" not found in configuration.`); } - const instanceName = groupConfig.instanceName as string; + const instanceName = groupConfig.instanceName ?? ''; + const serverless = groupConfig.serverless ?? false; + const baseURL = groupConfig.baseURL ?? ''; - if (!instanceName && !groupConfig.serverless) { + if (!instanceName && !serverless) { throw new Error( `Group "${groupName}" is missing an instanceName for non-serverless configuration.`, ); } - if (groupConfig.serverless && !groupConfig.baseURL) { + if (serverless && !baseURL) { throw new Error( `Group "${groupName}" is missing the required base URL for serverless configuration.`, ); @@ -311,25 +316,26 @@ export function mapGroupToAzureConfig({ const modelDetails = groupConfig.models[firstModelName]; const azureOptions: AzureOptions = { + azureOpenAIApiVersion: extractEnvVariable(groupConfig.version ?? ''), azureOpenAIApiKey: extractEnvVariable(groupConfig.apiKey), azureOpenAIApiInstanceName: extractEnvVariable(instanceName), // DeploymentName and Version set below }; - if (groupConfig.serverless) { + if (serverless) { return { azureOptions, - baseURL: extractEnvVariable(groupConfig.baseURL ?? ''), + baseURL: extractEnvVariable(baseURL), serverless: true, ...(groupConfig.additionalHeaders && { headers: groupConfig.additionalHeaders }), }; } - const { deploymentName, version } = + const { deploymentName = '', version = '' } = typeof modelDetails === 'object' ? { - deploymentName: modelDetails.deploymentName || groupConfig.deploymentName, - version: modelDetails.version || groupConfig.version, + deploymentName: modelDetails.deploymentName ?? groupConfig.deploymentName, + version: modelDetails.version ?? groupConfig.version, } : { deploymentName: groupConfig.deploymentName, @@ -347,8 +353,8 @@ export function mapGroupToAzureConfig({ const result: MappedAzureConfig = { azureOptions }; - if (groupConfig.baseURL) { - result.baseURL = extractEnvVariable(groupConfig.baseURL); + if (baseURL) { + result.baseURL = extractEnvVariable(baseURL); } if (groupConfig.additionalHeaders) { diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 04f3faf077..306cc3d80e 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -114,10 +114,10 @@ export type TAzureModelMapSchema = { group: string; }; -export type TAzureModelGroupMap = Record; +export type TAzureModelGroupMap = Record; export type TAzureGroupMap = Record< string, - TAzureBaseSchema & { models: Record } + (TAzureBaseSchema & { models: Record }) | undefined >; export type TValidatedAzureConfig = { @@ -1080,7 +1080,7 @@ export enum Constants { /** Key for the app's version. */ VERSION = 'v0.7.5', /** Key for the Custom Config's version (librechat.yaml). */ - CONFIG_VERSION = '1.1.7', + CONFIG_VERSION = '1.1.8', /** Standard value for the first message's `parentMessageId` value, to indicate no parent exists. */ NO_PARENT = '00000000-0000-0000-0000-000000000000', /** Standard value for the initial conversationId before a request is sent */ From 8178ae2a20f95525c3cb41e49409ffd7281ca743 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leon=20J=C3=BCnemann?= <32913978+leon-juenemann@users.noreply.github.com> Date: Tue, 26 Nov 2024 21:02:13 +0100 Subject: [PATCH 03/26] =?UTF-8?q?=20=F0=9F=A4=96=20fix:=20Collaborative=20?= =?UTF-8?q?Agents=20are=20only=20editable=20by=20ADMIN=20#4659?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Leon Jรผnemann --- api/server/controllers/agents/v1.js | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/api/server/controllers/agents/v1.js b/api/server/controllers/agents/v1.js index 863c52431e..5212e9795b 100644 --- a/api/server/controllers/agents/v1.js +++ b/api/server/controllers/agents/v1.js @@ -111,7 +111,6 @@ const getAgentHandler = async (req, res) => { isCollaborative: agent.isCollaborative, }); } - return res.status(200).json(agent); } catch (error) { logger.error('[/Agents/:id] Error retrieving agent', error); @@ -132,16 +131,24 @@ const updateAgentHandler = async (req, res) => { try { const id = req.params.id; const { projectIds, removeProjectIds, ...updateData } = req.body; + const isAdmin = req.user.role === SystemRoles.ADMIN; + const existingAgent = await getAgent({ id }); + const isAuthor = existingAgent.author.toString() === req.user.id; - let updatedAgent; - const query = { id, author: req.user.id }; - if (req.user.role === SystemRoles.ADMIN) { - delete query.author; + if (!existingAgent) { + return res.status(404).json({ error: 'Agent not found' }); } - if (Object.keys(updateData).length > 0) { - updatedAgent = await updateAgent(query, updateData); + const hasEditPermission = existingAgent.isCollaborative || isAdmin || isAuthor; + + if (!hasEditPermission) { + return res.status(403).json({ + error: 'You do not have permission to modify this non-collaborative agent', + }); } + let updatedAgent = + Object.keys(updateData).length > 0 ? await updateAgent({ id }, updateData) : existingAgent; + if (projectIds || removeProjectIds) { updatedAgent = await updateAgentProjects({ user: req.user, From d6f7279bce6c0eaf03689e8dd4e8a854bacfcc6c Mon Sep 17 00:00:00 2001 From: Dennis Benz Date: Tue, 3 Dec 2024 23:11:47 +0100 Subject: [PATCH 04/26] =?UTF-8?q?=F0=9F=93=9C=20feat:=20Add=20script=20to?= =?UTF-8?q?=20set=20balance=20for=20user=20(#4506)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Add script to set balance for user * Show current balance before updating --- config/set-balance.js | 124 ++++++++++++++++++++++++++++++++++++++++++ package.json | 1 + 2 files changed, 125 insertions(+) create mode 100644 config/set-balance.js diff --git a/config/set-balance.js b/config/set-balance.js new file mode 100644 index 0000000000..c28586ad2d --- /dev/null +++ b/config/set-balance.js @@ -0,0 +1,124 @@ +const path = require('path'); +require('module-alias')({ base: path.resolve(__dirname, '..', 'api') }); +const { askQuestion, silentExit } = require('./helpers'); +const { isEnabled } = require('~/server/utils/handleText'); +const User = require('~/models/User'); +const connect = require('./connect'); +const Balance = require('~/models/Balance'); + +(async () => { + await connect(); + + /** + * Show the welcome / help menu + */ + console.purple('--------------------------'); + console.purple('Set balance to a user account!'); + console.purple('--------------------------'); + /** + * Set up the variables we need and get the arguments if they were passed in + */ + let email = ''; + let amount = ''; + // If we have the right number of arguments, lets use them + if (process.argv.length >= 3) { + email = process.argv[2]; + amount = process.argv[3]; + } else { + console.orange('Usage: npm run set-balance '); + console.orange('Note: if you do not pass in the arguments, you will be prompted for them.'); + console.purple('--------------------------'); + // console.purple(`[DEBUG] Args Length: ${process.argv.length}`); + } + + if (!process.env.CHECK_BALANCE) { + console.red( + 'Error: CHECK_BALANCE environment variable is not set! Configure it to use it: `CHECK_BALANCE=true`', + ); + silentExit(1); + } + if (isEnabled(process.env.CHECK_BALANCE) === false) { + console.red( + 'Error: CHECK_BALANCE environment variable is set to `false`! Please configure: `CHECK_BALANCE=true`', + ); + silentExit(1); + } + + /** + * If we don't have the right number of arguments, lets prompt the user for them + */ + if (!email) { + email = await askQuestion('Email:'); + } + // Validate the email + if (!email.includes('@')) { + console.red('Error: Invalid email address!'); + silentExit(1); + } + + // Validate the user + const user = await User.findOne({ email }).lean(); + if (!user) { + console.red('Error: No user with that email was found!'); + silentExit(1); + } else { + console.purple(`Found user: ${user.email}`); + } + + let balance = await Balance.findOne({ user: user._id }).lean(); + if (!balance) { + console.purple('User has no balance!'); + } else { + console.purple(`Current Balance: ${balance.tokenCredits}`); + } + + if (!amount) { + amount = await askQuestion('amount:'); + } + // Validate the amount + if (!amount) { + console.red('Error: Please specify an amount!'); + silentExit(1); + } + + /** + * Now that we have all the variables we need, lets set the balance + */ + let result; + try { + result = await Balance.findOneAndUpdate( + { user: user._id }, + { tokenCredits: amount }, + { upsert: true, new: true }, + ).lean(); + } catch (error) { + console.red('Error: ' + error.message); + console.error(error); + silentExit(1); + } + + // Check the result + if (!result?.tokenCredits) { + console.red('Error: Something went wrong while updating the balance!'); + console.error(result); + silentExit(1); + } + + // Done! + console.green('Balance set successfully!'); + console.purple(`New Balance: ${result.tokenCredits}`); + silentExit(0); +})(); + +process.on('uncaughtException', (err) => { + if (!err.message.includes('fetch failed')) { + console.error('There was an uncaught error:'); + console.error(err); + } + + if (err.message.includes('fetch failed')) { + return; + } else { + process.exit(1); + } +}); diff --git a/package.json b/package.json index 989c04423a..213195184a 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "scripts": { "update": "node config/update.js", "add-balance": "node config/add-balance.js", + "set-balance": "node config/set-balance.js", "list-balances": "node config/list-balances.js", "user-stats": "node config/user-stats.js", "rebuild:package-lock": "node config/packages", From ebae49433750ec19db323578c313d96a81d29c23 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 3 Dec 2024 22:25:15 -0500 Subject: [PATCH 05/26] =?UTF-8?q?=F0=9F=A4=96=20feat:=20Support=20for=20ne?= =?UTF-8?q?w=20AWS=20Nova=20Models=20&=20Updated=20Anthropic=20Rates=20(#4?= =?UTF-8?q?852)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * updated Claude 3.5 Haiku pricing https://www.anthropic.com/pricing#anthropic-api Claude 3.5 Haiku $0.80 / MTok Input $1 / MTok Prompt caching write $0.08 / MTok Prompt caching read $4 / MTok Output * Update tx.js * refactor: fix tests for cache multiplier and add new AWS models --------- Co-authored-by: khfung <68192841+khfung@users.noreply.github.com> --- api/models/tx.js | 20 ++++++++++++++----- api/models/tx.spec.js | 29 ++++++++++++++++++++++------ api/utils/tokens.js | 4 ++++ packages/data-provider/src/config.ts | 1 + 4 files changed, 43 insertions(+), 11 deletions(-) diff --git a/api/models/tx.js b/api/models/tx.js index c9a88b6d9d..ff30bbcc48 100644 --- a/api/models/tx.js +++ b/api/models/tx.js @@ -30,6 +30,9 @@ const bedrockValues = { 'amazon.titan-text-lite-v1': { prompt: 0.15, completion: 0.2 }, 'amazon.titan-text-express-v1': { prompt: 0.2, completion: 0.6 }, 'amazon.titan-text-premier-v1:0': { prompt: 0.5, completion: 1.5 }, + 'amazon.nova-micro-v1:0': { prompt: 0.035, completion: 0.14 }, + 'amazon.nova-lite-v1:0': { prompt: 0.06, completion: 0.24 }, + 'amazon.nova-pro-v1:0': { prompt: 0.8, completion: 3.2 }, }; /** @@ -56,8 +59,8 @@ const tokenValues = Object.assign( 'claude-3-sonnet': { prompt: 3, completion: 15 }, 'claude-3-5-sonnet': { prompt: 3, completion: 15 }, 'claude-3.5-sonnet': { prompt: 3, completion: 15 }, - 'claude-3-5-haiku': { prompt: 1, completion: 5 }, - 'claude-3.5-haiku': { prompt: 1, completion: 5 }, + 'claude-3-5-haiku': { prompt: 0.8, completion: 4 }, + 'claude-3.5-haiku': { prompt: 0.8, completion: 4 }, 'claude-3-haiku': { prompt: 0.25, completion: 1.25 }, 'claude-2.1': { prompt: 8, completion: 24 }, 'claude-2': { prompt: 8, completion: 24 }, @@ -83,8 +86,8 @@ const tokenValues = Object.assign( const cacheTokenValues = { 'claude-3.5-sonnet': { write: 3.75, read: 0.3 }, 'claude-3-5-sonnet': { write: 3.75, read: 0.3 }, - 'claude-3.5-haiku': { write: 1.25, read: 0.1 }, - 'claude-3-5-haiku': { write: 1.25, read: 0.1 }, + 'claude-3.5-haiku': { write: 1, read: 0.08 }, + 'claude-3-5-haiku': { write: 1, read: 0.08 }, 'claude-3-haiku': { write: 0.3, read: 0.03 }, }; @@ -208,4 +211,11 @@ const getCacheMultiplier = ({ valueKey, cacheType, model, endpoint, endpointToke return cacheTokenValues[valueKey]?.[cacheType] ?? null; }; -module.exports = { tokenValues, getValueKey, getMultiplier, getCacheMultiplier, defaultRate }; +module.exports = { + tokenValues, + getValueKey, + getMultiplier, + getCacheMultiplier, + defaultRate, + cacheTokenValues, +}; diff --git a/api/models/tx.spec.js b/api/models/tx.spec.js index d9ffafcb1e..238ca7b895 100644 --- a/api/models/tx.spec.js +++ b/api/models/tx.spec.js @@ -4,6 +4,7 @@ const { tokenValues, getValueKey, getMultiplier, + cacheTokenValues, getCacheMultiplier, } = require('./tx'); @@ -211,6 +212,7 @@ describe('getMultiplier', () => { describe('AWS Bedrock Model Tests', () => { const awsModels = [ + 'anthropic.claude-3-5-haiku-20241022-v1:0', 'anthropic.claude-3-haiku-20240307-v1:0', 'anthropic.claude-3-sonnet-20240229-v1:0', 'anthropic.claude-3-opus-20240229-v1:0', @@ -237,6 +239,9 @@ describe('AWS Bedrock Model Tests', () => { 'ai21.j2-ultra-v1', 'amazon.titan-text-lite-v1', 'amazon.titan-text-express-v1', + 'amazon.nova-micro-v1:0', + 'amazon.nova-lite-v1:0', + 'amazon.nova-pro-v1:0', ]; it('should return the correct prompt multipliers for all models', () => { @@ -260,12 +265,24 @@ describe('AWS Bedrock Model Tests', () => { describe('getCacheMultiplier', () => { it('should return the correct cache multiplier for a given valueKey and cacheType', () => { - expect(getCacheMultiplier({ valueKey: 'claude-3-5-sonnet', cacheType: 'write' })).toBe(3.75); - expect(getCacheMultiplier({ valueKey: 'claude-3-5-sonnet', cacheType: 'read' })).toBe(0.3); - expect(getCacheMultiplier({ valueKey: 'claude-3-5-haiku', cacheType: 'write' })).toBe(1.25); - expect(getCacheMultiplier({ valueKey: 'claude-3-5-haiku', cacheType: 'read' })).toBe(0.1); - expect(getCacheMultiplier({ valueKey: 'claude-3-haiku', cacheType: 'write' })).toBe(0.3); - expect(getCacheMultiplier({ valueKey: 'claude-3-haiku', cacheType: 'read' })).toBe(0.03); + expect(getCacheMultiplier({ valueKey: 'claude-3-5-sonnet', cacheType: 'write' })).toBe( + cacheTokenValues['claude-3-5-sonnet'].write, + ); + expect(getCacheMultiplier({ valueKey: 'claude-3-5-sonnet', cacheType: 'read' })).toBe( + cacheTokenValues['claude-3-5-sonnet'].read, + ); + expect(getCacheMultiplier({ valueKey: 'claude-3-5-haiku', cacheType: 'write' })).toBe( + cacheTokenValues['claude-3-5-haiku'].write, + ); + expect(getCacheMultiplier({ valueKey: 'claude-3-5-haiku', cacheType: 'read' })).toBe( + cacheTokenValues['claude-3-5-haiku'].read, + ); + expect(getCacheMultiplier({ valueKey: 'claude-3-haiku', cacheType: 'write' })).toBe( + cacheTokenValues['claude-3-haiku'].write, + ); + expect(getCacheMultiplier({ valueKey: 'claude-3-haiku', cacheType: 'read' })).toBe( + cacheTokenValues['claude-3-haiku'].read, + ); }); it('should return null if cacheType is provided but not found in cacheTokenValues', () => { diff --git a/api/utils/tokens.js b/api/utils/tokens.js index b7ede61a47..7c2ff2298f 100644 --- a/api/utils/tokens.js +++ b/api/utils/tokens.js @@ -117,6 +117,10 @@ const amazonModels = { 'amazon.titan-text-lite-v1': 4000, 'amazon.titan-text-express-v1': 8000, 'amazon.titan-text-premier-v1:0': 31500, // -500 from max + // https://aws.amazon.com/ai/generative-ai/nova/ + 'amazon.nova-micro-v1:0': 127000, // -1000 from max, + 'amazon.nova-lite-v1:0': 295000, // -5000 from max, + 'amazon.nova-pro-v1:0': 295000, // -5000 from max, }; const bedrockModels = { diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 306cc3d80e..85dd7a3d72 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -629,6 +629,7 @@ const sharedAnthropicModels = [ export const bedrockModels = [ 'anthropic.claude-3-5-sonnet-20241022-v2:0', 'anthropic.claude-3-5-sonnet-20240620-v1:0', + 'anthropic.claude-3-5-haiku-20241022-v1:0', 'anthropic.claude-3-haiku-20240307-v1:0', 'anthropic.claude-3-opus-20240229-v1:0', 'anthropic.claude-3-sonnet-20240229-v1:0', From daa8e878d240624f3cfd1d9d590a73e661b54b36 Mon Sep 17 00:00:00 2001 From: Thinger Soft Date: Wed, 4 Dec 2024 04:35:31 +0100 Subject: [PATCH 06/26] =?UTF-8?q?=F0=9F=9B=A3=EF=B8=8F=20fix:=20Chat=20Str?= =?UTF-8?q?eam=20Hangup=20(#4822)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Embedded sse.js code converted into an external dependency. Custom access token refresh logic moved to useSSE.ts hook. Closes #4820 --- client/package.json | 1 + client/src/hooks/SSE/useSSE.ts | 69 +++++--- package-lock.json | 7 + packages/data-provider/src/index.ts | 7 +- packages/data-provider/src/request.ts | 13 +- packages/data-provider/src/sse.js | 242 -------------------------- 6 files changed, 64 insertions(+), 275 deletions(-) delete mode 100644 packages/data-provider/src/sse.js diff --git a/client/package.json b/client/package.json index 3910f7beda..fa78185a99 100644 --- a/client/package.json +++ b/client/package.json @@ -96,6 +96,7 @@ "remark-gfm": "^4.0.0", "remark-math": "^6.0.0", "remark-supersub": "^1.0.0", + "sse.js": "^2.5.0", "tailwind-merge": "^1.9.1", "tailwindcss-animate": "^1.0.5", "tailwindcss-radix": "^2.8.0", diff --git a/client/src/hooks/SSE/useSSE.ts b/client/src/hooks/SSE/useSSE.ts index 1c3e51d101..7d0a63757d 100644 --- a/client/src/hooks/SSE/useSSE.ts +++ b/client/src/hooks/SSE/useSSE.ts @@ -1,22 +1,23 @@ -import { v4 } from 'uuid'; -import { useSetRecoilState } from 'recoil'; -import { useEffect, useState } from 'react'; +import type { EventSubmission, TMessage, TPayload, TSubmission } from 'librechat-data-provider'; import { /* @ts-ignore */ - SSE, createPayload, isAgentsEndpoint, - removeNullishValues, isAssistantsEndpoint, + removeNullishValues, + request, } from 'librechat-data-provider'; -import { useGetUserBalance, useGetStartupConfig } from 'librechat-data-provider/react-query'; -import type { TMessage, TSubmission, TPayload, EventSubmission } from 'librechat-data-provider'; -import type { EventHandlerParams } from './useEventHandlers'; +import { useGetStartupConfig, useGetUserBalance } from 'librechat-data-provider/react-query'; +import { useEffect, useState } from 'react'; +import { useSetRecoilState } from 'recoil'; +import { SSE } from 'sse.js'; +import { v4 } from 'uuid'; import type { TResData } from '~/common'; import { useGenTitleMutation } from '~/data-provider'; import { useAuthContext } from '~/hooks/AuthContext'; -import useEventHandlers from './useEventHandlers'; import store from '~/store'; +import type { EventHandlerParams } from './useEventHandlers'; +import useEventHandlers from './useEventHandlers'; type ChatHelpers = Pick< EventHandlerParams, @@ -94,21 +95,21 @@ export default function useSSE( let textIndex = null; - const events = new SSE(payloadData.server, { + const sse = new SSE(payloadData.server, { payload: JSON.stringify(payload), headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${token}` }, }); - events.onattachment = (e: MessageEvent) => { + sse.addEventListener('attachment', (e: MessageEvent) => { try { const data = JSON.parse(e.data); attachmentHandler({ data, submission: submission as EventSubmission }); } catch (error) { console.error(error); } - }; + }); - events.onmessage = (e: MessageEvent) => { + sse.addEventListener('message', (e: MessageEvent) => { const data = JSON.parse(e.data); if (data.final != null) { @@ -155,14 +156,14 @@ export default function useSSE( messageHandler(text, { ...submission, plugin, plugins, userMessage, initialResponse }); } } - }; + }); - events.onopen = () => { + sse.addEventListener('open', () => { setAbortScroll(false); console.log('connection is opened'); - }; + }); - events.oncancel = async () => { + sse.addEventListener('cancel', async () => { const streamKey = (submission as TSubmission | null)?.['initialResponse']?.messageId; if (completed.has(streamKey)) { setIsSubmitting(false); @@ -181,9 +182,27 @@ export default function useSSE( submission as EventSubmission, latestMessages, ); - }; + }); + + sse.addEventListener('error', async (e: MessageEvent) => { + /* @ts-ignore */ + if (e.responseCode === 401) { + /* token expired, refresh and retry */ + try { + const refreshResponse = await request.refreshToken(); + sse.headers = { + 'Content-Type': 'application/json', + Authorization: `Bearer ${refreshResponse.token}`, + }; + request.dispatchTokenUpdatedEvent(refreshResponse.token); + sse.stream(); + return; + } catch (error) { + /* token refresh failed, continue handling the original 401 */ + console.log(error); + } + } - events.onerror = function (e: MessageEvent) { console.log('error in server stream.'); (startupConfig?.checkBalance ?? false) && balanceQuery.refetch(); @@ -197,18 +216,18 @@ export default function useSSE( } errorHandler({ data, submission: { ...submission, userMessage } as EventSubmission }); - }; + }); setIsSubmitting(true); - events.stream(); + sse.stream(); return () => { - const isCancelled = events.readyState <= 1; - events.close(); - // setSource(null); + const isCancelled = sse.readyState <= 1; + sse.close(); if (isCancelled) { const e = new Event('cancel'); - events.dispatchEvent(e); + /* @ts-ignore */ + sse.dispatchEvent(e); } }; // eslint-disable-next-line react-hooks/exhaustive-deps diff --git a/package-lock.json b/package-lock.json index d292fff47e..a3fb7bf472 100644 --- a/package-lock.json +++ b/package-lock.json @@ -733,6 +733,7 @@ "remark-gfm": "^4.0.0", "remark-math": "^6.0.0", "remark-supersub": "^1.0.0", + "sse.js": "^2.5.0", "tailwind-merge": "^1.9.1", "tailwindcss-animate": "^1.0.5", "tailwindcss-radix": "^2.8.0", @@ -32747,6 +32748,12 @@ "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", "dev": true }, + "node_modules/sse.js": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/sse.js/-/sse.js-2.5.0.tgz", + "integrity": "sha512-I7zYndqOOkNpz9KIdFZ8c8A7zs1YazNewBr8Nsi/tqThfJkVPuP1q7UE2h4B0RwoWZxbBYpd06uoW3NI3SaZXg==", + "license": "Apache-2.0" + }, "node_modules/stack-trace": { "version": "0.0.10", "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", diff --git a/packages/data-provider/src/index.ts b/packages/data-provider/src/index.ts index 1fe1c00337..59e0d60b71 100644 --- a/packages/data-provider/src/index.ts +++ b/packages/data-provider/src/index.ts @@ -8,26 +8,25 @@ export * from './artifacts'; /* schema helpers */ export * from './parsers'; /* custom/dynamic configurations */ -export * from './models'; export * from './generate'; +export * from './models'; /* RBAC */ export * from './roles'; /* types (exports schemas from `./types` as they contain needed in other defs) */ export * from './types'; export * from './types/agents'; export * from './types/assistants'; -export * from './types/queries'; export * from './types/files'; export * from './types/mutations'; +export * from './types/queries'; export * from './types/runs'; /* query/mutation keys */ export * from './keys'; /* api call helpers */ export * from './headers-helpers'; export { default as request } from './request'; -import * as dataService from './data-service'; export { dataService }; +import * as dataService from './data-service'; /* general helpers */ -export * from './sse'; export * from './actions'; export { default as createPayload } from './createPayload'; diff --git a/packages/data-provider/src/request.ts b/packages/data-provider/src/request.ts index eaa1896cdb..c19d60f78a 100644 --- a/packages/data-provider/src/request.ts +++ b/packages/data-provider/src/request.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import axios, { AxiosRequestConfig, AxiosError } from 'axios'; -import { setTokenHeader } from './headers-helpers'; +import axios, { AxiosError, AxiosRequestConfig } from 'axios'; import * as endpoints from './api-endpoints'; +import { setTokenHeader } from './headers-helpers'; async function _get(url: string, options?: AxiosRequestConfig): Promise { const response = await axios.get(url, { ...options }); @@ -65,6 +65,11 @@ let failedQueue: { resolve: (value?: any) => void; reject: (reason?: any) => voi const refreshToken = (retry?: boolean) => _post(endpoints.refreshToken(retry)); +const dispatchTokenUpdatedEvent = (token: string) => { + setTokenHeader(token); + window.dispatchEvent(new CustomEvent('tokenUpdated', { detail: token })); +}; + const processQueue = (error: AxiosError | null, token: string | null = null) => { failedQueue.forEach((prom) => { if (error) { @@ -109,8 +114,7 @@ axios.interceptors.response.use( if (token) { originalRequest.headers['Authorization'] = 'Bearer ' + token; - setTokenHeader(token); - window.dispatchEvent(new CustomEvent('tokenUpdated', { detail: token })); + dispatchTokenUpdatedEvent(token); processQueue(null, token); return await axios(originalRequest); } else { @@ -139,4 +143,5 @@ export default { deleteWithOptions: _deleteWithOptions, patch: _patch, refreshToken, + dispatchTokenUpdatedEvent, }; diff --git a/packages/data-provider/src/sse.js b/packages/data-provider/src/sse.js deleted file mode 100644 index f3705a450c..0000000000 --- a/packages/data-provider/src/sse.js +++ /dev/null @@ -1,242 +0,0 @@ -/* eslint-disable */ -/** - * Copyright (C) 2016 Maxime Petazzoni . - * All rights reserved. - */ - -import request from './request'; -import { setTokenHeader } from './headers-helpers'; - -var SSE = function (url, options) { - if (!(this instanceof SSE)) { - return new SSE(url, options); - } - - this.INITIALIZING = -1; - this.CONNECTING = 0; - this.OPEN = 1; - this.CLOSED = 2; - - this.url = url; - - options = options || {}; - this.headers = options.headers || {}; - this.payload = options.payload !== undefined ? options.payload : ''; - this.method = options.method || (this.payload && 'POST') || 'GET'; - this.withCredentials = !!options.withCredentials; - - this.FIELD_SEPARATOR = ':'; - this.listeners = {}; - - this.xhr = null; - this.readyState = this.INITIALIZING; - this.progress = 0; - this.chunk = ''; - - this.addEventListener = function (type, listener) { - if (this.listeners[type] === undefined) { - this.listeners[type] = []; - } - - if (this.listeners[type].indexOf(listener) === -1) { - this.listeners[type].push(listener); - } - }; - - this.removeEventListener = function (type, listener) { - if (this.listeners[type] === undefined) { - return; - } - - var filtered = []; - this.listeners[type].forEach(function (element) { - if (element !== listener) { - filtered.push(element); - } - }); - if (filtered.length === 0) { - delete this.listeners[type]; - } else { - this.listeners[type] = filtered; - } - }; - - this.dispatchEvent = function (e) { - if (!e) { - return true; - } - - e.source = this; - - var onHandler = 'on' + e.type; - if (this.hasOwnProperty(onHandler)) { - this[onHandler].call(this, e); - if (e.defaultPrevented) { - return false; - } - } - - if (this.listeners[e.type]) { - return this.listeners[e.type].every(function (callback) { - callback(e); - return !e.defaultPrevented; - }); - } - - return true; - }; - - this._setReadyState = function (state) { - var event = new CustomEvent('readystatechange'); - event.readyState = state; - this.readyState = state; - this.dispatchEvent(event); - }; - - this._onStreamFailure = function (e) { - var event = new CustomEvent('error'); - event.data = e.currentTarget.response; - this.dispatchEvent(event); - this.close(); - }; - - this._onStreamAbort = function (e) { - this.dispatchEvent(new CustomEvent('abort')); - this.close(); - }; - - this._onStreamProgress = async function (e) { - if (!this.xhr) { - return; - } - - if (this.xhr.status === 401 && !this._retry) { - this._retry = true; - try { - const refreshResponse = await request.refreshToken(); - this.headers = { - 'Content-Type': 'application/json', - Authorization: `Bearer ${refreshResponse.token}`, - }; - setTokenHeader(refreshResponse.token); - window.dispatchEvent(new CustomEvent('tokenUpdated', { detail: refreshResponse.token })); - this.stream(); - } catch (err) { - this._onStreamFailure(e); - return; - } - } else if (this.xhr.status !== 200) { - this._onStreamFailure(e); - return; - } - - if (this.readyState == this.CONNECTING) { - this.dispatchEvent(new CustomEvent('open')); - this._setReadyState(this.OPEN); - } - - var data = this.xhr.responseText.substring(this.progress); - this.progress += data.length; - data.split(/(\r\n|\r|\n){2}/g).forEach( - function (part) { - if (part.trim().length === 0) { - this.dispatchEvent(this._parseEventChunk(this.chunk.trim())); - this.chunk = ''; - } else { - this.chunk += part; - } - }.bind(this), - ); - }; - - this._onStreamLoaded = function (e) { - this._onStreamProgress(e); - - // Parse the last chunk. - this.dispatchEvent(this._parseEventChunk(this.chunk)); - this.chunk = ''; - }; - - /** - * Parse a received SSE event chunk into a constructed event object. - */ - this._parseEventChunk = function (chunk) { - if (!chunk || chunk.length === 0) { - return null; - } - - var e = { id: null, retry: null, data: '', event: 'message' }; - chunk.split(/\n|\r\n|\r/).forEach( - function (line) { - line = line.trimRight(); - var index = line.indexOf(this.FIELD_SEPARATOR); - if (index <= 0) { - // Line was either empty, or started with a separator and is a comment. - // Either way, ignore. - return; - } - - var field = line.substring(0, index); - if (!(field in e)) { - return; - } - - var value = line.substring(index + 1).trimLeft(); - if (field === 'data') { - e[field] += value; - } else { - e[field] = value; - } - }.bind(this), - ); - - var event = new CustomEvent(e.event); - event.data = e.data; - event.id = e.id; - return event; - }; - - this._checkStreamClosed = function () { - if (!this.xhr) { - return; - } - - if (this.xhr.readyState === XMLHttpRequest.DONE) { - this._setReadyState(this.CLOSED); - } - }; - - this.stream = function () { - this._setReadyState(this.CONNECTING); - - this.xhr = new XMLHttpRequest(); - this.xhr.addEventListener('progress', this._onStreamProgress.bind(this)); - this.xhr.addEventListener('load', this._onStreamLoaded.bind(this)); - this.xhr.addEventListener('readystatechange', this._checkStreamClosed.bind(this)); - this.xhr.addEventListener('error', this._onStreamFailure.bind(this)); - this.xhr.addEventListener('abort', this._onStreamAbort.bind(this)); - this.xhr.open(this.method, this.url); - for (var header in this.headers) { - this.xhr.setRequestHeader(header, this.headers[header]); - } - this.xhr.withCredentials = this.withCredentials; - this.xhr.send(this.payload); - }; - - this.close = function () { - if (this.readyState === this.CLOSED) { - return; - } - - this.xhr.abort(); - this.xhr = null; - this._setReadyState(this.CLOSED); - }; -}; - -export { SSE }; -// Export our SSE module for npm.js -// if (typeof exports !== 'undefined') { -// // exports.SSE = SSE; -// module.exports = { SSE }; -// } From 9f25afef91bc9f522d35885de4b1b6e76036000e Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 3 Dec 2024 22:42:03 -0500 Subject: [PATCH 07/26] =?UTF-8?q?=F0=9F=94=A7=20chore:=20bump=20mongoose?= =?UTF-8?q?=20to=208.8.3=20for=20CVE-2024-53900=20(#4854)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/package.json | 2 +- package-lock.json | 185 ++++++++++++++++++++++++++++++++++++---------- 2 files changed, 147 insertions(+), 40 deletions(-) diff --git a/api/package.json b/api/package.json index 81861ae648..d0377d6e0c 100644 --- a/api/package.json +++ b/api/package.json @@ -77,7 +77,7 @@ "meilisearch": "^0.38.0", "mime": "^3.0.0", "module-alias": "^2.2.3", - "mongoose": "^7.3.3", + "mongoose": "^8.8.3", "multer": "^1.4.5-lts.1", "nanoid": "^3.3.7", "nodejs-gpt": "^1.37.4", diff --git a/package-lock.json b/package-lock.json index a3fb7bf472..c80f6537e1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -86,7 +86,7 @@ "meilisearch": "^0.38.0", "mime": "^3.0.0", "module-alias": "^2.2.3", - "mongoose": "^7.3.3", + "mongoose": "^8.8.3", "multer": "^1.4.5-lts.1", "nanoid": "^3.3.7", "nodejs-gpt": "^1.37.4", @@ -463,6 +463,14 @@ "undici-types": "~5.26.4" } }, + "api/node_modules/@types/whatwg-url": { + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-11.0.5.tgz", + "integrity": "sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ==", + "dependencies": { + "@types/webidl-conversions": "*" + } + }, "api/node_modules/ansi-styles": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", @@ -474,6 +482,14 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "api/node_modules/bson": { + "version": "6.10.1", + "resolved": "https://registry.npmjs.org/bson/-/bson-6.10.1.tgz", + "integrity": "sha512-P92xmHDQjSKPLHqFxefqMxASNq/aWJMEZugpCjf+AF/pgcUpMMQCg7t7+ewko0/u8AapvF3luf/FoehddEK+sA==", + "engines": { + "node": ">=16.20.1" + } + }, "api/node_modules/cookie-parser": { "version": "1.4.7", "resolved": "https://registry.npmjs.org/cookie-parser/-/cookie-parser-1.4.7.tgz", @@ -557,6 +573,14 @@ "node": ">= 0.6" } }, + "api/node_modules/kareem": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/kareem/-/kareem-2.6.3.tgz", + "integrity": "sha512-C3iHfuGUXK2u8/ipq9LfjFfXFxAZMQJJq7vLS45r3D9Y2xQ/m4S8zaR4zMLFWh9AsNPXmcFfUDhTEO8UIC/V6Q==", + "engines": { + "node": ">=12.0.0" + } + }, "api/node_modules/langsmith": { "version": "0.1.59", "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.1.59.tgz", @@ -578,6 +602,117 @@ } } }, + "api/node_modules/mongodb": { + "version": "6.10.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.10.0.tgz", + "integrity": "sha512-gP9vduuYWb9ZkDM546M+MP2qKVk5ZG2wPF63OvSRuUbqCR+11ZCAE1mOfllhlAG0wcoJY5yDL/rV3OmYEwXIzg==", + "dependencies": { + "@mongodb-js/saslprep": "^1.1.5", + "bson": "^6.7.0", + "mongodb-connection-string-url": "^3.0.0" + }, + "engines": { + "node": ">=16.20.1" + }, + "peerDependencies": { + "@aws-sdk/credential-providers": "^3.188.0", + "@mongodb-js/zstd": "^1.1.0", + "gcp-metadata": "^5.2.0", + "kerberos": "^2.0.1", + "mongodb-client-encryption": ">=6.0.0 <7", + "snappy": "^7.2.2", + "socks": "^2.7.1" + }, + "peerDependenciesMeta": { + "@aws-sdk/credential-providers": { + "optional": true + }, + "@mongodb-js/zstd": { + "optional": true + }, + "gcp-metadata": { + "optional": true + }, + "kerberos": { + "optional": true + }, + "mongodb-client-encryption": { + "optional": true + }, + "snappy": { + "optional": true + }, + "socks": { + "optional": true + } + } + }, + "api/node_modules/mongodb-connection-string-url": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-3.0.1.tgz", + "integrity": "sha512-XqMGwRX0Lgn05TDB4PyG2h2kKO/FfWJyCzYQbIhXUxz7ETt0I/FqHjUeqj37irJ+Dl1ZtU82uYyj14u2XsZKfg==", + "dependencies": { + "@types/whatwg-url": "^11.0.2", + "whatwg-url": "^13.0.0" + } + }, + "api/node_modules/mongodb-connection-string-url/node_modules/tr46": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-4.1.1.tgz", + "integrity": "sha512-2lv/66T7e5yNyhAAC4NaKe5nVavzuGJQVVtRYLyQ2OI8tsJ61PMLlelehb0wi2Hx6+hT/OJUWZcw8MjlSRnxvw==", + "dependencies": { + "punycode": "^2.3.0" + }, + "engines": { + "node": ">=14" + } + }, + "api/node_modules/mongodb-connection-string-url/node_modules/webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "engines": { + "node": ">=12" + } + }, + "api/node_modules/mongodb-connection-string-url/node_modules/whatwg-url": { + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-13.0.0.tgz", + "integrity": "sha512-9WWbymnqj57+XEuqADHrCJ2eSXzn8WXIW/YSGaZtb2WKAInQ6CHfaUUcTyyver0p8BDg5StLQq8h1vtZuwmOig==", + "dependencies": { + "tr46": "^4.1.1", + "webidl-conversions": "^7.0.0" + }, + "engines": { + "node": ">=16" + } + }, + "api/node_modules/mongoose": { + "version": "8.8.3", + "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-8.8.3.tgz", + "integrity": "sha512-/I4n/DcXqXyIiLRfAmUIiTjj3vXfeISke8dt4U4Y8Wfm074Wa6sXnQrXN49NFOFf2mM1kUdOXryoBvkuCnr+Qw==", + "dependencies": { + "bson": "^6.7.0", + "kareem": "2.6.3", + "mongodb": "~6.10.0", + "mpath": "0.9.0", + "mquery": "5.0.0", + "ms": "2.1.3", + "sift": "17.1.3" + }, + "engines": { + "node": ">=16.20.1" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mongoose" + } + }, + "api/node_modules/mongoose/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, "api/node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -628,6 +763,11 @@ } } }, + "api/node_modules/sift": { + "version": "17.1.3", + "resolved": "https://registry.npmjs.org/sift/-/sift-17.1.3.tgz", + "integrity": "sha512-Rtlj66/b0ICeFzYTuNvX/EF1igRbbnGSvEyT79McoZa/DeGhMyC5pWKOEsZKnpkqtSeovd5FL/bjHWC3CIIvCQ==" + }, "api/node_modules/tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", @@ -10380,7 +10520,6 @@ "version": "1.1.8", "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.1.8.tgz", "integrity": "sha512-qKwC/M/nNNaKUBMQ0nuzm47b7ZYWQHN3pcXq4IIcoSBc2hOIrflAxJduIvvqmhoz3gR2TacTAs8vlsCVPkiEdQ==", - "devOptional": true, "dependencies": { "sparse-bitfield": "^3.0.3" } @@ -16476,6 +16615,8 @@ "version": "5.5.1", "resolved": "https://registry.npmjs.org/bson/-/bson-5.5.1.tgz", "integrity": "sha512-ix0EwukN2EpC0SRWIj/7B5+A6uQMQy6KMREI9qQqvgpkV2frH63T0UDVd1SYedL6dNCmDBYB3QtXi4ISk9YT+g==", + "optional": true, + "peer": true, "engines": { "node": ">=14.20.1" } @@ -23837,14 +23978,6 @@ "safe-buffer": "^5.0.1" } }, - "node_modules/kareem": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/kareem/-/kareem-2.5.1.tgz", - "integrity": "sha512-7jFxRVm+jD+rkq3kY0iZDJfsO2/t4BBPeEb2qKn2lR/9KhuksYk5hxzfRYWMPV8P/x2d0kHD306YyWLzjjH+uA==", - "engines": { - "node": ">=12.0.0" - } - }, "node_modules/katex": { "version": "0.16.10", "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.10.tgz", @@ -26068,8 +26201,7 @@ "node_modules/memory-pager": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", - "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", - "devOptional": true + "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==" }, "node_modules/merge-descriptors": { "version": "1.0.3", @@ -26904,6 +27036,8 @@ "version": "5.9.2", "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-5.9.2.tgz", "integrity": "sha512-H60HecKO4Bc+7dhOv4sJlgvenK4fQNqqUIlXxZYQNbfEWSALGAwGoyJd/0Qwk4TttFXUOHJ2ZJQe/52ScaUwtQ==", + "optional": true, + "peer": true, "dependencies": { "bson": "^5.5.0", "mongodb-connection-string-url": "^2.6.0", @@ -27110,27 +27244,6 @@ "node": ">=16" } }, - "node_modules/mongoose": { - "version": "7.8.1", - "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-7.8.1.tgz", - "integrity": "sha512-c3MY8P1mGUGO+0H8rqxMNmAmhP0xb2EPNItfr7tHAHkh52uB0owH4Gu6q1GTUYj8yoHEDG5MN2V1aBBR6aJPuA==", - "dependencies": { - "bson": "^5.5.0", - "kareem": "2.5.1", - "mongodb": "5.9.2", - "mpath": "0.9.0", - "mquery": "5.0.0", - "ms": "2.1.3", - "sift": "16.0.1" - }, - "engines": { - "node": ">=14.20.1" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/mongoose" - } - }, "node_modules/moo": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/moo/-/moo-0.5.2.tgz", @@ -32479,11 +32592,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/sift": { - "version": "16.0.1", - "resolved": "https://registry.npmjs.org/sift/-/sift-16.0.1.tgz", - "integrity": "sha512-Wv6BjQ5zbhW7VFefWusVP33T/EM0vYikCaQ2qR8yULbsilAT8/wQaXvuQ3ptGLpoKx+lihJE3y2UTgKDyyNHZQ==" - }, "node_modules/signal-exit": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", @@ -32697,7 +32805,6 @@ "version": "3.0.3", "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==", - "devOptional": true, "dependencies": { "memory-pager": "^1.0.2" } From affcebd48cdc0e7ab85a758c581a714e414d90a5 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Wed, 4 Dec 2024 17:44:00 +0100 Subject: [PATCH 08/26] =?UTF-8?q?=E2=9A=99=EF=B8=8F=20fix:=20update=20Azur?= =?UTF-8?q?e=20OpenAI=20STT/TTS=20env=20handling=20(#4859)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/server/services/Files/Audio/STTService.js | 6 +++--- api/server/services/Files/Audio/TTSService.js | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/api/server/services/Files/Audio/STTService.js b/api/server/services/Files/Audio/STTService.js index 84590cac11..ea8d6ffaac 100644 --- a/api/server/services/Files/Audio/STTService.js +++ b/api/server/services/Files/Audio/STTService.js @@ -121,9 +121,9 @@ class STTService { */ azureOpenAIProvider(sttSchema, audioBuffer, audioFile) { const url = `${genAzureEndpoint({ - azureOpenAIApiInstanceName: sttSchema?.instanceName, - azureOpenAIApiDeploymentName: sttSchema?.deploymentName, - })}/audio/transcriptions?api-version=${sttSchema?.apiVersion}`; + azureOpenAIApiInstanceName: extractEnvVariable(sttSchema?.instanceName), + azureOpenAIApiDeploymentName: extractEnvVariable(sttSchema?.deploymentName), + })}/audio/transcriptions?api-version=${extractEnvVariable(sttSchema?.apiVersion)}`; const apiKey = sttSchema.apiKey ? extractEnvVariable(sttSchema.apiKey) : ''; diff --git a/api/server/services/Files/Audio/TTSService.js b/api/server/services/Files/Audio/TTSService.js index d9b1e1d44f..bfb90843da 100644 --- a/api/server/services/Files/Audio/TTSService.js +++ b/api/server/services/Files/Audio/TTSService.js @@ -143,9 +143,9 @@ class TTSService { */ azureOpenAIProvider(ttsSchema, input, voice) { const url = `${genAzureEndpoint({ - azureOpenAIApiInstanceName: ttsSchema?.instanceName, - azureOpenAIApiDeploymentName: ttsSchema?.deploymentName, - })}/audio/speech?api-version=${ttsSchema?.apiVersion}`; + azureOpenAIApiInstanceName: extractEnvVariable(ttsSchema?.instanceName), + azureOpenAIApiDeploymentName: extractEnvVariable(ttsSchema?.deploymentName), + })}/audio/speech?api-version=${extractEnvVariable(ttsSchema?.apiVersion)}`; if ( ttsSchema?.voices && @@ -157,7 +157,7 @@ class TTSService { } const data = { - model: ttsSchema?.model, + model: extractEnvVariable(ttsSchema?.model), input, voice: ttsSchema?.voices && ttsSchema.voices.length > 0 ? voice : undefined, }; From 1a815f5e19fd53ca321e804192629563006ba0a2 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 4 Dec 2024 15:48:13 -0500 Subject: [PATCH 09/26] =?UTF-8?q?=F0=9F=8E=89=20feat:=20Code=20Interpreter?= =?UTF-8?q?=20API=20and=20Agents=20Release=20(#4860)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Code Interpreter API & File Search Agent Uploads chore: add back code files wip: first pass, abstract key dialog refactor: influence checkbox on key changes refactor: update localization keys for 'execute code' to 'run code' wip: run code button refactor: add throwError parameter to loadAuthValues and getUserPluginAuthValue functions feat: first pass, API tool calling fix: handle missing toolId in callTool function and return 404 for non-existent tools feat: show code outputs fix: improve error handling in callTool function and log errors fix: handle potential null value for filepath in attachment destructuring fix: normalize language before rendering and prevent null return fix: add loading indicator in RunCode component while executing code feat: add support for conditional code execution in Markdown components feat: attachments refactor: remove bash fix: pass abort signal to graph/run refactor: debounce and rate limit tool call refactor: increase debounce delay for execute function feat: set code output attachments feat: image attachments refactor: apply message context refactor: pass `partIndex` feat: toolCall schema/model/methods feat: block indexing feat: get tool calls chore: imports chore: typing chore: condense type imports feat: get tool calls fix: block indexing chore: typing refactor: update tool calls mapping to support multiple results fix: add unique key to nav link for rendering wip: first pass, tool call results refactor: update query cache from successful tool call mutation style: improve result switcher styling chore: note on using \`.toObject()\` feat: add agent_id field to conversation schema chore: typing refactor: rename agentMap to agentsMap for consistency feat: Agent Name as chat input placeholder chore: bump agents ๐Ÿ“ฆ chore: update @langchain dependencies to latest versions to match agents package ๐Ÿ“ฆ chore: update @librechat/agents dependency to version 1.8.0 fix: Aborting agent stream removes sender; fix(bedrock): completion removes preset name label refactor: remove direct file parameter to use req.file, add `processAgentFileUpload` for image uploads feat: upload menu feat: prime message_file resources feat: implement conversation access validation in chat route refactor: remove file parameter from processFileUpload and use req.file instead feat: add savedMessageIds set to track saved message IDs in BaseClient, to prevent unnecessary double-write to db feat: prevent duplicate message saves by checking savedMessageIds in AgentController refactor: skip legacy RAG API handling for agents feat: add files field to convoSchema refactor: update request type annotations from Express.Request to ServerRequest in file processing functions feat: track conversation files fix: resendFiles, addPreviousAttachments handling feat: add ID validation for session_id and file_id in download route feat: entity_id for code file uploads/downloads fix: code file edge cases feat: delete related tool calls feat: add stream rate handling for LLM configuration feat: enhance system content with attached file information fix: improve error logging in resource priming function * WIP: PoC, sequential agents WIP: PoC Sequential Agents, first pass content data + bump agents package fix: package-lock WIP: PoC, o1 support, refactor bufferString feat: convertJsonSchemaToZod fix: form issues and schema defining erroneous model fix: max length issue on agent form instructions, limit conversation messages to sequential agents feat: add abort signal support to createRun function and AgentClient feat: PoC, hide prior sequential agent steps fix: update parameter naming from config to metadata in event handlers for clarity, add model to usage data refactor: use only last contentData, track model for usage data chore: bump agents package fix: content parts issue refactor: filter contentParts to include tool calls and relevant indices feat: show function calls refactor: filter context messages to exclude tool calls when no tools are available to the agent fix: ensure tool call content is not undefined in formatMessages feat: add agent_id field to conversationPreset schema feat: hide sequential agents feat: increase upload toast duration to 10 seconds * refactor: tool context handling & update Code API Key Dialog feat: toolContextMap chore: skipSpecs -> useSpecs ci: fix handleTools tests feat: API Key Dialog * feat: Agent Permissions Admin Controls feat: replace label with button for prompt permission toggle feat: update agent permissions feat: enable experimental agents and streamline capability configuration feat: implement access control for agents and enhance endpoint menu items feat: add welcome message for agent selection in localization feat: add agents permission to access control and update version to 0.7.57 * fix: update types in useAssistantListMap and useMentions hooks for better null handling * feat: mention agents * fix: agent tool resource race conditions when deleting agent tool resource files * feat: add error handling for code execution with user feedback * refactor: rename AdminControls to AdminSettings for clarity * style: add gap to button in AdminSettings for improved layout * refactor: separate agent query hooks and check access to enable fetching * fix: remove unused provider from agent initialization options, creates issue with custom endpoints * refactor: remove redundant/deprecated modelOptions from AgentClient processes * chore: update @librechat/agents to version 1.8.5 in package.json and package-lock.json * fix: minor styling issues + agent panel uniformity * fix: agent edge cases when set endpoint is no longer defined * refactor: remove unused cleanup function call from AppService * fix: update link in ApiKeyDialog to point to pricing page * fix: improve type handling and layout calculations in SidePanel component * fix: add missing localization string for agent selection in SidePanel * chore: form styling and localizations for upload filesearch/code interpreter * fix: model selection placeholder logic in AgentConfig component * style: agent capabilities * fix: add localization for provider selection and improve dropdown styling in ModelPanel * refactor: use gpt-4o-mini > gpt-3.5-turbo * fix: agents configuration for loadDefaultInterface and update related tests * feat: DALLE Agents support --- .env.example | 4 +- api/app/clients/BaseClient.js | 45 +- api/app/clients/OpenAIClient.js | 6 +- api/app/clients/PluginsClient.js | 7 +- api/app/clients/llm/createLLM.js | 2 +- api/app/clients/memory/summaryBuffer.demo.js | 2 +- api/app/clients/prompts/formatMessages.js | 2 +- api/app/clients/specs/BaseClient.test.js | 2 +- api/app/clients/specs/OpenAIClient.test.js | 4 +- api/app/clients/tools/structured/DALLE3.js | 36 +- ...{createFileSearchTool.js => fileSearch.js} | 60 +- api/app/clients/tools/util/handleTools.js | 82 +- .../clients/tools/util/handleTools.test.js | 7 +- api/cache/getLogStores.js | 1 + api/models/Agent.js | 41 +- api/models/Conversation.js | 15 + api/models/Message.js | 21 + api/models/ToolCall.js | 96 ++ api/models/index.js | 2 + api/models/schema/agent.js | 9 + api/models/schema/convoSchema.js | 6 + api/models/schema/defaults.js | 4 + api/models/schema/toolCallSchema.js | 54 + api/package.json | 6 +- api/server/controllers/AskController.js | 13 +- api/server/controllers/UserController.js | 2 + api/server/controllers/agents/callbacks.js | 93 +- api/server/controllers/agents/client.js | 246 +++- api/server/controllers/agents/request.js | 22 +- api/server/controllers/agents/run.js | 15 +- api/server/controllers/tools.js | 136 +- api/server/middleware/buildEndpointOption.js | 36 +- api/server/middleware/limiters/index.js | 2 + .../middleware/limiters/toolCallLimiter.js | 25 + api/server/routes/agents/chat.js | 13 +- api/server/routes/agents/tools.js | 19 +- api/server/routes/convos.js | 2 + api/server/routes/files/files.js | 20 +- api/server/routes/files/images.js | 15 +- api/server/routes/roles.js | 34 + api/server/services/AppService.js | 2 - api/server/services/Config/EndpointService.js | 6 +- api/server/services/Endpoints/agents/build.js | 16 +- .../services/Endpoints/agents/initialize.js | 191 ++- .../services/Endpoints/bedrock/initialize.js | 33 +- .../services/Endpoints/custom/initialize.js | 15 +- .../services/Endpoints/openAI/initialize.js | 15 +- api/server/services/Files/Code/crud.js | 13 +- api/server/services/Files/Code/process.js | 55 +- api/server/services/Files/images/encode.js | 1 + api/server/services/Files/process.js | 84 +- api/server/services/PluginService.js | 6 +- api/server/services/ToolService.js | 35 +- api/server/services/start/interface.js | 3 + api/server/services/start/interface.spec.js | 90 +- api/server/utils/handleText.js | 5 +- api/typedefs.js | 33 + client/public/assets/c.svg | 1 + client/public/assets/cplusplus.svg | 1 + client/public/assets/fortran.svg | 1 + client/public/assets/go.svg | 1 + client/public/assets/nodedotjs.svg | 1 + client/public/assets/php.svg | 1 + client/public/assets/python.svg | 1 + client/public/assets/rust.svg | 1 + client/public/assets/tsnode.svg | 1 + client/src/Providers/CodeBlockContext.tsx | 34 + client/src/Providers/MessageContext.tsx | 9 + client/src/Providers/ToolCallsMapContext.tsx | 21 + client/src/Providers/index.ts | 3 + client/src/common/agents-types.ts | 3 + client/src/common/index.ts | 1 + client/src/common/tools.ts | 6 + client/src/common/types.ts | 147 +- client/src/components/Chat/AddMultiConvo.tsx | 1 - .../Chat/Input/Files/AttachFileMenu.tsx | 100 ++ .../Chat/Input/Files/FileFormWrapper.tsx | 37 +- client/src/components/Chat/Input/Mention.tsx | 24 +- .../Chat/Menus/Endpoints/MenuItems.tsx | 74 +- .../Chat/Messages/Content/ContentParts.tsx | 29 +- .../Chat/Messages/Content/Markdown.tsx | 76 +- .../Chat/Messages/Content/MarkdownLite.tsx | 75 +- .../components/Chat/Messages/Content/Part.tsx | 229 ++- .../Messages/Content/Parts/Attachment.tsx | 19 + .../Messages/Content/Parts/ExecuteCode.tsx | 34 +- .../Messages/Content/Parts/LogContent.tsx | 50 +- .../Chat/Messages/Content/Parts/Text.tsx | 8 +- .../Chat/Messages/Content/ToolCall.tsx | 9 +- .../Chat/Messages/Content/ToolPopover.tsx | 4 +- .../components/Chat/Messages/MessageParts.tsx | 7 +- .../Chat/Messages/ui/MessageRender.tsx | 45 +- .../Endpoints/SaveAsPresetDialog.tsx | 2 +- .../components/Messages/Content/CodeBlock.tsx | 184 ++- .../Messages/Content/ResultSwitcher.tsx | 69 + .../components/Messages/Content/RunCode.tsx | 109 ++ .../src/components/Messages/ContentRender.tsx | 11 +- .../src/components/Prompts/AdminSettings.tsx | 12 +- .../Prompts/Groups/VariableForm.tsx | 8 +- .../src/components/Prompts/PromptDetails.tsx | 14 +- .../src/components/Prompts/PromptEditor.tsx | 9 +- .../components/Prompts/PromptVariables.tsx | 3 + client/src/components/Share/Message.tsx | 52 +- .../SidePanel/Agents/AdminSettings.tsx | 163 +++ .../SidePanel/Agents/AgentConfig.tsx | 54 +- .../SidePanel/Agents/AgentPanel.tsx | 9 + .../SidePanel/Agents/AgentSelect.tsx | 2 + .../SidePanel/Agents/Code/Action.tsx | 96 +- .../SidePanel/Agents/Code/ApiKeyDialog.tsx | 106 ++ .../SidePanel/Agents/Code/Files.tsx | 12 +- .../components/SidePanel/Agents/Code/Form.tsx | 19 +- .../SidePanel/Agents/FileSearch.tsx | 10 +- .../SidePanel/Agents/ModelPanel.tsx | 11 +- .../Agents/Sequential/HideSequential.tsx | 74 + .../Agents/Sequential/SequentialAgents.tsx | 153 ++ .../SidePanel/Builder/CodeFiles.tsx | 2 +- client/src/components/SidePanel/Nav.tsx | 1 + client/src/components/SidePanel/SidePanel.tsx | 12 +- client/src/components/ui/DropdownPopup.tsx | 18 +- client/src/components/ui/MultiSearch.tsx | 4 +- client/src/components/ui/SelectDropDown.tsx | 8 +- client/src/components/ui/Tooltip.tsx | 1 + client/src/data-provider/Agents/index.ts | 1 + client/src/data-provider/Agents/queries.ts | 76 + client/src/data-provider/Tools/index.ts | 2 +- client/src/data-provider/Tools/mutations.ts | 42 + client/src/data-provider/Tools/queries.ts | 23 +- client/src/data-provider/index.ts | 1 + client/src/data-provider/queries.ts | 75 - client/src/data-provider/roles.ts | 48 +- .../hooks/Assistants/useAssistantListMap.ts | 2 +- client/src/hooks/Conversations/usePresets.ts | 10 +- .../src/hooks/Files/useDelayedUploadToast.ts | 2 +- client/src/hooks/Files/useFileHandling.ts | 18 + client/src/hooks/Input/useMentions.ts | 42 +- client/src/hooks/Input/useSelectMention.ts | 22 +- client/src/hooks/Input/useTextarea.ts | 80 +- .../src/hooks/Messages/useMessageActions.tsx | 10 +- .../src/hooks/Messages/useMessageHelpers.tsx | 6 +- client/src/hooks/Nav/useSideNavLinks.ts | 14 +- client/src/hooks/Plugins/index.ts | 1 + client/src/hooks/Plugins/useCodeApiKeyForm.ts | 43 + client/src/hooks/Plugins/useToolCallsMap.ts | 28 + client/src/hooks/SSE/useStepHandler.ts | 33 +- client/src/localization/languages/Ar.ts | 4 +- client/src/localization/languages/Br.ts | 2 +- client/src/localization/languages/De.ts | 4 +- client/src/localization/languages/Eng.ts | 23 +- client/src/localization/languages/Es.ts | 4 +- client/src/localization/languages/Fi.ts | 2 +- client/src/localization/languages/Fr.ts | 4 +- client/src/localization/languages/He.ts | 2 +- client/src/localization/languages/Id.ts | 2 +- client/src/localization/languages/It.ts | 4 +- client/src/localization/languages/Jp.ts | 4 +- client/src/localization/languages/Ko.ts | 4 +- client/src/localization/languages/Ru.ts | 4 +- client/src/localization/languages/Tr.ts | 2 +- client/src/localization/languages/Zh.ts | 4 +- .../localization/languages/ZhTraditional.ts | 4 +- .../localization/prompts/instructions/Br.md | 2 +- .../localization/prompts/instructions/De.md | 2 +- .../localization/prompts/instructions/Es.md | 2 +- .../localization/prompts/instructions/Fr.md | 2 +- .../localization/prompts/instructions/He.md | 2 +- .../localization/prompts/instructions/Id.md | 2 +- .../localization/prompts/instructions/It.md | 2 +- .../localization/prompts/instructions/Jp.md | 2 +- .../localization/prompts/instructions/Ru.md | 2 +- .../localization/prompts/instructions/Tr.md | 2 +- .../localization/prompts/instructions/Zh.md | 2 +- client/src/routes/ChatRoute.tsx | 19 +- client/src/utils/endpoints.ts | 14 +- client/src/utils/languages.ts | 77 +- client/src/utils/map.ts | 52 +- client/src/utils/messages.ts | 10 +- package-lock.json | 1264 ++++++++--------- packages/data-provider/package.json | 2 +- packages/data-provider/src/api-endpoints.ts | 4 +- packages/data-provider/src/config.ts | 10 + packages/data-provider/src/data-service.ts | 32 +- packages/data-provider/src/index.ts | 1 + packages/data-provider/src/keys.ts | 1 + packages/data-provider/src/schemas.ts | 11 +- packages/data-provider/src/types/agents.ts | 12 + .../data-provider/src/types/assistants.ts | 16 +- packages/data-provider/src/types/mutations.ts | 43 +- packages/data-provider/src/types/queries.ts | 4 + packages/data-provider/src/zod.spec.ts | 467 ++++++ packages/data-provider/src/zod.ts | 66 + 189 files changed, 5056 insertions(+), 1815 deletions(-) rename api/app/clients/tools/util/{createFileSearchTool.js => fileSearch.js} (61%) create mode 100644 api/models/ToolCall.js create mode 100644 api/models/schema/toolCallSchema.js create mode 100644 api/server/middleware/limiters/toolCallLimiter.js create mode 100644 client/public/assets/c.svg create mode 100644 client/public/assets/cplusplus.svg create mode 100644 client/public/assets/fortran.svg create mode 100644 client/public/assets/go.svg create mode 100644 client/public/assets/nodedotjs.svg create mode 100644 client/public/assets/php.svg create mode 100644 client/public/assets/python.svg create mode 100644 client/public/assets/rust.svg create mode 100644 client/public/assets/tsnode.svg create mode 100644 client/src/Providers/CodeBlockContext.tsx create mode 100644 client/src/Providers/MessageContext.tsx create mode 100644 client/src/Providers/ToolCallsMapContext.tsx create mode 100644 client/src/common/tools.ts create mode 100644 client/src/components/Chat/Input/Files/AttachFileMenu.tsx create mode 100644 client/src/components/Chat/Messages/Content/Parts/Attachment.tsx create mode 100644 client/src/components/Messages/Content/ResultSwitcher.tsx create mode 100644 client/src/components/Messages/Content/RunCode.tsx create mode 100644 client/src/components/SidePanel/Agents/AdminSettings.tsx create mode 100644 client/src/components/SidePanel/Agents/Code/ApiKeyDialog.tsx create mode 100644 client/src/components/SidePanel/Agents/Sequential/HideSequential.tsx create mode 100644 client/src/components/SidePanel/Agents/Sequential/SequentialAgents.tsx create mode 100644 client/src/data-provider/Agents/index.ts create mode 100644 client/src/data-provider/Agents/queries.ts create mode 100644 client/src/data-provider/Tools/mutations.ts create mode 100644 client/src/hooks/Plugins/useCodeApiKeyForm.ts create mode 100644 client/src/hooks/Plugins/useToolCallsMap.ts create mode 100644 packages/data-provider/src/zod.spec.ts create mode 100644 packages/data-provider/src/zod.ts diff --git a/.env.example b/.env.example index e4aa8a46f0..017712027e 100644 --- a/.env.example +++ b/.env.example @@ -177,10 +177,10 @@ OPENAI_API_KEY=user_provided DEBUG_OPENAI=false # TITLE_CONVO=false -# OPENAI_TITLE_MODEL=gpt-3.5-turbo +# OPENAI_TITLE_MODEL=gpt-4o-mini # OPENAI_SUMMARIZE=true -# OPENAI_SUMMARY_MODEL=gpt-3.5-turbo +# OPENAI_SUMMARY_MODEL=gpt-4o-mini # OPENAI_FORCE_PROMPT=true diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.js index 33e3df3ac6..8b9e89860c 100644 --- a/api/app/clients/BaseClient.js +++ b/api/app/clients/BaseClient.js @@ -50,6 +50,8 @@ class BaseClient { /** The key for the usage object's output tokens * @type {string} */ this.outputTokensKey = 'completion_tokens'; + /** @type {Set} */ + this.savedMessageIds = new Set(); } setOptions() { @@ -84,7 +86,7 @@ class BaseClient { return this.options.agent.id; } - return this.modelOptions.model; + return this.modelOptions?.model ?? this.model; } /** @@ -508,7 +510,7 @@ class BaseClient { conversationId, parentMessageId: userMessage.messageId, isCreatedByUser: false, - model: this.modelOptions.model, + model: this.modelOptions?.model ?? this.model, sender: this.sender, text: generation, }; @@ -545,6 +547,7 @@ class BaseClient { if (!isEdited && !this.skipSaveUserMessage) { this.userMessagePromise = this.saveMessageToDatabase(userMessage, saveOptions, user); + this.savedMessageIds.add(userMessage.messageId); if (typeof opts?.getReqData === 'function') { opts.getReqData({ userMessagePromise: this.userMessagePromise, @@ -563,8 +566,8 @@ class BaseClient { user: this.user, tokenType: 'prompt', amount: promptTokens, - model: this.modelOptions.model, endpoint: this.options.endpoint, + model: this.modelOptions?.model ?? this.model, endpointTokenConfig: this.options.endpointTokenConfig, }, }); @@ -574,6 +577,7 @@ class BaseClient { const completion = await this.sendCompletion(payload, opts); this.abortController.requestCompleted = true; + /** @type {TMessage} */ const responseMessage = { messageId: responseMessageId, conversationId, @@ -635,7 +639,16 @@ class BaseClient { responseMessage.attachments = (await Promise.all(this.artifactPromises)).filter((a) => a); } + if (this.options.attachments) { + try { + saveOptions.files = this.options.attachments.map((attachments) => attachments.file_id); + } catch (error) { + logger.error('[BaseClient] Error mapping attachments for conversation', error); + } + } + this.responsePromise = this.saveMessageToDatabase(responseMessage, saveOptions, user); + this.savedMessageIds.add(responseMessage.messageId); const messageCache = getLogStores(CacheKeys.MESSAGES); messageCache.set( responseMessageId, @@ -902,8 +915,9 @@ class BaseClient { // Note: gpt-3.5-turbo and gpt-4 may update over time. Use default for these as well as for unknown models let tokensPerMessage = 3; let tokensPerName = 1; + const model = this.modelOptions?.model ?? this.model; - if (this.modelOptions.model === 'gpt-3.5-turbo-0301') { + if (model === 'gpt-3.5-turbo-0301') { tokensPerMessage = 4; tokensPerName = -1; } @@ -961,6 +975,15 @@ class BaseClient { return _messages; } + const seen = new Set(); + const attachmentsProcessed = + this.options.attachments && !(this.options.attachments instanceof Promise); + if (attachmentsProcessed) { + for (const attachment of this.options.attachments) { + seen.add(attachment.file_id); + } + } + /** * * @param {TMessage} message @@ -971,7 +994,19 @@ class BaseClient { this.message_file_map = {}; } - const fileIds = message.files.map((file) => file.file_id); + const fileIds = []; + for (const file of message.files) { + if (seen.has(file.file_id)) { + continue; + } + fileIds.push(file.file_id); + seen.add(file.file_id); + } + + if (fileIds.length === 0) { + return message; + } + const files = await getFiles({ file_id: { $in: fileIds }, }); diff --git a/api/app/clients/OpenAIClient.js b/api/app/clients/OpenAIClient.js index 6a8377be6f..d3bf5ddd12 100644 --- a/api/app/clients/OpenAIClient.js +++ b/api/app/clients/OpenAIClient.js @@ -688,7 +688,7 @@ class OpenAIClient extends BaseClient { } initializeLLM({ - model = 'gpt-3.5-turbo', + model = 'gpt-4o-mini', modelName, temperature = 0.2, presence_penalty = 0, @@ -793,7 +793,7 @@ class OpenAIClient extends BaseClient { const { OPENAI_TITLE_MODEL } = process.env ?? {}; - let model = this.options.titleModel ?? OPENAI_TITLE_MODEL ?? 'gpt-3.5-turbo'; + let model = this.options.titleModel ?? OPENAI_TITLE_MODEL ?? 'gpt-4o-mini'; if (model === Constants.CURRENT_MODEL) { model = this.modelOptions.model; } @@ -982,7 +982,7 @@ ${convo} let prompt; // TODO: remove the gpt fallback and make it specific to endpoint - const { OPENAI_SUMMARY_MODEL = 'gpt-3.5-turbo' } = process.env ?? {}; + const { OPENAI_SUMMARY_MODEL = 'gpt-4o-mini' } = process.env ?? {}; let model = this.options.summaryModel ?? OPENAI_SUMMARY_MODEL; if (model === Constants.CURRENT_MODEL) { model = this.modelOptions.model; diff --git a/api/app/clients/PluginsClient.js b/api/app/clients/PluginsClient.js index b25412d676..09e3dc5adc 100644 --- a/api/app/clients/PluginsClient.js +++ b/api/app/clients/PluginsClient.js @@ -105,7 +105,7 @@ class PluginsClient extends OpenAIClient { chatHistory: new ChatMessageHistory(pastMessages), }); - this.tools = await loadTools({ + const { loadedTools } = await loadTools({ user, model, tools: this.options.tools, @@ -119,12 +119,15 @@ class PluginsClient extends OpenAIClient { processFileURL, message, }, + useSpecs: true, }); - if (this.tools.length === 0) { + if (loadedTools.length === 0) { return; } + this.tools = loadedTools; + logger.debug('[PluginsClient] Requested Tools', this.options.tools); logger.debug( '[PluginsClient] Loaded Tools', diff --git a/api/app/clients/llm/createLLM.js b/api/app/clients/llm/createLLM.js index c227a2bf36..7dc0d40ceb 100644 --- a/api/app/clients/llm/createLLM.js +++ b/api/app/clients/llm/createLLM.js @@ -17,7 +17,7 @@ const { isEnabled } = require('~/server/utils'); * * @example * const llm = createLLM({ - * modelOptions: { modelName: 'gpt-3.5-turbo', temperature: 0.2 }, + * modelOptions: { modelName: 'gpt-4o-mini', temperature: 0.2 }, * configOptions: { basePath: 'https://example.api/path' }, * callbacks: { onMessage: handleMessage }, * openAIApiKey: 'your-api-key' diff --git a/api/app/clients/memory/summaryBuffer.demo.js b/api/app/clients/memory/summaryBuffer.demo.js index 73f4182710..fc575c3032 100644 --- a/api/app/clients/memory/summaryBuffer.demo.js +++ b/api/app/clients/memory/summaryBuffer.demo.js @@ -3,7 +3,7 @@ const { ChatOpenAI } = require('@langchain/openai'); const { getBufferString, ConversationSummaryBufferMemory } = require('langchain/memory'); const chatPromptMemory = new ConversationSummaryBufferMemory({ - llm: new ChatOpenAI({ modelName: 'gpt-3.5-turbo', temperature: 0 }), + llm: new ChatOpenAI({ modelName: 'gpt-4o-mini', temperature: 0 }), maxTokenLimit: 10, returnMessages: true, }); diff --git a/api/app/clients/prompts/formatMessages.js b/api/app/clients/prompts/formatMessages.js index fff18fad32..d84e62cca8 100644 --- a/api/app/clients/prompts/formatMessages.js +++ b/api/app/clients/prompts/formatMessages.js @@ -204,7 +204,7 @@ const formatAgentMessages = (payload) => { new ToolMessage({ tool_call_id: tool_call.id, name: tool_call.name, - content: output, + content: output || '', }), ); } else { diff --git a/api/app/clients/specs/BaseClient.test.js b/api/app/clients/specs/BaseClient.test.js index c62a5e2f18..4db1c9822a 100644 --- a/api/app/clients/specs/BaseClient.test.js +++ b/api/app/clients/specs/BaseClient.test.js @@ -61,7 +61,7 @@ describe('BaseClient', () => { const options = { // debug: true, modelOptions: { - model: 'gpt-3.5-turbo', + model: 'gpt-4o-mini', temperature: 0, }, }; diff --git a/api/app/clients/specs/OpenAIClient.test.js b/api/app/clients/specs/OpenAIClient.test.js index 3021c1caf2..2fa37957d1 100644 --- a/api/app/clients/specs/OpenAIClient.test.js +++ b/api/app/clients/specs/OpenAIClient.test.js @@ -221,7 +221,7 @@ describe('OpenAIClient', () => { it('should set isChatCompletion based on useOpenRouter, reverseProxyUrl, or model', () => { client.setOptions({ reverseProxyUrl: null }); - // true by default since default model will be gpt-3.5-turbo + // true by default since default model will be gpt-4o-mini expect(client.isChatCompletion).toBe(true); client.isChatCompletion = undefined; @@ -230,7 +230,7 @@ describe('OpenAIClient', () => { expect(client.isChatCompletion).toBe(false); client.isChatCompletion = undefined; - client.setOptions({ modelOptions: { model: 'gpt-3.5-turbo' }, reverseProxyUrl: null }); + client.setOptions({ modelOptions: { model: 'gpt-4o-mini' }, reverseProxyUrl: null }); expect(client.isChatCompletion).toBe(true); }); diff --git a/api/app/clients/tools/structured/DALLE3.js b/api/app/clients/tools/structured/DALLE3.js index 8cfeaf8416..b604ad4ea4 100644 --- a/api/app/clients/tools/structured/DALLE3.js +++ b/api/app/clients/tools/structured/DALLE3.js @@ -19,6 +19,8 @@ class DALLE3 extends Tool { this.userId = fields.userId; this.fileStrategy = fields.fileStrategy; + /** @type {boolean} */ + this.isAgent = fields.isAgent; if (fields.processFileURL) { /** @type {processFileURL} Necessary for output to contain all image metadata. */ this.processFileURL = fields.processFileURL.bind(this); @@ -108,6 +110,19 @@ class DALLE3 extends Tool { return `![generated image](${imageUrl})`; } + returnValue(value) { + if (this.isAgent === true && typeof value === 'string') { + return [value, {}]; + } else if (this.isAgent === true && typeof value === 'object') { + return [ + 'DALL-E displayed an image. All generated images are already plainly visible, so don\'t repeat the descriptions in detail. Do not list download links as they are available in the UI already. The user may download the images by clicking on them, but do not mention anything about downloading to the user.', + value, + ]; + } + + return value; + } + async _call(data) { const { prompt, quality = 'standard', size = '1024x1024', style = 'vivid' } = data; if (!prompt) { @@ -126,18 +141,23 @@ class DALLE3 extends Tool { }); } catch (error) { logger.error('[DALL-E-3] Problem generating the image:', error); - return `Something went wrong when trying to generate the image. The DALL-E API may be unavailable: -Error Message: ${error.message}`; + return this + .returnValue(`Something went wrong when trying to generate the image. The DALL-E API may be unavailable: +Error Message: ${error.message}`); } if (!resp) { - return 'Something went wrong when trying to generate the image. The DALL-E API may be unavailable'; + return this.returnValue( + 'Something went wrong when trying to generate the image. The DALL-E API may be unavailable', + ); } const theImageUrl = resp.data[0].url; if (!theImageUrl) { - return 'No image URL returned from OpenAI API. There may be a problem with the API or your configuration.'; + return this.returnValue( + 'No image URL returned from OpenAI API. There may be a problem with the API or your configuration.', + ); } const imageBasename = getImageBasename(theImageUrl); @@ -157,11 +177,11 @@ Error Message: ${error.message}`; try { const result = await this.processFileURL({ - fileStrategy: this.fileStrategy, - userId: this.userId, URL: theImageUrl, - fileName: imageName, basePath: 'images', + userId: this.userId, + fileName: imageName, + fileStrategy: this.fileStrategy, context: FileContext.image_generation, }); @@ -175,7 +195,7 @@ Error Message: ${error.message}`; this.result = `Failed to save the image locally. ${error.message}`; } - return this.result; + return this.returnValue(this.result); } } diff --git a/api/app/clients/tools/util/createFileSearchTool.js b/api/app/clients/tools/util/fileSearch.js similarity index 61% rename from api/app/clients/tools/util/createFileSearchTool.js rename to api/app/clients/tools/util/fileSearch.js index f00e4757f6..2d1010bd3b 100644 --- a/api/app/clients/tools/util/createFileSearchTool.js +++ b/api/app/clients/tools/util/fileSearch.js @@ -10,20 +10,50 @@ const { logger } = require('~/config'); * @param {Object} options * @param {ServerRequest} options.req * @param {Agent['tool_resources']} options.tool_resources + * @returns {Promise<{ + * files: Array<{ file_id: string; filename: string }>, + * toolContext: string + * }>} + */ +const primeFiles = async (options) => { + const { tool_resources } = options; + const file_ids = tool_resources?.[EToolResources.file_search]?.file_ids ?? []; + const agentResourceIds = new Set(file_ids); + const resourceFiles = tool_resources?.[EToolResources.file_search]?.files ?? []; + const dbFiles = ((await getFiles({ file_id: { $in: file_ids } })) ?? []).concat(resourceFiles); + + let toolContext = `- Note: Semantic search is available through the ${Tools.file_search} tool but no files are currently loaded. Request the user to upload documents to search through.`; + + const files = []; + for (let i = 0; i < dbFiles.length; i++) { + const file = dbFiles[i]; + if (!file) { + continue; + } + if (i === 0) { + toolContext = `- Note: Use the ${Tools.file_search} tool to find relevant information within:`; + } + toolContext += `\n\t- ${file.filename}${ + agentResourceIds.has(file.file_id) ? '' : ' (just attached by user)' + }`; + files.push({ + file_id: file.file_id, + filename: file.filename, + }); + } + + return { files, toolContext }; +}; + +/** + * + * @param {Object} options + * @param {ServerRequest} options.req + * @param {Array<{ file_id: string; filename: string }>} options.files * @returns */ -const createFileSearchTool = async (options) => { - const { req, tool_resources } = options; - const file_ids = tool_resources?.[EToolResources.file_search]?.file_ids ?? []; - const files = (await getFiles({ file_id: { $in: file_ids } })).map((file) => ({ - file_id: file.file_id, - filename: file.filename, - })); - - const fileList = files.map((file) => `- ${file.filename}`).join('\n'); - const toolDescription = `Performs a semantic search based on a natural language query across the following files:\n${fileList}`; - - const FileSearch = tool( +const createFileSearchTool = async ({ req, files }) => { + return tool( async ({ query }) => { if (files.length === 0) { return 'No files to search. Instruct the user to add files for the search.'; @@ -87,7 +117,7 @@ const createFileSearchTool = async (options) => { }, { name: Tools.file_search, - description: toolDescription, + description: `Performs semantic search across attached "${Tools.file_search}" documents using natural language queries. This tool analyzes the content of uploaded files to find relevant information, quotes, and passages that best match your query. Use this to extract specific information or find relevant sections within the available documents.`, schema: z.object({ query: z .string() @@ -97,8 +127,6 @@ const createFileSearchTool = async (options) => { }), }, ); - - return FileSearch; }; -module.exports = createFileSearchTool; +module.exports = { createFileSearchTool, primeFiles }; diff --git a/api/app/clients/tools/util/handleTools.js b/api/app/clients/tools/util/handleTools.js index bcfd4b3468..401bef4f52 100644 --- a/api/app/clients/tools/util/handleTools.js +++ b/api/app/clients/tools/util/handleTools.js @@ -15,8 +15,8 @@ const { StructuredWolfram, TavilySearchResults, } = require('../'); -const { primeFiles } = require('~/server/services/Files/Code/process'); -const createFileSearchTool = require('./createFileSearchTool'); +const { primeFiles: primeCodeFiles } = require('~/server/services/Files/Code/process'); +const { createFileSearchTool, primeFiles: primeSearchFiles } = require('./fileSearch'); const { loadSpecs } = require('./loadSpecs'); const { logger } = require('~/config'); @@ -83,7 +83,7 @@ const validateTools = async (user, tools = []) => { } }; -const loadAuthValues = async ({ userId, authFields }) => { +const loadAuthValues = async ({ userId, authFields, throwError = true }) => { let authValues = {}; /** @@ -98,7 +98,7 @@ const loadAuthValues = async ({ userId, authFields }) => { return { authField: field, authValue: value }; } try { - value = await getUserPluginAuthValue(userId, field); + value = await getUserPluginAuthValue(userId, field, throwError); } catch (err) { if (field === fields[fields.length - 1] && !value) { throw err; @@ -122,15 +122,18 @@ const loadAuthValues = async ({ userId, authFields }) => { return authValues; }; +/** @typedef {typeof import('@langchain/core/tools').Tool} ToolConstructor */ +/** @typedef {import('@langchain/core/tools').Tool} Tool */ + /** * Initializes a tool with authentication values for the given user, supporting alternate authentication fields. * Authentication fields can have alternates separated by "||", and the first defined variable will be used. * * @param {string} userId The user ID for which the tool is being loaded. * @param {Array} authFields Array of strings representing the authentication fields. Supports alternate fields delimited by "||". - * @param {typeof import('langchain/tools').Tool} ToolConstructor The constructor function for the tool to be initialized. + * @param {ToolConstructor} ToolConstructor The constructor function for the tool to be initialized. * @param {Object} options Optional parameters to be passed to the tool constructor alongside authentication values. - * @returns {Function} An Async function that, when called, asynchronously initializes and returns an instance of the tool with authentication. + * @returns {() => Promise} An Async function that, when called, asynchronously initializes and returns an instance of the tool with authentication. */ const loadToolWithAuth = (userId, authFields, ToolConstructor, options = {}) => { return async function () { @@ -142,11 +145,12 @@ const loadToolWithAuth = (userId, authFields, ToolConstructor, options = {}) => const loadTools = async ({ user, model, - functions = true, - returnMap = false, + isAgent, + useSpecs, tools = [], options = {}, - skipSpecs = false, + functions = true, + returnMap = false, }) => { const toolConstructors = { calculator: Calculator, @@ -174,11 +178,12 @@ const loadTools = async ({ const requestedTools = {}; - if (functions) { + if (functions === true) { toolConstructors.dalle = DALLE3; } const imageGenOptions = { + isAgent, req: options.req, fileStrategy: options.fileStrategy, processFileURL: options.processFileURL, @@ -189,7 +194,6 @@ const loadTools = async ({ const toolOptions = { serpapi: { location: 'Austin,Texas,United States', hl: 'en', gl: 'us' }, dalle: imageGenOptions, - 'dall-e': imageGenOptions, 'stable-diffusion': imageGenOptions, }; @@ -203,24 +207,38 @@ const loadTools = async ({ toolAuthFields[tool.pluginKey] = tool.authConfig.map((auth) => auth.authField); }); + const toolContextMap = {}; const remainingTools = []; for (const tool of tools) { if (tool === Tools.execute_code) { - const authValues = await loadAuthValues({ - userId: user, - authFields: [EnvVar.CODE_API_KEY], - }); - const files = await primeFiles(options, authValues[EnvVar.CODE_API_KEY]); - requestedTools[tool] = () => - createCodeExecutionTool({ + requestedTools[tool] = async () => { + const authValues = await loadAuthValues({ + userId: user, + authFields: [EnvVar.CODE_API_KEY], + }); + const codeApiKey = authValues[EnvVar.CODE_API_KEY]; + const { files, toolContext } = await primeCodeFiles(options, codeApiKey); + if (toolContext) { + toolContextMap[tool] = toolContext; + } + const CodeExecutionTool = createCodeExecutionTool({ user_id: user, files, ...authValues, }); + CodeExecutionTool.apiKey = codeApiKey; + return CodeExecutionTool; + }; continue; } else if (tool === Tools.file_search) { - requestedTools[tool] = () => createFileSearchTool(options); + requestedTools[tool] = async () => { + const { files, toolContext } = await primeSearchFiles(options); + if (toolContext) { + toolContextMap[tool] = toolContext; + } + return createFileSearchTool({ req: options.req, files }); + }; continue; } @@ -241,13 +259,13 @@ const loadTools = async ({ continue; } - if (functions) { + if (functions === true) { remainingTools.push(tool); } } let specs = null; - if (functions && remainingTools.length > 0 && skipSpecs !== true) { + if (useSpecs === true && functions === true && remainingTools.length > 0) { specs = await loadSpecs({ llm: model, user, @@ -270,23 +288,21 @@ const loadTools = async ({ return requestedTools; } - // load tools - let result = []; + const toolPromises = []; for (const tool of tools) { const validTool = requestedTools[tool]; - if (!validTool) { - continue; - } - const plugin = await validTool(); - - if (Array.isArray(plugin)) { - result = [...result, ...plugin]; - } else if (plugin) { - result.push(plugin); + if (validTool) { + toolPromises.push( + validTool().catch((error) => { + logger.error(`Error loading tool ${tool}:`, error); + return null; + }), + ); } } - return result; + const loadedTools = (await Promise.all(toolPromises)).flatMap((plugin) => plugin || []); + return { loadedTools, toolContextMap }; }; module.exports = { diff --git a/api/app/clients/tools/util/handleTools.test.js b/api/app/clients/tools/util/handleTools.test.js index 6d1c2a2e94..6538ce9aa4 100644 --- a/api/app/clients/tools/util/handleTools.test.js +++ b/api/app/clients/tools/util/handleTools.test.js @@ -128,12 +128,14 @@ describe('Tool Handlers', () => { ); beforeAll(async () => { - toolFunctions = await loadTools({ + const toolMap = await loadTools({ user: fakeUser._id, model: BaseLLM, tools: sampleTools, returnMap: true, + useSpecs: true, }); + toolFunctions = toolMap; loadTool1 = toolFunctions[sampleTools[0]]; loadTool2 = toolFunctions[sampleTools[1]]; loadTool3 = toolFunctions[sampleTools[2]]; @@ -195,6 +197,7 @@ describe('Tool Handlers', () => { expect(mockPluginService.getUserPluginAuthValue).toHaveBeenCalledWith( 'userId', 'DALLE3_API_KEY', + true, ); }); @@ -224,6 +227,7 @@ describe('Tool Handlers', () => { user: fakeUser._id, model: BaseLLM, returnMap: true, + useSpecs: true, }); expect(toolFunctions).toEqual({}); }); @@ -235,6 +239,7 @@ describe('Tool Handlers', () => { tools: ['stable-diffusion'], functions: true, returnMap: true, + useSpecs: true, }); const structuredTool = await toolFunctions['stable-diffusion'](); expect(structuredTool).toBeInstanceOf(StructuredSD); diff --git a/api/cache/getLogStores.js b/api/cache/getLogStores.js index 1fdaee9006..273888f2f4 100644 --- a/api/cache/getLogStores.js +++ b/api/cache/getLogStores.js @@ -70,6 +70,7 @@ const namespaces = { [ViolationTypes.TTS_LIMIT]: createViolationInstance(ViolationTypes.TTS_LIMIT), [ViolationTypes.STT_LIMIT]: createViolationInstance(ViolationTypes.STT_LIMIT), [ViolationTypes.CONVO_ACCESS]: createViolationInstance(ViolationTypes.CONVO_ACCESS), + [ViolationTypes.TOOL_CALL_LIMIT]: createViolationInstance(ViolationTypes.TOOL_CALL_LIMIT), [ViolationTypes.FILE_UPLOAD_LIMIT]: createViolationInstance(ViolationTypes.FILE_UPLOAD_LIMIT), [ViolationTypes.VERIFY_EMAIL_LIMIT]: createViolationInstance(ViolationTypes.VERIFY_EMAIL_LIMIT), [ViolationTypes.RESET_PASSWORD_LIMIT]: createViolationInstance( diff --git a/api/models/Agent.js b/api/models/Agent.js index 7d599d3032..206c983f97 100644 --- a/api/models/Agent.js +++ b/api/models/Agent.js @@ -118,36 +118,43 @@ const addAgentResourceFile = async ({ agent_id, tool_resource, file_id }) => { }; /** - * Removes a resource file id from an agent. + * Removes multiple resource files from an agent in a single update. * @param {object} params - * @param {ServerRequest} params.req * @param {string} params.agent_id - * @param {string} params.tool_resource - * @param {string} params.file_id + * @param {Array<{tool_resource: string, file_id: string}>} params.files * @returns {Promise} The updated agent. */ -const removeAgentResourceFile = async ({ agent_id, tool_resource, file_id }) => { +const removeAgentResourceFiles = async ({ agent_id, files }) => { const searchParameter = { id: agent_id }; const agent = await getAgent(searchParameter); if (!agent) { - throw new Error('Agent not found for removing resource file'); + throw new Error('Agent not found for removing resource files'); } - const tool_resources = agent.tool_resources || {}; + const tool_resources = { ...agent.tool_resources } || {}; - if (tool_resources[tool_resource] && tool_resources[tool_resource].file_ids) { - tool_resources[tool_resource].file_ids = tool_resources[tool_resource].file_ids.filter( - (id) => id !== file_id, - ); - - if (tool_resources[tool_resource].file_ids.length === 0) { - delete tool_resources[tool_resource]; + const filesByResource = files.reduce((acc, { tool_resource, file_id }) => { + if (!acc[tool_resource]) { + acc[tool_resource] = new Set(); } - } + acc[tool_resource].add(file_id); + return acc; + }, {}); + + Object.entries(filesByResource).forEach(([resource, fileIds]) => { + if (tool_resources[resource] && tool_resources[resource].file_ids) { + tool_resources[resource].file_ids = tool_resources[resource].file_ids.filter( + (id) => !fileIds.has(id), + ); + + if (tool_resources[resource].file_ids.length === 0) { + delete tool_resources[resource]; + } + } + }); const updateData = { tool_resources }; - return await updateAgent(searchParameter, updateData); }; @@ -281,5 +288,5 @@ module.exports = { getListAgents, updateAgentProjects, addAgentResourceFile, - removeAgentResourceFile, + removeAgentResourceFiles, }; diff --git a/api/models/Conversation.js b/api/models/Conversation.js index 0850ed0a71..8231f4548f 100644 --- a/api/models/Conversation.js +++ b/api/models/Conversation.js @@ -15,6 +15,19 @@ const searchConversation = async (conversationId) => { throw new Error('Error searching conversation'); } }; +/** + * Searches for a conversation by conversationId and returns associated file ids. + * @param {string} conversationId - The conversation's ID. + * @returns {Promise} + */ +const getConvoFiles = async (conversationId) => { + try { + return (await Conversation.findOne({ conversationId }, 'files').lean())?.files ?? []; + } catch (error) { + logger.error('[getConvoFiles] Error getting conversation files', error); + throw new Error('Error getting conversation files'); + } +}; /** * Retrieves a single conversation for a given user and conversation ID. @@ -62,6 +75,7 @@ const deleteNullOrEmptyConversations = async () => { module.exports = { Conversation, + getConvoFiles, searchConversation, deleteNullOrEmptyConversations, /** @@ -82,6 +96,7 @@ module.exports = { update.conversationId = newConversationId; } + /** Note: the resulting Model object is necessary for Meilisearch operations */ const conversation = await Conversation.findOneAndUpdate( { conversationId, user: req.user.id }, update, diff --git a/api/models/Message.js b/api/models/Message.js index 0d807f6bfd..f8f4fa7bc4 100644 --- a/api/models/Message.js +++ b/api/models/Message.js @@ -265,6 +265,26 @@ async function getMessages(filter, select) { } } +/** + * Retrieves a single message from the database. + * @async + * @function getMessage + * @param {{ user: string, messageId: string }} params - The search parameters + * @returns {Promise} The message that matches the criteria or null if not found + * @throws {Error} If there is an error in retrieving the message + */ +async function getMessage({ user, messageId }) { + try { + return await Message.findOne({ + user, + messageId, + }).lean(); + } catch (err) { + logger.error('Error getting message:', err); + throw err; + } +} + /** * Deletes messages from the database. * @@ -292,5 +312,6 @@ module.exports = { updateMessage, deleteMessagesSince, getMessages, + getMessage, deleteMessages, }; diff --git a/api/models/ToolCall.js b/api/models/ToolCall.js new file mode 100644 index 0000000000..e1d7b0cc84 --- /dev/null +++ b/api/models/ToolCall.js @@ -0,0 +1,96 @@ +const ToolCall = require('./schema/toolCallSchema'); + +/** + * Create a new tool call + * @param {ToolCallData} toolCallData - The tool call data + * @returns {Promise} The created tool call document + */ +async function createToolCall(toolCallData) { + try { + return await ToolCall.create(toolCallData); + } catch (error) { + throw new Error(`Error creating tool call: ${error.message}`); + } +} + +/** + * Get a tool call by ID + * @param {string} id - The tool call document ID + * @returns {Promise} The tool call document or null if not found + */ +async function getToolCallById(id) { + try { + return await ToolCall.findById(id).lean(); + } catch (error) { + throw new Error(`Error fetching tool call: ${error.message}`); + } +} + +/** + * Get tool calls by message ID and user + * @param {string} messageId - The message ID + * @param {string} userId - The user's ObjectId + * @returns {Promise} Array of tool call documents + */ +async function getToolCallsByMessage(messageId, userId) { + try { + return await ToolCall.find({ messageId, user: userId }).lean(); + } catch (error) { + throw new Error(`Error fetching tool calls: ${error.message}`); + } +} + +/** + * Get tool calls by conversation ID and user + * @param {string} conversationId - The conversation ID + * @param {string} userId - The user's ObjectId + * @returns {Promise} Array of tool call documents + */ +async function getToolCallsByConvo(conversationId, userId) { + try { + return await ToolCall.find({ conversationId, user: userId }).lean(); + } catch (error) { + throw new Error(`Error fetching tool calls: ${error.message}`); + } +} + +/** + * Update a tool call + * @param {string} id - The tool call document ID + * @param {Partial} updateData - The data to update + * @returns {Promise} The updated tool call document or null if not found + */ +async function updateToolCall(id, updateData) { + try { + return await ToolCall.findByIdAndUpdate(id, updateData, { new: true }).lean(); + } catch (error) { + throw new Error(`Error updating tool call: ${error.message}`); + } +} + +/** + * Delete a tool call + * @param {string} userId - The related user's ObjectId + * @param {string} [conversationId] - The tool call conversation ID + * @returns {Promise<{ ok?: number; n?: number; deletedCount?: number }>} The result of the delete operation + */ +async function deleteToolCalls(userId, conversationId) { + try { + const query = { user: userId }; + if (conversationId) { + query.conversationId = conversationId; + } + return await ToolCall.deleteMany(query); + } catch (error) { + throw new Error(`Error deleting tool call: ${error.message}`); + } +} + +module.exports = { + createToolCall, + updateToolCall, + deleteToolCalls, + getToolCallById, + getToolCallsByConvo, + getToolCallsByMessage, +}; diff --git a/api/models/index.js b/api/models/index.js index 380c93cc42..73fc2f4ab9 100644 --- a/api/models/index.js +++ b/api/models/index.js @@ -18,6 +18,7 @@ const { updateFileUsage, } = require('./File'); const { + getMessage, getMessages, saveMessage, recordMessage, @@ -51,6 +52,7 @@ module.exports = { getFiles, updateFileUsage, + getMessage, getMessages, saveMessage, recordMessage, diff --git a/api/models/schema/agent.js b/api/models/schema/agent.js index d7c5762b53..2006859ab6 100644 --- a/api/models/schema/agent.js +++ b/api/models/schema/agent.js @@ -58,6 +58,15 @@ const agentSchema = mongoose.Schema( type: String, default: undefined, }, + hide_sequential_outputs: { + type: Boolean, + }, + end_after_tools: { + type: Boolean, + }, + agent_ids: { + type: [String], + }, isCollaborative: { type: Boolean, default: undefined, diff --git a/api/models/schema/convoSchema.js b/api/models/schema/convoSchema.js index 7b020e3309..85232ed6a2 100644 --- a/api/models/schema/convoSchema.js +++ b/api/models/schema/convoSchema.js @@ -26,6 +26,9 @@ const convoSchema = mongoose.Schema( type: mongoose.Schema.Types.Mixed, }, ...conversationPreset, + agent_id: { + type: String, + }, // for bingAI only bingConversationId: { type: String, @@ -47,6 +50,9 @@ const convoSchema = mongoose.Schema( default: [], meiliIndex: true, }, + files: { + type: [String], + }, }, { timestamps: true }, ); diff --git a/api/models/schema/defaults.js b/api/models/schema/defaults.js index 6dced3af86..7898482359 100644 --- a/api/models/schema/defaults.js +++ b/api/models/schema/defaults.js @@ -93,6 +93,10 @@ const conversationPreset = { imageDetail: { type: String, }, + /* agents */ + agent_id: { + type: String, + }, /* assistants */ assistant_id: { type: String, diff --git a/api/models/schema/toolCallSchema.js b/api/models/schema/toolCallSchema.js new file mode 100644 index 0000000000..2af4c67c1b --- /dev/null +++ b/api/models/schema/toolCallSchema.js @@ -0,0 +1,54 @@ +const mongoose = require('mongoose'); + +/** + * @typedef {Object} ToolCallData + * @property {string} conversationId - The ID of the conversation + * @property {string} messageId - The ID of the message + * @property {string} toolId - The ID of the tool + * @property {string | ObjectId} user - The user's ObjectId + * @property {unknown} [result] - Optional result data + * @property {TAttachment[]} [attachments] - Optional attachments data + * @property {number} [blockIndex] - Optional code block index + * @property {number} [partIndex] - Optional part index + */ + +/** @type {MongooseSchema} */ +const toolCallSchema = mongoose.Schema( + { + conversationId: { + type: String, + required: true, + }, + messageId: { + type: String, + required: true, + }, + toolId: { + type: String, + required: true, + }, + user: { + type: mongoose.Schema.Types.ObjectId, + ref: 'User', + required: true, + }, + result: { + type: mongoose.Schema.Types.Mixed, + }, + attachments: { + type: mongoose.Schema.Types.Mixed, + }, + blockIndex: { + type: Number, + }, + partIndex: { + type: Number, + }, + }, + { timestamps: true }, +); + +toolCallSchema.index({ messageId: 1, user: 1 }); +toolCallSchema.index({ conversationId: 1, user: 1 }); + +module.exports = mongoose.model('ToolCall', toolCallSchema); diff --git a/api/package.json b/api/package.json index d0377d6e0c..9184037fdf 100644 --- a/api/package.json +++ b/api/package.json @@ -39,12 +39,12 @@ "@google/generative-ai": "^0.21.0", "@keyv/mongo": "^2.1.8", "@keyv/redis": "^2.8.1", - "@langchain/community": "^0.3.13", - "@langchain/core": "^0.3.17", + "@langchain/community": "^0.3.14", + "@langchain/core": "^0.3.18", "@langchain/google-genai": "^0.1.4", "@langchain/google-vertexai": "^0.1.2", "@langchain/textsplitters": "^0.1.0", - "@librechat/agents": "^1.7.7", + "@librechat/agents": "^1.8.5", "axios": "^1.7.7", "bcryptjs": "^2.4.3", "cheerio": "^1.0.0-rc.12", diff --git a/api/server/controllers/AskController.js b/api/server/controllers/AskController.js index d2d774b009..6534d6b3b3 100644 --- a/api/server/controllers/AskController.js +++ b/api/server/controllers/AskController.js @@ -127,6 +127,7 @@ const AskController = async (req, res, next, initializeClient, addTitle) => { }, }; + /** @type {TMessage} */ let response = await client.sendMessage(text, messageOptions); response.endpoint = endpointOption.endpoint; @@ -150,11 +151,13 @@ const AskController = async (req, res, next, initializeClient, addTitle) => { }); res.end(); - await saveMessage( - req, - { ...response, user }, - { context: 'api/server/controllers/AskController.js - response end' }, - ); + if (!client.savedMessageIds.has(response.messageId)) { + await saveMessage( + req, + { ...response, user }, + { context: 'api/server/controllers/AskController.js - response end' }, + ); + } } if (!client.skipSaveUserMessage) { diff --git a/api/server/controllers/UserController.js b/api/server/controllers/UserController.js index f9ed887b15..9e01da38e5 100644 --- a/api/server/controllers/UserController.js +++ b/api/server/controllers/UserController.js @@ -14,6 +14,7 @@ const { updateUserPluginsService, deleteUserKey } = require('~/server/services/U const { verifyEmail, resendVerificationEmail } = require('~/server/services/AuthService'); const { processDeleteRequest } = require('~/server/services/Files/process'); const { deleteAllSharedLinks } = require('~/models/Share'); +const { deleteToolCalls } = require('~/models/ToolCall'); const { Transaction } = require('~/models/Transaction'); const { logger } = require('~/config'); @@ -123,6 +124,7 @@ const deleteUserController = async (req, res) => { await deleteAllSharedLinks(user.id); // delete user shared links await deleteUserFiles(req); // delete user files await deleteFiles(null, user.id); // delete database files in case of orphaned files from previous steps + await deleteToolCalls(user.id); // delete user tool calls /* TODO: queue job for cleaning actions and assistants of non-existant users */ logger.info(`User deleted account. Email: ${user.email} ID: ${user.id}`); res.status(200).send({ message: 'User deleted' }); diff --git a/api/server/controllers/agents/callbacks.js b/api/server/controllers/agents/callbacks.js index 209de71714..08fceeb3c8 100644 --- a/api/server/controllers/agents/callbacks.js +++ b/api/server/controllers/agents/callbacks.js @@ -1,4 +1,4 @@ -const { Tools } = require('librechat-data-provider'); +const { Tools, StepTypes, imageGenTools } = require('librechat-data-provider'); const { EnvVar, GraphEvents, @@ -57,6 +57,9 @@ class ModelEndHandler { } const usage = data?.output?.usage_metadata; + if (metadata?.model) { + usage.model = metadata.model; + } if (usage) { this.collectedUsage.push(usage); @@ -89,9 +92,27 @@ function getDefaultHandlers({ res, aggregateContent, toolEndCallback, collectedU * Handle ON_RUN_STEP event. * @param {string} event - The event name. * @param {StreamEventData} data - The event data. + * @param {GraphRunnableConfig['configurable']} [metadata] The runnable metadata. */ - handle: (event, data) => { - sendEvent(res, { event, data }); + handle: (event, data, metadata) => { + if (data?.stepDetails.type === StepTypes.TOOL_CALLS) { + sendEvent(res, { event, data }); + } else if (metadata?.last_agent_index === metadata?.agent_index) { + sendEvent(res, { event, data }); + } else if (!metadata?.hide_sequential_outputs) { + sendEvent(res, { event, data }); + } else { + const agentName = metadata?.name ?? 'Agent'; + const isToolCall = data?.stepDetails.type === StepTypes.TOOL_CALLS; + const action = isToolCall ? 'performing a task...' : 'thinking...'; + sendEvent(res, { + event: 'on_agent_update', + data: { + runId: metadata?.run_id, + message: `${agentName} is ${action}`, + }, + }); + } aggregateContent({ event, data }); }, }, @@ -100,9 +121,16 @@ function getDefaultHandlers({ res, aggregateContent, toolEndCallback, collectedU * Handle ON_RUN_STEP_DELTA event. * @param {string} event - The event name. * @param {StreamEventData} data - The event data. + * @param {GraphRunnableConfig['configurable']} [metadata] The runnable metadata. */ - handle: (event, data) => { - sendEvent(res, { event, data }); + handle: (event, data, metadata) => { + if (data?.delta.type === StepTypes.TOOL_CALLS) { + sendEvent(res, { event, data }); + } else if (metadata?.last_agent_index === metadata?.agent_index) { + sendEvent(res, { event, data }); + } else if (!metadata?.hide_sequential_outputs) { + sendEvent(res, { event, data }); + } aggregateContent({ event, data }); }, }, @@ -111,9 +139,16 @@ function getDefaultHandlers({ res, aggregateContent, toolEndCallback, collectedU * Handle ON_RUN_STEP_COMPLETED event. * @param {string} event - The event name. * @param {StreamEventData & { result: ToolEndData }} data - The event data. + * @param {GraphRunnableConfig['configurable']} [metadata] The runnable metadata. */ - handle: (event, data) => { - sendEvent(res, { event, data }); + handle: (event, data, metadata) => { + if (data?.result != null) { + sendEvent(res, { event, data }); + } else if (metadata?.last_agent_index === metadata?.agent_index) { + sendEvent(res, { event, data }); + } else if (!metadata?.hide_sequential_outputs) { + sendEvent(res, { event, data }); + } aggregateContent({ event, data }); }, }, @@ -122,9 +157,14 @@ function getDefaultHandlers({ res, aggregateContent, toolEndCallback, collectedU * Handle ON_MESSAGE_DELTA event. * @param {string} event - The event name. * @param {StreamEventData} data - The event data. + * @param {GraphRunnableConfig['configurable']} [metadata] The runnable metadata. */ - handle: (event, data) => { - sendEvent(res, { event, data }); + handle: (event, data, metadata) => { + if (metadata?.last_agent_index === metadata?.agent_index) { + sendEvent(res, { event, data }); + } else if (!metadata?.hide_sequential_outputs) { + sendEvent(res, { event, data }); + } aggregateContent({ event, data }); }, }, @@ -151,16 +191,41 @@ function createToolEndCallback({ req, res, artifactPromises }) { return; } + if (imageGenTools.has(output.name) && output.artifact) { + artifactPromises.push( + (async () => { + const fileMetadata = Object.assign(output.artifact, { + messageId: metadata.run_id, + toolCallId: output.tool_call_id, + conversationId: metadata.thread_id, + }); + if (!res.headersSent) { + return fileMetadata; + } + + if (!fileMetadata) { + return null; + } + + res.write(`event: attachment\ndata: ${JSON.stringify(fileMetadata)}\n\n`); + return fileMetadata; + })().catch((error) => { + logger.error('Error processing code output:', error); + return null; + }), + ); + return; + } + if (output.name !== Tools.execute_code) { return; } - const { tool_call_id, artifact } = output; - if (!artifact.files) { + if (!output.artifact.files) { return; } - for (const file of artifact.files) { + for (const file of output.artifact.files) { const { id, name } = file; artifactPromises.push( (async () => { @@ -173,10 +238,10 @@ function createToolEndCallback({ req, res, artifactPromises }) { id, name, apiKey: result[EnvVar.CODE_API_KEY], - toolCallId: tool_call_id, messageId: metadata.run_id, - session_id: artifact.session_id, + toolCallId: output.tool_call_id, conversationId: metadata.thread_id, + session_id: output.artifact.session_id, }); if (!res.headersSent) { return fileMetadata; diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js index 277d545baa..450accb8ae 100644 --- a/api/server/controllers/agents/client.js +++ b/api/server/controllers/agents/client.js @@ -12,9 +12,11 @@ const { Constants, VisionModes, openAISchema, + ContentTypes, EModelEndpoint, KnownEndpoints, anthropicSchema, + isAgentsEndpoint, bedrockOutputParser, removeNullishValues, } = require('librechat-data-provider'); @@ -30,10 +32,10 @@ const { createContextHandlers, } = require('~/app/clients/prompts'); const { encodeAndFormat } = require('~/server/services/Files/images/encode'); +const { getBufferString, HumanMessage } = require('@langchain/core/messages'); const Tokenizer = require('~/server/services/Tokenizer'); const { spendTokens } = require('~/models/spendTokens'); const BaseClient = require('~/app/clients/BaseClient'); -// const { sleep } = require('~/server/utils'); const { createRun } = require('./run'); const { logger } = require('~/config'); @@ -48,6 +50,12 @@ const providerParsers = { const legacyContentEndpoints = new Set([KnownEndpoints.groq, KnownEndpoints.deepseek]); +const noSystemModelRegex = [/\bo1\b/gi]; + +// const { processMemory, memoryInstructions } = require('~/server/services/Endpoints/agents/memory'); +// const { getFormattedMemories } = require('~/models/Memory'); +// const { getCurrentDateTime } = require('~/utils'); + class AgentClient extends BaseClient { constructor(options = {}) { super(null, options); @@ -62,15 +70,15 @@ class AgentClient extends BaseClient { this.run; const { + agentConfigs, contentParts, collectedUsage, artifactPromises, maxContextTokens, - modelOptions = {}, ...clientOptions } = options; - this.modelOptions = modelOptions; + this.agentConfigs = agentConfigs; this.maxContextTokens = maxContextTokens; /** @type {MessageContentComplex[]} */ this.contentParts = contentParts; @@ -80,6 +88,8 @@ class AgentClient extends BaseClient { this.artifactPromises = artifactPromises; /** @type {AgentClientOptions} */ this.options = Object.assign({ endpoint: options.endpoint }, clientOptions); + /** @type {string} */ + this.model = this.options.agent.model_parameters.model; } /** @@ -169,7 +179,7 @@ class AgentClient extends BaseClient { : {}; if (parseOptions) { - runOptions = parseOptions(this.modelOptions); + runOptions = parseOptions(this.options.agent.model_parameters); } return removeNullishValues( @@ -224,7 +234,28 @@ class AgentClient extends BaseClient { let promptTokens; /** @type {string} */ - let systemContent = `${instructions ?? ''}${additional_instructions ?? ''}`; + let systemContent = [instructions ?? '', additional_instructions ?? ''] + .filter(Boolean) + .join('\n') + .trim(); + // this.systemMessage = getCurrentDateTime(); + // const { withKeys, withoutKeys } = await getFormattedMemories({ + // userId: this.options.req.user.id, + // }); + // processMemory({ + // userId: this.options.req.user.id, + // message: this.options.req.body.text, + // parentMessageId, + // memory: withKeys, + // thread_id: this.conversationId, + // }).catch((error) => { + // logger.error('Memory Agent failed to process memory', error); + // }); + + // this.systemMessage += '\n\n' + memoryInstructions; + // if (withoutKeys) { + // this.systemMessage += `\n\n# Existing memory about the user:\n${withoutKeys}`; + // } if (this.options.attachments) { const attachments = await this.options.attachments; @@ -245,7 +276,8 @@ class AgentClient extends BaseClient { this.options.attachments = files; } - if (this.message_file_map) { + /** Note: Bedrock uses legacy RAG API handling */ + if (this.message_file_map && !isAgentsEndpoint(this.options.endpoint)) { this.contextHandlers = createContextHandlers( this.options.req, orderedMessages[orderedMessages.length - 1].text, @@ -319,7 +351,6 @@ class AgentClient extends BaseClient { /** @type {sendCompletion} */ async sendCompletion(payload, opts = {}) { - this.modelOptions.user = this.user; await this.chatCompletion({ payload, onProgress: opts.onProgress, @@ -339,10 +370,10 @@ class AgentClient extends BaseClient { await spendTokens( { context, - model: model ?? this.modelOptions.model, conversationId: this.conversationId, user: this.user ?? this.options.req.user?.id, endpointTokenConfig: this.options.endpointTokenConfig, + model: usage.model ?? model ?? this.model ?? this.options.agent.model_parameters.model, }, { promptTokens: usage.input_tokens, completionTokens: usage.output_tokens }, ); @@ -457,43 +488,190 @@ class AgentClient extends BaseClient { // }); // } - const run = await createRun({ - req: this.options.req, - agent: this.options.agent, - tools: this.options.tools, - runId: this.responseMessageId, - modelOptions: this.modelOptions, - customHandlers: this.options.eventHandlers, - }); - const config = { configurable: { thread_id: this.conversationId, + last_agent_index: this.agentConfigs?.size ?? 0, + hide_sequential_outputs: this.options.agent.hide_sequential_outputs, }, signal: abortController.signal, streamMode: 'values', version: 'v2', }; - if (!run) { - throw new Error('Failed to create run'); - } - - this.run = run; - - const messages = formatAgentMessages(payload); + const initialMessages = formatAgentMessages(payload); if (legacyContentEndpoints.has(this.options.agent.endpoint)) { - formatContentStrings(messages); + formatContentStrings(initialMessages); } - await run.processStream({ messages }, config, { - [Callback.TOOL_ERROR]: (graph, error, toolId) => { - logger.error( - '[api/server/controllers/agents/client.js #chatCompletion] Tool Error', - error, - toolId, - ); - }, + + /** @type {ReturnType} */ + let run; + + /** + * + * @param {Agent} agent + * @param {BaseMessage[]} messages + * @param {number} [i] + * @param {TMessageContentParts[]} [contentData] + */ + const runAgent = async (agent, messages, i = 0, contentData = []) => { + config.configurable.model = agent.model_parameters.model; + if (i > 0) { + this.model = agent.model_parameters.model; + } + config.configurable.agent_id = agent.id; + config.configurable.name = agent.name; + config.configurable.agent_index = i; + const noSystemMessages = noSystemModelRegex.some((regex) => + agent.model_parameters.model.match(regex), + ); + + const systemMessage = Object.values(agent.toolContextMap ?? {}) + .join('\n') + .trim(); + + let systemContent = [ + systemMessage, + agent.instructions ?? '', + i !== 0 ? agent.additional_instructions ?? '' : '', + ] + .join('\n') + .trim(); + + if (noSystemMessages === true) { + agent.instructions = undefined; + agent.additional_instructions = undefined; + } else { + agent.instructions = systemContent; + agent.additional_instructions = undefined; + } + + if (noSystemMessages === true && systemContent?.length) { + let latestMessage = messages.pop().content; + if (typeof latestMessage !== 'string') { + latestMessage = latestMessage[0].text; + } + latestMessage = [systemContent, latestMessage].join('\n'); + messages.push(new HumanMessage(latestMessage)); + } + + run = await createRun({ + agent, + req: this.options.req, + runId: this.responseMessageId, + signal: abortController.signal, + customHandlers: this.options.eventHandlers, + }); + + if (!run) { + throw new Error('Failed to create run'); + } + + if (i === 0) { + this.run = run; + } + + if (contentData.length) { + run.Graph.contentData = contentData; + } + + await run.processStream({ messages }, config, { + keepContent: i !== 0, + callbacks: { + [Callback.TOOL_ERROR]: (graph, error, toolId) => { + logger.error( + '[api/server/controllers/agents/client.js #chatCompletion] Tool Error', + error, + toolId, + ); + }, + }, + }); + }; + + await runAgent(this.options.agent, initialMessages); + + let finalContentStart = 0; + if (this.agentConfigs && this.agentConfigs.size > 0) { + let latestMessage = initialMessages.pop().content; + if (typeof latestMessage !== 'string') { + latestMessage = latestMessage[0].text; + } + let i = 1; + let runMessages = []; + + const lastFiveMessages = initialMessages.slice(-5); + for (const [agentId, agent] of this.agentConfigs) { + if (abortController.signal.aborted === true) { + break; + } + const currentRun = await run; + + if ( + i === this.agentConfigs.size && + config.configurable.hide_sequential_outputs === true + ) { + const content = this.contentParts.filter( + (part) => part.type === ContentTypes.TOOL_CALL, + ); + + this.options.res.write( + `event: message\ndata: ${JSON.stringify({ + event: 'on_content_update', + data: { + runId: this.responseMessageId, + content, + }, + })}\n\n`, + ); + } + const _runMessages = currentRun.Graph.getRunMessages(); + finalContentStart = this.contentParts.length; + runMessages = runMessages.concat(_runMessages); + const contentData = currentRun.Graph.contentData.slice(); + const bufferString = getBufferString([new HumanMessage(latestMessage), ...runMessages]); + if (i === this.agentConfigs.size) { + logger.debug(`SEQUENTIAL AGENTS: Last buffer string:\n${bufferString}`); + } + try { + const contextMessages = []; + for (const message of lastFiveMessages) { + const messageType = message._getType(); + if ( + (!agent.tools || agent.tools.length === 0) && + (messageType === 'tool' || (message.tool_calls?.length ?? 0) > 0) + ) { + continue; + } + + contextMessages.push(message); + } + const currentMessages = [...contextMessages, new HumanMessage(bufferString)]; + await runAgent(agent, currentMessages, i, contentData); + } catch (err) { + logger.error( + `[api/server/controllers/agents/client.js #chatCompletion] Error running agent ${agentId} (${i})`, + err, + ); + } + i++; + } + } + + if (config.configurable.hide_sequential_outputs !== true) { + finalContentStart = 0; + } + + this.contentParts = this.contentParts.filter((part, index) => { + // Include parts that are either: + // 1. At or after the finalContentStart index + // 2. Of type tool_call + // 3. Have tool_call_ids property + return ( + index >= finalContentStart || part.type === ContentTypes.TOOL_CALL || part.tool_call_ids + ); }); + this.recordCollectedUsage({ context: 'message' }).catch((err) => { logger.error( '[api/server/controllers/agents/client.js #chatCompletion] Error recording collected usage', @@ -586,7 +764,7 @@ class AgentClient extends BaseClient { } getEncoding() { - return this.modelOptions.model?.includes('gpt-4o') ? 'o200k_base' : 'cl100k_base'; + return this.model?.includes('gpt-4o') ? 'o200k_base' : 'cl100k_base'; } /** diff --git a/api/server/controllers/agents/request.js b/api/server/controllers/agents/request.js index 2006d4e6ea..8ceadd977d 100644 --- a/api/server/controllers/agents/request.js +++ b/api/server/controllers/agents/request.js @@ -94,8 +94,14 @@ const AgentController = async (req, res, next, initializeClient, addTitle) => { conversation.title = conversation && !conversation.title ? null : conversation?.title || 'New Chat'; - if (client.options.attachments) { - userMessage.files = client.options.attachments; + if (req.body.files && client.options.attachments) { + userMessage.files = []; + const messageFiles = new Set(req.body.files.map((file) => file.file_id)); + for (let attachment of client.options.attachments) { + if (messageFiles.has(attachment.file_id)) { + userMessage.files.push(attachment); + } + } delete userMessage.image_urls; } @@ -109,11 +115,13 @@ const AgentController = async (req, res, next, initializeClient, addTitle) => { }); res.end(); - await saveMessage( - req, - { ...response, user }, - { context: 'api/server/controllers/agents/request.js - response end' }, - ); + if (!client.savedMessageIds.has(response.messageId)) { + await saveMessage( + req, + { ...response, user }, + { context: 'api/server/controllers/agents/request.js - response end' }, + ); + } } if (!client.skipSaveUserMessage) { diff --git a/api/server/controllers/agents/run.js b/api/server/controllers/agents/run.js index 56cc46d5b3..db7f945ca2 100644 --- a/api/server/controllers/agents/run.js +++ b/api/server/controllers/agents/run.js @@ -3,8 +3,8 @@ const { providerEndpointMap } = require('librechat-data-provider'); /** * @typedef {import('@librechat/agents').t} t + * @typedef {import('@librechat/agents').StandardGraphConfig} StandardGraphConfig * @typedef {import('@librechat/agents').StreamEventData} StreamEventData - * @typedef {import('@librechat/agents').ClientOptions} ClientOptions * @typedef {import('@librechat/agents').EventHandler} EventHandler * @typedef {import('@librechat/agents').GraphEvents} GraphEvents * @typedef {import('@librechat/agents').IState} IState @@ -17,18 +17,16 @@ const { providerEndpointMap } = require('librechat-data-provider'); * @param {ServerRequest} [options.req] - The server request. * @param {string | undefined} [options.runId] - Optional run ID; otherwise, a new run ID will be generated. * @param {Agent} options.agent - The agent for this run. - * @param {StructuredTool[] | undefined} [options.tools] - The tools to use in the run. + * @param {AbortSignal} options.signal - The signal for this run. * @param {Record | undefined} [options.customHandlers] - Custom event handlers. - * @param {ClientOptions} [options.modelOptions] - Optional model to use; if not provided, it will use the default from modelMap. * @param {boolean} [options.streaming=true] - Whether to use streaming. * @param {boolean} [options.streamUsage=true] - Whether to stream usage information. * @returns {Promise>} A promise that resolves to a new Run instance. */ async function createRun({ runId, - tools, agent, - modelOptions, + signal, customHandlers, streaming = true, streamUsage = true, @@ -40,14 +38,17 @@ async function createRun({ streaming, streamUsage, }, - modelOptions, + agent.model_parameters, ); + /** @type {StandardGraphConfig} */ const graphConfig = { - tools, + signal, llmConfig, + tools: agent.tools, instructions: agent.instructions, additional_instructions: agent.additional_instructions, + // toolEnd: agent.end_after_tools, }; // TEMPORARY FOR TESTING diff --git a/api/server/controllers/tools.js b/api/server/controllers/tools.js index 9fd9cb2942..9460e66136 100644 --- a/api/server/controllers/tools.js +++ b/api/server/controllers/tools.js @@ -1,6 +1,12 @@ +const { nanoid } = require('nanoid'); const { EnvVar } = require('@librechat/agents'); -const { Tools, AuthType } = require('librechat-data-provider'); -const { loadAuthValues } = require('~/app/clients/tools/util'); +const { Tools, AuthType, ToolCallTypes } = require('librechat-data-provider'); +const { processFileURL, uploadImageBuffer } = require('~/server/services/Files/process'); +const { processCodeOutput } = require('~/server/services/Files/Code/process'); +const { loadAuthValues, loadTools } = require('~/app/clients/tools/util'); +const { createToolCall, getToolCallsByConvo } = require('~/models/ToolCall'); +const { getMessage } = require('~/models/Message'); +const { logger } = require('~/config'); const fieldsMap = { [Tools.execute_code]: [EnvVar.CODE_API_KEY], @@ -24,6 +30,7 @@ const verifyToolAuth = async (req, res) => { result = await loadAuthValues({ userId: req.user.id, authFields, + throwError: false, }); } catch (error) { res.status(200).json({ authenticated: false, message: AuthType.USER_PROVIDED }); @@ -48,6 +55,131 @@ const verifyToolAuth = async (req, res) => { } }; +/** + * @param {ServerRequest} req - The request object, containing information about the HTTP request. + * @param {ServerResponse} res - The response object, used to send back the desired HTTP response. + * @returns {Promise} A promise that resolves when the function has completed. + */ +const callTool = async (req, res) => { + try { + const { toolId = '' } = req.params; + if (!fieldsMap[toolId]) { + logger.warn(`[${toolId}/call] User ${req.user.id} attempted call to invalid tool`); + res.status(404).json({ message: 'Tool not found' }); + return; + } + + const { partIndex, blockIndex, messageId, conversationId, ...args } = req.body; + if (!messageId) { + logger.warn(`[${toolId}/call] User ${req.user.id} attempted call without message ID`); + res.status(400).json({ message: 'Message ID required' }); + return; + } + + const message = await getMessage({ user: req.user.id, messageId }); + if (!message) { + logger.debug(`[${toolId}/call] User ${req.user.id} attempted call with invalid message ID`); + res.status(404).json({ message: 'Message not found' }); + return; + } + logger.debug(`[${toolId}/call] User: ${req.user.id}`); + const { loadedTools } = await loadTools({ + user: req.user.id, + tools: [toolId], + functions: true, + options: { + req, + returnMetadata: true, + processFileURL, + uploadImageBuffer, + fileStrategy: req.app.locals.fileStrategy, + }, + }); + + const tool = loadedTools[0]; + const toolCallId = `${req.user.id}_${nanoid()}`; + const result = await tool.invoke({ + args, + name: toolId, + id: toolCallId, + type: ToolCallTypes.TOOL_CALL, + }); + + const { content, artifact } = result; + const toolCallData = { + toolId, + messageId, + partIndex, + blockIndex, + conversationId, + result: content, + user: req.user.id, + }; + + if (!artifact || !artifact.files || toolId !== Tools.execute_code) { + createToolCall(toolCallData).catch((error) => { + logger.error(`Error creating tool call: ${error.message}`); + }); + return res.status(200).json({ + result: content, + }); + } + + const artifactPromises = []; + for (const file of artifact.files) { + const { id, name } = file; + artifactPromises.push( + (async () => { + const fileMetadata = await processCodeOutput({ + req, + id, + name, + apiKey: tool.apiKey, + messageId, + toolCallId, + conversationId, + session_id: artifact.session_id, + }); + + if (!fileMetadata) { + return null; + } + + return fileMetadata; + })().catch((error) => { + logger.error('Error processing code output:', error); + return null; + }), + ); + } + const attachments = await Promise.all(artifactPromises); + toolCallData.attachments = attachments; + createToolCall(toolCallData).catch((error) => { + logger.error(`Error creating tool call: ${error.message}`); + }); + res.status(200).json({ + result: content, + attachments, + }); + } catch (error) { + logger.error('Error calling tool', error); + res.status(500).json({ message: 'Error calling tool' }); + } +}; + +const getToolCalls = async (req, res) => { + try { + const { conversationId } = req.query; + const toolCalls = await getToolCallsByConvo(conversationId, req.user.id); + res.status(200).json(toolCalls); + } catch (error) { + logger.error('Error getting tool calls', error); + res.status(500).json({ message: 'Error getting tool calls' }); + } +}; + module.exports = { + callTool, + getToolCalls, verifyToolAuth, }; diff --git a/api/server/middleware/buildEndpointOption.js b/api/server/middleware/buildEndpointOption.js index 25bb5a3c9c..139a0134d2 100644 --- a/api/server/middleware/buildEndpointOption.js +++ b/api/server/middleware/buildEndpointOption.js @@ -10,6 +10,7 @@ const openAI = require('~/server/services/Endpoints/openAI'); const agents = require('~/server/services/Endpoints/agents'); const custom = require('~/server/services/Endpoints/custom'); const google = require('~/server/services/Endpoints/google'); +const { getConvoFiles } = require('~/models/Conversation'); const { handleError } = require('~/server/utils'); const buildFunction = { @@ -72,21 +73,32 @@ async function buildEndpointOption(req, res, next) { } } - const endpointFn = buildFunction[endpointType ?? endpoint]; - const builder = isAgentsEndpoint(endpoint) ? (...args) => endpointFn(req, ...args) : endpointFn; + try { + const isAgents = isAgentsEndpoint(endpoint); + const endpointFn = buildFunction[endpointType ?? endpoint]; + const builder = isAgents ? (...args) => endpointFn(req, ...args) : endpointFn; - // TODO: use object params - req.body.endpointOption = builder(endpoint, parsedBody, endpointType); + // TODO: use object params + req.body.endpointOption = builder(endpoint, parsedBody, endpointType); - // TODO: use `getModelsConfig` only when necessary - const modelsConfig = await getModelsConfig(req); - req.body.endpointOption.modelsConfig = modelsConfig; - - if (req.body.files) { - // hold the promise - req.body.endpointOption.attachments = processFiles(req.body.files); + // TODO: use `getModelsConfig` only when necessary + const modelsConfig = await getModelsConfig(req); + const { resendFiles = true } = req.body.endpointOption; + req.body.endpointOption.modelsConfig = modelsConfig; + if (isAgents && resendFiles && req.body.conversationId) { + const fileIds = await getConvoFiles(req.body.conversationId); + const requestFiles = req.body.files ?? []; + if (requestFiles.length || fileIds.length) { + req.body.endpointOption.attachments = processFiles(requestFiles, fileIds); + } + } else if (req.body.files) { + // hold the promise + req.body.endpointOption.attachments = processFiles(req.body.files); + } + next(); + } catch (error) { + return handleError(res, { text: 'Error building endpoint option' }); } - next(); } module.exports = buildEndpointOption; diff --git a/api/server/middleware/limiters/index.js b/api/server/middleware/limiters/index.js index 0ae6bb5c5e..d1c11e0a12 100644 --- a/api/server/middleware/limiters/index.js +++ b/api/server/middleware/limiters/index.js @@ -5,6 +5,7 @@ const loginLimiter = require('./loginLimiter'); const importLimiters = require('./importLimiters'); const uploadLimiters = require('./uploadLimiters'); const registerLimiter = require('./registerLimiter'); +const toolCallLimiter = require('./toolCallLimiter'); const messageLimiters = require('./messageLimiters'); const verifyEmailLimiter = require('./verifyEmailLimiter'); const resetPasswordLimiter = require('./resetPasswordLimiter'); @@ -15,6 +16,7 @@ module.exports = { ...messageLimiters, loginLimiter, registerLimiter, + toolCallLimiter, createTTSLimiters, createSTTLimiters, verifyEmailLimiter, diff --git a/api/server/middleware/limiters/toolCallLimiter.js b/api/server/middleware/limiters/toolCallLimiter.js new file mode 100644 index 0000000000..47dcaeabb4 --- /dev/null +++ b/api/server/middleware/limiters/toolCallLimiter.js @@ -0,0 +1,25 @@ +const rateLimit = require('express-rate-limit'); +const { ViolationTypes } = require('librechat-data-provider'); +const logViolation = require('~/cache/logViolation'); + +const toolCallLimiter = rateLimit({ + windowMs: 1000, + max: 1, + handler: async (req, res) => { + const type = ViolationTypes.TOOL_CALL_LIMIT; + const errorMessage = { + type, + max: 1, + limiter: 'user', + windowInMinutes: 1, + }; + + await logViolation(req, res, type, errorMessage, 0); + res.status(429).json({ message: 'Too many tool call requests. Try again later' }); + }, + keyGenerator: function (req) { + return req.user?.id; + }, +}); + +module.exports = toolCallLimiter; diff --git a/api/server/routes/agents/chat.js b/api/server/routes/agents/chat.js index 8302abcde0..fdb2db54d3 100644 --- a/api/server/routes/agents/chat.js +++ b/api/server/routes/agents/chat.js @@ -1,19 +1,23 @@ const express = require('express'); - -const router = express.Router(); +const { PermissionTypes, Permissions } = require('librechat-data-provider'); const { setHeaders, handleAbort, // validateModel, - // validateEndpoint, + generateCheckAccess, + validateConvoAccess, buildEndpointOption, } = require('~/server/middleware'); const { initializeClient } = require('~/server/services/Endpoints/agents'); const AgentController = require('~/server/controllers/agents/request'); const addTitle = require('~/server/services/Endpoints/agents/title'); +const router = express.Router(); + router.post('/abort', handleAbort()); +const checkAgentAccess = generateCheckAccess(PermissionTypes.AGENTS, [Permissions.USE]); + /** * @route POST / * @desc Chat with an assistant @@ -25,7 +29,8 @@ router.post('/abort', handleAbort()); router.post( '/', // validateModel, - // validateEndpoint, + checkAgentAccess, + validateConvoAccess, buildEndpointOption, setHeaders, async (req, res, next) => { diff --git a/api/server/routes/agents/tools.js b/api/server/routes/agents/tools.js index b58fc21d4f..8e498b1db8 100644 --- a/api/server/routes/agents/tools.js +++ b/api/server/routes/agents/tools.js @@ -1,6 +1,7 @@ const express = require('express'); +const { callTool, verifyToolAuth, getToolCalls } = require('~/server/controllers/tools'); const { getAvailableTools } = require('~/server/controllers/PluginController'); -const { verifyToolAuth } = require('~/server/controllers/tools'); +const { toolCallLimiter } = require('~/server/middleware/limiters'); const router = express.Router(); @@ -11,6 +12,13 @@ const router = express.Router(); */ router.get('/', getAvailableTools); +/** + * Get a list of tool calls. + * @route GET /agents/tools/calls + * @returns {ToolCallData[]} 200 - application/json + */ +router.get('/calls', getToolCalls); + /** * Verify authentication for a specific tool * @route GET /agents/tools/:toolId/auth @@ -19,4 +27,13 @@ router.get('/', getAvailableTools); */ router.get('/:toolId/auth', verifyToolAuth); +/** + * Execute code for a specific tool + * @route POST /agents/tools/:toolId/call + * @param {string} toolId - The ID of the tool to execute + * @param {object} req.body - Request body + * @returns {object} Result of code execution + */ +router.post('/:toolId/call', toolCallLimiter, callTool); + module.exports = router; diff --git a/api/server/routes/convos.js b/api/server/routes/convos.js index d47e757fd8..0aec01b8ee 100644 --- a/api/server/routes/convos.js +++ b/api/server/routes/convos.js @@ -7,6 +7,7 @@ const requireJwtAuth = require('~/server/middleware/requireJwtAuth'); const { forkConversation } = require('~/server/utils/import/fork'); const { importConversations } = require('~/server/utils/import'); const { createImportLimiters } = require('~/server/middleware'); +const { deleteToolCalls } = require('~/models/ToolCall'); const getLogStores = require('~/cache/getLogStores'); const { sleep } = require('~/server/utils'); const { logger } = require('~/config'); @@ -105,6 +106,7 @@ router.post('/clear', async (req, res) => { try { const dbResponse = await deleteConvos(req.user.id, filter); + await deleteToolCalls(req.user.id, filter.conversationId); res.status(201).json(dbResponse); } catch (error) { logger.error('Error clearing conversations', error); diff --git a/api/server/routes/files/files.js b/api/server/routes/files/files.js index e177142908..c320f7705b 100644 --- a/api/server/routes/files/files.js +++ b/api/server/routes/files/files.js @@ -107,6 +107,10 @@ router.delete('/', async (req, res) => { } }); +function isValidID(str) { + return /^[A-Za-z0-9_-]{21}$/.test(str); +} + router.get('/code/download/:session_id/:fileId', async (req, res) => { try { const { session_id, fileId } = req.params; @@ -117,6 +121,11 @@ router.get('/code/download/:session_id/:fileId', async (req, res) => { return res.status(400).send('Bad request'); } + if (!isValidID(session_id) || !isValidID(fileId)) { + logger.debug(`${logPrefix} invalid session_id or fileId`); + return res.status(400).send('Bad request'); + } + const { getDownloadStream } = getStrategyFunctions(FileSources.execute_code); if (!getDownloadStream) { logger.warn( @@ -213,21 +222,20 @@ router.get('/download/:userId/:file_id', async (req, res) => { }); router.post('/', async (req, res) => { - const file = req.file; const metadata = req.body; let cleanup = true; try { - filterFile({ req, file }); + filterFile({ req }); metadata.temp_file_id = metadata.file_id; metadata.file_id = req.file_id; if (isAgentsEndpoint(metadata.endpoint)) { - return await processAgentFileUpload({ req, res, file, metadata }); + return await processAgentFileUpload({ req, res, metadata }); } - await processFileUpload({ req, res, file, metadata }); + await processFileUpload({ req, res, metadata }); } catch (error) { let message = 'Error processing file'; logger.error('[/files] Error processing file:', error); @@ -238,7 +246,7 @@ router.post('/', async (req, res) => { // TODO: delete remote file if it exists try { - await fs.unlink(file.path); + await fs.unlink(req.file.path); cleanup = false; } catch (error) { logger.error('[/files] Error deleting file:', error); @@ -248,7 +256,7 @@ router.post('/', async (req, res) => { if (cleanup) { try { - await fs.unlink(file.path); + await fs.unlink(req.file.path); } catch (error) { logger.error('[/files] Error deleting file after file processing:', error); } diff --git a/api/server/routes/files/images.js b/api/server/routes/files/images.js index 318ac91e22..d6d04446f8 100644 --- a/api/server/routes/files/images.js +++ b/api/server/routes/files/images.js @@ -1,7 +1,12 @@ const path = require('path'); const fs = require('fs').promises; const express = require('express'); -const { filterFile, processImageFile } = require('~/server/services/Files/process'); +const { isAgentsEndpoint } = require('librechat-data-provider'); +const { + filterFile, + processImageFile, + processAgentFileUpload, +} = require('~/server/services/Files/process'); const { logger } = require('~/config'); const router = express.Router(); @@ -10,12 +15,16 @@ router.post('/', async (req, res) => { const metadata = req.body; try { - filterFile({ req, file: req.file, image: true }); + filterFile({ req, image: true }); metadata.temp_file_id = metadata.file_id; metadata.file_id = req.file_id; - await processImageFile({ req, res, file: req.file, metadata }); + if (isAgentsEndpoint(metadata.endpoint) && metadata.tool_resource != null) { + return await processAgentFileUpload({ req, res, metadata }); + } + + await processImageFile({ req, res, metadata }); } catch (error) { // TODO: delete remote file if it exists logger.error('[/files/images] Error processing file:', error); diff --git a/api/server/routes/roles.js b/api/server/routes/roles.js index 36152e2c7e..e58ebb6fe7 100644 --- a/api/server/routes/roles.js +++ b/api/server/routes/roles.js @@ -1,6 +1,7 @@ const express = require('express'); const { promptPermissionsSchema, + agentPermissionsSchema, PermissionTypes, roleDefaults, SystemRoles, @@ -72,4 +73,37 @@ router.put('/:roleName/prompts', checkAdmin, async (req, res) => { } }); +/** + * PUT /api/roles/:roleName/agents + * Update agent permissions for a specific role + */ +router.put('/:roleName/agents', checkAdmin, async (req, res) => { + const { roleName: _r } = req.params; + // TODO: TEMP, use a better parsing for roleName + const roleName = _r.toUpperCase(); + /** @type {TRole['AGENTS']} */ + const updates = req.body; + + try { + const parsedUpdates = agentPermissionsSchema.partial().parse(updates); + + const role = await getRoleByName(roleName); + if (!role) { + return res.status(404).send({ message: 'Role not found' }); + } + + const mergedUpdates = { + [PermissionTypes.AGENTS]: { + ...role[PermissionTypes.AGENTS], + ...parsedUpdates, + }, + }; + + const updatedRole = await updateRoleByName(roleName, mergedUpdates); + res.status(200).send(updatedRole); + } catch (error) { + return res.status(400).send({ message: 'Invalid prompt permissions.', error: error.errors }); + } +}); + module.exports = router; diff --git a/api/server/services/AppService.js b/api/server/services/AppService.js index f99e962871..19a9fc91a9 100644 --- a/api/server/services/AppService.js +++ b/api/server/services/AppService.js @@ -8,7 +8,6 @@ const { loadDefaultInterface } = require('./start/interface'); const { azureConfigSetup } = require('./start/azureOpenAI'); const { loadAndFormatTools } = require('./ToolService'); const { initializeRoles } = require('~/models/Role'); -const { cleanup } = require('./cleanup'); const paths = require('~/config/paths'); /** @@ -18,7 +17,6 @@ const paths = require('~/config/paths'); * @param {Express.Application} app - The Express application object. */ const AppService = async (app) => { - cleanup(); await initializeRoles(); /** @type {TCustomConfig}*/ const config = (await loadCustomConfig()) ?? {}; diff --git a/api/server/services/Config/EndpointService.js b/api/server/services/Config/EndpointService.js index 49f9d8f548..dc055e2872 100644 --- a/api/server/services/Config/EndpointService.js +++ b/api/server/services/Config/EndpointService.js @@ -49,10 +49,6 @@ module.exports = { process.env.BEDROCK_AWS_SECRET_ACCESS_KEY ?? process.env.BEDROCK_AWS_DEFAULT_REGION, ), /* key will be part of separate config */ - [EModelEndpoint.agents]: generateConfig( - process.env.EXPERIMENTAL_AGENTS, - undefined, - EModelEndpoint.agents, - ), + [EModelEndpoint.agents]: generateConfig('true', undefined, EModelEndpoint.agents), }, }; diff --git a/api/server/services/Endpoints/agents/build.js b/api/server/services/Endpoints/agents/build.js index 853c9ba266..90e251a4ea 100644 --- a/api/server/services/Endpoints/agents/build.js +++ b/api/server/services/Endpoints/agents/build.js @@ -2,8 +2,14 @@ const { loadAgent } = require('~/models/Agent'); const { logger } = require('~/config'); const buildOptions = (req, endpoint, parsedBody) => { - const { agent_id, instructions, spec, ...model_parameters } = parsedBody; - + const { + agent_id, + instructions, + spec, + maxContextTokens, + resendFiles = true, + ...model_parameters + } = parsedBody; const agentPromise = loadAgent({ req, agent_id, @@ -13,12 +19,14 @@ const buildOptions = (req, endpoint, parsedBody) => { }); const endpointOption = { - agent: agentPromise, + spec, endpoint, agent_id, + resendFiles, instructions, - spec, + maxContextTokens, model_parameters, + agent: agentPromise, }; return endpointOption; diff --git a/api/server/services/Endpoints/agents/initialize.js b/api/server/services/Endpoints/agents/initialize.js index 796f69e4ac..507546a345 100644 --- a/api/server/services/Endpoints/agents/initialize.js +++ b/api/server/services/Endpoints/agents/initialize.js @@ -16,6 +16,8 @@ const { getCustomEndpointConfig } = require('~/server/services/Config'); const { loadAgentTools } = require('~/server/services/ToolService'); const AgentClient = require('~/server/controllers/agents/client'); const { getModelMaxTokens } = require('~/utils'); +const { getAgent } = require('~/models/Agent'); +const { logger } = require('~/config'); const providerConfigMap = { [EModelEndpoint.openAI]: initOpenAI, @@ -25,6 +27,113 @@ const providerConfigMap = { [Providers.OLLAMA]: initCustom, }; +/** + * + * @param {Promise> | undefined} _attachments + * @param {AgentToolResources | undefined} _tool_resources + * @returns {Promise<{ attachments: Array | undefined, tool_resources: AgentToolResources | undefined }>} + */ +const primeResources = async (_attachments, _tool_resources) => { + try { + if (!_attachments) { + return { attachments: undefined, tool_resources: _tool_resources }; + } + /** @type {Array | undefined} */ + const files = await _attachments; + const attachments = []; + const tool_resources = _tool_resources ?? {}; + + for (const file of files) { + if (!file) { + continue; + } + if (file.metadata?.fileIdentifier) { + const execute_code = tool_resources.execute_code ?? {}; + if (!execute_code.files) { + tool_resources.execute_code = { ...execute_code, files: [] }; + } + tool_resources.execute_code.files.push(file); + } else if (file.embedded === true) { + const file_search = tool_resources.file_search ?? {}; + if (!file_search.files) { + tool_resources.file_search = { ...file_search, files: [] }; + } + tool_resources.file_search.files.push(file); + } + + attachments.push(file); + } + return { attachments, tool_resources }; + } catch (error) { + logger.error('Error priming resources', error); + return { attachments: _attachments, tool_resources: _tool_resources }; + } +}; + +const initializeAgentOptions = async ({ + req, + res, + agent, + endpointOption, + tool_resources, + isInitialAgent = false, +}) => { + const { tools, toolContextMap } = await loadAgentTools({ + req, + tools: agent.tools, + agent_id: agent.id, + tool_resources, + }); + + const provider = agent.provider; + let getOptions = providerConfigMap[provider]; + + if (!getOptions) { + const customEndpointConfig = await getCustomEndpointConfig(provider); + if (!customEndpointConfig) { + throw new Error(`Provider ${provider} not supported`); + } + getOptions = initCustom; + agent.provider = Providers.OPENAI; + agent.endpoint = provider.toLowerCase(); + } + + const model_parameters = agent.model_parameters ?? { model: agent.model }; + const _endpointOption = isInitialAgent + ? endpointOption + : { + model_parameters, + }; + + const options = await getOptions({ + req, + res, + optionsOnly: true, + overrideEndpoint: provider, + overrideModel: agent.model, + endpointOption: _endpointOption, + }); + + agent.model_parameters = Object.assign(model_parameters, options.llmConfig); + if (options.configOptions) { + agent.model_parameters.configuration = options.configOptions; + } + + if (!agent.model_parameters.model) { + agent.model_parameters.model = agent.model; + } + + return { + ...agent, + tools, + toolContextMap, + maxContextTokens: + agent.max_context_tokens ?? + getModelMaxTokens(agent.model_parameters.model, providerEndpointMap[provider]) ?? + 4000, + }; +}; + const initializeClient = async ({ req, res, endpointOption }) => { if (!endpointOption) { throw new Error('Endpoint option not provided'); @@ -48,70 +157,68 @@ const initializeClient = async ({ req, res, endpointOption }) => { throw new Error('No agent promise provided'); } - /** @type {Agent | null} */ - const agent = await endpointOption.agent; - if (!agent) { + // Initialize primary agent + const primaryAgent = await endpointOption.agent; + if (!primaryAgent) { throw new Error('Agent not found'); } - const { tools } = await loadAgentTools({ - req, - tools: agent.tools, - agent_id: agent.id, - tool_resources: agent.tool_resources, - }); + const { attachments, tool_resources } = await primeResources( + endpointOption.attachments, + primaryAgent.tool_resources, + ); - const provider = agent.provider; - let modelOptions = { model: agent.model }; - let getOptions = providerConfigMap[provider]; - if (!getOptions) { - const customEndpointConfig = await getCustomEndpointConfig(provider); - if (!customEndpointConfig) { - throw new Error(`Provider ${provider} not supported`); - } - getOptions = initCustom; - agent.provider = Providers.OPENAI; - agent.endpoint = provider.toLowerCase(); - } + const agentConfigs = new Map(); - // TODO: pass-in override settings that are specific to current run - endpointOption.model_parameters.model = agent.model; - const options = await getOptions({ + // Handle primary agent + const primaryConfig = await initializeAgentOptions({ req, res, + agent: primaryAgent, endpointOption, - optionsOnly: true, - overrideEndpoint: provider, - overrideModel: agent.model, + tool_resources, + isInitialAgent: true, }); - modelOptions = Object.assign(modelOptions, options.llmConfig); - if (options.configOptions) { - modelOptions.configuration = options.configOptions; + const agent_ids = primaryConfig.agent_ids; + if (agent_ids?.length) { + for (const agentId of agent_ids) { + const agent = await getAgent({ id: agentId }); + if (!agent) { + throw new Error(`Agent ${agentId} not found`); + } + const config = await initializeAgentOptions({ + req, + res, + agent, + endpointOption, + }); + agentConfigs.set(agentId, config); + } } - const sender = getResponseSender({ - ...endpointOption, - model: endpointOption.model_parameters.model, - }); + const sender = + primaryAgent.name ?? + getResponseSender({ + ...endpointOption, + model: endpointOption.model_parameters.model, + }); const client = new AgentClient({ req, - agent, - tools, + agent: primaryConfig, sender, + attachments, contentParts, - modelOptions, eventHandlers, collectedUsage, artifactPromises, + spec: endpointOption.spec, + agentConfigs, endpoint: EModelEndpoint.agents, - attachments: endpointOption.attachments, - maxContextTokens: - agent.max_context_tokens ?? - getModelMaxTokens(modelOptions.model, providerEndpointMap[provider]) ?? - 4000, + maxContextTokens: primaryConfig.maxContextTokens, }); + return { client }; }; diff --git a/api/server/services/Endpoints/bedrock/initialize.js b/api/server/services/Endpoints/bedrock/initialize.js index 00630c41e6..d2be7e235b 100644 --- a/api/server/services/Endpoints/bedrock/initialize.js +++ b/api/server/services/Endpoints/bedrock/initialize.js @@ -5,7 +5,6 @@ const { getResponseSender, } = require('librechat-data-provider'); const { getDefaultHandlers } = require('~/server/controllers/agents/callbacks'); -// const { loadAgentTools } = require('~/server/services/ToolService'); const getOptions = require('~/server/services/Endpoints/bedrock/options'); const AgentClient = require('~/server/controllers/agents/client'); const { getModelMaxTokens } = require('~/utils'); @@ -20,8 +19,6 @@ const initializeClient = async ({ req, res, endpointOption }) => { const { contentParts, aggregateContent } = createContentAggregator(); const eventHandlers = getDefaultHandlers({ res, aggregateContent, collectedUsage }); - // const tools = [createTavilySearchTool()]; - /** @type {Agent} */ const agent = { id: EModelEndpoint.bedrock, @@ -36,8 +33,6 @@ const initializeClient = async ({ req, res, endpointOption }) => { agent.instructions = `${agent.instructions ?? ''}\n${endpointOption.artifactsPrompt}`.trim(); } - let modelOptions = { model: agent.model }; - // TODO: pass-in override settings that are specific to current run const options = await getOptions({ req, @@ -45,28 +40,34 @@ const initializeClient = async ({ req, res, endpointOption }) => { endpointOption, }); - modelOptions = Object.assign(modelOptions, options.llmConfig); - const maxContextTokens = - agent.max_context_tokens ?? - getModelMaxTokens(modelOptions.model, providerEndpointMap[agent.provider]); + agent.model_parameters = Object.assign(agent.model_parameters, options.llmConfig); + if (options.configOptions) { + agent.model_parameters.configuration = options.configOptions; + } - const sender = getResponseSender({ - ...endpointOption, - model: endpointOption.model_parameters.model, - }); + const sender = + agent.name ?? + getResponseSender({ + ...endpointOption, + model: endpointOption.model_parameters.model, + }); const client = new AgentClient({ req, agent, sender, // tools, - modelOptions, contentParts, eventHandlers, collectedUsage, - maxContextTokens, + spec: endpointOption.spec, endpoint: EModelEndpoint.bedrock, - configOptions: options.configOptions, + resendFiles: endpointOption.resendFiles, + maxContextTokens: + endpointOption.maxContextTokens ?? + agent.max_context_tokens ?? + getModelMaxTokens(agent.model_parameters.model, providerEndpointMap[agent.provider]) ?? + 4000, attachments: endpointOption.attachments, }); return { client }; diff --git a/api/server/services/Endpoints/custom/initialize.js b/api/server/services/Endpoints/custom/initialize.js index 2390ea368d..c88e6882f5 100644 --- a/api/server/services/Endpoints/custom/initialize.js +++ b/api/server/services/Endpoints/custom/initialize.js @@ -10,8 +10,8 @@ const { getUserKeyValues, checkUserKeyExpiry } = require('~/server/services/User const { getLLMConfig } = require('~/server/services/Endpoints/openAI/llm'); const { getCustomEndpointConfig } = require('~/server/services/Config'); const { fetchModels } = require('~/server/services/ModelService'); +const { isUserProvided, sleep } = require('~/server/utils'); const getLogStores = require('~/cache/getLogStores'); -const { isUserProvided } = require('~/server/utils'); const { OpenAIClient } = require('~/app'); const { PROXY } = process.env; @@ -141,7 +141,18 @@ const initializeClient = async ({ req, res, endpointOption, optionsOnly, overrid }, clientOptions, ); - return getLLMConfig(apiKey, requestOptions); + const options = getLLMConfig(apiKey, requestOptions); + if (!customOptions.streamRate) { + return options; + } + options.llmConfig.callbacks = [ + { + handleLLMNewToken: async () => { + await sleep(customOptions.streamRate); + }, + }, + ]; + return options; } if (clientOptions.reverseProxyUrl) { diff --git a/api/server/services/Endpoints/openAI/initialize.js b/api/server/services/Endpoints/openAI/initialize.js index a84be42b91..63abbfea9c 100644 --- a/api/server/services/Endpoints/openAI/initialize.js +++ b/api/server/services/Endpoints/openAI/initialize.js @@ -6,7 +6,7 @@ const { } = require('librechat-data-provider'); const { getUserKeyValues, checkUserKeyExpiry } = require('~/server/services/UserService'); const { getLLMConfig } = require('~/server/services/Endpoints/openAI/llm'); -const { isEnabled, isUserProvided } = require('~/server/utils'); +const { isEnabled, isUserProvided, sleep } = require('~/server/utils'); const { getAzureCredentials } = require('~/utils'); const { OpenAIClient } = require('~/app'); @@ -140,7 +140,18 @@ const initializeClient = async ({ }, clientOptions, ); - return getLLMConfig(apiKey, requestOptions); + const options = getLLMConfig(apiKey, requestOptions); + if (!clientOptions.streamRate) { + return options; + } + options.llmConfig.callbacks = [ + { + handleLLMNewToken: async () => { + await sleep(clientOptions.streamRate); + }, + }, + ]; + return options; } const client = new OpenAIClient(apiKey, Object.assign({ req, res }, clientOptions)); diff --git a/api/server/services/Files/Code/crud.js b/api/server/services/Files/Code/crud.js index 82b999b9bb..07d09548ab 100644 --- a/api/server/services/Files/Code/crud.js +++ b/api/server/services/Files/Code/crud.js @@ -40,12 +40,16 @@ async function getCodeOutputDownloadStream(fileIdentifier, apiKey) { * @param {import('fs').ReadStream | import('stream').Readable} params.stream - The read stream for the file. * @param {string} params.filename - The name of the file. * @param {string} params.apiKey - The API key for authentication. + * @param {string} [params.entity_id] - Optional entity ID for the file. * @returns {Promise} * @throws {Error} If there's an error during the upload process. */ -async function uploadCodeEnvFile({ req, stream, filename, apiKey }) { +async function uploadCodeEnvFile({ req, stream, filename, apiKey, entity_id = '' }) { try { const form = new FormData(); + if (entity_id.length > 0) { + form.append('entity_id', entity_id); + } form.append('file', stream, filename); const baseURL = getCodeBaseURL(); @@ -67,7 +71,12 @@ async function uploadCodeEnvFile({ req, stream, filename, apiKey }) { throw new Error(`Error uploading file: ${result.message}`); } - return `${result.session_id}/${result.files[0].fileId}`; + const fileIdentifier = `${result.session_id}/${result.files[0].fileId}`; + if (entity_id.length === 0) { + return fileIdentifier; + } + + return `${fileIdentifier}?entity_id=${entity_id}`; } catch (error) { throw new Error(`Error uploading file: ${error.message}`); } diff --git a/api/server/services/Files/Code/process.js b/api/server/services/Files/Code/process.js index 313b98f39b..2a941a4647 100644 --- a/api/server/services/Files/Code/process.js +++ b/api/server/services/Files/Code/process.js @@ -3,10 +3,11 @@ const { v4 } = require('uuid'); const axios = require('axios'); const { getCodeBaseURL } = require('@librechat/agents'); const { - EToolResources, + Tools, FileContext, - imageExtRegex, FileSources, + imageExtRegex, + EToolResources, } = require('librechat-data-provider'); const { getStrategyFunctions } = require('~/server/services/Files/strategies'); const { convertImage } = require('~/server/services/Files/images/convert'); @@ -110,12 +111,20 @@ function checkIfActive(dateString) { async function getSessionInfo(fileIdentifier, apiKey) { try { const baseURL = getCodeBaseURL(); - const session_id = fileIdentifier.split('/')[0]; + const [path, queryString] = fileIdentifier.split('?'); + const session_id = path.split('/')[0]; + + let queryParams = {}; + if (queryString) { + queryParams = Object.fromEntries(new URLSearchParams(queryString).entries()); + } + const response = await axios({ method: 'get', url: `${baseURL}/files/${session_id}`, params: { detail: 'summary', + ...queryParams, }, headers: { 'User-Agent': 'LibreChat/1.0', @@ -124,7 +133,7 @@ async function getSessionInfo(fileIdentifier, apiKey) { timeout: 5000, }); - return response.data.find((file) => file.name.startsWith(fileIdentifier))?.lastModified; + return response.data.find((file) => file.name.startsWith(path))?.lastModified; } catch (error) { logger.error(`Error fetching session info: ${error.message}`, error); return null; @@ -137,29 +146,56 @@ async function getSessionInfo(fileIdentifier, apiKey) { * @param {ServerRequest} options.req * @param {Agent['tool_resources']} options.tool_resources * @param {string} apiKey - * @returns {Promise>} + * @returns {Promise<{ + * files: Array<{ id: string; session_id: string; name: string }>, + * toolContext: string, + * }>} */ const primeFiles = async (options, apiKey) => { const { tool_resources } = options; const file_ids = tool_resources?.[EToolResources.execute_code]?.file_ids ?? []; - const dbFiles = await getFiles({ file_id: { $in: file_ids } }); + const agentResourceIds = new Set(file_ids); + const resourceFiles = tool_resources?.[EToolResources.execute_code]?.files ?? []; + const dbFiles = ((await getFiles({ file_id: { $in: file_ids } })) ?? []).concat(resourceFiles); const files = []; const sessions = new Map(); - for (const file of dbFiles) { + let toolContext = ''; + + for (let i = 0; i < dbFiles.length; i++) { + const file = dbFiles[i]; + if (!file) { + continue; + } + if (file.metadata.fileIdentifier) { - const [session_id, id] = file.metadata.fileIdentifier.split('/'); + const [path, queryString] = file.metadata.fileIdentifier.split('?'); + const [session_id, id] = path.split('/'); + const pushFile = () => { + if (!toolContext) { + toolContext = `- Note: The following files are available in the "${Tools.execute_code}" tool environment:`; + } + toolContext += `\n\t- /mnt/data/${file.filename}${ + agentResourceIds.has(file.file_id) ? '' : ' (just attached by user)' + }`; files.push({ id, session_id, name: file.filename, }); }; + if (sessions.has(session_id)) { pushFile(); continue; } + + let queryParams = {}; + if (queryString) { + queryParams = Object.fromEntries(new URLSearchParams(queryString).entries()); + } + const reuploadFile = async () => { try { const { getDownloadStream } = getStrategyFunctions(file.source); @@ -171,6 +207,7 @@ const primeFiles = async (options, apiKey) => { req: options.req, stream, filename: file.filename, + entity_id: queryParams.entity_id, apiKey, }); await updateFile({ file_id: file.file_id, metadata: { fileIdentifier } }); @@ -198,7 +235,7 @@ const primeFiles = async (options, apiKey) => { } } - return files; + return { files, toolContext }; }; module.exports = { diff --git a/api/server/services/Files/images/encode.js b/api/server/services/Files/images/encode.js index f457927019..94153ffc64 100644 --- a/api/server/services/Files/images/encode.js +++ b/api/server/services/Files/images/encode.js @@ -97,6 +97,7 @@ async function encodeAndFormat(req, files, endpoint, mode) { filepath: file.filepath, filename: file.filename, embedded: !!file.embedded, + metadata: file.metadata, }; if (file.height && file.width) { diff --git a/api/server/services/Files/process.js b/api/server/services/Files/process.js index 5436b7037a..ab401420f1 100644 --- a/api/server/services/Files/process.js +++ b/api/server/services/Files/process.js @@ -20,7 +20,7 @@ const { const { EnvVar } = require('@librechat/agents'); const { addResourceFileId, deleteResourceFileId } = require('~/server/controllers/assistants/v2'); const { convertImage, resizeAndConvert } = require('~/server/services/Files/images'); -const { addAgentResourceFile, removeAgentResourceFile } = require('~/models/Agent'); +const { addAgentResourceFile, removeAgentResourceFiles } = require('~/models/Agent'); const { getOpenAIClient } = require('~/server/controllers/assistants/helpers'); const { createFile, updateFileUsage, deleteFiles } = require('~/models/File'); const { loadAuthValues } = require('~/app/clients/tools/util'); @@ -29,10 +29,34 @@ const { getStrategyFunctions } = require('./strategies'); const { determineFileType } = require('~/server/utils'); const { logger } = require('~/config'); -const processFiles = async (files) => { +/** + * + * @param {Array} files + * @param {Array} [fileIds] + * @returns + */ +const processFiles = async (files, fileIds) => { const promises = []; + const seen = new Set(); + for (let file of files) { const { file_id } = file; + if (seen.has(file_id)) { + continue; + } + seen.add(file_id); + promises.push(updateFileUsage({ file_id })); + } + + if (!fileIds) { + return await Promise.all(promises); + } + + for (let file_id of fileIds) { + if (seen.has(file_id)) { + continue; + } + seen.add(file_id); promises.push(updateFileUsage({ file_id })); } @@ -44,7 +68,7 @@ const processFiles = async (files) => { * Enqueues the delete operation to the leaky bucket queue if necessary, or adds it directly to promises. * * @param {object} params - The passed parameters. - * @param {Express.Request} params.req - The express request object. + * @param {ServerRequest} params.req - The express request object. * @param {MongoFile} params.file - The file object to delete. * @param {Function} params.deleteFile - The delete file function. * @param {Promise[]} params.promises - The array of promises to await. @@ -91,7 +115,7 @@ function enqueueDeleteOperation({ req, file, deleteFile, promises, resolvedFileI * * @param {Object} params - The params object. * @param {MongoFile[]} params.files - The file objects to delete. - * @param {Express.Request} params.req - The express request object. + * @param {ServerRequest} params.req - The express request object. * @param {DeleteFilesBody} params.req.body - The request body. * @param {string} [params.req.body.agent_id] - The agent ID if file uploaded is associated to an agent. * @param {string} [params.req.body.assistant_id] - The assistant ID if file uploaded is associated to an assistant. @@ -128,18 +152,16 @@ const processDeleteRequest = async ({ req, files }) => { await initializeClients(); } + const agentFiles = []; + for (const file of files) { const source = file.source ?? FileSources.local; if (req.body.agent_id && req.body.tool_resource) { - promises.push( - removeAgentResourceFile({ - req, - file_id: file.file_id, - agent_id: req.body.agent_id, - tool_resource: req.body.tool_resource, - }), - ); + agentFiles.push({ + tool_resource: req.body.tool_resource, + file_id: file.file_id, + }); } if (checkOpenAIStorage(source) && !client[source]) { @@ -183,6 +205,15 @@ const processDeleteRequest = async ({ req, files }) => { enqueueDeleteOperation({ req, file, deleteFile, promises, resolvedFileIds, openai }); } + if (agentFiles.length > 0) { + promises.push( + removeAgentResourceFiles({ + agent_id: req.body.agent_id, + files: agentFiles, + }), + ); + } + await Promise.allSettled(promises); await deleteFiles(resolvedFileIds); }; @@ -242,14 +273,14 @@ const processFileURL = async ({ fileStrategy, userId, URL, fileName, basePath, c * Saves file metadata to the database with an expiry TTL. * * @param {Object} params - The parameters object. - * @param {Express.Request} params.req - The Express request object. + * @param {ServerRequest} params.req - The Express request object. * @param {Express.Response} [params.res] - The Express response object. - * @param {Express.Multer.File} params.file - The uploaded file. * @param {ImageMetadata} params.metadata - Additional metadata for the file. * @param {boolean} params.returnFile - Whether to return the file metadata or return response as normal. * @returns {Promise} */ -const processImageFile = async ({ req, res, file, metadata, returnFile = false }) => { +const processImageFile = async ({ req, res, metadata, returnFile = false }) => { + const { file } = req; const source = req.app.locals.fileStrategy; const { handleImageUpload } = getStrategyFunctions(source); const { file_id, temp_file_id, endpoint } = metadata; @@ -289,7 +320,7 @@ const processImageFile = async ({ req, res, file, metadata, returnFile = false } * returns minimal file metadata, without saving to the database. * * @param {Object} params - The parameters object. - * @param {Express.Request} params.req - The Express request object. + * @param {ServerRequest} params.req - The Express request object. * @param {FileContext} params.context - The context of the file (e.g., 'avatar', 'image_generation', etc.) * @param {boolean} [params.resize=true] - Whether to resize and convert the image to target format. Default is `true`. * @param {{ buffer: Buffer, width: number, height: number, bytes: number, filename: string, type: string, file_id: string }} [params.metadata] - Required metadata for the file if resize is false. @@ -335,13 +366,12 @@ const uploadImageBuffer = async ({ req, context, metadata = {}, resize = true }) * Files must be deleted from the server filesystem manually. * * @param {Object} params - The parameters object. - * @param {Express.Request} params.req - The Express request object. + * @param {ServerRequest} params.req - The Express request object. * @param {Express.Response} params.res - The Express response object. - * @param {Express.Multer.File} params.file - The uploaded file. * @param {FileMetadata} params.metadata - Additional metadata for the file. * @returns {Promise} */ -const processFileUpload = async ({ req, res, file, metadata }) => { +const processFileUpload = async ({ req, res, metadata }) => { const isAssistantUpload = isAssistantsEndpoint(metadata.endpoint); const assistantSource = metadata.endpoint === EModelEndpoint.azureAssistants ? FileSources.azure : FileSources.openai; @@ -355,6 +385,7 @@ const processFileUpload = async ({ req, res, file, metadata }) => { ({ openai } = await getOpenAIClient({ req })); } + const { file } = req; const { id, bytes, @@ -422,13 +453,13 @@ const processFileUpload = async ({ req, res, file, metadata }) => { * Files must be deleted from the server filesystem manually. * * @param {Object} params - The parameters object. - * @param {Express.Request} params.req - The Express request object. + * @param {ServerRequest} params.req - The Express request object. * @param {Express.Response} params.res - The Express response object. - * @param {Express.Multer.File} params.file - The uploaded file. * @param {FileMetadata} params.metadata - Additional metadata for the file. * @returns {Promise} */ -const processAgentFileUpload = async ({ req, res, file, metadata }) => { +const processAgentFileUpload = async ({ req, res, metadata }) => { + const { file } = req; const { agent_id, tool_resource } = metadata; if (agent_id && !tool_resource) { throw new Error('No tool resource provided for agent file upload'); @@ -453,6 +484,7 @@ const processAgentFileUpload = async ({ req, res, file, metadata }) => { stream, filename: file.originalname, apiKey: result[EnvVar.CODE_API_KEY], + entity_id: messageAttachment === true ? undefined : agent_id, }); fileInfoMetadata = { fileIdentifier }; } @@ -576,7 +608,7 @@ const processOpenAIFile = async ({ /** * Process OpenAI image files, convert to target format, save and return file metadata. * @param {object} params - The params object. - * @param {Express.Request} params.req - The Express request object. + * @param {ServerRequest} params.req - The Express request object. * @param {Buffer} params.buffer - The image buffer. * @param {string} params.file_id - The file ID. * @param {string} params.filename - The filename. @@ -708,20 +740,20 @@ async function retrieveAndProcessFile({ * Filters a file based on its size and the endpoint origin. * * @param {Object} params - The parameters for the function. - * @param {object} params.req - The request object from Express. + * @param {ServerRequest} params.req - The request object from Express. * @param {string} [params.req.endpoint] * @param {string} [params.req.file_id] * @param {number} [params.req.width] * @param {number} [params.req.height] * @param {number} [params.req.version] - * @param {Express.Multer.File} params.file - The file uploaded to the server via multer. * @param {boolean} [params.image] - Whether the file expected is an image. * @param {boolean} [params.isAvatar] - Whether the file expected is a user or entity avatar. * @returns {void} * * @throws {Error} If a file exception is caught (invalid file size or type, lack of metadata). */ -function filterFile({ req, file, image, isAvatar }) { +function filterFile({ req, image, isAvatar }) { + const { file } = req; const { endpoint, file_id, width, height } = req.body; if (!file_id && !isAvatar) { diff --git a/api/server/services/PluginService.js b/api/server/services/PluginService.js index 2b09da96a7..e03f7f89e9 100644 --- a/api/server/services/PluginService.js +++ b/api/server/services/PluginService.js @@ -7,6 +7,7 @@ const { logger } = require('~/config'); * * @param {string} userId - The unique identifier of the user for whom the plugin authentication value is to be retrieved. * @param {string} authField - The specific authentication field (e.g., 'API_KEY', 'URL') whose value is to be retrieved and decrypted. + * @param {boolean} throwError - Whether to throw an error if the authentication value does not exist. Defaults to `true`. * @returns {Promise} A promise that resolves to the decrypted authentication value if found, or `null` if no such authentication value exists for the given user and field. * * The function throws an error if it encounters any issue during the retrieval or decryption process, or if the authentication value does not exist. @@ -22,7 +23,7 @@ const { logger } = require('~/config'); * @throws {Error} Throws an error if there's an issue during the retrieval or decryption process, or if the authentication value does not exist. * @async */ -const getUserPluginAuthValue = async (userId, authField) => { +const getUserPluginAuthValue = async (userId, authField, throwError = true) => { try { const pluginAuth = await PluginAuth.findOne({ userId, authField }).lean(); if (!pluginAuth) { @@ -32,6 +33,9 @@ const getUserPluginAuthValue = async (userId, authField) => { const decryptedValue = await decrypt(pluginAuth.value); return decryptedValue; } catch (err) { + if (!throwError) { + return null; + } logger.error('[getUserPluginAuthValue]', err); throw err; } diff --git a/api/server/services/ToolService.js b/api/server/services/ToolService.js index 4ffaf3d5ba..91a5e7a6cf 100644 --- a/api/server/services/ToolService.js +++ b/api/server/services/ToolService.js @@ -1,8 +1,8 @@ const fs = require('fs'); const path = require('path'); const { zodToJsonSchema } = require('zod-to-json-schema'); -const { Calculator } = require('@langchain/community/tools/calculator'); const { tool: toolFn, Tool } = require('@langchain/core/tools'); +const { Calculator } = require('@langchain/community/tools/calculator'); const { Tools, ContentTypes, @@ -170,7 +170,7 @@ async function processRequiredActions(client, requiredActions) { requiredActions, ); const tools = requiredActions.map((action) => action.tool); - const loadedTools = await loadTools({ + const { loadedTools } = await loadTools({ user: client.req.user.id, model: client.req.body.model ?? 'gpt-4o-mini', tools, @@ -183,7 +183,6 @@ async function processRequiredActions(client, requiredActions) { fileStrategy: client.req.app.locals.fileStrategy, returnMetadata: true, }, - skipSpecs: true, }); const ToolMap = loadedTools.reduce((map, tool) => { @@ -378,21 +377,21 @@ async function loadAgentTools({ req, agent_id, tools, tool_resources, openAIApiK if (!tools || tools.length === 0) { return {}; } - const loadedTools = await loadTools({ + const { loadedTools, toolContextMap } = await loadTools({ user: req.user.id, // model: req.body.model ?? 'gpt-4o-mini', tools, functions: true, + isAgent: agent_id != null, options: { req, openAIApiKey, tool_resources, - returnMetadata: true, processFileURL, uploadImageBuffer, + returnMetadata: true, fileStrategy: req.app.locals.fileStrategy, }, - skipSpecs: true, }); const agentTools = []; @@ -403,16 +402,19 @@ async function loadAgentTools({ req, agent_id, tools, tool_resources, openAIApiK continue; } - const toolInstance = toolFn( - async (...args) => { - return tool['_call'](...args); - }, - { - name: tool.name, - description: tool.description, - schema: tool.schema, - }, - ); + const toolDefinition = { + name: tool.name, + schema: tool.schema, + description: tool.description, + }; + + if (imageGenTools.has(tool.name)) { + toolDefinition.responseFormat = 'content_and_artifact'; + } + + const toolInstance = toolFn(async (...args) => { + return tool['_call'](...args); + }, toolDefinition); agentTools.push(toolInstance); } @@ -476,6 +478,7 @@ async function loadAgentTools({ req, agent_id, tools, tool_resources, openAIApiK return { tools: agentTools, + toolContextMap, }; } diff --git a/api/server/services/start/interface.js b/api/server/services/start/interface.js index bf31eb78b8..10db2fd3a8 100644 --- a/api/server/services/start/interface.js +++ b/api/server/services/start/interface.js @@ -32,17 +32,20 @@ async function loadDefaultInterface(config, configDefaults, roleName = SystemRol bookmarks: interfaceConfig?.bookmarks ?? defaults.bookmarks, prompts: interfaceConfig?.prompts ?? defaults.prompts, multiConvo: interfaceConfig?.multiConvo ?? defaults.multiConvo, + agents: interfaceConfig?.agents ?? defaults.agents, }); await updateAccessPermissions(roleName, { [PermissionTypes.PROMPTS]: { [Permissions.USE]: loadedInterface.prompts }, [PermissionTypes.BOOKMARKS]: { [Permissions.USE]: loadedInterface.bookmarks }, [PermissionTypes.MULTI_CONVO]: { [Permissions.USE]: loadedInterface.multiConvo }, + [PermissionTypes.AGENTS]: { [Permissions.USE]: loadedInterface.agents }, }); await updateAccessPermissions(SystemRoles.ADMIN, { [PermissionTypes.PROMPTS]: { [Permissions.USE]: loadedInterface.prompts }, [PermissionTypes.BOOKMARKS]: { [Permissions.USE]: loadedInterface.bookmarks }, [PermissionTypes.MULTI_CONVO]: { [Permissions.USE]: loadedInterface.multiConvo }, + [PermissionTypes.AGENTS]: { [Permissions.USE]: loadedInterface.agents }, }); let i = 0; diff --git a/api/server/services/start/interface.spec.js b/api/server/services/start/interface.spec.js index 62239a6a29..0041246433 100644 --- a/api/server/services/start/interface.spec.js +++ b/api/server/services/start/interface.spec.js @@ -7,8 +7,15 @@ jest.mock('~/models/Role', () => ({ })); describe('loadDefaultInterface', () => { - it('should call updateAccessPermissions with the correct parameters when prompts and bookmarks are true', async () => { - const config = { interface: { prompts: true, bookmarks: true } }; + it('should call updateAccessPermissions with the correct parameters when permission types are true', async () => { + const config = { + interface: { + prompts: true, + bookmarks: true, + multiConvo: true, + agents: true, + }, + }; const configDefaults = { interface: {} }; await loadDefaultInterface(config, configDefaults); @@ -16,12 +23,20 @@ describe('loadDefaultInterface', () => { expect(updateAccessPermissions).toHaveBeenCalledWith(SystemRoles.USER, { [PermissionTypes.PROMPTS]: { [Permissions.USE]: true }, [PermissionTypes.BOOKMARKS]: { [Permissions.USE]: true }, - [PermissionTypes.MULTI_CONVO]: { [Permissions.USE]: undefined }, + [PermissionTypes.MULTI_CONVO]: { [Permissions.USE]: true }, + [PermissionTypes.AGENTS]: { [Permissions.USE]: true }, }); }); - it('should call updateAccessPermissions with false when prompts and bookmarks are false', async () => { - const config = { interface: { prompts: false, bookmarks: false } }; + it('should call updateAccessPermissions with false when permission types are false', async () => { + const config = { + interface: { + prompts: false, + bookmarks: false, + multiConvo: false, + agents: false, + }, + }; const configDefaults = { interface: {} }; await loadDefaultInterface(config, configDefaults); @@ -29,11 +44,12 @@ describe('loadDefaultInterface', () => { expect(updateAccessPermissions).toHaveBeenCalledWith(SystemRoles.USER, { [PermissionTypes.PROMPTS]: { [Permissions.USE]: false }, [PermissionTypes.BOOKMARKS]: { [Permissions.USE]: false }, - [PermissionTypes.MULTI_CONVO]: { [Permissions.USE]: undefined }, + [PermissionTypes.MULTI_CONVO]: { [Permissions.USE]: false }, + [PermissionTypes.AGENTS]: { [Permissions.USE]: false }, }); }); - it('should call updateAccessPermissions with undefined when prompts and bookmarks are not specified in config', async () => { + it('should call updateAccessPermissions with undefined when permission types are not specified in config', async () => { const config = {}; const configDefaults = { interface: {} }; @@ -43,11 +59,19 @@ describe('loadDefaultInterface', () => { [PermissionTypes.PROMPTS]: { [Permissions.USE]: undefined }, [PermissionTypes.BOOKMARKS]: { [Permissions.USE]: undefined }, [PermissionTypes.MULTI_CONVO]: { [Permissions.USE]: undefined }, + [PermissionTypes.AGENTS]: { [Permissions.USE]: undefined }, }); }); - it('should call updateAccessPermissions with undefined when prompts and bookmarks are explicitly undefined', async () => { - const config = { interface: { prompts: undefined, bookmarks: undefined } }; + it('should call updateAccessPermissions with undefined when permission types are explicitly undefined', async () => { + const config = { + interface: { + prompts: undefined, + bookmarks: undefined, + multiConvo: undefined, + agents: undefined, + }, + }; const configDefaults = { interface: {} }; await loadDefaultInterface(config, configDefaults); @@ -56,11 +80,19 @@ describe('loadDefaultInterface', () => { [PermissionTypes.PROMPTS]: { [Permissions.USE]: undefined }, [PermissionTypes.BOOKMARKS]: { [Permissions.USE]: undefined }, [PermissionTypes.MULTI_CONVO]: { [Permissions.USE]: undefined }, + [PermissionTypes.AGENTS]: { [Permissions.USE]: undefined }, }); }); - it('should call updateAccessPermissions with mixed values for prompts and bookmarks', async () => { - const config = { interface: { prompts: true, bookmarks: false } }; + it('should call updateAccessPermissions with mixed values for permission types', async () => { + const config = { + interface: { + prompts: true, + bookmarks: false, + multiConvo: undefined, + agents: true, + }, + }; const configDefaults = { interface: {} }; await loadDefaultInterface(config, configDefaults); @@ -69,19 +101,28 @@ describe('loadDefaultInterface', () => { [PermissionTypes.PROMPTS]: { [Permissions.USE]: true }, [PermissionTypes.BOOKMARKS]: { [Permissions.USE]: false }, [PermissionTypes.MULTI_CONVO]: { [Permissions.USE]: undefined }, + [PermissionTypes.AGENTS]: { [Permissions.USE]: true }, }); }); it('should call updateAccessPermissions with true when config is undefined', async () => { const config = undefined; - const configDefaults = { interface: { prompts: true, bookmarks: true } }; + const configDefaults = { + interface: { + prompts: true, + bookmarks: true, + multiConvo: true, + agents: true, + }, + }; await loadDefaultInterface(config, configDefaults); expect(updateAccessPermissions).toHaveBeenCalledWith(SystemRoles.USER, { [PermissionTypes.PROMPTS]: { [Permissions.USE]: true }, [PermissionTypes.BOOKMARKS]: { [Permissions.USE]: true }, - [PermissionTypes.MULTI_CONVO]: { [Permissions.USE]: undefined }, + [PermissionTypes.MULTI_CONVO]: { [Permissions.USE]: true }, + [PermissionTypes.AGENTS]: { [Permissions.USE]: true }, }); }); @@ -95,6 +136,7 @@ describe('loadDefaultInterface', () => { [PermissionTypes.PROMPTS]: { [Permissions.USE]: undefined }, [PermissionTypes.BOOKMARKS]: { [Permissions.USE]: undefined }, [PermissionTypes.MULTI_CONVO]: { [Permissions.USE]: true }, + [PermissionTypes.AGENTS]: { [Permissions.USE]: undefined }, }); }); @@ -108,6 +150,7 @@ describe('loadDefaultInterface', () => { [PermissionTypes.PROMPTS]: { [Permissions.USE]: undefined }, [PermissionTypes.BOOKMARKS]: { [Permissions.USE]: undefined }, [PermissionTypes.MULTI_CONVO]: { [Permissions.USE]: false }, + [PermissionTypes.AGENTS]: { [Permissions.USE]: undefined }, }); }); @@ -121,11 +164,19 @@ describe('loadDefaultInterface', () => { [PermissionTypes.PROMPTS]: { [Permissions.USE]: undefined }, [PermissionTypes.BOOKMARKS]: { [Permissions.USE]: undefined }, [PermissionTypes.MULTI_CONVO]: { [Permissions.USE]: undefined }, + [PermissionTypes.AGENTS]: { [Permissions.USE]: undefined }, }); }); it('should call updateAccessPermissions with all interface options including multiConvo', async () => { - const config = { interface: { prompts: true, bookmarks: false, multiConvo: true } }; + const config = { + interface: { + prompts: true, + bookmarks: false, + multiConvo: true, + agents: false, + }, + }; const configDefaults = { interface: {} }; await loadDefaultInterface(config, configDefaults); @@ -134,12 +185,20 @@ describe('loadDefaultInterface', () => { [PermissionTypes.PROMPTS]: { [Permissions.USE]: true }, [PermissionTypes.BOOKMARKS]: { [Permissions.USE]: false }, [PermissionTypes.MULTI_CONVO]: { [Permissions.USE]: true }, + [PermissionTypes.AGENTS]: { [Permissions.USE]: false }, }); }); it('should use default values for multiConvo when config is undefined', async () => { const config = undefined; - const configDefaults = { interface: { prompts: true, bookmarks: true, multiConvo: false } }; + const configDefaults = { + interface: { + prompts: true, + bookmarks: true, + multiConvo: false, + agents: undefined, + }, + }; await loadDefaultInterface(config, configDefaults); @@ -147,6 +206,7 @@ describe('loadDefaultInterface', () => { [PermissionTypes.PROMPTS]: { [Permissions.USE]: true }, [PermissionTypes.BOOKMARKS]: { [Permissions.USE]: true }, [PermissionTypes.MULTI_CONVO]: { [Permissions.USE]: false }, + [PermissionTypes.AGENTS]: { [Permissions.USE]: undefined }, }); }); }); diff --git a/api/server/utils/handleText.js b/api/server/utils/handleText.js index 5a32394fcd..92f8253fc7 100644 --- a/api/server/utils/handleText.js +++ b/api/server/utils/handleText.js @@ -196,14 +196,11 @@ function generateConfig(key, baseURL, endpoint) { if (agents) { config.capabilities = [ + AgentCapabilities.execute_code, AgentCapabilities.file_search, AgentCapabilities.actions, AgentCapabilities.tools, ]; - - if (key === 'EXPERIMENTAL_RUN_CODE') { - config.capabilities.push(AgentCapabilities.execute_code); - } } if (assistants && endpoint === EModelEndpoint.azureAssistants) { diff --git a/api/typedefs.js b/api/typedefs.js index 8c1af11a69..a80a25609d 100644 --- a/api/typedefs.js +++ b/api/typedefs.js @@ -56,12 +56,33 @@ * @memberof typedefs */ +/** + * @exports BaseMessage + * @typedef {import('@langchain/core/messages').BaseMessage} BaseMessage + * @memberof typedefs + */ + /** * @exports UsageMetadata * @typedef {import('@langchain/core/messages').UsageMetadata} UsageMetadata * @memberof typedefs */ +/** + * @exports GraphRunnableConfig + * @typedef {import('@langchain/core/runnables').RunnableConfig<{ + * req: ServerRequest; + * thread_id: string; + * run_id: string; + * agent_id: string; + * name: string; + * agent_index: number; + * last_agent_index: number; + * hide_sequential_outputs: boolean; + * }>} GraphRunnableConfig + * @memberof typedefs + */ + /** * @exports Ollama * @typedef {import('ollama').Ollama} Ollama @@ -689,6 +710,12 @@ * @memberof typedefs */ +/** + * @exports ToolCallData + * @typedef {import('~/models/schema/toolCallSchema.js').ToolCallData} ToolCallData + * @memberof typedefs + */ + /** * @exports MongoUser * @typedef {import('~/models/schema/userSchema.js').MongoUser} MongoUser @@ -803,6 +830,12 @@ * @memberof typedefs */ +/** + * @exports AgentToolResources + * @typedef {import('librechat-data-provider').AgentToolResources} AgentToolResources + * @memberof typedefs + */ + /** * @exports AgentCreateParams * @typedef {import('librechat-data-provider').AgentCreateParams} AgentCreateParams diff --git a/client/public/assets/c.svg b/client/public/assets/c.svg new file mode 100644 index 0000000000..fc75a6258b --- /dev/null +++ b/client/public/assets/c.svg @@ -0,0 +1 @@ +C \ No newline at end of file diff --git a/client/public/assets/cplusplus.svg b/client/public/assets/cplusplus.svg new file mode 100644 index 0000000000..fe2f58d6af --- /dev/null +++ b/client/public/assets/cplusplus.svg @@ -0,0 +1 @@ +C++ \ No newline at end of file diff --git a/client/public/assets/fortran.svg b/client/public/assets/fortran.svg new file mode 100644 index 0000000000..44ae0a8e5f --- /dev/null +++ b/client/public/assets/fortran.svg @@ -0,0 +1 @@ +Fortran \ No newline at end of file diff --git a/client/public/assets/go.svg b/client/public/assets/go.svg new file mode 100644 index 0000000000..0cadd56b11 --- /dev/null +++ b/client/public/assets/go.svg @@ -0,0 +1 @@ +Go \ No newline at end of file diff --git a/client/public/assets/nodedotjs.svg b/client/public/assets/nodedotjs.svg new file mode 100644 index 0000000000..281c829627 --- /dev/null +++ b/client/public/assets/nodedotjs.svg @@ -0,0 +1 @@ +Node.js \ No newline at end of file diff --git a/client/public/assets/php.svg b/client/public/assets/php.svg new file mode 100644 index 0000000000..a08156aff7 --- /dev/null +++ b/client/public/assets/php.svg @@ -0,0 +1 @@ +PHP \ No newline at end of file diff --git a/client/public/assets/python.svg b/client/public/assets/python.svg new file mode 100644 index 0000000000..30587d8164 --- /dev/null +++ b/client/public/assets/python.svg @@ -0,0 +1 @@ +Python \ No newline at end of file diff --git a/client/public/assets/rust.svg b/client/public/assets/rust.svg new file mode 100644 index 0000000000..b95ce42ae7 --- /dev/null +++ b/client/public/assets/rust.svg @@ -0,0 +1 @@ +Rust \ No newline at end of file diff --git a/client/public/assets/tsnode.svg b/client/public/assets/tsnode.svg new file mode 100644 index 0000000000..5cc1aadb0e --- /dev/null +++ b/client/public/assets/tsnode.svg @@ -0,0 +1 @@ +ts-node \ No newline at end of file diff --git a/client/src/Providers/CodeBlockContext.tsx b/client/src/Providers/CodeBlockContext.tsx new file mode 100644 index 0000000000..915e740840 --- /dev/null +++ b/client/src/Providers/CodeBlockContext.tsx @@ -0,0 +1,34 @@ +import { createContext, useContext, ReactNode, useCallback, useRef } from 'react'; + +type TCodeBlockContext = { + getNextIndex: (skip: boolean) => number; + resetCounter: () => void; + // codeBlocks: Map; +}; + +export const CodeBlockContext = createContext({} as TCodeBlockContext); +export const useCodeBlockContext = () => useContext(CodeBlockContext); + +export function CodeBlockProvider({ children }: { children: ReactNode }) { + const counterRef = useRef(0); + // const codeBlocks = useRef(new Map()).current; + + const getNextIndex = useCallback((skip: boolean) => { + if (skip) { + return counterRef.current; + } + const nextIndex = counterRef.current; + counterRef.current += 1; + return nextIndex; + }, []); + + const resetCounter = useCallback(() => { + counterRef.current = 0; + }, []); + + return ( + + {children} + + ); +} diff --git a/client/src/Providers/MessageContext.tsx b/client/src/Providers/MessageContext.tsx new file mode 100644 index 0000000000..6673dd2eb3 --- /dev/null +++ b/client/src/Providers/MessageContext.tsx @@ -0,0 +1,9 @@ +import { createContext, useContext } from 'react'; +type MessageContext = { + messageId: string; + partIndex?: number; + conversationId?: string | null; +}; + +export const MessageContext = createContext({} as MessageContext); +export const useMessageContext = () => useContext(MessageContext); diff --git a/client/src/Providers/ToolCallsMapContext.tsx b/client/src/Providers/ToolCallsMapContext.tsx new file mode 100644 index 0000000000..516d3d77f0 --- /dev/null +++ b/client/src/Providers/ToolCallsMapContext.tsx @@ -0,0 +1,21 @@ +import { createContext, useContext } from 'react'; +import useToolCallsMap from '~/hooks/Plugins/useToolCallsMap'; +type ToolCallsMapContextType = ReturnType; + +export const ToolCallsMapContext = createContext( + {} as ToolCallsMapContextType, +); +export const useToolCallsMapContext = () => useContext(ToolCallsMapContext); + +interface ToolCallsMapProviderProps { + children: React.ReactNode; + conversationId: string; +} + +export function ToolCallsMapProvider({ children, conversationId }: ToolCallsMapProviderProps) { + const toolCallsMap = useToolCallsMap({ conversationId }); + + return ( + {children} + ); +} diff --git a/client/src/Providers/index.ts b/client/src/Providers/index.ts index be9036a51c..d777b5bb76 100644 --- a/client/src/Providers/index.ts +++ b/client/src/Providers/index.ts @@ -9,9 +9,12 @@ export * from './FileMapContext'; export * from './AddedChatContext'; export * from './ChatFormContext'; export * from './BookmarkContext'; +export * from './MessageContext'; export * from './DashboardContext'; export * from './AssistantsContext'; export * from './AgentsContext'; export * from './AssistantsMapContext'; export * from './AnnouncerContext'; export * from './AgentsMapContext'; +export * from './CodeBlockContext'; +export * from './ToolCallsMapContext'; diff --git a/client/src/common/agents-types.ts b/client/src/common/agents-types.ts index fd268e8cb7..7f64f07882 100644 --- a/client/src/common/agents-types.ts +++ b/client/src/common/agents-types.ts @@ -11,6 +11,8 @@ export type TAgentOption = OptionWithIcon & export type TAgentCapabilities = { [AgentCapabilities.execute_code]: boolean; [AgentCapabilities.file_search]: boolean; + [AgentCapabilities.end_after_tools]?: boolean; + [AgentCapabilities.hide_sequential_outputs]?: boolean; }; export type AgentForm = { @@ -23,4 +25,5 @@ export type AgentForm = { model_parameters: AgentModelParameters; tools?: string[]; provider?: AgentProvider | OptionWithIcon; + agent_ids?: string[]; } & TAgentCapabilities; diff --git a/client/src/common/index.ts b/client/src/common/index.ts index 85dda0700c..428f01017d 100644 --- a/client/src/common/index.ts +++ b/client/src/common/index.ts @@ -1,5 +1,6 @@ export * from './a11y'; export * from './artifacts'; export * from './types'; +export * from './tools'; export * from './assistants-types'; export * from './agents-types'; diff --git a/client/src/common/tools.ts b/client/src/common/tools.ts new file mode 100644 index 0000000000..140f5678c1 --- /dev/null +++ b/client/src/common/tools.ts @@ -0,0 +1,6 @@ +import type { AuthType } from 'librechat-data-provider'; + +export type ApiKeyFormData = { + apiKey: string; + authType?: string | AuthType; +}; diff --git a/client/src/common/types.ts b/client/src/common/types.ts index 3590b279b8..fe732ce0c0 100644 --- a/client/src/common/types.ts +++ b/client/src/common/types.ts @@ -1,36 +1,21 @@ -import React from 'react'; +import { RefObject } from 'react'; import { FileSources } from 'librechat-data-provider'; import type * as InputNumberPrimitive from 'rc-input-number'; import type { ColumnDef } from '@tanstack/react-table'; import type { SetterOrUpdater } from 'recoil'; -import type { - TRole, - TUser, - Agent, - Action, - TPreset, - TPlugin, - TMessage, - Assistant, - TResPlugin, - TLoginUser, - AuthTypeEnum, - TModelsConfig, - TConversation, - TStartupConfig, - EModelEndpoint, - TEndpointsConfig, - ActionMetadata, - AssistantDocument, - AssistantsEndpoint, - TMessageContentParts, - AuthorizationTypeEnum, - TSetOption as SetOption, - TokenExchangeMethodEnum, -} from 'librechat-data-provider'; +import type * as t from 'librechat-data-provider'; import type { UseMutationResult } from '@tanstack/react-query'; import type { LucideIcon } from 'lucide-react'; +export type CodeBarProps = { + lang: string; + error?: boolean; + plugin?: boolean; + blockIndex?: number; + allowExecution?: boolean; + codeRef: RefObject; +}; + export enum PromptsEditorMode { SIMPLE = 'simple', ADVANCED = 'advanced', @@ -65,21 +50,21 @@ export type AudioChunk = { export type AssistantListItem = { id: string; name: string; - metadata: Assistant['metadata']; + metadata: t.Assistant['metadata']; model: string; }; export type AgentListItem = { id: string; name: string; - avatar: Agent['avatar']; + avatar: t.Agent['avatar']; }; -export type TPluginMap = Record; +export type TPluginMap = Record; export type GenericSetter = (value: T | ((currentValue: T) => T)) => void; -export type LastSelectedModels = Record; +export type LastSelectedModels = Record; export type LocalizeFunction = (phraseKey: string, ...values: string[]) => string; @@ -145,11 +130,11 @@ export type FileSetter = export type ActionAuthForm = { /* General */ - type: AuthTypeEnum; + type: t.AuthTypeEnum; saved_auth_fields: boolean; /* API key */ api_key: string; // not nested - authorization_type: AuthorizationTypeEnum; + authorization_type: t.AuthorizationTypeEnum; custom_auth_header: string; /* OAuth */ oauth_client_id: string; // not nested @@ -157,23 +142,23 @@ export type ActionAuthForm = { authorization_url: string; client_url: string; scope: string; - token_exchange_method: TokenExchangeMethodEnum; + token_exchange_method: t.TokenExchangeMethodEnum; }; -export type ActionWithNullableMetadata = Omit & { - metadata: ActionMetadata | null; +export type ActionWithNullableMetadata = Omit & { + metadata: t.ActionMetadata | null; }; export type AssistantPanelProps = { index?: number; action?: ActionWithNullableMetadata; - actions?: Action[]; + actions?: t.Action[]; assistant_id?: string; activePanel?: string; - endpoint: AssistantsEndpoint; + endpoint: t.AssistantsEndpoint; version: number | string; - documentsMap: Map | null; - setAction: React.Dispatch>; + documentsMap: Map | null; + setAction: React.Dispatch>; setCurrentAssistantId: React.Dispatch>; setActivePanel: React.Dispatch>; }; @@ -182,11 +167,11 @@ export type AgentPanelProps = { index?: number; agent_id?: string; activePanel?: string; - action?: Action; - actions?: Action[]; + action?: t.Action; + actions?: t.Action[]; setActivePanel: React.Dispatch>; - setAction: React.Dispatch>; - endpointsConfig?: TEndpointsConfig; + setAction: React.Dispatch>; + endpointsConfig?: t.TEndpointsConfig; setCurrentAgentId: React.Dispatch>; }; @@ -199,7 +184,7 @@ export type AgentModelPanelProps = { export type AugmentedColumnDef = ColumnDef & DataColumnMeta; -export type TSetOption = SetOption; +export type TSetOption = t.TSetOption; export type TSetExample = ( i: number, @@ -234,7 +219,7 @@ export type TShowToast = { }; export type TBaseSettingsProps = { - conversation: TConversation | TPreset | null; + conversation: t.TConversation | t.TPreset | null; className?: string; isPreset?: boolean; readonly?: boolean; @@ -255,7 +240,7 @@ export type TModelSelectProps = TSettingsProps & TModels; export type TEditPresetProps = { open: boolean; onOpenChange: React.Dispatch>; - preset: TPreset; + preset: t.TPreset; title?: string; }; @@ -266,18 +251,18 @@ export type TSetOptionsPayload = { addExample: () => void; removeExample: () => void; setAgentOption: TSetOption; - // getConversation: () => TConversation | TPreset | null; + // getConversation: () => t.TConversation | t.TPreset | null; checkPluginSelection: (value: string) => boolean; setTools: (newValue: string, remove?: boolean) => void; setOptions?: TSetOptions; }; export type TPresetItemProps = { - preset: TPreset; - value: TPreset; - onSelect: (preset: TPreset) => void; - onChangePreset: (preset: TPreset) => void; - onDeletePreset: (preset: TPreset) => void; + preset: t.TPreset; + value: t.TPreset; + onSelect: (preset: t.TPreset) => void; + onChangePreset: (preset: t.TPreset) => void; + onDeletePreset: (preset: t.TPreset) => void; }; export type TOnClick = (e: React.MouseEvent) => void; @@ -302,16 +287,16 @@ export type TOptions = { isRegenerate?: boolean; isContinued?: boolean; isEdited?: boolean; - overrideMessages?: TMessage[]; + overrideMessages?: t.TMessage[]; }; export type TAskFunction = (props: TAskProps, options?: TOptions) => void; export type TMessageProps = { - conversation?: TConversation | null; + conversation?: t.TConversation | null; messageId?: string | null; - message?: TMessage; - messagesTree?: TMessage[]; + message?: t.TMessage; + messagesTree?: t.TMessage[]; currentEditId: string | number | null; isSearchView?: boolean; siblingIdx?: number; @@ -330,7 +315,7 @@ export type TInitialProps = { }; export type TAdditionalProps = { ask: TAskFunction; - message: TMessage; + message: t.TMessage; isCreatedByUser: boolean; siblingIdx: number; enterEdit: (cancel: boolean) => void; @@ -354,7 +339,7 @@ export type TDisplayProps = TText & export type TConfigProps = { userKey: string; setUserKey: React.Dispatch>; - endpoint: EModelEndpoint | string; + endpoint: t.EModelEndpoint | string; }; export type TDangerButtonProps = { @@ -389,18 +374,18 @@ export type TResError = { }; export type TAuthContext = { - user: TUser | undefined; + user: t.TUser | undefined; token: string | undefined; isAuthenticated: boolean; error: string | undefined; - login: (data: TLoginUser) => void; + login: (data: t.TLoginUser) => void; logout: () => void; setError: React.Dispatch>; - roles?: Record; + roles?: Record; }; export type TUserContext = { - user?: TUser | undefined; + user?: t.TUser | undefined; token: string | undefined; isAuthenticated: boolean; redirect?: string; @@ -411,16 +396,16 @@ export type TAuthConfig = { test?: boolean; }; -export type IconProps = Pick & - Pick & { +export type IconProps = Pick & + Pick & { size?: number; button?: boolean; iconURL?: string; message?: boolean; className?: string; iconClassName?: string; - endpoint?: EModelEndpoint | string | null; - endpointType?: EModelEndpoint | null; + endpoint?: t.EModelEndpoint | string | null; + endpointType?: t.EModelEndpoint | null; assistantName?: string; agentName?: string; error?: boolean; @@ -440,7 +425,7 @@ export type VoiceOption = { export type TMessageAudio = { messageId?: string; - content?: TMessageContentParts[] | string; + content?: t.TMessageContentParts[] | string; className?: string; isLast: boolean; index: number; @@ -482,12 +467,12 @@ export interface ExtendedFile { export type ContextType = { navVisible: boolean; setNavVisible: (visible: boolean) => void }; export interface SwitcherProps { - endpoint?: EModelEndpoint | null; + endpoint?: t.EModelEndpoint | null; endpointKeyProvided: boolean; isCollapsed: boolean; } export type TLoginLayoutContext = { - startupConfig: TStartupConfig | null; + startupConfig: t.TStartupConfig | null; startupConfigError: unknown; isFetching: boolean; error: string | null; @@ -497,34 +482,34 @@ export type TLoginLayoutContext = { }; export type NewConversationParams = { - template?: Partial; - preset?: Partial; - modelsData?: TModelsConfig; + template?: Partial; + preset?: Partial; + modelsData?: t.TModelsConfig; buildDefault?: boolean; keepLatestMessage?: boolean; keepAddedConvos?: boolean; }; -export type ConvoGenerator = (params: NewConversationParams) => void | TConversation; +export type ConvoGenerator = (params: NewConversationParams) => void | t.TConversation; export type TBaseResData = { - plugin?: TResPlugin; + plugin?: t.TResPlugin; final?: boolean; initial?: boolean; - previousMessages?: TMessage[]; - conversation: TConversation; + previousMessages?: t.TMessage[]; + conversation: t.TConversation; conversationId?: string; - runMessages?: TMessage[]; + runMessages?: t.TMessage[]; }; export type TResData = TBaseResData & { - requestMessage: TMessage; - responseMessage: TMessage; + requestMessage: t.TMessage; + responseMessage: t.TMessage; }; export type TFinalResData = TBaseResData & { - requestMessage?: TMessage; - responseMessage?: TMessage; + requestMessage?: t.TMessage; + responseMessage?: t.TMessage; }; export type TVectorStore = { diff --git a/client/src/components/Chat/AddMultiConvo.tsx b/client/src/components/Chat/AddMultiConvo.tsx index 8ee85ebb3a..6cfeb04b9c 100644 --- a/client/src/components/Chat/AddMultiConvo.tsx +++ b/client/src/components/Chat/AddMultiConvo.tsx @@ -5,7 +5,6 @@ import { useChatContext, useAddedChatContext } from '~/Providers'; import { TooltipAnchor } from '~/components'; import { mainTextareaId } from '~/common'; import { useLocalize } from '~/hooks'; -import { cn } from '~/utils'; function AddMultiConvo() { const { conversation } = useChatContext(); diff --git a/client/src/components/Chat/Input/Files/AttachFileMenu.tsx b/client/src/components/Chat/Input/Files/AttachFileMenu.tsx new file mode 100644 index 0000000000..a6854d4a70 --- /dev/null +++ b/client/src/components/Chat/Input/Files/AttachFileMenu.tsx @@ -0,0 +1,100 @@ +import * as Ariakit from '@ariakit/react'; +import React, { useRef, useState } from 'react'; +import { FileSearch, ImageUpIcon, TerminalSquareIcon } from 'lucide-react'; +import { EToolResources } from 'librechat-data-provider'; +import { FileUpload, TooltipAnchor, DropdownPopup } from '~/components/ui'; +import { AttachmentIcon } from '~/components/svg'; +import { useLocalize } from '~/hooks'; +import { cn } from '~/utils'; + +interface AttachFileProps { + isRTL: boolean; + disabled?: boolean | null; + handleFileChange: (event: React.ChangeEvent) => void; + setToolResource?: React.Dispatch>; +} + +const AttachFile = ({ isRTL, disabled, setToolResource, handleFileChange }: AttachFileProps) => { + const localize = useLocalize(); + const isUploadDisabled = disabled ?? false; + const inputRef = useRef(null); + const [isPopoverActive, setIsPopoverActive] = useState(false); + + const handleUploadClick = (isImage?: boolean) => { + if (!inputRef.current) { + return; + } + inputRef.current.value = ''; + inputRef.current.accept = isImage === true ? 'image/*' : ''; + inputRef.current.click(); + inputRef.current.accept = ''; + }; + + const dropdownItems = [ + { + label: localize('com_ui_upload_image_input'), + onClick: () => { + setToolResource?.(undefined); + handleUploadClick(true); + }, + icon: , + }, + { + label: localize('com_ui_upload_file_search'), + onClick: () => { + setToolResource?.(EToolResources.file_search); + handleUploadClick(); + }, + icon: , + }, + { + label: localize('com_ui_upload_code_files'), + onClick: () => { + setToolResource?.(EToolResources.execute_code); + handleUploadClick(); + }, + icon: , + }, + ]; + + const menuTrigger = ( + +
+ +
+ + } + id="attach-file-menu-button" + description={localize('com_sidepanel_attach_files')} + disabled={isUploadDisabled} + /> + ); + + return ( + +
+ +
+ + ); +}; + +export default React.memo(AttachFile); diff --git a/client/src/components/Chat/Input/Files/FileFormWrapper.tsx b/client/src/components/Chat/Input/Files/FileFormWrapper.tsx index 230b303615..a0310cf7f2 100644 --- a/client/src/components/Chat/Input/Files/FileFormWrapper.tsx +++ b/client/src/components/Chat/Input/Files/FileFormWrapper.tsx @@ -1,12 +1,14 @@ -import { memo } from 'react'; +import { memo, useMemo } from 'react'; import { useRecoilValue } from 'recoil'; import { supportsFiles, mergeFileConfig, + isAgentsEndpoint, EndpointFileConfig, fileConfig as defaultFileConfig, } from 'librechat-data-provider'; import { useGetFileConfig } from '~/data-provider'; +import AttachFileMenu from './AttachFileMenu'; import { useChatContext } from '~/Providers'; import { useFileHandling } from '~/hooks'; import AttachFile from './AttachFile'; @@ -20,23 +22,46 @@ function FileFormWrapper({ disableInputs: boolean; children?: React.ReactNode; }) { - const { handleFileChange, abortUpload } = useFileHandling(); const chatDirection = useRecoilValue(store.chatDirection).toLowerCase(); - const { files, setFiles, conversation, setFilesLoading } = useChatContext(); + const { endpoint: _endpoint, endpointType } = conversation ?? { endpoint: null }; + const isAgents = useMemo(() => isAgentsEndpoint(_endpoint), [_endpoint]); + + const { handleFileChange, abortUpload, setToolResource } = useFileHandling(); + const { data: fileConfig = defaultFileConfig } = useGetFileConfig({ select: (data) => mergeFileConfig(data), }); const isRTL = chatDirection === 'rtl'; - const { endpoint: _endpoint, endpointType } = conversation ?? { endpoint: null }; const endpointFileConfig = fileConfig.endpoints[_endpoint ?? ''] as | EndpointFileConfig | undefined; + const endpointSupportsFiles: boolean = supportsFiles[endpointType ?? _endpoint ?? ''] ?? false; const isUploadDisabled = (disableInputs || endpointFileConfig?.disabled) ?? false; + const renderAttachFile = () => { + if (isAgents) { + return ( + + ); + } + if (endpointSupportsFiles && !isUploadDisabled) { + return ( + + ); + } + + return null; + }; + return ( <> {children} - {endpointSupportsFiles && !isUploadDisabled && ( - - )} + {renderAttachFile()} ); } diff --git a/client/src/components/Chat/Input/Mention.tsx b/client/src/components/Chat/Input/Mention.tsx index 65dd07c792..e268bba00f 100644 --- a/client/src/components/Chat/Input/Mention.tsx +++ b/client/src/components/Chat/Input/Mention.tsx @@ -26,8 +26,15 @@ export default function Mention({ }) { const localize = useLocalize(); const assistantMap = useAssistantsMapContext(); - const { options, presets, modelSpecs, modelsConfig, endpointsConfig, assistantListMap } = - useMentions({ assistantMap: assistantMap || {}, includeAssistants }); + const { + options, + presets, + modelSpecs, + agentsList, + modelsConfig, + endpointsConfig, + assistantListMap, + } = useMentions({ assistantMap: assistantMap || {}, includeAssistants }); const { onSelectMention } = useSelectMention({ presets, modelSpecs, @@ -62,18 +69,23 @@ export default function Mention({ } }; - if (mention.type === 'endpoint' && mention.value === EModelEndpoint.assistants) { + if (mention.type === 'endpoint' && mention.value === EModelEndpoint.agents) { setSearchValue(''); - setInputOptions(assistantListMap[EModelEndpoint.assistants]); + setInputOptions(agentsList ?? []); + setActiveIndex(0); + inputRef.current?.focus(); + } else if (mention.type === 'endpoint' && mention.value === EModelEndpoint.assistants) { + setSearchValue(''); + setInputOptions(assistantListMap[EModelEndpoint.assistants] ?? []); setActiveIndex(0); inputRef.current?.focus(); } else if (mention.type === 'endpoint' && mention.value === EModelEndpoint.azureAssistants) { setSearchValue(''); - setInputOptions(assistantListMap[EModelEndpoint.azureAssistants]); + setInputOptions(assistantListMap[EModelEndpoint.azureAssistants] ?? []); setActiveIndex(0); inputRef.current?.focus(); } else if (mention.type === 'endpoint') { - const models = (modelsConfig?.[mention.value ?? ''] ?? []).map((model) => ({ + const models = (modelsConfig?.[mention.value || ''] ?? []).map((model) => ({ value: mention.value, label: model, type: 'model', diff --git a/client/src/components/Chat/Menus/Endpoints/MenuItems.tsx b/client/src/components/Chat/Menus/Endpoints/MenuItems.tsx index f5ff608285..aa50f274d3 100644 --- a/client/src/components/Chat/Menus/Endpoints/MenuItems.tsx +++ b/client/src/components/Chat/Menus/Endpoints/MenuItems.tsx @@ -1,47 +1,57 @@ import type { FC } from 'react'; import { Close } from '@radix-ui/react-popover'; -import { EModelEndpoint, alternateName } from 'librechat-data-provider'; +import { + EModelEndpoint, + alternateName, + PermissionTypes, + Permissions, +} from 'librechat-data-provider'; import { useGetEndpointsQuery } from 'librechat-data-provider/react-query'; import MenuSeparator from '../UI/MenuSeparator'; import { getEndpointField } from '~/utils'; +import { useHasAccess } from '~/hooks'; import MenuItem from './MenuItem'; const EndpointItems: FC<{ - endpoints: EModelEndpoint[]; + endpoints: Array; selected: EModelEndpoint | ''; -}> = ({ endpoints, selected }) => { +}> = ({ endpoints = [], selected }) => { + const hasAccessToAgents = useHasAccess({ + permissionType: PermissionTypes.AGENTS, + permission: Permissions.USE, + }); const { data: endpointsConfig } = useGetEndpointsQuery(); return ( <> - {endpoints && - endpoints.map((endpoint, i) => { - if (!endpoint) { - return null; - } else if (!endpointsConfig?.[endpoint]) { - return null; - } - const userProvidesKey: boolean | null | undefined = getEndpointField( - endpointsConfig, - endpoint, - 'userProvide', - ); - return ( - -
- - {i !== endpoints.length - 1 && } -
- - ); - })} + {endpoints.map((endpoint, i) => { + if (!endpoint) { + return null; + } else if (!endpointsConfig?.[endpoint]) { + return null; + } + + if (endpoint === EModelEndpoint.agents && !hasAccessToAgents) { + return null; + } + const userProvidesKey: boolean | null | undefined = + getEndpointField(endpointsConfig, endpoint, 'userProvide') ?? false; + return ( + +
+ + {i !== endpoints.length - 1 && } +
+ + ); + })} ); }; diff --git a/client/src/components/Chat/Messages/Content/ContentParts.tsx b/client/src/components/Chat/Messages/Content/ContentParts.tsx index c2018d1905..a59093e160 100644 --- a/client/src/components/Chat/Messages/Content/ContentParts.tsx +++ b/client/src/components/Chat/Messages/Content/ContentParts.tsx @@ -4,12 +4,14 @@ import { ContentTypes } from 'librechat-data-provider'; import type { TMessageContentParts, TAttachment, Agents } from 'librechat-data-provider'; import EditTextPart from './Parts/EditTextPart'; import { mapAttachments } from '~/utils/map'; +import { MessageContext } from '~/Providers'; import store from '~/store'; import Part from './Part'; type ContentPartsProps = { content: Array | undefined; messageId: string; + conversationId?: string | null; attachments?: TAttachment[]; isCreatedByUser: boolean; isLast: boolean; @@ -27,6 +29,7 @@ const ContentParts = memo( ({ content, messageId, + conversationId, attachments, isCreatedByUser, isLast, @@ -79,15 +82,23 @@ const ContentParts = memo( const attachments = attachmentMap[toolCallId]; return ( - + + + ); })} diff --git a/client/src/components/Chat/Messages/Content/Markdown.tsx b/client/src/components/Chat/Messages/Content/Markdown.tsx index 5ef21c79d0..59cbc70481 100644 --- a/client/src/components/Chat/Messages/Content/Markdown.tsx +++ b/client/src/components/Chat/Messages/Content/Markdown.tsx @@ -1,4 +1,4 @@ -import React, { memo, useMemo } from 'react'; +import React, { memo, useMemo, useRef, useEffect } from 'react'; import remarkGfm from 'remark-gfm'; import remarkMath from 'remark-math'; import supersub from 'remark-supersub'; @@ -10,10 +10,10 @@ import remarkDirective from 'remark-directive'; import type { Pluggable } from 'unified'; import { Artifact, artifactPlugin } from '~/components/Artifacts/Artifact'; import { langSubset, preprocessLaTeX, handleDoubleClick } from '~/utils'; +import { useToastContext, CodeBlockProvider, useCodeBlockContext } from '~/Providers'; import CodeBlock from '~/components/Messages/Content/CodeBlock'; import { useFileDownload } from '~/data-provider'; import useLocalize from '~/hooks/useLocalize'; -import { useToastContext } from '~/Providers'; import store from '~/store'; type TCodeProps = { @@ -25,6 +25,32 @@ type TCodeProps = { export const code: React.ElementType = memo(({ className, children }: TCodeProps) => { const match = /language-(\w+)/.exec(className ?? ''); const lang = match && match[1]; + const isMath = lang === 'math'; + const isSingleLine = typeof children === 'string' && children.split('\n').length === 1; + + const { getNextIndex, resetCounter } = useCodeBlockContext(); + const blockIndex = useRef(getNextIndex(isMath || isSingleLine)).current; + + useEffect(() => { + resetCounter(); + }, [children, resetCounter]); + + if (isMath) { + return children; + } else if (isSingleLine) { + return ( + + {children} + + ); + } else { + return ; + } +}); + +export const codeNoExecution: React.ElementType = memo(({ className, children }: TCodeProps) => { + const match = /language-(\w+)/.exec(className ?? ''); + const lang = match && match[1]; if (lang === 'math') { return children; @@ -35,7 +61,7 @@ export const code: React.ElementType = memo(({ className, children }: TCodeProps ); } else { - return ; + return ; } }); @@ -45,7 +71,11 @@ export const a: React.ElementType = memo( const { showToast } = useToastContext(); const localize = useLocalize(); - const { file_id, filename, filepath } = useMemo(() => { + const { + file_id = '', + filename = '', + filepath, + } = useMemo(() => { const pattern = new RegExp(`(?:files|outputs)/${user?.id}/([^\\s]+)`); const match = href.match(pattern); if (match && match[0]) { @@ -164,25 +194,27 @@ const Markdown = memo(({ content = '', showCursor, isLatestMessage }: TContentPr : [supersub, remarkGfm, [remarkMath, { singleDollarTextMath: true }]]; return ( - + - {isLatestMessage && showCursor === true ? currentContent + cursor : currentContent} - + > + {isLatestMessage && showCursor === true ? currentContent + cursor : currentContent} + + ); }); diff --git a/client/src/components/Chat/Messages/Content/MarkdownLite.tsx b/client/src/components/Chat/Messages/Content/MarkdownLite.tsx index 0996f7f740..1571d23448 100644 --- a/client/src/components/Chat/Messages/Content/MarkdownLite.tsx +++ b/client/src/components/Chat/Messages/Content/MarkdownLite.tsx @@ -6,40 +6,51 @@ import supersub from 'remark-supersub'; import ReactMarkdown from 'react-markdown'; import rehypeHighlight from 'rehype-highlight'; import type { PluggableList } from 'unified'; +import { code, codeNoExecution, a, p } from './Markdown'; +import { CodeBlockProvider } from '~/Providers'; import { langSubset } from '~/utils'; -import { code, a, p } from './Markdown'; -const MarkdownLite = memo(({ content = '' }: { content?: string }) => { - const rehypePlugins: PluggableList = [ - [rehypeKatex, { output: 'mathml' }], - [ - rehypeHighlight, - { - detect: true, - ignoreMissing: true, - subset: langSubset, - }, - ], - ]; - - return ( - { + const rehypePlugins: PluggableList = [ + [rehypeKatex, { output: 'mathml' }], + [ + rehypeHighlight, { - code, - a, - p, - } as { - [nodeType: string]: React.ElementType; - } - } - > - {content} - - ); -}); + detect: true, + ignoreMissing: true, + subset: langSubset, + }, + ], + ]; + + return ( + + + {content} + + + ); + }, +); export default MarkdownLite; diff --git a/client/src/components/Chat/Messages/Content/Part.tsx b/client/src/components/Chat/Messages/Content/Part.tsx index 83241a4211..f64ab6f361 100644 --- a/client/src/components/Chat/Messages/Content/Part.tsx +++ b/client/src/components/Chat/Messages/Content/Part.tsx @@ -21,143 +21,130 @@ type PartProps = { part?: TMessageContentParts; isSubmitting: boolean; showCursor: boolean; - messageId: string; isCreatedByUser: boolean; attachments?: TAttachment[]; }; -const Part = memo( - ({ part, isSubmitting, attachments, showCursor, messageId, isCreatedByUser }: PartProps) => { - attachments && console.log(attachments); - if (!part) { +const Part = memo(({ part, isSubmitting, attachments, showCursor, isCreatedByUser }: PartProps) => { + if (!part) { + return null; + } + + if (part.type === ContentTypes.ERROR) { + return ; + } else if (part.type === ContentTypes.TEXT) { + const text = typeof part.text === 'string' ? part.text : part.text.value; + + if (typeof text !== 'string') { + return null; + } + if (part.tool_call_ids != null && !text) { + return null; + } + return ( + + + + ); + } else if (part.type === ContentTypes.TOOL_CALL) { + const toolCall = part[ContentTypes.TOOL_CALL]; + + if (!toolCall) { return null; } - if (part.type === ContentTypes.ERROR) { - return ; - } else if (part.type === ContentTypes.TEXT) { - const text = typeof part.text === 'string' ? part.text : part.text.value; - - if (typeof text !== 'string') { - return null; - } - if (part.tool_call_ids != null && !text) { - return null; - } + const isToolCall = + 'args' in toolCall && (!toolCall.type || toolCall.type === ToolCallTypes.TOOL_CALL); + if (isToolCall && toolCall.name === Tools.execute_code) { return ( - - - + ); - } else if (part.type === ContentTypes.TOOL_CALL) { - const toolCall = part[ContentTypes.TOOL_CALL]; - - if (!toolCall) { + } else if (isToolCall) { + return ( + + ); + } else if (toolCall.type === ToolCallTypes.CODE_INTERPRETER) { + const code_interpreter = toolCall[ToolCallTypes.CODE_INTERPRETER]; + return ( + + ); + } else if ( + toolCall.type === ToolCallTypes.RETRIEVAL || + toolCall.type === ToolCallTypes.FILE_SEARCH + ) { + return ( + + ); + } else if ( + toolCall.type === ToolCallTypes.FUNCTION && + ToolCallTypes.FUNCTION in toolCall && + imageGenTools.has(toolCall.function.name) + ) { + return ( + + ); + } else if (toolCall.type === ToolCallTypes.FUNCTION && ToolCallTypes.FUNCTION in toolCall) { + if (isImageVisionTool(toolCall)) { + if (isSubmitting && showCursor) { + return ( + + + + ); + } return null; } - const isToolCall = - 'args' in toolCall && (!toolCall.type || toolCall.type === ToolCallTypes.TOOL_CALL); - if (isToolCall && toolCall.name === Tools.execute_code) { - return ( - - ); - } else if (isToolCall) { - return ( - - ); - } else if (toolCall.type === ToolCallTypes.CODE_INTERPRETER) { - const code_interpreter = toolCall[ToolCallTypes.CODE_INTERPRETER]; - return ( - - ); - } else if ( - toolCall.type === ToolCallTypes.RETRIEVAL || - toolCall.type === ToolCallTypes.FILE_SEARCH - ) { - return ( - - ); - } else if ( - toolCall.type === ToolCallTypes.FUNCTION && - ToolCallTypes.FUNCTION in toolCall && - imageGenTools.has(toolCall.function.name) - ) { - return ( - - ); - } else if (toolCall.type === ToolCallTypes.FUNCTION && ToolCallTypes.FUNCTION in toolCall) { - if (isImageVisionTool(toolCall)) { - if (isSubmitting && showCursor) { - return ( - - - - ); - } - return null; - } - - return ( - - ); - } - } else if (part.type === ContentTypes.IMAGE_FILE) { - const imageFile = part[ContentTypes.IMAGE_FILE]; - const height = imageFile.height ?? 1920; - const width = imageFile.width ?? 1080; return ( - ); } + } else if (part.type === ContentTypes.IMAGE_FILE) { + const imageFile = part[ContentTypes.IMAGE_FILE]; + const height = imageFile.height ?? 1920; + const width = imageFile.width ?? 1080; + return ( + + ); + } - return null; - }, -); + return null; +}); export default Part; diff --git a/client/src/components/Chat/Messages/Content/Parts/Attachment.tsx b/client/src/components/Chat/Messages/Content/Parts/Attachment.tsx new file mode 100644 index 0000000000..785a37a343 --- /dev/null +++ b/client/src/components/Chat/Messages/Content/Parts/Attachment.tsx @@ -0,0 +1,19 @@ +import { imageExtRegex } from 'librechat-data-provider'; +import type { TAttachment, TFile, TAttachmentMetadata } from 'librechat-data-provider'; +import Image from '~/components/Chat/Messages/Content/Image'; + +export default function Attachment({ attachment }: { attachment?: TAttachment }) { + if (!attachment) { + return null; + } + const { width, height, filepath = null } = attachment as TFile & TAttachmentMetadata; + const isImage = + imageExtRegex.test(attachment.filename) && width != null && height != null && filepath != null; + + if (isImage) { + return ( + + ); + } + return null; +} diff --git a/client/src/components/Chat/Messages/Content/Parts/ExecuteCode.tsx b/client/src/components/Chat/Messages/Content/Parts/ExecuteCode.tsx index 559fc3cd2c..49a15fc71c 100644 --- a/client/src/components/Chat/Messages/Content/Parts/ExecuteCode.tsx +++ b/client/src/components/Chat/Messages/Content/Parts/ExecuteCode.tsx @@ -1,12 +1,11 @@ import React, { useMemo, useState } from 'react'; import { useRecoilValue } from 'recoil'; -import { CodeInProgress } from './CodeProgress'; -import { imageExtRegex } from 'librechat-data-provider'; -import type { TFile, TAttachment, TAttachmentMetadata } from 'librechat-data-provider'; +import type { TAttachment } from 'librechat-data-provider'; import ProgressText from '~/components/Chat/Messages/Content/ProgressText'; import FinishedIcon from '~/components/Chat/Messages/Content/FinishedIcon'; import MarkdownLite from '~/components/Chat/Messages/Content/MarkdownLite'; -import Image from '~/components/Chat/Messages/Content/Image'; +import { CodeInProgress } from './CodeProgress'; +import Attachment from './Attachment'; import LogContent from './LogContent'; import { useProgress } from '~/hooks'; import store from '~/store'; @@ -86,7 +85,10 @@ export default function ExecuteCode({
{showCode && (
- + {output.length > 0 && (
)} - {attachments?.map((attachment, index) => { - const { width, height, filepath } = attachment as TFile & TAttachmentMetadata; - const isImage = - imageExtRegex.test(attachment.filename) && - width != null && - height != null && - filepath != null; - if (isImage) { - return ( - - ); - } - })} + {attachments?.map((attachment, index) => ( + + ))} ); } diff --git a/client/src/components/Chat/Messages/Content/Parts/LogContent.tsx b/client/src/components/Chat/Messages/Content/Parts/LogContent.tsx index 9f9eceaa4f..9a6e3fc99d 100644 --- a/client/src/components/Chat/Messages/Content/Parts/LogContent.tsx +++ b/client/src/components/Chat/Messages/Content/Parts/LogContent.tsx @@ -1,17 +1,26 @@ import { isAfter } from 'date-fns'; import React, { useMemo } from 'react'; import { imageExtRegex } from 'librechat-data-provider'; -import type { TAttachment } from 'librechat-data-provider'; +import type { TFile, TAttachment, TAttachmentMetadata } from 'librechat-data-provider'; +import Image from '~/components/Chat/Messages/Content/Image'; import { useLocalize } from '~/hooks'; import LogLink from './LogLink'; interface LogContentProps { output?: string; + renderImages?: boolean; attachments?: TAttachment[]; } -const LogContent: React.FC = ({ output = '', attachments }) => { +type ImageAttachment = TFile & + TAttachmentMetadata & { + height: number; + width: number; + }; + +const LogContent: React.FC = ({ output = '', renderImages, attachments }) => { const localize = useLocalize(); + const processedContent = useMemo(() => { if (!output) { return ''; @@ -21,8 +30,29 @@ const LogContent: React.FC = ({ output = '', attachments }) => return parts[0].trim(); }, [output]); - const nonImageAttachments = - attachments?.filter((file) => !imageExtRegex.test(file.filename)) || []; + const { imageAttachments, nonImageAttachments } = useMemo(() => { + const imageAtts: ImageAttachment[] = []; + const nonImageAtts: TAttachment[] = []; + + attachments?.forEach((attachment) => { + const { width, height, filepath = null } = attachment as TFile & TAttachmentMetadata; + const isImage = + imageExtRegex.test(attachment.filename) && + width != null && + height != null && + filepath != null; + if (isImage) { + imageAtts.push(attachment as ImageAttachment); + } else { + nonImageAtts.push(attachment); + } + }); + + return { + imageAttachments: renderImages === true ? imageAtts : null, + nonImageAttachments: nonImageAtts, + }; + }, [attachments, renderImages]); const renderAttachment = (file: TAttachment) => { const now = new Date(); @@ -59,6 +89,18 @@ const LogContent: React.FC = ({ output = '', attachments }) => ))}
)} + {imageAttachments?.map((attachment, index) => { + const { width, height, filepath } = attachment; + return ( + + ); + })} ); }; diff --git a/client/src/components/Chat/Messages/Content/Parts/Text.tsx b/client/src/components/Chat/Messages/Content/Parts/Text.tsx index b52a0bfa52..5806f76aac 100644 --- a/client/src/components/Chat/Messages/Content/Parts/Text.tsx +++ b/client/src/components/Chat/Messages/Content/Parts/Text.tsx @@ -2,15 +2,14 @@ import { memo, useMemo, ReactElement } from 'react'; import { useRecoilValue } from 'recoil'; import MarkdownLite from '~/components/Chat/Messages/Content/MarkdownLite'; import Markdown from '~/components/Chat/Messages/Content/Markdown'; -import { useChatContext } from '~/Providers'; +import { useChatContext, useMessageContext } from '~/Providers'; import { cn } from '~/utils'; import store from '~/store'; type TextPartProps = { text: string; - isCreatedByUser: boolean; - messageId: string; showCursor: boolean; + isCreatedByUser: boolean; }; type ContentType = @@ -18,7 +17,8 @@ type ContentType = | ReactElement> | ReactElement; -const TextPart = memo(({ text, isCreatedByUser, messageId, showCursor }: TextPartProps) => { +const TextPart = memo(({ text, isCreatedByUser, showCursor }: TextPartProps) => { + const { messageId } = useMessageContext(); const { isSubmitting, latestMessage } = useChatContext(); const enableUserMsgMarkdown = useRecoilValue(store.enableUserMsgMarkdown); const showCursorState = useMemo(() => showCursor && isSubmitting, [showCursor, isSubmitting]); diff --git a/client/src/components/Chat/Messages/Content/ToolCall.tsx b/client/src/components/Chat/Messages/Content/ToolCall.tsx index 64b3210981..756fbd4878 100644 --- a/client/src/components/Chat/Messages/Content/ToolCall.tsx +++ b/client/src/components/Chat/Messages/Content/ToolCall.tsx @@ -1,9 +1,11 @@ import { useMemo } from 'react'; -import { actionDelimiter, actionDomainSeparator, Constants } from 'librechat-data-provider'; import * as Popover from '@radix-ui/react-popover'; +import { actionDelimiter, actionDomainSeparator, Constants } from 'librechat-data-provider'; +import type { TAttachment } from 'librechat-data-provider'; import useLocalize from '~/hooks/useLocalize'; import ProgressCircle from './ProgressCircle'; import InProgressCall from './InProgressCall'; +import Attachment from './Parts/Attachment'; import CancelledIcon from './CancelledIcon'; import ProgressText from './ProgressText'; import FinishedIcon from './FinishedIcon'; @@ -18,12 +20,14 @@ export default function ToolCall({ name, args: _args = '', output, + attachments, }: { initialProgress: number; isSubmitting: boolean; name: string; args: string | Record; output?: string | null; + attachments?: TAttachment[]; }) { const localize = useLocalize(); const progress = useProgress(initialProgress); @@ -106,6 +110,9 @@ export default function ToolCall({ /> )}
+ {attachments?.map((attachment, index) => ( + + ))} ); } diff --git a/client/src/components/Chat/Messages/Content/ToolPopover.tsx b/client/src/components/Chat/Messages/Content/ToolPopover.tsx index dbc203f7b6..97e5aefa41 100644 --- a/client/src/components/Chat/Messages/Content/ToolPopover.tsx +++ b/client/src/components/Chat/Messages/Content/ToolPopover.tsx @@ -33,7 +33,7 @@ export default function ToolPopover({
- {domain + {domain != null && domain ? localize('com_assistants_domain_info', domain) : localize('com_assistants_function_use', function_name)}
@@ -42,7 +42,7 @@ export default function ToolPopover({ {formatText(input)}
- {output && ( + {output != null && output && ( <>
{localize('com_ui_result')} diff --git a/client/src/components/Chat/Messages/MessageParts.tsx b/client/src/components/Chat/Messages/MessageParts.tsx index 52a2dbe2dd..0e9d7cc40a 100644 --- a/client/src/components/Chat/Messages/MessageParts.tsx +++ b/client/src/components/Chat/Messages/MessageParts.tsx @@ -82,11 +82,12 @@ export default function Message(props: TMessageProps) {
} - messageId={message.messageId} - isCreatedByUser={message.isCreatedByUser} isLast={isLast} isSubmitting={isSubmitting} + messageId={message.messageId} + isCreatedByUser={message.isCreatedByUser} + conversationId={conversation?.conversationId} + content={message.content as Array} />
diff --git a/client/src/components/Chat/Messages/ui/MessageRender.tsx b/client/src/components/Chat/Messages/ui/MessageRender.tsx index bf237dea44..f77d9acafe 100644 --- a/client/src/components/Chat/Messages/ui/MessageRender.tsx +++ b/client/src/components/Chat/Messages/ui/MessageRender.tsx @@ -9,6 +9,7 @@ import HoverButtons from '~/components/Chat/Messages/HoverButtons'; import Icon from '~/components/Chat/Messages/MessageIcon'; import { Plugin } from '~/components/Messages/Content'; import SubRow from '~/components/Chat/Messages/SubRow'; +import { MessageContext } from '~/Providers'; import { useMessageActions } from '~/hooks'; import { cn, logger } from '~/utils'; import store from '~/store'; @@ -59,9 +60,10 @@ const MessageRender = memo( const fontSize = useRecoilValue(store.fontSize); const handleRegenerateMessage = useCallback(() => regenerateMessage(), [regenerateMessage]); const { isCreatedByUser, error, unfinished } = msg ?? {}; + const hasNoChildren = !(msg?.children?.length ?? 0); const isLast = useMemo( - () => !msg?.children?.length && (msg?.depth === latestMessage?.depth || msg?.depth === -1), - [msg?.children, msg?.depth, latestMessage?.depth], + () => hasNoChildren && (msg?.depth === latestMessage?.depth || msg?.depth === -1), + [hasNoChildren, msg?.depth, latestMessage?.depth], ); if (!msg) { @@ -122,24 +124,31 @@ const MessageRender = memo(

{messageLabel}

- {msg.plugin && } - ({}))} - /> + + {msg.plugin && } + ({}))} + /> +
- {!msg.children?.length && (isSubmittingFamily === true || isSubmitting) ? ( + {hasNoChildren && (isSubmittingFamily === true || isSubmitting) ? ( ) : ( diff --git a/client/src/components/Endpoints/SaveAsPresetDialog.tsx b/client/src/components/Endpoints/SaveAsPresetDialog.tsx index 338c2b8729..b827a2d8c2 100644 --- a/client/src/components/Endpoints/SaveAsPresetDialog.tsx +++ b/client/src/components/Endpoints/SaveAsPresetDialog.tsx @@ -28,7 +28,7 @@ const SaveAsPresetDialog = ({ open, onOpenChange, preset }: TEditPresetProps) => createPresetMutation.mutate(_preset, { onSuccess: () => { showToast({ - message: `${toastTitle} ${localize('com_endpoint_preset_saved')}`, + message: `${toastTitle} ${localize('com_ui_saved')}`, }); onOpenChange(false); // Close the dialog on success }, diff --git a/client/src/components/Messages/Content/CodeBlock.tsx b/client/src/components/Messages/Content/CodeBlock.tsx index 7f1b4e1b39..df80c90ecb 100644 --- a/client/src/components/Messages/Content/CodeBlock.tsx +++ b/client/src/components/Messages/Content/CodeBlock.tsx @@ -1,81 +1,133 @@ import copy from 'copy-to-clipboard'; import { InfoIcon } from 'lucide-react'; -import React, { useRef, useState, RefObject } from 'react'; +import { Tools } from 'librechat-data-provider'; +import React, { useRef, useState, useMemo, useEffect } from 'react'; +import type { CodeBarProps } from '~/common'; +import LogContent from '~/components/Chat/Messages/Content/Parts/LogContent'; +import ResultSwitcher from '~/components/Messages/Content/ResultSwitcher'; +import { useToolCallsMapContext, useMessageContext } from '~/Providers'; +import RunCode from '~/components/Messages/Content/RunCode'; import Clipboard from '~/components/svg/Clipboard'; import CheckMark from '~/components/svg/CheckMark'; import useLocalize from '~/hooks/useLocalize'; import cn from '~/utils/cn'; -type CodeBarProps = { - lang: string; - codeRef: RefObject; - plugin?: boolean; - error?: boolean; -}; - -type CodeBlockProps = Pick & { +type CodeBlockProps = Pick< + CodeBarProps, + 'lang' | 'plugin' | 'error' | 'allowExecution' | 'blockIndex' +> & { codeChildren: React.ReactNode; classProp?: string; }; -const CodeBar: React.FC = React.memo(({ lang, codeRef, error, plugin = null }) => { - const localize = useLocalize(); - const [isCopied, setIsCopied] = useState(false); - return ( -
- {lang} - {plugin === true ? ( - - ) : ( - - )} -
- ); -}); + setTimeout(() => { + setIsCopied(false); + }, 3000); + } + }} + > + {isCopied ? ( + <> + + {error === true ? '' : localize('com_ui_copied')} + + ) : ( + <> + + {error === true ? '' : localize('com_ui_copy_code')} + + )} + +
+ )} +
+ ); + }, +); const CodeBlock: React.FC = ({ lang, + blockIndex, codeChildren, classProp = '', + allowExecution = true, plugin = null, error, }) => { const codeRef = useRef(null); + const toolCallsMap = useToolCallsMapContext(); + const { messageId, partIndex } = useMessageContext(); + const key = allowExecution + ? `${messageId}_${partIndex ?? 0}_${blockIndex ?? 0}_${Tools.execute_code}` + : ''; + const [currentIndex, setCurrentIndex] = useState(0); + + const fetchedToolCalls = toolCallsMap?.[key]; + const [toolCalls, setToolCalls] = useState(toolCallsMap?.[key] ?? null); + + useEffect(() => { + if (fetchedToolCalls) { + setToolCalls(fetchedToolCalls); + setCurrentIndex(fetchedToolCalls.length - 1); + } + }, [fetchedToolCalls]); + + const currentToolCall = useMemo(() => toolCalls?.[currentIndex], [toolCalls, currentIndex]); + + const next = () => { + if (!toolCalls) { + return; + } + if (currentIndex < toolCalls.length - 1) { + setCurrentIndex(currentIndex + 1); + } + }; + + const previous = () => { + if (currentIndex > 0) { + setCurrentIndex(currentIndex - 1); + } + }; + const isNonCode = !!(plugin === true || error === true); const language = isNonCode ? 'json' : lang; return (
- +
= ({ {codeChildren}
+ {allowExecution === true && toolCalls && toolCalls.length > 0 && ( + <> +
+
+ 
+                
+
+
+
+ {toolCalls.length > 1 && ( + + )} + + )}
); }; diff --git a/client/src/components/Messages/Content/ResultSwitcher.tsx b/client/src/components/Messages/Content/ResultSwitcher.tsx new file mode 100644 index 0000000000..eb8c59b568 --- /dev/null +++ b/client/src/components/Messages/Content/ResultSwitcher.tsx @@ -0,0 +1,69 @@ +interface ResultSwitcherProps { + currentIndex: number; + totalCount: number; + onPrevious: () => void; + onNext: () => void; +} + +const ResultSwitcher: React.FC = ({ + currentIndex, + totalCount, + onPrevious, + onNext, +}) => { + if (totalCount <= 1) { + return null; + } + + return ( +
+ + + {currentIndex + 1} / {totalCount} + + +
+ ); +}; + +export default ResultSwitcher; diff --git a/client/src/components/Messages/Content/RunCode.tsx b/client/src/components/Messages/Content/RunCode.tsx new file mode 100644 index 0000000000..b219083468 --- /dev/null +++ b/client/src/components/Messages/Content/RunCode.tsx @@ -0,0 +1,109 @@ +import debounce from 'lodash/debounce'; +import { Tools, AuthType } from 'librechat-data-provider'; +import { TerminalSquareIcon, Loader } from 'lucide-react'; +import React, { useMemo, useCallback, useEffect } from 'react'; +import type { CodeBarProps } from '~/common'; +import { useVerifyAgentToolAuth, useToolCallMutation } from '~/data-provider'; +import ApiKeyDialog from '~/components/SidePanel/Agents/Code/ApiKeyDialog'; +import { useLocalize, useCodeApiKeyForm } from '~/hooks'; +import { useMessageContext } from '~/Providers'; +import { cn, normalizeLanguage } from '~/utils'; +import { useToastContext } from '~/Providers'; + +const RunCode: React.FC = React.memo(({ lang, codeRef, blockIndex }) => { + const localize = useLocalize(); + const { showToast } = useToastContext(); + const execute = useToolCallMutation(Tools.execute_code, { + onError: () => { + showToast({ message: localize('com_ui_run_code_error'), status: 'error' }); + }, + }); + + const { messageId, conversationId, partIndex } = useMessageContext(); + const normalizedLang = useMemo(() => normalizeLanguage(lang), [lang]); + const { data } = useVerifyAgentToolAuth({ toolId: Tools.execute_code }); + const authType = useMemo(() => data?.message ?? false, [data?.message]); + const isAuthenticated = useMemo(() => data?.authenticated ?? false, [data?.authenticated]); + const { methods, onSubmit, isDialogOpen, setIsDialogOpen, handleRevokeApiKey } = + useCodeApiKeyForm({}); + + const handleExecute = useCallback(async () => { + if (!isAuthenticated) { + setIsDialogOpen(true); + return; + } + const codeString: string = codeRef.current?.textContent ?? ''; + if ( + typeof codeString !== 'string' || + codeString.length === 0 || + typeof normalizedLang !== 'string' || + normalizedLang.length === 0 + ) { + return; + } + + execute.mutate({ + partIndex, + messageId, + blockIndex, + conversationId: conversationId ?? '', + lang: normalizedLang, + code: codeString, + }); + }, [ + codeRef, + execute, + partIndex, + messageId, + blockIndex, + conversationId, + normalizedLang, + setIsDialogOpen, + isAuthenticated, + ]); + + const debouncedExecute = useMemo( + () => debounce(handleExecute, 1000, { leading: true }), + [handleExecute], + ); + + useEffect(() => { + return () => { + debouncedExecute.cancel(); + }; + }, [debouncedExecute]); + + if (typeof normalizedLang !== 'string' || normalizedLang.length === 0) { + return null; + } + + return ( + <> + + + + ); +}); + +export default RunCode; diff --git a/client/src/components/Messages/ContentRender.tsx b/client/src/components/Messages/ContentRender.tsx index 1d33ae5b63..5033b9a291 100644 --- a/client/src/components/Messages/ContentRender.tsx +++ b/client/src/components/Messages/ContentRender.tsx @@ -129,16 +129,17 @@ const ContentRender = memo(
} - messageId={msg.messageId} - isCreatedByUser={msg.isCreatedByUser} - isLast={isLast} - isSubmitting={isSubmitting} edit={edit} + isLast={isLast} enterEdit={enterEdit} siblingIdx={siblingIdx} + messageId={msg.messageId} + isSubmitting={isSubmitting} setSiblingIdx={setSiblingIdx} attachments={msg.attachments} + isCreatedByUser={msg.isCreatedByUser} + conversationId={conversation?.conversationId} + content={msg.content as Array} />
diff --git a/client/src/components/Prompts/AdminSettings.tsx b/client/src/components/Prompts/AdminSettings.tsx index 7c1f00daed..4ae85a1940 100644 --- a/client/src/components/Prompts/AdminSettings.tsx +++ b/client/src/components/Prompts/AdminSettings.tsx @@ -29,17 +29,19 @@ const LabelController: React.FC = ({ setValue, }) => (
- + = ({ {...field} checked={field.value} onCheckedChange={field.onChange} - value={field?.value?.toString()} + value={field.value.toString()} /> )} /> @@ -61,7 +63,7 @@ const AdminSettings = () => { const { showToast } = useToastContext(); const { mutate, isLoading } = useUpdatePromptPermissionsMutation({ onSuccess: () => { - showToast({ status: 'success', message: localize('com_endpoint_preset_saved') }); + showToast({ status: 'success', message: localize('com_ui_saved') }); }, onError: () => { showToast({ status: 'error', message: localize('com_ui_error_save_admin_settings') }); diff --git a/client/src/components/Prompts/Groups/VariableForm.tsx b/client/src/components/Prompts/Groups/VariableForm.tsx index 74103d7c46..eb8b6683ac 100644 --- a/client/src/components/Prompts/Groups/VariableForm.tsx +++ b/client/src/components/Prompts/Groups/VariableForm.tsx @@ -14,9 +14,9 @@ import { replaceSpecialVars, extractVariableInfo, } from '~/utils'; +import { codeNoExecution } from '~/components/Chat/Messages/Content/Markdown'; import { useAuthContext, useLocalize, useSubmitMessage } from '~/hooks'; import { TextareaAutosize, InputCombobox } from '~/components/ui'; -import { code } from '~/components/Chat/Messages/Content/Markdown'; type FieldType = 'text' | 'select'; @@ -143,12 +143,16 @@ export default function VariableForm({
{generateHighlightedMarkdown()} diff --git a/client/src/components/Prompts/PromptDetails.tsx b/client/src/components/Prompts/PromptDetails.tsx index 0f2a64a71c..4dec2dd4fe 100644 --- a/client/src/components/Prompts/PromptDetails.tsx +++ b/client/src/components/Prompts/PromptDetails.tsx @@ -6,7 +6,7 @@ import remarkMath from 'remark-math'; import supersub from 'remark-supersub'; import rehypeHighlight from 'rehype-highlight'; import type { TPromptGroup } from 'librechat-data-provider'; -import { code } from '~/components/Chat/Messages/Content/Markdown'; +import { codeNoExecution } from '~/components/Chat/Messages/Content/Markdown'; import { useLocalize, useAuthContext } from '~/hooks'; import CategoryIcon from './Groups/CategoryIcon'; import PromptVariables from './PromptVariables'; @@ -50,12 +50,20 @@ const PromptDetails = ({ group }: { group?: TPromptGroup }) => {
{mainText} diff --git a/client/src/components/Prompts/PromptEditor.tsx b/client/src/components/Prompts/PromptEditor.tsx index e47cbc0b11..d01edad32c 100644 --- a/client/src/components/Prompts/PromptEditor.tsx +++ b/client/src/components/Prompts/PromptEditor.tsx @@ -9,8 +9,8 @@ import rehypeKatex from 'rehype-katex'; import remarkMath from 'remark-math'; import supersub from 'remark-supersub'; import ReactMarkdown from 'react-markdown'; +import { codeNoExecution } from '~/components/Chat/Messages/Content/Markdown'; import AlwaysMakeProd from '~/components/Prompts/Groups/AlwaysMakeProd'; -import { code } from '~/components/Chat/Messages/Content/Markdown'; import { SaveIcon, CrossIcon } from '~/components/svg'; import { TextareaAutosize } from '~/components/ui'; import { PromptVariableGfm } from './Markdown'; @@ -75,7 +75,7 @@ const PromptEditor: React.FC = ({ name, isEditing, setIsEditing }) => { role="button" className={cn( 'min-h-[8rem] w-full rounded-b-lg border border-border-medium p-4 transition-all duration-150', - { 'bg-surface-secondary-alt cursor-pointer hover:bg-surface-tertiary': !isEditing }, + { 'cursor-pointer bg-surface-secondary-alt hover:bg-surface-tertiary': !isEditing }, )} onClick={() => !isEditing && setIsEditing(true)} onKeyDown={(e) => { @@ -107,9 +107,12 @@ const PromptEditor: React.FC = ({ name, isEditing, setIsEditing }) => { /> ) : ( {field.value} diff --git a/client/src/components/Prompts/PromptVariables.tsx b/client/src/components/Prompts/PromptVariables.tsx index 23e9540cb2..8fd25d3095 100644 --- a/client/src/components/Prompts/PromptVariables.tsx +++ b/client/src/components/Prompts/PromptVariables.tsx @@ -53,6 +53,7 @@ const PromptVariables = ({ ) : (
+ {/** @ts-ignore */} {localize('com_ui_variables_info')} @@ -68,6 +69,7 @@ const PromptVariables = ({ {'\u00A0'} + {/** @ts-ignore */} {localize('com_ui_special_variables_info')} @@ -79,6 +81,7 @@ const PromptVariables = ({ {'\u00A0'} + {/** @ts-ignore */} {localize('com_ui_dropdown_variables_info')} diff --git a/client/src/components/Share/Message.tsx b/client/src/components/Share/Message.tsx index 7bc112ba8c..9b72ede5aa 100644 --- a/client/src/components/Share/Message.tsx +++ b/client/src/components/Share/Message.tsx @@ -6,6 +6,7 @@ import SearchContent from '~/components/Chat/Messages/Content/SearchContent'; import SiblingSwitch from '~/components/Chat/Messages/SiblingSwitch'; import { Plugin } from '~/components/Messages/Content'; import SubRow from '~/components/Chat/Messages/SubRow'; +import { MessageContext } from '~/Providers'; // eslint-disable-next-line import/no-cycle import MultiMessage from './MultiMessage'; import { cn } from '~/utils'; @@ -31,10 +32,10 @@ export default function Message(props: TMessageProps) { const { text = '', children, - messageId = null, - isCreatedByUser = true, error = false, + messageId = '', unfinished = false, + isCreatedByUser = true, } = message; let messageLabel = ''; @@ -64,26 +65,33 @@ export default function Message(props: TMessageProps) {
{messageLabel}
- {/* Legacy Plugins */} - {message.plugin && } - {message.content ? ( - - ) : ( - ({})} - text={text} - message={message} - isSubmitting={false} - enterEdit={() => ({})} - unfinished={!!unfinished} - isCreatedByUser={isCreatedByUser} - siblingIdx={siblingIdx ?? 0} - setSiblingIdx={setSiblingIdx ?? (() => ({}))} - /> - )} + + {/* Legacy Plugins */} + {message.plugin && } + {message.content ? ( + + ) : ( + ({})} + text={text || ''} + message={message} + isSubmitting={false} + enterEdit={() => ({})} + unfinished={unfinished} + siblingIdx={siblingIdx ?? 0} + isCreatedByUser={isCreatedByUser} + setSiblingIdx={setSiblingIdx ?? (() => ({}))} + /> + )} +
diff --git a/client/src/components/SidePanel/Agents/AdminSettings.tsx b/client/src/components/SidePanel/Agents/AdminSettings.tsx new file mode 100644 index 0000000000..dcc48212df --- /dev/null +++ b/client/src/components/SidePanel/Agents/AdminSettings.tsx @@ -0,0 +1,163 @@ +import { useMemo, useEffect } from 'react'; +import { ShieldEllipsis } from 'lucide-react'; +import { useForm, Controller } from 'react-hook-form'; +import { Permissions, SystemRoles, roleDefaults, PermissionTypes } from 'librechat-data-provider'; +import type { Control, UseFormSetValue, UseFormGetValues } from 'react-hook-form'; +import { OGDialog, OGDialogTitle, OGDialogContent, OGDialogTrigger } from '~/components/ui'; +import { useUpdateAgentPermissionsMutation } from '~/data-provider'; +import { useLocalize, useAuthContext } from '~/hooks'; +import { Button, Switch } from '~/components/ui'; +import { useToastContext } from '~/Providers'; + +type FormValues = Record; + +type LabelControllerProps = { + label: string; + agentPerm: Permissions; + control: Control; + setValue: UseFormSetValue; + getValues: UseFormGetValues; +}; + +const defaultValues = roleDefaults[SystemRoles.USER]; + +const LabelController: React.FC = ({ + control, + agentPerm, + label, + getValues, + setValue, +}) => ( +
+ + ( + + )} + /> +
+); + +const AdminSettings = () => { + const localize = useLocalize(); + const { user, roles } = useAuthContext(); + const { showToast } = useToastContext(); + const { mutate, isLoading } = useUpdateAgentPermissionsMutation({ + onSuccess: () => { + showToast({ status: 'success', message: localize('com_ui_saved') }); + }, + onError: () => { + showToast({ status: 'error', message: localize('com_ui_error_save_admin_settings') }); + }, + }); + + const { + reset, + control, + setValue, + getValues, + handleSubmit, + formState: { isSubmitting }, + } = useForm({ + mode: 'onChange', + defaultValues: useMemo(() => { + if (roles?.[SystemRoles.USER]) { + return roles[SystemRoles.USER][PermissionTypes.AGENTS]; + } + + return defaultValues[PermissionTypes.AGENTS]; + }, [roles]), + }); + + useEffect(() => { + if (roles?.[SystemRoles.USER]?.[PermissionTypes.AGENTS]) { + reset(roles[SystemRoles.USER][PermissionTypes.AGENTS]); + } + }, [roles, reset]); + + if (user?.role !== SystemRoles.ADMIN) { + return null; + } + + const labelControllerData = [ + { + agentPerm: Permissions.SHARED_GLOBAL, + label: localize('com_ui_agents_allow_share_global'), + }, + { + agentPerm: Permissions.USE, + label: localize('com_ui_agents_allow_use'), + }, + { + agentPerm: Permissions.CREATE, + label: localize('com_ui_agents_allow_create'), + }, + ]; + + const onSubmit = (data: FormValues) => { + mutate({ roleName: SystemRoles.USER, updates: data }); + }; + + return ( + + + + + + {`${localize('com_ui_admin_settings')} - ${localize( + 'com_ui_agents', + )}`} + +
+ {labelControllerData.map(({ agentPerm, label }) => ( + + ))} +
+
+ +
+ + + + ); +}; + +export default AdminSettings; diff --git a/client/src/components/SidePanel/Agents/AgentConfig.tsx b/client/src/components/SidePanel/Agents/AgentConfig.tsx index fb14d7e5f3..798e75c1bd 100644 --- a/client/src/components/SidePanel/Agents/AgentConfig.tsx +++ b/client/src/components/SidePanel/Agents/AgentConfig.tsx @@ -1,24 +1,32 @@ import React, { useState, useMemo, useCallback } from 'react'; import { useQueryClient } from '@tanstack/react-query'; import { Controller, useWatch, useFormContext } from 'react-hook-form'; -import { QueryKeys, AgentCapabilities, EModelEndpoint, SystemRoles } from 'librechat-data-provider'; +import { + QueryKeys, + SystemRoles, + Permissions, + EModelEndpoint, + PermissionTypes, + AgentCapabilities, +} from 'librechat-data-provider'; import type { TConfig, TPlugin } from 'librechat-data-provider'; import type { AgentForm, AgentPanelProps } from '~/common'; import { cn, defaultTextProps, removeFocusOutlines, getEndpointField, getIconKey } from '~/utils'; import { useCreateAgentMutation, useUpdateAgentMutation } from '~/data-provider'; +import { useLocalize, useAuthContext, useHasAccess } from '~/hooks'; import { useToastContext, useFileMapContext } from '~/Providers'; import { icons } from '~/components/Chat/Menus/Endpoints/Icons'; import Action from '~/components/SidePanel/Builder/Action'; import { ToolSelectDialog } from '~/components/Tools'; -import { useLocalize, useAuthContext } from '~/hooks'; import { processAgentOption } from '~/utils'; +import AdminSettings from './AdminSettings'; import { Spinner } from '~/components/svg'; import DeleteButton from './DeleteButton'; import AgentAvatar from './AgentAvatar'; import FileSearch from './FileSearch'; import ShareAgent from './ShareAgent'; import AgentTool from './AgentTool'; -// import CodeForm from './Code/Form'; +import CodeForm from './Code/Form'; import { Panel } from '~/common'; const labelClass = 'mb-2 text-token-text-primary block font-medium'; @@ -55,6 +63,11 @@ export default function AgentConfig({ const tools = useWatch({ control, name: 'tools' }); const agent_id = useWatch({ control, name: 'id' }); + const hasAccessToShareAgents = useHasAccess({ + permissionType: PermissionTypes.AGENTS, + permission: Permissions.SHARED_GLOBAL, + }); + const toolsEnabled = useMemo( () => agentsConfig?.capabilities?.includes(AgentCapabilities.tools), [agentsConfig], @@ -263,7 +276,7 @@ export default function AgentConfig({ />
{/* Instructions */} -
+
@@ -275,7 +288,7 @@ export default function AgentConfig({