diff --git a/api/package.json b/api/package.json index 43d8609a8e..6fc209c1b1 100644 --- a/api/package.json +++ b/api/package.json @@ -43,7 +43,7 @@ "@langchain/core": "^0.2.18", "@langchain/google-genai": "^0.0.11", "@langchain/google-vertexai": "^0.0.17", - "@librechat/agents": "^1.4.1", + "@librechat/agents": "^1.4.2", "axios": "^1.3.4", "bcryptjs": "^2.4.3", "cheerio": "^1.0.0-rc.12", diff --git a/api/server/controllers/agents/request.js b/api/server/controllers/agents/request.js index 6480205979..842f2f68cc 100644 --- a/api/server/controllers/agents/request.js +++ b/api/server/controllers/agents/request.js @@ -22,7 +22,7 @@ const AgentController = async (req, res, next, initializeClient, addTitle) => { const sender = getResponseSender({ ...endpointOption, - model: endpointOption.modelOptions.model, + model: endpointOption.model_parameters.model, modelDisplayLabel, }); const newConvo = !conversationId; diff --git a/api/server/controllers/bedrock/client.js b/api/server/controllers/bedrock/client.js new file mode 100644 index 0000000000..0dcb1dda87 --- /dev/null +++ b/api/server/controllers/bedrock/client.js @@ -0,0 +1,16 @@ +const { EModelEndpoint } = require('librechat-data-provider'); +const AgentClient = require('~/server/controllers/agents/client'); +const { logger } = require('~/config'); + +class BedrockClient extends AgentClient { + constructor(options = {}) { + super(options); + this.options.endpoint = EModelEndpoint.bedrock; + } + + setOptions(options) { + logger.info('[api/server/controllers/bedrock/client.js] setOptions', options); + } +} + +module.exports = BedrockClient; diff --git a/api/server/index.js b/api/server/index.js index 3fa5778301..47ce354f2a 100644 --- a/api/server/index.js +++ b/api/server/index.js @@ -106,6 +106,7 @@ const startServer = async () => { app.use('/api/share', routes.share); app.use('/api/roles', routes.roles); app.use('/api/agents', routes.agents); + app.use('/api/bedrock', routes.bedrock); app.use('/api/tags', routes.tags); diff --git a/api/server/middleware/buildEndpointOption.js b/api/server/middleware/buildEndpointOption.js index 83e06d77c3..2b4ba40172 100644 --- a/api/server/middleware/buildEndpointOption.js +++ b/api/server/middleware/buildEndpointOption.js @@ -5,6 +5,7 @@ const assistants = require('~/server/services/Endpoints/assistants'); const gptPlugins = require('~/server/services/Endpoints/gptPlugins'); const { processFiles } = require('~/server/services/Files/process'); const anthropic = require('~/server/services/Endpoints/anthropic'); +const bedrock = require('~/server/services/Endpoints/bedrock'); const openAI = require('~/server/services/Endpoints/openAI'); const agents = require('~/server/services/Endpoints/agents'); const custom = require('~/server/services/Endpoints/custom'); @@ -17,6 +18,7 @@ const buildFunction = { [EModelEndpoint.google]: google.buildOptions, [EModelEndpoint.custom]: custom.buildOptions, [EModelEndpoint.agents]: agents.buildOptions, + [EModelEndpoint.bedrock]: bedrock.buildOptions, [EModelEndpoint.azureOpenAI]: openAI.buildOptions, [EModelEndpoint.anthropic]: anthropic.buildOptions, [EModelEndpoint.gptPlugins]: gptPlugins.buildOptions, diff --git a/api/server/routes/bedrock/chat.js b/api/server/routes/bedrock/chat.js new file mode 100644 index 0000000000..0135025709 --- /dev/null +++ b/api/server/routes/bedrock/chat.js @@ -0,0 +1,35 @@ +const express = require('express'); + +const router = express.Router(); +const { + setHeaders, + handleAbort, + // validateModel, + // validateEndpoint, + buildEndpointOption, +} = require('~/server/middleware'); +const { initializeClient } = require('~/server/services/Endpoints/bedrock'); +const AgentController = require('~/server/controllers/agents/request'); + +router.post('/abort', handleAbort()); + +/** + * @route POST / + * @desc Chat with an assistant + * @access Public + * @param {express.Request} req - The request object, containing the request data. + * @param {express.Response} res - The response object, used to send back a response. + * @returns {void} + */ +router.post( + '/', + // validateModel, + // validateEndpoint, + buildEndpointOption, + setHeaders, + async (req, res, next) => { + await AgentController(req, res, next, initializeClient); + }, +); + +module.exports = router; diff --git a/api/server/routes/bedrock/index.js b/api/server/routes/bedrock/index.js new file mode 100644 index 0000000000..b1a9efec4c --- /dev/null +++ b/api/server/routes/bedrock/index.js @@ -0,0 +1,19 @@ +const express = require('express'); +const router = express.Router(); +const { + uaParser, + checkBan, + requireJwtAuth, + // concurrentLimiter, + // messageIpLimiter, + // messageUserLimiter, +} = require('~/server/middleware'); + +const chat = require('./chat'); + +router.use(requireJwtAuth); +router.use(checkBan); +router.use(uaParser); +router.use('/chat', chat); + +module.exports = router; diff --git a/api/server/routes/index.js b/api/server/routes/index.js index 90ba5c73ad..3790aacd24 100644 --- a/api/server/routes/index.js +++ b/api/server/routes/index.js @@ -8,6 +8,7 @@ const presets = require('./presets'); const prompts = require('./prompts'); const balance = require('./balance'); const plugins = require('./plugins'); +const bedrock = require('./bedrock'); const search = require('./search'); const models = require('./models'); const convos = require('./convos'); @@ -36,6 +37,7 @@ module.exports = { files, share, agents, + bedrock, convos, search, prompts, diff --git a/api/server/services/Config/EndpointService.js b/api/server/services/Config/EndpointService.js index b2f82f383b..485c99f373 100644 --- a/api/server/services/Config/EndpointService.js +++ b/api/server/services/Config/EndpointService.js @@ -45,6 +45,7 @@ module.exports = { AZURE_ASSISTANTS_BASE_URL, EModelEndpoint.azureAssistants, ), + [EModelEndpoint.bedrock]: generateConfig(process.env.BEDROCK_AWS_SECRET_ACCESS_KEY), /* key will be part of separate config */ [EModelEndpoint.agents]: generateConfig(process.env.I_AM_A_TEAPOT), }, diff --git a/api/server/services/Config/loadDefaultEConfig.js b/api/server/services/Config/loadDefaultEConfig.js index df331d92fb..c11ddbe9d5 100644 --- a/api/server/services/Config/loadDefaultEConfig.js +++ b/api/server/services/Config/loadDefaultEConfig.js @@ -9,22 +9,13 @@ const { config } = require('./EndpointService'); */ async function loadDefaultEndpointsConfig(req) { const { google, gptPlugins } = await loadAsyncEndpoints(req); - const { - openAI, - agents, - assistants, - azureAssistants, - bingAI, - anthropic, - azureOpenAI, - chatGPTBrowser, - } = config; + const { assistants, azureAssistants, bingAI, azureOpenAI, chatGPTBrowser } = config; const enabledEndpoints = getEnabledEndpoints(); const endpointConfig = { - [EModelEndpoint.openAI]: openAI, - [EModelEndpoint.agents]: agents, + [EModelEndpoint.openAI]: config[EModelEndpoint.openAI], + [EModelEndpoint.agents]: config[EModelEndpoint.agents], [EModelEndpoint.assistants]: assistants, [EModelEndpoint.azureAssistants]: azureAssistants, [EModelEndpoint.azureOpenAI]: azureOpenAI, @@ -32,7 +23,8 @@ async function loadDefaultEndpointsConfig(req) { [EModelEndpoint.bingAI]: bingAI, [EModelEndpoint.chatGPTBrowser]: chatGPTBrowser, [EModelEndpoint.gptPlugins]: gptPlugins, - [EModelEndpoint.anthropic]: anthropic, + [EModelEndpoint.anthropic]: config[EModelEndpoint.anthropic], + [EModelEndpoint.bedrock]: config[EModelEndpoint.bedrock], }; const orderedAndFilteredEndpoints = enabledEndpoints.reduce((config, key, index) => { diff --git a/api/server/services/Config/loadDefaultModels.js b/api/server/services/Config/loadDefaultModels.js index e06b73c0c0..85987a494e 100644 --- a/api/server/services/Config/loadDefaultModels.js +++ b/api/server/services/Config/loadDefaultModels.js @@ -38,6 +38,8 @@ async function loadDefaultModels(req) { [EModelEndpoint.chatGPTBrowser]: chatGPTBrowser, [EModelEndpoint.assistants]: assistants, [EModelEndpoint.azureAssistants]: azureAssistants, + /* TODO: remove this, only for testing */ + [EModelEndpoint.bedrock]: ['anthropic.claude-3-sonnet-20240229-v1:0'], }; } diff --git a/api/server/services/Endpoints/agents/build.js b/api/server/services/Endpoints/agents/build.js index 256901057d..d04dee9a06 100644 --- a/api/server/services/Endpoints/agents/build.js +++ b/api/server/services/Endpoints/agents/build.js @@ -2,7 +2,7 @@ const { getAgent } = require('~/models/Agent'); const { logger } = require('~/config'); const buildOptions = (req, endpoint, parsedBody) => { - const { agent_id, instructions, spec, ...rest } = parsedBody; + const { agent_id, instructions, spec, ...model_parameters } = parsedBody; const agentPromise = getAgent({ id: agent_id, @@ -19,9 +19,7 @@ const buildOptions = (req, endpoint, parsedBody) => { agent_id, instructions, spec, - modelOptions: { - ...rest, - }, + model_parameters, }; return endpointOption; diff --git a/api/server/services/Endpoints/bedrock/build.js b/api/server/services/Endpoints/bedrock/build.js new file mode 100644 index 0000000000..ef5058ac99 --- /dev/null +++ b/api/server/services/Endpoints/bedrock/build.js @@ -0,0 +1,37 @@ +const { removeNullishValues } = require('librechat-data-provider'); +const generateArtifactsPrompt = require('~/app/clients/prompts/artifacts'); + +const buildOptions = (endpoint, parsedBody) => { + const { + modelLabel: name, + promptPrefix, + maxContextTokens, + resendFiles = true, + imageDetail, + iconURL, + greeting, + spec, + artifacts, + ...model_parameters + } = parsedBody; + const endpointOption = removeNullishValues({ + endpoint, + name, + resendFiles, + imageDetail, + iconURL, + greeting, + spec, + instructions: promptPrefix, + maxContextTokens, + model_parameters, + }); + + if (typeof artifacts === 'string') { + endpointOption.artifactsPrompt = generateArtifactsPrompt({ endpoint, artifacts }); + } + + return endpointOption; +}; + +module.exports = { buildOptions }; diff --git a/api/server/services/Endpoints/bedrock/index.js b/api/server/services/Endpoints/bedrock/index.js new file mode 100644 index 0000000000..8989f7df8c --- /dev/null +++ b/api/server/services/Endpoints/bedrock/index.js @@ -0,0 +1,7 @@ +const build = require('./build'); +const initialize = require('./initialize'); + +module.exports = { + ...build, + ...initialize, +}; diff --git a/api/server/services/Endpoints/bedrock/initialize.js b/api/server/services/Endpoints/bedrock/initialize.js new file mode 100644 index 0000000000..13d12004e5 --- /dev/null +++ b/api/server/services/Endpoints/bedrock/initialize.js @@ -0,0 +1,55 @@ +const { EModelEndpoint, providerEndpointMap } = require('librechat-data-provider'); +const { getDefaultHandlers } = require('~/server/controllers/agents/callbacks'); +// const { loadAgentTools } = require('~/server/services/ToolService'); +const getOptions = require('~/server/services/Endpoints/bedrock/options'); +const AgentClient = require('~/server/controllers/agents/client'); +const { getModelMaxTokens } = require('~/utils'); + +const initializeClient = async ({ req, res, endpointOption }) => { + if (!endpointOption) { + throw new Error('Endpoint option not provided'); + } + + // TODO: use endpointOption to determine options/modelOptions + const eventHandlers = getDefaultHandlers({ res }); + + // const tools = [createTavilySearchTool()]; + + /** @type {Agent} */ + const agent = { + id: EModelEndpoint.bedrock, + name: endpointOption.name, + instructions: endpointOption.instructions, + provider: EModelEndpoint.bedrock, + model: endpointOption.model_parameters.model, + model_parameters: endpointOption.model_parameters, + }; + + let modelOptions = { model: agent.model }; + + // TODO: pass-in override settings that are specific to current run + const options = await getOptions({ + req, + res, + endpointOption, + }); + + modelOptions = Object.assign(modelOptions, options.llmConfig); + const maxContextTokens = + agent.max_context_tokens ?? + getModelMaxTokens(modelOptions.model, providerEndpointMap[agent.provider]); + + const client = new AgentClient({ + req, + agent, + // tools, + // toolMap, + modelOptions, + eventHandlers, + maxContextTokens, + configOptions: options.configOptions, + }); + return { client }; +}; + +module.exports = { initializeClient }; diff --git a/api/server/services/Endpoints/bedrock/options.js b/api/server/services/Endpoints/bedrock/options.js new file mode 100644 index 0000000000..bbd860882f --- /dev/null +++ b/api/server/services/Endpoints/bedrock/options.js @@ -0,0 +1,72 @@ +const { HttpsProxyAgent } = require('https-proxy-agent'); +const { EModelEndpoint, AuthType, removeNullishValues } = require('librechat-data-provider'); +const { getUserKey, checkUserKeyExpiry } = require('~/server/services/UserService'); + +const getOptions = async ({ req, endpointOption }) => { + const { + BEDROCK_AWS_SECRET_ACCESS_KEY, + BEDROCK_AWS_ACCESS_KEY_ID, + BEDROCK_REVERSE_PROXY, + BEDROCK_AWS_REGION, + PROXY, + } = process.env; + const expiresAt = req.body.key; + const isUserProvided = BEDROCK_AWS_SECRET_ACCESS_KEY === AuthType.USER_PROVIDED; + + const credentials = isUserProvided + ? await getUserKey({ userId: req.user.id, name: EModelEndpoint.bedrock }) + : { + accessKeyId: BEDROCK_AWS_ACCESS_KEY_ID, + secretAccessKey: BEDROCK_AWS_SECRET_ACCESS_KEY, + }; + + if (!credentials) { + throw new Error('Bedrock credentials not provided. Please provide them again.'); + } + + if (expiresAt && isUserProvided) { + checkUserKeyExpiry(expiresAt, EModelEndpoint.bedrock); + } + + const clientOptions = {}; + + /** @type {undefined | TBaseEndpoint} */ + const bedrockConfig = req.app.locals[EModelEndpoint.bedrock]; + + if (bedrockConfig) { + clientOptions.streamRate = bedrockConfig.streamRate; + } + + /** @type {undefined | TBaseEndpoint} */ + const allConfig = req.app.locals.all; + if (allConfig) { + clientOptions.streamRate = allConfig.streamRate; + } + + const requestOptions = Object.assign( + { + credentials, + model: endpointOption.model, + region: BEDROCK_AWS_REGION, + streaming: true, + streamUsage: true, + }, + endpointOption.model_parameters, + ); + + const configOptions = {}; + if (PROXY) { + configOptions.httpAgent = new HttpsProxyAgent(PROXY); + } + + if (BEDROCK_REVERSE_PROXY) { + configOptions.endpointHost = BEDROCK_REVERSE_PROXY; + } + + return { + llmConfig: removeNullishValues(requestOptions), + configOptions, + }; +}; + +module.exports = getOptions; diff --git a/api/typedefs.js b/api/typedefs.js index 6591d192b1..93cc666a8f 100644 --- a/api/typedefs.js +++ b/api/typedefs.js @@ -26,6 +26,12 @@ * @memberof typedefs */ +/** + * @exports BedrockClientOptions + * @typedef {import('@librechat/agents').BedrockConverseClientOptions} BedrockClientOptions + * @memberof typedefs + */ + /** * @exports StreamEventData * @typedef {import('@librechat/agents').StreamEventData} StreamEventData diff --git a/api/utils/tokens.js b/api/utils/tokens.js index 83246c5b74..2dfa440367 100644 --- a/api/utils/tokens.js +++ b/api/utils/tokens.js @@ -72,6 +72,7 @@ const maxTokensMap = { [EModelEndpoint.custom]: aggregateModels, [EModelEndpoint.google]: googleModels, [EModelEndpoint.anthropic]: anthropicModels, + [EModelEndpoint.bedrock]: aggregateModels, }; /** diff --git a/package-lock.json b/package-lock.json index ab752abba3..08b4f0e857 100644 --- a/package-lock.json +++ b/package-lock.json @@ -52,7 +52,7 @@ "@langchain/core": "^0.2.18", "@langchain/google-genai": "^0.0.11", "@langchain/google-vertexai": "^0.0.17", - "@librechat/agents": "^1.4.1", + "@librechat/agents": "^1.4.2", "axios": "^1.3.4", "bcryptjs": "^2.4.3", "cheerio": "^1.0.0-rc.12", @@ -10014,9 +10014,9 @@ } }, "node_modules/@librechat/agents": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-1.4.1.tgz", - "integrity": "sha512-i8FEiIUal570rbBQ5G7WRjpxd6JwSnYMif/VkfVQkMxx/6ln8gULdbzh6q0zwvUGcHp90Rv4e+Ve9C2dtNk2Ww==", + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-1.4.2.tgz", + "integrity": "sha512-6uo+VxJUJV48MoZlAfedgKWVHtkgmXQKfeiD3MiJNYsxl7udulifjAmdEd7uQZaLfkVdds81yrkWfn11sGsTEQ==", "dependencies": { "@aws-crypto/sha256-js": "^5.2.0", "@aws-sdk/credential-provider-node": "^3.613.0", diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 882a73c217..7901339973 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -1116,6 +1116,7 @@ export enum SystemCategories { export const providerEndpointMap = { [EModelEndpoint.openAI]: EModelEndpoint.openAI, + [EModelEndpoint.bedrock]: EModelEndpoint.bedrock, [EModelEndpoint.azureOpenAI]: EModelEndpoint.openAI, [EModelEndpoint.anthropic]: EModelEndpoint.anthropic, };