feat: first pass, bedrock chat. note: AgentClient is returning agents as conversation.endpoint

api/package.json

@@ -43,7 +43,7 @@
     "@langchain/core": "^0.2.18",
     "@langchain/google-genai": "^0.0.11",
     "@langchain/google-vertexai": "^0.0.17",
-    "@librechat/agents": "^1.4.1",
+    "@librechat/agents": "^1.4.2",
     "axios": "^1.3.4",
     "bcryptjs": "^2.4.3",
     "cheerio": "^1.0.0-rc.12",

api/server/controllers/agents/request.js

@@ -22,7 +22,7 @@ const AgentController = async (req, res, next, initializeClient, addTitle) => {
 
   const sender = getResponseSender({
     ...endpointOption,
-    model: endpointOption.modelOptions.model,
+    model: endpointOption.model_parameters.model,
     modelDisplayLabel,
   });
   const newConvo = !conversationId;

api/server/controllers/bedrock/client.js

@@ -0,0 +1,16 @@
+const { EModelEndpoint } = require('librechat-data-provider');
+const AgentClient = require('~/server/controllers/agents/client');
+const { logger } = require('~/config');
+
+class BedrockClient extends AgentClient {
+  constructor(options = {}) {
+    super(options);
+    this.options.endpoint = EModelEndpoint.bedrock;
+  }
+
+  setOptions(options) {
+    logger.info('[api/server/controllers/bedrock/client.js] setOptions', options);
+  }
+}
+
+module.exports = BedrockClient;

api/server/index.js

@@ -106,6 +106,7 @@ const startServer = async () => {
   app.use('/api/share', routes.share);
   app.use('/api/roles', routes.roles);
   app.use('/api/agents', routes.agents);
+  app.use('/api/bedrock', routes.bedrock);
 
   app.use('/api/tags', routes.tags);
 

api/server/middleware/buildEndpointOption.js

@@ -5,6 +5,7 @@ const assistants = require('~/server/services/Endpoints/assistants');
 const gptPlugins = require('~/server/services/Endpoints/gptPlugins');
 const { processFiles } = require('~/server/services/Files/process');
 const anthropic = require('~/server/services/Endpoints/anthropic');
+const bedrock = require('~/server/services/Endpoints/bedrock');
 const openAI = require('~/server/services/Endpoints/openAI');
 const agents = require('~/server/services/Endpoints/agents');
 const custom = require('~/server/services/Endpoints/custom');
@@ -17,6 +18,7 @@ const buildFunction = {
   [EModelEndpoint.google]: google.buildOptions,
   [EModelEndpoint.custom]: custom.buildOptions,
   [EModelEndpoint.agents]: agents.buildOptions,
+  [EModelEndpoint.bedrock]: bedrock.buildOptions,
   [EModelEndpoint.azureOpenAI]: openAI.buildOptions,
   [EModelEndpoint.anthropic]: anthropic.buildOptions,
   [EModelEndpoint.gptPlugins]: gptPlugins.buildOptions,

api/server/routes/bedrock/chat.js

@@ -0,0 +1,35 @@
+const express = require('express');
+
+const router = express.Router();
+const {
+  setHeaders,
+  handleAbort,
+  // validateModel,
+  // validateEndpoint,
+  buildEndpointOption,
+} = require('~/server/middleware');
+const { initializeClient } = require('~/server/services/Endpoints/bedrock');
+const AgentController = require('~/server/controllers/agents/request');
+
+router.post('/abort', handleAbort());
+
+/**
+ * @route POST /
+ * @desc Chat with an assistant
+ * @access Public
+ * @param {express.Request} req - The request object, containing the request data.
+ * @param {express.Response} res - The response object, used to send back a response.
+ * @returns {void}
+ */
+router.post(
+  '/',
+  // validateModel,
+  // validateEndpoint,
+  buildEndpointOption,
+  setHeaders,
+  async (req, res, next) => {
+    await AgentController(req, res, next, initializeClient);
+  },
+);
+
+module.exports = router;

api/server/routes/bedrock/index.js

@@ -0,0 +1,19 @@
+const express = require('express');
+const router = express.Router();
+const {
+  uaParser,
+  checkBan,
+  requireJwtAuth,
+  // concurrentLimiter,
+  // messageIpLimiter,
+  // messageUserLimiter,
+} = require('~/server/middleware');
+
+const chat = require('./chat');
+
+router.use(requireJwtAuth);
+router.use(checkBan);
+router.use(uaParser);
+router.use('/chat', chat);
+
+module.exports = router;

api/server/routes/index.js

@@ -8,6 +8,7 @@ const presets = require('./presets');
 const prompts = require('./prompts');
 const balance = require('./balance');
 const plugins = require('./plugins');
+const bedrock = require('./bedrock');
 const search = require('./search');
 const models = require('./models');
 const convos = require('./convos');
@@ -36,6 +37,7 @@ module.exports = {
   files,
   share,
   agents,
+  bedrock,
   convos,
   search,
   prompts,

api/server/services/Config/EndpointService.js

@@ -45,6 +45,7 @@ module.exports = {
       AZURE_ASSISTANTS_BASE_URL,
       EModelEndpoint.azureAssistants,
     ),
+    [EModelEndpoint.bedrock]: generateConfig(process.env.BEDROCK_AWS_SECRET_ACCESS_KEY),
     /* key will be part of separate config */
     [EModelEndpoint.agents]: generateConfig(process.env.I_AM_A_TEAPOT),
   },

api/server/services/Config/loadDefaultEConfig.js

@@ -9,22 +9,13 @@ const { config } = require('./EndpointService');
  */
 async function loadDefaultEndpointsConfig(req) {
   const { google, gptPlugins } = await loadAsyncEndpoints(req);
-  const {
+  const { assistants, azureAssistants, bingAI, azureOpenAI, chatGPTBrowser } = config;
-    openAI,
-    agents,
-    assistants,
-    azureAssistants,
-    bingAI,
-    anthropic,
-    azureOpenAI,
-    chatGPTBrowser,
-  } = config;
 
   const enabledEndpoints = getEnabledEndpoints();
 
   const endpointConfig = {
-    [EModelEndpoint.openAI]: openAI,
+    [EModelEndpoint.openAI]: config[EModelEndpoint.openAI],
-    [EModelEndpoint.agents]: agents,
+    [EModelEndpoint.agents]: config[EModelEndpoint.agents],
     [EModelEndpoint.assistants]: assistants,
     [EModelEndpoint.azureAssistants]: azureAssistants,
     [EModelEndpoint.azureOpenAI]: azureOpenAI,
@@ -32,7 +23,8 @@ async function loadDefaultEndpointsConfig(req) {
     [EModelEndpoint.bingAI]: bingAI,
     [EModelEndpoint.chatGPTBrowser]: chatGPTBrowser,
     [EModelEndpoint.gptPlugins]: gptPlugins,
-    [EModelEndpoint.anthropic]: anthropic,
+    [EModelEndpoint.anthropic]: config[EModelEndpoint.anthropic],
+    [EModelEndpoint.bedrock]: config[EModelEndpoint.bedrock],
   };
 
   const orderedAndFilteredEndpoints = enabledEndpoints.reduce((config, key, index) => {

api/server/services/Config/loadDefaultModels.js

@@ -38,6 +38,8 @@ async function loadDefaultModels(req) {
     [EModelEndpoint.chatGPTBrowser]: chatGPTBrowser,
     [EModelEndpoint.assistants]: assistants,
     [EModelEndpoint.azureAssistants]: azureAssistants,
+    /* TODO: remove this, only for testing */
+    [EModelEndpoint.bedrock]: ['anthropic.claude-3-sonnet-20240229-v1:0'],
   };
 }
 

api/server/services/Endpoints/agents/build.js

@@ -2,7 +2,7 @@ const { getAgent } = require('~/models/Agent');
 const { logger } = require('~/config');
 
 const buildOptions = (req, endpoint, parsedBody) => {
-  const { agent_id, instructions, spec, ...rest } = parsedBody;
+  const { agent_id, instructions, spec, ...model_parameters } = parsedBody;
 
   const agentPromise = getAgent({
     id: agent_id,
@@ -19,9 +19,7 @@ const buildOptions = (req, endpoint, parsedBody) => {
     agent_id,
     instructions,
     spec,
-    modelOptions: {
+    model_parameters,
-      ...rest,
-    },
   };
 
   return endpointOption;

api/server/services/Endpoints/bedrock/build.js

@@ -0,0 +1,37 @@
+const { removeNullishValues } = require('librechat-data-provider');
+const generateArtifactsPrompt = require('~/app/clients/prompts/artifacts');
+
+const buildOptions = (endpoint, parsedBody) => {
+  const {
+    modelLabel: name,
+    promptPrefix,
+    maxContextTokens,
+    resendFiles = true,
+    imageDetail,
+    iconURL,
+    greeting,
+    spec,
+    artifacts,
+    ...model_parameters
+  } = parsedBody;
+  const endpointOption = removeNullishValues({
+    endpoint,
+    name,
+    resendFiles,
+    imageDetail,
+    iconURL,
+    greeting,
+    spec,
+    instructions: promptPrefix,
+    maxContextTokens,
+    model_parameters,
+  });
+
+  if (typeof artifacts === 'string') {
+    endpointOption.artifactsPrompt = generateArtifactsPrompt({ endpoint, artifacts });
+  }
+
+  return endpointOption;
+};
+
+module.exports = { buildOptions };

api/server/services/Endpoints/bedrock/index.js

@@ -0,0 +1,7 @@
+const build = require('./build');
+const initialize = require('./initialize');
+
+module.exports = {
+  ...build,
+  ...initialize,
+};

api/server/services/Endpoints/bedrock/initialize.js

@@ -0,0 +1,55 @@
+const { EModelEndpoint, providerEndpointMap } = require('librechat-data-provider');
+const { getDefaultHandlers } = require('~/server/controllers/agents/callbacks');
+// const { loadAgentTools } = require('~/server/services/ToolService');
+const getOptions = require('~/server/services/Endpoints/bedrock/options');
+const AgentClient = require('~/server/controllers/agents/client');
+const { getModelMaxTokens } = require('~/utils');
+
+const initializeClient = async ({ req, res, endpointOption }) => {
+  if (!endpointOption) {
+    throw new Error('Endpoint option not provided');
+  }
+
+  // TODO: use endpointOption to determine options/modelOptions
+  const eventHandlers = getDefaultHandlers({ res });
+
+  // const tools = [createTavilySearchTool()];
+
+  /** @type {Agent} */
+  const agent = {
+    id: EModelEndpoint.bedrock,
+    name: endpointOption.name,
+    instructions: endpointOption.instructions,
+    provider: EModelEndpoint.bedrock,
+    model: endpointOption.model_parameters.model,
+    model_parameters: endpointOption.model_parameters,
+  };
+
+  let modelOptions = { model: agent.model };
+
+  // TODO: pass-in override settings that are specific to current run
+  const options = await getOptions({
+    req,
+    res,
+    endpointOption,
+  });
+
+  modelOptions = Object.assign(modelOptions, options.llmConfig);
+  const maxContextTokens =
+    agent.max_context_tokens ??
+    getModelMaxTokens(modelOptions.model, providerEndpointMap[agent.provider]);
+
+  const client = new AgentClient({
+    req,
+    agent,
+    // tools,
+    // toolMap,
+    modelOptions,
+    eventHandlers,
+    maxContextTokens,
+    configOptions: options.configOptions,
+  });
+  return { client };
+};
+
+module.exports = { initializeClient };

api/server/services/Endpoints/bedrock/options.js

@@ -0,0 +1,72 @@
+const { HttpsProxyAgent } = require('https-proxy-agent');
+const { EModelEndpoint, AuthType, removeNullishValues } = require('librechat-data-provider');
+const { getUserKey, checkUserKeyExpiry } = require('~/server/services/UserService');
+
+const getOptions = async ({ req, endpointOption }) => {
+  const {
+    BEDROCK_AWS_SECRET_ACCESS_KEY,
+    BEDROCK_AWS_ACCESS_KEY_ID,
+    BEDROCK_REVERSE_PROXY,
+    BEDROCK_AWS_REGION,
+    PROXY,
+  } = process.env;
+  const expiresAt = req.body.key;
+  const isUserProvided = BEDROCK_AWS_SECRET_ACCESS_KEY === AuthType.USER_PROVIDED;
+
+  const credentials = isUserProvided
+    ? await getUserKey({ userId: req.user.id, name: EModelEndpoint.bedrock })
+    : {
+      accessKeyId: BEDROCK_AWS_ACCESS_KEY_ID,
+      secretAccessKey: BEDROCK_AWS_SECRET_ACCESS_KEY,
+    };
+
+  if (!credentials) {
+    throw new Error('Bedrock credentials not provided. Please provide them again.');
+  }
+
+  if (expiresAt && isUserProvided) {
+    checkUserKeyExpiry(expiresAt, EModelEndpoint.bedrock);
+  }
+
+  const clientOptions = {};
+
+  /** @type {undefined | TBaseEndpoint} */
+  const bedrockConfig = req.app.locals[EModelEndpoint.bedrock];
+
+  if (bedrockConfig) {
+    clientOptions.streamRate = bedrockConfig.streamRate;
+  }
+
+  /** @type {undefined | TBaseEndpoint} */
+  const allConfig = req.app.locals.all;
+  if (allConfig) {
+    clientOptions.streamRate = allConfig.streamRate;
+  }
+
+  const requestOptions = Object.assign(
+    {
+      credentials,
+      model: endpointOption.model,
+      region: BEDROCK_AWS_REGION,
+      streaming: true,
+      streamUsage: true,
+    },
+    endpointOption.model_parameters,
+  );
+
+  const configOptions = {};
+  if (PROXY) {
+    configOptions.httpAgent = new HttpsProxyAgent(PROXY);
+  }
+
+  if (BEDROCK_REVERSE_PROXY) {
+    configOptions.endpointHost = BEDROCK_REVERSE_PROXY;
+  }
+
+  return {
+    llmConfig: removeNullishValues(requestOptions),
+    configOptions,
+  };
+};
+
+module.exports = getOptions;

api/typedefs.js

@@ -26,6 +26,12 @@
  * @memberof typedefs
  */
 
+/**
+ * @exports BedrockClientOptions
+ * @typedef {import('@librechat/agents').BedrockConverseClientOptions} BedrockClientOptions
+ * @memberof typedefs
+ */
+
 /**
  * @exports StreamEventData
  * @typedef {import('@librechat/agents').StreamEventData} StreamEventData

api/utils/tokens.js

@@ -72,6 +72,7 @@ const maxTokensMap = {
   [EModelEndpoint.custom]: aggregateModels,
   [EModelEndpoint.google]: googleModels,
   [EModelEndpoint.anthropic]: anthropicModels,
+  [EModelEndpoint.bedrock]: aggregateModels,
 };
 
 /**

package-lock.json

@@ -52,7 +52,7 @@
         "@langchain/core": "^0.2.18",
         "@langchain/google-genai": "^0.0.11",
         "@langchain/google-vertexai": "^0.0.17",
-        "@librechat/agents": "^1.4.1",
+        "@librechat/agents": "^1.4.2",
         "axios": "^1.3.4",
         "bcryptjs": "^2.4.3",
         "cheerio": "^1.0.0-rc.12",
@@ -10014,9 +10014,9 @@
       }
     },
     "node_modules/@librechat/agents": {
-      "version": "1.4.1",
+      "version": "1.4.2",
-      "resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-1.4.1.tgz",
+      "resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-1.4.2.tgz",
-      "integrity": "sha512-i8FEiIUal570rbBQ5G7WRjpxd6JwSnYMif/VkfVQkMxx/6ln8gULdbzh6q0zwvUGcHp90Rv4e+Ve9C2dtNk2Ww==",
+      "integrity": "sha512-6uo+VxJUJV48MoZlAfedgKWVHtkgmXQKfeiD3MiJNYsxl7udulifjAmdEd7uQZaLfkVdds81yrkWfn11sGsTEQ==",
       "dependencies": {
         "@aws-crypto/sha256-js": "^5.2.0",
         "@aws-sdk/credential-provider-node": "^3.613.0",

packages/data-provider/src/config.ts

@@ -1116,6 +1116,7 @@ export enum SystemCategories {
 
 export const providerEndpointMap = {
   [EModelEndpoint.openAI]: EModelEndpoint.openAI,
+  [EModelEndpoint.bedrock]: EModelEndpoint.bedrock,
   [EModelEndpoint.azureOpenAI]: EModelEndpoint.openAI,
   [EModelEndpoint.anthropic]: EModelEndpoint.anthropic,
 };