💫 feat: Config File & Custom Endpoints (#1474)

* WIP(backend/api): custom endpoint

* WIP(frontend/client): custom endpoint

* chore: adjust typedefs for configs

* refactor: use data-provider for cache keys and rename enums and custom endpoint for better clarity and compatibility

* feat: loadYaml utility

* refactor: rename back to  from  and proof-of-concept for creating schemas from user-defined defaults

* refactor: remove custom endpoint from default endpointsConfig as it will be exclusively managed by yaml config

* refactor(EndpointController): rename variables for clarity

* feat: initial load custom config

* feat(server/utils): add simple `isUserProvided` helper

* chore(types): update TConfig type

* refactor: remove custom endpoint handling from model services as will be handled by config, modularize fetching of models

* feat: loadCustomConfig, loadConfigEndpoints, loadConfigModels

* chore: reorganize server init imports, invoke loadCustomConfig

* refactor(loadConfigEndpoints/Models): return each custom endpoint as standalone endpoint

* refactor(Endpoint/ModelController): spread config values after default (temporary)

* chore(client): fix type issues

* WIP: first pass for multiple custom endpoints
- add endpointType to Conversation schema
- add update zod schemas for both convo/presets to allow non-EModelEndpoint value as endpoint (also using type assertion)
- use `endpointType` value as `endpoint` where mapping to type is necessary using this field
- use custom defined `endpoint` value and not type for mapping to modelsConfig
- misc: add return type to `getDefaultEndpoint`
- in `useNewConvo`, add the endpointType if it wasn't already added to conversation
- EndpointsMenu: use user-defined endpoint name as Title in menu
- TODO: custom icon via custom config, change unknown to robot icon

* refactor(parseConvo): pass args as an object and change where used accordingly; chore: comment out 'create schema' code

* chore: remove unused availableModels field in TConfig type

* refactor(parseCompactConvo): pass args as an object and change where used accordingly

* feat: chat through custom endpoint

* chore(message/convoSchemas): avoid saving empty arrays

* fix(BaseClient/saveMessageToDatabase): save endpointType

* refactor(ChatRoute): show Spinner if endpointsQuery or modelsQuery are still loading, which is apparent with slow fetching of models/remote config on first serve

* fix(useConversation): assign endpointType if it's missing

* fix(SaveAsPreset): pass real endpoint and endpointType when saving Preset)

* chore: recorganize types order for TConfig, add `iconURL`

* feat: custom endpoint icon support:
- use UnknownIcon in all icon contexts
- add mistral and openrouter as known endpoints, and add their icons
- iconURL support

* fix(presetSchema): move endpointType to default schema definitions shared between convoSchema and defaults

* refactor(Settings/OpenAI): remove legacy `isOpenAI` flag

* fix(OpenAIClient): do not invoke abortCompletion on completion error

* feat: add responseSender/label support for custom endpoints:
- use defaultModelLabel field in endpointOption
- add model defaults for custom endpoints in `getResponseSender`
- add `useGetSender` hook which uses EndpointsQuery to determine `defaultModelLabel`
- include defaultModelLabel from endpointConfig in custom endpoint client options
- pass `endpointType` to `getResponseSender`

* feat(OpenAIClient): use custom options from config file

* refactor: rename `defaultModelLabel` to `modelDisplayLabel`

* refactor(data-provider): separate concerns from `schemas` into `parsers`, `config`, and fix imports elsewhere

* feat: `iconURL` and extract environment variables from custom endpoint config values

* feat: custom config validation via zod schema, rename and move to `./projectRoot/librechat.yaml`

* docs: custom config docs and examples

* fix(OpenAIClient/mistral): mistral does not allow singular system message, also add `useChatCompletion` flag to use openai-node for title completions

* fix(custom/initializeClient): extract env var and use `isUserProvided` function

* Update librechat.example.yaml

* feat(InputWithLabel): add className props, and forwardRef

* fix(streamResponse): handle error edge case where either messages or convos query throws an error

* fix(useSSE): handle errorHandler edge cases where error response is and is not properly formatted from API, especially when a conversationId is not yet provided, which ensures stream is properly closed on error

* feat: user_provided keys for custom endpoints

* fix(config/endpointSchema): do not allow default endpoint values in custom endpoint `name`

* feat(loadConfigModels): extract env variables and optimize fetching models

* feat: support custom endpoint iconURL for messages and Nav

* feat(OpenAIClient): add/dropParams support

* docs: update docs with default params, add/dropParams, and notes to use config file instead of `OPENAI_REVERSE_PROXY`

* docs: update docs with additional notes

* feat(maxTokensMap): add mistral models (32k context)

* docs: update openrouter notes

* Update ai_setup.md

* docs(custom_config): add table of contents and fix note about custom name

* docs(custom_config): reorder ToC

* Update custom_config.md

* Add note about `max_tokens` field in custom_config.md

api/server/controllers/AskController.js

@@ -9,6 +9,7 @@ const AskController = async (req, res, next, initializeClient, addTitle) => {
     text,
     endpointOption,
     conversationId,
+    modelDisplayLabel,
     parentMessageId = null,
     overrideParentMessageId = null,
   } = req.body;
@@ -22,7 +23,11 @@ const AskController = async (req, res, next, initializeClient, addTitle) => {
   let responseMessageId;
   let lastSavedTimestamp = 0;
   let saveDelay = 100;
-  const sender = getResponseSender({ ...endpointOption, model: endpointOption.modelOptions.model });
+  const sender = getResponseSender({
+    ...endpointOption,
+    model: endpointOption.modelOptions.model,
+    modelDisplayLabel,
+  });
   const newConvo = !conversationId;
   const user = req.user.id;
 

api/server/controllers/EditController.js

@@ -10,6 +10,7 @@ const EditController = async (req, res, next, initializeClient) => {
     generation,
     endpointOption,
     conversationId,
+    modelDisplayLabel,
     responseMessageId,
     isContinued = false,
     parentMessageId = null,
@@ -29,7 +30,11 @@ const EditController = async (req, res, next, initializeClient) => {
   let promptTokens;
   let lastSavedTimestamp = 0;
   let saveDelay = 100;
-  const sender = getResponseSender({ ...endpointOption, model: endpointOption.modelOptions.model });
+  const sender = getResponseSender({
+    ...endpointOption,
+    model: endpointOption.modelOptions.model,
+    modelDisplayLabel,
+  });
   const userMessageId = parentMessageId;
   const user = req.user.id;
 

api/server/controllers/EndpointController.js

@@ -1,17 +1,22 @@
+const { CacheKeys } = require('librechat-data-provider');
+const { loadDefaultEndpointsConfig, loadConfigEndpoints } = require('~/server/services/Config');
 const { getLogStores } = require('~/cache');
-const { CacheKeys } = require('~/common/enums');
-const { loadDefaultEndpointsConfig } = require('~/server/services/Config');
 
 async function endpointController(req, res) {
-  const cache = getLogStores(CacheKeys.CONFIG);
-  const config = await cache.get(CacheKeys.DEFAULT_CONFIG);
-  if (config) {
-    res.send(config);
+  const cache = getLogStores(CacheKeys.CONFIG_STORE);
+  const cachedEndpointsConfig = await cache.get(CacheKeys.ENDPOINT_CONFIG);
+  if (cachedEndpointsConfig) {
+    res.send(cachedEndpointsConfig);
     return;
   }
-  const defaultConfig = await loadDefaultEndpointsConfig();
-  await cache.set(CacheKeys.DEFAULT_CONFIG, defaultConfig);
-  res.send(JSON.stringify(defaultConfig));
+
+  const defaultEndpointsConfig = await loadDefaultEndpointsConfig();
+  const customConfigEndpoints = await loadConfigEndpoints();
+
+  const endpointsConfig = { ...defaultEndpointsConfig, ...customConfigEndpoints };
+
+  await cache.set(CacheKeys.ENDPOINT_CONFIG, endpointsConfig);
+  res.send(JSON.stringify(endpointsConfig));
 }
 
 module.exports = endpointController;

api/server/controllers/ModelController.js

@@ -1,15 +1,19 @@
+const { CacheKeys } = require('librechat-data-provider');
+const { loadDefaultModels, loadConfigModels } = require('~/server/services/Config');
 const { getLogStores } = require('~/cache');
-const { CacheKeys } = require('~/common/enums');
-const { loadDefaultModels } = require('~/server/services/Config');
 
 async function modelController(req, res) {
-  const cache = getLogStores(CacheKeys.CONFIG);
-  let modelConfig = await cache.get(CacheKeys.MODELS_CONFIG);
-  if (modelConfig) {
-    res.send(modelConfig);
+  const cache = getLogStores(CacheKeys.CONFIG_STORE);
+  const cachedModelsConfig = await cache.get(CacheKeys.MODELS_CONFIG);
+  if (cachedModelsConfig) {
+    res.send(cachedModelsConfig);
     return;
   }
-  modelConfig = await loadDefaultModels();
+  const defaultModelsConfig = await loadDefaultModels();
+  const customModelsConfig = await loadConfigModels();
+
+  const modelConfig = { ...defaultModelsConfig, ...customModelsConfig };
+
   await cache.set(CacheKeys.MODELS_CONFIG, modelConfig);
   res.send(modelConfig);
 }

api/server/controllers/OverrideController.js

@@ -1,9 +1,9 @@
-const { getLogStores } = require('~/cache');
-const { CacheKeys } = require('~/common/enums');
+const { CacheKeys } = require('librechat-data-provider');
 const { loadOverrideConfig } = require('~/server/services/Config');
+const { getLogStores } = require('~/cache');
 
 async function overrideController(req, res) {
-  const cache = getLogStores(CacheKeys.CONFIG);
+  const cache = getLogStores(CacheKeys.CONFIG_STORE);
   let overrideConfig = await cache.get(CacheKeys.OVERRIDE_CONFIG);
   if (overrideConfig) {
     res.send(overrideConfig);
@@ -15,7 +15,7 @@ async function overrideController(req, res) {
   overrideConfig = await loadOverrideConfig();
   const { endpointsConfig, modelsConfig } = overrideConfig;
   if (endpointsConfig) {
-    await cache.set(CacheKeys.DEFAULT_CONFIG, endpointsConfig);
+    await cache.set(CacheKeys.ENDPOINT_CONFIG, endpointsConfig);
   }
   if (modelsConfig) {
     await cache.set(CacheKeys.MODELS_CONFIG, modelsConfig);

api/server/controllers/PluginController.js

@@ -1,7 +1,7 @@
 const path = require('path');
 const { promises: fs } = require('fs');
+const { CacheKeys } = require('librechat-data-provider');
 const { addOpenAPISpecs } = require('~/app/clients/tools/util/addOpenAPISpecs');
-const { CacheKeys } = require('~/common/enums');
 const { getLogStores } = require('~/cache');
 
 const filterUniquePlugins = (plugins) => {
@@ -29,7 +29,7 @@ const isPluginAuthenticated = (plugin) => {
 
 const getAvailablePluginsController = async (req, res) => {
   try {
-    const cache = getLogStores(CacheKeys.CONFIG);
+    const cache = getLogStores(CacheKeys.CONFIG_STORE);
     const cachedPlugins = await cache.get(CacheKeys.PLUGINS);
     if (cachedPlugins) {
       res.status(200).json(cachedPlugins);

api/server/index.js

@@ -5,14 +5,15 @@ const express = require('express');
 const passport = require('passport');
 const mongoSanitize = require('express-mongo-sanitize');
 const { initializeFirebase } = require('~/server/services/Files/Firebase/initialize');
-const errorController = require('./controllers/ErrorController');
-const configureSocialLogins = require('./socialLogins');
+const loadCustomConfig = require('~/server/services/Config/loadCustomConfig');
+const errorController = require('~/server/controllers/ErrorController');
+const configureSocialLogins = require('~/server/socialLogins');
+const noIndex = require('~/server/middleware/noIndex');
 const { connectDb, indexSync } = require('~/lib/db');
 const { logger } = require('~/config');
-const noIndex = require('./middleware/noIndex');
 
+const routes = require('~/server/routes');
 const paths = require('~/config/paths');
-const routes = require('./routes');
 
 const { PORT, HOST, ALLOW_SOCIAL_LOGIN } = process.env ?? {};
 
@@ -24,6 +25,7 @@ const { jwtLogin, passportLogin } = require('~/strategies');
 const startServer = async () => {
   await connectDb();
   logger.info('Connected to MongoDB');
+  await loadCustomConfig();
   initializeFirebase();
   await indexSync();
 

api/server/middleware/buildEndpointOption.js

@@ -1,5 +1,6 @@
 const { processFiles } = require('~/server/services/Files');
 const openAI = require('~/server/services/Endpoints/openAI');
+const custom = require('~/server/services/Endpoints/custom');
 const google = require('~/server/services/Endpoints/google');
 const anthropic = require('~/server/services/Endpoints/anthropic');
 const gptPlugins = require('~/server/services/Endpoints/gptPlugins');
@@ -8,15 +9,20 @@ const { parseConvo, EModelEndpoint } = require('librechat-data-provider');
 const buildFunction = {
   [EModelEndpoint.openAI]: openAI.buildOptions,
   [EModelEndpoint.google]: google.buildOptions,
+  [EModelEndpoint.custom]: custom.buildOptions,
   [EModelEndpoint.azureOpenAI]: openAI.buildOptions,
   [EModelEndpoint.anthropic]: anthropic.buildOptions,
   [EModelEndpoint.gptPlugins]: gptPlugins.buildOptions,
 };
 
 function buildEndpointOption(req, res, next) {
-  const { endpoint } = req.body;
-  const parsedBody = parseConvo(endpoint, req.body);
-  req.body.endpointOption = buildFunction[endpoint](endpoint, parsedBody);
+  const { endpoint, endpointType } = req.body;
+  const parsedBody = parseConvo({ endpoint, endpointType, conversation: req.body });
+  req.body.endpointOption = buildFunction[endpointType ?? endpoint](
+    endpoint,
+    parsedBody,
+    endpointType,
+  );
   if (req.body.files) {
     // hold the promise
     req.body.endpointOption.attachments = processFiles(req.body.files);

api/server/middleware/validateEndpoint.js

@@ -1,7 +1,8 @@
 const { handleError } = require('../utils');
 
 function validateEndpoint(req, res, next) {
-  const { endpoint } = req.body;
+  const { endpoint: _endpoint, endpointType } = req.body;
+  const endpoint = endpointType ?? _endpoint;
 
   if (!req.body.text || req.body.text.length === 0) {
     return handleError(res, { text: 'Prompt empty or too short' });

api/server/routes/ask/custom.js

@@ -0,0 +1,20 @@
+const express = require('express');
+const AskController = require('~/server/controllers/AskController');
+const { initializeClient } = require('~/server/services/Endpoints/custom');
+const { addTitle } = require('~/server/services/Endpoints/openAI');
+const {
+  handleAbort,
+  setHeaders,
+  validateEndpoint,
+  buildEndpointOption,
+} = require('~/server/middleware');
+
+const router = express.Router();
+
+router.post('/abort', handleAbort());
+
+router.post('/', validateEndpoint, buildEndpointOption, setHeaders, async (req, res, next) => {
+  await AskController(req, res, next, initializeClient, addTitle);
+});
+
+module.exports = router;

api/server/routes/ask/index.js

@@ -1,5 +1,6 @@
 const express = require('express');
 const openAI = require('./openAI');
+const custom = require('./custom');
 const google = require('./google');
 const bingAI = require('./bingAI');
 const anthropic = require('./anthropic');
@@ -42,5 +43,6 @@ router.use(`/${EModelEndpoint.gptPlugins}`, gptPlugins);
 router.use(`/${EModelEndpoint.anthropic}`, anthropic);
 router.use(`/${EModelEndpoint.google}`, google);
 router.use(`/${EModelEndpoint.bingAI}`, bingAI);
+router.use(`/${EModelEndpoint.custom}`, custom);
 
 module.exports = router;

api/server/routes/edit/custom.js

@@ -0,0 +1,20 @@
+const express = require('express');
+const EditController = require('~/server/controllers/EditController');
+const { initializeClient } = require('~/server/services/Endpoints/custom');
+const { addTitle } = require('~/server/services/Endpoints/openAI');
+const {
+  handleAbort,
+  setHeaders,
+  validateEndpoint,
+  buildEndpointOption,
+} = require('~/server/middleware');
+
+const router = express.Router();
+
+router.post('/abort', handleAbort());
+
+router.post('/', validateEndpoint, buildEndpointOption, setHeaders, async (req, res, next) => {
+  await EditController(req, res, next, initializeClient, addTitle);
+});
+
+module.exports = router;

api/server/routes/edit/index.js

@@ -1,5 +1,6 @@
 const express = require('express');
 const openAI = require('./openAI');
+const custom = require('./custom');
 const google = require('./google');
 const anthropic = require('./anthropic');
 const gptPlugins = require('./gptPlugins');
@@ -38,5 +39,6 @@ router.use([`/${EModelEndpoint.azureOpenAI}`, `/${EModelEndpoint.openAI}`], open
 router.use(`/${EModelEndpoint.gptPlugins}`, gptPlugins);
 router.use(`/${EModelEndpoint.anthropic}`, anthropic);
 router.use(`/${EModelEndpoint.google}`, google);
+router.use(`/${EModelEndpoint.custom}`, custom);
 
 module.exports = router;

api/server/services/Config/index.js

@@ -1,13 +1,19 @@
 const { config } = require('./EndpointService');
+const loadCustomConfig = require('./loadCustomConfig');
+const loadConfigModels = require('./loadConfigModels');
 const loadDefaultModels = require('./loadDefaultModels');
 const loadOverrideConfig = require('./loadOverrideConfig');
 const loadAsyncEndpoints = require('./loadAsyncEndpoints');
+const loadConfigEndpoints = require('./loadConfigEndpoints');
 const loadDefaultEndpointsConfig = require('./loadDefaultEConfig');
 
 module.exports = {
   config,
+  loadCustomConfig,
+  loadConfigModels,
   loadDefaultModels,
   loadOverrideConfig,
   loadAsyncEndpoints,
+  loadConfigEndpoints,
   loadDefaultEndpointsConfig,
 };

api/server/services/Config/loadConfigEndpoints.js

@@ -0,0 +1,54 @@
+const { CacheKeys, EModelEndpoint } = require('librechat-data-provider');
+const { isUserProvided, extractEnvVariable } = require('~/server/utils');
+const loadCustomConfig = require('./loadCustomConfig');
+const { getLogStores } = require('~/cache');
+
+/**
+ * Load config endpoints from the cached configuration object
+ * @function loadConfigEndpoints */
+async function loadConfigEndpoints() {
+  const cache = getLogStores(CacheKeys.CONFIG_STORE);
+  let customConfig = await cache.get(CacheKeys.CUSTOM_CONFIG);
+
+  if (!customConfig) {
+    customConfig = await loadCustomConfig();
+  }
+
+  if (!customConfig) {
+    return {};
+  }
+
+  const { endpoints = {} } = customConfig ?? {};
+  const endpointsConfig = {};
+
+  if (Array.isArray(endpoints[EModelEndpoint.custom])) {
+    const customEndpoints = endpoints[EModelEndpoint.custom].filter(
+      (endpoint) =>
+        endpoint.baseURL &&
+        endpoint.apiKey &&
+        endpoint.name &&
+        endpoint.models &&
+        (endpoint.models.fetch || endpoint.models.default),
+    );
+
+    for (let i = 0; i < customEndpoints.length; i++) {
+      const endpoint = customEndpoints[i];
+      const { baseURL, apiKey, name, iconURL, modelDisplayLabel } = endpoint;
+
+      const resolvedApiKey = extractEnvVariable(apiKey);
+      const resolvedBaseURL = extractEnvVariable(baseURL);
+
+      endpointsConfig[name] = {
+        type: EModelEndpoint.custom,
+        userProvide: isUserProvided(resolvedApiKey),
+        userProvideURL: isUserProvided(resolvedBaseURL),
+        modelDisplayLabel,
+        iconURL,
+      };
+    }
+  }
+
+  return endpointsConfig;
+}
+
+module.exports = loadConfigEndpoints;

api/server/services/Config/loadConfigModels.js

@@ -0,0 +1,79 @@
+const { CacheKeys, EModelEndpoint } = require('librechat-data-provider');
+const { isUserProvided, extractEnvVariable } = require('~/server/utils');
+const { fetchModels } = require('~/server/services/ModelService');
+const loadCustomConfig = require('./loadCustomConfig');
+const { getLogStores } = require('~/cache');
+
+/**
+ * Load config endpoints from the cached configuration object
+ * @function loadConfigModels */
+async function loadConfigModels() {
+  const cache = getLogStores(CacheKeys.CONFIG_STORE);
+  let customConfig = await cache.get(CacheKeys.CUSTOM_CONFIG);
+
+  if (!customConfig) {
+    customConfig = await loadCustomConfig();
+  }
+
+  if (!customConfig) {
+    return {};
+  }
+
+  const { endpoints = {} } = customConfig ?? {};
+  const modelsConfig = {};
+
+  if (!Array.isArray(endpoints[EModelEndpoint.custom])) {
+    return modelsConfig;
+  }
+
+  const customEndpoints = endpoints[EModelEndpoint.custom].filter(
+    (endpoint) =>
+      endpoint.baseURL &&
+      endpoint.apiKey &&
+      endpoint.name &&
+      endpoint.models &&
+      (endpoint.models.fetch || endpoint.models.default),
+  );
+
+  const fetchPromisesMap = {}; // Map for promises keyed by baseURL
+  const baseUrlToNameMap = {}; // Map to associate baseURLs with names
+
+  for (let i = 0; i < customEndpoints.length; i++) {
+    const endpoint = customEndpoints[i];
+    const { models, name, baseURL, apiKey } = endpoint;
+
+    const API_KEY = extractEnvVariable(apiKey);
+    const BASE_URL = extractEnvVariable(baseURL);
+
+    modelsConfig[name] = [];
+
+    if (models.fetch && !isUserProvided(API_KEY) && !isUserProvided(BASE_URL)) {
+      fetchPromisesMap[BASE_URL] =
+        fetchPromisesMap[BASE_URL] || fetchModels({ baseURL: BASE_URL, apiKey: API_KEY });
+      baseUrlToNameMap[BASE_URL] = baseUrlToNameMap[BASE_URL] || [];
+      baseUrlToNameMap[BASE_URL].push(name);
+      continue;
+    }
+
+    if (Array.isArray(models.default)) {
+      modelsConfig[name] = models.default;
+    }
+  }
+
+  const fetchedData = await Promise.all(Object.values(fetchPromisesMap));
+  const baseUrls = Object.keys(fetchPromisesMap);
+
+  for (let i = 0; i < fetchedData.length; i++) {
+    const currentBaseUrl = baseUrls[i];
+    const modelData = fetchedData[i];
+    const associatedNames = baseUrlToNameMap[currentBaseUrl];
+
+    for (const name of associatedNames) {
+      modelsConfig[name] = modelData;
+    }
+  }
+
+  return modelsConfig;
+}
+
+module.exports = loadConfigModels;

api/server/services/Config/loadCustomConfig.js

@@ -0,0 +1,41 @@
+const path = require('path');
+const { CacheKeys, configSchema } = require('librechat-data-provider');
+const loadYaml = require('~/utils/loadYaml');
+const { getLogStores } = require('~/cache');
+const { logger } = require('~/config');
+
+const projectRoot = path.resolve(__dirname, '..', '..', '..', '..');
+const configPath = path.resolve(projectRoot, 'librechat.yaml');
+
+/**
+ * Load custom configuration files and caches the object if the `cache` field at root is true.
+ * Validation via parsing the config file with the config schema.
+ * @function loadCustomConfig
+ * @returns {Promise<null | Object>} A promise that resolves to null or the custom config object.
+ * */
+
+async function loadCustomConfig() {
+  const customConfig = loadYaml(configPath);
+  if (!customConfig) {
+    return null;
+  }
+
+  const result = configSchema.strict().safeParse(customConfig);
+  if (!result.success) {
+    logger.error(`Invalid custom config file at ${configPath}`, result.error);
+    return null;
+  } else {
+    logger.info('Loaded custom config file');
+  }
+
+  if (customConfig.cache) {
+    const cache = getLogStores(CacheKeys.CONFIG_STORE);
+    await cache.set(CacheKeys.CUSTOM_CONFIG, customConfig);
+  }
+
+  // TODO: handle remote config
+
+  return customConfig;
+}
+
+module.exports = loadCustomConfig;

api/server/services/Endpoints/custom/buildOptions.js

@@ -0,0 +1,16 @@
+const buildOptions = (endpoint, parsedBody, endpointType) => {
+  const { chatGptLabel, promptPrefix, ...rest } = parsedBody;
+  const endpointOption = {
+    endpoint,
+    endpointType,
+    chatGptLabel,
+    promptPrefix,
+    modelOptions: {
+      ...rest,
+    },
+  };
+
+  return endpointOption;
+};
+
+module.exports = buildOptions;

api/server/services/Endpoints/custom/index.js

@@ -0,0 +1,7 @@
+const initializeClient = require('./initializeClient');
+const buildOptions = require('./buildOptions');
+
+module.exports = {
+  initializeClient,
+  buildOptions,
+};

api/server/services/Endpoints/custom/initializeClient.js

@@ -0,0 +1,79 @@
+const { EModelEndpoint } = require('librechat-data-provider');
+const { getUserKey, checkUserKeyExpiry } = require('~/server/services/UserService');
+const { isUserProvided, extractEnvVariable } = require('~/server/utils');
+const getCustomConfig = require('~/cache/getCustomConfig');
+const { OpenAIClient } = require('~/app');
+
+const { PROXY } = process.env;
+
+const initializeClient = async ({ req, res, endpointOption }) => {
+  const { key: expiresAt, endpoint } = req.body;
+  const customConfig = await getCustomConfig();
+  if (!customConfig) {
+    throw new Error(`Config not found for the ${endpoint} custom endpoint.`);
+  }
+
+  const { endpoints = {} } = customConfig;
+  const customEndpoints = endpoints[EModelEndpoint.custom] ?? [];
+  const endpointConfig = customEndpoints.find((endpointConfig) => endpointConfig.name === endpoint);
+
+  const CUSTOM_API_KEY = extractEnvVariable(endpointConfig.apiKey);
+  const CUSTOM_BASE_URL = extractEnvVariable(endpointConfig.baseURL);
+
+  const customOptions = {
+    addParams: endpointConfig.addParams,
+    dropParams: endpointConfig.dropParams,
+    titleConvo: endpointConfig.titleConvo,
+    titleModel: endpointConfig.titleModel,
+    forcePrompt: endpointConfig.forcePrompt,
+    summaryModel: endpointConfig.summaryModel,
+    modelDisplayLabel: endpointConfig.modelDisplayLabel,
+    titleMethod: endpointConfig.titleMethod ?? 'completion',
+    contextStrategy: endpointConfig.summarize ? 'summarize' : null,
+  };
+
+  const useUserKey = isUserProvided(CUSTOM_API_KEY);
+  const useUserURL = isUserProvided(CUSTOM_BASE_URL);
+
+  let userValues = null;
+  if (expiresAt && (useUserKey || useUserURL)) {
+    checkUserKeyExpiry(
+      expiresAt,
+      `Your API values for ${endpoint} have expired. Please configure them again.`,
+    );
+    userValues = await getUserKey({ userId: req.user.id, name: endpoint });
+    try {
+      userValues = JSON.parse(userValues);
+    } catch (e) {
+      throw new Error(`Invalid JSON provided for ${endpoint} user values.`);
+    }
+  }
+
+  let apiKey = useUserKey ? userValues.apiKey : CUSTOM_API_KEY;
+  let baseURL = useUserURL ? userValues.baseURL : CUSTOM_BASE_URL;
+
+  if (!apiKey) {
+    throw new Error(`${endpoint} API key not provided.`);
+  }
+
+  if (!baseURL) {
+    throw new Error(`${endpoint} Base URL not provided.`);
+  }
+
+  const clientOptions = {
+    reverseProxyUrl: baseURL ?? null,
+    proxy: PROXY ?? null,
+    req,
+    res,
+    ...customOptions,
+    ...endpointOption,
+  };
+
+  const client = new OpenAIClient(apiKey, clientOptions);
+  return {
+    client,
+    openAIApiKey: apiKey,
+  };
+};
+
+module.exports = initializeClient;

api/server/services/Endpoints/openAI/addTitle.js

@@ -7,6 +7,10 @@ const addTitle = async (req, { text, response, client }) => {
     return;
   }
 
+  if (client.options.titleConvo === false) {
+    return;
+  }
+
   // If the request was aborted and is not azure, don't generate the title.
   if (!client.azure && client.abortController.signal.aborted) {
     return;

api/server/services/ModelService.js

@@ -24,15 +24,53 @@ const {
   PROXY,
 } = process.env ?? {};
 
+/**
+ * Fetches OpenAI models from the specified base API path or Azure, based on the provided configuration.
+ *
+ * @param {Object} params - The parameters for fetching the models.
+ * @param {string} params.apiKey - The API key for authentication with the API.
+ * @param {string} params.baseURL - The base path URL for the API.
+ * @param {string} [params.name='OpenAI'] - The name of the API; defaults to 'OpenAI'.
+ * @param {boolean} [params.azure=false] - Whether to fetch models from Azure.
+ * @returns {Promise<string[]>} A promise that resolves to an array of model identifiers.
+ * @async
+ */
+const fetchModels = async ({ apiKey, baseURL, name = 'OpenAI', azure = false }) => {
+  let models = [];
+
+  if (!baseURL && !azure) {
+    return models;
+  }
+
+  try {
+    const payload = {
+      headers: {
+        Authorization: `Bearer ${apiKey}`,
+      },
+    };
+
+    if (PROXY) {
+      payload.httpsAgent = new HttpsProxyAgent(PROXY);
+    }
+
+    const res = await axios.get(`${baseURL}${azure ? '' : '/models'}`, payload);
+    models = res.data.data.map((item) => item.id);
+  } catch (err) {
+    logger.error(`Failed to fetch models from ${azure ? 'Azure ' : ''}${name} API`, err);
+  }
+
+  return models;
+};
+
 const fetchOpenAIModels = async (opts = { azure: false, plugins: false }, _models = []) => {
   let models = _models.slice() ?? [];
   let apiKey = openAIApiKey;
-  let basePath = 'https://api.openai.com/v1';
+  let baseURL = 'https://api.openai.com/v1';
   let reverseProxyUrl = OPENAI_REVERSE_PROXY;
   if (opts.azure) {
     return models;
     // const azure = getAzureCredentials();
-    // basePath = (genAzureChatCompletion(azure))
+    // baseURL = (genAzureChatCompletion(azure))
     //   .split('/deployments')[0]
     //   .concat(`/models?api-version=${azure.azureOpenAIApiVersion}`);
     // apiKey = azureOpenAIApiKey;
@@ -42,32 +80,20 @@ const fetchOpenAIModels = async (opts = { azure: false, plugins: false }, _model
   }
 
   if (reverseProxyUrl) {
-    basePath = extractBaseURL(reverseProxyUrl);
+    baseURL = extractBaseURL(reverseProxyUrl);
   }
 
-  const cachedModels = await modelsCache.get(basePath);
+  const cachedModels = await modelsCache.get(baseURL);
   if (cachedModels) {
     return cachedModels;
   }
 
-  if (basePath || opts.azure) {
-    try {
-      const payload = {
-        headers: {
-          Authorization: `Bearer ${apiKey}`,
-        },
-      };
-
-      if (PROXY) {
-        payload.httpsAgent = new HttpsProxyAgent(PROXY);
-      }
-      const res = await axios.get(`${basePath}${opts.azure ? '' : '/models'}`, payload);
-
-      models = res.data.data.map((item) => item.id);
-      // logger.debug(`Fetched ${models.length} models from ${opts.azure ? 'Azure ' : ''}OpenAI API`);
-    } catch (err) {
-      logger.error(`Failed to fetch models from ${opts.azure ? 'Azure ' : ''}OpenAI API`, err);
-    }
+  if (baseURL || opts.azure) {
+    models = await fetchModels({
+      apiKey,
+      baseURL,
+      azure: opts.azure,
+    });
   }
 
   if (!reverseProxyUrl) {
@@ -75,7 +101,7 @@ const fetchOpenAIModels = async (opts = { azure: false, plugins: false }, _model
     models = models.filter((model) => regex.test(model));
   }
 
-  await modelsCache.set(basePath, models);
+  await modelsCache.set(baseURL, models);
   return models;
 };
 
@@ -142,6 +168,7 @@ const getGoogleModels = () => {
 };
 
 module.exports = {
+  fetchModels,
   getOpenAIModels,
   getChatGPTBrowserModels,
   getAnthropicModels,

api/server/utils/handleText.js

@@ -165,6 +165,27 @@ function isEnabled(value) {
   return false;
 }
 
+/**
+ * Checks if the provided value is 'user_provided'.
+ *
+ * @param {string} value - The value to check.
+ * @returns {boolean} - Returns true if the value is 'user_provided', otherwise false.
+ */
+const isUserProvided = (value) => value === 'user_provided';
+
+/**
+ * Extracts the value of an environment variable from a string.
+ * @param {string} value - The value to be processed, possibly containing an env variable placeholder.
+ * @returns {string} - The actual value from the environment variable or the original value.
+ */
+function extractEnvVariable(value) {
+  const envVarMatch = value.match(/^\${(.+)}$/);
+  if (envVarMatch) {
+    return process.env[envVarMatch[1]] || value;
+  }
+  return value;
+}
+
 module.exports = {
   createOnProgress,
   isEnabled,
@@ -172,4 +193,6 @@ module.exports = {
   formatSteps,
   formatAction,
   addSpaceIfNeeded,
+  isUserProvided,
+  extractEnvVariable,
 };

api/server/utils/handleText.spec.js

@@ -1,4 +1,4 @@
-const { isEnabled } = require('./handleText');
+const { isEnabled, extractEnvVariable } = require('./handleText');
 
 describe('isEnabled', () => {
   test('should return true when input is "true"', () => {
@@ -48,4 +48,51 @@ describe('isEnabled', () => {
   test('should return false when input is an array', () => {
     expect(isEnabled([])).toBe(false);
   });
+
+  describe('extractEnvVariable', () => {
+    const originalEnv = process.env;
+
+    beforeEach(() => {
+      jest.resetModules();
+      process.env = { ...originalEnv };
+    });
+
+    afterAll(() => {
+      process.env = originalEnv;
+    });
+
+    test('should return the value of the environment variable', () => {
+      process.env.TEST_VAR = 'test_value';
+      expect(extractEnvVariable('${TEST_VAR}')).toBe('test_value');
+    });
+
+    test('should return the original string if the envrionment variable is not defined correctly', () => {
+      process.env.TEST_VAR = 'test_value';
+      expect(extractEnvVariable('${ TEST_VAR }')).toBe('${ TEST_VAR }');
+    });
+
+    test('should return the original string if environment variable is not set', () => {
+      expect(extractEnvVariable('${NON_EXISTENT_VAR}')).toBe('${NON_EXISTENT_VAR}');
+    });
+
+    test('should return the original string if it does not contain an environment variable', () => {
+      expect(extractEnvVariable('some_string')).toBe('some_string');
+    });
+
+    test('should handle empty strings', () => {
+      expect(extractEnvVariable('')).toBe('');
+    });
+
+    test('should handle strings without variable format', () => {
+      expect(extractEnvVariable('no_var_here')).toBe('no_var_here');
+    });
+
+    test('should not process multiple variable formats', () => {
+      process.env.FIRST_VAR = 'first';
+      process.env.SECOND_VAR = 'second';
+      expect(extractEnvVariable('${FIRST_VAR} and ${SECOND_VAR}')).toBe(
+        '${FIRST_VAR} and ${SECOND_VAR}',
+      );
+    });
+  });
 });

api/server/utils/streamResponse.js

@@ -1,6 +1,8 @@
 const crypto = require('crypto');
+const { parseConvo } = require('librechat-data-provider');
 const { saveMessage, getMessages } = require('~/models/Message');
 const { getConvo } = require('~/models/Conversation');
+const { logger } = require('~/config');
 
 /**
  * Sends error data in Server Sent Events format and ends the response.
@@ -65,12 +67,21 @@ const sendError = async (res, options, callback) => {
 
   if (!errorMessage.error) {
     const requestMessage = { messageId: parentMessageId, conversationId };
-    const query = await getMessages(requestMessage);
+    let query = [],
+      convo = {};
+    try {
+      query = await getMessages(requestMessage);
+      convo = await getConvo(user, conversationId);
+    } catch (err) {
+      logger.error('[sendError] Error retrieving conversation data:', err);
+      convo = parseConvo(errorMessage);
+    }
+
     return sendMessage(res, {
       final: true,
       requestMessage: query?.[0] ? query[0] : requestMessage,
       responseMessage: errorMessage,
-      conversation: await getConvo(user, conversationId),
+      conversation: convo,
     });
   }