💫 feat: Config File & Custom Endpoints (#1474)

* WIP(backend/api): custom endpoint

* WIP(frontend/client): custom endpoint

* chore: adjust typedefs for configs

* refactor: use data-provider for cache keys and rename enums and custom endpoint for better clarity and compatibility

* feat: loadYaml utility

* refactor: rename back to  from  and proof-of-concept for creating schemas from user-defined defaults

* refactor: remove custom endpoint from default endpointsConfig as it will be exclusively managed by yaml config

* refactor(EndpointController): rename variables for clarity

* feat: initial load custom config

* feat(server/utils): add simple `isUserProvided` helper

* chore(types): update TConfig type

* refactor: remove custom endpoint handling from model services as will be handled by config, modularize fetching of models

* feat: loadCustomConfig, loadConfigEndpoints, loadConfigModels

* chore: reorganize server init imports, invoke loadCustomConfig

* refactor(loadConfigEndpoints/Models): return each custom endpoint as standalone endpoint

* refactor(Endpoint/ModelController): spread config values after default (temporary)

* chore(client): fix type issues

* WIP: first pass for multiple custom endpoints
- add endpointType to Conversation schema
- add update zod schemas for both convo/presets to allow non-EModelEndpoint value as endpoint (also using type assertion)
- use `endpointType` value as `endpoint` where mapping to type is necessary using this field
- use custom defined `endpoint` value and not type for mapping to modelsConfig
- misc: add return type to `getDefaultEndpoint`
- in `useNewConvo`, add the endpointType if it wasn't already added to conversation
- EndpointsMenu: use user-defined endpoint name as Title in menu
- TODO: custom icon via custom config, change unknown to robot icon

* refactor(parseConvo): pass args as an object and change where used accordingly; chore: comment out 'create schema' code

* chore: remove unused availableModels field in TConfig type

* refactor(parseCompactConvo): pass args as an object and change where used accordingly

* feat: chat through custom endpoint

* chore(message/convoSchemas): avoid saving empty arrays

* fix(BaseClient/saveMessageToDatabase): save endpointType

* refactor(ChatRoute): show Spinner if endpointsQuery or modelsQuery are still loading, which is apparent with slow fetching of models/remote config on first serve

* fix(useConversation): assign endpointType if it's missing

* fix(SaveAsPreset): pass real endpoint and endpointType when saving Preset)

* chore: recorganize types order for TConfig, add `iconURL`

* feat: custom endpoint icon support:
- use UnknownIcon in all icon contexts
- add mistral and openrouter as known endpoints, and add their icons
- iconURL support

* fix(presetSchema): move endpointType to default schema definitions shared between convoSchema and defaults

* refactor(Settings/OpenAI): remove legacy `isOpenAI` flag

* fix(OpenAIClient): do not invoke abortCompletion on completion error

* feat: add responseSender/label support for custom endpoints:
- use defaultModelLabel field in endpointOption
- add model defaults for custom endpoints in `getResponseSender`
- add `useGetSender` hook which uses EndpointsQuery to determine `defaultModelLabel`
- include defaultModelLabel from endpointConfig in custom endpoint client options
- pass `endpointType` to `getResponseSender`

* feat(OpenAIClient): use custom options from config file

* refactor: rename `defaultModelLabel` to `modelDisplayLabel`

* refactor(data-provider): separate concerns from `schemas` into `parsers`, `config`, and fix imports elsewhere

* feat: `iconURL` and extract environment variables from custom endpoint config values

* feat: custom config validation via zod schema, rename and move to `./projectRoot/librechat.yaml`

* docs: custom config docs and examples

* fix(OpenAIClient/mistral): mistral does not allow singular system message, also add `useChatCompletion` flag to use openai-node for title completions

* fix(custom/initializeClient): extract env var and use `isUserProvided` function

* Update librechat.example.yaml

* feat(InputWithLabel): add className props, and forwardRef

* fix(streamResponse): handle error edge case where either messages or convos query throws an error

* fix(useSSE): handle errorHandler edge cases where error response is and is not properly formatted from API, especially when a conversationId is not yet provided, which ensures stream is properly closed on error

* feat: user_provided keys for custom endpoints

* fix(config/endpointSchema): do not allow default endpoint values in custom endpoint `name`

* feat(loadConfigModels): extract env variables and optimize fetching models

* feat: support custom endpoint iconURL for messages and Nav

* feat(OpenAIClient): add/dropParams support

* docs: update docs with default params, add/dropParams, and notes to use config file instead of `OPENAI_REVERSE_PROXY`

* docs: update docs with additional notes

* feat(maxTokensMap): add mistral models (32k context)

* docs: update openrouter notes

* Update ai_setup.md

* docs(custom_config): add table of contents and fix note about custom name

* docs(custom_config): reorder ToC

* Update custom_config.md

* Add note about `max_tokens` field in custom_config.md

api/server/services/Config/index.js

@@ -1,13 +1,19 @@
 const { config } = require('./EndpointService');
+const loadCustomConfig = require('./loadCustomConfig');
+const loadConfigModels = require('./loadConfigModels');
 const loadDefaultModels = require('./loadDefaultModels');
 const loadOverrideConfig = require('./loadOverrideConfig');
 const loadAsyncEndpoints = require('./loadAsyncEndpoints');
+const loadConfigEndpoints = require('./loadConfigEndpoints');
 const loadDefaultEndpointsConfig = require('./loadDefaultEConfig');
 
 module.exports = {
   config,
+  loadCustomConfig,
+  loadConfigModels,
   loadDefaultModels,
   loadOverrideConfig,
   loadAsyncEndpoints,
+  loadConfigEndpoints,
   loadDefaultEndpointsConfig,
 };

api/server/services/Config/loadConfigEndpoints.js

@@ -0,0 +1,54 @@
+const { CacheKeys, EModelEndpoint } = require('librechat-data-provider');
+const { isUserProvided, extractEnvVariable } = require('~/server/utils');
+const loadCustomConfig = require('./loadCustomConfig');
+const { getLogStores } = require('~/cache');
+
+/**
+ * Load config endpoints from the cached configuration object
+ * @function loadConfigEndpoints */
+async function loadConfigEndpoints() {
+  const cache = getLogStores(CacheKeys.CONFIG_STORE);
+  let customConfig = await cache.get(CacheKeys.CUSTOM_CONFIG);
+
+  if (!customConfig) {
+    customConfig = await loadCustomConfig();
+  }
+
+  if (!customConfig) {
+    return {};
+  }
+
+  const { endpoints = {} } = customConfig ?? {};
+  const endpointsConfig = {};
+
+  if (Array.isArray(endpoints[EModelEndpoint.custom])) {
+    const customEndpoints = endpoints[EModelEndpoint.custom].filter(
+      (endpoint) =>
+        endpoint.baseURL &&
+        endpoint.apiKey &&
+        endpoint.name &&
+        endpoint.models &&
+        (endpoint.models.fetch || endpoint.models.default),
+    );
+
+    for (let i = 0; i < customEndpoints.length; i++) {
+      const endpoint = customEndpoints[i];
+      const { baseURL, apiKey, name, iconURL, modelDisplayLabel } = endpoint;
+
+      const resolvedApiKey = extractEnvVariable(apiKey);
+      const resolvedBaseURL = extractEnvVariable(baseURL);
+
+      endpointsConfig[name] = {
+        type: EModelEndpoint.custom,
+        userProvide: isUserProvided(resolvedApiKey),
+        userProvideURL: isUserProvided(resolvedBaseURL),
+        modelDisplayLabel,
+        iconURL,
+      };
+    }
+  }
+
+  return endpointsConfig;
+}
+
+module.exports = loadConfigEndpoints;

api/server/services/Config/loadConfigModels.js

@@ -0,0 +1,79 @@
+const { CacheKeys, EModelEndpoint } = require('librechat-data-provider');
+const { isUserProvided, extractEnvVariable } = require('~/server/utils');
+const { fetchModels } = require('~/server/services/ModelService');
+const loadCustomConfig = require('./loadCustomConfig');
+const { getLogStores } = require('~/cache');
+
+/**
+ * Load config endpoints from the cached configuration object
+ * @function loadConfigModels */
+async function loadConfigModels() {
+  const cache = getLogStores(CacheKeys.CONFIG_STORE);
+  let customConfig = await cache.get(CacheKeys.CUSTOM_CONFIG);
+
+  if (!customConfig) {
+    customConfig = await loadCustomConfig();
+  }
+
+  if (!customConfig) {
+    return {};
+  }
+
+  const { endpoints = {} } = customConfig ?? {};
+  const modelsConfig = {};
+
+  if (!Array.isArray(endpoints[EModelEndpoint.custom])) {
+    return modelsConfig;
+  }
+
+  const customEndpoints = endpoints[EModelEndpoint.custom].filter(
+    (endpoint) =>
+      endpoint.baseURL &&
+      endpoint.apiKey &&
+      endpoint.name &&
+      endpoint.models &&
+      (endpoint.models.fetch || endpoint.models.default),
+  );
+
+  const fetchPromisesMap = {}; // Map for promises keyed by baseURL
+  const baseUrlToNameMap = {}; // Map to associate baseURLs with names
+
+  for (let i = 0; i < customEndpoints.length; i++) {
+    const endpoint = customEndpoints[i];
+    const { models, name, baseURL, apiKey } = endpoint;
+
+    const API_KEY = extractEnvVariable(apiKey);
+    const BASE_URL = extractEnvVariable(baseURL);
+
+    modelsConfig[name] = [];
+
+    if (models.fetch && !isUserProvided(API_KEY) && !isUserProvided(BASE_URL)) {
+      fetchPromisesMap[BASE_URL] =
+        fetchPromisesMap[BASE_URL] || fetchModels({ baseURL: BASE_URL, apiKey: API_KEY });
+      baseUrlToNameMap[BASE_URL] = baseUrlToNameMap[BASE_URL] || [];
+      baseUrlToNameMap[BASE_URL].push(name);
+      continue;
+    }
+
+    if (Array.isArray(models.default)) {
+      modelsConfig[name] = models.default;
+    }
+  }
+
+  const fetchedData = await Promise.all(Object.values(fetchPromisesMap));
+  const baseUrls = Object.keys(fetchPromisesMap);
+
+  for (let i = 0; i < fetchedData.length; i++) {
+    const currentBaseUrl = baseUrls[i];
+    const modelData = fetchedData[i];
+    const associatedNames = baseUrlToNameMap[currentBaseUrl];
+
+    for (const name of associatedNames) {
+      modelsConfig[name] = modelData;
+    }
+  }
+
+  return modelsConfig;
+}
+
+module.exports = loadConfigModels;

api/server/services/Config/loadCustomConfig.js

@@ -0,0 +1,41 @@
+const path = require('path');
+const { CacheKeys, configSchema } = require('librechat-data-provider');
+const loadYaml = require('~/utils/loadYaml');
+const { getLogStores } = require('~/cache');
+const { logger } = require('~/config');
+
+const projectRoot = path.resolve(__dirname, '..', '..', '..', '..');
+const configPath = path.resolve(projectRoot, 'librechat.yaml');
+
+/**
+ * Load custom configuration files and caches the object if the `cache` field at root is true.
+ * Validation via parsing the config file with the config schema.
+ * @function loadCustomConfig
+ * @returns {Promise<null | Object>} A promise that resolves to null or the custom config object.
+ * */
+
+async function loadCustomConfig() {
+  const customConfig = loadYaml(configPath);
+  if (!customConfig) {
+    return null;
+  }
+
+  const result = configSchema.strict().safeParse(customConfig);
+  if (!result.success) {
+    logger.error(`Invalid custom config file at ${configPath}`, result.error);
+    return null;
+  } else {
+    logger.info('Loaded custom config file');
+  }
+
+  if (customConfig.cache) {
+    const cache = getLogStores(CacheKeys.CONFIG_STORE);
+    await cache.set(CacheKeys.CUSTOM_CONFIG, customConfig);
+  }
+
+  // TODO: handle remote config
+
+  return customConfig;
+}
+
+module.exports = loadCustomConfig;

api/server/services/Endpoints/custom/buildOptions.js

@@ -0,0 +1,16 @@
+const buildOptions = (endpoint, parsedBody, endpointType) => {
+  const { chatGptLabel, promptPrefix, ...rest } = parsedBody;
+  const endpointOption = {
+    endpoint,
+    endpointType,
+    chatGptLabel,
+    promptPrefix,
+    modelOptions: {
+      ...rest,
+    },
+  };
+
+  return endpointOption;
+};
+
+module.exports = buildOptions;

api/server/services/Endpoints/custom/index.js

@@ -0,0 +1,7 @@
+const initializeClient = require('./initializeClient');
+const buildOptions = require('./buildOptions');
+
+module.exports = {
+  initializeClient,
+  buildOptions,
+};

api/server/services/Endpoints/custom/initializeClient.js

@@ -0,0 +1,79 @@
+const { EModelEndpoint } = require('librechat-data-provider');
+const { getUserKey, checkUserKeyExpiry } = require('~/server/services/UserService');
+const { isUserProvided, extractEnvVariable } = require('~/server/utils');
+const getCustomConfig = require('~/cache/getCustomConfig');
+const { OpenAIClient } = require('~/app');
+
+const { PROXY } = process.env;
+
+const initializeClient = async ({ req, res, endpointOption }) => {
+  const { key: expiresAt, endpoint } = req.body;
+  const customConfig = await getCustomConfig();
+  if (!customConfig) {
+    throw new Error(`Config not found for the ${endpoint} custom endpoint.`);
+  }
+
+  const { endpoints = {} } = customConfig;
+  const customEndpoints = endpoints[EModelEndpoint.custom] ?? [];
+  const endpointConfig = customEndpoints.find((endpointConfig) => endpointConfig.name === endpoint);
+
+  const CUSTOM_API_KEY = extractEnvVariable(endpointConfig.apiKey);
+  const CUSTOM_BASE_URL = extractEnvVariable(endpointConfig.baseURL);
+
+  const customOptions = {
+    addParams: endpointConfig.addParams,
+    dropParams: endpointConfig.dropParams,
+    titleConvo: endpointConfig.titleConvo,
+    titleModel: endpointConfig.titleModel,
+    forcePrompt: endpointConfig.forcePrompt,
+    summaryModel: endpointConfig.summaryModel,
+    modelDisplayLabel: endpointConfig.modelDisplayLabel,
+    titleMethod: endpointConfig.titleMethod ?? 'completion',
+    contextStrategy: endpointConfig.summarize ? 'summarize' : null,
+  };
+
+  const useUserKey = isUserProvided(CUSTOM_API_KEY);
+  const useUserURL = isUserProvided(CUSTOM_BASE_URL);
+
+  let userValues = null;
+  if (expiresAt && (useUserKey || useUserURL)) {
+    checkUserKeyExpiry(
+      expiresAt,
+      `Your API values for ${endpoint} have expired. Please configure them again.`,
+    );
+    userValues = await getUserKey({ userId: req.user.id, name: endpoint });
+    try {
+      userValues = JSON.parse(userValues);
+    } catch (e) {
+      throw new Error(`Invalid JSON provided for ${endpoint} user values.`);
+    }
+  }
+
+  let apiKey = useUserKey ? userValues.apiKey : CUSTOM_API_KEY;
+  let baseURL = useUserURL ? userValues.baseURL : CUSTOM_BASE_URL;
+
+  if (!apiKey) {
+    throw new Error(`${endpoint} API key not provided.`);
+  }
+
+  if (!baseURL) {
+    throw new Error(`${endpoint} Base URL not provided.`);
+  }
+
+  const clientOptions = {
+    reverseProxyUrl: baseURL ?? null,
+    proxy: PROXY ?? null,
+    req,
+    res,
+    ...customOptions,
+    ...endpointOption,
+  };
+
+  const client = new OpenAIClient(apiKey, clientOptions);
+  return {
+    client,
+    openAIApiKey: apiKey,
+  };
+};
+
+module.exports = initializeClient;

api/server/services/Endpoints/openAI/addTitle.js

@@ -7,6 +7,10 @@ const addTitle = async (req, { text, response, client }) => {
     return;
   }
 
+  if (client.options.titleConvo === false) {
+    return;
+  }
+
   // If the request was aborted and is not azure, don't generate the title.
   if (!client.azure && client.abortController.signal.aborted) {
     return;

api/server/services/ModelService.js

@@ -24,15 +24,53 @@ const {
   PROXY,
 } = process.env ?? {};
 
+/**
+ * Fetches OpenAI models from the specified base API path or Azure, based on the provided configuration.
+ *
+ * @param {Object} params - The parameters for fetching the models.
+ * @param {string} params.apiKey - The API key for authentication with the API.
+ * @param {string} params.baseURL - The base path URL for the API.
+ * @param {string} [params.name='OpenAI'] - The name of the API; defaults to 'OpenAI'.
+ * @param {boolean} [params.azure=false] - Whether to fetch models from Azure.
+ * @returns {Promise<string[]>} A promise that resolves to an array of model identifiers.
+ * @async
+ */
+const fetchModels = async ({ apiKey, baseURL, name = 'OpenAI', azure = false }) => {
+  let models = [];
+
+  if (!baseURL && !azure) {
+    return models;
+  }
+
+  try {
+    const payload = {
+      headers: {
+        Authorization: `Bearer ${apiKey}`,
+      },
+    };
+
+    if (PROXY) {
+      payload.httpsAgent = new HttpsProxyAgent(PROXY);
+    }
+
+    const res = await axios.get(`${baseURL}${azure ? '' : '/models'}`, payload);
+    models = res.data.data.map((item) => item.id);
+  } catch (err) {
+    logger.error(`Failed to fetch models from ${azure ? 'Azure ' : ''}${name} API`, err);
+  }
+
+  return models;
+};
+
 const fetchOpenAIModels = async (opts = { azure: false, plugins: false }, _models = []) => {
   let models = _models.slice() ?? [];
   let apiKey = openAIApiKey;
-  let basePath = 'https://api.openai.com/v1';
+  let baseURL = 'https://api.openai.com/v1';
   let reverseProxyUrl = OPENAI_REVERSE_PROXY;
   if (opts.azure) {
     return models;
     // const azure = getAzureCredentials();
-    // basePath = (genAzureChatCompletion(azure))
+    // baseURL = (genAzureChatCompletion(azure))
     //   .split('/deployments')[0]
     //   .concat(`/models?api-version=${azure.azureOpenAIApiVersion}`);
     // apiKey = azureOpenAIApiKey;
@@ -42,32 +80,20 @@ const fetchOpenAIModels = async (opts = { azure: false, plugins: false }, _model
   }
 
   if (reverseProxyUrl) {
-    basePath = extractBaseURL(reverseProxyUrl);
+    baseURL = extractBaseURL(reverseProxyUrl);
   }
 
-  const cachedModels = await modelsCache.get(basePath);
+  const cachedModels = await modelsCache.get(baseURL);
   if (cachedModels) {
     return cachedModels;
   }
 
-  if (basePath || opts.azure) {
-    try {
-      const payload = {
-        headers: {
-          Authorization: `Bearer ${apiKey}`,
-        },
-      };
-
-      if (PROXY) {
-        payload.httpsAgent = new HttpsProxyAgent(PROXY);
-      }
-      const res = await axios.get(`${basePath}${opts.azure ? '' : '/models'}`, payload);
-
-      models = res.data.data.map((item) => item.id);
-      // logger.debug(`Fetched ${models.length} models from ${opts.azure ? 'Azure ' : ''}OpenAI API`);
-    } catch (err) {
-      logger.error(`Failed to fetch models from ${opts.azure ? 'Azure ' : ''}OpenAI API`, err);
-    }
+  if (baseURL || opts.azure) {
+    models = await fetchModels({
+      apiKey,
+      baseURL,
+      azure: opts.azure,
+    });
   }
 
   if (!reverseProxyUrl) {
@@ -75,7 +101,7 @@ const fetchOpenAIModels = async (opts = { azure: false, plugins: false }, _model
     models = models.filter((model) => regex.test(model));
   }
 
-  await modelsCache.set(basePath, models);
+  await modelsCache.set(baseURL, models);
   return models;
 };
 
@@ -142,6 +168,7 @@ const getGoogleModels = () => {
 };
 
 module.exports = {
+  fetchModels,
   getOpenAIModels,
   getChatGPTBrowserModels,
   getAnthropicModels,