💫 feat: Config File & Custom Endpoints (#1474)

* WIP(backend/api): custom endpoint

* WIP(frontend/client): custom endpoint

* chore: adjust typedefs for configs

* refactor: use data-provider for cache keys and rename enums and custom endpoint for better clarity and compatibility

* feat: loadYaml utility

* refactor: rename back to  from  and proof-of-concept for creating schemas from user-defined defaults

* refactor: remove custom endpoint from default endpointsConfig as it will be exclusively managed by yaml config

* refactor(EndpointController): rename variables for clarity

* feat: initial load custom config

* feat(server/utils): add simple `isUserProvided` helper

* chore(types): update TConfig type

* refactor: remove custom endpoint handling from model services as will be handled by config, modularize fetching of models

* feat: loadCustomConfig, loadConfigEndpoints, loadConfigModels

* chore: reorganize server init imports, invoke loadCustomConfig

* refactor(loadConfigEndpoints/Models): return each custom endpoint as standalone endpoint

* refactor(Endpoint/ModelController): spread config values after default (temporary)

* chore(client): fix type issues

* WIP: first pass for multiple custom endpoints
- add endpointType to Conversation schema
- add update zod schemas for both convo/presets to allow non-EModelEndpoint value as endpoint (also using type assertion)
- use `endpointType` value as `endpoint` where mapping to type is necessary using this field
- use custom defined `endpoint` value and not type for mapping to modelsConfig
- misc: add return type to `getDefaultEndpoint`
- in `useNewConvo`, add the endpointType if it wasn't already added to conversation
- EndpointsMenu: use user-defined endpoint name as Title in menu
- TODO: custom icon via custom config, change unknown to robot icon

* refactor(parseConvo): pass args as an object and change where used accordingly; chore: comment out 'create schema' code

* chore: remove unused availableModels field in TConfig type

* refactor(parseCompactConvo): pass args as an object and change where used accordingly

* feat: chat through custom endpoint

* chore(message/convoSchemas): avoid saving empty arrays

* fix(BaseClient/saveMessageToDatabase): save endpointType

* refactor(ChatRoute): show Spinner if endpointsQuery or modelsQuery are still loading, which is apparent with slow fetching of models/remote config on first serve

* fix(useConversation): assign endpointType if it's missing

* fix(SaveAsPreset): pass real endpoint and endpointType when saving Preset)

* chore: recorganize types order for TConfig, add `iconURL`

* feat: custom endpoint icon support:
- use UnknownIcon in all icon contexts
- add mistral and openrouter as known endpoints, and add their icons
- iconURL support

* fix(presetSchema): move endpointType to default schema definitions shared between convoSchema and defaults

* refactor(Settings/OpenAI): remove legacy `isOpenAI` flag

* fix(OpenAIClient): do not invoke abortCompletion on completion error

* feat: add responseSender/label support for custom endpoints:
- use defaultModelLabel field in endpointOption
- add model defaults for custom endpoints in `getResponseSender`
- add `useGetSender` hook which uses EndpointsQuery to determine `defaultModelLabel`
- include defaultModelLabel from endpointConfig in custom endpoint client options
- pass `endpointType` to `getResponseSender`

* feat(OpenAIClient): use custom options from config file

* refactor: rename `defaultModelLabel` to `modelDisplayLabel`

* refactor(data-provider): separate concerns from `schemas` into `parsers`, `config`, and fix imports elsewhere

* feat: `iconURL` and extract environment variables from custom endpoint config values

* feat: custom config validation via zod schema, rename and move to `./projectRoot/librechat.yaml`

* docs: custom config docs and examples

* fix(OpenAIClient/mistral): mistral does not allow singular system message, also add `useChatCompletion` flag to use openai-node for title completions

* fix(custom/initializeClient): extract env var and use `isUserProvided` function

* Update librechat.example.yaml

* feat(InputWithLabel): add className props, and forwardRef

* fix(streamResponse): handle error edge case where either messages or convos query throws an error

* fix(useSSE): handle errorHandler edge cases where error response is and is not properly formatted from API, especially when a conversationId is not yet provided, which ensures stream is properly closed on error

* feat: user_provided keys for custom endpoints

* fix(config/endpointSchema): do not allow default endpoint values in custom endpoint `name`

* feat(loadConfigModels): extract env variables and optimize fetching models

* feat: support custom endpoint iconURL for messages and Nav

* feat(OpenAIClient): add/dropParams support

* docs: update docs with default params, add/dropParams, and notes to use config file instead of `OPENAI_REVERSE_PROXY`

* docs: update docs with additional notes

* feat(maxTokensMap): add mistral models (32k context)

* docs: update openrouter notes

* Update ai_setup.md

* docs(custom_config): add table of contents and fix note about custom name

* docs(custom_config): reorder ToC

* Update custom_config.md

* Add note about `max_tokens` field in custom_config.md

client/src/hooks/Conversations/index.ts

@@ -1 +1,2 @@
 export { default as usePresets } from './usePresets';
+export { default as useGetSender } from './useGetSender';

client/src/hooks/Conversations/useGetSender.ts

@@ -0,0 +1,15 @@
+import { useCallback } from 'react';
+import { getResponseSender } from 'librechat-data-provider';
+import { useGetEndpointsQuery } from 'librechat-data-provider/react-query';
+import type { TEndpointOption, TEndpointsConfig } from 'librechat-data-provider';
+
+export default function useGetSender() {
+  const { data: endpointsConfig = {} as TEndpointsConfig } = useGetEndpointsQuery();
+  return useCallback(
+    (endpointOption: TEndpointOption) => {
+      const { modelDisplayLabel } = endpointsConfig[endpointOption.endpoint ?? ''] ?? {};
+      return getResponseSender({ ...endpointOption, modelDisplayLabel });
+    },
+    [endpointsConfig],
+  );
+}

client/src/hooks/Input/useTextarea.ts

@@ -1,7 +1,8 @@
-import { useEffect, useRef } from 'react';
 import debounce from 'lodash/debounce';
-import { TEndpointOption, getResponseSender } from 'librechat-data-provider';
+import { useEffect, useRef } from 'react';
+import { TEndpointOption } from 'librechat-data-provider';
 import type { KeyboardEvent } from 'react';
+import useGetSender from '~/hooks/Conversations/useGetSender';
 import { useChatContext } from '~/Providers/ChatContext';
 import useFileHandling from '~/hooks/useFileHandling';
 import useLocalize from '~/hooks/useLocalize';
@@ -14,6 +15,7 @@ export default function useTextarea({ setText, submitMessage, disabled = false }
   const isComposing = useRef(false);
   const inputRef = useRef<HTMLTextAreaElement | null>(null);
   const { handleFiles } = useFileHandling();
+  const getSender = useGetSender();
   const localize = useLocalize();
 
   const { conversationId, jailbreak } = conversation || {};
@@ -59,7 +61,7 @@ export default function useTextarea({ setText, submitMessage, disabled = false }
         return localize('com_endpoint_message_not_appendable');
       }
 
-      const sender = getResponseSender(conversation as TEndpointOption);
+      const sender = getSender(conversation as TEndpointOption);
 
       return `${localize('com_endpoint_message')} ${sender ? sender : 'ChatGPT'}…`;
     };
@@ -82,7 +84,7 @@ export default function useTextarea({ setText, submitMessage, disabled = false }
     debouncedSetPlaceholder();
 
     return () => debouncedSetPlaceholder.cancel();
-  }, [conversation, disabled, latestMessage, isNotAppendable, localize]);
+  }, [conversation, disabled, latestMessage, isNotAppendable, localize, getSender]);
 
   const handleKeyDown = (e: KeyEvent) => {
     if (e.key === 'Enter' && isSubmitting) {

client/src/hooks/Messages/useMessageHelpers.ts

@@ -1,5 +1,6 @@
-import { useEffect, useRef } from 'react';
 import copy from 'copy-to-clipboard';
+import { useEffect, useRef } from 'react';
+import { useGetEndpointsQuery } from 'librechat-data-provider/react-query';
 import type { TMessage } from 'librechat-data-provider';
 import type { TMessageProps } from '~/common';
 import Icon from '~/components/Endpoints/Icon';
@@ -7,6 +8,7 @@ import { useChatContext } from '~/Providers';
 
 export default function useMessageHelpers(props: TMessageProps) {
   const latestText = useRef('');
+  const { data: endpointsConfig } = useGetEndpointsQuery();
   const { message, currentEditId, setCurrentEditId } = props;
 
   const {
@@ -51,6 +53,7 @@ export default function useMessageHelpers(props: TMessageProps) {
   const icon = Icon({
     ...conversation,
     ...(message as TMessage),
+    iconURL: endpointsConfig?.[conversation?.endpoint ?? '']?.iconURL,
     model: message?.model ?? conversation?.model,
     size: 28.8,
   });

client/src/hooks/useChatHelpers.ts

@@ -1,18 +1,20 @@
 import { v4 } from 'uuid';
 import { useCallback, useState } from 'react';
 import { useQueryClient } from '@tanstack/react-query';
+import { QueryKeys, parseCompactConvo } from 'librechat-data-provider';
 import { useRecoilState, useResetRecoilState, useSetRecoilState } from 'recoil';
-import { QueryKeys, parseCompactConvo, getResponseSender } from 'librechat-data-provider';
-import { useGetMessagesByConvoId } from 'librechat-data-provider/react-query';
+import { useGetMessagesByConvoId, useGetEndpointsQuery } from 'librechat-data-provider/react-query';
 import type {
   TMessage,
   TSubmission,
   TEndpointOption,
   TConversation,
+  TEndpointsConfig,
   TGetConversationsResponse,
 } from 'librechat-data-provider';
 import type { TAskFunction } from '~/common';
 import useSetFilesToDelete from './useSetFilesToDelete';
+import useGetSender from './Conversations/useGetSender';
 import { useAuthContext } from './AuthContext';
 import useUserKey from './Input/useUserKey';
 import useNewConvo from './useNewConvo';
@@ -20,10 +22,12 @@ import store from '~/store';
 
 // this to be set somewhere else
 export default function useChatHelpers(index = 0, paramId: string | undefined) {
+  const { data: endpointsConfig = {} as TEndpointsConfig } = useGetEndpointsQuery();
   const [files, setFiles] = useRecoilState(store.filesByIndex(index));
   const [showStopButton, setShowStopButton] = useState(true);
   const [filesLoading, setFilesLoading] = useState(false);
   const setFilesToDelete = useSetFilesToDelete();
+  const getSender = useGetSender();
 
   const queryClient = useQueryClient();
   const { isAuthenticated } = useAuthContext();
@@ -31,7 +35,7 @@ export default function useChatHelpers(index = 0, paramId: string | undefined) {
   const { newConversation } = useNewConvo(index);
   const { useCreateConversationAtom } = store;
   const { conversation, setConversation } = useCreateConversationAtom(index);
-  const { conversationId, endpoint } = conversation ?? {};
+  const { conversationId, endpoint, endpointType } = conversation ?? {};
 
   const queryParam = paramId === 'new' ? paramId : conversationId ?? paramId ?? '';
 
@@ -151,13 +155,21 @@ export default function useChatHelpers(index = 0, paramId: string | undefined) {
     const isEditOrContinue = isEdited || isContinued;
 
     // set the endpoint option
-    const convo = parseCompactConvo(endpoint, conversation ?? {});
+    const convo = parseCompactConvo({
+      endpoint,
+      endpointType,
+      conversation: conversation ?? {},
+    });
+
+    const { modelDisplayLabel } = endpointsConfig[endpoint ?? ''] ?? {};
     const endpointOption = {
       ...convo,
       endpoint,
+      endpointType,
+      modelDisplayLabel,
       key: getExpiry(),
     } as TEndpointOption;
-    const responseSender = getResponseSender({ model: conversation?.model, ...endpointOption });
+    const responseSender = getSender({ model: conversation?.model, ...endpointOption });
 
     let currentMessages: TMessage[] | null = getMessages() ?? [];
 

client/src/hooks/useConversation.ts

@@ -7,6 +7,7 @@ import type {
   TSubmission,
   TPreset,
   TModelsConfig,
+  TEndpointsConfig,
 } from 'librechat-data-provider';
 import { buildDefaultConvo, getDefaultEndpoint } from '~/utils';
 import useOriginNavigate from './useOriginNavigate';
@@ -18,7 +19,7 @@ const useConversation = () => {
   const setMessages = useSetRecoilState<TMessagesAtom>(store.messages);
   const setSubmission = useSetRecoilState<TSubmission | null>(store.submission);
   const resetLatestMessage = useResetRecoilState(store.latestMessage);
-  const { data: endpointsConfig = {} } = useGetEndpointsQuery();
+  const { data: endpointsConfig = {} as TEndpointsConfig } = useGetEndpointsQuery();
 
   const switchToConversation = useRecoilCallback(
     ({ snapshot }) =>
@@ -37,6 +38,10 @@ const useConversation = () => {
             endpointsConfig,
           });
 
+          if (!conversation.endpointType && endpointsConfig[defaultEndpoint]?.type) {
+            conversation.endpointType = endpointsConfig[defaultEndpoint]?.type;
+          }
+
           const models = modelsConfig?.[defaultEndpoint] ?? [];
           conversation = buildDefaultConvo({
             conversation,

client/src/hooks/useDefaultConvo.ts

@@ -1,13 +1,13 @@
 import { useRecoilValue } from 'recoil';
 import { useGetEndpointsQuery } from 'librechat-data-provider/react-query';
-import type { TConversation, TPreset } from 'librechat-data-provider';
+import type { TConversation, TPreset, TEndpointsConfig } from 'librechat-data-provider';
 import { getDefaultEndpoint, buildDefaultConvo } from '~/utils';
 import store from '~/store';
 
 type TDefaultConvo = { conversation: Partial<TConversation>; preset?: Partial<TPreset> | null };
 
 const useDefaultConvo = () => {
-  const { data: endpointsConfig = {} } = useGetEndpointsQuery();
+  const { data: endpointsConfig = {} as TEndpointsConfig } = useGetEndpointsQuery();
   const modelsConfig = useRecoilValue(store.modelsConfig);
 
   const getDefaultConversation = ({ conversation, preset }: TDefaultConvo) => {

client/src/hooks/useGenerationsByLatest.ts

@@ -19,6 +19,7 @@ export default function useGenerationsByLatest({
   const { error, messageId, searchResult, finish_reason, isCreatedByUser } = message ?? {};
   const isEditableEndpoint = !![
     EModelEndpoint.openAI,
+    EModelEndpoint.custom,
     EModelEndpoint.google,
     EModelEndpoint.assistant,
     EModelEndpoint.anthropic,
@@ -39,6 +40,7 @@ export default function useGenerationsByLatest({
     !![
       EModelEndpoint.azureOpenAI,
       EModelEndpoint.openAI,
+      EModelEndpoint.custom,
       EModelEndpoint.chatGPTBrowser,
       EModelEndpoint.google,
       EModelEndpoint.bingAI,

client/src/hooks/useNewConvo.ts

@@ -7,7 +7,13 @@ import {
   useRecoilState,
   useRecoilValue,
 } from 'recoil';
-import type { TConversation, TSubmission, TPreset, TModelsConfig } from 'librechat-data-provider';
+import type {
+  TConversation,
+  TSubmission,
+  TPreset,
+  TModelsConfig,
+  TEndpointsConfig,
+} from 'librechat-data-provider';
 import { buildDefaultConvo, getDefaultEndpoint } from '~/utils';
 import { useDeleteFilesMutation } from '~/data-provider';
 import useOriginNavigate from './useOriginNavigate';
@@ -22,7 +28,7 @@ const useNewConvo = (index = 0) => {
   const [files, setFiles] = useRecoilState(store.filesByIndex(index));
   const setSubmission = useSetRecoilState<TSubmission | null>(store.submissionByIndex(index));
   const resetLatestMessage = useResetRecoilState(store.latestMessageFamily(index));
-  const { data: endpointsConfig = {} } = useGetEndpointsQuery();
+  const { data: endpointsConfig = {} as TEndpointsConfig } = useGetEndpointsQuery();
 
   const { mutateAsync } = useDeleteFilesMutation({
     onSuccess: () => {
@@ -62,6 +68,10 @@ const useNewConvo = (index = 0) => {
             endpointsConfig,
           });
 
+          if (!conversation.endpointType && endpointsConfig[defaultEndpoint]?.type) {
+            conversation.endpointType = endpointsConfig[defaultEndpoint]?.type;
+          }
+
           const models = modelsConfig?.[defaultEndpoint] ?? [];
           conversation = buildDefaultConvo({
             conversation,

client/src/hooks/useSSE.ts

@@ -1,3 +1,4 @@
+import { v4 } from 'uuid';
 import { useEffect } from 'react';
 import { useParams } from 'react-router-dom';
 import {
@@ -5,7 +6,7 @@ import {
   SSE,
   createPayload,
   tMessageSchema,
-  tConversationSchema,
+  tConvoUpdateSchema,
   EModelEndpoint,
   removeNullishValues,
 } from 'librechat-data-provider';
@@ -152,10 +153,10 @@ export default function useSSE(submission: TSubmission | null, index = 0) {
 
     let update = {} as TConversation;
     setConversation((prevState) => {
-      update = tConversationSchema.parse({
+      update = tConvoUpdateSchema.parse({
         ...prevState,
         conversationId,
-      });
+      }) as TConversation;
 
       setStorage(update);
       return update;
@@ -207,10 +208,37 @@ export default function useSSE(submission: TSubmission | null, index = 0) {
     setIsSubmitting(false);
   };
 
-  const errorHandler = (data: TResData, submission: TSubmission) => {
+  const errorHandler = ({ data, submission }: { data?: TResData; submission: TSubmission }) => {
     const { messages, message } = submission;
 
-    if (!data.conversationId) {
+    const conversationId = message?.conversationId ?? submission?.conversationId;
+    const parseErrorResponse = (data: TResData | Partial<TMessage>) => {
+      const metadata = data['responseMessage'] ?? data;
+      return tMessageSchema.parse({
+        ...metadata,
+        error: true,
+        parentMessageId: message?.messageId,
+      });
+    };
+
+    if (!data) {
+      const convoId = conversationId ?? v4();
+      const errorResponse = parseErrorResponse({
+        text: 'Error connecting to server',
+        ...submission,
+        conversationId: convoId,
+      });
+      setMessages([...messages, message, errorResponse]);
+      newConversation({ template: { conversationId: convoId } });
+      setIsSubmitting(false);
+      return;
+    }
+
+    if (!conversationId && !data.conversationId) {
+      const convoId = v4();
+      const errorResponse = parseErrorResponse(data);
+      setMessages([...messages, message, errorResponse]);
+      newConversation({ template: { conversationId: convoId } });
       setIsSubmitting(false);
       return;
     }
@@ -318,19 +346,20 @@ export default function useSSE(submission: TSubmission | null, index = 0) {
       abortConversation(message?.conversationId ?? submission?.conversationId, submission);
 
     events.onerror = function (e: MessageEvent) {
-      console.log('error in opening conn.');
+      console.log('error in server stream.');
       startupConfig?.checkBalance && balanceQuery.refetch();
       events.close();
 
-      let data = {} as TResData;
+      let data: TResData | undefined = undefined;
       try {
-        data = JSON.parse(e.data);
+        data = JSON.parse(e.data) as TResData;
       } catch (error) {
         console.error(error);
         console.log(e);
       }
 
-      errorHandler(data, { ...submission, message });
+      errorHandler({ data, submission: { ...submission, message } });
+      events.oncancel();
     };
 
     setIsSubmitting(true);

client/src/hooks/useSetIndexOptions.ts

@@ -1,5 +1,11 @@
 import { useRecoilValue, useSetRecoilState } from 'recoil';
-import { TPreset, TPlugin, tConversationSchema, EModelEndpoint } from 'librechat-data-provider';
+import {
+  TPreset,
+  TPlugin,
+  tConvoUpdateSchema,
+  EModelEndpoint,
+  TConversation,
+} from 'librechat-data-provider';
 import type { TSetExample, TSetOption, TSetOptionsPayload } from '~/common';
 import usePresetIndexOptions from './usePresetIndexOptions';
 import { useChatContext } from '~/Providers/ChatContext';
@@ -36,11 +42,12 @@ const useSetOptions: TUseSetOptions = (preset = false) => {
       setLastBingSettings({ ...lastBingSettings, jailbreak: newValue });
     }
 
-    setConversation((prevState) =>
-      tConversationSchema.parse({
-        ...prevState,
-        ...update,
-      }),
+    setConversation(
+      (prevState) =>
+        tConvoUpdateSchema.parse({
+          ...prevState,
+          ...update,
+        }) as TConversation,
     );
   };
 
@@ -51,11 +58,12 @@ const useSetOptions: TUseSetOptions = (preset = false) => {
     currentExample[type] = { content: newValue };
     current[i] = currentExample;
     update['examples'] = current;
-    setConversation((prevState) =>
-      tConversationSchema.parse({
-        ...prevState,
-        ...update,
-      }),
+    setConversation(
+      (prevState) =>
+        tConvoUpdateSchema.parse({
+          ...prevState,
+          ...update,
+        }) as TConversation,
     );
   };
 
@@ -64,11 +72,12 @@ const useSetOptions: TUseSetOptions = (preset = false) => {
     const current = conversation?.examples?.slice() || [];
     current.push({ input: { content: '' }, output: { content: '' } });
     update['examples'] = current;
-    setConversation((prevState) =>
-      tConversationSchema.parse({
-        ...prevState,
-        ...update,
-      }),
+    setConversation(
+      (prevState) =>
+        tConvoUpdateSchema.parse({
+          ...prevState,
+          ...update,
+        }) as TConversation,
     );
   };
 
@@ -77,21 +86,23 @@ const useSetOptions: TUseSetOptions = (preset = false) => {
     const current = conversation?.examples?.slice() || [];
     if (current.length <= 1) {
       update['examples'] = [{ input: { content: '' }, output: { content: '' } }];
-      setConversation((prevState) =>
-        tConversationSchema.parse({
-          ...prevState,
-          ...update,
-        }),
+      setConversation(
+        (prevState) =>
+          tConvoUpdateSchema.parse({
+            ...prevState,
+            ...update,
+          }) as TConversation,
       );
       return;
     }
     current.pop();
     update['examples'] = current;
-    setConversation((prevState) =>
-      tConversationSchema.parse({
-        ...prevState,
-        ...update,
-      }),
+    setConversation(
+      (prevState) =>
+        tConvoUpdateSchema.parse({
+          ...prevState,
+          ...update,
+        }) as TConversation,
     );
   };
 
@@ -113,11 +124,12 @@ const useSetOptions: TUseSetOptions = (preset = false) => {
       lastModelUpdate.secondaryModel = newValue;
       setLastModel(lastModelUpdate);
     }
-    setConversation((prevState) =>
-      tConversationSchema.parse({
-        ...prevState,
-        agentOptions,
-      }),
+    setConversation(
+      (prevState) =>
+        tConvoUpdateSchema.parse({
+          ...prevState,
+          agentOptions,
+        }) as TConversation,
     );
   };
 
@@ -139,11 +151,12 @@ const useSetOptions: TUseSetOptions = (preset = false) => {
     }
 
     localStorage.setItem('lastSelectedTools', JSON.stringify(update['tools']));
-    setConversation((prevState) =>
-      tConversationSchema.parse({
-        ...prevState,
-        ...update,
-      }),
+    setConversation(
+      (prevState) =>
+        tConvoUpdateSchema.parse({
+          ...prevState,
+          ...update,
+        }) as TConversation,
     );
   };