🚀 feat: Use Model Specs + Specific Endpoints, Limit Providers for Agents (#6650)

* 🔧 refactor: Remove modelSpecs prop from ModelSelector and related components

* fix: Update submission.conversationId references in SSE hooks and data types as was incorrectly typed

* feat: Allow showing specific endpoints alongside model specs via `addedEndpoints` field

* feat: allowed agents providers via `agents.allowedProviders` field

* fix: bump dicebear/sharp dependencies to resolve CVE-2024-12905 and improve avatar gen logic

* fix: rename variable for clarity in loadDefaultInterface function

* fix: add keepAddedConvos option to newConversation calls for modular chat support

* fix: include model information in endpoint selection for improved context

* fix: update data-provider version to 0.7.78 and increment config version to 1.2.4

api/package.json

@@ -104,7 +104,7 @@
     "passport-ldapauth": "^3.0.1",
     "passport-local": "^1.0.0",
     "rate-limit-redis": "^4.2.0",
-    "sharp": "^0.32.6",
+    "sharp": "^0.33.5",
     "tiktoken": "^1.0.15",
     "traverse": "^0.6.7",
     "ua-parser-js": "^1.0.36",

api/server/services/AppService.js

@@ -146,7 +146,7 @@ const AppService = async (app) => {
     ...defaultLocals,
     fileConfig: config?.fileConfig,
     secureImageLinks: config?.secureImageLinks,
-    modelSpecs: processModelSpecs(endpoints, config.modelSpecs),
+    modelSpecs: processModelSpecs(endpoints, config.modelSpecs, interfaceConfig),
     ...endpointLocals,
   };
 };

api/server/services/Config/getEndpointsConfig.js

@@ -33,10 +33,12 @@ async function getEndpointsConfig(req) {
     };
   }
   if (mergedConfig[EModelEndpoint.agents] && req.app.locals?.[EModelEndpoint.agents]) {
-    const { disableBuilder, capabilities, ..._rest } = req.app.locals[EModelEndpoint.agents];
+    const { disableBuilder, capabilities, allowedProviders, ..._rest } =
+      req.app.locals[EModelEndpoint.agents];
 
     mergedConfig[EModelEndpoint.agents] = {
       ...mergedConfig[EModelEndpoint.agents],
+      allowedProviders,
       disableBuilder,
       capabilities,
     };

api/server/services/Endpoints/agents/initialize.js

@@ -1,5 +1,6 @@
 const { createContentAggregator, Providers } = require('@librechat/agents');
 const {
+  ErrorTypes,
   EModelEndpoint,
   getResponseSender,
   AgentCapabilities,
@@ -117,6 +118,7 @@ function optionalChainWithEmptyCheck(...values) {
  * @param {ServerRequest} params.req
  * @param {ServerResponse} params.res
  * @param {Agent} params.agent
+ * @param {Set<string>} [params.allowedProviders]
  * @param {object} [params.endpointOption]
  * @param {boolean} [params.isInitialAgent]
  * @returns {Promise<Agent>}
@@ -126,8 +128,14 @@ const initializeAgentOptions = async ({
   res,
   agent,
   endpointOption,
+  allowedProviders,
   isInitialAgent = false,
 }) => {
+  if (allowedProviders.size > 0 && !allowedProviders.has(agent.provider)) {
+    throw new Error(
+      `{ "type": "${ErrorTypes.INVALID_AGENT_PROVIDER}", "info": "${agent.provider}" }`,
+    );
+  }
   let currentFiles;
   /** @type {Array<MongoFile>} */
   const requestFiles = req.body.files ?? [];
@@ -263,6 +271,8 @@ const initializeClient = async ({ req, res, endpointOption }) => {
   }
 
   const agentConfigs = new Map();
+  /** @type {Set<string>} */
+  const allowedProviders = new Set(req?.app?.locals?.[EModelEndpoint.agents]?.allowedProviders);
 
   // Handle primary agent
   const primaryConfig = await initializeAgentOptions({
@@ -270,6 +280,7 @@ const initializeClient = async ({ req, res, endpointOption }) => {
     res,
     agent: primaryAgent,
     endpointOption,
+    allowedProviders,
     isInitialAgent: true,
   });
 
@@ -285,6 +296,7 @@ const initializeClient = async ({ req, res, endpointOption }) => {
         res,
         agent,
         endpointOption,
+        allowedProviders,
       });
       agentConfigs.set(agentId, config);
     }

api/server/services/start/interface.js

@@ -18,12 +18,15 @@ async function loadDefaultInterface(config, configDefaults, roleName = SystemRol
   const { interface: interfaceConfig } = config ?? {};
   const { interface: defaults } = configDefaults;
   const hasModelSpecs = config?.modelSpecs?.list?.length > 0;
+  const includesAddedEndpoints = config?.modelSpecs?.addedEndpoints?.length > 0;
 
   /** @type {TCustomConfig['interface']} */
   const loadedInterface = removeNullishValues({
     endpointsMenu:
       interfaceConfig?.endpointsMenu ?? (hasModelSpecs ? false : defaults.endpointsMenu),
-    modelSelect: interfaceConfig?.modelSelect ?? (hasModelSpecs ? false : defaults.modelSelect),
+    modelSelect:
+      interfaceConfig?.modelSelect ??
+      (hasModelSpecs ? includesAddedEndpoints : defaults.modelSelect),
     parameters: interfaceConfig?.parameters ?? (hasModelSpecs ? false : defaults.parameters),
     presets: interfaceConfig?.presets ?? (hasModelSpecs ? false : defaults.presets),
     sidePanel: interfaceConfig?.sidePanel ?? defaults.sidePanel,

api/server/services/start/modelSpecs.js

@@ -6,9 +6,10 @@ const { logger } = require('~/config');
  * Sets up Model Specs from the config (`librechat.yaml`) file.
  * @param {TCustomConfig['endpoints']} [endpoints] - The loaded custom configuration for endpoints.
  * @param {TCustomConfig['modelSpecs'] | undefined} [modelSpecs] - The loaded custom configuration for model specs.
+ * @param {TCustomConfig['interface'] | undefined} [interfaceConfig] - The loaded interface configuration.
  * @returns {TCustomConfig['modelSpecs'] | undefined} The processed model specs, if any.
  */
-function processModelSpecs(endpoints, _modelSpecs) {
+function processModelSpecs(endpoints, _modelSpecs, interfaceConfig) {
   if (!_modelSpecs) {
     return undefined;
   }
@@ -20,6 +21,19 @@ function processModelSpecs(endpoints, _modelSpecs) {
 
   const customEndpoints = endpoints?.[EModelEndpoint.custom] ?? [];
 
+  if (interfaceConfig.modelSelect !== true && _modelSpecs.addedEndpoints.length > 0) {
+    logger.warn(
+      `To utilize \`addedEndpoints\`, which allows provider/model selections alongside model specs, set \`modelSelect: true\` in the interface configuration.
+
+      Example:
+      \`\`\`yaml
+      interface:
+        modelSelect: true
+      \`\`\`
+      `,
+    );
+  }
+
   for (const spec of list) {
     if (EModelEndpoint[spec.preset.endpoint] && spec.preset.endpoint !== EModelEndpoint.custom) {
       modelSpecs.push(spec);

client/package.json

@@ -31,8 +31,8 @@
     "@ariakit/react": "^0.4.15",
     "@ariakit/react-core": "^0.4.15",
     "@codesandbox/sandpack-react": "^2.19.10",
-    "@dicebear/collection": "^7.0.4",
-    "@dicebear/core": "^7.0.4",
+    "@dicebear/collection": "^9.2.2",
+    "@dicebear/core": "^9.2.2",
     "@headlessui/react": "^2.1.2",
     "@radix-ui/react-accordion": "^1.1.2",
     "@radix-ui/react-alert-dialog": "^1.0.2",

client/src/common/selector.ts

@@ -20,5 +20,4 @@ export interface SelectedValues {
 
 export interface ModelSelectorProps {
   startupConfig: TStartupConfig | undefined;
-  modelSpecs: TModelSpec[];
 }

client/src/components/Chat/Header.tsx

@@ -16,7 +16,6 @@ const defaultInterface = getConfigDefaults().interface;
 export default function Header() {
   const { data: startupConfig } = useGetStartupConfig();
   const { navVisible } = useOutletContext<ContextType>();
-  const modelSpecs = useMemo(() => startupConfig?.modelSpecs?.list ?? [], [startupConfig]);
   const interfaceConfig = useMemo(
     () => startupConfig?.interface ?? defaultInterface,
     [startupConfig],
@@ -39,7 +38,7 @@ export default function Header() {
       <div className="hide-scrollbar flex w-full items-center justify-between gap-2 overflow-x-auto">
         <div className="mx-2 flex items-center gap-2">
           {!navVisible && <HeaderNewChat />}
-          {<ModelSelector startupConfig={startupConfig} modelSpecs={modelSpecs} />}
+          {<ModelSelector startupConfig={startupConfig} />}
           {interfaceConfig.presets === true && interfaceConfig.modelSelect && <PresetsMenu />}
           {hasAccessToBookmarks === true && <BookmarkMenu />}
           {hasAccessToMultiConvo === true && <AddMultiConvo />}

client/src/components/Chat/Menus/Endpoints/ModelSelector.tsx

@@ -98,9 +98,9 @@ function ModelSelectorContent() {
   );
 }
 
-export default function ModelSelector({ startupConfig, modelSpecs }: ModelSelectorProps) {
+export default function ModelSelector({ startupConfig }: ModelSelectorProps) {
   return (
-    <ModelSelectorProvider modelSpecs={modelSpecs} startupConfig={startupConfig}>
+    <ModelSelectorProvider startupConfig={startupConfig}>
       <ModelSelectorContent />
     </ModelSelectorProvider>
   );

client/src/components/Chat/Menus/Endpoints/ModelSelectorContext.tsx

@@ -44,24 +44,20 @@ export function useModelSelectorContext() {
 
 interface ModelSelectorProviderProps {
   children: React.ReactNode;
-  modelSpecs: t.TModelSpec[];
   startupConfig: t.TStartupConfig | undefined;
 }
 
-export function ModelSelectorProvider({
-  children,
-  modelSpecs,
-  startupConfig,
-}: ModelSelectorProviderProps) {
+export function ModelSelectorProvider({ children, startupConfig }: ModelSelectorProviderProps) {
   const agentsMap = useAgentsMapContext();
   const assistantsMap = useAssistantsMapContext();
   const { data: endpointsConfig } = useGetEndpointsQuery();
   const { conversation, newConversation } = useChatContext();
+  const modelSpecs = useMemo(() => startupConfig?.modelSpecs?.list ?? [], [startupConfig]);
   const { mappedEndpoints, endpointRequiresUserKey } = useEndpoints({
     agentsMap,
     assistantsMap,
-    endpointsConfig,
     startupConfig,
+    endpointsConfig,
   });
   const { onSelectEndpoint, onSelectSpec } = useSelectMention({
     // presets,
@@ -146,6 +142,7 @@ export function ModelSelectorProvider({
     if (isAgentsEndpoint(endpoint.value)) {
       onSelectEndpoint?.(endpoint.value, {
         agent_id: model,
+        model: agentsMap?.[model]?.model ?? '',
       });
     } else if (isAssistantsEndpoint(endpoint.value)) {
       onSelectEndpoint?.(endpoint.value, {
@@ -157,7 +154,7 @@ export function ModelSelectorProvider({
     }
     setSelectedValues({
       endpoint: endpoint.value,
-      model: model,
+      model,
       modelSpec: '',
     });
   };

client/src/components/Messages/Content/Error.tsx

@@ -1,5 +1,5 @@
 // file deepcode ignore HardcodedNonCryptoSecret: No hardcoded secrets
-import { ViolationTypes, ErrorTypes } from 'librechat-data-provider';
+import { ViolationTypes, ErrorTypes, alternateName } from 'librechat-data-provider';
 import type { TOpenAIMessage } from 'librechat-data-provider';
 import type { LocalizeFunction } from '~/common';
 import { formatJSON, extractJson, isJson } from '~/utils/json';
@@ -53,6 +53,11 @@ const errorMessages = {
     const { info } = json;
     return localize('com_error_input_length', { 0: info });
   },
+  [ErrorTypes.INVALID_AGENT_PROVIDER]: (json: TGenericError, localize: LocalizeFunction) => {
+    const { info } = json;
+    const provider = (alternateName[info] as string | undefined) ?? info;
+    return localize('com_error_invalid_agent_provider', { 0: provider });
+  },
   [ErrorTypes.GOOGLE_ERROR]: (json: TGenericError) => {
     const { info } = json;
     return info;

client/src/components/Nav/AccountSettings.tsx

@@ -1,6 +1,6 @@
+import { useState, memo } from 'react';
 import { useRecoilState } from 'recoil';
 import * as Select from '@ariakit/react/select';
-import { Fragment, useState, memo } from 'react';
 import { FileText, LogOut } from 'lucide-react';
 import { LinkIcon, GearIcon, DropdownMenuSeparator } from '~/components';
 import { useGetStartupConfig, useGetUserBalance } from '~/data-provider';
@@ -23,7 +23,7 @@ function AccountSettings() {
   const [showFiles, setShowFiles] = useRecoilState(store.showFiles);
 
   const avatarSrc = useAvatar(user);
-  const name = user?.avatar ?? user?.username ?? '';
+  const avatarSeed = user?.avatar || user?.name || user?.username || '';
 
   return (
     <Select.SelectProvider>
@@ -34,7 +34,7 @@ function AccountSettings() {
       >
         <div className="-ml-0.9 -mt-0.8 h-8 w-8 flex-shrink-0">
           <div className="relative flex">
-            {name.length === 0 ? (
+            {avatarSeed.length === 0 ? (
               <div
                 style={{
                   backgroundColor: 'rgb(121, 137, 255)',
@@ -51,7 +51,7 @@ function AccountSettings() {
               <img
                 className="rounded-full"
                 src={(user?.avatar ?? '') || avatarSrc}
-                alt={`${name}'s avatar`}
+                alt={`${user?.name || user?.username || user?.email || ''}'s avatar`}
               />
             )}
           </div>

client/src/components/SidePanel/Agents/AgentPanel.tsx

@@ -56,18 +56,24 @@ export default function AgentPanel({
   const { control, handleSubmit, reset } = methods;
   const agent_id = useWatch({ control, name: 'id' });
 
+  const allowedProviders = useMemo(
+    () => new Set(agentsConfig?.allowedProviders),
+    [agentsConfig?.allowedProviders],
+  );
+
   const providers = useMemo(
     () =>
       Object.keys(endpointsConfig ?? {})
         .filter(
           (key) =>
             !isAssistantsEndpoint(key) &&
+            (allowedProviders.size > 0 ? allowedProviders.has(key) : true) &&
             key !== EModelEndpoint.agents &&
             key !== EModelEndpoint.chatGPTBrowser &&
             key !== EModelEndpoint.gptPlugins,
         )
         .map((provider) => createProviderOption(provider)),
-    [endpointsConfig],
+    [endpointsConfig, allowedProviders],
   );
 
   /* Mutations */

client/src/components/SidePanel/Agents/ModelPanel.tsx

@@ -12,7 +12,7 @@ import { getEndpointField, cn } from '~/utils';
 import { useLocalize } from '~/hooks';
 import { Panel } from '~/common';
 
-export default function Parameters({
+export default function ModelPanel({
   setActivePanel,
   providers,
   models: modelsData,

client/src/hooks/Endpoint/useEndpoints.ts

@@ -37,6 +37,10 @@ export const useEndpoints = ({
   const { data: endpoints = [] } = useGetEndpointsQuery({ select: mapEndpoints });
   const { instanceProjectId } = startupConfig ?? {};
   const interfaceConfig = startupConfig?.interface ?? {};
+  const includedEndpoints = useMemo(
+    () => new Set(startupConfig?.modelSpecs?.addedEndpoints ?? []),
+    [startupConfig?.modelSpecs?.addedEndpoints],
+  );
 
   const { endpoint } = conversation ?? {};
 
@@ -73,11 +77,14 @@ export const useEndpoints = ({
       if (endpoints[i] === EModelEndpoint.agents && !hasAgentAccess) {
         continue;
       }
+      if (includedEndpoints.size > 0 && !includedEndpoints.has(endpoints[i])) {
+        continue;
+      }
       result.push(endpoints[i]);
     }
 
     return result;
-  }, [endpoints, hasAgentAccess]);
+  }, [endpoints, hasAgentAccess, includedEndpoints]);
 
   const endpointRequiresUserKey = useCallback(
     (ep: string) => {

client/src/hooks/Input/useSelectMention.ts

@@ -172,13 +172,19 @@ export default function useSelectMention({
         });
 
         /* We don't reset the latest message, only when changing settings mid-converstion */
-        newConversation({ template: currentConvo, preset: currentConvo, keepLatestMessage: true });
+        newConversation({
+          template: currentConvo,
+          preset: currentConvo,
+          keepLatestMessage: true,
+          keepAddedConvos: true,
+        });
         return;
       }
 
       newConversation({
         template: { ...(template as Partial<TConversation>) },
         preset: { ...kwargs, spec: null, iconURL: null, modelLabel: null, endpoint: newEndpoint },
+        keepAddedConvos: isNewModular,
       });
     },
     [conversation, getDefaultConversation, modularChat, newConversation, endpointsConfig],
@@ -233,7 +239,7 @@ export default function useSelectMention({
         return;
       }
 
-      newConversation({ preset: newPreset, keepAddedConvos: true });
+      newConversation({ preset: newPreset, keepAddedConvos: isModular });
     },
     [
       modularChat,

client/src/hooks/Messages/useAvatar.ts

@@ -7,36 +7,35 @@ const avatarCache: Record<string, string> = {};
 
 const useAvatar = (user: TUser | undefined) => {
   return useMemo(() => {
-    if (!user?.username) {
+    const { username, name } = user ?? {};
+    const seed = name || username;
+    if (!seed) {
       return '';
     }
 
-    if (user.avatar) {
+    if (user?.avatar && user?.avatar !== '') {
       return user.avatar;
     }
 
-    const { username } = user;
-
-    if (avatarCache[username]) {
-      return avatarCache[username];
+    if (avatarCache[seed]) {
+      return avatarCache[seed];
     }
 
     const avatar = createAvatar(initials, {
-      seed: username,
+      seed,
       fontFamily: ['Verdana'],
       fontSize: 36,
     });
 
     let avatarDataUri = '';
-    avatar
-      .toDataUri()
-      .then((dataUri) => {
-        avatarDataUri = dataUri;
-        avatarCache[username] = dataUri; // Store in cache
-      })
-      .catch((error) => {
-        console.error('Failed to generate avatar:', error);
-      });
+    try {
+      avatarDataUri = avatar.toDataUri();
+      if (avatarDataUri) {
+        avatarCache[seed] = avatarDataUri;
+      }
+    } catch (error) {
+      console.error('Failed to generate avatar:', error);
+    }
 
     return avatarDataUri;
   }, [user]);

client/src/hooks/SSE/useEventHandlers.ts

@@ -528,7 +528,8 @@ export default function useEventHandlers({
 
       setCompleted((prev) => new Set(prev.add(initialResponse.messageId)));
 
-      const conversationId = userMessage.conversationId ?? submission.conversationId ?? '';
+      const conversationId =
+        userMessage.conversationId ?? submission.conversation?.conversationId ?? '';
 
       const parseErrorResponse = (data: TResData | Partial<TMessage>) => {
         const metadata = data['responseMessage'] ?? data;

client/src/hooks/SSE/useSSE.ts

@@ -124,7 +124,7 @@ export default function useSSE(
       const data = JSON.parse(e.data);
 
       if (data.final != null) {
-        clearDraft(submission.conversationId);
+        clearDraft(submission.conversation?.conversationId);
         const { plugins } = data;
         finalHandler(data, { ...submission, plugins } as EventSubmission);
         (startupConfig?.balance?.enabled ?? false) && balanceQuery.refetch();
@@ -190,7 +190,10 @@ export default function useSSE(
       const latestMessages = getMessages();
       const conversationId = latestMessages?.[latestMessages.length - 1]?.conversationId;
       return await abortConversation(
-        conversationId ?? userMessage.conversationId ?? submission.conversationId,
+        conversationId ??
+          userMessage.conversationId ??
+          submission.conversation?.conversationId ??
+          '',
         submission as EventSubmission,
         latestMessages,
       );

client/src/locales/en/translation.json

@@ -268,6 +268,7 @@
   "com_error_files_upload_canceled": "The file upload request was canceled. Note: the file upload may still be processing and will need to be manually deleted.",
   "com_error_files_validation": "An error occurred while validating the file.",
   "com_error_input_length": "The latest message token count is too long, exceeding the token limit, or your token limit parameters are misconfigured, adversely affecting the context window. More info: {{0}}. Please shorten your message, adjust the max context size from the conversation parameters, or fork the conversation to continue.",
+  "com_error_invalid_agent_provider": "The \"{{0}}\" provider is not available for use with Agents. Please go to your agent's settings and select a currently available provider.",
   "com_error_invalid_user_key": "Invalid key provided. Please provide a valid key and try again.",
   "com_error_moderation": "It appears that the content submitted has been flagged by our moderation system for not aligning with our community guidelines. We're unable to proceed with this specific topic. If you have any other questions or topics you'd like to explore, please edit your message, or create a new conversation.",
   "com_error_no_base_url": "No base URL found. Please provide one and try again.",

client/src/utils/forms.tsx

@@ -1,8 +1,8 @@
 import { EarthIcon } from 'lucide-react';
 import {
+  FileSources,
   alternateName,
   EModelEndpoint,
-  FileSources,
   EToolResources,
 } from 'librechat-data-provider';
 import type { Agent, TFile } from 'librechat-data-provider';

packages/data-provider/package.json

@@ -1,6 +1,6 @@
 {
   "name": "librechat-data-provider",
-  "version": "0.7.77",
+  "version": "0.7.78",
   "description": "data services for librechat apps",
   "main": "dist/index.js",
   "module": "dist/index.es.js",

packages/data-provider/src/config.ts

@@ -237,6 +237,7 @@ export const agentsEndpointSChema = baseEndpointSchema.merge(
     recursionLimit: z.number().optional(),
     disableBuilder: z.boolean().optional(),
     maxRecursionLimit: z.number().optional(),
+    allowedProviders: z.array(z.union([z.string(), eModelEndpointSchema])).optional(),
     capabilities: z
       .array(z.nativeEnum(AgentCapabilities))
       .optional()
@@ -1102,6 +1103,10 @@ export enum ErrorTypes {
    * Google provider returned an error
    */
   GOOGLE_ERROR = 'google_error',
+  /**
+   * Invalid Agent Provider (excluded by Admin)
+   */
+  INVALID_AGENT_PROVIDER = 'invalid_agent_provider',
 }
 
 /**
@@ -1214,7 +1219,7 @@ export enum Constants {
   /** Key for the app's version. */
   VERSION = 'v0.7.7',
   /** Key for the Custom Config's version (librechat.yaml). */
-  CONFIG_VERSION = '1.2.3',
+  CONFIG_VERSION = '1.2.4',
   /** Standard value for the first message's `parentMessageId` value, to indicate no parent exists. */
   NO_PARENT = '00000000-0000-0000-0000-000000000000',
   /** Standard value for the initial conversationId before a request is sent */

packages/data-provider/src/models.ts

@@ -38,6 +38,7 @@ export const specsConfigSchema = z.object({
   enforce: z.boolean().default(false),
   prioritize: z.boolean().default(true),
   list: z.array(tModelSpecSchema).min(1),
+  addedEndpoints: z.array(z.union([z.string(), eModelEndpointSchema])).optional(),
 });
 
 export type TSpecsConfig = z.infer<typeof specsConfigSchema>;

packages/data-provider/src/types.ts

@@ -57,7 +57,6 @@ export type TSubmission = {
   isTemporary: boolean;
   messages: TMessage[];
   isRegenerate?: boolean;
-  conversationId?: string;
   initialResponse?: TMessage;
   conversation: Partial<TConversation>;
   endpointOption: TEndpointOption;