🤖 feat: Model Specs & Save Tools per Convo/Preset (#2578)

* WIP: first pass ModelSpecs

* refactor(onSelectEndpoint): use `getConvoSwitchLogic`

* feat: introduce iconURL, greeting, frontend fields for conversations/presets/messages

* feat: conversation.iconURL & greeting in Landing

* feat: conversation.iconURL & greeting in New Chat button

* feat: message.iconURL

* refactor: ConversationIcon -> ConvoIconURL

* WIP: add spec as a conversation field

* refactor: useAppStartup, set spec on initial load for new chat, allow undefined spec, add localStorage keys enum, additional type fields for spec

* feat: handle `showIconInMenu`, `showIconInHeader`, undefined `iconURL` and no specs on initial load

* chore: handle undefined or empty modelSpecs

* WIP: first pass, modelSpec schema for custom config

* refactor: move default filtered tools definition to ToolService

* feat: pass modelSpecs from backend via startupConfig

* refactor: modelSpecs config, return and define list

* fix: react error and include iconURL in responseMessage

* refactor: add iconURL to responseMessage only

* refactor: getIconEndpoint

* refactor: pass TSpecsConfig

* fix(assistants): differentiate compactAssistantSchema, correctly resets shared conversation state with other endpoints

* refactor: assistant id prefix localStorage key

* refactor: add more LocalStorageKeys and replace hardcoded values

* feat: prioritize spec on new chat behavior: last selected modelSpec behavior (localStorage)

* feat: first pass, interface config

* chore: WIP, todo: add warnings based on config.modelSpecs settings.

* feat: enforce modelSpecs if configured

* feat: show config file yaml errors

* chore: delete unused legacy Plugins component

* refactor: set tools to localStorage from recoil store

* chore: add stable recoil setter to useEffect deps

* refactor: save tools to conversation documents

* style(MultiSelectPop): dynamic height, remove unused import

* refactor(react-query): use localstorage keys and pass config to useAvailablePluginsQuery

* feat(utils): add mapPlugins

* refactor(Convo): use conversation.tools if defined, lastSelectedTools if not

* refactor: remove unused legacy code using `useSetOptions`, remove conditional flag `isMultiChat` for using legacy settings

* refactor(PluginStoreDialog): add exhaustive-deps which are stable react state setters

* fix(HeaderOptions): pass `popover` as true

* refactor(useSetStorage): use project enums

* refactor: use LocalStorageKeys enum

* fix: prevent setConversation from setting falsy values in lastSelectedTools

* refactor: use map for availableTools state and available Plugins query

* refactor(updateLastSelectedModel): organize logic better and add note on purpose

* fix(setAgentOption): prevent reseting last model to secondary model for gptPlugins

* refactor(buildDefaultConvo): use enum

* refactor: remove `useSetStorage` and consolidate areas where conversation state is saved to localStorage

* fix: conversations retain tools on refresh

* fix(gptPlugins): prevent nullish tools from being saved

* chore: delete useServerStream

* refactor: move initial plugins logic to useAppStartup

* refactor(MultiSelectDropDown): add more pass-in className props

* feat: use tools in presets

* chore: delete unused usePresetOptions

* refactor: new agentOptions default handling

* chore: note

* feat: add label and custom instructions to agents

* chore: remove 'disabled with tools' message

* style: move plugins to 2nd column in parameters

* fix: TPreset type for agentOptions

* fix: interface controls

* refactor: add interfaceConfig, use Separator within Switcher

* refactor: hide Assistants panel if interface.parameters are disabled

* fix(Header): only modelSpecs if list is greater than 0

* refactor: separate MessageIcon logic from useMessageHelpers for better react rule-following

* fix(AppService): don't use reserved keyword 'interface'

* feat: set existing Icon for custom endpoints through iconURL

* fix(ci): tests passing for App Service

* docs: refactor custom_config.md for readability and better organization, also include missing values

* docs: interface section and re-organize docs

* docs: update modelSpecs info

* chore: remove unused files

* chore: remove unused files

* chore: move useSetIndexOptions

* chore: remove unused file

* chore: move useConversation(s)

* chore: move useDefaultConvo

* chore: move useNavigateToConvo

* refactor: use plugin install hook so it can be used elsewhere

* chore: import order

* update docs

* refactor(OpenAI/Plugins): allow modelLabel as an initial value for chatGptLabel

* chore: remove unused EndpointOptionsPopover and hide 'Save as Preset' button if preset UI visibility disabled

* feat(loadDefaultInterface): issue warnings based on values

* feat: changelog for custom config file

* docs: add additional changelog note

* fix: prevent unavailable tool selection from preset and update availableTools on Plugin installations

* feat: add `filteredTools` option in custom config

* chore: changelog

* fix(MessageIcon): always overwrite conversation.iconURL in messageSettings

* fix(ModelSpecsMenu): icon edge cases

* fix(NewChat): dynamic icon

* fix(PluginsClient): always include endpoint in responseMessage

* fix: always include endpoint and iconURL in responseMessage across different response methods

* feat: interchangeable keys for modelSpec enforcing

api/server/controllers/PluginController.js

@@ -55,6 +55,9 @@ const getAvailablePluginsController = async (req, res) => {
       return;
     }
 
+    /** @type {{ filteredTools: string[] }} */
+    const { filteredTools = [] } = req.app.locals;
+
     const pluginManifest = await fs.readFile(req.app.locals.paths.pluginManifest, 'utf8');
 
     const jsonData = JSON.parse(pluginManifest);
@@ -67,7 +70,10 @@ const getAvailablePluginsController = async (req, res) => {
         return plugin;
       }
     });
-    const plugins = await addOpenAPISpecs(authenticatedPlugins);
+
+    let plugins = await addOpenAPISpecs(authenticatedPlugins);
+    plugins = plugins.filter((plugin) => !filteredTools.includes(plugin.pluginKey));
+
     await cache.set(CacheKeys.PLUGINS, plugins);
     res.status(200).json(plugins);
   } catch (error) {

api/server/middleware/abortMiddleware.js

@@ -73,6 +73,8 @@ const createAbortController = (req, res, getAbortData) => {
       ...responseData,
       conversationId,
       finish_reason: 'incomplete',
+      endpoint: endpointOption.endpoint,
+      iconURL: endpointOption.iconURL,
       model: endpointOption.modelOptions.model,
       unfinished: false,
       error: false,

api/server/middleware/buildEndpointOption.js

@@ -7,6 +7,8 @@ const anthropic = require('~/server/services/Endpoints/anthropic');
 const openAI = require('~/server/services/Endpoints/openAI');
 const custom = require('~/server/services/Endpoints/custom');
 const google = require('~/server/services/Endpoints/google');
+const enforceModelSpec = require('./enforceModelSpec');
+const { handleError } = require('~/server/utils');
 
 const buildFunction = {
   [EModelEndpoint.openAI]: openAI.buildOptions,
@@ -21,6 +23,31 @@ const buildFunction = {
 async function buildEndpointOption(req, res, next) {
   const { endpoint, endpointType } = req.body;
   const parsedBody = parseConvo({ endpoint, endpointType, conversation: req.body });
+
+  if (req.app.locals.modelSpecs?.list && req.app.locals.modelSpecs?.enforce) {
+    /** @type {{ list: TModelSpec[] }}*/
+    const { list } = req.app.locals.modelSpecs;
+    const { spec } = parsedBody;
+
+    if (!spec) {
+      return handleError(res, { text: 'No model spec selected' });
+    }
+
+    const currentModelSpec = list.find((s) => s.name === spec);
+    if (!currentModelSpec) {
+      return handleError(res, { text: 'Invalid model spec' });
+    }
+
+    if (endpoint !== currentModelSpec.preset.endpoint) {
+      return handleError(res, { text: 'Model spec mismatch' });
+    }
+
+    const isValidModelSpec = enforceModelSpec(currentModelSpec, parsedBody);
+    if (!isValidModelSpec) {
+      return handleError(res, { text: 'Model spec mismatch' });
+    }
+  }
+
   req.body.endpointOption = buildFunction[endpointType ?? endpoint](
     endpoint,
     parsedBody,

api/server/middleware/enforceModelSpec.js

@@ -0,0 +1,47 @@
+const interchangeableKeys = new Map([
+  ['chatGptLabel', ['modelLabel']],
+  ['modelLabel', ['chatGptLabel']],
+]);
+
+/**
+ * Middleware to enforce the model spec for a conversation
+ * @param {TModelSpec} modelSpec - The model spec to enforce
+ * @param {TConversation} parsedBody - The parsed body of the conversation
+ * @returns {boolean} - Whether the model spec is enforced
+ */
+const enforceModelSpec = (modelSpec, parsedBody) => {
+  for (const [key, value] of Object.entries(modelSpec.preset)) {
+    if (key === 'endpoint') {
+      continue;
+    }
+
+    if (!checkMatch(key, value, parsedBody)) {
+      return false;
+    }
+  }
+  return true;
+};
+
+/**
+ * Checks if there is a match for the given key and value in the parsed body
+ * or any of its interchangeable keys.
+ * @param {string} key
+ * @param {any} value
+ * @param {TConversation} parsedBody
+ * @returns {boolean}
+ */
+const checkMatch = (key, value, parsedBody) => {
+  if (parsedBody[key] === value) {
+    return true;
+  }
+
+  if (interchangeableKeys.has(key)) {
+    return interchangeableKeys
+      .get(key)
+      .some((interchangeableKey) => parsedBody[interchangeableKey] === value);
+  }
+
+  return false;
+};
+
+module.exports = enforceModelSpec;

api/server/routes/config.js

@@ -14,6 +14,7 @@ router.get('/', async function (req, res) {
   };
 
   try {
+    /** @type {TStartupConfig} */
     const payload = {
       appTitle: process.env.APP_TITLE || 'LibreChat',
       socialLogins: req.app.locals.socialLogins ?? defaultSocialLogins,
@@ -44,7 +45,8 @@ router.get('/', async function (req, res) {
         isEnabled(process.env.SHOW_BIRTHDAY_ICON) ||
         process.env.SHOW_BIRTHDAY_ICON === '',
       helpAndFaqURL: process.env.HELP_AND_FAQ_URL || 'https://librechat.ai',
-      interface: req.app.locals.interface,
+      interface: req.app.locals.interfaceConfig,
+      modelSpecs: req.app.locals.modelSpecs,
     };
 
     if (typeof process.env.CUSTOM_FOOTER === 'string') {

api/server/services/AppService.js

@@ -1,14 +1,10 @@
-const {
-  FileSources,
-  EModelEndpoint,
-  EImageOutputType,
-  defaultSocialLogins,
-} = require('librechat-data-provider');
+const { FileSources, EModelEndpoint, getConfigDefaults } = require('librechat-data-provider');
 const { checkVariables, checkHealth, checkConfig, checkAzureVariables } = require('./start/checks');
 const { azureAssistantsDefaults, assistantsConfigSetup } = require('./start/assistants');
 const { initializeFirebase } = require('./Files/Firebase/initialize');
 const loadCustomConfig = require('./Config/loadCustomConfig');
 const handleRateLimits = require('./Config/handleRateLimits');
+const { loadDefaultInterface } = require('./start/interface');
 const { azureConfigSetup } = require('./start/azureOpenAI');
 const { loadAndFormatTools } = require('./ToolService');
 const paths = require('~/config/paths');
@@ -22,9 +18,12 @@ const paths = require('~/config/paths');
 const AppService = async (app) => {
   /** @type {TCustomConfig}*/
   const config = (await loadCustomConfig()) ?? {};
+  const configDefaults = getConfigDefaults();
+
+  const filteredTools = config.filteredTools;
+  const fileStrategy = config.fileStrategy ?? configDefaults.fileStrategy;
+  const imageOutputType = config?.imageOutputType ?? configDefaults.imageOutputType;
 
-  const fileStrategy = config.fileStrategy ?? FileSources.local;
-  const imageOutputType = config?.imageOutputType ?? EImageOutputType.PNG;
   process.env.CDN_PROVIDER = fileStrategy;
 
   checkVariables();
@@ -37,24 +36,22 @@ const AppService = async (app) => {
   /** @type {Record<string, FunctionTool} */
   const availableTools = loadAndFormatTools({
     directory: paths.structuredTools,
-    filter: new Set([
-      'ChatTool.js',
-      'CodeSherpa.js',
-      'CodeSherpaTools.js',
-      'E2BTools.js',
-      'extractionChain.js',
-    ]),
+    adminFilter: filteredTools,
   });
 
-  const socialLogins = config?.registration?.socialLogins ?? defaultSocialLogins;
+  const socialLogins =
+    config?.registration?.socialLogins ?? configDefaults?.registration?.socialLogins;
+  const interfaceConfig = loadDefaultInterface(config, configDefaults);
 
   if (!Object.keys(config).length) {
     app.locals = {
       paths,
       fileStrategy,
       socialLogins,
+      filteredTools,
       availableTools,
       imageOutputType,
+      interfaceConfig,
     };
 
     return;
@@ -85,9 +82,11 @@ const AppService = async (app) => {
     paths,
     socialLogins,
     fileStrategy,
+    filteredTools,
     availableTools,
     imageOutputType,
-    interface: config?.interface,
+    interfaceConfig,
+    modelSpecs: config.modelSpecs,
     fileConfig: config?.fileConfig,
     secureImageLinks: config?.secureImageLinks,
     ...endpointLocals,

api/server/services/AppService.spec.js

@@ -93,6 +93,16 @@ describe('AppService', () => {
     expect(app.locals).toEqual({
       socialLogins: ['testLogin'],
       fileStrategy: 'testStrategy',
+      interfaceConfig: expect.objectContaining({
+        privacyPolicy: undefined,
+        termsOfService: undefined,
+        endpointsMenu: true,
+        modelSelect: true,
+        parameters: true,
+        sidePanel: true,
+        presets: true,
+      }),
+      modelSpecs: undefined,
       availableTools: {
         ExampleTool: {
           type: 'function',
@@ -109,7 +119,6 @@ describe('AppService', () => {
       },
       paths: expect.anything(),
       imageOutputType: expect.any(String),
-      interface: undefined,
       fileConfig: undefined,
       secureImageLinks: undefined,
     });
@@ -181,7 +190,6 @@ describe('AppService', () => {
 
     expect(loadAndFormatTools).toHaveBeenCalledWith({
       directory: expect.anything(),
-      filter: expect.anything(),
     });
 
     expect(app.locals.availableTools.ExampleTool).toBeDefined();

api/server/services/Config/loadCustomConfig.js

@@ -42,6 +42,12 @@ async function loadCustomConfig() {
       i === 0 && i++;
       return null;
     }
+
+    if (customConfig.reason || customConfig.stack) {
+      i === 0 && logger.error('Config file YAML format is invalid:', customConfig);
+      i === 0 && i++;
+      return null;
+    }
   }
 
   if (typeof customConfig === 'string') {
@@ -84,6 +90,10 @@ Please specify a correct \`imageOutputType\` value (case-sensitive).
     await cache.set(CacheKeys.CUSTOM_CONFIG, customConfig);
   }
 
+  if (result.data.modelSpecs) {
+    customConfig.modelSpecs = result.data.modelSpecs;
+  }
+
   return customConfig;
 }
 

api/server/services/Endpoints/anthropic/buildOptions.js

@@ -1,10 +1,13 @@
 const buildOptions = (endpoint, parsedBody) => {
-  const { modelLabel, promptPrefix, resendFiles, ...rest } = parsedBody;
+  const { modelLabel, promptPrefix, resendFiles, iconURL, greeting, spec, ...rest } = parsedBody;
   const endpointOption = {
     endpoint,
     modelLabel,
     promptPrefix,
     resendFiles,
+    iconURL,
+    greeting,
+    spec,
     modelOptions: {
       ...rest,
     },

api/server/services/Endpoints/assistants/buildOptions.js

@@ -1,10 +1,13 @@
 const buildOptions = (endpoint, parsedBody) => {
   // eslint-disable-next-line no-unused-vars
-  const { promptPrefix, assistant_id, ...rest } = parsedBody;
+  const { promptPrefix, assistant_id, iconURL, greeting, spec, ...rest } = parsedBody;
   const endpointOption = {
     endpoint,
     promptPrefix,
     assistant_id,
+    iconURL,
+    greeting,
+    spec,
     modelOptions: {
       ...rest,
     },

api/server/services/Endpoints/custom/buildOptions.js

@@ -1,5 +1,6 @@
 const buildOptions = (endpoint, parsedBody, endpointType) => {
-  const { chatGptLabel, promptPrefix, resendFiles, imageDetail, ...rest } = parsedBody;
+  const { chatGptLabel, promptPrefix, resendFiles, imageDetail, iconURL, greeting, spec, ...rest } =
+    parsedBody;
   const endpointOption = {
     endpoint,
     endpointType,
@@ -7,6 +8,9 @@ const buildOptions = (endpoint, parsedBody, endpointType) => {
     promptPrefix,
     resendFiles,
     imageDetail,
+    iconURL,
+    greeting,
+    spec,
     modelOptions: {
       ...rest,
     },

api/server/services/Endpoints/google/buildOptions.js

@@ -1,10 +1,13 @@
 const buildOptions = (endpoint, parsedBody) => {
-  const { examples, modelLabel, promptPrefix, ...rest } = parsedBody;
+  const { examples, modelLabel, promptPrefix, iconURL, greeting, spec, ...rest } = parsedBody;
   const endpointOption = {
     examples,
     endpoint,
     modelLabel,
     promptPrefix,
+    iconURL,
+    greeting,
+    spec,
     modelOptions: {
       ...rest,
     },

api/server/services/Endpoints/gptPlugins/buildOptions.js

@@ -4,25 +4,24 @@ const buildOptions = (endpoint, parsedBody) => {
     promptPrefix,
     agentOptions,
     tools,
-    model,
-    temperature,
-    top_p,
-    presence_penalty,
-    frequency_penalty,
+    iconURL,
+    greeting,
+    spec,
+    ...modelOptions
   } = parsedBody;
   const endpointOption = {
     endpoint,
-    tools: tools.map((tool) => tool.pluginKey) ?? [],
+    tools:
+      tools
+        .map((tool) => tool?.pluginKey ?? tool)
+        .filter((toolName) => typeof toolName === 'string') ?? [],
     chatGptLabel,
     promptPrefix,
     agentOptions,
-    modelOptions: {
-      model,
-      temperature,
-      top_p,
-      presence_penalty,
-      frequency_penalty,
-    },
+    iconURL,
+    greeting,
+    spec,
+    modelOptions,
   };
 
   return endpointOption;

api/server/services/Endpoints/openAI/buildOptions.js

@@ -1,11 +1,15 @@
 const buildOptions = (endpoint, parsedBody) => {
-  const { chatGptLabel, promptPrefix, resendFiles, imageDetail, ...rest } = parsedBody;
+  const { chatGptLabel, promptPrefix, resendFiles, imageDetail, iconURL, greeting, spec, ...rest } =
+    parsedBody;
   const endpointOption = {
     endpoint,
     chatGptLabel,
     promptPrefix,
     resendFiles,
     imageDetail,
+    iconURL,
+    greeting,
+    spec,
     modelOptions: {
       ...rest,
     },

api/server/services/ToolService.js

@@ -20,6 +20,14 @@ const { redactMessage } = require('~/config/parsers');
 const { sleep } = require('~/server/utils');
 const { logger } = require('~/config');
 
+const filteredTools = new Set([
+  'ChatTool.js',
+  'CodeSherpa.js',
+  'CodeSherpaTools.js',
+  'E2BTools.js',
+  'extractionChain.js',
+]);
+
 /**
  * Loads and formats tools from the specified tool directory.
  *
@@ -30,10 +38,11 @@ const { logger } = require('~/config');
  *
  * @param {object} params - The parameters for the function.
  * @param {string} params.directory - The directory path where the tools are located.
- * @param {Set<string>} [params.filter=new Set()] - A set of filenames to exclude from loading.
+ * @param {Array<string>} [params.adminFilter=[]] - Array of admin-defined tool keys to exclude from loading.
  * @returns {Record<string, FunctionTool>} An object mapping each tool's plugin key to its instance.
  */
-function loadAndFormatTools({ directory, filter = new Set() }) {
+function loadAndFormatTools({ directory, adminFilter = [] }) {
+  const filter = new Set([...adminFilter, ...filteredTools]);
   const tools = [];
   /* Structured Tools Directory */
   const files = fs.readdirSync(directory);

api/server/services/start/interface.js

@@ -0,0 +1,74 @@
+const { logger } = require('~/config');
+
+/**
+ * Loads the default interface object.
+ * @param {TCustomConfig | undefined} config - The loaded custom configuration.
+ * @param {TConfigDefaults} configDefaults - The custom configuration default values.
+ * @returns {TCustomConfig['interface']} The default interface object.
+ */
+function loadDefaultInterface(config, configDefaults) {
+  const { interface: interfaceConfig } = config ?? {};
+  const { interface: defaults } = configDefaults;
+  const hasModelSpecs = config?.modelSpecs?.list?.length > 0;
+
+  const loadedInterface = {
+    endpointsMenu:
+      interfaceConfig?.endpointsMenu ?? (hasModelSpecs ? false : defaults.endpointsMenu),
+    modelSelect: interfaceConfig?.modelSelect ?? (hasModelSpecs ? false : defaults.modelSelect),
+    parameters: interfaceConfig?.parameters ?? (hasModelSpecs ? false : defaults.parameters),
+    presets: interfaceConfig?.presets ?? (hasModelSpecs ? false : defaults.presets),
+    sidePanel: interfaceConfig?.sidePanel ?? defaults.sidePanel,
+    privacyPolicy: interfaceConfig?.privacyPolicy ?? defaults.privacyPolicy,
+    termsOfService: interfaceConfig?.termsOfService ?? defaults.termsOfService,
+  };
+
+  let i = 0;
+  const logSettings = () => {
+    // log interface object and model specs object (without list) for reference
+    logger.warn(`\`interface\` settings:\n${JSON.stringify(loadedInterface, null, 2)}`);
+    logger.warn(
+      `\`modelSpecs\` settings:\n${JSON.stringify(
+        { ...(config?.modelSpecs ?? {}), list: undefined },
+        null,
+        2,
+      )}`,
+    );
+  };
+
+  // warn about config.modelSpecs.prioritize if true and presets are enabled, that default presets will conflict with prioritizing model specs.
+  if (config?.modelSpecs?.prioritize && loadedInterface.presets) {
+    logger.warn(
+      'Note: Prioritizing model specs can conflict with default presets if a default preset is set. It\'s recommended to disable presets from the interface or disable use of a default preset.',
+    );
+    i === 0 && i++;
+  }
+
+  // warn about config.modelSpecs.enforce if true and if any of these, endpointsMenu, modelSelect, presets, or parameters are enabled, that enforcing model specs can conflict with these options.
+  if (
+    config?.modelSpecs?.enforce &&
+    (loadedInterface.endpointsMenu ||
+      loadedInterface.modelSelect ||
+      loadedInterface.presets ||
+      loadedInterface.parameters)
+  ) {
+    logger.warn(
+      'Note: Enforcing model specs can conflict with the interface options: endpointsMenu, modelSelect, presets, and parameters. It\'s recommended to disable these options from the interface or disable enforcing model specs.',
+    );
+    i === 0 && i++;
+  }
+  // warn if enforce is true and prioritize is not, that enforcing model specs without prioritizing them can lead to unexpected behavior.
+  if (config?.modelSpecs?.enforce && !config?.modelSpecs?.prioritize) {
+    logger.warn(
+      'Note: Enforcing model specs without prioritizing them can lead to unexpected behavior. It\'s recommended to enable prioritizing model specs if enforcing them.',
+    );
+    i === 0 && i++;
+  }
+
+  if (i > 0) {
+    logSettings();
+  }
+
+  return loadedInterface;
+}
+
+module.exports = { loadDefaultInterface };