From 2e519f9b574693d3c202c1b5d3a6cc075a660658 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 4 Nov 2024 12:59:04 -0500 Subject: [PATCH] =?UTF-8?q?=F0=9F=A4=96=20feat:=20Custom=20Endpoint=20Agen?= =?UTF-8?q?ts=20(experimental)=20(#4627)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * wip: first pass, custom endpoint agents * chore: imports * chore: consolidate exports * fix: imports * feat: convert message.content array to strings for legacy format handling (deepseek/groq) * refactor: normalize ollama endpoint name * refactor: update mocking in isDomainAllowed.spec.js * refactor: update deepseekModels in tokens.js and tokens.spec.js --- api/app/clients/prompts/formatMessages.js | 32 ++++++++ api/server/controllers/agents/client.js | 8 ++ api/server/routes/files/multer.js | 2 +- api/server/services/Config/getCustomConfig.js | 19 ++++- api/server/services/Config/index.js | 2 +- .../services/Config/loadConfigEndpoints.js | 2 +- .../services/Config/loadConfigModels.js | 14 +++- .../services/Config/loadConfigModels.spec.js | 74 +++++++++++++++++-- .../services/Endpoints/agents/initialize.js | 57 +++++--------- .../services/Endpoints/custom/initialize.js | 27 ++++--- api/server/services/Files/Audio/STTService.js | 2 +- api/server/services/Files/Audio/TTSService.js | 6 +- .../Files/Audio/getCustomConfigSpeech.js | 2 +- api/server/services/Files/Audio/getVoices.js | 2 +- api/server/services/Files/Audio/index.js | 2 +- api/server/services/isDomainAllowed.js | 2 +- api/server/services/isDomainAllowed.spec.js | 6 +- api/typedefs.js | 23 ++++++ api/utils/tokens.js | 5 ++ api/utils/tokens.spec.js | 10 +++ package-lock.json | 2 +- packages/data-provider/package.json | 2 +- .../data-provider/src/types/assistants.ts | 2 + 23 files changed, 230 insertions(+), 73 deletions(-) diff --git a/api/app/clients/prompts/formatMessages.js b/api/app/clients/prompts/formatMessages.js index 29784d653..c9839102d 100644 --- a/api/app/clients/prompts/formatMessages.js +++ b/api/app/clients/prompts/formatMessages.js @@ -217,9 +217,41 @@ const formatAgentMessages = (payload) => { return messages; }; +/** + * Formats an array of messages for LangChain, making sure all content fields are strings + * @param {Array<(HumanMessage|AIMessage|SystemMessage|ToolMessage)>} payload - The array of messages to format. + * @returns {Array<(HumanMessage|AIMessage|SystemMessage|ToolMessage)>} - The array of formatted LangChain messages, including ToolMessages for tool calls. + */ +const formatContentStrings = (payload) => { + const messages = []; + + for (const message of payload) { + if (typeof message.content === 'string') { + continue; + } + + if (!Array.isArray(message.content)) { + continue; + } + + // Reduce text types to a single string, ignore all other types + const content = message.content.reduce((acc, curr) => { + if (curr.type === ContentTypes.TEXT) { + return `${acc}${curr[ContentTypes.TEXT]}\n`; + } + return acc; + }, ''); + + message.content = content.trim(); + } + + return messages; +}; + module.exports = { formatMessage, formatFromLangChain, formatAgentMessages, + formatContentStrings, formatLangChainMessages, }; diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js index 0534014be..c7f832db1 100644 --- a/api/server/controllers/agents/client.js +++ b/api/server/controllers/agents/client.js @@ -13,6 +13,7 @@ const { VisionModes, openAISchema, EModelEndpoint, + KnownEndpoints, anthropicSchema, bedrockOutputParser, removeNullishValues, @@ -25,6 +26,7 @@ const { const { formatMessage, formatAgentMessages, + formatContentStrings, createContextHandlers, } = require('~/app/clients/prompts'); const { encodeAndFormat } = require('~/server/services/Files/images/encode'); @@ -44,6 +46,8 @@ const providerParsers = { [EModelEndpoint.bedrock]: bedrockOutputParser, }; +const legacyContentEndpoints = new Set([KnownEndpoints.groq, KnownEndpoints.deepseek]); + class AgentClient extends BaseClient { constructor(options = {}) { super(null, options); @@ -74,6 +78,7 @@ class AgentClient extends BaseClient { this.collectedUsage = collectedUsage; /** @type {ArtifactPromises} */ this.artifactPromises = artifactPromises; + /** @type {AgentClientOptions} */ this.options = Object.assign({ endpoint: options.endpoint }, clientOptions); } @@ -478,6 +483,9 @@ class AgentClient extends BaseClient { this.run = run; const messages = formatAgentMessages(payload); + if (legacyContentEndpoints.has(this.options.agent.endpoint)) { + formatContentStrings(messages); + } await run.processStream({ messages }, config, { [Callback.TOOL_ERROR]: (graph, error, toolId) => { logger.error( diff --git a/api/server/routes/files/multer.js b/api/server/routes/files/multer.js index 76c4d50c3..4f0d38f22 100644 --- a/api/server/routes/files/multer.js +++ b/api/server/routes/files/multer.js @@ -3,7 +3,7 @@ const path = require('path'); const crypto = require('crypto'); const multer = require('multer'); const { fileConfig: defaultFileConfig, mergeFileConfig } = require('librechat-data-provider'); -const getCustomConfig = require('~/server/services/Config/getCustomConfig'); +const { getCustomConfig } = require('~/server/services/Config'); const storage = multer.diskStorage({ destination: function (req, file, cb) { diff --git a/api/server/services/Config/getCustomConfig.js b/api/server/services/Config/getCustomConfig.js index a479ca37b..6704475b5 100644 --- a/api/server/services/Config/getCustomConfig.js +++ b/api/server/services/Config/getCustomConfig.js @@ -1,4 +1,4 @@ -const { CacheKeys } = require('librechat-data-provider'); +const { CacheKeys, EModelEndpoint } = require('librechat-data-provider'); const loadCustomConfig = require('./loadCustomConfig'); const getLogStores = require('~/cache/getLogStores'); @@ -22,4 +22,19 @@ async function getCustomConfig() { return customConfig; } -module.exports = getCustomConfig; +/** + * + * @param {string | EModelEndpoint} endpoint + */ +const getCustomEndpointConfig = async (endpoint) => { + const customConfig = await getCustomConfig(); + if (!customConfig) { + throw new Error(`Config not found for the ${endpoint} custom endpoint.`); + } + + const { endpoints = {} } = customConfig; + const customEndpoints = endpoints[EModelEndpoint.custom] ?? []; + return customEndpoints.find((endpointConfig) => endpointConfig.name === endpoint); +}; + +module.exports = { getCustomConfig, getCustomEndpointConfig }; diff --git a/api/server/services/Config/index.js b/api/server/services/Config/index.js index 2e8ccb143..6dba63e8e 100644 --- a/api/server/services/Config/index.js +++ b/api/server/services/Config/index.js @@ -10,12 +10,12 @@ const loadDefaultEndpointsConfig = require('./loadDefaultEConfig'); module.exports = { config, - getCustomConfig, loadCustomConfig, loadConfigModels, loadDefaultModels, loadOverrideConfig, loadAsyncEndpoints, + ...getCustomConfig, loadConfigEndpoints, loadDefaultEndpointsConfig, }; diff --git a/api/server/services/Config/loadConfigEndpoints.js b/api/server/services/Config/loadConfigEndpoints.js index 203a461b0..25da3b427 100644 --- a/api/server/services/Config/loadConfigEndpoints.js +++ b/api/server/services/Config/loadConfigEndpoints.js @@ -1,6 +1,6 @@ const { EModelEndpoint, extractEnvVariable } = require('librechat-data-provider'); +const { getCustomConfig } = require('./getCustomConfig'); const { isUserProvided } = require('~/server/utils'); -const getCustomConfig = require('./getCustomConfig'); /** * Load config endpoints from the cached configuration object diff --git a/api/server/services/Config/loadConfigModels.js b/api/server/services/Config/loadConfigModels.js index cb0b800d7..5aca002b8 100644 --- a/api/server/services/Config/loadConfigModels.js +++ b/api/server/services/Config/loadConfigModels.js @@ -1,7 +1,16 @@ +const { Providers } = require('@librechat/agents'); const { EModelEndpoint, extractEnvVariable } = require('librechat-data-provider'); const { fetchModels } = require('~/server/services/ModelService'); +const { getCustomConfig } = require('./getCustomConfig'); const { isUserProvided } = require('~/server/utils'); -const getCustomConfig = require('./getCustomConfig'); + +/** + * @param {string} name + * @returns {string} + */ +function normalizeEndpointName(name = '') { + return name.toLowerCase() === Providers.OLLAMA ? Providers.OLLAMA : name; +} /** * Load config endpoints from the cached configuration object @@ -61,7 +70,8 @@ async function loadConfigModels(req) { for (let i = 0; i < customEndpoints.length; i++) { const endpoint = customEndpoints[i]; - const { models, name, baseURL, apiKey } = endpoint; + const { models, name: configName, baseURL, apiKey } = endpoint; + const name = normalizeEndpointName(configName); endpointsMap[name] = endpoint; const API_KEY = extractEnvVariable(apiKey); diff --git a/api/server/services/Config/loadConfigModels.spec.js b/api/server/services/Config/loadConfigModels.spec.js index 828ecd881..e7199c59d 100644 --- a/api/server/services/Config/loadConfigModels.spec.js +++ b/api/server/services/Config/loadConfigModels.spec.js @@ -1,6 +1,6 @@ const { fetchModels } = require('~/server/services/ModelService'); +const { getCustomConfig } = require('./getCustomConfig'); const loadConfigModels = require('./loadConfigModels'); -const getCustomConfig = require('./getCustomConfig'); jest.mock('~/server/services/ModelService'); jest.mock('./getCustomConfig'); @@ -253,13 +253,13 @@ describe('loadConfigModels', () => { }), ); - // For groq and Ollama, since the apiKey is "user_provided", models should not be fetched + // For groq and ollama, since the apiKey is "user_provided", models should not be fetched // Depending on your implementation's behavior regarding "default" models without fetching, // you may need to adjust the following assertions: expect(result.groq).toBe(exampleConfig.endpoints.custom[2].models.default); - expect(result.Ollama).toBe(exampleConfig.endpoints.custom[3].models.default); + expect(result.ollama).toBe(exampleConfig.endpoints.custom[3].models.default); - // Verifying fetchModels was not called for groq and Ollama + // Verifying fetchModels was not called for groq and ollama expect(fetchModels).not.toHaveBeenCalledWith( expect.objectContaining({ name: 'groq', @@ -267,7 +267,7 @@ describe('loadConfigModels', () => { ); expect(fetchModels).not.toHaveBeenCalledWith( expect.objectContaining({ - name: 'Ollama', + name: 'ollama', }), ); }); @@ -335,4 +335,68 @@ describe('loadConfigModels', () => { expect(result.FalsyFetchModel).toEqual(['defaultModel1', 'defaultModel2']); }); + + it('normalizes Ollama endpoint name to lowercase', async () => { + const testCases = [ + { + name: 'Ollama', + apiKey: 'user_provided', + baseURL: 'http://localhost:11434/v1/', + models: { + default: ['mistral', 'llama2'], + fetch: false, + }, + }, + { + name: 'OLLAMA', + apiKey: 'user_provided', + baseURL: 'http://localhost:11434/v1/', + models: { + default: ['mixtral', 'codellama'], + fetch: false, + }, + }, + { + name: 'OLLaMA', + apiKey: 'user_provided', + baseURL: 'http://localhost:11434/v1/', + models: { + default: ['phi', 'neural-chat'], + fetch: false, + }, + }, + ]; + + getCustomConfig.mockResolvedValue({ + endpoints: { + custom: testCases, + }, + }); + + const result = await loadConfigModels(mockRequest); + + // All variations of "Ollama" should be normalized to lowercase "ollama" + // and the last config in the array should override previous ones + expect(result.Ollama).toBeUndefined(); + expect(result.OLLAMA).toBeUndefined(); + expect(result.OLLaMA).toBeUndefined(); + expect(result.ollama).toEqual(['phi', 'neural-chat']); + + // Verify fetchModels was not called since these are user_provided + expect(fetchModels).not.toHaveBeenCalledWith( + expect.objectContaining({ + name: 'Ollama', + }), + ); + expect(fetchModels).not.toHaveBeenCalledWith( + expect.objectContaining({ + name: 'OLLAMA', + }), + ); + expect(fetchModels).not.toHaveBeenCalledWith( + expect.objectContaining({ + name: 'OLLaMA', + }), + ); + }); }); diff --git a/api/server/services/Endpoints/agents/initialize.js b/api/server/services/Endpoints/agents/initialize.js index 190972ac5..93bb83769 100644 --- a/api/server/services/Endpoints/agents/initialize.js +++ b/api/server/services/Endpoints/agents/initialize.js @@ -1,16 +1,3 @@ -// const { -// ErrorTypes, -// EModelEndpoint, -// resolveHeaders, -// mapModelToAzureConfig, -// } = require('librechat-data-provider'); -// const { getUserKeyValues, checkUserKeyExpiry } = require('~/server/services/UserService'); -// const { isEnabled, isUserProvided } = require('~/server/utils'); -// const { getAzureCredentials } = require('~/utils'); -// const { OpenAIClient } = require('~/app'); - -const { z } = require('zod'); -const { tool } = require('@langchain/core/tools'); const { createContentAggregator, Providers } = require('@librechat/agents'); const { EModelEndpoint, @@ -25,30 +12,11 @@ const initAnthropic = require('~/server/services/Endpoints/anthropic/initialize' const getBedrockOptions = require('~/server/services/Endpoints/bedrock/options'); const initOpenAI = require('~/server/services/Endpoints/openAI/initialize'); const initCustom = require('~/server/services/Endpoints/custom/initialize'); +const { getCustomEndpointConfig } = require('~/server/services/Config'); const { loadAgentTools } = require('~/server/services/ToolService'); const AgentClient = require('~/server/controllers/agents/client'); const { getModelMaxTokens } = require('~/utils'); -/* For testing errors */ -const _getWeather = tool( - async ({ location }) => { - if (location === 'SAN FRANCISCO') { - return 'It\'s 60 degrees and foggy'; - } else if (location.toLowerCase() === 'san francisco') { - throw new Error('Input queries must be all capitals'); - } else { - throw new Error('Invalid input.'); - } - }, - { - name: 'get_weather', - description: 'Call to get the current weather', - schema: z.object({ - location: z.string(), - }), - }, -); - const providerConfigMap = { [EModelEndpoint.openAI]: initOpenAI, [EModelEndpoint.azureOpenAI]: initOpenAI, @@ -85,18 +53,25 @@ const initializeClient = async ({ req, res, endpointOption }) => { if (!agent) { throw new Error('Agent not found'); } + const { tools, toolMap } = await loadAgentTools({ req, tools: agent.tools, agent_id: agent.id, tool_resources: agent.tool_resources, - // openAIApiKey: process.env.OPENAI_API_KEY, }); + const provider = agent.provider; let modelOptions = { model: agent.model }; - let getOptions = providerConfigMap[agent.provider]; + let getOptions = providerConfigMap[provider]; if (!getOptions) { - throw new Error(`Provider ${agent.provider} not supported`); + const customEndpointConfig = await getCustomEndpointConfig(provider); + if (!customEndpointConfig) { + throw new Error(`Provider ${provider} not supported`); + } + getOptions = initCustom; + agent.provider = Providers.OPENAI; + agent.endpoint = provider.toLowerCase(); } // TODO: pass-in override settings that are specific to current run @@ -106,10 +81,14 @@ const initializeClient = async ({ req, res, endpointOption }) => { res, endpointOption, optionsOnly: true, - overrideEndpoint: agent.provider, + overrideEndpoint: provider, overrideModel: agent.model, }); + modelOptions = Object.assign(modelOptions, options.llmConfig); + if (options.configOptions) { + modelOptions.configuration = options.configOptions; + } const sender = getResponseSender({ ...endpointOption, @@ -128,11 +107,11 @@ const initializeClient = async ({ req, res, endpointOption }) => { collectedUsage, artifactPromises, endpoint: EModelEndpoint.agents, - configOptions: options.configOptions, attachments: endpointOption.attachments, maxContextTokens: agent.max_context_tokens ?? - getModelMaxTokens(modelOptions.model, providerEndpointMap[agent.provider]), + getModelMaxTokens(modelOptions.model, providerEndpointMap[provider]) ?? + 4000, }); return { client }; }; diff --git a/api/server/services/Endpoints/custom/initialize.js b/api/server/services/Endpoints/custom/initialize.js index a2f410cdf..2390ea368 100644 --- a/api/server/services/Endpoints/custom/initialize.js +++ b/api/server/services/Endpoints/custom/initialize.js @@ -2,17 +2,17 @@ const { CacheKeys, ErrorTypes, envVarRegex, - EModelEndpoint, FetchTokenConfig, extractEnvVariable, } = require('librechat-data-provider'); +const { Providers } = require('@librechat/agents'); const { getUserKeyValues, checkUserKeyExpiry } = require('~/server/services/UserService'); -const getCustomConfig = require('~/server/services/Config/getCustomConfig'); +const { getLLMConfig } = require('~/server/services/Endpoints/openAI/llm'); +const { getCustomEndpointConfig } = require('~/server/services/Config'); const { fetchModels } = require('~/server/services/ModelService'); const getLogStores = require('~/cache/getLogStores'); const { isUserProvided } = require('~/server/utils'); const { OpenAIClient } = require('~/app'); -const { Providers } = require('@librechat/agents'); const { PROXY } = process.env; @@ -20,15 +20,11 @@ const initializeClient = async ({ req, res, endpointOption, optionsOnly, overrid const { key: expiresAt } = req.body; const endpoint = overrideEndpoint ?? req.body.endpoint; - const customConfig = await getCustomConfig(); - if (!customConfig) { + const endpointConfig = await getCustomEndpointConfig(endpoint); + if (!endpointConfig) { throw new Error(`Config not found for the ${endpoint} custom endpoint.`); } - const { endpoints = {} } = customConfig; - const customEndpoints = endpoints[EModelEndpoint.custom] ?? []; - const endpointConfig = customEndpoints.find((endpointConfig) => endpointConfig.name === endpoint); - const CUSTOM_API_KEY = extractEnvVariable(endpointConfig.apiKey); const CUSTOM_BASE_URL = extractEnvVariable(endpointConfig.baseURL); @@ -138,10 +134,21 @@ const initializeClient = async ({ req, res, endpointOption, optionsOnly, overrid if (optionsOnly) { const modelOptions = endpointOption.model_parameters; - if (endpoint === Providers.OLLAMA && clientOptions.reverseProxyUrl) { + if (endpoint !== Providers.OLLAMA) { + const requestOptions = Object.assign( + { + modelOptions, + }, + clientOptions, + ); + return getLLMConfig(apiKey, requestOptions); + } + + if (clientOptions.reverseProxyUrl) { modelOptions.baseUrl = clientOptions.reverseProxyUrl.split('/v1')[0]; delete clientOptions.reverseProxyUrl; } + return { llmConfig: modelOptions, }; diff --git a/api/server/services/Files/Audio/STTService.js b/api/server/services/Files/Audio/STTService.js index bf2b00c9d..03f6b2861 100644 --- a/api/server/services/Files/Audio/STTService.js +++ b/api/server/services/Files/Audio/STTService.js @@ -2,7 +2,7 @@ const axios = require('axios'); const FormData = require('form-data'); const { Readable } = require('stream'); const { extractEnvVariable, STTProviders } = require('librechat-data-provider'); -const getCustomConfig = require('~/server/services/Config/getCustomConfig'); +const { getCustomConfig } = require('~/server/services/Config'); const { genAzureEndpoint } = require('~/utils'); const { logger } = require('~/config'); diff --git a/api/server/services/Files/Audio/TTSService.js b/api/server/services/Files/Audio/TTSService.js index cf99633cb..d9b1e1d44 100644 --- a/api/server/services/Files/Audio/TTSService.js +++ b/api/server/services/Files/Audio/TTSService.js @@ -1,9 +1,9 @@ const axios = require('axios'); const { extractEnvVariable, TTSProviders } = require('librechat-data-provider'); -const { logger } = require('~/config'); -const getCustomConfig = require('~/server/services/Config/getCustomConfig'); -const { genAzureEndpoint } = require('~/utils'); const { getRandomVoiceId, createChunkProcessor, splitTextIntoChunks } = require('./streamAudio'); +const { getCustomConfig } = require('~/server/services/Config'); +const { genAzureEndpoint } = require('~/utils'); +const { logger } = require('~/config'); /** * Service class for handling Text-to-Speech (TTS) operations. diff --git a/api/server/services/Files/Audio/getCustomConfigSpeech.js b/api/server/services/Files/Audio/getCustomConfigSpeech.js index a0fd23e3f..36f97bc49 100644 --- a/api/server/services/Files/Audio/getCustomConfigSpeech.js +++ b/api/server/services/Files/Audio/getCustomConfigSpeech.js @@ -1,4 +1,4 @@ -const getCustomConfig = require('~/server/services/Config/getCustomConfig'); +const { getCustomConfig } = require('~/server/services/Config'); const { logger } = require('~/config'); /** diff --git a/api/server/services/Files/Audio/getVoices.js b/api/server/services/Files/Audio/getVoices.js index 07c050ab3..24612d85e 100644 --- a/api/server/services/Files/Audio/getVoices.js +++ b/api/server/services/Files/Audio/getVoices.js @@ -1,5 +1,5 @@ const { TTSProviders } = require('librechat-data-provider'); -const getCustomConfig = require('~/server/services/Config/getCustomConfig'); +const { getCustomConfig } = require('~/server/services/Config'); const { getProvider } = require('./TTSService'); /** diff --git a/api/server/services/Files/Audio/index.js b/api/server/services/Files/Audio/index.js index b698e9da8..4378391e2 100644 --- a/api/server/services/Files/Audio/index.js +++ b/api/server/services/Files/Audio/index.js @@ -1,7 +1,7 @@ -const getVoices = require('./getVoices'); const getCustomConfigSpeech = require('./getCustomConfigSpeech'); const TTSService = require('./TTSService'); const STTService = require('./STTService'); +const getVoices = require('./getVoices'); module.exports = { getVoices, diff --git a/api/server/services/isDomainAllowed.js b/api/server/services/isDomainAllowed.js index 48e074751..2eb6c0db2 100644 --- a/api/server/services/isDomainAllowed.js +++ b/api/server/services/isDomainAllowed.js @@ -1,4 +1,4 @@ -const getCustomConfig = require('~/server/services/Config/getCustomConfig'); +const { getCustomConfig } = require('~/server/services/Config'); async function isDomainAllowed(email) { if (!email) { diff --git a/api/server/services/isDomainAllowed.spec.js b/api/server/services/isDomainAllowed.spec.js index b1cf03a56..216b7d581 100644 --- a/api/server/services/isDomainAllowed.spec.js +++ b/api/server/services/isDomainAllowed.spec.js @@ -1,7 +1,9 @@ -const getCustomConfig = require('~/server/services/Config/getCustomConfig'); +const { getCustomConfig } = require('~/server/services/Config'); const isDomainAllowed = require('./isDomainAllowed'); -jest.mock('~/server/services/Config/getCustomConfig', () => jest.fn()); +jest.mock('~/server/services/Config', () => ({ + getCustomConfig: jest.fn(), +})); describe('isDomainAllowed', () => { afterEach(() => { diff --git a/api/typedefs.js b/api/typedefs.js index 8f0e9fef2..371fe9444 100644 --- a/api/typedefs.js +++ b/api/typedefs.js @@ -942,6 +942,29 @@ * @memberof typedefs */ +/** + * @typedef {Object} AgentClientOptions + * @property {Agent} agent - The agent configuration object + * @property {string} endpoint - The endpoint identifier for the agent + * @property {Object} req - The request object + * @property {string} [name] - The username + * @property {string} [modelLabel] - The label for the model being used + * @property {number} [maxContextTokens] - Maximum number of tokens allowed in context + * @property {Object} [endpointTokenConfig] - Token configuration for the endpoint + * @property {boolean} [resendFiles] - Whether to resend files + * @property {string} [imageDetail] - Detail level for image processing + * @property {Object} [spec] - Specification object + * @property {Promise} [attachments] - Promise resolving to file attachments + * @property {Object} [headers] - Additional headers for requests + * @property {string} [proxy] - Proxy configuration + * @property {Object} [tools] - Available tools for the agent + * @property {Object} [toolMap] - Mapping of tool configurations + * @property {Object} [eventHandlers] - Custom event handlers + * @property {Object} [addParams] - Additional parameters to add to requests + * @property {string[]} [dropParams] - Parameters to remove from requests + * @memberof typedefs + */ + /** * @exports ImportBatchBuilder * @typedef {import('./server/utils/import/importBatchBuilder.js').ImportBatchBuilder} ImportBatchBuilder diff --git a/api/utils/tokens.js b/api/utils/tokens.js index 11a29e31f..92833035a 100644 --- a/api/utils/tokens.js +++ b/api/utils/tokens.js @@ -76,6 +76,10 @@ const anthropicModels = { 'claude-3.5-sonnet-latest': 200000, }; +const deepseekModels = { + deepseek: 127500, +}; + const metaModels = { llama3: 8000, llama2: 4000, @@ -117,6 +121,7 @@ const bedrockModels = { ...mistralModels, ...cohereModels, ...ollamaModels, + ...deepseekModels, ...metaModels, ...ai21Models, ...amazonModels, diff --git a/api/utils/tokens.spec.js b/api/utils/tokens.spec.js index 4912d3c80..cacf72cb4 100644 --- a/api/utils/tokens.spec.js +++ b/api/utils/tokens.spec.js @@ -357,6 +357,11 @@ describe('Meta Models Tests', () => { expect(getModelMaxTokens('meta/llama3')).toBe(8000); expect(getModelMaxTokens('meta/llama2')).toBe(4000); }); + + test('should match Deepseek model variations', () => { + expect(getModelMaxTokens('deepseek-chat')).toBe(127500); + expect(getModelMaxTokens('deepseek-coder')).toBe(127500); + }); }); describe('matchModelName', () => { @@ -383,6 +388,11 @@ describe('Meta Models Tests', () => { expect(matchModelName('llama3', EModelEndpoint.bedrock)).toBe('llama3'); expect(matchModelName('llama3.1:8b', EModelEndpoint.bedrock)).toBe('llama3.1:8b'); }); + + test('should match Deepseek model variations', () => { + expect(matchModelName('deepseek-chat')).toBe('deepseek'); + expect(matchModelName('deepseek-coder')).toBe('deepseek'); + }); }); describe('processModelData with Meta models', () => { diff --git a/package-lock.json b/package-lock.json index f218774e7..b72673368 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36283,7 +36283,7 @@ }, "packages/data-provider": { "name": "librechat-data-provider", - "version": "0.7.51", + "version": "0.7.52", "license": "ISC", "dependencies": { "@types/js-yaml": "^4.0.9", diff --git a/packages/data-provider/package.json b/packages/data-provider/package.json index 0170ec740..b68c5ce19 100644 --- a/packages/data-provider/package.json +++ b/packages/data-provider/package.json @@ -1,6 +1,6 @@ { "name": "librechat-data-provider", - "version": "0.7.51", + "version": "0.7.52", "description": "data services for librechat apps", "main": "dist/index.js", "module": "dist/index.es.js", diff --git a/packages/data-provider/src/types/assistants.ts b/packages/data-provider/src/types/assistants.ts index fde34fd13..6e21c0b5b 100644 --- a/packages/data-provider/src/types/assistants.ts +++ b/packages/data-provider/src/types/assistants.ts @@ -184,6 +184,8 @@ export type Agent = { id: string; name: string | null; author?: string | null; + /** The original custom endpoint name, lowercased */ + endpoint?: string | null; authorName?: string | null; description: string | null; created_at: number;