Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-12-17 08:50:15 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

📋 feat: Log Custom Config File and Add Known Model Limits to Custom Endpoint (#1657)

Browse source 
* refactor(custom): add all recognized models to maxTokensMap for custom endpoint

* feat(librechat.yaml): log the custom config file on initial load

* fix(OpenAIClient): pass endpointType/endpoint to `getModelMaxTokens` call
This commit is contained in:
Danny Avila 2024-01-27 08:59:04 -05:00 • committed by GitHub
parent c470147ea2
commit f7f7f929a0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 27 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

3
api/app/clients/OpenAIClient.js
View file

@ -131,7 +131,8 @@ class OpenAIClient extends BaseClient {
const { isChatGptModel } = this; const { isChatGptModel } = this;
this.isUnofficialChatGptModel = this.isUnofficialChatGptModel =
model.startsWith('text-chat') || model.startsWith('text-davinci-002-render'); model.startsWith('text-chat') || model.startsWith('text-davinci-002-render');
this.maxContextTokens = getModelMaxTokens(model) ?? 4095; // 1 less than maximum this.maxContextTokens =
getModelMaxTokens(model, this.options.endpointType ?? this.options.endpoint) ?? 4095; // 1 less than maximum
if (this.shouldSummarize) { if (this.shouldSummarize) {
this.maxContextTokens = Math.floor(this.maxContextTokens / 2); this.maxContextTokens = Math.floor(this.maxContextTokens / 2);

3
api/server/services/Config/loadCustomConfig.js
View file

@ -25,7 +25,8 @@ async function loadCustomConfig() {
logger.error(`Invalid custom config file at ${configPath}`, result.error); logger.error(`Invalid custom config file at ${configPath}`, result.error);
return null; return null;
} else { } else {
logger.info('Loaded custom config file'); logger.info('Loaded custom config file:');
logger.info(JSON.stringify(customConfig, null, 2));
} }
if (customConfig.cache) { if (customConfig.cache) {

20
api/utils/tokens.js
View file

@ -57,11 +57,7 @@ const openAIModels = {
'mistral-': 31990, // -10 from max 'mistral-': 31990, // -10 from max
}; };
// Order is important here: by model series and context size (gpt-4 then gpt-3, ascending) const googleModels = {
const maxTokensMap = {
[EModelEndpoint.openAI]: openAIModels,
[EModelEndpoint.custom]: openAIModels,
[EModelEndpoint.google]: {
/* Max I/O is combined so we subtract the amount from max response tokens for actual total */ /* Max I/O is combined so we subtract the amount from max response tokens for actual total */
gemini: 32750, // -10 from max gemini: 32750, // -10 from max
'text-bison-32k': 32758, // -10 from max 'text-bison-32k': 32758, // -10 from max
@ -74,11 +70,19 @@ const maxTokensMap = {
/* PaLM2, -5 from max: 8192 */ /* PaLM2, -5 from max: 8192 */
'text-': 8187, 'text-': 8187,
'chat-': 8187, 'chat-': 8187,
}, };
[EModelEndpoint.anthropic]: {
const anthropicModels = {
'claude-2.1': 200000, 'claude-2.1': 200000,
'claude-': 100000, 'claude-': 100000,
}, };
// Order is important here: by model series and context size (gpt-4 then gpt-3, ascending)
const maxTokensMap = {
[EModelEndpoint.openAI]: openAIModels,
[EModelEndpoint.custom]: { ...openAIModels, ...googleModels, ...anthropicModels },
[EModelEndpoint.google]: googleModels,
[EModelEndpoint.anthropic]: anthropicModels,
}; };
/** /**