Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-12-17 08:50:15 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

🪙 feat: Use OpenRouter Model Data for Token Cost and Context (#1703)

Browse source 
* feat: use openrouter data for model token cost/context

* chore: add ttl for tokenConfig and refetch models if cache expired
This commit is contained in:
Danny Avila 2024-02-02 00:42:11 -05:00 • committed by GitHub
parent f1d974c513
commit 30e143e96d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 146 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

12
api/server/services/Endpoints/custom/initializeClient.js
View file

@ -1,7 +1,9 @@
const { EModelEndpoint } = require('librechat-data-provider');
const { EModelEndpoint, CacheKeys } = require('librechat-data-provider');
const { getUserKey, checkUserKeyExpiry } = require('~/server/services/UserService');
const { isUserProvided, extractEnvVariable } = require('~/server/utils');
const { fetchModels } = require('~/server/services/ModelService');
const getCustomConfig = require('~/cache/getCustomConfig');
const getLogStores = require('~/cache/getLogStores');
const { OpenAIClient } = require('~/app');
const envVarRegex = /^\${(.+)}$/;
@ -37,6 +39,13 @@ const initializeClient = async ({ req, res, endpointOption }) => {
throw new Error(`Missing Base URL for ${endpoint}.`);
}
const cache = getLogStores(CacheKeys.TOKEN_CONFIG);
let endpointTokenConfig = await cache.get(endpoint);
if (!endpointTokenConfig) {
await fetchModels({ apiKey: CUSTOM_API_KEY, baseURL: CUSTOM_BASE_URL, name: endpoint });
endpointTokenConfig = await cache.get(endpoint);
}
const customOptions = {
headers: resolvedHeaders,
addParams: endpointConfig.addParams,
@ -48,6 +57,7 @@ const initializeClient = async ({ req, res, endpointOption }) => {
modelDisplayLabel: endpointConfig.modelDisplayLabel,
titleMethod: endpointConfig.titleMethod ?? 'completion',
contextStrategy: endpointConfig.summarize ? 'summarize' : null,
endpointTokenConfig,
};
const useUserKey = isUserProvided(CUSTOM_API_KEY);