mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-01-07 11:08:52 +01:00
🧵 refactor: Migrate Endpoint Initialization to TypeScript (#10794)
* refactor: move endpoint initialization methods to typescript * refactor: move agent init to packages/api - Introduced `initialize.ts` for agent initialization, including file processing and tool loading. - Updated `resources.ts` to allow optional appConfig parameter. - Enhanced endpoint configuration handling in various initialization files to support model parameters. - Added new artifacts and prompts for React component generation. - Refactored existing code to improve type safety and maintainability. * refactor: streamline endpoint initialization and enhance type safety - Updated initialization functions across various endpoints to use a consistent request structure, replacing `unknown` types with `ServerResponse`. - Simplified request handling by directly extracting keys from the request body. - Improved type safety by ensuring user IDs are safely accessed with optional chaining. - Removed unnecessary parameters and streamlined model options handling for better clarity and maintainability. * refactor: moved ModelService and extractBaseURL to packages/api - Added comprehensive tests for the models fetching functionality, covering scenarios for OpenAI, Anthropic, Google, and Ollama models. - Updated existing endpoint index to include the new models module. - Enhanced utility functions for URL extraction and model data processing. - Improved type safety and error handling across the models fetching logic. * refactor: consolidate utility functions and remove unused files - Merged `deriveBaseURL` and `extractBaseURL` into the `@librechat/api` module for better organization. - Removed redundant utility files and their associated tests to streamline the codebase. - Updated imports across various client files to utilize the new consolidated functions. - Enhanced overall maintainability by reducing the number of utility modules. * refactor: replace ModelService references with direct imports from @librechat/api and remove ModelService file * refactor: move encrypt/decrypt methods and key db methods to data-schemas, use `getProviderConfig` from `@librechat/api` * chore: remove unused 'res' from options in AgentClient * refactor: file model imports and methods - Updated imports in various controllers and services to use the unified file model from '~/models' instead of '~/models/File'. - Consolidated file-related methods into a new file methods module in the data-schemas package. - Added comprehensive tests for file methods including creation, retrieval, updating, and deletion. - Enhanced the initializeAgent function to accept dependency injection for file-related methods. - Improved error handling and logging in file methods. * refactor: streamline database method references in agent initialization * refactor: enhance file method tests and update type references to IMongoFile * refactor: consolidate database method imports in agent client and initialization * chore: remove redundant import of initializeAgent from @librechat/api * refactor: move checkUserKeyExpiry utility to @librechat/api and update references across endpoints * refactor: move updateUserPlugins logic to user.ts and simplify UserController * refactor: update imports for user key management and remove UserService * refactor: remove unused Anthropics and Bedrock endpoint files and clean up imports * refactor: consolidate and update encryption imports across various files to use @librechat/data-schemas * chore: update file model mock to use unified import from '~/models' * chore: import order * refactor: remove migrated to TS agent.js file and its associated logic from the endpoints * chore: add reusable function to extract imports from source code in unused-packages workflow * chore: enhance unused-packages workflow to include @librechat/api dependencies and improve dependency extraction * chore: improve dependency extraction in unused-packages workflow with enhanced error handling and debugging output * chore: add detailed debugging output to unused-packages workflow for better visibility into unused dependencies and exclusion lists * chore: refine subpath handling in unused-packages workflow to correctly process scoped and non-scoped package imports * chore: clean up unused debug output in unused-packages workflow and reorganize type imports in initialize.ts
This commit is contained in:
Danny Avila
parent
1a11b64266
commit
04a4a2aa44
103 changed files with 4135 additions and 2647 deletions
|
|
@ -1,2 +1,3 @@
|
|||
export * from './helpers';
|
||||
export * from './llm';
|
||||
export * from './initialize';
|
||||
|
|
|
|||
73
packages/api/src/endpoints/anthropic/initialize.ts
Normal file
73
packages/api/src/endpoints/anthropic/initialize.ts
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
import { EModelEndpoint } from 'librechat-data-provider';
|
||||
import type { BaseInitializeParams, InitializeResultBase, AnthropicConfigOptions } from '~/types';
|
||||
import { checkUserKeyExpiry } from '~/utils';
|
||||
import { getLLMConfig } from './llm';
|
||||
|
||||
/**
|
||||
* Initializes Anthropic endpoint configuration.
|
||||
*
|
||||
* @param params - Configuration parameters
|
||||
* @returns Promise resolving to Anthropic configuration options
|
||||
* @throws Error if API key is not provided
|
||||
*/
|
||||
export async function initializeAnthropic({
|
||||
req,
|
||||
endpoint,
|
||||
model_parameters,
|
||||
db,
|
||||
}: BaseInitializeParams): Promise<InitializeResultBase> {
|
||||
void endpoint;
|
||||
const appConfig = req.config;
|
||||
const { ANTHROPIC_API_KEY, ANTHROPIC_REVERSE_PROXY, PROXY } = process.env;
|
||||
const { key: expiresAt } = req.body;
|
||||
const isUserProvided = ANTHROPIC_API_KEY === 'user_provided';
|
||||
|
||||
const anthropicApiKey = isUserProvided
|
||||
? await db.getUserKey({ userId: req.user?.id ?? '', name: EModelEndpoint.anthropic })
|
||||
: ANTHROPIC_API_KEY;
|
||||
|
||||
if (!anthropicApiKey) {
|
||||
throw new Error('Anthropic API key not provided. Please provide it again.');
|
||||
}
|
||||
|
||||
if (expiresAt && isUserProvided) {
|
||||
checkUserKeyExpiry(expiresAt, EModelEndpoint.anthropic);
|
||||
}
|
||||
|
||||
let clientOptions: AnthropicConfigOptions = {};
|
||||
|
||||
/** @type {undefined | TBaseEndpoint} */
|
||||
const anthropicConfig = appConfig?.endpoints?.[EModelEndpoint.anthropic];
|
||||
|
||||
if (anthropicConfig) {
|
||||
clientOptions = {
|
||||
...clientOptions,
|
||||
// Note: _lc_stream_delay is set on modelOptions in the result
|
||||
};
|
||||
}
|
||||
|
||||
const allConfig = appConfig?.endpoints?.all;
|
||||
|
||||
clientOptions = {
|
||||
proxy: PROXY ?? undefined,
|
||||
reverseProxyUrl: ANTHROPIC_REVERSE_PROXY ?? undefined,
|
||||
modelOptions: {
|
||||
...(model_parameters ?? {}),
|
||||
user: req.user?.id,
|
||||
},
|
||||
...clientOptions,
|
||||
};
|
||||
|
||||
const result = getLLMConfig(anthropicApiKey, clientOptions);
|
||||
|
||||
// Apply stream rate delay
|
||||
if (anthropicConfig?.streamRate) {
|
||||
(result.llmConfig as Record<string, unknown>)._lc_stream_delay = anthropicConfig.streamRate;
|
||||
}
|
||||
|
||||
if (allConfig?.streamRate) {
|
||||
(result.llmConfig as Record<string, unknown>)._lc_stream_delay = allConfig.streamRate;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
1
packages/api/src/endpoints/bedrock/index.ts
Normal file
1
packages/api/src/endpoints/bedrock/index.ts
Normal file
|
|
@ -0,0 +1 @@
|
|||
export * from './initialize';
|
||||
148
packages/api/src/endpoints/bedrock/initialize.ts
Normal file
148
packages/api/src/endpoints/bedrock/initialize.ts
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
import { HttpsProxyAgent } from 'https-proxy-agent';
|
||||
import { NodeHttpHandler } from '@smithy/node-http-handler';
|
||||
import { BedrockRuntimeClient } from '@aws-sdk/client-bedrock-runtime';
|
||||
import {
|
||||
AuthType,
|
||||
EModelEndpoint,
|
||||
bedrockInputParser,
|
||||
bedrockOutputParser,
|
||||
removeNullishValues,
|
||||
} from 'librechat-data-provider';
|
||||
import type { BaseInitializeParams, InitializeResultBase, BedrockCredentials } from '~/types';
|
||||
import { checkUserKeyExpiry } from '~/utils';
|
||||
|
||||
/**
|
||||
* Initializes Bedrock endpoint configuration.
|
||||
*
|
||||
* This module handles configuration for AWS Bedrock endpoints, including support for
|
||||
* HTTP/HTTPS proxies and reverse proxies.
|
||||
*
|
||||
* Proxy Support:
|
||||
* - When the PROXY environment variable is set, creates a custom BedrockRuntimeClient
|
||||
* with an HttpsProxyAgent to route all Bedrock API calls through the specified proxy
|
||||
* - The custom client is fully configured with credentials, region, and endpoint,
|
||||
* and is passed directly to ChatBedrockConverse via the 'client' parameter
|
||||
*
|
||||
* Reverse Proxy Support:
|
||||
* - When BEDROCK_REVERSE_PROXY is set, routes Bedrock API calls through a custom endpoint
|
||||
* - Works with or without the PROXY setting
|
||||
*
|
||||
* Without Proxy:
|
||||
* - Credentials and endpoint configuration are passed separately to ChatBedrockConverse,
|
||||
* which creates its own BedrockRuntimeClient internally
|
||||
*
|
||||
* @param params - Configuration parameters
|
||||
* @returns Promise resolving to Bedrock configuration options
|
||||
* @throws Error if credentials are not provided when required
|
||||
*/
|
||||
export async function initializeBedrock({
|
||||
req,
|
||||
endpoint,
|
||||
model_parameters,
|
||||
db,
|
||||
}: BaseInitializeParams): Promise<InitializeResultBase> {
|
||||
void endpoint;
|
||||
const {
|
||||
BEDROCK_AWS_SECRET_ACCESS_KEY,
|
||||
BEDROCK_AWS_ACCESS_KEY_ID,
|
||||
BEDROCK_AWS_SESSION_TOKEN,
|
||||
BEDROCK_REVERSE_PROXY,
|
||||
BEDROCK_AWS_DEFAULT_REGION,
|
||||
PROXY,
|
||||
} = process.env;
|
||||
|
||||
const { key: expiresAt } = req.body;
|
||||
const isUserProvided = BEDROCK_AWS_SECRET_ACCESS_KEY === AuthType.USER_PROVIDED;
|
||||
|
||||
let credentials: BedrockCredentials | undefined = isUserProvided
|
||||
? await db
|
||||
.getUserKey({ userId: req.user?.id ?? '', name: EModelEndpoint.bedrock })
|
||||
.then((key) => JSON.parse(key) as BedrockCredentials)
|
||||
: {
|
||||
accessKeyId: BEDROCK_AWS_ACCESS_KEY_ID,
|
||||
secretAccessKey: BEDROCK_AWS_SECRET_ACCESS_KEY,
|
||||
...(BEDROCK_AWS_SESSION_TOKEN && { sessionToken: BEDROCK_AWS_SESSION_TOKEN }),
|
||||
};
|
||||
|
||||
if (!credentials) {
|
||||
throw new Error('Bedrock credentials not provided. Please provide them again.');
|
||||
}
|
||||
|
||||
if (
|
||||
!isUserProvided &&
|
||||
(credentials.accessKeyId === undefined || credentials.accessKeyId === '') &&
|
||||
(credentials.secretAccessKey === undefined || credentials.secretAccessKey === '')
|
||||
) {
|
||||
credentials = undefined;
|
||||
}
|
||||
|
||||
if (expiresAt && isUserProvided) {
|
||||
checkUserKeyExpiry(expiresAt, EModelEndpoint.bedrock);
|
||||
}
|
||||
|
||||
const requestOptions: Record<string, unknown> = {
|
||||
model: model_parameters?.model as string | undefined,
|
||||
region: BEDROCK_AWS_DEFAULT_REGION,
|
||||
};
|
||||
|
||||
const configOptions: Record<string, unknown> = {};
|
||||
|
||||
const llmConfig = bedrockOutputParser(
|
||||
bedrockInputParser.parse(
|
||||
removeNullishValues({ ...requestOptions, ...(model_parameters ?? {}) }),
|
||||
),
|
||||
) as InitializeResultBase['llmConfig'] & {
|
||||
region?: string;
|
||||
client?: BedrockRuntimeClient;
|
||||
credentials?: BedrockCredentials;
|
||||
endpointHost?: string;
|
||||
};
|
||||
|
||||
/** Only include credentials if they're complete (accessKeyId and secretAccessKey are both set) */
|
||||
const hasCompleteCredentials =
|
||||
credentials &&
|
||||
typeof credentials.accessKeyId === 'string' &&
|
||||
credentials.accessKeyId !== '' &&
|
||||
typeof credentials.secretAccessKey === 'string' &&
|
||||
credentials.secretAccessKey !== '';
|
||||
|
||||
if (PROXY) {
|
||||
const proxyAgent = new HttpsProxyAgent(PROXY);
|
||||
|
||||
// Create a custom BedrockRuntimeClient with proxy-enabled request handler.
|
||||
// ChatBedrockConverse will use this pre-configured client directly instead of
|
||||
// creating its own. Credentials are only set if explicitly provided; otherwise
|
||||
// the AWS SDK's default credential provider chain is used (instance profiles,
|
||||
// AWS profiles, environment variables, etc.)
|
||||
const customClient = new BedrockRuntimeClient({
|
||||
region: (llmConfig.region as string) ?? BEDROCK_AWS_DEFAULT_REGION,
|
||||
...(hasCompleteCredentials && {
|
||||
credentials: credentials as { accessKeyId: string; secretAccessKey: string },
|
||||
}),
|
||||
requestHandler: new NodeHttpHandler({
|
||||
httpAgent: proxyAgent,
|
||||
httpsAgent: proxyAgent,
|
||||
}),
|
||||
...(BEDROCK_REVERSE_PROXY && {
|
||||
endpoint: `https://${BEDROCK_REVERSE_PROXY}`,
|
||||
}),
|
||||
});
|
||||
|
||||
llmConfig.client = customClient;
|
||||
} else {
|
||||
// When not using a proxy, let ChatBedrockConverse create its own client
|
||||
// by providing credentials and endpoint separately
|
||||
if (credentials) {
|
||||
llmConfig.credentials = credentials;
|
||||
}
|
||||
|
||||
if (BEDROCK_REVERSE_PROXY) {
|
||||
llmConfig.endpointHost = BEDROCK_REVERSE_PROXY;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
llmConfig,
|
||||
configOptions,
|
||||
};
|
||||
}
|
||||
99
packages/api/src/endpoints/config.ts
Normal file
99
packages/api/src/endpoints/config.ts
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
import { Providers } from '@librechat/agents';
|
||||
import { EModelEndpoint } from 'librechat-data-provider';
|
||||
import type { TEndpoint } from 'librechat-data-provider';
|
||||
import type { AppConfig } from '@librechat/data-schemas';
|
||||
import type { BaseInitializeParams, InitializeResultBase } from '~/types';
|
||||
import { initializeAnthropic } from './anthropic/initialize';
|
||||
import { initializeBedrock } from './bedrock/initialize';
|
||||
import { initializeCustom } from './custom/initialize';
|
||||
import { initializeGoogle } from './google/initialize';
|
||||
import { initializeOpenAI } from './openai/initialize';
|
||||
import { getCustomEndpointConfig } from '~/app/config';
|
||||
|
||||
/**
|
||||
* Type for initialize functions
|
||||
*/
|
||||
export type InitializeFn = (params: BaseInitializeParams) => Promise<InitializeResultBase>;
|
||||
|
||||
/**
|
||||
* Check if the provider is a known custom provider
|
||||
* @param provider - The provider string
|
||||
* @returns True if the provider is a known custom provider, false otherwise
|
||||
*/
|
||||
export function isKnownCustomProvider(provider?: string): boolean {
|
||||
return [Providers.XAI, Providers.DEEPSEEK, Providers.OPENROUTER].includes(
|
||||
(provider?.toLowerCase() ?? '') as Providers,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Provider configuration map mapping providers to their initialization functions
|
||||
*/
|
||||
export const providerConfigMap: Record<string, InitializeFn> = {
|
||||
[Providers.XAI]: initializeCustom,
|
||||
[Providers.DEEPSEEK]: initializeCustom,
|
||||
[Providers.OPENROUTER]: initializeCustom,
|
||||
[EModelEndpoint.openAI]: initializeOpenAI,
|
||||
[EModelEndpoint.google]: initializeGoogle,
|
||||
[EModelEndpoint.bedrock]: initializeBedrock,
|
||||
[EModelEndpoint.azureOpenAI]: initializeOpenAI,
|
||||
[EModelEndpoint.anthropic]: initializeAnthropic,
|
||||
};
|
||||
|
||||
/**
|
||||
* Result from getProviderConfig
|
||||
*/
|
||||
export interface ProviderConfigResult {
|
||||
/** The initialization function for this provider */
|
||||
getOptions: InitializeFn;
|
||||
/** The resolved provider name (may be different from input if normalized) */
|
||||
overrideProvider: string;
|
||||
/** Custom endpoint configuration (if applicable) */
|
||||
customEndpointConfig?: Partial<TEndpoint>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the provider configuration and override endpoint based on the provider string
|
||||
*
|
||||
* @param params - Configuration parameters
|
||||
* @param params.provider - The provider string
|
||||
* @param params.appConfig - The application configuration
|
||||
* @returns Provider configuration including getOptions function, override provider, and custom config
|
||||
* @throws Error if provider is not supported
|
||||
*/
|
||||
export function getProviderConfig({
|
||||
provider,
|
||||
appConfig,
|
||||
}: {
|
||||
provider: string;
|
||||
appConfig?: AppConfig;
|
||||
}): ProviderConfigResult {
|
||||
let getOptions = providerConfigMap[provider];
|
||||
let overrideProvider = provider;
|
||||
let customEndpointConfig: Partial<TEndpoint> | undefined;
|
||||
|
||||
if (!getOptions && providerConfigMap[provider.toLowerCase()] != null) {
|
||||
overrideProvider = provider.toLowerCase();
|
||||
getOptions = providerConfigMap[overrideProvider];
|
||||
} else if (!getOptions) {
|
||||
customEndpointConfig = getCustomEndpointConfig({ endpoint: provider, appConfig });
|
||||
if (!customEndpointConfig) {
|
||||
throw new Error(`Provider ${provider} not supported`);
|
||||
}
|
||||
getOptions = initializeCustom;
|
||||
overrideProvider = Providers.OPENAI;
|
||||
}
|
||||
|
||||
if (isKnownCustomProvider(overrideProvider) && !customEndpointConfig) {
|
||||
customEndpointConfig = getCustomEndpointConfig({ endpoint: provider, appConfig });
|
||||
if (!customEndpointConfig) {
|
||||
throw new Error(`Provider ${provider} not supported`);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
getOptions,
|
||||
overrideProvider,
|
||||
customEndpointConfig,
|
||||
};
|
||||
}
|
||||
|
|
@ -1 +1,2 @@
|
|||
export * from './config';
|
||||
export * from './initialize';
|
||||
|
|
|
|||
180
packages/api/src/endpoints/custom/initialize.ts
Normal file
180
packages/api/src/endpoints/custom/initialize.ts
Normal file
|
|
@ -0,0 +1,180 @@
|
|||
import {
|
||||
CacheKeys,
|
||||
ErrorTypes,
|
||||
envVarRegex,
|
||||
FetchTokenConfig,
|
||||
extractEnvVariable,
|
||||
} from 'librechat-data-provider';
|
||||
import type { TEndpoint } from 'librechat-data-provider';
|
||||
import type { AppConfig } from '@librechat/data-schemas';
|
||||
import type { BaseInitializeParams, InitializeResultBase, EndpointTokenConfig } from '~/types';
|
||||
import { getOpenAIConfig } from '~/endpoints/openai/config';
|
||||
import { getCustomEndpointConfig } from '~/app/config';
|
||||
import { fetchModels } from '~/endpoints/models';
|
||||
import { isUserProvided, checkUserKeyExpiry } from '~/utils';
|
||||
import { standardCache } from '~/cache';
|
||||
|
||||
const { PROXY } = process.env;
|
||||
|
||||
/**
|
||||
* Builds custom options from endpoint configuration
|
||||
*/
|
||||
function buildCustomOptions(
|
||||
endpointConfig: Partial<TEndpoint>,
|
||||
appConfig?: AppConfig,
|
||||
endpointTokenConfig?: Record<string, unknown>,
|
||||
) {
|
||||
const customOptions: Record<string, unknown> = {
|
||||
headers: endpointConfig.headers,
|
||||
addParams: endpointConfig.addParams,
|
||||
dropParams: endpointConfig.dropParams,
|
||||
customParams: endpointConfig.customParams,
|
||||
titleConvo: endpointConfig.titleConvo,
|
||||
titleModel: endpointConfig.titleModel,
|
||||
forcePrompt: endpointConfig.forcePrompt,
|
||||
summaryModel: endpointConfig.summaryModel,
|
||||
modelDisplayLabel: endpointConfig.modelDisplayLabel,
|
||||
titleMethod: endpointConfig.titleMethod ?? 'completion',
|
||||
contextStrategy: endpointConfig.summarize ? 'summarize' : null,
|
||||
directEndpoint: endpointConfig.directEndpoint,
|
||||
titleMessageRole: endpointConfig.titleMessageRole,
|
||||
streamRate: endpointConfig.streamRate,
|
||||
endpointTokenConfig,
|
||||
};
|
||||
|
||||
const allConfig = appConfig?.endpoints?.all;
|
||||
if (allConfig) {
|
||||
customOptions.streamRate = allConfig.streamRate;
|
||||
}
|
||||
|
||||
return customOptions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes a custom endpoint client configuration.
|
||||
* This function handles custom endpoints defined in librechat.yaml, including
|
||||
* user-provided API keys and URLs.
|
||||
*
|
||||
* @param params - Configuration parameters
|
||||
* @returns Promise resolving to endpoint configuration options
|
||||
* @throws Error if config is missing, API key is not provided, or base URL is missing
|
||||
*/
|
||||
export async function initializeCustom({
|
||||
req,
|
||||
endpoint,
|
||||
model_parameters,
|
||||
db,
|
||||
}: BaseInitializeParams): Promise<InitializeResultBase> {
|
||||
const appConfig = req.config;
|
||||
const { key: expiresAt } = req.body;
|
||||
|
||||
const endpointConfig = getCustomEndpointConfig({
|
||||
endpoint,
|
||||
appConfig,
|
||||
});
|
||||
|
||||
if (!endpointConfig) {
|
||||
throw new Error(`Config not found for the ${endpoint} custom endpoint.`);
|
||||
}
|
||||
|
||||
const CUSTOM_API_KEY = extractEnvVariable(endpointConfig.apiKey ?? '');
|
||||
const CUSTOM_BASE_URL = extractEnvVariable(endpointConfig.baseURL ?? '');
|
||||
|
||||
if (CUSTOM_API_KEY.match(envVarRegex)) {
|
||||
throw new Error(`Missing API Key for ${endpoint}.`);
|
||||
}
|
||||
|
||||
if (CUSTOM_BASE_URL.match(envVarRegex)) {
|
||||
throw new Error(`Missing Base URL for ${endpoint}.`);
|
||||
}
|
||||
|
||||
const userProvidesKey = isUserProvided(CUSTOM_API_KEY);
|
||||
const userProvidesURL = isUserProvided(CUSTOM_BASE_URL);
|
||||
|
||||
let userValues = null;
|
||||
if (expiresAt && (userProvidesKey || userProvidesURL)) {
|
||||
checkUserKeyExpiry(expiresAt, endpoint);
|
||||
userValues = await db.getUserKeyValues({ userId: req.user?.id ?? '', name: endpoint });
|
||||
}
|
||||
|
||||
const apiKey = userProvidesKey ? userValues?.apiKey : CUSTOM_API_KEY;
|
||||
const baseURL = userProvidesURL ? userValues?.baseURL : CUSTOM_BASE_URL;
|
||||
|
||||
if (userProvidesKey && !apiKey) {
|
||||
throw new Error(
|
||||
JSON.stringify({
|
||||
type: ErrorTypes.NO_USER_KEY,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
if (userProvidesURL && !baseURL) {
|
||||
throw new Error(
|
||||
JSON.stringify({
|
||||
type: ErrorTypes.NO_BASE_URL,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
if (!apiKey) {
|
||||
throw new Error(`${endpoint} API key not provided.`);
|
||||
}
|
||||
|
||||
if (!baseURL) {
|
||||
throw new Error(`${endpoint} Base URL not provided.`);
|
||||
}
|
||||
|
||||
let endpointTokenConfig: EndpointTokenConfig | undefined;
|
||||
|
||||
const userId = req.user?.id ?? '';
|
||||
|
||||
const cache = standardCache(CacheKeys.TOKEN_CONFIG);
|
||||
/** tokenConfig is an optional extended property on custom endpoints */
|
||||
const hasTokenConfig = (endpointConfig as Record<string, unknown>).tokenConfig != null;
|
||||
const tokenKey =
|
||||
!hasTokenConfig && (userProvidesKey || userProvidesURL) ? `${endpoint}:${userId}` : endpoint;
|
||||
|
||||
const cachedConfig =
|
||||
!hasTokenConfig &&
|
||||
FetchTokenConfig[endpoint.toLowerCase() as keyof typeof FetchTokenConfig] &&
|
||||
(await cache.get(tokenKey));
|
||||
|
||||
endpointTokenConfig = (cachedConfig as EndpointTokenConfig) || undefined;
|
||||
|
||||
if (
|
||||
FetchTokenConfig[endpoint.toLowerCase() as keyof typeof FetchTokenConfig] &&
|
||||
endpointConfig &&
|
||||
endpointConfig.models?.fetch &&
|
||||
!endpointTokenConfig
|
||||
) {
|
||||
await fetchModels({ apiKey, baseURL, name: endpoint, user: userId, tokenKey });
|
||||
endpointTokenConfig = (await cache.get(tokenKey)) as EndpointTokenConfig | undefined;
|
||||
}
|
||||
|
||||
const customOptions = buildCustomOptions(endpointConfig, appConfig, endpointTokenConfig);
|
||||
|
||||
const clientOptions: Record<string, unknown> = {
|
||||
reverseProxyUrl: baseURL ?? null,
|
||||
proxy: PROXY ?? null,
|
||||
...customOptions,
|
||||
};
|
||||
|
||||
const modelOptions = { ...(model_parameters ?? {}), user: userId };
|
||||
const finalClientOptions = {
|
||||
modelOptions,
|
||||
...clientOptions,
|
||||
};
|
||||
|
||||
const options = getOpenAIConfig(apiKey, finalClientOptions, endpoint);
|
||||
if (options != null) {
|
||||
(options as InitializeResultBase).useLegacyContent = true;
|
||||
(options as InitializeResultBase).endpointTokenConfig = endpointTokenConfig;
|
||||
}
|
||||
|
||||
const streamRate = clientOptions.streamRate as number | undefined;
|
||||
if (streamRate) {
|
||||
(options.llmConfig as Record<string, unknown>)._lc_stream_delay = streamRate;
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
|
@ -1 +1,2 @@
|
|||
export * from './llm';
|
||||
export * from './initialize';
|
||||
|
|
|
|||
91
packages/api/src/endpoints/google/initialize.ts
Normal file
91
packages/api/src/endpoints/google/initialize.ts
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
import path from 'path';
|
||||
import { EModelEndpoint, AuthKeys } from 'librechat-data-provider';
|
||||
import type {
|
||||
BaseInitializeParams,
|
||||
InitializeResultBase,
|
||||
GoogleConfigOptions,
|
||||
GoogleCredentials,
|
||||
} from '~/types';
|
||||
import { isEnabled, loadServiceKey, checkUserKeyExpiry } from '~/utils';
|
||||
import { getGoogleConfig } from './llm';
|
||||
|
||||
/**
|
||||
* Initializes Google/Vertex AI endpoint configuration.
|
||||
* Supports both API key authentication and service account credentials.
|
||||
*
|
||||
* @param params - Configuration parameters
|
||||
* @returns Promise resolving to Google configuration options
|
||||
* @throws Error if no valid credentials are provided
|
||||
*/
|
||||
export async function initializeGoogle({
|
||||
req,
|
||||
endpoint,
|
||||
model_parameters,
|
||||
db,
|
||||
}: BaseInitializeParams): Promise<InitializeResultBase> {
|
||||
void endpoint;
|
||||
const appConfig = req.config;
|
||||
const { GOOGLE_KEY, GOOGLE_REVERSE_PROXY, GOOGLE_AUTH_HEADER, PROXY } = process.env;
|
||||
const isUserProvided = GOOGLE_KEY === 'user_provided';
|
||||
const { key: expiresAt } = req.body;
|
||||
|
||||
let userKey = null;
|
||||
if (expiresAt && isUserProvided) {
|
||||
checkUserKeyExpiry(expiresAt, EModelEndpoint.google);
|
||||
userKey = await db.getUserKey({ userId: req.user?.id, name: EModelEndpoint.google });
|
||||
}
|
||||
|
||||
let serviceKey: Record<string, unknown> = {};
|
||||
|
||||
/** Check if GOOGLE_KEY is provided at all (including 'user_provided') */
|
||||
const isGoogleKeyProvided =
|
||||
(GOOGLE_KEY && GOOGLE_KEY.trim() !== '') || (isUserProvided && userKey != null);
|
||||
|
||||
if (!isGoogleKeyProvided && loadServiceKey) {
|
||||
/** Only attempt to load service key if GOOGLE_KEY is not provided */
|
||||
try {
|
||||
const serviceKeyPath =
|
||||
process.env.GOOGLE_SERVICE_KEY_FILE || path.join(process.cwd(), 'data', 'auth.json');
|
||||
const loadedKey = await loadServiceKey(serviceKeyPath);
|
||||
if (loadedKey) {
|
||||
serviceKey = loadedKey;
|
||||
}
|
||||
} catch {
|
||||
// Service key loading failed, but that's okay if not required
|
||||
serviceKey = {};
|
||||
}
|
||||
}
|
||||
|
||||
const credentials: GoogleCredentials = isUserProvided
|
||||
? (userKey as GoogleCredentials)
|
||||
: {
|
||||
[AuthKeys.GOOGLE_SERVICE_KEY]: serviceKey,
|
||||
[AuthKeys.GOOGLE_API_KEY]: GOOGLE_KEY,
|
||||
};
|
||||
|
||||
let clientOptions: GoogleConfigOptions = {};
|
||||
|
||||
/** @type {undefined | TBaseEndpoint} */
|
||||
const allConfig = appConfig?.endpoints?.all;
|
||||
/** @type {undefined | TBaseEndpoint} */
|
||||
const googleConfig = appConfig?.endpoints?.[EModelEndpoint.google];
|
||||
|
||||
if (googleConfig) {
|
||||
clientOptions.streamRate = googleConfig.streamRate;
|
||||
clientOptions.titleModel = googleConfig.titleModel;
|
||||
}
|
||||
|
||||
if (allConfig) {
|
||||
clientOptions.streamRate = allConfig.streamRate;
|
||||
}
|
||||
|
||||
clientOptions = {
|
||||
reverseProxyUrl: GOOGLE_REVERSE_PROXY ?? undefined,
|
||||
authHeader: isEnabled(GOOGLE_AUTH_HEADER) ?? undefined,
|
||||
proxy: PROXY ?? undefined,
|
||||
modelOptions: model_parameters ?? {},
|
||||
...clientOptions,
|
||||
};
|
||||
|
||||
return getGoogleConfig(credentials, clientOptions);
|
||||
}
|
||||
|
|
@ -1,4 +1,7 @@
|
|||
export * from './anthropic';
|
||||
export * from './bedrock';
|
||||
export * from './config';
|
||||
export * from './custom';
|
||||
export * from './google';
|
||||
export * from './models';
|
||||
export * from './openai';
|
||||
export * from './anthropic';
|
||||
|
|
|
|||
628
packages/api/src/endpoints/models.spec.ts
Normal file
628
packages/api/src/endpoints/models.spec.ts
Normal file
|
|
@ -0,0 +1,628 @@
|
|||
import axios from 'axios';
|
||||
import { EModelEndpoint, defaultModels } from 'librechat-data-provider';
|
||||
import {
|
||||
fetchModels,
|
||||
splitAndTrim,
|
||||
getOpenAIModels,
|
||||
getGoogleModels,
|
||||
getBedrockModels,
|
||||
getAnthropicModels,
|
||||
} from './models';
|
||||
|
||||
jest.mock('axios');
|
||||
|
||||
jest.mock('~/cache', () => ({
|
||||
standardCache: jest.fn().mockImplementation(() => ({
|
||||
get: jest.fn().mockResolvedValue(undefined),
|
||||
set: jest.fn().mockResolvedValue(true),
|
||||
})),
|
||||
}));
|
||||
|
||||
jest.mock('~/utils', () => {
|
||||
const originalUtils = jest.requireActual('~/utils');
|
||||
return {
|
||||
...originalUtils,
|
||||
processModelData: jest.fn((...args) => originalUtils.processModelData(...args)),
|
||||
logAxiosError: jest.fn(),
|
||||
resolveHeaders: jest.fn((options) => options?.headers || {}),
|
||||
};
|
||||
});
|
||||
|
||||
jest.mock('@librechat/data-schemas', () => ({
|
||||
...jest.requireActual('@librechat/data-schemas'),
|
||||
logger: {
|
||||
error: jest.fn(),
|
||||
warn: jest.fn(),
|
||||
debug: jest.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
const mockedAxios = axios as jest.Mocked<typeof axios>;
|
||||
const { logAxiosError, resolveHeaders } = jest.requireMock('~/utils');
|
||||
|
||||
mockedAxios.get.mockResolvedValue({
|
||||
data: {
|
||||
data: [{ id: 'model-1' }, { id: 'model-2' }],
|
||||
},
|
||||
});
|
||||
|
||||
describe('fetchModels', () => {
|
||||
it('fetches models successfully from the API', async () => {
|
||||
const models = await fetchModels({
|
||||
user: 'user123',
|
||||
apiKey: 'testApiKey',
|
||||
baseURL: 'https://api.test.com',
|
||||
name: 'TestAPI',
|
||||
});
|
||||
|
||||
expect(models).toEqual(['model-1', 'model-2']);
|
||||
expect(mockedAxios.get).toHaveBeenCalledWith(
|
||||
expect.stringContaining('https://api.test.com/models'),
|
||||
expect.any(Object),
|
||||
);
|
||||
});
|
||||
|
||||
it('adds the user ID to the models query when option and ID are passed', async () => {
|
||||
const models = await fetchModels({
|
||||
user: 'user123',
|
||||
apiKey: 'testApiKey',
|
||||
baseURL: 'https://api.test.com',
|
||||
userIdQuery: true,
|
||||
name: 'TestAPI',
|
||||
});
|
||||
|
||||
expect(models).toEqual(['model-1', 'model-2']);
|
||||
expect(mockedAxios.get).toHaveBeenCalledWith(
|
||||
expect.stringContaining('https://api.test.com/models?user=user123'),
|
||||
expect.any(Object),
|
||||
);
|
||||
});
|
||||
|
||||
it('should pass custom headers to the API request', async () => {
|
||||
const customHeaders = {
|
||||
'X-Custom-Header': 'custom-value',
|
||||
'X-API-Version': 'v2',
|
||||
};
|
||||
|
||||
await fetchModels({
|
||||
user: 'user123',
|
||||
apiKey: 'testApiKey',
|
||||
baseURL: 'https://api.test.com',
|
||||
name: 'TestAPI',
|
||||
headers: customHeaders,
|
||||
});
|
||||
|
||||
expect(mockedAxios.get).toHaveBeenCalledWith(
|
||||
expect.stringContaining('https://api.test.com/models'),
|
||||
expect.objectContaining({
|
||||
headers: expect.objectContaining({
|
||||
'X-Custom-Header': 'custom-value',
|
||||
'X-API-Version': 'v2',
|
||||
Authorization: 'Bearer testApiKey',
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should handle null headers gracefully', async () => {
|
||||
await fetchModels({
|
||||
user: 'user123',
|
||||
apiKey: 'testApiKey',
|
||||
baseURL: 'https://api.test.com',
|
||||
name: 'TestAPI',
|
||||
headers: null,
|
||||
});
|
||||
|
||||
expect(mockedAxios.get).toHaveBeenCalledWith(
|
||||
expect.stringContaining('https://api.test.com/models'),
|
||||
expect.objectContaining({
|
||||
headers: expect.objectContaining({
|
||||
Authorization: 'Bearer testApiKey',
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should handle undefined headers gracefully', async () => {
|
||||
await fetchModels({
|
||||
user: 'user123',
|
||||
apiKey: 'testApiKey',
|
||||
baseURL: 'https://api.test.com',
|
||||
name: 'TestAPI',
|
||||
headers: undefined,
|
||||
});
|
||||
|
||||
expect(mockedAxios.get).toHaveBeenCalledWith(
|
||||
expect.stringContaining('https://api.test.com/models'),
|
||||
expect.objectContaining({
|
||||
headers: expect.objectContaining({
|
||||
Authorization: 'Bearer testApiKey',
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
});
|
||||
|
||||
describe('fetchModels with createTokenConfig true', () => {
|
||||
const data = {
|
||||
data: [
|
||||
{
|
||||
id: 'model-1',
|
||||
pricing: {
|
||||
prompt: '0.002',
|
||||
completion: '0.001',
|
||||
},
|
||||
context_length: 1024,
|
||||
},
|
||||
{
|
||||
id: 'model-2',
|
||||
pricing: {
|
||||
prompt: '0.003',
|
||||
completion: '0.0015',
|
||||
},
|
||||
context_length: 2048,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
mockedAxios.get.mockResolvedValue({ data });
|
||||
});
|
||||
|
||||
it('creates and stores token configuration if createTokenConfig is true', async () => {
|
||||
await fetchModels({
|
||||
user: 'user123',
|
||||
apiKey: 'testApiKey',
|
||||
baseURL: 'https://api.test.com',
|
||||
createTokenConfig: true,
|
||||
});
|
||||
|
||||
const { processModelData } = jest.requireMock('~/utils');
|
||||
expect(processModelData).toHaveBeenCalled();
|
||||
expect(processModelData).toHaveBeenCalledWith(data);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getOpenAIModels', () => {
|
||||
let originalEnv: NodeJS.ProcessEnv;
|
||||
|
||||
beforeEach(() => {
|
||||
originalEnv = { ...process.env };
|
||||
mockedAxios.get.mockRejectedValue(new Error('Network error'));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
process.env = originalEnv;
|
||||
mockedAxios.get.mockReset();
|
||||
});
|
||||
|
||||
it('returns default models when no environment configurations are provided (and fetch fails)', async () => {
|
||||
const models = await getOpenAIModels({ user: 'user456' });
|
||||
expect(models).toContain('gpt-4');
|
||||
});
|
||||
|
||||
it('returns `AZURE_OPENAI_MODELS` with `azure` flag (and fetch fails)', async () => {
|
||||
process.env.AZURE_OPENAI_MODELS = 'azure-model,azure-model-2';
|
||||
const models = await getOpenAIModels({ azure: true });
|
||||
expect(models).toEqual(expect.arrayContaining(['azure-model', 'azure-model-2']));
|
||||
});
|
||||
|
||||
it('returns `OPENAI_MODELS` with no flags (and fetch fails)', async () => {
|
||||
process.env.OPENAI_MODELS = 'openai-model,openai-model-2';
|
||||
const models = await getOpenAIModels({});
|
||||
expect(models).toEqual(expect.arrayContaining(['openai-model', 'openai-model-2']));
|
||||
});
|
||||
|
||||
it('utilizes proxy configuration when PROXY is set', async () => {
|
||||
mockedAxios.get.mockResolvedValue({
|
||||
data: {
|
||||
data: [],
|
||||
},
|
||||
});
|
||||
process.env.PROXY = 'http://localhost:8888';
|
||||
process.env.OPENAI_API_KEY = 'mockedApiKey';
|
||||
await getOpenAIModels({ user: 'user456' });
|
||||
|
||||
expect(mockedAxios.get).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
expect.objectContaining({
|
||||
httpsAgent: expect.anything(),
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getOpenAIModels sorting behavior', () => {
|
||||
let originalEnv: NodeJS.ProcessEnv;
|
||||
|
||||
beforeEach(() => {
|
||||
originalEnv = { ...process.env };
|
||||
process.env.OPENAI_API_KEY = 'mockedApiKey';
|
||||
mockedAxios.get.mockResolvedValue({
|
||||
data: {
|
||||
data: [
|
||||
{ id: 'gpt-3.5-turbo-instruct-0914' },
|
||||
{ id: 'gpt-3.5-turbo-instruct' },
|
||||
{ id: 'gpt-3.5-turbo' },
|
||||
{ id: 'gpt-4-0314' },
|
||||
{ id: 'gpt-4-turbo-preview' },
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
process.env = originalEnv;
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it('ensures instruct models are listed last', async () => {
|
||||
const models = await getOpenAIModels({ user: 'user456' });
|
||||
|
||||
expect(models[models.length - 1]).toMatch(/instruct/);
|
||||
|
||||
const instructIndexes = models
|
||||
.map((model, index) => (model.includes('instruct') ? index : -1))
|
||||
.filter((index) => index !== -1);
|
||||
const nonInstructIndexes = models
|
||||
.map((model, index) => (!model.includes('instruct') ? index : -1))
|
||||
.filter((index) => index !== -1);
|
||||
|
||||
expect(Math.max(...nonInstructIndexes)).toBeLessThan(Math.min(...instructIndexes));
|
||||
|
||||
const expectedOrder = [
|
||||
'gpt-3.5-turbo',
|
||||
'gpt-4-0314',
|
||||
'gpt-4-turbo-preview',
|
||||
'gpt-3.5-turbo-instruct-0914',
|
||||
'gpt-3.5-turbo-instruct',
|
||||
];
|
||||
expect(models).toEqual(expectedOrder);
|
||||
});
|
||||
});
|
||||
|
||||
describe('fetchModels with Ollama specific logic', () => {
|
||||
const mockOllamaData = {
|
||||
data: {
|
||||
models: [{ name: 'Ollama-Base' }, { name: 'Ollama-Advanced' }],
|
||||
},
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
mockedAxios.get.mockResolvedValue(mockOllamaData);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it('should fetch Ollama models when name starts with "ollama"', async () => {
|
||||
const models = await fetchModels({
|
||||
user: 'user789',
|
||||
apiKey: 'testApiKey',
|
||||
baseURL: 'https://api.ollama.test.com',
|
||||
name: 'OllamaAPI',
|
||||
});
|
||||
|
||||
expect(models).toEqual(['Ollama-Base', 'Ollama-Advanced']);
|
||||
expect(mockedAxios.get).toHaveBeenCalledWith('https://api.ollama.test.com/api/tags', {
|
||||
headers: {},
|
||||
timeout: 5000,
|
||||
});
|
||||
});
|
||||
|
||||
it('should pass headers and user object to Ollama fetchModels', async () => {
|
||||
const customHeaders = {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: 'Bearer custom-token',
|
||||
};
|
||||
const userObject = {
|
||||
id: 'user789',
|
||||
email: 'test@example.com',
|
||||
};
|
||||
|
||||
(resolveHeaders as jest.Mock).mockReturnValueOnce(customHeaders);
|
||||
|
||||
const models = await fetchModels({
|
||||
user: 'user789',
|
||||
apiKey: 'testApiKey',
|
||||
baseURL: 'https://api.ollama.test.com',
|
||||
name: 'ollama',
|
||||
headers: customHeaders,
|
||||
userObject,
|
||||
});
|
||||
|
||||
expect(models).toEqual(['Ollama-Base', 'Ollama-Advanced']);
|
||||
expect(resolveHeaders).toHaveBeenCalledWith({
|
||||
headers: customHeaders,
|
||||
user: userObject,
|
||||
});
|
||||
expect(mockedAxios.get).toHaveBeenCalledWith('https://api.ollama.test.com/api/tags', {
|
||||
headers: customHeaders,
|
||||
timeout: 5000,
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle errors gracefully when fetching Ollama models fails and fallback to OpenAI-compatible fetch', async () => {
|
||||
mockedAxios.get.mockRejectedValueOnce(new Error('Ollama API error'));
|
||||
mockedAxios.get.mockResolvedValueOnce({
|
||||
data: {
|
||||
data: [{ id: 'fallback-model-1' }, { id: 'fallback-model-2' }],
|
||||
},
|
||||
});
|
||||
|
||||
const models = await fetchModels({
|
||||
user: 'user789',
|
||||
apiKey: 'testApiKey',
|
||||
baseURL: 'https://api.ollama.test.com',
|
||||
name: 'OllamaAPI',
|
||||
});
|
||||
|
||||
expect(models).toEqual(['fallback-model-1', 'fallback-model-2']);
|
||||
expect(logAxiosError).toHaveBeenCalledWith({
|
||||
message:
|
||||
'Failed to fetch models from Ollama API. Attempting to fetch via OpenAI-compatible endpoint.',
|
||||
error: expect.any(Error),
|
||||
});
|
||||
expect(mockedAxios.get).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('should return an empty array if no baseURL is provided', async () => {
|
||||
const models = await fetchModels({
|
||||
user: 'user789',
|
||||
apiKey: 'testApiKey',
|
||||
name: 'OllamaAPI',
|
||||
});
|
||||
expect(models).toEqual([]);
|
||||
});
|
||||
|
||||
it('should not fetch Ollama models if the name does not start with "ollama"', async () => {
|
||||
mockedAxios.get.mockResolvedValue({
|
||||
data: {
|
||||
data: [{ id: 'model-1' }, { id: 'model-2' }],
|
||||
},
|
||||
});
|
||||
|
||||
const models = await fetchModels({
|
||||
user: 'user789',
|
||||
apiKey: 'testApiKey',
|
||||
baseURL: 'https://api.test.com',
|
||||
name: 'TestAPI',
|
||||
});
|
||||
|
||||
expect(models).toEqual(['model-1', 'model-2']);
|
||||
expect(mockedAxios.get).toHaveBeenCalledWith('https://api.test.com/models', expect.any(Object));
|
||||
});
|
||||
});
|
||||
|
||||
describe('fetchModels URL construction with trailing slashes', () => {
|
||||
beforeEach(() => {
|
||||
mockedAxios.get.mockResolvedValue({
|
||||
data: {
|
||||
data: [{ id: 'model-1' }, { id: 'model-2' }],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it('should not create double slashes when baseURL has a trailing slash', async () => {
|
||||
await fetchModels({
|
||||
user: 'user123',
|
||||
apiKey: 'testApiKey',
|
||||
baseURL: 'https://api.test.com/v1/',
|
||||
name: 'TestAPI',
|
||||
});
|
||||
|
||||
expect(mockedAxios.get).toHaveBeenCalledWith(
|
||||
'https://api.test.com/v1/models',
|
||||
expect.any(Object),
|
||||
);
|
||||
});
|
||||
|
||||
it('should handle baseURL without trailing slash normally', async () => {
|
||||
await fetchModels({
|
||||
user: 'user123',
|
||||
apiKey: 'testApiKey',
|
||||
baseURL: 'https://api.test.com/v1',
|
||||
name: 'TestAPI',
|
||||
});
|
||||
|
||||
expect(mockedAxios.get).toHaveBeenCalledWith(
|
||||
'https://api.test.com/v1/models',
|
||||
expect.any(Object),
|
||||
);
|
||||
});
|
||||
|
||||
it('should handle baseURL with multiple trailing slashes', async () => {
|
||||
await fetchModels({
|
||||
user: 'user123',
|
||||
apiKey: 'testApiKey',
|
||||
baseURL: 'https://api.test.com/v1///',
|
||||
name: 'TestAPI',
|
||||
});
|
||||
|
||||
expect(mockedAxios.get).toHaveBeenCalledWith(
|
||||
'https://api.test.com/v1/models',
|
||||
expect.any(Object),
|
||||
);
|
||||
});
|
||||
|
||||
it('should correctly append query params after stripping trailing slashes', async () => {
|
||||
await fetchModels({
|
||||
user: 'user123',
|
||||
apiKey: 'testApiKey',
|
||||
baseURL: 'https://api.test.com/v1/',
|
||||
name: 'TestAPI',
|
||||
userIdQuery: true,
|
||||
});
|
||||
|
||||
expect(mockedAxios.get).toHaveBeenCalledWith(
|
||||
'https://api.test.com/v1/models?user=user123',
|
||||
expect.any(Object),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('splitAndTrim', () => {
|
||||
it('should split a string by commas and trim each value', () => {
|
||||
const input = ' model1, model2 , model3,model4 ';
|
||||
const expected = ['model1', 'model2', 'model3', 'model4'];
|
||||
expect(splitAndTrim(input)).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should return an empty array for empty input', () => {
|
||||
expect(splitAndTrim('')).toEqual([]);
|
||||
});
|
||||
|
||||
it('should return an empty array for null input', () => {
|
||||
expect(splitAndTrim(null)).toEqual([]);
|
||||
});
|
||||
|
||||
it('should return an empty array for undefined input', () => {
|
||||
expect(splitAndTrim(undefined)).toEqual([]);
|
||||
});
|
||||
|
||||
it('should filter out empty values after trimming', () => {
|
||||
const input = 'model1,, ,model2,';
|
||||
const expected = ['model1', 'model2'];
|
||||
expect(splitAndTrim(input)).toEqual(expected);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getAnthropicModels', () => {
|
||||
let originalEnv: NodeJS.ProcessEnv;
|
||||
|
||||
beforeEach(() => {
|
||||
originalEnv = { ...process.env };
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
process.env = originalEnv;
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it('returns default models when ANTHROPIC_MODELS is not set', async () => {
|
||||
delete process.env.ANTHROPIC_MODELS;
|
||||
const models = await getAnthropicModels();
|
||||
expect(models).toEqual(defaultModels[EModelEndpoint.anthropic]);
|
||||
});
|
||||
|
||||
it('returns models from ANTHROPIC_MODELS when set', async () => {
|
||||
process.env.ANTHROPIC_MODELS = 'claude-1, claude-2 ';
|
||||
const models = await getAnthropicModels();
|
||||
expect(models).toEqual(['claude-1', 'claude-2']);
|
||||
});
|
||||
|
||||
it('should use Anthropic-specific headers when fetching models', async () => {
|
||||
delete process.env.ANTHROPIC_MODELS;
|
||||
process.env.ANTHROPIC_API_KEY = 'test-anthropic-key';
|
||||
|
||||
mockedAxios.get.mockResolvedValue({
|
||||
data: {
|
||||
data: [{ id: 'claude-3' }, { id: 'claude-4' }],
|
||||
},
|
||||
});
|
||||
|
||||
await fetchModels({
|
||||
user: 'user123',
|
||||
apiKey: 'test-anthropic-key',
|
||||
baseURL: 'https://api.anthropic.com/v1',
|
||||
name: EModelEndpoint.anthropic,
|
||||
});
|
||||
|
||||
expect(mockedAxios.get).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
expect.objectContaining({
|
||||
headers: {
|
||||
'x-api-key': 'test-anthropic-key',
|
||||
'anthropic-version': expect.any(String),
|
||||
},
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should pass custom headers for Anthropic endpoint', async () => {
|
||||
const customHeaders = {
|
||||
'X-Custom-Header': 'custom-value',
|
||||
};
|
||||
|
||||
mockedAxios.get.mockResolvedValue({
|
||||
data: {
|
||||
data: [{ id: 'claude-3' }],
|
||||
},
|
||||
});
|
||||
|
||||
await fetchModels({
|
||||
user: 'user123',
|
||||
apiKey: 'test-anthropic-key',
|
||||
baseURL: 'https://api.anthropic.com/v1',
|
||||
name: EModelEndpoint.anthropic,
|
||||
headers: customHeaders,
|
||||
});
|
||||
|
||||
expect(mockedAxios.get).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
expect.objectContaining({
|
||||
headers: {
|
||||
'x-api-key': 'test-anthropic-key',
|
||||
'anthropic-version': expect.any(String),
|
||||
},
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getGoogleModels', () => {
|
||||
let originalEnv: NodeJS.ProcessEnv;
|
||||
|
||||
beforeEach(() => {
|
||||
originalEnv = { ...process.env };
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
process.env = originalEnv;
|
||||
});
|
||||
|
||||
it('returns default models when GOOGLE_MODELS is not set', () => {
|
||||
delete process.env.GOOGLE_MODELS;
|
||||
const models = getGoogleModels();
|
||||
expect(models).toEqual(defaultModels[EModelEndpoint.google]);
|
||||
});
|
||||
|
||||
it('returns models from GOOGLE_MODELS when set', () => {
|
||||
process.env.GOOGLE_MODELS = 'gemini-pro, bard ';
|
||||
const models = getGoogleModels();
|
||||
expect(models).toEqual(['gemini-pro', 'bard']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getBedrockModels', () => {
|
||||
let originalEnv: NodeJS.ProcessEnv;
|
||||
|
||||
beforeEach(() => {
|
||||
originalEnv = { ...process.env };
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
process.env = originalEnv;
|
||||
});
|
||||
|
||||
it('returns default models when BEDROCK_AWS_MODELS is not set', () => {
|
||||
delete process.env.BEDROCK_AWS_MODELS;
|
||||
const models = getBedrockModels();
|
||||
expect(models).toEqual(defaultModels[EModelEndpoint.bedrock]);
|
||||
});
|
||||
|
||||
it('returns models from BEDROCK_AWS_MODELS when set', () => {
|
||||
process.env.BEDROCK_AWS_MODELS = 'anthropic.claude-v2, ai21.j2-ultra ';
|
||||
const models = getBedrockModels();
|
||||
expect(models).toEqual(['anthropic.claude-v2', 'ai21.j2-ultra']);
|
||||
});
|
||||
});
|
||||
383
packages/api/src/endpoints/models.ts
Normal file
383
packages/api/src/endpoints/models.ts
Normal file
|
|
@ -0,0 +1,383 @@
|
|||
import axios from 'axios';
|
||||
import { logger } from '@librechat/data-schemas';
|
||||
import { HttpsProxyAgent } from 'https-proxy-agent';
|
||||
import { CacheKeys, KnownEndpoints, EModelEndpoint, defaultModels } from 'librechat-data-provider';
|
||||
import type { IUser } from '@librechat/data-schemas';
|
||||
import {
|
||||
processModelData,
|
||||
extractBaseURL,
|
||||
isUserProvided,
|
||||
resolveHeaders,
|
||||
deriveBaseURL,
|
||||
logAxiosError,
|
||||
inputSchema,
|
||||
} from '~/utils';
|
||||
import { standardCache } from '~/cache';
|
||||
|
||||
export interface FetchModelsParams {
|
||||
/** User ID for API requests */
|
||||
user?: string;
|
||||
/** API key for authentication */
|
||||
apiKey: string;
|
||||
/** Base URL for the API */
|
||||
baseURL?: string;
|
||||
/** Endpoint name (defaults to 'openAI') */
|
||||
name?: string;
|
||||
/** Whether directEndpoint was configured */
|
||||
direct?: boolean;
|
||||
/** Whether to fetch from Azure */
|
||||
azure?: boolean;
|
||||
/** Whether to send user ID as query parameter */
|
||||
userIdQuery?: boolean;
|
||||
/** Whether to create token configuration from API response */
|
||||
createTokenConfig?: boolean;
|
||||
/** Cache key for token configuration (uses name if omitted) */
|
||||
tokenKey?: string;
|
||||
/** Optional headers for the request */
|
||||
headers?: Record<string, string> | null;
|
||||
/** Optional user object for header resolution */
|
||||
userObject?: Partial<IUser>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches Ollama models from the specified base API path.
|
||||
* @param baseURL - The Ollama server URL
|
||||
* @param options - Optional configuration
|
||||
* @returns Promise resolving to array of model names
|
||||
*/
|
||||
async function fetchOllamaModels(
|
||||
baseURL: string,
|
||||
options: { headers?: Record<string, string> | null; user?: Partial<IUser> } = {},
|
||||
): Promise<string[]> {
|
||||
if (!baseURL) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const ollamaEndpoint = deriveBaseURL(baseURL);
|
||||
|
||||
const resolvedHeaders = resolveHeaders({
|
||||
headers: options.headers ?? undefined,
|
||||
user: options.user,
|
||||
});
|
||||
|
||||
const response = await axios.get<{ models: Array<{ name: string }> }>(
|
||||
`${ollamaEndpoint}/api/tags`,
|
||||
{
|
||||
headers: resolvedHeaders,
|
||||
timeout: 5000,
|
||||
},
|
||||
);
|
||||
|
||||
return response.data.models.map((tag) => tag.name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits a string by commas and trims each resulting value.
|
||||
* @param input - The input string to split.
|
||||
* @returns An array of trimmed values.
|
||||
*/
|
||||
export function splitAndTrim(input: string | null | undefined): string[] {
|
||||
if (!input || typeof input !== 'string') {
|
||||
return [];
|
||||
}
|
||||
return input
|
||||
.split(',')
|
||||
.map((item) => item.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches models from the specified base API path or Azure, based on the provided configuration.
|
||||
*
|
||||
* @param params - The parameters for fetching the models.
|
||||
* @returns A promise that resolves to an array of model identifiers.
|
||||
*/
|
||||
export async function fetchModels({
|
||||
user,
|
||||
apiKey,
|
||||
baseURL: _baseURL,
|
||||
name = EModelEndpoint.openAI,
|
||||
direct = false,
|
||||
azure = false,
|
||||
userIdQuery = false,
|
||||
createTokenConfig = true,
|
||||
tokenKey,
|
||||
headers,
|
||||
userObject,
|
||||
}: FetchModelsParams): Promise<string[]> {
|
||||
let models: string[] = [];
|
||||
const baseURL = direct ? extractBaseURL(_baseURL ?? '') : _baseURL;
|
||||
|
||||
if (!baseURL && !azure) {
|
||||
return models;
|
||||
}
|
||||
|
||||
if (!apiKey) {
|
||||
return models;
|
||||
}
|
||||
|
||||
if (name && name.toLowerCase().startsWith(KnownEndpoints.ollama)) {
|
||||
try {
|
||||
return await fetchOllamaModels(baseURL ?? '', { headers, user: userObject });
|
||||
} catch (ollamaError) {
|
||||
const logMessage =
|
||||
'Failed to fetch models from Ollama API. Attempting to fetch via OpenAI-compatible endpoint.';
|
||||
logAxiosError({ message: logMessage, error: ollamaError as Error });
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const options: {
|
||||
headers: Record<string, string>;
|
||||
timeout: number;
|
||||
httpsAgent?: HttpsProxyAgent;
|
||||
} = {
|
||||
headers: {
|
||||
...(headers ?? {}),
|
||||
},
|
||||
timeout: 5000,
|
||||
};
|
||||
|
||||
if (name === EModelEndpoint.anthropic) {
|
||||
options.headers = {
|
||||
'x-api-key': apiKey,
|
||||
'anthropic-version': process.env.ANTHROPIC_VERSION || '2023-06-01',
|
||||
};
|
||||
} else {
|
||||
options.headers.Authorization = `Bearer ${apiKey}`;
|
||||
}
|
||||
|
||||
if (process.env.PROXY) {
|
||||
options.httpsAgent = new HttpsProxyAgent(process.env.PROXY);
|
||||
}
|
||||
|
||||
if (process.env.OPENAI_ORGANIZATION && baseURL?.includes('openai')) {
|
||||
options.headers['OpenAI-Organization'] = process.env.OPENAI_ORGANIZATION;
|
||||
}
|
||||
|
||||
const url = new URL(`${(baseURL ?? '').replace(/\/+$/, '')}${azure ? '' : '/models'}`);
|
||||
if (user && userIdQuery) {
|
||||
url.searchParams.append('user', user);
|
||||
}
|
||||
const res = await axios.get(url.toString(), options);
|
||||
|
||||
const input = res.data;
|
||||
|
||||
const validationResult = inputSchema.safeParse(input);
|
||||
if (validationResult.success && createTokenConfig) {
|
||||
const endpointTokenConfig = processModelData(input);
|
||||
const cache = standardCache(CacheKeys.TOKEN_CONFIG);
|
||||
await cache.set(tokenKey ?? name, endpointTokenConfig);
|
||||
}
|
||||
models = input.data.map((item: { id: string }) => item.id);
|
||||
} catch (error) {
|
||||
const logMessage = `Failed to fetch models from ${azure ? 'Azure ' : ''}${name} API`;
|
||||
logAxiosError({ message: logMessage, error: error as Error });
|
||||
}
|
||||
|
||||
return models;
|
||||
}
|
||||
|
||||
/** Options for fetching OpenAI models */
|
||||
export interface GetOpenAIModelsOptions {
|
||||
/** User ID for API requests */
|
||||
user?: string;
|
||||
/** Whether to fetch from Azure */
|
||||
azure?: boolean;
|
||||
/** Whether to fetch models for the Assistants endpoint */
|
||||
assistants?: boolean;
|
||||
/** OpenAI API key (if not using environment variable) */
|
||||
openAIApiKey?: string;
|
||||
/** Whether user provides their own API key */
|
||||
userProvidedOpenAI?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches models from OpenAI or Azure based on the provided options.
|
||||
* @param opts - Options for fetching models
|
||||
* @param _models - Fallback models array
|
||||
* @returns Promise resolving to array of model IDs
|
||||
*/
|
||||
export async function fetchOpenAIModels(
|
||||
opts: GetOpenAIModelsOptions,
|
||||
_models: string[] = [],
|
||||
): Promise<string[]> {
|
||||
let models = _models.slice() ?? [];
|
||||
const apiKey = opts.openAIApiKey ?? process.env.OPENAI_API_KEY;
|
||||
const openaiBaseURL = 'https://api.openai.com/v1';
|
||||
let baseURL = openaiBaseURL;
|
||||
let reverseProxyUrl = process.env.OPENAI_REVERSE_PROXY;
|
||||
|
||||
if (opts.assistants && process.env.ASSISTANTS_BASE_URL) {
|
||||
reverseProxyUrl = process.env.ASSISTANTS_BASE_URL;
|
||||
} else if (opts.azure) {
|
||||
return models;
|
||||
}
|
||||
|
||||
if (reverseProxyUrl) {
|
||||
baseURL = extractBaseURL(reverseProxyUrl) ?? openaiBaseURL;
|
||||
}
|
||||
|
||||
const modelsCache = standardCache(CacheKeys.MODEL_QUERIES);
|
||||
|
||||
const cachedModels = await modelsCache.get(baseURL);
|
||||
if (cachedModels) {
|
||||
return cachedModels as string[];
|
||||
}
|
||||
|
||||
if (baseURL || opts.azure) {
|
||||
models = await fetchModels({
|
||||
apiKey: apiKey ?? '',
|
||||
baseURL,
|
||||
azure: opts.azure,
|
||||
user: opts.user,
|
||||
name: EModelEndpoint.openAI,
|
||||
});
|
||||
}
|
||||
|
||||
if (models.length === 0) {
|
||||
return _models;
|
||||
}
|
||||
|
||||
if (baseURL === openaiBaseURL) {
|
||||
const regex = /(text-davinci-003|gpt-|o\d+)/;
|
||||
const excludeRegex = /audio|realtime/;
|
||||
models = models.filter((model) => regex.test(model) && !excludeRegex.test(model));
|
||||
const instructModels = models.filter((model) => model.includes('instruct'));
|
||||
const otherModels = models.filter((model) => !model.includes('instruct'));
|
||||
models = otherModels.concat(instructModels);
|
||||
}
|
||||
|
||||
await modelsCache.set(baseURL, models);
|
||||
return models;
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads the default models for OpenAI or Azure.
|
||||
* @param opts - Options for getting models
|
||||
* @returns Promise resolving to array of model IDs
|
||||
*/
|
||||
export async function getOpenAIModels(opts: GetOpenAIModelsOptions = {}): Promise<string[]> {
|
||||
let models = defaultModels[EModelEndpoint.openAI];
|
||||
|
||||
if (opts.assistants) {
|
||||
models = defaultModels[EModelEndpoint.assistants];
|
||||
} else if (opts.azure) {
|
||||
models = defaultModels[EModelEndpoint.azureAssistants];
|
||||
}
|
||||
|
||||
let key: string;
|
||||
if (opts.assistants) {
|
||||
key = 'ASSISTANTS_MODELS';
|
||||
} else if (opts.azure) {
|
||||
key = 'AZURE_OPENAI_MODELS';
|
||||
} else {
|
||||
key = 'OPENAI_MODELS';
|
||||
}
|
||||
|
||||
if (process.env[key]) {
|
||||
return splitAndTrim(process.env[key]);
|
||||
}
|
||||
|
||||
if (opts.userProvidedOpenAI) {
|
||||
return models;
|
||||
}
|
||||
|
||||
return await fetchOpenAIModels(opts, models);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches models from the Anthropic API.
|
||||
* @param opts - Options for fetching models
|
||||
* @param _models - Fallback models array
|
||||
* @returns Promise resolving to array of model IDs
|
||||
*/
|
||||
export async function fetchAnthropicModels(
|
||||
opts: { user?: string } = {},
|
||||
_models: string[] = [],
|
||||
): Promise<string[]> {
|
||||
let models = _models.slice() ?? [];
|
||||
const apiKey = process.env.ANTHROPIC_API_KEY;
|
||||
const anthropicBaseURL = 'https://api.anthropic.com/v1';
|
||||
let baseURL = anthropicBaseURL;
|
||||
const reverseProxyUrl = process.env.ANTHROPIC_REVERSE_PROXY;
|
||||
|
||||
if (reverseProxyUrl) {
|
||||
baseURL = extractBaseURL(reverseProxyUrl) ?? anthropicBaseURL;
|
||||
}
|
||||
|
||||
if (!apiKey) {
|
||||
return models;
|
||||
}
|
||||
|
||||
const modelsCache = standardCache(CacheKeys.MODEL_QUERIES);
|
||||
|
||||
const cachedModels = await modelsCache.get(baseURL);
|
||||
if (cachedModels) {
|
||||
return cachedModels as string[];
|
||||
}
|
||||
|
||||
if (baseURL) {
|
||||
models = await fetchModels({
|
||||
apiKey,
|
||||
baseURL,
|
||||
user: opts.user,
|
||||
name: EModelEndpoint.anthropic,
|
||||
tokenKey: EModelEndpoint.anthropic,
|
||||
});
|
||||
}
|
||||
|
||||
if (models.length === 0) {
|
||||
return _models;
|
||||
}
|
||||
|
||||
await modelsCache.set(baseURL, models);
|
||||
return models;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets Anthropic models from environment or API.
|
||||
* @param opts - Options for fetching models
|
||||
* @returns Promise resolving to array of model IDs
|
||||
*/
|
||||
export async function getAnthropicModels(opts: { user?: string } = {}): Promise<string[]> {
|
||||
const models = defaultModels[EModelEndpoint.anthropic];
|
||||
if (process.env.ANTHROPIC_MODELS) {
|
||||
return splitAndTrim(process.env.ANTHROPIC_MODELS);
|
||||
}
|
||||
|
||||
if (isUserProvided(process.env.ANTHROPIC_API_KEY)) {
|
||||
return models;
|
||||
}
|
||||
|
||||
try {
|
||||
return await fetchAnthropicModels(opts, models);
|
||||
} catch (error) {
|
||||
logger.error('Error fetching Anthropic models:', error);
|
||||
return models;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets Google models from environment or defaults.
|
||||
* @returns Array of model IDs
|
||||
*/
|
||||
export function getGoogleModels(): string[] {
|
||||
let models = defaultModels[EModelEndpoint.google];
|
||||
if (process.env.GOOGLE_MODELS) {
|
||||
models = splitAndTrim(process.env.GOOGLE_MODELS);
|
||||
}
|
||||
return models;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets Bedrock models from environment or defaults.
|
||||
* @returns Array of model IDs
|
||||
*/
|
||||
export function getBedrockModels(): string[] {
|
||||
let models = defaultModels[EModelEndpoint.bedrock];
|
||||
if (process.env.BEDROCK_AWS_MODELS) {
|
||||
models = splitAndTrim(process.env.BEDROCK_AWS_MODELS);
|
||||
}
|
||||
return models;
|
||||
}
|
||||
|
|
@ -1,13 +1,11 @@
|
|||
import { ErrorTypes, EModelEndpoint, mapModelToAzureConfig } from 'librechat-data-provider';
|
||||
import type {
|
||||
InitializeOpenAIOptionsParams,
|
||||
BaseInitializeParams,
|
||||
InitializeResultBase,
|
||||
OpenAIConfigOptions,
|
||||
LLMConfigResult,
|
||||
UserKeyValues,
|
||||
} from '~/types';
|
||||
import { getAzureCredentials } from '~/utils/azure';
|
||||
import { isUserProvided } from '~/utils/common';
|
||||
import { resolveHeaders } from '~/utils/env';
|
||||
import { getAzureCredentials, resolveHeaders, isUserProvided, checkUserKeyExpiry } from '~/utils';
|
||||
import { getOpenAIConfig } from './config';
|
||||
|
||||
/**
|
||||
|
|
@ -18,25 +16,18 @@ import { getOpenAIConfig } from './config';
|
|||
* @returns Promise resolving to OpenAI configuration options
|
||||
* @throws Error if API key is missing or user key has expired
|
||||
*/
|
||||
export const initializeOpenAI = async ({
|
||||
export async function initializeOpenAI({
|
||||
req,
|
||||
appConfig,
|
||||
overrideModel,
|
||||
endpointOption,
|
||||
overrideEndpoint,
|
||||
getUserKeyValues,
|
||||
checkUserKeyExpiry,
|
||||
}: InitializeOpenAIOptionsParams): Promise<LLMConfigResult> => {
|
||||
endpoint,
|
||||
model_parameters,
|
||||
db,
|
||||
}: BaseInitializeParams): Promise<InitializeResultBase> {
|
||||
const appConfig = req.config;
|
||||
const { PROXY, OPENAI_API_KEY, AZURE_API_KEY, OPENAI_REVERSE_PROXY, AZURE_OPENAI_BASEURL } =
|
||||
process.env;
|
||||
|
||||
const { key: expiresAt } = req.body;
|
||||
const modelName = overrideModel ?? req.body.model;
|
||||
const endpoint = overrideEndpoint ?? req.body.endpoint;
|
||||
|
||||
if (!endpoint) {
|
||||
throw new Error('Endpoint is required');
|
||||
}
|
||||
const modelName = model_parameters?.model as string | undefined;
|
||||
|
||||
const credentials = {
|
||||
[EModelEndpoint.openAI]: OPENAI_API_KEY,
|
||||
|
|
@ -54,7 +45,7 @@ export const initializeOpenAI = async ({
|
|||
let userValues: UserKeyValues | null = null;
|
||||
if (expiresAt && (userProvidesKey || userProvidesURL)) {
|
||||
checkUserKeyExpiry(expiresAt, endpoint);
|
||||
userValues = await getUserKeyValues({ userId: req.user.id, name: endpoint });
|
||||
userValues = await db.getUserKeyValues({ userId: req.user?.id ?? '', name: endpoint });
|
||||
}
|
||||
|
||||
let apiKey = userProvidesKey
|
||||
|
|
@ -71,7 +62,8 @@ export const initializeOpenAI = async ({
|
|||
};
|
||||
|
||||
const isAzureOpenAI = endpoint === EModelEndpoint.azureOpenAI;
|
||||
const azureConfig = isAzureOpenAI && appConfig.endpoints?.[EModelEndpoint.azureOpenAI];
|
||||
const azureConfig = isAzureOpenAI && appConfig?.endpoints?.[EModelEndpoint.azureOpenAI];
|
||||
let isServerless = false;
|
||||
|
||||
if (isAzureOpenAI && azureConfig) {
|
||||
const { modelGroupMap, groupMap } = azureConfig;
|
||||
|
|
@ -85,6 +77,7 @@ export const initializeOpenAI = async ({
|
|||
modelGroupMap,
|
||||
groupMap,
|
||||
});
|
||||
isServerless = serverless === true;
|
||||
|
||||
clientOptions.reverseProxyUrl = configBaseURL ?? clientOptions.reverseProxyUrl;
|
||||
clientOptions.headers = resolveHeaders({
|
||||
|
|
@ -99,9 +92,9 @@ export const initializeOpenAI = async ({
|
|||
}
|
||||
|
||||
apiKey = azureOptions.azureOpenAIApiKey;
|
||||
clientOptions.azure = !serverless ? azureOptions : undefined;
|
||||
clientOptions.azure = !isServerless ? azureOptions : undefined;
|
||||
|
||||
if (serverless === true) {
|
||||
if (isServerless) {
|
||||
clientOptions.defaultQuery = azureOptions.azureOpenAIApiVersion
|
||||
? { 'api-version': azureOptions.azureOpenAIApiVersion }
|
||||
: undefined;
|
||||
|
|
@ -130,9 +123,9 @@ export const initializeOpenAI = async ({
|
|||
}
|
||||
|
||||
const modelOptions = {
|
||||
...endpointOption.model_parameters,
|
||||
...(model_parameters ?? {}),
|
||||
model: modelName,
|
||||
user: req.user.id,
|
||||
user: req.user?.id,
|
||||
};
|
||||
|
||||
const finalClientOptions: OpenAIConfigOptions = {
|
||||
|
|
@ -142,8 +135,13 @@ export const initializeOpenAI = async ({
|
|||
|
||||
const options = getOpenAIConfig(apiKey, finalClientOptions, endpoint);
|
||||
|
||||
const openAIConfig = appConfig.endpoints?.[EModelEndpoint.openAI];
|
||||
const allConfig = appConfig.endpoints?.all;
|
||||
/** Set useLegacyContent for Azure serverless deployments */
|
||||
if (isServerless) {
|
||||
(options as InitializeResultBase).useLegacyContent = true;
|
||||
}
|
||||
|
||||
const openAIConfig = appConfig?.endpoints?.[EModelEndpoint.openAI];
|
||||
const allConfig = appConfig?.endpoints?.all;
|
||||
const azureRate = modelName?.includes('gpt-4') ? 30 : 17;
|
||||
|
||||
let streamRate: number | undefined;
|
||||
|
|
@ -163,4 +161,4 @@ export const initializeOpenAI = async ({
|
|||
}
|
||||
|
||||
return options;
|
||||
};
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue