mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-01-24 11:16:12 +01:00
🎚️ feat: Anthropic Parameter Set Support via Custom Endpoints (#9415)
* refactor: modularize openai llm config logic into new getOpenAILLMConfig function (#9412) * ✈️ refactor: Migrate Anthropic's getLLMConfig to TypeScript (#9413) * refactor: move tokens.js over to packages/api and update imports * refactor: port tokens.js to typescript * refactor: move helpers.js over to packages/api and update imports * refactor: port helpers.js to typescript * refactor: move anthropic/llm.js over to packages/api and update imports * refactor: port anthropic/llm.js to typescript with supporting types in types/anthropic.ts and updated tests in llm.spec.js * refactor: move llm.spec.js over to packages/api and update import * refactor: port llm.spec.js over to typescript * 📝 Add Prompt Parameter Support for Anthropic Custom Endpoints (#9414) feat: add anthropic llm config support for openai-like (custom) endpoints * fix: missed compiler / type issues from addition of getAnthropicLLMConfig * refactor: update tokens.ts to export constants and functions, enhance type definitions, and adjust default values * WIP: first pass, decouple `llmConfig` from `configOptions` * chore: update import path for OpenAI configuration from 'llm' to 'config' * refactor: enhance type definitions for ThinkingConfig and update modelOptions in AnthropicConfigOptions * refactor: cleanup type, introduce openai transform from alt provider * chore: integrate removeNullishValues in Google llmConfig and update OpenAI exports * chore: bump version of @librechat/api to 1.3.5 in package.json and package-lock.json * refactor: update customParams type in OpenAIConfigOptions to use TConfig['customParams'] * refactor: enhance transformToOpenAIConfig to include fromEndpoint and improve config extraction * refactor: conform userId field for anthropic/openai, cleanup anthropic typing * ci: add backward compatibility tests for getOpenAIConfig with various endpoints and configurations * ci: replace userId with user in clientOptions for getLLMConfig * test: add Azure OpenAI endpoint tests for various configurations in getOpenAIConfig * refactor: defaultHeaders retrieval for prompt caching for anthropic-based custom endpoint (litellm) * test: add unit tests for getOpenAIConfig with various Anthropic model configurations * test: enhance Anthropic compatibility tests with addParams and dropParams handling * chore: update @librechat/agents dependency to version 2.4.78 in package.json and package-lock.json * chore: update @librechat/agents dependency to version 2.4.79 in package.json and package-lock.json --------- Co-authored-by: Danny Avila <danny@librechat.ai>
This commit is contained in:
Dustin Healy
GitHub
parent
7de6f6e44c
commit
c6ecf0095b
40 changed files with 1736 additions and 432 deletions
551
packages/api/src/endpoints/openai/config.anthropic.spec.ts
Normal file
551
packages/api/src/endpoints/openai/config.anthropic.spec.ts
Normal file
|
|
@ -0,0 +1,551 @@
|
|||
import { getOpenAIConfig } from './config';
|
||||
|
||||
describe('getOpenAIConfig - Anthropic Compatibility', () => {
|
||||
describe('Anthropic via LiteLLM', () => {
|
||||
it('should handle basic Anthropic configuration with defaultParamsEndpoint', () => {
|
||||
const apiKey = 'sk-xxxx';
|
||||
const endpoint = 'Anthropic (via LiteLLM)';
|
||||
const options = {
|
||||
modelOptions: {
|
||||
model: 'claude-sonnet-4',
|
||||
user: 'some_user_id',
|
||||
},
|
||||
reverseProxyUrl: 'http://host.docker.internal:4000/v1',
|
||||
proxy: '',
|
||||
headers: {},
|
||||
addParams: undefined,
|
||||
dropParams: undefined,
|
||||
customParams: {
|
||||
defaultParamsEndpoint: 'anthropic',
|
||||
paramDefinitions: [],
|
||||
},
|
||||
endpoint: 'Anthropic (via LiteLLM)',
|
||||
endpointType: 'custom',
|
||||
};
|
||||
|
||||
const result = getOpenAIConfig(apiKey, options, endpoint);
|
||||
|
||||
expect(result).toEqual({
|
||||
llmConfig: {
|
||||
apiKey: 'sk-xxxx',
|
||||
model: 'claude-sonnet-4',
|
||||
stream: true,
|
||||
maxTokens: 8192,
|
||||
modelKwargs: {
|
||||
metadata: {
|
||||
user_id: 'some_user_id',
|
||||
},
|
||||
thinking: {
|
||||
type: 'enabled',
|
||||
budget_tokens: 2000,
|
||||
},
|
||||
},
|
||||
},
|
||||
configOptions: {
|
||||
baseURL: 'http://host.docker.internal:4000/v1',
|
||||
defaultHeaders: {
|
||||
'anthropic-beta': 'prompt-caching-2024-07-31,context-1m-2025-08-07',
|
||||
},
|
||||
},
|
||||
tools: [],
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle Claude 3.7 model with thinking enabled', () => {
|
||||
const apiKey = 'sk-yyyy';
|
||||
const endpoint = 'Anthropic (via LiteLLM)';
|
||||
const options = {
|
||||
modelOptions: {
|
||||
model: 'claude-3.7-sonnet-20241022',
|
||||
user: 'user123',
|
||||
temperature: 0.7,
|
||||
thinking: true,
|
||||
thinkingBudget: 3000,
|
||||
},
|
||||
reverseProxyUrl: 'http://localhost:4000/v1',
|
||||
customParams: {
|
||||
defaultParamsEndpoint: 'anthropic',
|
||||
},
|
||||
endpoint: 'Anthropic (via LiteLLM)',
|
||||
endpointType: 'custom',
|
||||
};
|
||||
|
||||
const result = getOpenAIConfig(apiKey, options, endpoint);
|
||||
|
||||
expect(result).toEqual({
|
||||
llmConfig: {
|
||||
apiKey: 'sk-yyyy',
|
||||
model: 'claude-3.7-sonnet-20241022',
|
||||
stream: true,
|
||||
temperature: 0.7,
|
||||
maxTokens: 8192,
|
||||
modelKwargs: {
|
||||
metadata: {
|
||||
user_id: 'user123',
|
||||
},
|
||||
thinking: {
|
||||
type: 'enabled',
|
||||
budget_tokens: 3000,
|
||||
},
|
||||
},
|
||||
},
|
||||
configOptions: {
|
||||
baseURL: 'http://localhost:4000/v1',
|
||||
defaultHeaders: {
|
||||
'anthropic-beta':
|
||||
'token-efficient-tools-2025-02-19,output-128k-2025-02-19,prompt-caching-2024-07-31',
|
||||
},
|
||||
},
|
||||
tools: [],
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle Claude 3.7 model with thinking disabled (topP and topK included)', () => {
|
||||
const apiKey = 'sk-yyyy';
|
||||
const endpoint = 'Anthropic (via LiteLLM)';
|
||||
const options = {
|
||||
modelOptions: {
|
||||
model: 'claude-3.7-sonnet-20241022',
|
||||
user: 'user123',
|
||||
temperature: 0.7,
|
||||
topP: 0.9,
|
||||
topK: 50,
|
||||
thinking: false,
|
||||
},
|
||||
reverseProxyUrl: 'http://localhost:4000/v1',
|
||||
customParams: {
|
||||
defaultParamsEndpoint: 'anthropic',
|
||||
},
|
||||
endpoint: 'Anthropic (via LiteLLM)',
|
||||
endpointType: 'custom',
|
||||
};
|
||||
|
||||
const result = getOpenAIConfig(apiKey, options, endpoint);
|
||||
|
||||
expect(result).toEqual({
|
||||
llmConfig: {
|
||||
apiKey: 'sk-yyyy',
|
||||
model: 'claude-3.7-sonnet-20241022',
|
||||
stream: true,
|
||||
temperature: 0.7,
|
||||
topP: 0.9,
|
||||
maxTokens: 8192,
|
||||
modelKwargs: {
|
||||
metadata: {
|
||||
user_id: 'user123',
|
||||
},
|
||||
topK: 50,
|
||||
},
|
||||
},
|
||||
configOptions: {
|
||||
baseURL: 'http://localhost:4000/v1',
|
||||
defaultHeaders: {
|
||||
'anthropic-beta':
|
||||
'token-efficient-tools-2025-02-19,output-128k-2025-02-19,prompt-caching-2024-07-31',
|
||||
},
|
||||
},
|
||||
tools: [],
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle Claude 3.5 sonnet with special headers', () => {
|
||||
const apiKey = 'sk-zzzz';
|
||||
const endpoint = 'Anthropic (via LiteLLM)';
|
||||
const options = {
|
||||
modelOptions: {
|
||||
model: 'claude-3.5-sonnet-20240620',
|
||||
user: 'user456',
|
||||
maxOutputTokens: 4096,
|
||||
},
|
||||
reverseProxyUrl: 'https://api.anthropic.proxy.com/v1',
|
||||
customParams: {
|
||||
defaultParamsEndpoint: 'anthropic',
|
||||
},
|
||||
endpoint: 'Anthropic (via LiteLLM)',
|
||||
endpointType: 'custom',
|
||||
};
|
||||
|
||||
const result = getOpenAIConfig(apiKey, options, endpoint);
|
||||
|
||||
expect(result).toEqual({
|
||||
llmConfig: {
|
||||
apiKey: 'sk-zzzz',
|
||||
model: 'claude-3.5-sonnet-20240620',
|
||||
stream: true,
|
||||
maxTokens: 4096,
|
||||
modelKwargs: {
|
||||
metadata: {
|
||||
user_id: 'user456',
|
||||
},
|
||||
},
|
||||
},
|
||||
configOptions: {
|
||||
baseURL: 'https://api.anthropic.proxy.com/v1',
|
||||
defaultHeaders: {
|
||||
'anthropic-beta': 'max-tokens-3-5-sonnet-2024-07-15,prompt-caching-2024-07-31',
|
||||
},
|
||||
},
|
||||
tools: [],
|
||||
});
|
||||
});
|
||||
|
||||
it('should apply anthropic-beta headers based on model pattern', () => {
|
||||
const apiKey = 'sk-custom';
|
||||
const endpoint = 'Anthropic (via LiteLLM)';
|
||||
const options = {
|
||||
modelOptions: {
|
||||
model: 'claude-3-sonnet',
|
||||
},
|
||||
reverseProxyUrl: 'http://custom.proxy/v1',
|
||||
headers: {
|
||||
'Custom-Header': 'custom-value',
|
||||
Authorization: 'Bearer custom-token',
|
||||
},
|
||||
customParams: {
|
||||
defaultParamsEndpoint: 'anthropic',
|
||||
},
|
||||
endpoint: 'Anthropic (via LiteLLM)',
|
||||
endpointType: 'custom',
|
||||
};
|
||||
|
||||
const result = getOpenAIConfig(apiKey, options, endpoint);
|
||||
|
||||
expect(result).toEqual({
|
||||
llmConfig: {
|
||||
apiKey: 'sk-custom',
|
||||
model: 'claude-3-sonnet',
|
||||
stream: true,
|
||||
maxTokens: 8192,
|
||||
modelKwargs: {
|
||||
metadata: {
|
||||
user_id: undefined,
|
||||
},
|
||||
},
|
||||
},
|
||||
configOptions: {
|
||||
baseURL: 'http://custom.proxy/v1',
|
||||
defaultHeaders: {
|
||||
'Custom-Header': 'custom-value',
|
||||
Authorization: 'Bearer custom-token',
|
||||
'anthropic-beta': 'prompt-caching-2024-07-31',
|
||||
},
|
||||
},
|
||||
tools: [],
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle models that do not match Claude patterns', () => {
|
||||
const apiKey = 'sk-other';
|
||||
const endpoint = 'Anthropic (via LiteLLM)';
|
||||
const options = {
|
||||
modelOptions: {
|
||||
model: 'gpt-4-turbo',
|
||||
user: 'userGPT',
|
||||
temperature: 0.8,
|
||||
},
|
||||
reverseProxyUrl: 'http://litellm:4000/v1',
|
||||
customParams: {
|
||||
defaultParamsEndpoint: 'anthropic',
|
||||
},
|
||||
endpoint: 'Anthropic (via LiteLLM)',
|
||||
endpointType: 'custom',
|
||||
};
|
||||
|
||||
const result = getOpenAIConfig(apiKey, options, endpoint);
|
||||
|
||||
expect(result).toEqual({
|
||||
llmConfig: {
|
||||
apiKey: 'sk-other',
|
||||
model: 'gpt-4-turbo',
|
||||
stream: true,
|
||||
temperature: 0.8,
|
||||
maxTokens: 8192,
|
||||
modelKwargs: {
|
||||
metadata: {
|
||||
user_id: 'userGPT',
|
||||
},
|
||||
},
|
||||
},
|
||||
configOptions: {
|
||||
baseURL: 'http://litellm:4000/v1',
|
||||
},
|
||||
tools: [],
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle dropParams correctly in Anthropic path', () => {
|
||||
const apiKey = 'sk-drop';
|
||||
const endpoint = 'Anthropic (via LiteLLM)';
|
||||
const options = {
|
||||
modelOptions: {
|
||||
model: 'claude-3-opus-20240229',
|
||||
user: 'userDrop',
|
||||
temperature: 0.5,
|
||||
maxOutputTokens: 2048,
|
||||
topP: 0.9,
|
||||
topK: 40,
|
||||
},
|
||||
reverseProxyUrl: 'http://proxy.litellm/v1',
|
||||
dropParams: ['temperature', 'topK', 'metadata'],
|
||||
customParams: {
|
||||
defaultParamsEndpoint: 'anthropic',
|
||||
},
|
||||
endpoint: 'Anthropic (via LiteLLM)',
|
||||
endpointType: 'custom',
|
||||
};
|
||||
|
||||
const result = getOpenAIConfig(apiKey, options, endpoint);
|
||||
|
||||
expect(result).toEqual({
|
||||
llmConfig: {
|
||||
apiKey: 'sk-drop',
|
||||
model: 'claude-3-opus-20240229',
|
||||
stream: true,
|
||||
topP: 0.9,
|
||||
maxTokens: 2048,
|
||||
// temperature is dropped
|
||||
// modelKwargs.topK is dropped
|
||||
// modelKwargs.metadata is dropped completely
|
||||
},
|
||||
configOptions: {
|
||||
baseURL: 'http://proxy.litellm/v1',
|
||||
defaultHeaders: {
|
||||
'anthropic-beta': 'prompt-caching-2024-07-31',
|
||||
},
|
||||
},
|
||||
tools: [],
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle empty user string', () => {
|
||||
const apiKey = 'sk-edge';
|
||||
const endpoint = 'Anthropic (via LiteLLM)';
|
||||
const options = {
|
||||
modelOptions: {
|
||||
model: 'claude-2.1',
|
||||
user: '',
|
||||
temperature: 0,
|
||||
},
|
||||
reverseProxyUrl: 'http://litellm/v1',
|
||||
customParams: {
|
||||
defaultParamsEndpoint: 'anthropic',
|
||||
},
|
||||
endpoint: 'Anthropic (via LiteLLM)',
|
||||
endpointType: 'custom',
|
||||
};
|
||||
|
||||
const result = getOpenAIConfig(apiKey, options, endpoint);
|
||||
|
||||
expect(result).toEqual({
|
||||
llmConfig: {
|
||||
apiKey: 'sk-edge',
|
||||
model: 'claude-2.1',
|
||||
stream: true,
|
||||
temperature: 0,
|
||||
maxTokens: 8192,
|
||||
modelKwargs: {
|
||||
metadata: {
|
||||
user_id: '',
|
||||
},
|
||||
},
|
||||
},
|
||||
configOptions: {
|
||||
baseURL: 'http://litellm/v1',
|
||||
},
|
||||
tools: [],
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle web_search tool', () => {
|
||||
const apiKey = 'sk-search';
|
||||
const endpoint = 'Anthropic (via LiteLLM)';
|
||||
const options = {
|
||||
modelOptions: {
|
||||
model: 'claude-3-opus-20240229',
|
||||
user: 'searchUser',
|
||||
web_search: true,
|
||||
},
|
||||
reverseProxyUrl: 'http://litellm/v1',
|
||||
customParams: {
|
||||
defaultParamsEndpoint: 'anthropic',
|
||||
},
|
||||
endpoint: 'Anthropic (via LiteLLM)',
|
||||
endpointType: 'custom',
|
||||
};
|
||||
|
||||
const result = getOpenAIConfig(apiKey, options, endpoint);
|
||||
|
||||
expect(result).toEqual({
|
||||
llmConfig: {
|
||||
apiKey: 'sk-search',
|
||||
model: 'claude-3-opus-20240229',
|
||||
stream: true,
|
||||
maxTokens: 8192,
|
||||
modelKwargs: {
|
||||
metadata: {
|
||||
user_id: 'searchUser',
|
||||
},
|
||||
},
|
||||
},
|
||||
configOptions: {
|
||||
baseURL: 'http://litellm/v1',
|
||||
defaultHeaders: {
|
||||
'anthropic-beta': 'prompt-caching-2024-07-31',
|
||||
},
|
||||
},
|
||||
tools: [
|
||||
{
|
||||
type: 'web_search_20250305',
|
||||
name: 'web_search',
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it('should properly transform Anthropic config with invocationKwargs', () => {
|
||||
const apiKey = 'sk-test';
|
||||
const endpoint = 'Anthropic (via LiteLLM)';
|
||||
const options = {
|
||||
modelOptions: {
|
||||
model: 'claude-3.5-haiku-20241022',
|
||||
user: 'testUser',
|
||||
topP: 0.9,
|
||||
topK: 40,
|
||||
},
|
||||
reverseProxyUrl: 'http://litellm/v1',
|
||||
customParams: {
|
||||
defaultParamsEndpoint: 'anthropic',
|
||||
},
|
||||
endpoint: 'Anthropic (via LiteLLM)',
|
||||
endpointType: 'custom',
|
||||
};
|
||||
|
||||
const result = getOpenAIConfig(apiKey, options, endpoint);
|
||||
|
||||
expect(result).toEqual({
|
||||
llmConfig: {
|
||||
apiKey: 'sk-test',
|
||||
model: 'claude-3.5-haiku-20241022',
|
||||
stream: true,
|
||||
topP: 0.9,
|
||||
maxTokens: 8192,
|
||||
modelKwargs: {
|
||||
metadata: {
|
||||
user_id: 'testUser',
|
||||
},
|
||||
topK: 40,
|
||||
},
|
||||
},
|
||||
configOptions: {
|
||||
baseURL: 'http://litellm/v1',
|
||||
defaultHeaders: {
|
||||
'anthropic-beta': 'prompt-caching-2024-07-31',
|
||||
},
|
||||
},
|
||||
tools: [],
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle addParams with Anthropic defaults', () => {
|
||||
const apiKey = 'sk-add';
|
||||
const endpoint = 'Anthropic (via LiteLLM)';
|
||||
const options = {
|
||||
modelOptions: {
|
||||
model: 'claude-3-opus-20240229',
|
||||
user: 'addUser',
|
||||
temperature: 0.7,
|
||||
},
|
||||
reverseProxyUrl: 'http://litellm/v1',
|
||||
addParams: {
|
||||
customParam1: 'value1',
|
||||
customParam2: 42,
|
||||
frequencyPenalty: 0.5, // Known OpenAI param
|
||||
},
|
||||
customParams: {
|
||||
defaultParamsEndpoint: 'anthropic',
|
||||
},
|
||||
endpoint: 'Anthropic (via LiteLLM)',
|
||||
endpointType: 'custom',
|
||||
};
|
||||
|
||||
const result = getOpenAIConfig(apiKey, options, endpoint);
|
||||
|
||||
expect(result).toEqual({
|
||||
llmConfig: {
|
||||
apiKey: 'sk-add',
|
||||
model: 'claude-3-opus-20240229',
|
||||
stream: true,
|
||||
temperature: 0.7,
|
||||
frequencyPenalty: 0.5, // Known param added to main config
|
||||
maxTokens: 8192,
|
||||
modelKwargs: {
|
||||
metadata: {
|
||||
user_id: 'addUser',
|
||||
},
|
||||
customParam1: 'value1', // Unknown params added to modelKwargs
|
||||
customParam2: 42,
|
||||
},
|
||||
},
|
||||
configOptions: {
|
||||
baseURL: 'http://litellm/v1',
|
||||
defaultHeaders: {
|
||||
'anthropic-beta': 'prompt-caching-2024-07-31',
|
||||
},
|
||||
},
|
||||
tools: [],
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle both addParams and dropParams together', () => {
|
||||
const apiKey = 'sk-both';
|
||||
const endpoint = 'Anthropic (via LiteLLM)';
|
||||
const options = {
|
||||
modelOptions: {
|
||||
model: 'claude-3.5-sonnet-20240620',
|
||||
user: 'bothUser',
|
||||
temperature: 0.6,
|
||||
topP: 0.9,
|
||||
topK: 40,
|
||||
},
|
||||
reverseProxyUrl: 'http://litellm/v1',
|
||||
addParams: {
|
||||
customParam: 'customValue',
|
||||
maxRetries: 3, // Known OpenAI param
|
||||
},
|
||||
dropParams: ['temperature', 'topK'], // Drop one known and one unknown param
|
||||
customParams: {
|
||||
defaultParamsEndpoint: 'anthropic',
|
||||
},
|
||||
endpoint: 'Anthropic (via LiteLLM)',
|
||||
endpointType: 'custom',
|
||||
};
|
||||
|
||||
const result = getOpenAIConfig(apiKey, options, endpoint);
|
||||
|
||||
expect(result).toEqual({
|
||||
llmConfig: {
|
||||
apiKey: 'sk-both',
|
||||
model: 'claude-3.5-sonnet-20240620',
|
||||
stream: true,
|
||||
topP: 0.9,
|
||||
maxRetries: 3,
|
||||
maxTokens: 8192,
|
||||
modelKwargs: {
|
||||
metadata: {
|
||||
user_id: 'bothUser',
|
||||
},
|
||||
customParam: 'customValue',
|
||||
// topK is dropped
|
||||
},
|
||||
},
|
||||
configOptions: {
|
||||
baseURL: 'http://litellm/v1',
|
||||
defaultHeaders: {
|
||||
'anthropic-beta': 'max-tokens-3-5-sonnet-2024-07-15,prompt-caching-2024-07-31',
|
||||
},
|
||||
},
|
||||
tools: [],
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
431
packages/api/src/endpoints/openai/config.backward-compat.spec.ts
Normal file
431
packages/api/src/endpoints/openai/config.backward-compat.spec.ts
Normal file
|
|
@ -0,0 +1,431 @@
|
|||
import {
|
||||
Verbosity,
|
||||
EModelEndpoint,
|
||||
ReasoningEffort,
|
||||
ReasoningSummary,
|
||||
} from 'librechat-data-provider';
|
||||
import { getOpenAIConfig } from './config';
|
||||
|
||||
describe('getOpenAIConfig - Backward Compatibility', () => {
|
||||
describe('OpenAI endpoint', () => {
|
||||
it('should handle GPT-5 model with reasoning and web search', () => {
|
||||
const apiKey = 'sk-proj-somekey';
|
||||
const endpoint = undefined;
|
||||
const options = {
|
||||
modelOptions: {
|
||||
model: 'gpt-5-nano',
|
||||
verbosity: Verbosity.high,
|
||||
reasoning_effort: ReasoningEffort.high,
|
||||
reasoning_summary: ReasoningSummary.detailed,
|
||||
useResponsesApi: true,
|
||||
web_search: true,
|
||||
user: 'some-user',
|
||||
},
|
||||
proxy: '',
|
||||
reverseProxyUrl: null,
|
||||
endpoint: EModelEndpoint.openAI,
|
||||
};
|
||||
|
||||
const result = getOpenAIConfig(apiKey, options, endpoint);
|
||||
|
||||
expect(result).toEqual({
|
||||
llmConfig: {
|
||||
streaming: true,
|
||||
model: 'gpt-5-nano',
|
||||
useResponsesApi: true,
|
||||
user: 'some-user',
|
||||
apiKey: 'sk-proj-somekey',
|
||||
reasoning: {
|
||||
effort: ReasoningEffort.high,
|
||||
summary: ReasoningSummary.detailed,
|
||||
},
|
||||
modelKwargs: {
|
||||
text: {
|
||||
verbosity: Verbosity.high,
|
||||
},
|
||||
},
|
||||
},
|
||||
configOptions: {},
|
||||
tools: [
|
||||
{
|
||||
type: 'web_search_preview',
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('OpenRouter endpoint', () => {
|
||||
it('should handle OpenRouter configuration with dropParams and custom headers', () => {
|
||||
const apiKey = 'sk-xxxx';
|
||||
const endpoint = 'OpenRouter';
|
||||
const options = {
|
||||
modelOptions: {
|
||||
model: 'qwen/qwen3-max',
|
||||
user: 'some-user',
|
||||
},
|
||||
reverseProxyUrl: 'https://gateway.ai.cloudflare.com/v1/account-id/gateway-id/openrouter',
|
||||
headers: {
|
||||
'x-librechat-thread-id': '{{LIBRECHAT_BODY_CONVERSATIONID}}',
|
||||
'x-test-key': '{{TESTING_USER_VAR}}',
|
||||
},
|
||||
proxy: '',
|
||||
dropParams: ['user'],
|
||||
};
|
||||
|
||||
const result = getOpenAIConfig(apiKey, options, endpoint);
|
||||
|
||||
expect(result).toEqual({
|
||||
llmConfig: {
|
||||
streaming: true,
|
||||
model: 'qwen/qwen3-max',
|
||||
include_reasoning: true,
|
||||
apiKey: 'sk-xxxx',
|
||||
},
|
||||
configOptions: {
|
||||
baseURL: 'https://gateway.ai.cloudflare.com/v1/account-id/gateway-id/openrouter',
|
||||
defaultHeaders: {
|
||||
'HTTP-Referer': 'https://librechat.ai',
|
||||
'X-Title': 'LibreChat',
|
||||
'x-librechat-thread-id': '{{LIBRECHAT_BODY_CONVERSATIONID}}',
|
||||
'x-test-key': '{{TESTING_USER_VAR}}',
|
||||
},
|
||||
},
|
||||
tools: [],
|
||||
provider: 'openrouter',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('Azure OpenAI endpoint', () => {
|
||||
it('should handle basic Azure OpenAI configuration', () => {
|
||||
const apiKey = 'some_key';
|
||||
const endpoint = undefined;
|
||||
const options = {
|
||||
modelOptions: {
|
||||
model: 'gpt-4o',
|
||||
user: 'some_user_id',
|
||||
},
|
||||
reverseProxyUrl: null,
|
||||
endpoint: 'azureOpenAI',
|
||||
azure: {
|
||||
azureOpenAIApiKey: 'some_azure_key',
|
||||
azureOpenAIApiInstanceName: 'some_instance_name',
|
||||
azureOpenAIApiDeploymentName: 'gpt-4o',
|
||||
azureOpenAIApiVersion: '2024-02-15-preview',
|
||||
},
|
||||
};
|
||||
|
||||
const result = getOpenAIConfig(apiKey, options, endpoint);
|
||||
|
||||
expect(result).toEqual({
|
||||
llmConfig: {
|
||||
streaming: true,
|
||||
model: 'gpt-4o',
|
||||
user: 'some_user_id',
|
||||
azureOpenAIApiKey: 'some_azure_key',
|
||||
azureOpenAIApiInstanceName: 'some_instance_name',
|
||||
azureOpenAIApiDeploymentName: 'gpt-4o',
|
||||
azureOpenAIApiVersion: '2024-02-15-preview',
|
||||
},
|
||||
configOptions: {},
|
||||
tools: [],
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle Azure OpenAI with Responses API and reasoning', () => {
|
||||
const apiKey = 'some_azure_key';
|
||||
const endpoint = undefined;
|
||||
const options = {
|
||||
modelOptions: {
|
||||
model: 'gpt-5',
|
||||
reasoning_effort: ReasoningEffort.high,
|
||||
reasoning_summary: ReasoningSummary.detailed,
|
||||
verbosity: Verbosity.high,
|
||||
useResponsesApi: true,
|
||||
user: 'some_user_id',
|
||||
},
|
||||
endpoint: 'azureOpenAI',
|
||||
azure: {
|
||||
azureOpenAIApiKey: 'some_azure_key',
|
||||
azureOpenAIApiInstanceName: 'some_instance_name',
|
||||
azureOpenAIApiDeploymentName: 'gpt-5',
|
||||
azureOpenAIApiVersion: '2024-12-01-preview',
|
||||
},
|
||||
};
|
||||
|
||||
const result = getOpenAIConfig(apiKey, options, endpoint);
|
||||
|
||||
expect(result).toEqual({
|
||||
llmConfig: {
|
||||
streaming: true,
|
||||
model: 'gpt-5',
|
||||
useResponsesApi: true,
|
||||
user: 'some_user_id',
|
||||
apiKey: 'some_azure_key',
|
||||
reasoning: {
|
||||
effort: ReasoningEffort.high,
|
||||
summary: ReasoningSummary.detailed,
|
||||
},
|
||||
modelKwargs: {
|
||||
text: {
|
||||
verbosity: Verbosity.high,
|
||||
},
|
||||
},
|
||||
},
|
||||
configOptions: {
|
||||
baseURL: 'https://some_instance_name.openai.azure.com/openai/v1',
|
||||
defaultHeaders: {
|
||||
'api-key': 'some_azure_key',
|
||||
},
|
||||
defaultQuery: {
|
||||
'api-version': 'preview',
|
||||
},
|
||||
},
|
||||
tools: [],
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle Azure serverless configuration with dropParams', () => {
|
||||
const apiKey = 'some_azure_key';
|
||||
const endpoint = undefined;
|
||||
const options = {
|
||||
modelOptions: {
|
||||
model: 'jais-30b-chat',
|
||||
user: 'some_user_id',
|
||||
},
|
||||
reverseProxyUrl: 'https://some_endpoint_name.services.ai.azure.com/models',
|
||||
endpoint: 'azureOpenAI',
|
||||
headers: {
|
||||
'api-key': 'some_azure_key',
|
||||
},
|
||||
dropParams: ['stream_options', 'user'],
|
||||
azure: false as const,
|
||||
defaultQuery: {
|
||||
'api-version': '2024-05-01-preview',
|
||||
},
|
||||
};
|
||||
|
||||
const result = getOpenAIConfig(apiKey, options, endpoint);
|
||||
|
||||
expect(result).toEqual({
|
||||
llmConfig: {
|
||||
streaming: true,
|
||||
model: 'jais-30b-chat',
|
||||
apiKey: 'some_azure_key',
|
||||
},
|
||||
configOptions: {
|
||||
baseURL: 'https://some_endpoint_name.services.ai.azure.com/models',
|
||||
defaultHeaders: {
|
||||
'api-key': 'some_azure_key',
|
||||
},
|
||||
defaultQuery: {
|
||||
'api-version': '2024-05-01-preview',
|
||||
},
|
||||
},
|
||||
tools: [],
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle Azure serverless with user-provided key configuration', () => {
|
||||
const apiKey = 'some_azure_key';
|
||||
const endpoint = undefined;
|
||||
const options = {
|
||||
modelOptions: {
|
||||
model: 'grok-3',
|
||||
user: 'some_user_id',
|
||||
},
|
||||
reverseProxyUrl: 'https://some_endpoint_name.services.ai.azure.com/models',
|
||||
endpoint: 'azureOpenAI',
|
||||
headers: {
|
||||
'api-key': 'some_azure_key',
|
||||
},
|
||||
dropParams: ['stream_options', 'user'],
|
||||
azure: false as const,
|
||||
defaultQuery: {
|
||||
'api-version': '2024-05-01-preview',
|
||||
},
|
||||
};
|
||||
|
||||
const result = getOpenAIConfig(apiKey, options, endpoint);
|
||||
|
||||
expect(result).toEqual({
|
||||
llmConfig: {
|
||||
streaming: true,
|
||||
model: 'grok-3',
|
||||
apiKey: 'some_azure_key',
|
||||
},
|
||||
configOptions: {
|
||||
baseURL: 'https://some_endpoint_name.services.ai.azure.com/models',
|
||||
defaultHeaders: {
|
||||
'api-key': 'some_azure_key',
|
||||
},
|
||||
defaultQuery: {
|
||||
'api-version': '2024-05-01-preview',
|
||||
},
|
||||
},
|
||||
tools: [],
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle Azure serverless with Mistral model configuration', () => {
|
||||
const apiKey = 'some_azure_key';
|
||||
const endpoint = undefined;
|
||||
const options = {
|
||||
modelOptions: {
|
||||
model: 'Mistral-Large-2411',
|
||||
user: 'some_user_id',
|
||||
},
|
||||
reverseProxyUrl: 'https://some_endpoint_name.services.ai.azure.com/models',
|
||||
endpoint: 'azureOpenAI',
|
||||
headers: {
|
||||
'api-key': 'some_azure_key',
|
||||
},
|
||||
dropParams: ['stream_options', 'user'],
|
||||
azure: false as const,
|
||||
defaultQuery: {
|
||||
'api-version': '2024-05-01-preview',
|
||||
},
|
||||
};
|
||||
|
||||
const result = getOpenAIConfig(apiKey, options, endpoint);
|
||||
|
||||
expect(result).toEqual({
|
||||
llmConfig: {
|
||||
streaming: true,
|
||||
model: 'Mistral-Large-2411',
|
||||
apiKey: 'some_azure_key',
|
||||
},
|
||||
configOptions: {
|
||||
baseURL: 'https://some_endpoint_name.services.ai.azure.com/models',
|
||||
defaultHeaders: {
|
||||
'api-key': 'some_azure_key',
|
||||
},
|
||||
defaultQuery: {
|
||||
'api-version': '2024-05-01-preview',
|
||||
},
|
||||
},
|
||||
tools: [],
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle Azure serverless with DeepSeek model without dropParams', () => {
|
||||
const apiKey = 'some_azure_key';
|
||||
const endpoint = undefined;
|
||||
const options = {
|
||||
modelOptions: {
|
||||
model: 'DeepSeek-R1',
|
||||
user: 'some_user_id',
|
||||
},
|
||||
reverseProxyUrl: 'https://some_endpoint_name.models.ai.azure.com/v1/',
|
||||
endpoint: 'azureOpenAI',
|
||||
headers: {
|
||||
'api-key': 'some_azure_key',
|
||||
},
|
||||
azure: false as const,
|
||||
defaultQuery: {
|
||||
'api-version': '2024-08-01-preview',
|
||||
},
|
||||
};
|
||||
|
||||
const result = getOpenAIConfig(apiKey, options, endpoint);
|
||||
|
||||
expect(result).toEqual({
|
||||
llmConfig: {
|
||||
streaming: true,
|
||||
model: 'DeepSeek-R1',
|
||||
user: 'some_user_id',
|
||||
apiKey: 'some_azure_key',
|
||||
},
|
||||
configOptions: {
|
||||
baseURL: 'https://some_endpoint_name.models.ai.azure.com/v1/',
|
||||
defaultHeaders: {
|
||||
'api-key': 'some_azure_key',
|
||||
},
|
||||
defaultQuery: {
|
||||
'api-version': '2024-08-01-preview',
|
||||
},
|
||||
},
|
||||
tools: [],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('Custom endpoints', () => {
|
||||
it('should handle Groq custom endpoint configuration', () => {
|
||||
const apiKey = 'gsk_somekey';
|
||||
const endpoint = 'groq';
|
||||
const options = {
|
||||
modelOptions: {
|
||||
model: 'qwen/qwen3-32b',
|
||||
user: 'some-user',
|
||||
},
|
||||
reverseProxyUrl: 'https://api.groq.com/openai/v1/',
|
||||
proxy: '',
|
||||
headers: {},
|
||||
endpoint: 'groq',
|
||||
endpointType: 'custom',
|
||||
};
|
||||
|
||||
const result = getOpenAIConfig(apiKey, options, endpoint);
|
||||
|
||||
expect(result).toEqual({
|
||||
llmConfig: {
|
||||
streaming: true,
|
||||
model: 'qwen/qwen3-32b',
|
||||
user: 'some-user',
|
||||
apiKey: 'gsk_somekey',
|
||||
},
|
||||
configOptions: {
|
||||
baseURL: 'https://api.groq.com/openai/v1/',
|
||||
defaultHeaders: {},
|
||||
},
|
||||
tools: [],
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle Cloudflare Workers AI with custom headers and addParams', () => {
|
||||
const apiKey = 'someKey';
|
||||
const endpoint = 'Cloudflare Workers AI';
|
||||
const options = {
|
||||
modelOptions: {
|
||||
model: '@cf/deepseek-ai/deepseek-r1-distill-qwen-32b',
|
||||
user: 'some-user',
|
||||
},
|
||||
reverseProxyUrl:
|
||||
'https://gateway.ai.cloudflare.com/v1/${CF_ACCOUNT_ID}/${CF_GATEWAY_ID}/workers-ai/v1',
|
||||
proxy: '',
|
||||
headers: {
|
||||
'x-librechat-thread-id': '{{LIBRECHAT_BODY_CONVERSATIONID}}',
|
||||
'x-test-key': '{{TESTING_USER_VAR}}',
|
||||
},
|
||||
addParams: {
|
||||
disableStreaming: true,
|
||||
},
|
||||
endpoint: 'Cloudflare Workers AI',
|
||||
endpointType: 'custom',
|
||||
};
|
||||
|
||||
const result = getOpenAIConfig(apiKey, options, endpoint);
|
||||
|
||||
expect(result).toEqual({
|
||||
llmConfig: {
|
||||
streaming: true,
|
||||
model: '@cf/deepseek-ai/deepseek-r1-distill-qwen-32b',
|
||||
user: 'some-user',
|
||||
disableStreaming: true,
|
||||
apiKey: 'someKey',
|
||||
},
|
||||
configOptions: {
|
||||
baseURL:
|
||||
'https://gateway.ai.cloudflare.com/v1/${CF_ACCOUNT_ID}/${CF_GATEWAY_ID}/workers-ai/v1',
|
||||
defaultHeaders: {
|
||||
'x-librechat-thread-id': '{{LIBRECHAT_BODY_CONVERSATIONID}}',
|
||||
'x-test-key': '{{TESTING_USER_VAR}}',
|
||||
},
|
||||
},
|
||||
tools: [],
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -1,7 +1,8 @@
|
|||
import { Verbosity, ReasoningEffort, ReasoningSummary } from 'librechat-data-provider';
|
||||
import type { RequestInit } from 'undici';
|
||||
import type { OpenAIParameters, AzureOptions } from '~/types';
|
||||
import { getOpenAIConfig, knownOpenAIParams } from './llm';
|
||||
import { getOpenAIConfig } from './config';
|
||||
import { knownOpenAIParams } from './llm';
|
||||
|
||||
describe('getOpenAIConfig', () => {
|
||||
const mockApiKey = 'test-api-key';
|
||||
150
packages/api/src/endpoints/openai/config.ts
Normal file
150
packages/api/src/endpoints/openai/config.ts
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
import { ProxyAgent } from 'undici';
|
||||
import { Providers } from '@librechat/agents';
|
||||
import { KnownEndpoints, EModelEndpoint } from 'librechat-data-provider';
|
||||
import type * as t from '~/types';
|
||||
import { getLLMConfig as getAnthropicLLMConfig } from '~/endpoints/anthropic/llm';
|
||||
import { transformToOpenAIConfig } from './transform';
|
||||
import { constructAzureURL } from '~/utils/azure';
|
||||
import { createFetch } from '~/utils/generators';
|
||||
import { getOpenAILLMConfig } from './llm';
|
||||
|
||||
type Fetch = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
|
||||
|
||||
/**
|
||||
* Generates configuration options for creating a language model (LLM) instance.
|
||||
* @param apiKey - The API key for authentication.
|
||||
* @param options - Additional options for configuring the LLM.
|
||||
* @param endpoint - The endpoint name
|
||||
* @returns Configuration options for creating an LLM instance.
|
||||
*/
|
||||
export function getOpenAIConfig(
|
||||
apiKey: string,
|
||||
options: t.OpenAIConfigOptions = {},
|
||||
endpoint?: string | null,
|
||||
): t.OpenAIConfigResult {
|
||||
const {
|
||||
proxy,
|
||||
addParams,
|
||||
dropParams,
|
||||
defaultQuery,
|
||||
directEndpoint,
|
||||
streaming = true,
|
||||
modelOptions = {},
|
||||
reverseProxyUrl: baseURL,
|
||||
} = options;
|
||||
|
||||
let llmConfig: t.OAIClientOptions;
|
||||
let tools: t.LLMConfigResult['tools'];
|
||||
const isAnthropic = options.customParams?.defaultParamsEndpoint === EModelEndpoint.anthropic;
|
||||
|
||||
const useOpenRouter =
|
||||
!isAnthropic &&
|
||||
((baseURL && baseURL.includes(KnownEndpoints.openrouter)) ||
|
||||
(endpoint != null && endpoint.toLowerCase().includes(KnownEndpoints.openrouter)));
|
||||
|
||||
let azure = options.azure;
|
||||
let headers = options.headers;
|
||||
if (isAnthropic) {
|
||||
const anthropicResult = getAnthropicLLMConfig(apiKey, {
|
||||
modelOptions,
|
||||
proxy: options.proxy,
|
||||
});
|
||||
const transformed = transformToOpenAIConfig({
|
||||
addParams,
|
||||
dropParams,
|
||||
llmConfig: anthropicResult.llmConfig,
|
||||
fromEndpoint: EModelEndpoint.anthropic,
|
||||
});
|
||||
llmConfig = transformed.llmConfig;
|
||||
tools = anthropicResult.tools;
|
||||
if (transformed.configOptions?.defaultHeaders) {
|
||||
headers = Object.assign(headers ?? {}, transformed.configOptions?.defaultHeaders);
|
||||
}
|
||||
} else {
|
||||
const openaiResult = getOpenAILLMConfig({
|
||||
azure,
|
||||
apiKey,
|
||||
baseURL,
|
||||
streaming,
|
||||
addParams,
|
||||
dropParams,
|
||||
modelOptions,
|
||||
useOpenRouter,
|
||||
});
|
||||
llmConfig = openaiResult.llmConfig;
|
||||
azure = openaiResult.azure;
|
||||
tools = openaiResult.tools;
|
||||
}
|
||||
|
||||
const configOptions: t.OpenAIConfiguration = {};
|
||||
if (baseURL) {
|
||||
configOptions.baseURL = baseURL;
|
||||
}
|
||||
if (useOpenRouter) {
|
||||
configOptions.defaultHeaders = Object.assign(
|
||||
{
|
||||
'HTTP-Referer': 'https://librechat.ai',
|
||||
'X-Title': 'LibreChat',
|
||||
},
|
||||
headers,
|
||||
);
|
||||
} else if (headers) {
|
||||
configOptions.defaultHeaders = headers;
|
||||
}
|
||||
|
||||
if (defaultQuery) {
|
||||
configOptions.defaultQuery = defaultQuery;
|
||||
}
|
||||
|
||||
if (proxy) {
|
||||
const proxyAgent = new ProxyAgent(proxy);
|
||||
configOptions.fetchOptions = {
|
||||
dispatcher: proxyAgent,
|
||||
};
|
||||
}
|
||||
|
||||
if (azure && !isAnthropic) {
|
||||
const constructAzureResponsesApi = () => {
|
||||
if (!llmConfig.useResponsesApi || !azure) {
|
||||
return;
|
||||
}
|
||||
|
||||
configOptions.baseURL = constructAzureURL({
|
||||
baseURL: configOptions.baseURL || 'https://${INSTANCE_NAME}.openai.azure.com/openai/v1',
|
||||
azureOptions: azure,
|
||||
});
|
||||
|
||||
configOptions.defaultHeaders = {
|
||||
...configOptions.defaultHeaders,
|
||||
'api-key': apiKey,
|
||||
};
|
||||
configOptions.defaultQuery = {
|
||||
...configOptions.defaultQuery,
|
||||
'api-version': configOptions.defaultQuery?.['api-version'] ?? 'preview',
|
||||
};
|
||||
};
|
||||
|
||||
constructAzureResponsesApi();
|
||||
}
|
||||
|
||||
if (process.env.OPENAI_ORGANIZATION && !isAnthropic) {
|
||||
configOptions.organization = process.env.OPENAI_ORGANIZATION;
|
||||
}
|
||||
|
||||
if (directEndpoint === true && configOptions?.baseURL != null) {
|
||||
configOptions.fetch = createFetch({
|
||||
directEndpoint: directEndpoint,
|
||||
reverseProxyUrl: configOptions?.baseURL,
|
||||
}) as unknown as Fetch;
|
||||
}
|
||||
|
||||
const result: t.OpenAIConfigResult = {
|
||||
llmConfig,
|
||||
configOptions,
|
||||
tools,
|
||||
};
|
||||
if (useOpenRouter) {
|
||||
result.provider = Providers.OPENROUTER;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
|
@ -1,2 +1,3 @@
|
|||
export * from './llm';
|
||||
export * from './config';
|
||||
export * from './initialize';
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import { createHandleLLMNewToken } from '~/utils/generators';
|
|||
import { getAzureCredentials } from '~/utils/azure';
|
||||
import { isUserProvided } from '~/utils/common';
|
||||
import { resolveHeaders } from '~/utils/env';
|
||||
import { getOpenAIConfig } from './llm';
|
||||
import { getOpenAIConfig } from './config';
|
||||
|
||||
/**
|
||||
* Initializes OpenAI options for agent usage. This function always returns configuration
|
||||
|
|
@ -115,7 +115,7 @@ export const initializeOpenAI = async ({
|
|||
} else if (isAzureOpenAI) {
|
||||
clientOptions.azure =
|
||||
userProvidesKey && userValues?.apiKey ? JSON.parse(userValues.apiKey) : getAzureCredentials();
|
||||
apiKey = clientOptions.azure?.azureOpenAIApiKey;
|
||||
apiKey = clientOptions.azure ? clientOptions.azure.azureOpenAIApiKey : undefined;
|
||||
}
|
||||
|
||||
if (userProvidesKey && !apiKey) {
|
||||
|
|
|
|||
|
|
@ -1,16 +1,11 @@
|
|||
import { ProxyAgent } from 'undici';
|
||||
import { Providers } from '@librechat/agents';
|
||||
import { KnownEndpoints, removeNullishValues } from 'librechat-data-provider';
|
||||
import { removeNullishValues } from 'librechat-data-provider';
|
||||
import type { BindToolsInput } from '@langchain/core/language_models/chat_models';
|
||||
import type { AzureOpenAIInput } from '@langchain/openai';
|
||||
import type { OpenAI } from 'openai';
|
||||
import type * as t from '~/types';
|
||||
import { sanitizeModelName, constructAzureURL } from '~/utils/azure';
|
||||
import { createFetch } from '~/utils/generators';
|
||||
import { isEnabled } from '~/utils/common';
|
||||
|
||||
type Fetch = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
|
||||
|
||||
export const knownOpenAIParams = new Set([
|
||||
// Constructor/Instance Parameters
|
||||
'model',
|
||||
|
|
@ -80,47 +75,44 @@ function hasReasoningParams({
|
|||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates configuration options for creating a language model (LLM) instance.
|
||||
* @param apiKey - The API key for authentication.
|
||||
* @param options - Additional options for configuring the LLM.
|
||||
* @param endpoint - The endpoint name
|
||||
* @returns Configuration options for creating an LLM instance.
|
||||
*/
|
||||
export function getOpenAIConfig(
|
||||
apiKey: string,
|
||||
options: t.OpenAIConfigOptions = {},
|
||||
endpoint?: string | null,
|
||||
): t.LLMConfigResult {
|
||||
const {
|
||||
modelOptions: _modelOptions = {},
|
||||
reverseProxyUrl,
|
||||
directEndpoint,
|
||||
defaultQuery,
|
||||
headers,
|
||||
proxy,
|
||||
azure,
|
||||
streaming = true,
|
||||
addParams,
|
||||
dropParams,
|
||||
} = options;
|
||||
export function getOpenAILLMConfig({
|
||||
azure,
|
||||
apiKey,
|
||||
baseURL,
|
||||
streaming,
|
||||
addParams,
|
||||
dropParams,
|
||||
useOpenRouter,
|
||||
modelOptions: _modelOptions,
|
||||
}: {
|
||||
apiKey: string;
|
||||
streaming: boolean;
|
||||
baseURL?: string | null;
|
||||
modelOptions: Partial<t.OpenAIParameters>;
|
||||
addParams?: Record<string, unknown>;
|
||||
dropParams?: string[];
|
||||
useOpenRouter?: boolean;
|
||||
azure?: false | t.AzureOptions;
|
||||
}): Pick<t.LLMConfigResult, 'llmConfig' | 'tools'> & {
|
||||
azure?: t.AzureOptions;
|
||||
} {
|
||||
const {
|
||||
reasoning_effort,
|
||||
reasoning_summary,
|
||||
verbosity,
|
||||
web_search,
|
||||
frequency_penalty,
|
||||
presence_penalty,
|
||||
...modelOptions
|
||||
} = _modelOptions;
|
||||
const llmConfig: Partial<t.ClientOptions> &
|
||||
Partial<t.OpenAIParameters> &
|
||||
Partial<AzureOpenAIInput> = Object.assign(
|
||||
|
||||
const llmConfig = Object.assign(
|
||||
{
|
||||
streaming,
|
||||
model: modelOptions.model ?? '',
|
||||
},
|
||||
modelOptions,
|
||||
);
|
||||
) as Partial<t.OAIClientOptions> & Partial<t.OpenAIParameters> & Partial<AzureOpenAIInput>;
|
||||
|
||||
if (frequency_penalty != null) {
|
||||
llmConfig.frequencyPenalty = frequency_penalty;
|
||||
|
|
@ -148,104 +140,8 @@ export function getOpenAIConfig(
|
|||
}
|
||||
}
|
||||
|
||||
let useOpenRouter = false;
|
||||
const configOptions: t.OpenAIConfiguration = {};
|
||||
|
||||
if (
|
||||
(reverseProxyUrl && reverseProxyUrl.includes(KnownEndpoints.openrouter)) ||
|
||||
(endpoint && endpoint.toLowerCase().includes(KnownEndpoints.openrouter))
|
||||
) {
|
||||
useOpenRouter = true;
|
||||
if (useOpenRouter) {
|
||||
llmConfig.include_reasoning = true;
|
||||
configOptions.baseURL = reverseProxyUrl;
|
||||
configOptions.defaultHeaders = Object.assign(
|
||||
{
|
||||
'HTTP-Referer': 'https://librechat.ai',
|
||||
'X-Title': 'LibreChat',
|
||||
},
|
||||
headers,
|
||||
);
|
||||
} else if (reverseProxyUrl) {
|
||||
configOptions.baseURL = reverseProxyUrl;
|
||||
if (headers) {
|
||||
configOptions.defaultHeaders = headers;
|
||||
}
|
||||
}
|
||||
|
||||
if (defaultQuery) {
|
||||
configOptions.defaultQuery = defaultQuery;
|
||||
}
|
||||
|
||||
if (proxy) {
|
||||
const proxyAgent = new ProxyAgent(proxy);
|
||||
configOptions.fetchOptions = {
|
||||
dispatcher: proxyAgent,
|
||||
};
|
||||
}
|
||||
|
||||
if (azure) {
|
||||
const useModelName = isEnabled(process.env.AZURE_USE_MODEL_AS_DEPLOYMENT_NAME);
|
||||
const updatedAzure = { ...azure };
|
||||
updatedAzure.azureOpenAIApiDeploymentName = useModelName
|
||||
? sanitizeModelName(llmConfig.model || '')
|
||||
: azure.azureOpenAIApiDeploymentName;
|
||||
|
||||
if (process.env.AZURE_OPENAI_DEFAULT_MODEL) {
|
||||
llmConfig.model = process.env.AZURE_OPENAI_DEFAULT_MODEL;
|
||||
}
|
||||
|
||||
const constructBaseURL = () => {
|
||||
if (!configOptions.baseURL) {
|
||||
return;
|
||||
}
|
||||
const azureURL = constructAzureURL({
|
||||
baseURL: configOptions.baseURL,
|
||||
azureOptions: updatedAzure,
|
||||
});
|
||||
updatedAzure.azureOpenAIBasePath = azureURL.split(
|
||||
`/${updatedAzure.azureOpenAIApiDeploymentName}`,
|
||||
)[0];
|
||||
};
|
||||
|
||||
constructBaseURL();
|
||||
Object.assign(llmConfig, updatedAzure);
|
||||
|
||||
const constructAzureResponsesApi = () => {
|
||||
if (!llmConfig.useResponsesApi) {
|
||||
return;
|
||||
}
|
||||
|
||||
configOptions.baseURL = constructAzureURL({
|
||||
baseURL: configOptions.baseURL || 'https://${INSTANCE_NAME}.openai.azure.com/openai/v1',
|
||||
azureOptions: llmConfig,
|
||||
});
|
||||
|
||||
delete llmConfig.azureOpenAIApiDeploymentName;
|
||||
delete llmConfig.azureOpenAIApiInstanceName;
|
||||
delete llmConfig.azureOpenAIApiVersion;
|
||||
delete llmConfig.azureOpenAIBasePath;
|
||||
delete llmConfig.azureOpenAIApiKey;
|
||||
llmConfig.apiKey = apiKey;
|
||||
|
||||
configOptions.defaultHeaders = {
|
||||
...configOptions.defaultHeaders,
|
||||
'api-key': apiKey,
|
||||
};
|
||||
configOptions.defaultQuery = {
|
||||
...configOptions.defaultQuery,
|
||||
'api-version': configOptions.defaultQuery?.['api-version'] ?? 'preview',
|
||||
};
|
||||
};
|
||||
|
||||
constructAzureResponsesApi();
|
||||
|
||||
llmConfig.model = updatedAzure.azureOpenAIApiDeploymentName;
|
||||
} else {
|
||||
llmConfig.apiKey = apiKey;
|
||||
}
|
||||
|
||||
if (process.env.OPENAI_ORGANIZATION && azure) {
|
||||
configOptions.organization = process.env.OPENAI_ORGANIZATION;
|
||||
}
|
||||
|
||||
if (
|
||||
|
|
@ -270,7 +166,7 @@ export function getOpenAIConfig(
|
|||
|
||||
const tools: BindToolsInput[] = [];
|
||||
|
||||
if (modelOptions.web_search) {
|
||||
if (web_search) {
|
||||
llmConfig.useResponsesApi = true;
|
||||
tools.push({ type: 'web_search_preview' });
|
||||
}
|
||||
|
|
@ -278,7 +174,7 @@ export function getOpenAIConfig(
|
|||
/**
|
||||
* Note: OpenAI Web Search models do not support any known parameters besides `max_tokens`
|
||||
*/
|
||||
if (modelOptions.model && /gpt-4o.*search/.test(modelOptions.model)) {
|
||||
if (modelOptions.model && /gpt-4o.*search/.test(modelOptions.model as string)) {
|
||||
const searchExcludeParams = [
|
||||
'frequency_penalty',
|
||||
'presence_penalty',
|
||||
|
|
@ -301,13 +197,13 @@ export function getOpenAIConfig(
|
|||
|
||||
combinedDropParams.forEach((param) => {
|
||||
if (param in llmConfig) {
|
||||
delete llmConfig[param as keyof t.ClientOptions];
|
||||
delete llmConfig[param as keyof t.OAIClientOptions];
|
||||
}
|
||||
});
|
||||
} else if (dropParams && Array.isArray(dropParams)) {
|
||||
dropParams.forEach((param) => {
|
||||
if (param in llmConfig) {
|
||||
delete llmConfig[param as keyof t.ClientOptions];
|
||||
delete llmConfig[param as keyof t.OAIClientOptions];
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
@ -329,20 +225,52 @@ export function getOpenAIConfig(
|
|||
llmConfig.modelKwargs = modelKwargs;
|
||||
}
|
||||
|
||||
if (directEndpoint === true && configOptions?.baseURL != null) {
|
||||
configOptions.fetch = createFetch({
|
||||
directEndpoint: directEndpoint,
|
||||
reverseProxyUrl: configOptions?.baseURL,
|
||||
}) as unknown as Fetch;
|
||||
if (!azure) {
|
||||
llmConfig.apiKey = apiKey;
|
||||
return { llmConfig, tools };
|
||||
}
|
||||
|
||||
const result: t.LLMConfigResult = {
|
||||
llmConfig,
|
||||
configOptions,
|
||||
tools,
|
||||
};
|
||||
if (useOpenRouter) {
|
||||
result.provider = Providers.OPENROUTER;
|
||||
const useModelName = isEnabled(process.env.AZURE_USE_MODEL_AS_DEPLOYMENT_NAME);
|
||||
const updatedAzure = { ...azure };
|
||||
updatedAzure.azureOpenAIApiDeploymentName = useModelName
|
||||
? sanitizeModelName(llmConfig.model || '')
|
||||
: azure.azureOpenAIApiDeploymentName;
|
||||
|
||||
if (process.env.AZURE_OPENAI_DEFAULT_MODEL) {
|
||||
llmConfig.model = process.env.AZURE_OPENAI_DEFAULT_MODEL;
|
||||
}
|
||||
return result;
|
||||
|
||||
const constructAzureOpenAIBasePath = () => {
|
||||
if (!baseURL) {
|
||||
return;
|
||||
}
|
||||
const azureURL = constructAzureURL({
|
||||
baseURL,
|
||||
azureOptions: updatedAzure,
|
||||
});
|
||||
updatedAzure.azureOpenAIBasePath = azureURL.split(
|
||||
`/${updatedAzure.azureOpenAIApiDeploymentName}`,
|
||||
)[0];
|
||||
};
|
||||
|
||||
constructAzureOpenAIBasePath();
|
||||
Object.assign(llmConfig, updatedAzure);
|
||||
|
||||
const constructAzureResponsesApi = () => {
|
||||
if (!llmConfig.useResponsesApi) {
|
||||
return;
|
||||
}
|
||||
|
||||
delete llmConfig.azureOpenAIApiDeploymentName;
|
||||
delete llmConfig.azureOpenAIApiInstanceName;
|
||||
delete llmConfig.azureOpenAIApiVersion;
|
||||
delete llmConfig.azureOpenAIBasePath;
|
||||
delete llmConfig.azureOpenAIApiKey;
|
||||
llmConfig.apiKey = apiKey;
|
||||
};
|
||||
|
||||
constructAzureResponsesApi();
|
||||
|
||||
llmConfig.model = updatedAzure.azureOpenAIApiDeploymentName;
|
||||
return { llmConfig, tools, azure: updatedAzure };
|
||||
}
|
||||
|
|
|
|||
95
packages/api/src/endpoints/openai/transform.ts
Normal file
95
packages/api/src/endpoints/openai/transform.ts
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
import { EModelEndpoint } from 'librechat-data-provider';
|
||||
import type { ClientOptions } from '@librechat/agents';
|
||||
import type * as t from '~/types';
|
||||
import { knownOpenAIParams } from './llm';
|
||||
|
||||
const anthropicExcludeParams = new Set(['anthropicApiUrl']);
|
||||
|
||||
/**
|
||||
* Transforms a Non-OpenAI LLM config to an OpenAI-conformant config.
|
||||
* Non-OpenAI parameters are moved to modelKwargs.
|
||||
* Also extracts configuration options that belong in configOptions.
|
||||
* Handles addParams and dropParams for parameter customization.
|
||||
*/
|
||||
export function transformToOpenAIConfig({
|
||||
addParams,
|
||||
dropParams,
|
||||
llmConfig,
|
||||
fromEndpoint,
|
||||
}: {
|
||||
addParams?: Record<string, unknown>;
|
||||
dropParams?: string[];
|
||||
llmConfig: ClientOptions;
|
||||
fromEndpoint: string;
|
||||
}): {
|
||||
llmConfig: t.OAIClientOptions;
|
||||
configOptions: Partial<t.OpenAIConfiguration>;
|
||||
} {
|
||||
const openAIConfig: Partial<t.OAIClientOptions> = {};
|
||||
let configOptions: Partial<t.OpenAIConfiguration> = {};
|
||||
let modelKwargs: Record<string, unknown> = {};
|
||||
let hasModelKwargs = false;
|
||||
|
||||
const isAnthropic = fromEndpoint === EModelEndpoint.anthropic;
|
||||
const excludeParams = isAnthropic ? anthropicExcludeParams : new Set();
|
||||
|
||||
for (const [key, value] of Object.entries(llmConfig)) {
|
||||
if (value === undefined || value === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (excludeParams.has(key)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (isAnthropic && key === 'clientOptions') {
|
||||
configOptions = Object.assign({}, configOptions, value as Partial<t.OpenAIConfiguration>);
|
||||
continue;
|
||||
} else if (isAnthropic && key === 'invocationKwargs') {
|
||||
modelKwargs = Object.assign({}, modelKwargs, value as Record<string, unknown>);
|
||||
hasModelKwargs = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (knownOpenAIParams.has(key)) {
|
||||
(openAIConfig as Record<string, unknown>)[key] = value;
|
||||
} else {
|
||||
modelKwargs[key] = value;
|
||||
hasModelKwargs = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (addParams && typeof addParams === 'object') {
|
||||
for (const [key, value] of Object.entries(addParams)) {
|
||||
if (knownOpenAIParams.has(key)) {
|
||||
(openAIConfig as Record<string, unknown>)[key] = value;
|
||||
} else {
|
||||
modelKwargs[key] = value;
|
||||
hasModelKwargs = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (hasModelKwargs) {
|
||||
openAIConfig.modelKwargs = modelKwargs;
|
||||
}
|
||||
|
||||
if (dropParams && Array.isArray(dropParams)) {
|
||||
dropParams.forEach((param) => {
|
||||
if (param in openAIConfig) {
|
||||
delete openAIConfig[param as keyof t.OAIClientOptions];
|
||||
}
|
||||
if (openAIConfig.modelKwargs && param in openAIConfig.modelKwargs) {
|
||||
delete openAIConfig.modelKwargs[param];
|
||||
if (Object.keys(openAIConfig.modelKwargs).length === 0) {
|
||||
delete openAIConfig.modelKwargs;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
llmConfig: openAIConfig as t.OAIClientOptions,
|
||||
configOptions,
|
||||
};
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue