diff --git a/api/app/clients/OpenAIClient.js b/api/app/clients/OpenAIClient.js index 7d0baa40d7..ca0c8d8424 100644 --- a/api/app/clients/OpenAIClient.js +++ b/api/app/clients/OpenAIClient.js @@ -2,8 +2,13 @@ const OpenAI = require('openai'); const { HttpsProxyAgent } = require('https-proxy-agent'); const { getResponseSender, ImageDetailCost, ImageDetail } = require('librechat-data-provider'); const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken'); +const { + getModelMaxTokens, + genAzureChatCompletion, + extractBaseURL, + constructAzureURL, +} = require('~/utils'); const { encodeAndFormat, validateVisionModel } = require('~/server/services/Files/images'); -const { getModelMaxTokens, genAzureChatCompletion, extractBaseURL } = require('~/utils'); const { truncateText, formatMessage, CUT_OFF_PROMPT } = require('./prompts'); const { handleOpenAIErrors } = require('./tools/util'); const spendTokens = require('~/models/spendTokens'); @@ -32,6 +37,7 @@ class OpenAIClient extends BaseClient { ? options.contextStrategy.toLowerCase() : 'discard'; this.shouldSummarize = this.contextStrategy === 'summarize'; + /** @type {AzureOptions} */ this.azure = options.azure || false; this.setOptions(options); } @@ -104,10 +110,10 @@ class OpenAIClient extends BaseClient { } if (this.azure && process.env.AZURE_OPENAI_DEFAULT_MODEL) { - this.azureEndpoint = genAzureChatCompletion(this.azure, this.modelOptions.model); + this.azureEndpoint = genAzureChatCompletion(this.azure, this.modelOptions.model, this); this.modelOptions.model = process.env.AZURE_OPENAI_DEFAULT_MODEL; } else if (this.azure) { - this.azureEndpoint = genAzureChatCompletion(this.azure, this.modelOptions.model); + this.azureEndpoint = genAzureChatCompletion(this.azure, this.modelOptions.model, this); } const { model } = this.modelOptions; @@ -711,7 +717,7 @@ class OpenAIClient extends BaseClient { if (this.azure) { modelOptions.model = process.env.AZURE_OPENAI_DEFAULT_MODEL ?? modelOptions.model; - this.azureEndpoint = genAzureChatCompletion(this.azure, modelOptions.model); + this.azureEndpoint = genAzureChatCompletion(this.azure, modelOptions.model, this); } const instructionsPayload = [ @@ -949,7 +955,12 @@ ${convo} // Azure does not accept `model` in the body, so we need to remove it. delete modelOptions.model; - opts.baseURL = this.azureEndpoint.split('/chat')[0]; + opts.baseURL = this.langchainProxy + ? constructAzureURL({ + baseURL: this.langchainProxy, + azure: this.azure, + }) + : this.azureEndpoint.split(/\/(chat|completion)/)[0]; opts.defaultQuery = { 'api-version': this.azure.azureOpenAIApiVersion }; opts.defaultHeaders = { ...opts.defaultHeaders, 'api-key': this.apiKey }; } diff --git a/api/app/clients/llm/createLLM.js b/api/app/clients/llm/createLLM.js index de5fa18e77..62f2fe86f9 100644 --- a/api/app/clients/llm/createLLM.js +++ b/api/app/clients/llm/createLLM.js @@ -1,6 +1,6 @@ const { ChatOpenAI } = require('langchain/chat_models/openai'); -const { sanitizeModelName } = require('../../../utils'); -const { isEnabled } = require('../../../server/utils'); +const { sanitizeModelName, constructAzureURL } = require('~/utils'); +const { isEnabled } = require('~/server/utils'); /** * Creates a new instance of a language model (LLM) for chat interactions. @@ -36,6 +36,7 @@ function createLLM({ apiKey: openAIApiKey, }; + /** @type {AzureOptions} */ let azureOptions = {}; if (azure) { const useModelName = isEnabled(process.env.AZURE_USE_MODEL_AS_DEPLOYMENT_NAME); @@ -53,8 +54,12 @@ function createLLM({ modelOptions.modelName = process.env.AZURE_OPENAI_DEFAULT_MODEL; } - // console.debug('createLLM: configOptions'); - // console.debug(configOptions); + if (azure && configOptions.basePath) { + configOptions.basePath = constructAzureURL({ + baseURL: configOptions.basePath, + azure: azureOptions, + }); + } return new ChatOpenAI( { diff --git a/api/server/services/Endpoints/gptPlugins/initializeClient.js b/api/server/services/Endpoints/gptPlugins/initializeClient.js index 4abb2d2de5..54ea822e49 100644 --- a/api/server/services/Endpoints/gptPlugins/initializeClient.js +++ b/api/server/services/Endpoints/gptPlugins/initializeClient.js @@ -1,7 +1,8 @@ -const { PluginsClient } = require('~/app'); -const { isEnabled } = require('~/server/utils'); -const { getAzureCredentials } = require('~/utils'); +const { EModelEndpoint } = require('librechat-data-provider'); const { getUserKey, checkUserKeyExpiry } = require('~/server/services/UserService'); +const { getAzureCredentials } = require('~/utils'); +const { isEnabled } = require('~/server/utils'); +const { PluginsClient } = require('~/app'); const initializeClient = async ({ req, res, endpointOption }) => { const { @@ -10,26 +11,40 @@ const initializeClient = async ({ req, res, endpointOption }) => { AZURE_API_KEY, PLUGINS_USE_AZURE, OPENAI_REVERSE_PROXY, + AZURE_OPENAI_BASEURL, OPENAI_SUMMARIZE, DEBUG_PLUGINS, } = process.env; + const { key: expiresAt } = req.body; const contextStrategy = isEnabled(OPENAI_SUMMARIZE) ? 'summarize' : null; + + const useAzure = isEnabled(PLUGINS_USE_AZURE); + const endpoint = useAzure ? EModelEndpoint.azureOpenAI : EModelEndpoint.openAI; + + const baseURLOptions = { + [EModelEndpoint.openAI]: OPENAI_REVERSE_PROXY, + [EModelEndpoint.azureOpenAI]: AZURE_OPENAI_BASEURL, + }; + + const reverseProxyUrl = baseURLOptions[endpoint] ?? null; + const clientOptions = { contextStrategy, debug: isEnabled(DEBUG_PLUGINS), - reverseProxyUrl: OPENAI_REVERSE_PROXY ?? null, + reverseProxyUrl, proxy: PROXY ?? null, req, res, ...endpointOption, }; - const useAzure = isEnabled(PLUGINS_USE_AZURE); + const credentials = { + [EModelEndpoint.openAI]: OPENAI_API_KEY, + [EModelEndpoint.azureOpenAI]: AZURE_API_KEY, + }; - const isUserProvided = useAzure - ? AZURE_API_KEY === 'user_provided' - : OPENAI_API_KEY === 'user_provided'; + const isUserProvided = credentials[endpoint] === 'user_provided'; let userKey = null; if (expiresAt && isUserProvided) { @@ -39,11 +54,11 @@ const initializeClient = async ({ req, res, endpointOption }) => { ); userKey = await getUserKey({ userId: req.user.id, - name: useAzure ? 'azureOpenAI' : 'openAI', + name: endpoint, }); } - let apiKey = isUserProvided ? userKey : OPENAI_API_KEY; + let apiKey = isUserProvided ? userKey : credentials[endpoint]; if (useAzure || (apiKey && apiKey.includes('azure') && !clientOptions.azure)) { clientOptions.azure = isUserProvided ? JSON.parse(userKey) : getAzureCredentials(); diff --git a/api/server/services/Endpoints/openAI/initializeClient.js b/api/server/services/Endpoints/openAI/initializeClient.js index 37681485b2..b6427823e1 100644 --- a/api/server/services/Endpoints/openAI/initializeClient.js +++ b/api/server/services/Endpoints/openAI/initializeClient.js @@ -1,7 +1,8 @@ -const { OpenAIClient } = require('~/app'); -const { isEnabled } = require('~/server/utils'); -const { getAzureCredentials } = require('~/utils'); +const { EModelEndpoint } = require('librechat-data-provider'); const { getUserKey, checkUserKeyExpiry } = require('~/server/services/UserService'); +const { getAzureCredentials } = require('~/utils'); +const { isEnabled } = require('~/server/utils'); +const { OpenAIClient } = require('~/app'); const initializeClient = async ({ req, res, endpointOption }) => { const { @@ -9,15 +10,24 @@ const initializeClient = async ({ req, res, endpointOption }) => { OPENAI_API_KEY, AZURE_API_KEY, OPENAI_REVERSE_PROXY, + AZURE_OPENAI_BASEURL, OPENAI_SUMMARIZE, DEBUG_OPENAI, } = process.env; const { key: expiresAt, endpoint } = req.body; const contextStrategy = isEnabled(OPENAI_SUMMARIZE) ? 'summarize' : null; + + const baseURLOptions = { + [EModelEndpoint.openAI]: OPENAI_REVERSE_PROXY, + [EModelEndpoint.azureOpenAI]: AZURE_OPENAI_BASEURL, + }; + + const reverseProxyUrl = baseURLOptions[endpoint] ?? null; + const clientOptions = { debug: isEnabled(DEBUG_OPENAI), contextStrategy, - reverseProxyUrl: OPENAI_REVERSE_PROXY ?? null, + reverseProxyUrl, proxy: PROXY ?? null, req, res, @@ -25,8 +35,8 @@ const initializeClient = async ({ req, res, endpointOption }) => { }; const credentials = { - openAI: OPENAI_API_KEY, - azureOpenAI: AZURE_API_KEY, + [EModelEndpoint.openAI]: OPENAI_API_KEY, + [EModelEndpoint.azureOpenAI]: AZURE_API_KEY, }; const isUserProvided = credentials[endpoint] === 'user_provided'; @@ -42,7 +52,7 @@ const initializeClient = async ({ req, res, endpointOption }) => { let apiKey = isUserProvided ? userKey : credentials[endpoint]; - if (endpoint === 'azureOpenAI') { + if (endpoint === EModelEndpoint.azureOpenAI) { clientOptions.azure = isUserProvided ? JSON.parse(userKey) : getAzureCredentials(); apiKey = clientOptions.azure.azureOpenAIApiKey; } diff --git a/api/utils/azureUtils.js b/api/utils/azureUtils.js index a735a6b4f7..638357872f 100644 --- a/api/utils/azureUtils.js +++ b/api/utils/azureUtils.js @@ -1,11 +1,3 @@ -/** - * @typedef {Object} AzureCredentials - * @property {string} azureOpenAIApiKey - The Azure OpenAI API key. - * @property {string} azureOpenAIApiInstanceName - The Azure OpenAI API instance name. - * @property {string} azureOpenAIApiDeploymentName - The Azure OpenAI API deployment name. - * @property {string} azureOpenAIApiVersion - The Azure OpenAI API version. - */ - const { isEnabled } = require('~/server/utils'); /** @@ -37,22 +29,29 @@ const genAzureEndpoint = ({ azureOpenAIApiInstanceName, azureOpenAIApiDeployment * @param {string} [AzureConfig.azureOpenAIApiDeploymentName] - The Azure OpenAI API deployment name (optional). * @param {string} AzureConfig.azureOpenAIApiVersion - The Azure OpenAI API version. * @param {string} [modelName] - The model name to be included in the deployment name (optional). + * @param {Object} [client] - The API Client class for optionally setting properties (optional). * @returns {string} The complete chat completion endpoint URL for the Azure OpenAI API. * @throws {Error} If neither azureOpenAIApiDeploymentName nor modelName is provided. */ const genAzureChatCompletion = ( { azureOpenAIApiInstanceName, azureOpenAIApiDeploymentName, azureOpenAIApiVersion }, modelName, + client, ) => { // Determine the deployment segment of the URL based on provided modelName or azureOpenAIApiDeploymentName let deploymentSegment; if (isEnabled(process.env.AZURE_USE_MODEL_AS_DEPLOYMENT_NAME) && modelName) { const sanitizedModelName = sanitizeModelName(modelName); deploymentSegment = `${sanitizedModelName}`; + client && + typeof client === 'object' && + (client.azure.azureOpenAIApiDeploymentName = sanitizedModelName); } else if (azureOpenAIApiDeploymentName) { deploymentSegment = azureOpenAIApiDeploymentName; - } else { - throw new Error('Either a model name or a deployment name must be provided.'); + } else if (!process.env.AZURE_OPENAI_BASEURL) { + throw new Error( + 'Either a model name with the `AZURE_USE_MODEL_AS_DEPLOYMENT_NAME` setting or a deployment name must be provided if `AZURE_OPENAI_BASEURL` is omitted.', + ); } return `https://${azureOpenAIApiInstanceName}.openai.azure.com/openai/deployments/${deploymentSegment}/chat/completions?api-version=${azureOpenAIApiVersion}`; @@ -60,7 +59,7 @@ const genAzureChatCompletion = ( /** * Retrieves the Azure OpenAI API credentials from environment variables. - * @returns {AzureCredentials} An object containing the Azure OpenAI API credentials. + * @returns {AzureOptions} An object containing the Azure OpenAI API credentials. */ const getAzureCredentials = () => { return { @@ -71,9 +70,33 @@ const getAzureCredentials = () => { }; }; +/** + * Constructs a URL by replacing placeholders in the baseURL with values from the azure object. + * It specifically looks for '${INSTANCE_NAME}' and '${DEPLOYMENT_NAME}' within the baseURL and replaces + * them with 'azureOpenAIApiInstanceName' and 'azureOpenAIApiDeploymentName' from the azure object. + * If the respective azure property is not provided, the placeholder is replaced with an empty string. + * + * @param {Object} params - The parameters object. + * @param {string} params.baseURL - The baseURL to inspect for replacement placeholders. + * @param {AzureOptions} params.azure - The baseURL to inspect for replacement placeholders. + * @returns {string} The complete baseURL with credentials injected for the Azure OpenAI API. + */ +function constructAzureURL({ baseURL, azure }) { + let finalURL = baseURL; + + // Replace INSTANCE_NAME and DEPLOYMENT_NAME placeholders with actual values if available + if (azure) { + finalURL = finalURL.replace('${INSTANCE_NAME}', azure.azureOpenAIApiInstanceName ?? ''); + finalURL = finalURL.replace('${DEPLOYMENT_NAME}', azure.azureOpenAIApiDeploymentName ?? ''); + } + + return finalURL; +} + module.exports = { sanitizeModelName, genAzureEndpoint, genAzureChatCompletion, getAzureCredentials, + constructAzureURL, }; diff --git a/api/utils/azureUtils.spec.js b/api/utils/azureUtils.spec.js new file mode 100644 index 0000000000..77db26b091 --- /dev/null +++ b/api/utils/azureUtils.spec.js @@ -0,0 +1,268 @@ +const { + sanitizeModelName, + genAzureEndpoint, + genAzureChatCompletion, + getAzureCredentials, + constructAzureURL, +} = require('./azureUtils'); + +describe('sanitizeModelName', () => { + test('removes periods from the model name', () => { + const sanitized = sanitizeModelName('model.name'); + expect(sanitized).toBe('modelname'); + }); + + test('leaves model name unchanged if no periods are present', () => { + const sanitized = sanitizeModelName('modelname'); + expect(sanitized).toBe('modelname'); + }); +}); + +describe('genAzureEndpoint', () => { + test('generates correct endpoint URL', () => { + const url = genAzureEndpoint({ + azureOpenAIApiInstanceName: 'instanceName', + azureOpenAIApiDeploymentName: 'deploymentName', + }); + expect(url).toBe('https://instanceName.openai.azure.com/openai/deployments/deploymentName'); + }); +}); + +describe('genAzureChatCompletion', () => { + // Test with both deployment name and model name provided + test('prefers model name over deployment name when both are provided and feature enabled', () => { + process.env.AZURE_USE_MODEL_AS_DEPLOYMENT_NAME = 'true'; + const url = genAzureChatCompletion( + { + azureOpenAIApiInstanceName: 'instanceName', + azureOpenAIApiDeploymentName: 'deploymentName', + azureOpenAIApiVersion: 'v1', + }, + 'modelName', + ); + expect(url).toBe( + 'https://instanceName.openai.azure.com/openai/deployments/modelName/chat/completions?api-version=v1', + ); + }); + + // Test with only deployment name provided + test('uses deployment name when model name is not provided', () => { + const url = genAzureChatCompletion({ + azureOpenAIApiInstanceName: 'instanceName', + azureOpenAIApiDeploymentName: 'deploymentName', + azureOpenAIApiVersion: 'v1', + }); + expect(url).toBe( + 'https://instanceName.openai.azure.com/openai/deployments/deploymentName/chat/completions?api-version=v1', + ); + }); + + // Test with only model name provided + test('uses model name when deployment name is not provided and feature enabled', () => { + process.env.AZURE_USE_MODEL_AS_DEPLOYMENT_NAME = 'true'; + const url = genAzureChatCompletion( + { + azureOpenAIApiInstanceName: 'instanceName', + azureOpenAIApiVersion: 'v1', + }, + 'modelName', + ); + expect(url).toBe( + 'https://instanceName.openai.azure.com/openai/deployments/modelName/chat/completions?api-version=v1', + ); + }); + + // Test with neither deployment name nor model name provided + test('throws error if neither deployment name nor model name is provided', () => { + expect(() => { + genAzureChatCompletion({ + azureOpenAIApiInstanceName: 'instanceName', + azureOpenAIApiVersion: 'v1', + }); + }).toThrow( + 'Either a model name with the `AZURE_USE_MODEL_AS_DEPLOYMENT_NAME` setting or a deployment name must be provided if `AZURE_OPENAI_BASEURL` is omitted.', + ); + }); + + // Test with feature disabled but model name provided + test('ignores model name and uses deployment name when feature is disabled', () => { + process.env.AZURE_USE_MODEL_AS_DEPLOYMENT_NAME = 'false'; + const url = genAzureChatCompletion( + { + azureOpenAIApiInstanceName: 'instanceName', + azureOpenAIApiDeploymentName: 'deploymentName', + azureOpenAIApiVersion: 'v1', + }, + 'modelName', + ); + expect(url).toBe( + 'https://instanceName.openai.azure.com/openai/deployments/deploymentName/chat/completions?api-version=v1', + ); + }); + + // Test with sanitized model name + test('sanitizes model name when used in URL', () => { + process.env.AZURE_USE_MODEL_AS_DEPLOYMENT_NAME = 'true'; + const url = genAzureChatCompletion( + { + azureOpenAIApiInstanceName: 'instanceName', + azureOpenAIApiVersion: 'v1', + }, + 'model.name', + ); + expect(url).toBe( + 'https://instanceName.openai.azure.com/openai/deployments/modelname/chat/completions?api-version=v1', + ); + }); + + // Test with client parameter and model name + test('updates client with sanitized model name when provided and feature enabled', () => { + process.env.AZURE_USE_MODEL_AS_DEPLOYMENT_NAME = 'true'; + const clientMock = { azure: {} }; + const url = genAzureChatCompletion( + { + azureOpenAIApiInstanceName: 'instanceName', + azureOpenAIApiVersion: 'v1', + }, + 'model.name', + clientMock, + ); + expect(url).toBe( + 'https://instanceName.openai.azure.com/openai/deployments/modelname/chat/completions?api-version=v1', + ); + expect(clientMock.azure.azureOpenAIApiDeploymentName).toBe('modelname'); + }); + + // Test with client parameter but without model name + test('does not update client when model name is not provided', () => { + const clientMock = { azure: {} }; + const url = genAzureChatCompletion( + { + azureOpenAIApiInstanceName: 'instanceName', + azureOpenAIApiDeploymentName: 'deploymentName', + azureOpenAIApiVersion: 'v1', + }, + undefined, + clientMock, + ); + expect(url).toBe( + 'https://instanceName.openai.azure.com/openai/deployments/deploymentName/chat/completions?api-version=v1', + ); + expect(clientMock.azure.azureOpenAIApiDeploymentName).toBeUndefined(); + }); + + // Test with client parameter and deployment name when feature is disabled + test('does not update client when feature is disabled', () => { + process.env.AZURE_USE_MODEL_AS_DEPLOYMENT_NAME = 'false'; + const clientMock = { azure: {} }; + const url = genAzureChatCompletion( + { + azureOpenAIApiInstanceName: 'instanceName', + azureOpenAIApiDeploymentName: 'deploymentName', + azureOpenAIApiVersion: 'v1', + }, + 'modelName', + clientMock, + ); + expect(url).toBe( + 'https://instanceName.openai.azure.com/openai/deployments/deploymentName/chat/completions?api-version=v1', + ); + expect(clientMock.azure.azureOpenAIApiDeploymentName).toBeUndefined(); + }); + + // Reset environment variable after tests + afterEach(() => { + delete process.env.AZURE_USE_MODEL_AS_DEPLOYMENT_NAME; + }); +}); + +describe('getAzureCredentials', () => { + beforeEach(() => { + process.env.AZURE_API_KEY = 'testApiKey'; + process.env.AZURE_OPENAI_API_INSTANCE_NAME = 'instanceName'; + process.env.AZURE_OPENAI_API_DEPLOYMENT_NAME = 'deploymentName'; + process.env.AZURE_OPENAI_API_VERSION = 'v1'; + }); + + test('retrieves Azure OpenAI API credentials from environment variables', () => { + const credentials = getAzureCredentials(); + expect(credentials).toEqual({ + azureOpenAIApiKey: 'testApiKey', + azureOpenAIApiInstanceName: 'instanceName', + azureOpenAIApiDeploymentName: 'deploymentName', + azureOpenAIApiVersion: 'v1', + }); + }); +}); + +describe('constructAzureURL', () => { + test('replaces both placeholders when both properties are provided', () => { + const url = constructAzureURL({ + baseURL: 'https://example.com/${INSTANCE_NAME}/${DEPLOYMENT_NAME}', + azure: { + azureOpenAIApiInstanceName: 'instance1', + azureOpenAIApiDeploymentName: 'deployment1', + }, + }); + expect(url).toBe('https://example.com/instance1/deployment1'); + }); + + test('replaces only INSTANCE_NAME when only azureOpenAIApiInstanceName is provided', () => { + const url = constructAzureURL({ + baseURL: 'https://example.com/${INSTANCE_NAME}/${DEPLOYMENT_NAME}', + azure: { + azureOpenAIApiInstanceName: 'instance2', + }, + }); + expect(url).toBe('https://example.com/instance2/'); + }); + + test('replaces only DEPLOYMENT_NAME when only azureOpenAIApiDeploymentName is provided', () => { + const url = constructAzureURL({ + baseURL: 'https://example.com/${INSTANCE_NAME}/${DEPLOYMENT_NAME}', + azure: { + azureOpenAIApiDeploymentName: 'deployment2', + }, + }); + expect(url).toBe('https://example.com//deployment2'); + }); + + test('does not replace any placeholders when azure object is empty', () => { + const url = constructAzureURL({ + baseURL: 'https://example.com/${INSTANCE_NAME}/${DEPLOYMENT_NAME}', + azure: {}, + }); + expect(url).toBe('https://example.com//'); + }); + + test('returns baseURL as is when azure object is not provided', () => { + const url = constructAzureURL({ + baseURL: 'https://example.com/${INSTANCE_NAME}/${DEPLOYMENT_NAME}', + }); + expect(url).toBe('https://example.com/${INSTANCE_NAME}/${DEPLOYMENT_NAME}'); + }); + + test('returns baseURL as is when no placeholders are set', () => { + const url = constructAzureURL({ + baseURL: 'https://example.com/my_custom_instance/my_deployment', + azure: { + azureOpenAIApiInstanceName: 'instance1', + azureOpenAIApiDeploymentName: 'deployment1', + }, + }); + expect(url).toBe('https://example.com/my_custom_instance/my_deployment'); + }); + + test('returns regular Azure OpenAI baseURL with placeholders set', () => { + const baseURL = + 'https://${INSTANCE_NAME}.openai.azure.com/openai/deployments/${DEPLOYMENT_NAME}'; + const url = constructAzureURL({ + baseURL, + azure: { + azureOpenAIApiInstanceName: 'instance1', + azureOpenAIApiDeploymentName: 'deployment1', + }, + }); + expect(url).toBe('https://instance1.openai.azure.com/openai/deployments/deployment1'); + }); +}); diff --git a/api/utils/extractBaseURL.js b/api/utils/extractBaseURL.js index cc95f4481d..730473c410 100644 --- a/api/utils/extractBaseURL.js +++ b/api/utils/extractBaseURL.js @@ -1,13 +1,15 @@ /** - * Extracts a valid OpenAI baseURL from a given string, matching "url/v1," also an added suffix, - * ending with "/openai" (to allow the Cloudflare, LiteLLM pattern). - * Returns the original URL if no match is found. + * Extracts a valid OpenAI baseURL from a given string, matching "url/v1," followed by an optional suffix. + * The suffix can be one of several predefined values (e.g., 'openai', 'azure-openai', etc.), + * accommodating different proxy patterns like Cloudflare, LiteLLM, etc. + * Returns the original URL if no valid pattern is found. * * Examples: * - `https://open.ai/v1/chat` -> `https://open.ai/v1` * - `https://open.ai/v1/chat/completions` -> `https://open.ai/v1` - * - `https://open.ai/v1/ACCOUNT/GATEWAY/openai/completions` -> `https://open.ai/v1/ACCOUNT/GATEWAY/openai` + * - `https://gateway.ai.cloudflare.com/v1/account/gateway/azure-openai/completions` -> `https://gateway.ai.cloudflare.com/v1/account/gateway/azure-openai` * - `https://open.ai/v1/hi/openai` -> `https://open.ai/v1/hi/openai` + * - `https://api.example.com/v1/replicate` -> `https://api.example.com/v1/replicate` * * @param {string} url - The URL to be processed. * @returns {string} The matched pattern or input if no match is found. @@ -23,8 +25,27 @@ function extractBaseURL(url) { // Extract the part of the URL up to and including '/v1'. let baseUrl = url.substring(0, v1Index + 3); + const openai = 'openai'; + // Find which suffix is present. + const suffixes = [ + 'azure-openai', + openai, + 'replicate', + 'huggingface', + 'workers-ai', + 'aws-bedrock', + ]; + const suffixUsed = suffixes.find((suffix) => url.includes(`/${suffix}`)); + + if (suffixUsed === 'azure-openai') { + return url.split(/\/(chat|completion)/)[0]; + } + // Check if the URL has '/openai' immediately after '/v1'. - const openaiIndex = url.indexOf('/openai', v1Index + 3); + const openaiIndex = url.indexOf(`/${openai}`, v1Index + 3); + // Find which suffix is present in the URL, if any. + const suffixIndex = + suffixUsed === openai ? openaiIndex : url.indexOf(`/${suffixUsed}`, v1Index + 3); // If '/openai' is found right after '/v1', include it in the base URL. if (openaiIndex === v1Index + 3) { @@ -37,9 +58,9 @@ function extractBaseURL(url) { // If there is a next slash, the base URL goes up to but not including the slash. baseUrl = url.substring(0, nextSlashIndex); } - } else if (openaiIndex > 0) { - // If '/openai' is present but not immediately after '/v1', we need to include the reverse proxy pattern. - baseUrl = url.substring(0, openaiIndex + 7); + } else if (suffixIndex > 0) { + // If a suffix is present but not immediately after '/v1', we need to include the reverse proxy pattern. + baseUrl = url.substring(0, suffixIndex + suffixUsed.length + 1); } return baseUrl; diff --git a/api/utils/extractBaseURL.spec.js b/api/utils/extractBaseURL.spec.js index 299b9c1397..fe647b0699 100644 --- a/api/utils/extractBaseURL.spec.js +++ b/api/utils/extractBaseURL.spec.js @@ -53,4 +53,59 @@ describe('extractBaseURL', () => { const url = 'https://open.ai/v1/hi/openai'; expect(extractBaseURL(url)).toBe('https://open.ai/v1/hi/openai'); }); + + test('should handle Azure OpenAI Cloudflare endpoint correctly', () => { + const url = 'https://gateway.ai.cloudflare.com/v1/account/gateway/azure-openai/completions'; + expect(extractBaseURL(url)).toBe( + 'https://gateway.ai.cloudflare.com/v1/account/gateway/azure-openai', + ); + }); + + test('should include various suffixes in the extracted URL when present', () => { + const urls = [ + 'https://api.example.com/v1/azure-openai/something', + 'https://api.example.com/v1/replicate/anotherthing', + 'https://api.example.com/v1/huggingface/yetanotherthing', + 'https://api.example.com/v1/workers-ai/differentthing', + 'https://api.example.com/v1/aws-bedrock/somethingelse', + ]; + + const expected = [ + /* Note: exception for azure-openai to allow credential injection */ + 'https://api.example.com/v1/azure-openai/something', + 'https://api.example.com/v1/replicate', + 'https://api.example.com/v1/huggingface', + 'https://api.example.com/v1/workers-ai', + 'https://api.example.com/v1/aws-bedrock', + ]; + + urls.forEach((url, index) => { + expect(extractBaseURL(url)).toBe(expected[index]); + }); + }); + + test('should handle URLs with suffixes not immediately after /v1', () => { + const url = 'https://api.example.com/v1/some/path/azure-openai'; + expect(extractBaseURL(url)).toBe('https://api.example.com/v1/some/path/azure-openai'); + }); + + test('should handle URLs with complex paths after the suffix', () => { + const url = 'https://api.example.com/v1/replicate/deep/path/segment'; + expect(extractBaseURL(url)).toBe('https://api.example.com/v1/replicate'); + }); + + test('should leave a regular Azure OpenAI baseURL as is', () => { + const url = 'https://instance-name.openai.azure.com/openai/deployments/deployment-name'; + expect(extractBaseURL(url)).toBe(url); + }); + + test('should leave a regular Azure OpenAI baseURL with placeholders as is', () => { + const url = 'https://${INSTANCE_NAME}.openai.azure.com/openai/deployments/${DEPLOYMENT_NAME}'; + expect(extractBaseURL(url)).toBe(url); + }); + + test('should leave an alternate Azure OpenAI baseURL with placeholders as is', () => { + const url = 'https://${INSTANCE_NAME}.com/resources/deployments/${DEPLOYMENT_NAME}'; + expect(extractBaseURL(url)).toBe(url); + }); }); diff --git a/docs/install/configuration/ai_setup.md b/docs/install/configuration/ai_setup.md index 9a412c936f..3c9ad08a02 100644 --- a/docs/install/configuration/ai_setup.md +++ b/docs/install/configuration/ai_setup.md @@ -278,6 +278,45 @@ AZURE_OPENAI_DEFAULT_MODEL=gpt-3.5-turbo # do include periods in the model name ``` +### Using a Specified Base URL with Azure + +The base URL for Azure OpenAI API requests can be dynamically configured. This is useful for proxying services such as [Cloudflare AI Gateway](https://developers.cloudflare.com/ai-gateway/providers/azureopenai/), or if you wish to explicitly override the baseURL handling of the app. + +LibreChat will use the `AZURE_OPENAI_BASEURL` environment variable, which can include placeholders for the Azure OpenAI API instance and deployment names. + +In the application's environment configuration, the base URL is set like this: + +```bash +# .env file +AZURE_OPENAI_BASEURL=https://example.azure-api.net/${INSTANCE_NAME}/${DEPLOYMENT_NAME} + +# OR +AZURE_OPENAI_BASEURL=https://${INSTANCE_NAME}.openai.azure.com/openai/deployments/${DEPLOYMENT_NAME} + +# Cloudflare example +AZURE_OPENAI_BASEURL=https://gateway.ai.cloudflare.com/v1/ACCOUNT_TAG/GATEWAY/azure-openai/${INSTANCE_NAME}/${DEPLOYMENT_NAME} +``` + +The application replaces `${INSTANCE_NAME}` and `${DEPLOYMENT_NAME}` in the `AZURE_OPENAI_BASEURL`, processed according to the other settings discussed in the guide. + +**You can also omit the placeholders completely and simply construct the baseURL with your credentials:** + +```bash +# .env file +AZURE_OPENAI_BASEURL=https://instance-1.openai.azure.com/openai/deployments/deployment-1 + +# Cloudflare example +AZURE_OPENAI_BASEURL=https://gateway.ai.cloudflare.com/v1/ACCOUNT_TAG/GATEWAY/azure-openai/instance-1/deployment-1 +``` + +Setting these values will override all of the application's internal handling of the instance and deployment names and use your specified base URL. + +**Notes:** +- You should still provide the `AZURE_OPENAI_API_VERSION` and `AZURE_API_KEY` via the .env file as they are programmatically added to the requests. +- When specifying instance and deployment names in the `AZURE_OPENAI_BASEURL`, their respective environment variables can be omitted (`AZURE_OPENAI_API_INSTANCE_NAME` and `AZURE_OPENAI_API_DEPLOYMENT_NAME`) except for use with Plugins. +- Specifying instance and deployment names in the `AZURE_OPENAI_BASEURL` instead of placeholders creates conflicts with "plugins," "vision," "default-model," and "model-as-deployment-name" support. +- Due to the conflicts that arise with other features, it is recommended to use placeholder for instance and deployment names in the `AZURE_OPENAI_BASEURL` + ### Enabling Auto-Generated Titles with Azure The default titling model is set to `gpt-3.5-turbo`. @@ -294,7 +333,10 @@ This will work seamlessly as it does with the [OpenAI endpoint](#openai) (no nee Alternatively, you can set the [required variables](#required-variables) to explicitly use your vision deployment, but this may limit you to exclusively using your vision deployment for all Azure chat settings. -As of December 18th, 2023, Vision models seem to have degraded performance with Azure OpenAI when compared to [OpenAI](#openai) + +**Notes:** +- If using `AZURE_OPENAI_BASEURL`, you should not specify instance and deployment names instead of placeholders as the vision request will fail. +- As of December 18th, 2023, Vision models seem to have degraded performance with Azure OpenAI when compared to [OpenAI](#openai) ![image](https://github.com/danny-avila/LibreChat/assets/110412045/7306185f-c32c-4483-9167-af514cc1c2dd) @@ -361,6 +403,9 @@ To use Azure with the Plugins endpoint, make sure the following environment vari * `PLUGINS_USE_AZURE`: If set to "true" or any truthy value, this will enable the program to use Azure with the Plugins endpoint. * `AZURE_API_KEY`: Your Azure API key must be set with an environment variable. +**Important:** +- If using `AZURE_OPENAI_BASEURL`, you should not specify instance and deployment names instead of placeholders as the plugin request will fail. + --- ## [OpenRouter](https://openrouter.ai/) diff --git a/docs/install/configuration/custom_config.md b/docs/install/configuration/custom_config.md index b7bba50309..ff6b793df5 100644 --- a/docs/install/configuration/custom_config.md +++ b/docs/install/configuration/custom_config.md @@ -4,6 +4,21 @@ description: Comprehensive guide for configuring the `librechat.yaml` file AKA t weight: -10 --- + + # LibreChat Configuration Guide Welcome to the guide for configuring the **librechat.yaml** file in LibreChat. diff --git a/docs/install/configuration/dotenv.md b/docs/install/configuration/dotenv.md index 981656780d..802837d002 100644 --- a/docs/install/configuration/dotenv.md +++ b/docs/install/configuration/dotenv.md @@ -177,6 +177,20 @@ AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME= - Identify the available models, separated by commas *without spaces*. The first will be default. Leave it blank or as is to use internal settings. +- **The base URL for Azure OpenAI API requests can be dynamically configured.** + +```bash +# .env file +AZURE_OPENAI_BASEURL=https://${INSTANCE_NAME}.openai.azure.com/openai/deployments/${DEPLOYMENT_NAME} + +# Cloudflare example +AZURE_OPENAI_BASEURL=https://gateway.ai.cloudflare.com/v1/ACCOUNT_TAG/GATEWAY/azure-openai/${INSTANCE_NAME}/${DEPLOYMENT_NAME} +``` +- Sets the base URL for Azure OpenAI API requests. +- Can include `${INSTANCE_NAME}` and `${DEPLOYMENT_NAME}` placeholders or specific credentials. +- Example: "https://gateway.ai.cloudflare.com/v1/ACCOUNT_TAG/GATEWAY/azure-openai/${INSTANCE_NAME}/${DEPLOYMENT_NAME}" +- [More info about `AZURE_OPENAI_BASEURL` here](./ai_setup.md#using-a-specified-base-url-with-azure) + > Note: as deployment names can't have periods, they will be removed when the endpoint is generated. ```bash diff --git a/packages/data-provider/src/types.ts b/packages/data-provider/src/types.ts index de3bd88933..07a5ace6d2 100644 --- a/packages/data-provider/src/types.ts +++ b/packages/data-provider/src/types.ts @@ -1,5 +1,5 @@ import OpenAI from 'openai'; -import type { TResPlugin, TMessage, TConversation, EModelEndpoint } from './schemas'; +import type { TResPlugin, TMessage, TConversation, EModelEndpoint, ImageDetail } from './schemas'; export type TOpenAIMessage = OpenAI.Chat.ChatCompletionMessageParam; export type TOpenAIFunction = OpenAI.Chat.ChatCompletionCreateParams.Function; @@ -11,10 +11,13 @@ export type TMessages = TMessage[]; export type TMessagesAtom = TMessages | null; +/* TODO: Cleanup EndpointOption types */ export type TEndpointOption = { endpoint: EModelEndpoint; endpointType?: EModelEndpoint; modelDisplayLabel?: string; + resendImages?: boolean; + imageDetail?: ImageDetail; model?: string | null; promptPrefix?: string; temperature?: number;