diff --git a/.env.example b/.env.example index d82395189c..2d3d9b51cd 100644 --- a/.env.example +++ b/.env.example @@ -169,11 +169,23 @@ DEBUG_OPENAI=false # Set to true to enable debug mode for the OpenAI endpoint # AZURE_OPENAI_API_COMPLETIONS_DEPLOYMENT_NAME= # AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME= -# Identify the available models, separated by commas *without spaces*. -# The first will be default. -# Leave it blank to use internal settings. +# NOTE: As of 2023-11-10, the Azure API only allows one model per deployment, +# It's recommended to name your deployments after the model name, e.g. "gpt-35-turbo," +# which allows for fast deployment switching and AZURE_USE_MODEL_AS_DEPLOYMENT_NAME enabled. +# However, you can use non-model deployment names and setting the AZURE_OPENAI_DEFAULT_MODEL to ensure it works as expected. + +# Identify the available models, separated by commas *without spaces*. +# The first will be default. Leave it blank or as is to use internal settings. +# NOTE: as deployment names can't have periods, they will be removed when the endpoint is generated. AZURE_OPENAI_MODELS=gpt-3.5-turbo,gpt-4 +# (Advanced) this enables the use of the model name as the deployment name, e.g. "gpt-3.5-turbo" as the deployment name +AZURE_USE_MODEL_AS_DEPLOYMENT_NAME=TRUE + +# (Advanced) this overrides the model setting for Azure, in case you want to use your custom deployment names +# as the values for AZURE_OPENAI_MODELS +# AZURE_OPENAI_DEFAULT_MODEL=gpt-3.5-turbo + # To use Azure with the Plugins endpoint, you need the variables above, and uncomment the following variable: # NOTE: This may not work as expected and Azure OpenAI may not support OpenAI Functions yet # Omit/leave it commented to use the default OpenAI API diff --git a/api/app/clients/OpenAIClient.js b/api/app/clients/OpenAIClient.js index b16c070dc1..124d05e68f 100644 --- a/api/app/clients/OpenAIClient.js +++ b/api/app/clients/OpenAIClient.js @@ -30,9 +30,6 @@ class OpenAIClient extends BaseClient { : 'discard'; this.shouldSummarize = this.contextStrategy === 'summarize'; this.azure = options.azure || false; - if (this.azure) { - this.azureEndpoint = genAzureChatCompletion(this.azure); - } this.setOptions(options); } @@ -86,6 +83,13 @@ class OpenAIClient extends BaseClient { isEnabled(OPENAI_FORCE_PROMPT) || (reverseProxy && reverseProxy.includes('completions') && !reverseProxy.includes('chat')); + if (this.azure && process.env.AZURE_OPENAI_DEFAULT_MODEL) { + this.azureEndpoint = genAzureChatCompletion(this.azure, this.modelOptions.model); + this.modelOptions.model = process.env.AZURE_OPENAI_DEFAULT_MODEL; + } else if (this.azure) { + this.azureEndpoint = genAzureChatCompletion(this.azure, this.modelOptions.model); + } + const { model } = this.modelOptions; this.isChatCompletion = this.useOpenRouter || !!reverseProxy || model.includes('gpt-'); @@ -533,6 +537,7 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s this.options.debug && console.error(e.message, e); modelOptions.model = OPENAI_TITLE_MODEL ?? 'gpt-3.5-turbo'; if (this.azure) { + modelOptions.model = process.env.AZURE_OPENAI_DEFAULT_MODEL ?? modelOptions.model; this.azureEndpoint = genAzureChatCompletion(this.azure, modelOptions.model); } const instructionsPayload = [ diff --git a/api/app/clients/llm/createLLM.js b/api/app/clients/llm/createLLM.js index 28a8d86de2..020fba6503 100644 --- a/api/app/clients/llm/createLLM.js +++ b/api/app/clients/llm/createLLM.js @@ -1,6 +1,60 @@ const { ChatOpenAI } = require('langchain/chat_models/openai'); const { sanitizeModelName } = require('../../../utils'); +const { isEnabled } = require('../../../server/utils'); +/** + * @typedef {Object} ModelOptions + * @property {string} modelName - The name of the model. + * @property {number} [temperature] - The temperature setting for the model. + * @property {number} [presence_penalty] - The presence penalty setting. + * @property {number} [frequency_penalty] - The frequency penalty setting. + * @property {number} [max_tokens] - The maximum number of tokens to generate. + */ + +/** + * @typedef {Object} ConfigOptions + * @property {string} [basePath] - The base path for the API requests. + * @property {Object} [baseOptions] - Base options for the API requests, including headers. + * @property {Object} [httpAgent] - The HTTP agent for the request. + * @property {Object} [httpsAgent] - The HTTPS agent for the request. + */ + +/** + * @typedef {Object} Callbacks + * @property {Function} [handleChatModelStart] - A callback function for handleChatModelStart + * @property {Function} [handleLLMEnd] - A callback function for handleLLMEnd + * @property {Function} [handleLLMError] - A callback function for handleLLMError + */ + +/** + * @typedef {Object} AzureOptions + * @property {string} [azureOpenAIApiKey] - The Azure OpenAI API key. + * @property {string} [azureOpenAIApiInstanceName] - The Azure OpenAI API instance name. + * @property {string} [azureOpenAIApiDeploymentName] - The Azure OpenAI API deployment name. + * @property {string} [azureOpenAIApiVersion] - The Azure OpenAI API version. + */ + +/** + * Creates a new instance of a language model (LLM) for chat interactions. + * + * @param {Object} options - The options for creating the LLM. + * @param {ModelOptions} options.modelOptions - The options specific to the model, including modelName, temperature, presence_penalty, frequency_penalty, and other model-related settings. + * @param {ConfigOptions} options.configOptions - Configuration options for the API requests, including proxy settings and custom headers. + * @param {Callbacks} options.callbacks - Callback functions for managing the lifecycle of the LLM, including token buffers, context, and initial message count. + * @param {boolean} [options.streaming=false] - Determines if the LLM should operate in streaming mode. + * @param {string} options.openAIApiKey - The API key for OpenAI, used for authentication. + * @param {AzureOptions} [options.azure={}] - Optional Azure-specific configurations. If provided, Azure configurations take precedence over OpenAI configurations. + * + * @returns {ChatOpenAI} An instance of the ChatOpenAI class, configured with the provided options. + * + * @example + * const llm = createLLM({ + * modelOptions: { modelName: 'gpt-3.5-turbo', temperature: 0.2 }, + * configOptions: { basePath: 'https://example.api/path' }, + * callbacks: { onMessage: handleMessage }, + * openAIApiKey: 'your-api-key' + * }); + */ function createLLM({ modelOptions, configOptions, @@ -16,10 +70,19 @@ function createLLM({ let azureOptions = {}; if (azure) { + const useModelName = isEnabled(process.env.AZURE_USE_MODEL_AS_DEPLOYMENT_NAME); + credentials = {}; configuration = {}; azureOptions = azure; - azureOptions.azureOpenAIApiDeploymentName = sanitizeModelName(modelOptions.modelName); + + azureOptions.azureOpenAIApiDeploymentName = useModelName + ? sanitizeModelName(modelOptions.modelName) + : azureOptions.azureOpenAIApiDeploymentName; + } + + if (azure && process.env.AZURE_OPENAI_DEFAULT_MODEL) { + modelOptions.modelName = process.env.AZURE_OPENAI_DEFAULT_MODEL; } // console.debug('createLLM: configOptions'); diff --git a/api/app/clients/specs/OpenAIClient.test.js b/api/app/clients/specs/OpenAIClient.test.js index 6c54a6b09c..ff7ab2f6de 100644 --- a/api/app/clients/specs/OpenAIClient.test.js +++ b/api/app/clients/specs/OpenAIClient.test.js @@ -12,6 +12,14 @@ describe('OpenAIClient', () => { { role: 'assistant', sender: 'Assistant', text: 'Hi', messageId: '2' }, ]; + beforeAll(() => { + jest.spyOn(console, 'warn').mockImplementation(() => {}); + }); + + afterAll(() => { + console.warn.mockRestore(); + }); + beforeEach(() => { const options = { // debug: true, @@ -90,6 +98,96 @@ describe('OpenAIClient', () => { }); }); + describe('setOptions with Simplified Azure Integration', () => { + afterEach(() => { + delete process.env.AZURE_OPENAI_DEFAULT_MODEL; + delete process.env.AZURE_USE_MODEL_AS_DEPLOYMENT_NAME; + }); + + const azureOpenAIApiInstanceName = 'test-instance'; + const azureOpenAIApiDeploymentName = 'test-deployment'; + const azureOpenAIApiVersion = '2020-07-01-preview'; + + const createOptions = (model) => ({ + modelOptions: { model }, + azure: { + azureOpenAIApiInstanceName, + azureOpenAIApiDeploymentName, + azureOpenAIApiVersion, + }, + }); + + it('should set model from AZURE_OPENAI_DEFAULT_MODEL when Azure is enabled', () => { + process.env.AZURE_OPENAI_DEFAULT_MODEL = 'gpt-4-azure'; + const options = createOptions('test'); + client.azure = options.azure; + client.setOptions(options); + expect(client.modelOptions.model).toBe('gpt-4-azure'); + }); + + it('should not change model if Azure is not enabled', () => { + process.env.AZURE_OPENAI_DEFAULT_MODEL = 'gpt-4-azure'; + const originalModel = 'test'; + client.azure = false; + client.setOptions(createOptions('test')); + expect(client.modelOptions.model).toBe(originalModel); + }); + + it('should not change model if AZURE_OPENAI_DEFAULT_MODEL is not set and model is passed', () => { + const originalModel = 'GROK-LLM'; + const options = createOptions(originalModel); + client.azure = options.azure; + client.setOptions(options); + expect(client.modelOptions.model).toBe(originalModel); + }); + + it('should change model if AZURE_OPENAI_DEFAULT_MODEL is set and model is passed', () => { + process.env.AZURE_OPENAI_DEFAULT_MODEL = 'gpt-4-azure'; + const originalModel = 'GROK-LLM'; + const options = createOptions(originalModel); + client.azure = options.azure; + client.setOptions(options); + expect(client.modelOptions.model).toBe(process.env.AZURE_OPENAI_DEFAULT_MODEL); + }); + + it('should include model in deployment name if AZURE_USE_MODEL_AS_DEPLOYMENT_NAME is set', () => { + process.env.AZURE_USE_MODEL_AS_DEPLOYMENT_NAME = 'true'; + const model = 'gpt-4-azure'; + + const AzureClient = new OpenAIClient('test-api-key', createOptions(model)); + + const expectedValue = `https://${azureOpenAIApiInstanceName}.openai.azure.com/openai/deployments/${model}/chat/completions?api-version=${azureOpenAIApiVersion}`; + + expect(AzureClient.modelOptions.model).toBe(model); + expect(AzureClient.azureEndpoint).toBe(expectedValue); + }); + + it('should include model in deployment name if AZURE_USE_MODEL_AS_DEPLOYMENT_NAME and default model is set', () => { + const defaultModel = 'gpt-4-azure'; + process.env.AZURE_USE_MODEL_AS_DEPLOYMENT_NAME = 'true'; + process.env.AZURE_OPENAI_DEFAULT_MODEL = defaultModel; + const model = 'gpt-4-this-is-a-test-model-name'; + + const AzureClient = new OpenAIClient('test-api-key', createOptions(model)); + + const expectedValue = `https://${azureOpenAIApiInstanceName}.openai.azure.com/openai/deployments/${model}/chat/completions?api-version=${azureOpenAIApiVersion}`; + + expect(AzureClient.modelOptions.model).toBe(defaultModel); + expect(AzureClient.azureEndpoint).toBe(expectedValue); + }); + + it('should not include model in deployment name if AZURE_USE_MODEL_AS_DEPLOYMENT_NAME is not set', () => { + const model = 'gpt-4-azure'; + + const AzureClient = new OpenAIClient('test-api-key', createOptions(model)); + + const expectedValue = `https://${azureOpenAIApiInstanceName}.openai.azure.com/openai/deployments/${azureOpenAIApiDeploymentName}/chat/completions?api-version=${azureOpenAIApiVersion}`; + + expect(AzureClient.modelOptions.model).toBe(model); + expect(AzureClient.azureEndpoint).toBe(expectedValue); + }); + }); + describe('selectTokenizer', () => { it('should get the correct tokenizer based on the instance state', () => { const tokenizer = client.selectTokenizer(); diff --git a/api/server/routes/endpoints/gptPlugins/initializeClient.js b/api/server/routes/endpoints/gptPlugins/initializeClient.js index f952f4feb5..2ab04ec098 100644 --- a/api/server/routes/endpoints/gptPlugins/initializeClient.js +++ b/api/server/routes/endpoints/gptPlugins/initializeClient.js @@ -1,6 +1,6 @@ const { PluginsClient } = require('../../../../app'); const { isEnabled } = require('../../../utils'); -const { getAzureCredentials, sanitizeModelName } = require('../../../../utils'); +const { getAzureCredentials } = require('../../../../utils'); const { getUserKey, checkUserKeyExpiry } = require('../../../services/UserService'); const initializeClient = async ({ req, res, endpointOption }) => { @@ -47,9 +47,6 @@ const initializeClient = async ({ req, res, endpointOption }) => { if (useAzure || (apiKey && apiKey.includes('azure') && !clientOptions.azure)) { clientOptions.azure = isUserProvided ? JSON.parse(userKey) : getAzureCredentials(); - clientOptions.azure.azureOpenAIApiDeploymentName = sanitizeModelName( - clientOptions.modelOptions.model, - ); apiKey = clientOptions.azure.azureOpenAIApiKey; } diff --git a/api/server/routes/endpoints/gptPlugins/initializeClient.spec.js b/api/server/routes/endpoints/gptPlugins/initializeClient.spec.js index 8ff7a2e5df..b8d76ced25 100644 --- a/api/server/routes/endpoints/gptPlugins/initializeClient.spec.js +++ b/api/server/routes/endpoints/gptPlugins/initializeClient.spec.js @@ -43,7 +43,12 @@ describe('gptPlugins/initializeClient', () => { test('should initialize PluginsClient with Azure credentials when PLUGINS_USE_AZURE is true', async () => { process.env.AZURE_API_KEY = 'test-azure-api-key'; - process.env.PLUGINS_USE_AZURE = 'true'; + (process.env.AZURE_OPENAI_API_INSTANCE_NAME = 'some-value'), + (process.env.AZURE_OPENAI_API_DEPLOYMENT_NAME = 'some-value'), + (process.env.AZURE_OPENAI_API_VERSION = 'some-value'), + (process.env.AZURE_OPENAI_API_COMPLETIONS_DEPLOYMENT_NAME = 'some-value'), + (process.env.AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME = 'some-value'), + (process.env.PLUGINS_USE_AZURE = 'true'); process.env.DEBUG_PLUGINS = 'false'; process.env.OPENAI_SUMMARIZE = 'false'; @@ -174,31 +179,6 @@ describe('gptPlugins/initializeClient', () => { ); }); - test('should sanitize model name for Azure when modelOptions is provided', async () => { - process.env.AZURE_API_KEY = 'azure-provided-api-key'; - process.env.PLUGINS_USE_AZURE = 'true'; - - const modelName = 'test-3.5-model'; - const sanitizedModelName = 'test-35-model'; - const req = { - body: { key: new Date(Date.now() + 10000).toISOString() }, - user: { id: '123' }, - }; - const res = {}; - const endpointOption = { modelOptions: { model: modelName } }; - - getUserKey.mockResolvedValue( - JSON.stringify({ - azureOpenAIApiKey: 'test-user-provided-azure-api-key', - azureOpenAIApiDeploymentName: modelName, - }), - ); - - const { azure } = await initializeClient({ req, res, endpointOption }); - - expect(azure.azureOpenAIApiDeploymentName).toBe(sanitizedModelName); - }); - test('should throw an error if the user-provided Azure key is invalid JSON', async () => { process.env.AZURE_API_KEY = 'user_provided'; process.env.PLUGINS_USE_AZURE = 'true'; diff --git a/api/server/routes/endpoints/openAI/initializeClient.js b/api/server/routes/endpoints/openAI/initializeClient.js index 3604198cf9..c84eb4050b 100644 --- a/api/server/routes/endpoints/openAI/initializeClient.js +++ b/api/server/routes/endpoints/openAI/initializeClient.js @@ -1,6 +1,6 @@ const { OpenAIClient } = require('../../../../app'); const { isEnabled } = require('../../../utils'); -const { getAzureCredentials, sanitizeModelName } = require('../../../../utils'); +const { getAzureCredentials } = require('../../../../utils'); const { getUserKey, checkUserKeyExpiry } = require('../../../services/UserService'); const initializeClient = async ({ req, res, endpointOption }) => { @@ -44,9 +44,6 @@ const initializeClient = async ({ req, res, endpointOption }) => { if (endpoint === 'azureOpenAI') { clientOptions.azure = isUserProvided ? JSON.parse(userKey) : getAzureCredentials(); - clientOptions.azure.azureOpenAIApiDeploymentName = sanitizeModelName( - clientOptions.modelOptions.model, - ); apiKey = clientOptions.azure.azureOpenAIApiKey; } diff --git a/api/server/routes/endpoints/openAI/initializeClient.spec.js b/api/server/routes/endpoints/openAI/initializeClient.spec.js index 213f42f071..731d42e06e 100644 --- a/api/server/routes/endpoints/openAI/initializeClient.spec.js +++ b/api/server/routes/endpoints/openAI/initializeClient.spec.js @@ -41,7 +41,12 @@ describe('initializeClient', () => { test('should initialize client with Azure credentials when endpoint is azureOpenAI', async () => { process.env.AZURE_API_KEY = 'test-azure-api-key'; - process.env.OPENAI_API_KEY = 'test-openai-api-key'; + (process.env.AZURE_OPENAI_API_INSTANCE_NAME = 'some-value'), + (process.env.AZURE_OPENAI_API_DEPLOYMENT_NAME = 'some-value'), + (process.env.AZURE_OPENAI_API_VERSION = 'some-value'), + (process.env.AZURE_OPENAI_API_COMPLETIONS_DEPLOYMENT_NAME = 'some-value'), + (process.env.AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME = 'some-value'), + (process.env.OPENAI_API_KEY = 'test-openai-api-key'); process.env.DEBUG_OPENAI = 'false'; process.env.OPENAI_SUMMARIZE = 'false'; @@ -191,21 +196,4 @@ describe('initializeClient', () => { /Your OpenAI API key has expired/, ); }); - - test('should sanitize model name for Azure when modelOptions is provided', async () => { - const modelName = 'test-3.5-model'; - const sanitizedModelName = 'test-35-model'; - const req = { - body: { key: new Date(Date.now() + 10000).toISOString(), endpoint: 'azureOpenAI' }, - user: { id: '123' }, - }; - const res = {}; - const endpointOption = { modelOptions: { model: modelName } }; - process.env.AZURE_API_KEY = 'azure-provided-api-key'; - getUserKey.mockResolvedValue('test-user-provided-openai-api-key'); - - const result = await initializeClient({ req, res, endpointOption }); - - expect(result.client.options.azure.azureOpenAIApiDeploymentName).toBe(sanitizedModelName); - }); }); diff --git a/api/utils/azureUtils.js b/api/utils/azureUtils.js index 13956199c6..3c4a891bec 100644 --- a/api/utils/azureUtils.js +++ b/api/utils/azureUtils.js @@ -6,6 +6,8 @@ * @property {string} azureOpenAIApiVersion - The Azure OpenAI API version. */ +const { isEnabled } = require('../server/utils'); + /** * Sanitizes the model name to be used in the URL by removing or replacing disallowed characters. * @param {string} modelName - The model name to be sanitized. @@ -44,7 +46,7 @@ const genAzureChatCompletion = ( ) => { // Determine the deployment segment of the URL based on provided modelName or azureOpenAIApiDeploymentName let deploymentSegment; - if (modelName) { + if (isEnabled(process.env.AZURE_USE_MODEL_AS_DEPLOYMENT_NAME) && modelName) { const sanitizedModelName = sanitizeModelName(modelName); deploymentSegment = `${sanitizedModelName}`; } else if (azureOpenAIApiDeploymentName) { diff --git a/docs/install/apis_and_tokens.md b/docs/install/apis_and_tokens.md index 2a7838a831..1007a6e2d2 100644 --- a/docs/install/apis_and_tokens.md +++ b/docs/install/apis_and_tokens.md @@ -84,6 +84,46 @@ https://{AZURE_OPENAI_API_INSTANCE_NAME}.openai.azure.com/openai/deployments/{AZ ``` You should also consider changing the `AZURE_OPENAI_MODELS` variable to the models available in your deployment. +#### Additional Configuration Notes + +- **Endpoint Construction**: The provided variables help customize the construction of the API URL for Azure. + +- **Model Deployment Naming**: As of 2023-11-10, the Azure API allows only one model per deployment. It's advisable to name your deployments after the model name (e.g., "gpt-3.5-turbo") for easy deployment switching. This is facilitated by setting `AZURE_USE_MODEL_AS_DEPLOYMENT_NAME` to `TRUE`. + +Alternatively, use custom deployment names and set `AZURE_OPENAI_DEFAULT_MODEL` for expected functionality. + +- **`AZURE_OPENAI_MODELS`**: List the available models, separated by commas without spaces. The first listed model will be the default. If left blank, internal settings will be used. Note that deployment names can't have periods, which are removed when generating the endpoint. + +Example use: + +```bash +# .env file +AZURE_OPENAI_MODELS=gpt-3.5-turbo,gpt-4,gpt-5 + +``` + +- **`AZURE_USE_MODEL_AS_DEPLOYMENT_NAME`**: Enable using the model name as the deployment name for the API URL. + +Example use: + +```bash +# .env file +AZURE_USE_MODEL_AS_DEPLOYMENT_NAME=TRUE + +``` + +Note: Azure API does not use the `model` in the payload and is more of an identifying field for the LibreChat App. If using non-model deployment names, but you're having issues with the model not being recognized, you should set this field. It will also not be used as the deployment name if AZURE_USE_MODEL_AS_DEPLOYMENT_NAME is enabled, which will prioritize what the user selects as the model. + +- **`AZURE_OPENAI_DEFAULT_MODEL`**: Override the model setting for Azure, useful if using custom deployment names. + +Example use: + +```bash +# .env file +AZURE_OPENAI_DEFAULT_MODEL=gpt-3.5-turbo # do include periods in the model name here + +``` + ### Optional Variables * `AZURE_OPENAI_API_COMPLETIONS_DEPLOYMENT_NAME`: The deployment name for completion. This is currently not in use but may be used in future.