From 5ab9802aa912ef2e0d5768e93ee2d943c8de9f51 Mon Sep 17 00:00:00 2001 From: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Thu, 9 Nov 2023 14:04:36 -0500 Subject: [PATCH] fix(OpenAIClient): use official SDK to identify client and avoid false Rate Limit Error (#1161) * chore: add eslint ignore unused var pattern * feat: add extractBaseURL helper for valid OpenAI reverse proxies, with tests * feat(OpenAIClient): add new chatCompletion using official OpenAI node SDK * fix(ci): revert change to FORCE_PROMPT condition --- .eslintrc.js | 1 + api/app/clients/OpenAIClient.js | 160 +++++++++++++++++- api/app/clients/PluginsClient.js | 3 +- api/app/clients/specs/OpenAIClient.test.js | 2 +- .../clients/tools/util/handleOpenAIErrors.js | 30 ++++ api/app/clients/tools/util/index.js | 2 + api/server/services/ModelService.js | 3 +- api/utils/extractBaseURL.js | 48 ++++++ api/utils/extractBaseURL.spec.js | 56 ++++++ api/utils/index.js | 4 +- 10 files changed, 297 insertions(+), 12 deletions(-) create mode 100644 api/app/clients/tools/util/handleOpenAIErrors.js create mode 100644 api/utils/extractBaseURL.js create mode 100644 api/utils/extractBaseURL.spec.js diff --git a/.eslintrc.js b/.eslintrc.js index 9e7858375e..53277e02c7 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -61,6 +61,7 @@ module.exports = { 'no-restricted-syntax': 'off', 'react/prop-types': ['off'], 'react/display-name': ['off'], + 'no-unused-vars': ['error', { varsIgnorePattern: '^_' }], quotes: ['error', 'single'], }, overrides: [ diff --git a/api/app/clients/OpenAIClient.js b/api/app/clients/OpenAIClient.js index ccb3646e8f..b16c070dc1 100644 --- a/api/app/clients/OpenAIClient.js +++ b/api/app/clients/OpenAIClient.js @@ -1,15 +1,17 @@ -const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken'); +const OpenAI = require('openai'); const { HttpsProxyAgent } = require('https-proxy-agent'); -const ChatGPTClient = require('./ChatGPTClient'); -const BaseClient = require('./BaseClient'); -const { getModelMaxTokens, genAzureChatCompletion } = require('../../utils'); +const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken'); +const { getModelMaxTokens, genAzureChatCompletion, extractBaseURL } = require('../../utils'); const { truncateText, formatMessage, CUT_OFF_PROMPT } = require('./prompts'); const spendTokens = require('../../models/spendTokens'); +const { handleOpenAIErrors } = require('./tools/util'); const { isEnabled } = require('../../server/utils'); const { createLLM, RunManager } = require('./llm'); +const ChatGPTClient = require('./ChatGPTClient'); const { summaryBuffer } = require('./memory'); const { runTitleChain } = require('./chains'); const { tokenSplit } = require('./document'); +const BaseClient = require('./BaseClient'); // Cache to store Tiktoken instances const tokenizersCache = {}; @@ -74,7 +76,7 @@ class OpenAIClient extends BaseClient { } const { OPENROUTER_API_KEY, OPENAI_FORCE_PROMPT } = process.env ?? {}; - if (OPENROUTER_API_KEY) { + if (OPENROUTER_API_KEY && !this.azure) { this.apiKey = OPENROUTER_API_KEY; this.useOpenRouter = true; } @@ -88,7 +90,11 @@ class OpenAIClient extends BaseClient { this.isChatCompletion = this.useOpenRouter || !!reverseProxy || model.includes('gpt-'); this.isChatGptModel = this.isChatCompletion; - if (model.includes('text-davinci-003') || model.includes('instruct') || this.FORCE_PROMPT) { + if ( + model.includes('text-davinci') || + model.includes('gpt-3.5-turbo-instruct') || + this.FORCE_PROMPT + ) { this.isChatCompletion = false; this.isChatGptModel = false; } @@ -134,7 +140,7 @@ class OpenAIClient extends BaseClient { if (reverseProxy) { this.completionsUrl = reverseProxy; - this.langchainProxy = reverseProxy.match(/.*v1/)?.[0]; + this.langchainProxy = extractBaseURL(reverseProxy); !this.langchainProxy && console.warn(`The reverse proxy URL ${reverseProxy} is not valid for Plugins. The url must follow OpenAI specs, for example: https://localhost:8080/v1/chat/completions @@ -356,7 +362,9 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s let result = null; let streamResult = null; this.modelOptions.user = this.user; - if (typeof opts.onProgress === 'function') { + const invalidBaseUrl = this.completionsUrl && extractBaseURL(this.completionsUrl) === null; + const useOldMethod = !!(this.azure || invalidBaseUrl); + if (typeof opts.onProgress === 'function' && useOldMethod) { await this.getCompletion( payload, (progressMessage) => { @@ -399,6 +407,13 @@ If your reverse proxy is compatible to OpenAI specs in every other way, it may s }, opts.abortController || new AbortController(), ); + } else if (typeof opts.onProgress === 'function') { + reply = await this.chatCompletion({ + payload, + clientOptions: opts, + onProgress: opts.onProgress, + abortController: opts.abortController, + }); } else { result = await this.getCompletion( payload, @@ -669,6 +684,135 @@ ${convo} content: response.text, }); } + + async chatCompletion({ payload, onProgress, clientOptions, abortController = null }) { + let error = null; + const errorCallback = (err) => (error = err); + let intermediateReply = ''; + try { + if (!abortController) { + abortController = new AbortController(); + } + const modelOptions = { ...this.modelOptions }; + if (typeof onProgress === 'function') { + modelOptions.stream = true; + } + if (this.isChatGptModel) { + modelOptions.messages = payload; + } else { + modelOptions.prompt = payload; + } + + const { debug } = this.options; + const url = extractBaseURL(this.completionsUrl); + if (debug) { + console.debug('baseURL', url); + console.debug('modelOptions', modelOptions); + } + const opts = { + baseURL: url, + }; + + if (this.useOpenRouter) { + opts.defaultHeaders = { + 'HTTP-Referer': 'https://librechat.ai', + 'X-Title': 'LibreChat', + }; + } + + if (this.options.headers) { + opts.defaultHeaders = { ...opts.defaultHeaders, ...this.options.headers }; + } + + if (this.options.proxy) { + opts.httpAgent = new HttpsProxyAgent(this.options.proxy); + } + + let chatCompletion; + const openai = new OpenAI({ + apiKey: this.apiKey, + ...opts, + }); + + if (modelOptions.stream) { + const stream = await openai.beta.chat.completions + .stream({ + ...modelOptions, + stream: true, + }) + .on('abort', () => { + /* Do nothing here */ + }) + .on('error', (err) => { + handleOpenAIErrors(err, errorCallback, 'stream'); + }); + + for await (const chunk of stream) { + const token = chunk.choices[0]?.delta?.content || ''; + intermediateReply += token; + onProgress(token); + if (abortController.signal.aborted) { + stream.controller.abort(); + break; + } + } + + chatCompletion = await stream.finalChatCompletion().catch((err) => { + handleOpenAIErrors(err, errorCallback, 'finalChatCompletion'); + }); + } + // regular completion + else { + chatCompletion = await openai.chat.completions + .create({ + ...modelOptions, + }) + .catch((err) => { + handleOpenAIErrors(err, errorCallback, 'create'); + }); + } + + if (!chatCompletion && error) { + throw new Error(error); + } else if (!chatCompletion) { + throw new Error('Chat completion failed'); + } + + const { message, finish_reason } = chatCompletion.choices[0]; + if (chatCompletion && typeof clientOptions.addMetadata === 'function') { + clientOptions.addMetadata({ finish_reason }); + } + + return message.content; + } catch (err) { + if ( + err?.message?.includes('abort') || + (err instanceof OpenAI.APIError && err?.message?.includes('abort')) + ) { + return ''; + } + if ( + err?.message?.includes('missing finish_reason') || + (err instanceof OpenAI.OpenAIError && err?.message?.includes('missing finish_reason')) + ) { + await abortController.abortCompletion(); + return intermediateReply; + } else if (err instanceof OpenAI.APIError) { + console.log(err.name); + console.log(err.status); + console.log(err.headers); + if (intermediateReply) { + return intermediateReply; + } else { + throw err; + } + } else { + console.warn('[OpenAIClient.chatCompletion] Unhandled error type'); + console.error(err); + throw err; + } + } + } } module.exports = OpenAIClient; diff --git a/api/app/clients/PluginsClient.js b/api/app/clients/PluginsClient.js index 5e9bde1435..801219c842 100644 --- a/api/app/clients/PluginsClient.js +++ b/api/app/clients/PluginsClient.js @@ -6,6 +6,7 @@ const { addImages, buildErrorInput, buildPromptPrefix } = require('./output_pars const checkBalance = require('../../models/checkBalance'); const { formatLangChainMessages } = require('./prompts'); const { isEnabled } = require('../../server/utils'); +const { extractBaseURL } = require('../../utils'); const { SelfReflectionTool } = require('./tools'); const { loadTools } = require('./tools/util'); @@ -34,7 +35,7 @@ class PluginsClient extends OpenAIClient { this.isGpt3 = this.modelOptions?.model?.includes('gpt-3'); if (this.options.reverseProxyUrl) { - this.langchainProxy = this.options.reverseProxyUrl.match(/.*v1/)?.[0]; + this.langchainProxy = extractBaseURL(this.options.reverseProxyUrl); !this.langchainProxy && console.warn(`The reverse proxy URL ${this.options.reverseProxyUrl} is not valid for Plugins. The url must follow OpenAI specs, for example: https://localhost:8080/v1/chat/completions diff --git a/api/app/clients/specs/OpenAIClient.test.js b/api/app/clients/specs/OpenAIClient.test.js index 7257aec36a..6c54a6b09c 100644 --- a/api/app/clients/specs/OpenAIClient.test.js +++ b/api/app/clients/specs/OpenAIClient.test.js @@ -86,7 +86,7 @@ describe('OpenAIClient', () => { client.setOptions({ reverseProxyUrl: 'https://example.com/completions' }); expect(client.completionsUrl).toBe('https://example.com/completions'); - expect(client.langchainProxy).toBeUndefined(); + expect(client.langchainProxy).toBe(null); }); }); diff --git a/api/app/clients/tools/util/handleOpenAIErrors.js b/api/app/clients/tools/util/handleOpenAIErrors.js new file mode 100644 index 0000000000..b5a31f7f40 --- /dev/null +++ b/api/app/clients/tools/util/handleOpenAIErrors.js @@ -0,0 +1,30 @@ +const OpenAI = require('openai'); + +/** + * Handles errors that may occur when making requests to OpenAI's API. + * It checks the instance of the error and prints a specific warning message + * to the console depending on the type of error encountered. + * It then calls an optional error callback function with the error object. + * + * @param {Error} err - The error object thrown by OpenAI API. + * @param {Function} errorCallback - A callback function that is called with the error object. + * @param {string} [context='stream'] - A string providing context where the error occurred, defaults to 'stream'. + */ +async function handleOpenAIErrors(err, errorCallback, context = 'stream') { + if (err instanceof OpenAI.APIError && err?.message?.includes('abort')) { + console.warn(`[OpenAIClient.chatCompletion][${context}] Aborted Message`); + } + if (err instanceof OpenAI.OpenAIError && err?.message?.includes('missing finish_reason')) { + console.warn(`[OpenAIClient.chatCompletion][${context}] Missing finish_reason`); + } else if (err instanceof OpenAI.APIError) { + console.warn(`[OpenAIClient.chatCompletion][${context}] API Error`); + } else { + console.warn(`[OpenAIClient.chatCompletion][${context}] Unhandled error type`); + } + + if (errorCallback) { + errorCallback(err); + } +} + +module.exports = handleOpenAIErrors; diff --git a/api/app/clients/tools/util/index.js b/api/app/clients/tools/util/index.js index 9c96fb50f3..ea67bb4ced 100644 --- a/api/app/clients/tools/util/index.js +++ b/api/app/clients/tools/util/index.js @@ -1,6 +1,8 @@ const { validateTools, loadTools } = require('./handleTools'); +const handleOpenAIErrors = require('./handleOpenAIErrors'); module.exports = { + handleOpenAIErrors, validateTools, loadTools, }; diff --git a/api/server/services/ModelService.js b/api/server/services/ModelService.js index 893e348e03..4dbc780693 100644 --- a/api/server/services/ModelService.js +++ b/api/server/services/ModelService.js @@ -1,6 +1,7 @@ const Keyv = require('keyv'); const axios = require('axios'); const { isEnabled } = require('../utils'); +const { extractBaseURL } = require('../../utils'); const keyvRedis = require('../../cache/keyvRedis'); // const { getAzureCredentials, genAzureChatCompletion } = require('../../utils/'); const { openAIApiKey, userProvidedOpenAI } = require('./EndpointService').config; @@ -30,7 +31,7 @@ const fetchOpenAIModels = async (opts = { azure: false, plugins: false }, _model } if (reverseProxyUrl) { - basePath = reverseProxyUrl.match(/.*v1/)?.[0]; + basePath = extractBaseURL(reverseProxyUrl); } const cachedModels = await modelsCache.get(basePath); diff --git a/api/utils/extractBaseURL.js b/api/utils/extractBaseURL.js new file mode 100644 index 0000000000..83565a4caf --- /dev/null +++ b/api/utils/extractBaseURL.js @@ -0,0 +1,48 @@ +/** + * Extracts a valid OpenAI baseURL from a given string, matching "url/v1," also an added suffix, + * ending with "/openai" (to allow the Cloudflare, LiteLLM pattern). + * + * Examples: + * - `https://open.ai/v1/chat` -> `https://open.ai/v1` + * - `https://open.ai/v1/chat/completions` -> `https://open.ai/v1` + * - `https://open.ai/v1/ACCOUNT/GATEWAY/openai/completions` -> `https://open.ai/v1/ACCOUNT/GATEWAY/openai` + * - `https://open.ai/v1/hi/openai` -> `https://open.ai/v1/hi/openai` + * + * @param {string} url - The URL to be processed. + * @returns {string|null} The matched pattern or null if no match is found. + */ +function extractBaseURL(url) { + // First, let's make sure the URL contains '/v1'. + if (!url.includes('/v1')) { + return null; + } + + // Find the index of '/v1' to use it as a reference point. + const v1Index = url.indexOf('/v1'); + + // Extract the part of the URL up to and including '/v1'. + let baseUrl = url.substring(0, v1Index + 3); + + // Check if the URL has '/openai' immediately after '/v1'. + const openaiIndex = url.indexOf('/openai', v1Index + 3); + + // If '/openai' is found right after '/v1', include it in the base URL. + if (openaiIndex === v1Index + 3) { + // Find the next slash or the end of the URL after '/openai'. + const nextSlashIndex = url.indexOf('/', openaiIndex + 7); + if (nextSlashIndex === -1) { + // If there is no next slash, the rest of the URL is the base URL. + baseUrl = url.substring(0, openaiIndex + 7); + } else { + // If there is a next slash, the base URL goes up to but not including the slash. + baseUrl = url.substring(0, nextSlashIndex); + } + } else if (openaiIndex > 0) { + // If '/openai' is present but not immediately after '/v1', we need to include the reverse proxy pattern. + baseUrl = url.substring(0, openaiIndex + 7); + } + + return baseUrl; +} + +module.exports = extractBaseURL; // Export the function for use in your test file. diff --git a/api/utils/extractBaseURL.spec.js b/api/utils/extractBaseURL.spec.js new file mode 100644 index 0000000000..1403ac9968 --- /dev/null +++ b/api/utils/extractBaseURL.spec.js @@ -0,0 +1,56 @@ +const extractBaseURL = require('./extractBaseURL'); + +describe('extractBaseURL', () => { + test('should extract base URL up to /v1 for standard endpoints', () => { + const url = 'https://localhost:8080/v1/chat/completions'; + expect(extractBaseURL(url)).toBe('https://localhost:8080/v1'); + }); + + test('should include /openai in the extracted URL when present', () => { + const url = 'https://localhost:8080/v1/openai'; + expect(extractBaseURL(url)).toBe('https://localhost:8080/v1/openai'); + }); + + test('should stop at /openai and not include any additional paths', () => { + const url = 'https://fake.open.ai/v1/openai/you-are-cool'; + expect(extractBaseURL(url)).toBe('https://fake.open.ai/v1/openai'); + }); + + test('should return the correct base URL for official openai endpoints', () => { + const url = 'https://api.openai.com/v1/chat/completions'; + expect(extractBaseURL(url)).toBe('https://api.openai.com/v1'); + }); + + test('should handle URLs with reverse proxy pattern correctly', () => { + const url = 'https://gateway.ai.cloudflare.com/v1/ACCOUNT_TAG/GATEWAY/openai/completions'; + expect(extractBaseURL(url)).toBe( + 'https://gateway.ai.cloudflare.com/v1/ACCOUNT_TAG/GATEWAY/openai', + ); + }); + + test('should return null if the URL does not match the expected pattern', () => { + const url = 'https://someotherdomain.com/notv1'; + expect(extractBaseURL(url)).toBeNull(); + }); + + // Test our JSDoc examples. + test('should extract base URL up to /v1 for open.ai standard endpoint', () => { + const url = 'https://open.ai/v1/chat'; + expect(extractBaseURL(url)).toBe('https://open.ai/v1'); + }); + + test('should extract base URL up to /v1 for open.ai standard endpoint with additional path', () => { + const url = 'https://open.ai/v1/chat/completions'; + expect(extractBaseURL(url)).toBe('https://open.ai/v1'); + }); + + test('should handle URLs with ACCOUNT/GATEWAY pattern followed by /openai', () => { + const url = 'https://open.ai/v1/ACCOUNT/GATEWAY/openai/completions'; + expect(extractBaseURL(url)).toBe('https://open.ai/v1/ACCOUNT/GATEWAY/openai'); + }); + + test('should include /openai in the extracted URL with additional segments', () => { + const url = 'https://open.ai/v1/hi/openai'; + expect(extractBaseURL(url)).toBe('https://open.ai/v1/hi/openai'); + }); +}); diff --git a/api/utils/index.js b/api/utils/index.js index 3e1c4d0c44..f9194858e8 100644 --- a/api/utils/index.js +++ b/api/utils/index.js @@ -1,9 +1,11 @@ -const azureUtils = require('./azureUtils'); const tokenHelpers = require('./tokens'); +const azureUtils = require('./azureUtils'); +const extractBaseURL = require('./extractBaseURL'); const findMessageContent = require('./findMessageContent'); module.exports = { ...azureUtils, ...tokenHelpers, + extractBaseURL, findMessageContent, };