mirror of
https://github.com/danny-avila/LibreChat.git
synced 2025-09-22 06:00:56 +02:00
🍞 fix: Minor fixes and improved Bun support (#1916)
* fix(bun): fix bun compatibility to allow gzip header: https://github.com/oven-sh/bun/issues/267#issuecomment-1854460357 * chore: update custom config examples * fix(OpenAIClient.chatCompletion): remove redundant call of stream.controller.abort() as `break` aborts the request and prevents abort errors when not called redundantly * chore: bump bun.lockb * fix: remove result-thinking class when message is no longer streaming * fix(bun): improve Bun support by forcing use of old method in bun env, also update old methods with new customizable params * fix(ci): pass tests
This commit is contained in:
Danny Avila
GitHub
parent
5d887492ea
commit
c37d5568bf
9 changed files with 175 additions and 59 deletions
|
|
@ -1,9 +1,16 @@
|
|||
const crypto = require('crypto');
|
||||
const Keyv = require('keyv');
|
||||
const crypto = require('crypto');
|
||||
const {
|
||||
EModelEndpoint,
|
||||
resolveHeaders,
|
||||
mapModelToAzureConfig,
|
||||
} = require('librechat-data-provider');
|
||||
const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken');
|
||||
const { fetchEventSource } = require('@waylaidwanderer/fetch-event-source');
|
||||
const { Agent, ProxyAgent } = require('undici');
|
||||
const BaseClient = require('./BaseClient');
|
||||
const { logger } = require('~/config');
|
||||
const { extractBaseURL, constructAzureURL, genAzureChatCompletion } = require('~/utils');
|
||||
|
||||
const CHATGPT_MODEL = 'gpt-3.5-turbo';
|
||||
const tokenizersCache = {};
|
||||
|
|
@ -144,7 +151,8 @@ class ChatGPTClient extends BaseClient {
|
|||
if (!abortController) {
|
||||
abortController = new AbortController();
|
||||
}
|
||||
const modelOptions = { ...this.modelOptions };
|
||||
|
||||
let modelOptions = { ...this.modelOptions };
|
||||
if (typeof onProgress === 'function') {
|
||||
modelOptions.stream = true;
|
||||
}
|
||||
|
|
@ -159,56 +167,171 @@ class ChatGPTClient extends BaseClient {
|
|||
}
|
||||
|
||||
const { debug } = this.options;
|
||||
const url = this.completionsUrl;
|
||||
let baseURL = this.completionsUrl;
|
||||
if (debug) {
|
||||
console.debug();
|
||||
console.debug(url);
|
||||
console.debug(baseURL);
|
||||
console.debug(modelOptions);
|
||||
console.debug();
|
||||
}
|
||||
|
||||
if (this.azure || this.options.azure) {
|
||||
// Azure does not accept `model` in the body, so we need to remove it.
|
||||
delete modelOptions.model;
|
||||
}
|
||||
|
||||
const opts = {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify(modelOptions),
|
||||
dispatcher: new Agent({
|
||||
bodyTimeout: 0,
|
||||
headersTimeout: 0,
|
||||
}),
|
||||
};
|
||||
|
||||
if (this.apiKey && this.options.azure) {
|
||||
opts.headers['api-key'] = this.apiKey;
|
||||
if (this.isVisionModel) {
|
||||
modelOptions.max_tokens = 4000;
|
||||
}
|
||||
|
||||
/** @type {TAzureConfig | undefined} */
|
||||
const azureConfig = this.options?.req?.app?.locals?.[EModelEndpoint.azureOpenAI];
|
||||
|
||||
const isAzure = this.azure || this.options.azure;
|
||||
if (
|
||||
(isAzure && this.isVisionModel && azureConfig) ||
|
||||
(azureConfig && this.isVisionModel && this.options.endpoint === EModelEndpoint.azureOpenAI)
|
||||
) {
|
||||
const { modelGroupMap, groupMap } = azureConfig;
|
||||
const {
|
||||
azureOptions,
|
||||
baseURL,
|
||||
headers = {},
|
||||
serverless,
|
||||
} = mapModelToAzureConfig({
|
||||
modelName: modelOptions.model,
|
||||
modelGroupMap,
|
||||
groupMap,
|
||||
});
|
||||
opts.headers = resolveHeaders(headers);
|
||||
this.langchainProxy = extractBaseURL(baseURL);
|
||||
this.apiKey = azureOptions.azureOpenAIApiKey;
|
||||
|
||||
const groupName = modelGroupMap[modelOptions.model].group;
|
||||
this.options.addParams = azureConfig.groupMap[groupName].addParams;
|
||||
this.options.dropParams = azureConfig.groupMap[groupName].dropParams;
|
||||
// Note: `forcePrompt` not re-assigned as only chat models are vision models
|
||||
|
||||
this.azure = !serverless && azureOptions;
|
||||
this.azureEndpoint =
|
||||
!serverless && genAzureChatCompletion(this.azure, modelOptions.model, this);
|
||||
}
|
||||
|
||||
if (this.options.headers) {
|
||||
opts.headers = { ...opts.headers, ...this.options.headers };
|
||||
}
|
||||
|
||||
if (isAzure) {
|
||||
// Azure does not accept `model` in the body, so we need to remove it.
|
||||
delete modelOptions.model;
|
||||
|
||||
baseURL = this.langchainProxy
|
||||
? constructAzureURL({
|
||||
baseURL: this.langchainProxy,
|
||||
azure: this.azure,
|
||||
})
|
||||
: this.azureEndpoint.split(/\/(chat|completion)/)[0];
|
||||
|
||||
if (this.options.forcePrompt) {
|
||||
baseURL += '/completions';
|
||||
} else {
|
||||
baseURL += '/chat/completions';
|
||||
}
|
||||
|
||||
opts.defaultQuery = { 'api-version': this.azure.azureOpenAIApiVersion };
|
||||
opts.headers = { ...opts.headers, 'api-key': this.apiKey };
|
||||
} else if (this.apiKey) {
|
||||
opts.headers.Authorization = `Bearer ${this.apiKey}`;
|
||||
}
|
||||
|
||||
if (process.env.OPENAI_ORGANIZATION) {
|
||||
opts.headers['OpenAI-Organization'] = process.env.OPENAI_ORGANIZATION;
|
||||
}
|
||||
|
||||
if (this.useOpenRouter) {
|
||||
opts.headers['HTTP-Referer'] = 'https://librechat.ai';
|
||||
opts.headers['X-Title'] = 'LibreChat';
|
||||
}
|
||||
|
||||
if (this.options.headers) {
|
||||
opts.headers = { ...opts.headers, ...this.options.headers };
|
||||
}
|
||||
|
||||
if (this.options.proxy) {
|
||||
opts.dispatcher = new ProxyAgent(this.options.proxy);
|
||||
}
|
||||
|
||||
/* hacky fixes for Mistral AI API:
|
||||
- Re-orders system message to the top of the messages payload, as not allowed anywhere else
|
||||
- If there is only one message and it's a system message, change the role to user
|
||||
*/
|
||||
if (baseURL.includes('https://api.mistral.ai/v1') && modelOptions.messages) {
|
||||
const { messages } = modelOptions;
|
||||
|
||||
const systemMessageIndex = messages.findIndex((msg) => msg.role === 'system');
|
||||
|
||||
if (systemMessageIndex > 0) {
|
||||
const [systemMessage] = messages.splice(systemMessageIndex, 1);
|
||||
messages.unshift(systemMessage);
|
||||
}
|
||||
|
||||
modelOptions.messages = messages;
|
||||
|
||||
if (messages.length === 1 && messages[0].role === 'system') {
|
||||
modelOptions.messages[0].role = 'user';
|
||||
}
|
||||
}
|
||||
|
||||
if (this.options.addParams && typeof this.options.addParams === 'object') {
|
||||
modelOptions = {
|
||||
...modelOptions,
|
||||
...this.options.addParams,
|
||||
};
|
||||
logger.debug('[ChatGPTClient] chatCompletion: added params', {
|
||||
addParams: this.options.addParams,
|
||||
modelOptions,
|
||||
});
|
||||
}
|
||||
|
||||
if (this.options.dropParams && Array.isArray(this.options.dropParams)) {
|
||||
this.options.dropParams.forEach((param) => {
|
||||
delete modelOptions[param];
|
||||
});
|
||||
logger.debug('[ChatGPTClient] chatCompletion: dropped params', {
|
||||
dropParams: this.options.dropParams,
|
||||
modelOptions,
|
||||
});
|
||||
}
|
||||
|
||||
if (baseURL.includes('v1') && !baseURL.includes('/completions') && !this.isChatCompletion) {
|
||||
baseURL = baseURL.split('v1')[0] + 'v1/completions';
|
||||
} else if (
|
||||
baseURL.includes('v1') &&
|
||||
!baseURL.includes('/chat/completions') &&
|
||||
this.isChatCompletion
|
||||
) {
|
||||
baseURL = baseURL.split('v1')[0] + 'v1/chat/completions';
|
||||
}
|
||||
|
||||
const BASE_URL = new URL(baseURL);
|
||||
if (opts.defaultQuery) {
|
||||
Object.entries(opts.defaultQuery).forEach(([key, value]) => {
|
||||
BASE_URL.searchParams.append(key, value);
|
||||
});
|
||||
delete opts.defaultQuery;
|
||||
}
|
||||
|
||||
const completionsURL = BASE_URL.toString();
|
||||
opts.body = JSON.stringify(modelOptions);
|
||||
|
||||
if (modelOptions.stream) {
|
||||
// eslint-disable-next-line no-async-promise-executor
|
||||
return new Promise(async (resolve, reject) => {
|
||||
try {
|
||||
let done = false;
|
||||
await fetchEventSource(url, {
|
||||
await fetchEventSource(completionsURL, {
|
||||
...opts,
|
||||
signal: abortController.signal,
|
||||
async onopen(response) {
|
||||
|
|
@ -236,7 +359,6 @@ class ChatGPTClient extends BaseClient {
|
|||
// workaround for private API not sending [DONE] event
|
||||
if (!done) {
|
||||
onProgress('[DONE]');
|
||||
abortController.abort();
|
||||
resolve();
|
||||
}
|
||||
},
|
||||
|
|
@ -249,14 +371,13 @@ class ChatGPTClient extends BaseClient {
|
|||
},
|
||||
onmessage(message) {
|
||||
if (debug) {
|
||||
// console.debug(message);
|
||||
console.debug(message);
|
||||
}
|
||||
if (!message.data || message.event === 'ping') {
|
||||
return;
|
||||
}
|
||||
if (message.data === '[DONE]') {
|
||||
onProgress('[DONE]');
|
||||
abortController.abort();
|
||||
resolve();
|
||||
done = true;
|
||||
return;
|
||||
|
|
@ -269,7 +390,7 @@ class ChatGPTClient extends BaseClient {
|
|||
}
|
||||
});
|
||||
}
|
||||
const response = await fetch(url, {
|
||||
const response = await fetch(completionsURL, {
|
||||
...opts,
|
||||
signal: abortController.signal,
|
||||
});
|
||||
|
|
|
|||
|
|
@ -560,7 +560,7 @@ class OpenAIClient extends BaseClient {
|
|||
let streamResult = null;
|
||||
this.modelOptions.user = this.user;
|
||||
const invalidBaseUrl = this.completionsUrl && extractBaseURL(this.completionsUrl) === null;
|
||||
const useOldMethod = !!(invalidBaseUrl || !this.isChatCompletion);
|
||||
const useOldMethod = !!(invalidBaseUrl || !this.isChatCompletion || typeof Bun !== 'undefined');
|
||||
if (typeof opts.onProgress === 'function' && useOldMethod) {
|
||||
await this.getCompletion(
|
||||
payload,
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ require('dotenv').config();
|
|||
const path = require('path');
|
||||
require('module-alias')({ base: path.resolve(__dirname, '..') });
|
||||
const cors = require('cors');
|
||||
const axios = require('axios');
|
||||
const express = require('express');
|
||||
const passport = require('passport');
|
||||
const mongoSanitize = require('express-mongo-sanitize');
|
||||
|
|
@ -22,6 +23,9 @@ const port = Number(PORT) || 3080;
|
|||
const host = HOST || 'localhost';
|
||||
|
||||
const startServer = async () => {
|
||||
if (typeof Bun !== 'undefined') {
|
||||
axios.defaults.headers.common['Accept-Encoding'] = 'gzip';
|
||||
}
|
||||
await connectDb();
|
||||
logger.info('Connected to MongoDB');
|
||||
await indexSync();
|
||||
|
|
|
|||
BIN
bun.lockb
BIN
bun.lockb
Binary file not shown.
|
|
@ -10,7 +10,7 @@ import rehypeHighlight from 'rehype-highlight';
|
|||
import type { TMessage } from 'librechat-data-provider';
|
||||
import type { PluggableList } from 'unified';
|
||||
import CodeBlock from '~/components/Messages/Content/CodeBlock';
|
||||
import { langSubset, validateIframe, processLaTeX } from '~/utils';
|
||||
import { cn, langSubset, validateIframe, processLaTeX } from '~/utils';
|
||||
import { useChatContext } from '~/Providers';
|
||||
import store from '~/store';
|
||||
|
||||
|
|
@ -75,7 +75,7 @@ const Markdown = memo(({ content, message, showCursor }: TContentProps) => {
|
|||
return (
|
||||
<div className="absolute">
|
||||
<p className="relative">
|
||||
<span className="result-thinking" />
|
||||
<span className={cn(isSubmitting ? 'result-thinking' : '')} />
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
|
|
|
|||
|
|
@ -35,7 +35,6 @@ Some of the endpoints are marked as **Known,** which means they might have speci
|
|||
]
|
||||
fetch: false
|
||||
titleConvo: true
|
||||
titleMethod: "completion"
|
||||
titleModel: "mixtral-8x7b-32768"
|
||||
modelDisplayLabel: "groq"
|
||||
iconURL: "https://raw.githubusercontent.com/fuegovic/lc-config-yaml/main/icons/groq.png"
|
||||
|
|
@ -64,7 +63,6 @@ Some of the endpoints are marked as **Known,** which means they might have speci
|
|||
default: ["mistral-tiny", "mistral-small", "mistral-medium", "mistral-large-latest"]
|
||||
fetch: true
|
||||
titleConvo: true
|
||||
titleMethod: "completion"
|
||||
titleModel: "mistral-tiny"
|
||||
modelDisplayLabel: "Mistral"
|
||||
# Drop Default params parameters from the request. See default params in guide linked below.
|
||||
|
|
@ -81,7 +79,7 @@ Some of the endpoints are marked as **Known,** which means they might have speci
|
|||
|
||||
- **Known:** icon provided, fetching list of models is recommended as API token rates and pricing used for token credit balances when models are fetched.
|
||||
|
||||
- API may be strict for some models, and may not allow fields like `stop`, in which case, you should use [`dropParams`.](./custom_config.md#dropparams)
|
||||
- It's recommended, and for some models required, to use [`dropParams`](./custom_config.md#dropparams) to drop the `stop` as Openrouter models use a variety of stop tokens.
|
||||
|
||||
- Known issue: you should not use `OPENROUTER_API_KEY` as it will then override the `openAI` endpoint to use OpenRouter as well.
|
||||
|
||||
|
|
@ -95,9 +93,10 @@ Some of the endpoints are marked as **Known,** which means they might have speci
|
|||
default: ["gpt-3.5-turbo"]
|
||||
fetch: true
|
||||
titleConvo: true
|
||||
titleMethod: "completion"
|
||||
titleModel: "gpt-3.5-turbo" # change to your preferred model
|
||||
modelDisplayLabel: "OpenRouter"
|
||||
# Recommended: Drop the stop parameter from the request as Openrouter models use a variety of stop tokens.
|
||||
dropParams: ["stop"]
|
||||
```
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -80,18 +80,18 @@ fileConfig:
|
|||
fileLimit: 5
|
||||
fileSizeLimit: 10 # Maximum size for an individual file in MB
|
||||
totalSizeLimit: 50 # Maximum total size for all files in a single request in MB
|
||||
supportedMimeTypes:
|
||||
- "image/.*"
|
||||
- "application/pdf"
|
||||
# supportedMimeTypes: # In case you wish to limit certain filetypes
|
||||
# - "image/.*"
|
||||
# - "application/pdf"
|
||||
openAI:
|
||||
disabled: true # Disables file uploading to the OpenAI endpoint
|
||||
default:
|
||||
totalSizeLimit: 20
|
||||
YourCustomEndpointName:
|
||||
fileLimit: 2
|
||||
fileSizeLimit: 5
|
||||
# YourCustomEndpointName: # Example for custom endpoints
|
||||
# fileLimit: 2
|
||||
# fileSizeLimit: 5
|
||||
serverFileSizeLimit: 100 # Global server file size limit in MB
|
||||
avatarSizeLimit: 2 # Limit for user avatar image size in MB
|
||||
avatarSizeLimit: 4 # Limit for user avatar image size in MB, default: 2 MB
|
||||
rateLimits:
|
||||
fileUploads:
|
||||
ipMax: 100
|
||||
|
|
@ -116,19 +116,15 @@ endpoints:
|
|||
apiKey: "${MISTRAL_API_KEY}"
|
||||
baseURL: "https://api.mistral.ai/v1"
|
||||
models:
|
||||
default: ["mistral-tiny", "mistral-small", "mistral-medium"]
|
||||
default: ["mistral-tiny", "mistral-small", "mistral-medium", "mistral-large-latest"]
|
||||
fetch: true # Attempt to dynamically fetch available models
|
||||
userIdQuery: false
|
||||
iconURL: "https://example.com/mistral-icon.png"
|
||||
titleConvo: true
|
||||
titleMethod: "completion"
|
||||
titleModel: "mistral-tiny"
|
||||
summarize: true
|
||||
summaryModel: "mistral-summary"
|
||||
forcePrompt: false
|
||||
modelDisplayLabel: "Mistral AI"
|
||||
addParams:
|
||||
safe_prompt: true
|
||||
# addParams:
|
||||
# safe_prompt: true # Mistral specific value for moderating messages
|
||||
dropParams:
|
||||
- "stop"
|
||||
- "user"
|
||||
|
|
@ -144,10 +140,9 @@ endpoints:
|
|||
fetch: false
|
||||
titleConvo: true
|
||||
titleModel: "gpt-3.5-turbo"
|
||||
summarize: false
|
||||
forcePrompt: false
|
||||
modelDisplayLabel: "OpenRouter"
|
||||
dropParams:
|
||||
- "stop"
|
||||
- "frequency_penalty"
|
||||
```
|
||||
|
||||
|
|
@ -521,15 +516,12 @@ endpoints:
|
|||
apiKey: "${YOUR_ENV_VAR_KEY}"
|
||||
baseURL: "https://api.mistral.ai/v1"
|
||||
models:
|
||||
default: ["mistral-tiny", "mistral-small", "mistral-medium"]
|
||||
default: ["mistral-tiny", "mistral-small", "mistral-medium", "mistral-large-latest"]
|
||||
titleConvo: true
|
||||
titleModel: "mistral-tiny"
|
||||
summarize: false
|
||||
summaryModel: "mistral-tiny"
|
||||
forcePrompt: false
|
||||
modelDisplayLabel: "Mistral"
|
||||
addParams:
|
||||
safe_prompt: true
|
||||
# addParams:
|
||||
# safe_prompt: true # Mistral specific value for moderating messages
|
||||
# NOTE: For Mistral, it is necessary to drop the following parameters or you will encounter a 422 Error:
|
||||
dropParams: ["stop", "user", "frequency_penalty", "presence_penalty"]
|
||||
```
|
||||
|
|
|
|||
|
|
@ -68,26 +68,26 @@ endpoints:
|
|||
titleConvo: true # Set to true to enable title conversation
|
||||
|
||||
# Title Method: Choose between "completion" or "functions".
|
||||
titleMethod: "completion" # Defaults to "completion" if omitted.
|
||||
# titleMethod: "completion" # Defaults to "completion" if omitted.
|
||||
|
||||
# Title Model: Specify the model to use for titles.
|
||||
titleModel: "mistral-tiny" # Defaults to "gpt-3.5-turbo" if omitted.
|
||||
|
||||
# Summarize setting: Set to true to enable summarization.
|
||||
summarize: false
|
||||
# summarize: false
|
||||
|
||||
# Summary Model: Specify the model to use if summarization is enabled.
|
||||
summaryModel: "mistral-tiny" # Defaults to "gpt-3.5-turbo" if omitted.
|
||||
# summaryModel: "mistral-tiny" # Defaults to "gpt-3.5-turbo" if omitted.
|
||||
|
||||
# Force Prompt setting: If true, sends a `prompt` parameter instead of `messages`.
|
||||
forcePrompt: false
|
||||
# forcePrompt: false
|
||||
|
||||
# The label displayed for the AI model in messages.
|
||||
modelDisplayLabel: "Mistral" # Default is "AI" when not set.
|
||||
|
||||
# Add additional parameters to the request. Default params will be overwritten.
|
||||
addParams:
|
||||
safe_prompt: true # This field is specific to Mistral AI: https://docs.mistral.ai/api/
|
||||
# addParams:
|
||||
# safe_prompt: true # This field is specific to Mistral AI: https://docs.mistral.ai/api/
|
||||
|
||||
# Drop Default params parameters from the request. See default params in guide linked below.
|
||||
# NOTE: For Mistral, it is necessary to drop the following parameters or you will encounter a 422 Error:
|
||||
|
|
@ -105,9 +105,8 @@ endpoints:
|
|||
fetch: true
|
||||
titleConvo: true
|
||||
titleModel: "gpt-3.5-turbo"
|
||||
summarize: false
|
||||
summaryModel: "gpt-3.5-turbo"
|
||||
forcePrompt: false
|
||||
# Recommended: Drop the stop parameter from the request as Openrouter models use a variety of stop tokens.
|
||||
dropParams: ["stop"]
|
||||
modelDisplayLabel: "OpenRouter"
|
||||
|
||||
# See the Custom Configuration Guide for more information:
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@
|
|||
"lint": "eslint \"{,!(node_modules)/**/}*.{js,jsx,ts,tsx}\"",
|
||||
"format": "prettier-eslint --write \"{,!(node_modules)/**/}*.{js,jsx,ts,tsx}\"",
|
||||
"b:api": "NODE_ENV=production bun run api/server/index.js",
|
||||
"b:api-inspect": "NODE_ENV=production bun --inspect run api/server/index.js",
|
||||
"b:api:dev": "NODE_ENV=production bun run --watch api/server/index.js",
|
||||
"b:data": "cd packages/data-provider && bun run b:build",
|
||||
"b:client": "bun --bun run b:data && cd client && bun --bun run b:build",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue