Merge branch 'main' into feat/user-groups

This commit is contained in:
Ruben Talstra 2025-02-22 10:08:12 +01:00 committed by GitHub
commit 0cdccb617c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
34 changed files with 1016 additions and 369 deletions

View file

@ -51,7 +51,7 @@ class GoogleClient extends BaseClient {
const serviceKey = creds[AuthKeys.GOOGLE_SERVICE_KEY] ?? {};
this.serviceKey =
serviceKey && typeof serviceKey === 'string' ? JSON.parse(serviceKey) : serviceKey ?? {};
serviceKey && typeof serviceKey === 'string' ? JSON.parse(serviceKey) : (serviceKey ?? {});
/** @type {string | null | undefined} */
this.project_id = this.serviceKey.project_id;
this.client_email = this.serviceKey.client_email;
@ -73,6 +73,8 @@ class GoogleClient extends BaseClient {
* @type {string} */
this.outputTokensKey = 'output_tokens';
this.visionMode = VisionModes.generative;
/** @type {string} */
this.systemMessage;
if (options.skipSetOptions) {
return;
}
@ -184,7 +186,7 @@ class GoogleClient extends BaseClient {
if (typeof this.options.artifactsPrompt === 'string' && this.options.artifactsPrompt) {
promptPrefix = `${promptPrefix ?? ''}\n${this.options.artifactsPrompt}`.trim();
}
this.options.promptPrefix = promptPrefix;
this.systemMessage = promptPrefix;
this.initializeClient();
return this;
}
@ -314,7 +316,7 @@ class GoogleClient extends BaseClient {
}
this.augmentedPrompt = await this.contextHandlers.createContext();
this.options.promptPrefix = this.augmentedPrompt + this.options.promptPrefix;
this.systemMessage = this.augmentedPrompt + this.systemMessage;
}
}
@ -361,8 +363,8 @@ class GoogleClient extends BaseClient {
throw new Error('[GoogleClient] PaLM 2 and Codey models are no longer supported.');
}
if (this.options.promptPrefix) {
const instructionsTokenCount = this.getTokenCount(this.options.promptPrefix);
if (this.systemMessage) {
const instructionsTokenCount = this.getTokenCount(this.systemMessage);
this.maxContextTokens = this.maxContextTokens - instructionsTokenCount;
if (this.maxContextTokens < 0) {
@ -417,8 +419,8 @@ class GoogleClient extends BaseClient {
],
};
if (this.options.promptPrefix) {
payload.instances[0].context = this.options.promptPrefix;
if (this.systemMessage) {
payload.instances[0].context = this.systemMessage;
}
logger.debug('[GoogleClient] buildMessages', payload);
@ -464,7 +466,7 @@ class GoogleClient extends BaseClient {
identityPrefix = `${identityPrefix}\nYou are ${this.options.modelLabel}`;
}
let promptPrefix = (this.options.promptPrefix ?? '').trim();
let promptPrefix = (this.systemMessage ?? '').trim();
if (identityPrefix) {
promptPrefix = `${identityPrefix}${promptPrefix}`;
@ -639,7 +641,7 @@ class GoogleClient extends BaseClient {
let error;
try {
if (!EXCLUDED_GENAI_MODELS.test(modelName) && !this.project_id) {
/** @type {GenAI} */
/** @type {GenerativeModel} */
const client = this.client;
/** @type {GenerateContentRequest} */
const requestOptions = {
@ -648,7 +650,7 @@ class GoogleClient extends BaseClient {
generationConfig: googleGenConfigSchema.parse(this.modelOptions),
};
const promptPrefix = (this.options.promptPrefix ?? '').trim();
const promptPrefix = (this.systemMessage ?? '').trim();
if (promptPrefix.length) {
requestOptions.systemInstruction = {
parts: [
@ -663,7 +665,17 @@ class GoogleClient extends BaseClient {
/** @type {GenAIUsageMetadata} */
let usageMetadata;
const result = await client.generateContentStream(requestOptions);
abortController.signal.addEventListener(
'abort',
() => {
logger.warn('[GoogleClient] Request was aborted', abortController.signal.reason);
},
{ once: true },
);
const result = await client.generateContentStream(requestOptions, {
signal: abortController.signal,
});
for await (const chunk of result.stream) {
usageMetadata = !usageMetadata
? chunk?.usageMetadata

View file

@ -1,15 +1,81 @@
const fs = require('fs');
const ioredis = require('ioredis');
const KeyvRedis = require('@keyv/redis');
const { isEnabled } = require('~/server/utils');
const logger = require('~/config/winston');
const { REDIS_URI, USE_REDIS } = process.env;
const { REDIS_URI, USE_REDIS, USE_REDIS_CLUSTER, REDIS_CA, REDIS_KEY_PREFIX, REDIS_MAX_LISTENERS } =
process.env;
let keyvRedis;
const redis_prefix = REDIS_KEY_PREFIX || '';
const redis_max_listeners = REDIS_MAX_LISTENERS || 10;
function mapURI(uri) {
const regex =
/^(?:(?<scheme>\w+):\/\/)?(?:(?<user>[^:@]+)(?::(?<password>[^@]+))?@)?(?<host>[\w.-]+)(?::(?<port>\d{1,5}))?$/;
const match = uri.match(regex);
if (match) {
const { scheme, user, password, host, port } = match.groups;
return {
scheme: scheme || 'none',
user: user || null,
password: password || null,
host: host || null,
port: port || null,
};
} else {
const parts = uri.split(':');
if (parts.length === 2) {
return {
scheme: 'none',
user: null,
password: null,
host: parts[0],
port: parts[1],
};
}
return {
scheme: 'none',
user: null,
password: null,
host: uri,
port: null,
};
}
}
if (REDIS_URI && isEnabled(USE_REDIS)) {
keyvRedis = new KeyvRedis(REDIS_URI, { useRedisSets: false });
let redisOptions = null;
let keyvOpts = {
useRedisSets: false,
keyPrefix: redis_prefix,
};
if (REDIS_CA) {
const ca = fs.readFileSync(REDIS_CA);
redisOptions = { tls: { ca } };
}
if (isEnabled(USE_REDIS_CLUSTER)) {
const hosts = REDIS_URI.split(',').map((item) => {
var value = mapURI(item);
return {
host: value.host,
port: value.port,
};
});
const cluster = new ioredis.Cluster(hosts, { redisOptions });
keyvRedis = new KeyvRedis(cluster, keyvOpts);
} else {
keyvRedis = new KeyvRedis(REDIS_URI, keyvOpts);
}
keyvRedis.on('error', (err) => logger.error('KeyvRedis connection error:', err));
keyvRedis.setMaxListeners(20);
keyvRedis.setMaxListeners(redis_max_listeners);
logger.info(
'[Optional] Redis initialized. Note: Redis support is experimental. If you have issues, disable it. Cache needs to be flushed for values to refresh.',
);

View file

@ -1,9 +1,11 @@
const { MeiliSearch } = require('meilisearch');
const Conversation = require('~/models/schema/convoSchema');
const Message = require('~/models/schema/messageSchema');
const { isEnabled } = require('~/server/utils');
const { logger } = require('~/config');
const searchEnabled = process.env?.SEARCH?.toLowerCase() === 'true';
const searchEnabled = isEnabled(process.env.SEARCH);
const indexingDisabled = isEnabled(process.env.MEILI_NO_SYNC);
let currentTimeout = null;
class MeiliSearchClient {
@ -23,8 +25,7 @@ class MeiliSearchClient {
}
}
// eslint-disable-next-line no-unused-vars
async function indexSync(req, res, next) {
async function indexSync() {
if (!searchEnabled) {
return;
}
@ -33,10 +34,15 @@ async function indexSync(req, res, next) {
const client = MeiliSearchClient.getInstance();
const { status } = await client.health();
if (status !== 'available' || !process.env.SEARCH) {
if (status !== 'available') {
throw new Error('Meilisearch not available');
}
if (indexingDisabled === true) {
logger.info('[indexSync] Indexing is disabled, skipping...');
return;
}
const messageCount = await Message.countDocuments();
const convoCount = await Conversation.countDocuments();
const messages = await client.index('messages').getStats();
@ -71,7 +77,6 @@ async function indexSync(req, res, next) {
logger.info('[indexSync] Meilisearch not configured, search will be disabled.');
} else {
logger.error('[indexSync] error', err);
// res.status(500).json({ error: 'Server error' });
}
}
}

View file

@ -3,40 +3,40 @@ const { logger } = require('~/config');
const options = [
{
label: 'idea',
value: 'com_ui_idea',
label: 'com_ui_idea',
value: 'idea',
},
{
label: 'travel',
value: 'com_ui_travel',
label: 'com_ui_travel',
value: 'travel',
},
{
label: 'teach_or_explain',
value: 'com_ui_teach_or_explain',
label: 'com_ui_teach_or_explain',
value: 'teach_or_explain',
},
{
label: 'write',
value: 'com_ui_write',
label: 'com_ui_write',
value: 'write',
},
{
label: 'shop',
value: 'com_ui_shop',
label: 'com_ui_shop',
value: 'shop',
},
{
label: 'code',
value: 'com_ui_code',
label: 'com_ui_code',
value: 'code',
},
{
label: 'misc',
value: 'com_ui_misc',
label: 'com_ui_misc',
value: 'misc',
},
{
label: 'roleplay',
value: 'com_ui_roleplay',
label: 'com_ui_roleplay',
value: 'roleplay',
},
{
label: 'finance',
value: 'com_ui_finance',
label: 'com_ui_finance',
value: 'finance',
},
];

View file

@ -3,9 +3,11 @@ const {
verifyBackupCode,
generateTOTPSecret,
generateBackupCodes,
getTOTPSecret,
} = require('~/server/services/twoFactorService');
const { updateUser, getUserById } = require('~/models');
const { logger } = require('~/config');
const { encryptV2 } = require('~/server/utils/crypto');
const enable2FAController = async (req, res) => {
const safeAppTitle = (process.env.APP_TITLE || 'LibreChat').replace(/\s+/g, '');
@ -15,7 +17,8 @@ const enable2FAController = async (req, res) => {
const secret = generateTOTPSecret();
const { plainCodes, codeObjects } = await generateBackupCodes();
const user = await updateUser(userId, { totpSecret: secret, backupCodes: codeObjects });
const encryptedSecret = await encryptV2(secret);
const user = await updateUser(userId, { totpSecret: encryptedSecret, backupCodes: codeObjects });
const otpauthUrl = `otpauth://totp/${safeAppTitle}:${user.email}?secret=${secret}&issuer=${safeAppTitle}`;
@ -38,14 +41,16 @@ const verify2FAController = async (req, res) => {
return res.status(400).json({ message: '2FA not initiated' });
}
let verified = false;
if (token && (await verifyTOTP(user.totpSecret, token))) {
// Retrieve the plain TOTP secret using getTOTPSecret.
const secret = await getTOTPSecret(user.totpSecret);
if (token && (await verifyTOTP(secret, token))) {
return res.status(200).json();
} else if (backupCode) {
verified = await verifyBackupCode({ user, backupCode });
}
if (verified) {
return res.status(200).json();
const verified = await verifyBackupCode({ user, backupCode });
if (verified) {
return res.status(200).json();
}
}
return res.status(400).json({ message: 'Invalid token.' });
@ -65,7 +70,10 @@ const confirm2FAController = async (req, res) => {
return res.status(400).json({ message: '2FA not initiated' });
}
if (await verifyTOTP(user.totpSecret, token)) {
// Retrieve the plain TOTP secret using getTOTPSecret.
const secret = await getTOTPSecret(user.totpSecret);
if (await verifyTOTP(secret, token)) {
return res.status(200).json();
}

View file

@ -1,5 +1,5 @@
const jwt = require('jsonwebtoken');
const { verifyTOTP, verifyBackupCode } = require('~/server/services/twoFactorService');
const { verifyTOTP, verifyBackupCode, getTOTPSecret } = require('~/server/services/twoFactorService');
const { setAuthTokens } = require('~/server/services/AuthService');
const { getUserById } = require('~/models/userMethods');
const { logger } = require('~/config');
@ -24,9 +24,11 @@ const verify2FA = async (req, res) => {
return res.status(400).json({ message: '2FA is not enabled for this user' });
}
let verified = false;
// Use the new getTOTPSecret function to retrieve (and decrypt if necessary) the TOTP secret.
const secret = await getTOTPSecret(user.totpSecret);
if (token && (await verifyTOTP(user.totpSecret, token))) {
let verified = false;
if (token && (await verifyTOTP(secret, token))) {
verified = true;
} else if (backupCode) {
verified = await verifyBackupCode({ user, backupCode });
@ -39,7 +41,7 @@ const verify2FA = async (req, res) => {
// Prepare user data for response.
// If the user is a plain object (from lean queries), we create a shallow copy.
const userData = user.toObject ? user.toObject() : { ...user };
// Remove sensitive fields
// Remove sensitive fields.
delete userData.password;
delete userData.__v;
delete userData.totpSecret;

View file

@ -22,10 +22,11 @@ const staticCache = require('./utils/staticCache');
const noIndex = require('./middleware/noIndex');
const routes = require('./routes');
const { PORT, HOST, ALLOW_SOCIAL_LOGIN, DISABLE_COMPRESSION } = process.env ?? {};
const { PORT, HOST, ALLOW_SOCIAL_LOGIN, DISABLE_COMPRESSION, TRUST_PROXY } = process.env ?? {};
const port = Number(PORT) || 3080;
const host = HOST || 'localhost';
const trusted_proxy = Number(TRUST_PROXY) || 1; /* trust first proxy by default */
const startServer = async () => {
if (typeof Bun !== 'undefined') {
@ -53,7 +54,7 @@ const startServer = async () => {
app.use(staticCache(app.locals.paths.dist));
app.use(staticCache(app.locals.paths.fonts));
app.use(staticCache(app.locals.paths.assets));
app.set('trust proxy', 1); /* trust first proxy */
app.set('trust proxy', trusted_proxy);
app.use(cors());
app.use(cookieParser());
@ -145,6 +146,18 @@ process.on('uncaughtException', (err) => {
logger.error('There was an uncaught error:', err);
}
if (err.message.includes('abort')) {
logger.warn('There was an uncatchable AbortController error.');
return;
}
if (err.message.includes('GoogleGenerativeAI')) {
logger.warn(
'\n\n`GoogleGenerativeAI` errors cannot be caught due to an upstream issue, see: https://github.com/google-gemini/generative-ai-js/issues/303',
);
return;
}
if (err.message.includes('fetch failed')) {
if (messageCount === 0) {
logger.warn('Meilisearch error, search will be disabled');

View file

@ -209,7 +209,7 @@ const primeFiles = async (options, apiKey) => {
const { handleFileUpload: uploadCodeEnvFile } = getStrategyFunctions(
FileSources.execute_code,
);
const stream = await getDownloadStream(file.filepath);
const stream = await getDownloadStream(options.req, file.filepath);
const fileIdentifier = await uploadCodeEnvFile({
req: options.req,
stream,

View file

@ -224,10 +224,11 @@ async function uploadFileToFirebase({ req, file, file_id }) {
/**
* Retrieves a readable stream for a file from Firebase storage.
*
* @param {ServerRequest} _req
* @param {string} filepath - The filepath.
* @returns {Promise<ReadableStream>} A readable stream of the file.
*/
async function getFirebaseFileStream(filepath) {
async function getFirebaseFileStream(_req, filepath) {
try {
const storage = getFirebaseStorage();
if (!storage) {

View file

@ -286,11 +286,31 @@ async function uploadLocalFile({ req, file, file_id }) {
/**
* Retrieves a readable stream for a file from local storage.
*
* @param {ServerRequest} req - The request object from Express
* @param {string} filepath - The filepath.
* @returns {ReadableStream} A readable stream of the file.
*/
function getLocalFileStream(filepath) {
function getLocalFileStream(req, filepath) {
try {
if (filepath.includes('/uploads/')) {
const basePath = filepath.split('/uploads/')[1];
if (!basePath) {
logger.warn(`Invalid base path: ${filepath}`);
throw new Error(`Invalid file path: ${filepath}`);
}
const fullPath = path.join(req.app.locals.paths.uploads, basePath);
const uploadsDir = req.app.locals.paths.uploads;
const rel = path.relative(uploadsDir, fullPath);
if (rel.startsWith('..') || path.isAbsolute(rel) || rel.includes(`..${path.sep}`)) {
logger.warn(`Invalid relative file path: ${filepath}`);
throw new Error(`Invalid file path: ${filepath}`);
}
return fs.createReadStream(fullPath);
}
return fs.createReadStream(filepath);
} catch (error) {
logger.error('Error getting local file stream:', error);

View file

@ -1,14 +1,15 @@
const { sign } = require('jsonwebtoken');
const { webcrypto } = require('node:crypto');
const { hashBackupCode } = require('~/server/utils/crypto');
const { hashBackupCode, decryptV2 } = require('~/server/utils/crypto');
const { updateUser } = require('~/models/userMethods');
const BASE32_ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567';
/**
* Encodes a Buffer into a Base32 string using RFC 4648 alphabet.
* Encodes a Buffer into a Base32 string using the RFC 4648 alphabet.
*
* @param {Buffer} buffer - The buffer to encode.
* @returns {string} - The Base32 encoded string.
* @returns {string} The Base32 encoded string.
*/
const encodeBase32 = (buffer) => {
let bits = 0;
@ -30,8 +31,9 @@ const encodeBase32 = (buffer) => {
/**
* Decodes a Base32-encoded string back into a Buffer.
* @param {string} base32Str
* @returns {Buffer}
*
* @param {string} base32Str - The Base32-encoded string.
* @returns {Buffer} The decoded buffer.
*/
const decodeBase32 = (base32Str) => {
const cleaned = base32Str.replace(/=+$/, '').toUpperCase();
@ -54,15 +56,20 @@ const decodeBase32 = (base32Str) => {
};
/**
* Generate a temporary token for 2FA verification.
* This token is signed with JWT_SECRET and expires in 5 minutes.
* Generates a temporary token for 2FA verification.
* The token is signed with the JWT_SECRET and expires in 5 minutes.
*
* @param {string} userId - The unique identifier of the user.
* @returns {string} The signed JWT token.
*/
const generate2FATempToken = (userId) =>
sign({ userId, twoFAPending: true }, process.env.JWT_SECRET, { expiresIn: '5m' });
/**
* Generate a TOTP secret.
* Generates 10 random bytes using WebCrypto and encodes them into a Base32 string.
* Generates a TOTP secret.
* Creates 10 random bytes using WebCrypto and encodes them into a Base32 string.
*
* @returns {string} A Base32-encoded secret for TOTP.
*/
const generateTOTPSecret = () => {
const randomArray = new Uint8Array(10);
@ -71,12 +78,12 @@ const generateTOTPSecret = () => {
};
/**
* Generate a TOTP code based on the secret and current time.
* Uses a 30-second time step and generates a 6-digit code.
* Generates a Time-based One-Time Password (TOTP) based on the provided secret and time.
* This implementation uses a 30-second time step and produces a 6-digit code.
*
* @param {string} secret - Base32-encoded secret
* @param {number} [forTime=Date.now()] - Time in milliseconds
* @returns {Promise<string>} - The 6-digit TOTP code.
* @param {string} secret - The Base32-encoded TOTP secret.
* @param {number} [forTime=Date.now()] - The time (in milliseconds) for which to generate the TOTP.
* @returns {Promise<string>} A promise that resolves to the 6-digit TOTP code.
*/
const generateTOTP = async (secret, forTime = Date.now()) => {
const timeStep = 30; // seconds
@ -106,6 +113,7 @@ const generateTOTP = async (secret, forTime = Date.now()) => {
const signatureBuffer = await webcrypto.subtle.sign('HMAC', cryptoKey, counterBuffer);
const hmac = new Uint8Array(signatureBuffer);
// Dynamic truncation as per RFC 4226
const offset = hmac[hmac.length - 1] & 0xf;
const slice = hmac.slice(offset, offset + 4);
const view = new DataView(slice.buffer, slice.byteOffset, slice.byteLength);
@ -115,12 +123,12 @@ const generateTOTP = async (secret, forTime = Date.now()) => {
};
/**
* Verify a provided TOTP token against the secret.
* Allows for a ±1 time-step window.
* Verifies a provided TOTP token against the secret.
* It allows for a ±1 time-step window to account for slight clock discrepancies.
*
* @param {string} secret
* @param {string} token
* @returns {Promise<boolean>}
* @param {string} secret - The Base32-encoded TOTP secret.
* @param {string} token - The TOTP token provided by the user.
* @returns {Promise<boolean>} A promise that resolves to true if the token is valid; otherwise, false.
*/
const verifyTOTP = async (secret, token) => {
const timeStepMS = 30 * 1000;
@ -135,12 +143,13 @@ const verifyTOTP = async (secret, token) => {
};
/**
* Generate backup codes.
* Generates `count` backup code objects and returns an object with both plain codes
* (for one-time download) and their objects (for secure storage). Uses WebCrypto for randomness and hashing.
* Generates backup codes for two-factor authentication.
* Each backup code is an 8-character hexadecimal string along with its SHA-256 hash.
* The plain codes are returned for one-time download, while the hashed objects are meant for secure storage.
*
* @param {number} count - Number of backup codes to generate (default: 10).
* @returns {Promise<Object>} - Contains `plainCodes` (array of strings) and `codeObjects` (array of objects).
* @param {number} [count=10] - The number of backup codes to generate.
* @returns {Promise<{ plainCodes: string[], codeObjects: Array<{ codeHash: string, used: boolean, usedAt: Date | null }> }>}
* A promise that resolves to an object containing both plain backup codes and their corresponding code objects.
*/
const generateBackupCodes = async (count = 10) => {
const plainCodes = [];
@ -165,11 +174,12 @@ const generateBackupCodes = async (count = 10) => {
};
/**
* Verifies a backup code and updates the user's backup codes if valid
* @param {Object} params
* @param {TUser | undefined} [params.user] - The user object
* @param {string | undefined} [params.backupCode] - The backup code to verify
* @returns {Promise<boolean>} - Whether the backup code was valid
* Verifies a backup code for a user and updates its status as used if valid.
*
* @param {Object} params - The parameters object.
* @param {TUser | undefined} [params.user] - The user object containing backup codes.
* @param {string | undefined} [params.backupCode] - The backup code to verify.
* @returns {Promise<boolean>} A promise that resolves to true if the backup code is valid and updated; otherwise, false.
*/
const verifyBackupCode = async ({ user, backupCode }) => {
if (!backupCode || !user || !Array.isArray(user.backupCodes)) {
@ -195,9 +205,32 @@ const verifyBackupCode = async ({ user, backupCode }) => {
return false;
};
/**
* Retrieves and, if necessary, decrypts a stored TOTP secret.
* If the secret contains a colon, it is assumed to be in the format "iv:encryptedData" and will be decrypted.
* If the secret is exactly 16 characters long, it is assumed to be a legacy plain secret.
*
* @param {string|null} storedSecret - The stored TOTP secret (which may be encrypted).
* @returns {Promise<string|null>} A promise that resolves to the plain TOTP secret, or null if none is provided.
*/
const getTOTPSecret = async (storedSecret) => {
if (!storedSecret) { return null; }
// Check for a colon marker (encrypted secrets are stored as "iv:encryptedData")
if (storedSecret.includes(':')) {
return await decryptV2(storedSecret);
}
// If it's exactly 16 characters, assume it's already plain (legacy secret)
if (storedSecret.length === 16) {
return storedSecret;
}
// Fallback in case it doesn't meet our criteria.
return storedSecret;
};
module.exports = {
verifyTOTP,
generateTOTP,
getTOTPSecret,
verifyBackupCode,
generateTOTPSecret,
generateBackupCodes,