mirror of
https://github.com/danny-avila/LibreChat.git
synced 2025-09-22 06:00:56 +02:00

* Basic implementation of ChatGPT conversation import * remove debug code * Handle citations * Fix updatedAt in import * update default model * Use job scheduler to handle import requests * import job status endpoint * Add wrapper around Agenda * Rate limits for import endpoint * rename import api path * Batch save import to mongo * Improve naming * Add documenting comments * Test for importers * Change button for importing conversations * Frontend changes * Import job status endpoint * Import endpoint response * Add translations to new phrases * Fix conversations refreshing * cleanup unused functions * set timeout for import job status polling * Add documentation * get extra spaces back * Improve error message * Fix translation files after merge * fix translation files 2 * Add zh translation for import functionality * Sync mailisearch index after import * chore: add dummy uri for jest tests, as MONGO_URI should only be real for E2E tests * docs: fix links * docs: fix conversationsImport section * fix: user role issue for librechat imports * refactor: import conversations from json - organize imports - add additional jsdocs - use multer with diskStorage to avoid loading file into memory outside of job - use filepath instead of loading data string for imports - replace console logs and some logger.info() with logger.debug - only use multer for import route * fix: undefined metadata edge case and replace ChatGtp -> ChatGpt * Refactor importChatGptConvo function to handle undefined metadata edge case and replace ChatGtp with ChatGpt * fix: chatgpt importer * feat: maintain tree relationship for librechat messages * chore: use enum * refactor: saveMessage to use single object arg, replace console logs, add userId to log message * chore: additional comment * chore: multer edge case * feat: first pass, maintain tree relationship * chore: organize * chore: remove log * ci: add heirarchy test for chatgpt * ci: test maintaining of heirarchy for librechat * wip: allow non-text content type messages * refactor: import content part object json string * refactor: more content types to format * chore: consolidate messageText formatting * docs: update on changes, bump data-provider/config versions, update readme * refactor(indexSync): singleton pattern for MeiliSearchClient * refactor: debug log after batch is done * chore: add back indexSync error handling --------- Co-authored-by: jakubmieszczak <jakub.mieszczak@zendesk.com> Co-authored-by: Danny Avila <danny@librechat.ai>
69 lines
2.2 KiB
JavaScript
69 lines
2.2 KiB
JavaScript
const rateLimit = require('express-rate-limit');
|
|
const { ViolationTypes } = require('librechat-data-provider');
|
|
const logViolation = require('~/cache/logViolation');
|
|
|
|
const getEnvironmentVariables = () => {
|
|
const IMPORT_IP_MAX = parseInt(process.env.IMPORT_IP_MAX) || 100;
|
|
const IMPORT_IP_WINDOW = parseInt(process.env.IMPORT_IP_WINDOW) || 15;
|
|
const IMPORT_USER_MAX = parseInt(process.env.IMPORT_USER_MAX) || 50;
|
|
const IMPORT_USER_WINDOW = parseInt(process.env.IMPORT_USER_WINDOW) || 15;
|
|
|
|
const importIpWindowMs = IMPORT_IP_WINDOW * 60 * 1000;
|
|
const importIpMax = IMPORT_IP_MAX;
|
|
const importIpWindowInMinutes = importIpWindowMs / 60000;
|
|
|
|
const importUserWindowMs = IMPORT_USER_WINDOW * 60 * 1000;
|
|
const importUserMax = IMPORT_USER_MAX;
|
|
const importUserWindowInMinutes = importUserWindowMs / 60000;
|
|
|
|
return {
|
|
importIpWindowMs,
|
|
importIpMax,
|
|
importIpWindowInMinutes,
|
|
importUserWindowMs,
|
|
importUserMax,
|
|
importUserWindowInMinutes,
|
|
};
|
|
};
|
|
|
|
const createImportHandler = (ip = true) => {
|
|
const { importIpMax, importIpWindowInMinutes, importUserMax, importUserWindowInMinutes } =
|
|
getEnvironmentVariables();
|
|
|
|
return async (req, res) => {
|
|
const type = ViolationTypes.FILE_UPLOAD_LIMIT;
|
|
const errorMessage = {
|
|
type,
|
|
max: ip ? importIpMax : importUserMax,
|
|
limiter: ip ? 'ip' : 'user',
|
|
windowInMinutes: ip ? importIpWindowInMinutes : importUserWindowInMinutes,
|
|
};
|
|
|
|
await logViolation(req, res, type, errorMessage);
|
|
res.status(429).json({ message: 'Too many conversation import requests. Try again later' });
|
|
};
|
|
};
|
|
|
|
const createImportLimiters = () => {
|
|
const { importIpWindowMs, importIpMax, importUserWindowMs, importUserMax } =
|
|
getEnvironmentVariables();
|
|
|
|
const importIpLimiter = rateLimit({
|
|
windowMs: importIpWindowMs,
|
|
max: importIpMax,
|
|
handler: createImportHandler(),
|
|
});
|
|
|
|
const importUserLimiter = rateLimit({
|
|
windowMs: importUserWindowMs,
|
|
max: importUserMax,
|
|
handler: createImportHandler(false),
|
|
keyGenerator: function (req) {
|
|
return req.user?.id; // Use the user ID or NULL if not available
|
|
},
|
|
});
|
|
|
|
return { importIpLimiter, importUserLimiter };
|
|
};
|
|
|
|
module.exports = { createImportLimiters };
|