📥 feat: Import Conversations from LibreChat, ChatGPT, Chatbot UI (#2355)

* Basic implementation of ChatGPT conversation import

* remove debug code

* Handle citations

* Fix updatedAt in import

* update default model

* Use job scheduler to handle import requests

* import job status endpoint

* Add wrapper around Agenda

* Rate limits for import endpoint

* rename import api path

* Batch save import to mongo

* Improve naming

* Add documenting comments

* Test for importers

* Change button for importing conversations

* Frontend changes

* Import job status endpoint

* Import endpoint response

* Add translations to new phrases

* Fix conversations refreshing

* cleanup unused functions

* set timeout for import job status polling

* Add documentation

* get extra spaces back

* Improve error message

* Fix translation files after merge

* fix translation files 2

* Add zh translation for import functionality

* Sync mailisearch index after import

* chore: add dummy uri for jest tests, as MONGO_URI should only be real for E2E tests

* docs: fix links

* docs: fix conversationsImport section

* fix: user role issue for librechat imports

* refactor: import conversations from json
- organize imports
- add additional jsdocs
- use multer with diskStorage to avoid loading file into memory outside of job
- use filepath instead of loading data string for imports
- replace console logs and some logger.info() with logger.debug
- only use multer for import route

* fix: undefined metadata edge case and replace ChatGtp -> ChatGpt

* Refactor importChatGptConvo function to handle undefined metadata edge case and replace ChatGtp with ChatGpt

* fix: chatgpt importer

* feat: maintain tree relationship for librechat messages

* chore: use enum

* refactor: saveMessage to use single object arg, replace console logs, add userId to log message

* chore: additional comment

* chore: multer edge case

* feat: first pass, maintain tree relationship

* chore: organize

* chore: remove log

* ci: add heirarchy test for chatgpt

* ci: test maintaining of heirarchy for librechat

* wip: allow non-text content type messages

* refactor: import content part object json string

* refactor: more content types to format

* chore: consolidate messageText formatting

* docs: update on changes, bump data-provider/config versions, update readme

* refactor(indexSync): singleton pattern for MeiliSearchClient

* refactor: debug log after batch is done

* chore: add back indexSync error handling

---------

Co-authored-by: jakubmieszczak <jakub.mieszczak@zendesk.com>
Co-authored-by: Danny Avila <danny@librechat.ai>

api/server/middleware/importLimiters.js

@@ -0,0 +1,69 @@
+const rateLimit = require('express-rate-limit');
+const { ViolationTypes } = require('librechat-data-provider');
+const logViolation = require('~/cache/logViolation');
+
+const getEnvironmentVariables = () => {
+  const IMPORT_IP_MAX = parseInt(process.env.IMPORT_IP_MAX) || 100;
+  const IMPORT_IP_WINDOW = parseInt(process.env.IMPORT_IP_WINDOW) || 15;
+  const IMPORT_USER_MAX = parseInt(process.env.IMPORT_USER_MAX) || 50;
+  const IMPORT_USER_WINDOW = parseInt(process.env.IMPORT_USER_WINDOW) || 15;
+
+  const importIpWindowMs = IMPORT_IP_WINDOW * 60 * 1000;
+  const importIpMax = IMPORT_IP_MAX;
+  const importIpWindowInMinutes = importIpWindowMs / 60000;
+
+  const importUserWindowMs = IMPORT_USER_WINDOW * 60 * 1000;
+  const importUserMax = IMPORT_USER_MAX;
+  const importUserWindowInMinutes = importUserWindowMs / 60000;
+
+  return {
+    importIpWindowMs,
+    importIpMax,
+    importIpWindowInMinutes,
+    importUserWindowMs,
+    importUserMax,
+    importUserWindowInMinutes,
+  };
+};
+
+const createImportHandler = (ip = true) => {
+  const { importIpMax, importIpWindowInMinutes, importUserMax, importUserWindowInMinutes } =
+    getEnvironmentVariables();
+
+  return async (req, res) => {
+    const type = ViolationTypes.FILE_UPLOAD_LIMIT;
+    const errorMessage = {
+      type,
+      max: ip ? importIpMax : importUserMax,
+      limiter: ip ? 'ip' : 'user',
+      windowInMinutes: ip ? importIpWindowInMinutes : importUserWindowInMinutes,
+    };
+
+    await logViolation(req, res, type, errorMessage);
+    res.status(429).json({ message: 'Too many conversation import requests. Try again later' });
+  };
+};
+
+const createImportLimiters = () => {
+  const { importIpWindowMs, importIpMax, importUserWindowMs, importUserMax } =
+    getEnvironmentVariables();
+
+  const importIpLimiter = rateLimit({
+    windowMs: importIpWindowMs,
+    max: importIpMax,
+    handler: createImportHandler(),
+  });
+
+  const importUserLimiter = rateLimit({
+    windowMs: importUserWindowMs,
+    max: importUserMax,
+    handler: createImportHandler(false),
+    keyGenerator: function (req) {
+      return req.user?.id; // Use the user ID or NULL if not available
+    },
+  });
+
+  return { importIpLimiter, importUserLimiter };
+};
+
+module.exports = { createImportLimiters };

api/server/middleware/index.js

@@ -18,6 +18,7 @@ const validateRegistration = require('./validateRegistration');
 const validateImageRequest = require('./validateImageRequest');
 const moderateText = require('./moderateText');
 const noIndex = require('./noIndex');
+const importLimiters = require('./importLimiters');
 
 module.exports = {
   ...uploadLimiters,
@@ -39,5 +40,6 @@ module.exports = {
   validateModel,
   moderateText,
   noIndex,
+  ...importLimiters,
   checkDomainAllowed,
 };