📥 feat: Import Conversations from LibreChat, ChatGPT, Chatbot UI (#2355)

* Basic implementation of ChatGPT conversation import

* remove debug code

* Handle citations

* Fix updatedAt in import

* update default model

* Use job scheduler to handle import requests

* import job status endpoint

* Add wrapper around Agenda

* Rate limits for import endpoint

* rename import api path

* Batch save import to mongo

* Improve naming

* Add documenting comments

* Test for importers

* Change button for importing conversations

* Frontend changes

* Import job status endpoint

* Import endpoint response

* Add translations to new phrases

* Fix conversations refreshing

* cleanup unused functions

* set timeout for import job status polling

* Add documentation

* get extra spaces back

* Improve error message

* Fix translation files after merge

* fix translation files 2

* Add zh translation for import functionality

* Sync mailisearch index after import

* chore: add dummy uri for jest tests, as MONGO_URI should only be real for E2E tests

* docs: fix links

* docs: fix conversationsImport section

* fix: user role issue for librechat imports

* refactor: import conversations from json
- organize imports
- add additional jsdocs
- use multer with diskStorage to avoid loading file into memory outside of job
- use filepath instead of loading data string for imports
- replace console logs and some logger.info() with logger.debug
- only use multer for import route

* fix: undefined metadata edge case and replace ChatGtp -> ChatGpt

* Refactor importChatGptConvo function to handle undefined metadata edge case and replace ChatGtp with ChatGpt

* fix: chatgpt importer

* feat: maintain tree relationship for librechat messages

* chore: use enum

* refactor: saveMessage to use single object arg, replace console logs, add userId to log message

* chore: additional comment

* chore: multer edge case

* feat: first pass, maintain tree relationship

* chore: organize

* chore: remove log

* ci: add heirarchy test for chatgpt

* ci: test maintaining of heirarchy for librechat

* wip: allow non-text content type messages

* refactor: import content part object json string

* refactor: more content types to format

* chore: consolidate messageText formatting

* docs: update on changes, bump data-provider/config versions, update readme

* refactor(indexSync): singleton pattern for MeiliSearchClient

* refactor: debug log after batch is done

* chore: add back indexSync error handling

---------

Co-authored-by: jakubmieszczak <jakub.mieszczak@zendesk.com>
Co-authored-by: Danny Avila <danny@librechat.ai>

api/server/services/AppService.spec.js

@@ -347,6 +347,69 @@ describe('AppService', () => {
     expect(process.env.FILE_UPLOAD_USER_MAX).toEqual('initialUserMax');
     expect(process.env.FILE_UPLOAD_USER_WINDOW).toEqual('initialUserWindow');
   });
+
+  it('should not modify IMPORT environment variables without rate limits', async () => {
+    // Setup initial environment variables
+    process.env.IMPORT_IP_MAX = '10';
+    process.env.IMPORT_IP_WINDOW = '15';
+    process.env.IMPORT_USER_MAX = '5';
+    process.env.IMPORT_USER_WINDOW = '20';
+
+    const initialEnv = { ...process.env };
+
+    await AppService(app);
+
+    // Expect environment variables to remain unchanged
+    expect(process.env.IMPORT_IP_MAX).toEqual(initialEnv.IMPORT_IP_MAX);
+    expect(process.env.IMPORT_IP_WINDOW).toEqual(initialEnv.IMPORT_IP_WINDOW);
+    expect(process.env.IMPORT_USER_MAX).toEqual(initialEnv.IMPORT_USER_MAX);
+    expect(process.env.IMPORT_USER_WINDOW).toEqual(initialEnv.IMPORT_USER_WINDOW);
+  });
+
+  it('should correctly set IMPORT environment variables based on rate limits', async () => {
+    // Define and mock a custom configuration with rate limits
+    const importLimitsConfig = {
+      rateLimits: {
+        conversationsImport: {
+          ipMax: '150',
+          ipWindowInMinutes: '60',
+          userMax: '50',
+          userWindowInMinutes: '30',
+        },
+      },
+    };
+
+    require('./Config/loadCustomConfig').mockImplementationOnce(() =>
+      Promise.resolve(importLimitsConfig),
+    );
+
+    await AppService(app);
+
+    // Verify that process.env has been updated according to the rate limits config
+    expect(process.env.IMPORT_IP_MAX).toEqual('150');
+    expect(process.env.IMPORT_IP_WINDOW).toEqual('60');
+    expect(process.env.IMPORT_USER_MAX).toEqual('50');
+    expect(process.env.IMPORT_USER_WINDOW).toEqual('30');
+  });
+
+  it('should fallback to default IMPORT environment variables when rate limits are unspecified', async () => {
+    // Setup initial environment variables to non-default values
+    process.env.IMPORT_IP_MAX = 'initialMax';
+    process.env.IMPORT_IP_WINDOW = 'initialWindow';
+    process.env.IMPORT_USER_MAX = 'initialUserMax';
+    process.env.IMPORT_USER_WINDOW = 'initialUserWindow';
+
+    // Mock a custom configuration without specific rate limits
+    require('./Config/loadCustomConfig').mockImplementationOnce(() => Promise.resolve({}));
+
+    await AppService(app);
+
+    // Verify that process.env falls back to the initial values
+    expect(process.env.IMPORT_IP_MAX).toEqual('initialMax');
+    expect(process.env.IMPORT_IP_WINDOW).toEqual('initialWindow');
+    expect(process.env.IMPORT_USER_MAX).toEqual('initialUserMax');
+    expect(process.env.IMPORT_USER_WINDOW).toEqual('initialUserWindow');
+  });
 });
 
 describe('AppService updating app.locals and issuing warnings', () => {