📥 feat: Import Conversations from LibreChat, ChatGPT, Chatbot UI (#2355)

* Basic implementation of ChatGPT conversation import

* remove debug code

* Handle citations

* Fix updatedAt in import

* update default model

* Use job scheduler to handle import requests

* import job status endpoint

* Add wrapper around Agenda

* Rate limits for import endpoint

* rename import api path

* Batch save import to mongo

* Improve naming

* Add documenting comments

* Test for importers

* Change button for importing conversations

* Frontend changes

* Import job status endpoint

* Import endpoint response

* Add translations to new phrases

* Fix conversations refreshing

* cleanup unused functions

* set timeout for import job status polling

* Add documentation

* get extra spaces back

* Improve error message

* Fix translation files after merge

* fix translation files 2

* Add zh translation for import functionality

* Sync mailisearch index after import

* chore: add dummy uri for jest tests, as MONGO_URI should only be real for E2E tests

* docs: fix links

* docs: fix conversationsImport section

* fix: user role issue for librechat imports

* refactor: import conversations from json
- organize imports
- add additional jsdocs
- use multer with diskStorage to avoid loading file into memory outside of job
- use filepath instead of loading data string for imports
- replace console logs and some logger.info() with logger.debug
- only use multer for import route

* fix: undefined metadata edge case and replace ChatGtp -> ChatGpt

* Refactor importChatGptConvo function to handle undefined metadata edge case and replace ChatGtp with ChatGpt

* fix: chatgpt importer

* feat: maintain tree relationship for librechat messages

* chore: use enum

* refactor: saveMessage to use single object arg, replace console logs, add userId to log message

* chore: additional comment

* chore: multer edge case

* feat: first pass, maintain tree relationship

* chore: organize

* chore: remove log

* ci: add heirarchy test for chatgpt

* ci: test maintaining of heirarchy for librechat

* wip: allow non-text content type messages

* refactor: import content part object json string

* refactor: more content types to format

* chore: consolidate messageText formatting

* docs: update on changes, bump data-provider/config versions, update readme

* refactor(indexSync): singleton pattern for MeiliSearchClient

* refactor: debug log after batch is done

* chore: add back indexSync error handling

---------

Co-authored-by: jakubmieszczak <jakub.mieszczak@zendesk.com>
Co-authored-by: Danny Avila <danny@librechat.ai>

api/server/routes/convos.js

@@ -1,8 +1,13 @@
+const multer = require('multer');
 const express = require('express');
 const { CacheKeys } = require('librechat-data-provider');
 const { initializeClient } = require('~/server/services/Endpoints/assistants');
 const { getConvosByPage, deleteConvos, getConvo, saveConvo } = require('~/models/Conversation');
+const { IMPORT_CONVERSATION_JOB_NAME } = require('~/server/utils/import/jobDefinition');
+const { storage, importFileFilter } = require('~/server/routes/files/multer');
 const requireJwtAuth = require('~/server/middleware/requireJwtAuth');
+const { createImportLimiters } = require('~/server/middleware');
+const jobScheduler = require('~/server/utils/jobScheduler');
 const getLogStores = require('~/cache/getLogStores');
 const { sleep } = require('~/server/utils');
 const { logger } = require('~/config');
@@ -99,4 +104,51 @@ router.post('/update', async (req, res) => {
   }
 });
 
+const { importIpLimiter, importUserLimiter } = createImportLimiters();
+const upload = multer({ storage: storage, fileFilter: importFileFilter });
+
+/**
+ * Imports a conversation from a JSON file and saves it to the database.
+ * @route POST /import
+ * @param {Express.Multer.File} req.file - The JSON file to import.
+ * @returns {object} 201 - success response - application/json
+ */
+router.post(
+  '/import',
+  importIpLimiter,
+  importUserLimiter,
+  upload.single('file'),
+  async (req, res) => {
+    try {
+      const filepath = req.file.path;
+      const job = await jobScheduler.now(IMPORT_CONVERSATION_JOB_NAME, filepath, req.user.id);
+
+      res.status(201).json({ message: 'Import started', jobId: job.id });
+    } catch (error) {
+      logger.error('Error processing file', error);
+      res.status(500).send('Error processing file');
+    }
+  },
+);
+
+// Get the status of an import job for polling
+router.get('/import/jobs/:jobId', async (req, res) => {
+  try {
+    const { jobId } = req.params;
+    const { userId, ...jobStatus } = await jobScheduler.getJobStatus(jobId);
+    if (!jobStatus) {
+      return res.status(404).json({ message: 'Job not found.' });
+    }
+
+    if (userId !== req.user.id) {
+      return res.status(403).json({ message: 'Unauthorized' });
+    }
+
+    res.json(jobStatus);
+  } catch (error) {
+    logger.error('Error getting job details', error);
+    res.status(500).send('Error getting job details');
+  }
+});
+
 module.exports = router;