📥 feat: Import Conversations from LibreChat, ChatGPT, Chatbot UI (#2355)

* Basic implementation of ChatGPT conversation import

* remove debug code

* Handle citations

* Fix updatedAt in import

* update default model

* Use job scheduler to handle import requests

* import job status endpoint

* Add wrapper around Agenda

* Rate limits for import endpoint

* rename import api path

* Batch save import to mongo

* Improve naming

* Add documenting comments

* Test for importers

* Change button for importing conversations

* Frontend changes

* Import job status endpoint

* Import endpoint response

* Add translations to new phrases

* Fix conversations refreshing

* cleanup unused functions

* set timeout for import job status polling

* Add documentation

* get extra spaces back

* Improve error message

* Fix translation files after merge

* fix translation files 2

* Add zh translation for import functionality

* Sync mailisearch index after import

* chore: add dummy uri for jest tests, as MONGO_URI should only be real for E2E tests

* docs: fix links

* docs: fix conversationsImport section

* fix: user role issue for librechat imports

* refactor: import conversations from json
- organize imports
- add additional jsdocs
- use multer with diskStorage to avoid loading file into memory outside of job
- use filepath instead of loading data string for imports
- replace console logs and some logger.info() with logger.debug
- only use multer for import route

* fix: undefined metadata edge case and replace ChatGtp -> ChatGpt

* Refactor importChatGptConvo function to handle undefined metadata edge case and replace ChatGtp with ChatGpt

* fix: chatgpt importer

* feat: maintain tree relationship for librechat messages

* chore: use enum

* refactor: saveMessage to use single object arg, replace console logs, add userId to log message

* chore: additional comment

* chore: multer edge case

* feat: first pass, maintain tree relationship

* chore: organize

* chore: remove log

* ci: add heirarchy test for chatgpt

* ci: test maintaining of heirarchy for librechat

* wip: allow non-text content type messages

* refactor: import content part object json string

* refactor: more content types to format

* chore: consolidate messageText formatting

* docs: update on changes, bump data-provider/config versions, update readme

* refactor(indexSync): singleton pattern for MeiliSearchClient

* refactor: debug log after batch is done

* chore: add back indexSync error handling

---------

Co-authored-by: jakubmieszczak <jakub.mieszczak@zendesk.com>
Co-authored-by: Danny Avila <danny@librechat.ai>

api/lib/db/indexSync.js

@@ -1,11 +1,28 @@
 const { MeiliSearch } = require('meilisearch');
-const Message = require('~/models/schema/messageSchema');
 const Conversation = require('~/models/schema/convoSchema');
+const Message = require('~/models/schema/messageSchema');
 const { logger } = require('~/config');
 
 const searchEnabled = process.env?.SEARCH?.toLowerCase() === 'true';
 let currentTimeout = null;
 
+class MeiliSearchClient {
+  static instance = null;
+
+  static getInstance() {
+    if (!MeiliSearchClient.instance) {
+      if (!process.env.MEILI_HOST || !process.env.MEILI_MASTER_KEY) {
+        throw new Error('Meilisearch configuration is missing.');
+      }
+      MeiliSearchClient.instance = new MeiliSearch({
+        host: process.env.MEILI_HOST,
+        apiKey: process.env.MEILI_MASTER_KEY,
+      });
+    }
+    return MeiliSearchClient.instance;
+  }
+}
+
 // eslint-disable-next-line no-unused-vars
 async function indexSync(req, res, next) {
   if (!searchEnabled) {
@@ -13,20 +30,10 @@ async function indexSync(req, res, next) {
   }
 
   try {
-    if (!process.env.MEILI_HOST || !process.env.MEILI_MASTER_KEY || !searchEnabled) {
-      throw new Error('Meilisearch not configured, search will be disabled.');
-    }
-
-    const client = new MeiliSearch({
-      host: process.env.MEILI_HOST,
-      apiKey: process.env.MEILI_MASTER_KEY,
-    });
+    const client = MeiliSearchClient.getInstance();
 
     const { status } = await client.health();
-    // logger.debug(`[indexSync] Meilisearch: ${status}`);
-    const result = status === 'available' && !!process.env.SEARCH;
-
-    if (!result) {
+    if (status !== 'available' || !process.env.SEARCH) {
       throw new Error('Meilisearch not available');
     }
 
@@ -37,12 +44,8 @@ async function indexSync(req, res, next) {
     const messagesIndexed = messages.numberOfDocuments;
     const convosIndexed = convos.numberOfDocuments;
 
-    logger.debug(
-      `[indexSync] There are ${messageCount} messages in the database, ${messagesIndexed} indexed`,
-    );
-    logger.debug(
-      `[indexSync] There are ${convoCount} convos in the database, ${convosIndexed} indexed`,
-    );
+    logger.debug(`[indexSync] There are ${messageCount} messages and ${messagesIndexed} indexed`);
+    logger.debug(`[indexSync] There are ${convoCount} convos and ${convosIndexed} indexed`);
 
     if (messageCount !== messagesIndexed) {
       logger.debug('[indexSync] Messages out of sync, indexing');
@@ -54,7 +57,6 @@ async function indexSync(req, res, next) {
       Conversation.syncWithMeili();
     }
   } catch (err) {
-    // logger.debug('[indexSync] in index sync');
     if (err.message.includes('not found')) {
       logger.debug('[indexSync] Creating indices...');
       currentTimeout = setTimeout(async () => {