📥 feat: Import Conversations from LibreChat, ChatGPT, Chatbot UI (#2355)

* Basic implementation of ChatGPT conversation import

* remove debug code

* Handle citations

* Fix updatedAt in import

* update default model

* Use job scheduler to handle import requests

* import job status endpoint

* Add wrapper around Agenda

* Rate limits for import endpoint

* rename import api path

* Batch save import to mongo

* Improve naming

* Add documenting comments

* Test for importers

* Change button for importing conversations

* Frontend changes

* Import job status endpoint

* Import endpoint response

* Add translations to new phrases

* Fix conversations refreshing

* cleanup unused functions

* set timeout for import job status polling

* Add documentation

* get extra spaces back

* Improve error message

* Fix translation files after merge

* fix translation files 2

* Add zh translation for import functionality

* Sync mailisearch index after import

* chore: add dummy uri for jest tests, as MONGO_URI should only be real for E2E tests

* docs: fix links

* docs: fix conversationsImport section

* fix: user role issue for librechat imports

* refactor: import conversations from json
- organize imports
- add additional jsdocs
- use multer with diskStorage to avoid loading file into memory outside of job
- use filepath instead of loading data string for imports
- replace console logs and some logger.info() with logger.debug
- only use multer for import route

* fix: undefined metadata edge case and replace ChatGtp -> ChatGpt

* Refactor importChatGptConvo function to handle undefined metadata edge case and replace ChatGtp with ChatGpt

* fix: chatgpt importer

* feat: maintain tree relationship for librechat messages

* chore: use enum

* refactor: saveMessage to use single object arg, replace console logs, add userId to log message

* chore: additional comment

* chore: multer edge case

* feat: first pass, maintain tree relationship

* chore: organize

* chore: remove log

* ci: add heirarchy test for chatgpt

* ci: test maintaining of heirarchy for librechat

* wip: allow non-text content type messages

* refactor: import content part object json string

* refactor: more content types to format

* chore: consolidate messageText formatting

* docs: update on changes, bump data-provider/config versions, update readme

* refactor(indexSync): singleton pattern for MeiliSearchClient

* refactor: debug log after batch is done

* chore: add back indexSync error handling

---------

Co-authored-by: jakubmieszczak <jakub.mieszczak@zendesk.com>
Co-authored-by: Danny Avila <danny@librechat.ai>

client/src/data-provider/mutations.ts

@@ -6,8 +6,10 @@ import type {
   TFile,
   BatchFile,
   TFileUpload,
+  TImportStartResponse,
   AssistantListResponse,
   UploadMutationOptions,
+  UploadConversationsMutationOptions,
   DeleteFilesResponse,
   DeleteFilesBody,
   DeleteMutationOptions,
@@ -131,6 +133,89 @@ export const useDeleteConversationMutation = (
   );
 };
 
+export const useUploadConversationsMutation = (_options?: UploadConversationsMutationOptions) => {
+  const queryClient = useQueryClient();
+  const { onSuccess, onError } = _options || {};
+
+  // returns the job status or reason of failure
+  const checkJobStatus = async (jobId) => {
+    try {
+      const response = await dataService.queryImportConversationJobStatus(jobId);
+      return response;
+    } catch (error) {
+      throw new Error('Failed to check job status');
+    }
+  };
+
+  // Polls the job status until it is completed, failed, or timed out
+  const pollJobStatus = (jobId, onSuccess, onError) => {
+    let timeElapsed = 0;
+    const timeout = 60000; // Timeout after a minute
+    const pollInterval = 500; // Poll every 500ms
+    const intervalId = setInterval(async () => {
+      try {
+        const statusResponse = await checkJobStatus(jobId);
+        console.log('Polling job status', statusResponse);
+        if (statusResponse.status === 'completed' || statusResponse.status === 'failed') {
+          clearInterval(intervalId);
+          if (statusResponse.status === 'completed') {
+            onSuccess && onSuccess(statusResponse);
+          } else {
+            onError &&
+              onError(
+                new Error(
+                  statusResponse.failReason
+                    ? statusResponse.failReason
+                    : 'Failed to import conversations',
+                ),
+              );
+          }
+        }
+        timeElapsed += pollInterval; // Increment time elapsed by polling interval
+        if (timeElapsed >= timeout) {
+          clearInterval(intervalId);
+          onError && onError(new Error('Polling timed out'));
+        }
+      } catch (error) {
+        clearInterval(intervalId);
+        onError && onError(error);
+      }
+    }, pollInterval);
+  };
+  return useMutation<TImportStartResponse, unknown, FormData>({
+    mutationFn: (formData: FormData) => dataService.importConversationsFile(formData),
+    onSuccess: (data, variables, context) => {
+      queryClient.invalidateQueries([QueryKeys.allConversations]);
+      // Assuming the job ID is in the response data
+      const jobId = data.jobId;
+      if (jobId) {
+        // Start polling for job status
+        pollJobStatus(
+          jobId,
+          (statusResponse) => {
+            // This is the final success callback when the job is completed
+            queryClient.invalidateQueries([QueryKeys.allConversations]); // Optionally refresh conversations query
+            if (onSuccess) {
+              onSuccess(statusResponse, variables, context);
+            }
+          },
+          (error) => {
+            // This is the error callback for job failure or polling errors
+            if (onError) {
+              onError(error, variables, context);
+            }
+          },
+        );
+      }
+    },
+    onError: (err, variables, context) => {
+      if (onError) {
+        onError(err, variables, context);
+      }
+    },
+  });
+};
+
 export const useUploadFileMutation = (
   _options?: UploadMutationOptions,
 ): UseMutationResult<