📥 feat: Import Conversations from LibreChat, ChatGPT, Chatbot UI (#2355)

* Basic implementation of ChatGPT conversation import

* remove debug code

* Handle citations

* Fix updatedAt in import

* update default model

* Use job scheduler to handle import requests

* import job status endpoint

* Add wrapper around Agenda

* Rate limits for import endpoint

* rename import api path

* Batch save import to mongo

* Improve naming

* Add documenting comments

* Test for importers

* Change button for importing conversations

* Frontend changes

* Import job status endpoint

* Import endpoint response

* Add translations to new phrases

* Fix conversations refreshing

* cleanup unused functions

* set timeout for import job status polling

* Add documentation

* get extra spaces back

* Improve error message

* Fix translation files after merge

* fix translation files 2

* Add zh translation for import functionality

* Sync mailisearch index after import

* chore: add dummy uri for jest tests, as MONGO_URI should only be real for E2E tests

* docs: fix links

* docs: fix conversationsImport section

* fix: user role issue for librechat imports

* refactor: import conversations from json
- organize imports
- add additional jsdocs
- use multer with diskStorage to avoid loading file into memory outside of job
- use filepath instead of loading data string for imports
- replace console logs and some logger.info() with logger.debug
- only use multer for import route

* fix: undefined metadata edge case and replace ChatGtp -> ChatGpt

* Refactor importChatGptConvo function to handle undefined metadata edge case and replace ChatGtp with ChatGpt

* fix: chatgpt importer

* feat: maintain tree relationship for librechat messages

* chore: use enum

* refactor: saveMessage to use single object arg, replace console logs, add userId to log message

* chore: additional comment

* chore: multer edge case

* feat: first pass, maintain tree relationship

* chore: organize

* chore: remove log

* ci: add heirarchy test for chatgpt

* ci: test maintaining of heirarchy for librechat

* wip: allow non-text content type messages

* refactor: import content part object json string

* refactor: more content types to format

* chore: consolidate messageText formatting

* docs: update on changes, bump data-provider/config versions, update readme

* refactor(indexSync): singleton pattern for MeiliSearchClient

* refactor: debug log after batch is done

* chore: add back indexSync error handling

---------

Co-authored-by: jakubmieszczak <jakub.mieszczak@zendesk.com>
Co-authored-by: Danny Avila <danny@librechat.ai>

client/src/components/Nav/SettingsTabs/Data/Data.tsx

@@ -7,6 +7,7 @@ import { SettingsTabValues } from 'librechat-data-provider';
 import React, { useState, useCallback, useRef } from 'react';
 import { useOnClickOutside } from '~/hooks';
 import DangerButton from '../DangerButton';
+import ImportConversations from './ImportConversations';
 
 export const RevokeKeysButton = ({
   showText = true,
@@ -76,6 +77,9 @@ function Data() {
         <div className="border-b pb-3 last-of-type:border-b-0 dark:border-gray-700">
           <RevokeKeysButton all={true} />
         </div>
+        <div className="border-b pb-3 last-of-type:border-b-0 dark:border-gray-700">
+          <ImportConversations />
+        </div>
       </div>
     </Tabs.Content>
   );

client/src/components/Nav/SettingsTabs/Data/ImportConversations.tsx

@@ -0,0 +1,87 @@
+import { Import } from 'lucide-react';
+import { cn } from '~/utils';
+import { useUploadConversationsMutation } from '~/data-provider';
+import { useLocalize, useConversations } from '~/hooks';
+import { useState } from 'react';
+import { useToastContext } from '~/Providers';
+
+function ImportConversations() {
+  const localize = useLocalize();
+
+  const { showToast } = useToastContext();
+  const [, setErrors] = useState<string[]>([]);
+  const setError = (error: string) => setErrors((prevErrors) => [...prevErrors, error]);
+  const { refreshConversations } = useConversations();
+
+  const uploadFile = useUploadConversationsMutation({
+    onSuccess: () => {
+      refreshConversations();
+      showToast({ message: localize('com_ui_import_conversation_success') });
+    },
+    onError: (error) => {
+      console.error('Error: ', error);
+      setError(
+        (error as { response: { data: { message?: string } } })?.response?.data?.message ??
+          'An error occurred while uploading the file.',
+      );
+      if (error?.toString().includes('Unsupported import type')) {
+        showToast({
+          message: localize('com_ui_import_conversation_file_type_error'),
+          status: 'error',
+        });
+      } else {
+        showToast({ message: localize('com_ui_import_conversation_error'), status: 'error' });
+      }
+    },
+  });
+
+  const startUpload = async (file: File) => {
+    const formData = new FormData();
+    formData.append('file', file, encodeURIComponent(file?.name || 'File'));
+
+    uploadFile.mutate(formData);
+  };
+
+  const handleFiles = async (_file: File) => {
+    console.log('Handling files...');
+
+    /* Process files */
+    try {
+      await startUpload(_file);
+    } catch (error) {
+      console.log('file handling error', error);
+      setError('An error occurred while processing the file.');
+    }
+  };
+
+  const handleFileChange = (event) => {
+    console.log('file change');
+    const file = event.target.files[0];
+    if (file) {
+      handleFiles(file);
+    }
+  };
+
+  return (
+    <div className="flex items-center justify-between">
+      <span>{localize('com_ui_import_conversation_info')}</span>
+      <label
+        htmlFor={'import-conversations-file'}
+        className="flex h-auto cursor-pointer items-center rounded bg-transparent px-2 py-1 text-xs font-medium font-normal transition-colors hover:bg-gray-100 hover:text-green-700 dark:bg-transparent dark:text-white dark:hover:bg-gray-600 dark:hover:text-green-500"
+      >
+        <Import className="mr-1 flex w-[22px] items-center stroke-1" />
+        <span>{localize('com_ui_import_conversation')}</span>
+        <input
+          id={'import-conversations-file'}
+          value=""
+          type="file"
+          className={cn('hidden')}
+          accept=".json"
+          onChange={handleFileChange}
+        />
+      </label>
+    </div>
+  );
+}
+
+export default ImportConversations;