📥 feat: Import Conversations from LibreChat, ChatGPT, Chatbot UI (#2355)

* Basic implementation of ChatGPT conversation import

* remove debug code

* Handle citations

* Fix updatedAt in import

* update default model

* Use job scheduler to handle import requests

* import job status endpoint

* Add wrapper around Agenda

* Rate limits for import endpoint

* rename import api path

* Batch save import to mongo

* Improve naming

* Add documenting comments

* Test for importers

* Change button for importing conversations

* Frontend changes

* Import job status endpoint

* Import endpoint response

* Add translations to new phrases

* Fix conversations refreshing

* cleanup unused functions

* set timeout for import job status polling

* Add documentation

* get extra spaces back

* Improve error message

* Fix translation files after merge

* fix translation files 2

* Add zh translation for import functionality

* Sync mailisearch index after import

* chore: add dummy uri for jest tests, as MONGO_URI should only be real for E2E tests

* docs: fix links

* docs: fix conversationsImport section

* fix: user role issue for librechat imports

* refactor: import conversations from json
- organize imports
- add additional jsdocs
- use multer with diskStorage to avoid loading file into memory outside of job
- use filepath instead of loading data string for imports
- replace console logs and some logger.info() with logger.debug
- only use multer for import route

* fix: undefined metadata edge case and replace ChatGtp -> ChatGpt

* Refactor importChatGptConvo function to handle undefined metadata edge case and replace ChatGtp with ChatGpt

* fix: chatgpt importer

* feat: maintain tree relationship for librechat messages

* chore: use enum

* refactor: saveMessage to use single object arg, replace console logs, add userId to log message

* chore: additional comment

* chore: multer edge case

* feat: first pass, maintain tree relationship

* chore: organize

* chore: remove log

* ci: add heirarchy test for chatgpt

* ci: test maintaining of heirarchy for librechat

* wip: allow non-text content type messages

* refactor: import content part object json string

* refactor: more content types to format

* chore: consolidate messageText formatting

* docs: update on changes, bump data-provider/config versions, update readme

* refactor(indexSync): singleton pattern for MeiliSearchClient

* refactor: debug log after batch is done

* chore: add back indexSync error handling

---------

Co-authored-by: jakubmieszczak <jakub.mieszczak@zendesk.com>
Co-authored-by: Danny Avila <danny@librechat.ai>

packages/data-provider/package.json

@@ -1,6 +1,6 @@
 {
   "name": "librechat-data-provider",
-  "version": "0.5.8",
+  "version": "0.5.9",
   "description": "data services for librechat apps",
   "main": "dist/index.js",
   "module": "dist/index.es.js",

packages/data-provider/src/api-endpoints.ts

@@ -19,15 +19,23 @@ export const revokeAllUserKeys = () => `${keysEndpoint}?all=true`;
 
 export const abortRequest = (endpoint: string) => `/api/ask/${endpoint}/abort`;
 
-export const conversations = (pageNumber: string) => `/api/convos?pageNumber=${pageNumber}`;
+export const conversationsRoot = '/api/convos';
 
-export const conversationById = (id: string) => `/api/convos/${id}`;
+export const conversations = (pageNumber: string) =>
+  `${conversationsRoot}?pageNumber=${pageNumber}`;
 
-export const genTitle = () => '/api/convos/gen_title';
+export const conversationById = (id: string) => `${conversationsRoot}/${id}`;
 
-export const updateConversation = () => '/api/convos/update';
+export const genTitle = () => `${conversationsRoot}/gen_title`;
 
-export const deleteConversation = () => '/api/convos/clear';
+export const updateConversation = () => `${conversationsRoot}/update`;
+
+export const deleteConversation = () => `${conversationsRoot}/clear`;
+
+export const importConversation = () => `${conversationsRoot}/import`;
+
+export const importConversationJobStatus = (jobId: string) =>
+  `${conversationsRoot}/import/jobs/${jobId}`;
 
 export const search = (q: string, pageNumber: string) =>
   `/api/search?q=${q}&pageNumber=${pageNumber}`;

packages/data-provider/src/config.ts

@@ -224,6 +224,14 @@ export const rateLimitSchema = z.object({
       userWindowInMinutes: z.number().optional(),
     })
     .optional(),
+  conversationsImport: z
+    .object({
+      ipMax: z.number().optional(),
+      ipWindowInMinutes: z.number().optional(),
+      userMax: z.number().optional(),
+      userWindowInMinutes: z.number().optional(),
+    })
+    .optional(),
 });
 
 export enum EImageOutputType {
@@ -660,7 +668,7 @@ export enum Constants {
   /** Key for the app's version. */
   VERSION = 'v0.7.1',
   /** Key for the Custom Config's version (librechat.yaml). */
-  CONFIG_VERSION = '1.0.8',
+  CONFIG_VERSION = '1.0.9',
   /** Standard value for the first message's `parentMessageId` value, to indicate no parent exists. */
   NO_PARENT = '00000000-0000-0000-0000-000000000000',
   /** Fixed, encoded domain length for Azure OpenAI Assistants Function name parsing. */

packages/data-provider/src/data-service.ts

@@ -191,6 +191,28 @@ export const uploadFile = (data: FormData): Promise<f.TFileUpload> => {
   return request.postMultiPart(endpoints.files(), data);
 };
 
+/**
+ * Imports a conversations file.
+ *
+ * @param data - The FormData containing the file to import.
+ * @returns A Promise that resolves to the import start response.
+ */
+export const importConversationsFile = (data: FormData): Promise<t.TImportStartResponse> => {
+  return request.postMultiPart(endpoints.importConversation(), data);
+};
+
+/**
+ * Retrieves the status of an import conversation job.
+ *
+ * @param jobId - The ID of the import conversation job.
+ * @returns A promise that resolves to the import job status.
+ */
+export const queryImportConversationJobStatus = async (
+  jobId: string,
+): Promise<t.TImportJobStatus> => {
+  return request.get(endpoints.importConversationJobStatus(jobId));
+};
+
 export const uploadAvatar = (data: FormData): Promise<f.AvatarUploadResponse> => {
   return request.postMultiPart(endpoints.avatar(), data);
 };

packages/data-provider/src/types.ts

@@ -247,3 +247,43 @@ export type TRequestPasswordResetResponse = {
   link?: string;
   message?: string;
 };
+
+/**
+ * Represents the response from the import endpoint.
+ */
+export type TImportStartResponse = {
+  /**
+   * The message associated with the response.
+   */
+  message: string;
+
+  /**
+   * The ID of the job associated with the import.
+   */
+  jobId: string;
+};
+
+/**
+ * Represents the status of an import job.
+ */
+export type TImportJobStatus = {
+  /**
+   * The name of the job.
+   */
+  name: string;
+
+  /**
+   * The ID of the job.
+   */
+  id: string;
+
+  /**
+   * The status of the job.
+   */
+  status: 'scheduled' | 'running' | 'completed' | 'failed';
+
+  /**
+   * The reason the job failed, if applicable.
+   */
+  failReason?: string;
+};