LibreChat/api/server/utils/import/importBatchBuilder.js
Denis Palnitsky ab6fbe48f1
📥 feat: Import Conversations from LibreChat, ChatGPT, Chatbot UI (#2355)
* Basic implementation of ChatGPT conversation import

* remove debug code

* Handle citations

* Fix updatedAt in import

* update default model

* Use job scheduler to handle import requests

* import job status endpoint

* Add wrapper around Agenda

* Rate limits for import endpoint

* rename import api path

* Batch save import to mongo

* Improve naming

* Add documenting comments

* Test for importers

* Change button for importing conversations

* Frontend changes

* Import job status endpoint

* Import endpoint response

* Add translations to new phrases

* Fix conversations refreshing

* cleanup unused functions

* set timeout for import job status polling

* Add documentation

* get extra spaces back

* Improve error message

* Fix translation files after merge

* fix translation files 2

* Add zh translation for import functionality

* Sync mailisearch index after import

* chore: add dummy uri for jest tests, as MONGO_URI should only be real for E2E tests

* docs: fix links

* docs: fix conversationsImport section

* fix: user role issue for librechat imports

* refactor: import conversations from json
- organize imports
- add additional jsdocs
- use multer with diskStorage to avoid loading file into memory outside of job
- use filepath instead of loading data string for imports
- replace console logs and some logger.info() with logger.debug
- only use multer for import route

* fix: undefined metadata edge case and replace ChatGtp -> ChatGpt

* Refactor importChatGptConvo function to handle undefined metadata edge case and replace ChatGtp with ChatGpt

* fix: chatgpt importer

* feat: maintain tree relationship for librechat messages

* chore: use enum

* refactor: saveMessage to use single object arg, replace console logs, add userId to log message

* chore: additional comment

* chore: multer edge case

* feat: first pass, maintain tree relationship

* chore: organize

* chore: remove log

* ci: add heirarchy test for chatgpt

* ci: test maintaining of heirarchy for librechat

* wip: allow non-text content type messages

* refactor: import content part object json string

* refactor: more content types to format

* chore: consolidate messageText formatting

* docs: update on changes, bump data-provider/config versions, update readme

* refactor(indexSync): singleton pattern for MeiliSearchClient

* refactor: debug log after batch is done

* chore: add back indexSync error handling

---------

Co-authored-by: jakubmieszczak <jakub.mieszczak@zendesk.com>
Co-authored-by: Danny Avila <danny@librechat.ai>
2024-05-02 02:48:26 -04:00

149 lines
5 KiB
JavaScript

const { v4: uuidv4 } = require('uuid');
const { EModelEndpoint, Constants, openAISettings } = require('librechat-data-provider');
const { bulkSaveConvos } = require('~/models/Conversation');
const { bulkSaveMessages } = require('~/models/Message');
const { logger } = require('~/config');
/**
* Factory function for creating an instance of ImportBatchBuilder.
* @param {string} requestUserId - The ID of the user making the request.
* @returns {ImportBatchBuilder} - The newly created ImportBatchBuilder instance.
*/
function createImportBatchBuilder(requestUserId) {
return new ImportBatchBuilder(requestUserId);
}
/**
* Class for building a batch of conversations and messages and pushing them to DB for Conversation Import functionality
*/
class ImportBatchBuilder {
/**
* Creates an instance of ImportBatchBuilder.
* @param {string} requestUserId - The ID of the user making the import request.
*/
constructor(requestUserId) {
this.requestUserId = requestUserId;
this.conversations = [];
this.messages = [];
}
/**
* Starts a new conversation in the batch.
* @param {string} [endpoint=EModelEndpoint.openAI] - The endpoint for the conversation. Defaults to EModelEndpoint.openAI.
* @returns {void}
*/
startConversation(endpoint) {
// we are simplifying by using a single model for the entire conversation
this.endpoint = endpoint || EModelEndpoint.openAI;
this.conversationId = uuidv4();
this.lastMessageId = Constants.NO_PARENT;
}
/**
* Adds a user message to the current conversation.
* @param {string} text - The text of the user message.
* @returns {object} The saved message object.
*/
addUserMessage(text) {
const message = this.saveMessage({ text, sender: 'user', isCreatedByUser: true });
return message;
}
/**
* Adds a GPT message to the current conversation.
* @param {string} text - The text of the GPT message.
* @param {string} [model='defaultModel'] - The model used for generating the GPT message. Defaults to 'defaultModel'.
* @param {string} [sender='GPT-3.5'] - The sender of the GPT message. Defaults to 'GPT-3.5'.
* @returns {object} The saved message object.
*/
addGptMessage(text, model, sender = 'GPT-3.5') {
const message = this.saveMessage({
text,
sender,
isCreatedByUser: false,
model: model || openAISettings.model.default,
});
return message;
}
/**
* Finishes the current conversation and adds it to the batch.
* @param {string} [title='Imported Chat'] - The title of the conversation. Defaults to 'Imported Chat'.
* @param {Date} [createdAt] - The creation date of the conversation.
* @returns {object} The added conversation object.
*/
finishConversation(title, createdAt) {
const convo = {
user: this.requestUserId,
conversationId: this.conversationId,
title: title || 'Imported Chat',
createdAt: createdAt,
updatedAt: createdAt,
overrideTimestamp: true,
endpoint: this.endpoint,
model: openAISettings.model.default,
};
this.conversations.push(convo);
return convo;
}
/**
* Saves the batch of conversations and messages to the DB.
* @returns {Promise<void>} A promise that resolves when the batch is saved.
* @throws {Error} If there is an error saving the batch.
*/
async saveBatch() {
try {
await bulkSaveConvos(this.conversations);
await bulkSaveMessages(this.messages);
logger.debug(
`user: ${this.requestUserId} | Added ${this.conversations.length} conversations and ${this.messages.length} messages to the DB.`,
);
} catch (error) {
logger.error('Error saving batch', error);
throw error;
}
}
/**
* Saves a message to the current conversation.
* @param {object} messageDetails - The details of the message.
* @param {string} messageDetails.text - The text of the message.
* @param {string} messageDetails.sender - The sender of the message.
* @param {string} [messageDetails.messageId] - The ID of the current message.
* @param {boolean} messageDetails.isCreatedByUser - Indicates whether the message is created by the user.
* @param {string} [messageDetails.model] - The model used for generating the message.
* @param {string} [messageDetails.parentMessageId=this.lastMessageId] - The ID of the parent message.
* @returns {object} The saved message object.
*/
saveMessage({
text,
sender,
isCreatedByUser,
model,
messageId,
parentMessageId = this.lastMessageId,
}) {
const newMessageId = messageId ?? uuidv4();
const message = {
parentMessageId,
messageId: newMessageId,
conversationId: this.conversationId,
isCreatedByUser: isCreatedByUser,
model: model || this.model,
user: this.requestUserId,
endpoint: this.endpoint,
unfinished: false,
isEdited: false,
error: false,
sender,
text,
};
this.lastMessageId = newMessageId;
this.messages.push(message);
return message;
}
}
module.exports = { ImportBatchBuilder, createImportBatchBuilder };