LibreChat/api/server/utils/import/importBatchBuilder.js

150 lines
5 KiB
JavaScript
Raw Normal View History

📥 feat: Import Conversations from LibreChat, ChatGPT, Chatbot UI (#2355) * Basic implementation of ChatGPT conversation import * remove debug code * Handle citations * Fix updatedAt in import * update default model * Use job scheduler to handle import requests * import job status endpoint * Add wrapper around Agenda * Rate limits for import endpoint * rename import api path * Batch save import to mongo * Improve naming * Add documenting comments * Test for importers * Change button for importing conversations * Frontend changes * Import job status endpoint * Import endpoint response * Add translations to new phrases * Fix conversations refreshing * cleanup unused functions * set timeout for import job status polling * Add documentation * get extra spaces back * Improve error message * Fix translation files after merge * fix translation files 2 * Add zh translation for import functionality * Sync mailisearch index after import * chore: add dummy uri for jest tests, as MONGO_URI should only be real for E2E tests * docs: fix links * docs: fix conversationsImport section * fix: user role issue for librechat imports * refactor: import conversations from json - organize imports - add additional jsdocs - use multer with diskStorage to avoid loading file into memory outside of job - use filepath instead of loading data string for imports - replace console logs and some logger.info() with logger.debug - only use multer for import route * fix: undefined metadata edge case and replace ChatGtp -> ChatGpt * Refactor importChatGptConvo function to handle undefined metadata edge case and replace ChatGtp with ChatGpt * fix: chatgpt importer * feat: maintain tree relationship for librechat messages * chore: use enum * refactor: saveMessage to use single object arg, replace console logs, add userId to log message * chore: additional comment * chore: multer edge case * feat: first pass, maintain tree relationship * chore: organize * chore: remove log * ci: add heirarchy test for chatgpt * ci: test maintaining of heirarchy for librechat * wip: allow non-text content type messages * refactor: import content part object json string * refactor: more content types to format * chore: consolidate messageText formatting * docs: update on changes, bump data-provider/config versions, update readme * refactor(indexSync): singleton pattern for MeiliSearchClient * refactor: debug log after batch is done * chore: add back indexSync error handling --------- Co-authored-by: jakubmieszczak <jakub.mieszczak@zendesk.com> Co-authored-by: Danny Avila <danny@librechat.ai>
2024-05-02 08:48:26 +02:00
const { v4: uuidv4 } = require('uuid');
const { EModelEndpoint, Constants, openAISettings } = require('librechat-data-provider');
const { bulkSaveConvos } = require('~/models/Conversation');
const { bulkSaveMessages } = require('~/models/Message');
const { logger } = require('~/config');
/**
* Factory function for creating an instance of ImportBatchBuilder.
* @param {string} requestUserId - The ID of the user making the request.
* @returns {ImportBatchBuilder} - The newly created ImportBatchBuilder instance.
*/
function createImportBatchBuilder(requestUserId) {
return new ImportBatchBuilder(requestUserId);
}
/**
* Class for building a batch of conversations and messages and pushing them to DB for Conversation Import functionality
*/
class ImportBatchBuilder {
/**
* Creates an instance of ImportBatchBuilder.
* @param {string} requestUserId - The ID of the user making the import request.
*/
constructor(requestUserId) {
this.requestUserId = requestUserId;
this.conversations = [];
this.messages = [];
}
/**
* Starts a new conversation in the batch.
* @param {string} [endpoint=EModelEndpoint.openAI] - The endpoint for the conversation. Defaults to EModelEndpoint.openAI.
* @returns {void}
*/
startConversation(endpoint) {
// we are simplifying by using a single model for the entire conversation
this.endpoint = endpoint || EModelEndpoint.openAI;
this.conversationId = uuidv4();
this.lastMessageId = Constants.NO_PARENT;
}
/**
* Adds a user message to the current conversation.
* @param {string} text - The text of the user message.
* @returns {object} The saved message object.
*/
addUserMessage(text) {
const message = this.saveMessage({ text, sender: 'user', isCreatedByUser: true });
return message;
}
/**
* Adds a GPT message to the current conversation.
* @param {string} text - The text of the GPT message.
* @param {string} [model='defaultModel'] - The model used for generating the GPT message. Defaults to 'defaultModel'.
* @param {string} [sender='GPT-3.5'] - The sender of the GPT message. Defaults to 'GPT-3.5'.
* @returns {object} The saved message object.
*/
addGptMessage(text, model, sender = 'GPT-3.5') {
const message = this.saveMessage({
text,
sender,
isCreatedByUser: false,
model: model || openAISettings.model.default,
});
return message;
}
/**
* Finishes the current conversation and adds it to the batch.
* @param {string} [title='Imported Chat'] - The title of the conversation. Defaults to 'Imported Chat'.
* @param {Date} [createdAt] - The creation date of the conversation.
* @returns {object} The added conversation object.
*/
finishConversation(title, createdAt) {
const convo = {
user: this.requestUserId,
conversationId: this.conversationId,
title: title || 'Imported Chat',
createdAt: createdAt,
updatedAt: createdAt,
overrideTimestamp: true,
endpoint: this.endpoint,
model: openAISettings.model.default,
};
this.conversations.push(convo);
return convo;
}
/**
* Saves the batch of conversations and messages to the DB.
* @returns {Promise<void>} A promise that resolves when the batch is saved.
* @throws {Error} If there is an error saving the batch.
*/
async saveBatch() {
try {
await bulkSaveConvos(this.conversations);
await bulkSaveMessages(this.messages);
logger.debug(
`user: ${this.requestUserId} | Added ${this.conversations.length} conversations and ${this.messages.length} messages to the DB.`,
);
} catch (error) {
logger.error('Error saving batch', error);
throw error;
}
}
/**
* Saves a message to the current conversation.
* @param {object} messageDetails - The details of the message.
* @param {string} messageDetails.text - The text of the message.
* @param {string} messageDetails.sender - The sender of the message.
* @param {string} [messageDetails.messageId] - The ID of the current message.
* @param {boolean} messageDetails.isCreatedByUser - Indicates whether the message is created by the user.
* @param {string} [messageDetails.model] - The model used for generating the message.
* @param {string} [messageDetails.parentMessageId=this.lastMessageId] - The ID of the parent message.
* @returns {object} The saved message object.
*/
saveMessage({
text,
sender,
isCreatedByUser,
model,
messageId,
parentMessageId = this.lastMessageId,
}) {
const newMessageId = messageId ?? uuidv4();
const message = {
parentMessageId,
messageId: newMessageId,
conversationId: this.conversationId,
isCreatedByUser: isCreatedByUser,
model: model || this.model,
user: this.requestUserId,
endpoint: this.endpoint,
unfinished: false,
isEdited: false,
error: false,
sender,
text,
};
this.lastMessageId = newMessageId;
this.messages.push(message);
return message;
}
}
module.exports = { ImportBatchBuilder, createImportBatchBuilder };