mirror of
https://github.com/danny-avila/LibreChat.git
synced 2025-09-21 21:50:49 +02:00

* Basic implementation of ChatGPT conversation import * remove debug code * Handle citations * Fix updatedAt in import * update default model * Use job scheduler to handle import requests * import job status endpoint * Add wrapper around Agenda * Rate limits for import endpoint * rename import api path * Batch save import to mongo * Improve naming * Add documenting comments * Test for importers * Change button for importing conversations * Frontend changes * Import job status endpoint * Import endpoint response * Add translations to new phrases * Fix conversations refreshing * cleanup unused functions * set timeout for import job status polling * Add documentation * get extra spaces back * Improve error message * Fix translation files after merge * fix translation files 2 * Add zh translation for import functionality * Sync mailisearch index after import * chore: add dummy uri for jest tests, as MONGO_URI should only be real for E2E tests * docs: fix links * docs: fix conversationsImport section * fix: user role issue for librechat imports * refactor: import conversations from json - organize imports - add additional jsdocs - use multer with diskStorage to avoid loading file into memory outside of job - use filepath instead of loading data string for imports - replace console logs and some logger.info() with logger.debug - only use multer for import route * fix: undefined metadata edge case and replace ChatGtp -> ChatGpt * Refactor importChatGptConvo function to handle undefined metadata edge case and replace ChatGtp with ChatGpt * fix: chatgpt importer * feat: maintain tree relationship for librechat messages * chore: use enum * refactor: saveMessage to use single object arg, replace console logs, add userId to log message * chore: additional comment * chore: multer edge case * feat: first pass, maintain tree relationship * chore: organize * chore: remove log * ci: add heirarchy test for chatgpt * ci: test maintaining of heirarchy for librechat * wip: allow non-text content type messages * refactor: import content part object json string * refactor: more content types to format * chore: consolidate messageText formatting * docs: update on changes, bump data-provider/config versions, update readme * refactor(indexSync): singleton pattern for MeiliSearchClient * refactor: debug log after batch is done * chore: add back indexSync error handling --------- Co-authored-by: jakubmieszczak <jakub.mieszczak@zendesk.com> Co-authored-by: Danny Avila <danny@librechat.ai>
149 lines
5 KiB
JavaScript
149 lines
5 KiB
JavaScript
const { v4: uuidv4 } = require('uuid');
|
|
const { EModelEndpoint, Constants, openAISettings } = require('librechat-data-provider');
|
|
const { bulkSaveConvos } = require('~/models/Conversation');
|
|
const { bulkSaveMessages } = require('~/models/Message');
|
|
const { logger } = require('~/config');
|
|
|
|
/**
|
|
* Factory function for creating an instance of ImportBatchBuilder.
|
|
* @param {string} requestUserId - The ID of the user making the request.
|
|
* @returns {ImportBatchBuilder} - The newly created ImportBatchBuilder instance.
|
|
*/
|
|
function createImportBatchBuilder(requestUserId) {
|
|
return new ImportBatchBuilder(requestUserId);
|
|
}
|
|
|
|
/**
|
|
* Class for building a batch of conversations and messages and pushing them to DB for Conversation Import functionality
|
|
*/
|
|
class ImportBatchBuilder {
|
|
/**
|
|
* Creates an instance of ImportBatchBuilder.
|
|
* @param {string} requestUserId - The ID of the user making the import request.
|
|
*/
|
|
constructor(requestUserId) {
|
|
this.requestUserId = requestUserId;
|
|
this.conversations = [];
|
|
this.messages = [];
|
|
}
|
|
|
|
/**
|
|
* Starts a new conversation in the batch.
|
|
* @param {string} [endpoint=EModelEndpoint.openAI] - The endpoint for the conversation. Defaults to EModelEndpoint.openAI.
|
|
* @returns {void}
|
|
*/
|
|
startConversation(endpoint) {
|
|
// we are simplifying by using a single model for the entire conversation
|
|
this.endpoint = endpoint || EModelEndpoint.openAI;
|
|
this.conversationId = uuidv4();
|
|
this.lastMessageId = Constants.NO_PARENT;
|
|
}
|
|
|
|
/**
|
|
* Adds a user message to the current conversation.
|
|
* @param {string} text - The text of the user message.
|
|
* @returns {object} The saved message object.
|
|
*/
|
|
addUserMessage(text) {
|
|
const message = this.saveMessage({ text, sender: 'user', isCreatedByUser: true });
|
|
return message;
|
|
}
|
|
|
|
/**
|
|
* Adds a GPT message to the current conversation.
|
|
* @param {string} text - The text of the GPT message.
|
|
* @param {string} [model='defaultModel'] - The model used for generating the GPT message. Defaults to 'defaultModel'.
|
|
* @param {string} [sender='GPT-3.5'] - The sender of the GPT message. Defaults to 'GPT-3.5'.
|
|
* @returns {object} The saved message object.
|
|
*/
|
|
addGptMessage(text, model, sender = 'GPT-3.5') {
|
|
const message = this.saveMessage({
|
|
text,
|
|
sender,
|
|
isCreatedByUser: false,
|
|
model: model || openAISettings.model.default,
|
|
});
|
|
return message;
|
|
}
|
|
|
|
/**
|
|
* Finishes the current conversation and adds it to the batch.
|
|
* @param {string} [title='Imported Chat'] - The title of the conversation. Defaults to 'Imported Chat'.
|
|
* @param {Date} [createdAt] - The creation date of the conversation.
|
|
* @returns {object} The added conversation object.
|
|
*/
|
|
finishConversation(title, createdAt) {
|
|
const convo = {
|
|
user: this.requestUserId,
|
|
conversationId: this.conversationId,
|
|
title: title || 'Imported Chat',
|
|
createdAt: createdAt,
|
|
updatedAt: createdAt,
|
|
overrideTimestamp: true,
|
|
endpoint: this.endpoint,
|
|
model: openAISettings.model.default,
|
|
};
|
|
this.conversations.push(convo);
|
|
|
|
return convo;
|
|
}
|
|
|
|
/**
|
|
* Saves the batch of conversations and messages to the DB.
|
|
* @returns {Promise<void>} A promise that resolves when the batch is saved.
|
|
* @throws {Error} If there is an error saving the batch.
|
|
*/
|
|
async saveBatch() {
|
|
try {
|
|
await bulkSaveConvos(this.conversations);
|
|
await bulkSaveMessages(this.messages);
|
|
logger.debug(
|
|
`user: ${this.requestUserId} | Added ${this.conversations.length} conversations and ${this.messages.length} messages to the DB.`,
|
|
);
|
|
} catch (error) {
|
|
logger.error('Error saving batch', error);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Saves a message to the current conversation.
|
|
* @param {object} messageDetails - The details of the message.
|
|
* @param {string} messageDetails.text - The text of the message.
|
|
* @param {string} messageDetails.sender - The sender of the message.
|
|
* @param {string} [messageDetails.messageId] - The ID of the current message.
|
|
* @param {boolean} messageDetails.isCreatedByUser - Indicates whether the message is created by the user.
|
|
* @param {string} [messageDetails.model] - The model used for generating the message.
|
|
* @param {string} [messageDetails.parentMessageId=this.lastMessageId] - The ID of the parent message.
|
|
* @returns {object} The saved message object.
|
|
*/
|
|
saveMessage({
|
|
text,
|
|
sender,
|
|
isCreatedByUser,
|
|
model,
|
|
messageId,
|
|
parentMessageId = this.lastMessageId,
|
|
}) {
|
|
const newMessageId = messageId ?? uuidv4();
|
|
const message = {
|
|
parentMessageId,
|
|
messageId: newMessageId,
|
|
conversationId: this.conversationId,
|
|
isCreatedByUser: isCreatedByUser,
|
|
model: model || this.model,
|
|
user: this.requestUserId,
|
|
endpoint: this.endpoint,
|
|
unfinished: false,
|
|
isEdited: false,
|
|
error: false,
|
|
sender,
|
|
text,
|
|
};
|
|
this.lastMessageId = newMessageId;
|
|
this.messages.push(message);
|
|
return message;
|
|
}
|
|
}
|
|
|
|
module.exports = { ImportBatchBuilder, createImportBatchBuilder };
|