LibreChat/api/server/routes/convos.js
Denis Palnitsky ab6fbe48f1
📥 feat: Import Conversations from LibreChat, ChatGPT, Chatbot UI (#2355)
* Basic implementation of ChatGPT conversation import

* remove debug code

* Handle citations

* Fix updatedAt in import

* update default model

* Use job scheduler to handle import requests

* import job status endpoint

* Add wrapper around Agenda

* Rate limits for import endpoint

* rename import api path

* Batch save import to mongo

* Improve naming

* Add documenting comments

* Test for importers

* Change button for importing conversations

* Frontend changes

* Import job status endpoint

* Import endpoint response

* Add translations to new phrases

* Fix conversations refreshing

* cleanup unused functions

* set timeout for import job status polling

* Add documentation

* get extra spaces back

* Improve error message

* Fix translation files after merge

* fix translation files 2

* Add zh translation for import functionality

* Sync mailisearch index after import

* chore: add dummy uri for jest tests, as MONGO_URI should only be real for E2E tests

* docs: fix links

* docs: fix conversationsImport section

* fix: user role issue for librechat imports

* refactor: import conversations from json
- organize imports
- add additional jsdocs
- use multer with diskStorage to avoid loading file into memory outside of job
- use filepath instead of loading data string for imports
- replace console logs and some logger.info() with logger.debug
- only use multer for import route

* fix: undefined metadata edge case and replace ChatGtp -> ChatGpt

* Refactor importChatGptConvo function to handle undefined metadata edge case and replace ChatGtp with ChatGpt

* fix: chatgpt importer

* feat: maintain tree relationship for librechat messages

* chore: use enum

* refactor: saveMessage to use single object arg, replace console logs, add userId to log message

* chore: additional comment

* chore: multer edge case

* feat: first pass, maintain tree relationship

* chore: organize

* chore: remove log

* ci: add heirarchy test for chatgpt

* ci: test maintaining of heirarchy for librechat

* wip: allow non-text content type messages

* refactor: import content part object json string

* refactor: more content types to format

* chore: consolidate messageText formatting

* docs: update on changes, bump data-provider/config versions, update readme

* refactor(indexSync): singleton pattern for MeiliSearchClient

* refactor: debug log after batch is done

* chore: add back indexSync error handling

---------

Co-authored-by: jakubmieszczak <jakub.mieszczak@zendesk.com>
Co-authored-by: Danny Avila <danny@librechat.ai>
2024-05-02 02:48:26 -04:00

154 lines
4.6 KiB
JavaScript

const multer = require('multer');
const express = require('express');
const { CacheKeys } = require('librechat-data-provider');
const { initializeClient } = require('~/server/services/Endpoints/assistants');
const { getConvosByPage, deleteConvos, getConvo, saveConvo } = require('~/models/Conversation');
const { IMPORT_CONVERSATION_JOB_NAME } = require('~/server/utils/import/jobDefinition');
const { storage, importFileFilter } = require('~/server/routes/files/multer');
const requireJwtAuth = require('~/server/middleware/requireJwtAuth');
const { createImportLimiters } = require('~/server/middleware');
const jobScheduler = require('~/server/utils/jobScheduler');
const getLogStores = require('~/cache/getLogStores');
const { sleep } = require('~/server/utils');
const { logger } = require('~/config');
const router = express.Router();
router.use(requireJwtAuth);
router.get('/', async (req, res) => {
let pageNumber = req.query.pageNumber || 1;
pageNumber = parseInt(pageNumber, 10);
if (isNaN(pageNumber) || pageNumber < 1) {
return res.status(400).json({ error: 'Invalid page number' });
}
res.status(200).send(await getConvosByPage(req.user.id, pageNumber));
});
router.get('/:conversationId', async (req, res) => {
const { conversationId } = req.params;
const convo = await getConvo(req.user.id, conversationId);
if (convo) {
res.status(200).json(convo);
} else {
res.status(404).end();
}
});
router.post('/gen_title', async (req, res) => {
const { conversationId } = req.body;
const titleCache = getLogStores(CacheKeys.GEN_TITLE);
const key = `${req.user.id}-${conversationId}`;
let title = await titleCache.get(key);
if (!title) {
await sleep(2500);
title = await titleCache.get(key);
}
if (title) {
await titleCache.delete(key);
res.status(200).json({ title });
} else {
res.status(404).json({
message: 'Title not found or method not implemented for the conversation\'s endpoint',
});
}
});
router.post('/clear', async (req, res) => {
let filter = {};
const { conversationId, source, thread_id } = req.body.arg;
if (conversationId) {
filter = { conversationId };
}
if (source === 'button' && !conversationId) {
return res.status(200).send('No conversationId provided');
}
if (thread_id) {
/** @type {{ openai: OpenAI}} */
const { openai } = await initializeClient({ req, res });
try {
const response = await openai.beta.threads.del(thread_id);
logger.debug('Deleted OpenAI thread:', response);
} catch (error) {
logger.error('Error deleting OpenAI thread:', error);
}
}
// for debugging deletion source
// logger.debug('source:', source);
try {
const dbResponse = await deleteConvos(req.user.id, filter);
res.status(201).json(dbResponse);
} catch (error) {
logger.error('Error clearing conversations', error);
res.status(500).send('Error clearing conversations');
}
});
router.post('/update', async (req, res) => {
const update = req.body.arg;
try {
const dbResponse = await saveConvo(req.user.id, update);
res.status(201).json(dbResponse);
} catch (error) {
logger.error('Error updating conversation', error);
res.status(500).send('Error updating conversation');
}
});
const { importIpLimiter, importUserLimiter } = createImportLimiters();
const upload = multer({ storage: storage, fileFilter: importFileFilter });
/**
* Imports a conversation from a JSON file and saves it to the database.
* @route POST /import
* @param {Express.Multer.File} req.file - The JSON file to import.
* @returns {object} 201 - success response - application/json
*/
router.post(
'/import',
importIpLimiter,
importUserLimiter,
upload.single('file'),
async (req, res) => {
try {
const filepath = req.file.path;
const job = await jobScheduler.now(IMPORT_CONVERSATION_JOB_NAME, filepath, req.user.id);
res.status(201).json({ message: 'Import started', jobId: job.id });
} catch (error) {
logger.error('Error processing file', error);
res.status(500).send('Error processing file');
}
},
);
// Get the status of an import job for polling
router.get('/import/jobs/:jobId', async (req, res) => {
try {
const { jobId } = req.params;
const { userId, ...jobStatus } = await jobScheduler.getJobStatus(jobId);
if (!jobStatus) {
return res.status(404).json({ message: 'Job not found.' });
}
if (userId !== req.user.id) {
return res.status(403).json({ message: 'Unauthorized' });
}
res.json(jobStatus);
} catch (error) {
logger.error('Error getting job details', error);
res.status(500).send('Error getting job details');
}
});
module.exports = router;