Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-12-21 19:00:13 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

⬇️ fix: JSON LibreChat Imports (#2897)

Browse source 
* chore: remove unused code

* refactor: Update NewChatButtonIcon component to use JSX syntax

The NewChatButtonIcon component in the Nav folder has been updated to use JSX syntax instead of calling the Icon function directly. This change improves code readability and maintainability.

* remove use memo

* refactor: allow passing `select` to messages db query

* fix: initial fix for non-recursive messages

* ci: first pass, importers test rewrite

* fix(groupConversationsByDate): handle edge case of conversation.updatedAt being null

* fix: correctly handle non-recursive uploads

* feat: imports non-recursive conversations with branches correctly

* feat: support retaining original options on import

* refactor: Allow `messageTree` field for Import of non-recursive conversations
This commit is contained in:
Danny Avila 2024-05-29 09:15:05 -04:00 • committed by GitHub
parent c9e7d4ac18
commit 3f600f0d3f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 516 additions and 175 deletions
Download patch file Download diff file Expand all files Collapse all files

2
api/models/Conversation.js
View file

@ -21,7 +21,7 @@ module.exports = {
Conversation,
saveConvo: async (user, { conversationId, newConversationId, ...convo }) => {
try {
const messages = await getMessages({ conversationId });
const messages = await getMessages({ conversationId }, '_id');
const update = { ...convo, messages, user };
if (newConversationId) {
update.conversationId = newConversationId;

12
api/models/Message.js
View file

@ -179,8 +179,18 @@ module.exports = {
}
},
async getMessages(filter) {
/**
* Retrieves messages from the database.
* @param {Record<string, unknown>} filter
* @param {string | undefined} [select]
* @returns
*/
async getMessages(filter, select) {
try {
if (select) {
return await Message.find(filter).select(select).sort({ createdAt: 1 }).lean();
}
return await Message.find(filter).sort({ createdAt: 1 }).lean();
} catch (err) {
logger.error('Error getting messages:', err);

4
api/server/routes/messages.js
View file

@ -14,7 +14,7 @@ router.use(requireJwtAuth);
router.get('/:conversationId', validateMessageReq, async (req, res) => {
const { conversationId } = req.params;
res.status(200).send(await getMessages({ conversationId }));
res.status(200).send(await getMessages({ conversationId }, '-_id -__v -user'));
});
// CREATE
@ -28,7 +28,7 @@ router.post('/:conversationId', validateMessageReq, async (req, res) => {
// READ
router.get('/:conversationId/:messageId', validateMessageReq, async (req, res) => {
const { conversationId, messageId } = req.params;
res.status(200).send(await getMessages({ conversationId, messageId }));
res.status(200).send(await getMessages({ conversationId, messageId }, '-_id -__v -user'));
});
// UPDATE

2
api/server/utils/import/__data__/librechat-export.json
View file

@ -19,7 +19,7 @@
"endpoint": "openAI",
"title": "VW Transporter 2014 Fuel Consumption. Web Search"
},
"messagesTree": [
"messages": [
{
"_id": "6615516574dc2ddcdebe40b6",
"messageId": "b123942f-ca1a-4b16-9e1f-ea4af5171168",

40
api/server/utils/import/__data__/librechat-linear.json Normal file
View file

@ -0,0 +1,40 @@
{
"conversationId": "4c9e1402-f746-479d-8a18-2dfab1bf750f",
"endpoint": "azureOpenAI",
"title": "Adventure Across Time and Space",
"exportAt": "20:33:31 GMT-0400 (Eastern Daylight Time)",
"branches": false,
"recursive": false,
"messages": [
{
"messageId": "ed3fdaaa-e174-4d70-89a9-b06e70939723",
"conversationId": "4c9e1402-f746-479d-8a18-2dfab1bf750f",
"createdAt": "2024-05-29T00:33:03.661Z",
"endpoint": "azureOpenAI",
"error": false,
"isCreatedByUser": true,
"isEdited": false,
"model": null,
"parentMessageId": null,
"sender": "user",
"text": "tell me a long story",
"unfinished": false,
"updatedAt": "2024-05-29T00:33:03.661Z"
},
{
"messageId": "113b8336-4b19-4f91-8eed-741b3dc2940e",
"conversationId": "4c9e1402-f746-479d-8a18-2dfab1bf750f",
"createdAt": "2024-05-29T00:33:03.661Z",
"endpoint": "azureOpenAI",
"error": false,
"isCreatedByUser": false,
"isEdited": false,
"model": "gpt-4o",
"parentMessageId": "ed3fdaaa-e174-4d70-89a9-b06e70939723",
"sender": "GPT-4",
"text": "Of course! Here’s a detailed and engaging story:\n\n---\n\n### The Legend of Eldoria\n\nNestled between towering mountains and dense, ancient forests was the enigmatic kingdom of Eldoria. This realm, clo aked in perpetual twilight, was the stuff of legends. It was said that the land was blessed by the gods and guarded by mythical creatures. Eldoria was a place where magic and realism intertwined seamlessly, creating a land of beauty, wonder, and peril.\n\nIn the heart of this kingdom lay the grand city of Lumina, known",
"unfinished": false,
"updatedAt": "2024-05-29T00:33:03.661Z"
}
]
}

104
api/server/utils/import/__data__/librechat-opts-nonr-branches.json Normal file
View file

@ -0,0 +1,104 @@
{
"conversationId": "27b593be-9500-479c-94cb-050cab8f5033",
"endpoint": "azureOpenAI",
"title": "Original",
"exportAt": "08:12:16 GMT-0400 (Eastern Daylight Time)",
"branches": true,
"recursive": false,
"options": {
"presetId": null,
"model": "gpt-4o",
"chatGptLabel": null,
"promptPrefix": null,
"temperature": 1,
"top_p": 1,
"presence_penalty": 0,
"frequency_penalty": 0,
"resendFiles": true,
"imageDetail": "auto",
"endpoint": "azureOpenAI",
"title": "Original"
},
"messages": [
{
"messageId": "115a6247-8fb0-4937-a536-12956669098d",
"conversationId": "27b593be-9500-479c-94cb-050cab8f5033",
"createdAt": "2024-05-28T18:08:55.014Z",
"endpoint": "azureOpenAI",
"error": false,
"isCreatedByUser": true,
"isEdited": false,
"model": null,
"parentMessageId": "00000000-0000-0000-0000-000000000000",
"sender": "User",
"text": "tell me a long story",
"tokenCount": 9,
"unfinished": false,
"updatedAt": "2024-05-28T18:09:27.193Z"
},
{
"messageId": "069b9c22-7649-45a9-b90b-fc050533ea21",
"conversationId": "27b593be-9500-479c-94cb-050cab8f5033",
"createdAt": "2024-05-28T18:08:55.390Z",
"error": false,
"isCreatedByUser": false,
"isEdited": false,
"model": "gpt-4o",
"parentMessageId": "115a6247-8fb0-4937-a536-12956669098d",
"sender": "GPT-4",
"text": "Of course! Settle in for a tale of adventure across time and space.\n\n---\n\nOnce upon a time in the small, sleepy village of Eldoria, there was a young woman named Elara who longed for adventure. Eldoria was a place of routine and simplicity, nestled between rolling hills and dense forests, but Elara always felt that there was more to the world than the boundaries",
"unfinished": false,
"updatedAt": "2024-05-28T18:08:58.669Z",
"endpoint": "azureOpenAI",
"tokenCount": 78,
"finish_reason": "incomplete"
},
{
"messageId": "303e4c2c-f03e-4e0a-8551-c96ec73be5fe",
"conversationId": "27b593be-9500-479c-94cb-050cab8f5033",
"createdAt": "2024-05-28T18:09:27.444Z",
"error": false,
"isCreatedByUser": false,
"isEdited": false,
"model": "gpt-4o",
"parentMessageId": "115a6247-8fb0-4937-a536-12956669098d",
"sender": "GPT-4",
"text": "Sure, I can craft a long story for you. Here it goes:\n\n### The Chronicles of Elenor: The Luminary of Anduril\n\nIn an age long forgotten by men, in a world kissed by the glow of dual suns, the Kingdom of Anduril flourished. Verdant valleys graced its land, majestic mountains shielded",
"unfinished": true,
"updatedAt": "2024-05-28T18:09:30.448Z"
},
{
"messageId": "599e1908-8c52-4a73-ba6b-f6dffbd79ba0",
"conversationId": "27b593be-9500-479c-94cb-050cab8f5033",
"createdAt": "2024-05-28T18:14:07.988Z",
"endpoint": "azureOpenAI",
"error": false,
"isCreatedByUser": true,
"isEdited": false,
"model": null,
"parentMessageId": "00000000-0000-0000-0000-000000000000",
"sender": "User",
"text": "tell me a long long story",
"tokenCount": 9,
"unfinished": false,
"updatedAt": "2024-05-28T18:14:07.988Z"
},
{
"messageId": "de9a4e7c-020d-4856-a5a6-ce6794efef99",
"conversationId": "27b593be-9500-479c-94cb-050cab8f5033",
"createdAt": "2024-05-28T18:14:08.403Z",
"error": false,
"isCreatedByUser": false,
"isEdited": true,
"model": "gpt-4o",
"parentMessageId": "599e1908-8c52-4a73-ba6b-f6dffbd79ba0",
"sender": "GPT-4",
"text": "Of course! Here’s a detailed and engaging story:\n\n---\n\n### The Legend of Eldoria\n\nNestled between towering mountains and dense, ancient forests was the enigmatic kingdom of Eldoria. This realm, clo aked in perpetual twilight, was the stuff of legends. It was said that the land was blessed by the gods and guarded by mythical creatures. Eldoria was a place where magic and realism intertwined seamlessly, creating a land of beauty, wonder, and peril.\n\nIn the heart of this kingdom lay the grand city of Lumina, known",
"unfinished": false,
"updatedAt": "2024-05-28T18:14:20.349Z",
"endpoint": "azureOpenAI",
"finish_reason": "incomplete",
"tokenCount": 110
}
]
}

2
api/server/utils/import/__data__/librechat-tree.json
View file

@ -19,7 +19,7 @@
"endpoint": "openAI",
"title": "Troubleshooting Python Virtual Environment Activation Issue"
},
"messagesTree": [
"messages": [
{
"_id": "66326f3f04bed94b7f5be68d",
"messageId": "9501f99d-9bbb-40cb-bbb2-16d79aeceb72",

70
api/server/utils/import/importers.js
View file

@ -1,6 +1,7 @@
const { v4: uuidv4 } = require('uuid');
const { EModelEndpoint, Constants, openAISettings } = require('librechat-data-provider');
const { EModelEndpoint, Constants, openAISettings, CacheKeys } = require('librechat-data-provider');
const { createImportBatchBuilder } = require('./importBatchBuilder');
const getLogStores = require('~/cache/getLogStores');
const logger = require('~/config/winston');
/**
@ -24,7 +25,7 @@ function getImporter(jsonData) {
}
// For LibreChat
if (jsonData.conversationId && jsonData.messagesTree) {
if (jsonData.conversationId && (jsonData.messagesTree || jsonData.messages)) {
logger.info('Importing LibreChat conversation');
return importLibreChatConvo;
}
@ -85,30 +86,51 @@ async function importLibreChatConvo(
try {
/** @type {ImportBatchBuilder} */
const importBatchBuilder = builderFactory(requestUserId);
importBatchBuilder.startConversation(EModelEndpoint.openAI);
const options = jsonData.options || {};
/* Endpoint configuration */
let endpoint = jsonData.endpoint ?? options.endpoint ?? EModelEndpoint.openAI;
const cache = getLogStores(CacheKeys.CONFIG_STORE);
const endpointsConfig = await cache.get(CacheKeys.ENDPOINT_CONFIG);
const endpointConfig = endpointsConfig?.[endpoint];
if (!endpointConfig && endpointsConfig) {
endpoint = Object.keys(endpointsConfig)[0];
} else if (!endpointConfig) {
endpoint = EModelEndpoint.openAI;
}
importBatchBuilder.startConversation(endpoint);
let firstMessageDate = null;
const traverseMessages = (messages, parentMessageId = null) => {
const messagesToImport = jsonData.messagesTree || jsonData.messages;
if (jsonData.recursive) {
/**
* Recursively traverse the messages tree and save each message to the database.
* @param {TMessage[]} messages
* @param {string} parentMessageId
*/
const traverseMessages = async (messages, parentMessageId = null) => {
for (const message of messages) {
if (!message.text) {
continue;
}
let savedMessage;
if (message.sender?.toLowerCase() === 'user') {
savedMessage = importBatchBuilder.saveMessage({
if (message.sender?.toLowerCase() === 'user' || message.isCreatedByUser) {
savedMessage = await importBatchBuilder.saveMessage({
text: message.text,
sender: 'user',
isCreatedByUser: true,
parentMessageId: parentMessageId,
});
} else {
savedMessage = importBatchBuilder.saveMessage({
savedMessage = await importBatchBuilder.saveMessage({
text: message.text,
sender: message.sender,
isCreatedByUser: false,
model: jsonData.options.model,
model: options.model,
parentMessageId: parentMessageId,
});
}
@ -117,15 +139,39 @@ async function importLibreChatConvo(
firstMessageDate = new Date(message.createdAt);
}
if (message.children) {
traverseMessages(message.children, savedMessage.messageId);
if (message.children && message.children.length > 0) {
await traverseMessages(message.children, savedMessage.messageId);
}
}
};
traverseMessages(jsonData.messagesTree);
await traverseMessages(messagesToImport);
} else if (messagesToImport) {
const idMapping = new Map();
importBatchBuilder.finishConversation(jsonData.title, firstMessageDate);
for (const message of messagesToImport) {
if (!firstMessageDate) {
firstMessageDate = new Date(message.createdAt);
}
const newMessageId = uuidv4();
idMapping.set(message.messageId, newMessageId);
const clonedMessage = {
...message,
messageId: newMessageId,
parentMessageId:
message.parentMessageId && message.parentMessageId !== Constants.NO_PARENT
? idMapping.get(message.parentMessageId) || Constants.NO_PARENT
: Constants.NO_PARENT,
};
importBatchBuilder.saveMessage(clonedMessage);
}
} else {
throw new Error('Invalid LibreChat file format');
}
importBatchBuilder.finishConversation(jsonData.title, firstMessageDate ?? new Date(), options);
await importBatchBuilder.saveBatch();
logger.debug(`user: ${requestUserId} | Conversation "${jsonData.title}" imported`);
} catch (error) {

358
api/server/utils/import/importers.spec.js
View file

@ -1,70 +1,68 @@
const fs = require('fs');
const path = require('path');
const { EModelEndpoint, Constants } = require('librechat-data-provider');
const { EModelEndpoint, Constants, openAISettings } = require('librechat-data-provider');
const { bulkSaveConvos: _bulkSaveConvos } = require('~/models/Conversation');
const { ImportBatchBuilder } = require('./importBatchBuilder');
const { bulkSaveMessages } = require('~/models/Message');
const getLogStores = require('~/cache/getLogStores');
const { getImporter } = require('./importers');
// Mocking the ImportBatchBuilder class and its methods
jest.mock('./importBatchBuilder', () => {
return {
ImportBatchBuilder: jest.fn().mockImplementation(() => {
return {
startConversation: jest.fn().mockResolvedValue(undefined),
addUserMessage: jest.fn().mockResolvedValue(undefined),
addGptMessage: jest.fn().mockResolvedValue(undefined),
saveMessage: jest.fn().mockResolvedValue(undefined),
finishConversation: jest.fn().mockResolvedValue(undefined),
saveBatch: jest.fn().mockResolvedValue(undefined),
};
}),
};
jest.mock('~/cache/getLogStores');
const mockedCacheGet = jest.fn();
getLogStores.mockImplementation(() => ({
get: mockedCacheGet,
}));
// Mock the database methods
jest.mock('~/models/Conversation', () => ({
bulkSaveConvos: jest.fn(),
}));
jest.mock('~/models/Message', () => ({
bulkSaveMessages: jest.fn(),
}));
afterEach(() => {
jest.clearAllMocks();
});
describe('importChatGptConvo', () => {
it('should import conversation correctly', async () => {
const expectedNumberOfMessages = 19;
const expectedNumberOfConversations = 2;
// Given
const jsonData = JSON.parse(
fs.readFileSync(path.join(__dirname, '__data__', 'chatgpt-export.json'), 'utf8'),
);
const requestUserId = 'user-123';
const mockedBuilderFactory = jest.fn().mockReturnValue(new ImportBatchBuilder(requestUserId));
const importBatchBuilder = new ImportBatchBuilder(requestUserId);
// Spy on instance methods
jest.spyOn(importBatchBuilder, 'startConversation');
jest.spyOn(importBatchBuilder, 'saveMessage');
jest.spyOn(importBatchBuilder, 'finishConversation');
jest.spyOn(importBatchBuilder, 'saveBatch');
// When
const importer = getImporter(jsonData);
await importer(jsonData, requestUserId, mockedBuilderFactory);
await importer(jsonData, requestUserId, () => importBatchBuilder);
// Then
expect(mockedBuilderFactory).toHaveBeenCalledWith(requestUserId);
const mockImportBatchBuilder = mockedBuilderFactory.mock.results[0].value;
expect(mockImportBatchBuilder.startConversation).toHaveBeenCalledWith(EModelEndpoint.openAI);
expect(mockImportBatchBuilder.saveMessage).toHaveBeenCalledTimes(expectedNumberOfMessages); // Adjust expected number
expect(mockImportBatchBuilder.finishConversation).toHaveBeenCalledTimes(
expectedNumberOfConversations,
); // Adjust expected number
expect(mockImportBatchBuilder.saveBatch).toHaveBeenCalled();
expect(importBatchBuilder.startConversation).toHaveBeenCalledWith(EModelEndpoint.openAI);
expect(importBatchBuilder.saveMessage).toHaveBeenCalledTimes(expectedNumberOfMessages);
expect(importBatchBuilder.finishConversation).toHaveBeenCalledTimes(jsonData.length);
expect(importBatchBuilder.saveBatch).toHaveBeenCalled();
});
it('should maintain correct message hierarchy (tree parent/children relationship)', async () => {
// Prepare test data with known hierarchy
const jsonData = JSON.parse(
fs.readFileSync(path.join(__dirname, '__data__', 'chatgpt-tree.json'), 'utf8'),
);
const requestUserId = 'user-123';
const mockedBuilderFactory = jest.fn().mockReturnValue(new ImportBatchBuilder(requestUserId));
const importBatchBuilder = new ImportBatchBuilder(requestUserId);
jest.spyOn(importBatchBuilder, 'saveMessage');
jest.spyOn(importBatchBuilder, 'saveBatch');
// When
const importer = getImporter(jsonData);
await importer(jsonData, requestUserId, mockedBuilderFactory);
// Then
expect(mockedBuilderFactory).toHaveBeenCalledWith(requestUserId);
const mockImportBatchBuilder = mockedBuilderFactory.mock.results[0].value;
await importer(jsonData, requestUserId, () => importBatchBuilder);
const entries = Object.keys(jsonData[0].mapping);
// Filter entries that should be processed (not system and have content)
const messageEntries = entries.filter(
(id) =>
jsonData[0].mapping[id].message &&
@ -72,20 +70,16 @@ describe('importChatGptConvo', () => {
jsonData[0].mapping[id].message.content,
);
// Expect the saveMessage to be called for each valid entry
expect(mockImportBatchBuilder.saveMessage).toHaveBeenCalledTimes(messageEntries.length);
expect(importBatchBuilder.saveMessage).toHaveBeenCalledTimes(messageEntries.length);
const idToUUIDMap = new Map();
// Map original IDs to dynamically generated UUIDs
mockImportBatchBuilder.saveMessage.mock.calls.forEach((call, index) => {
importBatchBuilder.saveMessage.mock.calls.forEach((call, index) => {
const originalId = messageEntries[index];
idToUUIDMap.set(originalId, call[0].messageId);
});
// Validate the UUID map contains all expected entries
expect(idToUUIDMap.size).toBe(messageEntries.length);
// Validate correct parent-child relationships
messageEntries.forEach((id) => {
const { parent } = jsonData[0].mapping[id];
@ -93,72 +87,110 @@ describe('importChatGptConvo', () => {
? idToUUIDMap.get(parent) ?? Constants.NO_PARENT
: Constants.NO_PARENT;
const actualParentId = idToUUIDMap.get(id)
? mockImportBatchBuilder.saveMessage.mock.calls.find(
(call) => call[0].messageId === idToUUIDMap.get(id),
const actualMessageId = idToUUIDMap.get(id);
const actualParentId = actualMessageId
? importBatchBuilder.saveMessage.mock.calls.find(
(call) => call[0].messageId === actualMessageId,
)[0].parentMessageId
: Constants.NO_PARENT;
expect(actualParentId).toBe(expectedParentId);
});
expect(mockImportBatchBuilder.saveBatch).toHaveBeenCalled();
expect(importBatchBuilder.saveBatch).toHaveBeenCalled();
});
});
describe('importLibreChatConvo', () => {
it('should import conversation correctly', async () => {
const expectedNumberOfMessages = 6;
const expectedNumberOfConversations = 1;
const jsonDataNonRecursiveBranches = JSON.parse(
fs.readFileSync(path.join(__dirname, '__data__', 'librechat-opts-nonr-branches.json'), 'utf8'),
);
// Given
it('should import conversation correctly', async () => {
mockedCacheGet.mockResolvedValue({
[EModelEndpoint.openAI]: {},
});
const expectedNumberOfMessages = 6;
const jsonData = JSON.parse(
fs.readFileSync(path.join(__dirname, '__data__', 'librechat-export.json'), 'utf8'),
);
const requestUserId = 'user-123';
const mockedBuilderFactory = jest.fn().mockReturnValue(new ImportBatchBuilder(requestUserId));
const importBatchBuilder = new ImportBatchBuilder(requestUserId);
// Spy on instance methods
jest.spyOn(importBatchBuilder, 'startConversation');
jest.spyOn(importBatchBuilder, 'saveMessage');
jest.spyOn(importBatchBuilder, 'finishConversation');
jest.spyOn(importBatchBuilder, 'saveBatch');
// When
const importer = getImporter(jsonData);
await importer(jsonData, requestUserId, mockedBuilderFactory);
await importer(jsonData, requestUserId, () => importBatchBuilder);
// Then
const mockImportBatchBuilder = mockedBuilderFactory.mock.results[0].value;
expect(mockImportBatchBuilder.startConversation).toHaveBeenCalledWith(EModelEndpoint.openAI);
expect(mockImportBatchBuilder.saveMessage).toHaveBeenCalledTimes(expectedNumberOfMessages); // Adjust expected number
expect(mockImportBatchBuilder.finishConversation).toHaveBeenCalledTimes(
expectedNumberOfConversations,
); // Adjust expected number
expect(mockImportBatchBuilder.saveBatch).toHaveBeenCalled();
expect(importBatchBuilder.startConversation).toHaveBeenCalledWith(EModelEndpoint.openAI);
expect(importBatchBuilder.saveMessage).toHaveBeenCalledTimes(expectedNumberOfMessages);
expect(importBatchBuilder.finishConversation).toHaveBeenCalledTimes(1);
expect(importBatchBuilder.saveBatch).toHaveBeenCalled();
});
it('should import linear, non-recursive thread correctly with correct endpoint', async () => {
mockedCacheGet.mockResolvedValue({
[EModelEndpoint.azureOpenAI]: {},
});
const jsonData = JSON.parse(
fs.readFileSync(path.join(__dirname, '__data__', 'librechat-linear.json'), 'utf8'),
);
const requestUserId = 'user-123';
const importBatchBuilder = new ImportBatchBuilder(requestUserId);
jest.spyOn(importBatchBuilder, 'startConversation');
jest.spyOn(importBatchBuilder, 'saveMessage');
jest.spyOn(importBatchBuilder, 'finishConversation');
jest.spyOn(importBatchBuilder, 'saveBatch');
const importer = getImporter(jsonData);
await importer(jsonData, requestUserId, () => importBatchBuilder);
expect(bulkSaveMessages).toHaveBeenCalledTimes(1);
const messages = bulkSaveMessages.mock.calls[0][0];
let lastMessageId = Constants.NO_PARENT;
for (const message of messages) {
expect(message.parentMessageId).toBe(lastMessageId);
lastMessageId = message.messageId;
}
expect(importBatchBuilder.startConversation).toHaveBeenCalledWith(EModelEndpoint.azureOpenAI);
expect(importBatchBuilder.saveMessage).toHaveBeenCalledTimes(jsonData.messages.length);
expect(importBatchBuilder.finishConversation).toHaveBeenCalled();
expect(importBatchBuilder.saveBatch).toHaveBeenCalled();
});
it('should maintain correct message hierarchy (tree parent/children relationship)', async () => {
// Load test data
const jsonData = JSON.parse(
fs.readFileSync(path.join(__dirname, '__data__', 'librechat-tree.json'), 'utf8'),
);
const requestUserId = 'user-123';
const mockedBuilderFactory = jest.fn().mockReturnValue(new ImportBatchBuilder(requestUserId));
const importBatchBuilder = new ImportBatchBuilder(requestUserId);
jest.spyOn(importBatchBuilder, 'saveMessage');
jest.spyOn(importBatchBuilder, 'saveBatch');
// When
const importer = getImporter(jsonData);
await importer(jsonData, requestUserId, mockedBuilderFactory);
// Then
const mockImportBatchBuilder = mockedBuilderFactory.mock.results[0].value;
await importer(jsonData, requestUserId, () => importBatchBuilder);
// Create a map to track original message IDs to new UUIDs
const idToUUIDMap = new Map();
mockImportBatchBuilder.saveMessage.mock.calls.forEach((call) => {
importBatchBuilder.saveMessage.mock.calls.forEach((call) => {
const message = call[0];
idToUUIDMap.set(message.originalMessageId, message.messageId);
});
// Function to recursively check children
const checkChildren = (children, parentId) => {
children.forEach((child) => {
const childUUID = idToUUIDMap.get(child.messageId);
const expectedParentId = idToUUIDMap.get(parentId) ?? null;
const messageCall = mockImportBatchBuilder.saveMessage.mock.calls.find(
const messageCall = importBatchBuilder.saveMessage.mock.calls.find(
(call) => call[0].messageId === childUUID,
);
@ -172,75 +204,203 @@ describe('importLibreChatConvo', () => {
};
// Start hierarchy validation from root messages
checkChildren(jsonData.messagesTree, null); // Assuming root messages have no parent
checkChildren(jsonData.messages, null);
expect(mockImportBatchBuilder.saveBatch).toHaveBeenCalled();
expect(importBatchBuilder.saveBatch).toHaveBeenCalled();
});
it('should maintain correct message hierarchy (non-recursive)', async () => {
const jsonData = jsonDataNonRecursiveBranches;
const requestUserId = 'user-123';
const importBatchBuilder = new ImportBatchBuilder(requestUserId);
jest.spyOn(importBatchBuilder, 'saveMessage');
jest.spyOn(importBatchBuilder, 'saveBatch');
const importer = getImporter(jsonData);
await importer(jsonData, requestUserId, () => importBatchBuilder);
const textToMessageMap = new Map();
importBatchBuilder.saveMessage.mock.calls.forEach((call) => {
const message = call[0];
textToMessageMap.set(message.text, message);
});
const relationships = {
'tell me a long story': [
'Of course! Settle in for a tale of adventure across time and space.\n\n---\n\nOnce upon a time in the small, sleepy village of Eldoria, there was a young woman named Elara who longed for adventure. Eldoria was a place of routine and simplicity, nestled between rolling hills and dense forests, but Elara always felt that there was more to the world than the boundaries',
'Sure, I can craft a long story for you. Here it goes:\n\n### The Chronicles of Elenor: The Luminary of Anduril\n\nIn an age long forgotten by men, in a world kissed by the glow of dual suns, the Kingdom of Anduril flourished. Verdant valleys graced its land, majestic mountains shielded',
],
'tell me a long long story': [
'Of course! Here’s a detailed and engaging story:\n\n---\n\n### The Legend of Eldoria\n\nNestled between towering mountains and dense, ancient forests was the enigmatic kingdom of Eldoria. This realm, clo aked in perpetual twilight, was the stuff of legends. It was said that the land was blessed by the gods and guarded by mythical creatures. Eldoria was a place where magic and realism intertwined seamlessly, creating a land of beauty, wonder, and peril.\n\nIn the heart of this kingdom lay the grand city of Lumina, known',
],
};
Object.keys(relationships).forEach((parentText) => {
const parentMessage = textToMessageMap.get(parentText);
const childrenTexts = relationships[parentText];
childrenTexts.forEach((childText) => {
const childMessage = textToMessageMap.get(childText);
expect(childMessage.parentMessageId).toBe(parentMessage.messageId);
});
});
expect(importBatchBuilder.saveBatch).toHaveBeenCalled();
});
it('should retain properties from the original conversation as well as new settings', async () => {
mockedCacheGet.mockResolvedValue({
[EModelEndpoint.azureOpenAI]: {},
});
const requestUserId = 'user-123';
const importBatchBuilder = new ImportBatchBuilder(requestUserId);
jest.spyOn(importBatchBuilder, 'finishConversation');
const importer = getImporter(jsonDataNonRecursiveBranches);
await importer(jsonDataNonRecursiveBranches, requestUserId, () => importBatchBuilder);
expect(importBatchBuilder.finishConversation).toHaveBeenCalledTimes(1);
const [_title, createdAt, originalConvo] = importBatchBuilder.finishConversation.mock.calls[0];
const convo = importBatchBuilder.conversations[0];
expect(convo).toEqual({
...jsonDataNonRecursiveBranches.options,
user: requestUserId,
conversationId: importBatchBuilder.conversationId,
title: originalConvo.title || 'Imported Chat',
createdAt: createdAt,
updatedAt: createdAt,
overrideTimestamp: true,
endpoint: importBatchBuilder.endpoint,
model: originalConvo.model || openAISettings.model.default,
});
expect(convo.title).toBe('Original');
expect(convo.createdAt).toBeInstanceOf(Date);
expect(convo.endpoint).toBe(EModelEndpoint.azureOpenAI);
expect(convo.model).toBe('gpt-4o');
});
describe('finishConversation', () => {
it('should retain properties from the original conversation as well as update with new settings', () => {
const requestUserId = 'user-123';
const builder = new ImportBatchBuilder(requestUserId);
builder.conversationId = 'conv-id-123';
builder.messages = [{ text: 'Hello, world!' }];
const originalConvo = {
_id: 'old-convo-id',
model: 'custom-model',
};
builder.endpoint = 'test-endpoint';
const title = 'New Chat Title';
const createdAt = new Date('2023-10-01T00:00:00Z');
const result = builder.finishConversation(title, createdAt, originalConvo);
expect(result).toEqual({
conversation: {
user: requestUserId,
conversationId: builder.conversationId,
title: 'New Chat Title',
createdAt: createdAt,
updatedAt: createdAt,
overrideTimestamp: true,
endpoint: 'test-endpoint',
model: 'custom-model',
},
messages: builder.messages,
});
expect(builder.conversations).toContainEqual({
user: requestUserId,
conversationId: builder.conversationId,
title: 'New Chat Title',
createdAt: createdAt,
updatedAt: createdAt,
overrideTimestamp: true,
endpoint: 'test-endpoint',
model: 'custom-model',
});
});
it('should use default values if not provided in the original conversation or as parameters', () => {
const requestUserId = 'user-123';
const builder = new ImportBatchBuilder(requestUserId);
builder.conversationId = 'conv-id-123';
builder.messages = [{ text: 'Hello, world!' }];
builder.endpoint = 'test-endpoint';
const result = builder.finishConversation();
expect(result.conversation.title).toBe('Imported Chat');
expect(result.conversation.model).toBe(openAISettings.model.default);
});
});
});
describe('importChatBotUiConvo', () => {
it('should import custom conversation correctly', async () => {
// Given
const jsonData = JSON.parse(
fs.readFileSync(path.join(__dirname, '__data__', 'chatbotui-export.json'), 'utf8'),
);
const requestUserId = 'custom-user-456';
const mockedBuilderFactory = jest.fn().mockReturnValue(new ImportBatchBuilder(requestUserId));
const importBatchBuilder = new ImportBatchBuilder(requestUserId);
// Spy on instance methods
jest.spyOn(importBatchBuilder, 'startConversation');
jest.spyOn(importBatchBuilder, 'saveMessage');
jest.spyOn(importBatchBuilder, 'addUserMessage');
jest.spyOn(importBatchBuilder, 'addGptMessage');
jest.spyOn(importBatchBuilder, 'finishConversation');
jest.spyOn(importBatchBuilder, 'saveBatch');
// When
const importer = getImporter(jsonData);
await importer(jsonData, requestUserId, mockedBuilderFactory);
await importer(jsonData, requestUserId, () => importBatchBuilder);
// Then
const mockImportBatchBuilder = mockedBuilderFactory.mock.results[0].value;
expect(mockImportBatchBuilder.startConversation).toHaveBeenCalledWith('openAI');
// User messages
expect(mockImportBatchBuilder.addUserMessage).toHaveBeenCalledTimes(3);
expect(mockImportBatchBuilder.addUserMessage).toHaveBeenNthCalledWith(
expect(importBatchBuilder.startConversation).toHaveBeenCalledWith(EModelEndpoint.openAI);
expect(importBatchBuilder.addUserMessage).toHaveBeenCalledTimes(3);
expect(importBatchBuilder.addUserMessage).toHaveBeenNthCalledWith(
1,
'Hello what are you able to do?',
);
expect(mockImportBatchBuilder.addUserMessage).toHaveBeenNthCalledWith(
expect(importBatchBuilder.addUserMessage).toHaveBeenNthCalledWith(
3,
'Give me the code that inverts binary tree in COBOL',
);
// GPT messages
expect(mockImportBatchBuilder.addGptMessage).toHaveBeenCalledTimes(3);
expect(mockImportBatchBuilder.addGptMessage).toHaveBeenNthCalledWith(
expect(importBatchBuilder.addGptMessage).toHaveBeenCalledTimes(3);
expect(importBatchBuilder.addGptMessage).toHaveBeenNthCalledWith(
1,
expect.stringMatching(/^Hello! As an AI developed by OpenAI/),
'gpt-4-1106-preview',
);
expect(mockImportBatchBuilder.addGptMessage).toHaveBeenNthCalledWith(
expect(importBatchBuilder.addGptMessage).toHaveBeenNthCalledWith(
3,
expect.stringContaining('```cobol'),
'gpt-3.5-turbo',
);
expect(mockImportBatchBuilder.finishConversation).toHaveBeenCalledTimes(2);
expect(mockImportBatchBuilder.finishConversation).toHaveBeenNthCalledWith(
expect(importBatchBuilder.finishConversation).toHaveBeenCalledTimes(2);
expect(importBatchBuilder.finishConversation).toHaveBeenNthCalledWith(
1,
'Hello what are you able to do?',
expect.any(Date),
);
expect(mockImportBatchBuilder.finishConversation).toHaveBeenNthCalledWith(
expect(importBatchBuilder.finishConversation).toHaveBeenNthCalledWith(
2,
'Give me the code that inverts ...',
expect.any(Date),
);
expect(mockImportBatchBuilder.saveBatch).toHaveBeenCalled();
expect(importBatchBuilder.saveBatch).toHaveBeenCalled();
});
});
describe('getImporter', () => {
it('should throw an error if the import type is not supported', () => {
// Given
const jsonData = { unsupported: 'data' };
// When
expect(() => getImporter(jsonData)).toThrow('Unsupported import type');
});
});

17
client/src/components/Nav/NavLinks.tsx
View file

@ -1,10 +1,8 @@
import { useLocation } from 'react-router-dom';
import { Fragment, useState, memo } from 'react';
import { FileText } from 'lucide-react';
import { useRecoilState } from 'recoil';
import { Fragment, useState, memo } from 'react';
import { Menu, Transition } from '@headlessui/react';
import { useRecoilValue, useRecoilState } from 'recoil';
import { useGetUserBalance, useGetStartupConfig } from 'librechat-data-provider/react-query';
import type { TConversation } from 'librechat-data-provider';
import FilesView from '~/components/Chat/Input/Files/FilesView';
import { useAuthContext } from '~/hooks/AuthContext';
import useAvatar from '~/hooks/Messages/useAvatar';
@ -19,7 +17,6 @@ import store from '~/store';
function NavLinks() {
const localize = useLocalize();
const location = useLocation();
const { user, isAuthenticated } = useAuthContext();
const { data: startupConfig } = useGetStartupConfig();
const balanceQuery = useGetUserBalance({
@ -28,18 +25,8 @@ function NavLinks() {
const [showSettings, setShowSettings] = useState(false);
const [showFiles, setShowFiles] = useRecoilState(store.showFiles);
const activeConvo = useRecoilValue(store.conversationByIndex(0));
const globalConvo = useRecoilValue(store.conversation) ?? ({} as TConversation);
const avatarSrc = useAvatar(user);
let conversation: TConversation | null | undefined;
if (location.state?.from?.pathname.includes('/chat')) {
conversation = globalConvo;
} else {
conversation = activeConvo;
}
return (
<>
<Menu as="div" className="group relative">

20
client/src/components/Nav/NewChat.tsx
View file

@ -38,16 +38,16 @@ const NewChatButtonIcon = ({ conversation }: { conversation: TConversation | nul
<ConvoIconURL preset={conversation} endpointIconURL={iconURL} context="nav" />
) : (
<div className="shadow-stroke relative flex h-full items-center justify-center rounded-full bg-white text-black dark:bg-white">
{endpoint &&
Icon &&
Icon({
size: 41,
context: 'nav',
className: 'h-2/3 w-2/3',
endpoint,
endpointType,
iconURL: endpointIconURL,
})}
{endpoint && Icon && (
<Icon
size={41}
context="nav"
className="h-2/3 w-2/3"
endpoint={endpoint}
endpointType={endpointType}
iconURL={endpointIconURL}
/>
)}
</div>
)}
</div>

9
client/src/components/ui/Combobox.tsx
View file

@ -35,19 +35,12 @@ export default function ComboboxComponent({
isCollapsed: boolean;
SelectIcon?: React.ReactNode;
}) {
const options: OptionWithIcon[] = useMemo(() => {
if (!items) {
return [];
}
return items.map((option: string | OptionWithIcon) => {
const options: OptionWithIcon[] = (items ?? []).map((option: string | OptionWithIcon) => {
if (typeof option === 'string') {
return { label: option, value: option };
}
return option;
});
}, [items]);
const { open, setOpen, setSearchValue, matches } = useCombobox({
value: selectedValue,

3
client/src/utils/convos.ts
View file

@ -6,6 +6,7 @@ import {
parseISO,
startOfDay,
startOfYear,
startOfToday,
isWithinInterval,
} from 'date-fns';
import { EModelEndpoint, LocalStorageKeys } from 'librechat-data-provider';
@ -76,7 +77,7 @@ export const groupConversationsByDate = (conversations: TConversation[]): Groupe
}
seenConversationIds.add(conversation.conversationId);
const date = parseISO(conversation.updatedAt);
const date = conversation.updatedAt ? parseISO(conversation.updatedAt) : startOfToday();
const groupName = getGroupName(date);
if (!acc[groupName]) {
acc[groupName] = [];