mirror of
https://github.com/danny-avila/LibreChat.git
synced 2025-12-17 17:00:15 +01:00
🔍 refactor: Search & Message Retrieval (#6903)
* refactor: conversation search fetch * refactor: Message and Convo fetch with paramters and search * refactor: update search states and cleanup old store states * refactor: re-enable search API; fix: search conversation * fix: message's convo fetch * fix: redirect when searching * chore: use logger instead of console * fix: search message loading * feat: small optimizations * feat(Message): remove cache for search path * fix: handle delete of all archivedConversation and sharedLinks * chore: cleanup * fix: search messages * style: update ConvoOptions styles * refactor(SearchButtons): streamline conversation fetching and remove unused state * fix: ensure messages are invalidated after fetching conversation data * fix: add iconURL to conversation query selection --------- Co-authored-by: Danny Avila <danny@librechat.ai>
This commit is contained in:
Marco Beretta
GitHub
parent
851938e7a6
commit
88f4ad7c47
30 changed files with 489 additions and 576 deletions
|
|
@ -10,6 +10,7 @@ const balance = require('./balance');
|
|||
const plugins = require('./plugins');
|
||||
const bedrock = require('./bedrock');
|
||||
const actions = require('./actions');
|
||||
const banner = require('./banner');
|
||||
const search = require('./search');
|
||||
const models = require('./models');
|
||||
const convos = require('./convos');
|
||||
|
|
@ -25,7 +26,6 @@ const edit = require('./edit');
|
|||
const keys = require('./keys');
|
||||
const user = require('./user');
|
||||
const ask = require('./ask');
|
||||
const banner = require('./banner');
|
||||
|
||||
module.exports = {
|
||||
ask,
|
||||
|
|
@ -38,13 +38,14 @@ module.exports = {
|
|||
oauth,
|
||||
files,
|
||||
share,
|
||||
banner,
|
||||
agents,
|
||||
bedrock,
|
||||
convos,
|
||||
search,
|
||||
prompts,
|
||||
config,
|
||||
models,
|
||||
bedrock,
|
||||
prompts,
|
||||
plugins,
|
||||
actions,
|
||||
presets,
|
||||
|
|
@ -55,5 +56,4 @@ module.exports = {
|
|||
assistants,
|
||||
categories,
|
||||
staticRoute,
|
||||
banner,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -10,12 +10,90 @@ const {
|
|||
} = require('~/models');
|
||||
const { findAllArtifacts, replaceArtifactContent } = require('~/server/services/Artifacts/update');
|
||||
const { requireJwtAuth, validateMessageReq } = require('~/server/middleware');
|
||||
const { cleanUpPrimaryKeyValue } = require('~/lib/utils/misc');
|
||||
const { getConvosQueried } = require('~/models/Conversation');
|
||||
const { countTokens } = require('~/server/utils');
|
||||
const { Message } = require('~/models/Message');
|
||||
const { logger } = require('~/config');
|
||||
|
||||
const router = express.Router();
|
||||
router.use(requireJwtAuth);
|
||||
|
||||
router.get('/', async (req, res) => {
|
||||
try {
|
||||
const user = req.user.id ?? '';
|
||||
const {
|
||||
cursor = null,
|
||||
sortBy = 'createdAt',
|
||||
sortDirection = 'desc',
|
||||
pageSize: pageSizeRaw,
|
||||
conversationId,
|
||||
messageId,
|
||||
search,
|
||||
} = req.query;
|
||||
const pageSize = parseInt(pageSizeRaw, 10) || 25;
|
||||
|
||||
let response;
|
||||
const sortField = ['endpoint', 'createdAt', 'updatedAt'].includes(sortBy)
|
||||
? sortBy
|
||||
: 'createdAt';
|
||||
const sortOrder = sortDirection === 'asc' ? 1 : -1;
|
||||
|
||||
if (conversationId && messageId) {
|
||||
const message = await Message.findOne({ conversationId, messageId, user: user }).lean();
|
||||
response = { messages: message ? [message] : [], nextCursor: null };
|
||||
} else if (conversationId) {
|
||||
const filter = { conversationId, user: user };
|
||||
if (cursor) {
|
||||
filter[sortField] = sortOrder === 1 ? { $gt: cursor } : { $lt: cursor };
|
||||
}
|
||||
const messages = await Message.find(filter)
|
||||
.sort({ [sortField]: sortOrder })
|
||||
.limit(pageSize + 1)
|
||||
.lean();
|
||||
const nextCursor = messages.length > pageSize ? messages.pop()[sortField] : null;
|
||||
response = { messages, nextCursor };
|
||||
} else if (search) {
|
||||
const searchResults = await Message.meiliSearch(search, undefined, true);
|
||||
|
||||
const messages = searchResults.hits || [];
|
||||
|
||||
const result = await getConvosQueried(req.user.id, messages, cursor);
|
||||
|
||||
const activeMessages = [];
|
||||
for (let i = 0; i < messages.length; i++) {
|
||||
let message = messages[i];
|
||||
if (message.conversationId.includes('--')) {
|
||||
message.conversationId = cleanUpPrimaryKeyValue(message.conversationId);
|
||||
}
|
||||
if (result.convoMap[message.conversationId]) {
|
||||
const convo = result.convoMap[message.conversationId];
|
||||
|
||||
const dbMessage = await getMessage({ user, messageId: message.messageId });
|
||||
activeMessages.push({
|
||||
...message,
|
||||
title: convo.title,
|
||||
conversationId: message.conversationId,
|
||||
model: convo.model,
|
||||
isCreatedByUser: dbMessage?.isCreatedByUser,
|
||||
endpoint: dbMessage?.endpoint,
|
||||
iconURL: dbMessage?.iconURL,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
response = { messages: activeMessages, nextCursor: null };
|
||||
} else {
|
||||
response = { messages: [], nextCursor: null };
|
||||
}
|
||||
|
||||
res.status(200).json(response);
|
||||
} catch (error) {
|
||||
logger.error('Error fetching messages:', error);
|
||||
res.status(500).json({ error: 'Internal server error' });
|
||||
}
|
||||
});
|
||||
|
||||
router.post('/artifact/:messageId', async (req, res) => {
|
||||
try {
|
||||
const { messageId } = req.params;
|
||||
|
|
|
|||
|
|
@ -1,14 +1,11 @@
|
|||
const { Keyv } = require('keyv');
|
||||
const express = require('express');
|
||||
const { MeiliSearch } = require('meilisearch');
|
||||
const { Conversation, getConvosQueried } = require('~/models/Conversation');
|
||||
const { Conversation } = require('~/models/Conversation');
|
||||
const requireJwtAuth = require('~/server/middleware/requireJwtAuth');
|
||||
const { cleanUpPrimaryKeyValue } = require('~/lib/utils/misc');
|
||||
const { Message, getMessage } = require('~/models/Message');
|
||||
const { reduceHits } = require('~/lib/utils/reduceHits');
|
||||
const { Message } = require('~/models/Message');
|
||||
const { isEnabled } = require('~/server/utils');
|
||||
const keyvRedis = require('~/cache/keyvRedis');
|
||||
const { logger } = require('~/config');
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
|
|
@ -25,88 +22,6 @@ router.get('/sync', async function (req, res) {
|
|||
res.send('synced');
|
||||
});
|
||||
|
||||
router.get('/', async function (req, res) {
|
||||
try {
|
||||
const user = req.user.id ?? '';
|
||||
const { q, cursor = 'start' } = req.query;
|
||||
const key = `${user}:search:${q}:${cursor}`;
|
||||
const cached = await cache.get(key);
|
||||
if (cached) {
|
||||
logger.debug('[/search] cache hit: ' + key);
|
||||
return res.status(200).send(cached);
|
||||
}
|
||||
|
||||
const [messageResults, titleResults] = await Promise.all([
|
||||
Message.meiliSearch(q, undefined, true),
|
||||
Conversation.meiliSearch(q),
|
||||
]);
|
||||
const messages = messageResults.hits;
|
||||
const titles = titleResults.hits;
|
||||
|
||||
const sortedHits = reduceHits(messages, titles);
|
||||
const result = await getConvosQueried(user, sortedHits, cursor);
|
||||
|
||||
const activeMessages = [];
|
||||
for (let i = 0; i < messages.length; i++) {
|
||||
let message = messages[i];
|
||||
if (message.conversationId.includes('--')) {
|
||||
message.conversationId = cleanUpPrimaryKeyValue(message.conversationId);
|
||||
}
|
||||
if (result.convoMap[message.conversationId]) {
|
||||
const convo = result.convoMap[message.conversationId];
|
||||
|
||||
const dbMessage = await getMessage({ user, messageId: message.messageId });
|
||||
activeMessages.push({
|
||||
...message,
|
||||
title: convo.title,
|
||||
conversationId: message.conversationId,
|
||||
model: convo.model,
|
||||
isCreatedByUser: dbMessage?.isCreatedByUser,
|
||||
endpoint: dbMessage?.endpoint,
|
||||
iconURL: dbMessage?.iconURL,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const activeConversations = [];
|
||||
for (const convId in result.convoMap) {
|
||||
const convo = result.convoMap[convId];
|
||||
|
||||
if (convo.isArchived) {
|
||||
continue;
|
||||
}
|
||||
|
||||
activeConversations.push({
|
||||
title: convo.title,
|
||||
user: convo.user,
|
||||
conversationId: convo.conversationId,
|
||||
endpoint: convo.endpoint,
|
||||
endpointType: convo.endpointType,
|
||||
model: convo.model,
|
||||
createdAt: convo.createdAt,
|
||||
updatedAt: convo.updatedAt,
|
||||
});
|
||||
}
|
||||
|
||||
if (result.cache) {
|
||||
result.cache.messages = activeMessages;
|
||||
result.cache.conversations = activeConversations;
|
||||
cache.set(key, result.cache, expiration);
|
||||
}
|
||||
|
||||
const response = {
|
||||
nextCursor: result.nextCursor ?? null,
|
||||
messages: activeMessages,
|
||||
conversations: activeConversations,
|
||||
};
|
||||
|
||||
res.status(200).send(response);
|
||||
} catch (error) {
|
||||
logger.error('[/search] Error while searching messages & conversations', error);
|
||||
res.status(500).send({ message: 'Error searching' });
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/test', async function (req, res) {
|
||||
const { q } = req.query;
|
||||
const messages = (
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue