Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2026-01-28 21:26:13 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

📊 fix: MeiliSearch Sync Threshold & Document Count Accuracy (#11406)
Some checks are pending
Docker Dev Branch Images Build / build (Dockerfile, lc-dev, node) (push) Waiting to run
Details
Docker Dev Branch Images Build / build (Dockerfile.multi, lc-dev-api, api-build) (push) Waiting to run
Details
Docker Dev Images Build / build (Dockerfile, librechat-dev, node) (push) Waiting to run
Details
Docker Dev Images Build / build (Dockerfile.multi, librechat-dev-api, api-build) (push) Waiting to run
Details
Sync Locize Translations & Create Translation PR / Sync Translation Keys with Locize (push) Waiting to run
Details
Sync Locize Translations & Create Translation PR / Create Translation PR on Version Published (push) Blocked by required conditions
Details

Browse source 
* 🔧 fix: meilisearch incorrect count of total documents & performance improvement

Temporary documents were counted & removed 2 redundant heavy calls to the database, use known information instead

🔧 fix: respect MEILI_SYNC_THRESHOLD value

Do not sync with meili if threshold was not reached

* refactor: reformat lint

* fix: forces update if meili index settingsUpdated
This commit is contained in:
Andrei Blizorukov 2026-01-19 22:32:57 +01:00 • committed by GitHub
parent 9134471143
commit 4a1d2b0d94
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 487 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

39
api/db/indexSync.js
View file

@ -13,6 +13,11 @@ const searchEnabled = isEnabled(process.env.SEARCH);
const indexingDisabled = isEnabled(process.env.MEILI_NO_SYNC);
let currentTimeout = null;
const defaultSyncThreshold = 1000;
const syncThreshold = process.env.MEILI_SYNC_THRESHOLD
? parseInt(process.env.MEILI_SYNC_THRESHOLD, 10)
: defaultSyncThreshold;
class MeiliSearchClient {
static instance = null;
@ -221,25 +226,25 @@ async function performSync(flowManager, flowId, flowType) {
}
// Check if we need to sync messages
logger.info('[indexSync] Requesting message sync progress...');
const messageProgress = await Message.getSyncProgress();
if (!messageProgress.isComplete || settingsUpdated) {
logger.info(
`[indexSync] Messages need syncing: ${messageProgress.totalProcessed}/${messageProgress.totalDocuments} indexed`,
);
// Check if we should do a full sync or incremental
const messageCount = await Message.countDocuments();
const messageCount = messageProgress.totalDocuments;
const messagesIndexed = messageProgress.totalProcessed;
const syncThreshold = parseInt(process.env.MEILI_SYNC_THRESHOLD || '1000', 10);
const unindexedMessages = messageCount - messagesIndexed;
if (messageCount - messagesIndexed > syncThreshold) {
logger.info('[indexSync] Starting full message sync due to large difference');
await Message.syncWithMeili();
messagesSync = true;
} else if (messageCount !== messagesIndexed) {
logger.warn('[indexSync] Messages out of sync, performing incremental sync');
if (settingsUpdated || unindexedMessages > syncThreshold) {
logger.info(`[indexSync] Starting message sync (${unindexedMessages} unindexed)`);
await Message.syncWithMeili();
messagesSync = true;
} else if (unindexedMessages > 0) {
logger.info(
`[indexSync] ${unindexedMessages} messages unindexed (below threshold: ${syncThreshold}, skipping)`,
);
}
} else {
logger.info(
@ -254,18 +259,18 @@ async function performSync(flowManager, flowId, flowType) {
`[indexSync] Conversations need syncing: ${convoProgress.totalProcessed}/${convoProgress.totalDocuments} indexed`,
);
const convoCount = await Conversation.countDocuments();
const convoCount = convoProgress.totalDocuments;
const convosIndexed = convoProgress.totalProcessed;
const syncThreshold = parseInt(process.env.MEILI_SYNC_THRESHOLD || '1000', 10);
if (convoCount - convosIndexed > syncThreshold) {
logger.info('[indexSync] Starting full conversation sync due to large difference');
await Conversation.syncWithMeili();
convosSync = true;
} else if (convoCount !== convosIndexed) {
logger.warn('[indexSync] Convos out of sync, performing incremental sync');
const unindexedConvos = convoCount - convosIndexed;
if (settingsUpdated || unindexedConvos > syncThreshold) {
logger.info(`[indexSync] Starting convos sync (${unindexedConvos} unindexed)`);
await Conversation.syncWithMeili();
convosSync = true;
} else if (unindexedConvos > 0) {
logger.info(
`[indexSync] ${unindexedConvos} convos unindexed (below threshold: ${syncThreshold}, skipping)`,
);
}
} else {
logger.info(