Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-12-20 18:30:15 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

refactor(data-schemas): remove legacy mongoMeili plugin and related schemas

Browse source 
- Deleted the mongoMeili plugin and its associated schemas (messageSchema, pluginAuthSchema) to streamline the codebase.
- Updated PluginService to import PluginAuth directly from data-schemas.
- Introduced a new meiliLogger configuration file for improved logging functionality.
This commit is contained in:
Danny Avila 2025-05-30 00:34:28 -04:00
parent ea459749f9
commit 848cb6f871
No known key found for this signature in database
GPG key ID: BF31EEB2C5CA0956
6 changed files with 259 additions and 659 deletions
Download patch file Download diff file Expand all files Collapse all files

475
api/models/plugins/mongoMeili.js
View file

@ -1,475 +0,0 @@
const _ = require('lodash');
const mongoose = require('mongoose');
const { MeiliSearch } = require('meilisearch');
const { parseTextParts, ContentTypes } = require('librechat-data-provider');
const { cleanUpPrimaryKeyValue } = require('~/lib/utils/misc');
const logger = require('~/config/meiliLogger');
// Environment flags
/**
* Flag to indicate if search is enabled based on environment variables.
* @type {boolean}
*/
const searchEnabled = process.env.SEARCH && process.env.SEARCH.toLowerCase() === 'true';
/**
* Flag to indicate if MeiliSearch is enabled based on required environment variables.
* @type {boolean}
*/
const meiliEnabled = process.env.MEILI_HOST && process.env.MEILI_MASTER_KEY && searchEnabled;
/**
* Validates the required options for configuring the mongoMeili plugin.
*
* @param {Object} options - The configuration options.
* @param {string} options.host - The MeiliSearch host.
* @param {string} options.apiKey - The MeiliSearch API key.
* @param {string} options.indexName - The name of the index.
* @throws {Error} Throws an error if any required option is missing.
*/
const validateOptions = function (options) {
const requiredKeys = ['host', 'apiKey', 'indexName'];
requiredKeys.forEach((key) => {
if (!options[key]) {
throw new Error(`Missing mongoMeili Option: ${key}`);
}
});
};
/**
* Factory function to create a MeiliMongooseModel class which extends a Mongoose model.
* This class contains static and instance methods to synchronize and manage the MeiliSearch index
* corresponding to the MongoDB collection.
*
* @param {Object} config - Configuration object.
* @param {Object} config.index - The MeiliSearch index object.
* @param {Array<string>} config.attributesToIndex - List of attributes to index.
* @returns {Function} A class definition that will be loaded into the Mongoose schema.
*/
const createMeiliMongooseModel = function ({ index, attributesToIndex }) {
// The primary key is assumed to be the first attribute in the attributesToIndex array.
const primaryKey = attributesToIndex[0];
class MeiliMongooseModel {
/**
* Synchronizes the data between the MongoDB collection and the MeiliSearch index.
*
* The synchronization process involves:
* 1. Fetching all documents from the MongoDB collection and MeiliSearch index.
* 2. Comparing documents from both sources.
* 3. Deleting documents from MeiliSearch that no longer exist in MongoDB.
* 4. Adding documents to MeiliSearch that exist in MongoDB but not in the index.
* 5. Updating documents in MeiliSearch if key fields (such as `text` or `title`) differ.
* 6. Updating the `_meiliIndex` field in MongoDB to indicate the indexing status.
*
* Note: The function processes documents in batches because MeiliSearch's
* `index.getDocuments` requires an exact limit and `index.addDocuments` does not handle
* partial failures in a batch.
*
* @returns {Promise<void>} Resolves when the synchronization is complete.
*/
static async syncWithMeili() {
try {
let moreDocuments = true;
// Retrieve all MongoDB documents from the collection as plain JavaScript objects.
const mongoDocuments = await this.find().lean();
// Helper function to format a document by selecting only the attributes to index
// and omitting keys starting with '$'.
const format = (doc) =>
_.omitBy(_.pick(doc, attributesToIndex), (v, k) => k.startsWith('$'));
// Build a map of MongoDB documents for quick lookup based on the primary key.
const mongoMap = new Map(mongoDocuments.map((doc) => [doc[primaryKey], format(doc)]));
const indexMap = new Map();
let offset = 0;
const batchSize = 1000;
// Fetch documents from the MeiliSearch index in batches.
while (moreDocuments) {
const batch = await index.getDocuments({ limit: batchSize, offset });
if (batch.results.length === 0) {
moreDocuments = false;
}
for (const doc of batch.results) {
indexMap.set(doc[primaryKey], format(doc));
}
offset += batchSize;
}
logger.debug('[syncWithMeili]', { indexMap: indexMap.size, mongoMap: mongoMap.size });
const updateOps = [];
// Process documents present in the MeiliSearch index.
for (const [id, doc] of indexMap) {
const update = {};
update[primaryKey] = id;
if (mongoMap.has(id)) {
// If document exists in MongoDB, check for discrepancies in key fields.
if (
(doc.text && doc.text !== mongoMap.get(id).text) ||
(doc.title && doc.title !== mongoMap.get(id).title)
) {
logger.debug(
`[syncWithMeili] ${id} had document discrepancy in ${
doc.text ? 'text' : 'title'
} field`,
);
updateOps.push({
updateOne: { filter: update, update: { $set: { _meiliIndex: true } } },
});
await index.addDocuments([doc]);
}
} else {
// If the document does not exist in MongoDB, delete it from MeiliSearch.
await index.deleteDocument(id);
updateOps.push({
updateOne: { filter: update, update: { $set: { _meiliIndex: false } } },
});
}
}
// Process documents present in MongoDB.
for (const [id, doc] of mongoMap) {
const update = {};
update[primaryKey] = id;
// If the document is missing in the Meili index, add it.
if (!indexMap.has(id)) {
await index.addDocuments([doc]);
updateOps.push({
updateOne: { filter: update, update: { $set: { _meiliIndex: true } } },
});
} else if (doc._meiliIndex === false) {
// If the document exists but is marked as not indexed, update the flag.
updateOps.push({
updateOne: { filter: update, update: { $set: { _meiliIndex: true } } },
});
}
}
// Execute bulk update operations in MongoDB to update the _meiliIndex flags.
if (updateOps.length > 0) {
await this.collection.bulkWrite(updateOps);
logger.debug(
`[syncWithMeili] Finished indexing ${
primaryKey === 'messageId' ? 'messages' : 'conversations'
}`,
);
}
} catch (error) {
logger.error('[syncWithMeili] Error adding document to Meili', error);
}
}
/**
* Updates settings for the MeiliSearch index.
*
* @param {Object} settings - The settings to update on the MeiliSearch index.
* @returns {Promise<Object>} Promise resolving to the update result.
*/
static async setMeiliIndexSettings(settings) {
return await index.updateSettings(settings);
}
/**
* Searches the MeiliSearch index and optionally populates the results with data from MongoDB.
*
* @param {string} q - The search query.
* @param {Object} params - Additional search parameters for MeiliSearch.
* @param {boolean} populate - Whether to populate search hits with full MongoDB documents.
* @returns {Promise<Object>} The search results with populated hits if requested.
*/
static async meiliSearch(q, params, populate) {
const data = await index.search(q, params);
if (populate) {
// Build a query using the primary key values from the search hits.
const query = {};
query[primaryKey] = _.map(data.hits, (hit) => cleanUpPrimaryKeyValue(hit[primaryKey]));
// Build a projection object, including only keys that do not start with '$'.
const projection = Object.keys(this.schema.obj).reduce(
(results, key) => {
if (!key.startsWith('$')) {
results[key] = 1;
}
return results;
},
{ _id: 1, __v: 1 },
);
// Retrieve the full documents from MongoDB.
const hitsFromMongoose = await this.find(query, projection).lean();
// Merge the MongoDB documents with the search hits.
const populatedHits = data.hits.map(function (hit) {
const query = {};
query[primaryKey] = hit[primaryKey];
const originalHit = _.find(hitsFromMongoose, query);
return {
...(originalHit ?? {}),
...hit,
};
});
data.hits = populatedHits;
}
return data;
}
/**
* Preprocesses the current document for indexing.
*
* This method:
* - Picks only the defined attributes to index.
* - Omits any keys starting with '$'.
* - Replaces pipe characters ('|') in `conversationId` with '--'.
* - Extracts and concatenates text from an array of content items.
*
* @returns {Object} The preprocessed object ready for indexing.
*/
preprocessObjectForIndex() {
const object = _.omitBy(_.pick(this.toJSON(), attributesToIndex), (v, k) =>
k.startsWith('$'),
);
if (object.conversationId && object.conversationId.includes('|')) {
object.conversationId = object.conversationId.replace(/\|/g, '--');
}
if (object.content && Array.isArray(object.content)) {
object.text = parseTextParts(object.content);
delete object.content;
}
return object;
}
/**
* Adds the current document to the MeiliSearch index.
*
* The method preprocesses the document, adds it to MeiliSearch, and then updates
* the MongoDB document's `_meiliIndex` flag to true.
*
* @returns {Promise<void>}
*/
async addObjectToMeili() {
const object = this.preprocessObjectForIndex();
try {
await index.addDocuments([object]);
} catch (error) {
// Error handling can be enhanced as needed.
logger.error('[addObjectToMeili] Error adding document to Meili', error);
}
await this.collection.updateMany({ _id: this._id }, { $set: { _meiliIndex: true } });
}
/**
* Updates the current document in the MeiliSearch index.
*
* @returns {Promise<void>}
*/
async updateObjectToMeili() {
const object = _.omitBy(_.pick(this.toJSON(), attributesToIndex), (v, k) =>
k.startsWith('$'),
);
await index.updateDocuments([object]);
}
/**
* Deletes the current document from the MeiliSearch index.
*
* @returns {Promise<void>}
*/
async deleteObjectFromMeili() {
await index.deleteDocument(this._id);
}
/**
* Post-save hook to synchronize the document with MeiliSearch.
*
* If the document is already indexed (i.e. `_meiliIndex` is true), it updates it;
* otherwise, it adds the document to the index.
*/
postSaveHook() {
if (this._meiliIndex) {
this.updateObjectToMeili();
} else {
this.addObjectToMeili();
}
}
/**
* Post-update hook to update the document in MeiliSearch.
*
* This hook is triggered after a document update, ensuring that changes are
* propagated to the MeiliSearch index if the document is indexed.
*/
postUpdateHook() {
if (this._meiliIndex) {
this.updateObjectToMeili();
}
}
/**
* Post-remove hook to delete the document from MeiliSearch.
*
* This hook is triggered after a document is removed, ensuring that the document
* is also removed from the MeiliSearch index if it was previously indexed.
*/
postRemoveHook() {
if (this._meiliIndex) {
this.deleteObjectFromMeili();
}
}
}
return MeiliMongooseModel;
};
/**
* Mongoose plugin to synchronize MongoDB collections with a MeiliSearch index.
*
* This plugin:
* - Validates the provided options.
* - Adds a `_meiliIndex` field to the schema to track indexing status.
* - Sets up a MeiliSearch client and creates an index if it doesn't already exist.
* - Loads class methods for syncing, searching, and managing documents in MeiliSearch.
* - Registers Mongoose hooks (post-save, post-update, post-remove, etc.) to maintain index consistency.
*
* @param {mongoose.Schema} schema - The Mongoose schema to which the plugin is applied.
* @param {Object} options - Configuration options.
* @param {string} options.host - The MeiliSearch host.
* @param {string} options.apiKey - The MeiliSearch API key.
* @param {string} options.indexName - The name of the MeiliSearch index.
* @param {string} options.primaryKey - The primary key field for indexing.
*/
module.exports = function mongoMeili(schema, options) {
validateOptions(options);
// Add _meiliIndex field to the schema to track if a document has been indexed in MeiliSearch.
schema.add({
_meiliIndex: {
type: Boolean,
required: false,
select: false,
default: false,
},
});
const { host, apiKey, indexName, primaryKey } = options;
// Setup the MeiliSearch client.
const client = new MeiliSearch({ host, apiKey });
// Create the index asynchronously if it doesn't exist.
client.createIndex(indexName, { primaryKey });
// Setup the MeiliSearch index for this schema.
const index = client.index(indexName);
// Collect attributes from the schema that should be indexed.
const attributesToIndex = [
..._.reduce(
schema.obj,
function (results, value, key) {
return value.meiliIndex ? [...results, key] : results;
},
[],
),
];
// Load the class methods into the schema.
schema.loadClass(createMeiliMongooseModel({ index, indexName, client, attributesToIndex }));
// Register Mongoose hooks to synchronize with MeiliSearch.
// Post-save: synchronize after a document is saved.
schema.post('save', function (doc) {
doc.postSaveHook();
});
// Post-update: synchronize after a document is updated.
schema.post('update', function (doc) {
doc.postUpdateHook();
});
// Post-remove: synchronize after a document is removed.
schema.post('remove', function (doc) {
doc.postRemoveHook();
});
// Pre-deleteMany hook: remove corresponding documents from MeiliSearch when multiple documents are deleted.
schema.pre('deleteMany', async function (next) {
if (!meiliEnabled) {
return next();
}
try {
// Check if the schema has a "messages" field to determine if it's a conversation schema.
if (Object.prototype.hasOwnProperty.call(schema.obj, 'messages')) {
const convoIndex = client.index('convos');
const deletedConvos = await mongoose.model('Conversation').find(this._conditions).lean();
const promises = deletedConvos.map((convo) =>
convoIndex.deleteDocument(convo.conversationId),
);
await Promise.all(promises);
}
// Check if the schema has a "messageId" field to determine if it's a message schema.
if (Object.prototype.hasOwnProperty.call(schema.obj, 'messageId')) {
const messageIndex = client.index('messages');
const deletedMessages = await mongoose.model('Message').find(this._conditions).lean();
const promises = deletedMessages.map((message) =>
messageIndex.deleteDocument(message.messageId),
);
await Promise.all(promises);
}
return next();
} catch (error) {
if (meiliEnabled) {
logger.error(
'[MeiliMongooseModel.deleteMany] There was an issue deleting conversation indexes upon deletion. Next startup may be slow due to syncing.',
error,
);
}
return next();
}
});
// Post-findOneAndUpdate hook: update MeiliSearch index after a document is updated via findOneAndUpdate.
schema.post('findOneAndUpdate', async function (doc) {
if (!meiliEnabled) {
return;
}
// If the document is unfinished, do not update the index.
if (doc.unfinished) {
return;
}
let meiliDoc;
// For conversation documents, try to fetch the document from the "convos" index.
if (doc.messages) {
try {
meiliDoc = await client.index('convos').getDocument(doc.conversationId);
} catch (error) {
logger.debug(
'[MeiliMongooseModel.findOneAndUpdate] Convo not found in MeiliSearch and will index ' +
doc.conversationId,
error,
);
}
}
// If the MeiliSearch document exists and the title is unchanged, do nothing.
if (meiliDoc && meiliDoc.title === doc.title) {
return;
}
// Otherwise, trigger a post-save hook to synchronize the document.
doc.postSaveHook();
});
};

15
api/models/schema/messageSchema.js
View file

@ -1,15 +0,0 @@
const mongoMeili = require('~/models/plugins/mongoMeili');
const { messageSchema } = require('@librechat/data-schemas');
if (process.env.MEILI_HOST && process.env.MEILI_MASTER_KEY) {
messageSchema.plugin(mongoMeili, {
host: process.env.MEILI_HOST,
apiKey: process.env.MEILI_MASTER_KEY,
indexName: 'messages',
primaryKey: 'messageId',
});
}
// const Message = mongoose.models.Message || mongoose.model('Message', messageSchema);
module.exports = Message;

6
api/models/schema/pluginAuthSchema.js
View file

@ -1,6 +0,0 @@
const mongoose = require('mongoose');
const { pluginAuthSchema } = require('@librechat/data-schemas');
const PluginAuth = mongoose.models.Plugin || mongoose.model('PluginAuth', pluginAuthSchema);
module.exports = PluginAuth;

2
api/server/services/PluginService.js
View file

@ -1,4 +1,4 @@
const PluginAuth = require('~/models/schema/pluginAuthSchema'); const { PluginAuth } = require('@librechat/data-schemas');
const { encrypt, decrypt } = require('~/server/utils/'); const { encrypt, decrypt } = require('~/server/utils/');
const { logger } = require('~/config'); const { logger } = require('~/config');

75
packages/data-schemas/src/config/meiliLogger.ts Normal file
View file

@ -0,0 +1,75 @@
import path from 'path';
import winston from 'winston';
import 'winston-daily-rotate-file';
const logDir = path.join(__dirname, '..', 'logs');
const { NODE_ENV, DEBUG_LOGGING = 'false' } = process.env;
const useDebugLogging =
(typeof DEBUG_LOGGING === 'string' && DEBUG_LOGGING.toLowerCase() === 'true') ||
DEBUG_LOGGING === 'true';
const levels: winston.config.AbstractConfigSetLevels = {
error: 0,
warn: 1,
info: 2,
http: 3,
verbose: 4,
debug: 5,
activity: 6,
silly: 7,
};
winston.addColors({
info: 'green',
warn: 'italic yellow',
error: 'red',
debug: 'blue',
});
const level = (): string => {
const env = NODE_ENV || 'development';
const isDevelopment = env === 'development';
return isDevelopment ? 'debug' : 'warn';
};
const fileFormat = winston.format.combine(
winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
winston.format.errors({ stack: true }),
winston.format.splat(),
);
const logLevel = useDebugLogging ? 'debug' : 'error';
const transports: winston.transport[] = [
new winston.transports.DailyRotateFile({
level: logLevel,
filename: `${logDir}/meiliSync-%DATE%.log`,
datePattern: 'YYYY-MM-DD',
zippedArchive: true,
maxSize: '20m',
maxFiles: '14d',
format: fileFormat,
}),
];
const consoleFormat = winston.format.combine(
winston.format.colorize({ all: true }),
winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
winston.format.printf((info) => `${info.timestamp} ${info.level}: ${info.message}`),
);
transports.push(
new winston.transports.Console({
level: 'info',
format: consoleFormat,
}),
);
const logger = winston.createLogger({
level: level(),
levels,
transports,
});
export default logger;

345
packages/data-schemas/src/models/plugins/mongoMeili.ts
View file

@ -1,8 +1,7 @@
import _ from 'lodash'; import _ from 'lodash';
import mongoose, { Schema, Document, Model } from 'mongoose'; import mongoose, { Schema, Document, Model, Query } from 'mongoose';
import { MeiliSearch, Index } from 'meilisearch'; import { MeiliSearch, Index } from 'meilisearch';
const { parseTextParts } = require('librechat-data-provider'); import logger from '../../config/meiliLogger';
const logger = require('~/config/meiliLogger');
interface MongoMeiliOptions { interface MongoMeiliOptions {
host: string; host: string;
@ -12,34 +11,79 @@ interface MongoMeiliOptions {
} }
interface MeiliIndexable { interface MeiliIndexable {
[key: string]: any; [key: string]: unknown;
_meiliIndex?: boolean; _meiliIndex?: boolean;
} }
interface ContentItem {
type: string;
text?: string;
}
interface DocumentWithMeiliIndex extends Document {
_meiliIndex?: boolean;
preprocessObjectForIndex?: () => Record<string, unknown>;
addObjectToMeili?: () => Promise<void>;
updateObjectToMeili?: () => Promise<void>;
deleteObjectFromMeili?: () => Promise<void>;
postSaveHook?: () => void;
postUpdateHook?: () => void;
postRemoveHook?: () => void;
conversationId?: string;
content?: ContentItem[];
messageId?: string;
unfinished?: boolean;
messages?: unknown[];
title?: string;
toJSON(): Record<string, unknown>;
}
interface SchemaWithMeiliMethods extends Model<DocumentWithMeiliIndex> {
syncWithMeili(): Promise<void>;
setMeiliIndexSettings(settings: Record<string, unknown>): Promise<unknown>;
meiliSearch(q: string, params: Record<string, unknown>, populate: boolean): Promise<unknown>;
}
// Environment flags // Environment flags
/** /**
* Flag to indicate if search is enabled based on environment variables. * Flag to indicate if search is enabled based on environment variables.
* @type {boolean}
*/ */
const searchEnabled = process.env.SEARCH && process.env.SEARCH.toLowerCase() === 'true'; const searchEnabled = process.env.SEARCH != null && process.env.SEARCH.toLowerCase() === 'true';
/** /**
* Flag to indicate if MeiliSearch is enabled based on required environment variables. * Flag to indicate if MeiliSearch is enabled based on required environment variables.
* @type {boolean}
*/ */
const meiliEnabled = process.env.MEILI_HOST && process.env.MEILI_MASTER_KEY && searchEnabled; const meiliEnabled =
process.env.MEILI_HOST != null && process.env.MEILI_MASTER_KEY != null && searchEnabled;
/**
* Local implementation of parseTextParts to avoid dependency on librechat-data-provider
* Extracts text content from an array of content items
*/
const parseTextParts = (content: ContentItem[]): string => {
if (!Array.isArray(content)) {
return '';
}
return content
.filter((item) => item.type === 'text' && typeof item.text === 'string')
.map((item) => item.text)
.join(' ')
.trim();
};
/**
* Local implementation to handle Bing convoId conversion
*/
const cleanUpPrimaryKeyValue = (value: string): string => {
return value.replace(/--/g, '|');
};
/** /**
* Validates the required options for configuring the mongoMeili plugin. * Validates the required options for configuring the mongoMeili plugin.
*
* @param {Object} options - The configuration options.
* @param {string} options.host - The MeiliSearch host.
* @param {string} options.apiKey - The MeiliSearch API key.
* @param {string} options.indexName - The name of the index.
* @throws {Error} Throws an error if any required option is missing.
*/ */
const validateOptions = function (options: any) { const validateOptions = (options: Partial<MongoMeiliOptions>): void => {
const requiredKeys = ['host', 'apiKey', 'indexName']; const requiredKeys: (keyof MongoMeiliOptions)[] = ['host', 'apiKey', 'indexName'];
requiredKeys.forEach((key) => { requiredKeys.forEach((key) => {
if (!options[key]) { if (!options[key]) {
throw new Error(`Missing mongoMeili Option: ${key}`); throw new Error(`Missing mongoMeili Option: ${key}`);
@ -52,19 +96,18 @@ const validateOptions = function (options: any) {
* This class contains static and instance methods to synchronize and manage the MeiliSearch index * This class contains static and instance methods to synchronize and manage the MeiliSearch index
* corresponding to the MongoDB collection. * corresponding to the MongoDB collection.
* *
* @param {Object} config - Configuration object. * @param config - Configuration object.
* @param {Object} config.index - The MeiliSearch index object. * @param config.index - The MeiliSearch index object.
* @param {Array<string>} config.attributesToIndex - List of attributes to index. * @param config.attributesToIndex - List of attributes to index.
* @returns {Function} A class definition that will be loaded into the Mongoose schema. * @returns A class definition that will be loaded into the Mongoose schema.
*/ */
const createMeiliMongooseModel = function ({ const createMeiliMongooseModel = ({
index, index,
attributesToIndex, attributesToIndex,
}: { }: {
index: Index<MeiliIndexable>; index: Index<MeiliIndexable>;
attributesToIndex: string[]; attributesToIndex: string[];
}) { }) => {
// The primary key is assumed to be the first attribute in the attributesToIndex array.
const primaryKey = attributesToIndex[0]; const primaryKey = attributesToIndex[0];
class MeiliMongooseModel { class MeiliMongooseModel {
@ -85,24 +128,24 @@ const createMeiliMongooseModel = function ({
* *
* @returns {Promise<void>} Resolves when the synchronization is complete. * @returns {Promise<void>} Resolves when the synchronization is complete.
*/ */
static async syncWithMeili(this: Model<any>) { static async syncWithMeili(this: SchemaWithMeiliMethods): Promise<void> {
try { try {
let moreDocuments = true; let moreDocuments = true;
// Retrieve all MongoDB documents from the collection as plain JavaScript objects.
const mongoDocuments = await this.find().lean(); const mongoDocuments = await this.find().lean();
// Helper function to format a document by selecting only the attributes to index const format = (doc: Record<string, unknown>) =>
// and omitting keys starting with '$'.
const format = (doc: Record<string, any>) =>
_.omitBy(_.pick(doc, attributesToIndex), (v, k) => k.startsWith('$')); _.omitBy(_.pick(doc, attributesToIndex), (v, k) => k.startsWith('$'));
// Build a map of MongoDB documents for quick lookup based on the primary key. const mongoMap = new Map(
const mongoMap = new Map(mongoDocuments.map((doc) => [doc[primaryKey], format(doc)])); mongoDocuments.map((doc) => {
const indexMap = new Map(); const typedDoc = doc as Record<string, unknown>;
return [typedDoc[primaryKey], format(typedDoc)];
}),
);
const indexMap = new Map<unknown, Record<string, unknown>>();
let offset = 0; let offset = 0;
const batchSize = 1000; const batchSize = 1000;
// Fetch documents from the MeiliSearch index in batches.
while (moreDocuments) { while (moreDocuments) {
const batch = await index.getDocuments({ limit: batchSize, offset }); const batch = await index.getDocuments({ limit: batchSize, offset });
if (batch.results.length === 0) { if (batch.results.length === 0) {
@ -116,17 +159,22 @@ const createMeiliMongooseModel = function ({
logger.debug('[syncWithMeili]', { indexMap: indexMap.size, mongoMap: mongoMap.size }); logger.debug('[syncWithMeili]', { indexMap: indexMap.size, mongoMap: mongoMap.size });
const updateOps = []; const updateOps: Array<{
updateOne: {
filter: Record<string, unknown>;
update: { $set: { _meiliIndex: boolean } };
};
}> = [];
// Process documents present in the MeiliSearch index. // Process documents present in the MeiliSearch index
for (const [id, doc] of indexMap) { for (const [id, doc] of indexMap) {
const update: any = {}; const update: Record<string, unknown> = {};
update[primaryKey] = id; update[primaryKey] = id;
if (mongoMap.has(id)) { if (mongoMap.has(id)) {
// If document exists in MongoDB, check for discrepancies in key fields. const mongoDoc = mongoMap.get(id);
if ( if (
(doc.text && doc.text !== mongoMap.get(id)?.text) || (doc.text && doc.text !== mongoDoc?.text) ||
(doc.title && doc.title !== mongoMap.get(id)?.title) (doc.title && doc.title !== mongoDoc?.title)
) { ) {
logger.debug( logger.debug(
`[syncWithMeili] ${id} had document discrepancy in ${ `[syncWithMeili] ${id} had document discrepancy in ${
@ -139,33 +187,29 @@ const createMeiliMongooseModel = function ({
await index.addDocuments([doc]); await index.addDocuments([doc]);
} }
} else { } else {
// If the document does not exist in MongoDB, delete it from MeiliSearch. await index.deleteDocument(id as string);
await index.deleteDocument(id);
updateOps.push({ updateOps.push({
updateOne: { filter: update, update: { $set: { _meiliIndex: false } } }, updateOne: { filter: update, update: { $set: { _meiliIndex: false } } },
}); });
} }
} }
// Process documents present in MongoDB. // Process documents present in MongoDB
for (const [id, doc] of mongoMap) { for (const [id, doc] of mongoMap) {
const update: any = {}; const update: Record<string, unknown> = {};
update[primaryKey] = id; update[primaryKey] = id;
// If the document is missing in the Meili index, add it.
if (!indexMap.has(id)) { if (!indexMap.has(id)) {
await index.addDocuments([doc]); await index.addDocuments([doc]);
updateOps.push({ updateOps.push({
updateOne: { filter: update, update: { $set: { _meiliIndex: true } } }, updateOne: { filter: update, update: { $set: { _meiliIndex: true } } },
}); });
} else if (doc._meiliIndex === false) { } else if (doc._meiliIndex === false) {
// If the document exists but is marked as not indexed, update the flag.
updateOps.push({ updateOps.push({
updateOne: { filter: update, update: { $set: { _meiliIndex: true } } }, updateOne: { filter: update, update: { $set: { _meiliIndex: true } } },
}); });
} }
} }
// Execute bulk update operations in MongoDB to update the _meiliIndex flags.
if (updateOps.length > 0) { if (updateOps.length > 0) {
await this.collection.bulkWrite(updateOps); await this.collection.bulkWrite(updateOps);
logger.debug( logger.debug(
@ -180,51 +224,51 @@ const createMeiliMongooseModel = function ({
} }
/** /**
* Updates settings for the MeiliSearch index. * Updates settings for the MeiliSearch index
*
* @param {Object} settings - The settings to update on the MeiliSearch index.
* @returns {Promise<Object>} Promise resolving to the update result.
*/ */
static async setMeiliIndexSettings(settings: any) { static async setMeiliIndexSettings(settings: Record<string, unknown>): Promise<unknown> {
return await index.updateSettings(settings); return await index.updateSettings(settings);
} }
/** /**
* Searches the MeiliSearch index and optionally populates the results with data from MongoDB. * Searches the MeiliSearch index and optionally populates results
*
* @param {string} q - The search query.
* @param {Object} params - Additional search parameters for MeiliSearch.
* @param {boolean} populate - Whether to populate search hits with full MongoDB documents.
* @returns {Promise<Object>} The search results with populated hits if requested.
*/ */
static async meiliSearch(this: Model<any>, q: string, params: any, populate: boolean) { static async meiliSearch(
this: SchemaWithMeiliMethods,
q: string,
params: Record<string, unknown>,
populate: boolean,
): Promise<unknown> {
const data = await index.search(q, params); const data = await index.search(q, params);
if (populate) { if (populate) {
// Build a query using the primary key values from the search hits. const query: Record<string, unknown> = {};
const query: Record<string, any> = {}; query[primaryKey] = _.map(data.hits, (hit) =>
query[primaryKey] = _.map(data.hits, (hit) => cleanUpPrimaryKeyValue(hit[primaryKey])); cleanUpPrimaryKeyValue(hit[primaryKey] as string),
);
// Build a projection object, including only keys that do not start with '$'.
const projection = Object.keys(this.schema.obj).reduce<Record<string, number>>( const projection = Object.keys(this.schema.obj).reduce<Record<string, number>>(
(acc, key) => { (results, key) => {
if (!key.startsWith('$')) acc[key] = 1; if (!key.startsWith('$')) {
return acc; results[key] = 1;
}
return results;
}, },
{ _id: 1, __v: 1 }, { _id: 1, __v: 1 },
); );
// Retrieve the full documents from MongoDB.
const hitsFromMongoose = await this.find(query, projection).lean(); const hitsFromMongoose = await this.find(query, projection).lean();
// Merge the MongoDB documents with the search hits. const populatedHits = data.hits.map((hit) => {
const populatedHits = data.hits.map(function (hit) { const queryObj: Record<string, unknown> = {};
const query = {}; queryObj[primaryKey] = hit[primaryKey];
query[primaryKey] = hit[primaryKey]; const originalHit = _.find(hitsFromMongoose, (item) => {
const originalHit = _.find(hitsFromMongoose, query); const typedItem = item as Record<string, unknown>;
return typedItem[primaryKey] === hit[primaryKey];
});
return { return {
...(originalHit ?? {}), ...(originalHit && typeof originalHit === 'object' ? originalHit : {}),
...hit, ...hit,
}; };
}); });
@ -235,21 +279,18 @@ const createMeiliMongooseModel = function ({
} }
/** /**
* Preprocesses the current document for indexing. * Preprocesses the current document for indexing
*
* This method:
* - Picks only the defined attributes to index.
* - Omits any keys starting with '$'.
* - Replaces pipe characters ('|') in `conversationId` with '--'.
* - Extracts and concatenates text from an array of content items.
*
* @returns {Object} The preprocessed object ready for indexing.
*/ */
preprocessObjectForIndex(this: Document) { preprocessObjectForIndex(this: DocumentWithMeiliIndex): Record<string, unknown> {
const object = _.omitBy(_.pick(this.toJSON(), attributesToIndex), (v, k) => const object = _.omitBy(_.pick(this.toJSON(), attributesToIndex), (v, k) =>
k.startsWith('$'), k.startsWith('$'),
); );
if (object.conversationId && object.conversationId.includes('|')) {
if (
object.conversationId &&
typeof object.conversationId === 'string' &&
object.conversationId.includes('|')
) {
object.conversationId = object.conversationId.replace(/\|/g, '--'); object.conversationId = object.conversationId.replace(/\|/g, '--');
} }
@ -262,31 +303,26 @@ const createMeiliMongooseModel = function ({
} }
/** /**
* Adds the current document to the MeiliSearch index. * Adds the current document to the MeiliSearch index
*
* The method preprocesses the document, adds it to MeiliSearch, and then updates
* the MongoDB document's `_meiliIndex` flag to true.
*
* @returns {Promise<void>}
*/ */
async addObjectToMeili(this: Document) { async addObjectToMeili(this: DocumentWithMeiliIndex): Promise<void> {
const object = this.preprocessObjectForIndex(); const object = this.preprocessObjectForIndex!();
try { try {
await index.addDocuments([object]); await index.addDocuments([object]);
} catch (error) { } catch (error) {
// Error handling can be enhanced as needed.
logger.error('[addObjectToMeili] Error adding document to Meili', error); logger.error('[addObjectToMeili] Error adding document to Meili', error);
} }
await this.collection.updateMany({ _id: this._id }, { $set: { _meiliIndex: true } }); await this.collection.updateMany(
{ _id: this._id as mongoose.Types.ObjectId },
{ $set: { _meiliIndex: true } },
);
} }
/** /**
* Updates the current document in the MeiliSearch index. * Updates the current document in the MeiliSearch index
*
* @returns {Promise<void>}
*/ */
async updateObjectToMeili(this: Document) { async updateObjectToMeili(this: DocumentWithMeiliIndex): Promise<void> {
const object = _.omitBy(_.pick(this.toJSON(), attributesToIndex), (v, k) => const object = _.omitBy(_.pick(this.toJSON(), attributesToIndex), (v, k) =>
k.startsWith('$'), k.startsWith('$'),
); );
@ -298,8 +334,8 @@ const createMeiliMongooseModel = function ({
* *
* @returns {Promise<void>} * @returns {Promise<void>}
*/ */
async deleteObjectFromMeili(this: Document) { async deleteObjectFromMeili(this: DocumentWithMeiliIndex): Promise<void> {
await index.deleteDocument(this._id); await index.deleteDocument(this._id as string);
} }
/** /**
@ -308,11 +344,11 @@ const createMeiliMongooseModel = function ({
* If the document is already indexed (i.e. `_meiliIndex` is true), it updates it; * If the document is already indexed (i.e. `_meiliIndex` is true), it updates it;
* otherwise, it adds the document to the index. * otherwise, it adds the document to the index.
*/ */
postSaveHook(this: Document) { postSaveHook(this: DocumentWithMeiliIndex): void {
if (this._meiliIndex) { if (this._meiliIndex) {
this.updateObjectToMeili(); this.updateObjectToMeili!();
} else { } else {
this.addObjectToMeili(); this.addObjectToMeili!();
} }
} }
@ -322,9 +358,9 @@ const createMeiliMongooseModel = function ({
* This hook is triggered after a document update, ensuring that changes are * This hook is triggered after a document update, ensuring that changes are
* propagated to the MeiliSearch index if the document is indexed. * propagated to the MeiliSearch index if the document is indexed.
*/ */
postUpdateHook() { postUpdateHook(this: DocumentWithMeiliIndex): void {
if (this._meiliIndex) { if (this._meiliIndex) {
this.updateObjectToMeili(); this.updateObjectToMeili!();
} }
} }
@ -334,9 +370,9 @@ const createMeiliMongooseModel = function ({
* This hook is triggered after a document is removed, ensuring that the document * This hook is triggered after a document is removed, ensuring that the document
* is also removed from the MeiliSearch index if it was previously indexed. * is also removed from the MeiliSearch index if it was previously indexed.
*/ */
postRemoveHook(this: Document) { postRemoveHook(this: DocumentWithMeiliIndex): void {
if (this._meiliIndex) { if (this._meiliIndex) {
this.deleteObjectFromMeili(); this.deleteObjectFromMeili!();
} }
} }
} }
@ -344,10 +380,6 @@ const createMeiliMongooseModel = function ({
return MeiliMongooseModel; return MeiliMongooseModel;
}; };
const cleanUpPrimaryKeyValue = (value) => {
// For Bing convoId handling
return value.replace(/--/g, '|');
};
/** /**
* Mongoose plugin to synchronize MongoDB collections with a MeiliSearch index. * Mongoose plugin to synchronize MongoDB collections with a MeiliSearch index.
* *
@ -358,14 +390,14 @@ const cleanUpPrimaryKeyValue = (value) => {
* - Loads class methods for syncing, searching, and managing documents in MeiliSearch. * - Loads class methods for syncing, searching, and managing documents in MeiliSearch.
* - Registers Mongoose hooks (post-save, post-update, post-remove, etc.) to maintain index consistency. * - Registers Mongoose hooks (post-save, post-update, post-remove, etc.) to maintain index consistency.
* *
* @param {mongoose.Schema} schema - The Mongoose schema to which the plugin is applied. * @param schema - The Mongoose schema to which the plugin is applied.
* @param {Object} options - Configuration options. * @param options - Configuration options.
* @param {string} options.host - The MeiliSearch host. * @param options.host - The MeiliSearch host.
* @param {string} options.apiKey - The MeiliSearch API key. * @param options.apiKey - The MeiliSearch API key.
* @param {string} options.indexName - The name of the MeiliSearch index. * @param options.indexName - The name of the MeiliSearch index.
* @param {string} options.primaryKey - The primary key field for indexing. * @param options.primaryKey - The primary key field for indexing.
*/ */
export default function mongoMeili(schema: Schema, options: MongoMeiliOptions) { export default function mongoMeili(schema: Schema, options: MongoMeiliOptions): void {
validateOptions(options); validateOptions(options);
// Add _meiliIndex field to the schema to track if a document has been indexed in MeiliSearch. // Add _meiliIndex field to the schema to track if a document has been indexed in MeiliSearch.
@ -380,44 +412,31 @@ export default function mongoMeili(schema: Schema, options: MongoMeiliOptions) {
const { host, apiKey, indexName, primaryKey } = options; const { host, apiKey, indexName, primaryKey } = options;
// Setup the MeiliSearch client.
const client = new MeiliSearch({ host, apiKey }); const client = new MeiliSearch({ host, apiKey });
// Create the index asynchronously if it doesn't exist.
client.createIndex(indexName, { primaryKey }); client.createIndex(indexName, { primaryKey });
// Setup the MeiliSearch index for this schema.
const index = client.index<MeiliIndexable>(indexName); const index = client.index<MeiliIndexable>(indexName);
// Collect attributes from the schema that should be indexed. // Collect attributes from the schema that should be indexed
const attributesToIndex = [ const attributesToIndex: string[] = [
..._.reduce( ...Object.entries(schema.obj).reduce<string[]>((results, [key, value]) => {
schema.obj, const schemaValue = value as { meiliIndex?: boolean };
function (results, value, key) { return schemaValue.meiliIndex ? [...results, key] : results;
return value.meiliIndex ? [...results, key] : results; }, []),
},
[],
),
]; ];
// Load the class methods into the schema. schema.loadClass(createMeiliMongooseModel({ index, attributesToIndex }));
schema.loadClass(createMeiliMongooseModel({ index, client, attributesToIndex }));
// Register Mongoose hooks to synchronize with MeiliSearch. // Register Mongoose hooks
schema.post('save', function (doc: DocumentWithMeiliIndex) {
// Post-save: synchronize after a document is saved. doc.postSaveHook?.();
schema.post('save', function (doc) {
doc.postSaveHook();
}); });
// Post-update: synchronize after a document is updated. schema.post('updateOne', function (doc: DocumentWithMeiliIndex) {
schema.post('update', function (doc) { doc.postUpdateHook?.();
doc.postUpdateHook();
}); });
// Post-remove: synchronize after a document is removed. schema.post('deleteOne', function (doc: DocumentWithMeiliIndex) {
schema.post('remove', function (doc) { doc.postRemoveHook?.();
doc.postRemoveHook();
}); });
// Pre-deleteMany hook: remove corresponding documents from MeiliSearch when multiple documents are deleted. // Pre-deleteMany hook: remove corresponding documents from MeiliSearch when multiple documents are deleted.
@ -427,22 +446,28 @@ export default function mongoMeili(schema: Schema, options: MongoMeiliOptions) {
} }
try { try {
// Check if the schema has a "messages" field to determine if it's a conversation schema. const conditions = (this as Query<unknown, unknown>).getQuery();
if (Object.prototype.hasOwnProperty.call(schema.obj, 'messages')) { if (Object.prototype.hasOwnProperty.call(schema.obj, 'messages')) {
const convoIndex = client.index('convos'); const convoIndex = client.index('convos');
const deletedConvos = await mongoose.model('Conversation').find(this._conditions).lean(); const deletedConvos = await mongoose
const promises = deletedConvos.map((convo) => .model('Conversation')
convoIndex.deleteDocument(convo.conversationId), .find(conditions as mongoose.FilterQuery<unknown>)
.lean();
const promises = deletedConvos.map((convo: Record<string, unknown>) =>
convoIndex.deleteDocument(convo.conversationId as string),
); );
await Promise.all(promises); await Promise.all(promises);
} }
// Check if the schema has a "messageId" field to determine if it's a message schema.
if (Object.prototype.hasOwnProperty.call(schema.obj, 'messageId')) { if (Object.prototype.hasOwnProperty.call(schema.obj, 'messageId')) {
const messageIndex = client.index('messages'); const messageIndex = client.index('messages');
const deletedMessages = await mongoose.model('Message').find(this._conditions).lean(); const deletedMessages = await mongoose
const promises = deletedMessages.map((message) => .model('Message')
messageIndex.deleteDocument(message.messageId), .find(conditions as mongoose.FilterQuery<unknown>)
.lean();
const promises = deletedMessages.map((message: Record<string, unknown>) =>
messageIndex.deleteDocument(message.messageId as string),
); );
await Promise.all(promises); await Promise.all(promises);
} }
@ -458,37 +483,33 @@ export default function mongoMeili(schema: Schema, options: MongoMeiliOptions) {
} }
}); });
// Post-findOneAndUpdate hook: update MeiliSearch index after a document is updated via findOneAndUpdate. // Post-findOneAndUpdate hook
schema.post('findOneAndUpdate', async function (doc) { schema.post('findOneAndUpdate', async function (doc: DocumentWithMeiliIndex) {
if (!meiliEnabled) { if (!meiliEnabled) {
return; return;
} }
// If the document is unfinished, do not update the index.
if (doc.unfinished) { if (doc.unfinished) {
return; return;
} }
let meiliDoc; let meiliDoc: Record<string, unknown> | undefined;
// For conversation documents, try to fetch the document from the "convos" index.
if (doc.messages) { if (doc.messages) {
try { try {
meiliDoc = await client.index('convos').getDocument(doc.conversationId); meiliDoc = await client.index('convos').getDocument(doc.conversationId as string);
} catch (error) { } catch (error: unknown) {
logger.debug( logger.debug(
'[MeiliMongooseModel.findOneAndUpdate] Convo not found in MeiliSearch and will index ' + '[MeiliMongooseModel.findOneAndUpdate] Convo not found in MeiliSearch and will index ' +
doc.conversationId, doc.conversationId,
error, error as Record<string, unknown>,
); );
} }
} }
// If the MeiliSearch document exists and the title is unchanged, do nothing.
if (meiliDoc && meiliDoc.title === doc.title) { if (meiliDoc && meiliDoc.title === doc.title) {
return; return;
} }
// Otherwise, trigger a post-save hook to synchronize the document. doc.postSaveHook?.();
doc.postSaveHook();
}); });
} }