diff --git a/api/models/plugins/mongoMeili.js b/api/models/plugins/mongoMeili.js deleted file mode 100644 index 75e3738e5d..0000000000 --- a/api/models/plugins/mongoMeili.js +++ /dev/null @@ -1,475 +0,0 @@ -const _ = require('lodash'); -const mongoose = require('mongoose'); -const { MeiliSearch } = require('meilisearch'); -const { parseTextParts, ContentTypes } = require('librechat-data-provider'); -const { cleanUpPrimaryKeyValue } = require('~/lib/utils/misc'); -const logger = require('~/config/meiliLogger'); - -// Environment flags -/** - * Flag to indicate if search is enabled based on environment variables. - * @type {boolean} - */ -const searchEnabled = process.env.SEARCH && process.env.SEARCH.toLowerCase() === 'true'; - -/** - * Flag to indicate if MeiliSearch is enabled based on required environment variables. - * @type {boolean} - */ -const meiliEnabled = process.env.MEILI_HOST && process.env.MEILI_MASTER_KEY && searchEnabled; - -/** - * Validates the required options for configuring the mongoMeili plugin. - * - * @param {Object} options - The configuration options. - * @param {string} options.host - The MeiliSearch host. - * @param {string} options.apiKey - The MeiliSearch API key. - * @param {string} options.indexName - The name of the index. - * @throws {Error} Throws an error if any required option is missing. - */ -const validateOptions = function (options) { - const requiredKeys = ['host', 'apiKey', 'indexName']; - requiredKeys.forEach((key) => { - if (!options[key]) { - throw new Error(`Missing mongoMeili Option: ${key}`); - } - }); -}; - -/** - * Factory function to create a MeiliMongooseModel class which extends a Mongoose model. - * This class contains static and instance methods to synchronize and manage the MeiliSearch index - * corresponding to the MongoDB collection. - * - * @param {Object} config - Configuration object. - * @param {Object} config.index - The MeiliSearch index object. - * @param {Array} config.attributesToIndex - List of attributes to index. - * @returns {Function} A class definition that will be loaded into the Mongoose schema. - */ -const createMeiliMongooseModel = function ({ index, attributesToIndex }) { - // The primary key is assumed to be the first attribute in the attributesToIndex array. - const primaryKey = attributesToIndex[0]; - - class MeiliMongooseModel { - /** - * Synchronizes the data between the MongoDB collection and the MeiliSearch index. - * - * The synchronization process involves: - * 1. Fetching all documents from the MongoDB collection and MeiliSearch index. - * 2. Comparing documents from both sources. - * 3. Deleting documents from MeiliSearch that no longer exist in MongoDB. - * 4. Adding documents to MeiliSearch that exist in MongoDB but not in the index. - * 5. Updating documents in MeiliSearch if key fields (such as `text` or `title`) differ. - * 6. Updating the `_meiliIndex` field in MongoDB to indicate the indexing status. - * - * Note: The function processes documents in batches because MeiliSearch's - * `index.getDocuments` requires an exact limit and `index.addDocuments` does not handle - * partial failures in a batch. - * - * @returns {Promise} Resolves when the synchronization is complete. - */ - static async syncWithMeili() { - try { - let moreDocuments = true; - // Retrieve all MongoDB documents from the collection as plain JavaScript objects. - const mongoDocuments = await this.find().lean(); - - // Helper function to format a document by selecting only the attributes to index - // and omitting keys starting with '$'. - const format = (doc) => - _.omitBy(_.pick(doc, attributesToIndex), (v, k) => k.startsWith('$')); - - // Build a map of MongoDB documents for quick lookup based on the primary key. - const mongoMap = new Map(mongoDocuments.map((doc) => [doc[primaryKey], format(doc)])); - const indexMap = new Map(); - let offset = 0; - const batchSize = 1000; - - // Fetch documents from the MeiliSearch index in batches. - while (moreDocuments) { - const batch = await index.getDocuments({ limit: batchSize, offset }); - if (batch.results.length === 0) { - moreDocuments = false; - } - for (const doc of batch.results) { - indexMap.set(doc[primaryKey], format(doc)); - } - offset += batchSize; - } - - logger.debug('[syncWithMeili]', { indexMap: indexMap.size, mongoMap: mongoMap.size }); - - const updateOps = []; - - // Process documents present in the MeiliSearch index. - for (const [id, doc] of indexMap) { - const update = {}; - update[primaryKey] = id; - if (mongoMap.has(id)) { - // If document exists in MongoDB, check for discrepancies in key fields. - if ( - (doc.text && doc.text !== mongoMap.get(id).text) || - (doc.title && doc.title !== mongoMap.get(id).title) - ) { - logger.debug( - `[syncWithMeili] ${id} had document discrepancy in ${ - doc.text ? 'text' : 'title' - } field`, - ); - updateOps.push({ - updateOne: { filter: update, update: { $set: { _meiliIndex: true } } }, - }); - await index.addDocuments([doc]); - } - } else { - // If the document does not exist in MongoDB, delete it from MeiliSearch. - await index.deleteDocument(id); - updateOps.push({ - updateOne: { filter: update, update: { $set: { _meiliIndex: false } } }, - }); - } - } - - // Process documents present in MongoDB. - for (const [id, doc] of mongoMap) { - const update = {}; - update[primaryKey] = id; - // If the document is missing in the Meili index, add it. - if (!indexMap.has(id)) { - await index.addDocuments([doc]); - updateOps.push({ - updateOne: { filter: update, update: { $set: { _meiliIndex: true } } }, - }); - } else if (doc._meiliIndex === false) { - // If the document exists but is marked as not indexed, update the flag. - updateOps.push({ - updateOne: { filter: update, update: { $set: { _meiliIndex: true } } }, - }); - } - } - - // Execute bulk update operations in MongoDB to update the _meiliIndex flags. - if (updateOps.length > 0) { - await this.collection.bulkWrite(updateOps); - logger.debug( - `[syncWithMeili] Finished indexing ${ - primaryKey === 'messageId' ? 'messages' : 'conversations' - }`, - ); - } - } catch (error) { - logger.error('[syncWithMeili] Error adding document to Meili', error); - } - } - - /** - * Updates settings for the MeiliSearch index. - * - * @param {Object} settings - The settings to update on the MeiliSearch index. - * @returns {Promise} Promise resolving to the update result. - */ - static async setMeiliIndexSettings(settings) { - return await index.updateSettings(settings); - } - - /** - * Searches the MeiliSearch index and optionally populates the results with data from MongoDB. - * - * @param {string} q - The search query. - * @param {Object} params - Additional search parameters for MeiliSearch. - * @param {boolean} populate - Whether to populate search hits with full MongoDB documents. - * @returns {Promise} The search results with populated hits if requested. - */ - static async meiliSearch(q, params, populate) { - const data = await index.search(q, params); - - if (populate) { - // Build a query using the primary key values from the search hits. - const query = {}; - query[primaryKey] = _.map(data.hits, (hit) => cleanUpPrimaryKeyValue(hit[primaryKey])); - - // Build a projection object, including only keys that do not start with '$'. - const projection = Object.keys(this.schema.obj).reduce( - (results, key) => { - if (!key.startsWith('$')) { - results[key] = 1; - } - return results; - }, - { _id: 1, __v: 1 }, - ); - - // Retrieve the full documents from MongoDB. - const hitsFromMongoose = await this.find(query, projection).lean(); - - // Merge the MongoDB documents with the search hits. - const populatedHits = data.hits.map(function (hit) { - const query = {}; - query[primaryKey] = hit[primaryKey]; - const originalHit = _.find(hitsFromMongoose, query); - - return { - ...(originalHit ?? {}), - ...hit, - }; - }); - data.hits = populatedHits; - } - - return data; - } - - /** - * Preprocesses the current document for indexing. - * - * This method: - * - Picks only the defined attributes to index. - * - Omits any keys starting with '$'. - * - Replaces pipe characters ('|') in `conversationId` with '--'. - * - Extracts and concatenates text from an array of content items. - * - * @returns {Object} The preprocessed object ready for indexing. - */ - preprocessObjectForIndex() { - const object = _.omitBy(_.pick(this.toJSON(), attributesToIndex), (v, k) => - k.startsWith('$'), - ); - if (object.conversationId && object.conversationId.includes('|')) { - object.conversationId = object.conversationId.replace(/\|/g, '--'); - } - - if (object.content && Array.isArray(object.content)) { - object.text = parseTextParts(object.content); - delete object.content; - } - - return object; - } - - /** - * Adds the current document to the MeiliSearch index. - * - * The method preprocesses the document, adds it to MeiliSearch, and then updates - * the MongoDB document's `_meiliIndex` flag to true. - * - * @returns {Promise} - */ - async addObjectToMeili() { - const object = this.preprocessObjectForIndex(); - try { - await index.addDocuments([object]); - } catch (error) { - // Error handling can be enhanced as needed. - logger.error('[addObjectToMeili] Error adding document to Meili', error); - } - - await this.collection.updateMany({ _id: this._id }, { $set: { _meiliIndex: true } }); - } - - /** - * Updates the current document in the MeiliSearch index. - * - * @returns {Promise} - */ - async updateObjectToMeili() { - const object = _.omitBy(_.pick(this.toJSON(), attributesToIndex), (v, k) => - k.startsWith('$'), - ); - await index.updateDocuments([object]); - } - - /** - * Deletes the current document from the MeiliSearch index. - * - * @returns {Promise} - */ - async deleteObjectFromMeili() { - await index.deleteDocument(this._id); - } - - /** - * Post-save hook to synchronize the document with MeiliSearch. - * - * If the document is already indexed (i.e. `_meiliIndex` is true), it updates it; - * otherwise, it adds the document to the index. - */ - postSaveHook() { - if (this._meiliIndex) { - this.updateObjectToMeili(); - } else { - this.addObjectToMeili(); - } - } - - /** - * Post-update hook to update the document in MeiliSearch. - * - * This hook is triggered after a document update, ensuring that changes are - * propagated to the MeiliSearch index if the document is indexed. - */ - postUpdateHook() { - if (this._meiliIndex) { - this.updateObjectToMeili(); - } - } - - /** - * Post-remove hook to delete the document from MeiliSearch. - * - * This hook is triggered after a document is removed, ensuring that the document - * is also removed from the MeiliSearch index if it was previously indexed. - */ - postRemoveHook() { - if (this._meiliIndex) { - this.deleteObjectFromMeili(); - } - } - } - - return MeiliMongooseModel; -}; - -/** - * Mongoose plugin to synchronize MongoDB collections with a MeiliSearch index. - * - * This plugin: - * - Validates the provided options. - * - Adds a `_meiliIndex` field to the schema to track indexing status. - * - Sets up a MeiliSearch client and creates an index if it doesn't already exist. - * - Loads class methods for syncing, searching, and managing documents in MeiliSearch. - * - Registers Mongoose hooks (post-save, post-update, post-remove, etc.) to maintain index consistency. - * - * @param {mongoose.Schema} schema - The Mongoose schema to which the plugin is applied. - * @param {Object} options - Configuration options. - * @param {string} options.host - The MeiliSearch host. - * @param {string} options.apiKey - The MeiliSearch API key. - * @param {string} options.indexName - The name of the MeiliSearch index. - * @param {string} options.primaryKey - The primary key field for indexing. - */ -module.exports = function mongoMeili(schema, options) { - validateOptions(options); - - // Add _meiliIndex field to the schema to track if a document has been indexed in MeiliSearch. - schema.add({ - _meiliIndex: { - type: Boolean, - required: false, - select: false, - default: false, - }, - }); - - const { host, apiKey, indexName, primaryKey } = options; - - // Setup the MeiliSearch client. - const client = new MeiliSearch({ host, apiKey }); - - // Create the index asynchronously if it doesn't exist. - client.createIndex(indexName, { primaryKey }); - - // Setup the MeiliSearch index for this schema. - const index = client.index(indexName); - - // Collect attributes from the schema that should be indexed. - const attributesToIndex = [ - ..._.reduce( - schema.obj, - function (results, value, key) { - return value.meiliIndex ? [...results, key] : results; - }, - [], - ), - ]; - - // Load the class methods into the schema. - schema.loadClass(createMeiliMongooseModel({ index, indexName, client, attributesToIndex })); - - // Register Mongoose hooks to synchronize with MeiliSearch. - - // Post-save: synchronize after a document is saved. - schema.post('save', function (doc) { - doc.postSaveHook(); - }); - - // Post-update: synchronize after a document is updated. - schema.post('update', function (doc) { - doc.postUpdateHook(); - }); - - // Post-remove: synchronize after a document is removed. - schema.post('remove', function (doc) { - doc.postRemoveHook(); - }); - - // Pre-deleteMany hook: remove corresponding documents from MeiliSearch when multiple documents are deleted. - schema.pre('deleteMany', async function (next) { - if (!meiliEnabled) { - return next(); - } - - try { - // Check if the schema has a "messages" field to determine if it's a conversation schema. - if (Object.prototype.hasOwnProperty.call(schema.obj, 'messages')) { - const convoIndex = client.index('convos'); - const deletedConvos = await mongoose.model('Conversation').find(this._conditions).lean(); - const promises = deletedConvos.map((convo) => - convoIndex.deleteDocument(convo.conversationId), - ); - await Promise.all(promises); - } - - // Check if the schema has a "messageId" field to determine if it's a message schema. - if (Object.prototype.hasOwnProperty.call(schema.obj, 'messageId')) { - const messageIndex = client.index('messages'); - const deletedMessages = await mongoose.model('Message').find(this._conditions).lean(); - const promises = deletedMessages.map((message) => - messageIndex.deleteDocument(message.messageId), - ); - await Promise.all(promises); - } - return next(); - } catch (error) { - if (meiliEnabled) { - logger.error( - '[MeiliMongooseModel.deleteMany] There was an issue deleting conversation indexes upon deletion. Next startup may be slow due to syncing.', - error, - ); - } - return next(); - } - }); - - // Post-findOneAndUpdate hook: update MeiliSearch index after a document is updated via findOneAndUpdate. - schema.post('findOneAndUpdate', async function (doc) { - if (!meiliEnabled) { - return; - } - - // If the document is unfinished, do not update the index. - if (doc.unfinished) { - return; - } - - let meiliDoc; - // For conversation documents, try to fetch the document from the "convos" index. - if (doc.messages) { - try { - meiliDoc = await client.index('convos').getDocument(doc.conversationId); - } catch (error) { - logger.debug( - '[MeiliMongooseModel.findOneAndUpdate] Convo not found in MeiliSearch and will index ' + - doc.conversationId, - error, - ); - } - } - - // If the MeiliSearch document exists and the title is unchanged, do nothing. - if (meiliDoc && meiliDoc.title === doc.title) { - return; - } - - // Otherwise, trigger a post-save hook to synchronize the document. - doc.postSaveHook(); - }); -}; diff --git a/api/models/schema/messageSchema.js b/api/models/schema/messageSchema.js deleted file mode 100644 index ebdd911fef..0000000000 --- a/api/models/schema/messageSchema.js +++ /dev/null @@ -1,15 +0,0 @@ -const mongoMeili = require('~/models/plugins/mongoMeili'); -const { messageSchema } = require('@librechat/data-schemas'); - -if (process.env.MEILI_HOST && process.env.MEILI_MASTER_KEY) { - messageSchema.plugin(mongoMeili, { - host: process.env.MEILI_HOST, - apiKey: process.env.MEILI_MASTER_KEY, - indexName: 'messages', - primaryKey: 'messageId', - }); -} - -// const Message = mongoose.models.Message || mongoose.model('Message', messageSchema); - -module.exports = Message; diff --git a/api/models/schema/pluginAuthSchema.js b/api/models/schema/pluginAuthSchema.js deleted file mode 100644 index 2066eda4c4..0000000000 --- a/api/models/schema/pluginAuthSchema.js +++ /dev/null @@ -1,6 +0,0 @@ -const mongoose = require('mongoose'); -const { pluginAuthSchema } = require('@librechat/data-schemas'); - -const PluginAuth = mongoose.models.Plugin || mongoose.model('PluginAuth', pluginAuthSchema); - -module.exports = PluginAuth; diff --git a/api/server/services/PluginService.js b/api/server/services/PluginService.js index 03e90bce41..3c7af612a9 100644 --- a/api/server/services/PluginService.js +++ b/api/server/services/PluginService.js @@ -1,4 +1,4 @@ -const PluginAuth = require('~/models/schema/pluginAuthSchema'); +const { PluginAuth } = require('@librechat/data-schemas'); const { encrypt, decrypt } = require('~/server/utils/'); const { logger } = require('~/config'); diff --git a/packages/data-schemas/src/config/meiliLogger.ts b/packages/data-schemas/src/config/meiliLogger.ts new file mode 100644 index 0000000000..1341c328fd --- /dev/null +++ b/packages/data-schemas/src/config/meiliLogger.ts @@ -0,0 +1,75 @@ +import path from 'path'; +import winston from 'winston'; +import 'winston-daily-rotate-file'; + +const logDir = path.join(__dirname, '..', 'logs'); + +const { NODE_ENV, DEBUG_LOGGING = 'false' } = process.env; + +const useDebugLogging = + (typeof DEBUG_LOGGING === 'string' && DEBUG_LOGGING.toLowerCase() === 'true') || + DEBUG_LOGGING === 'true'; + +const levels: winston.config.AbstractConfigSetLevels = { + error: 0, + warn: 1, + info: 2, + http: 3, + verbose: 4, + debug: 5, + activity: 6, + silly: 7, +}; + +winston.addColors({ + info: 'green', + warn: 'italic yellow', + error: 'red', + debug: 'blue', +}); + +const level = (): string => { + const env = NODE_ENV || 'development'; + const isDevelopment = env === 'development'; + return isDevelopment ? 'debug' : 'warn'; +}; + +const fileFormat = winston.format.combine( + winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), + winston.format.errors({ stack: true }), + winston.format.splat(), +); + +const logLevel = useDebugLogging ? 'debug' : 'error'; +const transports: winston.transport[] = [ + new winston.transports.DailyRotateFile({ + level: logLevel, + filename: `${logDir}/meiliSync-%DATE%.log`, + datePattern: 'YYYY-MM-DD', + zippedArchive: true, + maxSize: '20m', + maxFiles: '14d', + format: fileFormat, + }), +]; + +const consoleFormat = winston.format.combine( + winston.format.colorize({ all: true }), + winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), + winston.format.printf((info) => `${info.timestamp} ${info.level}: ${info.message}`), +); + +transports.push( + new winston.transports.Console({ + level: 'info', + format: consoleFormat, + }), +); + +const logger = winston.createLogger({ + level: level(), + levels, + transports, +}); + +export default logger; diff --git a/packages/data-schemas/src/models/plugins/mongoMeili.ts b/packages/data-schemas/src/models/plugins/mongoMeili.ts index 470f1f5bdb..0eabc22f7f 100644 --- a/packages/data-schemas/src/models/plugins/mongoMeili.ts +++ b/packages/data-schemas/src/models/plugins/mongoMeili.ts @@ -1,8 +1,7 @@ import _ from 'lodash'; -import mongoose, { Schema, Document, Model } from 'mongoose'; +import mongoose, { Schema, Document, Model, Query } from 'mongoose'; import { MeiliSearch, Index } from 'meilisearch'; -const { parseTextParts } = require('librechat-data-provider'); -const logger = require('~/config/meiliLogger'); +import logger from '../../config/meiliLogger'; interface MongoMeiliOptions { host: string; @@ -12,34 +11,79 @@ interface MongoMeiliOptions { } interface MeiliIndexable { - [key: string]: any; + [key: string]: unknown; _meiliIndex?: boolean; } +interface ContentItem { + type: string; + text?: string; +} + +interface DocumentWithMeiliIndex extends Document { + _meiliIndex?: boolean; + preprocessObjectForIndex?: () => Record; + addObjectToMeili?: () => Promise; + updateObjectToMeili?: () => Promise; + deleteObjectFromMeili?: () => Promise; + postSaveHook?: () => void; + postUpdateHook?: () => void; + postRemoveHook?: () => void; + conversationId?: string; + content?: ContentItem[]; + messageId?: string; + unfinished?: boolean; + messages?: unknown[]; + title?: string; + toJSON(): Record; +} + +interface SchemaWithMeiliMethods extends Model { + syncWithMeili(): Promise; + setMeiliIndexSettings(settings: Record): Promise; + meiliSearch(q: string, params: Record, populate: boolean): Promise; +} + // Environment flags /** * Flag to indicate if search is enabled based on environment variables. - * @type {boolean} */ -const searchEnabled = process.env.SEARCH && process.env.SEARCH.toLowerCase() === 'true'; +const searchEnabled = process.env.SEARCH != null && process.env.SEARCH.toLowerCase() === 'true'; /** * Flag to indicate if MeiliSearch is enabled based on required environment variables. - * @type {boolean} */ -const meiliEnabled = process.env.MEILI_HOST && process.env.MEILI_MASTER_KEY && searchEnabled; +const meiliEnabled = + process.env.MEILI_HOST != null && process.env.MEILI_MASTER_KEY != null && searchEnabled; + +/** + * Local implementation of parseTextParts to avoid dependency on librechat-data-provider + * Extracts text content from an array of content items + */ +const parseTextParts = (content: ContentItem[]): string => { + if (!Array.isArray(content)) { + return ''; + } + + return content + .filter((item) => item.type === 'text' && typeof item.text === 'string') + .map((item) => item.text) + .join(' ') + .trim(); +}; + +/** + * Local implementation to handle Bing convoId conversion + */ +const cleanUpPrimaryKeyValue = (value: string): string => { + return value.replace(/--/g, '|'); +}; /** * Validates the required options for configuring the mongoMeili plugin. - * - * @param {Object} options - The configuration options. - * @param {string} options.host - The MeiliSearch host. - * @param {string} options.apiKey - The MeiliSearch API key. - * @param {string} options.indexName - The name of the index. - * @throws {Error} Throws an error if any required option is missing. */ -const validateOptions = function (options: any) { - const requiredKeys = ['host', 'apiKey', 'indexName']; +const validateOptions = (options: Partial): void => { + const requiredKeys: (keyof MongoMeiliOptions)[] = ['host', 'apiKey', 'indexName']; requiredKeys.forEach((key) => { if (!options[key]) { throw new Error(`Missing mongoMeili Option: ${key}`); @@ -52,19 +96,18 @@ const validateOptions = function (options: any) { * This class contains static and instance methods to synchronize and manage the MeiliSearch index * corresponding to the MongoDB collection. * - * @param {Object} config - Configuration object. - * @param {Object} config.index - The MeiliSearch index object. - * @param {Array} config.attributesToIndex - List of attributes to index. - * @returns {Function} A class definition that will be loaded into the Mongoose schema. + * @param config - Configuration object. + * @param config.index - The MeiliSearch index object. + * @param config.attributesToIndex - List of attributes to index. + * @returns A class definition that will be loaded into the Mongoose schema. */ -const createMeiliMongooseModel = function ({ +const createMeiliMongooseModel = ({ index, attributesToIndex, }: { index: Index; attributesToIndex: string[]; -}) { - // The primary key is assumed to be the first attribute in the attributesToIndex array. +}) => { const primaryKey = attributesToIndex[0]; class MeiliMongooseModel { @@ -85,24 +128,24 @@ const createMeiliMongooseModel = function ({ * * @returns {Promise} Resolves when the synchronization is complete. */ - static async syncWithMeili(this: Model) { + static async syncWithMeili(this: SchemaWithMeiliMethods): Promise { try { let moreDocuments = true; - // Retrieve all MongoDB documents from the collection as plain JavaScript objects. const mongoDocuments = await this.find().lean(); - // Helper function to format a document by selecting only the attributes to index - // and omitting keys starting with '$'. - const format = (doc: Record) => + const format = (doc: Record) => _.omitBy(_.pick(doc, attributesToIndex), (v, k) => k.startsWith('$')); - // Build a map of MongoDB documents for quick lookup based on the primary key. - const mongoMap = new Map(mongoDocuments.map((doc) => [doc[primaryKey], format(doc)])); - const indexMap = new Map(); + const mongoMap = new Map( + mongoDocuments.map((doc) => { + const typedDoc = doc as Record; + return [typedDoc[primaryKey], format(typedDoc)]; + }), + ); + const indexMap = new Map>(); let offset = 0; const batchSize = 1000; - // Fetch documents from the MeiliSearch index in batches. while (moreDocuments) { const batch = await index.getDocuments({ limit: batchSize, offset }); if (batch.results.length === 0) { @@ -116,17 +159,22 @@ const createMeiliMongooseModel = function ({ logger.debug('[syncWithMeili]', { indexMap: indexMap.size, mongoMap: mongoMap.size }); - const updateOps = []; + const updateOps: Array<{ + updateOne: { + filter: Record; + update: { $set: { _meiliIndex: boolean } }; + }; + }> = []; - // Process documents present in the MeiliSearch index. + // Process documents present in the MeiliSearch index for (const [id, doc] of indexMap) { - const update: any = {}; + const update: Record = {}; update[primaryKey] = id; if (mongoMap.has(id)) { - // If document exists in MongoDB, check for discrepancies in key fields. + const mongoDoc = mongoMap.get(id); if ( - (doc.text && doc.text !== mongoMap.get(id)?.text) || - (doc.title && doc.title !== mongoMap.get(id)?.title) + (doc.text && doc.text !== mongoDoc?.text) || + (doc.title && doc.title !== mongoDoc?.title) ) { logger.debug( `[syncWithMeili] ${id} had document discrepancy in ${ @@ -139,33 +187,29 @@ const createMeiliMongooseModel = function ({ await index.addDocuments([doc]); } } else { - // If the document does not exist in MongoDB, delete it from MeiliSearch. - await index.deleteDocument(id); + await index.deleteDocument(id as string); updateOps.push({ updateOne: { filter: update, update: { $set: { _meiliIndex: false } } }, }); } } - // Process documents present in MongoDB. + // Process documents present in MongoDB for (const [id, doc] of mongoMap) { - const update: any = {}; + const update: Record = {}; update[primaryKey] = id; - // If the document is missing in the Meili index, add it. if (!indexMap.has(id)) { await index.addDocuments([doc]); updateOps.push({ updateOne: { filter: update, update: { $set: { _meiliIndex: true } } }, }); } else if (doc._meiliIndex === false) { - // If the document exists but is marked as not indexed, update the flag. updateOps.push({ updateOne: { filter: update, update: { $set: { _meiliIndex: true } } }, }); } } - // Execute bulk update operations in MongoDB to update the _meiliIndex flags. if (updateOps.length > 0) { await this.collection.bulkWrite(updateOps); logger.debug( @@ -180,51 +224,51 @@ const createMeiliMongooseModel = function ({ } /** - * Updates settings for the MeiliSearch index. - * - * @param {Object} settings - The settings to update on the MeiliSearch index. - * @returns {Promise} Promise resolving to the update result. + * Updates settings for the MeiliSearch index */ - static async setMeiliIndexSettings(settings: any) { + static async setMeiliIndexSettings(settings: Record): Promise { return await index.updateSettings(settings); } /** - * Searches the MeiliSearch index and optionally populates the results with data from MongoDB. - * - * @param {string} q - The search query. - * @param {Object} params - Additional search parameters for MeiliSearch. - * @param {boolean} populate - Whether to populate search hits with full MongoDB documents. - * @returns {Promise} The search results with populated hits if requested. + * Searches the MeiliSearch index and optionally populates results */ - static async meiliSearch(this: Model, q: string, params: any, populate: boolean) { + static async meiliSearch( + this: SchemaWithMeiliMethods, + q: string, + params: Record, + populate: boolean, + ): Promise { const data = await index.search(q, params); if (populate) { - // Build a query using the primary key values from the search hits. - const query: Record = {}; - query[primaryKey] = _.map(data.hits, (hit) => cleanUpPrimaryKeyValue(hit[primaryKey])); + const query: Record = {}; + query[primaryKey] = _.map(data.hits, (hit) => + cleanUpPrimaryKeyValue(hit[primaryKey] as string), + ); - // Build a projection object, including only keys that do not start with '$'. const projection = Object.keys(this.schema.obj).reduce>( - (acc, key) => { - if (!key.startsWith('$')) acc[key] = 1; - return acc; + (results, key) => { + if (!key.startsWith('$')) { + results[key] = 1; + } + return results; }, { _id: 1, __v: 1 }, ); - // Retrieve the full documents from MongoDB. const hitsFromMongoose = await this.find(query, projection).lean(); - // Merge the MongoDB documents with the search hits. - const populatedHits = data.hits.map(function (hit) { - const query = {}; - query[primaryKey] = hit[primaryKey]; - const originalHit = _.find(hitsFromMongoose, query); + const populatedHits = data.hits.map((hit) => { + const queryObj: Record = {}; + queryObj[primaryKey] = hit[primaryKey]; + const originalHit = _.find(hitsFromMongoose, (item) => { + const typedItem = item as Record; + return typedItem[primaryKey] === hit[primaryKey]; + }); return { - ...(originalHit ?? {}), + ...(originalHit && typeof originalHit === 'object' ? originalHit : {}), ...hit, }; }); @@ -235,21 +279,18 @@ const createMeiliMongooseModel = function ({ } /** - * Preprocesses the current document for indexing. - * - * This method: - * - Picks only the defined attributes to index. - * - Omits any keys starting with '$'. - * - Replaces pipe characters ('|') in `conversationId` with '--'. - * - Extracts and concatenates text from an array of content items. - * - * @returns {Object} The preprocessed object ready for indexing. + * Preprocesses the current document for indexing */ - preprocessObjectForIndex(this: Document) { + preprocessObjectForIndex(this: DocumentWithMeiliIndex): Record { const object = _.omitBy(_.pick(this.toJSON(), attributesToIndex), (v, k) => k.startsWith('$'), ); - if (object.conversationId && object.conversationId.includes('|')) { + + if ( + object.conversationId && + typeof object.conversationId === 'string' && + object.conversationId.includes('|') + ) { object.conversationId = object.conversationId.replace(/\|/g, '--'); } @@ -262,31 +303,26 @@ const createMeiliMongooseModel = function ({ } /** - * Adds the current document to the MeiliSearch index. - * - * The method preprocesses the document, adds it to MeiliSearch, and then updates - * the MongoDB document's `_meiliIndex` flag to true. - * - * @returns {Promise} + * Adds the current document to the MeiliSearch index */ - async addObjectToMeili(this: Document) { - const object = this.preprocessObjectForIndex(); + async addObjectToMeili(this: DocumentWithMeiliIndex): Promise { + const object = this.preprocessObjectForIndex!(); try { await index.addDocuments([object]); } catch (error) { - // Error handling can be enhanced as needed. logger.error('[addObjectToMeili] Error adding document to Meili', error); } - await this.collection.updateMany({ _id: this._id }, { $set: { _meiliIndex: true } }); + await this.collection.updateMany( + { _id: this._id as mongoose.Types.ObjectId }, + { $set: { _meiliIndex: true } }, + ); } /** - * Updates the current document in the MeiliSearch index. - * - * @returns {Promise} + * Updates the current document in the MeiliSearch index */ - async updateObjectToMeili(this: Document) { + async updateObjectToMeili(this: DocumentWithMeiliIndex): Promise { const object = _.omitBy(_.pick(this.toJSON(), attributesToIndex), (v, k) => k.startsWith('$'), ); @@ -298,8 +334,8 @@ const createMeiliMongooseModel = function ({ * * @returns {Promise} */ - async deleteObjectFromMeili(this: Document) { - await index.deleteDocument(this._id); + async deleteObjectFromMeili(this: DocumentWithMeiliIndex): Promise { + await index.deleteDocument(this._id as string); } /** @@ -308,11 +344,11 @@ const createMeiliMongooseModel = function ({ * If the document is already indexed (i.e. `_meiliIndex` is true), it updates it; * otherwise, it adds the document to the index. */ - postSaveHook(this: Document) { + postSaveHook(this: DocumentWithMeiliIndex): void { if (this._meiliIndex) { - this.updateObjectToMeili(); + this.updateObjectToMeili!(); } else { - this.addObjectToMeili(); + this.addObjectToMeili!(); } } @@ -322,9 +358,9 @@ const createMeiliMongooseModel = function ({ * This hook is triggered after a document update, ensuring that changes are * propagated to the MeiliSearch index if the document is indexed. */ - postUpdateHook() { + postUpdateHook(this: DocumentWithMeiliIndex): void { if (this._meiliIndex) { - this.updateObjectToMeili(); + this.updateObjectToMeili!(); } } @@ -334,9 +370,9 @@ const createMeiliMongooseModel = function ({ * This hook is triggered after a document is removed, ensuring that the document * is also removed from the MeiliSearch index if it was previously indexed. */ - postRemoveHook(this: Document) { + postRemoveHook(this: DocumentWithMeiliIndex): void { if (this._meiliIndex) { - this.deleteObjectFromMeili(); + this.deleteObjectFromMeili!(); } } } @@ -344,10 +380,6 @@ const createMeiliMongooseModel = function ({ return MeiliMongooseModel; }; -const cleanUpPrimaryKeyValue = (value) => { - // For Bing convoId handling - return value.replace(/--/g, '|'); -}; /** * Mongoose plugin to synchronize MongoDB collections with a MeiliSearch index. * @@ -358,14 +390,14 @@ const cleanUpPrimaryKeyValue = (value) => { * - Loads class methods for syncing, searching, and managing documents in MeiliSearch. * - Registers Mongoose hooks (post-save, post-update, post-remove, etc.) to maintain index consistency. * - * @param {mongoose.Schema} schema - The Mongoose schema to which the plugin is applied. - * @param {Object} options - Configuration options. - * @param {string} options.host - The MeiliSearch host. - * @param {string} options.apiKey - The MeiliSearch API key. - * @param {string} options.indexName - The name of the MeiliSearch index. - * @param {string} options.primaryKey - The primary key field for indexing. + * @param schema - The Mongoose schema to which the plugin is applied. + * @param options - Configuration options. + * @param options.host - The MeiliSearch host. + * @param options.apiKey - The MeiliSearch API key. + * @param options.indexName - The name of the MeiliSearch index. + * @param options.primaryKey - The primary key field for indexing. */ -export default function mongoMeili(schema: Schema, options: MongoMeiliOptions) { +export default function mongoMeili(schema: Schema, options: MongoMeiliOptions): void { validateOptions(options); // Add _meiliIndex field to the schema to track if a document has been indexed in MeiliSearch. @@ -380,44 +412,31 @@ export default function mongoMeili(schema: Schema, options: MongoMeiliOptions) { const { host, apiKey, indexName, primaryKey } = options; - // Setup the MeiliSearch client. const client = new MeiliSearch({ host, apiKey }); - - // Create the index asynchronously if it doesn't exist. client.createIndex(indexName, { primaryKey }); - - // Setup the MeiliSearch index for this schema. const index = client.index(indexName); - // Collect attributes from the schema that should be indexed. - const attributesToIndex = [ - ..._.reduce( - schema.obj, - function (results, value, key) { - return value.meiliIndex ? [...results, key] : results; - }, - [], - ), + // Collect attributes from the schema that should be indexed + const attributesToIndex: string[] = [ + ...Object.entries(schema.obj).reduce((results, [key, value]) => { + const schemaValue = value as { meiliIndex?: boolean }; + return schemaValue.meiliIndex ? [...results, key] : results; + }, []), ]; - // Load the class methods into the schema. - schema.loadClass(createMeiliMongooseModel({ index, client, attributesToIndex })); + schema.loadClass(createMeiliMongooseModel({ index, attributesToIndex })); - // Register Mongoose hooks to synchronize with MeiliSearch. - - // Post-save: synchronize after a document is saved. - schema.post('save', function (doc) { - doc.postSaveHook(); + // Register Mongoose hooks + schema.post('save', function (doc: DocumentWithMeiliIndex) { + doc.postSaveHook?.(); }); - // Post-update: synchronize after a document is updated. - schema.post('update', function (doc) { - doc.postUpdateHook(); + schema.post('updateOne', function (doc: DocumentWithMeiliIndex) { + doc.postUpdateHook?.(); }); - // Post-remove: synchronize after a document is removed. - schema.post('remove', function (doc) { - doc.postRemoveHook(); + schema.post('deleteOne', function (doc: DocumentWithMeiliIndex) { + doc.postRemoveHook?.(); }); // Pre-deleteMany hook: remove corresponding documents from MeiliSearch when multiple documents are deleted. @@ -427,22 +446,28 @@ export default function mongoMeili(schema: Schema, options: MongoMeiliOptions) { } try { - // Check if the schema has a "messages" field to determine if it's a conversation schema. + const conditions = (this as Query).getQuery(); + if (Object.prototype.hasOwnProperty.call(schema.obj, 'messages')) { const convoIndex = client.index('convos'); - const deletedConvos = await mongoose.model('Conversation').find(this._conditions).lean(); - const promises = deletedConvos.map((convo) => - convoIndex.deleteDocument(convo.conversationId), + const deletedConvos = await mongoose + .model('Conversation') + .find(conditions as mongoose.FilterQuery) + .lean(); + const promises = deletedConvos.map((convo: Record) => + convoIndex.deleteDocument(convo.conversationId as string), ); await Promise.all(promises); } - // Check if the schema has a "messageId" field to determine if it's a message schema. if (Object.prototype.hasOwnProperty.call(schema.obj, 'messageId')) { const messageIndex = client.index('messages'); - const deletedMessages = await mongoose.model('Message').find(this._conditions).lean(); - const promises = deletedMessages.map((message) => - messageIndex.deleteDocument(message.messageId), + const deletedMessages = await mongoose + .model('Message') + .find(conditions as mongoose.FilterQuery) + .lean(); + const promises = deletedMessages.map((message: Record) => + messageIndex.deleteDocument(message.messageId as string), ); await Promise.all(promises); } @@ -458,37 +483,33 @@ export default function mongoMeili(schema: Schema, options: MongoMeiliOptions) { } }); - // Post-findOneAndUpdate hook: update MeiliSearch index after a document is updated via findOneAndUpdate. - schema.post('findOneAndUpdate', async function (doc) { + // Post-findOneAndUpdate hook + schema.post('findOneAndUpdate', async function (doc: DocumentWithMeiliIndex) { if (!meiliEnabled) { return; } - // If the document is unfinished, do not update the index. if (doc.unfinished) { return; } - let meiliDoc; - // For conversation documents, try to fetch the document from the "convos" index. + let meiliDoc: Record | undefined; if (doc.messages) { try { - meiliDoc = await client.index('convos').getDocument(doc.conversationId); - } catch (error) { + meiliDoc = await client.index('convos').getDocument(doc.conversationId as string); + } catch (error: unknown) { logger.debug( '[MeiliMongooseModel.findOneAndUpdate] Convo not found in MeiliSearch and will index ' + doc.conversationId, - error, + error as Record, ); } } - // If the MeiliSearch document exists and the title is unchanged, do nothing. if (meiliDoc && meiliDoc.title === doc.title) { return; } - // Otherwise, trigger a post-save hook to synchronize the document. - doc.postSaveHook(); + doc.postSaveHook?.(); }); }