Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-12-17 08:50:15 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

refactor(api): Central Logging 📜 (#1348)

Browse source 
* WIP: initial logging changes
add several transports in ~/config/winston
omit messages in logs, truncate long strings
add short blurb in dotenv for debug logging
GoogleClient: using logger
OpenAIClient: using logger, handleOpenAIErrors
Adding typedef for payload message
bumped winston and using winston-daily-rotate-file
moved config for server paths to ~/config dir
Added `DEBUG_LOGGING=true` to .env.example

* WIP: Refactor logging statements in code

* WIP: Refactor logging statements and import configurations

* WIP: Refactor logging statements and import configurations

* refactor: broadcast Redis initialization message with `info` not `debug`

* refactor: complete Refactor logging statements and import configurations

* chore: delete unused tools

* fix: circular dependencies due to accessing logger

* refactor(handleText): handle booleans and write tests

* refactor: redact sensitive values, better formatting

* chore: improve log formatting, avoid passing strings to 2nd arg

* fix(ci): fix jest tests due to logger changes

* refactor(getAvailablePluginsController): cache plugins as they are static and avoids async addOpenAPISpecs call every time

* chore: update docs

* chore: update docs

* chore: create separate meiliSync logger, clean up logs to avoid being unnecessarily verbose

* chore: spread objects where they are commonly logged to allow string truncation

* chore: improve error log formatting
This commit is contained in:
Danny Avila 2023-12-14 07:49:27 -05:00 • committed by GitHub
parent 49571ac635
commit ea1dd59ef4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
115 changed files with 1271 additions and 1001 deletions
Download patch file Download diff file Expand all files Collapse all files

42
api/models/plugins/mongoMeili.js
View file

@ -1,7 +1,9 @@
const _ = require('lodash');
const mongoose = require('mongoose');
const { MeiliSearch } = require('meilisearch');
const { cleanUpPrimaryKeyValue } = require('../../lib/utils/misc');
const _ = require('lodash');
const { cleanUpPrimaryKeyValue } = require('~/lib/utils/misc');
const logger = require('~/config/meiliLogger');
const searchEnabled = process.env.SEARCH && process.env.SEARCH.toLowerCase() === 'true';
const meiliEnabled = process.env.MEILI_HOST && process.env.MEILI_MASTER_KEY && searchEnabled;
@ -64,8 +66,7 @@ const createMeiliMongooseModel = function ({ index, attributesToIndex }) {
offset += batchSize;
}
console.log('indexMap', indexMap.size);
console.log('mongoMap', mongoMap.size);
logger.debug('[syncWithMeili]', { indexMap: indexMap.size, mongoMap: mongoMap.size });
const updateOps = [];
@ -80,7 +81,11 @@ const createMeiliMongooseModel = function ({ index, attributesToIndex }) {
(doc.text && doc.text !== mongoMap.get(id).text) ||
(doc.title && doc.title !== mongoMap.get(id).title)
) {
console.log(`${id} had document discrepancy in ${doc.text ? 'text' : 'title'} field`);
logger.debug(
`[syncWithMeili] ${id} had document discrepancy in ${
doc.text ? 'text' : 'title'
} field`,
);
updateOps.push({
updateOne: { filter: update, update: { $set: { _meiliIndex: true } } },
});
@ -116,15 +121,14 @@ const createMeiliMongooseModel = function ({ index, attributesToIndex }) {
if (updateOps.length > 0) {
await this.collection.bulkWrite(updateOps);
console.log(
`[Meilisearch] Finished indexing ${
logger.debug(
`[syncWithMeili] Finished indexing ${
primaryKey === 'messageId' ? 'messages' : 'conversations'
}`,
);
}
} catch (error) {
console.log('[Meilisearch] Error adding document to Meili');
console.error(error);
logger.error('[syncWithMeili] Error adding document to Meili', error);
}
}
@ -143,7 +147,7 @@ const createMeiliMongooseModel = function ({ index, attributesToIndex }) {
const query = {};
// query[primaryKey] = { $in: _.map(data.hits, primaryKey) };
query[primaryKey] = _.map(data.hits, (hit) => cleanUpPrimaryKeyValue(hit[primaryKey]));
// console.log('query', query);
// logger.debug('query', query);
const hitsFromMongoose = await this.find(
query,
_.reduce(
@ -186,11 +190,11 @@ const createMeiliMongooseModel = function ({ index, attributesToIndex }) {
async addObjectToMeili() {
const object = this.preprocessObjectForIndex();
try {
// console.log('Adding document to Meili', object);
// logger.debug('Adding document to Meili', object);
await index.addDocuments([object]);
} catch (error) {
// console.log('Error adding document to Meili');
// console.error(error);
// logger.debug('Error adding document to Meili');
// logger.error(error);
}
await this.collection.updateMany({ _id: this._id }, { $set: { _meiliIndex: true } });
@ -311,10 +315,10 @@ module.exports = function mongoMeili(schema, options) {
return next();
} catch (error) {
if (meiliEnabled) {
console.log(
'[Meilisearch] There was an issue deleting conversation indexes upon deletion, next startup may be slow due to syncing',
logger.error(
'[MeiliMongooseModel.deleteMany] There was an issue deleting conversation indexes upon deletion, next startup may be slow due to syncing',
error,
);
console.error(error);
}
return next();
}
@ -335,7 +339,11 @@ module.exports = function mongoMeili(schema, options) {
try {
meiliDoc = await client.index('convos').getDocument(doc.conversationId);
} catch (error) {
console.log('[Meilisearch] Convo not found and will index', doc.conversationId);
logger.error(
'[MeiliMongooseModel.findOneAndUpdate] Convo not found in MeiliSearch and will index ' +
doc.conversationId,
error,
);
}
}