Feature/logging system with pino and sanitization (#214) (#227)

* feat: Create structured data logging system with Pino

This commit creates a new feature that enables structured data logging using the Pino logging library. The structured data logging feature allows for more granular and customizable logging, making it easier to analyze and debug issues in the application.

The changes made in this commit include:
- Adding support for structured data logging using the Pino API
- Adding support to redact sensible data from logging output using pino
  redact.
- Pino integrate natively with fluentd, logstash, Docker Logging Drivers
  and other JSON based system.

* Add pino package to project

* Logging-System: Add support for an array of regex to redact

* Logging-Systems: Add Redact Patterns and Pino Redact Paths + Boolean logics wasn't right.

Co-authored-by: Olivier Contant <ocontant@users.noreply.github.com>
This commit is contained in:
Dan Orlando 2023-05-10 20:59:26 -07:00 committed by GitHub
parent c72a3a0362
commit 53ea3dd9fb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 470 additions and 14 deletions

125
api/utils/LoggingSystem.js Normal file
View file

@ -0,0 +1,125 @@
const pino = require('pino');
const logger = pino({
level: 'info',
redact: {
paths: [ // List of Paths to redact from the logs (https://getpino.io/#/docs/redaction)
'env.OPENAI_KEY',
'env.BINGAI_TOKEN',
'env.CHATGPT_TOKEN',
'env.MEILI_MASTER_KEY',
'env.GOOGLE_CLIENT_SECRET',
'env.JWT_SECRET_DEV',
'env.JWT_SECRET_PROD',
'newUser.password'], // See example to filter object class instances
censor: '***', // Redaction character
},
});
// Sanitize outside the logger paths. This is useful for sanitizing variables directly with Regex and patterns.
const redactPatterns = [ // Array of regular expressions for redacting patterns
/api[-_]?key/i,
/password/i,
/token/i,
/secret/i,
/key/i,
/certificate/i,
/client[-_]?id/i,
/authorization[-_]?code/i,
/authorization[-_]?login[-_]?hint/i,
/authorization[-_]?acr[-_]?values/i,
/authorization[-_]?response[-_]?mode/i,
/authorization[-_]?nonce/i
];
/*
// Example of redacting sensitive data from object class instances
function redactSensitiveData(obj) {
if (obj instanceof User) {
return {
...obj.toObject(),
password: '***', // Redact the password field
};
}
return obj;
}
// Example of redacting sensitive data from object class instances
logger.info({ newUser: redactSensitiveData(newUser) }, 'newUser');
*/
const levels = {
TRACE: 10,
DEBUG: 20,
INFO: 30,
WARN: 40,
ERROR: 50,
FATAL: 60
};
let level = levels.INFO;
module.exports = {
levels,
setLevel: (l) => (level = l),
log: {
trace: (msg) => {
if (level <= levels.TRACE) return;
logger.trace(msg);
},
debug: (msg) => {
if (level <= levels.DEBUG) return;
logger.debug(msg);
},
info: (msg) => {
if (level <= levels.INFO) return;
logger.info(msg);
},
warn: (msg) => {
if (level <= levels.WARN) return;
logger.warn(msg);
},
error: (msg) => {
if (level <= levels.ERROR) return;
logger.error(msg);
},
fatal: (msg) => {
if (level <= levels.FATAL) return;
logger.fatal(msg);
},
// Custom loggers
parameters: (parameters) => {
if (level <= levels.TRACE) return;
logger.debug({ parameters }, 'Function Parameters');
},
functionName: (name) => {
if (level <= levels.TRACE) return;
logger.debug(`EXECUTING: ${name}`);
},
flow: (flow) => {
if (level <= levels.INFO) return;
logger.debug(`BEGIN FLOW: ${flow}`);
},
variable: ({ name, value }) => {
if (level <= levels.DEBUG) return;
// Check if the variable name matches any of the redact patterns and redact the value
let sanitizedValue = value;
for (const pattern of redactPatterns) {
if (pattern.test(name)) {
sanitizedValue = '***';
break;
}
}
logger.debug({ variable: { name, value: sanitizedValue } }, `VARIABLE ${name}`);
},
request: () => (req, res, next) => {
if (level < levels.DEBUG) return next();
logger.debug({ query: req.query, body: req.body }, `Hit URL ${req.url} with following`);
return next();
}
}
};