refactor(api): Central Logging 📜 (#1348)

* WIP: initial logging changes
add several transports in ~/config/winston
omit messages in logs, truncate long strings
add short blurb in dotenv for debug logging
GoogleClient: using logger
OpenAIClient: using logger, handleOpenAIErrors
Adding typedef for payload message
bumped winston and using winston-daily-rotate-file
moved config for server paths to ~/config dir
Added `DEBUG_LOGGING=true` to .env.example

* WIP: Refactor logging statements in code

* WIP: Refactor logging statements and import configurations

* WIP: Refactor logging statements and import configurations

* refactor: broadcast Redis initialization message with `info` not `debug`

* refactor: complete Refactor logging statements and import configurations

* chore: delete unused tools

* fix: circular dependencies due to accessing logger

* refactor(handleText): handle booleans and write tests

* refactor: redact sensitive values, better formatting

* chore: improve log formatting, avoid passing strings to 2nd arg

* fix(ci): fix jest tests due to logger changes

* refactor(getAvailablePluginsController): cache plugins as they are static and avoids async addOpenAPISpecs call every time

* chore: update docs

* chore: update docs

* chore: create separate meiliSync logger, clean up logs to avoid being unnecessarily verbose

* chore: spread objects where they are commonly logged to allow string truncation

* chore: improve error log formatting

api/server/utils/countTokens.js

@@ -1,7 +1,8 @@
-const { Tiktoken } = require('tiktoken/lite');
 const { load } = require('tiktoken/load');
+const { Tiktoken } = require('tiktoken/lite');
 const registry = require('tiktoken/registry.json');
 const models = require('tiktoken/model_to_encoding.json');
+const logger = require('~/config/winston');
 
 const countTokens = async (text = '', modelName = 'gpt-3.5-turbo') => {
   let encoder = null;
@@ -12,7 +13,7 @@ const countTokens = async (text = '', modelName = 'gpt-3.5-turbo') => {
     encoder.free();
     return tokens.length;
   } catch (e) {
-    console.error(e);
+    logger.error('[countTokens]', e);
     if (encoder) {
       encoder.free();
     }

api/server/utils/crypto.js

@@ -1,3 +1,5 @@
+require('dotenv').config();
+
 const crypto = require('crypto');
 const key = Buffer.from(process.env.CREDS_KEY, 'hex');
 const iv = Buffer.from(process.env.CREDS_IV, 'hex');

api/server/utils/handleText.js

@@ -1,6 +1,6 @@
 const partialRight = require('lodash/partialRight');
-const { getCitations, citeText } = require('./citations');
 const { sendMessage } = require('./streamResponse');
+const { getCitations, citeText } = require('./citations');
 const cursor = '<span className="result-streaming">█</span>';
 const citationRegex = /\[\^\d+?\^]/g;
 
@@ -138,21 +138,31 @@ function formatAction(action) {
 }
 
 /**
- * Checks if the given string value is truthy by comparing it to the string 'true' (case-insensitive).
+ * Checks if the given value is truthy by being either the boolean `true` or a string
+ * that case-insensitively matches 'true'.
  *
  * @function
- * @param {string|null|undefined} value - The string value to check.
- * @returns {boolean} Returns `true` if the value is a case-insensitive match for the string 'true', otherwise returns `false`.
+ * @param {string|boolean|null|undefined} value - The value to check.
+ * @returns {boolean} Returns `true` if the value is the boolean `true` or a case-insensitive
+ *                    match for the string 'true', otherwise returns `false`.
  * @example
  *
  * isEnabled("True");  // returns true
  * isEnabled("TRUE");  // returns true
+ * isEnabled(true);    // returns true
  * isEnabled("false"); // returns false
+ * isEnabled(false);   // returns false
  * isEnabled(null);    // returns false
  * isEnabled();        // returns false
  */
 function isEnabled(value) {
-  return value?.toLowerCase()?.trim() === 'true';
+  if (typeof value === 'boolean') {
+    return value;
+  }
+  if (typeof value === 'string') {
+    return value.toLowerCase().trim() === 'true';
+  }
+  return false;
 }
 
 module.exports = {

api/server/utils/handleText.spec.js

@@ -0,0 +1,51 @@
+const { isEnabled } = require('./handleText');
+
+describe('isEnabled', () => {
+  test('should return true when input is "true"', () => {
+    expect(isEnabled('true')).toBe(true);
+  });
+
+  test('should return true when input is "TRUE"', () => {
+    expect(isEnabled('TRUE')).toBe(true);
+  });
+
+  test('should return true when input is true', () => {
+    expect(isEnabled(true)).toBe(true);
+  });
+
+  test('should return false when input is "false"', () => {
+    expect(isEnabled('false')).toBe(false);
+  });
+
+  test('should return false when input is false', () => {
+    expect(isEnabled(false)).toBe(false);
+  });
+
+  test('should return false when input is null', () => {
+    expect(isEnabled(null)).toBe(false);
+  });
+
+  test('should return false when input is undefined', () => {
+    expect(isEnabled()).toBe(false);
+  });
+
+  test('should return false when input is an empty string', () => {
+    expect(isEnabled('')).toBe(false);
+  });
+
+  test('should return false when input is a whitespace string', () => {
+    expect(isEnabled('   ')).toBe(false);
+  });
+
+  test('should return false when input is a number', () => {
+    expect(isEnabled(123)).toBe(false);
+  });
+
+  test('should return false when input is an object', () => {
+    expect(isEnabled({})).toBe(false);
+  });
+
+  test('should return false when input is an array', () => {
+    expect(isEnabled([])).toBe(false);
+  });
+});

api/server/utils/math.js

@@ -38,8 +38,7 @@ function math(str, fallbackValue) {
     if (fallback) {
       return fallbackValue;
     }
-    console.error('str', str);
-    throw new Error(`str did not evaluate to a number but to a ${typeof value}`);
+    throw new Error(`[math] str did not evaluate to a number but to a ${typeof value}`);
   }
 
   return value;

api/server/utils/sendEmail.js

@@ -1,7 +1,8 @@
-const nodemailer = require('nodemailer');
-const handlebars = require('handlebars');
 const fs = require('fs');
 const path = require('path');
+const nodemailer = require('nodemailer');
+const handlebars = require('handlebars');
+const logger = require('~/config/winston');
 
 const sendEmail = async (email, subject, payload, template) => {
   try {
@@ -58,15 +59,15 @@ const sendEmail = async (email, subject, payload, template) => {
     // Send email
     transporter.sendMail(options(), (error, info) => {
       if (error) {
-        console.log(error);
+        logger.error('[sendEmail]', error);
         return error;
       } else {
-        console.log(info);
+        logger.debug('[sendEmail]', info);
         return info;
       }
     });
   } catch (error) {
-    console.log(error);
+    logger.error('[sendEmail]', error);
     return error;
   }
 };

api/server/utils/streamResponse.js

@@ -1,5 +1,5 @@
 const crypto = require('crypto');
-const { saveMessage } = require('../../models/Message');
+const { saveMessage } = require('~/models/Message');
 
 /**
  * Sends error data in Server Sent Events format and ends the response.