refactor(api): Central Logging 📜 (#1348)

* WIP: initial logging changes
add several transports in ~/config/winston
omit messages in logs, truncate long strings
add short blurb in dotenv for debug logging
GoogleClient: using logger
OpenAIClient: using logger, handleOpenAIErrors
Adding typedef for payload message
bumped winston and using winston-daily-rotate-file
moved config for server paths to ~/config dir
Added `DEBUG_LOGGING=true` to .env.example

* WIP: Refactor logging statements in code

* WIP: Refactor logging statements and import configurations

* WIP: Refactor logging statements and import configurations

* refactor: broadcast Redis initialization message with `info` not `debug`

* refactor: complete Refactor logging statements and import configurations

* chore: delete unused tools

* fix: circular dependencies due to accessing logger

* refactor(handleText): handle booleans and write tests

* refactor: redact sensitive values, better formatting

* chore: improve log formatting, avoid passing strings to 2nd arg

* fix(ci): fix jest tests due to logger changes

* refactor(getAvailablePluginsController): cache plugins as they are static and avoids async addOpenAPISpecs call every time

* chore: update docs

* chore: update docs

* chore: create separate meiliSync logger, clean up logs to avoid being unnecessarily verbose

* chore: spread objects where they are commonly logged to allow string truncation

* chore: improve error log formatting

api/server/services/AuthService.js

@@ -1,10 +1,12 @@
 const crypto = require('crypto');
 const bcrypt = require('bcryptjs');
-const User = require('../../models/User');
-const Session = require('../../models/Session');
-const Token = require('../../models/schema/tokenSchema');
-const { registerSchema, errorsToString } = require('../../strategies/validators');
-const { sendEmail } = require('../utils');
+const { registerSchema, errorsToString } = require('~/strategies/validators');
+const Token = require('~/models/schema/tokenSchema');
+const { sendEmail } = require('~/server/utils');
+const Session = require('~/models/Session');
+const { logger } = require('~/config');
+const User = require('~/models/User');
+
 const domains = {
   client: process.env.DOMAIN_CLIENT,
   server: process.env.DOMAIN_SERVER,
@@ -29,7 +31,7 @@ const logoutUser = async (userId, refreshToken) => {
       try {
         await Session.deleteOne({ _id: session._id });
       } catch (deleteErr) {
-        console.error(deleteErr);
+        logger.error('[logoutUser] Failed to delete session.', deleteErr);
         return { status: 500, message: 'Failed to delete session.' };
       }
     }
@@ -50,7 +52,7 @@ const registerUser = async (user) => {
   const { error } = registerSchema.safeParse(user);
   if (error) {
     const errorMessage = errorsToString(error.errors);
-    console.info(
+    logger.info(
       'Route: register - Validation Error',
       { name: 'Request params:', value: user },
       { name: 'Validation error:', value: errorMessage },
@@ -65,7 +67,7 @@ const registerUser = async (user) => {
     const existingUser = await User.findOne({ email }).lean();
 
     if (existingUser) {
-      console.info(
+      logger.info(
         'Register User - Email in use',
         { name: 'Request params:', value: user },
         { name: 'Existing user:', value: existingUser },
@@ -229,7 +231,7 @@ const setAuthTokens = async (userId, res, sessionId = null) => {
 
     return token;
   } catch (error) {
-    console.log('Error in setting authentication tokens:', error);
+    logger.error('[setAuthTokens] Error in setting authentication tokens:', error);
     throw error;
   }
 };

api/server/services/Files/save.js

@@ -1,5 +1,6 @@
 const fs = require('fs');
 const path = require('path');
+const { logger } = require('~/config');
 
 /**
  * Saves a file to a specified output path with a new filename.
@@ -24,7 +25,7 @@ async function saveFile(file, outputPath, outputFilename) {
 
     return outputFilePath;
   } catch (error) {
-    console.error('Error while saving the file:', error);
+    logger.error('[saveFile] Error while saving the file:', error);
     throw error;
   }
 }

api/server/services/ModelService.js

@@ -1,10 +1,13 @@
-const HttpsProxyAgent = require('https-proxy-agent');
-const axios = require('axios');
 const Keyv = require('keyv');
+const axios = require('axios');
+const HttpsProxyAgent = require('https-proxy-agent');
 const { isEnabled } = require('~/server/utils');
-const { extractBaseURL } = require('~/utils');
 const keyvRedis = require('~/cache/keyvRedis');
+const { extractBaseURL } = require('~/utils');
+const { logger } = require('~/config');
+
 // const { getAzureCredentials, genAzureChatCompletion } = require('~/utils/');
+
 const { openAIApiKey, userProvidedOpenAI } = require('./Config/EndpointService').config;
 
 const modelsCache = isEnabled(process.env.USE_REDIS)
@@ -54,9 +57,9 @@ const fetchOpenAIModels = async (opts = { azure: false, plugins: false }, _model
       const res = await axios.get(`${basePath}${opts.azure ? '' : '/models'}`, payload);
 
       models = res.data.data.map((item) => item.id);
-      // console.log(`Fetched ${models.length} models from ${opts.azure ? 'Azure ' : ''}OpenAI API`);
+      // logger.debug(`Fetched ${models.length} models from ${opts.azure ? 'Azure ' : ''}OpenAI API`);
     } catch (err) {
-      console.log(`Failed to fetch models from ${opts.azure ? 'Azure ' : ''}OpenAI API`);
+      logger.error(`Failed to fetch models from ${opts.azure ? 'Azure ' : ''}OpenAI API`, err);
     }
   }
 

api/server/services/PluginService.js

@@ -1,5 +1,6 @@
-const PluginAuth = require('../../models/schema/pluginAuthSchema');
-const { encrypt, decrypt } = require('../utils/');
+const PluginAuth = require('~/models/schema/pluginAuthSchema');
+const { encrypt, decrypt } = require('~/server/utils/');
+const { logger } = require('~/config');
 
 const getUserPluginAuthValue = async (user, authField) => {
   try {
@@ -11,7 +12,7 @@ const getUserPluginAuthValue = async (user, authField) => {
     const decryptedValue = decrypt(pluginAuth.value);
     return decryptedValue;
   } catch (err) {
-    console.log(err);
+    logger.error('[getUserPluginAuthValue]', err);
     return err;
   }
 };
@@ -36,7 +37,7 @@ const getUserPluginAuthValue = async (user, authField) => {
 
 //     return pluginAuth;
 //   } catch (err) {
-//     console.log(err);
+//     logger.error('[getUserPluginAuthValue]', err);
 //     return err;
 //   }
 // };
@@ -62,7 +63,7 @@ const updateUserPluginAuth = async (userId, authField, pluginKey, value) => {
       return newPluginAuth;
     }
   } catch (err) {
-    console.log(err);
+    logger.error('[updateUserPluginAuth]', err);
     return err;
   }
 };
@@ -72,7 +73,7 @@ const deleteUserPluginAuth = async (userId, authField) => {
     const response = await PluginAuth.deleteOne({ userId, authField });
     return response;
   } catch (err) {
-    console.log(err);
+    logger.error('[deleteUserPluginAuth]', err);
     return err;
   }
 };

api/server/services/Runs/RunMananger.js

@@ -1,3 +1,5 @@
+const { logger } = require('~/config');
+
 /**
  * @typedef {import('openai').OpenAI} OpenAI
  * @typedef {import('../AssistantService').RunStep} RunStep
@@ -84,8 +86,12 @@ class RunManager {
       return await this.handlers['final']({ step, runStatus, stepsByStatus: this.stepsByStatus });
     }
 
-    console.log(`Default handler for ${step.id} with status \`${runStatus}\``);
-    console.dir({ step, runStatus, final, isLast }, { depth: null });
+    logger.debug(`[RunManager] Default handler for ${step.id} with status \`${runStatus}\``, {
+      step,
+      runStatus,
+      final,
+      isLast,
+    });
     return step;
   }
 }

api/server/services/UserService.js

@@ -1,5 +1,6 @@
-const { User, Key } = require('../../models');
-const { encrypt, decrypt } = require('../utils');
+const { User, Key } = require('~/models');
+const { encrypt, decrypt } = require('~/server/utils');
+const { logger } = require('~/config');
 
 const updateUserPluginsService = async (user, pluginKey, action) => {
   try {
@@ -15,7 +16,7 @@ const updateUserPluginsService = async (user, pluginKey, action) => {
       );
     }
   } catch (err) {
-    console.log(err);
+    logger.error('[updateUserPluginsService]', err);
     return err;
   }
 };