refactor(api): Central Logging 📜 (#1348)

* WIP: initial logging changes
add several transports in ~/config/winston
omit messages in logs, truncate long strings
add short blurb in dotenv for debug logging
GoogleClient: using logger
OpenAIClient: using logger, handleOpenAIErrors
Adding typedef for payload message
bumped winston and using winston-daily-rotate-file
moved config for server paths to ~/config dir
Added `DEBUG_LOGGING=true` to .env.example

* WIP: Refactor logging statements in code

* WIP: Refactor logging statements and import configurations

* WIP: Refactor logging statements and import configurations

* refactor: broadcast Redis initialization message with `info` not `debug`

* refactor: complete Refactor logging statements and import configurations

* chore: delete unused tools

* fix: circular dependencies due to accessing logger

* refactor(handleText): handle booleans and write tests

* refactor: redact sensitive values, better formatting

* chore: improve log formatting, avoid passing strings to 2nd arg

* fix(ci): fix jest tests due to logger changes

* refactor(getAvailablePluginsController): cache plugins as they are static and avoids async addOpenAPISpecs call every time

* chore: update docs

* chore: update docs

* chore: create separate meiliSync logger, clean up logs to avoid being unnecessarily verbose

* chore: spread objects where they are commonly logged to allow string truncation

* chore: improve error log formatting

api/server/controllers/AskController.js

@@ -1,7 +1,8 @@
+const { getResponseSender } = require('librechat-data-provider');
 const { sendMessage, createOnProgress } = require('~/server/utils');
 const { saveMessage, getConvoTitle, getConvo } = require('~/models');
-const { getResponseSender } = require('librechat-data-provider');
 const { createAbortController, handleAbortError } = require('~/server/middleware');
+const { logger } = require('~/config');
 
 const AskController = async (req, res, next, initializeClient) => {
   let {
@@ -11,8 +12,7 @@ const AskController = async (req, res, next, initializeClient) => {
     parentMessageId = null,
     overrideParentMessageId = null,
   } = req.body;
-  console.log('ask log');
-  console.dir({ text, conversationId, endpointOption }, { depth: null });
+  logger.debug('[AskController]', { text, conversationId, ...endpointOption });
   let metadata;
   let userMessage;
   let promptTokens;

api/server/controllers/AuthController.js

@@ -1,13 +1,14 @@
 const crypto = require('crypto');
 const cookies = require('cookie');
 const jwt = require('jsonwebtoken');
-const { Session, User } = require('../../models');
+const { Session, User } = require('~/models');
 const {
   registerUser,
-  requestPasswordReset,
   resetPassword,
   setAuthTokens,
-} = require('../services/AuthService');
+  requestPasswordReset,
+} = require('~/server/services/AuthService');
+const { logger } = require('~/config');
 
 const registrationController = async (req, res) => {
   try {
@@ -27,7 +28,7 @@ const registrationController = async (req, res) => {
       res.status(status).send({ message });
     }
   } catch (err) {
-    console.log(err);
+    logger.error('[registrationController]', err);
     return res.status(500).json({ message: err.message });
   }
 };
@@ -45,7 +46,7 @@ const resetPasswordRequestController = async (req, res) => {
       return res.status(200).json(resetService);
     }
   } catch (e) {
-    console.log(e);
+    logger.error('[resetPasswordRequestController]', e);
     return res.status(400).json({ message: e.message });
   }
 };
@@ -63,7 +64,7 @@ const resetPasswordController = async (req, res) => {
       return res.status(200).json(resetPasswordService);
     }
   } catch (e) {
-    console.log(e);
+    logger.error('[resetPasswordController]', e);
     return res.status(400).json({ message: e.message });
   }
 };
@@ -108,8 +109,7 @@ const refreshController = async (req, res) => {
       res.status(401).send('Refresh token expired or not found for this user');
     }
   } catch (err) {
-    console.error('Refresh token error', refreshToken);
-    console.error(err);
+    logger.error(`[refreshController] Refresh token: ${refreshToken}`, err);
     res.status(403).send('Invalid refresh token');
   }
 };

api/server/controllers/EditController.js

@@ -1,7 +1,8 @@
+const { getResponseSender } = require('librechat-data-provider');
 const { sendMessage, createOnProgress } = require('~/server/utils');
 const { saveMessage, getConvoTitle, getConvo } = require('~/models');
-const { getResponseSender } = require('librechat-data-provider');
 const { createAbortController, handleAbortError } = require('~/server/middleware');
+const { logger } = require('~/config');
 
 const EditController = async (req, res, next, initializeClient) => {
   let {
@@ -14,8 +15,13 @@ const EditController = async (req, res, next, initializeClient) => {
     parentMessageId = null,
     overrideParentMessageId = null,
   } = req.body;
-  console.log('edit log');
-  console.dir({ text, generation, isContinued, conversationId, endpointOption }, { depth: null });
+  logger.debug('[EditController]', {
+    text,
+    generation,
+    isContinued,
+    conversationId,
+    ...endpointOption,
+  });
   let metadata;
   let userMessage;
   let promptTokens;

api/server/controllers/ErrorController.js

@@ -1,15 +1,17 @@
+const { logger } = require('~/config');
+
 //handle duplicates
 const handleDuplicateKeyError = (err, res) => {
+  logger.error('Duplicate key error:', err.keyValue);
   const field = Object.keys(err.keyValue);
   const code = 409;
   const error = `An document with that ${field} already exists.`;
-  console.log('congrats you hit the duped keys error');
   res.status(code).send({ messages: error, fields: field });
 };
 
 //handle validation errors
 const handleValidationError = (err, res) => {
-  console.log('congrats you hit the validation middleware');
+  logger.error('Validation error:', err.errors);
   let errors = Object.values(err.errors).map((el) => el.message);
   let fields = Object.values(err.errors).map((el) => el.path);
   let code = 400;
@@ -24,7 +26,6 @@ const handleValidationError = (err, res) => {
 // eslint-disable-next-line no-unused-vars
 module.exports = (err, req, res, next) => {
   try {
-    console.log('congrats you hit the error middleware');
     if (err.name === 'ValidationError') {
       return (err = handleValidationError(err, res));
     }
@@ -32,6 +33,7 @@ module.exports = (err, req, res, next) => {
       return (err = handleDuplicateKeyError(err, res));
     }
   } catch (err) {
+    logger.error('ErrorController => error', err);
     res.status(500).send('An unknown error occurred.');
   }
 };

api/server/controllers/PluginController.js

@@ -1,6 +1,8 @@
-const { promises: fs } = require('fs');
 const path = require('path');
-const { addOpenAPISpecs } = require('../../app/clients/tools/util/addOpenAPISpecs');
+const { promises: fs } = require('fs');
+const { addOpenAPISpecs } = require('~/app/clients/tools/util/addOpenAPISpecs');
+const { CacheKeys } = require('~/common/enums');
+const { getLogStores } = require('~/cache');
 
 const filterUniquePlugins = (plugins) => {
   const seen = new Set();
@@ -27,6 +29,13 @@ const isPluginAuthenticated = (plugin) => {
 
 const getAvailablePluginsController = async (req, res) => {
   try {
+    const cache = getLogStores(CacheKeys.CONFIG);
+    const cachedPlugins = await cache.get(CacheKeys.PLUGINS);
+    if (cachedPlugins) {
+      res.status(200).json(cachedPlugins);
+      return;
+    }
+
     const manifestFile = await fs.readFile(
       path.join(__dirname, '..', '..', 'app', 'clients', 'tools', 'manifest.json'),
       'utf8',
@@ -42,6 +51,7 @@ const getAvailablePluginsController = async (req, res) => {
       }
     });
     const plugins = await addOpenAPISpecs(authenticatedPlugins);
+    await cache.set(CacheKeys.PLUGINS, plugins);
     res.status(200).json(plugins);
   } catch (error) {
     res.status(500).json({ message: error.message });

api/server/controllers/UserController.js

@@ -1,5 +1,6 @@
-const { updateUserPluginsService } = require('../services/UserService');
-const { updateUserPluginAuth, deleteUserPluginAuth } = require('../services/PluginService');
+const { updateUserPluginsService } = require('~/server/services/UserService');
+const { updateUserPluginAuth, deleteUserPluginAuth } = require('~/server/services/PluginService');
+const { logger } = require('~/config');
 
 const getUserController = async (req, res) => {
   res.status(200).send(req.user);
@@ -13,7 +14,7 @@ const updateUserPluginsController = async (req, res) => {
     const userPluginsService = await updateUserPluginsService(user, pluginKey, action);
 
     if (userPluginsService instanceof Error) {
-      console.log(userPluginsService);
+      logger.error('[userPluginsService]', userPluginsService);
       const { status, message } = userPluginsService;
       res.status(status).send({ message });
     }
@@ -24,7 +25,7 @@ const updateUserPluginsController = async (req, res) => {
         for (let i = 0; i < keys.length; i++) {
           authService = await updateUserPluginAuth(user.id, keys[i], pluginKey, values[i]);
           if (authService instanceof Error) {
-            console.log(authService);
+            logger.error('[authService]', authService);
             const { status, message } = authService;
             res.status(status).send({ message });
           }
@@ -34,7 +35,7 @@ const updateUserPluginsController = async (req, res) => {
         for (let i = 0; i < keys.length; i++) {
           authService = await deleteUserPluginAuth(user.id, keys[i]);
           if (authService instanceof Error) {
-            console.log(authService);
+            logger.error('[authService]', authService);
             const { status, message } = authService;
             res.status(status).send({ message });
           }
@@ -44,7 +45,7 @@ const updateUserPluginsController = async (req, res) => {
 
     res.status(200).send();
   } catch (err) {
-    console.log(err);
+    logger.error('[updateUserPluginsController]', err);
     res.status(500).json({ message: err.message });
   }
 };

api/server/controllers/auth/LoginController.js

@@ -1,5 +1,6 @@
-const User = require('../../../models/User');
-const { setAuthTokens } = require('../../services/AuthService');
+const User = require('~/models/User');
+const { setAuthTokens } = require('~/server/services/AuthService');
+const { logger } = require('~/config');
 
 const loginController = async (req, res) => {
   try {
@@ -15,7 +16,7 @@ const loginController = async (req, res) => {
 
     return res.status(200).send({ token, user });
   } catch (err) {
-    console.log(err);
+    logger.error('[loginController]', err);
   }
 
   // Generic error messages are safer

api/server/controllers/auth/LogoutController.js

@@ -1,5 +1,6 @@
-const { logoutUser } = require('../../services/AuthService');
 const cookies = require('cookie');
+const { logoutUser } = require('~/server/services/AuthService');
+const { logger } = require('~/config');
 
 const logoutController = async (req, res) => {
   const refreshToken = req.headers.cookie ? cookies.parse(req.headers.cookie).refreshToken : null;
@@ -9,7 +10,7 @@ const logoutController = async (req, res) => {
     res.clearCookie('refreshToken');
     return res.status(status).send({ message });
   } catch (err) {
-    console.log(err);
+    logger.error('[logoutController]', err);
     return res.status(500).json({ message: err.message });
   }
 };