Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-12-24 04:10:15 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor(api): Central Logging 📜 (#1348)

Browse source 
* WIP: initial logging changes
add several transports in ~/config/winston
omit messages in logs, truncate long strings
add short blurb in dotenv for debug logging
GoogleClient: using logger
OpenAIClient: using logger, handleOpenAIErrors
Adding typedef for payload message
bumped winston and using winston-daily-rotate-file
moved config for server paths to ~/config dir
Added `DEBUG_LOGGING=true` to .env.example

* WIP: Refactor logging statements in code

* WIP: Refactor logging statements and import configurations

* WIP: Refactor logging statements and import configurations

* refactor: broadcast Redis initialization message with `info` not `debug`

* refactor: complete Refactor logging statements and import configurations

* chore: delete unused tools

* fix: circular dependencies due to accessing logger

* refactor(handleText): handle booleans and write tests

* refactor: redact sensitive values, better formatting

* chore: improve log formatting, avoid passing strings to 2nd arg

* fix(ci): fix jest tests due to logger changes

* refactor(getAvailablePluginsController): cache plugins as they are static and avoids async addOpenAPISpecs call every time

* chore: update docs

* chore: update docs

* chore: create separate meiliSync logger, clean up logs to avoid being unnecessarily verbose

* chore: spread objects where they are commonly logged to allow string truncation

* chore: improve error log formatting
This commit is contained in:
Danny Avila 2023-12-14 07:49:27 -05:00 • committed by GitHub
parent 49571ac635
commit ea1dd59ef4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
115 changed files with 1271 additions and 1001 deletions
Download patch file Download diff file Expand all files Collapse all files

3
api/server/routes/ask/addToCache.js
View file

@ -1,5 +1,6 @@
const Keyv = require('keyv');
const { KeyvFile } = require('keyv-file');
const { logger } = require('~/config');
const addToCache = async ({ endpoint, endpointOption, userMessage, responseMessage }) => {
try {
@ -57,7 +58,7 @@ const addToCache = async ({ endpoint, endpointOption, userMessage, responseMessa
await conversationsCache.set(conversationId, conversation);
} catch (error) {
console.error('Trouble adding to cache', error);
logger.error('[addToCache] Error adding conversation to cache', error);
}
};

18
api/server/routes/ask/askChatGPTBrowser.js
View file

@ -1,10 +1,12 @@
const express = require('express');
const crypto = require('crypto');
const express = require('express');
const { saveMessage, getConvoTitle, saveConvo, getConvo } = require('~/models');
const { handleError, sendMessage, createOnProgress, handleText } = require('~/server/utils');
const { setHeaders } = require('~/server/middleware');
const { browserClient } = require('~/app/');
const { logger } = require('~/config');
const router = express.Router();
const { browserClient } = require('../../../app/');
const { saveMessage, getConvoTitle, saveConvo, getConvo } = require('../../../models');
const { handleError, sendMessage, createOnProgress, handleText } = require('../../utils');
const { setHeaders } = require('../../middleware');
router.post('/', setHeaders, async (req, res) => {
const {
@ -41,10 +43,10 @@ router.post('/', setHeaders, async (req, res) => {
key: req.body?.key ?? null,
};
console.log('ask log', {
logger.debug('[/ask/chatGPTBrowser]', {
userMessage,
endpointOption,
conversationId,
...endpointOption,
});
if (!overrideParentMessageId) {
@ -136,7 +138,7 @@ const ask = async ({
},
});
console.log('CLIENT RESPONSE', response);
logger.debug('[/ask/chatGPTBrowser]', response);
const newConversationId = response.conversationId || conversationId;
const newUserMassageId = response.parentMessageId || userMessageId;

20
api/server/routes/ask/bingAI.js
View file

@ -1,10 +1,12 @@
const express = require('express');
const crypto = require('crypto');
const { handleError, sendMessage, createOnProgress, handleText } = require('~/server/utils');
const { saveMessage, getConvoTitle, saveConvo, getConvo } = require('~/models');
const { setHeaders } = require('~/server/middleware');
const { titleConvoBing, askBing } = require('~/app');
const { logger } = require('~/config');
const router = express.Router();
const { titleConvoBing, askBing } = require('../../../app');
const { saveMessage, getConvoTitle, saveConvo, getConvo } = require('../../../models');
const { handleError, sendMessage, createOnProgress, handleText } = require('../../utils');
const { setHeaders } = require('../../middleware');
router.post('/', setHeaders, async (req, res) => {
const {
@ -60,7 +62,7 @@ router.post('/', setHeaders, async (req, res) => {
};
}
console.log('ask log', {
logger.debug('[/ask/bingAI] ask log', {
userMessage,
endpointOption,
conversationId,
@ -153,10 +155,10 @@ const ask = async ({
abortController,
});
console.log('BING RESPONSE', response);
logger.debug('[/ask/bingAI] BING RESPONSE', response);
if (response.details && response.details.scores) {
console.log('SCORES', response.details.scores);
logger.debug('[/ask/bingAI] SCORES', response.details.scores);
}
const newConversationId = endpointOption?.jailbreak
@ -250,7 +252,7 @@ const ask = async ({
});
}
} catch (error) {
console.error(error);
logger.error('[/ask/bingAI] Error handling BingAI response', error);
const partialText = getPartialText();
if (partialText?.length > 2) {
const responseMessage = {
@ -276,7 +278,7 @@ const ask = async ({
responseMessage: responseMessage,
};
} else {
console.log(error);
logger.error('[/ask/bingAI] Error handling BingAI response', error);
const errorMessage = {
messageId: responseMessageId,
sender: model,

8
api/server/routes/ask/gptPlugins.js
View file

@ -14,6 +14,7 @@ const {
validateEndpoint,
buildEndpointOption,
} = require('~/server/middleware');
const { logger } = require('~/config');
router.post('/abort', handleAbort());
@ -25,8 +26,7 @@ router.post('/', validateEndpoint, buildEndpointOption, setHeaders, async (req,
parentMessageId = null,
overrideParentMessageId = null,
} = req.body;
console.log('ask log');
console.dir({ text, conversationId, endpointOption }, { depth: null });
logger.debug('[/ask/gptPlugins]', { text, conversationId, ...endpointOption });
let metadata;
let userMessage;
let promptTokens;
@ -189,8 +189,8 @@ router.post('/', validateEndpoint, buildEndpointOption, setHeaders, async (req,
response = { ...response, ...metadata };
}
console.log('CLIENT RESPONSE');
console.dir(response, { depth: null });
logger.debug('[/ask/gptPlugins]', response);
response.plugins = plugins.map((p) => ({ ...p, loading: false }));
await saveMessage({ ...response, user });

6
api/server/routes/ask/openAI.js
View file

@ -12,6 +12,7 @@ const {
validateEndpoint,
buildEndpointOption,
} = require('~/server/middleware');
const { logger } = require('~/config');
router.post('/abort', handleAbort());
@ -23,8 +24,9 @@ router.post('/', validateEndpoint, buildEndpointOption, setHeaders, async (req,
parentMessageId = null,
overrideParentMessageId = null,
} = req.body;
console.log('ask log');
console.dir({ text, conversationId, endpointOption }, { depth: null });
logger.debug('[/ask/openAI]', { text, conversationId, ...endpointOption });
let metadata;
let userMessage;
let promptTokens;

4
api/server/routes/assistants/assistants.js
View file

@ -1,5 +1,7 @@
const OpenAI = require('openai');
const express = require('express');
const { logger } = require('~/config');
const router = express.Router();
/**
@ -13,7 +15,7 @@ router.post('/', async (req, res) => {
const openai = new OpenAI(process.env.OPENAI_API_KEY);
const assistantData = req.body;
const assistant = await openai.beta.assistants.create(assistantData);
console.log(assistant);
logger.debug('/assistants/', assistant);
res.status(201).json(assistant);
} catch (error) {
res.status(500).json({ error: error.message });

5
api/server/routes/assistants/chat.js
View file

@ -1,5 +1,6 @@
const crypto = require('crypto');
const OpenAI = require('openai');
const { logger } = require('~/config');
const { sendMessage } = require('../../utils');
const { initThread, createRun, handleRun } = require('../../services/AssistantService');
const express = require('express');
@ -23,7 +24,7 @@ const {
*/
router.post('/', setHeaders, async (req, res) => {
try {
console.log(req.body);
logger.debug('[/assistants/chat/] req.body', req.body);
// test message:
// How many polls of 500 ms intervals are there in 18 seconds?
@ -100,7 +101,7 @@ router.post('/', setHeaders, async (req, res) => {
res.end();
} catch (error) {
// res.status(500).json({ error: error.message });
console.error(error);
logger.error('[/assistants/chat/]', error);
res.end();
}
});

6
api/server/routes/config.js
View file

@ -1,6 +1,8 @@
const express = require('express');
const { isEnabled } = require('~/server/utils');
const { logger } = require('~/config');
const router = express.Router();
const { isEnabled } = require('../utils');
const emailLoginEnabled =
process.env.ALLOW_EMAIL_LOGIN === undefined || isEnabled(process.env.ALLOW_EMAIL_LOGIN);
@ -38,7 +40,7 @@ router.get('/', async function (req, res) {
return res.status(200).send(payload);
} catch (err) {
console.error(err);
logger.error('Error in startup config', err);
return res.status(500).send({ error: err.message });
}
});

13
api/server/routes/convos.js
View file

@ -1,8 +1,9 @@
const express = require('express');
const router = express.Router();
const { getConvo, saveConvo } = require('../../models');
const { getConvosByPage, deleteConvos } = require('../../models/Conversation');
const requireJwtAuth = require('../middleware/requireJwtAuth');
const { getConvosByPage, deleteConvos } = require('~/models/Conversation');
const requireJwtAuth = require('~/server/middleware/requireJwtAuth');
const { getConvo, saveConvo } = require('~/models');
const { logger } = require('~/config');
router.use(requireJwtAuth);
@ -30,7 +31,7 @@ router.post('/clear', async (req, res) => {
}
// for debugging deletion source
// console.log('source:', source);
// logger.debug('source:', source);
if (source === 'button' && !conversationId) {
return res.status(200).send('No conversationId provided');
@ -40,7 +41,7 @@ router.post('/clear', async (req, res) => {
const dbResponse = await deleteConvos(req.user.id, filter);
res.status(201).send(dbResponse);
} catch (error) {
console.error(error);
logger.error('Error clearing conversations', error);
res.status(500).send(error);
}
});
@ -52,7 +53,7 @@ router.post('/update', async (req, res) => {
const dbResponse = await saveConvo(req.user.id, update);
res.status(201).send(dbResponse);
} catch (error) {
console.error(error);
logger.error('Error updating conversation', error);
res.status(500).send(error);
}
});

20
api/server/routes/edit/gptPlugins.js
View file

@ -1,8 +1,8 @@
const express = require('express');
const router = express.Router();
const { validateTools } = require('~/app');
const { saveMessage, getConvoTitle, getConvo } = require('~/models');
const { getResponseSender } = require('librechat-data-provider');
const { saveMessage, getConvoTitle, getConvo } = require('~/models');
const { initializeClient } = require('~/server/services/Endpoints/gptPlugins');
const { sendMessage, createOnProgress, formatSteps, formatAction } = require('~/server/utils');
const {
@ -13,6 +13,7 @@ const {
validateEndpoint,
buildEndpointOption,
} = require('~/server/middleware');
const { logger } = require('~/config');
router.post('/abort', handleAbort());
@ -27,8 +28,14 @@ router.post('/', validateEndpoint, buildEndpointOption, setHeaders, async (req,
parentMessageId = null,
overrideParentMessageId = null,
} = req.body;
console.log('edit log');
console.dir({ text, generation, isContinued, conversationId, endpointOption }, { depth: null });
logger.debug('[/edit/gptPlugins]', {
text,
generation,
isContinued,
conversationId,
...endpointOption,
});
let metadata;
let userMessage;
let promptTokens;
@ -102,7 +109,7 @@ router.post('/', validateEndpoint, buildEndpointOption, setHeaders, async (req,
saveMessage({ ...userMessage, user });
}
sendIntermediateMessage(res, { plugin });
// console.log('PLUGIN ACTION', formattedAction);
// logger.debug('PLUGIN ACTION', formattedAction);
};
const onChainEnd = (data) => {
@ -111,7 +118,7 @@ router.post('/', validateEndpoint, buildEndpointOption, setHeaders, async (req,
plugin.loading = false;
saveMessage({ ...userMessage, user });
sendIntermediateMessage(res, { plugin });
// console.log('CHAIN END', plugin.outputs);
// logger.debug('CHAIN END', plugin.outputs);
};
const getAbortData = () => ({
@ -162,8 +169,7 @@ router.post('/', validateEndpoint, buildEndpointOption, setHeaders, async (req,
response = { ...response, ...metadata };
}
console.log('CLIENT RESPONSE');
console.dir(response, { depth: null });
logger.debug('[/edit/gptPlugins] CLIENT RESPONSE', response);
response.plugin = { ...plugin, loading: false };
await saveMessage({ ...response, user });

12
api/server/routes/edit/openAI.js
View file

@ -12,6 +12,7 @@ const {
validateEndpoint,
buildEndpointOption,
} = require('~/server/middleware');
const { logger } = require('~/config');
router.post('/abort', handleAbort());
@ -26,8 +27,15 @@ router.post('/', validateEndpoint, buildEndpointOption, setHeaders, async (req,
parentMessageId = null,
overrideParentMessageId = null,
} = req.body;
console.log('edit log');
console.dir({ text, generation, isContinued, conversationId, endpointOption }, { depth: null });
logger.debug('[/edit/openAI]', {
text,
generation,
isContinued,
conversationId,
...endpointOption,
});
let metadata;
let userMessage;
let promptTokens;

5
api/server/routes/files/files.js
View file

@ -1,8 +1,9 @@
const { z } = require('zod');
const path = require('path');
const fs = require('fs').promises;
const express = require('express');
const { deleteFiles } = require('~/models');
const path = require('path');
const { logger } = require('~/config');
const router = express.Router();
@ -55,7 +56,7 @@ router.delete('/', async (req, res) => {
await Promise.all(promises);
res.status(200).json({ message: 'Files deleted successfully' });
} catch (error) {
console.error('Error deleting files:', error);
logger.error('[/files] Error deleting files:', error);
res.status(400).json({ message: 'Error in request', error: error.message });
}
});

7
api/server/routes/files/images.js
View file

@ -3,6 +3,7 @@ const fs = require('fs').promises;
const express = require('express');
const upload = require('./multer');
const { localStrategy } = require('~/server/services/Files');
const { logger } = require('~/config');
const router = express.Router();
@ -35,11 +36,11 @@ router.post('/', upload.single('file'), async (req, res) => {
metadata.file_id = req.file_id;
await localStrategy({ req, res, file, metadata });
} catch (error) {
console.error('Error processing file:', error);
logger.error('[/files/images] Error processing file:', error);
try {
await fs.unlink(file.path);
} catch (error) {
console.error('Error deleting file:', error);
logger.error('[/files/images] Error deleting file:', error);
}
res.status(500).json({ message: 'Error processing file' });
}
@ -49,7 +50,7 @@ router.post('/', upload.single('file'), async (req, res) => {
// try {
// // await fs.unlink(file.path);
// } catch (error) {
// console.error('Error deleting file:', error);
// logger.error('[/files/images] Error deleting file:', error);
// }
// }

8
api/server/routes/oauth.js
View file

@ -1,8 +1,10 @@
const passport = require('passport');
const express = require('express');
const router = express.Router();
const { setAuthTokens } = require('../services/AuthService');
const { loginLimiter, checkBan } = require('../middleware');
const { setAuthTokens } = require('~/server/services/AuthService');
const { loginLimiter, checkBan } = require('~/server/middleware');
const { logger } = require('~/config');
const domains = {
client: process.env.DOMAIN_CLIENT,
server: process.env.DOMAIN_SERVER,
@ -19,7 +21,7 @@ const oauthHandler = async (req, res) => {
await setAuthTokens(req.user._id, res);
res.redirect(domains.client);
} catch (err) {
console.error('Error in setting authentication tokens:', err);
logger.error('Error in setting authentication tokens:', err);
}
};

14
api/server/routes/presets.js
View file

@ -1,8 +1,10 @@
const express = require('express');
const router = express.Router();
const { getPresets, savePreset, deletePresets } = require('../../models');
const crypto = require('crypto');
const requireJwtAuth = require('../middleware/requireJwtAuth');
const { getPresets, savePreset, deletePresets } = require('~/models');
const requireJwtAuth = require('~/server/middleware/requireJwtAuth');
const { logger } = require('~/config');
const router = express.Router();
router.get('/', requireJwtAuth, async (req, res) => {
const presets = (await getPresets(req.user.id)).map((preset) => preset);
@ -18,7 +20,7 @@ router.post('/', requireJwtAuth, async (req, res) => {
const preset = await savePreset(req.user.id, update);
res.status(201).send(preset);
} catch (error) {
console.error(error);
logger.error('[/presets] error saving preset', error);
res.status(500).send(error);
}
});
@ -31,13 +33,13 @@ router.post('/delete', requireJwtAuth, async (req, res) => {
filter = { presetId };
}
console.log('delete preset filter', filter);
logger.debug('[/presets/delete] delete preset filter', filter);
try {
const deleteCount = await deletePresets(req.user.id, filter);
res.status(201).send(deleteCount);
} catch (error) {
console.error(error);
logger.error('[/presets/delete] error deleting presets', error);
res.status(500).send(error);
}
});

31
api/server/routes/search.js
View file

@ -1,14 +1,16 @@
const Keyv = require('keyv');
const express = require('express');
const router = express.Router();
const { MeiliSearch } = require('meilisearch');
const { Message } = require('../../models/Message');
const { Conversation, getConvosQueried } = require('../../models/Conversation');
const { reduceHits } = require('../../lib/utils/reduceHits');
const { cleanUpPrimaryKeyValue } = require('../../lib/utils/misc');
const requireJwtAuth = require('../middleware/requireJwtAuth');
const keyvRedis = require('../../cache/keyvRedis');
const { isEnabled } = require('../utils');
const { Conversation, getConvosQueried } = require('~/models/Conversation');
const requireJwtAuth = require('~/server/middleware/requireJwtAuth');
const { cleanUpPrimaryKeyValue } = require('~/lib/utils/misc');
const { reduceHits } = require('~/lib/utils/reduceHits');
const { isEnabled } = require('~/server/utils');
const { Message } = require('~/models/Message');
const keyvRedis = require('~/cache/keyvRedis');
const { logger } = require('~/config');
const router = express.Router();
const expiration = 60 * 1000;
const cache = isEnabled(process.env.USE_REDIS)
@ -31,7 +33,7 @@ router.get('/', async function (req, res) {
const key = `${user}:search:${q}`;
const cached = await cache.get(key);
if (cached) {
console.log('cache hit', key);
logger.debug('[/search] cache hit: ' + key);
const { pages, pageSize, messages } = cached;
res
.status(200)
@ -39,7 +41,6 @@ router.get('/', async function (req, res) {
return;
}
// const message = await Message.meiliSearch(q);
const messages = (
await Message.meiliSearch(
q,
@ -61,8 +62,8 @@ router.get('/', async function (req, res) {
const titles = (await Conversation.meiliSearch(q)).hits;
const sortedHits = reduceHits(messages, titles);
// debugging:
// console.log('user:', user, 'message hits:', messages.length, 'convo hits:', titles.length);
// console.log('sorted hits:', sortedHits.length);
// logger.debug('user:', user, 'message hits:', messages.length, 'convo hits:', titles.length);
// logger.debug('sorted hits:', sortedHits.length);
const result = await getConvosQueried(user, sortedHits, pageNumber);
const activeMessages = [];
@ -86,10 +87,10 @@ router.get('/', async function (req, res) {
}
delete result.convoMap;
// for debugging
// console.log(result, messages.length);
// logger.debug(result, messages.length);
res.status(200).send(result);
} catch (error) {
console.log(error);
logger.error('[/search] Error while searching messages & conversations', error);
res.status(500).send({ message: 'Error searching' });
}
});
@ -114,11 +115,9 @@ router.get('/enable', async function (req, res) {
});
const { status } = await client.health();
// console.log(`Meilisearch: ${status}`);
result = status === 'available' && !!process.env.SEARCH;
return res.send(result);
} catch (error) {
// console.error(error);
return res.send(false);
}
});

7
api/server/routes/tokenizer.js
View file

@ -1,7 +1,8 @@
const express = require('express');
const router = express.Router();
const { countTokens } = require('../utils');
const requireJwtAuth = require('../middleware/requireJwtAuth');
const requireJwtAuth = require('~/server/middleware/requireJwtAuth');
const { countTokens } = require('~/server/utils');
const { logger } = require('~/config');
router.post('/', requireJwtAuth, async (req, res) => {
try {
@ -9,7 +10,7 @@ router.post('/', requireJwtAuth, async (req, res) => {
const count = await countTokens(arg?.text ?? arg);
res.send({ count });
} catch (e) {
console.error(e);
logger.error('[/tokenizer] Error counting tokens', e);
res.status(500).send(e.message);
}
});