Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-12-17 00:40:14 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

style(endpoints.js): fix indentation and add semicolons

Browse source 
fix(tokenizer.js): add try-catch block and error handling
style(SetTokenDialog/index.jsx): fix typo in sentence
refactor(data-service.ts): change argument format to match server API
This commit is contained in:
Daniel Avila 2023-04-11 10:28:11 -04:00
parent e0d5e75e73
commit 5c5871afd8
4 changed files with 20 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

22
api/server/routes/tokenizer.js
View file

@ -6,13 +6,21 @@ const registry = require('@dqbd/tiktoken/registry.json');
const models = require('@dqbd/tiktoken/model_to_encoding.json');
router.post('/', async (req, res) => {
const { arg } = req.body;
// console.log(typeof req.body === 'object' ? { ...req.body, ...req.query } : req.query);
const model = await load(registry[models['gpt-3.5-turbo']]);
const encoder = new Tiktoken(model.bpe_ranks, model.special_tokens, model.pat_str);
const tokens = encoder.encode(arg.text);
encoder.free();
res.send({ count: tokens.length });
try {
const { arg } = req.body;
console.log('context:', arg, req.body);
// console.log(typeof req.body === 'object' ? { ...req.body, ...req.query } : req.query);
const model = await load(registry[models['gpt-3.5-turbo']]);
const encoder = new Tiktoken(model.bpe_ranks, model.special_tokens, model.pat_str);
const tokens = encoder.encode(arg.text);
encoder.free();
res.send({ count: tokens.length });
} catch (e) {
console.error(e);
res.status(500).send(e.message);
}
});
module.exports = router;