Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-12-16 16:30:15 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

feat: count tokens for context

Browse source 
feat(BingAI): convert toneStyle to lowercase before setting it in state
feat(BingAI): pass setToneStyle function to Settings component
refactor(BingAI): remove unused import and change setOption to setToneStyle in Settings component
refactor(fetchers.js): add axiosPost function for debounced axios post requests
This commit is contained in:
Danny Avila 2023-04-04 10:04:21 -04:00
parent cef98070e9
commit 1a196580b2
5 changed files with 56 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

33
api/server/routes/tokenizer.js
View file

@ -1,33 +1,18 @@
const express = require('express');
const router = express.Router();
const { Tiktoken } = require("@dqbd/tiktoken/lite");
const { load } = require("@dqbd/tiktoken/load");
const registry = require("@dqbd/tiktoken/registry.json");
const models = require("@dqbd/tiktoken/model_to_encoding.json");
const { Tiktoken } = require('@dqbd/tiktoken/lite');
const { load } = require('@dqbd/tiktoken/load');
const registry = require('@dqbd/tiktoken/registry.json');
const models = require('@dqbd/tiktoken/model_to_encoding.json');
router.post('/', async (req, res) => {
console.log('hit');
const input = req.body;
const { arg } = req.body;
console.log(typeof req.body === 'object' ? { ...req.body, ...req.query } : req.query);
const model = await load(registry[models["gpt-3.5-turbo"]]);
const encoder = new Tiktoken(
model.bpe_ranks,
model.special_tokens,
model.pat_str
);
// work in progress
const tokens = encoder.encode('dsfsdf sdf sdfsdf sdf sdf sdf sdf ');
res.status(201).send({
tokens,
count: tokens.length
});
const model = await load(registry[models['gpt-3.5-turbo']]);
const encoder = new Tiktoken(model.bpe_ranks, model.special_tokens, model.pat_str);
const tokens = encoder.encode(arg.text);
encoder.free();
res.send({ count: tokens.length });
});
module.exports = router;