LibreChat/api/server/routes/tokenizer.js

const express = require('express');
const router = express.Router();
const { Tiktoken } = require("@dqbd/tiktoken/lite");
const { load } = require("@dqbd/tiktoken/load");
const registry = require("@dqbd/tiktoken/registry.json");
const models = require("@dqbd/tiktoken/model_to_encoding.json");

router.post('/', async (req, res) => {
  console.log('hit');

  const input = req.body;

  console.log(typeof req.body === 'object' ? { ...req.body, ...req.query } : req.query);

  const model = await load(registry[models["gpt-3.5-turbo"]]);
  const encoder = new Tiktoken(
    model.bpe_ranks,
    model.special_tokens,
    model.pat_str
  );
  
  // work in progress
  const tokens = encoder.encode('dsfsdf sdf sdfsdf sdf sdf sdf sdf  ');
  
  res.status(201).send({
    tokens,
    count: tokens.length
  });

  encoder.free();
});

module.exports = router;