feat(bing-settings): Work in Progress, will finish tomorrow

feat(api): add @dqbd/tiktoken package as a dependency
feat(api): add /api/tokenizer endpoint to tokenize text using @dqbd/tiktoken
feat(client): add toneStyle dropdown to BingAI Settings component
feat(client): add token count to BingAI Settings component

style(ui): add z-index to Dropdown and EndpointOptionsPopover components

Add z-index to Dropdown and EndpointOptionsPopover components to ensure they are displayed above other components.

api/server/routes/index.js

@@ -5,6 +5,7 @@ const presets = require('./presets');
 const customGpts = require('./customGpts');
 const prompts = require('./prompts');
 const search = require('./search');
+const tokenizer = require('./tokenizer');
 const { router: auth, authenticatedOr401, authenticatedOrRedirect } = require('./auth');
 
 module.exports = {
@@ -16,6 +17,7 @@ module.exports = {
   customGpts,
   prompts,
   auth,
+  tokenizer,
   authenticatedOr401,
   authenticatedOrRedirect
 };

api/server/routes/tokenizer.js

@@ -0,0 +1,33 @@
+const express = require('express');
+const router = express.Router();
+const { Tiktoken } = require("@dqbd/tiktoken/lite");
+const { load } = require("@dqbd/tiktoken/load");
+const registry = require("@dqbd/tiktoken/registry.json");
+const models = require("@dqbd/tiktoken/model_to_encoding.json");
+
+router.post('/', async (req, res) => {
+  console.log('hit');
+
+  const input = req.body;
+
+  console.log(typeof req.body === 'object' ? { ...req.body, ...req.query } : req.query);
+
+  const model = await load(registry[models["gpt-3.5-turbo"]]);
+  const encoder = new Tiktoken(
+    model.bpe_ranks,
+    model.special_tokens,
+    model.pat_str
+  );
+  
+  // work in progress
+  const tokens = encoder.encode('dsfsdf sdf sdfsdf sdf sdf sdf sdf  ');
+  
+  res.status(201).send({
+    tokens,
+    count: tokens.length
+  });
+
+  encoder.free();
+});
+
+module.exports = router;