feat: count tokens for context

feat(BingAI): convert toneStyle to lowercase before setting it in state
feat(BingAI): pass setToneStyle function to Settings component
refactor(BingAI): remove unused import and change setOption to setToneStyle in Settings component
refactor(fetchers.js): add axiosPost function for debounced axios post requests

api/server/routes/tokenizer.js

@@ -1,33 +1,18 @@
 const express = require('express');
 const router = express.Router();
-const { Tiktoken } = require("@dqbd/tiktoken/lite");
-const { load } = require("@dqbd/tiktoken/load");
-const registry = require("@dqbd/tiktoken/registry.json");
-const models = require("@dqbd/tiktoken/model_to_encoding.json");
+const { Tiktoken } = require('@dqbd/tiktoken/lite');
+const { load } = require('@dqbd/tiktoken/load');
+const registry = require('@dqbd/tiktoken/registry.json');
+const models = require('@dqbd/tiktoken/model_to_encoding.json');
 
 router.post('/', async (req, res) => {
-  console.log('hit');
-
-  const input = req.body;
-
+  const { arg } = req.body;
   console.log(typeof req.body === 'object' ? { ...req.body, ...req.query } : req.query);
-
-  const model = await load(registry[models["gpt-3.5-turbo"]]);
-  const encoder = new Tiktoken(
-    model.bpe_ranks,
-    model.special_tokens,
-    model.pat_str
-  );
-  
-  // work in progress
-  const tokens = encoder.encode('dsfsdf sdf sdfsdf sdf sdf sdf sdf  ');
-  
-  res.status(201).send({
-    tokens,
-    count: tokens.length
-  });
-
+  const model = await load(registry[models['gpt-3.5-turbo']]);
+  const encoder = new Tiktoken(model.bpe_ranks, model.special_tokens, model.pat_str);
+  const tokens = encoder.encode(arg.text);
   encoder.free();
+  res.send({ count: tokens.length });
 });
 
 module.exports = router;

client/src/components/Endpoints/BingAI/Settings.jsx

@@ -1,10 +1,12 @@
-import React, { useState } from 'react';
+import React, { useEffect, useState } from 'react';
 import TextareaAutosize from 'react-textarea-autosize';
 import { Input } from '~/components/ui/Input.tsx';
 import { Label } from '~/components/ui/Label.tsx';
 import { Checkbox } from '~/components/ui/Checkbox.tsx';
 import Dropdown from '~/components/ui/Dropdown';
+import { axiosPost } from '~/utils/fetchers.js';
 import { cn } from '~/utils/';
+import debounce from 'lodash/debounce';
 // import ModelDropDown from '../../ui/ModelDropDown';
 // import { Slider } from '~/components/ui/Slider.tsx';
 // import OptionHover from './OptionHover';
@@ -16,12 +18,38 @@ const optionText =
   'p-0 shadow-none text-right pr-1 h-8 border-transparent focus:ring-[#10a37f] focus:ring-offset-0 focus:ring-opacity-100 hover:bg-gray-800/10 dark:hover:bg-white/10 focus:bg-gray-800/10 dark:focus:bg-white/10 transition-colors';
 
 function Settings(props) {
-  const { readonly, context, systemMessage, jailbreak, toneStyle, setOption } = props;
+  const { readonly, context, systemMessage, jailbreak, toneStyle, setOption, setToneStyle } = props;
+  const [tokenCount, setTokenCount] = useState(0);
   const showSystemMessage = jailbreak;
   const setContext = setOption('context');
   const setSystemMessage = setOption('systemMessage');
   const setJailbreak = setOption('jailbreak');
 
+  // useEffect to update token count
+  
+  useEffect(() => {
+    if (!context) return;
+
+    if (context.trim() === '') {
+      setTokenCount(0);
+      return;
+    }
+
+    const debouncedPost = debounce(axiosPost, 500);
+    const handleTextChange = context => {
+      debouncedPost({
+        url: '/api/tokenizer',
+        arg: { text: context},
+        callback: data => {
+          setTokenCount(data.count);
+        }
+      });
+    };
+
+    handleTextChange(context);
+    return () => debouncedPost.cancel(); 
+  }, [context]);
+
   // console.log('data', data);
 
   return (
@@ -31,9 +59,9 @@ function Settings(props) {
           <div className="grid w-full items-center gap-2">
             <Dropdown
               id="toneStyle-dropdown"
-              value={toneStyle}
-              onChange={setOption('toneStyle')}
-              options={['creative', 'fast', 'balanced', 'precise']}
+              value={`${toneStyle.charAt(0).toUpperCase()}${toneStyle.slice(1)}`}
+              onChange={setToneStyle}
+              options={['Creative', 'Fast', 'Balanced', 'Precise']}
               className={cn(
                 defaultTextProps,
                 'flex h-10 max-h-10 w-full resize-none focus:outline-none focus:ring-0 focus:ring-opacity-0 focus:ring-offset-0'
@@ -58,7 +86,7 @@ function Settings(props) {
               )}
             />
             <small className="mb-5 text-black dark:text-white">
-              {'Token count: 1200 (work in progress)'}
+              {`Token count: ${tokenCount}`}
             </small>
           </div>
         </div>
@@ -70,11 +98,7 @@ function Settings(props) {
                 disabled={readonly}
                 checked={jailbreak}
                 className="focus:ring-opacity-20 dark:border-gray-500 dark:bg-gray-700 dark:text-gray-50 dark:focus:ring-gray-600 dark:focus:ring-opacity-50 dark:focus:ring-offset-0"
-                // onCheckedChange={setJailbreak}
-                onCheckedChange={checked => {
-                  setJailbreak(checked);
-                  // setShowSystemMessage(checked);
-                }}
+                onCheckedChange={setJailbreak}
               />
               <label
                 htmlFor="jailbreak"
@@ -99,12 +123,6 @@ function Settings(props) {
             </div>
             {showSystemMessage && (
               <>
-                {/* <Label
-                  htmlFor="systemMessage"
-                  className="text-left text-sm font-medium"
-                >
-                  System Message <small className="opacity-40">(default: blank)</small>
-                </Label> */}
                 <TextareaAutosize
                   id="systemMessage"
                   disabled={readonly}

client/src/components/Input/BingAIOptions/index.jsx

@@ -21,7 +21,7 @@ function BingAIOptions() {
   if (conversationId !== 'new') return null;
 
   const changeHandler = value => {
-    setConversation(prevState => ({ ...prevState, toneStyle: value }));
+    setConversation(prevState => ({ ...prevState, toneStyle: value.toLowerCase() }));
   };
 
   const triggerAdvancedMode = () => setAdvancedMode(prev => !prev);
@@ -120,6 +120,7 @@ function BingAIOptions() {
               systemMessage={systemMessage}
               jailbreak={jailbreak}
               toneStyle={toneStyle}
+              setToneStyle={changeHandler}
               setOption={setOption}
             />
           </div>

client/src/utils/fetchers.js

@@ -18,6 +18,15 @@ export const postRequest = async (url, { arg }) => {
   });
 };
 
+export const axiosPost = async ({ url, arg, callback }) => {
+  try {
+    const response = await axios.post(url, { arg }, { withCredentials: true });
+    callback(response.data);
+  } catch (error) {
+    console.error('An error occurred while making the axios post request:', error);
+  }
+};
+
 export const searchFetcher = async (pre, q, pageNumber, callback) => {
   pre();
   const { data } = await axios.get(`/api/search?q=${q}&pageNumber=${pageNumber}`);

package-lock.json

@@ -6,7 +6,7 @@
   "packages": {
     "": {
       "name": "chatgpt-clone",
-      "version": "0.1.0",
+      "version": "0.3.0",
       "license": "ISC",
       "devDependencies": {
         "@playwright/test": "^1.32.1",