diff --git a/api/server/routes/tokenizer.js b/api/server/routes/tokenizer.js index acc13b2062..050919fb81 100644 --- a/api/server/routes/tokenizer.js +++ b/api/server/routes/tokenizer.js @@ -1,33 +1,18 @@ const express = require('express'); const router = express.Router(); -const { Tiktoken } = require("@dqbd/tiktoken/lite"); -const { load } = require("@dqbd/tiktoken/load"); -const registry = require("@dqbd/tiktoken/registry.json"); -const models = require("@dqbd/tiktoken/model_to_encoding.json"); +const { Tiktoken } = require('@dqbd/tiktoken/lite'); +const { load } = require('@dqbd/tiktoken/load'); +const registry = require('@dqbd/tiktoken/registry.json'); +const models = require('@dqbd/tiktoken/model_to_encoding.json'); router.post('/', async (req, res) => { - console.log('hit'); - - const input = req.body; - + const { arg } = req.body; console.log(typeof req.body === 'object' ? { ...req.body, ...req.query } : req.query); - - const model = await load(registry[models["gpt-3.5-turbo"]]); - const encoder = new Tiktoken( - model.bpe_ranks, - model.special_tokens, - model.pat_str - ); - - // work in progress - const tokens = encoder.encode('dsfsdf sdf sdfsdf sdf sdf sdf sdf '); - - res.status(201).send({ - tokens, - count: tokens.length - }); - + const model = await load(registry[models['gpt-3.5-turbo']]); + const encoder = new Tiktoken(model.bpe_ranks, model.special_tokens, model.pat_str); + const tokens = encoder.encode(arg.text); encoder.free(); + res.send({ count: tokens.length }); }); module.exports = router; diff --git a/client/src/components/Endpoints/BingAI/Settings.jsx b/client/src/components/Endpoints/BingAI/Settings.jsx index 4b7ad3efcc..cd8edcdb03 100644 --- a/client/src/components/Endpoints/BingAI/Settings.jsx +++ b/client/src/components/Endpoints/BingAI/Settings.jsx @@ -1,10 +1,12 @@ -import React, { useState } from 'react'; +import React, { useEffect, useState } from 'react'; import TextareaAutosize from 'react-textarea-autosize'; import { Input } from '~/components/ui/Input.tsx'; import { Label } from '~/components/ui/Label.tsx'; import { Checkbox } from '~/components/ui/Checkbox.tsx'; import Dropdown from '~/components/ui/Dropdown'; +import { axiosPost } from '~/utils/fetchers.js'; import { cn } from '~/utils/'; +import debounce from 'lodash/debounce'; // import ModelDropDown from '../../ui/ModelDropDown'; // import { Slider } from '~/components/ui/Slider.tsx'; // import OptionHover from './OptionHover'; @@ -16,12 +18,38 @@ const optionText = 'p-0 shadow-none text-right pr-1 h-8 border-transparent focus:ring-[#10a37f] focus:ring-offset-0 focus:ring-opacity-100 hover:bg-gray-800/10 dark:hover:bg-white/10 focus:bg-gray-800/10 dark:focus:bg-white/10 transition-colors'; function Settings(props) { - const { readonly, context, systemMessage, jailbreak, toneStyle, setOption } = props; + const { readonly, context, systemMessage, jailbreak, toneStyle, setOption, setToneStyle } = props; + const [tokenCount, setTokenCount] = useState(0); const showSystemMessage = jailbreak; const setContext = setOption('context'); const setSystemMessage = setOption('systemMessage'); const setJailbreak = setOption('jailbreak'); + // useEffect to update token count + + useEffect(() => { + if (!context) return; + + if (context.trim() === '') { + setTokenCount(0); + return; + } + + const debouncedPost = debounce(axiosPost, 500); + const handleTextChange = context => { + debouncedPost({ + url: '/api/tokenizer', + arg: { text: context}, + callback: data => { + setTokenCount(data.count); + } + }); + }; + + handleTextChange(context); + return () => debouncedPost.cancel(); + }, [context]); + // console.log('data', data); return ( @@ -31,9 +59,9 @@ function Settings(props) {