diff --git a/api/package-lock.json b/api/package-lock.json index 362646cb85..803b49f298 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -6,9 +6,10 @@ "packages": { "": { "name": "chatgpt-clone", - "version": "0.2.0", + "version": "0.3.0", "license": "ISC", "dependencies": { + "@dqbd/tiktoken": "^1.0.2", "@keyv/mongo": "^2.1.8", "@waylaidwanderer/chatgpt-api": "^1.33.2", "axios": "^1.3.4", diff --git a/api/package.json b/api/package.json index 85e702ff9b..dd0df5dcf8 100644 --- a/api/package.json +++ b/api/package.json @@ -19,6 +19,7 @@ }, "homepage": "https://github.com/danny-avila/chatgpt-clone#readme", "dependencies": { + "@dqbd/tiktoken": "^1.0.2", "@keyv/mongo": "^2.1.8", "@waylaidwanderer/chatgpt-api": "^1.33.2", "axios": "^1.3.4", diff --git a/api/server/index.js b/api/server/index.js index 8516edb945..48823a036e 100644 --- a/api/server/index.js +++ b/api/server/index.js @@ -59,6 +59,7 @@ const projectPath = path.join(__dirname, '..', '..', 'client'); app.use('/api/customGpts', routes.authenticatedOr401, routes.customGpts); app.use('/api/presets', routes.authenticatedOr401, routes.presets); app.use('/api/prompts', routes.authenticatedOr401, routes.prompts); + app.use('/api/tokenizer', routes.authenticatedOr401, routes.tokenizer); app.use('/auth', routes.auth); app.get('/api/endpoints', function (req, res) { diff --git a/api/server/routes/index.js b/api/server/routes/index.js index e517b0bf48..bce6327a0e 100644 --- a/api/server/routes/index.js +++ b/api/server/routes/index.js @@ -5,6 +5,7 @@ const presets = require('./presets'); const customGpts = require('./customGpts'); const prompts = require('./prompts'); const search = require('./search'); +const tokenizer = require('./tokenizer'); const { router: auth, authenticatedOr401, authenticatedOrRedirect } = require('./auth'); module.exports = { @@ -16,6 +17,7 @@ module.exports = { customGpts, prompts, auth, + tokenizer, authenticatedOr401, authenticatedOrRedirect }; diff --git a/api/server/routes/tokenizer.js b/api/server/routes/tokenizer.js new file mode 100644 index 0000000000..acc13b2062 --- /dev/null +++ b/api/server/routes/tokenizer.js @@ -0,0 +1,33 @@ +const express = require('express'); +const router = express.Router(); +const { Tiktoken } = require("@dqbd/tiktoken/lite"); +const { load } = require("@dqbd/tiktoken/load"); +const registry = require("@dqbd/tiktoken/registry.json"); +const models = require("@dqbd/tiktoken/model_to_encoding.json"); + +router.post('/', async (req, res) => { + console.log('hit'); + + const input = req.body; + + console.log(typeof req.body === 'object' ? { ...req.body, ...req.query } : req.query); + + const model = await load(registry[models["gpt-3.5-turbo"]]); + const encoder = new Tiktoken( + model.bpe_ranks, + model.special_tokens, + model.pat_str + ); + + // work in progress + const tokens = encoder.encode('dsfsdf sdf sdfsdf sdf sdf sdf sdf '); + + res.status(201).send({ + tokens, + count: tokens.length + }); + + encoder.free(); +}); + +module.exports = router; diff --git a/client/src/components/Endpoints/BingAI/Settings.jsx b/client/src/components/Endpoints/BingAI/Settings.jsx index e8cd219f40..4c1e7bc4f1 100644 --- a/client/src/components/Endpoints/BingAI/Settings.jsx +++ b/client/src/components/Endpoints/BingAI/Settings.jsx @@ -1,8 +1,9 @@ -import React from 'react'; +import React, { useState } from 'react'; import TextareaAutosize from 'react-textarea-autosize'; import { Input } from '~/components/ui/Input.tsx'; import { Label } from '~/components/ui/Label.tsx'; import { Checkbox } from '~/components/ui/Checkbox.tsx'; +import Dropdown from '~/components/ui/Dropdown'; import { cn } from '~/utils/'; // import ModelDropDown from '../../ui/ModelDropDown'; // import { Slider } from '~/components/ui/Slider.tsx'; @@ -15,15 +16,30 @@ const optionText = 'p-0 shadow-none text-right pr-1 h-8 border-transparent focus:ring-[#10a37f] focus:ring-offset-0 focus:ring-opacity-100 hover:bg-gray-800/10 dark:hover:bg-white/10 focus:bg-gray-800/10 dark:focus:bg-white/10 transition-colors'; function Settings(props) { - // const [showSystemMessage, setShowSystemMessage] = React.useState(false); - const { readonly, context, setContext, systemMessage, setSystemMessage, jailbreak, setJailbreak } = props; + const { readonly, context, systemMessage, jailbreak, toneStyle, setOption } = props; const showSystemMessage = jailbreak; + const setContext = setOption('context'); + const setSystemMessage = setOption('systemMessage'); + const setJailbreak = setOption('jailbreak'); + + // console.log('data', data); return ( <>
+
@@ -50,7 +67,7 @@ function Settings(props) { id="jailbreak" disabled={readonly} checked={jailbreak} - className="dark:border-gray-500 dark:bg-gray-700 dark:text-gray-50 focus:ring-opacity-20 dark:focus:ring-opacity-50 dark:focus:ring-offset-0 dark:focus:ring-gray-600" + className="focus:ring-opacity-20 dark:border-gray-500 dark:bg-gray-700 dark:text-gray-50 dark:focus:ring-gray-600 dark:focus:ring-opacity-50 dark:focus:ring-offset-0" // onCheckedChange={setJailbreak} onCheckedChange={checked => { setJailbreak(checked); @@ -65,10 +82,17 @@ function Settings(props) {
{showSystemMessage && ( @@ -84,10 +108,10 @@ function Settings(props) { disabled={readonly} value={systemMessage || ''} onChange={e => setSystemMessage(e.target.value || null)} - placeholder="Set custom instructions. Defaults to: 'You are ChatGPT, a large language model trained by OpenAI.'" + placeholder="WARNING: Misuse of this feature can get you BANNED from using Bing! Click on 'System Message' for full instructions and the default message if omitted, which is the 'Sydney' preset that is considered safe." className={cn( defaultTextProps, - 'flex max-h-[300px] min-h-[100px] w-full resize-none px-3 py-2 ' + 'flex max-h-[300px] min-h-[148px] w-full resize-none px-3 py-2 ' )} /> diff --git a/client/src/components/Input/BingAIOptions/index.jsx b/client/src/components/Input/BingAIOptions/index.jsx index 00068051db..46941a8f7b 100644 --- a/client/src/components/Input/BingAIOptions/index.jsx +++ b/client/src/components/Input/BingAIOptions/index.jsx @@ -114,13 +114,13 @@ function BingAIOptions() {
+
} diff --git a/client/src/components/ui/Dropdown.jsx b/client/src/components/ui/Dropdown.jsx index 812ee94edd..72544125dc 100644 --- a/client/src/components/ui/Dropdown.jsx +++ b/client/src/components/ui/Dropdown.jsx @@ -39,7 +39,7 @@ function Dropdown({ value, onChange, options, className, containerClassName }) { - + {options.map((item, i) => (