Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2026-03-13 19:36:16 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

fix: Match OpenAI Token Counting Strategy 🪙 (#945)

Browse source 
* wip token fix

* fix: complete token count refactor to match OpenAI example

* chore: add back sendPayload method (accidentally deleted)

* chore: revise JSDoc for getTokenCountForMessage
This commit is contained in:
Danny Avila 2023-09-14 19:40:21 -04:00 • committed by GitHub
parent b3afd562b9
commit 9491b753c3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 115 additions and 76 deletions
Download patch file Download diff file Expand all files Collapse all files

7
package-lock.json generated
View file

@ -44,7 +44,6 @@
"dependencies": {
"@anthropic-ai/sdk": "^0.5.4",
"@azure/search-documents": "^11.3.2",
"@dqbd/tiktoken": "^1.0.7",
"@keyv/mongo": "^2.1.8",
"@waylaidwanderer/chatgpt-api": "^1.37.2",
"axios": "^1.3.4",
@ -81,6 +80,7 @@
"passport-local": "^1.0.0",
"pino": "^8.12.1",
"sharp": "^0.32.5",
"tiktoken": "^1.0.10",
"ua-parser-js": "^1.0.36",
"zod": "^3.22.2"
},
@ -21886,6 +21886,11 @@
"real-require": "^0.2.0"
}
},
"node_modules/tiktoken": {
"version": "1.0.10",
"resolved": "https://registry.npmjs.org/tiktoken/-/tiktoken-1.0.10.tgz",
"integrity": "sha512-gF8ndTCNu7WcRFbl1UUWaFIB4CTXmHzS3tRYdyUYF7x3C6YR6Evoao4zhKDmWIwv2PzNbzoQMV8Pxt+17lEDbA=="
},
"node_modules/tmp": {
"version": "0.0.33",
"resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz",