mirror of
https://github.com/danny-avila/LibreChat.git
synced 2025-12-16 16:30:15 +01:00
431fc6284f
* fix(Message): avoid overwriting unprovided properties * fix(OpenAIClient): return intermediateReply on user abort * fix(AskController): do not send/save final message if abort was triggered * fix(countTokens): avoid fetching remote registry and exclusively use cl100k_base or p50k_base weights for token counting * refactor(Message/messageSchema): rely on messageSchema for default values when saving messages * fix(EditController): do not send/save final message if abort was triggered * fix(config/helpers): fix module resolution error
23 lines
730 B
JavaScript
23 lines
730 B
JavaScript
const { Tiktoken } = require('tiktoken/lite');
|
|
const p50k_base = require('tiktoken/encoders/p50k_base.json');
|
|
const cl100k_base = require('tiktoken/encoders/cl100k_base.json');
|
|
const logger = require('~/config/winston');
|
|
|
|
const countTokens = async (text = '', modelName = 'gpt-3.5-turbo') => {
|
|
let encoder = null;
|
|
try {
|
|
const model = modelName.includes('text-davinci-003') ? p50k_base : cl100k_base;
|
|
encoder = new Tiktoken(model.bpe_ranks, model.special_tokens, model.pat_str);
|
|
const tokens = encoder.encode(text);
|
|
encoder.free();
|
|
return tokens.length;
|
|
} catch (e) {
|
|
logger.error('[countTokens]', e);
|
|
if (encoder) {
|
|
encoder.free();
|
|
}
|
|
return 0;
|
|
}
|
|
};
|
|
|
|
module.exports = countTokens;
|