Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-12-17 08:50:15 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

fix(getMultiplier): correct rate for gpt-4 context

Browse source
This commit is contained in:
Danny Avila 2023-10-06 13:21:44 -04:00 • committed by Danny Avila
parent ce966419f7
commit 599d70f1de
2 changed files with 14 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

13
api/models/tx.js
View file

@ -1,12 +1,13 @@
const { matchModelName } = require('../utils'); const { matchModelName } = require('../utils');
const defaultRate = 6;
/** /**
* Mapping of model token sizes to their respective multipliers for prompt and completion. * Mapping of model token sizes to their respective multipliers for prompt and completion.
* @type {Object.<string, {prompt: number, completion: number}>} * @type {Object.<string, {prompt: number, completion: number}>}
*/ */
const tokenValues = { const tokenValues = {
'8k': { prompt: 3, completion: 6 }, '8k': { prompt: 30, completion: 60 },
'32k': { prompt: 6, completion: 12 }, '32k': { prompt: 60, completion: 120 },
'4k': { prompt: 1.5, completion: 2 }, '4k': { prompt: 1.5, completion: 2 },
'16k': { prompt: 3, completion: 4 }, '16k': { prompt: 3, completion: 4 },
}; };
@ -48,7 +49,7 @@ const getValueKey = (model) => {
*/ */
const getMultiplier = ({ valueKey, tokenType, model }) => { const getMultiplier = ({ valueKey, tokenType, model }) => {
if (valueKey && tokenType) { if (valueKey && tokenType) {
return tokenValues[valueKey][tokenType] ?? 4.5; return tokenValues[valueKey][tokenType] ?? defaultRate;
} }
if (!tokenType || !model) { if (!tokenType || !model) {
@ -57,11 +58,11 @@ const getMultiplier = ({ valueKey, tokenType, model }) => {
valueKey = getValueKey(model); valueKey = getValueKey(model);
if (!valueKey) { if (!valueKey) {
return 4.5; return defaultRate;
} }
// If we got this far, and values[tokenType] is undefined somehow, return a rough average of default multipliers // If we got this far, and values[tokenType] is undefined somehow, return a rough average of default multipliers
return tokenValues[valueKey][tokenType] ?? 4.5; return tokenValues[valueKey][tokenType] ?? defaultRate;
}; };
module.exports = { tokenValues, getValueKey, getMultiplier }; module.exports = { tokenValues, getValueKey, getMultiplier, defaultRate };

12
api/models/tx.spec.js
View file

@ -1,4 +1,4 @@
const { getValueKey, getMultiplier } = require('./tx'); const { getValueKey, getMultiplier, defaultRate } = require('./tx');
describe('getValueKey', () => { describe('getValueKey', () => {
it('should return "16k" for model name containing "gpt-3.5-turbo-16k"', () => { it('should return "16k" for model name containing "gpt-3.5-turbo-16k"', () => {
@ -28,8 +28,8 @@ describe('getMultiplier', () => {
expect(getMultiplier({ valueKey: '8k', tokenType: 'completion' })).toBe(6); expect(getMultiplier({ valueKey: '8k', tokenType: 'completion' })).toBe(6);
}); });
it('should return 4.5 if tokenType is provided but not found in tokenValues', () => { it('should return defaultRate if tokenType is provided but not found in tokenValues', () => {
expect(getMultiplier({ valueKey: '8k', tokenType: 'unknownType' })).toBe(4.5); expect(getMultiplier({ valueKey: '8k', tokenType: 'unknownType' })).toBe(defaultRate);
}); });
it('should derive the valueKey from the model if not provided', () => { it('should derive the valueKey from the model if not provided', () => {
@ -41,7 +41,9 @@ describe('getMultiplier', () => {
expect(getMultiplier({ model: 'gpt-4-some-other-info' })).toBe(1); expect(getMultiplier({ model: 'gpt-4-some-other-info' })).toBe(1);
}); });
it('should return 4.5 if derived valueKey does not match any known patterns', () => { it('should return defaultRate if derived valueKey does not match any known patterns', () => {
expect(getMultiplier({ tokenType: 'prompt', model: 'gpt-5-some-other-info' })).toBe(4.5); expect(getMultiplier({ tokenType: 'prompt', model: 'gpt-5-some-other-info' })).toBe(
defaultRate,
);
}); });
}); });