mirror of
https://github.com/danny-avila/LibreChat.git
synced 2025-09-22 06:00:56 +02:00
a45b384bbc
* wip: initial cache control implementation, add typing for transactions handling * feat: first pass of Anthropic Prompt Caching * feat: standardize stream usage as pass in when calculating token counts * feat: Add getCacheMultiplier function to calculate cache multiplier for different valueKeys and cacheTypes * chore: imports order * refactor: token usage recording in AnthropicClient, no need to "correct" as we have the correct amount * feat: more accurate token counting using stream usage data * feat: Improve token counting accuracy with stream usage data * refactor: ensure more accurate than not token estimations if custom instructions or files are not being resent with every request * refactor: cleanup updateUserMessageTokenCount to allow transactions to be as accurate as possible even if we shouldn't update user message token counts * ci: fix tests
43 lines
1.3 KiB
JavaScript
43 lines
1.3 KiB
JavaScript
/**
|
|
* Anthropic API: Adds cache control to the appropriate user messages in the payload.
|
|
* @param {Array<AnthropicMessage>} messages - The array of message objects.
|
|
* @returns {Array<AnthropicMessage>} - The updated array of message objects with cache control added.
|
|
*/
|
|
function addCacheControl(messages) {
|
|
if (!Array.isArray(messages) || messages.length < 2) {
|
|
return messages;
|
|
}
|
|
|
|
const updatedMessages = [...messages];
|
|
let userMessagesFound = 0;
|
|
|
|
for (let i = updatedMessages.length - 1; i >= 0 && userMessagesFound < 2; i--) {
|
|
if (updatedMessages[i].role === 'user') {
|
|
if (typeof updatedMessages[i].content === 'string') {
|
|
updatedMessages[i] = {
|
|
...updatedMessages[i],
|
|
content: [
|
|
{
|
|
type: 'text',
|
|
text: updatedMessages[i].content,
|
|
cache_control: { type: 'ephemeral' },
|
|
},
|
|
],
|
|
};
|
|
} else if (Array.isArray(updatedMessages[i].content)) {
|
|
updatedMessages[i] = {
|
|
...updatedMessages[i],
|
|
content: updatedMessages[i].content.map((item) => ({
|
|
...item,
|
|
cache_control: { type: 'ephemeral' },
|
|
})),
|
|
};
|
|
}
|
|
userMessagesFound++;
|
|
}
|
|
}
|
|
|
|
return updatedMessages;
|
|
}
|
|
|
|
module.exports = addCacheControl;
|