Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-12-17 17:00:15 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

🐛 fix: Anthropic Prompt Caching Edge Case (#3690)

Browse source
This commit is contained in:
Danny Avila 2024-08-18 19:03:03 -04:00 • committed by GitHub
parent 87d95a9d82
commit 8ca1e4f94f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 89 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

46
api/app/clients/prompts/addCacheControl.js
View file

@ -9,31 +9,31 @@ function addCacheControl(messages) {
} }
const updatedMessages = [...messages]; const updatedMessages = [...messages];
let userMessagesFound = 0; let userMessagesModified = 0;
for (let i = updatedMessages.length - 1; i >= 0 && userMessagesFound < 2; i--) { for (let i = updatedMessages.length - 1; i >= 0 && userMessagesModified < 2; i--) {
if (updatedMessages[i].role === 'user') { const message = updatedMessages[i];
if (typeof updatedMessages[i].content === 'string') { if (message.role !== 'user') {
updatedMessages[i] = { continue;
...updatedMessages[i], }
content: [
{ if (typeof message.content === 'string') {
type: 'text', message.content = [
text: updatedMessages[i].content, {
cache_control: { type: 'ephemeral' }, type: 'text',
}, text: message.content,
], cache_control: { type: 'ephemeral' },
}; },
} else if (Array.isArray(updatedMessages[i].content)) { ];
updatedMessages[i] = { userMessagesModified++;
...updatedMessages[i], } else if (Array.isArray(message.content)) {
content: updatedMessages[i].content.map((item) => ({ for (let j = message.content.length - 1; j >= 0; j--) {
...item, if (message.content[j].type === 'text') {
cache_control: { type: 'ephemeral' }, message.content[j].cache_control = { type: 'ephemeral' };
})), userMessagesModified++;
}; break;
}
} }
userMessagesFound++;
} }
} }

69
api/app/clients/prompts/addCacheControl.spec.js
View file

@ -116,6 +116,7 @@ describe('addCacheControl', () => {
content: [ content: [
{ type: 'text', text: 'Hello' }, { type: 'text', text: 'Hello' },
{ type: 'image', url: 'http://example.com/image.jpg' }, { type: 'image', url: 'http://example.com/image.jpg' },
{ type: 'text', text: 'This is an image' },
], ],
}, },
{ role: 'assistant', content: 'Hi there' }, { role: 'assistant', content: 'Hi there' },
@ -124,8 +125,9 @@ describe('addCacheControl', () => {
const result = addCacheControl(messages); const result = addCacheControl(messages);
expect(result[0].content[0].cache_control).toEqual({ type: 'ephemeral' }); expect(result[0].content[0]).not.toHaveProperty('cache_control');
expect(result[0].content[1].cache_control).toEqual({ type: 'ephemeral' }); expect(result[0].content[1]).not.toHaveProperty('cache_control');
expect(result[0].content[2].cache_control).toEqual({ type: 'ephemeral' });
expect(result[2].content[0]).toEqual({ expect(result[2].content[0]).toEqual({
type: 'text', type: 'text',
text: 'How are you?', text: 'How are you?',
@ -143,7 +145,6 @@ describe('addCacheControl', () => {
]; ];
const result = addCacheControl(messages); const result = addCacheControl(messages);
console.dir(result, { depth: null });
expect(result[0].content).toEqual('Hello'); expect(result[0].content).toEqual('Hello');
expect(result[2].content[0]).toEqual({ expect(result[2].content[0]).toEqual({
@ -161,4 +162,66 @@ describe('addCacheControl', () => {
expect(result[1].content).toBe('Hi there'); expect(result[1].content).toBe('Hi there');
expect(result[3].content).toBe('I\'m doing well, thanks!'); expect(result[3].content).toBe('I\'m doing well, thanks!');
}); });
test('should handle edge case with multiple content types', () => {
const messages = [
{
role: 'user',
content: [
{
type: 'image',
source: { type: 'base64', media_type: 'image/png', data: 'some_base64_string' },
},
{
type: 'image',
source: { type: 'base64', media_type: 'image/png', data: 'another_base64_string' },
},
{ type: 'text', text: 'what do all these images have in common' },
],
},
{ role: 'assistant', content: 'I see multiple images.' },
{ role: 'user', content: 'Correct!' },
];
const result = addCacheControl(messages);
expect(result[0].content[0]).not.toHaveProperty('cache_control');
expect(result[0].content[1]).not.toHaveProperty('cache_control');
expect(result[0].content[2].cache_control).toEqual({ type: 'ephemeral' });
expect(result[2].content[0]).toEqual({
type: 'text',
text: 'Correct!',
cache_control: { type: 'ephemeral' },
});
});
test('should handle user message with no text block', () => {
const messages = [
{
role: 'user',
content: [
{
type: 'image',
source: { type: 'base64', media_type: 'image/png', data: 'some_base64_string' },
},
{
type: 'image',
source: { type: 'base64', media_type: 'image/png', data: 'another_base64_string' },
},
],
},
{ role: 'assistant', content: 'I see two images.' },
{ role: 'user', content: 'Correct!' },
];
const result = addCacheControl(messages);
expect(result[0].content[0]).not.toHaveProperty('cache_control');
expect(result[0].content[1]).not.toHaveProperty('cache_control');
expect(result[2].content[0]).toEqual({
type: 'text',
text: 'Correct!',
cache_control: { type: 'ephemeral' },
});
});
}); });