Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-12-17 00:40:14 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

🪨 feat: Add Bedrock Prompt Caching Support (#8271)

Browse source 
* feat: Add Bedrock Cache Control Functionality

- fix: Update Bedrock Cache Control to Require cachePoint as a Separate Content Block

- Modified the addBedrockCacheControl function to ensure cachePoint is added as a separate content block in the content array, rather than as a property of text objects.

- refactor: move addBedrockCacheControl over to packages/api

- ci: add tests for addBedrockCacheControl until full coverage reached

* ci: add test similar to example from the langchain PR

* refactor: move addBedrockCacheControl logic and tests to agents repository

* chore: remove extraneous comment

* chore: update @librechat/agents dependency to version 3.0.12

* chore: update @librechat/agents dependency to version 3.0.13

* chore: update @librechat/agents dependency to version 3.0.14

* chore: update @librechat/agents to v3.0.15

* chore: update default value for prompt cache setting to true

* refactor: set default promptCache to true for claude and nova models

---------

Co-authored-by: Danny Avila <danny@librechat.ai>
This commit is contained in:
Dustin Healy 2025-11-13 07:21:50 -08:00 • committed by GitHub
parent bc561840bb
commit c9ee0f138a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 37 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

2
api/package.json
View file

@ -47,7 +47,7 @@
"@langchain/google-genai": "^0.2.13",
"@langchain/google-vertexai": "^0.2.13",
"@langchain/textsplitters": "^0.1.0",
"@librechat/agents": "^3.0.11",
"@librechat/agents": "^3.0.15",
"@librechat/api": "*",
"@librechat/data-schemas": "*",
"@microsoft/microsoft-graph-client": "^3.0.7",

12
package-lock.json generated
View file

@ -63,7 +63,7 @@
"@langchain/google-genai": "^0.2.13",
"@langchain/google-vertexai": "^0.2.13",
"@langchain/textsplitters": "^0.1.0",
"@librechat/agents": "^3.0.11",
"@librechat/agents": "^3.0.15",
"@librechat/api": "*",
"@librechat/data-schemas": "*",
"@microsoft/microsoft-graph-client": "^3.0.7",
@ -16666,13 +16666,13 @@
}
},
"node_modules/@librechat/agents": {
"version": "3.0.11",
"resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-3.0.11.tgz",
"integrity": "sha512-Qey74ZptzKRBHVjei4m/dhplBDgZ5lcNPz1MugYWmVJHTJ7AK+m6zJYtSLDAYal10Yz/RP+oFIo2zJJS+VbFXQ==",
"version": "3.0.15",
"resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-3.0.15.tgz",
"integrity": "sha512-iJyUZnfZrPgCeuhCEMOTJW5rp2z6QBsERwytNAr6haSXKiN4F7A6yfzJUkSTTLAu49+xoMIANAOMi8POaFdZ9Q==",
"license": "MIT",
"dependencies": {
"@langchain/anthropic": "^0.3.26",
"@langchain/aws": "^0.1.12",
"@langchain/aws": "^0.1.15",
"@langchain/core": "^0.3.79",
"@langchain/deepseek": "^0.0.2",
"@langchain/google-genai": "^0.2.13",
@ -46470,7 +46470,7 @@
"@azure/storage-blob": "^12.27.0",
"@keyv/redis": "^4.3.3",
"@langchain/core": "^0.3.79",
"@librechat/agents": "^3.0.11",
"@librechat/agents": "^3.0.15",
"@librechat/data-schemas": "*",
"@modelcontextprotocol/sdk": "^1.21.0",
"axios": "^1.12.1",

2
packages/api/package.json
View file

@ -83,7 +83,7 @@
"@azure/storage-blob": "^12.27.0",
"@keyv/redis": "^4.3.3",
"@langchain/core": "^0.3.79",
"@librechat/agents": "^3.0.11",
"@librechat/agents": "^3.0.15",
"@librechat/data-schemas": "*",
"@modelcontextprotocol/sdk": "^1.21.0",
"axios": "^1.12.1",

12
packages/data-provider/src/bedrock.ts
View file

@ -37,6 +37,7 @@ export const bedrockInputSchema = s.tConversationSchema
stop: true,
thinking: true,
thinkingBudget: true,
promptCache: true,
/* Catch-all fields */
topK: true,
additionalModelRequestFields: true,
@ -78,6 +79,7 @@ export const bedrockInputParser = s.tConversationSchema
stop: true,
thinking: true,
thinkingBudget: true,
promptCache: true,
/* Catch-all fields */
topK: true,
additionalModelRequestFields: true,
@ -100,6 +102,7 @@ export const bedrockInputParser = s.tConversationSchema
'temperature',
'topP',
'stop',
'promptCache',
];
const additionalFields: Record<string, unknown> = {};
@ -140,6 +143,15 @@ export const bedrockInputParser = s.tConversationSchema
delete additionalFields.thinkingBudget;
}
/** Default promptCache for claude and nova models, if not defined */
if (
typeof typedData.model === 'string' &&
(typedData.model.includes('claude') || typedData.model.includes('nova')) &&
typedData.promptCache === undefined
) {
typedData.promptCache = true;
}
if (Object.keys(additionalFields).length > 0) {
typedData.additionalModelRequestFields = {
...((typedData.additionalModelRequestFields as Record<string, unknown> | undefined) || {}),

17
packages/data-provider/src/parameterSettings.ts
View file

@ -492,6 +492,19 @@ const bedrock: Record<string, SettingDefinition> = {
default: 0.999,
range: { min: 0, max: 1, step: 0.01 },
}),
promptCache: {
key: 'promptCache',
label: 'com_endpoint_prompt_cache',
labelCode: true,
type: 'boolean',
description: 'com_endpoint_anthropic_prompt_cache',
descriptionCode: true,
default: true,
component: 'switch',
optionType: 'conversation',
showDefault: false,
columnSpan: 2,
},
};
const mistral: Record<string, SettingDefinition> = {
@ -752,6 +765,7 @@ const bedrockAnthropic: SettingsConfiguration = [
baseDefinitions.stop,
librechat.resendFiles,
bedrock.region,
bedrock.promptCache,
anthropic.thinking,
anthropic.thinkingBudget,
librechat.fileTokenLimit,
@ -789,6 +803,7 @@ const bedrockGeneral: SettingsConfiguration = [
meta.topP,
librechat.resendFiles,
bedrock.region,
bedrock.promptCache,
librechat.fileTokenLimit,
];
@ -807,6 +822,7 @@ const bedrockAnthropicCol2: SettingsConfiguration = [
bedrock.topK,
librechat.resendFiles,
bedrock.region,
bedrock.promptCache,
anthropic.thinking,
anthropic.thinkingBudget,
librechat.fileTokenLimit,
@ -856,6 +872,7 @@ const bedrockGeneralCol2: SettingsConfiguration = [
meta.topP,
librechat.resendFiles,
bedrock.region,
bedrock.promptCache,
librechat.fileTokenLimit,
];