Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2026-02-08 02:24:24 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

👤 feat: AWS Bedrock Custom Inference Profiles (#11308)

Browse source 
* feat: add support for inferenceProfiles mapping

* fix: remove friendly name since api requires actual model id for validation alongside inference profile

* docs: more generic description in docs

* chore: address comments

* chore: update peer dependency versions in package.json

- Bump @aws-sdk/client-bedrock-runtime from ^3.941.0 to ^3.970.0
- Update @librechat/agents from ^3.0.78 to ^3.0.79

* fix: update @librechat/agents dependency to version 3.0.80

* test: add unit tests for inference profile configuration in initializeBedrock function

- Introduced tests to validate the applicationInferenceProfile setting based on model configuration.
- Ensured correct handling of environment variables and fallback scenarios for inference profile ARNs.
- Added cases for empty inferenceProfiles and absence of bedrock config to confirm expected behavior.

* fix: update bedrock endpoint schema reference in config

- Changed the bedrock endpoint reference from baseEndpointSchema to bedrockEndpointSchema for improved clarity and accuracy in configuration.

* test: add unit tests for Bedrock endpoint configuration

- Introduced tests to validate the configuration of Bedrock endpoints with models and inference profiles.
- Added scenarios for both complete and minimal configurations to ensure expected behavior.
- Enhanced coverage for the handling of inference profiles without a models array.

---------

Co-authored-by: Danny Avila <danny@librechat.ai>
This commit is contained in:
Dustin Healy 2026-01-16 10:52:58 -08:00 • committed by Danny Avila
parent 75c02a1a18
commit bb220f1af9
No known key found for this signature in database
GPG key ID: BF31EEB2C5CA0956
10 changed files with 2081 additions and 1776 deletions
Download patch file Download diff file Expand all files Collapse all files

72
packages/api/src/app/AppService.spec.ts
View file

@ -611,6 +611,78 @@ describe('AppService', () => {
);
});
it('should correctly configure Bedrock endpoint with models and inferenceProfiles', async () => {
const config: Partial<TCustomConfig> = {
endpoints: {
[EModelEndpoint.bedrock]: {
models: [
'us.anthropic.claude-3-7-sonnet-20250219-v1:0',
'us.anthropic.claude-sonnet-4-5-20250929-v1:0',
'global.anthropic.claude-opus-4-5-20251101-v1:0',
],
inferenceProfiles: {
'us.anthropic.claude-3-7-sonnet-20250219-v1:0':
'arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123',
'us.anthropic.claude-sonnet-4-5-20250929-v1:0': '${BEDROCK_SONNET_45_PROFILE}',
},
availableRegions: ['us-east-1', 'us-west-2'],
titleConvo: true,
titleModel: 'us.anthropic.claude-3-7-sonnet-20250219-v1:0',
},
},
};
const result = await AppService({ config });
expect(result).toEqual(
expect.objectContaining({
endpoints: expect.objectContaining({
[EModelEndpoint.bedrock]: expect.objectContaining({
models: expect.arrayContaining([
'us.anthropic.claude-3-7-sonnet-20250219-v1:0',
'us.anthropic.claude-sonnet-4-5-20250929-v1:0',
'global.anthropic.claude-opus-4-5-20251101-v1:0',
]),
inferenceProfiles: expect.objectContaining({
'us.anthropic.claude-3-7-sonnet-20250219-v1:0':
'arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123',
'us.anthropic.claude-sonnet-4-5-20250929-v1:0': '${BEDROCK_SONNET_45_PROFILE}',
}),
availableRegions: expect.arrayContaining(['us-east-1', 'us-west-2']),
titleConvo: true,
titleModel: 'us.anthropic.claude-3-7-sonnet-20250219-v1:0',
}),
}),
}),
);
});
it('should configure Bedrock endpoint with only inferenceProfiles (no models array)', async () => {
const config: Partial<TCustomConfig> = {
endpoints: {
[EModelEndpoint.bedrock]: {
inferenceProfiles: {
'us.anthropic.claude-3-7-sonnet-20250219-v1:0': '${BEDROCK_INFERENCE_PROFILE_ARN}',
},
},
},
};
const result = await AppService({ config });
expect(result).toEqual(
expect.objectContaining({
endpoints: expect.objectContaining({
[EModelEndpoint.bedrock]: expect.objectContaining({
inferenceProfiles: expect.objectContaining({
'us.anthropic.claude-3-7-sonnet-20250219-v1:0': '${BEDROCK_INFERENCE_PROFILE_ARN}',
}),
}),
}),
}),
);
});
it('should correctly configure all endpoint when specified', async () => {
const config: Partial<TCustomConfig> = {
endpoints: {