Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-12-19 18:00:15 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

🔗 feat: Custom Jina API URL for Web Search Reranking (#9236)

Browse source 
* feat: added support for custom JINA_API_URL

* fixed tests

* chore: Update @librechat/agents dependency to version 2.4.77 in package-lock.json and package.json files

* fix: Update Jina API URL to use environment variable in configuration files

* Refactor AppService, web.ts, and config.ts to replace hardcoded Jina API URL with an environment variable placeholder.
* Ensure consistency across tests and configuration for Jina API URL.

* chore: alphabetical order translation.json

* fix: alphabetical order

---------

Co-authored-by: Danny Avila <danny@librechat.ai>
This commit is contained in:
Daniel Andersen 2025-09-06 14:39:20 +02:00 • committed by GitHub
parent fff1f1cf27
commit 6f6a34d126
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 68 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

19
packages/api/src/web/web.spec.ts
View file

@ -81,6 +81,7 @@ describe('web.ts', () => {
firecrawlApiKey: '${FIRECRAWL_API_KEY}',
firecrawlApiUrl: '${FIRECRAWL_API_URL}',
jinaApiKey: '${JINA_API_KEY}',
jinaApiUrl: '${JINA_API_URL}',
cohereApiKey: '${COHERE_API_KEY}',
safeSearch: SafeSearchTypes.MODERATE,
};
@ -300,6 +301,7 @@ describe('web.ts', () => {
firecrawlApiKey: '${FIRECRAWL_API_KEY}',
firecrawlApiUrl: '${FIRECRAWL_API_URL}',
jinaApiKey: '${JINA_API_KEY}',
jinaApiUrl: '${JINA_API_URL}',
cohereApiKey: '${COHERE_API_KEY}',
safeSearch: SafeSearchTypes.MODERATE,
};
@ -351,6 +353,7 @@ describe('web.ts', () => {
firecrawlApiKey: '${FIRECRAWL_API_KEY}',
firecrawlApiUrl: '${FIRECRAWL_API_URL}',
jinaApiKey: '${JINA_API_KEY}',
jinaApiUrl: '${JINA_API_URL}',
cohereApiKey: '${COHERE_API_KEY}',
safeSearch: SafeSearchTypes.MODERATE,
// Specify which services to use
@ -432,6 +435,7 @@ describe('web.ts', () => {
CUSTOM_FIRECRAWL_URL: 'https://custom.firecrawl.dev',
CUSTOM_JINA_KEY: 'custom-jina-key',
CUSTOM_COHERE_KEY: 'custom-cohere-key',
CUSTOM_JINA_URL: 'https://custom.jina.ai',
};
// Initialize webSearchConfig with custom variable names
@ -442,6 +446,7 @@ describe('web.ts', () => {
firecrawlApiKey: '${CUSTOM_FIRECRAWL_KEY}',
firecrawlApiUrl: '${CUSTOM_FIRECRAWL_URL}',
jinaApiKey: '${CUSTOM_JINA_KEY}',
jinaApiUrl: '${CUSTOM_JINA_URL}',
cohereApiKey: '${CUSTOM_COHERE_KEY}',
safeSearch: SafeSearchTypes.MODERATE,
// Specify which services to use
@ -512,6 +517,7 @@ describe('web.ts', () => {
firecrawlApiKey: '${FIRECRAWL_API_KEY}',
firecrawlApiUrl: '${FIRECRAWL_API_URL}',
jinaApiKey: '${JINA_API_KEY}',
jinaApiUrl: '${JINA_API_URL}',
cohereApiKey: '${COHERE_API_KEY}',
safeSearch: SafeSearchTypes.MODERATE,
};
@ -573,6 +579,7 @@ describe('web.ts', () => {
firecrawlApiKey: '${FIRECRAWL_API_KEY}',
firecrawlApiUrl: '${FIRECRAWL_API_URL}',
jinaApiKey: '${JINA_API_KEY}',
jinaApiUrl: '${JINA_API_URL}',
cohereApiKey: '${COHERE_API_KEY}',
safeSearch: SafeSearchTypes.MODERATE,
};
@ -682,6 +689,7 @@ describe('web.ts', () => {
firecrawlApiKey: '${FIRECRAWL_API_KEY}',
firecrawlApiUrl: '${FIRECRAWL_API_URL}',
jinaApiKey: '${JINA_API_KEY}',
jinaApiUrl: '${JINA_API_URL}',
cohereApiKey: '${COHERE_API_KEY}',
safeSearch: SafeSearchTypes.MODERATE,
searchProvider: 'serper' as SearchProviders,
@ -722,6 +730,7 @@ describe('web.ts', () => {
firecrawlApiKey: '${FIRECRAWL_API_KEY}',
firecrawlApiUrl: '${FIRECRAWL_API_URL}',
jinaApiKey: '${JINA_API_KEY}',
jinaApiUrl: '${JINA_API_URL}',
cohereApiKey: '${COHERE_API_KEY}',
safeSearch: SafeSearchTypes.MODERATE,
scraperType: 'firecrawl' as ScraperTypes,
@ -762,6 +771,7 @@ describe('web.ts', () => {
firecrawlApiKey: '${FIRECRAWL_API_KEY}',
firecrawlApiUrl: '${FIRECRAWL_API_URL}',
jinaApiKey: '${JINA_API_KEY}',
jinaApiUrl: '${JINA_API_URL}',
cohereApiKey: '${COHERE_API_KEY}',
safeSearch: SafeSearchTypes.MODERATE,
rerankerType: 'jina' as RerankerTypes,
@ -808,6 +818,7 @@ describe('web.ts', () => {
firecrawlApiKey: '${FIRECRAWL_API_KEY}',
firecrawlApiUrl: '${FIRECRAWL_API_URL}',
jinaApiKey: '${JINA_API_KEY}',
jinaApiUrl: '${JINA_API_URL}',
cohereApiKey: '${COHERE_API_KEY}',
safeSearch: SafeSearchTypes.MODERATE,
searchProvider: 'invalid-provider' as SearchProviders,
@ -842,6 +853,7 @@ describe('web.ts', () => {
firecrawlApiKey: '${FIRECRAWL_API_KEY}',
firecrawlApiUrl: '${FIRECRAWL_API_URL}',
jinaApiKey: '${JINA_API_KEY}',
jinaApiUrl: '${JINA_API_URL}',
cohereApiKey: '${COHERE_API_KEY}',
safeSearch: SafeSearchTypes.MODERATE,
rerankerType: 'jina' as RerankerTypes,
@ -892,6 +904,7 @@ describe('web.ts', () => {
firecrawlApiKey: '${FIRECRAWL_API_KEY}',
firecrawlApiUrl: '${FIRECRAWL_API_URL}',
jinaApiKey: '${JINA_API_KEY}',
jinaApiUrl: '${JINA_API_URL}',
cohereApiKey: '${COHERE_API_KEY}',
safeSearch: SafeSearchTypes.MODERATE,
};
@ -932,6 +945,7 @@ describe('web.ts', () => {
firecrawlApiKey: '${FIRECRAWL_API_KEY}',
firecrawlApiUrl: '${FIRECRAWL_API_URL}',
jinaApiKey: '${JINA_API_KEY}',
jinaApiUrl: '${JINA_API_URL}',
cohereApiKey: '${COHERE_API_KEY}',
safeSearch: SafeSearchTypes.MODERATE,
firecrawlOptions: {
@ -991,6 +1005,7 @@ describe('web.ts', () => {
firecrawlApiKey: '${FIRECRAWL_API_KEY}',
firecrawlApiUrl: '${FIRECRAWL_API_URL}',
jinaApiKey: '${JINA_API_KEY}',
jinaApiUrl: '${JINA_API_URL}',
safeSearch: SafeSearchTypes.MODERATE,
scraperTimeout: 15000, // This should take priority
firecrawlOptions: {
@ -1032,6 +1047,7 @@ describe('web.ts', () => {
firecrawlApiKey: '${FIRECRAWL_API_KEY}',
firecrawlApiUrl: '${FIRECRAWL_API_URL}',
jinaApiKey: '${JINA_API_KEY}',
jinaApiUrl: '${JINA_API_URL}',
safeSearch: SafeSearchTypes.MODERATE,
firecrawlOptions: {
includeTags: ['p'],
@ -1070,6 +1086,7 @@ describe('web.ts', () => {
firecrawlApiKey: '${FIRECRAWL_API_KEY}',
firecrawlApiUrl: '${FIRECRAWL_API_URL}',
jinaApiKey: '${JINA_API_KEY}',
jinaApiUrl: '${JINA_API_URL}',
safeSearch: SafeSearchTypes.MODERATE,
firecrawlOptions: {
timeout: 12000, // Only timeout provided
@ -1106,6 +1123,7 @@ describe('web.ts', () => {
firecrawlApiKey: '${FIRECRAWL_API_KEY}',
firecrawlApiUrl: '${FIRECRAWL_API_URL}',
jinaApiKey: '${JINA_API_KEY}',
jinaApiUrl: '${JINA_API_URL}',
safeSearch: SafeSearchTypes.MODERATE,
firecrawlOptions: {
formats: ['html', 'markdown'], // Only formats provided
@ -1142,6 +1160,7 @@ describe('web.ts', () => {
firecrawlApiKey: '${FIRECRAWL_API_KEY}',
firecrawlApiUrl: '${FIRECRAWL_API_URL}',
jinaApiKey: '${JINA_API_KEY}',
jinaApiUrl: '${JINA_API_URL}',
safeSearch: SafeSearchTypes.MODERATE,
firecrawlOptions: {
timeout: 8000,

9
packages/api/src/web/web.ts
View file

@ -21,6 +21,7 @@ export function loadWebSearchConfig(
const firecrawlApiKey = config?.firecrawlApiKey ?? '${FIRECRAWL_API_KEY}';
const firecrawlApiUrl = config?.firecrawlApiUrl ?? '${FIRECRAWL_API_URL}';
const jinaApiKey = config?.jinaApiKey ?? '${JINA_API_KEY}';
const jinaApiUrl = config?.jinaApiUrl ?? '${JINA_API_URL}';
const cohereApiKey = config?.cohereApiKey ?? '${COHERE_API_KEY}';
const safeSearch = config?.safeSearch ?? SafeSearchTypes.MODERATE;
@ -28,6 +29,7 @@ export function loadWebSearchConfig(
...config,
safeSearch,
jinaApiKey,
jinaApiUrl,
cohereApiKey,
serperApiKey,
searxngInstanceUrl,
@ -44,6 +46,7 @@ export type TWebSearchKeys =
| 'firecrawlApiKey'
| 'firecrawlApiUrl'
| 'jinaApiKey'
| 'jinaApiUrl'
| 'cohereApiKey';
export type TWebSearchCategories =
@ -70,7 +73,11 @@ export const webSearchAuth = {
},
},
rerankers: {
jina: { jinaApiKey: 1 as const },
jina: {
jinaApiKey: 1 as const,
/** Optional (0) */
jinaApiUrl: 0 as const,
},
cohere: { cohereApiKey: 1 as const },
},
};