mirror of
https://github.com/danny-avila/LibreChat.git
synced 2026-03-12 11:02:37 +01:00
9a5d7eaa4e
Some checks are pending
Docker Dev Branch Images Build / build (Dockerfile, lc-dev, node) (push) Waiting to run
Docker Dev Branch Images Build / build (Dockerfile.multi, lc-dev-api, api-build) (push) Waiting to run
Docker Dev Images Build / build (Dockerfile, librechat-dev, node) (push) Waiting to run
Docker Dev Images Build / build (Dockerfile.multi, librechat-dev-api, api-build) (push) Waiting to run
Sync Locize Translations & Create Translation PR / Sync Translation Keys with Locize (push) Waiting to run
Sync Locize Translations & Create Translation PR / Create Translation PR on Version Published (push) Blocked by required conditions
* chore: Update dependencies by adding ai-tokenizer and removing tiktoken - Added ai-tokenizer version 1.0.6 to package.json and package-lock.json across multiple packages. - Removed tiktoken version 1.0.15 from package.json and package-lock.json in the same locations, streamlining dependency management. * refactor: replace js-tiktoken with ai-tokenizer - Added support for 'claude' encoding in the AgentClient class to improve model compatibility. - Updated Tokenizer class to utilize 'ai-tokenizer' for both 'o200k_base' and 'claude' encodings, replacing the previous 'tiktoken' dependency. - Refactored tests to reflect changes in tokenizer behavior and ensure accurate token counting for both encoding types. - Removed deprecated references to 'tiktoken' and adjusted related tests for improved clarity and functionality. * chore: remove tiktoken mocks from DALLE3 tests - Eliminated mock implementations of 'tiktoken' from DALLE3-related test files to streamline test setup and align with recent dependency updates. - Adjusted related test structures to ensure compatibility with the new tokenizer implementation. * chore: Add distinct encoding support for Anthropic Claude models - Introduced a new method `getEncoding` in the AgentClient class to handle the specific BPE tokenizer for Claude models, ensuring compatibility with the distinct encoding requirements. - Updated documentation to clarify the encoding logic for Claude and other models. * docs: Update return type documentation for getEncoding method in AgentClient - Clarified the return type of the getEncoding method to specify that it can return an EncodingName or undefined, enhancing code readability and type safety. * refactor: Tokenizer class and error handling - Exported the EncodingName type for broader usage. - Renamed encodingMap to encodingData for clarity. - Improved error handling in getTokenCount method to ensure recovery attempts are logged and return 0 on failure. - Updated countTokens function documentation to specify the use of 'o200k_base' encoding. * refactor: Simplify encoding documentation and export type - Updated the getEncoding method documentation to clarify the default behavior for non-Anthropic Claude models. - Exported the EncodingName type separately from the Tokenizer module for improved clarity and usage. * test: Update text processing tests for token limits - Adjusted test cases to handle smaller text sizes, changing scenarios from ~120k tokens to ~20k tokens for both the real tokenizer and countTokens functions. - Updated token limits in tests to reflect new constraints, ensuring tests accurately assess performance and call reduction. - Enhanced console log messages for clarity regarding token counts and reductions in the updated scenarios. * refactor: Update Tokenizer imports and exports - Moved Tokenizer and countTokens exports to the tokenizer module for better organization. - Adjusted imports in memory.ts to reflect the new structure, ensuring consistent usage across the codebase. - Updated memory.test.ts to mock the Tokenizer from the correct module path, enhancing test accuracy. * refactor: Tokenizer initialization and error handling - Introduced an async `initEncoding` method to preload tokenizers, improving performance and accuracy in token counting. - Updated `getTokenCount` to handle uninitialized tokenizers more gracefully, ensuring proper recovery and logging on errors. - Removed deprecated synchronous tokenizer retrieval, streamlining the overall tokenizer management process. * test: Enhance tokenizer tests with initialization and encoding checks - Added `beforeAll` hooks to initialize tokenizers for 'o200k_base' and 'claude' encodings before running tests, ensuring proper setup. - Updated tests to validate the loading of encodings and the correctness of token counts for both 'o200k_base' and 'claude'. - Improved test structure to deduplicate concurrent initialization calls, enhancing performance and reliability.
124 lines
3.6 KiB
JSON
124 lines
3.6 KiB
JSON
{
|
|
"name": "@librechat/backend",
|
|
"version": "v0.8.3",
|
|
"description": "",
|
|
"scripts": {
|
|
"start": "echo 'please run this from the root directory'",
|
|
"server-dev": "echo 'please run this from the root directory'",
|
|
"test": "cross-env NODE_ENV=test jest",
|
|
"b:test": "NODE_ENV=test bun jest",
|
|
"test:ci": "jest --ci --logHeapUsage",
|
|
"add-balance": "node ./add-balance.js",
|
|
"list-balances": "node ./list-balances.js",
|
|
"user-stats": "node ./user-stats.js",
|
|
"create-user": "node ./create-user.js",
|
|
"invite-user": "node ./invite-user.js",
|
|
"ban-user": "node ./ban-user.js",
|
|
"delete-user": "node ./delete-user.js"
|
|
},
|
|
"repository": {
|
|
"type": "git",
|
|
"url": "git+https://github.com/danny-avila/LibreChat.git"
|
|
},
|
|
"keywords": [],
|
|
"author": "",
|
|
"license": "ISC",
|
|
"_moduleAliases": {
|
|
"~": "."
|
|
},
|
|
"imports": {
|
|
"~/*": "./*"
|
|
},
|
|
"bugs": {
|
|
"url": "https://github.com/danny-avila/LibreChat/issues"
|
|
},
|
|
"homepage": "https://librechat.ai",
|
|
"dependencies": {
|
|
"@anthropic-ai/vertex-sdk": "^0.14.3",
|
|
"@aws-sdk/client-bedrock-runtime": "^3.980.0",
|
|
"@aws-sdk/client-s3": "^3.980.0",
|
|
"@aws-sdk/s3-request-presigner": "^3.758.0",
|
|
"@azure/identity": "^4.7.0",
|
|
"@azure/search-documents": "^12.0.0",
|
|
"@azure/storage-blob": "^12.30.0",
|
|
"@google/genai": "^1.19.0",
|
|
"@keyv/redis": "^4.3.3",
|
|
"@langchain/core": "^0.3.80",
|
|
"@librechat/agents": "^3.1.55",
|
|
"@librechat/api": "*",
|
|
"@librechat/data-schemas": "*",
|
|
"@microsoft/microsoft-graph-client": "^3.0.7",
|
|
"@modelcontextprotocol/sdk": "^1.27.1",
|
|
"@node-saml/passport-saml": "^5.1.0",
|
|
"@smithy/node-http-handler": "^4.4.5",
|
|
"ai-tokenizer": "^1.0.6",
|
|
"axios": "^1.13.5",
|
|
"bcryptjs": "^2.4.3",
|
|
"compression": "^1.8.1",
|
|
"connect-redis": "^8.1.0",
|
|
"cookie": "^0.7.2",
|
|
"cookie-parser": "^1.4.7",
|
|
"cors": "^2.8.5",
|
|
"dedent": "^1.5.3",
|
|
"dotenv": "^16.0.3",
|
|
"eventsource": "^3.0.2",
|
|
"express": "^5.2.1",
|
|
"express-mongo-sanitize": "^2.2.0",
|
|
"express-rate-limit": "^8.3.0",
|
|
"express-session": "^1.18.2",
|
|
"express-static-gzip": "^2.2.0",
|
|
"file-type": "^18.7.0",
|
|
"firebase": "^11.0.2",
|
|
"form-data": "^4.0.4",
|
|
"handlebars": "^4.7.7",
|
|
"https-proxy-agent": "^7.0.6",
|
|
"ioredis": "^5.3.2",
|
|
"js-yaml": "^4.1.1",
|
|
"jsonwebtoken": "^9.0.0",
|
|
"jwks-rsa": "^3.2.0",
|
|
"keyv": "^5.3.2",
|
|
"keyv-file": "^5.1.2",
|
|
"klona": "^2.0.6",
|
|
"librechat-data-provider": "*",
|
|
"lodash": "^4.17.23",
|
|
"mammoth": "^1.11.0",
|
|
"mathjs": "^15.1.0",
|
|
"meilisearch": "^0.38.0",
|
|
"memorystore": "^1.6.7",
|
|
"mime": "^3.0.0",
|
|
"module-alias": "^2.2.3",
|
|
"mongoose": "^8.12.1",
|
|
"multer": "^2.1.1",
|
|
"nanoid": "^3.3.7",
|
|
"node-fetch": "^2.7.0",
|
|
"nodemailer": "^7.0.11",
|
|
"ollama": "^0.5.0",
|
|
"openai": "5.8.2",
|
|
"openid-client": "^6.5.0",
|
|
"passport": "^0.6.0",
|
|
"passport-apple": "^2.0.2",
|
|
"passport-discord": "^0.1.4",
|
|
"passport-facebook": "^3.0.0",
|
|
"passport-github2": "^0.1.12",
|
|
"passport-google-oauth20": "^2.0.0",
|
|
"passport-jwt": "^4.0.1",
|
|
"passport-ldapauth": "^3.0.1",
|
|
"passport-local": "^1.0.0",
|
|
"pdfjs-dist": "^5.4.624",
|
|
"rate-limit-redis": "^4.2.0",
|
|
"sharp": "^0.33.5",
|
|
"traverse": "^0.6.7",
|
|
"ua-parser-js": "^1.0.36",
|
|
"undici": "^7.18.2",
|
|
"winston": "^3.11.0",
|
|
"winston-daily-rotate-file": "^5.0.0",
|
|
"xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz",
|
|
"zod": "^3.22.4"
|
|
},
|
|
"devDependencies": {
|
|
"jest": "^30.2.0",
|
|
"mongodb-memory-server": "^10.1.4",
|
|
"nodemon": "^3.0.3",
|
|
"supertest": "^7.1.0"
|
|
}
|
|
}
|