Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-12-22 03:10:15 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: add bottleneck for tts routes

Browse source 
refactor: slightly increase request threshold
This commit is contained in:
Danny Avila 2024-05-30 20:05:01 -04:00
parent b8e35002f4
commit b76233bd66
No known key found for this signature in database
GPG key ID: 2DD9CC89B9B50364
3 changed files with 28 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

1
api/package.json
View file

@ -44,6 +44,7 @@
"agenda": "^5.0.0", "agenda": "^5.0.0",
"axios": "^1.3.4", "axios": "^1.3.4",
"bcryptjs": "^2.4.3", "bcryptjs": "^2.4.3",
"bottleneck": "^2.19.5",
"cheerio": "^1.0.0-rc.12", "cheerio": "^1.0.0-rc.12",
"cohere-ai": "^7.9.1", "cohere-ai": "^7.9.1",
"connect-redis": "^7.1.0", "connect-redis": "^7.1.0",

23
api/server/routes/files/tts.js
View file

@ -1,5 +1,6 @@
const multer = require('multer'); const multer = require('multer');
const express = require('express'); const express = require('express');
const Bottleneck = require('bottleneck');
const { CacheKeys } = require('librechat-data-provider'); const { CacheKeys } = require('librechat-data-provider');
const { getVoices, streamAudio, textToSpeech } = require('~/server/services/Files/Audio'); const { getVoices, streamAudio, textToSpeech } = require('~/server/services/Files/Audio');
const { getLogStores } = require('~/cache'); const { getLogStores } = require('~/cache');
@ -8,8 +9,25 @@ const { logger } = require('~/config');
const router = express.Router(); const router = express.Router();
const upload = multer(); const upload = multer();
// todo: can add Redis support for limiter
const limiter = new Bottleneck({
minTime: 240, // Minimum time between requests (240ms per request = 250 requests per minute)
maxConcurrent: 100, // Maximum number of concurrent requests
reservoir: 250, // Initial number of available requests
reservoirRefreshAmount: 250, // Number of requests replenished in each interval
reservoirRefreshInterval: 60 * 1000, // Reservoir refresh interval (60 seconds)
});
const limitedStreamAudio = limiter.wrap(streamAudio);
const limitedTextToSpeech = limiter.wrap(textToSpeech);
router.post('/manual', upload.none(), async (req, res) => { router.post('/manual', upload.none(), async (req, res) => {
await textToSpeech(req, res); try {
await limitedTextToSpeech(req, res);
} catch (error) {
logger.error(`[textToSpeech] user: ${req.user.id} | Failed to process textToSpeech: ${error}`);
res.status(500).json({ error: 'Failed to process textToSpeech' });
}
}); });
const logDebugMessage = (req, message) => const logDebugMessage = (req, message) =>
@ -26,7 +44,7 @@ router.post('/', async (req, res) => {
return res.status(401).json({ error: 'Audio stream already running' }); return res.status(401).json({ error: 'Audio stream already running' });
} }
audioRunsCache.set(req.body.runId, true); audioRunsCache.set(req.body.runId, true);
await streamAudio(req, res); await limitedStreamAudio(req, res);
logDebugMessage(req, 'end stream audio'); logDebugMessage(req, 'end stream audio');
res.status(200).end(); res.status(200).end();
} catch (error) { } catch (error) {
@ -35,6 +53,7 @@ router.post('/', async (req, res) => {
} }
}); });
// todo: cache voices
router.get('/voices', async (req, res) => { router.get('/voices', async (req, res) => {
await getVoices(req, res); await getVoices(req, res);
}); });

6
package-lock.json generated
View file

@ -52,6 +52,7 @@
"agenda": "^5.0.0", "agenda": "^5.0.0",
"axios": "^1.3.4", "axios": "^1.3.4",
"bcryptjs": "^2.4.3", "bcryptjs": "^2.4.3",
"bottleneck": "^2.19.5",
"cheerio": "^1.0.0-rc.12", "cheerio": "^1.0.0-rc.12",
"cohere-ai": "^7.9.1", "cohere-ai": "^7.9.1",
"connect-redis": "^7.1.0", "connect-redis": "^7.1.0",
@ -11700,6 +11701,11 @@
"integrity": "sha512-ma2q0Tc760dW54CdOyJjhrg/a54317o1zYADQJFgperNGKIKgAUGIcKnuMiff8z57+yGlrGNEt4lPgZfCgTJgA==", "integrity": "sha512-ma2q0Tc760dW54CdOyJjhrg/a54317o1zYADQJFgperNGKIKgAUGIcKnuMiff8z57+yGlrGNEt4lPgZfCgTJgA==",
"dev": true "dev": true
}, },
"node_modules/bottleneck": {
"version": "2.19.5",
"resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz",
"integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw=="
},
"node_modules/bowser": { "node_modules/bowser": {
"version": "2.11.0", "version": "2.11.0",
"resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz", "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz",