Andreas/LibreChat
1
0
Fork 0
mirror of https://github.com/danny-avila/LibreChat.git synced 2025-12-21 19:00:13 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: add bottleneck for tts routes

Browse source 
refactor: slightly increase request threshold
This commit is contained in:
Danny Avila 2024-05-30 20:05:01 -04:00
parent b8e35002f4
commit b76233bd66
No known key found for this signature in database
GPG key ID: 2DD9CC89B9B50364
3 changed files with 28 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

23
api/server/routes/files/tts.js
View file

@ -1,5 +1,6 @@
const multer = require('multer');
const express = require('express');
const Bottleneck = require('bottleneck');
const { CacheKeys } = require('librechat-data-provider');
const { getVoices, streamAudio, textToSpeech } = require('~/server/services/Files/Audio');
const { getLogStores } = require('~/cache');
@ -8,8 +9,25 @@ const { logger } = require('~/config');
const router = express.Router();
const upload = multer();
// todo: can add Redis support for limiter
const limiter = new Bottleneck({
minTime: 240, // Minimum time between requests (240ms per request = 250 requests per minute)
maxConcurrent: 100, // Maximum number of concurrent requests
reservoir: 250, // Initial number of available requests
reservoirRefreshAmount: 250, // Number of requests replenished in each interval
reservoirRefreshInterval: 60 * 1000, // Reservoir refresh interval (60 seconds)
});
const limitedStreamAudio = limiter.wrap(streamAudio);
const limitedTextToSpeech = limiter.wrap(textToSpeech);
router.post('/manual', upload.none(), async (req, res) => {
await textToSpeech(req, res);
try {
await limitedTextToSpeech(req, res);
} catch (error) {
logger.error(`[textToSpeech] user: ${req.user.id} | Failed to process textToSpeech: ${error}`);
res.status(500).json({ error: 'Failed to process textToSpeech' });
}
});
const logDebugMessage = (req, message) =>
@ -26,7 +44,7 @@ router.post('/', async (req, res) => {
return res.status(401).json({ error: 'Audio stream already running' });
}
audioRunsCache.set(req.body.runId, true);
await streamAudio(req, res);
await limitedStreamAudio(req, res);
logDebugMessage(req, 'end stream audio');
res.status(200).end();
} catch (error) {
@ -35,6 +53,7 @@ router.post('/', async (req, res) => {
}
});
// todo: cache voices
router.get('/voices', async (req, res) => {
await getVoices(req, res);
});