From b76233bd668d8b189aec941ab352948fe37b7d9f Mon Sep 17 00:00:00 2001
From: Danny Avila <danny@librechat.ai>
Date: Thu, 30 May 2024 20:05:01 -0400
Subject: [PATCH] feat: add `bottleneck` for tts routes

refactor: slightly increase request threshold
---
 api/package.json               |  1 +
 api/server/routes/files/tts.js | 23 +++++++++++++++++++++--
 package-lock.json              |  6 ++++++
 3 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/api/package.json b/api/package.json
index ca1a62daf8..d6a8d53f0f 100644
--- a/api/package.json
+++ b/api/package.json
@@ -44,6 +44,7 @@
     "agenda": "^5.0.0",
     "axios": "^1.3.4",
     "bcryptjs": "^2.4.3",
+    "bottleneck": "^2.19.5",
     "cheerio": "^1.0.0-rc.12",
     "cohere-ai": "^7.9.1",
     "connect-redis": "^7.1.0",
diff --git a/api/server/routes/files/tts.js b/api/server/routes/files/tts.js
index 1ee540874f..e745a4b8c9 100644
--- a/api/server/routes/files/tts.js
+++ b/api/server/routes/files/tts.js
@@ -1,5 +1,6 @@
 const multer = require('multer');
 const express = require('express');
+const Bottleneck = require('bottleneck');
 const { CacheKeys } = require('librechat-data-provider');
 const { getVoices, streamAudio, textToSpeech } = require('~/server/services/Files/Audio');
 const { getLogStores } = require('~/cache');
@@ -8,8 +9,25 @@ const { logger } = require('~/config');
 const router = express.Router();
 const upload = multer();
 
+// todo: can add Redis support for limiter
+const limiter = new Bottleneck({
+  minTime: 240, // Minimum time between requests (240ms per request = 250 requests per minute)
+  maxConcurrent: 100, // Maximum number of concurrent requests
+  reservoir: 250, // Initial number of available requests
+  reservoirRefreshAmount: 250, // Number of requests replenished in each interval
+  reservoirRefreshInterval: 60 * 1000, // Reservoir refresh interval (60 seconds)
+});
+
+const limitedStreamAudio = limiter.wrap(streamAudio);
+const limitedTextToSpeech = limiter.wrap(textToSpeech);
+
 router.post('/manual', upload.none(), async (req, res) => {
-  await textToSpeech(req, res);
+  try {
+    await limitedTextToSpeech(req, res);
+  } catch (error) {
+    logger.error(`[textToSpeech] user: ${req.user.id} | Failed to process textToSpeech: ${error}`);
+    res.status(500).json({ error: 'Failed to process textToSpeech' });
+  }
 });
 
 const logDebugMessage = (req, message) =>
@@ -26,7 +44,7 @@ router.post('/', async (req, res) => {
       return res.status(401).json({ error: 'Audio stream already running' });
     }
     audioRunsCache.set(req.body.runId, true);
-    await streamAudio(req, res);
+    await limitedStreamAudio(req, res);
     logDebugMessage(req, 'end stream audio');
     res.status(200).end();
   } catch (error) {
@@ -35,6 +53,7 @@ router.post('/', async (req, res) => {
   }
 });
 
+// todo: cache voices
 router.get('/voices', async (req, res) => {
   await getVoices(req, res);
 });
diff --git a/package-lock.json b/package-lock.json
index f39075f7f6..f312ce3716 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -52,6 +52,7 @@
         "agenda": "^5.0.0",
         "axios": "^1.3.4",
         "bcryptjs": "^2.4.3",
+        "bottleneck": "^2.19.5",
         "cheerio": "^1.0.0-rc.12",
         "cohere-ai": "^7.9.1",
         "connect-redis": "^7.1.0",
@@ -11700,6 +11701,11 @@
       "integrity": "sha512-ma2q0Tc760dW54CdOyJjhrg/a54317o1zYADQJFgperNGKIKgAUGIcKnuMiff8z57+yGlrGNEt4lPgZfCgTJgA==",
       "dev": true
     },
+    "node_modules/bottleneck": {
+      "version": "2.19.5",
+      "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz",
+      "integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw=="
+    },
     "node_modules/bowser": {
       "version": "2.11.0",
       "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz",