diff --git a/api/server/services/Files/Audio/STTService.js b/api/server/services/Files/Audio/STTService.js new file mode 100644 index 0000000000..49de1c84d6 --- /dev/null +++ b/api/server/services/Files/Audio/STTService.js @@ -0,0 +1,248 @@ +const { Readable } = require('stream'); +const axios = require('axios'); +const { extractEnvVariable, STTProviders } = require('librechat-data-provider'); +const getCustomConfig = require('~/server/services/Config/getCustomConfig'); +const { genAzureEndpoint } = require('~/utils'); +const { logger } = require('~/config'); + +/** + * Service class for handling Speech-to-Text (STT) operations. + * @class + */ +class STTService { + /** + * Creates an instance of STTService. + * @param {Object} customConfig - The custom configuration object. + */ + constructor(customConfig) { + this.customConfig = customConfig; + this.providerStrategies = { + [STTProviders.OPENAI]: this.openAIProvider, + [STTProviders.AZURE_OPENAI]: this.azureOpenAIProvider, + }; + } + + /** + * Creates a singleton instance of STTService. + * @static + * @async + * @returns {Promise} The STTService instance. + * @throws {Error} If the custom config is not found. + */ + static async getInstance() { + const customConfig = await getCustomConfig(); + if (!customConfig) { + throw new Error('Custom config not found'); + } + return new STTService(customConfig); + } + + /** + * Retrieves the configured STT provider and its schema. + * @returns {Promise<[string, Object]>} A promise that resolves to an array containing the provider name and its schema. + * @throws {Error} If no STT schema is set, multiple providers are set, or no provider is set. + */ + async getProviderSchema() { + const sttSchema = this.customConfig.speech.stt; + + if (!sttSchema) { + throw new Error( + 'No STT schema is set. Did you configure STT in the custom config (librechat.yaml)?', + ); + } + + const providers = Object.entries(sttSchema).filter( + ([, value]) => Object.keys(value).length > 0, + ); + + if (providers.length !== 1) { + throw new Error( + providers.length > 1 + ? 'Multiple providers are set. Please set only one provider.' + : 'No provider is set. Please set a provider.', + ); + } + + const [provider, schema] = providers[0]; + return [provider, schema]; + } + + /** + * Recursively removes undefined properties from an object. + * @param {Object} obj - The object to clean. + * @returns {void} + */ + removeUndefined(obj) { + Object.keys(obj).forEach((key) => { + if (obj[key] && typeof obj[key] === 'object') { + this.removeUndefined(obj[key]); + if (Object.keys(obj[key]).length === 0) { + delete obj[key]; + } + } else if (obj[key] === undefined) { + delete obj[key]; + } + }); + } + + /** + * Prepares the request for the OpenAI STT provider. + * @param {Object} sttSchema - The STT schema for OpenAI. + * @param {Stream} audioReadStream - The audio data to be transcribed. + * @returns {Array} An array containing the URL, data, and headers for the request. + */ + openAIProvider(sttSchema, audioReadStream) { + const url = sttSchema?.url || 'https://api.openai.com/v1/audio/transcriptions'; + const apiKey = extractEnvVariable(sttSchema.apiKey) || ''; + + const data = { + file: audioReadStream, + model: sttSchema.model, + }; + + const headers = { + 'Content-Type': 'multipart/form-data', + ...(apiKey && { Authorization: `Bearer ${apiKey}` }), + }; + [headers].forEach(this.removeUndefined); + + return [url, data, headers]; + } + + /** + * Prepares the request for the Azure OpenAI STT provider. + * @param {Object} sttSchema - The STT schema for Azure OpenAI. + * @param {Buffer} audioBuffer - The audio data to be transcribed. + * @param {Object} audioFile - The audio file object containing originalname, mimetype, and size. + * @returns {Array} An array containing the URL, data, and headers for the request. + * @throws {Error} If the audio file size exceeds 25MB or the audio file format is not accepted. + */ + azureOpenAIProvider(sttSchema, audioBuffer, audioFile) { + const url = `${genAzureEndpoint({ + azureOpenAIApiInstanceName: sttSchema?.instanceName, + azureOpenAIApiDeploymentName: sttSchema?.deploymentName, + })}/audio/transcriptions?api-version=${sttSchema?.apiVersion}`; + + const apiKey = sttSchema.apiKey ? extractEnvVariable(sttSchema.apiKey) : ''; + + if (audioBuffer.byteLength > 25 * 1024 * 1024) { + throw new Error('The audio file size exceeds the limit of 25MB'); + } + + const acceptedFormats = ['flac', 'mp3', 'mp4', 'mpeg', 'mpga', 'm4a', 'ogg', 'wav', 'webm']; + const fileFormat = audioFile.mimetype.split('/')[1]; + if (!acceptedFormats.includes(fileFormat)) { + throw new Error(`The audio file format ${fileFormat} is not accepted`); + } + + const formData = new FormData(); + const audioBlob = new Blob([audioBuffer], { type: audioFile.mimetype }); + formData.append('file', audioBlob, audioFile.originalname); + + const headers = { + 'Content-Type': 'multipart/form-data', + ...(apiKey && { 'api-key': apiKey }), + }; + + [headers].forEach(this.removeUndefined); + + return [url, formData, headers]; + } + + /** + * Sends an STT request to the specified provider. + * @async + * @param {string} provider - The STT provider to use. + * @param {Object} sttSchema - The STT schema for the provider. + * @param {Object} requestData - The data required for the STT request. + * @param {Buffer} requestData.audioBuffer - The audio data to be transcribed. + * @param {Object} requestData.audioFile - The audio file object containing originalname, mimetype, and size. + * @returns {Promise} A promise that resolves to the transcribed text. + * @throws {Error} If the provider is invalid, the response status is not 200, or the response data is missing. + */ + async sttRequest(provider, sttSchema, { audioBuffer, audioFile }) { + const strategy = this.providerStrategies[provider]; + if (!strategy) { + throw new Error('Invalid provider'); + } + + const audioReadStream = Readable.from(audioBuffer); + audioReadStream.path = 'audio.wav'; + + const [url, data, headers] = strategy.call(this, sttSchema, audioReadStream, audioFile); + + if (!Readable.from && data instanceof FormData) { + const audioBlob = new Blob([audioBuffer], { type: audioFile.mimetype }); + data.set('file', audioBlob, audioFile.originalname); + } + + try { + const response = await axios.post(url, data, { headers }); + + if (response.status !== 200) { + throw new Error('Invalid response from the STT API'); + } + + if (!response.data || !response.data.text) { + throw new Error('Missing data in response from the STT API'); + } + + return response.data.text.trim(); + } catch (error) { + logger.error(`STT request failed for provider ${provider}:`, error); + throw error; + } + } + + /** + * Processes a speech-to-text request. + * @async + * @param {Object} req - The request object. + * @param {Object} res - The response object. + * @returns {Promise} + */ + async processTextToSpeech(req, res) { + if (!req.file || !req.file.buffer) { + return res.status(400).json({ message: 'No audio file provided in the FormData' }); + } + + const audioBuffer = req.file.buffer; + const audioFile = { + originalname: req.file.originalname, + mimetype: req.file.mimetype, + size: req.file.size, + }; + + try { + const [provider, sttSchema] = await this.getProviderSchema(); + const text = await this.sttRequest(provider, sttSchema, { audioBuffer, audioFile }); + res.json({ text }); + } catch (error) { + logger.error('An error occurred while processing the audio:', error); + res.sendStatus(500); + } + } +} + +/** + * Factory function to create an STTService instance. + * @async + * @returns {Promise} A promise that resolves to an STTService instance. + */ +async function createSTTService() { + return STTService.getInstance(); +} + +/** + * Wrapper function for speech-to-text processing. + * @async + * @param {Object} req - The request object. + * @param {Object} res - The response object. + * @returns {Promise} + */ +async function speechToText(req, res) { + const sttService = await createSTTService(); + await sttService.processTextToSpeech(req, res); +} + +module.exports = speechToText; diff --git a/api/server/services/Files/Audio/TTSService.js b/api/server/services/Files/Audio/TTSService.js new file mode 100644 index 0000000000..cf99633cb7 --- /dev/null +++ b/api/server/services/Files/Audio/TTSService.js @@ -0,0 +1,477 @@ +const axios = require('axios'); +const { extractEnvVariable, TTSProviders } = require('librechat-data-provider'); +const { logger } = require('~/config'); +const getCustomConfig = require('~/server/services/Config/getCustomConfig'); +const { genAzureEndpoint } = require('~/utils'); +const { getRandomVoiceId, createChunkProcessor, splitTextIntoChunks } = require('./streamAudio'); + +/** + * Service class for handling Text-to-Speech (TTS) operations. + * @class + */ +class TTSService { + /** + * Creates an instance of TTSService. + * @param {Object} customConfig - The custom configuration object. + */ + constructor(customConfig) { + this.customConfig = customConfig; + this.providerStrategies = { + [TTSProviders.OPENAI]: this.openAIProvider.bind(this), + [TTSProviders.AZURE_OPENAI]: this.azureOpenAIProvider.bind(this), + [TTSProviders.ELEVENLABS]: this.elevenLabsProvider.bind(this), + [TTSProviders.LOCALAI]: this.localAIProvider.bind(this), + }; + } + + /** + * Creates a singleton instance of TTSService. + * @static + * @async + * @returns {Promise} The TTSService instance. + * @throws {Error} If the custom config is not found. + */ + static async getInstance() { + const customConfig = await getCustomConfig(); + if (!customConfig) { + throw new Error('Custom config not found'); + } + return new TTSService(customConfig); + } + + /** + * Retrieves the configured TTS provider. + * @returns {string} The name of the configured provider. + * @throws {Error} If no provider is set or multiple providers are set. + */ + getProvider() { + const ttsSchema = this.customConfig.speech.tts; + if (!ttsSchema) { + throw new Error( + 'No TTS schema is set. Did you configure TTS in the custom config (librechat.yaml)?', + ); + } + const providers = Object.entries(ttsSchema).filter( + ([, value]) => Object.keys(value).length > 0, + ); + + if (providers.length !== 1) { + throw new Error( + providers.length > 1 + ? 'Multiple providers are set. Please set only one provider.' + : 'No provider is set. Please set a provider.', + ); + } + return providers[0][0]; + } + + /** + * Selects a voice for TTS based on provider schema and request. + * @async + * @param {Object} providerSchema - The schema for the selected provider. + * @param {string} requestVoice - The requested voice. + * @returns {Promise} The selected voice. + */ + async getVoice(providerSchema, requestVoice) { + const voices = providerSchema.voices.filter((voice) => voice && voice.toUpperCase() !== 'ALL'); + let voice = requestVoice; + if (!voice || !voices.includes(voice) || (voice.toUpperCase() === 'ALL' && voices.length > 1)) { + voice = getRandomVoiceId(voices); + } + return voice; + } + + /** + * Recursively removes undefined properties from an object. + * @param {Object} obj - The object to clean. + */ + removeUndefined(obj) { + Object.keys(obj).forEach((key) => { + if (obj[key] && typeof obj[key] === 'object') { + this.removeUndefined(obj[key]); + if (Object.keys(obj[key]).length === 0) { + delete obj[key]; + } + } else if (obj[key] === undefined) { + delete obj[key]; + } + }); + } + + /** + * Prepares the request for OpenAI TTS provider. + * @param {Object} ttsSchema - The TTS schema for OpenAI. + * @param {string} input - The input text. + * @param {string} voice - The selected voice. + * @returns {Array} An array containing the URL, data, and headers for the request. + * @throws {Error} If the selected voice is not available. + */ + openAIProvider(ttsSchema, input, voice) { + const url = ttsSchema?.url || 'https://api.openai.com/v1/audio/speech'; + + if ( + ttsSchema?.voices && + ttsSchema.voices.length > 0 && + !ttsSchema.voices.includes(voice) && + !ttsSchema.voices.includes('ALL') + ) { + throw new Error(`Voice ${voice} is not available.`); + } + + const data = { + input, + model: ttsSchema?.model, + voice: ttsSchema?.voices && ttsSchema.voices.length > 0 ? voice : undefined, + backend: ttsSchema?.backend, + }; + + const headers = { + 'Content-Type': 'application/json', + Authorization: `Bearer ${extractEnvVariable(ttsSchema?.apiKey)}`, + }; + + return [url, data, headers]; + } + + /** + * Prepares the request for Azure OpenAI TTS provider. + * @param {Object} ttsSchema - The TTS schema for Azure OpenAI. + * @param {string} input - The input text. + * @param {string} voice - The selected voice. + * @returns {Array} An array containing the URL, data, and headers for the request. + * @throws {Error} If the selected voice is not available. + */ + azureOpenAIProvider(ttsSchema, input, voice) { + const url = `${genAzureEndpoint({ + azureOpenAIApiInstanceName: ttsSchema?.instanceName, + azureOpenAIApiDeploymentName: ttsSchema?.deploymentName, + })}/audio/speech?api-version=${ttsSchema?.apiVersion}`; + + if ( + ttsSchema?.voices && + ttsSchema.voices.length > 0 && + !ttsSchema.voices.includes(voice) && + !ttsSchema.voices.includes('ALL') + ) { + throw new Error(`Voice ${voice} is not available.`); + } + + const data = { + model: ttsSchema?.model, + input, + voice: ttsSchema?.voices && ttsSchema.voices.length > 0 ? voice : undefined, + }; + + const headers = { + 'Content-Type': 'application/json', + 'api-key': ttsSchema.apiKey ? extractEnvVariable(ttsSchema.apiKey) : '', + }; + + return [url, data, headers]; + } + + /** + * Prepares the request for ElevenLabs TTS provider. + * @param {Object} ttsSchema - The TTS schema for ElevenLabs. + * @param {string} input - The input text. + * @param {string} voice - The selected voice. + * @param {boolean} stream - Whether to use streaming. + * @returns {Array} An array containing the URL, data, and headers for the request. + * @throws {Error} If the selected voice is not available. + */ + elevenLabsProvider(ttsSchema, input, voice, stream) { + let url = + ttsSchema?.url || + `https://api.elevenlabs.io/v1/text-to-speech/${voice}${stream ? '/stream' : ''}`; + + if (!ttsSchema?.voices.includes(voice) && !ttsSchema?.voices.includes('ALL')) { + throw new Error(`Voice ${voice} is not available.`); + } + + const data = { + model_id: ttsSchema?.model, + text: input, + voice_settings: { + similarity_boost: ttsSchema?.voice_settings?.similarity_boost, + stability: ttsSchema?.voice_settings?.stability, + style: ttsSchema?.voice_settings?.style, + use_speaker_boost: ttsSchema?.voice_settings?.use_speaker_boost, + }, + pronunciation_dictionary_locators: ttsSchema?.pronunciation_dictionary_locators, + }; + + const headers = { + 'Content-Type': 'application/json', + 'xi-api-key': extractEnvVariable(ttsSchema?.apiKey), + Accept: 'audio/mpeg', + }; + + return [url, data, headers]; + } + + /** + * Prepares the request for LocalAI TTS provider. + * @param {Object} ttsSchema - The TTS schema for LocalAI. + * @param {string} input - The input text. + * @param {string} voice - The selected voice. + * @returns {Array} An array containing the URL, data, and headers for the request. + * @throws {Error} If the selected voice is not available. + */ + localAIProvider(ttsSchema, input, voice) { + const url = ttsSchema?.url; + + if ( + ttsSchema?.voices && + ttsSchema.voices.length > 0 && + !ttsSchema.voices.includes(voice) && + !ttsSchema.voices.includes('ALL') + ) { + throw new Error(`Voice ${voice} is not available.`); + } + + const data = { + input, + model: ttsSchema?.voices && ttsSchema.voices.length > 0 ? voice : undefined, + backend: ttsSchema?.backend, + }; + + const headers = { + 'Content-Type': 'application/json', + Authorization: `Bearer ${extractEnvVariable(ttsSchema?.apiKey)}`, + }; + + if (extractEnvVariable(ttsSchema.apiKey) === '') { + delete headers.Authorization; + } + + return [url, data, headers]; + } + + /** + * Sends a TTS request to the specified provider. + * @async + * @param {string} provider - The TTS provider to use. + * @param {Object} ttsSchema - The TTS schema for the provider. + * @param {Object} options - The options for the TTS request. + * @param {string} options.input - The input text. + * @param {string} options.voice - The voice to use. + * @param {boolean} [options.stream=true] - Whether to use streaming. + * @returns {Promise} The axios response object. + * @throws {Error} If the provider is invalid or the request fails. + */ + async ttsRequest(provider, ttsSchema, { input, voice, stream = true }) { + const strategy = this.providerStrategies[provider]; + if (!strategy) { + throw new Error('Invalid provider'); + } + + const [url, data, headers] = strategy.call(this, ttsSchema, input, voice, stream); + + [data, headers].forEach(this.removeUndefined.bind(this)); + + const options = { headers, responseType: stream ? 'stream' : 'arraybuffer' }; + + try { + return await axios.post(url, data, options); + } catch (error) { + logger.error(`TTS request failed for provider ${provider}:`, error); + throw error; + } + } + + /** + * Processes a text-to-speech request. + * @async + * @param {Object} req - The request object. + * @param {Object} res - The response object. + * @returns {Promise} + */ + async processTextToSpeech(req, res) { + const { input, voice: requestVoice } = req.body; + + if (!input) { + return res.status(400).send('Missing text in request body'); + } + + try { + res.setHeader('Content-Type', 'audio/mpeg'); + const provider = this.getProvider(); + const ttsSchema = this.customConfig.speech.tts[provider]; + const voice = await this.getVoice(ttsSchema, requestVoice); + + if (input.length < 4096) { + const response = await this.ttsRequest(provider, ttsSchema, { input, voice }); + response.data.pipe(res); + return; + } + + const textChunks = splitTextIntoChunks(input, 1000); + + for (const chunk of textChunks) { + try { + const response = await this.ttsRequest(provider, ttsSchema, { + voice, + input: chunk.text, + stream: true, + }); + + logger.debug(`[textToSpeech] user: ${req?.user?.id} | writing audio stream`); + await new Promise((resolve) => { + response.data.pipe(res, { end: chunk.isFinished }); + response.data.on('end', resolve); + }); + + if (chunk.isFinished) { + break; + } + } catch (innerError) { + logger.error('Error processing manual update:', chunk, innerError); + if (!res.headersSent) { + return res.status(500).end(); + } + return; + } + } + + if (!res.headersSent) { + res.end(); + } + } catch (error) { + logger.error('Error creating the audio stream:', error); + if (!res.headersSent) { + return res.status(500).send('An error occurred'); + } + } + } + + /** + * Streams audio data from the TTS provider. + * @async + * @param {Object} req - The request object. + * @param {Object} res - The response object. + * @returns {Promise} + */ + async streamAudio(req, res) { + res.setHeader('Content-Type', 'audio/mpeg'); + const provider = this.getProvider(); + const ttsSchema = this.customConfig.speech.tts[provider]; + const voice = await this.getVoice(ttsSchema, req.body.voice); + + let shouldContinue = true; + + req.on('close', () => { + logger.warn('[streamAudio] Audio Stream Request closed by client'); + shouldContinue = false; + }); + + const processChunks = createChunkProcessor(req.body.messageId); + + try { + while (shouldContinue) { + const updates = await processChunks(); + if (typeof updates === 'string') { + logger.error(`Error processing audio stream updates: ${updates}`); + return res.status(500).end(); + } + + if (updates.length === 0) { + await new Promise((resolve) => setTimeout(resolve, 1250)); + continue; + } + + for (const update of updates) { + try { + const response = await this.ttsRequest(provider, ttsSchema, { + voice, + input: update.text, + stream: true, + }); + + if (!shouldContinue) { + break; + } + + logger.debug(`[streamAudio] user: ${req?.user?.id} | writing audio stream`); + await new Promise((resolve) => { + response.data.pipe(res, { end: update.isFinished }); + response.data.on('end', resolve); + }); + + if (update.isFinished) { + shouldContinue = false; + break; + } + } catch (innerError) { + logger.error('Error processing audio stream update:', update, innerError); + if (!res.headersSent) { + return res.status(500).end(); + } + return; + } + } + + if (!shouldContinue) { + break; + } + } + + if (!res.headersSent) { + res.end(); + } + } catch (error) { + logger.error('Failed to fetch audio:', error); + if (!res.headersSent) { + res.status(500).end(); + } + } + } +} + +/** + * Factory function to create a TTSService instance. + * @async + * @returns {Promise} A promise that resolves to a TTSService instance. + */ +async function createTTSService() { + return TTSService.getInstance(); +} + +/** + * Wrapper function for text-to-speech processing. + * @async + * @param {Object} req - The request object. + * @param {Object} res - The response object. + * @returns {Promise} + */ +async function textToSpeech(req, res) { + const ttsService = await createTTSService(); + await ttsService.processTextToSpeech(req, res); +} + +/** + * Wrapper function for audio streaming. + * @async + * @param {Object} req - The request object. + * @param {Object} res - The response object. + * @returns {Promise} + */ +async function streamAudio(req, res) { + const ttsService = await createTTSService(); + await ttsService.streamAudio(req, res); +} + +/** + * Wrapper function to get the configured TTS provider. + * @async + * @returns {Promise} A promise that resolves to the name of the configured provider. + */ +async function getProvider() { + const ttsService = await createTTSService(); + return ttsService.getProvider(); +} + +module.exports = { + textToSpeech, + streamAudio, + getProvider, +}; diff --git a/api/server/services/Files/Audio/getVoices.js b/api/server/services/Files/Audio/getVoices.js index 6a543bb4a4..3a0618b772 100644 --- a/api/server/services/Files/Audio/getVoices.js +++ b/api/server/services/Files/Audio/getVoices.js @@ -1,6 +1,6 @@ const { TTSProviders } = require('librechat-data-provider'); const getCustomConfig = require('~/server/services/Config/getCustomConfig'); -const { getProvider } = require('./textToSpeech'); +const { getProvider } = require('./TTSService'); /** * This function retrieves the available voices for the current TTS provider diff --git a/api/server/services/Files/Audio/index.js b/api/server/services/Files/Audio/index.js index 75882f2397..b698e9da87 100644 --- a/api/server/services/Files/Audio/index.js +++ b/api/server/services/Files/Audio/index.js @@ -1,11 +1,11 @@ const getVoices = require('./getVoices'); const getCustomConfigSpeech = require('./getCustomConfigSpeech'); -const textToSpeech = require('./textToSpeech'); -const speechToText = require('./speechToText'); +const TTSService = require('./TTSService'); +const STTService = require('./STTService'); module.exports = { getVoices, getCustomConfigSpeech, - speechToText, - ...textToSpeech, + ...STTService, + ...TTSService, }; diff --git a/api/server/services/Files/Audio/speechToText.js b/api/server/services/Files/Audio/speechToText.js deleted file mode 100644 index f6b52f8b7c..0000000000 --- a/api/server/services/Files/Audio/speechToText.js +++ /dev/null @@ -1,225 +0,0 @@ -const { Readable } = require('stream'); -const axios = require('axios'); -const { extractEnvVariable, STTProviders } = require('librechat-data-provider'); -const getCustomConfig = require('~/server/services/Config/getCustomConfig'); -const { genAzureEndpoint } = require('~/utils'); -const { logger } = require('~/config'); - -/** - * Handle the response from the STT API - * @param {Object} response - The response from the STT API - * - * @returns {string} The text from the response data - * - * @throws Will throw an error if the response status is not 200 or the response data is missing - */ -async function handleResponse(response) { - if (response.status !== 200) { - throw new Error('Invalid response from the STT API'); - } - - if (!response.data || !response.data.text) { - throw new Error('Missing data in response from the STT API'); - } - - return response.data.text.trim(); -} - -/** - * getProviderSchema function - * This function takes the customConfig object and returns the name of the provider and its schema - * If more than one provider is set or no provider is set, it throws an error - * - * @param {Object} customConfig - The custom configuration containing the STT schema - * @returns {Promise<[string, Object]>} The name of the provider and its schema - * @throws {Error} Throws an error if multiple providers are set or no provider is set - */ -async function getProviderSchema(customConfig) { - const sttSchema = customConfig.speech.stt; - - if (!sttSchema) { - throw new Error(`No STT schema is set. Did you configure STT in the custom config (librechat.yaml)? - - https://www.librechat.ai/docs/configuration/stt_tts#stt`); - } - - const providers = Object.entries(sttSchema).filter(([, value]) => Object.keys(value).length > 0); - - if (providers.length > 1) { - throw new Error('Multiple providers are set. Please set only one provider.'); - } else if (providers.length === 0) { - throw new Error('No provider is set. Please set a provider.'); - } else { - const provider = providers[0][0]; - return [provider, sttSchema[provider]]; - } -} - -function removeUndefined(obj) { - Object.keys(obj).forEach((key) => { - if (obj[key] && typeof obj[key] === 'object') { - removeUndefined(obj[key]); - if (Object.keys(obj[key]).length === 0) { - delete obj[key]; - } - } else if (obj[key] === undefined) { - delete obj[key]; - } - }); -} - -/** - * This function prepares the necessary data and headers for making a request to the OpenAI API - * It uses the provided speech-to-text schema and audio stream to create the request - * - * @param {Object} sttSchema - The speech-to-text schema containing the OpenAI configuration - * @param {Stream} audioReadStream - The audio data to be transcribed - * - * @returns {Array} An array containing the URL for the API request, the data to be sent, and the headers for the request - * If an error occurs, it returns an array with three null values and logs the error with logger - */ -function openAIProvider(sttSchema, audioReadStream) { - try { - const url = sttSchema.openai?.url || 'https://api.openai.com/v1/audio/transcriptions'; - const apiKey = sttSchema.openai.apiKey ? extractEnvVariable(sttSchema.openai.apiKey) : ''; - - let data = { - file: audioReadStream, - model: sttSchema.openai.model, - }; - - let headers = { - 'Content-Type': 'multipart/form-data', - }; - - [headers].forEach(removeUndefined); - - if (apiKey) { - headers.Authorization = 'Bearer ' + apiKey; - } - - return [url, data, headers]; - } catch (error) { - logger.error('An error occurred while preparing the OpenAI API STT request: ', error); - return [null, null, null]; - } -} - -/** - * Prepares the necessary data and headers for making a request to the Azure API. - * It uses the provided Speech-to-Text (STT) schema and audio file to create the request. - * - * @param {Object} sttSchema - The STT schema object, which should contain instanceName, deploymentName, apiVersion, and apiKey. - * @param {Buffer} audioBuffer - The audio data to be transcribed - * @param {Object} audioFile - The audio file object, which should contain originalname, mimetype, and size. - * - * @returns {Array} An array containing the URL for the API request, the data to be sent, and the headers for the request. - * If an error occurs, it logs the error with logger and returns an array with three null values. - */ -function azureOpenAIProvider(sttSchema, audioBuffer, audioFile) { - try { - const instanceName = sttSchema?.instanceName; - const deploymentName = sttSchema?.deploymentName; - const apiVersion = sttSchema?.apiVersion; - - const url = - genAzureEndpoint({ - azureOpenAIApiInstanceName: instanceName, - azureOpenAIApiDeploymentName: deploymentName, - }) + - '/audio/transcriptions?api-version=' + - apiVersion; - - const apiKey = sttSchema.apiKey ? extractEnvVariable(sttSchema.apiKey) : ''; - - if (audioBuffer.byteLength > 25 * 1024 * 1024) { - throw new Error('The audio file size exceeds the limit of 25MB'); - } - const acceptedFormats = ['flac', 'mp3', 'mp4', 'mpeg', 'mpga', 'm4a', 'ogg', 'wav', 'webm']; - const fileFormat = audioFile.mimetype.split('/')[1]; - if (!acceptedFormats.includes(fileFormat)) { - throw new Error(`The audio file format ${fileFormat} is not accepted`); - } - - const formData = new FormData(); - - const audioBlob = new Blob([audioBuffer], { type: audioFile.mimetype }); - - formData.append('file', audioBlob, audioFile.originalname); - - let data = formData; - - let headers = { - 'Content-Type': 'multipart/form-data', - }; - - [headers].forEach(removeUndefined); - - if (apiKey) { - headers['api-key'] = apiKey; - } - - return [url, data, headers]; - } catch (error) { - logger.error('An error occurred while preparing the Azure OpenAI API STT request: ', error); - throw error; - } -} - -/** - * Convert speech to text - * @param {Object} req - The request object - * @param {Object} res - The response object - * - * @returns {Object} The response object with the text from the STT API - * - * @throws Will throw an error if an error occurs while processing the audio - */ - -async function speechToText(req, res) { - const customConfig = await getCustomConfig(); - if (!customConfig) { - return res.status(500).send('Custom config not found'); - } - - if (!req.file || !req.file.buffer) { - return res.status(400).json({ message: 'No audio file provided in the FormData' }); - } - - const audioBuffer = req.file.buffer; - const audioReadStream = Readable.from(audioBuffer); - audioReadStream.path = 'audio.wav'; - - const [provider, sttSchema] = await getProviderSchema(customConfig); - - let [url, data, headers] = []; - - switch (provider) { - case STTProviders.OPENAI: - [url, data, headers] = openAIProvider(sttSchema, audioReadStream); - break; - case STTProviders.AZURE_OPENAI: - [url, data, headers] = azureOpenAIProvider(sttSchema, audioBuffer, req.file); - break; - default: - throw new Error('Invalid provider'); - } - - if (!Readable.from) { - const audioBlob = new Blob([audioBuffer], { type: req.file.mimetype }); - delete data['file']; - data['file'] = audioBlob; - } - - try { - const response = await axios.post(url, data, { headers: headers }); - const text = await handleResponse(response); - - res.json({ text }); - } catch (error) { - logger.error('An error occurred while processing the audio:', error); - res.sendStatus(500); - } -} - -module.exports = speechToText; diff --git a/api/server/services/Files/Audio/textToSpeech.js b/api/server/services/Files/Audio/textToSpeech.js deleted file mode 100644 index 2d04a75236..0000000000 --- a/api/server/services/Files/Audio/textToSpeech.js +++ /dev/null @@ -1,473 +0,0 @@ -const axios = require('axios'); -const { extractEnvVariable, TTSProviders } = require('librechat-data-provider'); -const { logger } = require('~/config'); -const getCustomConfig = require('~/server/services/Config/getCustomConfig'); -const { genAzureEndpoint } = require('~/utils'); -const { getRandomVoiceId, createChunkProcessor, splitTextIntoChunks } = require('./streamAudio'); - -/** - * getProvider function - * This function takes the ttsSchema object and returns the name of the provider - * If more than one provider is set or no provider is set, it throws an error - * - * @param {Object} ttsSchema - The TTS schema containing the provider configuration - * @returns {string} The name of the provider - * @throws {Error} Throws an error if multiple providers are set or no provider is set - */ -function getProvider(ttsSchema) { - if (!ttsSchema) { - throw new Error(`No TTS schema is set. Did you configure TTS in the custom config (librechat.yaml)? - - https://www.librechat.ai/docs/configuration/stt_tts#tts`); - } - const providers = Object.entries(ttsSchema).filter(([, value]) => Object.keys(value).length > 0); - - if (providers.length > 1) { - throw new Error('Multiple providers are set. Please set only one provider.'); - } else if (providers.length === 0) { - throw new Error('No provider is set. Please set a provider.'); - } else { - return providers[0][0]; - } -} - -/** - * removeUndefined function - * This function takes an object and removes all keys with undefined values - * It also removes keys with empty objects as values - * - * @param {Object} obj - The object to be cleaned - * @returns {void} This function does not return a value. It modifies the input object directly - */ -function removeUndefined(obj) { - Object.keys(obj).forEach((key) => { - if (obj[key] && typeof obj[key] === 'object') { - removeUndefined(obj[key]); - if (Object.keys(obj[key]).length === 0) { - delete obj[key]; - } - } else if (obj[key] === undefined) { - delete obj[key]; - } - }); -} - -/** - * This function prepares the necessary data and headers for making a request to the OpenAI TTS - * It uses the provided TTS schema, input text, and voice to create the request - * - * @param {TCustomConfig['tts']['openai']} ttsSchema - The TTS schema containing the OpenAI configuration - * @param {string} input - The text to be converted to speech - * @param {string} voice - The voice to be used for the speech - * - * @returns {Array} An array containing the URL for the API request, the data to be sent, and the headers for the request - * If an error occurs, it throws an error with a message indicating that the selected voice is not available - */ -function openAIProvider(ttsSchema, input, voice) { - const url = ttsSchema?.url || 'https://api.openai.com/v1/audio/speech'; - - if ( - ttsSchema?.voices && - ttsSchema.voices.length > 0 && - !ttsSchema.voices.includes(voice) && - !ttsSchema.voices.includes('ALL') - ) { - throw new Error(`Voice ${voice} is not available.`); - } - - let data = { - input, - model: ttsSchema?.model, - voice: ttsSchema?.voices && ttsSchema.voices.length > 0 ? voice : undefined, - backend: ttsSchema?.backend, - }; - - let headers = { - 'Content-Type': 'application/json', - Authorization: 'Bearer ' + extractEnvVariable(ttsSchema?.apiKey), - }; - - [data, headers].forEach(removeUndefined); - - return [url, data, headers]; -} - -/** - * Generates the necessary parameters for making a request to Azure's OpenAI Text-to-Speech API. - * - * @param {TCustomConfig['tts']['azureOpenAI']} ttsSchema - The TTS schema containing the AzureOpenAI configuration - * @param {string} input - The text to be converted to speech - * @param {string} voice - The voice to be used for the speech - * - * @returns {Array} An array containing the URL for the API request, the data to be sent, and the headers for the request - * If an error occurs, it throws an error with a message indicating that the selected voice is not available - */ -function azureOpenAIProvider(ttsSchema, input, voice) { - const instanceName = ttsSchema?.instanceName; - const deploymentName = ttsSchema?.deploymentName; - const apiVersion = ttsSchema?.apiVersion; - - const url = - genAzureEndpoint({ - azureOpenAIApiInstanceName: instanceName, - azureOpenAIApiDeploymentName: deploymentName, - }) + - '/audio/speech?api-version=' + - apiVersion; - - const apiKey = ttsSchema.apiKey ? extractEnvVariable(ttsSchema.apiKey) : ''; - - if ( - ttsSchema?.voices && - ttsSchema.voices.length > 0 && - !ttsSchema.voices.includes(voice) && - !ttsSchema.voices.includes('ALL') - ) { - throw new Error(`Voice ${voice} is not available.`); - } - - let data = { - model: ttsSchema?.model, - input, - voice: ttsSchema?.voices && ttsSchema.voices.length > 0 ? voice : undefined, - }; - - let headers = { - 'Content-Type': 'application/json', - }; - - [data, headers].forEach(removeUndefined); - - if (apiKey) { - headers['api-key'] = apiKey; - } - - return [url, data, headers]; -} - -/** - * elevenLabsProvider function - * This function prepares the necessary data and headers for making a request to the Eleven Labs TTS - * It uses the provided TTS schema, input text, and voice to create the request - * - * @param {TCustomConfig['tts']['elevenLabs']} ttsSchema - The TTS schema containing the Eleven Labs configuration - * @param {string} input - The text to be converted to speech - * @param {string} voice - The voice to be used for the speech - * @param {boolean} stream - Whether to stream the audio or not - * - * @returns {Array} An array containing the URL for the API request, the data to be sent, and the headers for the request - * @throws {Error} Throws an error if the selected voice is not available - */ -function elevenLabsProvider(ttsSchema, input, voice, stream) { - let url = - ttsSchema?.url || - `https://api.elevenlabs.io/v1/text-to-speech/{voice_id}${stream ? '/stream' : ''}`; - - if (!ttsSchema?.voices.includes(voice) && !ttsSchema?.voices.includes('ALL')) { - throw new Error(`Voice ${voice} is not available.`); - } - - url = url.replace('{voice_id}', voice); - - let data = { - model_id: ttsSchema?.model, - text: input, - // voice_id: voice, - voice_settings: { - similarity_boost: ttsSchema?.voice_settings?.similarity_boost, - stability: ttsSchema?.voice_settings?.stability, - style: ttsSchema?.voice_settings?.style, - use_speaker_boost: ttsSchema?.voice_settings?.use_speaker_boost || undefined, - }, - pronunciation_dictionary_locators: ttsSchema?.pronunciation_dictionary_locators, - }; - - let headers = { - 'Content-Type': 'application/json', - 'xi-api-key': extractEnvVariable(ttsSchema?.apiKey), - Accept: 'audio/mpeg', - }; - - [data, headers].forEach(removeUndefined); - - return [url, data, headers]; -} - -/** - * localAIProvider function - * This function prepares the necessary data and headers for making a request to the LocalAI TTS - * It uses the provided TTS schema, input text, and voice to create the request - * - * @param {TCustomConfig['tts']['localai']} ttsSchema - The TTS schema containing the LocalAI configuration - * @param {string} input - The text to be converted to speech - * @param {string} voice - The voice to be used for the speech - * - * @returns {Array} An array containing the URL for the API request, the data to be sent, and the headers for the request - * @throws {Error} Throws an error if the selected voice is not available - */ -function localAIProvider(ttsSchema, input, voice) { - let url = ttsSchema?.url; - - if ( - ttsSchema?.voices && - ttsSchema.voices.length > 0 && - !ttsSchema.voices.includes(voice) && - !ttsSchema.voices.includes('ALL') - ) { - throw new Error(`Voice ${voice} is not available.`); - } - - let data = { - input, - model: ttsSchema?.voices && ttsSchema.voices.length > 0 ? voice : undefined, - backend: ttsSchema?.backend, - }; - - let headers = { - 'Content-Type': 'application/json', - Authorization: 'Bearer ' + extractEnvVariable(ttsSchema?.apiKey), - }; - - [data, headers].forEach(removeUndefined); - - if (extractEnvVariable(ttsSchema.apiKey) === '') { - delete headers.Authorization; - } - - return [url, data, headers]; -} - -/** - * - * Returns provider and its schema for use with TTS requests - * @param {TCustomConfig} customConfig - * @param {string} _voice - * @returns {Promise<[string, TProviderSchema]>} - */ -async function getProviderSchema(customConfig) { - const provider = getProvider(customConfig.speech.tts); - return [provider, customConfig.speech.tts[provider]]; -} - -/** - * - * Returns a tuple of the TTS schema as well as the voice for the TTS request - * @param {TProviderSchema} providerSchema - * @param {string} requestVoice - * @returns {Promise} - */ -async function getVoice(providerSchema, requestVoice) { - const voices = providerSchema.voices.filter((voice) => voice && voice.toUpperCase() !== 'ALL'); - let voice = requestVoice; - if (!voice || !voices.includes(voice) || (voice.toUpperCase() === 'ALL' && voices.length > 1)) { - voice = getRandomVoiceId(voices); - } - - return voice; -} - -/** - * - * @param {string} provider - * @param {TProviderSchema} ttsSchema - * @param {object} params - * @param {string} params.voice - * @param {string} params.input - * @param {boolean} [params.stream] - * @returns {Promise} - */ -async function ttsRequest(provider, ttsSchema, { input, voice, stream = true } = { stream: true }) { - let [url, data, headers] = []; - switch (provider) { - case TTSProviders.OPENAI: - [url, data, headers] = openAIProvider(ttsSchema, input, voice); - break; - case TTSProviders.AZURE_OPENAI: - [url, data, headers] = azureOpenAIProvider(ttsSchema, input, voice); - break; - case TTSProviders.ELEVENLABS: - [url, data, headers] = elevenLabsProvider(ttsSchema, input, voice, stream); - break; - case TTSProviders.LOCALAI: - [url, data, headers] = localAIProvider(ttsSchema, input, voice); - break; - default: - throw new Error('Invalid provider'); - } - - if (stream) { - return await axios.post(url, data, { headers, responseType: 'stream' }); - } - - return await axios.post(url, data, { headers, responseType: 'arraybuffer' }); -} - -/** - * Handles a text-to-speech request. Extracts input and voice from the request, retrieves the TTS configuration, - * and sends a request to the appropriate provider. The resulting audio data is sent in the response - * - * @param {Object} req - The request object, which should contain the input text and voice in its body - * @param {Object} res - The response object, used to send the audio data or an error message - * - * @returns {Promise} This function does not return a value. It sends the audio data or an error message in the response - * - * @throws {Error} Throws an error if the provider is invalid - */ -async function textToSpeech(req, res) { - const { input } = req.body; - - if (!input) { - return res.status(400).send('Missing text in request body'); - } - - const customConfig = await getCustomConfig(); - if (!customConfig) { - res.status(500).send('Custom config not found'); - } - - try { - res.setHeader('Content-Type', 'audio/mpeg'); - const [provider, ttsSchema] = await getProviderSchema(customConfig); - const voice = await getVoice(ttsSchema, req.body.voice); - if (input.length < 4096) { - const response = await ttsRequest(provider, ttsSchema, { input, voice }); - response.data.pipe(res); - return; - } - - const textChunks = splitTextIntoChunks(input, 1000); - - for (const chunk of textChunks) { - try { - const response = await ttsRequest(provider, ttsSchema, { - voice, - input: chunk.text, - stream: true, - }); - - logger.debug(`[textToSpeech] user: ${req?.user?.id} | writing audio stream`); - await new Promise((resolve) => { - response.data.pipe(res, { end: chunk.isFinished }); - response.data.on('end', () => { - resolve(); - }); - }); - - if (chunk.isFinished) { - break; - } - } catch (innerError) { - logger.error('Error processing manual update:', chunk, innerError); - if (!res.headersSent) { - res.status(500).end(); - } - return; - } - } - - if (!res.headersSent) { - res.end(); - } - } catch (error) { - logger.error( - 'Error creating the audio stream. Suggestion: check your provider quota. Error:', - error, - ); - res.status(500).send('An error occurred'); - } -} - -async function streamAudio(req, res) { - res.setHeader('Content-Type', 'audio/mpeg'); - const customConfig = await getCustomConfig(); - if (!customConfig) { - return res.status(500).send('Custom config not found'); - } - - const [provider, ttsSchema] = await getProviderSchema(customConfig); - const voice = await getVoice(ttsSchema, req.body.voice); - - try { - let shouldContinue = true; - - req.on('close', () => { - logger.warn('[streamAudio] Audio Stream Request closed by client'); - shouldContinue = false; - }); - - const processChunks = createChunkProcessor(req.body.messageId); - - while (shouldContinue) { - // example updates - // const updates = [ - // { text: 'This is a test.', isFinished: false }, - // { text: 'This is only a test.', isFinished: false }, - // { text: 'Your voice is like a combination of Fergie and Jesus!', isFinished: true }, - // ]; - - const updates = await processChunks(); - if (typeof updates === 'string') { - logger.error(`Error processing audio stream updates: ${JSON.stringify(updates)}`); - res.status(500).end(); - return; - } - - if (updates.length === 0) { - await new Promise((resolve) => setTimeout(resolve, 1250)); - continue; - } - - for (const update of updates) { - try { - const response = await ttsRequest(provider, ttsSchema, { - voice, - input: update.text, - stream: true, - }); - - if (!shouldContinue) { - break; - } - - logger.debug(`[streamAudio] user: ${req?.user?.id} | writing audio stream`); - await new Promise((resolve) => { - response.data.pipe(res, { end: update.isFinished }); - response.data.on('end', () => { - resolve(); - }); - }); - - if (update.isFinished) { - shouldContinue = false; - break; - } - } catch (innerError) { - logger.error('Error processing audio stream update:', update, innerError); - if (!res.headersSent) { - res.status(500).end(); - } - return; - } - } - - if (!shouldContinue) { - break; - } - } - - if (!res.headersSent) { - res.end(); - } - } catch (error) { - logger.error('Failed to fetch audio:', error); - if (!res.headersSent) { - res.status(500).end(); - } - } -} - -module.exports = { - textToSpeech, - getProvider, - streamAudio, -}; diff --git a/client/src/components/Nav/SettingsTabs/Speech/STT/DecibelSelector.tsx b/client/src/components/Nav/SettingsTabs/Speech/STT/DecibelSelector.tsx index 30ff66dc0d..12836775e5 100755 --- a/client/src/components/Nav/SettingsTabs/Speech/STT/DecibelSelector.tsx +++ b/client/src/components/Nav/SettingsTabs/Speech/STT/DecibelSelector.tsx @@ -15,7 +15,7 @@ export default function DecibelSelector() {
{localize('com_nav_db_sensitivity')}
- ({localize('com_endpoint_default_with_num', '0.45')}) + ({localize('com_endpoint_default_with_num', '-45')})
void const cleanup = () => { if (mediaRecorderRef.current) { - mediaRecorderRef.current.removeEventListener('dataavailable', handleDataAvailable); + mediaRecorderRef.current.removeEventListener('dataavailable', (event: BlobEvent) => { + audioChunks.push(event.data); + }); mediaRecorderRef.current.removeEventListener('stop', handleStop); mediaRecorderRef.current = null; } @@ -68,14 +70,6 @@ const useSpeechToTextExternal = (onTranscriptionComplete: (text: string) => void } }; - const handleDataAvailable = (event: BlobEvent) => { - if (event.data.size > 0) { - audioChunks.push(event.data); - } else { - showToast({ message: 'No audio data available', status: 'warning' }); - } - }; - const handleStop = () => { if (audioChunks.length > 0) { const audioBlob = new Blob(audioChunks, { type: 'audio/wav' }); @@ -139,7 +133,9 @@ const useSpeechToTextExternal = (onTranscriptionComplete: (text: string) => void try { setAudioChunks([]); mediaRecorderRef.current = new MediaRecorder(audioStream.current); - mediaRecorderRef.current.addEventListener('dataavailable', handleDataAvailable); + mediaRecorderRef.current.addEventListener('dataavailable', (event: BlobEvent) => { + audioChunks.push(event.data); + }); mediaRecorderRef.current.addEventListener('stop', handleStop); mediaRecorderRef.current.start(100); if (!audioContextRef.current && autoTranscribeAudio && speechToText) {