✨ WIP: Implement Realtime Ephemeral Token functionality and update related components

api/server/routes/files/speech/customConfigSpeech.js

@@ -3,7 +3,7 @@ const router = express.Router();
 
 const { getCustomConfigSpeech } = require('~/server/services/Files/Audio');
 
-router.get('/get', async (req, res) => {
+router.get('/', async (req, res) => {
   await getCustomConfigSpeech(req, res);
 });
 

api/server/routes/files/speech/index.js

@@ -4,6 +4,7 @@ const { createTTSLimiters, createSTTLimiters } = require('~/server/middleware');
 const stt = require('./stt');
 const tts = require('./tts');
 const customConfigSpeech = require('./customConfigSpeech');
+const realtime = require('./realtime');
 
 const router = express.Router();
 
@@ -14,4 +15,6 @@ router.use('/tts', ttsIpLimiter, ttsUserLimiter, tts);
 
 router.use('/config', customConfigSpeech);
 
+router.use('/realtime', realtime);
+
 module.exports = router;

api/server/routes/files/speech/realtime.js

@@ -0,0 +1,10 @@
+const express = require('express');
+const router = express.Router();
+
+const { getRealtimeConfig } = require('~/server/services/Files/Audio');
+
+router.get('/', async (req, res) => {
+  await getRealtimeConfig(req, res);
+});
+
+module.exports = router;

api/server/services/Files/Audio/getRealtimeConfig.js

@@ -0,0 +1,102 @@
+const { extractEnvVariable, RealtimeVoiceProviders } = require('librechat-data-provider');
+const { getCustomConfig } = require('~/server/services/Config');
+const { logger } = require('~/config');
+
+class RealtimeService {
+  constructor(customConfig) {
+    this.customConfig = customConfig;
+    this.providerStrategies = {
+      [RealtimeVoiceProviders.OPENAI]: this.openaiProvider.bind(this),
+    };
+  }
+
+  static async getInstance() {
+    const customConfig = await getCustomConfig();
+    if (!customConfig) {
+      throw new Error('Custom config not found');
+    }
+    return new RealtimeService(customConfig);
+  }
+
+  async getProviderSchema() {
+    const realtimeSchema = this.customConfig.speech.realtime;
+    if (!realtimeSchema) {
+      throw new Error('No Realtime schema is set in config');
+    }
+
+    const providers = Object.entries(realtimeSchema).filter(
+      ([, value]) => Object.keys(value).length > 0,
+    );
+
+    if (providers.length !== 1) {
+      throw new Error(providers.length > 1 ? 'Multiple providers set' : 'No provider set');
+    }
+
+    return providers[0];
+  }
+
+  async openaiProvider(schema, voice) {
+    const defaultRealtimeUrl = 'https://api.openai.com/v1/realtime';
+    const allowedVoices = ['alloy', 'ash', 'ballad', 'coral', 'echo', 'sage', 'shimmer', 'verse'];
+
+    if (!voice) {
+      throw new Error('Voice not specified');
+    }
+
+    if (!allowedVoices.includes(voice)) {
+      throw new Error(`Invalid voice: ${voice}`);
+    }
+
+    const apiKey = extractEnvVariable(schema.apiKey);
+    if (!apiKey) {
+      throw new Error('OpenAI API key not configured');
+    }
+
+    const response = await fetch('https://api.openai.com/v1/realtime/sessions', {
+      method: 'POST',
+      headers: {
+        Authorization: `Bearer ${apiKey}`,
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        model: 'gpt-4o-realtime-preview-2024-12-17',
+        modalities: ['audio', 'text'],
+        voice: voice,
+      }),
+    });
+
+    const token = response.json();
+
+    return {
+      provider: RealtimeVoiceProviders.OPENAI,
+      token: token,
+      url: schema.url || defaultRealtimeUrl,
+    };
+  }
+
+  async getRealtimeConfig(req, res) {
+    try {
+      const [provider, schema] = await this.getProviderSchema();
+      const strategy = this.providerStrategies[provider];
+
+      if (!strategy) {
+        throw new Error(`Unsupported provider: ${provider}`);
+      }
+
+      const voice = req.query.voice;
+
+      const config = strategy(schema, voice);
+      res.json(config);
+    } catch (error) {
+      logger.error('[RealtimeService] Config generation failed:', error);
+      res.status(500).json({ error: error.message });
+    }
+  }
+}
+
+async function getRealtimeConfig(req, res) {
+  const service = await RealtimeService.getInstance();
+  await service.getRealtimeConfig(req, res);
+}
+
+module.exports = getRealtimeConfig;

api/server/services/Files/Audio/index.js

@@ -1,4 +1,5 @@
 const getCustomConfigSpeech = require('./getCustomConfigSpeech');
+const getRealtimeConfig = require('./getRealtimeConfig');
 const TTSService = require('./TTSService');
 const STTService = require('./STTService');
 const getVoices = require('./getVoices');
@@ -6,6 +7,7 @@ const getVoices = require('./getVoices');
 module.exports = {
   getVoices,
   getCustomConfigSpeech,
+  getRealtimeConfig,
   ...STTService,
   ...TTSService,
 };