mirror of
https://github.com/danny-avila/LibreChat.git
synced 2025-12-19 09:50:15 +01:00
✨ fix: both webrtc-client and webrtc-server
This commit is contained in:
Marco Beretta
Danny Avila
parent
9c0c341dee
commit
964d47cfa3
8 changed files with 948 additions and 390 deletions
|
|
@ -1,157 +1,205 @@
|
|||
const { Server } = require('socket.io');
|
||||
const { RTCPeerConnection } = require('wrtc');
|
||||
const { RTCPeerConnection, RTCIceCandidate } = require('wrtc');
|
||||
|
||||
module.exports.SocketIOService = class {
|
||||
constructor(httpServer) {
|
||||
this.io = new Server(httpServer, { path: '/socket.io' });
|
||||
this.log('Socket.IO Server initialized');
|
||||
this.activeClients = new Map();
|
||||
this.iceServers = [
|
||||
{ urls: 'stun:stun.l.google.com:19302' },
|
||||
{ urls: 'stun:stun1.l.google.com:19302' },
|
||||
];
|
||||
this.setupHandlers();
|
||||
class WebRTCConnection {
|
||||
constructor(socket, config) {
|
||||
this.socket = socket;
|
||||
this.config = config;
|
||||
this.peerConnection = null;
|
||||
this.audioTransceiver = null;
|
||||
this.pendingCandidates = [];
|
||||
this.state = 'idle';
|
||||
this.log = config.log || console.log;
|
||||
}
|
||||
|
||||
log(msg) {
|
||||
console.log(`[Socket.IO ${new Date().toISOString()}] ${msg}`);
|
||||
}
|
||||
async handleOffer(offer) {
|
||||
try {
|
||||
// Create new peer connection if needed
|
||||
if (!this.peerConnection) {
|
||||
this.peerConnection = new RTCPeerConnection(this.config.rtcConfig);
|
||||
this.setupPeerConnectionListeners();
|
||||
}
|
||||
|
||||
setupHandlers() {
|
||||
this.io.on('connection', (socket) => {
|
||||
const clientId = socket.id;
|
||||
this.activeClients.set(clientId, {
|
||||
socket,
|
||||
state: 'idle',
|
||||
audioBuffer: [],
|
||||
currentTranscription: '',
|
||||
isProcessing: false,
|
||||
// Set the remote description (client's offer)
|
||||
await this.peerConnection.setRemoteDescription(offer);
|
||||
|
||||
// Set up audio transceiver for two-way audio
|
||||
this.audioTransceiver = this.peerConnection.addTransceiver('audio', {
|
||||
direction: 'sendrecv',
|
||||
});
|
||||
|
||||
this.log(`Client connected: ${clientId}`);
|
||||
// Create and set local description (answer)
|
||||
const answer = await this.peerConnection.createAnswer();
|
||||
await this.peerConnection.setLocalDescription(answer);
|
||||
|
||||
socket.on('call-start', () => this.handleCallStart(clientId));
|
||||
socket.on('audio-chunk', (data) => this.handleAudioChunk(clientId, data));
|
||||
socket.on('processing-start', () => this.processAudioStream(clientId));
|
||||
socket.on('audio-received', () => this.confirmAudioReceived(clientId));
|
||||
socket.on('call-ended', () => this.handleCallEnd(clientId));
|
||||
// Send answer to client
|
||||
this.socket.emit('webrtc-answer', answer);
|
||||
|
||||
// Process any pending ICE candidates
|
||||
while (this.pendingCandidates.length) {
|
||||
const candidate = this.pendingCandidates.shift();
|
||||
await this.addIceCandidate(candidate);
|
||||
}
|
||||
|
||||
this.state = 'connecting';
|
||||
} catch (error) {
|
||||
this.log(`Error handling offer: ${error}`, 'error');
|
||||
this.socket.emit('error', { message: 'Failed to process offer' });
|
||||
this.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
setupPeerConnectionListeners() {
|
||||
if (!this.peerConnection) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Handle incoming audio tracks
|
||||
this.peerConnection.ontrack = ({ track, streams }) => {
|
||||
this.log(`Received ${track.kind} track from client`);
|
||||
|
||||
// For testing: Echo the audio back after a delay
|
||||
if (track.kind === 'audio') {
|
||||
this.handleIncomingAudio(track, streams[0]);
|
||||
}
|
||||
|
||||
track.onended = () => {
|
||||
this.log(`${track.kind} track ended`);
|
||||
};
|
||||
};
|
||||
|
||||
this.peerConnection.onicecandidate = ({ candidate }) => {
|
||||
if (candidate) {
|
||||
this.socket.emit('icecandidate', candidate);
|
||||
}
|
||||
};
|
||||
|
||||
this.peerConnection.onconnectionstatechange = () => {
|
||||
if (!this.peerConnection) {
|
||||
return;
|
||||
}
|
||||
const state = this.peerConnection.connectionState;
|
||||
this.log(`Connection state changed to ${state}`);
|
||||
this.state = state;
|
||||
if (state === 'failed' || state === 'closed') {
|
||||
this.cleanup();
|
||||
}
|
||||
};
|
||||
|
||||
this.peerConnection.oniceconnectionstatechange = () => {
|
||||
if (this.peerConnection) {
|
||||
this.log(`ICE connection state: ${this.peerConnection.iceConnectionState}`);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
handleIncomingAudio(inputTrack) {
|
||||
// For testing: Echo back the input track directly
|
||||
this.peerConnection.addTrack(inputTrack);
|
||||
|
||||
// Log the track info for debugging
|
||||
this.log(`Audio track added: ${inputTrack.id}, enabled: ${inputTrack.enabled}`);
|
||||
}
|
||||
|
||||
async addIceCandidate(candidate) {
|
||||
try {
|
||||
if (this.peerConnection?.remoteDescription) {
|
||||
if (candidate && candidate.candidate) {
|
||||
await this.peerConnection.addIceCandidate(new RTCIceCandidate(candidate));
|
||||
} else {
|
||||
this.log('Invalid ICE candidate', 'warn');
|
||||
}
|
||||
} else {
|
||||
this.pendingCandidates.push(candidate);
|
||||
}
|
||||
} catch (error) {
|
||||
this.log(`Error adding ICE candidate: ${error}`, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
if (this.peerConnection) {
|
||||
try {
|
||||
this.peerConnection.close();
|
||||
} catch (error) {
|
||||
this.log(`Error closing peer connection: ${error}`, 'error');
|
||||
}
|
||||
this.peerConnection = null;
|
||||
}
|
||||
this.audioTransceiver = null;
|
||||
this.pendingCandidates = [];
|
||||
this.state = 'idle';
|
||||
}
|
||||
}
|
||||
|
||||
class SocketIOService {
|
||||
constructor(httpServer, config = {}) {
|
||||
this.config = {
|
||||
rtcConfig: {
|
||||
iceServers: [
|
||||
{
|
||||
urls: ['stun:stun.l.google.com:19302', 'stun:stun1.l.google.com:19302'],
|
||||
},
|
||||
],
|
||||
iceCandidatePoolSize: 10,
|
||||
bundlePolicy: 'max-bundle',
|
||||
rtcpMuxPolicy: 'require',
|
||||
},
|
||||
...config,
|
||||
};
|
||||
|
||||
this.io = new Server(httpServer, {
|
||||
path: '/socket.io',
|
||||
cors: {
|
||||
origin: '*',
|
||||
methods: ['GET', 'POST'],
|
||||
},
|
||||
});
|
||||
|
||||
this.connections = new Map();
|
||||
this.setupSocketHandlers();
|
||||
}
|
||||
|
||||
log(message, level = 'info') {
|
||||
const timestamp = new Date().toISOString();
|
||||
console.log(`[WebRTC ${timestamp}] [${level.toUpperCase()}] ${message}`);
|
||||
}
|
||||
|
||||
setupSocketHandlers() {
|
||||
this.io.on('connection', (socket) => {
|
||||
this.log(`Client connected: ${socket.id}`);
|
||||
|
||||
// Create a new WebRTC connection for this socket
|
||||
const rtcConnection = new WebRTCConnection(socket, {
|
||||
...this.config,
|
||||
log: this.log.bind(this),
|
||||
});
|
||||
this.connections.set(socket.id, rtcConnection);
|
||||
|
||||
socket.on('webrtc-offer', (offer) => {
|
||||
this.log(`Received WebRTC offer from ${socket.id}`);
|
||||
rtcConnection.handleOffer(offer);
|
||||
});
|
||||
|
||||
socket.on('icecandidate', (candidate) => {
|
||||
rtcConnection.addIceCandidate(candidate);
|
||||
});
|
||||
|
||||
socket.on('disconnect', () => {
|
||||
this.handleCallEnd(clientId);
|
||||
this.activeClients.delete(clientId);
|
||||
this.log(`Client disconnected: ${clientId}`);
|
||||
});
|
||||
|
||||
socket.on('error', (error) => {
|
||||
this.log(`Error for client ${clientId}: ${error.message}`);
|
||||
this.handleCallEnd(clientId);
|
||||
this.log(`Client disconnected: ${socket.id}`);
|
||||
rtcConnection.cleanup();
|
||||
this.connections.delete(socket.id);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async handleCallStart(clientId) {
|
||||
const client = this.activeClients.get(clientId);
|
||||
if (!client) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
client.state = 'active';
|
||||
client.audioBuffer = [];
|
||||
client.currentTranscription = '';
|
||||
client.isProcessing = false;
|
||||
|
||||
const peerConnection = new RTCPeerConnection({
|
||||
iceServers: this.iceServers,
|
||||
sdpSemantics: 'unified-plan',
|
||||
});
|
||||
|
||||
client.peerConnection = peerConnection;
|
||||
client.dataChannel = peerConnection.createDataChannel('audio', {
|
||||
ordered: true,
|
||||
maxRetransmits: 3,
|
||||
});
|
||||
|
||||
client.dataChannel.onopen = () => this.log(`Data channel opened for ${clientId}`);
|
||||
client.dataChannel.onmessage = async (event) => {
|
||||
await this.handleAudioChunk(clientId, event.data);
|
||||
};
|
||||
|
||||
peerConnection.onicecandidate = (event) => {
|
||||
if (event.candidate) {
|
||||
client.socket.emit('ice-candidate', { candidate: event.candidate });
|
||||
}
|
||||
};
|
||||
|
||||
peerConnection.onnegotiationneeded = async () => {
|
||||
try {
|
||||
const offer = await peerConnection.createOffer();
|
||||
await peerConnection.setLocalDescription(offer);
|
||||
client.socket.emit('webrtc-offer', { sdp: peerConnection.localDescription });
|
||||
} catch (error) {
|
||||
this.log(`Negotiation failed for ${clientId}: ${error}`);
|
||||
}
|
||||
};
|
||||
|
||||
this.log(`Call started for client ${clientId}`);
|
||||
} catch (error) {
|
||||
this.log(`Error starting call for ${clientId}: ${error.message}`);
|
||||
this.handleCallEnd(clientId);
|
||||
shutdown() {
|
||||
for (const connection of this.connections.values()) {
|
||||
connection.cleanup();
|
||||
}
|
||||
this.connections.clear();
|
||||
this.io.close();
|
||||
}
|
||||
}
|
||||
|
||||
async handleAudioChunk(clientId, data) {
|
||||
const client = this.activeClients.get(clientId);
|
||||
if (!client || client.state !== 'active') {
|
||||
return;
|
||||
}
|
||||
|
||||
client.audioBuffer.push(data);
|
||||
client.socket.emit('audio-received');
|
||||
}
|
||||
|
||||
async processAudioStream(clientId) {
|
||||
const client = this.activeClients.get(clientId);
|
||||
if (!client || client.state !== 'active' || client.isProcessing) {
|
||||
return;
|
||||
}
|
||||
|
||||
client.isProcessing = true;
|
||||
|
||||
try {
|
||||
// Process transcription
|
||||
client.socket.emit('transcription', { data: 'Processing audio...' });
|
||||
|
||||
// Stream LLM response
|
||||
client.socket.emit('llm-response', { data: 'Processing response...' });
|
||||
|
||||
// Stream TTS chunks
|
||||
client.socket.emit('tts-chunk', { data: 'audio_data_here' });
|
||||
} catch (error) {
|
||||
this.log(`Processing error for client ${clientId}: ${error.message}`);
|
||||
} finally {
|
||||
client.isProcessing = false;
|
||||
client.audioBuffer = [];
|
||||
}
|
||||
}
|
||||
|
||||
confirmAudioReceived(clientId) {
|
||||
const client = this.activeClients.get(clientId);
|
||||
if (!client) {
|
||||
return;
|
||||
}
|
||||
|
||||
client.socket.emit('audio-received', { data: null });
|
||||
}
|
||||
|
||||
handleCallEnd(clientId) {
|
||||
const client = this.activeClients.get(clientId);
|
||||
if (!client) {
|
||||
return;
|
||||
}
|
||||
|
||||
client.state = 'idle';
|
||||
client.audioBuffer = [];
|
||||
client.currentTranscription = '';
|
||||
}
|
||||
};
|
||||
module.exports = { SocketIOService };
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue