From 7aed891838b920079723d359948ab20f242a4372 Mon Sep 17 00:00:00 2001
From: Marco Beretta <81851188+berry-13@users.noreply.github.com>
Date: Fri, 3 Jan 2025 23:14:18 +0100
Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20feat:=20stream=20back=20audio=20to?=
 =?UTF-8?q?=20user=20(test)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 api/package.json                              |  1 -
 .../services/WebSocket/WebSocketServer.js     | 27 +++----
 client/src/components/Chat/Input/Call.tsx     | 38 +++++++++-
 client/src/hooks/useCall.ts                   | 60 ++++++++++++++-
 client/src/services/WebRTC/WebRTCService.ts   | 73 ++++++++++++-------
 package-lock.json                             |  1 -
 6 files changed, 151 insertions(+), 49 deletions(-)

diff --git a/api/package.json b/api/package.json
index 8fc6b73384..d2021dc7d6 100644
--- a/api/package.json
+++ b/api/package.json
@@ -82,7 +82,6 @@
     "mongoose": "^8.9.5",
     "multer": "^1.4.5-lts.1",
     "nanoid": "^3.3.7",
-    "node-pre-gyp": "^0.17.0",
     "nodemailer": "^6.9.15",
     "ollama": "^0.5.0",
     "openai": "^4.47.1",
diff --git a/api/server/services/WebSocket/WebSocketServer.js b/api/server/services/WebSocket/WebSocketServer.js
index b5339377c2..268073bb7c 100644
--- a/api/server/services/WebSocket/WebSocketServer.js
+++ b/api/server/services/WebSocket/WebSocketServer.js
@@ -1,5 +1,5 @@
 const { Server } = require('socket.io');
-const { RTCPeerConnection, RTCIceCandidate } = require('wrtc');
+const { RTCPeerConnection, RTCIceCandidate, MediaStream } = require('wrtc');
 
 class WebRTCConnection {
   constructor(socket, config) {
@@ -14,38 +14,31 @@ class WebRTCConnection {
 
   async handleOffer(offer) {
     try {
-      // Create new peer connection if needed
       if (!this.peerConnection) {
         this.peerConnection = new RTCPeerConnection(this.config.rtcConfig);
         this.setupPeerConnectionListeners();
       }
 
-      // Set the remote description (client's offer)
       await this.peerConnection.setRemoteDescription(offer);
 
-      // Set up audio transceiver for two-way audio
+      // Create MediaStream instance properly
+      const mediaStream = new MediaStream();
+
       this.audioTransceiver = this.peerConnection.addTransceiver('audio', {
         direction: 'sendrecv',
+        streams: [mediaStream],
       });
 
-      // Create and set local description (answer)
       const answer = await this.peerConnection.createAnswer();
       await this.peerConnection.setLocalDescription(answer);
-
-      // Send answer to client
       this.socket.emit('webrtc-answer', answer);
-
-      // Process any pending ICE candidates
-      while (this.pendingCandidates.length) {
-        const candidate = this.pendingCandidates.shift();
-        await this.addIceCandidate(candidate);
-      }
-
-      this.state = 'connecting';
     } catch (error) {
       this.log(`Error handling offer: ${error}`, 'error');
-      this.socket.emit('error', { message: 'Failed to process offer' });
-      this.cleanup();
+      // Don't throw, handle gracefully
+      this.socket.emit('webrtc-error', {
+        message: error.message,
+        code: 'OFFER_ERROR',
+      });
     }
   }
 
diff --git a/client/src/components/Chat/Input/Call.tsx b/client/src/components/Chat/Input/Call.tsx
index 15021a6801..c60900a68a 100644
--- a/client/src/components/Chat/Input/Call.tsx
+++ b/client/src/components/Chat/Input/Call.tsx
@@ -43,6 +43,8 @@ export const Call: React.FC = () => {
   useEffect(() => {
     if (remoteAudioRef.current && remoteStream) {
       remoteAudioRef.current.srcObject = remoteStream;
+
+      remoteAudioRef.current.play().catch((err) => console.error('Error playing audio:', err));
     }
   }, [remoteStream]);
 
@@ -98,6 +100,36 @@ export const Call: React.FC = () => {
   const isActive = callState === CallState.ACTIVE;
   const isError = callState === CallState.ERROR;
 
+  // TESTS
+
+  useEffect(() => {
+    if (remoteAudioRef.current && remoteStream) {
+      console.log('Setting up remote audio:', {
+        tracks: remoteStream.getTracks().length,
+        active: remoteStream.active,
+      });
+
+      remoteAudioRef.current.srcObject = remoteStream;
+      remoteAudioRef.current.muted = false;
+      remoteAudioRef.current.volume = 1.0;
+
+      const playPromise = remoteAudioRef.current.play();
+      if (playPromise) {
+        playPromise.catch((err) => {
+          console.error('Error playing audio:', err);
+          // Retry play on user interaction
+          document.addEventListener(
+            'click',
+            () => {
+              remoteAudioRef.current?.play();
+            },
+            { once: true },
+          );
+        });
+      }
+    }
+  }, [remoteStream]);
+
   return (
     <OGDialog open={open} onOpenChange={setOpen}>
       <OGDialogContent className="w-[28rem] p-8">
@@ -200,7 +232,11 @@ export const Call: React.FC = () => {
           </div>
 
           {/* Hidden Audio Element */}
-          <audio ref={remoteAudioRef} autoPlay>
+          <audio
+            ref={remoteAudioRef}
+            autoPlay
+            playsInline
+          >
             <track kind="captions" />
           </audio>
         </div>
diff --git a/client/src/hooks/useCall.ts b/client/src/hooks/useCall.ts
index 713c79be2a..abcfbf9d45 100644
--- a/client/src/hooks/useCall.ts
+++ b/client/src/hooks/useCall.ts
@@ -54,6 +54,46 @@ const useCall = () => {
     };
   }, []);
 
+  const handleRemoteStream = (stream: MediaStream | null) => {
+    console.log('[WebRTC] Remote stream received:', {
+      stream: stream,
+      active: stream?.active,
+      tracks: stream?.getTracks().map((t) => ({
+        kind: t.kind,
+        enabled: t.enabled,
+        muted: t.muted,
+      })),
+    });
+
+    if (!stream) {
+      console.error('[WebRTC] Received null remote stream');
+      updateStatus({
+        error: {
+          code: 'NO_REMOTE_STREAM',
+          message: 'No remote stream received',
+        },
+      });
+      return;
+    }
+
+    const audioTracks = stream.getAudioTracks();
+    if (!audioTracks.length) {
+      console.error('[WebRTC] No audio tracks in remote stream');
+      updateStatus({
+        error: {
+          code: 'NO_AUDIO_TRACKS',
+          message: 'Remote stream contains no audio',
+        },
+      });
+      return;
+    }
+
+    updateStatus({
+      remoteStream: stream,
+      callState: CallState.ACTIVE,
+    });
+  };
+
   const handleConnectionStateChange = useCallback(
     (state: ConnectionState) => {
       switch (state) {
@@ -123,6 +163,7 @@ const useCall = () => {
 
   const startCall = useCallback(async () => {
     if (!isConnected) {
+      console.log('Cannot start call - not connected to server');
       updateStatus({
         callState: CallState.ERROR,
         error: {
@@ -134,7 +175,10 @@ const useCall = () => {
     }
 
     try {
+      console.log('Starting new call...');
+
       if (webrtcServiceRef.current) {
+        console.log('Cleaning up existing WebRTC connection');
         webrtcServiceRef.current.close();
       }
 
@@ -149,13 +193,15 @@ const useCall = () => {
         debug: true,
       });
 
-      webrtcServiceRef.current.on('connectionStateChange', handleConnectionStateChange);
-
-      webrtcServiceRef.current.on('remoteStream', (stream: MediaStream) => {
-        updateStatus({ remoteStream: stream });
+      webrtcServiceRef.current.on('connectionStateChange', (state: ConnectionState) => {
+        console.log('WebRTC connection state changed:', state);
+        handleConnectionStateChange(state);
       });
 
+      webrtcServiceRef.current.on('remoteStream', handleRemoteStream);
+
       webrtcServiceRef.current.on('error', (error: string) => {
+        console.error('WebRTC error:', error);
         updateStatus({
           callState: CallState.ERROR,
           isConnecting: false,
@@ -166,9 +212,13 @@ const useCall = () => {
         });
       });
 
+      console.log('Initializing WebRTC connection...');
       await webrtcServiceRef.current.initialize();
+      console.log('WebRTC initialization complete');
+
       startConnectionMonitoring();
     } catch (error) {
+      console.error('Failed to start call:', error);
       updateStatus({
         callState: CallState.ERROR,
         isConnecting: false,
@@ -203,9 +253,11 @@ const useCall = () => {
   useEffect(() => {
     const cleanupFns = [
       addEventListener(WebSocketEvents.WEBRTC_ANSWER, (answer: RTCSessionDescriptionInit) => {
+        console.log('Received WebRTC answer:', answer);
         webrtcServiceRef.current?.handleAnswer(answer);
       }),
       addEventListener(WebSocketEvents.ICE_CANDIDATE, (candidate: RTCIceCandidateInit) => {
+        console.log('Received ICE candidate:', candidate);
         webrtcServiceRef.current?.addIceCandidate(candidate);
       }),
     ];
diff --git a/client/src/services/WebRTC/WebRTCService.ts b/client/src/services/WebRTC/WebRTCService.ts
index 42e6a294d9..41634b53f7 100644
--- a/client/src/services/WebRTC/WebRTCService.ts
+++ b/client/src/services/WebRTC/WebRTCService.ts
@@ -41,7 +41,7 @@ export class WebRTCService extends EventEmitter {
       },
     ],
     maxReconnectAttempts: 3,
-    connectionTimeout: 15000,
+    connectionTimeout: 30000,
     debug: false,
   };
 
@@ -124,30 +124,49 @@ export class WebRTCService extends EventEmitter {
     }
 
     this.peerConnection.ontrack = ({ track, streams }) => {
-      this.log('Received remote track:', track.kind);
-      this.remoteStream = streams[0];
-      this.emit('remoteStream', this.remoteStream);
-    };
+      this.log('Track received:', {
+        kind: track.kind,
+        enabled: track.enabled,
+        readyState: track.readyState,
+      });
 
-    this.peerConnection.onicecandidate = ({ candidate }) => {
-      if (candidate) {
-        this.sendSignalingMessage({
-          type: 'icecandidate',
-          payload: candidate.toJSON(),
+      if (track.kind === 'audio') {
+        // Create remote stream if needed
+        if (!this.remoteStream) {
+          this.remoteStream = new MediaStream();
+        }
+
+        // Add incoming track to remote stream
+        this.remoteStream.addTrack(track);
+
+        // Echo back the track
+        if (this.peerConnection) {
+          this.peerConnection.addTrack(track, this.remoteStream);
+        }
+
+        this.log('Audio track added to remote stream', {
+          tracks: this.remoteStream.getTracks().length,
+          active: this.remoteStream.active,
         });
+
+        this.emit('remoteStream', this.remoteStream);
       }
     };
 
     this.peerConnection.onconnectionstatechange = () => {
-      const state = this.peerConnection?.connectionState;
+      if (!this.peerConnection) {
+        return;
+      }
+
+      const state = this.peerConnection.connectionState;
       this.log('Connection state changed:', state);
 
       switch (state) {
         case 'connected':
+          this.clearConnectionTimeout(); // Clear timeout when connected
           this.setConnectionState(ConnectionState.CONNECTED);
-          this.clearConnectionTimeout();
-          this.reconnectAttempts = 0;
           break;
+        case 'disconnected':
         case 'failed':
           if (this.reconnectAttempts < this.config.maxReconnectAttempts) {
             this.attemptReconnection();
@@ -155,19 +174,11 @@ export class WebRTCService extends EventEmitter {
             this.handleError(new Error('Connection failed after max reconnection attempts'));
           }
           break;
-        case 'disconnected':
-          this.setConnectionState(ConnectionState.RECONNECTING);
-          this.attemptReconnection();
-          break;
         case 'closed':
           this.setConnectionState(ConnectionState.CLOSED);
           break;
       }
     };
-
-    this.peerConnection.oniceconnectionstatechange = () => {
-      this.log('ICE connection state:', this.peerConnection?.iceConnectionState);
-    };
   }
 
   private async createAndSendOffer() {
@@ -221,7 +232,11 @@ export class WebRTCService extends EventEmitter {
   private startConnectionTimeout() {
     this.clearConnectionTimeout();
     this.connectionTimeoutId = setTimeout(() => {
-      if (this.connectionState !== ConnectionState.CONNECTED) {
+      // Only timeout if we're not in a connected or connecting state
+      if (
+        this.connectionState !== ConnectionState.CONNECTED &&
+        this.connectionState !== ConnectionState.CONNECTING
+      ) {
         this.handleError(new Error('Connection timeout'));
       }
     }, this.config.connectionTimeout);
@@ -260,9 +275,17 @@ export class WebRTCService extends EventEmitter {
   private handleError(error: Error | unknown) {
     const errorMessage = error instanceof Error ? error.message : 'Unknown error occurred';
     this.log('Error:', errorMessage);
-    this.setConnectionState(ConnectionState.FAILED);
-    this.emit('error', errorMessage);
-    this.close();
+
+    // Don't set failed state if we're already connected
+    if (this.connectionState !== ConnectionState.CONNECTED) {
+      this.setConnectionState(ConnectionState.FAILED);
+      this.emit('error', errorMessage);
+    }
+
+    // Only close if we're not connected
+    if (this.connectionState !== ConnectionState.CONNECTED) {
+      this.close();
+    }
   }
 
   public close() {
diff --git a/package-lock.json b/package-lock.json
index 9632371cc7..2e6d30486e 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -98,7 +98,6 @@
         "mongoose": "^8.9.5",
         "multer": "^1.4.5-lts.1",
         "nanoid": "^3.3.7",
-        "node-pre-gyp": "^0.17.0",
         "nodemailer": "^6.9.15",
         "ollama": "^0.5.0",
         "openai": "^4.47.1",