Flutter WebRTC audio but no video

1,970

So my problem was that I wasn't adding queued candidates to the caller.

I added

sendMessage(answerMessage);
if (_remoteCandidates.isNotEmpty) {
  _remoteCandidates
      .forEach((candidate) => _peerConnection.addCandidate(candidate));
  _remoteCandidates.clear();
}

to the processAnswer method and it works just fine!

Share:
1,970
Cate Daniel
Author by

Cate Daniel

I'm a simple Physics student trying to find my way in the world of programing

Updated on December 21, 2022

Comments

  • Cate Daniel
    Cate Daniel over 1 year

    So I'm building a video calling application using flutter, flutterWeb, and the WebRTC package.

    I have a spring boot server sitting in the middle to pass the messages between the two clients.

    Each side shows the local video, but neither shows the remote. Audio does work though. I got som nasty feedback loops. Testing with headphones showed that audio does indeed work.

    My singaling code

    typedef void StreamStateCallback(MediaStream stream);
    
    class CallingService {
      String sendToUserId;
      String currentUserId;
      final String authToken;
      final StompClient _client;
      final StreamStateCallback onAddRemoteStream;
      final StreamStateCallback onRemoveRemoteStream;
      final StreamStateCallback onAddLocalStream;
      RTCPeerConnection _peerConnection;
      List<RTCIceCandidate> _remoteCandidates = [];
      String destination;
      var hasOffer = false;
      var isNegotiating = false;
      MediaStream _localStream;
    
      final Map<String, dynamic> _constraints = {
        'mandatory': {
          'OfferToReceiveAudio': true,
          'OfferToReceiveVideo': true,
        },
        'optional': [],
      };
    
      CallingService(
          this._client,
          this.sendToUserId,
          this.currentUserId,
          this.authToken,
          this.onAddRemoteStream,
          this.onRemoveRemoteStream,
          this.onAddLocalStream) {
        destination = '/app/start-call/$sendToUserId';
        print("destination $destination");
        _client.subscribe(
            destination: destination,
            headers: {'Authorization': "$authToken"},
            callback: (StompFrame frame) => processMessage(jsonDecode(frame.body)));
      }
    
      Future<void> startCall() async {
        await processRemoteStream();
        RTCSessionDescription description =
            await _peerConnection.createOffer(_constraints);
        await _peerConnection.setLocalDescription(description);
        var message = RtcMessage(RtcMessageType.OFFER, currentUserId, {
          'description': {'sdp': description.sdp, 'type': description.type},
        });
        sendMessage(message);
      }
    
      Future<void> processMessage(Map<String, dynamic> messageJson) async {
        var message = RtcMessage.fromJson(messageJson);
        if (message.from == currentUserId) {
          return;
        }
        print("processing ${message.messageType.toString()}");
        switch (message.messageType) {
          case RtcMessageType.BYE:
            // TODO: Handle this case.
            break;
          case RtcMessageType.LEAVE:
            // TODO: Handle this case.
            break;
          case RtcMessageType.CANDIDATE:
            await processCandidate(message);
            break;
          case RtcMessageType.ANSWER:
            await processAnswer(message);
            break;
          case RtcMessageType.OFFER:
            await processOffer(message);
            break;
        }
      }
    
      Future<void> processCandidate(RtcMessage candidate) async {
        Map<String, dynamic> map = candidate.data['candidate'];
        var rtcCandidate = RTCIceCandidate(
          map['candidate'],
          map['sdpMid'],
          map['sdpMLineIndex'],
        );
        if (_peerConnection != null) {
          _peerConnection.addCandidate(rtcCandidate);
        } else {
          _remoteCandidates.add(rtcCandidate);
        }
      }
    
      Future<void> processAnswer(RtcMessage answer) async {
        if (isNegotiating) {
          return;
        }
        isNegotiating = true;
        var description = answer.data['description'];
        if (_peerConnection == null) {
          return;
        }
        await _peerConnection.setRemoteDescription(
            RTCSessionDescription(description['sdp'], description['type']));
      }
    
      Future<void> processOffer(RtcMessage offer) async {
        await processRemoteStream();
        var description = offer.data['description'];
        await _peerConnection.setRemoteDescription(
            new RTCSessionDescription(description['sdp'], description['type']));
        var answerDescription = await _peerConnection.createAnswer(_constraints);
        await _peerConnection.setLocalDescription(answerDescription);
        var answerMessage = RtcMessage(RtcMessageType.ANSWER, currentUserId, {
          'description': {
            'sdp': answerDescription.sdp,
            'type': answerDescription.type
          },
        });
        sendMessage(answerMessage);
        if (_remoteCandidates.isNotEmpty) {
          _remoteCandidates
              .forEach((candidate) => _peerConnection.addCandidate(candidate));
          _remoteCandidates.clear();
        }
      }
    
      Future<void> processRemoteStream() async {
        _localStream = await createStream();
        _peerConnection = await createPeerConnection(_iceServers, _config);
        _peerConnection.addStream(_localStream);
        _peerConnection.onSignalingState = (state) {
          //isNegotiating = state != RTCSignalingState.RTCSignalingStateStable;
        };
    
        _peerConnection.onAddStream = (MediaStream stream) {
          this.onAddRemoteStream(stream);
        };
        _peerConnection.onRemoveStream =
            (MediaStream stream) => this.onRemoveRemoteStream(stream);
        _peerConnection.onIceCandidate = (RTCIceCandidate candidate) {
          var data = {
            'candidate': {
              'sdpMLineIndex': candidate.sdpMlineIndex,
              'sdpMid': candidate.sdpMid,
              'candidate': candidate.candidate,
            },
          };
          var message = RtcMessage(RtcMessageType.CANDIDATE, currentUserId, data);
          sendMessage(message);
        };
      }
    
      void sendMessage(RtcMessage message) {
        _client.send(
            destination: destination,
            headers: {'Authorization': "$authToken"},
            body: jsonEncode(message.toJson()));
      }
    
      Map<String, dynamic> _iceServers = {
        'iceServers': [
          {'urls': 'stun:stun.l.google.com:19302'},
          /*
           * turn server configuration example.
          {
            'url': 'turn:123.45.67.89:3478',
            'username': 'change_to_real_user',
            'credential': 'change_to_real_secret'
          },
           */
        ]
      };
    
      final Map<String, dynamic> _config = {
        'mandatory': {},
        'optional': [
          {'DtlsSrtpKeyAgreement': true},
        ],
      };
    
      Future<MediaStream> createStream() async {
        final Map<String, dynamic> mediaConstraints = {
          'audio': true,
          'video': {
            'mandatory': {
              'minWidth': '640',
              'minHeight': '480',
              'minFrameRate': '30',
            },
            'facingMode': 'user',
            'optional': [],
          }
        };
    
        MediaStream stream = await navigator.getUserMedia(mediaConstraints);
        if (this.onAddLocalStream != null) {
          this.onAddLocalStream(stream);
        }
        return stream;
      }
    }
    

    Here are my widgets

    class _CallScreenState extends State<CallScreen> {
      StompClient _client;
      CallingService _callingService;
      RTCVideoRenderer _localRenderer = new RTCVideoRenderer();
      RTCVideoRenderer _remoteRenderer = new RTCVideoRenderer();
      final UserService userService = GetIt.instance.get<UserService>();
    
      void onConnectCallback(StompClient client, StompFrame connectFrame) async {
        var currentUser = await userService.getCurrentUser();
        _callingService = CallingService(
            _client,
            widget.intent.toUserId.toString(),
            currentUser.id.toString(),
            widget.intent.authToken,
            onAddRemoteStream,
            onRemoveRemoteStream,
            onAddLocalStream);
        if (widget.intent.initialMessage != null) {
          _callingService.processMessage(jsonDecode(widget.intent.initialMessage));
        } else {
          _callingService.startCall();
        }
      }
    
      void onAddRemoteStream(MediaStream stream) {
        _remoteRenderer.srcObject = stream;
      }
    
      void onRemoveRemoteStream(MediaStream steam) {
        _remoteRenderer.srcObject = null;
      }
    
      void onAddLocalStream(MediaStream stream) {
        _localRenderer.srcObject = stream;
      }
    
      @override
      void initState() {
        super.initState();
        _localRenderer.initialize();
        _remoteRenderer.initialize();
        _client = StompClient(
          config: StompConfig(
            url: 'ws://${DomainService.getDomainBase()}/stomp',
            onConnect: onConnectCallback,
            onWebSocketError: (dynamic error) => print(error.toString()),
            stompConnectHeaders: {'Authorization': "${widget.intent.authToken}"},
            onDisconnect: (message) => print("disconnected ${message.body}"),),
        );
        _client.activate();
      }
    
      @override
      Widget build(BuildContext context) {
        return PlatformScaffold(
          pageTitle: "",
          child: Flex(
            direction: Axis.vertical,
            children: [
              Flexible(
                flex: 1,
                child: RTCVideoView(_localRenderer),
              ),
              Flexible(
                flex: 1,
                child: RTCVideoView(_remoteRenderer),
              )
            ],
          ),
        );
      }
    }
    

    I put a print statment in the the widget on the addRemoteStream callback, and it's getting called. So some kind of stream is being sent. I'm just not sure why the video isnt' showing.