Recieved video muting for 1-2 minutes before starting to play

Hello,

I am trying to send a camera stream with aiortc through a self-implemented golang SFU (using github.com/pions/webrtc). The intended setup is aiortc client (streaming video) -> SFU -> browser client (displaying video). However I have an issue where the recieved video is first unmuted for ~3 seconds, then muted for 1-2 minutes before starting to play. I have tried some different combinations:

1. Using OpenCV cv2.VideoCapture to stream the video as in #99 . This results in the video being muted for around 1-2 minutes and then playing.
2. Using aiortc.MediaPlayer with a mp4-file. Same result as 1.
3. Using aiortc.MediaPlayer('0', format='avfoundation', options={'framerate': '30'}). So far this one does not work (never unmutes, even after several minutes). However, this option works in the Webcam example.
4. Using a browser as client instead of aiortc. This works fine without any muting.

Since 4 works I suspect there might be something aiortc related. Looking at the specification muted seems to mean that there is no video avaliable in the stream, however using wireshark I can see UDP traffic being both sent to the SFU and forwarded to the browser.

[tobiasfriden]

Managed to get aiortc based SFU working, and now it works fine! Works with both cv2.VideoCapture and aiortc.MediaPlayer('0', format='avfoundation', options={'framerate': '30'}), however the first option seems to be much better latency-wise.

Edit: Working (very naive) SFU implementation:

import argparse
import asyncio
import json
import logging
import os
import ssl

import cv2
from aiohttp import web
import aiohttp_cors
from av import VideoFrame

from aiortc import RTCPeerConnection, RTCSessionDescription, VideoStreamTrack
from aiortc.mediastreams import MediaStreamError
from aiortc.contrib.media import MediaBlackhole, MediaPlayer, MediaRecorder

clients = set()
listeners = set()
listenerTracks = set()

class TrackMux:
    def __init__(self, track):
        self.listeners = set()
        self.track = track

    def addListener(self, listener):
        self.listeners.add(listener)

    async def run(self):
        asyncio.create_task(self.__run_mux())

    async def __run_mux(self):
        while True:
            frame = await self.track.recv()
            print('Track got frame: ', frame)
            for listener in self.listeners:
                await listener.put(frame)

class ListenerTrack(VideoStreamTrack):
    def __init__(self):
        super().__init__()
        self.__queue = asyncio.Queue()

    def queue(self):
        return self.__queue

    async def recv(self):
        frame = await self.__queue.get()
        print('Recieved frame: ', frame)
        return frame


async def listener_sdp(request):
    params = await request.json()
    offer = RTCSessionDescription(sdp=params['sdp'], type='offer')
    print(params)
    pc = RTCPeerConnection()
    print('Number of listeners: ', len(listeners))
    track = ListenerTrack()
    listenerTracks.add(track)
    pc.addTrack(track)
    listeners.add(pc)

    await pc.setRemoteDescription(offer)
    answer = await pc.createAnswer()
    print(answer)
    await pc.setLocalDescription(answer)

    return web.Response(
        content_type='application/json',
        text=json.dumps({
            'sdp': pc.localDescription.sdp,
            'type': 'answer'
        })
    )

async def client_sdp(request):
    params = await request.json()
    offer = RTCSessionDescription(sdp=params['sdp'], type='offer')

    pc = RTCPeerConnection()
    clients.add(pc)
    print('Number of clients: ', len(clients))
    
    @pc.on('track')
    async def on_track(track):
        tm = TrackMux(track)
        for lt in listenerTracks:
            print('Added track to listener: ', track, lt)
            tm.addListener(lt.queue())
        await tm.run()

    await pc.setRemoteDescription(offer)
    answer = await pc.createAnswer()
    await pc.setLocalDescription(answer)

    return web.Response(
        content_type='application/json',
        text=json.dumps({
            'sdp': pc.localDescription.sdp,
            'type': 'answer'
        })
    )

async def on_shutdown(app):
    # close peer connections
    coros = [pc.close() for pc in clients] + [pc.close() for pc in listeners]
    await asyncio.gather(*coros)
    clients.clear()
    listeners.clear()

if __name__ == '__main__':
    app = web.Application()
    app.on_shutdown.append(on_shutdown)

    app.router.add_post('/client', client_sdp)
    app.router.add_post('/listener', listener_sdp)

    cors = aiohttp_cors.setup(app, defaults={
        "*": aiohttp_cors.ResourceOptions(
                allow_credentials=True,
                expose_headers="*",
                allow_headers="*",
            )
    })

    for route in list(app.router.routes()):
        cors.add(route)

    web.run_app(app, port='4000')

[tamert]

I solved and I understood… Mux is not necessary for me only track is enough 😃

    @pc.on("track")
    async def on_track(track):
        log_info("broadcast: Track %s received" % track.kind)
        senderStream.add(track)