Delay in displaying the video onto the browser using media-server

[RashmiAD]

Hi,

I'm using this library to stream the video to a browser which sends the encoded H.264 frames through RTP to the Janus WebRTC gateway to the browser. The problem I'm facing with this library is that the video isn't displayed as soon as browser is opened. Similarly I tested with GStreamer and this works pretty well and displays video as the browser is opened. I'm using the test example form librtsp/test/media folder to implement the above mentioned scenario.

[ireader]

Could you show me the sdp contents of test program and gstreamer? As you known, unlike audio, video must decode from the key frame. Video buffering may be cause by the video key frame receive delay.

I will try to do you test scenario. It is beffer if you could upload the wireshark pcap files (librtsp test program / gstreamer).

[RashmiAD]

Attached are the pcap files for gstreamer and librtsp and also the modified code as per my use case. librtsp.zip gstreamer.zip test_code.zip

[RashmiAD]

Could you please respond to this issue?

[ireader]

I have tested rtp streaming to janus gateway, the video show very quicky.

Config: h264-sample: { type = "rtp" id = 10 description = "H.264 live stream coming from gstreamer" audio = false video = true videoport = 8004 videopt = 126 videortpmap = "H264/90000" videofmtp = "profile-level-id=42e01f;packetization-mode=1"

secret = "adminpwd"

}

First click H.264 live stream on janus demo page, then start rtp streaming.

[RashmiAD]

Was the Janus tested with the modified code I attached earlier?

[RashmiAD]

I'm passing the encoded H.264 frames to the media-server library which is sent to the Janus server.

[ireader]

Below is my test code: read a mp4 file and streaming to janus gateway server.

Copy and save to media-server/test folder. (Just test on windows10 vs2019, not sure for linux)

Replace the "IP" with your server ip address.

#include "mov-reader.h"
#include "mov-format.h"
#include "mpeg4-hevc.h"
#include "mpeg4-avc.h"
#include "mpeg4-aac.h"
#include "webm-vpx.h"
#include "aom-av1.h"
#include "rtp-profile.h"
#include "rtsp-muxer.h"
#include "sockutil.h"
#include "sys/system.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <time.h>

#define IP "192.168.1.100"

#define SOCKET_STORAGE_TO_ADDR(storage) (const struct sockaddr*)(storage), socket_addr_len((const struct sockaddr*)(storage))

extern "C" const struct mov_buffer_t* mov_file_buffer(void);

static uint8_t s_packet[2 * 1024 * 1024];
static uint8_t s_buffer[4 * 1024 * 1024];
static struct mpeg4_hevc_t s_hevc;
static struct mpeg4_avc_t s_avc;
static struct mpeg4_aac_t s_aac;
static struct webm_vpx_t s_vpx;
static struct aom_av1_t s_av1;

struct rtp_streaming_test_t;
struct rtp_streaming_test_stream_t
{
    struct rtp_streaming_test_t* ctx;

    int av;
    int object;
    int track;
    int psi;
    int64_t dts;

    int mid;
    struct rtsp_muxer_t* rtp;

    socket_t udp[2];
    struct sockaddr_storage addr[2];
};

struct rtp_streaming_test_t
{
    struct rtp_streaming_test_stream_t a, v;
    uint64_t clock;
};

static int rtp_encode_packet(void* param, int pid, const void* packet, int bytes, uint32_t timestamp, int /*flags*/)
{
    static uint8_t rtcp[1500];
    struct rtp_streaming_test_stream_t* ctx = (struct rtp_streaming_test_stream_t*)param;
    assert(bytes == socket_sendto(ctx->udp[0], packet, bytes, 0, SOCKET_STORAGE_TO_ADDR(&ctx->addr[0])));

    int r = rtsp_muxer_rtcp(ctx->rtp, ctx->mid, rtcp, sizeof(rtcp));
    if (r > 0)
    {
        assert(r == socket_sendto(ctx->udp[1], rtcp, r, 0, SOCKET_STORAGE_TO_ADDR(&ctx->addr[1])));
    }

    return bytes;
}

inline const char* ftimestamp(uint32_t t, char* buf)
{
    sprintf(buf, "%02u:%02u:%02u.%03u", t / 3600000, (t / 60000) % 60, (t / 1000) % 60, t % 1000);
    return buf;
}

static void onread(void* param, uint32_t track, const void* buffer, size_t bytes, int64_t pts, int64_t dts, int flags)
{
    static char s_pts[64], s_dts[64];
    static int64_t v_pts, v_dts;
    static int64_t a_pts, a_dts;
    struct rtp_streaming_test_t* ctx = (struct rtp_streaming_test_t*)param;
    int n;

    uint64_t clock = system_clock();
    if (clock - ctx->clock + 5 < dts)
        system_sleep(dts - (clock - ctx->clock + 5));

    if (ctx->v.track == track)
    {
        if (MOV_OBJECT_H264 == ctx->v.object)
        {
            n = h264_mp4toannexb(&s_avc, buffer, bytes, s_packet, sizeof(s_packet));
        }
        else if (MOV_OBJECT_HEVC == ctx->v.object)
        {
            n = h265_mp4toannexb(&s_hevc, buffer, bytes, s_packet, sizeof(s_packet));
        }
        else if (MOV_OBJECT_AV1 == ctx->v.object)
        {
            n = aom_av1_codec_configuration_record_save(&s_av1, s_packet, sizeof(s_packet));
        }
        else if (MOV_OBJECT_VP9 == ctx->v.object || MOV_OBJECT_VP8 == ctx->v.object)
        {
            n = aom_av1_codec_configuration_record_save(&s_av1, s_packet, sizeof(s_packet));
        }
        else
        {
            assert(0);
        }

        printf("[V] pts: %s, dts: %s, diff: %03d/%03d, %d%s\n", ftimestamp(pts, s_pts), ftimestamp(dts, s_dts), (int)(pts - v_pts), (int)(dts - v_dts), (int)n, flags ? " [I]" : "");
        v_pts = pts;
        v_dts = dts;
        assert(0 == rtsp_muxer_input(ctx->v.rtp, ctx->v.mid, pts, dts, s_packet, n, 0));
    }
    else if (ctx->a.track == track)
    {
        if (MOV_OBJECT_AAC == ctx->a.object)
        {
            n = mpeg4_aac_adts_save(&s_aac, bytes, s_packet, sizeof(s_packet));
        }
        else if (MOV_OBJECT_OPUS == ctx->a.object)
        {
            //assert(0);
        }
        else
        {
            assert(0);
        }

        printf("[A] pts: %s, dts: %s, diff: %03d/%03d, %d\n", ftimestamp(pts, s_pts), ftimestamp(dts, s_dts), (int)(pts - a_pts), (int)(dts - a_dts), (int)n);
        a_pts = pts;
        a_dts = dts;
        assert(0 == rtsp_muxer_input(ctx->a.rtp, ctx->a.mid, pts, dts, s_packet, n, 0));
    }
    else
    {
        assert(0);
    }
}

static void mov_video_info(void* param, uint32_t track, uint8_t object, int /*width*/, int /*height*/, const void* extra, size_t bytes)
{
    struct rtp_streaming_test_t* ctx = (struct rtp_streaming_test_t*)param;
    ctx->v.track = track;
    ctx->v.object = object;
    ctx->v.av = 1;
    ctx->v.udp[0] = socket_udp_bind(NULL, 0);
    ctx->v.udp[1] = socket_udp_bind(NULL, 0);
    assert(0 == socket_addr_from(&ctx->v.addr[0], NULL, IP, 8004));
    assert(0 == socket_addr_from(&ctx->v.addr[1], NULL, IP, 8005));
    ctx->v.rtp = rtsp_muxer_create(rtp_encode_packet, &ctx->v);

    if (MOV_OBJECT_H264 == object)
    {
        assert(bytes == mpeg4_avc_decoder_configuration_record_load((const uint8_t*)extra, bytes, &s_avc));
        int pid = rtsp_muxer_add_payload(ctx->v.rtp, 90000, 126, "H264", 0, 0, 0, extra, bytes);
        ctx->v.mid = rtsp_muxer_add_media(ctx->v.rtp, pid, RTP_PAYLOAD_H264, extra, bytes);
    }
    else if (MOV_OBJECT_HEVC == object)
    {
        assert(bytes == mpeg4_hevc_decoder_configuration_record_load((const uint8_t*)extra, bytes, &s_hevc));
        int pid = rtsp_muxer_add_payload(ctx->v.rtp, 90000, RTP_PAYLOAD_H265, "H265", 0, 0, 0, extra, bytes);
        ctx->v.mid = rtsp_muxer_add_media(ctx->v.rtp, pid, RTP_PAYLOAD_H265, extra, bytes);
    }
    else if (MOV_OBJECT_AV1 == object)
    {
        assert(bytes == aom_av1_codec_configuration_record_load((const uint8_t*)extra, bytes, &s_av1));
        int pid = rtsp_muxer_add_payload(ctx->v.rtp, 90000, 96, "AV1", 0, 0, 0, extra, bytes);
        ctx->v.mid = rtsp_muxer_add_media(ctx->v.rtp, pid, RTP_PAYLOAD_AV1, extra, bytes);
    }
    else if (MOV_OBJECT_VP9 == object)
    {
        assert(bytes == webm_vpx_codec_configuration_record_load((const uint8_t*)extra, bytes, &s_vpx));
        int pid = rtsp_muxer_add_payload(ctx->v.rtp, 90000, RTP_PAYLOAD_VP9, "VP9", 0, 0, 0, extra, bytes);
        ctx->v.mid = rtsp_muxer_add_media(ctx->v.rtp, pid, RTP_PAYLOAD_VP9, extra, bytes);
    }
    else if (MOV_OBJECT_VP8 == object)
    {
        assert(bytes == webm_vpx_codec_configuration_record_load((const uint8_t*)extra, bytes, &s_vpx));
        int pid = rtsp_muxer_add_payload(ctx->v.rtp, 90000, RTP_PAYLOAD_VP8, "VP8", 0, 0, 0, extra, bytes);
        ctx->v.mid = rtsp_muxer_add_media(ctx->v.rtp, pid, RTP_PAYLOAD_VP8, extra, bytes);
    }
    else
    {
        assert(0);
    }
}

static void mov_audio_info(void* param, uint32_t track, uint8_t object, int /*channel_count*/, int /*bit_per_sample*/, int sample_rate, const void* extra, size_t bytes)
{
    struct rtp_streaming_test_t* ctx = (struct rtp_streaming_test_t*)param;
    ctx->a.track = track;
    ctx->a.object = object;
    ctx->a.av = 0;
    ctx->a.udp[0] = socket_udp_bind(NULL, 0);
    ctx->a.udp[1] = socket_udp_bind(NULL, 0);
    assert(0 == socket_addr_from(&ctx->a.addr[0], NULL, IP, 5002));
    assert(0 == socket_addr_from(&ctx->a.addr[1], NULL, IP, 5003));
    ctx->a.rtp = rtsp_muxer_create(rtp_encode_packet, &ctx->a);

    if (MOV_OBJECT_AAC == object)
    {
        assert(bytes == mpeg4_aac_audio_specific_config_load((const uint8_t*)extra, bytes, &s_aac));
        int pid = rtsp_muxer_add_payload(ctx->a.rtp, sample_rate, RTP_PAYLOAD_MP4A, "MP4A-LATM", 0, 0, 0, extra, bytes);
        ctx->a.mid = rtsp_muxer_add_media(ctx->a.rtp, pid, RTP_PAYLOAD_MP4A, extra, bytes);
    }
    else if (MOV_OBJECT_OPUS == object)
    {
        assert(48000 == sample_rate);
        int pid = rtsp_muxer_add_payload(ctx->a.rtp, sample_rate, RTP_PAYLOAD_OPUS, "OPUS", 0, 0, 0, extra, bytes);
        ctx->a.mid = rtsp_muxer_add_media(ctx->a.rtp, pid, RTP_PAYLOAD_OPUS, extra, bytes);
    }
    else
    {
        assert(0);
    }
}

void rtp_streaming_test(const char* mp4)
{
    struct rtp_streaming_test_t ctx;
    memset(&ctx, 0, sizeof(ctx));
    ctx.a.ctx = &ctx;
    ctx.v.ctx = &ctx;

    FILE* fp = fopen(mp4, "rb");
    mov_reader_t* mov = mov_reader_create(mov_file_buffer(), fp);
    uint64_t duration = mov_reader_getduration(mov);

    struct mov_reader_trackinfo_t info = { mov_video_info, mov_audio_info };
    mov_reader_getinfo(mov, &info, &ctx);

    ctx.clock = system_clock();
    while (mov_reader_read(mov, s_buffer, sizeof(s_buffer), onread, &ctx) > 0)
    {
        int n = 0;
        socket_t udp[4];
        if (ctx.v.udp[0] && socket_invalid != ctx.v.udp[0])
        {
            udp[n] = ctx.v.udp[0];
            udp[n++] = ctx.v.udp[1];
        }

        if (ctx.a.udp[0] && socket_invalid != ctx.a.udp[0])
        {
            udp[n] = ctx.a.udp[0];
            udp[n++] = ctx.a.udp[1];
        }

        socklen_t addrlen;
        struct sockaddr_storage addr;
        int64_t flags = socket_poll_readv(0, n, udp);
        for (int i = 0; i < 4; i++)
        {
            // discard rtcp
            if (flags & (1LL << i))
            {
                socket_recvfrom(udp[i], s_buffer, sizeof(s_buffer), 0, (struct sockaddr*)&addr, &addrlen);
            }
        }
    }

    if (ctx.a.rtp)
        rtsp_muxer_destroy(ctx.a.rtp);
    if (ctx.v.rtp)
        rtsp_muxer_destroy(ctx.v.rtp);

    mov_reader_destroy(mov);
    fclose(fp);
}

[RashmiAD]

  #include "h264-ici-source.h"
  #include <assert.h>
  #include <string.h>
  #include <algorithm>
  #include <iostream>

  //Socket
  #include <sys/socket.h>
  #include <netinet/in.h>
  #include <arpa/inet.h>

  #include "cstringext.h"
  #include "base64.h"
  #include "rtp-profile.h"
  #include "rtp-payload.h"
  #include "../rtp-udp-transport.h"

  #define H264_NAL(v)   (v & 0x1F)

  enum { NAL_IDR = 5, NAL_SEI = 6, NAL_SPS = 7, NAL_PPS = 8 };

  extern "C" uint32_t rtp_ssrc(void);

  H264IciSource::H264IciSource()
  {
      m_speed = 1.0;
      m_status = 0;
      m_rtp_clock = 0;
      m_rtcp_clock = 0;
      m_timestamp = 0;

      m_rtp = nullptr;
      m_rtppacker = nullptr;

      const char* ip = GetHostIP();

      unsigned  short port[2] = { 8004, 0 };

      auto udp = std::make_shared<RTPUdpTransport>();
      udp->Init(ip, port);
      SetTransport("", udp);

      uint32_t ssrc = rtp_ssrc();
      static struct rtp_payload_t s_rtpfunc = {
          H264IciSource::RTPAlloc,
          H264IciSource::RTPFree,
          H264IciSource::RTPPacket,
      };

      m_rtppacker = rtp_payload_encode_create(RTP_PAYLOAD_H264, "H264", (uint16_t)ssrc, ssrc, &s_rtpfunc, this);

      struct rtp_event_t event;
      event.on_rtcp = OnRTCPEvent;
      m_rtp = rtp_create(&event, this, ssrc, m_timestamp, 90000, 4 * 1024, 1);
      rtp_set_info(m_rtp, "RTSPServer", "szj.h264");
  }

  H264IciSource::~H264IciSource()
  {
      if (m_rtp)
          rtp_destroy(m_rtp);

      if (m_rtppacker)
          rtp_payload_encode_destroy(m_rtppacker);
  }

  static inline const uint8_t* search_start_code(const uint8_t* ptr, const uint8_t* end)
  {
      for (const uint8_t *p = ptr; p + 3 < end; p++)
      {
          if (0x00 == p[0] && 0x00 == p[1] && (0x01 == p[2] || (0x00 == p[2] && 0x01 == p[3])))
          {
              return p;
          }
      }
      return end;
  }

  static inline int h264_nal_type(const unsigned char* ptr)
  {
      int i = 2;
      assert(0x00 == ptr[0] && 0x00 == ptr[1]);
      if (0x00 == ptr[2])
          ++i;
      assert(0x01 == ptr[i]);
      return H264_NAL(ptr[i + 1]);
  }

  static inline int h264_nal_new_access(const unsigned char* ptr, const uint8_t* end)
  {
      int i = 2;
      if (end - ptr < 4)
          return 1;
      assert(0x00 == ptr[0] && 0x00 == ptr[1]);
      if (0x00 == ptr[2])
          ++i;
      assert(0x01 == ptr[i]);
      int nal_unit_type = H264_NAL(ptr[i + 1]);
      if (nal_unit_type < 1 || nal_unit_type > 5)
          return 1;

      if (ptr + i + 2 > end)
          return 1;

      // Live555 H264or5VideoStreamParser::parse
      // The high-order bit of the byte after the "nal_unit_header" tells us whether it's
      // the start of a new 'access unit' (and thus the current NAL unit ends an 'access unit'):
      return (ptr[i + 2] & 0x80) != 0 ? 1 : 0;
  }

  // #define toOffset(ptr) (n - (m_bytes - (ptr - m_ptr)))

  void H264IciSource::destroyRtpAndRtpPacker()
  {
      if (m_rtp)
      {
          rtp_destroy(m_rtp);
          m_rtp = nullptr;
      }

      if (m_rtppacker)
      {
          rtp_payload_encode_destroy(m_rtppacker);
          m_rtppacker = nullptr;
      }
  }

  void H264IciSource::Init(const std::vector<uint8_t>& h264_frame, size_t frame_size)
  {
      try
      {
          m_capacity = frame_size;

          size_t count = 0;
          bool spspps = true;

          const uint8_t* end = h264_frame.data() + m_capacity;
          const uint8_t* nalu = search_start_code(h264_frame.data(), end);
          const uint8_t* p = nalu;

          int counter = 0;

          while (p < end)
          {
              const unsigned char* pn = search_start_code(p + 4, end);
              size_t bytes = pn - p;

              int nal_unit_type = h264_nal_type(p);
              assert(0 != nal_unit_type);
              if (nal_unit_type <= 5  && h264_nal_new_access(pn, end))
              {
                  //if (m_sps.size() > 0) spspps = false; // don't need more sps/pps

                  vframe_t frame;
                  frame.bytes = bytes;
                  frame.nalu.assign(nalu, nalu + bytes);
                  frame.time = 40 * count++;
                  frame.idr = 5 == nal_unit_type; // IDR-frame
                  m_videos.push_back(frame);
                  nalu = pn;
              }
              // else if (NAL_SPS == nal_unit_type || NAL_PPS == nal_unit_type)
              // {
              //    if (spspps)
              //    {
              //        size_t n = 0x01 == p[2] ? 3 : 4;
              //        std::pair<const uint8_t*, size_t> pr;
              //        pr.first = p + n;
              //        pr.second = bytes;
              //        m_sps.push_back(pr);
              //    }
              // }

              p = pn;
          }

          m_duration = 40 * count;
      }
      catch(const std::exception& ex)
      {
          std::cout << "Exception occured : " << ex.what() << std::endl;
          std::cout << "[ERROR] : Exception occured in function " << __func__ << " in file " << __FILE__ << std::endl;
      }
  }

  int H264IciSource::SetTransport(const char* /*track*/, const std::shared_ptr<IRTPTransport>& transport)
  {
      m_transport = transport;
      return 0;
  }

  int H264IciSource::GetNextFrame(int64_t &dts, const uint8_t* &ptr, size_t &bytes)
  {
      if (m_videos.empty())
          return -1; //no frames

      vframe_t next_frame = m_videos.front();

      ptr = next_frame.nalu.data();
      dts = next_frame.time;
      bytes = next_frame.bytes;

      next_frame.nalu.clear();
      next_frame.nalu.shrink_to_fit();

      m_videos.pop_front();
      m_videos.shrink_to_fit();

      return 0;
  }

  int H264IciSource::Play()
  {
      try
      {
          m_status = 1;

          time64_t clock = time64_now();
          if (0 == m_rtp_clock)
              m_rtp_clock = clock;

          if (m_rtp_clock + 40 < clock)
          {
              size_t bytes;
              const uint8_t* ptr;
              if (0 == GetNextFrame(m_pos, ptr, bytes))
              {
                  if(m_rtppacker != NULL && ptr != NULL)
                  {
                      int enc_val = rtp_payload_encode_input(m_rtppacker, ptr, bytes, m_timestamp * 90 /*kHz*/);
                  }
                  m_rtp_clock = clock;
                  m_timestamp += 40;

                  SendRTCP();
                  return 1;
              }
          }

          return 0;
      }
      catch(const std::exception& ex)
      {
          std::cout << "Exception occured : " << ex.what() << std::endl;
          std::cout << "[ERROR] : Exception occured in function " << __func__ << " in file " << __FILE__ << std::endl;
      }
      catch (...)
      {
          std::cout << "[ERROR] : Exception occured in function " << __func__ << " in file " << __FILE__ << std::endl;
          return 0;
      }
  }

  void H264IciSource::OnRTCPEvent(const struct rtcp_msg_t* msg)
  {
      msg;
  }

  void H264IciSource::OnRTCPEvent(void* param, const struct rtcp_msg_t* msg)
  {
      H264IciSource *self = (H264IciSource *)param;
      self->OnRTCPEvent(msg);
  }

  int H264IciSource::SendRTCP()
  {
      // make sure have sent RTP packet

      time64_t clock = time64_now();
      int interval = rtp_rtcp_interval(m_rtp);
      if (0 == m_rtcp_clock || m_rtcp_clock + interval < clock)
      {
          char rtcp[1024] = { 0 };
          size_t n = rtp_rtcp_report(m_rtp, rtcp, sizeof(rtcp));

          // send RTCP packet
          m_transport->Send(true, rtcp, n);

          m_rtcp_clock = clock;
      }

      return 0;
  }

  void* H264IciSource::RTPAlloc(void* param, int bytes)
  {
      H264IciSource *self = (H264IciSource*)param;
      assert(bytes <= sizeof(self->m_packet));
      return self->m_packet;
  }

  void H264IciSource::RTPFree(void* param, void *packet)
  {
      H264IciSource *self = (H264IciSource*)param;
      assert(self->m_packet == packet);
  }

  void H264IciSource::RTPPacket(void* param, const void *packet, int bytes, uint32_t /*timestamp*/, int /*flags*/)
  {
      H264IciSource *self = (H264IciSource*)param;
      assert(self->m_packet == packet);

      int r = self->m_transport->Send(false, packet, bytes);
      assert(r == (int)bytes);
      rtp_onsend(self->m_rtp, packet, bytes/*, time*/);
  }

  const char* H264IciSource::GetHostIP()
  {
      char hostbuffer[256];
      struct hostent* host_entry;

      // To retrieve hostname 
      gethostname(hostbuffer, sizeof(hostbuffer));

      // To retrieve host information 
      host_entry = gethostbyname(hostbuffer);

      // To convert an Internet network 
      // address into ASCII string 
      const char* ip = inet_ntoa(*((struct in_addr*)host_entry->h_addr_list[0]));

      return ip;
  }

[RashmiAD]

I'm using the above code as follows where I'm sending H.264 encoded frames from the camera to stream live video to the browser through Janus.

std::vector<uint8_t> h264Image;
std::shared_ptr<H264IciSource> src = h264IciSrc;
src->Init(h264Image, h264Image.size());
src->Play();

[ireader]

1. make sure janue h264-sample be uncomment:

2. rtp payload change to 126 (same as janus.plugin.streaming.jcfg)

m_rtppacker = rtp_payload_encode_create(RTP_PAYLOAD_H264, "H264", (uint16_t)ssrc, ssrc, &s_rtpfunc, this);

--> m_rtppacker = rtp_payload_encode_create(126, "H264", (uint16_t)ssrc, ssrc, &s_rtpfunc, this);

3. change udp port

unsigned short port[2] = { 5004, 0 };

--> unsigned short port[2] = { 8004, 8005 };

4. fix GetNextFrame memory invalid

int H264IciSource::GetNextFrame(int64_t &dts, const uint8_t* &ptr, size_t &bytes)
{
    if (m_videos.empty())
        return -1; //no frames

    vframe_t& next_frame = m_videos.front();

    ptr = next_frame.nalu.data();
    dts = next_frame.time;
    bytes = next_frame.bytes;
    return 0;
}

int H264IciSource::PopFrame()
{
    if (m_videos.empty())
        return -1; //no frames

    vframe_t next_frame = m_videos.front();
    next_frame.nalu.clear();
    next_frame.nalu.shrink_to_fit();

    m_videos.pop_front();
    m_videos.shrink_to_fit();
    return 0;
}

5. call PopFrame() after play per frame

SendRTCP();
PopFrame();
return 1;

[RashmiAD]

Thanks for the above code changes. It's working now for 320 x 240 resolution camera.

But, for 640 x 512 resolution camera the video isn't streaming properly. There are retransmission errors "Got 76 SRTP/SRTCP errors in the last few seconds" on janus and there are lags in video.

[ireader]

SRTP/SRTCP error between web browser and janus，try to restart janus。

Check janus video fmtp profile-level-id value with your camera h.264 video profile.

[RashmiAD]

Parsing compound packet (total of 68 bytes)

1 RR (201)

jitter=17199.000000, fraction=0, loss=0 rtt=39 RTCP PT 201, length: 32 bytes

2 PLI -- PSFB (206)

RTCP PT 206, length: 12 bytes

3 REMB -- PSFB (206)

-- -- -- REMB: 197717 * 2^0 = 197717 (1 SSRCs, 3940002786) RTCP PT 206, length: 24 bytes

Above is the Janus logs. After starting the camera initially I'm getting 12 bytes of PLI (Packet Loss Indicator) for few seconds and then it disappears and I get the logs as follows:

Parsing compound packet (total of 56 bytes)

1 RR (201)

jitter=16533.000000, fraction=0, loss=0 rtt=5 RTCP PT 201, length: 32 bytes

2 REMB -- PSFB (206)

-- -- -- REMB: 145176 * 2^1 = 290352 (1 SSRCs, 3203619403) RTCP PT 206, length: 24 bytes

After which the video starts displaying on the browser.

Please let me know why is there a 12 bytes of PLI initially and what could be the reason for it?

[ireader]

Browser send RTCP/PLI on miss/lost IDR frame. Janus find IDR frame in local memory buffer, or forward PLI to camera.

librtp don't support rtcp/pli extension.

[RashmiAD]

Then how can we resolve this, because due to this PLI there is a delay in displaying video on to the browser.

[ireader]

Make sure your camera streaming start with IDR frame.

Or, config camera video GOP interval 1s(25 frames)

[RashmiAD]

The encoding is done frame by frame and each frame is an IDR frame. Below is the information of single frame.

So, in this case Janus should be able to identify IDR frame as all the frames are IDR frames.

[ireader]

Why ffmpeg output "decodint for stream 0 failed" ?

Could you upload the h264 file?

[RashmiAD]

I have attached the dumped encoded H.264 file and updated ffprobe -show_frames output of that file. ffprobe_show_frames_output.txt h264_frames.txt

[RashmiAD]

h264_frames.txt ffprobe_show_frames_output.txt These are the most recent dumps.

[RashmiAD]

Hi, could you please update your findings regarding this issue?

[ireader]

i got nothing, janus output log. test with your file 'h264_frames.txt'

Got a Janus API request from janus.transport.http (0x7f29c40050c0)
Session 658050493279329 found... returning up to 10 messages
[transports/janus_http.c:janus_http_request_timeout_create:260] [0x7f29c4006200] create (30)
Got a keep-alive on session 658050493279329
Sending Janus API response to janus.transport.http (0x7f29c40050c0)
Got a Janus API response to send (0x7f29c40050c0)
[6118274127759478]  Got an RTCP packet
[6118274127759478] Incoming RTCP, bundling: this is video (no audio has been negotiated)
[6118274127759478]  Got an RTCP packet
[6118274127759478] Incoming RTCP, bundling: this is video (no audio has been negotiated)
[6118274127759478]  Got an RTCP packet
[6118274127759478] Incoming RTCP, bundling: this is video (no audio has been negotiated)
[6118274127759478]  Got an RTCP packet
[6118274127759478] Incoming RTCP, bundling: this is video (no audio has been negotiated)
[6118274127759478]  Got an RTCP packet
[6118274127759478] Incoming RTCP, bundling: this is video (no audio has been negotiated)
[h264-sample] New video stream! (ssrc=1257223418, index 0)
Video SSRC changed, 1972381153 --> 1257223418
Computed offset for video RTP timestamp: 1190301
Video SSRC changed, 0 --> 1257223418
[6118274127759478] Retransmissions will have payload type 127
[6118274127759478]  Got an RTCP packet
[6118274127759478] Incoming RTCP, bundling: this is video (no audio has been negotiated)
[6118274127759478] Incoming RTCP, bundling: this is video (local SSRC: video=1004598754, got 1004598754)
   Parsing compound packet (total of 48 bytes)
     #1 RR (201)
jitter=11.000000, fraction=0, loss=0
       RTCP PT 201, length: 32 bytes
     #2 NACK -- RTPFB (205)
[rtcp.c:janus_rtcp_fix_ssrc:591]         Got 1 nacks
[rtcp.c:janus_rtcp_fix_ssrc:606] [0] 46 / 0000000000000000

       RTCP PT 205, length: 16 bytes
[6118274127759478] Got video RTCP (48 bytes)
[rtcp.c:janus_rtcp_get_nacks:1191]         Got 1 nacks
[rtcp.c:janus_rtcp_get_nacks:1209] [0] 46 / 0000000000000000

[6118274127759478]     Just got some NACKS (1) we should handle...
[ice.c:janus_ice_cb_nice_recv:3000] [6118274127759478]   >> 46
[6118274127759478]   >> >> Scheduling 46 for retransmission due to NACK
[6118274127759478] Retransmitted 1 packets due to NACK (video stream #0)
[6118274127759478]  Got an RTCP packet
Switched incoming RTCP Report Blocks 2507191866(rtx) <--> 1004598754
[6118274127759478] Incoming RTCP, bundling: this is video (no audio has been negotiated)
[6118274127759478] Incoming RTCP, bundling: this is video (local SSRC: video=1004598754, got 1004598754)
   Parsing compound packet (total of 68 bytes)
     #1 RR (201)
jitter=122.000000, fraction=0, loss=0
       RTCP PT 201, length: 56 bytes
     #2 PLI -- PSFB (206)
       RTCP PT 206, length: 12 bytes
[6118274127759478] Got video RTCP (68 bytes)
[6118274127759478]  Got an RTCP packet
Switched incoming RTCP Report Blocks 2507191866(rtx) <--> 1004598754
[6118274127759478] Incoming RTCP, bundling: this is video (no audio has been negotiated)

[RashmiAD]

In the above janus.log file you can see total packet size as 68 bytes out of 12 bytes are PLI (Packet Loss Indicator) when there is PLI then there is no video till it is gone and I get the video when there are total of 56 bytes of RR only.

Parsing compound packet (total of 68 bytes)

1 RR (201)

jitter=122.000000, fraction=0, loss=0 RTCP PT 201, length: 56 bytes

2 PLI -- PSFB (206)

   RTCP PT 206, length: 12 bytes