Output raw bytes on esp ADF pipeline

[ayazalavi]

Hi,

I am trying to get frames from audio connection and write them to esp pipeline as follows:

CHK_STATUS((rtp_transceiver_onFrame(pStreamingSession->pAudioRtcRtpTransceiver, (UINT64) pAppConfiguration->pMediaContext, app_common_onFrame)));

typedef struct {
....
....
...
    AudioElementHandle raw_stream_writer;
    AudioElementHandle encoder;
    AudioElementHandle decoder;
    AudioPipelineHandle pipeline;
    AudioPipelineHandle pipeline_writer;
    AudioEventHandle evt;
} FileSrcContext, *PFileSrcContext;

static void app_common_onFrame(UINT64 userData, PFrame frame) {
    DLOGD("receiving frame of size %d", frame->size); 
    PFileSrcContext pFileSrcContext = (PFileSrcContext) userData;
    if(pFileSrcContext->raw_stream_writer != NULL) {
        raw_stream_write(pFileSrcContext->raw_stream_writer, (char *)frame->frameData, frame->size);
    }
    else {
        DLOGD("no stream writer");
    }
}

 const char *link_tag_[3] = {"raww", "ulawdec", "i2sw"};
    audio_pipeline_link(pFileSrcContext->pipeline_writer, &link_tag_[0], 3);

I need help making it work. Currently I am hearing only squeeks when I try to say anything. I need to make sure I send/receive sound properly from webrtc web app.

[ayazalavi]

Also I am trying to read raw buffer and write send over rtc connection. Below is the code:

PVOID sendAudioPackets(PVOID args)
{
    STATUS_WEBRTC retStatus = STATUS_SUCCESS;
    PFileSrcContext pFileSrcContext = (PFileSrcContext) args;
    PCodecStreamConf pCodecStreamConf = NULL;
    Frame frame;
    UINT32 fileIndex = 0, frameSize;
    PAppConfiguration pAppConfiguration = (PAppConfiguration) pFileSrcContext->mediaSinkHookUserdata;
    PStreamingSession pStreamingSession = NULL;
    PRtcRtpTransceiver pRtcRtpTransceiver = NULL;
    UINT32 i;
    CHK(pAppConfiguration != NULL, STATUS_APP_COMMON_NULL_ARG);
    CHK(pFileSrcContext != NULL, STATUS_MEDIA_NULL_ARG);
    // pCodecStreamConf = &pFileSrcContext->codecConfiguration.audioStream;
    // pCodecStreamConf->pFrameBuffer = NULL;
    // pCodecStreamConf->frameBufferSize = 0;
    frame.presentationTs = 0;
    DLOGD("[ 7 ] Listen for all pipeline events");
    DLOGD("[6.0] Run pipeline");
    audio_pipeline_run(pFileSrcContext->pipeline);
    audio_pipeline_run(pFileSrcContext->pipeline_writer);
    i2s_stream_set_clk(pFileSrcContext->i2s_stream_writer, 8000, 16, 1);
    i2s_stream_set_clk(pFileSrcContext->i2s_stream_reader, 8000, 16, 1);
    while (!ATOMIC_LOAD_BOOL(&pFileSrcContext->shutdownFileSrc)) {
      //  fileIndex = fileIndex % NUMBER_OF_OPUS_FRAME_FILES + 1;
        int buf_size = audio_element_get_output_ringbuf_size(pFileSrcContext->raw_stream_reader);
        //  DLOGD("Bytes read %d", buf_size);
        //  THREAD_SLEEP(FILESRC_AUDIO_FRAME_DURATION);
        //  continue;
        char *buffer = malloc( sizeof(char) * ( buf_size + 1 ) );
        frameSize = raw_stream_read(pFileSrcContext->raw_stream_reader, (char *)buffer, buf_size);
        frame.frameData = (PBYTE)buffer;
        frame.size = frameSize;
        frame.trackId = DEFAULT_AUDIO_TRACK_ID;
        frame.duration = 0;
        MUTEX_LOCK(pAppConfiguration->streamingSessionListReadLock);
        for (i = 0; i < pAppConfiguration->streamingSessionCount; ++i) {
            pStreamingSession = pAppConfiguration->streamingSessionList[i];
            pRtcRtpTransceiver = pStreamingSession->pAudioRtcRtpTransceiver;
            DLOGD("Writing frame to buffer %d", frame->size);
            retStatus = rtp_writeFrame(pRtcRtpTransceiver, &frame);            
            if (retStatus != STATUS_SUCCESS && retStatus != STATUS_SRTP_NOT_READY_YET) {  
                DLOGW("rtp_writeFrame() failed with 0x%08x", retStatus);
                retStatus = STATUS_SUCCESS;
            }

        }
        MUTEX_UNLOCK(pAppConfiguration->streamingSessionListReadLock);
        THREAD_SLEEP(FILESRC_AUDIO_FRAME_DURATION);
    }

CleanUp:

    if(pCodecStreamConf != NULL){
       // SAFE_MEMFREE(pCodecStreamConf->pFrameBuffer);
    }
    CHK_LOG_ERR(retStatus);
    /* free resources */
    DLOGD("terminating media source");
    if (pFileSrcContext->mediaEosHook != NULL) {
        retStatus = pFileSrcContext->mediaEosHook(pFileSrcContext->mediaEosHookUserdata);
    }
    return (PVOID) (ULONG_PTR) retStatus;
}

Any bug in it?

[ayazalavi]

I figured out the issue. Anyone there to discuss?

[ayazalavi]

typedef struct {
    UINT32 version;

    // Id of the frame
    UINT32 index;

    // Flags associated with the frame (ex. IFrame for frames)
    FRAME_FLAGS flags;

    // The decoding timestamp of the frame in 100ns precision
    UINT64 decodingTs;

    // The presentation timestamp of the frame in 100ns precision
    UINT64 presentationTs;

    // The duration of the frame in 100ns precision. Can be 0.
    UINT64 duration;

    // Size of the frame data in bytes
    UINT32 size;

    // The frame bits
    PBYTE frameData;

    // Id of the track this frame belong to
    UINT64 trackId;
} Frame, *PFrame;

In above struct how do you compute presentationTs, decodingTs for real time microphone input? can anyone explain please?