Split video input into video and audio, work on both video and audio frame by frame, save updated frames into separate output files

[codekrolik2]

Hello guys, I have this prototype and I'm trying to create 2 pipe outputs for video and audio, but video part doesn't work. Do you know how this problem could be solved? Audio works as is, and if you comment out audio-related stuff, video works, too.

import ffmpeg
import numpy as np

if __name__ == '__main__':
    # Input opts for ffmpeg.probe() and ffmpeg.input()
    input_opts = {'threads': 1, 'fflags': 'nobuffer'}

    # Probe - discover parameters of the stream
    rtsp_stream_link = 'rtsp://SOME_LINK'
    probe = ffmpeg.probe(rtsp_stream_link, **input_opts)

    # Get video stream info (video frame width, height)
    video_stream_info = None
    for substream in probe['streams']:
        if substream['codec_type'] == 'video':
            video_stream_info = substream
    width = int(video_stream_info['width'])
    height = int(video_stream_info['height'])

    # Open RTSP stream
    rtsp_stream = ffmpeg.input(rtsp_stream_link, **input_opts)

    video_output_opts = {'f': 'rawvideo', 'pix_fmt': 'rgb24'}
    rtsp_video_pipe_output = (
        rtsp_stream.video
        .output('pipe:v', **video_output_opts)
        .global_args('-y', '-loglevel', 'panic')
        .run_async(pipe_stdout=True)
    )

    save_output_opts = {'vcodec': 'h264'}
    video_file_output = (
        ffmpeg
        .input('pipe:v', format='rawvideo', pix_fmt='rgb24', s='{}x{}'.format(width, height))
        .output('video_out.mp4', **save_output_opts)
        .overwrite_output()
        .run_async(pipe_stdin=True)
    )

    rtsp_audio_pipe_output = (
        rtsp_stream.audio
        .output('pipe:a', format='f32le', acodec='pcm_s32le', ac=1, ar='48k')
        .global_args('-y', '-loglevel', 'panic')
        .run_async(pipe_stdout=True)
   )

    save_output_opts = {'loglevel': 'error', 'codec:a': 'libmp3lame', 'f': 'mp3'}
    audio_file_output = (
        ffmpeg
        .input('pipe:a', format='f32le', acodec='pcm_s32le', ac=1, ar='48k')
        .output('audio_out.mp3', **save_output_opts)
        .overwrite_output()
        .run_async(pipe_stdin=True)
    )

    start_timestamp_sec = 0
    while True:
        # Video frame bytes logic
        poll_video_data = rtsp_video_pipe_output.poll()
        if poll_video_data is None:
            # Read bytes from pipe_output
            in_video_bytes = rtsp_video_pipe_output.stdout.read(width * height * 3)
            if in_video_bytes:
                # Transform bytes to pixels
                pixels = (
                    np.frombuffer(in_video_bytes, np.uint8)
                    .reshape([height, width, 3])
                )

                video_file_output.stdin.write(
                    pixels
                    .astype(np.uint8)
                    .tobytes()
                )

        # Audio frame bytes logic
        poll_audio_data = rtsp_audio_pipe_output.poll()
        if poll_audio_data is None:
            in_audio_bytes = rtsp_audio_pipe_output.stdout.read(4*48000)
            if in_audio_bytes:
                if len(in_audio_bytes) == 4*48000:
                    audio_file_output.stdin.write(in_audio_bytes)

[codekrolik2]

The following processes are created:

Video only:

john      165227  8.4  0.2 375456 33984 pts/0    Sl+  20:27   0:02 /home/john/PycharmProjects/RtspStreamer/.venv/bin/python /home/john/PycharmProjects/RtspStreamer/ffmpeg_approach_combined.py
john      165254  6.4  0.3 295864 54124 pts/0    SLl+ 20:27   0:01 ffmpeg -fflags nobuffer -threads 1 -i rtsp://rtspstream:48aa741b4fe778cfb4235270bb11bbb4@zephyr.rtsp.stream/movie -map 0:v -f rawvideo -pix_fmt rgb24 pipe:v -y -loglevel panic
john      165255 49.4  1.6 799336 266712 pts/0   SLl+ 20:27   0:11 ffmpeg -f rawvideo -pix_fmt rgb24 -s 720x480 -i pipe:v -vcodec h264 video_out.mp4 -y

Audio only:

john      166208 13.5  0.2 373804 33244 pts/0    Sl+  20:30   0:00 /home/john/PycharmProjects/RtspStreamer/.venv/bin/python /home/john/PycharmProjects/RtspStreamer/ffmpeg_approach_combined.py
john      166228  1.2  0.2 288700 47796 pts/0    SLl+ 20:30   0:00 ffmpeg -fflags nobuffer -threads 1 -i rtsp://rtspstream:48aa741b4fe778cfb4235270bb11bbb4@zephyr.rtsp.stream/movie -map 0:a -f f32le -ac 1 -acodec pcm_s32le -ar 48k pipe:a -y -loglevel panic
john      166229  0.7  0.2 221456 42752 pts/0    SL+  20:30   0:00 ffmpeg -f f32le -ac 1 -acodec pcm_s32le -ar 48k -i pipe:a -f mp3 -codec:a libmp3lame -loglevel error audio_out.mp3 -y

Video and Audio:

john      166284  8.3  0.2 374872 34536 pts/0    Sl+  20:31   0:00 /home/john/PycharmProjects/RtspStreamer/.venv/bin/python /home/john/PycharmProjects/RtspStreamer/ffmpeg_approach_combined.py
john      166293  1.2  0.3 294724 54708 pts/0    SLl+ 20:31   0:00 ffmpeg -fflags nobuffer -threads 1 -i rtsp://rtspstream:48aa741b4fe778cfb4235270bb11bbb4@zephyr.rtsp.stream/movie -map 0:v -f rawvideo -pix_fmt rgb24 pipe:v -y -loglevel panic
john      166294  0.7  0.4 558808 65628 pts/0    SLl+ 20:31   0:00 ffmpeg -f rawvideo -pix_fmt rgb24 -s 720x480 -i pipe:v -vcodec h264 video_out.mp4 -y
john      166295  1.1  0.2 289944 47196 pts/0    SLl+ 20:31   0:00 ffmpeg -fflags nobuffer -threads 1 -i rtsp://rtspstream:48aa741b4fe778cfb4235270bb11bbb4@zephyr.rtsp.stream/movie -map 0:a -f f32le -ac 1 -acodec pcm_s32le -ar 48k pipe:a -y -loglevel panic
john      166296  1.5  0.2 287884 43008 pts/0    SLl+ 20:31   0:00 ffmpeg -f f32le -ac 1 -acodec pcm_s32le -ar 48k -i pipe:a -f mp3 -codec:a libmp3lame -loglevel error audio_out.mp3 -y

[codekrolik2]

Named pipes can be used as a workaround