search

ossrs / srs

SRS is a simple, high-efficiency, real-time video server supporting RTMP, WebRTC, HLS, HTTP-FLV, SRT, MPEG-DASH, and GB28181.
https://ossrs.io
MIT License
24.79k stars 5.28k forks source link

RTMP streaming issue with SRS cluster #3819

Open 5centscdn opened 9 months ago

5centscdn commented 9 months ago

Describe the bug

We have configured an SRS cluster for RTMP delivery. However, RTMP streams become stuck when played from the edge SRS cluster. If we publish the streams directly to SRS, they work without any issues. The problem appears to be specific to the cluster setup.

Version

version: 6.0.75

Error Logs from SRS:

[2023-09-29 20:25:48.235][WARN][38060][qn238533][11] EdgeIngester: Ignore error, code=1022(FastStreamGrow)(Fast stream buffer grow failed) : recv message : recv interlaced message : read message payload : read 180455 bytes payload : overflow, required=180455, max=131072, left=130953 thread [38060][qn238533]: ingest() [./src/app/srs_app_edge.cpp:618][errno=11] thread [38060][qn238533]: recv_message() [./src/protocol/srs_protocol_rtmp_stack.cpp:341][errno=11] thread [38060][qn238533]: recv_interlaced_message() [./src/protocol/srs_protocol_rtmp_stack.cpp:825][errno=11] thread [38060][qn238533]: read_message_payload() [./src/protocol/srs_protocol_rtmp_stack.cpp:1208][errno=11] thread [38060][qn238533]: grow() [./src/protocol/srs_protocol_stream.cpp:154][errno=11]

ffprobe output from cluster

ffprobe -hide_banner -v quiet -of json -read_intervals '%+20' -show_error -show_format -show_streams rtmp://10.0.0.10:19351/cctvkkm/cinema { "streams": [ { "index": 0, "codec_type": "data", "codec_tag_string": "[0][0][0][0]", "codec_tag": "0x0000", "r_frame_rate": "0/0", "avg_frame_rate": "0/0", "time_base": "1/1000", "start_pts": 0, "start_time": "0.000000", "disposition": { "default": 0, "dub": 0, "original": 0, "comment": 0, "lyrics": 0, "karaoke": 0, "forced": 0, "hearing_impaired": 0, "visual_impaired": 0, "clean_effects": 0, "attached_pic": 0, "timed_thumbnails": 0, "captions": 0, "descriptions": 0, "metadata": 0, "dependent": 0, "still_image": 0 } }, { "index": 1, "codec_name": "aac", "codec_long_name": "AAC (Advanced Audio Coding)", "profile": "LC", "codec_type": "audio", "codec_tag_string": "[0][0][0][0]", "codec_tag": "0x0000", "sample_fmt": "fltp", "sample_rate": "44100", "channels": 4, "channel_layout": "4.0", "bits_per_sample": 0, "initial_padding": 0, "r_frame_rate": "0/0", "avg_frame_rate": "0/0", "time_base": "1/1000", "start_pts": 9, "start_time": "0.009000", "bit_rate": "128000", "extradata_size": 5, "disposition": { "default": 0, "dub": 0, "original": 0, "comment": 0, "lyrics": 0, "karaoke": 0, "forced": 0, "hearing_impaired": 0, "visual_impaired": 0, "clean_effects": 0, "attached_pic": 0, "timed_thumbnails": 0, "captions": 0, "descriptions": 0, "metadata": 0, "dependent": 0, "still_image": 0 } }, { "index": 2, "codec_name": "h264", "codec_long_name": "H.264 / AVC / MPEG-4 AVC / MPEG-4 part 10", "profile": "High", "codec_type": "video", "codec_tag_string": "[0][0][0][0]", "codec_tag": "0x0000", "width": 720, "height": 576, "coded_width": 720, "coded_height": 576, "closed_captions": 0, "film_grain": 0, "has_b_frames": 0, "sample_aspect_ratio": "64:45", "display_aspect_ratio": "16:9", "pix_fmt": "yuv420p", "level": 30, "chroma_location": "left", "field_order": "progressive", "refs": 1, "is_avc": "true", "nal_length_size": "4", "r_frame_rate": "250/1", "avg_frame_rate": "25/1", "time_base": "1/1000", "start_pts": 3931, "start_time": "3.931000", "bit_rate": "8000000", "bits_per_raw_sample": "8", "extradata_size": 57, "disposition": { "default": 0, "dub": 0, "original": 0, "comment": 0, "lyrics": 0, "karaoke": 0, "forced": 0, "hearing_impaired": 0, "visual_impaired": 0, "clean_effects": 0, "attached_pic": 0, "timed_thumbnails": 0, "captions": 0, "descriptions": 0, "metadata": 0, "dependent": 0, "still_image": 0 } } ], "format": { "filename": "rtmp://10.0.0.10:19351/cctvkkm/cinema", "nb_streams": 3, "nb_programs": 0, "format_name": "flv", "format_long_name": "FLV (Flash Video)", "start_time": "0.000000", "bit_rate": "8128000", "probe_score": 100, "tags": { "|RtmpSampleAccess": "true", "encoder": "Lavf58.76.100", "server": "SRS/6.0.75(Bee)", "timecode": "01:31:40:11", "server_version": "6.0.75" } } }

ffprobe from source stream (no issue with this stream)

ffprobe -hide_banner -v quiet -of json -read_intervals '%+20' -show_error -show_format -show_streams rtmp://10.0.0.21:1935/cctvkkm/cinema { "streams": [ { "index": 0, "codec_name": "aac", "codec_long_name": "AAC (Advanced Audio Coding)", "profile": "LC", "codec_type": "audio", "codec_tag_string": "[0][0][0][0]", "codec_tag": "0x0000", "sample_fmt": "fltp", "sample_rate": "44100", "channels": 4, "channel_layout": "4.0", "bits_per_sample": 0, "initial_padding": 0, "r_frame_rate": "0/0", "avg_frame_rate": "0/0", "time_base": "1/1000", "start_pts": 847522, "start_time": "847.522000", "bit_rate": "128000", "extradata_size": 5, "disposition": { "default": 0, "dub": 0, "original": 0, "comment": 0, "lyrics": 0, "karaoke": 0, "forced": 0, "hearing_impaired": 0, "visual_impaired": 0, "clean_effects": 0, "attached_pic": 0, "timed_thumbnails": 0, "captions": 0, "descriptions": 0, "metadata": 0, "dependent": 0, "still_image": 0 } }, { "index": 1, "codec_name": "h264", "codec_long_name": "H.264 / AVC / MPEG-4 AVC / MPEG-4 part 10", "profile": "High", "codec_type": "video", "codec_tag_string": "[0][0][0][0]", "codec_tag": "0x0000", "width": 720, "height": 576, "coded_width": 720, "coded_height": 576, "closed_captions": 0, "film_grain": 0, "has_b_frames": 0, "sample_aspect_ratio": "64:45", "display_aspect_ratio": "16:9", "pix_fmt": "yuv420p", "level": 30, "chroma_location": "left", "field_order": "progressive", "refs": 1, "is_avc": "true", "nal_length_size": "4", "r_frame_rate": "25/1", "avg_frame_rate": "25/1", "time_base": "1/1000", "start_pts": 848440, "start_time": "848.440000", "bit_rate": "8000000", "bits_per_raw_sample": "8", "extradata_size": 57, "disposition": { "default": 0, "dub": 0, "original": 0, "comment": 0, "lyrics": 0, "karaoke": 0, "forced": 0, "hearing_impaired": 0, "visual_impaired": 0, "clean_effects": 0, "attached_pic": 0, "timed_thumbnails": 0, "captions": 0, "descriptions": 0, "metadata": 0, "dependent": 0, "still_image": 0 } } ], "format": { "filename": "rtmp://10.0.0.21:1935/cctvkkm/cinema", "nb_streams": 2, "nb_programs": 0, "format_name": "flv", "format_long_name": "FLV (Flash Video)", "start_time": "847.522000", "duration": "0.000000", "probe_score": 100, "tags": { "|RtmpSampleAccess": "false", "timecode": "01:31:40:11", "encoder": "Lavf58.76.100" } } }

SRS configuration:

listen 19351; max_connections 1000; daemon on; srs_log_tank file; srs_log_file /usr/local/srs/objs/srs.log; pid /usr/local/srs/objs/srs.pid; http_api { enabled on; listen 10.0.0.10:1985; } vhost defaultVhost { cluster { mode remote; origin 10.0.0.21:1935; } }

Please help us to fix this issue.

winlinvip commented 3 months ago

The SRS Edge server is a regular RTMP client, pulling and consuming stream from any RTMP server. It does not change the stream or insert additional metadata. With currently provided information, reproducing the issue is very difficult.