Closed brenc closed 1 year ago
OK I figured out why this wasn't happening in dev: the playlists on dev are limited to just 100 entries. I copied over the full production playlists and was able to recreate this very odd issue. Here's the config:
settings.request.metadata_decoders.set([])
# Squelch useless ffmpeg errors.
settings.ffmpeg.log.verbosity.set('quiet')
settings.init.allow_root.set(false)
settings.log.file.set(false)
settings.log.level.set(4)
settings.log.stdout.set(true)
settings.request.leak_warning.set(200)
settings.harbor.ssl.private_key.set('/app/star_newgroundsradio_com.key')
settings.harbor.ssl.certificate.set('/app/star_newgroundsradio_com.cert')
settings.prometheus.server.set(true)
settings.prometheus.server.port.set(8002)
settings.server.telnet.set(true)
settings.server.telnet.port.set(8001)
%include "secrets.liq"
let ngr = ()
let ngr.inputs = ()
##
## INPUTS
##
let ngr.inputs.harbor_ssl = input.harbor.ssl(
"radio",
buffer=6.0,
port=8000,
timeout=10.0,
user=icecast_mount_radio_usr,
password=icecast_mount_radio_psw,
)
let ngr.inputs.harbor_ssl = normalize(
debug=1.0,
down=0.5,
lookahead=2.0,
# lufs=true,
# target=-24.0,
window=4.0,
ngr.inputs.harbor_ssl
)
let ngr.inputs.harbor_ssl = blank.strip(max_blank=10.0, ngr.inputs.harbor_ssl)
##
## TRANSITIONS
##
let ngr.transitions = ()
def ngr.transitions.to_auto_dj(_, b)
add(
normalize=false,
[
once(single('/ngradio/jingles/piper_welcome_1.mp3')),
fade.in(duration=6.0, type='sin', b),
]
)
end
def ngr.transitions.to_live(a, b)
add(
normalize=false,
[
fade.out(duration=3.0, type='sin', a),
fade.in(duration=3.0, type='sin', b),
]
)
end
##
## PLAYLISTS
##
jingles = playlist(
'/ngradio/jingles/jingles_electronic.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=1,
reload_mode='rounds',
)
chipstepnewhigh = playlist(
'/ngradio/playlists/chipstep_newhigh.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
chipstepnewlow = playlist(
'/ngradio/playlists/chipstep_newlow.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
chipstepoldhigh = playlist(
'/ngradio/playlists/chipstep_oldhigh.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
chipstepoldlow = playlist(
'/ngradio/playlists/chipstep_oldlow.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
chipstep = random(
[
chipstepnewlow,
chipstepoldlow,
chipstepoldhigh,
chipstepnewhigh,
],
weights = [
1,
1,
10,
20,
],
)
dancenewhigh = playlist(
'/ngradio/playlists/dance_newhigh.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
dancenewlow = playlist(
'/ngradio/playlists/dance_newlow.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
danceoldhigh = playlist(
'/ngradio/playlists/dance_oldhigh.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
danceoldlow = playlist(
'/ngradio/playlists/dance_oldlow.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
dance = random(
[
dancenewlow,
danceoldlow,
danceoldhigh,
dancenewhigh
],
weights = [
1,
1,
10,
20,
],
)
drumnbassnewhigh = playlist(
'/ngradio/playlists/drum-n-bass_newhigh.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
drumnbassnewlow = playlist(
'/ngradio/playlists/drum-n-bass_newlow.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
drumnbassoldhigh = playlist(
'/ngradio/playlists/drum-n-bass_oldhigh.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
drumnbassoldlow = playlist(
'/ngradio/playlists/drum-n-bass_oldlow.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
drumnbass = random(
[
drumnbassnewlow,
drumnbassoldlow,
drumnbassoldhigh,
drumnbassnewhigh
],
weights = [
1,
1,
10,
20,
],
)
dubstepnewhigh = playlist(
'/ngradio/playlists/dubstep_newhigh.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
dubstepnewlow = playlist(
'/ngradio/playlists/dubstep_newlow.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
dubstepoldhigh = playlist(
'/ngradio/playlists/dubstep_oldhigh.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
dubstepoldlow = playlist(
'/ngradio/playlists/dubstep_oldlow.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
dubstep = random(
[
dubstepnewlow,
dubstepoldlow,
dubstepoldhigh,
dubstepnewhigh
],
weights = [
1,
1,
10,
20,
],
)
housenewhigh = playlist(
'/ngradio/playlists/house_newhigh.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
housenewlow = playlist(
'/ngradio/playlists/house_newlow.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
houseoldhigh = playlist(
'/ngradio/playlists/house_oldhigh.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
houseoldlow = playlist(
'/ngradio/playlists/house_oldlow.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
house = random(
[
housenewlow,
houseoldlow,
houseoldhigh,
housenewhigh,
],
weights = [
1,
1,
10,
20,
],
)
industrialnewhigh = playlist(
'/ngradio/playlists/industrial_newhigh.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
industrialnewlow = playlist(
'/ngradio/playlists/industrial_newlow.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
industrialoldhigh = playlist(
'/ngradio/playlists/industrial_oldhigh.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
industrialoldlow = playlist(
'/ngradio/playlists/industrial_oldlow.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
industrial = random(
[
industrialnewlow,
industrialoldlow,
industrialoldhigh,
industrialnewhigh,
],
weights = [
1,
1,
10,
20,
],
)
newwavenewhigh = playlist(
'/ngradio/playlists/new-wave_newhigh.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
newwavenewlow = playlist(
'/ngradio/playlists/new-wave_newlow.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
newwaveoldhigh = playlist(
'/ngradio/playlists/new-wave_oldhigh.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
newwaveoldlow = playlist(
'/ngradio/playlists/new-wave_oldlow.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
newwave = random(
[
newwavenewlow,
newwaveoldlow,
newwaveoldhigh,
newwavenewhigh,
],
weights = [
1,
1,
10,
20,
],
)
technonewhigh = playlist(
'/ngradio/playlists/techno_newhigh.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
technonewlow = playlist(
'/ngradio/playlists/techno_newlow.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
technooldhigh = playlist(
'/ngradio/playlists/techno_oldhigh.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
technooldlow = playlist(
'/ngradio/playlists/techno_oldlow.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
techno = random(
[
technonewlow,
technooldlow,
technooldhigh,
technonewhigh,
],
weights = [
1,
1,
10,
20,
],
)
trancenewhigh = playlist(
'/ngradio/playlists/trance_newhigh.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
trancenewlow = playlist(
'/ngradio/playlists/trance_newlow.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
tranceoldhigh = playlist(
'/ngradio/playlists/trance_oldhigh.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
tranceoldlow = playlist(
'/ngradio/playlists/trance_oldlow.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
trance = random(
[
trancenewlow,
tranceoldlow,
tranceoldhigh,
trancenewhigh,
],
weights = [
1,
1,
10,
20,
],
)
videogamenewhigh = playlist(
'/ngradio/playlists/video-game_newhigh.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
videogamenewlow = playlist(
'/ngradio/playlists/video-game_newlow.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
videogameoldhigh = playlist(
'/ngradio/playlists/video-game_oldhigh.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
videogameoldlow = playlist(
'/ngradio/playlists/video-game_oldlow.m3u',
mime_type='application/x-mpegURL',
mode='randomize',
reload=86400,
)
videogame = random(
[
videogamenewlow,
videogameoldlow,
videogameoldhigh,
videogamenewhigh,
],
weights = [
1,
1,
10,
20,
],
)
##
## Station Mixes
##
electronic = random(
[
chipstep,
dance,
drumnbass,
dubstep,
house,
industrial,
newwave,
techno,
trance,
videogame,
],
weights=[
1, # chipstep
10, # dance
10, # drumnbass
5, # dubstep
5, # house
5, # industrial
1, # newwave
20, # techno
15, # trance
25, # videogame
],
)
##
## autodj
##
let ngr.autodj = random(
[
electronic,
],
weights = [
10, # electronic
],
)
let ngr.autodj = amplify(
# Base multiplicative factor.
1.0,
ngr.autodj,
override='replay_gain',
)
let ngr.autodj = blank.skip(ngr.autodj)
let ngr.autodj = fallback(
[
request.queue(id='request'),
delay(900.0, jingles),
ngr.autodj,
]
)
let ngr.autodj_last_metadata= ref([])
ngr.autodj.on_track(fun (m) -> ngr.autodj_last_metadata := m)
##
## RADIO
##
let ngr.radio = fallback(
track_sensitive = false,
transition_length = 10.0,
transitions = [
ngr.transitions.to_live,
ngr.transitions.to_auto_dj,
],
[
ngr.inputs.harbor_ssl,
ngr.autodj,
],
)
let ngr.radio = rms(ngr.radio)
let ngr.rms = ngr.radio.rms
let ngr.radio = lufs(ngr.radio)
let ngr.lufs = ngr.radio.lufs
let ngr.radio = bpm(ngr.radio)
let ngr.bpm = ngr.radio.bpm
let ngr.radio = mksafe(ngr.radio)
let ngr.radio = insert_metadata(ngr.radio)
##
## OUTPUTS
##
let ngr.outputs = ()
let ngr.outputs.aac_lofi = %ffmpeg(
format="mp4",
movflags="+dash+skip_sidx+skip_trailer+frag_custom",
frag_duration=10,
%audio(
codec="aac",
channels=2,
ar=44100,
b="32k",
)
)
let ngr.outputs.aac_midfi = %ffmpeg(
format="mp4",
movflags="+dash+skip_sidx+skip_trailer+frag_custom",
frag_duration=10,
%audio(
codec="aac",
channels=2,
ar=44100,
b="64k",
)
)
let ngr.outputs.aac_hifi = %ffmpeg(
format="mp4",
movflags="+dash+skip_sidx+skip_trailer+frag_custom",
frag_duration=10,
%audio(
codec="aac",
channels=2,
ar=44100,
b="128k",
)
)
def ngr.outputs.hls_segment_name(~position, ~extname, stream_name)
timestamp = int_of_float(time())
duration = 2
"#{stream_name}_#{duration}_#{timestamp}_#{position}.#{extname}"
end
output.file.hls(
persist_at="state.config",
playlist="radio.m3u8",
segments=5,
segment_duration=2.0,
segments_overhead=5,
segment_name=ngr.outputs.hls_segment_name,
"/ngradio/hls/aac",
[
("aac_lofi", ngr.outputs.aac_lofi),
("aac_midfi", ngr.outputs.aac_midfi),
("aac_hifi", ngr.outputs.aac_hifi),
],
ngr.radio
)
Here are the lines counts of all the referenced playlists:
0 chipstep_newlow.m3u
0 new-wave_newlow.m3u
1 chipstep_newhigh.m3u
1 industrial_newlow.m3u
1 new-wave_newhigh.m3u
1 trance_newlow.m3u
3 industrial_newhigh.m3u
4 drum-n-bass_newlow.m3u
5 house_newlow.m3u
7 techno_newlow.m3u
9 dubstep_newlow.m3u
10 techno_newhigh.m3u
11 trance_newhigh.m3u
12 dance_newlow.m3u
12 video-game_newlow.m3u
19 house_newhigh.m3u
23 drum-n-bass_newhigh.m3u
31 dance_newhigh.m3u
36 dubstep_newhigh.m3u
36 video-game_newhigh.m3u
330 chipstep_oldlow.m3u
1067 new-wave_oldlow.m3u
1953 new-wave_oldhigh.m3u
1996 chipstep_oldhigh.m3u
2408 industrial_oldlow.m3u
4346 industrial_oldhigh.m3u
4735 house_oldlow.m3u
5216 dubstep_oldlow.m3u
6312 drum-n-bass_oldlow.m3u
8011 dance_oldlow.m3u
10386 trance_oldlow.m3u
10950 video-game_oldlow.m3u
12042 house_oldhigh.m3u
14591 drum-n-bass_oldhigh.m3u
16331 dubstep_oldhigh.m3u
16371 techno_oldlow.m3u
17594 trance_oldhigh.m3u
18995 dance_oldhigh.m3u
21671 techno_oldhigh.m3u
37854 video-game_oldhigh.m3u
213381 total
So it seems that the length of the playlists is somehow affecting this. If I use the same playlists with 100 tracks or less this doesn't happen.
Here's what happens just before the crash:
[switch_25:3] Switch to blank.strip_0 with transition.
[source:4] Source sequence_0 gets up with content kind: {audio=pcm(stereo),video=none,midi=none}.
[source:4] Source max_duration_0 gets up with content kind: {audio=pcm(stereo),video=none,midi=none}.
[source:4] Source add_0 gets up with content kind: {audio=pcm(stereo),video=none,midi=none}.
[add_0:4] Content type is {audio=pcm(stereo),video=none,midi=none}.
[source:4] Source fade.out gets up with content kind: {audio=pcm(stereo),video=none,midi=none}.
[fade.out:4] Content type is {audio=pcm(stereo),video=none,midi=none}.
[source:4] Source fade.out gets up with content kind: {audio=pcm(stereo),video=none,midi=none}.
[fade.out:4] Content type is {audio=pcm(stereo),video=none,midi=none}.
[source:4] Source on_end_0 gets up with content kind: {audio=pcm(stereo),video=none,midi=none}.
[on_end_0:4] Content type is {audio=pcm(stereo),video=none,midi=none}.
[source:4] Source on_track_13 gets up with content kind: {audio=pcm(stereo),video=none,midi=none}.
[on_track_13:4] Content type is {audio=pcm(stereo),video=none,midi=none}.
[source:4] Source on_metadata_0 gets up with content kind: {audio=pcm(stereo),video=none,midi=none}.
[on_metadata_0:4] Content type is {audio=pcm(stereo),video=none,midi=none}.
[source:4] Source fade.in gets up with content kind: {audio=pcm(stereo),video=none,midi=none}.
[fade.in:4] Content type is {audio=pcm(stereo),video=none,midi=none}.
[source:4] Source fade.in gets up with content kind: {audio=pcm(stereo),video=none,midi=none}.
[fade.in:4] Content type is {audio=pcm(stereo),video=none,midi=none}.
[source:4] Source on_track_14 gets up with content kind: {audio=pcm(stereo),video=none,midi=none}.
[on_track_14:4] Content type is {audio=pcm(stereo),video=none,midi=none}.
[source:4] Source on_metadata_1 gets up with content kind: {audio=pcm(stereo),video=none,midi=none}.
[on_metadata_1:4] Content type is {audio=pcm(stereo),video=none,midi=none}.
[blank.strip_0:4] Fading in (type: sin, duration: 3.s).
[clock.main:4] Source /ngradio/hls/aac/radio.m3u8 failed while streaming: Avutil.Error(Invalid argument)!
Hi! Any chance you could reproduce with the ffmpeg logs enabled? You can add this to your script:
settings.ffmpeg.log.verbosity.set("info")
Yep. Here's the output before and during the crash:
[switch_25:3] Switch to blank.strip_0 with transition.
[source:4] Source sequence_0 gets up with content kind: {audio=pcm(stereo),video=none,midi=none}.
[source:4] Source max_duration_0 gets up with content kind: {audio=pcm(stereo),video=none,midi=none}.
[source:4] Source add_0 gets up with content kind: {audio=pcm(stereo),video=none,midi=none}.
[add_0:4] Content type is {audio=pcm(stereo),video=none,midi=none}.
[source:4] Source fade.out gets up with content kind: {audio=pcm(stereo),video=none,midi=none}.
[fade.out:4] Content type is {audio=pcm(stereo),video=none,midi=none}.
[source:4] Source fade.out gets up with content kind: {audio=pcm(stereo),video=none,midi=none}.
[fade.out:4] Content type is {audio=pcm(stereo),video=none,midi=none}.
[source:4] Source on_end_0 gets up with content kind: {audio=pcm(stereo),video=none,midi=none}.
[on_end_0:4] Content type is {audio=pcm(stereo),video=none,midi=none}.
[source:4] Source on_track_13 gets up with content kind: {audio=pcm(stereo),video=none,midi=none}.
[on_track_13:4] Content type is {audio=pcm(stereo),video=none,midi=none}.
[source:4] Source on_metadata_0 gets up with content kind: {audio=pcm(stereo),video=none,midi=none}.
[on_metadata_0:4] Content type is {audio=pcm(stereo),video=none,midi=none}.
[source:4] Source fade.in gets up with content kind: {audio=pcm(stereo),video=none,midi=none}.
[fade.in:4] Content type is {audio=pcm(stereo),video=none,midi=none}.
[source:4] Source fade.in gets up with content kind: {audio=pcm(stereo),video=none,midi=none}.
[fade.in:4] Content type is {audio=pcm(stereo),video=none,midi=none}.
[source:4] Source on_track_14 gets up with content kind: {audio=pcm(stereo),video=none,midi=none}.
[on_track_14:4] Content type is {audio=pcm(stereo),video=none,midi=none}.
[source:4] Source on_metadata_1 gets up with content kind: {audio=pcm(stereo),video=none,midi=none}.
[on_metadata_1:4] Content type is {audio=pcm(stereo),video=none,midi=none}.
[blank.strip_0:4] Fading in (type: sin, duration: 3.s).
[aac @ 0x5584142d8040] Input contains (near) NaN/+-Inf
[aac @ 0x5584142d8040] Input contains (near) NaN/+-Inf
[clock.main:4] Source /ngradio/hls/aac/radio.m3u8 failed while streaming: Avutil.Error(Invalid argument)!
[clock.main:4] Raised by primitive operation at Avfilter.Utils.convert_audio.flush in file "avfilter/avfilter.ml", line 496, characters 8-26
[clock.main:4] Called from Avfilter.Utils.convert_audio.flush in file "avfilter/avfilter.ml", line 497, characters 8-16
[clock.main:4] Called from Stdlib__List.iter in file "list.ml", line 110, characters 12-15
[clock.main:4] Called from Stdlib__Option.map in file "option.ml", line 24, characters 57-62
[clock.main:4] Called from Ffmpeg_encoder_common.encode in file "encoder/ffmpeg_encoder_common.ml", line 103, characters 4-80
[clock.main:4] Called from Ffmpeg_encoder_common.encoder.encode in file "encoder/ffmpeg_encoder_common.ml", line 226, characters 4-44
[clock.main:4] Called from Hls_output.hls_output#encode.(fun) in file "outputs/hls_output.ml", line 759, characters 33-63
[clock.main:4] Called from Stdlib__List.map in file "list.ml", line 92, characters 20-23
[clock.main:4] Called from Output.encoded#send_frame.output_chunks.f in file "outputs/output.ml", line 239, characters 21-59
[clock.main:4] Called from Output.output#output in file "outputs/output.ml", line 174, characters 10-35
[clock.main:4] Called from Clock.clock#end_tick.(fun) in file "clock.ml", line 299, characters 14-22
Thanks. Clearly, the issue comes from these lines:
[aac @ 0x5584142d8040] Input contains (near) NaN/+-Inf
[aac @ 0x5584142d8040] Input contains (near) NaN/+-Inf
I'm gonna filter out these values in the ffmpeg binding and will report back to you when it's ready for another test.
Hi @brenc. Just pushed a fixed to ocaml-ffmpeg
filtering NaN
when converting to and from OCaml. Are you able to test it? How do you install liquidsoap?
I use Docker so I can test it once it reaches rolling release!
Ok! Gonna do a rebuild of the rolling release then. Which image in particular?
v2.1.x
I mean which architecture/OS: amd64/arm64/armhf and debian or alpine?
amd64 and Debian.
Coming up!
Looks like this is fixed in the latest rolling release. Thanks!
Great!
Describe the bug This one is a weird one. I've been trying to phase out Icecast in favor of HLS. This is one I can't seem to recreate in development for some odd reason (using docker-compose). This only happens in staging and production which is using Docker Swarm. Absolutely everything else is as identical as possible.
The script starts up and starts producing HLS files and connects to Icecast. But seconds after I connect to input.harbor/ssl, ls crashes:
Then #2630 happens and the process has to be restarted manually because it gets caught in a loop (I'm using the same basic script I posted there).
To Reproduce This is the tough part. I can only reproduce this on a Docker Swarm for some reason. I'm still working on getting a minimal test case / reproduction of this.
Expected behavior I should be able to connect to input.harbor etc. and not have ls crash.
Version details