search

warren-bank / HLS-Proxy

Node.js server to proxy HLS video streams
http://webcast-reloaded.surge.sh/proxy.html
GNU General Public License v2.0
238 stars 68 forks source link

How to limit segment on playlist. #43

Open siberkolosis opened 4 months ago

siberkolosis commented 4 months ago

sometimes live video streaming has 2k segments in the playlist. The advantage is that the video can be rewinded. however the m3u gets very large at around 1MB. I want it to only have 5 or 6 segments, is that possible?

warren-bank commented 4 months ago

it isn't elegant.. but you could quickly and easily write something yourself to shorten your playlists using a hook function.

ex: hlsd --hooks /path/to/hooks.js

// hooks.js

module.exports = {
  "modify_m3u8_content": function(m3u8_content, m3u8_url) {
    // if (!some_conditional_test(m3u8_url)) return m3u8_content

    const max_segments = 6
    const token = "\nhttp"
    const all_segments = m3u8_content.split(token)

    return all_segments.shift() + token + all_segments.slice(-1 * max_segments).join(token)
  }
}

note: i haven't tested this code.. just a quick scribble.. but it looks about right

warren-bank commented 4 months ago

actually.. on 2nd thought.. that previous example assumes that the manifest uses absolute URLs. a better example might be:

// hooks.js

module.exports = {
  "modify_m3u8_content": function(m3u8_content, m3u8_url) {
    // if (!some_conditional_test(m3u8_url)) return m3u8_content

    const max_segments = 6
    const token = "\n#EXTINF:"
    const all_segments = m3u8_content.split(token)

    return all_segments.shift() + token + all_segments.slice(-1 * max_segments).join(token)
  }
}
warren-bank commented 4 months ago

unit test:

{

  const m3u8_url = 'https://docs.aws.amazon.com/mediatailor/latest/ug/manifest-hls-example.html'
  const m3u8_content = `#EXTM3U
#EXT-X-VERSION:3
#EXT-X-TARGETDURATION:7
#EXT-X-MEDIA-SEQUENCE:8779957
#EXTINF:6.006,
index_1_8779957.ts?m=1566416212
#EXTINF:6.006,
index_1_8779958.ts?m=1566416212
#EXTINF:5.372,
index_1_8779959.ts?m=1566416212
#EXT-OATCLS-SCTE35:/DAlAAAAAsvhAP/wFAXwAAAGf+/+AdLfiP4AG3dAAAEBAQAAXytxmQ==
#EXT-X-CUE-OUT:20.020
#EXTINF:0.634,
index_1_8779960.ts?m=1566416212
#EXT-X-CUE-OUT-CONT:ElapsedTime=0.634,Duration=21,SCTE35=/DAlAAAAAsvhAP/wFAXwAAAGf+/+AdLfiP4AG3dAAAEBAQAAXytxmQ==
#EXTINF:6.006,
index_1_8779961.ts?m=1566416212
#EXT-X-CUE-OUT-CONT:ElapsedTime=6.640,Duration=21,SCTE35=/DAlAAAAAsvhAP/wFAXwAAAGf+/+AdLfiP4AG3dAAAEBAQAAXytxmQ==
#EXTINF:6.006,
index_1_8779962.ts?m=1566416212
#EXT-X-CUE-OUT-CONT:ElapsedTime=12.646,Duration=21,SCTE35=/DAlAAAAAsvhAP/wFAXwAAAGf+/+AdLfiP4AG3dAAAEBAQAAXytxmQ==
#EXTINF:6.006,
index_1_8779963.ts?m=1566416212
#EXT-X-CUE-OUT-CONT:ElapsedTime=18.652,Duration=21,SCTE35=/DAlAAAAAsvhAP/wFAXwAAAGf+/+AdLfiP4AG3dAAAEBAQAAXytxmQ==
#EXTINF:1.368,
index_1_8779964.ts?m=1566416212
#EXT-X-CUE-IN
#EXTINF:4.638,
index_1_8779965.ts?m=1566416212
#EXTINF:6.006,
index_1_8779966.ts?m=1566416212
#EXTINF:6.006,
index_1_8779967.ts?m=1566416212
#EXTINF:6.006,
index_1_8779968.ts?m=1566416212`

  const modify_m3u8_content = function(m3u8_content, m3u8_url) {
    // if (!some_conditional_test(m3u8_url)) return m3u8_content

    const max_segments = 6
    const token = "\n#EXTINF:"
    const all_segments = m3u8_content.split(token)

    return all_segments.shift() + token + all_segments.slice(-1 * max_segments).join(token)
  }

  console.log(
    modify_m3u8_content(m3u8_content, m3u8_url)
  )
}

outputs:

#EXTM3U
#EXT-X-VERSION:3
#EXT-X-TARGETDURATION:7
#EXT-X-MEDIA-SEQUENCE:8779957
#EXTINF:6.006,
index_1_8779963.ts?m=1566416212
#EXT-X-CUE-OUT-CONT:ElapsedTime=18.652,Duration=21,SCTE35=/DAlAAAAAsvhAP/wFAXwAAAGf+/+AdLfiP4AG3dAAAEBAQAAXytxmQ==
#EXTINF:1.368,
index_1_8779964.ts?m=1566416212
#EXT-X-CUE-IN
#EXTINF:4.638,
index_1_8779965.ts?m=1566416212
#EXTINF:6.006,
index_1_8779966.ts?m=1566416212
#EXTINF:6.006,
index_1_8779967.ts?m=1566416212
#EXTINF:6.006,
index_1_8779968.ts?m=1566416212

success.

siberkolosis commented 4 months ago

thanks...

warren-bank commented 4 months ago

I don't know if those "#EXT-X-CUE-*" tags could be problematic. Since they (or possibly other tags) may be, I tweaked the example.. this one shows you how you could filter unwanted tags and only keep the ones that you're interested in:

// hooks.js

const filter_partial_playlist = function(pp) {
  const token = '#EXTINF:'
  const all_lines = pp.split(/[\r\n]+/g)

  return all_lines
    .filter(function(line) {
      return (!line || (line[0] !== '#') || line.startsWith(token))
    })
    .join("\n")
}

const modify_m3u8_content = function(m3u8_content, m3u8_url) {
  // if (!some_conditional_test(m3u8_url)) return m3u8_content

  const max_segments = 6
  const token = "\n#EXTINF:"
  const all_segments = m3u8_content.split(token)

  return all_segments.shift() + token + filter_partial_playlist( all_segments.slice(-1 * max_segments).join(token) )
}

module.exports = {
  modify_m3u8_content
}

unit test:

{

  const m3u8_url = 'https://docs.aws.amazon.com/mediatailor/latest/ug/manifest-hls-example.html'
  const m3u8_content = `#EXTM3U
#EXT-X-VERSION:3
#EXT-X-TARGETDURATION:7
#EXT-X-MEDIA-SEQUENCE:8779957
#EXTINF:6.006,
index_1_8779957.ts?m=1566416212
#EXTINF:6.006,
index_1_8779958.ts?m=1566416212
#EXTINF:5.372,
index_1_8779959.ts?m=1566416212
#EXT-OATCLS-SCTE35:/DAlAAAAAsvhAP/wFAXwAAAGf+/+AdLfiP4AG3dAAAEBAQAAXytxmQ==
#EXT-X-CUE-OUT:20.020
#EXTINF:0.634,
index_1_8779960.ts?m=1566416212
#EXT-X-CUE-OUT-CONT:ElapsedTime=0.634,Duration=21,SCTE35=/DAlAAAAAsvhAP/wFAXwAAAGf+/+AdLfiP4AG3dAAAEBAQAAXytxmQ==
#EXTINF:6.006,
index_1_8779961.ts?m=1566416212
#EXT-X-CUE-OUT-CONT:ElapsedTime=6.640,Duration=21,SCTE35=/DAlAAAAAsvhAP/wFAXwAAAGf+/+AdLfiP4AG3dAAAEBAQAAXytxmQ==
#EXTINF:6.006,
index_1_8779962.ts?m=1566416212
#EXT-X-CUE-OUT-CONT:ElapsedTime=12.646,Duration=21,SCTE35=/DAlAAAAAsvhAP/wFAXwAAAGf+/+AdLfiP4AG3dAAAEBAQAAXytxmQ==
#EXTINF:6.006,
index_1_8779963.ts?m=1566416212
#EXT-X-CUE-OUT-CONT:ElapsedTime=18.652,Duration=21,SCTE35=/DAlAAAAAsvhAP/wFAXwAAAGf+/+AdLfiP4AG3dAAAEBAQAAXytxmQ==
#EXTINF:1.368,
index_1_8779964.ts?m=1566416212
#EXT-X-CUE-IN
#EXTINF:4.638,
index_1_8779965.ts?m=1566416212
#EXTINF:6.006,
index_1_8779966.ts?m=1566416212
#EXTINF:6.006,
index_1_8779967.ts?m=1566416212
#EXTINF:6.006,
index_1_8779968.ts?m=1566416212`

  const filter_partial_playlist = function(pp) {
    const token = '#EXTINF:'
    const all_lines = pp.split(/[\r\n]+/g)

    return all_lines
      .filter(function(line) {
        return (!line || (line[0] !== '#') || line.startsWith(token))
      })
      .join("\n")
  }

  const modify_m3u8_content = function(m3u8_content, m3u8_url) {
    // if (!some_conditional_test(m3u8_url)) return m3u8_content

    const max_segments = 6
    const token = "\n#EXTINF:"
    const all_segments = m3u8_content.split(token)

    return all_segments.shift() + token + filter_partial_playlist( all_segments.slice(-1 * max_segments).join(token) )
  }

  console.log(
    modify_m3u8_content(m3u8_content, m3u8_url)
  )
}

outputs:

#EXTM3U
#EXT-X-VERSION:3
#EXT-X-TARGETDURATION:7
#EXT-X-MEDIA-SEQUENCE:8779957
#EXTINF:6.006,
index_1_8779963.ts?m=1566416212
#EXTINF:1.368,
index_1_8779964.ts?m=1566416212
#EXTINF:4.638,
index_1_8779965.ts?m=1566416212
#EXTINF:6.006,
index_1_8779966.ts?m=1566416212
#EXTINF:6.006,
index_1_8779967.ts?m=1566416212
#EXTINF:6.006,
index_1_8779968.ts?m=1566416212

success.

siberkolosis commented 4 months ago

working perfect on my case. problem is on master playlist multibitrate, no "EXTINF:", but adding EXTINF: in last line. how to combine with my hook.js
module.exports = { "redirect_final": (url) =>https://images-blogger-opensocial.googleusercontent.com/gadgets/proxy?refresh=4&container=focus&gadget=a&url=${encodeURIComponent(url)} }

warren-bank commented 4 months ago

you wouldn't want to modify your master playlist.. since it only contains a few links.. to child manifests having alternate bitrates.

you could either:

// hooks.js

const filter_partial_playlist = function(pp) {
  const token = '#EXTINF:'
  const all_lines = pp.split(/[\r\n]+/g)

  return all_lines
    .filter(function(line) {
      return (!line || (line[0] !== '#') || line.startsWith(token))
    })
    .join("\n")
}

const modify_m3u8_content = function(m3u8_content, m3u8_url) {
  // if (!some_conditional_test(m3u8_url)) return m3u8_content

  const max_segments = 6
  const token = "\n#EXTINF:"
  const all_segments = m3u8_content.split(token)

  if (all_segments.length === 1) return m3u8_content

  return all_segments.shift() + token + filter_partial_playlist( all_segments.slice(-1 * max_segments).join(token) )
}

module.exports = {
  modify_m3u8_content
}

..that should do the trick

warren-bank commented 4 months ago

I should probably mention that this example hook does not support manifests using rotating encryption keys.

It would be easy enough to whitelist the #EXT-X-KEY: tag in filter_partial_playlist. The difficulty is that the last instance of this tag in the part of the manifest that is being removed.. should be cherry-picked and re-inserted:

  return all_segments.shift() + "\n" + get_last_key(all_segments) + token + filter_partial_playlist( all_segments.slice(-1 * max_segments).join(token) )
warren-bank commented 4 months ago

here is an implementation that does support changing encryption keys:

// hooks.js

const modify_m3u8_content = function(m3u8_content, m3u8_url) {
  // if (!some_conditional_test(m3u8_url)) return m3u8_content

  const max_segments = 6
  const token = "\n#EXTINF:"
  const all_segments = m3u8_content.split(token)

  if (all_segments.length === 1) return m3u8_content

  return all_segments.shift() + get_last_encryption_key(all_segments, max_segments) + token + filter_partial_playlist( all_segments.slice(-1 * max_segments).join(token) )
}

const get_last_encryption_key = function(segments, count_exclude) {
  if (count_exclude >= segments.length) return ''

  const token = /\n(#EXT-X-KEY:.*?)(?:[\r\n]|$)/i

  for (let i = (segments.length - count_exclude - 1); i >= 0; i--) {
    const match = token.exec(segments[i])
    if (match) return ("\n" + match[1])
  }

  return ''
}

const filter_partial_playlist = function(pp) {
  const token_whitelist = /^#(?:EXTINF|EXT-X-KEY):/i
  const all_lines = pp.split(/[\r\n]+/g)

  return all_lines
    .filter(function(line) {
      return (!line || (line[0] !== '#') || token_whitelist.test(line))
    })
    .join("\n")
}

module.exports = {
  modify_m3u8_content
}

unit test:

{

  const m3u8_url = 'https://github.com/google/ExoPlayer/issues/3725'
  const m3u8_content = `#EXTM3U
#EXT-X-VERSION:3
#EXT-X-TARGETDURATION:5
#EXT-X-PROGRAM-DATE-TIME:2018-01-18T20:42:59+00:00
#EXT-X-KEY:METHOD=AES-128,URI="https://foo.com/321630/keys/id/2348028",IV=0x66AA04B5A372A3D24D91A44C194BD050
#EXT-X-MEDIA-SEQUENCE:259371
#EXTINF:5.005,
018/20/42/59_405.ts
#EXT-X-KEY:METHOD=AES-128,URI="https://foo.com/321630/keys/id/2348028",IV=0xAD9A75768DADF6684C4A55AF649484F2
#EXTINF:5.005,
018/20/43/04_410.ts
#EXTINF:5.005,
018/20/43/09_415.ts
#EXTINF:5.005,
018/20/43/14_420.ts
#EXTINF:5.005,
018/20/43/19_425.ts
#EXTINF:5.005,
018/20/43/24_430.ts
#EXTINF:5.005,
018/20/43/29_435.ts
#EXT-X-KEY:METHOD=AES-128,URI="https://foo.com/321630/keys/id/2348028",IV=0xC4467EF6216F8E93A2D241FCAD2FC486
#EXTINF:5.005,
018/20/43/34_440.ts
#EXTINF:5.005,
018/20/43/39_445.ts
#EXTINF:5.005,
018/20/43/44_450.ts`

  const modify_m3u8_content = function(m3u8_content, m3u8_url) {
    // if (!some_conditional_test(m3u8_url)) return m3u8_content

    const max_segments = 6
    const token = "\n#EXTINF:"
    const all_segments = m3u8_content.split(token)

    if (all_segments.length === 1) return m3u8_content

    return all_segments.shift() + get_last_encryption_key(all_segments, max_segments) + token + filter_partial_playlist( all_segments.slice(-1 * max_segments).join(token) )
  }

  const get_last_encryption_key = function(segments, count_exclude) {
    if (count_exclude >= segments.length) return ''

    const token = /\n(#EXT-X-KEY:.*?)(?:[\r\n]|$)/i

    for (let i = (segments.length - count_exclude - 1); i >= 0; i--) {
      const match = token.exec(segments[i])
      if (match) return ("\n" + match[1])
    }

    return ''
  }

  const filter_partial_playlist = function(pp) {
    const token_whitelist = /^#(?:EXTINF|EXT-X-KEY):/i
    const all_lines = pp.split(/[\r\n]+/g)

    return all_lines
      .filter(function(line) {
        return (!line || (line[0] !== '#') || token_whitelist.test(line))
      })
      .join("\n")
  }

  console.log(
    modify_m3u8_content(m3u8_content, m3u8_url)
  )
}

outputs:

#EXTM3U
#EXT-X-VERSION:3
#EXT-X-TARGETDURATION:5
#EXT-X-PROGRAM-DATE-TIME:2018-01-18T20:42:59+00:00
#EXT-X-KEY:METHOD=AES-128,URI="https://foo.com/321630/keys/id/2348028",IV=0x66AA04B5A372A3D24D91A44C194BD050
#EXT-X-MEDIA-SEQUENCE:259371
#EXT-X-KEY:METHOD=AES-128,URI="https://foo.com/321630/keys/id/2348028",IV=0xAD9A75768DADF6684C4A55AF649484F2
#EXTINF:5.005,
018/20/43/19_425.ts
#EXTINF:5.005,
018/20/43/24_430.ts
#EXTINF:5.005,
018/20/43/29_435.ts
#EXT-X-KEY:METHOD=AES-128,URI="https://foo.com/321630/keys/id/2348028",IV=0xC4467EF6216F8E93A2D241FCAD2FC486
#EXTINF:5.005,
018/20/43/34_440.ts
#EXTINF:5.005,
018/20/43/39_445.ts
#EXTINF:5.005,
018/20/43/44_450.ts
siberkolosis commented 4 months ago

thank you. work perfect. i can easely share hls or iptv with hls-proxy. with vpn split i can bypass geo blocking. someday don't want to create a dash proxy version? Many paid videos now use dash. with Widevine license server. Is it possible that the Widevine license server can be proxied too?

warren-bank commented 4 months ago

I wrote a little proxy server that can convert DASH manifests to HLS manifests on-the-fly. I haven't looked at that project in a very long time, but it did work.. and passed whatever testing that I performed at the time. You might be able to daisy-chain it together with this HLS proxy.. I don't see why that wouldn't work, but I haven't played around with it at all. Generally speaking.. I try to avoid DASH; I just don't like working with it.. too verbose.