Open MaskOfDeath opened 2 years ago
I believe that must be so.
This patch gets a video on the free pages but I have no idea about the new subscription pages:
--- old/youtube-dl/youtube_dl/extractor/raywenderlich.py
+++ new/youtube-dl/youtube_dl/extractor/raywenderlich.py
@@ -7,11 +7,17 @@
from ..compat import compat_str
from ..utils import (
ExtractorError,
+ clean_html,
+ get_element_by_attribute,
+ get_element_by_class,
int_or_none,
merge_dicts,
+ parse_duration,
+ str_or_none,
try_get,
unescapeHTML,
unified_timestamp,
+ unified_strdate,
urljoin,
)
@@ -82,7 +88,10 @@
webpage, default=None) or self._html_search_meta(
'twitter:image', webpage, 'thumbnail')
- if '>Subscribe to unlock' in webpage:
+ if self._search_regex(
+ ('>Subscribe to unlock',
+ r'<a\s[^>]*?\bdata-subscription-required\s*=\s*(?:\'|"|\b)true(?:\'|"|\b)'),
+ webpage, 'subscription needed', default=False, group=0):
raise ExtractorError(
'This content is only available for subscribers',
expected=True)
@@ -91,8 +100,25 @@
'thumbnail': thumbnail,
}
- vimeo_id = self._search_regex(
- r'data-vimeo-id=["\'](\d+)', webpage, 'vimeo id', default=None)
+ vimeo_url = self._og_search_property('video:url', webpage, fatal=False)
+ if vimeo_url:
+ info.update({
+ '_type': 'url_transparent',
+ 'title': self._og_search_title(webpage),
+ 'description': clean_html(self._og_search_description(webpage)),
+ 'duration': parse_duration(get_element_by_class('c-box-list__item-duration', webpage)),
+ 'upload_date': unified_strdate(
+ re.split(r'[^\w\s]',
+ get_element_by_class('c-written-tutorial__meta', webpage))[0].strip()),
+ 'creator': clean_html(str_or_none(get_element_by_class('c-content-author__title', webpage)))
+ })
+
+ if not vimeo_url:
+ vimeo_id = self._search_regex(
+ r'data-vimeo-id=["\'](\d+)', webpage, 'vimeo id', default=None)
+ else:
+ vimeo_id = self._search_regex(
+ r'https?://player\.vimeo\.com/external/(\d+)', vimeo_url, 'vimeo id', default=None)
if not vimeo_id:
data = self._parse_json(
I believe that must be so.
This patch gets a video on the free pages but I have no idea about the new subscription pages:
Hi, dirkf! Yes, now this code allow to download the free part of the video, but the subscription video is not downloaded. Please tell me what needs to be changed in the code for this? where to dig?
Try passing --cookies
extracted from the browser with a logged-in account that has the relevant subscription.
Try passing
--cookies
extracted from the browser with a logged-in account that has the relevant subscription.
I was passing --cookies cookies.txt with a extension "Get cookies.txt"
Sounds good: verbose log (-F -v
, eg)?
@dirkf
[debug] Command-line config: ['--cookies-from-browser', 'chrome', 'https://www.raywenderlich.com/22246944-cloud-firestore-on-android-getting-started/', '-v']
[Cookies] Extracting cookies from chrome
[debug] Extracting cookies from: "C:\Users\User\AppData\Local\Google\Chrome\User Data\Default\Network\Cookies"
[Cookies] Extracted 4326 cookies from chrome
[debug] cookie version breakdown: {'v10': 4326, 'other': 0, 'unencrypted': 0}
[debug] Encodings: locale cp1251, fs utf-8, out utf-8, err utf-8, pref cp1251
[debug] yt-dlp version 2021.12.27 [6223f67a8] (win_exe)
[debug] Python version 3.10.1 (CPython 64bit) - Windows-10-10.0.19044-SP0
[debug] exe versions: ffmpeg 2021-12-12-git-996b13fac4-full_build-www.gyan.dev (setts)
[debug] Optional libraries: Cryptodome, mutagen, sqlite, websockets
[debug] Proxy map: {}
[debug] [RayWenderlichCourse] Extracting URL: https://www.raywenderlich.com/22246944-cloud-firestore-on-android-getting-started/
[RayWenderlichCourse] 22246944-cloud-firestore-on-android-getting-started: Downloading webpage
[download] Downloading playlist: Cloud Firestore on Android - Getting Started
[RayWenderlichCourse] playlist Cloud Firestore on Android - Getting Started: Collected 10 videos; downloading 10 of them[download] Downloading video 1 of 10
[debug] [RayWenderlich] Extracting URL: https://www.raywenderlich.com/22246944-cloud-firestore-on-android-getting-started/lessons/1
[RayWenderlich] 22246944-cloud-firestore-on-android-getting-started/1: Downloading webpage
[debug] [vimeo] Extracting URL: https://player.vimeo.com/video/647287328#__youtubedl_smuggle=%7B%22http_headers%22%3A+%7B%22Referer%22%3A+%22https%3A%2F%2Fwww.raywenderlich.com%2F22246944-cloud-firestore-on-android-getting-started%2Flessons%2F1%22%7D%7D
[vimeo] 647287328: Downloading webpage
[vimeo] 647287328: Downloading akfire_interconnect_quic m3u8 information
WARNING: [vimeo] Ignoring subtitle tracks found in the HLS manifest; if any subtitle tracks are missing, please report this issue on https://github.com/yt-dlp/yt-dlp . Make sure you are using the latest version; type yt-dlp -U to update. Be sure to call yt-dlp with the --verbose flag and include its complete output.
[vimeo] 647287328: Downloading akfire_interconnect_quic m3u8 information
[vimeo] 647287328: Downloading fastly_skyfire m3u8 information
[vimeo] 647287328: Downloading fastly_skyfire m3u8 information
[vimeo] 647287328: Downloading akfire_interconnect_quic MPD information
[vimeo] 647287328: Downloading akfire_interconnect_quic MPD information
[vimeo] 647287328: Downloading fastly_skyfire MPD information
[vimeo] 647287328: Downloading fastly_skyfire MPD information
[debug] Sort order given by extractor: quality, res, fps, hdr:12, source
[debug] Formats sorted by: hasvid, ie_pref, quality, res, fps, hdr:12(7), source, lang, vcodec:vp9.2(10), acodec, filesize, fs_approx, tbr, vbr, abr, asr, proto, vext, aext, hasaud, id
[debug] Default format spec: bestvideo*+bestaudio/best
[info] 647287328: Downloading 1 format(s): http-720p
[debug] Invoking downloader on "https://vod-progressive.akamaized.net/exp=1641135551~acl=%2Fvimeo-prod-skyfire-std-us%2F01%2F4457%2F25%2F647287328%2F2969148271.mp4~hmac=af8cf5a820c2cd821072f01d9613949536d344e7e2c951266eb6db4dee604212/vimeo-prod-skyfire-std-us/01/4457/25/647287328/2969148271.mp4"
[download] Cloud Firestore on Android - Getting Started, Episode 1 - What is Cloud Firestore [647287328].mp4 has already been downloaded
[download] 100% of 7.17MiB
[download] Downloading video 2 of 10
[debug] [RayWenderlich] Extracting URL: https://www.raywenderlich.com/22246944-cloud-firestore-on-android-getting-started/lessons/2
[RayWenderlich] 22246944-cloud-firestore-on-android-getting-started/2: Downloading webpage
[debug] [vimeo] Extracting URL: https://player.vimeo.com/video/647287409#__youtubedl_smuggle=%7B%22http_headers%22%3A+%7B%22Referer%22%3A+%22https%3A%2F%2Fwww.raywenderlich.com%2F22246944-cloud-firestore-on-android-getting-started%2Flessons%2F2%22%7D%7D
[vimeo] 647287409: Downloading webpage
[vimeo] 647287409: Downloading akfire_interconnect_quic m3u8 information
WARNING: [vimeo] Ignoring subtitle tracks found in the HLS manifest; if any subtitle tracks are missing, please report this issue on https://github.com/yt-dlp/yt-dlp . Make sure you are using the latest version; type yt-dlp -U to update. Be sure to call yt-dlp with the --verbose flag and include its complete output.
[vimeo] 647287409: Downloading akfire_interconnect_quic m3u8 information
[vimeo] 647287409: Downloading fastly_skyfire m3u8 information
[vimeo] 647287409: Downloading fastly_skyfire m3u8 information
[vimeo] 647287409: Downloading akfire_interconnect_quic MPD information
[vimeo] 647287409: Downloading akfire_interconnect_quic MPD information
[vimeo] 647287409: Downloading fastly_skyfire MPD information
[vimeo] 647287409: Downloading fastly_skyfire MPD information
[debug] Sort order given by extractor: quality, res, fps, hdr:12, source
[debug] Formats sorted by: hasvid, ie_pref, quality, res, fps, hdr:12(7), source, lang, vcodec:vp9.2(10), acodec, filesize, fs_approx, tbr, vbr, abr, asr, proto, vext, aext, hasaud, id
[debug] Default format spec: bestvideo*+bestaudio/best
[info] 647287409: Downloading 1 format(s): http-1080p
[debug] Invoking downloader on "https://vod-progressive.akamaized.net/exp=1641135558~acl=%2Fvimeo-prod-skyfire-std-us%2F01%2F4457%2F25%2F647287409%2F2969150312.mp4~hmac=4c359feb60f93bd2f19ceb6ba103abb180332eeb9b85bb40448b435395f2e562/vimeo-prod-skyfire-std-us/01/4457/25/647287409/2969150312.mp4"
[download] Cloud Firestore on Android - Getting Started, Episode 2 - Set Up Firebase [647287409].mp4 has already been downloaded
[download] 100% of 17.10MiB
[download] Downloading video 3 of 10
[debug] [RayWenderlich] Extracting URL: https://www.raywenderlich.com/22246944-cloud-firestore-on-android-getting-started/lessons/3
[RayWenderlich] 22246944-cloud-firestore-on-android-getting-started/3: Downloading webpage
[debug] [vimeo] Extracting URL: https://player.vimeo.com/video/647287744#__youtubedl_smuggle=%7B%22http_headers%22%3A+%7B%22Referer%22%3A+%22https%3A%2F%2Fwww.raywenderlich.com%2F22246944-cloud-firestore-on-android-getting-started%2Flessons%2F3%22%7D%7D
[vimeo] 647287744: Downloading webpage
[vimeo] 647287744: Downloading akfire_interconnect_quic m3u8 information
WARNING: [vimeo] Ignoring subtitle tracks found in the HLS manifest; if any subtitle tracks are missing, please report this issue on https://github.com/yt-dlp/yt-dlp . Make sure you are using the latest version; type yt-dlp -U to update. Be sure to call yt-dlp with the --verbose flag and include its complete output.
[vimeo] 647287744: Downloading akfire_interconnect_quic m3u8 information
[vimeo] 647287744: Downloading fastly_skyfire m3u8 information
[vimeo] 647287744: Downloading fastly_skyfire m3u8 information
[vimeo] 647287744: Downloading akfire_interconnect_quic MPD information
[vimeo] 647287744: Downloading akfire_interconnect_quic MPD information
[vimeo] 647287744: Downloading fastly_skyfire MPD information
[vimeo] 647287744: Downloading fastly_skyfire MPD information
[debug] Sort order given by extractor: quality, res, fps, hdr:12, source
[debug] Formats sorted by: hasvid, ie_pref, quality, res, fps, hdr:12(7), source, lang, vcodec:vp9.2(10), acodec, filesize, fs_approx, tbr, vbr, abr, asr, proto, vext, aext, hasaud, id
[debug] Default format spec: bestvideo*+bestaudio/best
[info] 647287744: Downloading 1 format(s): http-720p
[debug] Invoking downloader on "https://vod-progressive.akamaized.net/exp=1641135566~acl=%2Fvimeo-prod-skyfire-std-us%2F01%2F4457%2F25%2F647287744%2F2969150545.mp4~hmac=41de2732a7f545e7c547bf2c041573fe5c70701d8ab291310953d10404f829d0/vimeo-prod-skyfire-std-us/01/4457/25/647287744/2969150545.mp4"
[download] Cloud Firestore on Android - Getting Started, Episode 3 - Run the Sample Project [647287744].mp4 has already been downloaded
[download] 100% of 15.23MiB
[download] Downloading video 4 of 10
[debug] [RayWenderlich] Extracting URL: https://www.raywenderlich.com/22246944-cloud-firestore-on-android-getting-started/lessons/4
[RayWenderlich] 22246944-cloud-firestore-on-android-getting-started/4: Downloading webpage
WARNING: [RayWenderlich] unable to extract OpenGraph video:url; please report this issue on https://github.com/yt-dlp/yt-dlp . Make sure you are using the latest version; type yt-dlp -U to update. Be sure to call yt-dlp with the --verbose flag and include its complete output.
ERROR: [RayWenderlich] 4: Unable to extract video id; please report this issue on https://github.com/yt-dlp/yt-dlp . Make sure you are using the latest version; type yt-dlp -U to update. Be sure to call yt-dlp with the --verbose flag and include its complete output.; please report this issue on https://github.com/yt-dlp/yt-dlp . Make sure you are using the latest version; type yt-dlp -U to update. Be sure to call yt-dlp with the --verbose flag and include its complete output.
File "yt_dlp\extractor\common.py", line 599, in extract
File "yt_dlp\extractor\raywenderlich.py", line 130, in _real_extract
File "yt_dlp\extractor\common.py", line 1173, in _search_regex
[download] Downloading video 5 of 10
[debug] [RayWenderlich] Extracting URL: https://www.raywenderlich.com/22246944-cloud-firestore-on-android-getting-started/lessons/5
[RayWenderlich] 22246944-cloud-firestore-on-android-getting-started/5: Downloading webpage
WARNING: [RayWenderlich] unable to extract OpenGraph video:url; please report this issue on https://github.com/yt-dlp/yt-dlp . Make sure you are using the latest version; type yt-dlp -U to update. Be sure to call yt-dlp with the --verbose flag and include its complete output.
ERROR: [RayWenderlich] 5: Unable to extract video id; please report this issue on https://github.com/yt-dlp/yt-dlp . Make sure you are using the latest version; type yt-dlp -U to update. Be sure to call yt-dlp with the --verbose flag and include its complete output.; please report this issue on https://github.com/yt-dlp/yt-dlp . Make sure you are using the latest version; type yt-dlp -U to update. Be sure to call yt-dlp with the --verbose flag and include its complete output.
File "yt_dlp\extractor\common.py", line 599, in extract
File "yt_dlp\extractor\raywenderlich.py", line 130, in _real_extract
File "yt_dlp\extractor\common.py", line 1173, in _search_regex
[download] Downloading video 6 of 10
[debug] [RayWenderlich] Extracting URL: https://www.raywenderlich.com/22246944-cloud-firestore-on-android-getting-started/lessons/6
[RayWenderlich] 22246944-cloud-firestore-on-android-getting-started/6: Downloading webpage
WARNING: [RayWenderlich] unable to extract OpenGraph video:url; please report this issue on https://github.com/yt-dlp/yt-dlp . Make sure you are using the latest version; type yt-dlp -U to update. Be sure to call yt-dlp with the --verbose flag and include its complete output.
ERROR: [RayWenderlich] 6: Unable to extract video id; please report this issue on https://github.com/yt-dlp/yt-dlp . Make sure you are using the latest version; type yt-dlp -U to update. Be sure to call yt-dlp with the --verbose flag and include its complete output.; please report this issue on https://github.com/yt-dlp/yt-dlp . Make sure you are using the latest version; type yt-dlp -U to update. Be sure to call yt-dlp with the --verbose flag and include its complete output.
File "yt_dlp\extractor\common.py", line 599, in extract
File "yt_dlp\extractor\raywenderlich.py", line 130, in _real_extract
File "yt_dlp\extractor\common.py", line 1173, in _search_regex
[download] Downloading video 7 of 10
[debug] [RayWenderlich] Extracting URL: https://www.raywenderlich.com/22246944-cloud-firestore-on-android-getting-started/lessons/7
[RayWenderlich] 22246944-cloud-firestore-on-android-getting-started/7: Downloading webpage
WARNING: [RayWenderlich] unable to extract OpenGraph video:url; please report this issue on https://github.com/yt-dlp/yt-dlp . Make sure you are using the latest version; type yt-dlp -U to update. Be sure to call yt-dlp with the --verbose flag and include its complete output.
ERROR: [RayWenderlich] 7: Unable to extract video id; please report this issue on https://github.com/yt-dlp/yt-dlp . Make sure you are using the latest version; type yt-dlp -U to update. Be sure to call yt-dlp with the --verbose flag and include its complete output.; please report this issue on https://github.com/yt-dlp/yt-dlp . Make sure you are using the latest version; type yt-dlp -U to update. Be sure to call yt-dlp with the --verbose flag and include its complete output.
File "yt_dlp\extractor\common.py", line 599, in extract
File "yt_dlp\extractor\raywenderlich.py", line 130, in _real_extract
File "yt_dlp\extractor\common.py", line 1173, in _search_regex
[download] Downloading video 8 of 10
[debug] [RayWenderlich] Extracting URL: https://www.raywenderlich.com/22246944-cloud-firestore-on-android-getting-started/lessons/8
[RayWenderlich] 22246944-cloud-firestore-on-android-getting-started/8: Downloading webpage
WARNING: [RayWenderlich] unable to extract OpenGraph video:url; please report this issue on https://github.com/yt-dlp/yt-dlp . Make sure you are using the latest version; type yt-dlp -U to update. Be sure to call yt-dlp with the --verbose flag and include its complete output.
ERROR: [RayWenderlich] 8: Unable to extract video id; please report this issue on https://github.com/yt-dlp/yt-dlp . Make sure you are using the latest version; type yt-dlp -U to update. Be sure to call yt-dlp with the --verbose flag and include its complete output.; please report this issue on https://github.com/yt-dlp/yt-dlp . Make sure you are using the latest version; type yt-dlp -U to update. Be sure to call yt-dlp with the --verbose flag and include its complete output.
File "yt_dlp\extractor\common.py", line 599, in extract
File "yt_dlp\extractor\raywenderlich.py", line 130, in _real_extract
File "yt_dlp\extractor\common.py", line 1173, in _search_regex
[download] Downloading video 9 of 10
[debug] [RayWenderlich] Extracting URL: https://www.raywenderlich.com/22246944-cloud-firestore-on-android-getting-started/lessons/9
[RayWenderlich] 22246944-cloud-firestore-on-android-getting-started/9: Downloading webpage
WARNING: [RayWenderlich] unable to extract OpenGraph video:url; please report this issue on https://github.com/yt-dlp/yt-dlp . Make sure you are using the latest version; type yt-dlp -U to update. Be sure to call yt-dlp with the --verbose flag and include its complete output.
ERROR: [RayWenderlich] 9: Unable to extract video id; please report this issue on https://github.com/yt-dlp/yt-dlp . Make sure you are using the latest version; type yt-dlp -U to update. Be sure to call yt-dlp with the --verbose flag and include its complete output.; please report this issue on https://github.com/yt-dlp/yt-dlp . Make sure you are using the latest version; type yt-dlp -U to update. Be sure to call yt-dlp with the --verbose flag and include its complete output.
File "yt_dlp\extractor\common.py", line 599, in extract
File "yt_dlp\extractor\raywenderlich.py", line 130, in _real_extract
File "yt_dlp\extractor\common.py", line 1173, in _search_regex
[download] Downloading video 10 of 10
[debug] [RayWenderlich] Extracting URL: https://www.raywenderlich.com/22246944-cloud-firestore-on-android-getting-started/lessons/10
[RayWenderlich] 22246944-cloud-firestore-on-android-getting-started/10: Downloading webpage
WARNING: [RayWenderlich] unable to extract OpenGraph video:url; please report this issue on https://github.com/yt-dlp/yt-dlp . Make sure you are using the latest version; type yt-dlp -U to update. Be sure to call yt-dlp with the --verbose flag and include its complete output.
ERROR: [RayWenderlich] 10: Unable to extract video id; please report this issue on https://github.com/yt-dlp/yt-dlp . Make sure you are using the latest version; type yt-dlp -U to update. Be sure to call yt-dlp with the --verbose flag and include its complete output.; please report this issue on https://github.com/yt-dlp/yt-dlp . Make sure you are using the latest version; type yt-dlp -U to update. Be sure to call yt-dlp with the --verbose flag and include its complete output.
File "yt_dlp\extractor\common.py", line 599, in extract
File "yt_dlp\extractor\raywenderlich.py", line 130, in _real_extract
File "yt_dlp\extractor\common.py", line 1173, in _search_regex
[download] Finished downloading playlist: Cloud Firestore on Android - Getting Started
OK, so the tell that the extractor uses to detect subscriber content is still there. If you visit one of the subscription pages with your login you should be able to see if that's also true. The patterns for the tell are in the hunk '@@ -82,7 +88,10 @@' of the patch.
Then:
--print-traffic
) or some further headers (a User-Agent that matches the cookies, Referer, or whatever) are being required by the site.@dirkf, sorry, I updated the cookies and the errors have changed:
WARNING: [RayWenderlich] unable to extract OpenGraph video:url; please report this issue on https://github.com/yt-dlp/yt-dlp . Make sure you are using the latest version; type yt-dlp -U to update. Be sure to call yt-dlp with the --verbose flag and include its complete output.
ERROR: [RayWenderlich] 9: Unable to extract video id; please report this issue on https://github.com/yt-dlp/yt-dlp . Make sure you are using the latest version; type yt-dlp -U to update. Be sure to call yt-dlp with the --verbose flag and include its complete output.; please report this issue on https://github.com/yt-dlp/yt-dlp . Make sure you are using the latest version; type yt-dlp -U to update. Be sure to call yt-dlp with the --verbose flag and include its complete output.
File "yt_dlp\extractor\common.py", line 599, in extract
File "yt_dlp\extractor\raywenderlich.py", line 130, in _real_extract
File "yt_dlp\extractor\common.py", line 1173, in _search_regex
I modified the raywenderlich.py file manually, I think was not mistaken in this.
It's the same as the patched yt-dl file, except for Windows line endings and this line, which only works if you're using a modified extractor/common.py
, such as the one from yt-dlp:
- mobj = re.match(self._VALID_URL, url)
+ mobj = self._match_valid_url(url)
But as you're actually using that version, you should carry on the issue in the yt-dlp tracker, where there is the benefit that probably more eyes will see it.
@dirkf ok, thanks!
@dirkf, free videos stopped loading again, if you can help with the paid ones too, I can provide you with cookies.
[debug] System config: []
[debug] User config: []
[debug] Custom config: []
[debug] Command-line args: ['--cookies', 'rw.txt', 'https://www.raywenderlich.com/28092310-material-motion-for-android', '-v']
[debug] Encodings: locale cp1251, fs utf-8, out utf-8, pref cp1251
[debug] youtube-dl version 2021.12.17
[debug] Python version 3.10.1 (CPython) - Windows-10-10.0.19044-SP0
[debug] exe versions: none
[debug] Proxy map: {}
[RayWenderlichCourse] 28092310-material-motion-for-android: Downloading webpage
[download] Downloading playlist: Material Motion for Android
[RayWenderlichCourse] playlist Material Motion for Android: Collected 6 video ids (downloading 6 of them)
[download] Downloading video 1 of 6
[RayWenderlich] 28092310-material-motion-for-android/1: Downloading webpage
ERROR: Unable to extract video id; please report this issue on https://yt-dl.org/bug . Make sure you are using the latest version; see https://yt-dl.org/update on how to update. Be sure to call youtube-dl with the --verbose flag and include its complete output.
Traceback (most recent call last):
File "C:\Program Files\_ytdl\youtube-dl-2021.12.17\youtube_dl\YoutubeDL.py", line 815, in wrapper
return func(self, *args, **kwargs)
File "C:\Program Files\_ytdl\youtube-dl-2021.12.17\youtube_dl\YoutubeDL.py", line 836, in __extract_info
ie_result = ie.extract(url)
File "C:\Program Files\_ytdl\youtube-dl-2021.12.17\youtube_dl\extractor\common.py", line 534, in extract
ie_result = self._real_extract(url)
File "C:\Program Files\_ytdl\youtube-dl-2021.12.17\youtube_dl\extractor\raywenderlich.py", line 130, in _real_extract
data, lesson_id) or self._search_regex(
File "C:\Program Files\_ytdl\youtube-dl-2021.12.17\youtube_dl\extractor\common.py", line 1012, in _search_regex
raise RegexNotFoundError('Unable to extract %s' % _name)
youtube_dl.utils.RegexNotFoundError: Unable to extract video id; please report this issue on https://yt-dl.org/bug . Make sure you are using the latest version; see https://yt-dl.org/update on how to update. Be sure to call youtube-dl with the --verbose flag and include its complete output.
Help :]
Checklist
Verbose log
Description
It looks like the site has changed the output of the video id and now the extractor\raywenderlich.py does not work correctly. Can someone help?