Closed rEnr3n closed 3 years ago
Interesting, my chrome extension for downloading twitter videos isn't working on it either. The tweet is from yesterday, is this a new video embed style that needs to be accounted for?
Here's a json output of the tweet from the official API:
{
"created_at": "Thu Sep 17 02:21:52 +0000 2020",
"id": 1306418059867860992,
"id_str": "1306418059867860992",
"text": "【💗#BLACKPINK🖤】\n\n⚠️お急ぎください💨\n\n⏳申し込み明日\n 9/18まで‼️\n\n\✨初めての\nオリジナルフルアルバム/\n💿「THE ALBUM」\n10月リリース‼️\n\n🌏UNIVERSAL MUSIC STORE限… https://t.co/QDjb49CyDs",
"truncated": true,
"entities": {
"hashtags": [
{
"text": "BLACKPINK",
"indices": [
2,
12
]
}
],
"symbols": [],
"user_mentions": [],
"urls": [
{
"url": "https://t.co/QDjb49CyDs",
"expanded_url": "https://twitter.com/i/web/status/1306418059867860992",
"display_url": "twitter.com/i/web/status/1…",
"indices": [
117,
140
]
}
]
},
"source": "<a href=\"https://ads-api.twitter.com\" rel=\"nofollow\">Twitter for Advertisers</a>",
"in_reply_to_status_id": null,
"in_reply_to_status_id_str": null,
"in_reply_to_user_id": null,
"in_reply_to_user_id_str": null,
"in_reply_to_screen_name": null,
"user": {
"id": 2361981054,
"id_str": "2361981054",
"name": "umusicjapan",
"screen_name": "umusicjapan",
"location": "東京都渋谷区神宮前1-5-8",
"description": "ユニバーサル ミュージック邦楽部門所属アーティストの情報をお届けします。\n\nユニバーサル ミュージック ジャパンが運営するTwitterリストはこちら → https://t.co/rfYAJItG0V",
"url": "http://t.co/1HWBWzcaEB",
"entities": {
"url": {
"urls": [
{
"url": "http://t.co/1HWBWzcaEB",
"expanded_url": "http://www.universal-music.co.jp/",
"display_url": "universal-music.co.jp",
"indices": [
0,
22
]
}
]
},
"description": {
"urls": [
{
"url": "https://t.co/rfYAJItG0V",
"expanded_url": "https://twitter.com/umusicjapan/lists/universal-music-japan/members",
"display_url": "twitter.com/umusicjapan/li…",
"indices": [
78,
101
]
}
]
}
},
"protected": false,
"followers_count": 92400,
"friends_count": 674,
"listed_count": 330,
"created_at": "Wed Feb 26 02:25:37 +0000 2014",
"favourites_count": 1671,
"utc_offset": null,
"time_zone": null,
"geo_enabled": true,
"verified": true,
"statuses_count": 39798,
"lang": null,
"contributors_enabled": false,
"is_translator": false,
"is_translation_enabled": false,
"profile_background_color": "C0DEED",
"profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png",
"profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png",
"profile_background_tile": false,
"profile_image_url": "http://pbs.twimg.com/profile_images/1210075621973356544/S1pdv78__normal.png",
"profile_image_url_https": "https://pbs.twimg.com/profile_images/1210075621973356544/S1pdv78__normal.png",
"profile_banner_url": "https://pbs.twimg.com/profile_banners/2361981054/1600222513",
"profile_link_color": "1DA1F2",
"profile_sidebar_border_color": "C0DEED",
"profile_sidebar_fill_color": "DDEEF6",
"profile_text_color": "333333",
"profile_use_background_image": true,
"has_extended_profile": false,
"default_profile": true,
"default_profile_image": false,
"following": false,
"follow_request_sent": false,
"notifications": false,
"translator_type": "none"
},
"geo": null,
"coordinates": null,
"place": null,
"contributors": null,
"is_quote_status": false,
"retweet_count": 1631,
"favorite_count": 5026,
"favorited": false,
"retweeted": false,
"lang": "ja"
}
(Separate post so the very long JSON outputs don't get mixed up) To compare, here is the API output of the example of a working extraction
{
"created_at": "Thu Sep 17 05:06:29 +0000 2020",
"id": 1306459490242396165,
"id_str": "1306459490242396165",
"text": "BLACKPINK พูดถึง PEPSI\n>https://t.co/bQ03xw9Bqg\n#BLACKPINK @BLACKPINK https://t.co/oDGsZ2lmoP",
"truncated": false,
"entities": {
"hashtags": [
{
"text": "BLACKPINK",
"indices": [
51,
61
]
}
],
"symbols": [],
"user_mentions": [
{
"screen_name": "BLACKPINK",
"name": "BLACKPINKOFFICIAL",
"id": 1273850774380883975,
"id_str": "1273850774380883975",
"indices": [
62,
72
]
}
],
"urls": [
{
"url": "https://t.co/bQ03xw9Bqg",
"expanded_url": "https://www.facebook.com/Pepsivietnam/posts/2103364229795902",
"display_url": "facebook.com/Pepsivietnam/p…",
"indices": [
27,
50
]
}
],
"media": [
{
"id": 1306459448844652544,
"id_str": "1306459448844652544",
"indices": [
73,
96
],
"media_url": "http://pbs.twimg.com/ext_tw_video_thumb/1306459448844652544/pu/img/-P1A5RT0tzbzEVqI.jpg",
"media_url_https": "https://pbs.twimg.com/ext_tw_video_thumb/1306459448844652544/pu/img/-P1A5RT0tzbzEVqI.jpg",
"url": "https://t.co/oDGsZ2lmoP",
"display_url": "pic.twitter.com/oDGsZ2lmoP",
"expanded_url": "https://twitter.com/BlackpinkWines2/status/1306459490242396165/video/1",
"type": "photo",
"sizes": {
"thumb": {
"w": 150,
"h": 150,
"resize": "crop"
},
"medium": {
"w": 1200,
"h": 675,
"resize": "fit"
},
"small": {
"w": 680,
"h": 383,
"resize": "fit"
},
"large": {
"w": 1280,
"h": 720,
"resize": "fit"
}
}
}
]
},
"extended_entities": {
"media": [
{
"id": 1306459448844652544,
"id_str": "1306459448844652544",
"indices": [
73,
96
],
"media_url": "http://pbs.twimg.com/ext_tw_video_thumb/1306459448844652544/pu/img/-P1A5RT0tzbzEVqI.jpg",
"media_url_https": "https://pbs.twimg.com/ext_tw_video_thumb/1306459448844652544/pu/img/-P1A5RT0tzbzEVqI.jpg",
"url": "https://t.co/oDGsZ2lmoP",
"display_url": "pic.twitter.com/oDGsZ2lmoP",
"expanded_url": "https://twitter.com/BlackpinkWines2/status/1306459490242396165/video/1",
"type": "video",
"sizes": {
"thumb": {
"w": 150,
"h": 150,
"resize": "crop"
},
"medium": {
"w": 1200,
"h": 675,
"resize": "fit"
},
"small": {
"w": 680,
"h": 383,
"resize": "fit"
},
"large": {
"w": 1280,
"h": 720,
"resize": "fit"
}
},
"video_info": {
"aspect_ratio": [
16,
9
],
"duration_millis": 16960,
"variants": [
{
"bitrate": 256000,
"content_type": "video/mp4",
"url": "https://video.twimg.com/ext_tw_video/1306459448844652544/pu/vid/480x270/uPmXVE0BDkvuXNWd.mp4?tag=10"
},
{
"bitrate": 2176000,
"content_type": "video/mp4",
"url": "https://video.twimg.com/ext_tw_video/1306459448844652544/pu/vid/1280x720/hCG6h7T02ksNR67q.mp4?tag=10"
},
{
"content_type": "application/x-mpegURL",
"url": "https://video.twimg.com/ext_tw_video/1306459448844652544/pu/pl/MZAaun3piod3xxED.m3u8?tag=10"
},
{
"bitrate": 832000,
"content_type": "video/mp4",
"url": "https://video.twimg.com/ext_tw_video/1306459448844652544/pu/vid/640x360/DDzuCjCv2qSvV3vR.mp4?tag=10"
}
]
},
"additional_media_info": {
"monetizable": false
}
}
]
},
"source": "<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>",
"in_reply_to_status_id": null,
"in_reply_to_status_id_str": null,
"in_reply_to_user_id": null,
"in_reply_to_user_id_str": null,
"in_reply_to_screen_name": null,
"user": {
"id": 901671651296788480,
"id_str": "901671651296788480",
"name": "BlackpinkWines",
"screen_name": "BlackpinkWines2",
"location": "",
"description": "BLACKPINK NEWS",
"url": null,
"entities": {
"description": {
"urls": []
}
},
"protected": false,
"followers_count": 55164,
"friends_count": 16,
"listed_count": 156,
"created_at": "Sun Aug 27 05:04:08 +0000 2017",
"favourites_count": 185,
"utc_offset": null,
"time_zone": null,
"geo_enabled": false,
"verified": false,
"statuses_count": 3610,
"lang": null,
"contributors_enabled": false,
"is_translator": false,
"is_translation_enabled": false,
"profile_background_color": "000000",
"profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png",
"profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png",
"profile_background_tile": false,
"profile_image_url": "http://pbs.twimg.com/profile_images/1116623819580436481/1Dnnt85p_normal.jpg",
"profile_image_url_https": "https://pbs.twimg.com/profile_images/1116623819580436481/1Dnnt85p_normal.jpg",
"profile_banner_url": "https://pbs.twimg.com/profile_banners/901671651296788480/1555058866",
"profile_link_color": "1B95E0",
"profile_sidebar_border_color": "000000",
"profile_sidebar_fill_color": "000000",
"profile_text_color": "000000",
"profile_use_background_image": false,
"has_extended_profile": false,
"default_profile": false,
"default_profile_image": false,
"following": false,
"follow_request_sent": false,
"notifications": false,
"translator_type": "none"
},
"geo": null,
"coordinates": null,
"place": null,
"contributors": null,
"is_quote_status": false,
"retweet_count": 3465,
"favorite_count": 3895,
"favorited": false,
"retweeted": false,
"possibly_sensitive": false,
"possibly_sensitive_appealable": false,
"lang": "et"
}
So, this is pretty bad. This JSON output has download links, but the tweet that can't be extracted does not. I really hope this isn't Twitter working in a way to protect videos from being downloaded without a Javascript Engine activating it first.
Seems to work here? Was able to play it too.
[gallery-dl][debug] Version 1.15.0-dev
[gallery-dl][debug] Python 3.8.5 - Linux-5.8.9-zen2-1-zen-x86_64-with-glibc2.2.5
[gallery-dl][debug] requests 2.24.0 - urllib3 1.25.10
[gallery-dl][debug] Starting DownloadJob for 'https://twitter.com/BlackpinkWines2/status/1306459490242396165'
[twitter][debug] Using TwitterTweetExtractor for 'https://twitter.com/BlackpinkWines2/status/1306459490242396165'
[urllib3.connectionpool][debug] Starting new HTTPS connection (1): api.twitter.com:443
[urllib3.connectionpool][debug] https://api.twitter.com:443 "GET /2/timeline/conversation/1306459490242396165.json?include_profile_interstitial_type=1&include_blocking=1&include_blocked_by=1&include_followed_by=1&include_want_retweets=1&include_mute_edge=1&include_can_dm=1&include_can_media_tag=1&skip_status=1&cards_platform=Web-12&include_cards=1&include_composer_source=true&include_ext_alt_text=true&include_reply_count=1&tweet_mode=extended&include_entities=true&include_user_entities=true&include_ext_media_color=true&include_ext_media_availability=true&send_error_codes=true&simple_quoted_tweet=true&count=100&ext=mediaStats%2ChighlightedLabel%2CcameraMoment&include_quote_count=true HTTP/1.1" 200 4562
[urllib3.connectionpool][debug] Starting new HTTPS connection (1): video.twimg.com:443
[urllib3.connectionpool][debug] https://video.twimg.com:443 "GET /ext_tw_video/1306459448844652544/pu/vid/1280x720/hCG6h7T02ksNR67q.mp4?tag=10 HTTP/1.1" 200 2101930
./gallery-dl/twitter/BlackpinkWines2/1306459490242396165_1.mp4
Seems to work here? Was able to play it too.
[gallery-dl][debug] Version 1.15.0-dev [gallery-dl][debug] Python 3.8.5 - Linux-5.8.9-zen2-1-zen-x86_64-with-glibc2.2.5 [gallery-dl][debug] requests 2.24.0 - urllib3 1.25.10 [gallery-dl][debug] Starting DownloadJob for 'https://twitter.com/BlackpinkWines2/status/1306459490242396165' [twitter][debug] Using TwitterTweetExtractor for 'https://twitter.com/BlackpinkWines2/status/1306459490242396165' [urllib3.connectionpool][debug] Starting new HTTPS connection (1): api.twitter.com:443 [urllib3.connectionpool][debug] https://api.twitter.com:443 "GET /2/timeline/conversation/1306459490242396165.json?include_profile_interstitial_type=1&include_blocking=1&include_blocked_by=1&include_followed_by=1&include_want_retweets=1&include_mute_edge=1&include_can_dm=1&include_can_media_tag=1&skip_status=1&cards_platform=Web-12&include_cards=1&include_composer_source=true&include_ext_alt_text=true&include_reply_count=1&tweet_mode=extended&include_entities=true&include_user_entities=true&include_ext_media_color=true&include_ext_media_availability=true&send_error_codes=true&simple_quoted_tweet=true&count=100&ext=mediaStats%2ChighlightedLabel%2CcameraMoment&include_quote_count=true HTTP/1.1" 200 4562 [urllib3.connectionpool][debug] Starting new HTTPS connection (1): video.twimg.com:443 [urllib3.connectionpool][debug] https://video.twimg.com:443 "GET /ext_tw_video/1306459448844652544/pu/vid/1280x720/hCG6h7T02ksNR67q.mp4?tag=10 HTTP/1.1" 200 2101930 ./gallery-dl/twitter/BlackpinkWines2/1306459490242396165_1.mp4
That's the example of the working tweet. It's the other one that doesn't work.
Namely, this one: https://twitter.com/umusicjapan/status/1306418059867860992
This video is part of a Twitter Card.
Those aren't part of the extended_entities
and media
fields, but have their own card
entry:
"card": {
"binding_values": {
"card_url": {
"scribe_key": "card_url",
"string_value": "https://twitter.com",
"type": "STRING"
},
"content_duration_seconds": {
"string_value": "34",
"type": "STRING"
},
"player_content_id": {
"string_value": "13_1302878075265282048",
"type": "STRING"
},
"player_height": {
"string_value": "9",
"type": "STRING"
},
"player_hls_url": {
"string_value": "https://video.twimg.com/amplify_video/1302878075265282048/pl/zu3Ziwpz30ixT3DX.m3u8",
"type": "STRING"
},
"player_image": {
"image_value": {
"height": 720,
"url": "https://pbs.twimg.com/amplify_video_thumb/1302878075265282048/img/Z8IG8d4JsL3Y4cP2.jpg",
"width": 1280
},
"type": "IMAGE"
},
"player_image_large": {
"image_value": {
"height": 720,
"url": "https://pbs.twimg.com/amplify_video_thumb/1302878075265282048/img/Z8IG8d4JsL3Y4cP2.jpg",
"width": 1280
},
"type": "IMAGE"
},
"player_image_original": {
"image_value": {
"height": 720,
"url": "https://pbs.twimg.com/amplify_video_thumb/1302878075265282048/img/Z8IG8d4JsL3Y4cP2.jpg",
"width": 1280
},
"type": "IMAGE"
},
"player_image_small": {
"image_value": {
"height": 720,
"url": "https://pbs.twimg.com/amplify_video_thumb/1302878075265282048/img/Z8IG8d4JsL3Y4cP2.jpg",
"width": 1280
},
"type": "IMAGE"
},
"player_image_x_large": {
"image_value": {
"height": 720,
"url": "https://pbs.twimg.com/amplify_video_thumb/1302878075265282048/img/Z8IG8d4JsL3Y4cP2.jpg",
"width": 1280
},
"type": "IMAGE"
},
"player_owner_id": {
"string_value": "2361981054",
"type": "STRING"
},
"player_stream_url": {
"string_value": "https://video.twimg.com/amplify_video/vmap/1302878075265282048.vmap",
"type": "STRING"
},
"player_width": {
"string_value": "16",
"type": "STRING"
},
"title": {
"string_value": "\u521d\u30aa\u30ea\u30b8\u30ca\u30eb\u30d5\u30eb\u30a2\u30eb\u30d0\u30e0\u300cTHE ALBUM\u300dUM\u30b9\u30c8\u30a2 \u9650\u5b9a\u7279\u51789/18\u307e\u3067\uff01",
"type": "STRING"
},
"vanity_url": {
"string_value": "store.universal-music.co.jp",
"type": "STRING"
},
"website_dest_url": {
"string_value": "https://store.universal-music.co.jp/artist/blackpink/",
"type": "STRING"
},
"website_shortened_url": {
"string_value": "https://t.co/y9qzhgZnxw",
"type": "STRING"
},
"website_url": {
"string_value": "https://store.universal-music.co.jp/artist/blackpink/",
"type": "STRING"
}
},
"card_platform": {
"platform": {
"audience": {
"name": "production"
},
"device": {
"name": "Swift",
"version": "12"
}
}
},
"card_type_url": "http://card-type-url-is-deprecated.invalid",
"name": "2586390716:promo_video_website",
"url": "card://1302880970513686534"
}
youtube-dl supports these, so forwarding the Tweet URL to the ytdl
downloader for any Tweets with a card
field should work.
This video is part of a Twitter Card. Those aren't part of the
extended_entities
andmedia
fields, but have there owncard
entry:youtube-dl supports these, so forwarding the Tweet URL to the
ytdl
downloader for any Tweets with acard
field should work.
That... sounds complicated. How would you set up the configuration to do that without having it download all twitter videos with ytdl?
That's the example of the working tweet. It's the other one that doesn't work.
Namely, this one: https://twitter.com/umusicjapan/status/1306418059867860992
Ah crap, my bad.
youtube-dl supports these, so forwarding the Tweet URL to the
ytdl
downloader for any Tweets with acard
field should work.
Another example but it's a picture, not a video. Added to OP. https://twitter.com/billboard/status/1306599586602135555
https://github.com/mikf/gallery-dl/commit/1686dc17573363f3b03e3298a17d7a8bdcb80c4a adds support for Twitter Cards, but you need to explicitly enable it at the moment and downloading videos from cards requires youtube-dl:
$ gallery-dl -o cards=true https://twitter.com/umusicjapan/status/1306418059867860992
/tmp/twitter/umusicjapan/1306418059867860992_1.mp4
$ gallery-dl -o cards=true https://twitter.com/billboard/status/1306599586602135555
/tmp/twitter/billboard/1306599586602135555_1.jpg
Most of the pictures and videos I see work.
These do not.