Benjamin-Loison
commented
8 months ago Let us try to proceed by hand first for a web-scraping usage:
https://www.youtube.com/playlist?list=UUWeg2Pkate69NFdBeuRFTAw (with a private Firefox window)
minimizeCURL curl.sh 'XrruklOv8X0'Note that minimizing on field name playlistVideoRenderer make us remove pagination related stuff in the following.
curl https://www.youtube.com/youtubei/v1/browse -H 'Content-Type: application/json' --data-raw '{"context": {"client": {"clientName": "WEB", "clientVersion": "2.20240313.05.00"}}, "continuation": "4qmFsgLmARIaVkxVVVdlZzJQa2F0ZTY5TkZkQmV1UkZUQXcarAFDQUY2ZlZCVU9rTkhVV2xGUkU1RVQwUlpNRTlWVlhoT1ZGVXlUMFJyTVZFd1VXOUJWV3BvYkdaTFpEaFFkVVZCTVVGQ1YycG5hVkV5YUc5V2JGcFhXa2Q0WVdWcmNGSlpWRXBIVFVad1ZWZFVWbFZoTVhCeVZWY3hWMDFXVm5KWGJGWlNWMGRPVkZKRlJuQk5WRTAxWlZoYVEyRkZVblpOYlZFeVYwVkdia2xumgIYVVVXZWcyUGthdGU2OU5GZEJldVJGVEF3"}'Python script using hardcoded continuation base64 encoded string:
```py
import requests
url = 'https://www.youtube.com/youtubei/v1/browse'
headers = {
'Content-Type': 'application/json'
}
data = {
'context': {
'client': {
'clientName': 'WEB',
'clientVersion': '2.20240313.05.00'
}
},
'continuation': '4qmFsgLmARIaVkxVVVdlZzJQa2F0ZTY5TkZkQmV1UkZUQXcarAFDQUY2ZlZCVU9rTkhVV2xGUkU1RVQwUlpNRTlWVlhoT1ZGVXlUMFJyTVZFd1VXOUJWV3BvYkdaTFpEaFFkVVZCTVVGQ1YycG5hVkV5YUc5V2JGcFhXa2Q0WVdWcmNGSlpWRXBIVFVad1ZWZFVWbFZoTVhCeVZWY3hWMDFXVm5KWGJGWlNWMGRPVkZKRlJuQk5WRTAxWlZoYVEyRkZVblpOYlZFeVYwVkdia2xumgIYVVVXZWcyUGthdGU2OU5GZEJldVJGVEF3'
}
data = requests.post(url, headers = headers, json = data).json()
print('XrruklOv8X0' in str(data))
```
Python script first level decoded Protobuf:
```py import requests import blackboxprotobuf import base64 def getBase64Protobuf(message, typedef): data = blackboxprotobuf.encode_message(message, typedef) return base64.b64encode(data).decode('ascii') message = { '80226972': { '2': 'VLUUWeg2Pkate69NFdBeuRFTAw', '3': 'CAF6fVBUOkNHUWlFRE5ET0RZME9VVXhOVFUyT0RrMVEwUW9BVWpobGZLZDhQdUVBMUFCV2pnaVEyaG9WbFpXWkd4YWVrcFJZVEpHTUZwVVdUVlVhMXByVVcxV01WVnJXbFZSV0dOVFJFRnBNVE01ZVhaQ2FFUnZNbVEyV0VGbkln', '35': 'UUWeg2Pkate69NFdBeuRFTAw' } } typedef = { '80226972': { 'type': 'message', 'message_typedef': { '2': { 'type': 'string' }, '3': { 'type': 'string' }, '35': { 'type': 'string' } }, 'field_order': [ '2', '3', '35' ] } } continuation = getBase64Protobuf(message, typedef) url = 'https://www.youtube.com/youtubei/v1/browse' headers = { 'Content-Type': 'application/json' } data = { 'context': { 'client': { 'clientName': 'WEB', 'clientVersion': '2.20240313.05.00' } }, 'continuation': continuation } data = requests.post(url, headers = headers, json = data).json() print('XrruklOv8X0' in str(data)) ```Just modifying message values seem to show that only 2 and 3 are actually needed, then can simplify typedef.
So get:
Python script first level decoded Protobuf simplified:
```py import requests import blackboxprotobuf import base64 def getBase64Protobuf(message, typedef): data = blackboxprotobuf.encode_message(message, typedef) return base64.b64encode(data).decode('ascii') message = { '80226972': { '2': 'VLUUWeg2Pkate69NFdBeuRFTAw', '3': 'CAF6fVBUOkNHUWlFRE5ET0RZME9VVXhOVFUyT0RrMVEwUW9BVWpobGZLZDhQdUVBMUFCV2pnaVEyaG9WbFpXWkd4YWVrcFJZVEpHTUZwVVdUVlVhMXByVVcxV01WVnJXbFZSV0dOVFJFRnBNVE01ZVhaQ2FFUnZNbVEyV0VGbkln', } } typedef = { '80226972': { 'type': 'message', 'message_typedef': { '2': { 'type': 'string' }, '3': { 'type': 'string' }, }, 'field_order': [ '2', '3', ] } } continuation = getBase64Protobuf(message, typedef) url = 'https://www.youtube.com/youtubei/v1/browse' headers = { 'Content-Type': 'application/json' } data = { 'context': { 'client': { 'clientName': 'WEB', 'clientVersion': '2.20240313.05.00' } }, 'continuation': continuation } data = requests.post(url, headers = headers, json = data).json() print('XrruklOv8X0' in str(data)) ```but in my case 3 is the pagination Protobuf, so have to proceed recursively:
Python script second level decoded Protobuf:
```py import requests import blackboxprotobuf import base64 def getBase64Protobuf(message, typedef): data = blackboxprotobuf.encode_message(message, typedef) return base64.b64encode(data).decode('ascii') message = { '1': 1, '15': 'PT:CGQiEDNDODY0OUUxNTU2ODk1Q0QoAUjhlfKd8PuEA1ABWjgiQ2hoVlZWZGxaekpRYTJGMFpUWTVUa1prUW1WMVVrWlVRWGNTREFpMTM5eXZCaERvMmQ2WEFnIg' } typedef = { '1': { 'type': 'int' }, '15': { 'type': 'string' } } three = getBase64Protobuf(message, typedef) message = { '80226972': { '2': 'VLUUWeg2Pkate69NFdBeuRFTAw', '3': three, } } typedef = { '80226972': { 'type': 'message', 'message_typedef': { '2': { 'type': 'string' }, '3': { 'type': 'string' }, }, 'field_order': [ '2', '3', ] } } continuation = getBase64Protobuf(message, typedef) url = 'https://www.youtube.com/youtubei/v1/browse' headers = { 'Content-Type': 'application/json' } data = { 'context': { 'client': { 'clientName': 'WEB', 'clientVersion': '2.20240313.05.00' } }, 'continuation': continuation } data = requests.post(url, headers = headers, json = data).json() print('XrruklOv8X0' in str(data)) ```1 is useless, hence get:
Python script second level decoded Protobuf simplified:
```py import requests import blackboxprotobuf import base64 def getBase64Protobuf(message, typedef): data = blackboxprotobuf.encode_message(message, typedef) return base64.b64encode(data).decode('ascii') message = { '15': 'PT:CGQiEDNDODY0OUUxNTU2ODk1Q0QoAUjhlfKd8PuEA1ABWjgiQ2hoVlZWZGxaekpRYTJGMFpUWTVUa1prUW1WMVVrWlVRWGNTREFpMTM5eXZCaERvMmQ2WEFnIg' } typedef = { '15': { 'type': 'string' } } three = getBase64Protobuf(message, typedef) message = { '80226972': { '2': 'VLUUWeg2Pkate69NFdBeuRFTAw', '3': three, } } typedef = { '80226972': { 'type': 'message', 'message_typedef': { '2': { 'type': 'string' }, '3': { 'type': 'string' }, }, 'field_order': [ '2', '3', ] } } continuation = getBase64Protobuf(message, typedef) url = 'https://www.youtube.com/youtubei/v1/browse' headers = { 'Content-Type': 'application/json' } data = { 'context': { 'client': { 'clientName': 'WEB', 'clientVersion': '2.20240313.05.00' } }, 'continuation': continuation } data = requests.post(url, headers = headers, json = data).json() print('XrruklOv8X0' in str(data)) ```15 is another Protobuf, hence use:
Python script third level decoded Protobuf:
```py import requests import blackboxprotobuf import base64 def getBase64Protobuf(message, typedef): data = blackboxprotobuf.encode_message(message, typedef) return base64.b64encode(data).decode('ascii') message = { '1': 100, '4': '3C8649E1556895CD', '5': 1, '9': 1710698421586657, '10': 1, '11': '"ChhVVVdlZzJQa2F0ZTY5TkZkQmV1UkZUQXcSDAi139yvBhDo2d6XAg"' } typedef = { '1': { 'type': 'int' }, '4': { 'type': 'string' }, '5': { 'type': 'int' }, '9': { 'type': 'int' }, '10': { 'type': 'int' }, '11': { 'type': 'string' } } fifteen = getBase64Protobuf(message, typedef) message = { '15': f'PT:{fifteen}' } typedef = { '15': { 'type': 'string' } } three = getBase64Protobuf(message, typedef) message = { '80226972': { '2': 'VLUUWeg2Pkate69NFdBeuRFTAw', '3': three, } } typedef = { '80226972': { 'type': 'message', 'message_typedef': { '2': { 'type': 'string' }, '3': { 'type': 'string' }, }, 'field_order': [ '2', '3', ] } } continuation = getBase64Protobuf(message, typedef) url = 'https://www.youtube.com/youtubei/v1/browse' headers = { 'Content-Type': 'application/json' } data = { 'context': { 'client': { 'clientName': 'WEB', 'clientVersion': '2.20240313.05.00' } }, 'continuation': continuation } data = requests.post(url, headers = headers, json = data).json() print('XrruklOv8X0' in str(data)) ```After simplifying get:
Related to #190, #69, #255 and Benjamin-Loison/cpython/issues/16.
+8