search

Benjamin-Loison / YouTube-operational-API

YouTube operational API works when YouTube Data API v3 fails.
397 stars 50 forks source link

Add live reactions extraction #273

Open Benjamin-Loison opened 6 months ago

Benjamin-Loison commented 6 months ago

As requested by someone as the Stack Overflow question 78419245.

https://www.youtube.com/live_chat?v=VIDEO_ID

When fetch a reaction:

{
    "responseContext": {
        "visitorData": "Cgtwb3pTdHlBS2ZlbyiXjIOyBjIOCgJGUhIIEgQSAgsMIEw%3D",
        "serviceTrackingParams": [
            {
                "service": "CSI",
                "params": [
                    {
                        "key": "c",
                        "value": "WEB"
                    },
                    {
                        "key": "cver",
                        "value": "2.20240509.00.00"
                    },
                    {
                        "key": "yt_li",
                        "value": "0"
                    },
                    {
                        "key": "GetLiveChat_rid",
                        "value": "0x79fe5ce4a7dad43d"
                    }
                ]
            },
            {
                "service": "GFEEDBACK",
                "params": [
                    {
                        "key": "logged_in",
                        "value": "0"
                    },
                    {
                        "key": "e",
                        "value": "23804281,23946420,23966208,23983296,23986023,2
3998056,24004644,24036948,24077241,24080738,24120820,24135310,24166867,24181174,
24187377,24241378,24290971,24377598,24439361,24453989,24456089,24468724,24524562
,24542367,24548627,24548629,24550458,24560416,24566687,24699899,39325345,3932576
2,39325801,39325815,39325854,39325955,39325971,39326025,39326075,39326086,393260
99,39326140,39326157,39326201,39326223,39326227,39326237,39326256,39326323,39326
376,39326398,39326418,51009781,51010235,51012659,51016856,51017346,51020570,5102
5415,51033765,51039200,51048489,51050361,51052758,51052970,51053689,51054765,510
59573,51060353,51063643,51064835,51072748,51074183,51079239,51089441,51091331,51
092661,51094388,51095478,51098297,51098299,51100401,51101454,51106995,51111738,5
1114662,51115184,51116067,51116370,51117319,51118932,51124104,51124407,51124411,
51127504,51129021,51129419,51133103,51137042,51139379,51142675,51146962,51147558
,51147728,51149480,51149607,51152050,51152207,51152530,51156055,51156385,5115741
1,51157430,51157432,51157838,51158470,51158514,51158535,51159506,51160401,511605
45,51162170,51162384,51162619,51162722,51165466,51165568,51165592,51166561,51166
922,51167358,51168225,51168248,51168280,51168408,51168918,51168926,51168967,5116
8970,51169118,51169258,51169260,51169338,51169947,51170248,51170373,51171439,511
72054,51172963,51172971,51173132,51173803,51173924,51174019,51174254,51174430,51
174435,51174473,51175022,51175604,51175732,51177013,51178490,51178982,51179279,5
1180639,51181955,51184106,51184113,51184161,51184805,51185184,51186212,51186236,
51186336,51186627"
                    }
                ]
            },
            {
                "service": "GUIDED_HELP",
                "params": [
                    {
                        "key": "logged_in",
                        "value": "0"
                    }
                ]
            },
            {
                "service": "ECATCHER",
                "params": [
                    {
                        "key": "client.version",
                        "value": "2.20240509"
                    },
                    {
                        "key": "client.name",
                        "value": "WEB"
                    }
                ]
            }
        ],
        "mainAppWebResponseContext": {
            "loggedOut": true,
            "trackingParam": "kV9fmPxhojXCz39TnrslLKPQSQR"
        },
        "webResponseContextExtensionData": {
            "hasDecorated": true
        }
    },
    "continuationContents": {
        "liveChatContinuation": {
            "continuations": [
                {
                    "invalidationContinuationData": {
                        "invalidationId": {
                            "objectSource": 1056,
                            "objectId": "Y2hhdH45MXEzbnNoRVV4cw==",
                            "topic": "chat~91q3nshEUxs",
                            "subscribeToGcmTopics": true,
                            "protoCreationTimestampMs": "1715521047942"
                        },
                        "timeoutMs": 10000,
                        "continuation": "0ofMyAOUAhpeQ2lrcUp3b1lWVU4wTlZWVFdYQjZlazFEV1docmFYSldVVWRJZDB0UkVnczVNWEV6Ym5Ob1JWVjRjeG9UNnFqZHVRRU5DZ3M1TVhFemJuTm9SVlY0Y3lBQ01BQSUzRCjluNHtlIiGAzAAQAJKbggAGAAgAEoKCAEQABgAIAAwAFDJxOSKl4iGA1gDeACiAQCqAQwQABoAIgAqBAgAEACwAQDAAQDIAcnE5IqXiIYD4gEMCO39grIGEI7Y058D6AEA8AEA-AEAiAIAkAIAmgIMCJeMg7IGEJOmv8ADULCX_u2UiIYDWOPLlOyUiIYDggEECAQYAYgBAJoBAggAoAGhqKv1nYiGA7oBAggK0AGWjIOyBg%3D%3D"
                    }
                }
            ]
        }
    },
    "frameworkUpdates": {
        "entityBatchUpdate": {
            "mutations": [
                {
                    "entityKey": "EiV5b3V0dWJlL2FwcC9lbW9qaV9mb3VudGFpbl9lbnRpdHlfa2V5IIkDKAE%3D",
                    "type": "ENTITY_MUTATION_TYPE_REPLACE",
                    "payload": {
                        "emojiFountainDataEntity": {
                            "key": "EiV5b3V0dWJlL2FwcC9lbW9qaV9mb3VudGFpbl9lbnRpdHlfa2V5IIkDKAE%3D",
                            "reactionBuckets": [
                                {
                                    "totalReactions": 1,
                                    "duration": {
                                        "seconds": "1"
                                    },
                                    "intensityScore": 0.75,
                                    "reactionsData": [
                                        {
                                            "unicodeEmojiId": "\ud83c\udf89",
                                            "reactionCount": 1
                                        }
                                    ]
                                }
                            ],
                            "updateTimeUsec": "1715521047942141"
                        }
                    }
                }
            ],
            "timestamp": {
                "seconds": "1715521047",
                "nanos": 942506111
            }
        }
    }
}

When no reaction are to fetch:

{
    "responseContext": {
        "visitorData": "Cgs1U3k2U0Y1WEVWWSijjIOyBjIOCgJGUhIIEgQSAgsMIBI%3D",
        "serviceTrackingParams": [
            {
                "service": "CSI",
                "params": [
                    {
                        "key": "c",
                        "value": "WEB"
                    },
                    {
                        "key": "cver",
                        "value": "2.20240509.00.00"
                    },
                    {
                        "key": "yt_li",
                        "value": "0"
                    },
                    {
                        "key": "GetLiveChat_rid",
                        "value": "0xacd5a6d85affc268"
                    }
                ]
            },
            {
                "service": "GFEEDBACK",
                "params": [
                    {
                        "key": "logged_in",
                        "value": "0"
                    },
                    {
                        "key": "e",
                        "value": "23804281,23843877,23946420,23966208,23983296,2
3986015,23998056,24004644,24036948,24077241,24080738,24108448,24120819,24135310,
24166867,24181174,24187377,24241378,24290971,24377598,24439361,24453989,24456089
,24468724,24499533,24515388,24524562,24542367,24548627,24548629,24550458,2456041
6,24566687,24697011,24699899,39325345,39325762,39325801,39325815,39325854,393259
55,39325971,39326025,39326075,39326086,39326099,39326140,39326157,39326201,39326
223,39326227,39326237,39326256,39326323,39326376,39326398,39326418,51009781,5101
0235,51012659,51016856,51017346,51020570,51025415,51030101,51033765,51037330,510
37342,51037353,51039200,51043775,51044000,51048489,51050361,51053689,51054765,51
057842,51057857,51059571,51060353,51063643,51064835,51072748,51074183,51079239,5
1089177,51089441,51091331,51092661,51094175,51094200,51094207,51095478,51098297,
51098299,51100401,51101454,51106995,51107333,51107338,51109704,51111738,51113656
,51113661,51115184,51116067,51117319,51118932,51124104,51126710,51127505,5112941
9,51133103,51134507,51137045,51139379,51141537,51141798,51142884,51142891,511449
26,51146962,51147558,51147728,51148978,51148983,51149607,51152050,51152207,51152
530,51156385,51157411,51157430,51157432,51157838,51158470,51158514,51158535,5115
8963,51160401,51160545,51162170,51162384,51162619,51162722,51162863,51163641,511
65126,51165467,51165567,51165592,51166562,51166921,51167067,51168212,51168225,51
168248,51168280,51168918,51168926,51168967,51168970,51169033,51169038,51169117,5
1169258,51169260,51169338,51169947,51170248,51170373,51171439,51171536,51171909,
51172670,51172684,51172695,51172700,51172707,51172712,51172719,51172730,51172963
,51172971,51173132,51173803,51173895,51173924,51174254,51174430,51174435,5117447
3,51175606,51175733,51176868,51177013,51177564,51178312,51178335,51178346,511783
57,51178490,51178983,51179279,51179842,51180166,51180639,51181954,51184105,51184
114,51184161,51184526,51185143,51185183,51186212"
                    }
                ]
            },
            {
                "service": "GUIDED_HELP",
                "params": [
                    {
                        "key": "logged_in",
                        "value": "0"
                    }
                ]
            },
            {
                "service": "ECATCHER",
                "params": [
                    {
                        "key": "client.version",
                        "value": "2.20240509"
                    },
                    {
                        "key": "client.name",
                        "value": "WEB"
                    }
                ]
            }
        ],
        "mainAppWebResponseContext": {
            "loggedOut": true,
            "trackingParam": "kV9fmPxhojXCz39TnrslLKPQSQR"
        },
        "webResponseContextExtensionData": {
            "hasDecorated": true
        }
    },
    "continuationContents": {
        "liveChatContinuation": {
            "continuations": [
                {
                    "invalidationContinuationData": {
                        "invalidationId": {
                            "objectSource": 1056,
                            "objectId": "Y2hhdH45MXEzbnNoRVV4cw==",
                            "topic": "chat~91q3nshEUxs",
                            "subscribeToGcmTopics": true,
                            "protoCreationTimestampMs": "1715521059736"
                        },
                        "timeoutMs": 10000,
                        "continuation": "0ofMyAOUAhpeQ2lrcUp3b1lWVU4wTlZWVFdYQjZlazFEV1docmFYSldVVWRJZDB0UkVnczVNWEV6Ym5Ob1JWVjRjeG9UNnFqZHVRRU5DZ3M1TVhFemJuTm9SVlY0Y3lBQ01BQSUzRCjluNHtlIiGAzAAQAJKbggAGAAgAEoKCAEQABgAIAAwAFDJxOSKl4iGA1gDeACiAQCqAQwQABoAIgAqBAgAEACwAQDAAQDIAcnE5IqXiIYD4gEMCO39grIGEI7Y058D6AEA8AEA-AEAiAIAkAIAmgIMCKOMg7IGEOHzr8cCULCX_u2UiIYDWOPLlOyUiIYDggEECAQYAYgBAJoBAggAoAHWmvv6nYiGA7oBAggK0AGTiYOyBg%3D%3D"
                    }
                }
            ]
        }
    }
}

When continuation seems to have expired:

{
    "responseContext": {
        "visitorData": "Cgs0eUlZc2pmM0NCVSiEjoOyBjIOCgJGUhIIEgQSAgsMIDQ%3D",
        "serviceTrackingParams": [
            {
                "service": "CSI",
                "params": [
                    {
                        "key": "c",
                        "value": "WEB"
                    },
                    {
                        "key": "cver",
                        "value": "2.20240509.00.00"
                    },
                    {
                        "key": "yt_li",
                        "value": "0"
                    },
                    {
                        "key": "GetLiveChat_rid",
                        "value": "0xda7d7c8a7922033d"
                    }
                ]
            },
            {
                "service": "GFEEDBACK",
                "params": [
                    {
                        "key": "logged_in",
                        "value": "0"
                    },
                    {
                        "key": "e",
                        "value": "23804281,23946420,23966208,23983296,23986024,2
3998056,24004644,24036948,24077241,24080738,24108448,24120819,24135310,24166867,
24181174,24186125,24187377,24241378,24290971,24377598,24407444,24428948,24439361
,24453989,24456089,24466622,24468724,24499533,24524562,24542367,24543668,2454862
7,24548629,24550458,24560416,24566687,24690006,24699899,39325345,39325762,393258
01,39325815,39325854,39325955,39325971,39326025,39326075,39326086,39326099,39326
140,39326157,39326201,39326223,39326227,39326237,39326256,39326323,39326376,3932
6398,39326418,51009781,51010235,51012659,51016856,51017346,51020077,51020570,510
25415,51030101,51033765,51037346,51037353,51039200,51043774,51048489,51050361,51
053689,51054765,51057846,51057855,51059571,51060353,51063643,51064835,51072748,5
1074183,51079239,51089175,51089441,51091331,51092661,51094175,51094197,51094200,
51094209,51095478,51098297,51098299,51100401,51101454,51105688,51106995,51107338
,51111738,51113658,51113661,51114660,51115184,51116067,51117319,51118932,5112410
4,51124406,51124411,51126576,51126710,51127505,51129419,51130324,51131427,511331
03,51134733,51135270,51137042,51139379,51141800,51142884,51142889,51144926,51146
962,51147558,51147728,51148687,51148976,51148985,51149607,51152050,51152207,5115
2530,51153947,51154420,51156385,51157280,51157411,51157430,51157432,51157838,511
58470,51158514,51158535,51160545,51161142,51162170,51162345,51162384,51162569,51
162722,51163641,51165466,51165567,51165592,51166057,51166562,51166922,51167065,5
1167956,51168225,51168248,51168280,51168918,51168926,51168967,51168970,51169118,
51169258,51169260,51169338,51169947,51170119,51170247,51170279,51170375,51171439
,51172670,51172684,51172691,51172702,51172705,51172712,51172719,51172728,5117296
3,51172971,51173132,51173803,51173887,51173891,51173924,51174254,51174430,511744
35,51174474,51174690,51175604,51175733,51176511,51177012,51178199,51178215,51178
314,51178337,51178342,51178351,51178490,51178982,51179278,51179638,51179894,5118
0167,51180639,51181507,51181954,51182202,51182595,51184065,51184105,51184114,511
84160,51185071,51185184,51186016,51186211,51186262"
                    }
                ]
            },
            {
                "service": "GUIDED_HELP",
                "params": [
                    {
                        "key": "logged_in",
                        "value": "0"
                    }
                ]
            },
            {
                "service": "ECATCHER",
                "params": [
                    {
                        "key": "client.version",
                        "value": "2.20240509"
                    },
                    {
                        "key": "client.name",
                        "value": "WEB"
                    }
                ]
            }
        ],
        "mainAppWebResponseContext": {
            "loggedOut": true,
            "trackingParam": "kV9fmPxhojXCz39TnrslLKPQSQR"
        },
        "webResponseContextExtensionData": {
            "hasDecorated": true
        }
    },
    "continuationContents": {
        "liveChatContinuation": {
            "continuations": [
                {
                    "reloadContinuationData": {
                        "continuation": "0ofMyAN4Gl5DaWtxSndvWVZVTjBOVlZUV1hCNmVrMURXV2hyYVhKV1VVZElkMHRSRWdzNU1YRXpibk5vUlZWNGN4b1Q2cWpkdVFFTkNnczVNWEV6Ym5Ob1JWVjRjeUFDTUFBJTNEMAGCAQYIBBgCIACgAeWu9-WeiIYDqAEA"
                    }
                }
            ]
        }
    }
}

So invalidationContinuationData seems to be a good string to look for the request validity.

Related to #256 as the request expires after a few minutes.

Benjamin-Loison commented 6 months ago

Current reduced continuation:

message = {
    "119693434": {
        "3": "CikqJwoYVUN0NVVTWXB6ek1DWWhraXJWUUdId0tREgs5MXEzbnNoRVV4cxoT6qjduQENCgs5MXEzbnNoRVV4cyACMAA%3D",
        "20": 1715521654955377,
    }
}

typedef = {
    "119693434": {
        "type": "message",
        "message_typedef": {
            "3": {
                "type": "string"
            },
            "20": {
                "type": "int"
            },
        },
        "field_order": [
            "3",
            "20"
        ]
    }
}
Benjamin-Loison commented 6 months ago

Seems to be the most minimized:

import requests
import json
import blackboxprotobuf
import base64

url = 'https://www.youtube.com/youtubei/v1/live_chat/get_live_chat'

headers = {
    'Content-Type': 'application/json',
}

def getBase64Protobuf(message, typedef):
    data = blackboxprotobuf.encode_message(message, typedef)
    return base64.b64encode(data, altchars = b'-_')

message = {
    '1': {
        '5': {
            '1': 'UCt5USYpzzMCYhkirVQGHwKQ',
            '2': {
                '7': 6144432111596302641,
                '15': 115
            }
        }
    },
    '4': 2,
}

typedef = {
    '1': {
        'type': 'message',
        'message_typedef': {
            '5': {
                'type': 'message',
                'message_typedef': {
                    '1': {
                        'type': 'string'
                    },
                    '2': {
                        'type': 'message',
                        'message_typedef': {
                            '7': {
                                'type': 'fixed64'
                            },
                            '15': {
                                'type': 'int'
                            }
                        },
                        'field_order': [
                            '7',
                            '15'
                        ]
                    }
                },
                'field_order': [
                    '1',
                    '2'
                ]
            }
        },
        'field_order': [
            '5'
        ]
    },
    '4': {
        'type': 'int'
    },
}

#three = 'CikqJwoYVUN0NVVTWXB6ek1DWWhraXJWUUdId0tREgs5MXEzbnNoRVV4cxoT6qjduQENCgs5MXEzbnNoRVV4cyACMAA%3D'
three = getBase64Protobuf(message, typedef)

message = {
    '119693434': {
        '3': three,
        '20': 1715522824544596,
    }
}

typedef = {
    '119693434': {
        'type': 'message',
        'message_typedef': {
            '3': {
                'type': 'string'
            },
            '20': {
                'type': 'int'
            },
        },
        'field_order': [
            '3',
            '20'
        ]
    }
}

#continuation = '0ofMyAOUAhpeQ2lrcUp3b1lWVU4wTlZWVFdYQjZlazFEV1docmFYSldVVWRJZDB0UkVnczVNWEV6Ym5Ob1JWVjRjeG9UNnFqZHVRRU5DZ3M1TVhFemJuTm9SVlY0Y3lBQ01BQSUzRCjluNHtlIiGAzAAQAJKbggAGAAgAEoKCAEQABgAIAAwAFDJxOSKl4iGA1gDeACiAQCqAQwQABoAIgAqBAgAEACwAQDAAQDIAcnE5IqXiIYD4gEMCO39grIGEI7Y058D6AEA8AEA-AEAiAIAkAIAmgIMCLCUg7IGEP3Q2fcBULCX_u2UiIYDWOPLlOyUiIYDggEECAQYAYgBAJoBAggAoAGMw6vpoYiGA7oBAggK0AH_kIOyBg=='
continuation = getBase64Protobuf(message, typedef)

json_ = {
    'context': {
        'client': {
            'clientName': 'WEB',
            'clientVersion': '2.20240509.00.00'
        }
    },
    'continuation': continuation
}

data = requests.post(url, headers = headers, json = json_).json()
dataStr = json.dumps(data, indent = 4)
#print(dataStr)
print('invalidationContinuationData' in dataStr)

However, now it seems to return a very large response but still containing invalidationContinuationData at least...

Well in fact it still contains what we look for:

                            "reactionBuckets": [
                                {
                                    "totalReactions": 0,
                                    "duration": {
                                        "seconds": "1"
                                    },
                                    "intensityScore": 1
                                },
                                {
                                    "totalReactions": 0,
                                    "duration": {
                                        "seconds": "1"
                                    },
                                    "intensityScore": 1
                                },
                                {
                                    "totalReactions": 0,
                                    "duration": {
                                        "seconds": "1"
                                    },
                                    "intensityScore": 1
                                },
                                {
                                    "totalReactions": 0,
                                    "duration": {
                                        "seconds": "1"
                                    },
                                    "intensityScore": 1
                                },
                                {
                                    "totalReactions": 4,
                                    "duration": {
                                        "seconds": "1"
                                    },
                                    "intensityScore": 0.75,
                                    "reactionsData": [
                                        {
                                            "unicodeEmojiId": "\u2764",
                                            "reactionCount": 1
                                        },
                                        {
                                            "unicodeEmojiId": "\ud83d\ude04",
                                            "reactionCount": 1
                                        },
                                        {
                                            "unicodeEmojiId": "\ud83c\udf89",
                                            "reactionCount": 1
                                        },
                                        {
                                            "unicodeEmojiId": "\ud83d\ude33",
                                            "reactionCount": 1
                                        }
                                    ]
                                }
                            ],

Note that only have to update 20 to keep it working.

Benjamin-Loison commented 6 months ago

To check that my solution is permanent:

import time
from datetime import datetime

initialDatetime = datetime.now()
print(initialDatetime)
while True:
    data = requests.post(url, headers = headers, json = json_).json()
    dataStr = json.dumps(data, indent = 4)
    #print(dataStr)
    isNeedleInData = 'invalidationContinuationData' in dataStr
    print(isNeedleInData)
    if not isNeedleInData:
        break
    time.sleep(10)
print(datetime.now() - initialDatetime)
2024-05-12 16:12:43.196524
True
...
True
False
0:04:34.450228
2024-05-12 16:19:14.196332
True
...
True
False
0:04:34.804377

So as there is a time for updating the Python script it seems pretty clear that the request expires 5 minutes after.

Benjamin-Loison commented 6 months ago 
data = base64.b64decode('0ofMyAOUAhpeQ2lrcUp3b1lWVU4wTlZWVFdYQjZlazFEV1docmFYSldVVWRJZDB0UkVnczVNWEV6Ym5Ob1JWVjRjeG9UNnFqZHVRRU5DZ3M1TVhFemJuTm9SVlY0Y3lBQ01BQSUzRCjluNHtlIiGAzAAQAJKbggAGAAgAEoKCAEQABgAIAAwAFDJxOSKl4iGA1gDeACiAQCqAQwQABoAIgAqBAgAEACwAQDAAQDIAcnE5IqXiIYD4gEMCO39grIGEI7Y058D6AEA8AEA-AEAiAIAkAIAmgIMCMuwg7IGEJmLrNsCULCX_u2UiIYDWOPLlOyUiIYDggEECAQYAYgBAJoBAggAoAHS24qor4iGA7oBAggK0AGBr4OyBg%3D%3D'.replace('%3D', '='), altchars = '-_')
message, typedef = blackboxprotobuf.decode_message(data)
encodedTimestamp = message['119693434']['20']
currentTimestamp = time.time() * 1_000_000
print(f'{currentTimestamp = } {encodedTimestamp = } {(encodedTimestamp - currentTimestamp) / 1_000_000 = }')
currentTimestamp = 1715525331895160.0 encodedTimestamp = 1715525717765586 (encodedTimestamp - currentTimestamp) / 1_000_000 = 385.870426

so let us use 400.

Benjamin-Loison commented 6 months ago 
import requests
import json
import blackboxprotobuf
import base64
import time
from datetime import datetime

url = 'https://www.youtube.com/youtubei/v1/live_chat/get_live_chat'

headers = {
    'Content-Type': 'application/json',
}

def getBase64Protobuf(message, typedef):
    data = blackboxprotobuf.encode_message(message, typedef)
    return base64.b64encode(data, altchars = b'-_')

message = {
    '1': {
        '5': {
            '1': 'UCt5USYpzzMCYhkirVQGHwKQ',
            '2': {
                '7': 6144432111596302641,
                '15': 115
            }
        }
    },
    '4': 2,
}

typedef = {
    '1': {
        'type': 'message',
        'message_typedef': {
            '5': {
                'type': 'message',
                'message_typedef': {
                    '1': {
                        'type': 'string'
                    },
                    '2': {
                        'type': 'message',
                        'message_typedef': {
                            '7': {
                                'type': 'fixed64'
                            },
                            '15': {
                                'type': 'int'
                            }
                        },
                        'field_order': [
                            '7',
                            '15'
                        ]
                    }
                },
                'field_order': [
                    '1',
                    '2'
                ]
            }
        },
        'field_order': [
            '5'
        ]
    },
    '4': {
        'type': 'int'
    },
}

three = getBase64Protobuf(message, typedef)

message = {
    '119693434': {
        '3': three,
    }
}

typedef = {
    '119693434': {
        'type': 'message',
        'message_typedef': {
            '3': {
                'type': 'string'
            },
            '20': {
                'type': 'int'
            },
        },
        'field_order': [
            '3',
            '20'
        ]
    }
}

json_ = {
    'context': {
        'client': {
            'clientName': 'WEB',
            'clientVersion': '2.20240509.00.00'
        }
    },
}

initialDatetime = datetime.now()
print(initialDatetime)

while True:
    message['119693434']['20'] = int((time.time() + 400) * 1_000_000)
    continuation = getBase64Protobuf(message, typedef)
    json_['continuation'] = continuation

    data = requests.post(url, headers = headers, json = json_).json()
    dataStr = json.dumps(data, indent = 4)
    #print(dataStr)
    isNeedleInData = 'invalidationContinuationData' in dataStr
    print(isNeedleInData, datetime.now() - initialDatetime)
    if not isNeedleInData:
        break
    time.sleep(10)
2024-05-12 16:53:58.399052
True 0:00:00.151504
True 0:00:10.303548
...
True 0:12:44.219750
True 0:12:54.459445

So my algorithm always work.

Benjamin-Loison commented 6 months ago

What about other livestreams? Then will have to consider extracting the given reactions.

print(json.dumps(data['frameworkUpdates']['entityBatchUpdate']['mutations'][0]['payload']['emojiFountainDataEntity']['reactionBuckets'], indent = 4))
[
    {
        "totalReactions": 0,
        "duration": {
            "seconds": "1"
        },
        "intensityScore": 1
    },
    {
        "totalReactions": 0,
        "duration": {
            "seconds": "1"
        },
        "intensityScore": 1
    },
    {
        "totalReactions": 1,
        "duration": {
            "seconds": "1"
        },
        "intensityScore": 0.75,
        "reactionsData": [
            {
                "unicodeEmojiId": "\u2764",
                "reactionCount": 1
            }
        ]
    },
    {
        "totalReactions": 3,
        "duration": {
            "seconds": "1"
        },
        "intensityScore": 0.75,
        "reactionsData": [
            {
                "unicodeEmojiId": "\ud83c\udf89",
                "reactionCount": 1
            },
            {
                "unicodeEmojiId": "\ud83d\ude04",
                "reactionCount": 1
            },
            {
                "unicodeEmojiId": "\ud83d\ude33",
                "reactionCount": 1
            }
        ]
    },
    {
        "totalReactions": 1,
        "duration": {
            "seconds": "1"
        },
        "intensityScore": 0.75,
        "reactionsData": [
            {
                "unicodeEmojiId": "\ud83d\udcaf",
                "reactionCount": 1
            }
        ]
    }
]
Benjamin-Loison commented 6 months ago

To test on another channel I have to wait 24 hours, the email I have in mind is:

-----BEGIN PGP MESSAGE-----

hF4DTQa9Wom5MBgSAQdAb2t1oINXCE1UIfQnLLKBHkKSRz8Zjs6b7Vq9HE3j/BYw
Xk84uwcYIeqUDZGy4HPwJ5P3Uffk+gDoVPXR1rKniIl6huUd7b6yofeVvwewGZMj
0lIBhyFmJeN0BwVs8Eud52pclueEine7IjmYMzNRHUONnyWAylGDwSjHRXiGVFrr
SDfcQ4MpVn7xZNycRXaSctYGOE9/0QJlhy0a+hKGx5geoH6V
=JTsI
-----END PGP MESSAGE-----

I set myself a reminder in 24 hours. Here I am back more than 24 hours later.

The channel I have been using according to above code snippets is UCt5USYpzzMCYhkirVQGHwKQ which is my personal channel using the email:

-----BEGIN PGP MESSAGE-----

hF4DTQa9Wom5MBgSAQdA4/4XJMm1ncdoVbGyg675TMGBaQd4eRdYsNPNOVdYzWEw
bmLCLaDkWjlygMoqAMyQ8r12omkrs1/ExFU1gOR3R9rxGJ1wdFQ3OnCM0aMfQgQM
0lgBoiBEPmo3R7N3QS5vRYZW8T8oAgm2Otb3IOiToDk84Qxi4riEuQkLUSMH0TCO
hr7SHBCXGEzULFil6yE+9/Hw/LL/3KxnIdeafjW6GP/K7CPYu2gbiTC9
=NCAb
-----END PGP MESSAGE-----

Above code snippet still seems to work as it returns True, I am just suspicious about just supporting another livestream from the same channel without modifying algorithm parameters as multiple livestreams can happen simultaneously as far as I know and without furher parameter the algorithm cannot select a precise one.

So previously I focused on the video id:

-----BEGIN PGP MESSAGE-----

hF4DTQa9Wom5MBgSAQdAgIxTq9Qvav8MZ2MLdcpCvwepeGPP7E+wahfhjsqZqCsw
2kAKspKMu5OYgoWpq7N0a/ANPzll/gUMV1/bso7zlegUPXB3ozWBxfBGK0Mv3EcJ
0kYBl9YGfL1pGN+VL9RD/3iiHHuvdHjKDQx/w1MC0n7YF7BXAy0Ch4IoON5Fjnas
Gf4G2Vmx07cZZGiDUdYHGrSWmC5emvEn
=8i1B
-----END PGP MESSAGE-----

let us generate and try another one.

Now testing the video id:

-----BEGIN PGP MESSAGE-----

hF4DTQa9Wom5MBgSAQdAh3sfOUkkVdkYBQKfLySgUpTkP421srGc9yEDnbl2f2Ew
u+qokOceuOuZ5ANrNi9FrAu3j4kKnLCvyNCFPhRv0YSZwwEnVFMeJw5JI1YUw2An
0kYBZj8VAkpcx56ViZXDjECKQVcM2qZQzVXyrpFpVAQNRu4tKncaimefStfBuESI
/JSz3tf19bbrbBpgq/xpn9B6oTc+N+r0
=Oxv+
-----END PGP MESSAGE-----

As with print(json.dumps(data['frameworkUpdates']['entityBatchUpdate']['mutations'][0]['payload']['emojiFountainDataEntity']['reactionBuckets'], indent = 4)) I added reactions to previous video id and noticed them correctly, so let us check what parameter we have to change to focus on the new video.

0ofMyAOSAhpeQ2lrcUp3b1lWVU4wTlZWVFdYQjZlazFEV1docmFYSldVVWRJZDB0UkVnczJaVkJHUzJscVZXVm5SUm9UNnFqZHVRRU5DZ3MyWlZCR1MybHFWV1ZuUlNBQ01BQSUzRCjHndHCr46GAzAAQAJKbAgAGAAgAEoIEAAYACAAMABQoL_CyLCOhgNYA3gAogEAqgEMEAAaACIAKgQIABAAsAEAwAEAyAGgv8LIsI6GA-IBDAj0_Y-yBhD0huy3AugBAPABAPgBAIgCAJACAJoCDAj0_Y-yBhD0huy3AlCT5djCr46GA1jm5NjCr46GA4IBBAgEGAGIAQCaAQIIAKABmfKmyrCOhgO6AQIICtAB8v2PsgY=
CikqJwoYVUN0NVVTWXB6ek1DWWhraXJWUUdId0tREgs2ZVBGS2lqVWVnRRoT6qjduQENCgs2ZVBGS2lqVWVnRSACMAA=

Needed this time:

message = {
    '1': {
        '5': {
            '1': 'UCt5USYpzzMCYhkirVQGHwKQ',
            '2': 'VIDEO_ID'
        }
    },
    '4': 2,
}

typedef = {
    '1': {
        'type': 'message',
        'message_typedef': {
            '5': {
                'type': 'message',
                'message_typedef': {
                    '1': {
                        'type': 'string'
                    },
                    '2': {
                        'type': 'string'
                    }
                },
                'field_order': [
                    '1',
                    '2'
                ]
            }
        },
        'field_order': [
            '5'
        ]
    },
    '4': {
        'type': 'int'
    },
}

three = getBase64Protobuf(message, typedef)

Can test with previous video as now get:

Le chat a été désactivé pour cette diffusion en direct.

Let us generate a new video id.

It seems to work as wanted.

Benjamin-Loison commented 5 months ago

I tested with another channel and it works fine too. As a result I answered the Stack Overflow question.