ytdl-org / youtube-dl

Command-line program to download videos from YouTube.com and other video sites
http://ytdl-org.github.io/youtube-dl/
The Unlicense
132.6k stars 10.05k forks source link

youtube-dl youtube:subscriptions Log-in. #24508

Open JaxonPlays opened 4 years ago

JaxonPlays commented 4 years ago

Checklist

Verbose log

youtube-dl --username=*********@gmail.com --password=********* https://youtube.com/feed/subscriptions
[youtube:subscriptions] Downloading login page
[youtube:subscriptions] Looking up account info
[youtube:subscriptions] Logging in
WARNING: Unable to log in: HTTP Error 400: Bad Request
[youtube:subscriptions] Youtube Subscriptions: Downloading webpage
[download] Downloading playlist: Youtube Subscriptions
[youtube:subscriptions] playlist Youtube Subscriptions: Downloading 0 videos
[download] Finished downloading playlist: Youtube Subscriptions

Description

Correct login details "HTTP Error 400".

Tatsh commented 4 years ago

This is what Google calls botguard. It expects a bgRequest parameter to be sent with the login which youtube-dl does not.

The code to generate this string is a bit complex. I've figured it out but it would be tricky to make into correct Python code.

If you look at accounts.google.com/ServiceLogin source, you'll see a data-initial-setup-data attribute (JSON embedded) which has the starting random string that's used to generate the challenge string. It is very long. Once the JSON is decoded, the field is index 18.

Using this string, the code calls window.botguard.bg(theString, void 0). This returns an object which has an invoke() method. The signature seems to be invoke(callback?: (x: any) => any, d: boolean, e: object), usually with b = false and e = {}.

The callback used in the code just passes the string around to assign it to various objects, so it is not important.

You can generate the string on demand by using this in console:

window.botguard.bg(JSON.parse('[' + document.querySelector('[data-initial-setup-data]').dataset.initialSetupData.substr(4))[18], void 0).invoke(null, false, {})

Below is the code from Chrome's VM from botguard, which generates the string (which starts with !).

Start from the definition of invoke() on line 839. Remember , in JavaScript means the last statement is what gets assigned or returned.

```javascript (function anonymous() { var G = function(u, H, J) { if (0 >= J.S || 1 < J.v || !J.Z && u || 0 != document.hidden || J.L() - J.I < J.S - H) return false; return !(f(23, J, (u = (J.M = true, J).G(u ? 203 : 23), J.$)), J.s.push([n, u]), 0) }, V = function(u, H, J) { if ("object" == (H = typeof u, H)) if (u) { if (u instanceof Array) return "array"; if (u instanceof Object) return H; if ("[object Window]" == (J = Object.prototype.toString.call(u), J)) return "object"; if ("[object Array]" == J || "number" == typeof u.length && "undefined" != typeof u.splice && "undefined" != typeof u.propertyIsEnumerable && !u.propertyIsEnumerable("splice")) return "array"; if ("[object Function]" == J || "undefined" != typeof u.call && "undefined" != typeof u.propertyIsEnumerable && !u.propertyIsEnumerable("call")) return "function" } else return "null"; else if ("function" == H && "undefined" == typeof u.call) return "object"; return H }, D = this || self, g = function(u, H) { for (H = []; u--; ) H.push(255 * Math.random() | 0); return H }, m = {}, f = function(u, H, J) { if (23 == u || 203 == u) if (H.Y[u]) H.Y[u][H.N](J); else H.Y[u] = H.Yt(J); else if (99 != u && 153 != u && 247 != u && 42 != u && 193 != u || !H.Y[u]) H.Y[u] = H.J(J, H.G); 116 == u && (H.f = void 0, f(23, H, H.G(23) + 32)) }, T = {}, uC = function(u, H, J, v) { try { for (v = 0; 79669387488 != v; ) u += (H << 4 ^ H >>> 5) + H ^ v + J[v & 3], v += 2489668359, H += (u << 4 ^ u >>> 5) + u ^ v + J[v >>> 11 & 3]; return [u >>> 24, u >> 16 & 255, u >> 8 & 255, u & 255, H >>> 24, H >> 16 & 255, H >> 8 & 255, H & 255] } catch (r) { throw r; } }, z = function(u, H, J, v, r) { if ((J = ((v = (0 == (void 0 != (r = u.G(203) >> 3, H = [H, r >> 8 & 255, r & 255], v) && H.push(v), u.G(42).length) && (u.Y[42] = void 0, f(42, u, H)), ""), J) && (J.message && (v += J.message), J.stack && (v += ":" + J.stack)), u).G(146), 3) < J) { u.m = (H = (v = (J -= (v = v.slice(0, J - 3), v.length + 3), iC(v.replace(/\r\n/g, "\n"))), u.m), u); try { p(u, 153, M(v.length, 2).concat(v), 12) } finally { u.m = H } } f(146, u, J) }, L = false, jo = function(u, H, J, v, r) { for (; H.s.length; ) { if (u = u && J) H.S && H.M ? (H.M = false, u = 0 != document.hidden ? false : true) : u = false; if (u) { r = H, H.lJ(function() { R(r, false, J, false) }); break } v = PJ((v = (u = true, H.s.pop()), v), H) } return v }, NW = function(u, H, J, v, r, S) { for (u.M = (r = (u.v = 0, u.D = 25, u.c = [], (u.B = void 0, u).S = 0, u.J = function(P, c, N, Z, h, k) { return ((k = (Z = (h = (N = this, function() { return Z() } ), function() { return Z[N.W + (h[N.H] === c) - !k[N.H]] } ), N).P, h)[N.N] = function(w) { Z[N.O] = w } , h)[N.N](P), P = h } , v = (u.Yt = function(P, c, N) { return (c = (N = function() { return P } , function() { return N() } ), c)[this.N] = function(Z) { P = Z } , c } , u.Z = !(u.l = void 0, 1), []), 0), false); 128 > r; r++) v[r] = String.fromCharCode(r); (f(50, (f(165, u, (S = (f(70, u, (f(112, u, (f((f(208, u, (f(28, u, (f(157, (f((f(88, (f(142, (f(68, (f(11, (f(221, u, (f(232, (f(193, u, (f(153, (f(179, u, (f(((f(244, u, (f(92, u, (f(175, (f(13, (f(229, (f(141, (f(168, (f(116, u, [(f(191, u, (f(113, ((f((f(42, u, (f(99, u, (f(174, (u.s = ((f(24, (f(217, (f(180, u, (f(73, u, (f((f(12, u, (f(203, u, (f(23, u, (u.w = (u.m = (u.Y = [], u.dS = function(P) { this.m = P } , u), []), 0)), 0)), function(P) { HJ(P, 4) } )), 146), u, 2048), function() {} )), function(P, c) { (P = (c = Y(P), P.G(c)), P[0]).removeEventListener(P[1], P[2], false) } )), u), function(P, c, N, Z) { f((N = (c = Y(P), Y(P)), Z = Y(P), Z), P, P.G(c) >> N) }), u), function(P, c, N, Z, h, k) { if (!G(true, 255, P)) { if (P = (Z = (N = (c = (h = Y((Z = (N = (c = Y(P), Y(P)), Y(P)), P)), P.G(c)), P.G(N)), P.G(Z)), P.G(h)), "object" == V(c)) { for (k in h = [], c) h.push(k); c = h } for (Z = 0 < Z ? Z : 1, k = c.length, h = 0; h < k; h += Z) N(c.slice(h, h + Z), P) } }), u).kt = function(P, c) { (c.push(P[0] << 24 | P[1] << 16 | P[2] << 8 | P[3]), c.push(P[4] << 24 | P[5] << 16 | P[6] << 8 | P[7]), c).push(P[8] << 24 | P[9] << 16 | P[10] << 8 | P[11]) } , []), u), function(P) { A(P, 1) }), [165, 0, 0])), [])), 186), u, function(P, c, N, Z) { f((Z = (N = Y((c = Y(P), P)), Y)(P), Z), P, P.G(c) || P.G(N)) }), f)(189, u, 0), u), function(P, c, N, Z, h, k, w) { if ((Z = (N = (c = Y(P), y(P)), ""), P.Y)[234]) for (h = P.G(234), k = 0, w = h.length; N--; ) k = (k + y(P)) % w, Z += v[h[k]]; else for (; N--; ) Z += v[Y(P)]; f(c, P, Z) }), function(P, c, N, Z, h) { for (Z = (h = (N = (c = Y(P), y(P)), 0), []); h < N; h++) Z.push(Y(P)); f(c, P, Z) } )), 0), 0, 0]), u), {}), u), function(P, c, N, Z, h) { f((N = (h = (Z = (N = Y((c = Y(P), P)), P).G(Y(P)), P).G(Y(P)), P.G(N)), c), P, oO(N, h, Z, P)) }), u), function(P) { A(P, 4) }), u), function(P, c, N, Z) { f((c = (N = (Z = Y((N = Y((c = Y(P), P)), P)), P.G(N)), P.G(c)), Z), P, c[N]) }), u), u), u.V = [], function(P, c, N) { (c = (N = (c = Y(P), Y(P)), P.G(c)), f)(N, P, V(c)) } )), function(P, c) { Jf(P, (c = P.G(Y(P)), c)) } )), f)(101, u, function(P, c, N, Z, h) { 0 !== (Z = (N = (h = (N = Y((c = Y(P), P)), Z = Y(P), c = P.G(c), P.G(Y(P))), P).G(N), P.G(Z)), c) && (Z = oO(Z, 1, h, P, c, N), c.addEventListener(N, Z, L), f(189, P, [c, N, Z])) }), 162), u, function(P, c, N, Z) { (Z = (N = (c = Y(P), Y(P)), Y)(P), P).m == P && (P.G(c)[P.G(N)] = P.G(Z), 116 == c && (P.T = void 0, 2 == P.G(N) && (P.f = void 0, f(23, P, P.G(23) + 32)))) }), function(P, c, N) { f((c = (N = Y((c = Y(P), P)), P).Y[c] && P.G(c), N), P, c) } )), r = J.h || function() {} , f(91, u, function(P, c, N, Z) { (Z = (N = (c = Y(P), Y(P)), Y)(P), f)(Z, P, (P.G(c)in P.G(N)) + 0) }), u), g(4)), [0, 0, 0])), u), function(P, c, N) { 0 != (N = Y((c = Y(P), P)), P.G(c)) && f(23, P, P.G(N)) }), function(P) { A(P, 2) } )), u), function(P, c, N) { c = Y(P), N = Y(P), f(N, P, P.G(N) + P.G(c)) }), u), function(P, c, N) { N = (c = Y(P), Y)(P), f(N, P, "" + P.G(c)) }), u), function(P, c, N, Z) { if (c = P.V.pop()) { for (N = Y(P); 0 < N; N--) Z = Y(P), c[Z] = P.Y[Z]; c[146] = P.Y[c[42] = P.Y[42], 146], P.Y = c } else f(23, P, P.$) }), u), function(P, c) { G(true, 5, P) || (c = vJ(P), f(c.A, P, c.F.apply(c.X, c.a))) }), 247), u, []), u), function(P, c, N, Z, h, k, w, X, E, e, K, d, q) { for (e = (X = (w = (k = (Z = (c = Y(P), N = 0), h = function(a, Q) { for (; Z < a; ) N |= Y(P) << Z, Z += 8; return Z -= (N >>= (Q = N & (1 << a) - 1, a), a), Q } , h(3) + 1), h(5)), []), E = 0); e < w; e++) K = h(1), X.push(K), E += K ? 0 : 1; for (d = (E = (E - (e = 0, 1)).toString(2).length, []); e < w; e++) X[e] || (d[e] = h(E)); for (e = 0; e < w; e++) X[e] && (d[e] = Y(P)); for (e = (q = [], k); e--; ) q.push(P.G(Y(P))); f(c, P, function(a, Q, x, F, I) { for (Q = (x = (a.v++, F = 0, []), []); F < w; F++) { if (I = d[F], !X[F]) { for (; I >= Q.length; ) Q.push(Y(a)); I = Q[I] } x.push(I) } a.B = (a.l = a.J(q.slice(), a.i), a.J(x, a.i)) }) }), function(P, c, N, Z) { f((c = (Z = (N = Y((c = Y(P), P)), Y(P)), P).G(c) == P.G(N), Z), P, +c) } )), f(151, u, D), function(P, c, N) { G(true, 5, P) || (c = Y(P), N = Y(P), f(N, P, function(Z) { return eval(Z) }(P.G(c)))) } )), 106), u, function(P) { HJ(P, 1) }), 0)), 93)), window.performance || {}), u.m4 = S.timeOrigin || (S.timing || {}).navigationStart || 0, function(P) { P.U(4) } )), u), function(P, c, N, Z, h, k, w) { G(true, 5, P) || (c = vJ(P), h = c.F, N = c.a, w = N.length, Z = c.X, 0 == w ? k = new Z[h] : 1 == w ? k = new Z[h](N[0]) : 2 == w ? k = new Z[h](N[0],N[1]) : 3 == w ? k = new Z[h](N[0],N[1],N[2]) : 4 == w ? k = new Z[h](N[0],N[1],N[2],N[3]) : z(P, 22), f(c.A, P, k)) }), H && "!" == H.charAt(0)) ? (u.j = H, r()) : (u.C = [], u.$ = 0, J = !!J.h, B(u, [m, H]), B(u, [cJ, r]), R(u, true, J, false)) }, n = {}, R = function(u, H, J, v, r) { if (0 != u.s.length) { if (r = 0 == u.v) u.I = u.L(); return H = jo(H, u, J), r && (r = u.L() - u.I, r < (v ? 10 : 0) || 0 >= u.D-- || u.c.push(254 >= r ? r : 254)), H } }, W = function(u, H, J) { return rs(u, ((J = u.G(23), u).C && J < u.$ ? (f(23, u, u.$), Jf(u, H)) : f(23, u, H), J)) }, iC = function(u, H, J, v, r) { for (H = [], v = J = 0; v < u.length; v++) r = u.charCodeAt(v), 128 > r ? H[J++] = r : (2048 > r ? H[J++] = r >> 6 | 192 : (55296 == (r & 64512) && v + 1 < u.length && 56320 == (u.charCodeAt(v + 1) & 64512) ? (r = 65536 + ((r & 1023) << 10) + (u.charCodeAt(++v) & 1023), H[J++] = r >> 18 | 240, H[J++] = r >> 12 & 63 | 128) : H[J++] = r >> 12 | 224, H[J++] = r >> 6 & 63 | 128), H[J++] = r & 63 | 128); return H }, A = function(u, H, J, v) { for (v = (J = Y(u), 0); 0 < H; H--) v = v << 8 | Y(u); f(J, u, v) }, l, M = function(u, H, J, v) { for (v = H - (J = [], 1); 0 <= v; v--) J[H - 1 - v] = u >> 8 * v & 255; return J }, t = function(u, H) { try { NW(this, u, H) } catch (J) { U(this, J) } }, p = function(u, H, J, v, r, S) { if (u.m == u) for (r = u.G(H), 153 == H ? (H = function(P, c, N, Z) { if ((N = (c = r.length, c - 4 >> 3), r).g != N) { N = (r.g = N, (N << (Z = [0, 0, S[1], S[2]], 3)) - 4); try { r.R = uC(O(r, N), O(r, N + 4), Z) } catch (h) { throw h; } } r.push(r.R[c & 7] ^ P) } , S = u.G(193)) : H = function(P) { r.push(P) } , v && H(v & 255), v = 0, u = J.length; v < u; v++) H(J[v]) }, U = function(u, H) { u.j = ("E:" + H.message + ":" + H.stack).slice(0, 2048) }, oO = ((t.prototype.K = false, t.prototype).L = (t.prototype.o = function(u, H, J, v, r, S) { if (this.j) return this.j; try { r = [], v = !!u, S = [], B(this, [b, r, H]), B(this, [C, u, r, S]), R(this, true, v, false), J = S[0] } catch (P) { U(this, P), J = this.j, u && u(J) } return J } , (t.prototype.lJ = D.requestIdleCallback ? function(u) { requestIdleCallback(u, { timeout: 4 }) } : D.setImmediate ? function(u) { setImmediate(u) } : function(u) { setTimeout(u, 0) } , t.prototype).P = function(u, H, J, v, r, S, P) { if ((H = u[0], H) == m) { J = u[1]; try { for (S = r = (v = atob(J), J = [], 0); S < v.length; S++) P = v.charCodeAt(S), 255 < P && (J[r++] = P & 255, P >>= 8), J[r++] = P; (this.C = J, this).$ = this.C.length << 3 } catch (c) { z(this, 17, c) } rs(this) } else if (H == b) v = u[1], v.push(this.G(99).length, this.G(153).length, this.G(247).length, this.G(146)), f(168, this, u[2]), this.Y[10] && W(this, this.G(10)); else { if (H == C) { this.m = (u = M((v = u[2], this.G(99).length + 2), 2), P = this.m, this); try { J = this.G(42), 0 < J.length && p(this, 99, M(J.length, 2).concat(J), 15), r = this.G(112) & 511, r -= this.G(99).length + 5, S = this.G(153), 4 < S.length && (r -= S.length + 3), 0 < r && p(this, 99, M(r, 2).concat(g(r)), 10), 4 < S.length && p(this, 99, M(S.length, 2).concat(S), 153) } finally { this.m = P } if (((P = g(2).concat(this.G(99)), P)[1] = P[0] ^ 3, P[3] = P[1] ^ u[0], P)[4] = P[1] ^ u[1], J = window.btoa) { for (r = "", S = 0; S < P.length; S += 8192) r += String.fromCharCode.apply(null, P.slice(S, S + 8192)); J = J(r).replace(/\+/g, "-").replace(/\//g, "_").replace(/=/g, "") } else J = void 0; if (J) J = "!" + J; else for (J = "", r = 0; r < P.length; r++) S = P[r][this.N](16), 1 == S.length && (S = "0" + S), J += S; return f(146, (((P = J, this).G(99).length = v[0], this.G(153)).length = v[1], this.G(247).length = v[2], this), v[3]), P } if (H == n) W(this, u[1]); else if (H == T) return W(this, u[1]) } } , t.prototype.N = (t.prototype.i = (t.prototype.G = function(u, H) { if (H = this.Y[u], void 0 === H) throw z(this, 30, 0, u), this.b; return H() } , function(u) { return (u = u().shift(), this.l().length) || this.B().length || (this.B = this.l = void 0, this.v--), u } ), "toString"), (t.prototype.pN = function(u, H, J, v, r, S) { for (S = v = (J = [], 0); S < u.length; S++) for (v += H, r = r << H | u[S]; 7 < v; ) v -= 8, J.push(r >> v & 255); return J } , t.prototype).$t = function(u, H, J) { return u ^ ((H ^= H << 13, H ^= H >> 17, H = (H ^ H << 5) & J) || (H = 1), H) } , window.performance || {}).now ? function() { return Math.floor(this.m4 + window.performance.now()) } : function() { return +new Date } , function(u, H, J, v, r, S) { return function() { if (v.m == v) { var P = H & 1 , c = [T, u, J, void 0, r, S, arguments]; if (H & 2) var N = R(v, false, false, !(B(v, c), 0)); else P && v.s.length ? B(v, c) : P ? (B(v, c), R(v, false, false, true)) : N = PJ(c, v); return N } } } ), HJ = (t.prototype.s$ = function(u, H, J, v) { try { v = u[(H + 2) % 3], u[H] = u[H] - u[(H + 1) % 3] - v ^ (1 == H ? v << J : v >>> J) } catch (r) { throw r; } } , function(u, H, J, v) { v = (J = Y(u), Y(u)), p(u, v, M(u.G(J), H)) } ), Jf = function(u, H) { f(23, u, ((u.V.push(u.Y.slice()), u).Y[23] = void 0, H)) }, PJ = function(u, H, J, v, r) { if ((H.Z = !(J = u[0], 1), J) == b) H.D = 25, H.P(u); else if (J == C) { J = u[v = u[1], 3]; try { r = H.P(u) } catch (S) { U(H, S), r = H.j } (v && v(r), J).push(r) } else if (J == n) H.P(u); else if (J == m) H.P(u); else if (J == cJ) { try { for (r = 0; r < H.w.length; r++) try { v = H.w[r], v[0][v[1]](v[2]) } catch (S) {} } catch (S) {} (0, (H.w = [], u)[1])() } else if (J == T) return r = u[2], f(26, H, u[6]), f(168, H, r), H.P(u) }, b = {}, C = (t.prototype.b = {}, t.prototype.va = (t.prototype.H = "caller", function(u, H, J, v) { for (; J--; ) 23 != J && 203 != J && H.Y[J] && (H.Y[J] = H[v](H[u](J), this)); H[u] = this } ), {}), vJ = function(u, H, J, v, r, S) { for (S = (r = ((J = (H = {}, Y(u)), H.A = Y(u), H).a = [], v = u.m == u ? Y(u) - 1 : 1, Y(u)), 0); S < v; S++) H.a.push(Y(u)); for (H.F = u.G(J), H.X = u.G(r); v--; ) H.a[v] = u.G(H.a[v]); return H }, O = (t.prototype.W = 35, t.prototype.O = 36, t.prototype.Pa = function(u, H, J, v, r) { for (r = v = 0; r < u.length; r++) v += u.charCodeAt(r), v += v << 10, v ^= v >> 6; return (v = new Number((u = (v += v << 3, v ^= v >> 11, v + (v << 15) >>> 0), u & (1 << H) - 1)), v)[0] = (u >>> H) % J, v } , function(u, H) { return u[H] << 24 | u[H + 1] << 16 | u[H + 2] << 8 | u[H + 3] } ), cJ = (t.prototype.U = function(u, H, J, v) { p(this, (((v = Y((u &= (H = u & 4, 3), J = Y(this), this)), J = this.G(J), H) && (J = iC(("" + J).replace(/\r\n/g, "\n"))), u) && p(this, v, M(J.length, 2)), v), J) } , {}), B = function(u, H) { u.s.splice(0, 0, H) }, Y = function(u, H, J, v) { if (u.l) return u.i(u.B); if ((J = (H = u.G(23), H) >> 3, H) >= u.$) throw z(u, 31), u.b; return f(23, (void 0 == u.f && (u.f = O(u.C, J - 4), u.T = void 0), u.T != J >> 3 && (u.T = J >> 3, v = u.G(116), u.Gw = uC(u.f, u.T, [0, 0, v[1], v[2]])), u), H + 8), u.C[J] ^ u.Gw[J % 8] }, rs = (t.prototype.eR = function(u, H, J) { if (3 == u.length) { for (J = 0; 3 > J; J++) H[J] += u[J]; for (J = (u = [13, 8, 13, 12, 16, 5, 3, 10, 15], 0); 9 > J; J++) H[3](H, J % 3, u[J]) } } , function(u, H, J, v, r, S, P) { u.v++; try { for (v = 5001, r = void 0, J = u.$, S = 0; (u.K || --v) && (u.l || (S = u.G(23)) < J); ) try { u.l ? r = u.i(u.l) : (f(203, u, S), P = Y(u), r = u.G(P)), r && r.call ? r(u) : z(u, 21, 0, P), u.Z = true, G(false, 2, u) } catch (c) { c != u.b && (u.G(70) ? z(u, 22, c) : f(70, u, c)) } v || z(u, 33) } catch (c) { try { z(u, 22, c) } catch (N) { U(u, N) } } return J = u.G(168), H && f(23, u, H), u.v--, J } ), y = (l = D.botguard || (D.botguard = {}), function(u, H) { return H = Y(u), H & 128 && (H = H & 127 | Y(u) << 7), H } ); l.wLT = (l.bg = function(u, H, J) { return u && u.substring && (J = l[u.substring(0, 3)]) ? new J(u.substring(3),H) : new l.wLT(u,H) } , function(u, H, J) { this.invoke = (J = new t(u,{ h: H }), function(v, r, S) { return S = J.o(r && v, S), v && !r && v(S), S } ) } ); try { l.u || (D.addEventListener("unload", function() {}, L), l.u = 1) } catch (u) {} try { D.addEventListener("test", null, Object.defineProperty({}, "passive", { get: function() { L = { passive: true } } })) } catch (u) {} ; } ) ```
Aruelius commented 4 years ago

@Tatsh The bgRequest is obviously very important,I tested. And bgRequest is tied to Google account.

kakserpom commented 4 years ago

Any news?

motiv8d commented 4 years ago

@Tatsh Thanks for that info about Login is broken and your patch https://github.com/Tatsh/youtube-unofficial/commit/ae749c50989e1ed4bd34d78a020450e2754efd58 I am having trouble ascertaining which init.py file it pertains to as none in 2020.03.24 in Arch installs seems to have a corresponding file that meets your changes. Is there another version that can be installed to incorporate your changes to test the login? And will this apply to all YouTube authenticated functions or only those related to "subscriptions"? This problem has numerous issues raised other than subscriptions so I hope there is some co-ordination with the other groups on it as it looks like you have broken the back of the problem.