Beautifier inserts Unicode Character 'END OF TRANSMISSION BLOCK' unexpectedly

advanceddeveloper commented 6 years ago

Description

Online beautifier sometimes inserts \u0017 character at random places in the source code.

Input

Click to expand:

Input

```js var g=(b,c)=>(a,C)=>ab[a(t=(t<<4)-(t^8),t^4)], l=(a,b,c)=>(a=k(...(a=[a+b^61],a[a[a&b]]=+c.a,a),c))&&m(a,c), o=(a,b)=>l((a=h+(b^=142))[b++]+a[b]^a,a[b^=186]+a[++b]^a,h(b^=49,b+=4)), d=(e,f,g,t)=>b(d,~(g*9-e+f-(t*=2)))(c(f=e%17/t,e=f-g-3,(e%10^f^-g)%(t+4)))(), n=(a,b)=>a[j((a,b,c)=>a^21+b[29]-~b[17+c[29]],(-~a<<3)*5+(a^2),(a&b|1<<6)+25,b)], m=(e,f,x)=>n(e,f)([~~e]+(m+x)[5]+(g``+e)[21]+~~b(e)+~~e+~~x+~~e[f]+~~f[e]+~~b(a)+5|0), i=(h,i,j)=>[d((i^=22,h^=3)*6,[i+8]+(h^2)^!h,~i-5,i-h>>2,(h=[i+=h])[~!i]=g(i,h),h[i--&h.i]=i^149),[j,h-i*2-4,h,[i*3^h.g]]], k=(a,b)=>i(i.b=b%~a*3+4,i.b^25)[~~a][a!=[a,b^=a][~~a]?j((a,b,c)=>a.b<0?b/2+c[29]-d^12:a(a.b,6,22)|0,(-~a<<3)*5+(a^2),(a&b|1<<6)+25,b):k(-~a,c(b,a^b.d))], e=(e,g,h,i,j,k)=>b(i+g,g+~h+(i^=h-g)+19)[(i=[i+52])[~i.g]=(h[~h.e]+g)[i%10],f(i.g=-~i+g+~h^41+(k?j-k(g%16/2,(i+1)%12):k&e),i.g+8,-~i-20,g+(h+6&2),i)](e), a=(b,c)=>(a.r=(b.o+8||a)+c)[b.n^c]+a.r[b=b^a.r[c*--b+~(c=a.r)]]+c[a.s^--b*2]+c[[b^=8]+b]+(a.k=(!!b+c)[b&c])+(a.v=c[b+=4])+(a.m+c)[a.r&1]+c[c^b-7]+a.k+c[b+5]+a.v, h=(a,b)=>([a,b]=i(a+b>>2,a+~b+34,a*b-9),a[e(...b,b[0]^=37)+e(...b,b[0]^=21)+e(...b,b[0]^=17)+(a=e(...b,b[0]^=11))][e(...b,b[0]^=1)+e(...b,b[0]^=24)+e(...b)])()*(1/(1-~a)+a+(~a-2))^a, f=(a,c,d,e,h)=>(a+=b(f.r^e.o,d.h)||f)[c+h%(e-h-c<>5)]+(h=[h,h[d-e>>6]])[1]+a[~-(++c^d-e+(h[0]^h[1]))]+a[f.z=h[a]^f.z]+a[c=++c+f.z]+h[1]+a[c+=d%10]+a[e=e^c+2]+a[e^195], c=(d,e,f)=>d(j=d-(k.d=h(b^=45,c^=82)),[...a+a.b][e(...j=i(j^b-3,j^c*3-4,j^109)[1],(j[0]^=12,k.d),h)+e(...j,j[0]^=17)+e(...j)]((a,b,c)=>1/a?j.a=j.a*10-a*~(j.c=d-(k.d=h((-~j.b<<3^j.c)+~j.c,j.c^11|j.c))):j.b+=1/c[--b]?e(...(j[j.c^j.d]^=j.a,j.a&=j.c,j),k.d,h):[j.d],j.a=0,j.b=''),j.b); o(); ```

Expected Output

The same as actual output, but without endoftransmissionblock characters.

Actual Output

Click to expand:

Output

```js var g = (b, c) => (a, C) => a < C ? b(a) : f(~b + c), b = (b, t) => b[a(t = (t << 4) - (t ^ 8), t ^ 4)], l = (a, b, c) => (a = k(...(a = [a + b ^ 61], a[a[a & b]] = +c.a, a), c)) && m(a, c), o = (a, b) => l((a = h + (b ^= 142))[b++] + a[b] ^ a, a[b ^= 186] + a[++b] ^ a, h(b ^= 49, b += 4)), d = (e, f, g, t) => b(d, ~(g * 9 - e + f - (t *= 2)))(c(f = e / t, e = f - g - 3, (e ^ f ^ -g) % (t + 4)))(), n = (a, b) => a[j((a, b, c) => a ^ 21 + b[29] - ~b[17 + c[29]], (-~a << 3) * 5 + (a ^ 2), (a & b | 1 << 6) + 25, b)], m = (e, f, x) => n(e, f)([~~e] + (m + x)[5] + (g `` + e)[21] + ~~b(e) + ~~e + ~~x + ~~e[f] + ~~f[e] + ~~b(a) + 5 | 0), i = (h, i, j) => [d((i ^= 22, h ^= 3) * 6, [i + 8] + (h ^ 2) ^ !h, ~i - 5, i - h >> 2, (h = [i += h])[~!i] = g(i, h), h[i-- & h.i] = i ^ 149), [j, h - i * 2 - 4, h, [i * 3 ^ h.g]]], k = (a, b) => i(i.b = b % ~a * 3 + 4, i.b ^ 25)[~~a][a != [a, b ^= a][~~a] ? j((a, b, c) => a.b < 0 ? b / 2 + c[29] - d ^ 12 : a(a.b, 6, 22) | 0, (-~a << 3) * 5 + (a ^ 2), (a & b | 1 << 6) + 25, b) : k(-~a, c(b, a ^ b.d))], e = (e, g, h, i, j, k) => b(i + g, g + ~h + (i ^= h - g) + 19)[(i = [i + 52])[~i.g] = (h[~h.e] + g)[i], f(i.g = -~i + g + ~h ^ 41 + (k ? j - k(g / 2, (i + 1)) : k & e), i.g + 8, -~i - 20, g + (h + 6 & 2), i)](e), a = (b, c) => (a.r = (b.o + 8 || a) + c)[b.n ^ c] + a.r[b = b ^ a.r[c * --b + ~(c = a.r)]] + c[a.s ^ --b * 2] + c[[b ^= 8] + b] + (a.k = (!!b + c)[b & c]) + (a.v = c[b += 4]) + (a.m + c)[a.r & 1] + c[c ^ b - 7] + a.k + c[b + 5] + a.v, h = (a, b) => ([a, b] = i(a + b >> 2, a + ~b + 34, a * b - 9), a[e(...b, b[0] ^= 37) + e(...b, b[0] ^= 21) + e(...b, b[0] ^= 17) + (a = e(...b, b[0] ^= 11))][e(...b, b[0] ^= 1) + e(...b, b[0] ^= 24) + e(...b)])() * (1 / (1 - ~a) + a + (~a - 2)) ^ a, f = (a, c, d, e, h) => (a += b(f.r ^ e.o, d.h) || f)[c + h % (e - h - c << h % 2)] + a[e - h] + a[h - d + 8] + a[(e.m ^ h) - ~(f.z = e >> 5)] + (h = [h, h[d - e >> 6]])[1] + a[~-(++c ^ d - e + (h[0] ^ h[1]))] + a[f.z = h[a] ^ f.z] + a[c = ++c + f.z] + h[1] + a[c += d] + a[e = e ^ c + 2] + a[e ^ 195], c = (d, e, f) => d < e.t / f.u ? a(b(c.i, f.h), d.n) + c(e.r ^ f.s, f, e) : (e -= d + ~(d += c + b([c], f)), (c.m = d[(e *= f -= 2) - f]) + d[f + (e & 1)] + d[f = f * 5 - e % 5 - 1] + d[e = f + (e & 4)] + c.m + d[e += f + 3] + (b(c, e - f ^ 20) + b(a, f - e))[f - 6] + d[f] + d[e -= 5] + d[e -= f / 2 - 3] + d[e + f + (f & 6)]), j = (a, b, c, d, j) => (j = d - (k.d = h(b ^= 45, c ^= 82)), [...a + a.b][e(...j = i(j ^ b - 3, j ^ c * 3 - 4, j ^ 109)[1], (j[0] ^= 12, k.d), h) + e(...j, j[0] ^= 17) + e(...j)]((a, b, c) => 1 / a ? j.a = j.a * 10 - a * ~(j.c = d - (k.d = h((-~j.b << 3 ^ j.c) + ~j.c, j.c ^ 11 | j.c))) : j.b += 1 / c[--b] ? e(...(j[j.c ^ j.d] ^= j.a, j.a &= j.c, j), k.d, h) : [j.d], j.a = 0, j.b = ''), j.b); o(); ```

Steps to Reproduce

Navigate to http://jsbeautifier.org/, paste the input, then click beautify javascript.

Environment: Irrelevant
Settings: ~~Irrelevant~~ Option Detect packers and obfuscators is enabled.

Here are screenshots:

Before

![](https://user-images.githubusercontent.com/38354752/39258680-d42ea002-48b4-11e8-9f1c-e9ced8cbedb0.png)

After

![](https://user-images.githubusercontent.com/38354752/39258693-daf4a846-48b4-11e8-83c6-4c278725dd79.png)

This literally breaks the code and causes SyntaxError.

bitwiseman commented 6 years ago

This has something to do with the "Detect packers and obfuscators" setting. Uncheck that setting and this does not occur.

Still a bug, but there is a work around.

MacKLess commented 6 years ago

I've been playing around with the input in v1.8.5 and noticed some interesting patterns as to when the endoftransmissionblock (EOTB) show up after beautification and when they don't.

EOTB marks appear to be % followed by two digit numbers before beautification
They seem to show up when some combination of characters in multiple lines (2 or more) are entered into input. I was able to trigger the EOTB when I used any combination of lines d, k, and e.

Example: Input:

var
d=(e,f,g,t)=>b(d,~(g*9-e+f-(t*=2)))(c(f=e%17/t,e=f-g-3,(e%10^f^-g)%(t+4)))(),
k=(a,b)=>i(i.b=b%~a*3+4,i.b^25)[~~a][a!=[a,b^=a][~~a]?j((a,b,c)=>a.b<0?b/2+c[29]-d^12:a(a.b,6,22)|0,(-~a<<3)*5+(a^2),(a&b|1<<6)+25,b):k(-~a,c(b,a^b.d))],
o();

Actual Output:

var
    d = (e, f, g, t) => b(d, ~(g * 9 - e + f - (t *= 2)))(c(f = e / t, e = f - g - 3, (e ^ f ^ -g) % (t + 4)))(),
    k = (a, b) => i(i.b = b % ~a * 3 + 4, i.b ^ 25)[~~a][a != [a, b ^= a][~~a] ? j((a, b, c) => a.b < 0 ? b / 2 + c[29] - d ^ 12 : a(a.b, 6, 22) | 0, (-~a << 3) * 5 + (a ^ 2), (a & b | 1 << 6) + 25, b) : k(-~a, c(b, a ^ b.d))],
    o();

Furthermore, if you start with the code snippet above and pull line "k=..." back to be inline with the preceding line (separated with a comma and a space) the EOTB will not appear. But if you remove the space after the comma between the two lines, the EOTB show up.

Input 1 (Inline with space):

var
d=(e,f,g,t)=>b(d,~(g*9-e+f-(t*=2)))(c(f=e%17/t,e=f-g-3,(e%10^f^-g)%(t+4)))(), k=(a,b)=>i(i.b=b%~a*3+4,i.b^25)[~~a][a!=[a,b^=a][~~a]?j((a,b,c)=>a.b<0?b/2+c[29]-d^12:a(a.b,6,22)|0,(-~a<<3)*5+(a^2),(a&b|1<<6)+25,b):k(-~a,c(b,a^b.d))],
o();

Actual Output 1:

var
    d = (e, f, g, t) => b(d, ~(g * 9 - e + f - (t *= 2)))(c(f = e % 17 / t, e = f - g - 3, (e % 10 ^ f ^ -g) % (t + 4)))(),
    k = (a, b) => i(i.b = b % ~a * 3 + 4, i.b ^ 25)[~~a][a != [a, b ^= a][~~a] ? j((a, b, c) => a.b < 0 ? b / 2 + c[29] - d ^ 12 : a(a.b, 6, 22) | 0, (-~a << 3) * 5 + (a ^ 2), (a & b | 1 << 6) + 25, b) : k(-~a, c(b, a ^ b.d))],
    o();

Input 2 (Inline no space):

var
d=(e,f,g,t)=>b(d,~(g*9-e+f-(t*=2)))(c(f=e%17/t,e=f-g-3,(e%10^f^-g)%(t+4)))(),k=(a,b)=>i(i.b=b%~a*3+4,i.b^25)[~~a][a!=[a,b^=a][~~a]?j((a,b,c)=>a.b<0?b/2+c[29]-d^12:a(a.b,6,22)|0,(-~a<<3)*5+(a^2),(a&b|1<<6)+25,b):k(-~a,c(b,a^b.d))],
o();

Actual Output 2:

var
    d = (e, f, g, t) => b(d, ~(g * 9 - e + f - (t *= 2)))(c(f = e / t, e = f - g - 3, (e ^ f ^ -g) % (t + 4)))(),
    k = (a, b) => i(i.b = b % ~a * 3 + 4, i.b ^ 25)[~~a][a != [a, b ^= a][~~a] ? j((a, b, c) => a.b < 0 ? b / 2 + c[29] - d ^ 12 : a(a.b, 6, 22) | 0, (-~a << 3) * 5 + (a ^ 2), (a & b | 1 << 6) + 25, b) : k(-~a, c(b, a ^ b.d))],
    o();

bitwiseman commented 5 years ago

I figured out what is triggering this - line 57 below:

https://github.com/beautify-web/js-beautify/blob/4ee8f02ae51576f77027c35cac4a4d0e0e619a30/js/src/unpackers/urlencode_unpacker.js#L52-L60

So, this specifically happens when there are no spaces, but sufficient % signs that the deobfuscator kicks in. Note, newlines don't count as spaces. Literally all it takes is one space and this issue won't repro.

beautifier / js-beautify