Applicability to dynamic languages?

[alex]

Last Thanksgiving, I had an idea to use libFuzzer as an input stream to a grammar to fuzz JS engines. Ultimately I didn't have a lot to show for it. There's a lot of success in fuzzing JS engines, but no success with coverage guided fuzzing as far as I'm aware.

I'm curious if you think the approach in prog-fuzz is applicable to dynamic languages, and if yes if you'd be interested in attempting to get one of the major engines going with prog-fuzz.

[vegard]

Hey, yeah, I don't really see why not. Since you're a Mozilla person, I guess you were thinking of SpiderMonkey in particular? I have no experience with it, but if there is a standalone executable without too many dependencies (like a graphical environment) then it shouldn't be too hard to set it up with AFL instrumentation and get something running.

[vegard]

Looks like SpiderMonkey has quite a bit of support for AFL already.

I pushed the branch js that has some rules for JS and it seems to be doing something:

However, JS does not have a terribly complicated syntax, I am guessing most bugs (if any) would be found in the interfaces/libraries rather than in the language itself. Well, I'll try running it for a bit :-)

[alex]

Yeah, almost certainly you need to teach it the name of builtin methods, or teach it how to discover them, for it to make real finds.

[alex]

FWIW, here's a list of identifiers that, at the time I was working on this, I thought were significant:

"$&"
"$'"
"$+"
"$1"
"$2"
"$3"
"$4"
"$5"
"$6"
"$7"
"$8"
"$9"
"$_"
"$`"
"Array"
"ArrayBuffer"
"EPSILON"
"Float64Array"
"Instance"
"MAX_SAFE_INTEGER"
"MAX_VALUE"
"MIN_SAFE_INTEGER"
"MIN_VALUE"
"Module"
"NEGATIVE_INFINITY"
"NaN"
"Number"
"Object"
"POSITIVE_INFINITY"
"Proxy"
"RegExp"
"Table"
"UTC"
"Uint32Array"
"Uint8Array"
"WebAssembly"
"__defineGetter__"
"__lookupGetter__"
"__proto__"
"apply"
"arguments"
"assign"
"byteLength"
"call"
"caller"
"concat"
"constructor"
"copyWithin"
"create"
"decodeURI"
"defineProperties"
"defineProperty"
"entries"
"eval"
"fill"
"for"
"freeze"
"from"
"fromCharCode"
"fromCodePoint"
"get"
"getOwnPropertyDescriptor"
"getOwnPropertyDescriptors"
"getOwnPropertyNames"
"getOwnPropertySymbols"
"getPrototypeOf"
"hasInstance"
"indexOf"
"input"
"is"
"isArray"
"isConcatSpreadable"
"isExtensible"
"isFinite"
"isFrozen"
"isInteger"
"isNaN"
"isSafeInteger"
"isSealed"
"iterator"
"join"
"keyFor"
"keys"
"lastMatch"
"lastParen"
"leftContext"
"length"
"link"
"map"
"match"
"name"
"now"
"of"
"parse"
"parseFloat"
"parseInt"
"preventExtensions"
"prototype"
"push"
"raw"
"repeat"
"replace"
"reverse"
"rightContext"
"seal"
"search"
"setPrototypeOf"
"shift"
"slice"
"sort"
"species"
"splice"
"split"
"subarray"
"this"
"toPrimitive"
"toString"
"toStringTag"
"unscopables"
"valueOf"
"values"

[alex]

@vegard Did anything interesting come out of this?

[vegard]

I ran it for a day or so just on my laptop, it came up with test cases like this:

parseInt(function main (Array) { new $1(assert.parseFloat([5 * 10])(function main (copyWithin) { if (0.9) { { assert.sameValue(x, x); ; new Date("2017-09-30") }; x ^= 0; setPrototypeOf } else { link; x ^= 0; }; document.getElementById(Array.construct(getOwnPropertySymbols, getPrototypeOf, Object)).x += Number; for (var keys in { Table }) { keyFor; return x; y:var x }; typeof x; }, isArray.length >>>= $8 == toPrimitive) **= 'x'.length); 0.9; $5 **= new setPrototypeOf(parseFloat($4))(new 5 * 10(fromCodePoint.isExtensible($8 *= undefined.NaN("x").innerHTML).getOwnPropertySymbols.sameValue(12e-1, ( 0, 0, 0 )) <<= { keys }), x >>= 0 / 1, 5 + 101e1.x($3, join, create >>= 0).$6).buffer({'x':( 0, concat, isFrozen ), 99999999:'x'.length}.length) &= ( x **= replacelastMatch / 1e-1, new DataView(new new {x:function main () { ; ; ; ; }, y:"y"}({x:x *= 1, isNaN:x -= 0} * { rightContext })(x /= repeat), x **= x &= $7, Number.NaN >>>= ArrayBuffer2isSafeInteger).valueOf = NEGATIVE_INFINITY %= ( 0, 0, RegExp ), {null:"x", y:{x:"x", y:"y"}} ); new 'x'.indexOf(x / 1)(new true(Instance &= {x:"x", y:"y"}), call <<= preventExtensions, {1e-12:values, $6:function NEGATIVE_INFINITY(x = 6) { $8++ }}).isInteger; })

function isFinite(RegExpObjectcopyWithin) { Reflect.construct(x >>>= 5 * 10 === x, "x", toPrimitive) = getOwnPropertyDescriptor === new new new Date(arguments)(new Number(reverse)(Number(1e-1)), POSITIVE_INFINITY.decodeURI, EPSILON -= keys).buffer(x().x **= 0(x += 0, MAX_SAFE_INTEGER **= 1e-1)).x++ / function slice (match) { 1(function () { return 0; })();; assert.$5(Object.x >>= new DataView(new MIN_SAFE_INTEGER === raw(hasInstance), $9, 0).parseFloat("0")(parseInt("0")), typeof new x()); reportCompare($_, 1e15 * 10); MIN_SAFE_INTEGERe-1; return MAX_VALUE.sort(( WebAssembly, UTC, 1e-1 ), POSITIVE_INFINITY99999999 -= prototype * get); [{ __lookupGetter__ } == parseFloat(eval <<= 0 * unscopables)]; MAX_VALUE %= isSealed( ( x -= 0, fromCharCode, x === Object ) ); } }

parseFloat(parseInt(Number(print(function Proxy(parseFloat = name$6.RegExp([Number("1")].Number.NEGATIVE_INFINITY(x += keys)).sort.parseFloat(3.14 * new Date("2017-09-30").split(push).lastParen)(function MAX_VALUE ($9) { __lookupGetter__; { lastParen >>>= Instance }; slice; getOwnPropertyNames.defineProperty %= ( function toString (getPrototypeOf) { new x().length.NaN; new subarray.MAX_VALUE([].length) + Array; x %= hasInstance.seal; Object = document.getElementById("x").innerHTML + 10(y == Float64Array); }, split, { parseInt }.Array('x'.Reflect.construct(x, [], null)('y'), x **= subarray) ); }, new DataView(new function __proto__ () { x >>>= 0; fromCodePoint; var rightContext = 0; var iter = function* (prototype) { ; throw new Number.NaN(x); assign; $9 += 1; }(); class C { defineProperty([,] = ( [$5], x === x, this.raw )) { } }; Table; var c = assert.setPrototypeOf(x **= 0, species |= 0); for (var x in []) { undefined; ; return 'x'.length }; return toString.throws(Test262Error, function() { 1e1.method(x === x); typeof ( 0, 0, 0 ); }); Float64Array.shift(prototype); lastMatch; assert.$&(following, 0, 'Iterator was properly closed.'); x === x; parseFloat(x *= input); $3; }(MAX_VALUE), {get:x == x, y:$_}.x >>= 0(x, document.getElementById(assert.prototype(x, x)).innerHTML), 10.9e1( search )).buffer++) >>= new call === of(new lastParenisInteger.prototype(concat)(search.MIN_VALUE(x -= 0, get) <<= rightContext / x == x), indexOf *= push.length, new Number("1").length(new function main ({ } = assert.sameValue(x, x)) { call; x %= reverse; split; typeof x; }.length(1), leftContext, x /= subarray).buffer &= print(input) -= link ^= 0).isFinite) { Instance |= search }))))

var split = 'x'.link(typeof 'x'.indexOf({99999999:print(Number(function RegExp (call) { __defineGetter__; lastMatch >>= document.join("x").innerHTML; new function POSITIVE_INFINITY(parseInt) { }(new { fromCharCode }(new DataView(new ArrayBuffer(freeze), split, MIN_VALUE).buffer.Number("1")), assert.apply(entries, print("x")), WebAssembly.defineProperties &= now(print(lastParen) + 10).isConcatSpreadable).parseFloat("0") -= Number(caller); if (1e-1) { reverse |= ( 0, 0, [Table] ).$7(isSafeInteger, x); split:Uint8Array; new seal.construct(decodeURI, iterator, null)(new new Float64Array(new x(new x())(false), new MIN_VALUE(document.getElementById("x").innerHTML), x === x).$&(x >>= x === x), shift, isSealed).raw } else { 3.140.toPrecision(); is; (function () { return x *= function main () { species.length; var x = 0; ; constructor; }; })(); }; })), length9999999999999999hasInstanceparseIntEPSILONthis:assign = new DataView(new ArrayBuffer(parseFloat(function undefined ($8) { seal %= $_.isArray == function x() { }.length; { x++; ( getOwnPropertySymbols, 'x'.length.__lookupGetter__ ^= 0(x, [], function x() { }), Number.NEGATIVE_INFINITYreverse ); leftContext }; getOwnPropertySymbols:__lookupGetter__; ( parseFloat(print(parseInt **= isFrozen = document.getElementById("x").innerHTML)++ + 5 * 10), [document.getElementById("x").$6] = 15 + of / 1e1, typeof $5(x()) ); })), 0.9, parseFloat).isArray /= "x".indexOf(new Float64Array(function map(getOwnPropertySymbols) { Number.MIN_VALUE += 0 })) * print("x")}))

typeof parseInt([new ArrayBuffer(999999990.90.toString().split(new DataView(new ArrayBuffer(this), 0, {search:function main () { x /= 1; concat; ; ; }, raw:Proxy}).indexOf **= NaN).valueOf(Number.MIN_VALUE1thise-15 * 10), print("x"))]).entries99999999$5++ * new Uint32Array({ parseInt } = 63.14).length * [new Uint8Array([parseFloat].MAX_VALUE >>>= print("x") * 'x'.__lookupGetter__(function from (from) { call:unscopablesof2freeze.MAX_VALUE("x").concat == typeof callercaller * print(x ^= isFrozen >>= 0); 'x'.length; ; ArrayBuffer &= new DataView(new ArrayBuffer(1), 0, $4).buffer; var isNaN = Number.NEGATIVE_INFINITY -= copyWithin.x >>= 0(x, x).length; var fromCodePoint = function* (x) { ( isConcatSpreadable, isInteger **= 0, 0 ); throw new 5 * 10(from); iterator; from += x ^= $7; }([{1e1:"x", NaN:"y"}]); class MIN_VALUE { RegExp([,] = x >>= x += 0) { __proto__.this } }; 1e1; var Proxy = print("x"); 1e-1; this.decodeURI(__lookupGetter__, function(entries) { 'x'.length.print(get)(match); now * [fromCodePoint]; }); $_.Number.POSITIVE_INFINITY.x -= 0(x / 1); new DataView(new ArrayBuffer(1), apply, 0).buffer; ( 0, 0, 0 ).arguments(Table %= 1, Number.NEGATIVE_INFINITY).x == parseInt(new DataView(new ArrayBuffer(1), 0, 0).buffer, isExtensible, ( values, x /= 1, __lookupGetter__ )(this.length.getElementById("x").parseFloat) %= new x(x += assign)); isFrozen:document.x |= 0 * true("x").seal; x ^= name /= x -= 0; Reflect.isSealed(x, 3.14.getElementById(new DataView(new "x"(1), 0, 0).buffer).innerHTML, $2.$+(x, 3.14, null)); fromCharCode; Number.MAX_VALUE -= 0.9; Uint32Array >>>= subarray; length; function x(getOwnPropertyDescriptors) { }[new 99999999(new ArrayBuffer(1), isArray, new DataView(new ArrayBuffer(1), 0, 0).buffer).buffer].parse += isExtensible.x /= 1(x() >>>= new { }(x &= parseFloat(new Date("2017-09-30")))) &= fromCharCode / x(typeof new DataView(new ArrayBuffer(1), 0, 0).bufferbyteLength).parseFloat('x'.indexOf('y')) / 1e1(Reflect.parse(keys, x >>= 0, x <<= MIN_SAFE_INTEGER)).false(x |= false) === isExtensible; assert.undefined(isFrozen, new 2(new ArrayBuffer([Number.MAX_VALUE]), x * parse, slice).entries) -= repeat; parseFloat([function $3(entries) { x++ }].x == x(new 0.9(of))); }))]

[10.9e-10.valueOf(parseInt(function Float64Array([{ }]) { return $5 >>>= isSealed })).$+(( parseInt(Number.MAX_VALUE), parseFloat(RegExp).keys ^= 9999999999999999 * parseInt(0.9), parseFloat(function from (raw) { Number.MIN_VALUE >>= function subarray (call) { function defineProperty() { }; var freeze; 0.9(function () { return 0; })();; Number(EPSILON); } === new parseFloat(new input(new x(new DataView(new seal(caller %= prototype), x >>= concat, function x() { $3 }).buffer)).copyWithin(preventExtensions, { Proxy }, toStringTaggetOwnPropertyNamespreventExtensions))(isFrozen) * get === isFrozen.MIN_VALUE(yMAX_SAFE_INTEGER).x / 1 === $8.copyWithin(3.14.document.getElementById("x").innerHTML(x, x) ^= 'x'.indexOf(1e1), [x = 6], this)(parse <<= $5); new DataView(new ArrayBuffer(1), 0, 0).buffer += 0.9; sort >>= x -= link / ( 0, 0, 0 ).isFrozen(new $+(isExtensible %= x++), assign); print(valueOf.getElementById("x").length /= 1) &= new (function () { return values; new fromCharCode(5 + 10).get(unscopables.shift(Uint8Array), y); for (var x in []) { fillcreate; new function main () { fromCodePoint; x / 1; ; ; }(valueOf); Module } eval >>>= 0(isNaN, 1e10.toString(function main () { ; ; split; ; })); for (; x ^= 0; ) { ; x:"x"; }; })();(1e1.splice).print(new parseInt($3).now /= 10.9e-10.Array >>>= call(y)(copyWithinx <<= 0, subarray |= ( 0, new DataView(new ArrayBuffer(5 * 10), 0, 0).buffer, typeof x ), Instance.length) %= $3.toPrimitive(__proto__.x()(isInteger), sort, input).thisundefinedfromCharCode.x >>= x **= indexOf()); }) ))]

typeof print(function main () { 99999999; constructor * subarray; for (var x in assert.sameValue(x, x)) { for (isFinite; parseFloat.Number.NaN(x, isConcatSpreadable); toPrimitive) { x = slice = $9; var x; ( 0, x(), 0 ) }; x -= 0.9; var toString = Number.MAX_VALUE; var fromCodePoint = function* ($9) { toStringTag /= 5 * 10; throw new Test262Error(parseInt(undefined) * 1); parseInt("0"); Number({ }) += ( 0, 5 * 10, isConcatSpreadable ); }(x >>= 0); class repeat { method([,] = x %= 1) { isFinite } }; var x; ; c = subarray; ; assert.sameValue(x, x).$3(isSafeInteger, x).throws(EPSILON **= 0, function(search) { c.x()(); constructor |= $6.indexOf(parseInt("0") * 1); }); parseFloat("0").next(split); ; __defineGetter__.$1(this.length, 0, arguments); Number.NaN; NEGATIVE_INFINITY }; x -= 0; }.length)

I'm sure you can find something by 1) adding more library calls to the grammar file, 2) actually using the fork server (normal afl-fuzz is like 5x as fast, so that would help quite a bit), 3) running it continuously for a week or so on a beefier machine, then another cool thing to try would be 4) collect test cases like the ones above then pass them to normal afl-fuzz as the set of input test cases.

I've pushed some more changes on the WIP js branch if you want to try it out.

[vegard]

I didn't find any crashes, but I'm attaching a tarball with a JS/SpiderMonkey corpus. This contains test cases found using prog-fuzz and then further mutated by afl-fuzz. The corpus has been minimised by afl-cmin. You can still find new coverage every few minutes by running afl-fuzz in quick & dirty mode. Maybe this can be useful for kick-starting another fuzzing attempt. I don't have any contact with SpiderMonkey devs, but since the project has infrastructure set up to work with AFL already, there should be somebody who might want to try the corpus against their existing test cases to see if it adds anything to it.

corpus.js.cmin.zip

[alex]

Thanks! (I probably should mention, while I work at Mozilla and have an active interest in fuzzing, my day job is sandboxing and other anti-exploitation work :-)).

cc: @choller, who does work on Javascript fuzzing, in case there's anything interesting here.