HaxeFoundation / haxe

Haxe - The Cross-Platform Toolkit
https://haxe.org
6.14k stars 655 forks source link

FPHelper float-int reinterpretation polyfill bug #7497

Open farteryhr opened 5 years ago

farteryhr commented 5 years ago

the conversion near power of two is mostly wrong. my fix and test code: https://github.com/HaxeFoundation/haxe/pull/6697#issuecomment-416920617

R32 commented 5 years ago

According to your comment, I made the following changes, it seems work well.

    static inline function _floatToI32(f: Float): Int {
        if( f == 0 ) return 0;
        var af = f < 0 ? -f : f;
        var exp = Math.floor(Math.log(af) / LN2);
        if (exp > 127) {
-           return 0x7F800000;
+           return f < 0 ? 0xFF800000 :0x7F800000;
        } else {
            if (exp <= -127 ) {
                exp = -127;
                af *= 7.1362384635298e+44;  // af * 0.5 * 0x800000 / Math.pow(2, -127)
            } else {
-               af = (af / Math.pow(2, exp) - 1.0) * 0x800000;
+               var epot = af / Math.pow(2, exp);
+               if (epot >= 2) {
+                   exp++;
+                   epot *= 0.5;
+               }
+               af = (epot - 1.0) * 0x800000;
            }
            return (f < 0 ? 0x80000000 : 0) | ((exp + 127) << 23) | Math.round(af);
        }
    }

    static inline function _doubleToI64(v: Float): Int64 @:privateAccess {
        var i64 = i64tmp;
        if( v == 0 ) {
@@ -90,17 +95,25 @@ class FPHelper {
            if (exp > 1023) {
                i64.set_low(0xFFFFFFFF);
                i64.set_high(0x7FEFFFFF);
            } else {
                if (exp <= -1023) {
                    exp = -1023;
                    av = av / 2.2250738585072014e-308;
                } else {
-                   av = av / Math.pow(2, exp) - 1.0;
+                   var epot = av / Math.pow(2, exp);
+                   if (epot >= 2) {
+                       exp++;
+                       epot *= 0.5;
+                   } else if (epot < 1) {
+                       exp--;
+                       epot *= 2.0;
+                   }
+                   av = epot - 1.0;
                }
                var sig = Math.fround(av * 4503599627370496.); // 2^52