make js code correct on little + big endian

timotheecour commented 3 years ago

some js code in nim (eg https://github.com/nim-lang/Nim/pull/16592) uses implicit assumption of little endian; this is likely true for most platforms (even more so for js code) but still, it's not always correct.

example

when defined(js):
  proc toBitsImpl(x: float): array[2, uint32] =
    asm """
    const buffer = new ArrayBuffer(8);
    const floatBuffer = new Float64Array(buffer);
    const uintBuffer = new Uint32Array(buffer);
    floatBuffer[0] = `x`;
    `result` = uintBuffer
    """

proposal

use this (or better) from https://stackoverflow.com/questions/7869752/javascript-typed-arrays-and-endianness

const isBigEndian = (() => {
    const array = new Uint8Array(4);
    const view = new Uint32Array(array.buffer);
    return !((view[0] = 1) & array[0]);
})();

then adapt code depending on isBigEndian

Additional Information

1.5.1 d34d023da1d7af972366c3af58a144b395964b4c low priority because big endian is not a common thing in js

/cc @xflywind @juancarlospaco

links

https://riptutorial.com/javascript/example/13317/little---big-endian-for-typed-arrays-when-using-bitwise-operators see Example where Edian type is important

let's assume littleEndian in JS backend by xflywind · Pull Request #16886 · nim-lang/Nim

make endians support JS backend by xflywind · Pull Request #16127 · nim-lang/Nim

endian independent views

https://stackoverflow.com/questions/63179879/should-i-care-about-big-endian-machines-while-using-uint32array

If you're concerned about endianness, you can use the DataView interface. This allows you to assert the endianness of a byte array, rather than relying on the platform byte order.

big endian machines

https://stackoverflow.com/questions/63179879/should-i-care-about-big-endian-machines-while-using-uint32array

This may be a concern if you expect your code to run on a PowerPC platform like an XBox or PS3.

timotheecour commented 3 years ago

swap endian-ness (to add support for std/endian in js):

https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/DataView/getInt32

littleEndian Optional Indicates whether the 32-bit int is stored in little- or big-endian format. If false or undefined, a big-endian value is read.

eg how to use it: https://www.i-programmer.info/programming/javascript/6151-javascript-data-structures-typed-arrays-ii.html?start=1

If you have a Big-endian array of two byte integers and want to work with it on a Little-endian machine you can use the equivalent idea for an array:

nodejs

https://nodejs.org/api/buffer.html#buffer_buf_swap16

ringabout commented 3 years ago

The full codebase

when defined(js):
  import std/jsbigints

  type
    ArrayBuffer* = ref object of JsRoot
    Float64Array* = ref object of JsRoot
    Uint32Array* = ref object of JsRoot
    BigUint64Array* = ref object of JsRoot

  func newArrayBuffer*(n: int): ArrayBuffer {.importjs: "new ArrayBuffer(#)".}
  func newFloat64Array*(buffer: ArrayBuffer): Float64Array {.importjs: "new Float64Array(#)".}
  func newUint32Array*(buffer: ArrayBuffer): Uint32Array {.importjs: "new Uint32Array(#)".}
  func newBigUint64Array*(buffer: ArrayBuffer): BigUint64Array {.importjs: "new BigUint64Array(#)".}

  func `[]`*(arr: Uint32Array, i: int): uint32 {.importjs: "#[#]".}
  func `[]`*(arr: BigUint64Array, i: int): JsBigInt {.importjs: "#[#]".}
  func `[]=`*(arr: Float64Array, i: int, v: float) {.importjs: "#[#] = #".}

  proc hasJsBigInt*(): bool =
    asm """`result` = typeof BigInt != 'undefined'"""

when defined(js):
  type
    Uint16Array* = ref object of JsRoot
    Uint8Array* = ref object of JsRoot

  func buffer(p: Uint32Array): ArrayBuffer {.importjs: "#.buffer".}

  func newUint32Array*(arr: openArray[uint32]): Uint32Array {.importjs: "new Uint32Array(#)".}

  func newUint8Array(p: ArrayBuffer): Uint8Array {.importjs: "new Uint8Array(#)".}

  func `[]`*(arr: Uint8Array, i: int): uint8 {.importjs: "#[#]".}

when defined(js):
  when defined(nodejs):
    {.emit: "const _nim_nodejs_os = require('os');".}

    proc endianness(): cstring {.importjs: "_nim_nodejs_os.endianness()".}

    template getEndiansJS(): Endianness =
      if endianness() == "LE":
        littleEndian
      else:
        bigEndian
  else:
    template getEndiansJS(): Endianness =
      let a = newUint32Array([0x12345678'u32])
      let b = newUint8Array(a.buffer)
      if b[0] == 0x78:
        littleEndian
      else:
        bigEndian

template getEndiansVM(): Endianness =
  when defined(i386) or defined(alpha) or defined(powerpc64el) or
      defined(ia64) or defined(amd64) or defined(mipsel) or defined(arm) or
      defined(arm64) or defined(avr) or defined(msp430) or defined(riscv32) or
      defined(riscv64) or defined(esp) or defined(wasm32) or defined(mips64el):
    littleEndian
  elif defined(m68k) or defined(powerpc) or defined(powerpc64) or defined(sparc) or
      defined(hppa) or defined(mips) or defined(sparc64) or defined(mips64):
    bigEndian
  else:
    littleEndian

proc getEndians(): Endianness {.inline.} =
  when nimvm:
    getEndiansVM()
  else:
    when defined(js):
      result = getEndiansJS()
    else:
      result = cpuEndian

const cpuEndiansVM* = getEndians()
let cpuEndiansRT* = getEndians()

#
#
#            Nim's Runtime Library
#        (c) Copyright 2012 Andreas Rumpf
#
#    See the file "copying.txt", included in this
#    distribution, for details about the copyright.
#

## This module contains helpers that deal with different byte orders
## (`endian`:idx:).
##
## Unstable API.
when not defined(js):
  template builtin_bswap(a: uint8): uint8 =
    a
  when defined(gcc) or defined(llvm_gcc) or defined(clang):
    func builtin_bswap(a: uint16): uint16 {.
        importc: "__builtin_bswap16", nodecl.}

    func builtin_bswap(a: uint32): uint32 {.
        importc: "__builtin_bswap32", nodecl.}

    func builtin_bswap(a: uint64): uint64 {.
        importc: "__builtin_bswap64", nodecl.}
  elif defined(icc):
    func builtin_bswap(a: uint16): uint16 {.
        importc: "_bswap16", nodecl.}

    func builtin_bswap(a: uint32): uint32 {.
        importc: "_bswap", nodecl.}

    func builtin_bswap(a: uint64): uint64 {.
        importc: "_bswap64", nodecl.}
  elif defined(vcc):
    func builtin_bswap(a: uint16): uint16 {.
        importc: "_byteswap_ushort", nodecl, header: "<intrin.h>".}

    func builtin_bswap(a: uint32): uint32 {.
        importc: "_byteswap_ulong", nodecl, header: "<intrin.h>".}

    func builtin_bswap(a: uint64): uint64 {.
        importc: "_byteswap_uint64", nodecl, header: "<intrin.h>".}

template builtin_bswapImpl(a: uint8): uint8 =
  a

template builtin_bswapImpl(a: uint16): uint16 =
  (a shl 8'u16) or (a shr 8'u16)

template builtin_bswapImpl(a: uint32): uint32 =
  ((a shl 24) and 0xff000000'u32) or
            ((a shl 8) and 0x00ff0000'u32) or
            ((a shr 8) and 0x0000ff00'u32) or
            ((a shr 24) and 0x000000ff'u32)

template builtin_bswapImpl(a: uint64): uint64 =
  var num = (a shl 32'u64) or (a shr 32'u64)
  num = ((num and 0x0000ffff0000ffff'u64) shl 16'u64) or
      ((num and 0xffff0000ffff0000'u64) shr 16'u64)

  ((num and 0x00ff00ff00ff00ff'u64) shl 8'u64) or
            ((num and 0xff00ff00ff00ff00'u64) shr 8'u64)

func swapEndian*[T: SomeUnsignedInt](a: T): T {.inline.} =
  when nimvm:
    result = builtin_bswapImpl(a)
  else:
    when defined(js):
      result = builtin_bswapImpl(a)
    else:
      result = builtin_bswap(a)

proc toEndian[T: SomeUnsignedInt](a: T, endianess: static Endianness): T {.inline.} =
  when nimvm:
    when cpuEndiansVM == endianess:
      result = swapEndian(a)
    else:
      result = a
  else:
    when defined(js):
      if cpuEndiansRT == endianess:
        result = swapEndian(a)
      else:
        result = a
    else:
      when cpuEndiansVM == endianess:
        result = swapEndian(a)
      else:
        result = a

proc toBigEndian*[T: SomeUnsignedInt](a: T): T {.inline.} =
  result = toEndian(a, littleEndian)

proc toLittleEndian*[T: SomeUnsignedInt](a: T): T {.inline.} =
  result = toEndian(a, bigEndian)

proc main() =
  doAssert toLittleEndian(12'u32) == 12
  doAssert toLittleEndian(12'u64) == 12
  doAssert toLittleEndian(12'u16) == 12
  doAssert toLittleEndian(12'u8) == 12

  echo toBigEndian(12'u64)
  doAssert toBigEndian(12'u32) == 201326592
  doAssert toBigEndian(12'u16) == 3072
  doAssert toBigEndian(12'u8) == 12

static: main()
main()

timotheecour commented 3 years ago

seems like a good start! is builtin_bswapImpl correct in nim js? eg for the whole range of uint32? ( i was expecting dataview.getUint32(byteOffset [, littleEndian]) would be needed for this, but maybe I'm wrong)

timotheecour / Nim