Styletransfer breaks - Githubissues

iam-yan commented 3 years ago

Setup

Web browser & version: Chrome 85.0.4183.83
Operating System: macOS 10.15.6
ml5 version you're using: 0.5
Any additional notes: use in Next.js project

Issue:

Similar with the issue #370 Requested texture size greater than WebGL maximum on this browser / GPU

When I use a smaller img, I got the following warn: ml5.min.js?255d:18 High memory usage in GPU: 3844.52 MB, most likely due to a memory leak and errs:

Couldn't parse line number in error: 
ml5.min.js?255d:18 #version 300 es
    precision highp float;
    precision highp int;
    precision highp sampler2D;
    in vec2 resultUV;
    out vec4 outputColor;
    const vec2 halfCR = vec2(0.5, 0.5);

    struct ivec5
    {
      int x;
      int y;
      int z;
      int w;
      int u;
    };

    struct ivec6
    {
      int x;
      int y;
      int z;
      int w;
      int u;
      int v;
    };

    uniform float NAN;
    #define isnan(value) isnan_custom(value)

      bool isnan_custom(float val) {
        return (val > 0. || val < 0. || val == 0.) ? false : true;
      }

    bvec4 isnan_custom(vec4 val) {
      return bvec4(isnan(val.x), isnan(val.y), isnan(val.z), isnan(val.w));
    }

      const float INFINITY = uintBitsToFloat(uint(0x7f800000));

      #define round(value) newRound(value)
      int newRound(float value) {
        return int(floor(value + 0.5));
      }

      ivec4 newRound(vec4 value) {
        return ivec4(floor(value + vec4(0.5)));
      }

    int imod(int x, int y) {
      return x - y * (x / y);
    }

    int idiv(int a, int b, float sign) {
      int res = a / b;
      int mod = imod(a, b);
      if (sign < 0. && mod != 0) {
        res -= 1;
      }
      return res;
    }

    //Based on the work of Dave Hoskins
    //https://www.shadertoy.com/view/4djSRW
    #define HASHSCALE1 443.8975
    float random(float seed){
      vec2 p = resultUV * seed;
      vec3 p3  = fract(vec3(p.xyx) * HASHSCALE1);
      p3 += dot(p3, p3.yzx + 19.19);
      return fract((p3.x + p3.y) * p3.z);
    }

vec2 uvFromFlat(int texNumR, int texNumC, int index) {
  int texR = index / texNumC;
  int texC = index - texR * texNumC;
  return (vec2(texC, texR) + halfCR) / vec2(texNumC, texNumR);
}
vec2 packedUVfrom1D(int texNumR, int texNumC, int index) {
  int texelIndex = index / 2;
  int texR = texelIndex / texNumC;
  int texC = texelIndex - texR * texNumC;
  return (vec2(texC, texR) + halfCR) / vec2(texNumC, texNumR);
}

vec2 packedUVfrom2D(int texelsInLogicalRow, int texNumR,
  int texNumC, int row, int col) {
  int texelIndex = (row / 2) * texelsInLogicalRow + (col / 2);
  int texR = texelIndex / texNumC;
  int texC = texelIndex - texR * texNumC;
  return (vec2(texC, texR) + halfCR) / vec2(texNumC, texNumR);
}

vec2 packedUVfrom3D(int texNumR, int texNumC,
    int texelsInBatch, int texelsInLogicalRow, int b,
    int row, int col) {
  int index = b * texelsInBatch + (row / 2) * texelsInLogicalRow + (col / 2);
  int texR = index / texNumC;
  int texC = index - texR * texNumC;
  return (vec2(texC, texR) + halfCR) / vec2(texNumC, texNumR);
}

  float getChannel(vec4 frag, vec2 innerDims) {
    vec2 modCoord = mod(innerDims, 2.);
    return modCoord.x == 0. ?
      (modCoord.y == 0. ? frag.r : frag.g) :
      (modCoord.y == 0. ? frag.b : frag.a);
  }
  float getChannel(vec4 frag, int dim) {
    float modCoord = mod(float(dim), 2.);
    return modCoord == 0. ? frag.r : frag.g;
  }

    float sampleTexture(sampler2D textureSampler, vec2 uv) {
      return texture(textureSampler, uv).r;
    }

    void setOutput(vec4 val) {
      outputColor = val;
    }

uniform sampler2D A;
uniform int offsetA;
uniform sampler2D B;
uniform int offsetB;

    ivec3 getOutputCoords() {
      ivec2 resTexRC = ivec2(resultUV.yx *
                             vec2(316, 1008));
      int index = resTexRC.x * 1008 + resTexRC.y;

      int b = index / 504;
      index -= b * 504;

      int r = 2 * (index / 2);
      int c = imod(index, 2) * 2;

      return ivec3(b, r, c);
    }

    vec4 getA(int b, int row, int col) {
      vec2 uv = packedUVfrom3D(
        565, 564, 504, 2, b, row, col);
      return texture(A, uv);
    }

    vec4 getAAtOutCoords() {
      ivec3 coords = getOutputCoords();

      vec4 outputValue = getA(coords.x, coords.y, coords.z);
      return outputValue;
    }

    vec4 getB() {
      return texture(B, halfCR);
    }

    vec4 getBAtOutCoords() {
      ivec3 coords = getOutputCoords();

      vec4 outputValue = getB();

        return vec4(outputValue.x);

    }

      vec4 binaryOperation(vec4 a, vec4 b) {

  // vec4 one = vec4(equal(a, b));
  // return one + (vec4(1.0) - one) * a / b;
  vec4 result = a / b;
  if(b.x == 0.0) {
    result.x = NAN;
  } else if(a.x == b.x) {
    result.x = 1.;
  }
  if(b.y == 0.0) {
    result.y = NAN;
  } else if(a.y == b.y) {
    result.y = 1.;
  }
  if(b.z == 0.0) {
    result.z = NAN;
  } else if(a.z == b.z) {
    result.z = 1.;
  }
  if(b.w == 0.0) {
    result.w = NAN;
  } else if(a.w == b.w) {
    result.w = 1.;
  }

  return result;

      }

      void main() {
        vec4 a = getAAtOutCoords();
        vec4 b = getBAtOutCoords();

        vec4 result = binaryOperation(a, b);

          ivec3 coords = getOutputCoords();

            bool nextRowOutOfBounds =
              (coords.y + 1) >= 504;
            bool nextColOutOfBounds =
              (coords.z + 1) >= 3;
            result.y = nextColOutOfBounds ? 0. : result.y;
            result.z = nextRowOutOfBounds ? 0. : result.z;
            result.w = nextColOutOfBounds || nextRowOutOfBounds ? 0. : result.w;

        setOutput(result);
      }

MlExperiment.tsx?5afc:48 Error: Failed to compile fragment shader.
    at Gt (ml5.min.js?255d:18)
    at t.createProgram (ml5.min.js?255d:18)
    at eval (ml5.min.js?255d:18)
    at eval (ml5.min.js?255d:18)
    at t.getAndSaveBinary (ml5.min.js?255d:18)
    at t.compileAndRun (ml5.min.js?255d:18)
    at t.packedBinaryOp (ml5.min.js?255d:18)
    at t.realDivide (ml5.min.js?255d:18)
    at jt.runKernel.$a (ml5.min.js?255d:18)
    at eval (ml5.min.js?255d:18)
    at t.scopedRun (ml5.min.js?255d:18)
    at t.runKernel (ml5.min.js?255d:18)
    at div_ (ml5.min.js?255d:18)
    at Object.div (ml5.min.js?255d:18)
    at t.div (ml5.min.js?255d:18)
    at eval (ml5.min.js?255d:18)
    at eval (ml5.min.js?255d:18)
    at eval (ml5.min.js?255d:18)
    at t.scopedRun (ml5.min.js?255d:18)
    at t.runKernel (ml5.min.js?255d:18)
    at eval (ml5.min.js?255d:18)
    at mean_ (ml5.min.js?255d:18)
    at Object.mean (ml5.min.js?255d:18)
    at t.mean (ml5.min.js?255d:18)
    at moments_ (ml5.min.js?255d:18)
    at Object.moments (ml5.min.js?255d:18)
    at eval (ml5.min.js?255d:125)
    at eval (ml5.min.js?255d:18)
    at t.scopedRun (ml5.min.js?255d:18)
    at t.tidy (ml5.min.js?255d:18)
    at Object.Le (ml5.min.js?255d:18)
    at e.value (ml5.min.js?255d:125)
    at e.value (ml5.min.js?255d:125)
    at eval (ml5.min.js?255d:125)
    at eval (ml5.min.js?255d:18)
    at t.scopedRun (ml5.min.js?255d:18)
    at t.tidy (ml5.min.js?255d:18)
    at Object.Le (ml5.min.js?255d:18)
    at eval (ml5.min.js?255d:125)
    at x (ml5.min.js?255d:175)
    at Generator.eval [as _invoke] (ml5.min.js?255d:175)
    at Generator.t.<computed> [as next] (ml5.min.js?255d:175)
    at i (ml5.min.js?255d:49)
    at eval (ml5.min.js?255d:49)
    at new Promise (<anonymous>)
    at new e (ml5.min.js?255d:56)
    at eval (ml5.min.js?255d:49)
    at eval (ml5.min.js?255d:125)
    at eval (ml5.min.js?255d:125)
    at x (ml5.min.js?255d:175)

lindapaiste commented 2 years ago

PR #1347 should resolve the second issue, re high memory usage / memory leak.

lindapaiste commented 2 years ago

I just got a crash with a very similar stack trace while running the video example. I'm pasting this here to investigate later.

High memory usage in GPU: 3695.68 MB, most likely due to a memory leak
e.acquireTexture @ tf-core.esm.js:17
e.uploadToGPU @ tf-core.esm.js:17
e.runWebGLProgram @ tf-core.esm.js:17
e.compileAndRun @ tf-core.esm.js:17
e.conv2dWithIm2Row @ tf-core.esm.js:17
e.conv2d @ tf-core.esm.js:17
Wt.runKernelFunc.x @ tf-core.esm.js:17
(anonymous) @ tf-core.esm.js:17
(anonymous) @ tf-core.esm.js:17
t.scopedRun @ tf-core.esm.js:17
t.tidy @ tf-core.esm.js:17
f @ tf-core.esm.js:17
(anonymous) @ tf-core.esm.js:17
t.scopedRun @ tf-core.esm.js:17
t.runKernelFunc @ tf-core.esm.js:17
conv2d_ @ tf-core.esm.js:17
conv2d @ tf-core.esm.js:17
value @ index.js:83
(anonymous) @ index.js:150
(anonymous) @ tf-core.esm.js:17
t.scopedRun @ tf-core.esm.js:17
t.tidy @ tf-core.esm.js:17
rn @ tf-core.esm.js:17
(anonymous) @ index.js:139
l @ runtime.js:63
(anonymous) @ runtime.js:294
(anonymous) @ runtime.js:119
n @ asyncToGenerator.js:3
s @ asyncToGenerator.js:25
(anonymous) @ asyncToGenerator.js:32
(anonymous) @ asyncToGenerator.js:21
(anonymous) @ index.js:137
(anonymous) @ index.js:134
l @ runtime.js:63
(anonymous) @ runtime.js:294
(anonymous) @ runtime.js:119
n @ asyncToGenerator.js:3
s @ asyncToGenerator.js:25
(anonymous) @ asyncToGenerator.js:32
(anonymous) @ asyncToGenerator.js:21
(anonymous) @ index.js:113
startStop @ sketch.js:56
(anonymous) @ p5.min.js:3
Show 13 more frames
tf-core.esm.js:17 Couldn't parse line number in error: 
tf-core.esm.js:17 #version 300 es
    precision highp float;
    precision highp int;
    precision highp sampler2D;
    in vec2 resultUV;
    out vec4 outputColor;
    const vec2 halfCR = vec2(0.5, 0.5);

    struct ivec5
    {
      int x;
      int y;
      int z;
      int w;
      int u;
    };

    struct ivec6
    {
      int x;
      int y;
      int z;
      int w;
      int u;
      int v;
    };

    uniform float NAN;

      bool isnan_custom(float val) {
        return (val > 0.0 || val < 0.0) ? false : val != 0.0;
      }

      bvec4 isnan_custom(vec4 val) {
        return bvec4(isnan_custom(val.x),
          isnan_custom(val.y), isnan_custom(val.z), isnan_custom(val.w));
      }

      #define isnan(value) isnan_custom(value)

      #define round(value) newRound(value)
      int newRound(float value) {
        return int(floor(value + 0.5));
      }

      ivec4 newRound(vec4 value) {
        return ivec4(floor(value + vec4(0.5)));
      }

    int imod(int x, int y) {
      return x - y * (x / y);
    }

    int idiv(int a, int b, float sign) {
      int res = a / b;
      int mod = imod(a, b);
      if (sign < 0. && mod != 0) {
        res -= 1;
      }
      return res;
    }

    //Based on the work of Dave Hoskins
    //https://www.shadertoy.com/view/4djSRW
    #define HASHSCALE1 443.8975
    float random(float seed){
      vec2 p = resultUV * seed;
      vec3 p3  = fract(vec3(p.xyx) * HASHSCALE1);
      p3 += dot(p3, p3.yzx + 19.19);
      return fract((p3.x + p3.y) * p3.z);
    }

vec2 uvFromFlat(int texNumR, int texNumC, int index) {
  int texR = index / texNumC;
  int texC = index - texR * texNumC;
  return (vec2(texC, texR) + halfCR) / vec2(texNumC, texNumR);
}
vec2 packedUVfrom1D(int texNumR, int texNumC, int index) {
  int texelIndex = index / 2;
  int texR = texelIndex / texNumC;
  int texC = texelIndex - texR * texNumC;
  return (vec2(texC, texR) + halfCR) / vec2(texNumC, texNumR);
}

vec2 packedUVfrom2D(int texelsInLogicalRow, int texNumR,
  int texNumC, int row, int col) {
  int texelIndex = (row / 2) * texelsInLogicalRow + (col / 2);
  int texR = texelIndex / texNumC;
  int texC = texelIndex - texR * texNumC;
  return (vec2(texC, texR) + halfCR) / vec2(texNumC, texNumR);
}

vec2 packedUVfrom3D(int texNumR, int texNumC,
    int texelsInBatch, int texelsInLogicalRow, int b,
    int row, int col) {
  int index = b * texelsInBatch + (row / 2) * texelsInLogicalRow + (col / 2);
  int texR = index / texNumC;
  int texC = index - texR * texNumC;
  return (vec2(texC, texR) + halfCR) / vec2(texNumC, texNumR);
}

  float getChannel(vec4 frag, vec2 innerDims) {
    vec2 modCoord = mod(innerDims, 2.);
    return modCoord.x == 0. ?
      (modCoord.y == 0. ? frag.r : frag.g) :
      (modCoord.y == 0. ? frag.b : frag.a);
  }
  float getChannel(vec4 frag, int dim) {
    float modCoord = mod(float(dim), 2.);
    return modCoord == 0. ? frag.r : frag.g;
  }

    float sampleTexture(sampler2D textureSampler, vec2 uv) {
      return texture(textureSampler, uv).r;
    }

    void setOutput(vec4 val) {
      outputColor = val;
    }

uniform sampler2D A;
uniform int offsetA;
uniform sampler2D B;
uniform int offsetB;

    ivec3 getOutputCoords() {
      ivec2 resTexRC = ivec2(resultUV.yx *
                             vec2(240, 1280));
      int index = resTexRC.x * 1280 + resTexRC.y;

      int b = index / 640;
      index -= b * 640;

      int r = 2 * (index / 2);
      int c = imod(index, 2) * 2;

      return ivec3(b, r, c);
    }

    vec4 getA(int b, int row, int col) {
      vec2 uv = packedUVfrom3D(
        555, 554, 640, 2, b, row, col);
      return texture(A, uv);
    }

    vec4 getAAtOutCoords() {
      ivec3 coords = getOutputCoords();

      vec4 outputValue = getA(coords.x, coords.y, coords.z);
      return outputValue;
    }

    vec4 getB() {
      return texture(B, halfCR);
    }

    vec4 getBAtOutCoords() {
      ivec3 coords = getOutputCoords();

      vec4 outputValue = getB();

        return vec4(outputValue.x);

    }

      vec4 binaryOperation(vec4 a, vec4 b) {

  // vec4 one = vec4(equal(a, b));
  // return one + (vec4(1.0) - one) * a / b;
  vec4 result = a / b;
  if(a.x == b.x) {
    result.x = 1.;
  }
  if(a.y == b.y) {
    result.y = 1.;
  }
  if(a.z == b.z) {
    result.z = 1.;
  }
  if(a.w == b.w) {
    result.w = 1.;
  }

  return result;

      }

      void main() {
        vec4 a = getAAtOutCoords();
        vec4 b = getBAtOutCoords();

        vec4 result = binaryOperation(a, b);

          ivec3 coords = getOutputCoords();

            bool nextRowOutOfBounds =
              (coords.y + 1) >= 640;
            bool nextColOutOfBounds =
              (coords.z + 1) >= 3;
            result.y = nextColOutOfBounds ? 0. : result.y;
            result.z = nextRowOutOfBounds ? 0. : result.z;
            result.w = nextColOutOfBounds || nextRowOutOfBounds ? 0. : result.w;

        setOutput(result);
      }

examples.ml5js.org/:1 [.WebGL-0x2000bca900] GL_INVALID_FRAMEBUFFER_OPERATION: Draw framebuffer is incomplete
examples.ml5js.org/:1 WebGL: CONTEXT_LOST_WEBGL: loseContext: context lost
examples.ml5js.org/:1 WebGL: CONTEXT_LOST_WEBGL: loseContext: context lost
sketch.js:63 Uncaught (in promise) TypeError: Cannot read properties of undefined (reading 'src')
    at gotResult (sketch.js:63:34)
    at callcallback.js:36:9

ml5js / ml5-library

Styletransfer breaks #1053

Setup

Issue: