search

tensorflow / tfjs

A WebGL accelerated JavaScript library for training and deploying ML models.
https://js.tensorflow.org
Apache License 2.0
18.37k stars 1.92k forks source link

Obscure error message when tensor memory exhausted: Couldn't parse line number in error #1499

Closed ToonTalk closed 5 years ago

ToonTalk commented 5 years ago

TensorFlow.js version

1.04

Browser version

Chrome Version 73.0.3683.103 (Official Build) (64-bit)

Describe the problem or feature request

When using mobilenet and knn repeatedly with a memory leak (see issue #1498) it produces the following error messages instead of something indicating out of tensor memory:

Couldn't parse line number in error: tfjs.js:2 #version 300 es precision highp float; precision highp int; precision highp sampler2D; in vec2 resultUV; out vec4 outputColor; const vec2 halfCR = vec2(0.5, 0.5);

struct ivec5
{
  int x;
  int y;
  int z;
  int w;
  int u;
};

struct ivec6
{
  int x;
  int y;
  int z;
  int w;
  int u;
  int v;
};

uniform float NAN;
#define isnan(value) isnan_custom(value)

  bool isnan_custom(float val) {
    return (val > 0. || val < 0. || val == 0.) ? false : true;
  }

bvec4 isnan_custom(vec4 val) {
  return bvec4(isnan(val.x), isnan(val.y), isnan(val.z), isnan(val.w));
}

  const float INFINITY = uintBitsToFloat(uint(0x7f800000));

  #define round(value) newRound(value)
  int newRound(float value) {
    return int(floor(value + 0.5));
  }

  ivec4 newRound(vec4 value) {
    return ivec4(floor(value + vec4(0.5)));
  }

int imod(int x, int y) {
  return x - y * (x / y);
}

int idiv(int a, int b, float sign) {
  int res = a / b;
  int mod = imod(a, b);
  if (sign < 0. && mod != 0) {
    res -= 1;
  }
  return res;
}

//Based on the work of Dave Hoskins
//https://www.shadertoy.com/view/4djSRW
#define HASHSCALE1 443.8975
float random(float seed){
  vec2 p = resultUV * seed;
  vec3 p3  = fract(vec3(p.xyx) * HASHSCALE1);
  p3 += dot(p3, p3.yzx + 19.19);
  return fract((p3.x + p3.y) * p3.z);
}

vec2 uvFromFlat(int texNumR, int texNumC, int index) { int texR = index / texNumC; int texC = index - texR texNumC; return (vec2(texC, texR) + halfCR) / vec2(texNumC, texNumR); } vec2 packedUVfrom1D(int texNumR, int texNumC, int index) { int texelIndex = index / 2; int texR = texelIndex / texNumC; int texC = texelIndex - texR texNumC; return (vec2(texC, texR) + halfCR) / vec2(texNumC, texNumR); }

vec2 packedUVfrom2D(int texelsInLogicalRow, int texNumR, int texNumC, int row, int col) { int texelIndex = (row / 2) texelsInLogicalRow + (col / 2); int texR = texelIndex / texNumC; int texC = texelIndex - texR texNumC; return (vec2(texC, texR) + halfCR) / vec2(texNumC, texNumR); }

vec2 packedUVfrom3D(int texNumR, int texNumC, int texelsInBatch, int texelsInLogicalRow, int b, int row, int col) { int index = b texelsInBatch + (row / 2) texelsInLogicalRow + (col / 2); int texR = index / texNumC; int texC = index - texR * texNumC; return (vec2(texC, texR) + halfCR) / vec2(texNumC, texNumR); }

float getChannel(vec4 frag, vec2 innerDims) { vec2 modCoord = mod(innerDims, 2.); return modCoord.x == 0. ? (modCoord.y == 0. ? frag.r : frag.g) : (modCoord.y == 0. ? frag.b : frag.a); } float getChannel(vec4 frag, int dim) { float modCoord = mod(float(dim), 2.); return modCoord == 0. ? frag.r : frag.g; }

float sampleTexture(sampler2D textureSampler, vec2 uv) {
  return texture(textureSampler, uv).r;
}

void setOutput(vec4 val) {
  outputColor = val;
}

uniform sampler2D T0; uniform int offsetT0; uniform sampler2D T1; uniform int offsetT1;

  ivec2 getOutputCoords() {
    return 2 * ivec2(resultUV.yx * vec2(773, 500));
  }

vec4 getT0(int row, int col) {
  vec2 uv = packedUVfrom2D(500, 773, 500, row, col);
  return texture(T0, uv);
}

vec4 getT0AtOutCoords() {
  ivec2 coords = getOutputCoords();

  vec4 outputValue = getT0(coords.x, coords.y);
  return outputValue;
}

vec4 getT1(int row, int col) {
  vec2 uv = packedUVfrom2D(500, 1, 500, row, col);
  return texture(T1, uv);
}

vec4 getT1AtOutCoords() {
  ivec2 coords = getOutputCoords();
  coords.x = 0;
  vec4 outputValue = getT1(coords.x, coords.y);
  return vec4(outputValue.x, outputValue.y, outputValue.x, outputValue.y);
}

  float getValue(int x,int y) {
    if (x < 1545)
      return getChannel(getT0(x,y), vec2(x,y));
    else {
      x -= 1545;
      return getChannel(getT1(x,y), vec2(x,y));
    }
  }

  void main() {
    ivec2 coords = getOutputCoords();
    vec4 result = vec4(getValue(coords.x,coords.y), 0., 0., 0.);
    if (++coords.y < 1000) {
      result.g = getValue(coords.x,coords.y);
    }
    if (++coords.x < 1546) {
      result.a = getValue(coords.x,coords.y);
    }
    if (coords.x < 1546 &&
        --coords.y < 1000) {
      result.b = getValue(coords.x,coords.y);
    }
    setOutput(result);
  }

tfjs.js:2 Uncaught Error: Failed to compile fragment shader. at kr (tfjs.js:2) at t.createProgram (tfjs.js:2) at tfjs.js:2 at tfjs.js:2 at t.getAndSaveBinary (tfjs.js:2) at t.compileAndRun (tfjs.js:2) at t.concat (tfjs.js:2) at tfjs.js:2 at tfjs.js:2 at t.scopedRun (tfjs.js:2) kr @ tfjs.js:2 t.createProgram @ tfjs.js:2 (anonymous) @ tfjs.js:2 (anonymous) @ tfjs.js:2 t.getAndSaveBinary @ tfjs.js:2 t.compileAndRun @ tfjs.js:2 t.concat @ tfjs.js:2 (anonymous) @ tfjs.js:2 (anonymous) @ tfjs.js:2 t.scopedRun @ tfjs.js:2 t.runKernel @ tfjs.js:2 concat_ @ tfjs.js:2 concat @ tfjs.js:2 t.concat @ tfjs.js:2 (anonymous) @ knn-classifier.js:2 (anonymous) @ tfjs.js:2 t.scopedRun @ tfjs.js:2 t.tidy @ tfjs.js:2 t.tidy @ tfjs.js:2 t.addExample @ knn-classifier.js:2 (anonymous) @ nails.js:305 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 load (async) load_image @ nails.js:217 add_image_to_training @ nails.js:302 next_image @ nails.js:241 (anonymous) @ nails.js:307 image.onload @ nails.js:218 localhost/:1 WebGL: CONTEXT_LOST_WEBGL: loseContext: context lost

Code to reproduce the bug / link to feature request

https://ecraft2learn.github.io/ai/onyx/memory-leak-index.html after a few minutes after loading - console shows the tf.memory on each iteration

vadimkantorov commented 5 years ago

Hitting the same issue with a speech recognition model in https://github.com/tensorflow/tfjs/issues/1657#issuecomment-506458895

tafsiri commented 5 years ago

@vadimkantorov Are you using the knn classifier? if not could you post a new issue if you see a memory leak in the speech recognition model.

vadimkantorov commented 5 years ago

@tafsiri No it's not a knn classifier, but I suspect it's also a memory issue

Leekao commented 5 years ago

@vadimkantorov Are you using the knn classifier? if not could you post a new issue if you see a memory leak in the speech recognition model.

I am running into the same issue and using knn classifier, only on mobile (ios and android)

rthadur commented 5 years ago

@Leekao please open a new issue here New Issue Template , thank you