input with big image error for webgl backend: context lost

[yyc-git]

Hello everyone, Thanks for your great work!

My OS

Mac OS Big Sur 10+ Win10

Description

set backend to webgl

When I use input with 256*258 size image and inference, it's OK and no error

But when I use input with 1280*720 size image and inference, it error, error info is:

webnn-polyfill.js:7384 Couldn't parse line number in error: 
#version 300 es
    precision highp float;
    precision highp int;
    precision highp sampler2D;
    in vec2 resultUV;
    out vec4 outputColor;
    const vec2 halfCR = vec2(0.5, 0.5);

    struct ivec5
    {
      int x;
      int y;
      int z;
      int w;
      int u;
    };

    struct ivec6
    {
      int x;
      int y;
      int z;
      int w;
      int u;
      int v;
    };

    uniform float NAN;

      bool isnan_custom(float val) {
        uint floatToUint = floatBitsToUint(val);
        return (floatToUint & 0x7fffffffu) > 0x7f800000u;
      }

      bvec4 isnan_custom(vec4 val) {
        return bvec4(isnan_custom(val.x),
          isnan_custom(val.y), isnan_custom(val.z), isnan_custom(val.w));
      }

      #define isnan(value) isnan_custom(value)

      #define round(value) newRound(value)
      int newRound(float value) {
        return int(floor(value + 0.5));
      }

      ivec4 newRound(vec4 value) {
        return ivec4(floor(value + vec4(0.5)));
      }

    int imod(int x, int y) {
      return x - y * (x / y);
    }

    int idiv(int a, int b, float sign) {
      int res = a / b;
      int mod = imod(a, b);
      if (sign < 0. && mod != 0) {
        res -= 1;
      }
      return res;
    }

    //Based on the work of Dave Hoskins
    //https://www.shadertoy.com/view/4djSRW
    #define HASHSCALE1 443.8975
    float random(float seed){
      vec2 p = resultUV * seed;
      vec3 p3  = fract(vec3(p.xyx) * HASHSCALE1);
      p3 += dot(p3, p3.yzx + 19.19);
      return fract((p3.x + p3.y) * p3.z);
    }

vec2 uvFromFlat(int texNumR, int texNumC, int index) {
  int texR = index / texNumC;
  int texC = index - texR * texNumC;
  return (vec2(texC, texR) + halfCR) / vec2(texNumC, texNumR);
}
vec2 packedUVfrom1D(int texNumR, int texNumC, int index) {
  int texelIndex = index / 2;
  int texR = texelIndex / texNumC;
  int texC = texelIndex - texR * texNumC;
  return (vec2(texC, texR) + halfCR) / vec2(texNumC, texNumR);
}

vec2 packedUVfrom2D(int texelsInLogicalRow, int texNumR,
  int texNumC, int row, int col) {
  int texelIndex = (row / 2) * texelsInLogicalRow + (col / 2);
  int texR = texelIndex / texNumC;
  int texC = texelIndex - texR * texNumC;
  return (vec2(texC, texR) + halfCR) / vec2(texNumC, texNumR);
}

vec2 packedUVfrom3D(int texNumR, int texNumC,
    int texelsInBatch, int texelsInLogicalRow, int b,
    int row, int col) {
  int index = b * texelsInBatch + (row / 2) * texelsInLogicalRow + (col / 2);
  int texR = index / texNumC;
  int texC = index - texR * texNumC;
  return (vec2(texC, texR) + halfCR) / vec2(texNumC, texNumR);
}

  float getChannel(vec4 frag, vec2 innerDims) {
    vec2 modCoord = mod(innerDims, 2.);
    return modCoord.x == 0. ?
      (modCoord.y == 0. ? frag.r : frag.g) :
      (modCoord.y == 0. ? frag.b : frag.a);
  }
  float getChannel(vec4 frag, int dim) {
    float modCoord = mod(float(dim), 2.);
    return modCoord == 0. ? frag.r : frag.g;
  }

    float sampleTexture(sampler2D textureSampler, vec2 uv) {
      return texture(textureSampler, uv).r;
    }

    void setOutput(vec4 val) {
      outputColor = val;
    }

uniform sampler2D matrixA;
uniform int offsetmatrixA;
uniform sampler2D matrixB;
uniform int offsetmatrixB;

    ivec3 getOutputCoords() {
      ivec2 resTexRC = ivec2(resultUV.yx *
                             vec2(679, 679));
      int index = resTexRC.x * 679 + resTexRC.y;

      int b = index / 460800;
      index -= b * 460800;

      int r = 2 * (index / 1);
      int c = imod(index, 1) * 2;

      return ivec3(b, r, c);
    }

    vec4 getMatrixA(int row, int col) {
      vec2 uv = packedUVfrom2D(460800, 5302, 5302, row, col);
      return texture(matrixA, uv);
    }

        vec4 getMatrixA(int b, int row, int col) {
          return getMatrixA(row, col);
        }

    vec4 getMatrixAAtOutCoords() {
      ivec3 coords = getOutputCoords();

      vec4 outputValue = getMatrixA(coords.x, coords.y, coords.z);
      return outputValue;
    }

    vec4 getMatrixB(int row, int col) {
      vec2 uv = packedUVfrom2D(1, 61, 1, row, col);
      return texture(matrixB, uv);
    }

        vec4 getMatrixB(int b, int row, int col) {
          return getMatrixB(row, col);
        }

    vec4 getMatrixBAtOutCoords() {
      ivec3 coords = getOutputCoords();

      vec4 outputValue = getMatrixB(coords.x, coords.y, coords.z);
      return outputValue;
    }

      vec4 activation(vec4 x) {
          return x;
        }
      // Don't use uniform for sharedDimensionPacked for performance.
      const float sharedDimension = 61.0;

      vec4 dot2x2ARowBCol(ivec3 rc) {
        vec4 result = vec4(0);
        for (int i = 0; i < 61; i++) {
          int batchA = rc.x;
          int batchB = rc.x;
          vec4 a = getMatrixA(batchA, i * 2, rc.y);
          vec4 b = getMatrixB(batchB, i * 2, rc.z);

          // These swizzled products need to be separately added.
          // See: https://github.com/tensorflow/tfjs/issues/1735
          result += (a.xxyy * b.xyxy);
          result += (a.zzww * b.zwzw);
        }
        return result;
      }

      void main() {
        ivec3 rc = getOutputCoords();
        vec4 result = dot2x2ARowBCol(rc);

        result = activation(result);

        setOutput(result);
      }

'Chrome':1 WebGL: CONTEXT_LOST_WEBGL: loseContext: context lost

webnn-polyfill.js:7465 Uncaught (in promise) Error: Failed to compile fragment shader.
    at eval (webnn-polyfill.js:7465:819)
    at oe (webnn-polyfill.js:7465:876)
    at eval (webnn-polyfill.js:7723:13564)
    at Dt.getAndSaveBinary (webnn-polyfill.js:7723:16891)
    at Dt.runWebGLProgram (webnn-polyfill.js:7723:13540)
    at Vr (webnn-polyfill.js:8885:3112)
    at Object.kernelFunc (webnn-polyfill.js:9561:1353)
    at i (webnn-polyfill.js:4550:6763)
    at eval (webnn-polyfill.js:4550:7657)
    at x.scopedRun (webnn-polyfill.js:4550:5264)

[BruceDai]

@yyc-git Thanks for using WebNN Polyfill API and reporting this issue. Could you please share us with your test code and reproduce steps, it would be helpful to find the root cause.

[yyc-git]

@BruceDai Hello, this is the code, please Download zip: Github

Then, please in the project dir and run the code:

npm install
npm run webpack:dev-server

Then, you can see error in Chrome

[yyc-git]

if change main.ts code:

    let [width, height] = [1280, 720]

to

    let [width, height] = [256, 256]

and change input.ts code:

import irradiance_img_path from './dataset/color0.png'
import albedo_img_path from './dataset/albedo0.png'
import depth_img_path from './dataset/depth0.png'
import normal_img_path from './dataset/shading_normal0.png'

to

import irradiance_img_path from './dataset/small/color0.png'
import albedo_img_path from './dataset/small/albedo0.png'
import depth_img_path from './dataset/small/depth0.png'
import normal_img_path from './dataset/small/shading_normal0.png'

Then run by the code:

npm run webpack:dev-server

it's OK and no error

[BruceDai]

Thanks @yyc-git . I can reproduce this issue and also get this message from console, I'm investigating it.

'Chrome':1 [.WebGL-0000173401D68000] GL_OUT_OF_MEMORY: Error: 0x00000505, in ....\third_party\angle\src\libANGLE\renderer\d3d\d3d11\ResourceManager11.cpp, allocate:496. Internal D3D11 error: HRESULT: 0x8007000E: Error allocating Texture2D

[BruceDai]

Thanks @qjia7 for pointing the root cause that in WebGL backend it default uses im2col algorithm with flag WEBGL_CONV_IM2COL default being true to speed up convolutions which has memory limitations, so it crashed with large size image. Please change to use experimental conv algorithm by setting flag WEBGL_EXP_CONV being true for this case.

Here's modification, PTAL, thanks.

// in src/wspk.ts
@@ -195,6 +195,7 @@ export let init = async (state, contextOptions) => {
     let tf = context.tf
     //TODO really use webgpu? or just webgl?
     // await tf.setBackend("webgpu")
+    tf.env().set('WEBGL_EXP_CONV', true);
     await tf.setBackend("webgl")
     await tf.ready()

Please see similar issue https://github.com/tensorflow/tfjs/issues/6678#issuecomment-1195764609.

[yyc-git]

Thanks @qjia7 for pointing the root cause that in WebGL backend it default uses im2col algorithm with flag WEBGL_CONV_IM2COL default being true to speed up convolutions which has memory limitations, so it crashed with large size image. Please change to use experimental conv algorithm by setting flag WEBGL_EXP_CONV being true for this case.

Here's modification, PTAL, thanks.

// in src/wspk.ts
@@ -195,6 +195,7 @@ export let init = async (state, contextOptions) => {
     let tf = context.tf
     //TODO really use webgpu? or just webgl?
     // await tf.setBackend("webgpu")
+    tf.env().set('WEBGL_EXP_CONV', true);
     await tf.setBackend("webgl")
     await tf.ready()

Please see similar issue tensorflow/tfjs#6678 (comment).

Thanks very much, it work and not error! Thanks!