alemart / encantar-js

encantAR.js: GPU-accelerated Augmented Reality for the web. Fast, easy to use and device-agnostic WebAR.
https://alemart.github.io/encantar-js/
GNU Lesser General Public License v3.0
56 stars 10 forks source link

live demos not working on Safari or Chrome IOS #4

Closed gavan1 closed 4 months ago

gavan1 commented 1 year ago

Video stream freezes and motion from device motion not being pickup up.

Feature tracking is working and 3d objects are appearing.#3

skeegan commented 10 months ago

I'm seeing the same result. This is the error-

martins.min.js:20 [martins-js] Tracking error: o: WebGL error.

---------- ERROR ----------

0:73: S0032: no default precision defined for variable 'vec4[9]' Program has not been successfully linked.

---------- SOURCE CODE ----------

  1. version 300 es

  2. if 0 == 0

  3. precision mediump float;
  4. precision mediump sampler2D;
  5. precision highp int;
  6. endif

  7. if 0 == 0

    1. define OUT_TYPE mediump vec4

    2. elif 0 == 1

    3. define OUT_TYPE mediump ivec4

    4. elif 0 == 2

    5. define OUT_TYPE mediump uvec4

    6. else

    7. error Unknown FS_OUTPUT_TYPE

    8. endif

    9. out OUT_TYPE color;
    10. in highp vec2 texCoord;
    11. uniform highp vec2 texSize;
    12. ifndef _GLOBAL_GLSL

    13. define _GLOBAL_GLSL

    14. define threadLocation() ivec2(texCoord * texSize)

    15. define outputSize() ivec2(texSize)

    16. define threadPixel(img) textureLod((img), texCoord, 0.0f)

    17. define pixelAt(img, pos) texelFetch((img), (pos), 0)

    18. define pixelAtShortOffset(img, offset) textureLodOffset((img), texCoord, 0.0f, (offset))

    19. define pixelAtLongOffset(img, offset) textureLod((img), texCoord + vec2(offset) / texSize, 0.0f)

    20. endif

    21. ifndef _PYRAMIDS_GLSL

    22. define _PYRAMIDS_GLSL

    23. define pyrPixel(pyr, lod) textureLod((pyr), texCoord, (lod))

    24. define pyrPixelAtOffset(pyr, lod, pot, offset) textureLod((pyr), texCoord + ((pot) * vec2(offset)) / texSize, (lod))

    25. define pyrPixelAt(pyr, pos, lod) textureLod((pyr), (vec2(pos) + vec2(0.5f)) / texSize, (lod))

    26. define pyrPixelAtEx(pyr, pos, lod, pyrBaseSize) textureLod((pyr), (vec2(pos) + vec2(0.5f)) / vec2(pyrBaseSize), (lod))

    27. define pyrSubpixelAtEx(pyr, pos, lod, pyrBaseSize) textureLod((pyr), ((pos) + vec2(0.5f)) / vec2(pyrBaseSize), (lod))

    28. define pyrSubpixelAtExOffset(pyr, pos, lod, pot, offset, pyrBaseSize) textureLod((pyr), (((pos) + vec2(0.5f)) + ((pot) * vec2(offset))) / vec2(pyrBaseSize), (lod))

    29. const int PYRAMID_MAX_LEVELS = int(8);
    30. const float F_PYRAMID_MAX_LEVELS = float(8);
    31. const float LOG2_PYRAMID_MAX_SCALE = float(0);
    32. define encodeLod(lod) ((LOG2_PYRAMID_MAX_SCALE + (lod)) / (LOG2_PYRAMID_MAX_SCALE + F_PYRAMID_MAX_LEVELS))

    33. float decodeLod(float encodedLod)
    34. {
    35. float lod = encodedLod * (LOG2_PYRAMID_MAX_SCALE + F_PYRAMID_MAX_LEVELS) - LOG2_PYRAMID_MAX_SCALE;
    36. return lod - lod * step(1.0f, encodedLod);
    37. }
    38. define LOD_EPS 0.0625f

    39. const float ENCODED_LOD_EPS = (LOD_EPS / (LOG2_PYRAMID_MAX_SCALE + F_PYRAMID_MAX_LEVELS));
    40. define isSameLod(lod1, lod2) (abs((lod1) - (lod2)) < LOD_EPS)

    41. define isSameEncodedLod(alpha1, alpha2) (abs((alpha1) - (alpha2)) < ENCODED_LOD_EPS)

    42. endif

    43. ifndef _FLOAT16_GLSL

    44. define _FLOAT16_GLSL

    45. define encodeFloat16(f) (vec2(packf16(f)) / 255.0f)

    46. define decodeFloat16(v) unpackf16(uvec2((v) * 255.0f))

    47. define encodePairOfFloat16(f) vec4(encodeFloat16((f).x), encodeFloat16((f).y))

    48. define decodePairOfFloat16(v) vec2(decodeFloat16((v).rg), decodeFloat16((v).ba))

    49. define encodeNullPairOfFloat16() vec4(1.0f)

    50. define isNullPairOfFloat16(v) all(equal((v), encodeNullPairOfFloat16()))

    51. define encodeDiscardedPairOfFloat16() vec4(0.0f, 1.0f, 0.0f, 1.0f)

    52. define isDiscardedPairOfFloat16(v) all(equal((v), encodeDiscardedPairOfFloat16()))

    53. define encodeFloat16NaN() vec2(0.5f, 1.0f)

    54. define isEncodedFloat16NaN(v) all(equal((v), encodeFloat16NaN()))

    55. uvec2 packf16( float f)
    56. {
    57. uint y = packHalf2x16(vec2(f, 0.0f));
    58. return uvec2(y, y >> 8u) & 0xFFu;
    59. }
    60. float unpackf16(uvec2 v)
    61. {
    62. v &= 0xFFu;
    63. return unpackHalf2x16(v.x | (v.y << 8u)).x;
    64. }
    65. bool isEncodedFloat16Zero(vec2 v)
    66. {
    67. uvec2 w = uvec2(v * 255.0f);
    68. return 0u == w.x + w.y * (0x80u - w.y);
    69. }
    70. endif

    71. uniform sampler2D corners;
    72. void main()
    73. {
    74. ivec2 thread = threadLocation();
    75. vec4 pixel = threadPixel(corners);
    76. float encodedLod = pixel.a;
    77. float score = decodeFloat16(pixel.rb);
    78. float lod = decodeLod(encodedLod);
    79. float pot = exp2(lod);
    80. color = pixel;
    81. if(score == 0.0f)
    82. return;
    83. if 1

    84. vec2 gridSize = vec2(pot);
    85. vec2 gridLocation = floor(mod(texCoord * texSize, gridSize));
    86. vec2 gridDelta = gridLocation / gridSize - vec2(0.5f);
    87. float gridStep = 1.0f / pot;
    88. const float adjustment = 1.25f;
    89. color.rb = encodeFloat16(0.0f);
    90. if(max(abs(gridDelta.x), abs(gridDelta.y)) > adjustment * gridStep)
    91. return;
    92. endif

    93. define P(x,y) textureLod(corners, texCoord + pot * vec2((x), (y)) / texSize, 0.0f)

    94. vec4 pix[9] = vec4[9](
    95. P(-1,-1), P(0,-1), P(1,-1),
    96. P(-1,0), pixel, P(1,0),
    97. P(-1,1), P(0,1), P(1,1)
    98. );
    99. define S(j) decodeFloat16(pix[j].rb)

    100. mat3 scores = mat3(
    101. S(0), S(1), S(2),
    102. S(3), S(4), S(5),
    103. S(6), S(7), S(8)
    104. );
    105. define B(j) float(isSameLod(decodeLod(pix[j].a), lod))

    106. mat3 sameLod = mat3(
    107. B(0), B(1), B(2),
    108. B(3), B(4), B(5),
    109. B(6), B(7), B(8)
    110. );
    111. mat3 sameLodScores = matrixCompMult(scores, sameLod);
    112. vec3 maxScore3 = max(sameLodScores[0], max(sameLodScores[1], sameLodScores[2]));
    113. float maxScore = max(maxScore3.x, max(maxScore3.y, maxScore3.z));
    114. color.rb = encodeFloat16(score * step(maxScore, score));
    115. }

-> [speedy-vision.js] warning @ martins.min.js:20

alemart commented 10 months ago

@skeegan Thanks for the feedback. I intend to get it working on Safari and iOS.

The error you are seeing has to do with a shader not being compiled due to a driver bug. I fix these kinds of errors in my library Speedy Vision. Once they are fixed, the AR demos here will work.

May I ask you to test the following Speedy Vision demos?

  1. https://alemart.github.io/speedy-vision/demos/best-features.html
  2. https://alemart.github.io/speedy-vision/demos/feature-matching.html
  3. https://alemart.github.io/speedy-vision/demos/feature-matching-lsh.html

You are expected to see points and lines. If you see errors in the console, please let me know.

skeegan commented 10 months ago

Hi @alemart , Thanks for your response. Each of those three works for me in Chrome on a Google Pixel 7A (no errors in console).

Any idea on when the other AR demo will be working?

alemart commented 10 months ago

@skeegan Thanks for testing. I have updated the Speedy Vision version used in the demos.

May you test the AR demos again? Try the basic webcam and the WebGL demos. You are expected to see an animation in augmented reality. If you see any errors in the console, let me know.

You should see MARTINS.js version 0.1.2-wip and Speedy Vision version 0.9.1-wip in the console. Please clear the browser cache if you don't.

skeegan commented 10 months ago

Hi @alemart,

I'm seeing the correct versions for MARTINS.js and for Speedy Vision. However, I'm now getting a different error:

martins.min.js:20 [martins-js] Tracking error: r: WebGL error.

_---------- ERROR ----------

0:207: S0032: no default precision defined for variable 'mat3[2]' Program has not been successfully linked._

[alemart.github.io-1704882393420.log](https://github.com/alemart/martins-js/files/13886362/alemart.github.io-1704882393420.log) (full log attached)

alemart commented 10 months ago

@skeegan This is a similar driver bug that has to do with an initializer. Would you try again? If you see the same error, please clear the browser cache.

skeegan commented 10 months ago

it works now- thanks.

Quick question: You say that Safari support is coming soon. Any idea when that might be ready?

alemart commented 10 months ago

it works now- thanks.

Awesome!

Quick question: You say that Safari support is coming soon. Any idea when that might be ready?

Yes: when a volunteer helps me test it on Safari. Support for Safari is probably just a tiny change ahead.

Since I started this project back in 2022, the Epiphany browser (WebKit-based) received WebGL2 support. I was able to run the demos on a version of Epiphany.

I don't own a Mac nor an iOS device at the present time, so I'm using Epiphany as a suitable alternative to Safari. I would like to have this project tested on actual Apple hardware as well.

skeegan commented 10 months ago

I don't have an iOS device either but I'll check it out when I can. Could your code be adapted to enable floor detection (for tap-to-place)?

On Thu, Jan 11, 2024 at 5:09 PM Alexandre Martins @.***> wrote:

it works now- thanks.

Awesome!

Quick question: You say that Safari support is coming soon. Any idea when that might be ready?

Yes: when a volunteer helps me test it on Safari. Support for Safari is probably just a tiny change ahead.

Since I started this project back in 2022, the Epiphany browser (WebKit-based) received WebGL2 support. I was able to run the demos on a version of Epiphany.

I don't own a Mac nor an iOS device at the present time, so I'm using Epiphany as a suitable alternative to Safari. I would like to have this project tested on actual Apple hardware as well.

— Reply to this email directly, view it on GitHub https://github.com/alemart/martins-js/issues/4#issuecomment-1887598405, or unsubscribe https://github.com/notifications/unsubscribe-auth/AAIYM35FJV5DHE5J4BU5VELYOAMEZAVCNFSM6AAAAAAXMZZBX6VHI2DSMVQWIX3LMV43OSLTON2WKQ3PNVWWK3TUHMYTQOBXGU4TQNBQGU . You are receiving this because you were mentioned.Message ID: @.***>

alemart commented 9 months ago

Could your code be adapted to enable floor detection (for tap-to-place)?

That would require implementing a new tracker. I don't have this feature in mind at this moment. I thank you for your interest in my work.

alemart commented 9 months ago

To whom it might interest: Safari 15 is the minimum required version.

If you're a Safari user, your feedback is appreciated.

alemart commented 9 months ago

I have good news: now we have Safari support.

I managed to test the WebAR demos on several iOS devices. After a small adjustment, I could get them to work on versions 15.2 or newer of the Safari browser (though not on Safari 15.0-15.1).

I have tested the demos on the following devices: iPhone 11 Pro (iOS 15.5), iPhone 12 (iOS 15.6.1), iPhone 12 Pro (iOS 15.0.2), iPhone 13 (iOS 15.0.2), iPhone 13 Pro (iOS 15.2), iPhone 13 Pro Max (iOS 15.1), iPhone 14 (iOS 16.1 and iOS 16.6.1), iPhone XR (iOS 16.4), iPad mini (iPadOS 15.1) and iPad Air (iPadOS 15.4.1). The demos work on every device with Safari 15.2 or newer. Tested on the AWS Device Farm.

If you're a iOS or a macOS user, your feedback is much appreciated. I was not able to test the camera feed on the cloud, and yet the engine seemed to work as expected.

Here is a video of a demo working on iOS (the FPS/GPU rates indicate good performance):

https://github.com/alemart/martins-js/assets/221477/f227bb42-cfdd-4ce4-b013-b6f3cba6ff8e

alemart commented 4 months ago

MARTINS.js version 0.2.0 works on iOS. I have tested it on more devices. I'm closing this issue now.