React Native App Dark Layer over Camera on some Android devices

[pkyipab]

Describe the current behavior

I'm encountering an issue on Pixel 6, Samsung and Redmi devices where there is a dark layer over the camera when using the above packages in my React Native app. This issue does not occur on Sony Xperia. I would appreciate any help or insights into resolving this problem.

Steps to Reproduce

1. Install the React Native app on a Pixel 6 or Redmi device.
2. Open the camera within the app.
3. Observe the dark layer over the camera feed.

Describe the expected behavior The camera feed should display normally without any dark layers or overlays on both Pixel 6 and Redmi devices. Additionally, key points on the detected human body should be visible.

System information

• OS Platform and Distribution : macOS Sonoma 14.2
• Mobile device : Pixel 6, Redmi, Samsung A13 etc.
• Expo CLI : 0.10.16
• package.json :

  "@mediapipe/pose": "^0.5.1675469404",
  "@tensorflow-models/pose-detection": "^2.1.3",
  "@tensorflow/tfjs": "^4.15.0",
  "@tensorflow/tfjs-react-native": "^1.0.0",
  "expo": "~49.0.15",
  "expo-camera": "~13.4.4",
  "expo-file-system": "~15.4.5",
  "expo-gl": "~13.0.1",
  "expo-status-bar": "~1.6.0",
  "react": "18.2.0",
  "react-native": "0.72.6",
  "react-native-gesture-handler": "~2.12.0",
  "react-native-svg": "13.9.0",
  "expo-splash-screen": "~0.20.5"

Source Code

App.js :

import { StatusBar } from "expo-status-bar";
import { Dimensions, Platform, StyleSheet, Text, View } from "react-native";

import { Camera } from "expo-camera";

import * as tf from "@tensorflow/tfjs";
import * as posedetection from "@tensorflow-models/pose-detection";
import { bundleResourceIO, cameraWithTensors } from "@tensorflow/tfjs-react-native";
import Svg, { Circle } from "react-native-svg";
import { useEffect, useRef, useState } from "react";

const TensorCamera = cameraWithTensors(Camera);

const IS_ANDROID = Platform.OS === "android";
const IS_IOS = Platform.OS === "ios";

// Camera preview size.
//
// From experiments, to render camera feed without distortion, 16:9 ratio
// should be used fo iOS devices and 4:3 ratio should be used for android
// devices.
//
// This might not cover all cases.
const CAM_PREVIEW_WIDTH = Dimensions.get("window").width;
const CAM_PREVIEW_HEIGHT = CAM_PREVIEW_WIDTH / (IS_IOS ? 9 / 16 : 3 / 4);

// The score threshold for pose detection results.
const MIN_KEYPOINT_SCORE = 0.3;

// The size of the resized output from TensorCamera.
//
// For movenet, the size here doesn't matter too much because the model will
// preprocess the input (crop, resize, etc). For best result, use the size that
// doesn't distort the image.
const OUTPUT_TENSOR_WIDTH = 180;
const OUTPUT_TENSOR_HEIGHT = OUTPUT_TENSOR_WIDTH / (IS_IOS ? 9 / 16 : 3 / 4);

// Whether to auto-render TensorCamera preview.
const AUTO_RENDER = false;

// Whether to load model from app bundle (true) or through network (false).
const LOAD_MODEL_FROM_BUNDLE = false;

export default function App() {
  const cameraRef = useRef(null);
  const [tfReady, setTfReady] = useState(false);
  const [model, setModel] = useState();
  const [poses, setPoses] = useState();
  const [fps, setFps] = useState(0);
  const [orientation, setOrientation] = useState();
  const [cameraType, setCameraType] = useState(Camera.Constants.Type.front);
  // Use `useRef` so that changing it won't trigger a re-render.
  //
  // - null: unset (initial value).
  // - 0: animation frame/loop has been canceled.
  // - >0: animation frame has been scheduled.
  const rafId = useRef();

  useEffect(() => {
    async function prepare() {
      rafId.current = null;

      // Camera permission.
      await Camera.requestCameraPermissionsAsync();

      // Wait for tfjs to initialize the backend.
      await tf.ready();

      // Load movenet model.
      // https://github.com/tensorflow/tfjs-models/tree/master/pose-detection
      // const movenetModelConfig = {
      //   modelType: posedetection.movenet.modelType.SINGLEPOSE_LIGHTNING,
      //   enableSmoothing: true,
      // };
      // if (LOAD_MODEL_FROM_BUNDLE) {
      //   const modelJson = require("./offline_model/model.json");
      //   const modelWeights1 = require("./offline_model/group1-shard1of2.bin");
      //   const modelWeights2 = require("./offline_model/group1-shard2of2.bin");
      //   movenetModelConfig.modelUrl = bundleResourceIO(modelJson, [modelWeights1, modelWeights2]);
      // }

      // Load movenet model.
      const movenetModelConfig = {
        modelType: posedetection.movenet.modelType.SINGLEPOSE_THUNDER,
        enableSmoothing: true,
      };

      const model = await posedetection.createDetector(posedetection.SupportedModels.MoveNet, movenetModelConfig);
      setModel(model);

      // Ready!
      setTfReady(true);
    }

    prepare();
  }, []);

  useEffect(() => {
    // Called when the app is unmounted.
    return () => {
      if (rafId.current != null && rafId.current !== 0) {
        cancelAnimationFrame(rafId.current);
        rafId.current = 0;
      }
    };
  }, []);

  const handleCameraStream = async (images, updatePreview, gl) => {
    const loop = async () => {
      // Get the tensor and run pose detection.
      const imageTensor = images.next().value;

      const poses = await model?.estimatePoses(imageTensor, undefined, Date.now());

      setPoses(poses);

      console.log("poses : ", poses);

      tf.dispose([imageTensor]);

      if (rafId.current === 0) {
        return;
      }

      // Render camera preview manually when autorender=false.
      if (!AUTO_RENDER) {
        updatePreview();
        gl.endFrameEXP();
      }

      rafId.current = requestAnimationFrame(loop);
    };

    loop();
  };

  const renderPose = () => {
    if (poses != null && poses.length > 0) {
      const keypoints = poses[0].keypoints
        .filter((k) => (k.score ?? 0) > MIN_KEYPOINT_SCORE)
        .map((k) => {
          // Flip horizontally on android or when using back camera on iOS.
          const flipX = true;
          const x = flipX ? getOutputTensorWidth() - k.x : k.x;
          const y = k.y;
          const cx = (x / getOutputTensorWidth()) * CAM_PREVIEW_WIDTH;
          const cy = (y / getOutputTensorHeight()) * CAM_PREVIEW_HEIGHT;
          return <Circle key={`skeletonkp_${k.name}`} cx={cx} cy={cy} r="4" strokeWidth="2" fill="#00AA00" stroke="white" />;
        });

      return <Svg style={styles.svg}>{keypoints}</Svg>;
    } else {
      return <View></View>;
    }
  };

  const isPortrait = () => {
    return true;
  };

  const getOutputTensorWidth = () => {
    return OUTPUT_TENSOR_WIDTH;
  };

  const getOutputTensorHeight = () => {
    return OUTPUT_TENSOR_HEIGHT;
  };

  if (!tfReady) {
    return (
      <View style={styles.loadingMsg}>
        <Text>Loading...</Text>
      </View>
    );
  } else {
    return (
      // Note that you don't need to specify `cameraTextureWidth` and
      // `cameraTextureHeight` prop in `TensorCamera` below.
      <View style={styles.containerPortrait}>
        <TensorCamera
          ref={cameraRef}
          style={styles.camera}
          autorender={false}
          type={cameraType}
          // tensor related props
          resizeWidth={getOutputTensorWidth()}
          resizeHeight={getOutputTensorHeight()}
          resizeDepth={3}
          rotation={0}
          onReady={handleCameraStream}
        />
        {renderPose()}
      </View>
    );
  }
}

Additional Information Video :

Thanks for your kindness and willingness to help!!

[gaikwadrahul8]

Hi, @pkyipab

Thank you for bringing this issue to our attention and could you please help us with your Github repo link and complete steps to run with model and package.json file to replicate the same behavior from our end also. Thank you for your understanding and patience.

[github-actions[bot]]

This issue has been marked stale because it has no recent activity since 7 days. It will be closed if no further activity occurs. Thank you.

[github-actions[bot]]

This issue was closed due to lack of activity after being marked stale for past 7 days.

[google-ml-butler[bot]]

Are you satisfied with the resolution of your issue? Yes No