Closed babvijayb closed 1 month ago
Hi, @babvijayb
I apologize for the delay in my response, thank you for bringing this issue to our attention, if possible could you please help us with your Github repo and complete steps to replicate the same behavior from our end to investigate this issue further ?
Thank you for your cooperation and patience.
This issue has been marked stale because it has no recent activity since 7 days. It will be closed if no further activity occurs. Thank you.
This issue was closed due to lack of activity after being marked stale for past 7 days.
The code below, which I provided, is used to measure the width of the shoulder and the height of the user of this application, built on React Native (0.72.10) and Expo (49.0.8). While the code works correctly on the Android application, it does not produce the expected output on iOS. The measurement of shoulder width and height is not accurate, and the captured image's detected human keypoints, indicated by red lines, are also incorrect.
It would be very helpful to know what considerations I should take into account when working with iOS platform-specific code to make things work correctly.
The Below Code Is Correctly working in Android Application.
import React, { useEffect, useState, useRef } from 'react'; import { StyleSheet, Text, View, Dimensions, Platform, TouchableOpacity, Image, ActivityIndicator, Button, PixelRatio } from 'react-native'; import { Camera } from 'expo-camera'; import as posenet from '@tensorflow-models/posenet'; import as tf from '@tensorflow/tfjs'; import { decodeJpeg } from '@tensorflow/tfjs-react-native'; import Svg, { Circle, Ellipse, Line } from 'react-native-svg'; import { CameraType } from 'expo-camera/build/Camera.types'; import { Icon } from '@react-native-material/core';
const IS_ANDROID = Platform.OS === 'android'; const IS_IOS = Platform.OS === 'ios'; const pixelDensity = PixelRatio.get() * 160; const CAM_PREVIEW_WIDTH = Dimensions.get('window').width; //const CAM_PREVIEW_HEIGHT = 600; const CAM_PREVIEW_HEIGHT = CAM_PREVIEW_WIDTH / (IS_IOS ? 9 / 16 : 3 / 4); const MIN_KEYPOINT_SCORE = 0.3; const MIN_KEYPOINT_SCORE_IOS = 0.1; const AUTO_RENDER = false; const KNOWN_DETAILS = { DEVICE_HEIGHT: 1600, DEVICE_WIDTH: 1200, FOCAL_LENGTH_MM: 2.216, DISTANCE_CM: 122,
HEIGHT_CM: 172, HEIGHT_PIXELS: 700, SHOULDER_DISTANCE_CM: 42, SHOULDER_DISTANCE_PIXELS: 150, } const KNOWN_DETAILS_IOS = { DEVICE_HEIGHT: 1300, DEVICE_WIDTH: 900, FOCAL_LENGTH_MM: 2.216, DISTANCE_CM: 122,
HEIGHT_CM: 172, HEIGHT_PIXELS: 700, SHOULDER_DISTANCE_CM: 42, SHOULDER_DISTANCE_PIXELS: 150, }
export default function CameraMatSelectionAfterTaken({ afterMeasured }) { const cameraRef = useRef(null); const [tfReady, setTfReady] = useState(false); const [model, setModel] = useState(null); const [poses, setPoses] = useState(null); const [fps, setFps] = useState(0); const [orientation, setOrientation] = useState(); const [cameraType, setCameraType] = useState(CameraType.front); const [timeLeft, setTimeLeft] = useState(null); const [capturedImage, setCapturedImage] = useState(null); const [processing, setProcessing] = useState(true); const [keyPoints, setKeyPoints] = useState(null); const [height, setHeight] = useState(null); const [shoulderWidth, setShoulderWidth] = useState(null); const [imageNotProper, setImageNotProper] = useState(false); const [outputTensorHeight, setOutputTensorHeight] = useState(0); const [outputTensorWidth, setOutputTensorWidth] = useState(0); console.log(CAM_PREVIEW_WIDTH)
useEffect(() => { async function prepare() { try { await Camera.requestCameraPermissionsAsync(); await tf.ready(); const model = await posenet.load(); setModel(model); setTfReady(true); console.log('Model loaded successfully'); } catch (error) { console.error('Error loading model:', error); } } prepare(); }, []);
useEffect(() => { if (timeLeft === 0) { takePhotoAsync(); setTimeLeft(null) } if (!timeLeft) return; const intervalId = setInterval(() => { setTimeLeft(timeLeft - 1); }, 1000); return () => clearInterval(intervalId); }, [timeLeft]); useEffect(() => { if (capturedImage && tfReady) { processImage(); } }, [capturedImage]); async function takePhotoAsync() { if (cameraRef.current) { let takenPhoto = await cameraRef.current.takePictureAsync({ base64: true, exif: true, }); setCapturedImage(takenPhoto); } } const processImage = async () => { console.log('Processing Image') //console.log(capturedImage.exif);
}; function calculateDistance(point1, point2) { const dx = point2.x - point1.x; const dy = point2.y - point1.y; return Math.sqrt(dx dx + dy dy); } const getFeetPosition = (leftFeet, rightFeet) => { const feet = { position: {} } feet.position.x = (leftFeet.position.x + rightFeet.position.x) / 2; feet.position.y = (leftFeet.position.y + rightFeet.position.y) / 2; return feet; } const renderPose = () => { if (keyPoints != null && keyPoints.length > 0) {
}; const onTakePhoto = () => { setTimeLeft(10); }
const handleSwitchCameraType = () => { if (cameraType === CameraType.front) { setCameraType(CameraType.back); } else { setCameraType(CameraType.front); } }; const reCapture = () => { setCapturedImage(null); setHeight(null); setShoulderWidth(null); setKeyPoints(null); } const onContinue = () => { if (height && shoulderWidth) { afterMeasured(Math.round(height), Math.round(shoulderWidth)); } }
if (!tfReady) { return (
} else { return ( <View style={styles.containerPortrait}
} }
const styles = StyleSheet.create({ containerPortrait: { position: 'relative', width: CAM_PREVIEW_WIDTH, height: CAM_PREVIEW_HEIGHT, marginTop: Dimensions.get('window').height / 2 - CAM_PREVIEW_HEIGHT / 2, }, containerLandscape: { position: 'relative', width: CAM_PREVIEW_HEIGHT, height: CAM_PREVIEW_WIDTH, marginLeft: Dimensions.get('window').height / 2 - CAM_PREVIEW_HEIGHT / 2, }, loadingMsg: { position: 'absolute', width: '100%', height: '100%', alignItems: 'center', justifyContent: 'center', }, camera: { width: '100%', height: '100%', zIndex: 1, }, imageMirror: { transform: [ { scaleX: -1 } ] }, svgMirror: { transform: [ { scaleX: -1 } ] }, svg: { width: '100%', height: '100%', position: 'absolute', zIndex: 2, }, timer: { position: 'absolute', bottom: CAM_PREVIEW_HEIGHT / 2, left: 0, justifyContent: 'center', alignItems: 'center', width: "100%", height: 100, zIndex: 3, }, fpsContainer: { backgroundColor: 'black', position: 'absolute', top: 10, left: 10, width: 80, alignItems: 'center', borderRadius: 2, padding: 8, zIndex: 3, }, cameraTypeSwitcher: { backgroundColor: 'black', position: 'absolute', top: 10, right: 10, width: 70, alignItems: 'center', borderRadius: 2, padding: 8, zIndex: 3, }, camera_icons: { color: 'white', }, camera_take_photo_icon: { fontSize: 56, }, camera_flip_camera_container: { backgroundColor: 'transparent', padding: 10, borderRadius: 50, marginBottom: 26, justifyContent: "center" }, camera_take_photo_container: { marginBottom: 28, }, camera_flip_camera_icon: { fontSize: 36, }, alignCenter: { position: 'absolute', bottom: CAM_PREVIEW_HEIGHT / 2, left: 0, justifyContent: 'center', alignItems: 'center', width: '100%', height: 100, zIndex: 3, backgroundColor: 'rgba(255, 255, 255, .7)', } });
And My Package.json file for reference:
{ "name": "camdetect.mobileapp", "version": "1.0.0", "main": "index.js", "scripts": { "start": "expo start --dev-client", "android": "expo run:android", "ios": "expo run:ios", "web": "expo start --web", "postinstall": "npx patch-package" }, "dependencies": { "@expo/vector-icons": "^13.0.0", "@mediapipe/pose": "^0.5.1675469404", "@react-native-async-storage/async-storage": "1.18.2", "@react-native-material/core": "^1.3.7", "@react-navigation/bottom-tabs": "^6.5.20", "@react-navigation/native": "^6.1.7", "@react-navigation/native-stack": "^6.9.13", "@reduxjs/toolkit": "^2.2.4", "@rneui/base": "^4.0.0-rc.8", "@rneui/themed": "^4.0.0-rc.8", "@tensorflow-models/pose-detection": "^2.1.3", "@tensorflow-models/posenet": "^2.2.2", "@tensorflow/tfjs": "4.10.0", "@tensorflow/tfjs-react-native": "0.8.0", "axios": "^1.6.8", "depcheck": "^1.4.7", "expo": "~49.0.8", "expo-camera": "~13.4.4", "expo-dev-client": "~2.4.13", "expo-file-system": "~15.4.5", "expo-font": "~11.4.0", "expo-gl": "~13.0.1", "expo-image-manipulator": "~11.3.0", "expo-permissions": "~14.2.1", "expo-screen-orientation": "~6.0.6", "expo-splash-screen": "~0.20.5", "expo-status-bar": "~1.6.0", "react": "18.2.0", "react-native": "0.72.10", "react-native-fs": "^2.20.0", "react-native-gesture-handler": "~2.12.1", "react-native-image-slider-box": "^2.0.7", "react-native-safe-area-context": "4.6.3", "react-native-screens": "~3.22.0", "react-native-svg": "13.9.0", "react-native-toast-message": "^2.2.0", "react-redux": "^9.1.2", "redux-persist": "^6.0.0" }, "devDependencies": { "@babel/core": "^7.20.0" }, "private": true }
Android Console output: Model loaded successfully LOG Processing Image LOG {"image_hight": 1440, "image_width": 1080} LOG imageTensor: {"dataId": {"id": 62}, "dtype": "int32", "id": 62, "isDisposedInternal": false, "kept": false, "rankType": "3", "shape": [1440, 1080, 3], "size": 4665600, "strides": [3240, 3]} LOG dtposes [{"part": "nose", "position": {"x": 545.1062059216925, "y": 191.49965041342412}, "score": 0.9810086488723755}, {"part": "leftEye", "position": {"x": 538.8779410870623, "y": 172.17822598112232}, "score": 0.9725439548492432}, {"part": "rightEye", "position": {"x": 532.0788146735165, "y": 160.94038788910504}, "score": 0.984584629535675}, {"part": "leftEar", "position": {"x": 584.5300245470573, "y": 179.49073880562986}, "score": 0.7300240397453308}, {"part": "rightEar", "position": {"x": 484.447311223249, "y": 178.01409294633086}, "score": 0.9097193479537964}, {"part": "leftShoulder", "position": {"x": 620.3027913728113, "y": 297.7782039419686}, "score": 0.997944176197052}, {"part": "rightShoulder", "position": {"x": 439.7834415658439, "y": 306.0812389804231}, "score": 0.9973758459091187}, {"part": "leftElbow", "position": {"x": 670.8984659989056, "y": 441.28194252340705}, "score": 0.9933682084083557}, {"part": "rightElbow", "position": {"x": 413.39004932210594, "y": 451.5963092017266}, "score": 0.992526650428772}, {"part": "leftWrist", "position": {"x": 675.0317192819795, "y": 540.9335386521158}, "score": 0.9703831076622009}, {"part": "rightWrist", "position": {"x": 404.9840976106517, "y": 531.7778234815783}, "score": 0.8649968504905701}, {"part": "leftHip", "position": {"x": 591.9822603812013, "y": 552.5793884514834}, "score": 0.9943028092384338}, {"part": "rightHip", "position": {"x": 483.0766706058487, "y": 554.6393463491002}, "score": 0.9938344359397888}, {"part": "leftKnee", "position": {"x": 583.9515537603355, "y": 790.9039662876945}, "score": 0.9841167330741882}, {"part": "rightKnee", "position": {"x": 506.98266411569796, "y": 807.5756550948443}, "score": 0.983876645565033}, {"part": "leftAnkle", "position": {"x": 581.4712085055934, "y": 1031.1535882022129}, "score": 0.5877557992935181}, {"part": "rightAnkle", "position": {"x": 546.8396518573686, "y": 1038.7305637463521}, "score": 0.7883061170578003}] LOG {"Pixel": 843.6575123715096, "conversionFactor": 0.2123456790123457, "currentFloor": 596.3424876284904, "known_pixels": 630, "knwonFloor": 810, "pixelDensity": 320, "pixelDensityCM": 125.98425196850394, "realHeight": "126.63"} LOG {"know_sholder_pixels": 135, "pixelDensity": 320, "pixelDensityCM": 125.98425196850394, "pixelShoulder": 180.7101990635284, "realShoulderWidth": "56.22", "shoulder_conversionFactor": 0.3111111111111111}
iOS Output: Model loaded successfully LOG Processing Image LOG {"image_hight": 4032, "image_width": 2268} LOG imageTensor: {"dataId": {"id": 62}, "dtype": "int32", "id": 62, "isDisposedInternal": false, "kept": false, "rankType": "3", "shape": [2268, 4032, 3], "size": 27433728, "strides": [12096, 3]} LOG dtposes [{"part": "nose", "position": {"x": 1597.000972762646, "y": 783.8229571984434}, "score": 0.01142120361328125}, {"part": "leftEye", "position": {"x": 1803.6201361867704, "y": 593.8270914396887}, "score": 0.023651123046875}, {"part": "rightEye", "position": {"x": 1933.0826848249026, "y": 452.52140077821014}, "score": 0.0143890380859375}, {"part": "leftEar", "position": {"x": 1825.4066147859921, "y": 604.2453793774318}, "score": 0.0218658447265625}, {"part": "rightEar", "position": {"x": 1742.0679717898831, "y": 568.9610894941634}, "score": 0.016021728515625}, {"part": "leftShoulder", "position": {"x": 1845.3239299610893, "y": 609.7992461089493}, "score": 0.041778564453125}, {"part": "rightShoulder", "position": {"x": 1801.1381322957197, "y": 538.135214007782}, "score": 0.0360107421875}, {"part": "leftElbow", "position": {"x": 1734.231274319066, "y": 769.727626459144}, "score": 0.021453857421875}, {"part": "rightElbow", "position": {"x": 1710.6215953307392, "y": 742.3336575875485}, "score": 0.0287628173828125}, {"part": "leftWrist", "position": {"x": 1738.6284046692606, "y": 1068.2422178988327}, "score": 0.019683837890625}, {"part": "rightWrist", "position": {"x": 1388.2062256809338, "y": 800.4922178988327}, "score": 0.01934814453125}, {"part": "leftHip", "position": {"x": 1799.7285992217899, "y": 1123.7464129377431}, "score": 0.03594970703125}, {"part": "rightHip", "position": {"x": 1812.567607003891, "y": 1129.0072653210116}, "score": 0.0274810791015625}, {"part": "leftKnee", "position": {"x": 1783.6875, "y": 1560.046692607004}, "score": 0.02386474609375}, {"part": "rightKnee", "position": {"x": 1799.7898832684825, "y": 1554.1634241245133}, "score": 0.0305023193359375}, {"part": "leftAnkle", "position": {"x": 1820.8715953307392, "y": 2132.062408803502}, "score": 0.03436279296875}, {"part": "rightAnkle", "position": {"x": 1005.2048729328793, "y": 2338.783073929961}, "score": 0.0352783203125}] LOG {"Pixel": 1463.2102455224874, "conversionFactor": 0.13482265334117186, "currentFloor": 804.7897544775126, "known_pixels": 992.25, "knwonFloor": 1275.75, "pixelDensity": 480, "pixelDensityCM": 188.9763779527559, "realHeight": "108.50"} LOG {"know_sholder_pixels": 504, "pixelDensity": 480, "pixelDensityCM": 188.9763779527559, "pixelShoulder": 84.19096277108441, "realShoulderWidth": "7.02", "shoulder_conversionFactor": 0.08333333333333333}