OpenCV version (if running on desktop): default, opencv4 and android opencv3
Describe the problem:
Fellowing Steps to build a MediaPipe AAR, I built face dection AAR and installed successfully. When I tryed to install hand tracking by aar, I encountered problem, the app crashed.
Specifically:
Create an empty android project through android studio, cropy mediapipe\examples\android\src\java\com\google\mediapipe\apps\basic\MainActivity.java, D:\project\mediapipe-master\mediapipe\examples\android\src\java\com\google\mediapipe\apps\basic\res to my project.
6. Copy the binary_graph to my assets
7. Copy the `hand_landmark_lite.tflite`, `hand_landmark_full.tflite`, `palm_detection_full.tflite` and `palm_detection_lite.tflite` to my assets.
8. Modify AndroidManifest.xml
10. Add some code to my MainActivity.java from `D:\project\mediapipe-master\mediapipe\examples\android\src\java\com\google\mediapipe\apps\handtrackinggpu\MainActivity.java `
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/* Main activity of MediaPipe basic app. /
public class MainActivity extends AppCompatActivity {
private static final String TAG = "MainActivity";
// Flips the camera-preview frames vertically by default, before sending them into FrameProcessor
// to be processed in a MediaPipe graph, and flips the processed frames back when they are
// displayed. This maybe needed because OpenGL represents images assuming the image origin is at
// the bottom-left corner, whereas MediaPipe in general assumes the image origin is at the
// top-left corner.
// NOTE: use "flipFramesVertically" in manifest metadata to override this behavior.
private static final boolean FLIP_FRAMES_VERTICALLY = true;
// Number of output frames allocated in ExternalTextureConverter.
// NOTE: use "converterNumBuffers" in manifest metadata to override number of buffers. For
// example, when there is a FlowLimiterCalculator in the graph, number of buffers should be at
// least `max_in_flight + max_in_queue + 1` (where max_in_flight and max_in_queue are used in
// FlowLimiterCalculator options). That's because we need buffers for all the frames that are in
// flight/queue plus one for the next frame from the camera.
private static final int NUM_BUFFERS = 2;
private static final String INPUT_NUM_HANDS_SIDE_PACKET_NAME = "num_hands";
private static final String INPUT_MODEL_COMPLEXITY = "model_complexity";
private static final String OUTPUT_LANDMARKS_STREAM_NAME = "hand_landmarks";
// Max number of hands to detect/process.
private static final int NUM_HANDS = 2;
static {
// Load all native libraries needed by the app.
System.loadLibrary("mediapipe_jni");
try {
System.loadLibrary("opencv_java3");
} catch (java.lang.UnsatisfiedLinkError e) {
// Some example apps (e.g. template matching) require OpenCV 4.
System.loadLibrary("opencv_java4");
}
}
// Sends camera-preview frames into a MediaPipe graph for processing, and displays the processed
// frames onto a {@link Surface}.
protected FrameProcessor processor;
// Handles camera access via the {@link CameraX} Jetpack support library.
protected CameraXPreviewHelper cameraHelper;
// {@link SurfaceTexture} where the camera-preview frames can be accessed.
private SurfaceTexture previewFrameTexture;
// {@link SurfaceView} that displays the camera-preview frames processed by a MediaPipe graph.
private SurfaceView previewDisplayView;
// Creates and manages an {@link EGLContext}.
private EglManager eglManager;
// Converts the GL_TEXTURE_EXTERNAL_OES texture from Android camera into a regular texture to be
// consumed by {@link FrameProcessor} and the underlying MediaPipe graph.
private ExternalTextureConverter converter;
// ApplicationInfo for retrieving metadata defined in the manifest.
private ApplicationInfo applicationInfo;
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(getContentViewLayoutResId());
try {
applicationInfo =
getPackageManager().getApplicationInfo(getPackageName(), PackageManager.GET_META_DATA);
} catch (NameNotFoundException e) {
Log.e(TAG, "Cannot find application info: " + e);
}
previewDisplayView = new SurfaceView(this);
setupPreviewDisplayView();
// Initialize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
// binary graphs.
AndroidAssetUtil.initializeNativeAssetManager(this);
eglManager = new EglManager(null);
processor =
new FrameProcessor(
this,
eglManager.getNativeContext(),
applicationInfo.metaData.getString("binaryGraphName"),
applicationInfo.metaData.getString("inputVideoStreamName"),
applicationInfo.metaData.getString("outputVideoStreamName"));
processor
.getVideoSurfaceOutput()
.setFlipY(
applicationInfo.metaData.getBoolean("flipFramesVertically", FLIP_FRAMES_VERTICALLY));
PermissionHelper.checkAndRequestCameraPermissions(this);
AndroidPacketCreator packetCreator = processor.getPacketCreator();
Map<String, Packet> inputSidePackets = new HashMap<>();
inputSidePackets.put(INPUT_NUM_HANDS_SIDE_PACKET_NAME, packetCreator.createInt32(NUM_HANDS));
if (applicationInfo.metaData.containsKey("modelComplexity")) {
inputSidePackets.put(
INPUT_MODEL_COMPLEXITY,
packetCreator.createInt32(applicationInfo.metaData.getInt("modelComplexity")));
}
processor.setInputSidePackets(inputSidePackets);
// To show verbose logging, run:
// adb shell setprop log.tag.MainActivity VERBOSE
if (Log.isLoggable(TAG, Log.VERBOSE)) {
processor.addPacketCallback(
OUTPUT_LANDMARKS_STREAM_NAME,
(packet) -> {
Log.v(TAG, "Received multi-hand landmarks packet.");
List<LandmarkProto.NormalizedLandmarkList> multiHandLandmarks =
PacketGetter.getProtoVector(packet, LandmarkProto.NormalizedLandmarkList.parser());
Log.v(
TAG,
"[TS:"
+ packet.getTimestamp()
+ "] "
+ getMultiHandLandmarksDebugString(multiHandLandmarks));
});
}
}
// Used to obtain the content view for this application. If you are extending this class, and
// have a custom layout, override this method and return the custom layout.
protected int getContentViewLayoutResId() {
return R.layout.activity_main;
}
@Override
protected void onResume() {
super.onResume();
converter =
new ExternalTextureConverter(
eglManager.getContext(),
applicationInfo.metaData.getInt("converterNumBuffers", NUM_BUFFERS));
converter.setFlipY(
applicationInfo.metaData.getBoolean("flipFramesVertically", FLIP_FRAMES_VERTICALLY));
converter.setConsumer(processor);
if (PermissionHelper.cameraPermissionsGranted(this)) {
startCamera();
}
}
@Override
protected void onPause() {
super.onPause();
converter.close();
// Hide preview display until we re-open the camera again.
previewDisplayView.setVisibility(View.GONE);
}
@Override
public void onRequestPermissionsResult(
int requestCode, String[] permissions, int[] grantResults) {
super.onRequestPermissionsResult(requestCode, permissions, grantResults);
PermissionHelper.onRequestPermissionsResult(requestCode, permissions, grantResults);
}
protected void onCameraStarted(SurfaceTexture surfaceTexture) {
previewFrameTexture = surfaceTexture;
// Make the display view visible to start showing the preview. This triggers the
// SurfaceHolder.Callback added to (the holder of) previewDisplayView.
previewDisplayView.setVisibility(View.VISIBLE);
}
protected Size cameraTargetResolution() {
return null; // No preference and let the camera (helper) decide.
}
public void startCamera() {
cameraHelper = new CameraXPreviewHelper();
previewFrameTexture = converter.getSurfaceTexture();
cameraHelper.setOnCameraStartedListener(
surfaceTexture -> {
onCameraStarted(surfaceTexture);
});
CameraHelper.CameraFacing cameraFacing =
applicationInfo.metaData.getBoolean("cameraFacingFront", false)
? CameraHelper.CameraFacing.FRONT
: CameraHelper.CameraFacing.BACK;
cameraHelper.startCamera(
this, cameraFacing, previewFrameTexture, cameraTargetResolution());
}
protected Size computeViewSize(int width, int height) {
return new Size(width, height);
}
protected void onPreviewDisplaySurfaceChanged(
SurfaceHolder holder, int format, int width, int height) {
// (Re-)Compute the ideal size of the camera-preview display (the area that the
// camera-preview frames get rendered onto, potentially with scaling and rotation)
// based on the size of the SurfaceView that contains the display.
Size viewSize = computeViewSize(width, height);
Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
boolean isCameraRotated = cameraHelper.isCameraRotated();
// Configure the output width and height as the computed display size.
converter.setDestinationSize(
isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
}
private void setupPreviewDisplayView() {
previewDisplayView.setVisibility(View.GONE);
ViewGroup viewGroup = findViewById(R.id.preview_display_layout);
viewGroup.addView(previewDisplayView);
previewDisplayView
.getHolder()
.addCallback(
new SurfaceHolder.Callback() {
@Override
public void surfaceCreated(SurfaceHolder holder) {
processor.getVideoSurfaceOutput().setSurface(holder.getSurface());
}
@Override
public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
onPreviewDisplaySurfaceChanged(holder, format, width, height);
}
@Override
public void surfaceDestroyed(SurfaceHolder holder) {
processor.getVideoSurfaceOutput().setSurface(null);
}
});
}
private String getMultiHandLandmarksDebugString(List<LandmarkProto.NormalizedLandmarkList> multiHandLandmarks) {
if (multiHandLandmarks.isEmpty()) {
return "No hand landmarks";
}
String multiHandLandmarksStr = "Number of hands detected: " + multiHandLandmarks.size() + "\n";
int handIndex = 0;
for (LandmarkProto.NormalizedLandmarkList landmarks : multiHandLandmarks) {
multiHandLandmarksStr +=
"\t#Hand landmarks for hand[" + handIndex + "]: " + landmarks.getLandmarkCount() + "\n";
int landmarkIndex = 0;
for (LandmarkProto.NormalizedLandmark landmark : landmarks.getLandmarkList()) {
multiHandLandmarksStr +=
"\t\tLandmark ["
+ landmarkIndex
+ "]: ("
+ landmark.getX()
+ ", "
+ landmark.getY()
+ ", "
+ landmark.getZ()
+ ")\n";
++landmarkIndex;
}
++handIndex;
}
return multiHandLandmarksStr;
}
}
11. Connect mobile phone and computer, click `Run app` in Android Studio. The app is installed to my phone, but it can't be run. When I get the permission of camera, it is rolled out.
I don't know which step is wrong, could you please give me a hand?
System information (Please provide as much relevant information as possible)
Describe the problem:
Fellowing Steps to build a MediaPipe AAR, I built face dection AAR and installed successfully. When I tryed to install hand tracking by aar, I encountered problem, the app crashed.
Specifically:
mediapipe\examples\android\src\java\com\google\mediapipe\apps\basic\MainActivity.java
,D:\project\mediapipe-master\mediapipe\examples\android\src\java\com\google\mediapipe\apps\basic\res
to my project.mediapipe_aar( name = "mediapipe_hand_tracking", calculators = ["//mediapipe/graphs/hand_tracking:mobile_calculators"], )
bazel build -c opt --strip=ALWAYS \ --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \ --fat_apk_cpu=arm64-v8a,armeabi-v7a \ --legacy_whole_archive=0 \ --features=-legacy_whole_archive \ --copt=-fvisibility=hidden \ --copt=-ffunction-sections \ --copt=-fdata-sections \ --copt=-fstack-protector \ --copt=-Oz \ --copt=-fomit-frame-pointer \ --copt=-DABSL_MIN_LOG_LEVEL=2 \ --linkopt=-Wl,--gc-sections,--strip-all \ //mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example:mediapipe_hand_tracking
bazel build -c opt mediapipe/graphs/hand_tracking:hand_tracking_mobile_gpu_binary_graph
<?xml version="1.0" encoding="utf-8"?> <manifest xmlns:android="http://schemas.android.com/apk/res/android" package="com.google.mediapipe.apps.basic">
plugins { id 'com.android.application' }
android { compileSdkVersion 30 buildToolsVersion "30.0.3"
}
dependencies { implementation fileTree(dir: 'libs', include: ['.jar', '.aar']) implementation 'androidx.appcompat:appcompat:1.0.2' implementation 'androidx.constraintlayout:constraintlayout:1.1.3' testImplementation 'junit:junit:4.12' androidTestImplementation 'androidx.test.ext:junit:1.1.0' androidTestImplementation 'androidx.test.espresso:espresso-core:3.1.1' // MediaPipe deps implementation 'com.google.flogger:flogger:latest.release' implementation 'com.google.flogger:flogger-system-backend:latest.release' implementation 'com.google.code.findbugs:jsr305:latest.release' implementation 'com.google.guava:guava:27.0.1-android' implementation 'com.google.protobuf:protobuf-javalite:3.19.1' // CameraX core library def camerax_version = "1.0.0-beta10" implementation "androidx.camera:camera-core:$camerax_version" implementation "androidx.camera:camera-camera2:$camerax_version" implementation "androidx.camera:camera-lifecycle:$camerax_version" // AutoValue def auto_value_version = "1.8.1" implementation "com.google.auto.value:auto-value-annotations:$auto_value_version" annotationProcessor "com.google.auto.value:auto-value:$auto_value_version" }
// Copyright 2019 The MediaPipe Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License.
package com.google.mediapipe.apps.basic;
import android.content.pm.ApplicationInfo; import android.content.pm.PackageManager; import android.content.pm.PackageManager.NameNotFoundException; import android.graphics.SurfaceTexture; import android.os.Bundle; import androidx.appcompat.app.AppCompatActivity; import android.util.Log; import android.util.Size; import android.view.SurfaceHolder; import android.view.SurfaceView; import android.view.View; import android.view.ViewGroup; import com.google.mediapipe.components.CameraHelper; import com.google.mediapipe.components.CameraXPreviewHelper; import com.google.mediapipe.components.ExternalTextureConverter; import com.google.mediapipe.components.FrameProcessor; import com.google.mediapipe.components.PermissionHelper; import com.google.mediapipe.formats.proto.LandmarkProto; import com.google.mediapipe.framework.AndroidAssetUtil; import com.google.mediapipe.framework.AndroidPacketCreator; import com.google.mediapipe.framework.Packet; import com.google.mediapipe.framework.PacketGetter; import com.google.mediapipe.glutil.EglManager;
import java.util.HashMap; import java.util.List; import java.util.Map;
/* Main activity of MediaPipe basic app. / public class MainActivity extends AppCompatActivity { private static final String TAG = "MainActivity";
}