JustinPeng2007
commented
8 months ago I have taken some time and I have modified my code to be similar to your example face detection code. See below.
Problems:
-
However, after adding various debug.log in many locations, I was able to spot that it was stuck at:
if (outputVideoStream.TryGetNext(out var outputVideo))
- Through my debug.log, I found it was also NOT calling OnCameraFrameReceived for some reason. It isn't even stuck, it was just never called at all.
P.S I have checked the logs, however, there was not a single error output. The previous error I have mentioned have been resolved. It just appears to be stuck at if (outputVideoStream.TryGetNext(out var outputVideo))
private IEnumerator Start()
{
_cameraManager.frameReceived += OnCameraFrameReceived;
yield return GpuManager.Initialize();
Debug.Log("after gpu manager");
if (!GpuManager.IsInitialized)
{
throw new System.Exception("Failed to initialize GPU resources");
}
_screen.rectTransform.sizeDelta = new Vector2(_width, _height);
_inputTexture = new Texture2D(_width, _height, TextureFormat.RGBA32, false);
_inputPixelData = new Color32[_width * _height];
_outputTexture = new Texture2D(_width, _height, TextureFormat.RGBA32, false);
_outputPixelData = new Color32[_width * _height];
_screen.texture = _outputTexture;
Debug.Log("after textures");
var _resourceManager = new StreamingAssetsResourceManager();
yield return _resourceManager.PrepareAssetAsync("face_detection_short_range.bytes");
yield return _resourceManager.PrepareAssetAsync("face_landmark_with_attention.bytes");
var stopwatch = new Stopwatch();
Debug.Log("before config");
//_graph = new CalculatorGraph(_configAsset.text);
var config = CalculatorGraphConfig.Parser.ParseFromTextFormat(_configAsset.text);
Debug.Log("before validatedGraphConfig");
using (var validatedGraphConfig = new ValidatedGraphConfig())
{
Debug.Log("before Initialize");
validatedGraphConfig.Initialize(config).AssertOk();
Debug.Log("before CalculatorGraph");
_graph = new CalculatorGraph(validatedGraphConfig.Config());
}
Debug.Log("before SetGpuResources");
_graph = new CalculatorGraph(_configAsset.text);
_graph.SetGpuResources(GpuManager.GpuResources).AssertOk();
Debug.Log("after SetGpuResources");
var outputVideoStream = new OutputStream<ImageFramePacket, ImageFrame>(_graph, "output_video");
var multiFaceLandmarksStream = new OutputStream<NormalizedLandmarkListVectorPacket, List<NormalizedLandmarkList>>(_graph, "multi_face_landmarks");
outputVideoStream.StartPolling().AssertOk();
multiFaceLandmarksStream.StartPolling().AssertOk();
_graph.StartRun().AssertOk();
stopwatch.Start();
var screenRect = _screen.GetComponent<RectTransform>().rect;
while (true)
{
Debug.Log("&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&");
yield return new WaitForEndOfFrame();
Debug.Log(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>waiting");
if (outputVideoStream.TryGetNext(out var outputVideo))
{
Debug.Log("----------------------------------------------------------------------------");
if (outputVideo.TryReadPixelData(_outputPixelData))
{
Debug.Log(")))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))");
_outputTexture.SetPixels32(_outputPixelData);
Debug.Log("READDD");
_outputTexture.Apply();
Debug.Log("==========================================================");
}
}
Debug.Log("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
}
}
private void OnDestroy()
{
_cameraManager.frameReceived -= OnCameraFrameReceived;
if (_graph != null)
{
try
{
_graph.CloseInputStream("input_video").AssertOk();
_graph.WaitUntilDone().AssertOk();
}
finally
{
_graph.Dispose();
}
}
GpuManager.Shutdown();
}
private unsafe void OnCameraFrameReceived(ARCameraFrameEventArgs eventArgs)
{
Debug.Log("*******************************************************************");
XRCpuImage image;
if (!_cameraManager.TryAcquireLatestCpuImage(out image))
return;
InitBuffer(image);
var conversionParams = new XRCpuImage.ConversionParams
(
image,
TextureFormat.RGBA32,
XRCpuImage.Transformation.MirrorY
);
if (m_Texture == null || m_Texture.width != image.width || m_Texture.height != image.height)
{
m_Texture = new Texture2D(conversionParams.outputDimensions.x,
conversionParams.outputDimensions.y,
conversionParams.outputFormat, false);
}
var ptr = (IntPtr)NativeArrayUnsafeUtility.GetUnsafePtr(_buffer);
image.Convert(conversionParams, ptr, _buffer.Length);
m_Texture.LoadRawTextureData(_buffer);
m_Texture.Apply();
//mRenderer.material.mainTexture = m_Texture;
image.Dispose();
var imageFrame = new ImageFrame(ImageFormat.Types.Format.Srgba, _width, _height, _width * 4, _buffer);
var currentTimestamp = stopwatch.ElapsedTicks / (System.TimeSpan.TicksPerMillisecond / 1000);
_graph.AddPacketToInputStream("input_video", new ImageFramePacket(imageFrame, new Timestamp(currentTimestamp))).AssertOk();
_buffer.Dispose();
}
private void InitBuffer(XRCpuImage image)
{
var length = image.width * image.height * 4;
if (_buffer == null || _buffer.Length != length)
{
_buffer = new NativeArray<byte>(length, Allocator.Persistent, NativeArrayOptions.UninitializedMemory);
}
}
homuler
Plugin Version or Commit ID
v0.12.0
Unity Version
2021.3.14f1
Your Host OS
Windows 11 Home
Target Platform
Android
Description
Hi, I understand that this might be a very similar problem to others, however after reading through the other issues, I have yet to resolve my problem. Like others, I have been trying to access the textures from the AR camera to use for media pipe unity.TryAcquireLatestCpuImage as well.
Below is CameraImageController.cs, which converts the AR camera to texture 2D. Than, I call it in FaceMesh.cs (see below), using
_inputTexture = _cameraTransfar.m_Texture;, which is then passed into ImageFrame. However,_cameraTransfar.m_Texturehas been always null when the program runs, which I am very confused about.Code to Reproduce the issue
Additional Context