search

luxonis / depthai

DepthAI Python API utilities, examples, and tutorials.
https://docs.luxonis.com
MIT License
926 stars 231 forks source link

[BUG] depthai-sdk Replay module - depth and color video asynrouse #881

Open li195111 opened 1 year ago

li195111 commented 1 year ago

Describe the bug I used the depthai-sdk to replay the video after using it to record but I got the async video between depth and color frames as follows image

My pencil is already down but the depth not following.

Minimal Reproducible Example

from depthai_sdk import OakCamera, RecordType

if name == 'main': record_dir = 'record_outputs/sdk' record_dir = os.path.abspath(record_dir) if not os.path.exists(record_dir): os.makedirs(record_dir, exist_ok=True)

with OakCamera() as oak: color = oak.create_camera('color', resolution='1080P', fps=30, encode='H264') left = oak.create_camera('left', resolution='480P', fps=30, encode='H264') right = oak.create_camera('right', resolution='480P', fps=30, encode='H264') depth = oak.create_stereo(resolution='1080P', fps=30, left=left, right=right)

Synchronize & save all (encoded) streams

oak.record([
    color.out.encoded, left.out.encoded, right.out.encoded,
    # depth.out.encoded
], record_dir, RecordType.VIDEO)
# Show color stream
oak.visualize([color.out.camera, depth.out.main], scale=2 / 3, fps=True)
# Blocking is necessary
oak.start(blocking=True)

- MRE replay

import os

from depthai_sdk import OakCamera

if name == 'main': record_dir = 'record_outputs/sdk/3-1844301041180E1300' record_dir = os.path.abspath(record_dir)

with OakCamera(replay=record_dir) as oak:

Created CameraComponent/StereoComponent will use streams from the recording

color = oak.create_camera('color',
                          resolution='1080P',
                          fps=30,
                          encode='H264')
left = oak.create_camera('left', resolution='480P', fps=30, encode='H264')
right = oak.create_camera('right',
                          resolution='480P',
                          fps=30,
                          encode='H264')
depth = oak.create_stereo(resolution='480p',
                          fps=30,
                          left=left,
                          right=right)
# Show color stream
oak.visualize([color.out.camera, depth.out.main], scale=1 / 3, fps=True)
# Blocking is necessary
oak.start(blocking=True)

**Expected behavior**
The depth image must correspond to the RGB image synchronously.

**Pipeline Graph**
- record
![image](https://user-images.githubusercontent.com/46022495/209082026-20b2eccf-7e96-4f21-b0ce-d2673077a383.png)

- replay
![image](https://user-images.githubusercontent.com/46022495/209082477-eaebf252-8f57-435b-9e04-4b3a76421025.png)

**Attach system log**
log_system_information.json

{ "architecture": "64bit WindowsPE", "machine": "AMD64", "platform": "Windows-10-10.0.22621-SP0", "processor": "Intel64 Family 6 Model 158 Stepping 10, GenuineIntel", "python_build": "default Nov 24 2022 14:38:14", "python_compiler": "MSC v.1916 64 bit (AMD64)", "python_implementation": "CPython", "python_version": "3.8.15", "release": "10", "system": "Windows", "version": "10.0.22621", "win32_ver": "10 10.0.22621 SP0 Multiprocessor Free", "uname": "Windows LI-DESKTOP 10 10.0.22621 AMD64 Intel64 Family 6 Model 158 Stepping 10, GenuineIntel", "packages": [ "anyio==3.6.2", "argon2-cffi==21.3.0", "argon2-cffi-bindings==21.2.0", "arrow==1.2.3", "asttokens==2.2.1", "attrs==22.1.0", "av==10.0.0", "backcall==0.2.0", "beautifulsoup4==4.11.1", "bleach==5.0.1", "blobconverter==1.3.0", "boto3==1.26.34", "botocore==1.29.34", "certifi @ file:///C:/b/abs_85o_6fm0se/croot/certifi_1671487778835/work/certifi", "cffi==1.15.1", "charset-normalizer==2.1.1", "colorama==0.4.6", "coloredlogs==15.0.1", "comm==0.1.2", "contourpy==1.0.6", "cycler==0.11.0", "debugpy==1.6.4", "decorator==5.1.1", "defusedxml==0.7.1", "depthai==2.19.1.0", "depthai-pipeline-graph @ git+https://github.com/geaxgx/depthai_pipeline_graph.git@b5f89bb06b5c421574d991bbdb31f1ebd42118e6", "depthai-sdk==1.9.1", "distinctipy==1.2.2", "entrypoints==0.4", "executing==1.2.0", "fastjsonschema==2.16.2", "flatbuffers==22.12.6", "fonttools==4.38.0", "fqdn==1.5.1", "humanfriendly==10.0", "idna==3.4", "importlib-metadata==5.2.0", "importlib-resources==5.10.1", "imutils==0.5.4", "ipykernel==6.19.4", "ipython==8.7.0", "ipython-genutils==0.2.0", "ipywidgets==8.0.3", "isoduration==20.11.0", "jedi==0.18.2", "Jinja2==3.1.2", "jmespath==1.0.1", "jsonpointer==2.3", "jsonschema==4.17.3", "jupyter==1.0.0", "jupyter-console==6.4.4", "jupyter-events==0.5.0", "jupyter_client==7.4.8", "jupyter_core==5.1.0", "jupyter_server==2.0.2", "jupyter_server_terminals==0.4.3", "jupyterlab-pygments==0.2.2", "jupyterlab-widgets==3.0.4", "kiwisolver==1.4.4", "MarkupSafe==2.1.1", "marshmallow==3.17.0", "matplotlib==3.6.2", "matplotlib-inline==0.1.6", "mistune==2.0.4", "mpmath==1.2.1", "nbclassic==0.4.8", "nbclient==0.7.2", "nbconvert==7.2.7", "nbformat==5.7.1", "nest-asyncio==1.5.6", "notebook==6.5.2", "notebook_shim==0.2.2", "numpy==1.24.0", "onnxruntime-gpu==1.13.1", "opencv-contrib-python==4.6.0.66", "opencv-python==4.6.0.66", "packaging==22.0", "pandas==1.5.2", "pandocfilters==1.5.0", "parso==0.8.3", "pickleshare==0.7.5", "Pillow==9.3.0", "pip==22.3.1", "pkgutil_resolve_name==1.3.10", "platformdirs==2.6.0", "prometheus-client==0.15.0", "prompt-toolkit==3.0.36", "protobuf==4.21.12", "psutil==5.9.4", "pure-eval==0.2.2", "pycparser==2.21", "pydantic==1.10.2", "pydbow==0.1.2", "Pygments==2.13.0", "pyparsing==3.0.9", "pyreadline3==3.4.1", "pyrsistent==0.19.2", "PySide2==5.15.2.1", "python-dateutil==2.8.2", "python-json-logger==2.0.4", "pytube==12.1.2", "PyTurboJPEG==1.6.4", "pytz==2022.7", "pyusb==1.2.1", "pywin32==305", "pywinpty==2.0.9", "PyYAML==6.0", "pyzmq==24.0.1", "Qt.py==1.3.7", "qtconsole==5.4.0", "QtPy==2.3.0", "requests==2.28.1", "rfc3339-validator==0.1.4", "rfc3986-validator==0.1.1", "s3transfer==0.6.0", "scipy==1.9.3", "seaborn==0.12.1", "Send2Trash==1.8.0", "setuptools==65.5.0", "shiboken2==5.15.2.1", "six==1.16.0", "sniffio==1.3.0", "soupsieve==2.3.2.post1", "stack-data==0.6.2", "sympy==1.11.1", "tensorrt @ file:///G:/TensorRT-8.4.3.1/python/tensorrt-8.4.3.1-cp38-none-win_amd64.whl", "terminado==0.17.1", "tinycss2==1.2.1", "torch==1.13.1+cu116", "torchaudio==0.13.1+cu116", "torchvision==0.14.1+cu116", "tornado==6.2", "tqdm==4.64.1", "traitlets==5.8.0", "typing_extensions==4.4.0", "uri-template==1.2.0", "urllib3==1.26.13", "wcwidth==0.2.5", "webcolors==1.12", "webencodings==0.5.1", "websocket-client==1.4.2", "wheel==0.37.1", "widgetsnbextension==4.0.4", "wincertstore==0.2", "xmltodict==0.13.0", "yapf==0.32.0", "zipp==3.11.0" ], "usb": [ "No USB backend found" ] }

Erol444 commented 1 year ago

Hi @li195111 , We apologize for the delay. After some debugging, I can confirm this issue, and it seems like it happens at Replay stage (not at Recording stage). I recorded a stopwatch app on a screen, then overlayed left/color frames with script below, and recording seems to be in sync: image

We will investigate the reason behind Replay out-of-sync issue. Thanks, Erik

import cv2

cap_color = cv2.VideoCapture('recordings/2-184430102127631200/color.mp4')
cap_mono = cv2.VideoCapture('recordings/2-184430102127631200/left.mp4')

read = True
while True:
    if read:
        ret_color, color_frame = cap_color.read()
        ret_mono, mono_frame = cap_mono.read()

    if ret_color and ret_mono:
        color_frame = cv2.cvtColor(color_frame, cv2.COLOR_BGR2RGB)
        mono_frame = cv2.cvtColor(mono_frame, cv2.COLOR_BGR2RGB)
        mono_frame = cv2.resize(mono_frame, (color_frame.shape[1], color_frame.shape[0]))
        print(color_frame.shape, mono_frame.shape)
        overlay = cv2.addWeighted(color_frame, 0.5, mono_frame, 0.5, 0)
        cv2.imshow('Overlay', cv2.pyrDown(overlay))
    else:
        break

    key = cv2.waitKey(33)
    if key == ord('q'):
        break

    if key == ord(' '):
        read = not read