Some problems with digital PTZ

[759401524]

Control ColorCamera with ImageManipConfig

1. Use RectCrop to crop the video node, and set the parameters according to setCropRect(self: depthai.ImageManipConfig, xmin: float, ymin: float, xmax: float, ymax: float) → None, it seems wrong

   2021-09-29 17:12:57.477 | INFO     | __main__:main:192 - Switch Rect Crop from [[0.3, 0.3], [0.7, 0.7]] to [[0.25, 0.25], [0.75, 0.75]]
   [14442C10E1C95ED700] [34.191] [ColorCamera(0)] [warning] Crop rectangle out of bounds: x,y (480, 270), w,h (1920, 1080). Using center crop
   2021-09-29 17:12:57.682 | INFO     | __main__:main:192 - Switch Rect Crop from [[0.25, 0.25], [0.75, 0.75]] to [[0.2, 0.2], [0.8, 0.8]]
   [14442C10E1C95ED700] [34.411] [ColorCamera(0)] [warning] Crop rectangle out of bounds: x,y (384, 216), w,h (1920, 1080). Using center crop
   2021-09-29 17:12:58.683 | INFO     | __main__:main:192 - Switch Rect Crop from [[0.2, 0.2], [0.8, 0.8]] to [[0.15, 0.15], [0.85, 0.85]]
   [14442C10E1C95ED700] [35.411] [ColorCamera(0)] [warning] Crop rectangle out of bounds: x,y (288, 162), w,h (1920, 1080). Using center crop

2. Use CenterCrop to crop the video node, set the parameters according to CenterCrop(self: depthai.ImageManipConfig, ratio: float, whRatio: float) → None, the picture remains unchanged

Control ImageManip with ImageManipConfig

1. Use Rect Crop or CenterCrop to crop the video node, some parameter values will have the following errors [ImageManip(3)] [error] Processing failed, potentially unsupported config

   Switch Rect Crop from [[0.25, 0.3], [0.8, 0.8]] to [[0.3, 0.35], [0.75, 0.75]]
   [ImageManip(3)] [error] Processing failed, potentially unsupported config
   Switch Rect Crop from [[0.3, 0.35], [0.75, 0.75]] to [[0.35, 0.4], [0.7, 0.7]]
   [ImageManip(3)] [error] Processing failed, potentially unsupported config

   Switch Center Crop from [0.8, 1.0] to [0.75, 1.0]
   [ImageManip(3)] [error] Processing failed, potentially unsupported config
   Switch Center Crop from [0.75, 1.0] to [0.7, 1.0]
   [ImageManip(3)] [error] Processing failed, potentially unsupported config

from collections import deque

import cv2
import depthai as dai
import numpy as np
import argparse

try:
    from loguru import logger
except ImportError:
    import logging as logger

    LOG_FORMAT = "%(asctime)s | %(levelname)s\t| %(message)s"  # type: str
    DATE_FORMAT = "%Y-%m-%d %H:%M:%S "
    logger.basicConfig(level=logger.DEBUG, format=LOG_FORMAT, datefmt=DATE_FORMAT)

parser = argparse.ArgumentParser()
parser.add_argument(
    "-c",
    "--CenterCrop",
    default=False,
    action="store_true",
    help="Use the center crop to adjust the field of view, default : %(default)s",
)
parser.add_argument(
    "-r",
    "--CropRect",
    default=False,
    action="store_true",
    help="Use rectangular cropping to adjust the field of view, default : %(default)s",
)
parser.add_argument(
    "-m",
    "--use_manip",
    default=False,
    action="store_true",
    help="Crop through the ImageManip node, default : %(default)s",
)
parser.add_argument(
    "-p",
    "--preview",
    default=False,
    action="store_true",
    help="Use preview node to display, default : %(default)s",
)
args = parser.parse_args()
if args.CenterCrop is False and args.CropRect is False:
    logger.warning(
        "Command line parameter error! '--CropRect' and '--CenterCrop' must choose one !"
        "\tCenterCrop is used by default."
    )
    args.CenterCrop = True
if args.CenterCrop and args.CropRect:
    logger.warning(
        "Command line parameter error! '--CropRect' cannot be used with '--CenterCrop'!"
        "\tCenterCrop is used by default."
    )
    args.CenterCrop = True

def center_zoom(boxes, zoom_in):
    zoom_list, step = np.linspace(start=0, stop=1, num=21, retstep=True)
    boxes_tmp = boxes.copy()
    if zoom_in:
        boxes_tmp[0] += step
    else:
        boxes_tmp[0] -= step
    boxes_tmp[0] = np.clip(boxes_tmp[0], 0, 1).round(3)
    return boxes_tmp

def zoom(boxes, zoom_in, limit_min=None, limit_max=None):
    zoom_list, step = np.linspace(start=0, stop=1, num=21, retstep=True)
    if limit_max is None:
        limit_max = np.array([[0, 0], [1, 1]])
    if limit_min is None:
        limit_min = limit_max[::-1]

    boxes = np.reshape(boxes, (-1, 2, 2))
    scale_size = (boxes / step).round() * step
    box_border = np.concatenate(
        [scale_size[:, 0] - limit_max[0], limit_max[1] - scale_size[:, 1]], 1
    ).reshape((-1, 2, 2))
    scale_size_round = (box_border / step).max(1).max(1).reshape(-1, 1)
    box_border = np.round((box_border / scale_size_round) / step) * step
    box_border = np.where(np.isnan(box_border), 0, box_border)
    if zoom_in:
        if box_border.sum() == 0:
            return scale_size.squeeze()
        scale_size[:, 0] -= box_border[:, 0]
        scale_size[:, 1] += box_border[:, 1]
    else:
        if box_border.sum() == 0:
            box_border[:] = step
        scale_size[:, 0] += box_border[:, 0]
        scale_size[:, 1] -= box_border[:, 1]
    scale_size[:, 0] = np.clip(scale_size[:, 0], limit_max[0], limit_min[0])
    scale_size[:, 1] = np.clip(scale_size[:, 1], limit_min[1], limit_max[1])
    scale_size = np.where((scale_size[:, 1] - scale_size[:, 0]) <= 0, boxes, scale_size)
    return scale_size.round(3).squeeze()

def create_pipeline():
    pipeline = dai.Pipeline()
    color_cam = pipeline.createColorCamera()
    color_cam.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)
    if args.preview:
        color_cam.setPreviewSize(color_cam.getResolutionSize())
        color_cam.setInterleaved(False)

    configIn = pipeline.createXLinkIn()
    configIn.setStreamName("config")

    cam_out = pipeline.createXLinkOut()
    cam_out.setStreamName("cam_out")

    if args.use_manip:
        config_manip = pipeline.createImageManip()
        config_manip.setMaxOutputFrameSize(
            color_cam.getResolutionHeight() * color_cam.getResolutionWidth() * 3
        )
        if args.preview:
            color_cam.preview.link(config_manip.inputImage)
        else:
            color_cam.video.link(config_manip.inputImage)

        config_manip.initialConfig.setFrameType(dai.RawImgFrame.Type.BGR888p)
        configIn.out.link(config_manip.inputConfig)
        config_manip.out.link(cam_out.input)

    else:
        configIn.out.link(color_cam.inputConfig)
        if args.preview:
            color_cam.preview.link(cam_out.input)
        else:
            color_cam.video.link(cam_out.input)

    return pipeline

def main():
    with dai.Device(create_pipeline()) as device:
        cam_out = device.getOutputQueue("cam_out")
        config = device.getInputQueue("config")
        CropRect = deque(maxlen=2)
        if args.CenterCrop:
            CropRect.append(np.array([0.5, 1]))
        else:
            CropRect.append(
                np.array(
                    [
                        [0.30835197649809043, 0.35194185934086797],
                        [0.7897466050543481, 0.7779800385758091],
                    ]
                )
            )
        crop = False
        while 1:
            frame = cam_out.get().getCvFrame()
            cv2.imshow("", cv2.resize(frame, (0, 0), fx=0.5, fy=0.5))
            key = cv2.waitKey(1)
            if args.CenterCrop:
                if key == ord("q"):
                    break
                elif key == ord("+"):
                    tmp = center_zoom(CropRect[-1], True)
                    CropRect.append(tmp)
                    crop = True
                elif key == ord("-"):
                    tmp = center_zoom(CropRect[-1], False)
                    CropRect.append(tmp)
                    crop = True
                if crop:
                    im_cfg = dai.ImageManipConfig()
                    im_cfg.setFrameType(dai.RawImgFrame.Type.BGR888p)
                    im_cfg.setCenterCrop(*CropRect[-1])
                    im_cfg.setResizeThumbnail(1920, 1080, 114, 114, 114)
                    # im_cfg.setResize(1920, 1080)
                    config.send(im_cfg)
                    logger.info(
                        f"Switch Center Crop from {CropRect[0].tolist()} to {CropRect[-1].tolist()}"
                    )
                    crop = False
            else:
                if key == ord("q"):
                    break
                elif key == ord("+"):
                    tmp = zoom(CropRect[-1], True)
                    tmp.sort(0)
                    CropRect.append(tmp)
                    crop = True
                elif key == ord("-"):
                    tmp = zoom(CropRect[-1], False)
                    tmp.sort(0)
                    CropRect.append(tmp)
                    crop = True
                if crop:
                    im_cfg = dai.ImageManipConfig()
                    im_cfg.setFrameType(dai.RawImgFrame.Type.BGR888p)
                    im_cfg.setResizeThumbnail(1920, 1080, 114, 114, 114)
                    # im_cfg.setResize(1920, 1080)
                    im_cfg.setCropRect(CropRect[-1].ravel())
                    config.send(im_cfg)
                    logger.info(
                        f"Switch Rect Crop from {CropRect[0].tolist()} to {CropRect[-1].tolist()}"
                    )
                    crop = False
        cv2.destroyAllWindows()

if __name__ == "__main__":
    main()

[themarpe]

Hi @759401524 Thanks for the report. As of right now, NV12 ImgFrames (that come from video output) aren't supported by ImageManip. One option would be to use isp output which is YUV420p type, but not all configurations are supported in that case either.

I am currently refactoring the ImageManip node to support more configurations as well as different types of inputs and outputs including NV12.

Will create a PR with fixes in the near future.