From 40928133e28071dce77ec8cc5e166840b862bc99 Mon Sep 17 00:00:00 2001 From: Guy Martin Date: Wed, 21 Feb 2024 14:03:01 -0500 Subject: [PATCH] SE-2715 updating feature demo --- disparity/README.md | 38 +++++------ disparity/demo.py | 14 ++++ keypoints/README.md | 6 +- keypoints/demo.py | 161 +++++++++++++++++++++++++++++++++++--------- sparse3d/README.md | 20 +++--- 5 files changed, 174 insertions(+), 65 deletions(-) diff --git a/disparity/README.md b/disparity/README.md index 2fb75a8..ac2c392 100644 --- a/disparity/README.md +++ b/disparity/README.md @@ -15,25 +15,25 @@ The Python script shows how to programmatically ## Setup -Change the following demo parameters to your desired settings in the ```demo.py``` file. - -| ***Parameter*** | ***Description*** | -|-----------------|------------------------------------------------------------------------------------------------| -| ```mac``` | (optional) The MAC address of the camera. Assumes the first available camera if not specified. | -| ```offsety1``` | (optional) The Y offset of the right image. Default 440. | -| ```image``` | (optional) Decide whether to stream the left image with disparity. | -| ```cost_path``` | (optional) Sets disparity cost path | -| ```crosscheck``` | (optional) Sets crosscheck value | -| ```subpixel``` | (optional) Set sub-pixel value | -| ```blockwidth``` | (optional) block size width | -| ```blockheight``` | (optional) block size height | -| ```mindisparity``` | (optional) minimum disparity | -| ```numdisparity``` | (optional) number of disparity | -| ```vsearch``` | (optional) Set vertical search | -| ```threshold``` | (optional) set threshold | -| ```uniqueness``` | (optional) set uniqueness ratio | -| ```penalty1``` | (optional) set penalty1 | -| ```penalty2``` | (optional) set penalty2 | +Set the following arguments to the ```demo.py``` file to change demo behavior. + +| ***Parameter*** | ***Description*** | +|----------------------|------------------------------------------------------------------------------------------------| +| ```--mac``` | (optional) The MAC address of the camera. Assumes the first available camera if not specified. | +| ```--offsety1``` | (optional) The Y offset of the right image. Default 440. | +| ```--image``` | (optional) Decide whether to stream the left image with disparity. | +| ```--cost_path``` | (optional) Sets disparity cost path | +| ```--crosscheck``` | (optional) Sets crosscheck value | +| ```--subpixel``` | (optional) Set sub-pixel value | +| ```--blockwidth``` | (optional) block size width | +| ```--blockheight``` | (optional) block size height | +| ```--mindisparity``` | (optional) minimum disparity | +| ```--numdisparity``` | (optional) number of disparity | +| ```--vsearch``` | (optional) Set vertical search | +| ```--threshold``` | (optional) set threshold | +| ```--uniqueness``` | (optional) set uniqueness ratio | +| ```--penalty1``` | (optional) set penalty1 | +| ```--penalty2``` | (optional) set penalty2 | ## Usage ```bash diff --git a/disparity/demo.py b/disparity/demo.py index baa24a7..a1e3319 100755 --- a/disparity/demo.py +++ b/disparity/demo.py @@ -92,6 +92,18 @@ def check_stereo(device: eb.PvDeviceGEV, image: bool): multipart = device_params.Get('GevSCCFGMultiPartEnabled') multipart.SetValue(image) + +def enable_undistort_rectify(device: eb.PvDeviceGEV): + """ + Enables undistortion and rectification + :param device: The device + """ + if not is_stereo(device): + raise RuntimeError('Sparse3D supported only on Bottlenose stereo.') + + # Get device parameters + device_params = device.GetParameters() + # Turn on rectification rectify = device_params.Get("Rectification") rectify.SetValue(True) @@ -316,6 +328,8 @@ def acquire_data(device, stream, invalid): if bn_device is not None: check_stereo(device=bn_device, image=args.image) set_y1_offset(device=bn_device, value=args.offsety1) + + enable_undistort_rectify(device=bn_device) enable_disparity(device=bn_device) configure_disparity(device=bn_device, params=args) diff --git a/keypoints/README.md b/keypoints/README.md index 66a3db9..6708a00 100644 --- a/keypoints/README.md +++ b/keypoints/README.md @@ -1,11 +1,11 @@ # Simple Keypoint Sample -This is a modified version of the streaming demo that shows how to transmit the key points from Bottlenose. +This demo shows how to stream keypoints from a Bottlenose camera. -Please use ```eBusPlayer``` first to configure the image quality, gain, and exposure settings. +This example assumes that any other settings related to image quality such as `exposure`, `gain`, and `CCM` is properly set. Please use `Stereo Viewer` or `eBusPlayer` to configure the image quality to your like. The Python script shows how to programmatically enable chunk data transmission for -[Fast9](https://en.wikipedia.org/wiki/Features_from_accelerated_segment_test) key points. +[FAST](https://en.wikipedia.org/wiki/Features_from_accelerated_segment_test) and GFTT key points. ## Setup diff --git a/keypoints/demo.py b/keypoints/demo.py index fea8fcb..447d2ab 100755 --- a/keypoints/demo.py +++ b/keypoints/demo.py @@ -23,8 +23,11 @@ import sys import warnings -import eBUS as eb +import argparse import cv2 +import numpy as np +import eBUS as eb + # reference common utility files sys.path.insert(1, '../common') @@ -34,6 +37,80 @@ import draw_chunkdata as chk +def parse_args(): + """ + parses and return the command-line arguments + """ + parser = argparse.ArgumentParser() + parser.add_argument("-m", "--mac", default=None, help="MAC address of the Bottlenose camera") + parser.add_argument("--corner_type", default='FAST', choices=['FAST', 'GFTT'], + help="type of corner to detect") + parser.add_argument("--gftt_detector", default='Harris', choices=['Harris', 'MinEigenValue'], + help="Set the detector for GFTT") + parser.add_argument("--max_features", default=1000, type=int, help="maximum number of features to detect") + parser.add_argument("--quality_level", default=500, type=int, help="quality level for GFTT") + parser.add_argument("--min_distance", default=15, type=int, help="minimum distance for GFTT") + + parser.add_argument("-k", "--paramk", type=float, default=0, help="free parameter K for Harris corner") + parser.add_argument("--threshold", type=int, default=100, help="set threshold for FAST") + parser.add_argument("--nms", action='store_true', help="use nms for FAST") + + return parser.parse_args() + + +def parse_validate_args(): + """ + Parse and validate the range of arguments + :return: the parsed argument object + """ + params = parse_args() + + max_num = 65535 if params.corner_type == 'FAST' else 8192 + assert 0 < params.max_features <= max_num, f'Invalid max_features, values in [1, {max_num}]' + + assert 0 <= params.threshold <= 255, 'Invalid threshold, values in [0, 255]' + assert 0 < params.quality_level <= 1023, f'Invalid quality_level, values in [1, 1023]' + assert 0 <= params.min_distance <= 30, 'Invalid min_distance, values in [0, 30]' + assert 0 <= params.paramk <= 1.0, 'Invalid paramk, values in [0, 1.0]' + + return params + + +def is_stereo(device: eb.PvDeviceGEV): + """ + Queries the device to determine if the camera is stereo. + :param device: The device to query + :return True if stereo, False otherwise + """ + reg = device.GetParameters().Get('DeviceModelName') + res, model = reg.GetValue() + + assert res.IsOK(), 'Error accessing camera' + + num_cameras = 1 if model[-1].upper() == 'M' else 2 + + return num_cameras == 2 + + +def set_image_streaming(device: eb.PvDeviceGEV): + """ + Checks that the device is a stereo camera and turns on multipart on if needed + :param device: The device + """ + + stereo = is_stereo(device) + + # Get device parameters + device_params = device.GetParameters() + + # turn multipart on if stereo + multipart = device_params.Get('GevSCCFGMultiPartEnabled') + multipart.SetValue(stereo) + + # set image mode + pixel_format = device_params.Get("PixelFormat") + pixel_format.SetValue("YUV422_8") + def handle_buffer(pvbuffer, device): payload_type = pvbuffer.GetPayloadType() if payload_type == eb.PvPayloadTypeImage: @@ -49,6 +126,24 @@ def handle_buffer(pvbuffer, device): image_data = chk.draw_keypoints(image_data, keypoints[0]) cv2.imshow("Keypoints", image_data) + elif payload_type == eb.PvPayloadTypeMultiPart: + image0 = pvbuffer.GetMultiPartContainer().GetPart(0).GetImage() + image1 = pvbuffer.GetMultiPartContainer().GetPart(1).GetImage() + + image_data0 = image0.GetDataPointer() + image_data1 = image1.GetDataPointer() + keypoints = decode_chunk(device=device, buffer=pvbuffer, chunk='FeaturePoints') + + cvimage0 = cv2.cvtColor(image_data0, cv2.COLOR_YUV2BGR_YUY2) + cvimage1 = cv2.cvtColor(image_data1, cv2.COLOR_YUV2BGR_YUY2) + + cvimage0 = chk.draw_keypoints(cvimage0, keypoints[0]) + cvimage1 = chk.draw_keypoints(cvimage1, keypoints[1]) + + display_image = np.hstack((cvimage0, cvimage1)) + + cv2.imshow("Keypoints", display_image) + def enable_feature_points(device): """ @@ -59,45 +154,44 @@ def enable_feature_points(device): device_params = device.GetParameters() # Enable keypoint detection and streaming - chunkMode = device_params.Get("ChunkModeActive") - chunkMode.SetValue(True) - chunkSelector = device_params.Get("ChunkSelector") - chunkSelector.SetValue("FeaturePoints") - chunkEnable = device_params.Get("ChunkEnable") - chunkEnable.SetValue(True) + chunk_mode = device_params.Get("ChunkModeActive") + chunk_mode.SetValue(True) + + chunk_selector = device_params.Get("ChunkSelector") + chunk_selector.SetValue("FeaturePoints") + chunk_enable = device_params.Get("ChunkEnable") + chunk_enable.SetValue(True) -def configure_fast9(device, max_num=1000, threshold=20, useNonMaxSuppression=True): + +def configure_fast9(device, max_num: int = 1000, threshold: int = 20, use_nms: bool = True): """ - Configure the Fast9n keypoint detector + Configure the Fast keypoint detector :param device: Device to configure :param max_num: Maximum number of features to consider. :param threshold: Quality threshold 0...100 - :param useNonMaxSuppression: Use non-maximum suppression + :param use_nms: Use non-maximum suppression """ # Get device parameters device_params = device.GetParameters() - KPCornerType = device_params.Get("KPCornerType") - KPCornerType.SetValue("Fast9n") + corner_type = device_params.Get("KPCornerType") + corner_type.SetValue("Fast9n") - KPFast9nMaxNum = device_params.Get("KPMaxNumber") - KPFast9nMaxNum.SetValue(int(max_num)) + max_features = device_params.Get("KPMaxNumber") + max_features.SetValue(max_num) - KPFast9nThreshold = device_params.Get("KPThreshold") - KPFast9nThreshold.SetValue(threshold) + fast_threshold = device_params.Get("KPThreshold") + fast_threshold.SetValue(threshold) - KPFast9nUseNonMaxSuppression = device_params.Get("KPUseNMS") - KPFast9nUseNonMaxSuppression.SetValue(useNonMaxSuppression) + fast_nms = device_params.Get("KPUseNMS") + fast_nms.SetValue(use_nms) -def run_demo(device, stream, max_fast_features=1000, fast_threshold=10, use_non_max_suppression=True): +def run_demo(device, stream): """ Run the demo :param device: The device to stream from :param stream: The stream to use for streaming - :param max_fast_features: The maximum number of features to extract from the image. - :param fast_threshold: The threshold to use for the Fast9 extractor - :param use_non_max_suppression: Use non-maximum suppression for the Fast9 extractor """ # Get device parameters need to control streaming device_params = device.GetParameters() @@ -106,10 +200,6 @@ def run_demo(device, stream, max_fast_features=1000, fast_threshold=10, use_non_ start = device_params.Get("AcquisitionStart") stop = device_params.Get("AcquisitionStop") - # Enable keypoint detection and streaming - enable_feature_points(device) - configure_fast9(device, max_fast_features, fast_threshold, use_non_max_suppression) - # Enable streaming and send the AcquisitionStart command device.StreamEnable() start.Execute() @@ -141,12 +231,17 @@ def run_demo(device, stream, max_fast_features=1000, fast_threshold=10, use_non_ if __name__ == '__main__': - mac_address = None - if len(sys.argv) >= 2: - mac_address = sys.argv[1] + args = parse_validate_args() + + bn_device, bn_stream, bn_buffers = init_bottlenose(args.mac) + if bn_device is not None: + # set device into image streaming mode + set_image_streaming(device=bn_device) + + # Enable keypoint detection and streaming + enable_feature_points(device=bn_device) + configure_fast9(device=bn_device, max_num=args.max_features, threshold=args.threshold, use_nms=args.nms) - device, stream, buffers = init_bottlenose(mac_address) - if device is not None: - run_demo(device, stream) + run_demo(bn_device, bn_stream) - deinit_bottlenose(device, stream, buffers) + deinit_bottlenose(bn_device, bn_stream, bn_buffers) diff --git a/sparse3d/README.md b/sparse3d/README.md index 2693305..29eeb62 100644 --- a/sparse3d/README.md +++ b/sparse3d/README.md @@ -7,22 +7,22 @@ This example assumes that: - all the image quality related settings such as `exposure`, `gain`, and `CCM` are set. Please use `Stereo Viewer` or `eBusPlayer` to configure the image quality to your like. The Python script shows how to programmatically -- set keypoint parameters, only [Fast9](https://en.wikipedia.org/wiki/Features_from_accelerated_segment_test) is shown, but can be adapted for [GFTT](https://ieeexplore.ieee.org/document/323794). +- set keypoint parameters, only [FAST](https://en.wikipedia.org/wiki/Features_from_accelerated_segment_test) is shown, but can be adapted for [GFTT](https://ieeexplore.ieee.org/document/323794). - set keypoint matching parameters - enable chunk data transmission for sparse point cloud ## Setup -Change the following demo parameters to your desired settings in the ```demo.py``` file. +Set the following arguments to the ```demo.py``` file to change demo behavior. -| ***Parameter*** | ***Description*** | -|----------------------|------------------------------------------------------------------------------------------------| -| ```mac``` | (optional) The MAC address of the camera. Assumes the first available camera if not specified. | -| ```max_keypoints``` | (optional) Maximum number of keypoints to detect. Default 100. | -| ```fast_threshold``` | (optional) Keypoint threshold for the Fast9 algorithm. Default 20. | -| ```match_xoffset``` | (optional) Matcher horizontal search range. Default 0. | -| ```match_yoffset``` | (optional) Matcher vertical search range. Default 0. | -| ```offsety1``` | (optional) The Y offset of the right image. Default 440. | +| ***Parameter*** | ***Description*** | +|------------------------|------------------------------------------------------------------------------------------------| +| ```--mac``` | (optional) The MAC address of the camera. Assumes the first available camera if not specified. | +| ```--max_keypoints``` | (optional) Maximum number of keypoints to detect. Default 100. | +| ```--fast_threshold``` | (optional) Keypoint threshold for the Fast9 algorithm. Default 20. | +| ```--match_xoffset``` | (optional) Matcher horizontal search range. Default 0. | +| ```--match_yoffset``` | (optional) Matcher vertical search range. Default 0. | +| ```--offsety1``` | (optional) The Y offset of the right image. Default 440. | ## Usage ```bash