From 447195b3f2ca06788652a67bf9c7890ea3d175d8 Mon Sep 17 00:00:00 2001 From: hcwinsemius Date: Fri, 28 Jun 2024 08:44:07 +0200 Subject: [PATCH] solved some deprecation issues and docs #171 --- pyorc/api/frames.py | 13 ++++----- pyorc/api/mask.py | 14 +++------- pyorc/helpers.py | 25 +++++++++-------- pyorc/project.py | 7 +++-- tests/test_cameraconfig.py | 17 +++++++----- tests/test_cli.py | 8 +++--- tests/test_frames.py | 55 ++++++++++++++++++++------------------ tests/test_mask.py | 1 - tests/test_video.py | 22 ++++++++------- 9 files changed, 85 insertions(+), 77 deletions(-) diff --git a/pyorc/api/frames.py b/pyorc/api/frames.py index f84fb41..3f7b52a 100644 --- a/pyorc/api/frames.py +++ b/pyorc/api/frames.py @@ -171,18 +171,15 @@ def project( With `cv` (opencv) resampling is performed by first undistorting images, and then by resampling to the desired grid. With heavily distorted images and part of the area of interest outside of the field of view, the orthoprojection of the corners may end up in the wrong space. With `numpy` each individual - image coordinate is mapped to the orthoprojected space, making the projection much more robust. + orthoprojected grid cell is mapped to the image pixels space. For oversampled areas, this is also done vice + versa. Undersampled areas result in nearest-neighbour interpolations, whilst for oversampled, this results + in a reduced value (which can be defined by the user). We recommend switching to `numpy` if you experience strange results with `cv`. resolution : float, optional resolution to project to. If not provided, this will be taken from the camera config in the metadata (Default value = None) - **kwargs: dict, optional - keyword arguments that can be passed to the different projection methods. Currently only used when - `method="numpy"`. kwargs in this case can be `stride` (sampling space used to estimate the real-world - location of each pixel) and `radius` (maximum search radius in pixels to use to fill in missing values - in areas with undersampling after interpolation. `stride` defaults to 10 anbd normally should not be changed. - `radius` defaults to 5. If a larger radius is needed, you are likely undersampling too much. Under normal - situations, these values should not be altered. + reducer: str, optional + Currently only used when `method="numpy"`. Default "mean", resulting in averaging of oversampled grid cells. Returns ------- diff --git a/pyorc/api/mask.py b/pyorc/api/mask.py index 3b1cd7e..18bc1f4 100644 --- a/pyorc/api/mask.py +++ b/pyorc/api/mask.py @@ -16,6 +16,8 @@ If ``inplace=True``, the dataset will be returned masked with ``mask``. """ + + def _base_mask(time_allowed=False, time_required=False): """ wrapper generator for creating generalized structure masking methods for velocimetry @@ -42,23 +44,15 @@ def wrapper_func(ref, inplace=False, reduce_time=False, *args, **kwargs): ds = ref._obj.mean(dim="time", keep_attrs=True) else: ds = ref._obj - if not(ds.velocimetry.is_velocimetry): + if not ds.velocimetry.is_velocimetry: raise AssertionError("Dataset is not a valid velocimetry dataset") if time_required: # then automatically time is also allowed - # time_allowed = True if not("time" in ds.dims): raise AssertionError( f'This mask requires dimension "time". The dataset does not contain dimension "time" or you have set' f'reduce_time=True. Apply this mask without applying any reducers in time.' ) - # if not(time_allowed) and not(time_required): - # if "time" in ref._obj.dims: - # raise AssertionError( - # f'This mask can only work without dimension "time". The dataset contains dimension "time".' - # f'Reduce this by applying a reducer or selecting a time step. ' - # f'Reducing can be done e.g. with ds.mean(dim="time", keep_attrs=True) or slicing with ds.isel(time=0)' - # ) if time_required: if not("time" in ds): raise AssertionError( @@ -67,7 +61,7 @@ def wrapper_func(ref, inplace=False, reduce_time=False, *args, **kwargs): ) if not(time_allowed or time_required) and "time" in ds: # function must be applied per time step - mask = ds.groupby("time").map(mask_func, **kwargs) + mask = ds.groupby("time", squeeze=False).map(mask_func, **kwargs) else: # apply the wrapped mask function as is mask = mask_func(ds, **kwargs) diff --git a/pyorc/helpers.py b/pyorc/helpers.py index 134cd4a..0388e6c 100644 --- a/pyorc/helpers.py +++ b/pyorc/helpers.py @@ -573,6 +573,7 @@ def staggered_index(start=0, end=100): idx_sort.sort() return idx_order + def velocity_log_fit(v, depth, dist_shore, dim="quantile"): """Fill missing surface velocities using a velocity depth profile with @@ -596,25 +597,27 @@ def velocity_log_fit(v, depth, dist_shore, dim="quantile"): """ def log_fit(_v): + idx_finite = np.isfinite(_v).values.flatten() pars = optimize_log_profile( - depth[np.isfinite(_v).values], - _v[np.isfinite(_v).values], - dist_shore[np.isfinite(_v).values] + depth[idx_finite], + _v[0, idx_finite], + dist_shore[idx_finite] ) - _v[np.isnan(_v).values] = log_profile( + idx_miss = np.where(np.isnan(_v[0]).values)[0] + _v[0, idx_miss] = log_profile( ( - depth[np.isnan(_v).values], - dist_shore[np.isnan(_v).values] + depth[idx_miss], + dist_shore[idx_miss] ), **pars ) # enforce that velocities are zero with zero depth - _v[depth <= 0] = 0. + _v[0, depth <= 0] = 0. return np.maximum(_v, 0) # fill per grouped dimension v.load() - v_group = copy.deepcopy(v).groupby(dim) + v_group = copy.deepcopy(v).groupby(dim, squeeze=False) return v_group.map(log_fit) @@ -642,16 +645,16 @@ def log_interp(_v): # scale with log depth c = xr.DataArray(_v / np.log(np.maximum(dist_wall, d_0) / d_0)) # fill dry points with the nearest valid value for c - c[dist_wall == 0] = c.interpolate_na(dim="points", method="nearest", fill_value="extrapolate")[dist_wall == 0] + c[0, np.where(dist_wall == 0)[0]] = c.interpolate_na(dim="points", method="nearest", fill_value="extrapolate")[0, np.where(dist_wall == 0)[0]] # interpolate with linear interpolation c = c.interpolate_na(dim="points") # use filled c to interpret missing v - _v[np.isnan(_v)] = (np.log(np.maximum(dist_wall, d_0) / d_0) * c)[np.isnan(_v)] + _v[0, np.isnan(_v[0])] = (np.log(np.maximum(dist_wall, d_0) / d_0) * c)[0, np.where(np.isnan(_v[0]))[0]] # (np.log(np.maximum(dist_wall, d_0) / d_0) * c)[np.isnan(_v)] return _v # fill per grouped dimension v.load() - v_group = copy.deepcopy(v).groupby(dim) + v_group = copy.deepcopy(v).groupby(dim, squeeze=False) return v_group.map(log_interp) diff --git a/pyorc/project.py b/pyorc/project.py index 18cd38a..e5c08d8 100644 --- a/pyorc/project.py +++ b/pyorc/project.py @@ -194,7 +194,10 @@ def project_numpy( ) # coerce 2D idxs to 1D idxs idx_back = np.array(points_cam[idx_in, 1]) * len(da.x) + np.array(points_cam[idx_in, 0]) - vals = da.stack(points=("y", "x")).isel(points=idx_back) + vals = da.stack(group=("y", "x")).isel(group=idx_back) + # overwrite the values group coordinates + vals = vals.drop_vars(['group', 'y', 'x']) + vals["group"] = da_new.group[idx_in] da_new[..., idx_in] = vals if reducer != "nearest": @@ -257,7 +260,7 @@ def project_numpy( classes = np.unique(da_idx) # group unique values and reduce with average da_point = xarray_reduce( - da_point.drop(["y", "x"]), + da_point.drop_vars(["y", "x", "points"]), da_idx,func=reducer, expected_groups=classes, engine="numba" diff --git a/tests/test_cameraconfig.py b/tests/test_cameraconfig.py index 580fdf6..3caab54 100644 --- a/tests/test_cameraconfig.py +++ b/tests/test_cameraconfig.py @@ -93,11 +93,15 @@ def test_get_M(cam_config, h_a, to_bbox_grid, M_expected): @pytest.mark.parametrize( "_cam_config, _corners, _bbox", [ - (pytest.lazy_fixture("cam_config_6gcps"), pytest.lazy_fixture("corners_6gcps"), pytest.lazy_fixture("bbox_6gcps")), - (pytest.lazy_fixture("cam_config"), pytest.lazy_fixture("corners"), pytest.lazy_fixture("bbox")) + ("cam_config_6gcps", "corners_6gcps", "bbox_6gcps"), + ("cam_config", "corners", "bbox") ] ) -def test_set_bbox_from_corners(_cam_config, _corners, _bbox): +def test_set_bbox_from_corners(_cam_config, _corners, _bbox, request): + _cam_config = request.getfixturevalue(_cam_config) + _corners = request.getfixturevalue(_corners) + _bbox = request.getfixturevalue(_bbox) + # check if this works _cam_config.set_bbox_from_corners(_corners) assert(np.allclose(_cam_config.bbox.bounds, _bbox.bounds)) @@ -181,11 +185,12 @@ def test_cv_undistort_points(cam_config): @pytest.mark.parametrize( "cur_cam_config", [ - pytest.lazy_fixture("cam_config_6gcps"), - pytest.lazy_fixture("cam_config"), + "cam_config_6gcps", + "cam_config", ] ) -def test_unproject_points(cur_cam_config): +def test_unproject_points(cur_cam_config, request): + cur_cam_config = request.getfixturevalue(cur_cam_config) src = cur_cam_config.gcps["src"] dst = cur_cam_config.gcps_dest # project x, y, z point to camera objective diff --git a/tests/test_cli.py b/tests/test_cli.py index 3887db9..c4f8cf0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -54,6 +54,7 @@ def test_cli_cam_config_video(cli_obj, vid_file, gcps_src, gcps_dst, lens_positi ) # assert result.exit_code == 0 + def test_cli_velocimetry(cli_obj, vid_file, cam_config_fn, cli_recipe_fn, cli_output_dir): # ensure we are in the right folder print(f"current file is: {os.path.dirname(__file__)}") @@ -88,11 +89,12 @@ def test_cli_velocimetry(cli_obj, vid_file, cam_config_fn, cli_recipe_fn, cli_ou @pytest.mark.parametrize( "recipe_", [ - pytest.lazy_fixture("recipe"), - pytest.lazy_fixture("recipe_geojson"), + "recipe", + "recipe_geojson", ] ) -def test_service_video(recipe_, vid_file, cam_config, cli_prefix, cli_output_dir): +def test_service_video(recipe_, vid_file, cam_config, cli_prefix, cli_output_dir, request): + recipe_ = request.getfixturevalue(recipe_) # ensure we are in the right folder print(f"current file is: {os.path.dirname(__file__)}") os.chdir(os.path.dirname(__file__)) diff --git a/tests/test_frames.py b/tests/test_frames.py index 8735771..3ccef05 100644 --- a/tests/test_frames.py +++ b/tests/test_frames.py @@ -5,19 +5,20 @@ @pytest.mark.parametrize( "frames, resolution, method, dims, shape, kwargs", [ - (pytest.lazy_fixture("frames_grayscale"), 0.1, "numpy", 3, (79, 88), {}), - (pytest.lazy_fixture("frames_grayscale"), 0.1, "numpy", 3, (79, 88), {"reducer": "mean"}), - (pytest.lazy_fixture("frames_rgb"), 0.1, "numpy", 4, (79, 88, 3), {"reducer": "mean"}), - (pytest.lazy_fixture("frames_rgb"), 0.1, "numpy", 4, (79, 88, 3), {}), - (pytest.lazy_fixture("frames_grayscale"), 0.1, "cv", 3, (79, 88), {}), - (pytest.lazy_fixture("frames_grayscale"), 0.01, "cv", 3, (786, 878), {}), - (pytest.lazy_fixture("frames_grayscale"), 0.05, "cv", 3, (157, 176), {}), - (pytest.lazy_fixture("frames_rgb"), 0.1, "cv", 4, (79, 88, 3), {}), + ("frames_grayscale", 0.1, "numpy", 3, (79, 88), {}), + ("frames_grayscale", 0.1, "numpy", 3, (79, 88), {"reducer": "mean"}), + ("frames_rgb", 0.1, "numpy", 4, (79, 88, 3), {"reducer": "mean"}), + ("frames_rgb", 0.1, "numpy", 4, (79, 88, 3), {}), + ("frames_grayscale", 0.1, "cv", 3, (79, 88), {}), + ("frames_grayscale", 0.01, "cv", 3, (786, 878), {}), + ("frames_grayscale", 0.05, "cv", 3, (157, 176), {}), + ("frames_rgb", 0.1, "cv", 4, (79, 88, 3), {}), ] ) -def test_project(frames, resolution, method, dims, shape, kwargs): +def test_project(frames, resolution, method, dims, shape, kwargs, request): # import matplotlib # matplotlib.use('Qt5Agg') + frames = request.getfixturevalue(frames) frames_proj = frames.frames.project(resolution=resolution, method=method, **kwargs) # check amount of time steps is equal assert(len(frames_proj.time) == len(frames.time)) @@ -25,7 +26,6 @@ def test_project(frames, resolution, method, dims, shape, kwargs): assert(len(frames_proj.dims) == dims), f"Expected nr of dims is {dims}, but {len(frames_proj.dims)} found" # check shape of x, y grids assert(frames_proj.isel(time=0).shape == shape), f"Projected frames shape {frames_proj.isel(time=0).shape} do not have expected shape {shape}" - # import matplotlib.pyplot as plt # plt.imshow(frames_proj[0]) # plt.colorbar() @@ -35,12 +35,13 @@ def test_project(frames, resolution, method, dims, shape, kwargs): @pytest.mark.parametrize( "frames, samples", [ - (pytest.lazy_fixture("frames_grayscale"), 2), - (pytest.lazy_fixture("frames_grayscale_shift"), 2), - (pytest.lazy_fixture("frames_proj"), 2), + ("frames_grayscale", 2), + ("frames_grayscale_shift", 2), + ("frames_proj", 2), ] ) -def test_normalize(frames, samples): +def test_normalize(frames, samples, request): + frames = request.getfixturevalue(frames) frames_norm = frames.frames.normalize(samples=samples) assert(frames_norm[0, 0, 0].values.dtype == "uint8"), f'dtype of result is {frames_norm[0, 0, 0].values.dtype}, expected "uint8"' @@ -52,7 +53,6 @@ def test_edge_detect(frames_proj): assert(np.allclose(frames_edge.values.flatten()[-4:], [-1.3828125, -4.3359375, 1.71875 , 7.234375 ])) - def test_reduce_rolling(frames_grayscale, samples=1): frames_reduced = frames_grayscale.frames.reduce_rolling(samples=samples) assert(frames_reduced.shape == frames_grayscale.shape) @@ -61,15 +61,16 @@ def test_reduce_rolling(frames_grayscale, samples=1): @pytest.mark.parametrize( "frames", [ - pytest.lazy_fixture("frames_grayscale"), - pytest.lazy_fixture("frames_rgb"), + "frames_grayscale", + "frames_rgb", ] ) @pytest.mark.parametrize( "idx", [0, -1] ) -def test_plot(frames, idx): +def test_plot(frames, idx, request): + frames = request.getfixturevalue(frames) frames[idx].frames.plot() frames[idx].frames.plot(mode="camera") plt.close("all") @@ -110,23 +111,25 @@ def test_get_piv(frames_proj, window_size, result): @pytest.mark.parametrize( "frames", [ - pytest.lazy_fixture("frames_grayscale"), - pytest.lazy_fixture("frames_rgb"), - pytest.lazy_fixture("frames_proj"), + "frames_grayscale", + "frames_rgb", + "frames_proj", ] ) -def test_to_ani(frames, ani_mp4): +def test_to_ani(frames, ani_mp4, request): + frames = request.getfixturevalue(frames) frames.frames.to_ani(ani_mp4, progress_bar=False) @pytest.mark.parametrize( "frames", [ - pytest.lazy_fixture("frames_grayscale"), - pytest.lazy_fixture("frames_rgb_stabilize"), - pytest.lazy_fixture("frames_proj"), + "frames_grayscale", + "frames_rgb_stabilize", + "frames_proj", ] ) -def test_to_video(frames, ani_mp4): +def test_to_video(frames, ani_mp4, request): + frames = request.getfixturevalue(frames) # only store the first 3 frames frames[0:3].frames.to_video(ani_mp4) diff --git a/tests/test_mask.py b/tests/test_mask.py index c401de6..3423d0e 100644 --- a/tests/test_mask.py +++ b/tests/test_mask.py @@ -39,7 +39,6 @@ def test_mask_window_nan(piv): # check if the method runs # first do filter that creates some missings piv.velocimetry.mask.minmax(s_max=0.6, inplace=True) - # piv = piv.isel(time=1) piv.velocimetry.mask.window_nan(inplace=True) diff --git a/tests/test_video.py b/tests/test_video.py index af33e09..96c40a4 100644 --- a/tests/test_video.py +++ b/tests/test_video.py @@ -30,13 +30,14 @@ def test_fps(vid): @pytest.mark.parametrize( "video, method, result", [ - (pytest.lazy_fixture("vid_cam_config"), "grayscale", [85, 71, 65, 80]), - (pytest.lazy_fixture("vid_cam_config_stabilize"), "grayscale", [5, 88, 78, 73]), - (pytest.lazy_fixture("vid_cam_config"), "rgb", [84, 91, 57, 70]), - (pytest.lazy_fixture("vid_cam_config"), "hsv", [36, 95, 91, 36]) + ("vid_cam_config", "grayscale", [85, 71, 65, 80]), + ("vid_cam_config_stabilize", "grayscale", [5, 88, 78, 73]), + ("vid_cam_config", "rgb", [84, 91, 57, 70]), + ("vid_cam_config", "hsv", [36, 95, 91, 36]) ] ) -def test_get_frame(video, method, result): +def test_get_frame(video, method, result, request): + video = request.getfixturevalue(video) frame = video.get_frame(1, method=method) assert(np.allclose(frame.flatten()[0:4], result)) @@ -44,13 +45,14 @@ def test_get_frame(video, method, result): @pytest.mark.parametrize( "video, method", [ - (pytest.lazy_fixture("vid_cam_config_nonlazy"), None), - (pytest.lazy_fixture("vid_cam_config"), "grayscale"), - (pytest.lazy_fixture("vid_cam_config"), "rgb"), - (pytest.lazy_fixture("vid_cam_config"), "hsv") + ("vid_cam_config_nonlazy", None), + ("vid_cam_config", "grayscale"), + ("vid_cam_config", "rgb"), + ("vid_cam_config", "hsv") ] ) -def test_get_frames(video, method): +def test_get_frames(video, method, request): + video = request.getfixturevalue(video) # check if the right amount of frames is extracted if method: frames = video.get_frames(method=method)