diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 92cd09d..6bdb62b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,7 +14,7 @@ on: - tests/* - pyorc/* - pyproject.toml -jobs: +jobs: Test-matrix: name: ${{ matrix.os }} - py${{ matrix.python-version }} runs-on: ${{ matrix.os }} @@ -26,7 +26,7 @@ jobs: matrix: os: ["ubuntu-latest" ] #, "macos-latest", "windows-latest"] python-version: ["3.10"] # fix tests to support older versions - + steps: - uses: actions/checkout@v4 @@ -43,7 +43,7 @@ jobs: - name: OpenCV dependencies run: | sudo apt update - sudo apt install libegl1 libopengl0 -y + sudo apt install libegl1 libopengl0 ffmpeg -y # build environment with pip - name: Install pyorc @@ -54,5 +54,8 @@ jobs: # run all tests - name: Test run: python -m pytest --verbose --cov=pyorc --cov-report xml - - - uses: codecov/codecov-action@v4 + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..73a0927 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,12 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 # Use the latest version + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-added-large-files + + - repo: https://github.com/kynan/nbstripout + rev: 0.7.1 # Use the latest version + hooks: + - id: nbstripout diff --git a/CHANGELOG.md b/CHANGELOG.md index e81cb9a..9337453 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,29 @@ +## [0.6.0] - 2024-09-20 +### Added +A logo with modifications in trademark guidelines in TRADEMARK.md and README.md. +Logo is also shown in the online documentation on https://localdevices.github.io/pyorc +### Changed +`Frames.project` with `method="numpy"` is improved so that it also works well in heavily undersampled areas. +`Video` instances defaulting with `lazy=False`. This in most cases increases the speed of video treatment significantly. +For large videos with large memory requirements, videos can be opened with `lazy=True`. +### Deprecated +### Removed +### Fixed +The legacy `setup.py` has been replaced by a `pyproject.toml` using flit installer. +### Security + + +## [0.5.6] - 2024-06-28 +### Added +### Changed +`Frames.project` with `method="numpy"` is improved so that it also works well in heavily undersampled areas. +### Deprecated +`Video` instances will default with `lazy=False` in v0.6.0. A warning message will appear for now +### Removed +### Fixed +### Security + + ## [0.5.5] - 2024-05-15 ### Added ### Changed @@ -23,7 +49,7 @@ Projection with `method="numpy"` sometimes results in missing values in the resu ## [0.5.3] - 2023-11-10 ### Added `frames.project` now has a `method` option which allows for choosing projection using opencv-methods (`method="cv"`) -which is currently still the default, or numpy-like operations (`method="numpy"`). `method="numpy"` is new and we +which is currently still the default, or numpy-like operations (`method="numpy"`). `method="numpy"` is new and we may change the default behaviour to this option in a later release as it is much more robust in cases with a lot of lens distortion and where part of the area of interest is outside of the field of view. @@ -32,8 +58,8 @@ Video rotation can be defined in the camera configuration, on the CLI with `--ro as measured in degrees, or for individual single videos as an additional input argument to `pyorc.Video` e.g. `pyorc.Video(..., rotation=90)`. ### Changed -Some default values for estimating the intrinsic lens parameters from control points are changed. We now estimate the -first two barrel distortion coefficients if enough information for constraining them is available. +Some default values for estimating the intrinsic lens parameters from control points are changed. We now estimate the +first two barrel distortion coefficients if enough information for constraining them is available. ### Deprecated ### Removed ### Fixed @@ -54,7 +80,7 @@ pyopenrivercam for scalable computation across a cloud. API / CLI users do not n ### Removed ### Fixed Notebook 02 in the examples folder contained a deprecation error with the stabilize option for opening videos. This -has been corrected and functionality description improved. +has been corrected and functionality description improved. ### Security ## [0.5.1] - 2023-06-27 @@ -63,7 +89,7 @@ has been corrected and functionality description improved. ### Deprecated ### Removed ### Fixed -- removed the strict cartopy dependency. This enables pip installation for users that are not interested in +- removed the strict cartopy dependency. This enables pip installation for users that are not interested in geographical plotting. Enables also installation on raspi platforms (only 64-bit!) - Transects sometimes gave infinite discharge when areas with zero depth received a small velocity. This has now been resolved. @@ -72,12 +98,12 @@ has been corrected and functionality description improved. ## [0.5.0] - 2023-05-24 ### Added -- make it a lot easier to get well-calibrated ground control and lens parameters at the same time. we now do this - by optimizing the lens' focal length and (if enough ground control is provided) barrel distortion whilst fitting +- make it a lot easier to get well-calibrated ground control and lens parameters at the same time. we now do this + by optimizing the lens' focal length and (if enough ground control is provided) barrel distortion whilst fitting the perspective to the user-provided ground control points. -- provide the fitted ground control points in the interface so that the user can immediately see if the ground control +- provide the fitted ground control points in the interface so that the user can immediately see if the ground control points are well fitted or if anything seems to be wrong with one or more control points. -- feature stabilization on command line which consequently provided user-interfacing to select non-moving areas by +- feature stabilization on command line which consequently provided user-interfacing to select non-moving areas by pointing and clicking. ### Changed - Much-improved stabilization for non-stable videos @@ -93,12 +119,12 @@ has been corrected and functionality description improved. ## [0.4.0] - 2023-03-10 ### Added The most notable change is that the code now includes an automatically installed command-line interface. This -will facilitate a much easier use by a large user group. Also the documentation is fully updated to include all +will facilitate a much easier use by a large user group. Also the documentation is fully updated to include all functionalities for both command-line users and API users. In detail we have the following additions: - First release of a command-line interface for the entire process of camera configuration, processing and preparing outputs and figures. - Service layer that makes it easy for developers to connect pyorc to apps such as GUIs or dashboards. -- Full user guide with description of both the command-line interface and API. +- Full user guide with description of both the command-line interface and API. ### Changed - Small modifications and additions in the API to accomodate the command-line interface building. ### Deprecated @@ -147,7 +173,7 @@ functionalities for both command-line users and API users. In detail we have the - Improved pytest code coverage ### Changed -- several API modifications to accommodate lens calibration and 6-point orthorectification +- several API modifications to accommodate lens calibration and 6-point orthorectification - CameraConfig format changed - CameraConfig.lens_parameters no longer used (replaced by camera_matrix and dist_coeffs) - CameraConfig.gcps extended orthorectification with 6(+)-point x, y, z perspective option @@ -178,7 +204,7 @@ functionalities for both command-line users and API users. In detail we have the ## [0.2.3] - 2022-08-10 ### Added ### Changed -- pyorc.transect.get_q added method="log_interp" using a log-depth normalized velocity and linear interpolation +- pyorc.transect.get_q added method="log_interp" using a log-depth normalized velocity and linear interpolation ### Deprecated ### Removed diff --git a/docs/conf.py b/docs/conf.py index d131dd1..3629124 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -49,7 +49,7 @@ def remove_dir_content(path: str) -> None: # -- Project information ----------------------------------------------------- project = 'pyorc' -copyright = '2023, Rainbow Sensing' +copyright = '2024, Rainbow Sensing' author = 'Hessel Winsemius' # The full version, including alpha/beta/rc tags diff --git a/envs/pyorc-test.yml b/envs/pyorc-test.yml index 388f3c0..3c39bf0 100644 --- a/envs/pyorc-test.yml +++ b/envs/pyorc-test.yml @@ -21,7 +21,7 @@ dependencies: - numba - opencv - pip - - pytest<8.0.0 # due to temporary problem with lazy_fixtures, see https://github.com/pytest-dev/pytest/issues/11890 + - pytest - pytest-cov - pytest-benchmark - pytest-lazy-fixture @@ -38,4 +38,3 @@ dependencies: - yaml - pip: - openpiv - diff --git a/examples/02_Process_velocimetry.ipynb b/examples/02_Process_velocimetry.ipynb index 02a313a..7d2ddc0 100644 --- a/examples/02_Process_velocimetry.ipynb +++ b/examples/02_Process_velocimetry.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "c5479fb0", + "id": "0", "metadata": {}, "source": [ "## Analyze surface velocities of a video with velocimetry\n", @@ -20,7 +20,7 @@ { "cell_type": "code", "execution_count": null, - "id": "0f24b24b", + "id": "1", "metadata": {}, "outputs": [], "source": [ @@ -36,7 +36,7 @@ }, { "cell_type": "markdown", - "id": "df9bc15b", + "id": "2", "metadata": {}, "source": [ "### load our camera configuration\n", @@ -50,10 +50,8 @@ { "cell_type": "code", "execution_count": null, - "id": "b6f78a4c", - "metadata": { - "scrolled": true - }, + "id": "3", + "metadata": {}, "outputs": [], "source": [ "cam_config = pyorc.load_camera_config(\"ngwerere/ngwerere.json\")\n", @@ -71,14 +69,14 @@ " start_frame=0,\n", " end_frame=125,\n", " stabilize=stabilize,\n", - " h_a=0.\n", + " h_a=0.,\n", ")\n", "video\n" ] }, { "cell_type": "markdown", - "id": "8dc8dbf7", + "id": "4", "metadata": {}, "source": [ "### Extract one frame with stabilization window\n", @@ -88,7 +86,7 @@ { "cell_type": "code", "execution_count": null, - "id": "89e0b441", + "id": "5", "metadata": {}, "outputs": [], "source": [ @@ -114,7 +112,7 @@ }, { "cell_type": "markdown", - "id": "b785ac1c", + "id": "6", "metadata": {}, "source": [ "### extracting gray scaled frames\n", @@ -124,7 +122,7 @@ { "cell_type": "code", "execution_count": null, - "id": "38da4a1a", + "id": "7", "metadata": {}, "outputs": [], "source": [ @@ -134,7 +132,7 @@ }, { "cell_type": "markdown", - "id": "e19c8d5d", + "id": "8", "metadata": {}, "source": [ "The frames object is really a `xarray.DataFrame` object, with some additional functionalities under the method `.frames`. The beauty of our API is that it also uses lazy dask arrays to prevent very lengthy runs that then result in gibberish because of a small mistake along the way. We can see the shape and datatype of the end result, without actually computing everything, until we request a sample. Let's have a look at only the first frame with the plotting functionalities. If you want to use the default plot functionalities of `xarray` simply replace the line below by:\n", @@ -146,16 +144,16 @@ { "cell_type": "code", "execution_count": null, - "id": "69675693", + "id": "9", "metadata": {}, "outputs": [], "source": [ - "da[0].frames.plot(cmap=\"gray\", ax=ax)\n" + "da[0].frames.plot(cmap=\"gray\")\n" ] }, { "cell_type": "markdown", - "id": "e20e74d7", + "id": "10", "metadata": {}, "source": [ "### normalize to add contrast\n", @@ -165,39 +163,40 @@ { "cell_type": "code", "execution_count": null, - "id": "2d60d734", - "metadata": { - "scrolled": true - }, + "id": "11", + "metadata": {}, "outputs": [], "source": [ + "# da_norm = da.frames.time_diff(abs=False, thres=0.)\n", + "# da_norm = da_norm.frames.minmax(min=0.)\n", "da_norm = da.frames.normalize()\n", - "da_norm[0].frames.plot(cmap=\"gray\")\n" + "p = da_norm[0].frames.plot(cmap=\"gray\")\n", + "plt.colorbar(p)\n" ] }, { "cell_type": "markdown", - "id": "266e1db0", + "id": "12", "metadata": {}, "source": [ - "A lot more contrast is visible now. We can now project the frames to an orthoprojected plane. The camera configuration, which is part of the `Video` object is used under the hood to do this." + "A lot more contrast is visible now. We can now project the frames to an orthoprojected plane. The camera configuration, which is part of the `Video` object is used under the hood to do this. We use the new numpy-based projection method. The default is to use OpenCV methods, remove `method=\"numpy\"` to try that." ] }, { "cell_type": "code", "execution_count": null, - "id": "d3f8813f", + "id": "13", "metadata": {}, "outputs": [], "source": [ "f = plt.figure(figsize=(16, 9))\n", - "da_norm_proj = da_norm.frames.project()\n", + "da_norm_proj = da_norm.frames.project(method=\"numpy\")\n", "da_norm_proj[0].frames.plot(cmap=\"gray\")\n" ] }, { "cell_type": "markdown", - "id": "590f401a", + "id": "14", "metadata": {}, "source": [ "You can see that the frames now also have x and y coordinates. These are in fact geographically aware, because we measured control points in real world coordinates and added a coordinate reference system to the `CameraConfig` object (see notebook 01). The `DataArray` therefore also contains coordinate grids for `lon` and `lat` for longitudes and latitudes. Hence we can also go through this entire pipeline with an RGB image and plot this in the real world by adding `mode=\"geographical\"` to the plotting functionalities. The grid is rotated so that its orientation always can follow the stream (in the local projection shown above, left is upstream, right downstream). \n", @@ -207,7 +206,7 @@ { "cell_type": "code", "execution_count": null, - "id": "efebc75f", + "id": "15", "metadata": {}, "outputs": [], "source": [ @@ -235,12 +234,12 @@ }, { "cell_type": "markdown", - "id": "8d832fc0", + "id": "16", "metadata": {}, "source": [ "### Velocimetry estimates\n", "Now that we have real-world projected frames, with contrast enhanced, let's do some velocimetry! For Particle Image Velocimetry, this is as simple as calling the `.get_piv` method on the frames. Again a lazy result is returned really fast. If you want to do the computations, you can either extract a single frame, or (as below) store the result in a nice NetCDF file. Note that this file can be loaded back into memory with the `xarray` API without any additional fuss. We use a delayed method for storing, just to see a progress bar. If you are not interested in that, you can also replace the last 3 lines by:\n", - "```\n", + "```python\n", "piv.to_netcdf(\"ngwerere_piv.nc\")\n", "```\n" ] @@ -248,19 +247,24 @@ { "cell_type": "code", "execution_count": null, - "id": "449237d2", + "id": "17", "metadata": {}, "outputs": [], "source": [ + "import time\n", + "t1 = time.time()\n", + "da_norm_proj = da_norm_proj.load()\n", "piv = da_norm_proj.frames.get_piv()\n", "delayed_obj = piv.to_netcdf(\"ngwerere_piv.nc\", compute=False)\n", "with ProgressBar():\n", - " results = delayed_obj.compute()\n" + " results = delayed_obj.compute()\n", + "t2 = time.time()\n", + "print(f\"write v took {t2-t1} secs.\")" ] }, { "cell_type": "markdown", - "id": "a2390213", + "id": "18", "metadata": {}, "source": [ "### Beautiful additions to your art gallery\n", @@ -284,7 +288,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.12.5" } }, "nbformat": 4, diff --git a/examples/03_Plotting_and_masking_velocimetry_results.ipynb b/examples/03_Plotting_and_masking_velocimetry_results.ipynb index 72b3204..ec75a2e 100644 --- a/examples/03_Plotting_and_masking_velocimetry_results.ipynb +++ b/examples/03_Plotting_and_masking_velocimetry_results.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "c4df4e0f", + "id": "0", "metadata": {}, "source": [ "## Immersive plotting and analyzing results\n", @@ -13,7 +13,7 @@ { "cell_type": "code", "execution_count": null, - "id": "7b9e90b1", + "id": "1", "metadata": {}, "outputs": [], "source": [ @@ -24,7 +24,7 @@ }, { "cell_type": "markdown", - "id": "2880e563", + "id": "2", "metadata": {}, "source": [ "You have a result stored in a NetCDF file after running notebook 02. Now you want to see if the results seem to make sense, and further analyze these. Especially post-processing of your results with masking invalid velocities is an essential step in most velocimetry analyses, given that sometimes hardly any tracers are visible, and only spuriously correlated results were found. With some intelligence these suspicious velocities can be removed, and `pyorc` has many methods available to do that. Most of these can be applied without any parameter changes for a good result. The order in which mask methods are applied can matter, which we will describe later in this notebook.\n", @@ -35,19 +35,16 @@ { "cell_type": "code", "execution_count": null, - "id": "0b0789d2", - "metadata": { - "scrolled": false - }, + "id": "3", + "metadata": {}, "outputs": [], "source": [ - "ds = xr.open_dataset(\"ngwerere/ngwerere_piv.nc\")\n", - "ds\n" + "ds = xr.open_dataset(\"ngwerere/ngwerere_piv.nc\")\n" ] }, { "cell_type": "markdown", - "id": "e9fd99b1", + "id": "4", "metadata": {}, "source": [ "As you can see, we have lots of coordinate variables at our disposal, these can be used in turn to plot our data in a local projection, with our bounding box top-left corner at the top-left. The `x` and `y` axes hold the local coordinates. We can also use the UTM35S coordinates, stored in `xs` and `ys`, the longitude and latitude coordinates stored in `lon` and `lat`, or....(very cool) the original row and column coordinate of the camera's objective. This allows us to plot the results as an augmented reality view.\n", @@ -61,7 +58,7 @@ }, { "cell_type": "markdown", - "id": "f8466af3", + "id": "5", "metadata": {}, "source": [ "### Plotting in local projection\n", @@ -73,7 +70,7 @@ { "cell_type": "code", "execution_count": null, - "id": "e9d6ef23", + "id": "6", "metadata": {}, "outputs": [], "source": [ @@ -116,7 +113,7 @@ }, { "cell_type": "markdown", - "id": "6050df61", + "id": "7", "metadata": {}, "source": [ "### Masking of results\n", @@ -135,10 +132,8 @@ { "cell_type": "code", "execution_count": null, - "id": "26f27309", - "metadata": { - "scrolled": false - }, + "id": "8", + "metadata": {}, "outputs": [], "source": [ "import copy\n", @@ -173,7 +168,7 @@ }, { "cell_type": "markdown", - "id": "4d3f31ca", + "id": "9", "metadata": {}, "source": [ "Interesting! We see that the velocities become a lot higher on average. Most likely because many spurious velocities are removed. We also see that the velocities seem to be more left-to-right oriented. In part this may be because we applied the `.mask.angle` method. This mask removes velocities that are in a direction, far off from the expected flow direction. The default direction of this mask is always left to right. Therefore take good care of this mask, if you decide to e.g. apply `pyorc` in a bottom to top direction.\n", @@ -186,7 +181,7 @@ { "cell_type": "code", "execution_count": null, - "id": "2e713b75", + "id": "10", "metadata": {}, "outputs": [], "source": [ @@ -225,7 +220,7 @@ }, { "cell_type": "markdown", - "id": "c0192f4d", + "id": "11", "metadata": {}, "source": [ "It looks more natural. Check for instance the pattern around the rock on the left side. Now we can also plot in a geographical view." @@ -234,7 +229,7 @@ { "cell_type": "code", "execution_count": null, - "id": "97952713", + "id": "12", "metadata": {}, "outputs": [], "source": [ @@ -270,7 +265,7 @@ }, { "cell_type": "markdown", - "id": "c43317e4", + "id": "13", "metadata": {}, "source": [ "### Immersive and intuitive augmented reality\n", @@ -280,7 +275,7 @@ { "cell_type": "code", "execution_count": null, - "id": "9828c26a", + "id": "14", "metadata": {}, "outputs": [], "source": [ @@ -289,7 +284,7 @@ "\n", "#...and then masked velocimetry again, but also camera. This gives us an augmented reality view. The quiver scale \n", "# needs to be adapted to fit in the screen properly\n", - "ds_mean_mask2.velocimetry.plot(\n", + "ds_mean_mask.velocimetry.plot(\n", " ax=p.axes,\n", " mode=\"camera\",\n", " alpha=0.4,\n", @@ -303,7 +298,7 @@ }, { "cell_type": "markdown", - "id": "096fd506", + "id": "15", "metadata": {}, "source": [ "### Store the final masked results\n", @@ -313,21 +308,13 @@ { "cell_type": "code", "execution_count": null, - "id": "993aab62", + "id": "16", "metadata": {}, "outputs": [], "source": [ "ds_mask2.velocimetry.set_encoding()\n", "ds_mask2.to_netcdf(\"ngwerere_masked.nc\")" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8071e5e6", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/examples/04_Extracting_crosssection_velocities_and_discharge.ipynb b/examples/04_Extracting_crosssection_velocities_and_discharge.ipynb index 254d89c..e55d65a 100644 --- a/examples/04_Extracting_crosssection_velocities_and_discharge.ipynb +++ b/examples/04_Extracting_crosssection_velocities_and_discharge.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "caf3068a", + "id": "0", "metadata": {}, "source": [ "## Obtain a discharge measurement over a cross section\n", @@ -13,7 +13,7 @@ { "cell_type": "code", "execution_count": null, - "id": "8066d219", + "id": "1", "metadata": {}, "outputs": [], "source": [ @@ -27,7 +27,7 @@ }, { "cell_type": "markdown", - "id": "76766bd5", + "id": "2", "metadata": {}, "source": [ "First we read the masked results from notebook 03 back into memory. We also load the first frame of our original video in rgb format.\n" @@ -36,10 +36,8 @@ { "cell_type": "code", "execution_count": null, - "id": "231dce0d", - "metadata": { - "scrolled": true - }, + "id": "3", + "metadata": {}, "outputs": [], "source": [ "ds = xr.open_dataset(\"ngwerere/ngwerere_masked.nc\")\n", @@ -57,7 +55,7 @@ }, { "cell_type": "markdown", - "id": "ea25d0a9", + "id": "4", "metadata": {}, "source": [ "We need a cross section. We have stored two cross sections in comma-separated text files (.csv) along with the examples. Below we load these in memory and we extract the x, y and z values from it. These are all measured in the same vertical reference as the water level. Let's first investigate if the cross sections are correctly referenced, by plotting them in the camera configuration." @@ -66,7 +64,7 @@ { "cell_type": "code", "execution_count": null, - "id": "7def8776", + "id": "5", "metadata": {}, "outputs": [], "source": [ @@ -92,7 +90,7 @@ }, { "cell_type": "markdown", - "id": "1c05b88e", + "id": "6", "metadata": {}, "source": [ "The cross sections are clearly in the area of interest, this means we can sample velocities from them.\n", @@ -102,7 +100,7 @@ { "cell_type": "code", "execution_count": null, - "id": "2beb38e0", + "id": "7", "metadata": {}, "outputs": [], "source": [ @@ -113,7 +111,7 @@ }, { "cell_type": "markdown", - "id": "993fced3", + "id": "8", "metadata": {}, "source": [ "You can see that all coordinates and variables now only have a `quantile` and `points` dimension. The `point` dimension represents all bathymetry points. By default, quantiles [0.05, 0.25, 0.5, 0.75, 0.95] are derived, but this can also be modified with the `quantile` parameter. During the sampling, the bathymetry points are resampled to more or less match the resolution of the velocimetry results, in order to get a dense enough sampling. You can also impose a resampling distance using the `distance` parameter. If you would set this to 0.1, then a velocity will be sampled each 0.1 meters. Because our velocimetry grid has a 0.13 m resolution, the distance will by default be 0.13. The variables `v_eff_nofill` and `v_dir` are added, which is the effective velocity scalar, and its angle, perpendicular to the cross-section direction. `v_eff_nofill` at this stage may contain gaps because of unknown velocities in moments and locations where the mask methods did not provide satisfactory samples.\n", @@ -124,7 +122,7 @@ { "cell_type": "code", "execution_count": null, - "id": "d813d989", + "id": "9", "metadata": {}, "outputs": [], "source": [ @@ -135,7 +133,7 @@ }, { "cell_type": "markdown", - "id": "27299cd9", + "id": "10", "metadata": {}, "source": [ "Now we have variables `v_eff` with velocities (filled with zeros where no data was found, but log profiles are also possible), and `q` and `q_nofill` which hold the depth integrated velocities. During depth integration the default assumption is that the depth average velocities is 0.9 times the surface velocity. This can be controlled by the `v_corr` parameter. Below, we make a quick plot of the `q` results for both profiles." @@ -144,7 +142,7 @@ { "cell_type": "code", "execution_count": null, - "id": "2183697c", + "id": "11", "metadata": {}, "outputs": [], "source": [ @@ -156,7 +154,7 @@ }, { "cell_type": "markdown", - "id": "777f16d0", + "id": "12", "metadata": {}, "source": [ "We can also plot the sampled surface velocities in combination with the velocity grid with bespoke plotting functions, giving intuitive graphics. We do this in the camera perspective below, similar to notebook 03." @@ -165,7 +163,7 @@ { "cell_type": "code", "execution_count": null, - "id": "9e7af514", + "id": "13", "metadata": {}, "outputs": [], "source": [ @@ -210,7 +208,7 @@ }, { "cell_type": "markdown", - "id": "b00fbbfc", + "id": "14", "metadata": {}, "source": [ "Of course we can also plot our results in a local projection and add a selected plotting style to it. Below this is shown with a streamplot as example." @@ -219,7 +217,7 @@ { "cell_type": "code", "execution_count": null, - "id": "479bb55e", + "id": "15", "metadata": {}, "outputs": [], "source": [ @@ -265,7 +263,7 @@ }, { "cell_type": "markdown", - "id": "ef5c8ad1", + "id": "16", "metadata": {}, "source": [ "Finally, we can extract discharge estimates from the cross section" @@ -274,7 +272,7 @@ { "cell_type": "code", "execution_count": null, - "id": "515f82b0", + "id": "17", "metadata": {}, "outputs": [], "source": [ @@ -286,7 +284,7 @@ }, { "cell_type": "markdown", - "id": "bd3ca3b9", + "id": "18", "metadata": {}, "source": [ "You can see that the different quantiles give very diverse values, but that even with this very shallow and difficult example, the flow estimates are quite close to each other. The bathymetry was recorded at only 10 cm accuracy and the site is far from uniform, and so not ideal for cross-sectional discharge observations. This can easily explain the differences between two sampled cross sections. \n", @@ -301,7 +299,7 @@ { "cell_type": "code", "execution_count": null, - "id": "47d61888", + "id": "19", "metadata": {}, "outputs": [], "source": [] diff --git a/pyorc/__init__.py b/pyorc/__init__.py index 4d69587..609a64b 100644 --- a/pyorc/__init__.py +++ b/pyorc/__init__.py @@ -1,9 +1,18 @@ -__version__ = "0.5.6" -from .api.cameraconfig import CameraConfig, load_camera_config, get_camera_config -from .api.video import Video -from .api.frames import Frames -from .api.velocimetry import Velocimetry -from .api.transect import Transect -from . import service -from . import cli -from .project import * \ No newline at end of file +"""pyorc: free and open-source image-based surface velocity and discharge.""" + +__version__ = "0.6.0" + +from .api import * +from .project import * + +__all__ = [ + "CameraConfig", + "load_camera_config", + "get_camera_config", + "Video", + "Frames", + "Velocimetry", + "Transect", + "service", + "cli" +] diff --git a/pyorc/api/__init__.py b/pyorc/api/__init__.py index e69de29..2fe3cc4 100644 --- a/pyorc/api/__init__.py +++ b/pyorc/api/__init__.py @@ -0,0 +1,15 @@ +from .cameraconfig import CameraConfig, load_camera_config, get_camera_config +from .video import Video +from .frames import Frames +from .velocimetry import Velocimetry +from .transect import Transect + +__all__ = [ + "CameraConfig", + "load_camera_config", + "get_camera_config", + "Video", + "Frames", + "Velocimetry", + "Transect" +] diff --git a/pyorc/api/cameraconfig.py b/pyorc/api/cameraconfig.py index d575f73..f9f7296 100644 --- a/pyorc/api/cameraconfig.py +++ b/pyorc/api/cameraconfig.py @@ -407,6 +407,15 @@ def set_lens_calibration( self.camera_matrix = camera_matrix self.dist_coeffs = dist_coeffs + def estimate_lens_position(self): + """estimate lens position from distortion and intrinsec/extrinsic matrix.""" + _, rvec, tvec = self.pnp + rmat = cv2.Rodrigues(rvec)[0] + # determine lens position related to center of objective + lens_pos_centroid = (np.array(-rmat).T @ tvec).flatten() + lens_pos = np.array(lens_pos_centroid) + self.gcps_mean + return lens_pos + def get_bbox( self, camera: Optional[bool] = False, @@ -654,8 +663,6 @@ def get_M( z_a = self.get_z_a(h_a) z_a -= self.gcps_mean[-1] # treating 3D homography - print(dst_a) - # print(z_a) return cv.get_M_3D( src=src, dst=dst_a, @@ -710,7 +717,6 @@ def set_bbox_from_corners( ) self.bbox = bbox - def set_intrinsic( self, camera_matrix: Optional[List[List]] = None, @@ -721,7 +727,6 @@ def set_intrinsic( self.set_lens_pars() # default parameters use width of frame if hasattr(self, "gcps"): if len(self.gcps["src"]) >= 4: - # if self.gcp_dims == 3: self.camera_matrix, self.dist_coeffs, err = cv.optimize_intrinsic( self.gcps["src"], self.gcps_dest, @@ -983,7 +988,7 @@ def plot( **tiles_kwargs additional keyword arguments to pass to ax.add_image when tiles are added 8) : - + Returns ------- @@ -1146,7 +1151,7 @@ def to_file( fn: str ): """Write the CameraConfig object to json structure - + Parameters ---------- fn : str @@ -1159,7 +1164,7 @@ def to_file( def to_json(self) -> str: """Convert CameraConfig object to string - + Returns ------- json_str : str @@ -1169,7 +1174,7 @@ def to_json(self) -> str: depr_warning_height_width = """ -Your camera configuration does not have a property "height" and/or "width", probably because your configuration file is +Your camera configuration does not have a property "height" and/or "width", probably because your configuration file is from an older < 0.3.0 version. Please rectify this by editing your .json config file. The top of your file should e.g. look as follows for a HD video: { diff --git a/pyorc/api/frames.py b/pyorc/api/frames.py index 3f7b52a..1ba5b94 100644 --- a/pyorc/api/frames.py +++ b/pyorc/api/frames.py @@ -8,10 +8,13 @@ from matplotlib.animation import FuncAnimation from tqdm import tqdm -import pyorc.project from .orcbase import ORCBase from .plot import _frames_plot -from .. import cv, helpers, const, piv_process +from .. import cv, helpers, const, piv_process, project + +__all__ = [ + "Frames" +] @xr.register_dataarray_accessor("frames") @@ -223,9 +226,9 @@ def project( ## PROJECTION PREPARATIONS # ======================== z = cc.get_z_a(self.h_a) - if not(hasattr(pyorc.project, f"project_{method}")): + if not(hasattr(project, f"project_{method}")): raise ValueError(f"Selected projection method {method} does not exist.") - proj_method = getattr(pyorc.project, f"project_{method}") + proj_method = getattr(project, f"project_{method}") da_proj = proj_method( self._obj, cc, diff --git a/pyorc/api/mask.py b/pyorc/api/mask.py index 18bc1f4..b5255a0 100644 --- a/pyorc/api/mask.py +++ b/pyorc/api/mask.py @@ -4,8 +4,8 @@ import warnings import xarray as xr -from pyorc.const import v_x, v_y, s2n, corr -from pyorc import helpers +from ..const import v_x, v_y, s2n, corr +from .. import helpers commondoc = """ @@ -14,7 +14,7 @@ mask : xr.DataArray mask applicable to input dataset with ``ds.velocimetry.filter(mask)``. If ``inplace=True``, the dataset will be returned masked with ``mask``. - + """ @@ -347,4 +347,3 @@ def window_replace(self, wdw=1, iter=1, **kwargs): ds_mean = ds_wdw.mean(dim="stride") ds = ds.fillna(ds_mean) return ds - diff --git a/pyorc/api/orcbase.py b/pyorc/api/orcbase.py index c06bbef..8dab809 100644 --- a/pyorc/api/orcbase.py +++ b/pyorc/api/orcbase.py @@ -1,15 +1,15 @@ import json import numpy as np import xarray as xr -from pyorc import helpers depr_warning = """ -The camera configuration of this pyorc output does not have a property "height" and/or "width", because it has been +The camera configuration of this pyorc output does not have a property "height" and/or "width", because it has been established with version < 0.3.0 version. Adding height and width property. This behaviour is deprecated. Please resave -your results with ``.to_netcdf()`` to make them compatible with later versions. +your results with ``.to_netcdf()`` to make them compatible with later versions. """ + class ORCBase(object): def __init__(self, xarray_obj): self._obj = xarray_obj @@ -128,4 +128,3 @@ def add_xy_coords( frames_coord[k].attrs = v.attrs # update the DataArray return frames_coord - diff --git a/pyorc/api/plot.py b/pyorc/api/plot.py index fbbeaa9..5d39309 100644 --- a/pyorc/api/plot.py +++ b/pyorc/api/plot.py @@ -6,7 +6,7 @@ from matplotlib.collections import QuadMesh import matplotlib.ticker as mticker -from pyorc import helpers, cv +from .. import helpers, cv def _base_plot(plot_func): @@ -26,13 +26,13 @@ def _base_plot(plot_func): If None (default), use the current axes. Not applicable when using facets. *args : additional arguments, passed to wrapped Matplotlib function. **kwargs : additional keyword arguments to wrapped Matplotlib function. - + Returns ------- artist : matplotlib mappable The same type of primitive artist that the wrapped Matplotlib function returns. - + """ # This function is largely based on xarray.Dataset function _dsplot # Build on the original docstring @@ -80,7 +80,7 @@ def get_plot_method( (Default value = {}) *args : additional arguments, passed to wrapped Matplotlib function. **kwargs : additional keyword arguments to wrapped Matplotlib function. - + Returns ------- p : matplotlib mappable @@ -193,7 +193,7 @@ def get_plot_method( def _frames_plot(ref, ax=None, mode="local", *args, **kwargs): """Creates QuadMesh plot from a RGB or grayscale frame on a new or existing (if ax is not None) axes - + Wraps :py:func:`matplotlib:matplotlib.collections.QuadMesh`. Parameters @@ -532,7 +532,7 @@ def get_uv_camera(self, dt=0.1): @_base_plot def quiver(_, x, y, u, v, s=None, ax=None, *args, **kwargs): """Creates quiver plot from velocimetry results on new or existing axes - + Wraps :py:func:`matplotlib:matplotlib.pyplot.quiver`. """ if "color" in kwargs: @@ -547,7 +547,7 @@ def quiver(_, x, y, u, v, s=None, ax=None, *args, **kwargs): @_base_plot def scatter(_, x, y, c=None, ax=None, *args, **kwargs): """Creates scatter plot of velocimetry or transect results on new or existing axes - + Wraps :py:func:`matplotlib:matplotlib.pyplot.scatter`. """ primitive = ax.scatter(x, y, c=c, *args, **kwargs) @@ -557,7 +557,7 @@ def scatter(_, x, y, c=None, ax=None, *args, **kwargs): @_base_plot def streamplot(_, x, y, u, v, s=None, ax=None, linewidth_scale=None, *args, **kwargs): """Creates streamplot of velocimetry results on new or existing axes - + Wraps :py:func:`matplotlib:matplotlib.pyplot.streamplot`. Additional input arguments: """ if linewidth_scale is not None: @@ -572,7 +572,7 @@ def streamplot(_, x, y, u, v, s=None, ax=None, linewidth_scale=None, *args, **kw @_base_plot def pcolormesh(_, x, y, s=None, ax=None, *args, **kwargs): """Creates pcolormesh plot from velocimetry results on new or existing axes - + Wraps :py:func:`matplotlib:matplotlib.pyplot.pcolormesh`. """ primitive = ax.pcolormesh(x, y, s, *args, **kwargs) @@ -593,7 +593,7 @@ def cbar(ax, p, size=12, **kwargs): kwargs : dict, additional settings passed to plt.colorbar **kwargs : - + Returns ------- diff --git a/pyorc/api/transect.py b/pyorc/api/transect.py index 625b0d8..ba31573 100644 --- a/pyorc/api/transect.py +++ b/pyorc/api/transect.py @@ -3,7 +3,7 @@ from xarray.core import utils -from pyorc import helpers +from .. import helpers from .plot import _Transect_PlotMethods from .orcbase import ORCBase @@ -68,7 +68,7 @@ def vector_to_scalar(self, v_x="v_x", v_y="v_y"): def get_xyz_perspective(self, M=None, xs=None, ys=None, mask_outside=True): """Get camera-perspective column, row coordinates from cross-section locations. - + Parameters ---------- M : np.ndarray, optional diff --git a/pyorc/api/video.py b/pyorc/api/video.py index 516af33..3b155af 100644 --- a/pyorc/api/video.py +++ b/pyorc/api/video.py @@ -43,7 +43,7 @@ def __init__( end_frame: Optional[int] = None, freq: Optional[int] = 1, stabilize: Optional[List[List]] = None, - lazy: bool = True, + lazy: bool = False, rotation: Optional[int] = None, fps: Optional[float] = None, ): diff --git a/pyorc/cli/cli_elements.py b/pyorc/cli/cli_elements.py index 99aa05e..67d5cfb 100644 --- a/pyorc/cli/cli_elements.py +++ b/pyorc/cli/cli_elements.py @@ -16,8 +16,9 @@ from matplotlib.widgets import Button from matplotlib.patches import Polygon from mpl_toolkits.axes_grid1 import Divider, Size -from pyorc import helpers -from pyorc.cli import cli_utils + +from .. import helpers +from . import cli_utils path_effects = [ patheffects.Stroke(linewidth=2, foreground="w"), @@ -479,7 +480,7 @@ def __init__(self, img, logger=logging): size=12, path_effects=path_effects ) - self.ax.legend() + # self.ax.legend() # add dst coords in the intended CRS self.required_clicks = 4 # minimum 4 points needed for a satisfactory ROI diff --git a/pyorc/cli/cli_utils.py b/pyorc/cli/cli_utils.py index 6d2dd37..be113bc 100644 --- a/pyorc/cli/cli_utils.py +++ b/pyorc/cli/cli_utils.py @@ -7,13 +7,13 @@ import matplotlib.pyplot as plt import numpy as np import os -import pyorc import yaml - -from pyorc import Video, helpers, CameraConfig, cv, load_camera_config -from pyorc.cli.cli_elements import GcpSelect, AoiSelect, StabilizeSelect from shapely.geometry import Point +import pyorc.api +from .. import Video, CameraConfig, cv, load_camera_config, helpers +from .cli_elements import GcpSelect, AoiSelect, StabilizeSelect + def get_corners_interactive( fn, @@ -85,7 +85,6 @@ def get_stabilize_pol( return selector.src - def get_file_hash(fn): hash256 = hashlib.sha256() with open(fn, "rb") as f: @@ -95,6 +94,7 @@ def get_file_hash(fn): hash256.update(byte_block) return hash256 + def get_gcps_optimized_fit(src, dst, height, width, c=2., lens_position=None): # optimize cam matrix and dist coeffs with provided control points if np.array(dst).shape == (4, 2): @@ -123,6 +123,7 @@ def get_gcps_optimized_fit(src, dst, height, width, c=2., lens_position=None): dst_est = np.array(dst_est)[:, 0:len(coord_mean)] + coord_mean return src_est, dst_est, camera_matrix, dist_coeffs, err + def parse_json(ctx, param, value): if value is None: return None @@ -138,6 +139,7 @@ def parse_json(ctx, param, value): raise ValueError(f'Could not decode JSON "{value}"') return kwargs + def parse_corners(ctx, param, value): if value is None: return None @@ -162,12 +164,14 @@ def validate_dir(ctx, param, value): os.makedirs(value) return value + def validate_rotation(ctx, param, value): if value is not None: if not(value in [90, 180, 270, None]): raise click.UsageError(f"Rotation value must be either 90, 180 or 270") return value + def parse_camconfig(ctx, param, camconfig_file): """ Read and validate cam config file @@ -268,6 +272,7 @@ def read_shape(fn=None, geojson=None): crs = gdf.crs.to_wkt() return coords, crs + def validate_dst(value): if value is not None: if len(value) in [2, 4]: @@ -282,6 +287,7 @@ def validate_dst(value): assert(len(val) == len_points), f"--dst value {n} must contain 3 coordinates (x, y, z) but consists of {len(val)} numbers, value is {val}" return value + def validate_recipe(recipe): valid_classes = ["video", "frames", "velocimetry", "mask", "transect", "plot"] # allowed classes required_classes = ["video", "frames", "velocimetry"] # mandatory classes (if not present, these are added) @@ -303,7 +309,7 @@ def validate_recipe(recipe): recipe[k][m] = {} if m not in process_methods and k in check_args: # get the subclass that is called within the section of interest - cls = getattr(pyorc, check_args[k].capitalize()) + cls = getattr(pyorc.api, check_args[k].capitalize()) if (not hasattr(cls, m)): raise ValueError(f"Class '{check_args[k].capitalize()}' does not have a method or property '{m}'") method = getattr(cls, m) @@ -325,4 +331,3 @@ def validate_recipe(recipe): # add empties for compulsory recipe components recipe[_c] = {} return recipe - diff --git a/pyorc/cli/log.py b/pyorc/cli/log.py index 8cb8907..cb61755 100644 --- a/pyorc/cli/log.py +++ b/pyorc/cli/log.py @@ -7,6 +7,7 @@ FMT = "%(asctime)s - %(name)s - %(module)s - %(levelname)s - %(message)s" # logger = logging.getLogger(__name__) + def setuplog( name: str = "pyorc", path: str = None, @@ -49,6 +50,7 @@ def setuplog( return logger + def add_filehandler(logger, path, log_level=20, fmt=FMT): """Add file handler to logger.""" if not os.path.isdir(os.path.dirname(path)): diff --git a/pyorc/cli/main.py b/pyorc/cli/main.py index 90d3789..48eec2b 100644 --- a/pyorc/cli/main.py +++ b/pyorc/cli/main.py @@ -7,11 +7,9 @@ import yaml # import CLI components -from pyorc.cli import cli_utils -from pyorc.cli import log # import pyorc api below -from pyorc import __version__ -import pyorc +from .. import service, __version__ +from . import log, cli_utils # import cli components below @@ -265,7 +263,7 @@ def camera_config( ) else: stabilize=None - pyorc.service.camera_config( + service.camera_config( video_file=videofile, cam_config_file=output, gcps=gcps, @@ -360,7 +358,7 @@ def velocimetry( ) logger.info(f"Preparing your velocimetry result in {output}") # load in recipe and camera config - pyorc.service.velocity_flow( + service.velocity_flow( recipe=recipe, videofile=videofile, cameraconfig=cameraconfig, @@ -371,18 +369,6 @@ def velocimetry( concurrency=not(lowmem), logger=logger ) - # processor = pyorc.service.VelocityFlowProcessor( - # recipe, - # videofile, - # cameraconfig, - # prefix, - # output, - # update=update, - # concurrency=not(lowmem), - # logger=logger - # ) - # # process video following the settings - # processor.process() pass if __name__ == "__main__": diff --git a/pyorc/cv.py b/pyorc/cv.py index d0856b7..2e31a18 100644 --- a/pyorc/cv.py +++ b/pyorc/cv.py @@ -3,7 +3,7 @@ import numpy as np import os import rasterio -from pyorc import helpers +from . import helpers from shapely.geometry import Polygon, LineString from shapely.affinity import rotate from tqdm import tqdm @@ -106,6 +106,7 @@ def get_ms_gftt(cap, start_frame=0, end_frame=None, n_pts=None, split=2, mask=No # Read first frame _, img_key = cap.read() + _, img_key = cap.read() # Convert frame to grayscale img1 = cv2.cvtColor(img_key, cv2.COLOR_BGR2GRAY) img_key = img1 @@ -395,9 +396,6 @@ def calibrate_camera( jpg = os.path.join(dir, "frame_{:06d}.png".format(int(f))) cv2.imwrite(jpg, imS) - # print(corners) - # skip 25 frames - # cap.set(cv2.CAP_PROP_POS_FRAMES, cur_f + df) if len(imgs) == max_imgs: print(f"Maximum required images {max_imgs} found") break @@ -572,6 +570,7 @@ def get_M_3D(src, dst, camera_matrix, dist_coeffs=np.zeros((1, 4)), z=0., revers success, rvec, tvec = solvepnp(dst, src, camera_matrix, dist_coeffs) return _Rt_to_M(rvec, tvec, camera_matrix, z=z, reverse=reverse) + def optimize_intrinsic(src, dst, height, width, c=2., lens_position=None): def error_intrinsic(x, src, dst, height, width, c=2., lens_position=None, dist_coeffs=DIST_COEFFS): """ @@ -599,8 +598,6 @@ def error_intrinsic(x, src, dst, height, width, c=2., lens_position=None, dist_c dist_xy = np.array(_dst)[:, 0:2] - np.array(dst_est)[:, 0:2] dist = (dist_xy ** 2).sum(axis=1) ** 0.5 gcp_err = dist.mean() - # print(f"Error: {gcp_err}") - # print(f"Parameters: {x}") if lens_position is not None: rmat = cv2.Rodrigues(rvec)[0] lens_pos2 = np.array(-rmat).T @ tvec @@ -634,8 +631,8 @@ def error_intrinsic(x, src, dst, height, width, c=2., lens_position=None, dist_c dist_coeffs[1][0] = opt.x[2] # dist_coeffs[4][0] = opt.x[3] # dist_coeffs[3][0] = opt.x[4] - print(f"CAMERA MATRIX: {camera_matrix}") - print(f"DIST COEFFS: {dist_coeffs}") + # print(f"CAMERA MATRIX: {camera_matrix}") + # print(f"DIST COEFFS: {dist_coeffs}") return camera_matrix, dist_coeffs, opt.fun @@ -936,4 +933,3 @@ def undistort_points(points, camera_matrix, dist_coeffs, reverse=False): P=camera_matrix ) return points_undistort[:, 0].tolist() - diff --git a/pyorc/helpers.py b/pyorc/helpers.py index 0388e6c..ec2f40b 100644 --- a/pyorc/helpers.py +++ b/pyorc/helpers.py @@ -227,7 +227,7 @@ def get_xs_ys(cols, rows, transform): def get_lons_lats(xs, ys, src_crs, dst_crs=CRS.from_epsg(4326)): """Computes raster of longitude and latitude coordinates (default) of a certain raster set of coordinates in a local coordinate reference system. User can supply an alternative coordinate reference system if projection other than - WGS84 Lat Lon is needed. + WGS84 Lat Lon is needed. Parameters ---------- @@ -794,6 +794,10 @@ def xyz_transform(points, crs_from, crs_to): y = points[:, 1] transform = Transformer.from_crs(crs_from, crs_to, always_xy=True) + # with only one point, transformer must not provide an array of points, to prevent a deprecation warning numpy>=1.25 + if len(points) == 1: + x = x[0] + y = y[0] # transform dst coordinates to local projection x_trans, y_trans = transform.transform(x, y) # check if finites are found, if not raise error @@ -802,7 +806,7 @@ def xyz_transform(points, crs_from, crs_to): np.all(np.isinf(x_trans)) ) ), "Transformation did not give valid results, please check if the provided crs of input coordinates is correct." - points[:, 0] = x_trans - points[:, 1] = y_trans + points[:, 0] = np.atleast_1d(x_trans) + points[:, 1] = np.atleast_1d(y_trans) return points.tolist() # return transform.transform(x, y) diff --git a/pyorc/project.py b/pyorc/project.py index e5c08d8..6d8ecca 100644 --- a/pyorc/project.py +++ b/pyorc/project.py @@ -1,20 +1,25 @@ +from __future__ import annotations + import cv2 import dask import numpy as np import xarray as xr + from rasterio.features import rasterize -from typing import Optional +from typing import Optional, Any from flox.xarray import xarray_reduce -import pyorc -from pyorc import helpers, cv + +from . import helpers, cv + +# from . import CameraConfig __all__ = ["project_numpy", "project_cv"] def project_cv( da: xr.DataArray, - cc: pyorc.CameraConfig, + cc: CameraConfig, x: np.ndarray, y: np.ndarray, z: np.ndarray @@ -107,7 +112,7 @@ def project_cv( def project_numpy( da: xr.DataArray, - cc: pyorc.CameraConfig, + cc: CameraConfig, x: np.ndarray, y: np.ndarray, z: np.ndarray, diff --git a/pyorc/service/camera_config.py b/pyorc/service/camera_config.py index 0c54090..c1bbcf2 100644 --- a/pyorc/service/camera_config.py +++ b/pyorc/service/camera_config.py @@ -1,7 +1,8 @@ import os.path -from pyorc import CameraConfig, Video +from .. import CameraConfig, Video import matplotlib.pyplot as plt + def camera_config( video_file, cam_config_file, @@ -20,6 +21,15 @@ def camera_config( Path to file with sample video containing objective of interest cam_config_file : str, Path to output file containing json camera config + lens_position : array-like + x, y, z position of lens in real-world coordinates + corners : list of lists + col, row corner points for definition of bounding box + frame_sample : int + frame number to use for sampling the control points and bounding box + rotation : Literal[90, 180, 270] + rotation in degrees to apply on frame + **kwargs: dict, Keyword arguments to pass to pyorc.CameraConfig (height and width are added on-the-fly from `video_file`) @@ -57,4 +67,4 @@ def camera_config( ax = plt.axes() ax.imshow(img_rgb) cam_config.plot(ax=ax, camera=True) - f.savefig(fn_cam)#, bbox_inches="tight") \ No newline at end of file + f.savefig(fn_cam)#, bbox_inches="tight") diff --git a/pyorc/service/velocimetry.py b/pyorc/service/velocimetry.py index 3c8a952..899927f 100644 --- a/pyorc/service/velocimetry.py +++ b/pyorc/service/velocimetry.py @@ -3,20 +3,22 @@ import functools import logging import os.path -import pyorc import subprocess import xarray as xr import yaml -from pyorc.cli import cli_utils from dask.diagnostics import ProgressBar from matplotlib.colors import Normalize from typing import Dict +from ..cli import cli_utils +from .. import Video, CameraConfig + __all__ = ["velocity_flow", "velocity_flow_subprocess"] logger = logging.getLogger(__name__) + def vmin_vmax_to_norm(opts): """ Check if opts contains vmin and/or vmax. If so change that into a norm option which works for all plotting methods @@ -221,7 +223,7 @@ def __init__( self.read = True self.write = False self.fn_video = videofile - self.cam_config = pyorc.CameraConfig(**cameraconfig) + self.cam_config = CameraConfig(**cameraconfig) self.logger = logger # TODO: perform checks, minimum steps required self.logger.info("pyorc velocimetry processor initialized") @@ -316,7 +318,7 @@ def process(self): # -y is provided, do with user intervention if stale file is present or -y is not provided def video(self, **kwargs): - self.video_obj = pyorc.Video( + self.video_obj = Video( self.fn_video, camera_config=self.cam_config, **kwargs @@ -566,7 +568,7 @@ def velocity_flow_subprocess( fn_cam_config = os.path.join(output, "camera_config.json") with open(fn_recipe, "w") as f: yaml.dump(recipe, f, default_flow_style=False, sort_keys=False) - pyorc.CameraConfig(**cameraconfig).to_file(fn_cam_config) + CameraConfig(**cameraconfig).to_file(fn_cam_config) cmd = [ "pyorc", "velocimetry", @@ -596,4 +598,4 @@ def velocity_flow_subprocess( cmd = cmd + cmd_suffix # call subprocess result = subprocess.run(cmd) - return result \ No newline at end of file + return result diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..bd2dfc1 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,108 @@ +[build-system] +requires = ["flit_core >=3.4.0,<4"] +build-backend = "flit_core.buildapi" + +[project] +name = "pyopenrivercam" +authors = [ + { name = "Hessel Winsemius", email = "winsemius@rainbowsensing.com" }, +] +packages = [ + { include = "pyorc" } +] + +dependencies = [ + "click", + "cython; platform_machine == 'armv7l'", + "dask", + "descartes", + "flox", + "geojson", + "geopandas", + "matplotlib", + "netCDF4", + "numba", + "numpy>=1.23, <2", # pin version to ensure compatibility with C-headers + "opencv-python", + "openpiv", + "packaging; platform_machine == 'armv7l'", + "pip", + "pyproj", + "pythran; platform_machine == 'armv7l'", + "pyyaml", + "rasterio", + "scikit-image", + "scipy", + "shapely", + "tqdm", + "typeguard", + "xarray" +] + +requires-python =">=3.9" +readme = "README.md" +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Topic :: Scientific/Engineering :: Hydrology", + "Topic :: Scientific/Engineering :: Image Processing", + "License :: OSI Approved :: GNU Affero General Public License v3", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11" +] +dynamic = ['version', 'description'] + +[project.optional-dependencies] +io = [ +] +extra = [ + "cartopy", + "notebook" +] +test = [ + "pytest", + "pytest-cov", +] +docs = [ + "sphinx==5.3", # >6.0 causes bug in pydata_sphinx_theme + "sphinx_autosummary_accessors", + "sphinxcontrib-programoutput", + "sphinx_rtd_theme", + "sphinx-design", + "pydata_sphinx_theme", + "ipykernel", + "nbsphinx", + "sphinx-gallery", + "pandoc", + "matplotlib", +] + +full = ["pyopenrivercam[io,extra,dev,test,doc]"] +slim = ["pyopenrivercam[io]"] + +[project.urls] +Source = "https://github.com/localdevices/pyorc" + +[project.scripts] +pyorc = "pyorc.cli.main:cli" + +[tool.flit.sdist] +include = ["pyorc"] + +[tool.flit.module] +name = "pyorc" + +[tool.pytest.ini_options] +addopts = "--ff " +testpaths = ["tests"] + +filterwarnings = [ + "ignore:This process *:DeprecationWarning:multiprocessing", # dask issue, related to python 3.12 stricter multiprocessing checks. + "ignore:All-NaN slice encountered*:RuntimeWarning", # expected behaviour when only NaNs occur in slicing in velocimetry results. + "ignore:invalid value encountered*:RuntimeWarning", # linestrings issue with plotting transects. + "ignore:Degrees of freedom *:RuntimeWarning", # not fully clear why this appears in user interfacing, test with future updates. + "ignore:numpy.ndarray size changed, may indicate binary incompatibility:RuntimeWarning", # likely caused by incompatibility in used numpy version across libraries. May resolve with future updates. +] diff --git a/setup.py b/setup.py deleted file mode 100644 index bd4ec47..0000000 --- a/setup.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -import os - -from setuptools import setup, find_packages - -here = os.path.abspath(os.path.dirname(__file__)) - -with open("README.md") as readme_file: - readme = readme_file.read() - -setup( - name="pyopenrivercam", - description="pyopenrivercam (pyorc) is a front and backend to control river camera observation locations", - version="0.5.5", - long_description=readme + "\n\n", - long_description_content_type="text/markdown", - url="https://github.com/localdevices/pyorc", - author="Hessel Winsemius", - author_email="winsemius@rainbowsensing.com", - packages=find_packages(), - package_dir={"pyorc": "pyorc"}, - test_suite="tests", - python_requires=">=3.9", - install_requires=[ - "click", - "cython; platform_machine == 'armv7l'", - "dask", - "descartes", - "flox", - "geojson", - "geopandas", - "matplotlib", - "netCDF4", - "numba", - "numpy", - "opencv-python", - "openpiv", - "packaging; platform_machine == 'armv7l'", - "pip", - "pyproj", - "pythran; platform_machine == 'armv7l'", - "pyyaml", - "rasterio", - "scikit-image", - "scipy", - "shapely", - "tqdm", - "typeguard", - "xarray" - ], - extras_require={ - "dev": ["pytest", "pytest-cov"], - "optional": [], - }, - entry_points={ - "console_scripts": [ - "pyorc = pyorc.cli.main:cli" - ] - }, - include_package_data=True, - license="AGPLv3", - zip_safe=False, - classifiers=[ - # https://pypi.python.org/pypi?%3Aaction=list_classifiers - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - "Topic :: Scientific/Engineering :: Hydrology", - "Topic :: Scientific/Engineering :: Image Processing", - "License :: OSI Approved :: GNU Affero General Public License v3", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - ], - keywords="hydrology, hydrometry, river-flow, pyorc", -) diff --git a/tests/conftest.py b/tests/conftest.py index 20f65b9..8a5d86e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -135,10 +135,14 @@ def bbox_6gcps(): @pytest.fixture def corners(): return [ - [292, 817], - [50, 166], - [1200, 236], - [1600, 834] + [500, 800], + [400, 600], + [1200, 550], + [1350, 650] + # [292, 817], + # [50, 166], + # [1200, 236], + # [1600, 834] ] @pytest.fixture diff --git a/tests/test_cameraconfig.py b/tests/test_cameraconfig.py index 3caab54..1ca6b06 100644 --- a/tests/test_cameraconfig.py +++ b/tests/test_cameraconfig.py @@ -32,13 +32,13 @@ def test_get_bbox(cam_config, vid): def test_shape(cam_config): - assert(cam_config.shape == (786, 878)) + assert(cam_config.shape == (475, 371)) def test_transform(cam_config): assert(np.allclose(cam_config.transform, Affine( - 0.0014443784253907177, 0.009895138754169435, 642730.233168765, - 0.009895138754169435, -0.0014443784253907175, 8304293.351276383 + -0.001107604584241635, 0.009938471315296278, 642732.3625957984, + 0.009938471315296278, 0.001107604584241631, 8304293.51724592 ))) @@ -68,18 +68,18 @@ def test_z_to_h(cam_config, cross_section): ( True, np.array( [ - [-2.83249013e-01, -8.93908572e-01, 7.95238051e+02], - [7.44402125e-01, -4.02349005e-01, -4.19808711e+02], - [-1.21275429e-04, 6.33985134e-04, 1.00000000e+00] + [-4.62466994e-01, -7.62938375e-01, 8.75609302e+02], + [ 6.48451357e-01, -6.15534992e-01, -2.04821521e+02], + [-1.21275313e-04, 6.33985726e-04, 1.00000000e+00] ] ) ), ( False, np.array( [ - [6.95684503e-03, -5.27244231e-03, -3.00544137e+00], - [-3.87798711e-03, -8.26420874e-03, 8.47535569e+00], - [-1.21275338e-04, 6.33985524e-04, 1.00000000e+00] + [ 6.95684503e-03, -5.27244231e-03, -3.00544137e+00], + [-3.87798711e-03, -8.26420874e-03, 8.47535569e+00], + [-1.21275338e-04, 6.33985524e-04, 1.00000000e+00] ] ) ) @@ -129,6 +129,22 @@ def test_lens_position(cam_config, lens_position): assert(np.allclose(cam_config.lens_position, lens_position)) +def test_estimate_lens_position(cam_config): + lens_pos = cam_config.estimate_lens_position() + assert np.allclose(lens_pos, [6.42731099e+05, 8.30429131e+06, 1.18996749e+03]) + + +def test_optimize_intrinsic(cam_config): + camera_matrix, dist_coeffs, err = cv.optimize_intrinsic( + cam_config.gcps["src"], + cam_config.gcps_dest, + cam_config.height, + cam_config.width, + lens_position=cam_config.lens_position + ) + print(camera_matrix, dist_coeffs, err) + + def test_to_file(tmpdir, cam_config, cam_config_str): fn = os.path.join(tmpdir, "cam_config.json") cam_config.to_file(fn) diff --git a/tests/test_cli.py b/tests/test_cli.py index c4f8cf0..da408a1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,15 +1,16 @@ +import json +from matplotlib import backend_bases +import matplotlib.pyplot as plt import os.path - +import pytest +import warnings from click.testing import CliRunner -from matplotlib import backend_bases import pyorc.service from pyorc.cli.main import cli from pyorc.cli.cli_elements import GcpSelect, AoiSelect, StabilizeSelect from pyorc.cli import cli_utils from pyorc.helpers import xyz_transform -import pytest -import json def test_cli_cam_config(cli_obj): result = cli_obj.invoke( @@ -79,13 +80,6 @@ def test_cli_velocimetry(cli_obj, vid_file, cam_config_fn, cli_recipe_fn, cli_ou assert result.exit_code == 0 -# def test_service_video(velocity_flow_processor): -# # ensure we are in the right folder -# print(f"current file is: {os.path.dirname(__file__)}") -# os.chdir(os.path.dirname(__file__)) -# # just test if everything is running -# velocity_flow_processor.process() - @pytest.mark.parametrize( "recipe_", [ @@ -129,10 +123,9 @@ def test_gcps_interact(gcps_dst, frame_rgb): selector.on_left_click(event) selector.on_right_click(event) selector.on_release(event) - + plt.close("all") def test_aoi_interact(frame_rgb, cam_config_without_aoi): - import matplotlib.pyplot as plt # convert dst to # del cam_config_without_aoi.crs src = cam_config_without_aoi.gcps["src"] @@ -153,6 +146,7 @@ def test_aoi_interact(frame_rgb, cam_config_without_aoi): selector.on_left_click(event) selector.on_right_click(event) selector.on_release(event) + plt.close("all") def test_stabilize_interact(frame_rgb): @@ -171,10 +165,9 @@ def test_stabilize_interact(frame_rgb): selector.close_window(event) # uncomment below to test the interaction, not suitable for automated unit test # plt.show(block=True) - + plt.close("all") def test_read_shape(gcps_fn): coords, wkt = cli_utils.read_shape(gcps_fn) assert(isinstance(wkt, str)) assert(isinstance(coords, list)) - diff --git a/tests/test_frames.py b/tests/test_frames.py index 3ccef05..5a18401 100644 --- a/tests/test_frames.py +++ b/tests/test_frames.py @@ -5,14 +5,14 @@ @pytest.mark.parametrize( "frames, resolution, method, dims, shape, kwargs", [ - ("frames_grayscale", 0.1, "numpy", 3, (79, 88), {}), - ("frames_grayscale", 0.1, "numpy", 3, (79, 88), {"reducer": "mean"}), - ("frames_rgb", 0.1, "numpy", 4, (79, 88, 3), {"reducer": "mean"}), - ("frames_rgb", 0.1, "numpy", 4, (79, 88, 3), {}), - ("frames_grayscale", 0.1, "cv", 3, (79, 88), {}), - ("frames_grayscale", 0.01, "cv", 3, (786, 878), {}), - ("frames_grayscale", 0.05, "cv", 3, (157, 176), {}), - ("frames_rgb", 0.1, "cv", 4, (79, 88, 3), {}), + ("frames_grayscale", 0.25, "numpy", 3, (19, 15), {}), + ("frames_grayscale", 0.25, "numpy", 3, (19, 15), {"reducer": "mean"}), + ("frames_rgb", 0.25, "numpy", 4, (19, 15, 3), {"reducer": "mean"}), + ("frames_rgb", 0.25, "numpy", 4, (19, 15, 3), {}), + ("frames_grayscale", 0.25, "cv", 3, (19, 15), {}), + ("frames_grayscale", 0.01, "cv", 3, (475, 371), {}), + ("frames_grayscale", 0.05, "cv", 3, (95, 74), {}), + ("frames_rgb", 0.25, "cv", 4, (19, 15, 3), {}), ] ) def test_project(frames, resolution, method, dims, shape, kwargs, request): @@ -50,7 +50,8 @@ def test_edge_detect(frames_proj): frames_edge = frames_proj.frames.edge_detect() assert(frames_edge.shape == frames_proj.shape) assert(frames_edge[0, 0, 0].values.dtype == "float32"), f'dtype of result is {frames_edge[0, 0, 0].values.dtype}, expected "float32"' - assert(np.allclose(frames_edge.values.flatten()[-4:], [-1.3828125, -4.3359375, 1.71875 , 7.234375 ])) + # assert(np.allclose(frames_edge.values.flatten()[-4:], [-1.3828125, -4.3359375, 1.71875 , 7.234375 ])) + assert (np.allclose(frames_edge.values.flatten()[-4:], [-6.0390625, 0.8671875, 6.4765625, 4.40625])) def test_reduce_rolling(frames_grayscale, samples=1): @@ -97,7 +98,7 @@ def test_plot_proj(frames_proj, idx): "window_size, result", [ # (5, [np.nan, np.nan, np.nan, 0.06877007]), - (10, [0.1623803 , 0.127019 , 0.26826966, 0.13940813]), + (10, [0.11740075, 0.09619355, 0.16204849, 0.14154269]), # (15, [0.21774408, 0.21398547, 0.25068682, 0.26456946]) ] ) diff --git a/tests/test_transect.py b/tests/test_transect.py index 6ce23de..730338a 100644 --- a/tests/test_transect.py +++ b/tests/test_transect.py @@ -7,7 +7,7 @@ def test_get_river_flow(piv_transect): piv_transect.transect.get_q() piv_transect.transect.get_river_flow() # we allow for 0.001 m3/s deviation for differences in versions of libs - assert(np.allclose(piv_transect.river_flow.values, [0.12069583, 0.12513554, 0.13068518, 0.13623481, 0.14067452], atol=0.001)) + assert(np.allclose(piv_transect.river_flow.values, [0.0821733 , 0.08626413, 0.09137767, 0.09649121, 0.10058204], atol=0.001)) @pytest.mark.parametrize( @@ -20,6 +20,7 @@ def test_get_river_flow(piv_transect): ] ) def test_get_q(piv_transect, fill_method): + piv_transect.load() f = piv_transect.transect.get_q(fill_method=fill_method) # assert if filled values are more complete than non-filled diff --git a/tests/test_velocimetry.py b/tests/test_velocimetry.py index 9f19da8..c0e5747 100644 --- a/tests/test_velocimetry.py +++ b/tests/test_velocimetry.py @@ -17,7 +17,7 @@ def test_get_transect(piv, cross_section, distance, nr_points): x, y, z = cross_section["x"], cross_section["y"], cross_section["z"] ds_points = piv.velocimetry.get_transect(x, y, z, crs=32735, rolling=4, distance=distance) # check if the angle is computed correctly - assert(np.isclose(ds_points["v_dir"][0].values, -4.67532165)) + assert(np.isclose(ds_points["v_dir"][0].values, -4.41938864)) assert(len(ds_points.points)) == nr_points diff --git a/tests/test_video.py b/tests/test_video.py index 96c40a4..bcd5c6b 100644 --- a/tests/test_video.py +++ b/tests/test_video.py @@ -31,7 +31,7 @@ def test_fps(vid): "video, method, result", [ ("vid_cam_config", "grayscale", [85, 71, 65, 80]), - ("vid_cam_config_stabilize", "grayscale", [5, 88, 78, 73]), + ("vid_cam_config_stabilize", "grayscale", [60, 78, 70, 76]), ("vid_cam_config", "rgb", [84, 91, 57, 70]), ("vid_cam_config", "hsv", [36, 95, 91, 36]) ] @@ -61,4 +61,3 @@ def test_get_frames(video, method, request): assert(len(frames) == video.end_frame - video.start_frame + 1) # check if the time difference is well taken from the fps of the video assert(np.allclose(np.diff(frames.time.values), [1./video.fps])) -