Merge pull request #31 from AgPipeline/develop

Merging develop to master - no review

.github/workflows/testing_checks.yaml

@@ -24,8 +24,7 @@ jobs:
               test_command: (cat action_test_files.txt | xargs python3 -m pylint --rcfile ./pylint.rc) && (find ./tests | grep '\.py' | xargs python3 -m pylint -d duplicate-code --rcfile ./pylint.rc)
             - app: pytest
               pip_installs: pytest pytest-cov
-              test_command: python3 -m pytest --cov=. -rpP --cov-report=xml > coverage.txt
-              artifacts: coverage.txt
+              test_command: python3 -m pytest --cov=. -rpP --cov-report=xml
     steps:
       - name: Current python version
         run: python3 --version || echo python3 not installed

README.md

@@ -38,7 +38,7 @@ An explanation of the command line options used follows.
 Be sure to read up on the [docker run](https://docs.docker.com/engine/reference/run/) command line for more information.
 
 ```bash
-docker run --rm --mount "src=${PWD}/test_data,target=/mnt,type=bind" agdrone/transformer-canopycover:1.1 --working_space "/mnt" --metadata "/mnt/experiment.yaml" --citation_author "Me Myself" --citation_title "Something in the green" --citation_year "2019" --germplasm_name "Big Plant" "/mnt/rgb_1_2_E.tif"
+docker run --rm --mount "src=${PWD}/test_data,target=/mnt,type=bind" agdrone/transformer-canopycover:1.1 --working_space "/mnt" --metadata "/mnt/experiment.yaml" --citation_author "Me Myself" --citation_title "Something in the green" --citation_year "2019" --species "Big Plant" "/mnt/rgb_1_2_E.tif"
 ```
 
 This example command line assumes the source files are located in the `test_data` folder off the current folder.
@@ -75,8 +75,13 @@ with a completely white image, make sure that 0 is returned.
 
 The reason this should be done is in order to test the extremes for image data.
 
-Next test on these [sample plot images](https://drive.google.com/file/d/1xWRU0YgK3Y9aUy5TdRxj14gmjLlozGxo/view) and make sure
-that reasonable values are returned.
+Next test on sample plot images and make sure that reasonable values are returned.
+The following commands can be used to retrieve the plot images:
+```bash
+mkdir test_data
+curl https://de.cyverse.org/dl/d/4108BB75-AAA3-48E1-BBD4-E10B06CADF54/sample_plot_images.zip -o test_data/sample_plot_images.zip
+unzip test_data/sample_plot_images.zip -d test_data/
+```
 
 **Deploying the Transformer** \
 Once you have used the transformer on your image data, you can upload your docker image to [Docker Hub](https://hub.docker.com)

REVIEW.adoc

@@ -44,7 +44,7 @@ usage: canopycover.py [-h] [--debug] [--info] [--result [RESULT]]
                       [--citation_author [CITATIONAUTHOR]]
                       [--citation_title [CITATIONTITLE]]
                       [--citation_year [CITATIONYEAR]]
-                      [--germplasm_name [GERMPLASMNAME]]
+                      [--species [SPECIES]]
                       ...
 
 Canopy Cover by Plot (Percentage of Green Pixels)
@@ -69,8 +69,8 @@ optional arguments:
                         measurements
   --citation_year [CITATIONYEAR]
                         year of citation to use when generating measurements
-  --germplasm_name [GERMPLASMNAME]
-                        name of the germplasm associated with the canopy cover
+  --species [SPECIES]
+                        name of the species associated with the canopy cover
 
 terra.stereo-rgb.canopycover version 3.0 author Chris Schnaufer
 schnaufer@email.arizona.edu
@@ -138,7 +138,7 @@ usage: canopycover.py [-h] [--debug] [--info] [--result [RESULT]] -m METADATA
                       [--citation_author [CITATIONAUTHOR]]
                       [--citation_title [CITATIONTITLE]]
                       [--citation_year [CITATIONYEAR]]
-                      [--germplasm_name [GERMPLASMNAME]]
+                      [--species [SPECIES]]
                       ...
 canopycover.py: error: the following arguments are required: -m/--metadata
 ----
@@ -152,7 +152,7 @@ usage: canopycover.py [-h] [--debug] [--info] [--result [RESULT]] -m METADATA
                       [--citation_author [CITATIONAUTHOR]]
                       [--citation_title [CITATIONTITLE]]
                       [--citation_year [CITATIONYEAR]]
-                      [--germplasm_name [GERMPLASMNAME]]
+                      [--species [SPECIES]]
                       ...
 canopycover.py: error: argument -m/--metadata: can't open 'blargh': \
 [Errno 2] No such file or directory: 'blargh'
@@ -267,7 +267,7 @@ usage: canopycover.py [-h] [--debug] [--info] [--result [RESULT]]
                       [--citation_author [CITATIONAUTHOR]]
                       [--citation_title [CITATIONTITLE]]
                       [--citation_year [CITATIONYEAR]]
-                      [--germplasm_name [GERMPLASMNAME]]
+                      [--species [SPECIES]]
                       file_list [file_list ...]
 canopycover.py: error: the following arguments are required: file_list
 ----
@@ -293,7 +293,7 @@ usage: canopycover.py [-h] [--debug] [--info] [--result [RESULT]]
                       [--citation_author [CITATIONAUTHOR]]
                       [--citation_title [CITATIONTITLE]]
                       [--citation_year [CITATIONYEAR]]
-                      [--germplasm_name [GERMPLASMNAME]]
+                      [--species [SPECIES]]
                       ...
 canopycover.py: error: Invalid files: foo, bar
 ----

canopycover.py

@@ -167,13 +167,13 @@ def setup_default_traits(traits: dict, args: argparse.Namespace, full_md: list)
     # Check metadata
     if full_md:
         for one_md in full_md:
-            if 'germplasmName' in one_md:
-                new_traits['species'] = one_md['germplasmName']
+            if 'species' in one_md:
+                new_traits['species'] = one_md['species']
                 traits_modified = True
 
     # Check command line parameters
-    if args.germplasmName is not None:
-        new_traits['species'] = args.germplasmName
+    if args.species is not None:
+        new_traits['species'] = args.species
         traits_modified = True
     if args.citationAuthor is not None:
         new_traits['citation_author'] = args.citationAuthor
@@ -198,12 +198,8 @@ def calculate_canopycover_masked(pxarray: np.ndarray) -> float:
     Notes:
         From TERRA REF canopy cover: https://github.com/terraref/extractors-stereo-rgb/tree/master/canopycover
     """
-    # If > 75% is NoData, return a -1 ccvalue for omission later
     total_size = pxarray.shape[0] * pxarray.shape[1]
     nodata = np.count_nonzero(pxarray[:, :, 3] == 0)
-    nodata_ratio = nodata/float(total_size)
-    if nodata_ratio > 0.75:
-        return -1
 
     # For masked images, all pixels with rgb>0,0,0 are considered canopy
     data = pxarray[pxarray[:, :, 3] == 255]
@@ -249,8 +245,8 @@ def add_parameters(self, parser: argparse.ArgumentParser) -> None:
         parser.add_argument('--citation_year', dest="citationYear", type=str, nargs='?',
                             help="year of citation to use when generating measurements")
 
-        parser.add_argument('--germplasm_name', dest="germplasmName", type=str, nargs='?',
-                            help="name of the germplasm associated with the canopy cover")
+        parser.add_argument('--species', dest="species", type=str, nargs='?',
+                            help="name of the species associated with the canopy cover")
 
     def check_continue(self, environment: Environment, check_md: dict, transformer_md: list, full_md: list) -> tuple:
         """Checks if conditions are right for continuing processing
@@ -356,8 +352,6 @@ def perform_process(self, environment: Environment, check_md: dict, transformer_
                         else:
                             logging.info('Adding missing alpha channel to loaded image from "%s"', one_file)
                             image_to_use = _add_image_mask(one_file)
-#                            mask = np.where(np.sum(pxarray, axis=0) == 0, 0, 255).astype(pxarray.dtype)
-#                            image_to_use = np.stack((pxarray[0], pxarray[1], pxarray[2], mask))
                             del pxarray     # Potentially free up memory
 
                         logging.debug("Calculating canopy cover")
@@ -389,6 +383,8 @@ def perform_process(self, environment: Environment, check_md: dict, transformer_
                     else:
                         continue
                 except Exception as ex:
+                    if logging.getLogger().level == logging.DEBUG:
+                        logging.exception("Exception caught while processing canopy")
                     logging.warning("Exception caught while processing canopy cover: %s", str(ex))
                     logging.warning("Error generating canopy cover for '%s'", one_file)
                     logging.warning("    plot name: '%s'", plot_name)
@@ -398,7 +394,7 @@ def perform_process(self, environment: Environment, check_md: dict, transformer_
         if not num_files:
             return {'code': -1000, 'error': "No files were processed"}
         if not total_plots_calculated:
-            return {'code': -1001, 'error': "No plots intersected with the images provided"}
+            return {'code': -1001, 'error': "No images were able to have their canopy cover calculated"}
 
         # Setup the metadata for returning files
         file_md = []

test_data/meta.yaml

@@ -3,6 +3,6 @@
 pipeline:
     studyName: 'S7_20181011'
     season: 'S7_20181011'
-    germplasmName: Sorghum bicolor
+    species: Sorghum bicolor
     collectingSite: Maricopa
     observationTimeStamp: '2018-10-11T13:01:02-08:00'

tests/test_canopy_cover.py

@@ -9,7 +9,9 @@
 import os
 import random
 import re
+import shutil
 import string
+import tempfile
 from shutil import rmtree
 from subprocess import getstatusoutput
 #import pytest
@@ -53,15 +55,17 @@ def test_no_args():
     changed to some non-zero value to indicate a failure.
     """
     ret_val, out = getstatusoutput(SOURCE_PATH)
-    assert ret_val == 0
-    assert re.search('No metadata paths were specified', out)
+    assert ret_val == 1
+    assert re.search('FileNotFoundError', out)
 
 
 def test_no_metadata():
     """ Run with a file but no metadata"""
-    ret_val, out = getstatusoutput(f'{SOURCE_PATH} {INPUT1}')
+    temp_dir = tempfile.mkdtemp()
+    ret_val, out = getstatusoutput(f'{SOURCE_PATH} --working_space {temp_dir}  {INPUT1}')
+    shutil.rmtree(temp_dir)
     assert ret_val == 0
-    assert re.search('No metadata paths were specified', out)
+    assert re.search('"code": 0', out)
 
 
 def test_get_fields():