Merge pull request #26 from AgPipeline/develop

Merge develop to main branch - no review

.github/workflows/shellscript_checks.yaml

@@ -0,0 +1,41 @@
+name: Enforcing shell script tests
+on:
+  push:
+    branches:
+      - master
+      - develop
+  pull_request:
+    branches:
+      - master
+      - develop
+    tags:
+      - v*
+
+jobs:
+  testing:
+    runs-on: ubuntu-latest
+    name: Running testing
+    strategy:
+      matrix:
+          app: [shellcheck, shfmt]
+          include:
+            - app: shellcheck
+              shellcheck_opts:
+              shellcheck_disable: false
+              shfmt_disable: true
+            - app: shfmt
+              shfmt_opts: -i 2 -ci -w
+              shellcheck_disable: true
+              shfmt_disable: false
+    steps:
+      - name: Fetch source code
+        uses: actions/checkout@v2
+      - name: shell check
+        uses: luizm/action-sh-checker@v0.1.8
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SHELLCHECK_OPTS: ${{ matrix.shellcheck_opts }}
+          SHFMT_OPTS: ${{ matrix.shfmt_opts }}
+        with:
+          sh_checker_shellcheck_disable: ${{ matrix.shellcheck_disable }}
+          sh_checker_shfmt_disable: ${{ matrix.shfmt_disable }}

.github/workflows/short_workflow_check.sh

@@ -18,45 +18,45 @@ CHECK_FILE="${TARGET_FOLDER}/orthomosaic_mask.tif"
 if [[ -f "${CHECK_FILE}" ]]; then
   echo "Mask file found: ${CHECK_FILE}"
 else
-  echo "Unable to find mask file: ${CHECK_FILE}";
-  exit 1;
+  echo "Unable to find mask file: ${CHECK_FILE}"
+  exit 1
 fi
 
 # Get all the folders and check the count
-FOLDER_LIST=(`find "${TARGET_FOLDER}/" -maxdepth 1 -type d`)
+# shellcheck disable=SC2207
+FOLDER_LIST=($(find "${TARGET_FOLDER}/" -maxdepth 1 -type d))
 if [[ "${#FOLDER_LIST[@]}" == "${EXPECTED_NUM_FOLDERS}" ]]; then
   echo "Found expected number of folders: ${EXPECTED_NUM_FOLDERS}"
 else
   echo "Expected ${EXPECTED_NUM_FOLDERS} folders and found ${#FOLDER_LIST[@]}"
-  for i in $(seq 0 $(( ${#FOLDER_LIST[@]} - 1 )))
-  do
-    echo "$(( ${i} + 1 )): ${FOLDER_LIST[$i]}"
+  for i in $(seq 0 $((${#FOLDER_LIST[@]} - 1))); do
+    echo "$((i + 1)): ${FOLDER_LIST[$i]}"
   done
   exit 10
 fi
 
 # Check the expected number of output files
-EXPECTED_CSV=(`find "${TARGET_FOLDER}/" -type f | grep 'canopycover\.csv'`)
+# shellcheck disable=SC2207
+EXPECTED_CSV=($(find "${TARGET_FOLDER}/" -type f | grep 'canopycover\.csv'))
 if [[ "${#EXPECTED_CSV[@]}" == "${EXPECTED_NUM_CANOPYCOVER_CSV}" ]]; then
   echo "Found expected number of canopycover.csv files: ${EXPECTED_NUM_CANOPYCOVER_CSV}"
 else
   echo "Expected ${EXPECTED_NUM_CANOPYCOVER_CSV} canopycover.csv files but found ${#EXPECTED_CSV[@]}"
-  for i in $(seq 0 $(( ${#EXPECTED_CSV[@]} - 1 )))
-  do
-    echo "$(( ${i} + 1 )): ${EXPECTED_CSV[$i]}"
+  for i in $(seq 0 $((${#EXPECTED_CSV[@]} - 1))); do
+    echo "$((i + 1)): ${EXPECTED_CSV[$i]}"
   done
   exit 20
 fi
 
 # Check the expected number of image mask files
-EXPECTED_MASK=(`find "${TARGET_FOLDER}/" -type f | grep 'orthomosaic_mask\.tif'`)
+# shellcheck disable=SC2207
+EXPECTED_MASK=($(find "${TARGET_FOLDER}/" -type f | grep 'orthomosaic_mask\.tif'))
 if [[ "${#EXPECTED_MASK[@]}" == "${EXPECTED_NUM_MASK_TIF}" ]]; then
   echo "Found expected number of orthomosaic_mask.tif files: ${EXPECTED_NUM_MASK_TIF}"
 else
   echo "Expected ${EXPECTED_NUM_MASK_TIF} orthomosaic_mask.tif files but found ${#EXPECTED_MASK[@]}"
-  for i in $(seq 0 $(( ${#EXPECTED_MASK[@]} - 1 )))
-  do
-    echo "$(( ${i} + 1 )): ${EXPECTED_MASK[$i]}"
+  for i in $(seq 0 $((${#EXPECTED_MASK[@]} - 1))); do
+    echo "$((i + 1)): ${EXPECTED_MASK[$i]}"
   done
   exit 30
 fi

.github/workflows/testing_checks.yaml

@@ -0,0 +1,93 @@
+name: Enforcing tests
+on:
+  push:
+    branches:
+      - master
+      - develop
+  pull_request:
+    branches:
+      - master
+      - develop
+    tags:
+      - v*
+
+jobs:
+  testing:
+    runs-on: ubuntu-latest
+    name: Running testing
+    strategy:
+      matrix:
+          app: [pylint, pytest]
+          include:
+            - app: pylint
+              pip_installs: pylint pytest
+              test_command: (cat action_pylint_files.txt | xargs python3 -m pylint --rcfile ./pylint.rc) && (find ./tests | grep '\.py' | xargs python3 -m pylint -d duplicate-code --rcfile ./pylint.rc)
+            - app: pytest
+              pip_installs: pytest pytest-cov
+              test_command: python3 -m pytest --cov=. -rpP --cov-report=xml > coverage.txt
+              artifacts: coverage.txt
+    steps:
+      - name: Current python version
+        run: python3 --version || echo python3 not installed
+      - name: Install Python 3.7
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.7'
+      - name: Updated python version
+        run: python3 --version
+      - name: PYTHONPATH environment variable
+        run: echo "PYTHONPATH is ${PYTHONPATH}"
+      - name: Update pip
+        run: python3 -m pip install --upgrade --no-cache-dir pip
+      - name: Fetch/update setuptools
+        run: python3 -m pip install --upgrade --no-cache-dir setuptools
+      - name: Install python-apt
+        run: sudo apt-get install -y python-apt
+      - name: HACK to fix apt-get update problem w/ different python versions
+        run: 'cd /usr/lib/python3/dist-packages && sudo cp apt_pkg.cpython-36m-x86_64-linux-gnu.so apt_pkg.so'
+      - name: Update apt-get
+        run: sudo apt-get update
+      - name: Fetch/update testing pip installations
+        run: python3 -m pip install --upgrade --no-cache-dir ${{ matrix.pip_installs }}
+      - name: Fetch source code
+        uses: actions/checkout@v2
+      - name: Finding files
+        run: find . -type f -name "*.py" | grep -v 'tests/' > action_pylint_files.txt
+      - name: Install system requirements
+        shell: bash
+        run: 'sudo apt-get install -y python3-gdal gdal-bin libgdal-dev gcc g++ python3.7-dev'
+      - name: Install Python numpy and other modules
+        shell: bash
+        run: 'python3 -m pip install --upgrade --no-cache-dir numpy wheel requests'
+      - name: Install Python pygdal
+        shell: bash
+        run: 'python3 -m pip install --no-cache-dir pygdal==2.2.3.5'
+      - name: Install system requirements from source
+        shell: bash
+        run: 'if [ -s "packages.txt" ]; then (cat packages.txt | sudo xargs apt-get install -y --no-install-recommends) || (echo "Failed to install additional packages" && exit 1); fi'
+      - name: Install Python requirements from source
+        shell: bash
+        run: 'if [ -s "requirements.txt" ]; then (python3 -m pip install --no-cache-dir -r requirements.txt) || (echo "Failed to install Python requirements" && exit 1); fi'
+      - name: Run action pylint script
+        shell: bash
+        run: 'if [ -s ".github/workflows/action_pylint.sh" ]; then (chmod +x ".github/workflows/action_pylint.sh" && ./.github/workflows/action_pylint.sh) || (echo "Error running shell script" && exit 1); fi'
+      - name: Fetching pylint.rc file
+        run: wget https://raw.githubusercontent.com/AgPipeline/Organization-info/master/pylint.rc
+        if: ${{ matrix.name }} == "pylint"
+      - name: Set execution permission for testing
+        run: chmod +x betydb2geojson.py
+      - name: Listing
+        run: ls -la
+      - name: Files to be tested
+        run: cat action_pylint_files.txt
+      - name: Running test
+        run: ${{ matrix.test_command }}
+      - name: Upload testing artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: testing_artifacts
+          path: ${{ matrix.artifacts }}
+        if: ${{ matrix.artifacts }}
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v1
+        if: ${{ matrix.app == 'pytest' }}