From fdc03affc4e6104f515bb55dbebbf7c950b9ac8e Mon Sep 17 00:00:00 2001 From: Chris Schnaufer Date: Mon, 1 Mar 2021 12:59:26 -0700 Subject: [PATCH 1/3] Removing incorrect files --- .github/workflows/docker_test_check_rgb.sh | 67 --------------------- .github/workflows/testing_docker.yaml | 69 ---------------------- 2 files changed, 136 deletions(-) delete mode 100755 .github/workflows/docker_test_check_rgb.sh delete mode 100644 .github/workflows/testing_docker.yaml diff --git a/.github/workflows/docker_test_check_rgb.sh b/.github/workflows/docker_test_check_rgb.sh deleted file mode 100755 index ebeabc3..0000000 --- a/.github/workflows/docker_test_check_rgb.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash - -# Checking the results of running the docker commands - -# Define expected results -EXPECTED_FILES=("result.json") -EXPECTED_CANOPYCOVER_VALUES=(99.8 20.9) - -# What folder are we looking in for outputs -if [[ ! "${1}" == "" ]]; then - TARGET_FOLDER="${1}" -else - TARGET_FOLDER="./outputs" -fi - -# What our target file to read is -if [[ ! "${2}" == "" ]]; then - CHECK_FILE="${2}" -else - CHECK_FILE="canopycover.csv" -fi -EXPECTED_FILES+=("${CHECK_FILE}") - -# Check if expected files are found -for i in $(seq 0 $(( ${#EXPECTED_FILES[@]} - 1 ))) -do - if [[ ! -f "${TARGET_FOLDER}/${EXPECTED_FILES[$i]}" ]]; then - echo "Expected file ${EXPECTED_FILES[$i]} is missing" - exit 10 - fi -done - -# Check the results of the canopy cover calculation -RESULT_VALUES=(`gawk ' -BEGIN { - FPAT = "([^,]+)|(\"[^\"]+\")" -} -{ - if ($1 != "local_datetime") { # Skipping the header line - printf("%s\n", $2) - } -} -END { -} -' "${TARGET_FOLDER}/${CHECK_FILE}"`) - -echo "Result counts: ${#EXPECTED_CANOPYCOVER_VALUES[@]} vs ${#RESULT_VALUES[@]}" -if [[ ${#EXPECTED_CANOPYCOVER_VALUES[@]} != ${#RESULT_VALUES[@]} ]]; then - echo "Number of results found in file (${#RESULT_VALUES[@]}) don't match expected count (${#EXPECTED_CANOPYCOVER_VALUES[@]})" - if [[ ${#RESULT_VALUES[@]} > 0 ]]; then - for i in $(seq 0 $(( ${#RESULT_VALUES[@]} - 1 ))) - do - echo "${i}: ${RESULT_VALUES[$i]}" - done - fi - exit 20 -fi - -for i in $(seq 0 $(( ${#EXPECTED_CANOPYCOVER_VALUES[@]} - 1 ))) -do - if [[ "${EXPECTED_CANOPYCOVER_VALUES[$i]}" == "${RESULT_VALUES[$i]}" ]]; then - echo "Values for index ${i} match: '${RESULT_VALUES[$i]}' '${EXPECTED_CANOPYCOVER_VALUES[$i]}'" - else - echo "Result value for index ${i}: '${RESULT_VALUES[$i]}' doesn't match expected: '${EXPECTED_CANOPYCOVER_VALUES[$i]}'" - exit 30 - fi -done diff --git a/.github/workflows/testing_docker.yaml b/.github/workflows/testing_docker.yaml deleted file mode 100644 index 0e3c590..0000000 --- a/.github/workflows/testing_docker.yaml +++ /dev/null @@ -1,69 +0,0 @@ -name: Testing Docker image -on: - push: - branches: - - master - - develop - pull_request: - branches: - - master - - develop - tags: - - v* - schedule: - # Every 01:00 Sunday re-run the test on the main branch - - cron: '0 1 * * 0' - workflow_dispatch: - -jobs: - docker_testing: - runs-on: ubuntu-latest - name: Running Docker testing - steps: - - name: Fetch source code - uses: actions/checkout@v2 - - name: Create folders - run: | - mkdir ./inputs && chmod 777 ./inputs - mkdir ./outputs && chmod 777 ./outputs - - name: List folder contents - run: | - echo "Current folder" && ls -la - echo "test_data" && ls -l ./test_data - - name: Copy testing data files - run: | - cp "${PWD}/test_data"/* "${PWD}/inputs/" - echo "inputs" && ls -l ./inputs - - name: Folder contents - run: | - echo "Current folder" && ls -l - echo "Inputs folder" && ls -l ./inputs - echo "Outputs folder" && ls -l ./outputs - - name: Build docker image - run: docker build -t canopycover_test:latest ./ - - name: Compress docker image - run: docker save canopycover_test:latest | gzip -7 -c - > canopycover_test_image.tar.gz - - name: Upload docker image - uses: actions/upload-artifact@v2 - with: - name: canopycover_test_image - path: canopycover_test_image.tar.gz - - name: Run docker test - run: docker run --rm -v "${PWD}/inputs:/inputs" -v "${PWD}/outputs:/outputs" canopycover_test:latest --working_space /outputs --metadata /inputs/meta.yaml /inputs/rgb_17_7_W.tif /inputs/mask_1.tif - - name: Output folder contents - run: echo "Outputs folder" && ls -l ./outputs - - name: Check outputs - run: | - cat outputs/canopycover.csv - chmod +x "./.github/workflows/docker_test_check_rgb.sh" - "./.github/workflows/docker_test_check_rgb.sh" - - artifact_cleanup: - runs-on: ubuntu-latest - needs: [docker_testing] - name: Cleanup artifacts upon success - steps: - - name: Remove docker artifact - uses: geekyeggo/delete-artifact@v1 - with: - name: canopycover_test_image From 8db7e09e040f4f1539c72490e9e217d8f52f369b Mon Sep 17 00:00:00 2001 From: Chris Schnaufer Date: Mon, 1 Mar 2021 13:00:33 -0700 Subject: [PATCH 2/3] Fixing wrong repo changes --- .github/workflows/docker_test_check_rgb.sh | 67 +++++++++++++++++++++ .github/workflows/testing_docker.yaml | 69 ++++++++++++++++++++++ 2 files changed, 136 insertions(+) create mode 100755 .github/workflows/docker_test_check_rgb.sh create mode 100644 .github/workflows/testing_docker.yaml diff --git a/.github/workflows/docker_test_check_rgb.sh b/.github/workflows/docker_test_check_rgb.sh new file mode 100755 index 0000000..ebeabc3 --- /dev/null +++ b/.github/workflows/docker_test_check_rgb.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +# Checking the results of running the docker commands + +# Define expected results +EXPECTED_FILES=("result.json") +EXPECTED_CANOPYCOVER_VALUES=(99.8 20.9) + +# What folder are we looking in for outputs +if [[ ! "${1}" == "" ]]; then + TARGET_FOLDER="${1}" +else + TARGET_FOLDER="./outputs" +fi + +# What our target file to read is +if [[ ! "${2}" == "" ]]; then + CHECK_FILE="${2}" +else + CHECK_FILE="canopycover.csv" +fi +EXPECTED_FILES+=("${CHECK_FILE}") + +# Check if expected files are found +for i in $(seq 0 $(( ${#EXPECTED_FILES[@]} - 1 ))) +do + if [[ ! -f "${TARGET_FOLDER}/${EXPECTED_FILES[$i]}" ]]; then + echo "Expected file ${EXPECTED_FILES[$i]} is missing" + exit 10 + fi +done + +# Check the results of the canopy cover calculation +RESULT_VALUES=(`gawk ' +BEGIN { + FPAT = "([^,]+)|(\"[^\"]+\")" +} +{ + if ($1 != "local_datetime") { # Skipping the header line + printf("%s\n", $2) + } +} +END { +} +' "${TARGET_FOLDER}/${CHECK_FILE}"`) + +echo "Result counts: ${#EXPECTED_CANOPYCOVER_VALUES[@]} vs ${#RESULT_VALUES[@]}" +if [[ ${#EXPECTED_CANOPYCOVER_VALUES[@]} != ${#RESULT_VALUES[@]} ]]; then + echo "Number of results found in file (${#RESULT_VALUES[@]}) don't match expected count (${#EXPECTED_CANOPYCOVER_VALUES[@]})" + if [[ ${#RESULT_VALUES[@]} > 0 ]]; then + for i in $(seq 0 $(( ${#RESULT_VALUES[@]} - 1 ))) + do + echo "${i}: ${RESULT_VALUES[$i]}" + done + fi + exit 20 +fi + +for i in $(seq 0 $(( ${#EXPECTED_CANOPYCOVER_VALUES[@]} - 1 ))) +do + if [[ "${EXPECTED_CANOPYCOVER_VALUES[$i]}" == "${RESULT_VALUES[$i]}" ]]; then + echo "Values for index ${i} match: '${RESULT_VALUES[$i]}' '${EXPECTED_CANOPYCOVER_VALUES[$i]}'" + else + echo "Result value for index ${i}: '${RESULT_VALUES[$i]}' doesn't match expected: '${EXPECTED_CANOPYCOVER_VALUES[$i]}'" + exit 30 + fi +done diff --git a/.github/workflows/testing_docker.yaml b/.github/workflows/testing_docker.yaml new file mode 100644 index 0000000..0e3c590 --- /dev/null +++ b/.github/workflows/testing_docker.yaml @@ -0,0 +1,69 @@ +name: Testing Docker image +on: + push: + branches: + - master + - develop + pull_request: + branches: + - master + - develop + tags: + - v* + schedule: + # Every 01:00 Sunday re-run the test on the main branch + - cron: '0 1 * * 0' + workflow_dispatch: + +jobs: + docker_testing: + runs-on: ubuntu-latest + name: Running Docker testing + steps: + - name: Fetch source code + uses: actions/checkout@v2 + - name: Create folders + run: | + mkdir ./inputs && chmod 777 ./inputs + mkdir ./outputs && chmod 777 ./outputs + - name: List folder contents + run: | + echo "Current folder" && ls -la + echo "test_data" && ls -l ./test_data + - name: Copy testing data files + run: | + cp "${PWD}/test_data"/* "${PWD}/inputs/" + echo "inputs" && ls -l ./inputs + - name: Folder contents + run: | + echo "Current folder" && ls -l + echo "Inputs folder" && ls -l ./inputs + echo "Outputs folder" && ls -l ./outputs + - name: Build docker image + run: docker build -t canopycover_test:latest ./ + - name: Compress docker image + run: docker save canopycover_test:latest | gzip -7 -c - > canopycover_test_image.tar.gz + - name: Upload docker image + uses: actions/upload-artifact@v2 + with: + name: canopycover_test_image + path: canopycover_test_image.tar.gz + - name: Run docker test + run: docker run --rm -v "${PWD}/inputs:/inputs" -v "${PWD}/outputs:/outputs" canopycover_test:latest --working_space /outputs --metadata /inputs/meta.yaml /inputs/rgb_17_7_W.tif /inputs/mask_1.tif + - name: Output folder contents + run: echo "Outputs folder" && ls -l ./outputs + - name: Check outputs + run: | + cat outputs/canopycover.csv + chmod +x "./.github/workflows/docker_test_check_rgb.sh" + "./.github/workflows/docker_test_check_rgb.sh" + + artifact_cleanup: + runs-on: ubuntu-latest + needs: [docker_testing] + name: Cleanup artifacts upon success + steps: + - name: Remove docker artifact + uses: geekyeggo/delete-artifact@v1 + with: + name: canopycover_test_image From e2de0307d1fd11bc07908c8827a4f1e92020603a Mon Sep 17 00:00:00 2001 From: Chris Schnaufer Date: Fri, 19 Mar 2021 12:14:49 -0700 Subject: [PATCH 3/3] Fixing Actions YAML branch names --- .github/workflows/testing_checks.yaml | 4 ++-- .github/workflows/testing_docker.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/testing_checks.yaml b/.github/workflows/testing_checks.yaml index 4bed72f..dd730aa 100644 --- a/.github/workflows/testing_checks.yaml +++ b/.github/workflows/testing_checks.yaml @@ -2,11 +2,11 @@ name: Enforcing tests on: push: branches: - - master + - main - develop pull_request: branches: - - master + - main - develop tags: - v* diff --git a/.github/workflows/testing_docker.yaml b/.github/workflows/testing_docker.yaml index 0e3c590..5df4583 100644 --- a/.github/workflows/testing_docker.yaml +++ b/.github/workflows/testing_docker.yaml @@ -2,11 +2,11 @@ name: Testing Docker image on: push: branches: - - master + - main - develop pull_request: branches: - - master + - main - develop tags: - v*