Display coverage checklist usable on GitHub. (#697)

* Display coverage checklist usable on GitHub. Signed-off-by: dblock <dblock@amazon.com> * Display missing test paths in a GitHub compatible format. Signed-off-by: dblock <dblock@amazon.com> --------- Signed-off-by: dblock <dblock@amazon.com>
opensearch-project · Nov 26, 2024 · 4f27c67 · 4f27c67
1 parent d7d6520
commit 4f27c67
Show file tree

Hide file tree

Showing 4 changed files with 35 additions and 21 deletions.
diff --git a/.github/workflows/analyze-pr-changes.yml b/.github/workflows/analyze-pr-changes.yml
@@ -22,7 +22,7 @@ jobs:
           BEFORE_COVERAGE=/tmp/coverage-api-${BEFORE_SHA}.json
           AFTER_COVERAGE=/tmp/coverage-api-${AFTER_SHA}.json
           COVERAGE_DIFF=/tmp/coverage-api-${BEFORE_SHA}-${AFTER_SHA}-DIFF.json
-          
+
           vars=(
             BEFORE_SHA
             AFTER_SHA
@@ -33,7 +33,7 @@ jobs:
             AFTER_COVERAGE
             COVERAGE_DIFF
           )
-          
+
           { 
             for var in "${vars[@]}"
             do
@@ -48,19 +48,19 @@ jobs:
         shell: bash -eo pipefail {0}
         run: |
           docker build coverage --tag opensearch-with-api-plugin
-          
+
           docker run \
             --name opensearch \
             --rm -d \
             -p 9200:9200 -p 9600:9600 \
             -e "discovery.type=single-node" \
             -e OPENSEARCH_INITIAL_ADMIN_PASSWORD="$OPENSEARCH_PASSWORD" \
             opensearch-with-api-plugin
-          
+
           npm ci
-          
+
           npm run dump-cluster-spec -- --opensearch-insecure --output $CLUSTER_SPEC
-          
+
           docker stop opensearch
         env:
           OPENSEARCH_PASSWORD: BobgG7YrtsdKf9M
@@ -92,14 +92,14 @@ jobs:
               --cluster $CLUSTER_SPEC \
               --specification $BEFORE_SPEC \
               --output $BEFORE_COVERAGE
-          
+
           npm run coverage:spec -- \
               --cluster $CLUSTER_SPEC \
               --specification $AFTER_SPEC \
               --output $AFTER_COVERAGE
-          
+
           jq . $AFTER_COVERAGE
-          
+
           jq --slurp '
             [ .[].counts ]
             | {
@@ -116,6 +116,18 @@ jobs:
             $AFTER_COVERAGE \
             | tee $COVERAGE_DIFF
 
+      - name: Display Coverage Checklist
+        shell: bash -eo pipefail {0}
+        run: |
+          jq -sc '
+            (map(.endpoints .uncovered | to_entries | sort_by(.value) | map((.value[] | ascii_upcase) + " " + .key)) | .[] | to_entries | map({path: .value, covered: false})) as $uncovered |
+            (map(.endpoints .covered | to_entries | sort_by(.value) | map((.value[] | ascii_upcase) + " " + .key)) | .[] | to_entries | map({path: .value, covered: true})) as $covered |
+            $covered+$uncovered |
+            sort_by(.path) |
+            map(if .covered then "- [x] \(.path)" else "- [ ] \(.path)" end) |
+            .[]
+          ' $AFTER_COVERAGE | jq -r
+
       - name: Upload Coverage Data
         id: upload-coverage
         uses: actions/upload-artifact@v4
@@ -150,7 +162,7 @@ jobs:
               exit 1
             fi
           fi
-          
+
           gawk '
             BEGIN {
               RS = "(\r|\n|\r\n)"
@@ -176,7 +188,7 @@ jobs:
               HAD_CHANGES = 1
               print
             }
-          
+
             END {
               if (!HAD_CHANGES) {
                 print "**NO CHANGES**\n"

diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml
@@ -126,14 +126,16 @@ jobs:
 
       - name: Display Missing Test Paths
         run: |
-          jq -sc '
-            (map(.operations) | add | unique) as $all |
-            (map(.evaluated_operations) | add | unique) as $evaluated |
-            $all-$evaluated |
-            sort_by(.path) |
-            .[] |
-            "\(.method) \(.path)"
-          ' $(find ./ -name "test-spec-coverage-*.json")
+          jq -r -sc '
+              (map(.operations) | add | unique) as $all |
+              (map(.evaluated_operations) | add | unique) as $evaluated |
+              ($all-$evaluated | map({path: .path, method: .method, evaluated: false})) as $unevaluated |
+              ($evaluated | map({path: .path, method: .method, evaluated: true})) as $evaluated |
+              $evaluated+$unevaluated |
+              sort_by(.path) |
+              map(if .evaluated then "- [x] \(.method) \(.path)" else "- [ ] \(.method) \(.path)" end) |
+              .[]
+            ' $(find ./ -name "test-spec-coverage-*.json")
 
       - name: Combine Test Coverage Data
         shell: bash -eo pipefail {0}

diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md
@@ -355,7 +355,7 @@ npm run lint--fix
 
 ### [Analyze PR Changes](.github/workflows/analyze-pr-changes.yml)
 
-This workflow runs on all pull requests to analyze any potential changes to the specification. It uses the [coverage](#coverage) tool and [openapi-changes](https://pb33f.io/openapi-changes/) to calculate coverage metrics and provide a report on the changes when comparing with the commit at which the PR was branched off.
+This workflow runs on all pull requests to analyze any potential changes to the specification. It uses the [coverage](#coverage) tool and [openapi-changes](https://pb33f.io/openapi-changes/) to calculate coverage metrics and provide a report on the changes when comparing with the commit at which the PR was branched off. In addition, the "Display Coverage Checklist" step will output a list of OpenSearch APIs with a checkbox indicating whether the API exists in the spec that can be directly copy-pasted into the [Add Missing API Specs issue](https://github.com/opensearch-project/opensearch-api-specification/issues/168).
 
 ### [Build](.github/workflows/build.yml)
 

diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md
@@ -422,7 +422,7 @@ entry:
 
 Snapshot builds of OpenSearch are available on Docker Hub under [opensearchstaging/opensearch/tags](https://hub.docker.com/r/opensearchstaging/opensearch/tags).
 
-The following example in the test matrix will use [a snapshot build of OpenSearch 2.18](https://hub.docker.com/layers/opensearchstaging/opensearch/2.18.0/images/sha256-504a9c42bc1b13cb47b39a29db8a9d300d01b8851fb95dbb9db6770f478e45b5?context=explore) to execute the default test suite in [tests/default](tests/default/).
+The following example in the test matrix will use [a snapshot build of OpenSearch 2.18](https://hub.docker.com/r/opensearchstaging/opensearch/tags?name=2.18) to execute the default test suite in [tests/default](tests/default/).
 
 ```yaml
 - version: 2.18.0