feat: add collection validation

Includes: - Python dependencies file and instructions - CI - Actual fixes
NASA-IMPACT · Sep 19, 2023 · a3bf6a5 · a3bf6a5
1 parent 2a6b114
commit a3bf6a5
Show file tree

Hide file tree

Showing 17 changed files with 190 additions and 50 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -0,0 +1,23 @@
+name: Continuous integration
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+          cache: "pip"
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+      - name: Validate collections
+        run: python scripts/validate_collection.py
diff --git a/README.md b/README.md
@@ -105,3 +105,35 @@ Should follow the following format:
     "dry_run": "<true/false>",
 }
 ```
+
+## Validation
+
+This repository provides a script for validating all collections.
+First, install the requirements (preferably in a virtual environment):
+
+```shell
+pip install -r requirements.txt
+```
+
+Then:
+
+```shell
+python scripts/validate_collections.py
+```
+
+## Development
+
+If you need to add new dependencies, first install the requirements:
+
+```shell
+pip install -r requirements.txt
+```
+
+Add your dependency to `requirements.in` *without a version specifier* (unless you really need one).
+Then run:
+
+```shell
+pip-compile
+```
+
+This will update `requirements.txt` with a complete, realized set of Python dependencies.
diff --git a/ingestion-data/collections/caldor-fire-behavior.json b/ingestion-data/collections/caldor-fire-behavior.json
@@ -8,10 +8,10 @@
         "spatial":{
             "bbox":[
                 [
-                    "-180",
-                    "90",
-                    "-90",
-                    "180"
+                    -180,
+                    90,
+                    -90,
+                    180
                 ]
             ]
         },

diff --git a/ingestion-data/collections/caldor-fire-burn-severity.json b/ingestion-data/collections/caldor-fire-burn-severity.json
@@ -8,10 +8,10 @@
         "spatial":{
             "bbox":[
                 [
-                    "-180",
-                    "90",
-                    "-90",
-                    "180"
+                    -180,
+                    90,
+                    -90,
+                    180
                 ]
             ]
         },

diff --git a/ingestion-data/collections/disturbance-probability.json b/ingestion-data/collections/disturbance-probability.json
@@ -1,16 +1,26 @@
 {
-    "collection": "disturbance-probability-percentile",
+    "type": "Collection",
+    "stac_version": "1.0.0",
+    "id": "disturbance-probability-percentile",
     "title": "Near Real-time Disturbance probability map (%)",
     "data_type": "cog",
-    "spatial_extent": {
-        "xmin": -84.132,
-        "ymin": 25.224,
-        "xmax": -79.853,
-        "ymax": 30.728
-    },
-    "temporal_extent": {
-        "startdate": "2022-10-03T00:00:00Z",
-        "enddate": "2022-10-03T23:59:59Z"
+    "extent": {
+        "spatial": {
+            "bbox": [
+                -84.132,
+                25.224,
+                -79.853,
+                30.728
+            ]
+        },
+        "temporal": {
+            "interval": [
+                [
+                    "2022-10-03T00:00:00Z",
+                    "2022-10-03T23:59:59Z"
+                ]
+            ]
+        }
     },
     "license": "CC-BY-NC-SA-1.0",
     "providers": [
@@ -52,5 +62,6 @@
             "bucket": "veda-data-store-staging",
             "filename_regex": "(.*)spec_prob_mosaic_2022-10-03_day.tif$"
         }
-    ]
+    ],
+    "links": []
 }
diff --git a/ingestion-data/collections/ecco-surface-height-change.json b/ingestion-data/collections/ecco-surface-height-change.json
@@ -8,10 +8,10 @@
         "spatial":{
             "bbox":[
                 [
-                    "-180",
-                    "90",
-                    "-90",
-                    "180"
+                    -180,
+                    90,
+                    -90,
+                    180
                 ]
             ]
         },

diff --git a/ingestion-data/collections/lis-tws-nonstationarity-index.json b/ingestion-data/collections/lis-tws-nonstationarity-index.json
@@ -20,7 +20,7 @@
             ]
         }
     },
-    "license": "Creative Commons Zero (CC0-1.0)",
+    "license": "CC0-1.0",
     "description": "The global Terrestrial Water Storage (TWS) non-stationarity index integrates the trend, seasonal shifts, and variability change of TWS for the period of 2003 - 2020. TWS is derived by jointly assimilating the MODIS Leaf Area Index, the ESA CCI surface soil moisture, and the GSFC GRACE mascon-based TWS anomalies into the Noah-MP land surface model within the NASA Land Information System (LIS) at 10 km spatial resolution forced by the combination of MERRA2 and IMERG meteorological fields. The smaller the non-stationarity index is, the more the water cycle is under a non-stationary process. Glaciers and Greenland are excluded from the analysis.",
     "item_assets": {
         "cog_default": {

diff --git a/ingestion-data/collections/modis-fire-anomalies-diff-covid-19-changing-landscape.json b/ingestion-data/collections/modis-fire-anomalies-diff-covid-19-changing-landscape.json
@@ -8,10 +8,10 @@
         "spatial":{
             "bbox":[
                 [ 
-                    "-125.0",
-                    "24.0",
-                    "-75.0",
-                    "43.0"
+                    -125.0,
+                    24.0,
+                    -75.0,
+                    43.0
                 ]
             ]
         },

diff --git a/ingestion-data/collections/mtbs-burn-severity.json b/ingestion-data/collections/mtbs-burn-severity.json
@@ -8,10 +8,10 @@
         "spatial":{
             "bbox":[
                 [
-                    "-126.49459612498832",
-                    "24.0478678762251",
-                    "-71.50752568733597",
-                    "50.55916724898132"
+                    -126.494596,
+                    24.047867,
+                    -71.507525,
+                    50.559167
                 ]
             ]
         },

diff --git a/ingestion-data/collections/nceo-africa-2017.json b/ingestion-data/collections/nceo-africa-2017.json
@@ -12,7 +12,7 @@
     "title": "NCEO Africa Aboveground Woody Biomass 2017",
     "extent": {
         "spatial": {
-        "   bbox": [[-18.2735295, -35.054059, 51.8642329, 37.7310386]]
+            "bbox": [[-18.2735295, -35.054059, 51.8642329, 37.7310386]]
         },
         "temporal": {
             "interval": [["2017-01-01T00:00:00Z", "2018-01-01T00:00:00Z"]]

diff --git a/ingestion-data/collections/pzd-anomaly-covid-19.json b/ingestion-data/collections/pzd-anomaly-covid-19.json
@@ -8,18 +8,18 @@
         "spatial":{
             "bbox":[
                 [
-                    "-92.2",
-                    "40.9975",
-                    "-76.0",
-                    "49.09"
+                    -92.2,
+                    40.9975,
+                    -76.0,
+                    49.09
                 ]
             ]
         },
         "temporal":{
             "interval":[
                 [
                     "2020-01-01T00:00:00Z",
-                    "2021-23-31T23:59:59Z"
+                    "2021-12-31T23:59:59Z"
                 ]
             ]
         }

diff --git a/ingestion-data/collections/recovery-proxy-maps-covid-19.json b/ingestion-data/collections/recovery-proxy-maps-covid-19.json
@@ -8,10 +8,10 @@
         "spatial":{
             "bbox":[
                 [
-                    "-180",
-                    "90",
-                    "-90",
-                    "180"
+                    -180,
+                    90,
+                    -90,
+                    180
                 ]
             ]
         },

diff --git a/ingestion-data/collections/slowdown-proxy-covid-19-changing-landscapes.json b/ingestion-data/collections/slowdown-proxy-covid-19-changing-landscapes.json
@@ -8,10 +8,10 @@
         "spatial":{
             "bbox":[
                 [
-                    "-180",
-                    "90",
-                    "-90",
-                    "180"
+                    -180,
+                    90,
+                    -90,
+                    180
                 ]
             ]
         },

diff --git a/ingestion-data/collections/togo-agriculture-covid-19.json b/ingestion-data/collections/togo-agriculture-covid-19.json
@@ -8,10 +8,10 @@
         "spatial":{
             "bbox":[
                 [ 
-                    "-0.14",
-                    "6.10",
-                    "1.80",
-                    "11.13"
+                    -0.14,
+                    6.10,
+                    1.80,
+                    11.13
                 ]
             ]
         },

diff --git a/requirements.in b/requirements.in
@@ -0,0 +1,2 @@
+pip-tools
+pystac[validation]
diff --git a/requirements.txt b/requirements.txt
@@ -0,0 +1,34 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile
+#
+attrs==23.1.0
+    # via jsonschema
+build==1.0.3
+    # via pip-tools
+click==8.1.7
+    # via pip-tools
+jsonschema==4.17.3
+    # via pystac
+packaging==23.1
+    # via build
+pip-tools==7.3.0
+    # via -r requirements.in
+pyproject-hooks==1.0.0
+    # via build
+pyrsistent==0.19.3
+    # via jsonschema
+pystac[validation]==1.8.3
+    # via -r requirements.in
+python-dateutil==2.8.2
+    # via pystac
+six==1.16.0
+    # via python-dateutil
+wheel==0.41.2
+    # via pip-tools
+
+# The following packages are considered to be unsafe in a requirements file:
+# pip
+# setuptools
diff --git a/scripts/validate_collections.py b/scripts/validate_collections.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+
+"""Validates all collections in ingestion-data/collections"""
+
+import json
+import sys
+from pathlib import Path
+
+from pystac import Collection, STACValidationError
+
+root = Path(__file__).parents[1]
+collections = root / "ingestion-data" / "collections"
+
+errors = dict()
+for path in collections.rglob("*.json"):
+    try:
+        collection = Collection.from_file(str(path))
+    except Exception as error:
+        errors[path.name] = {
+            "type": "error",
+            "message": f"cannot read collection, {type(error)}: {error}",
+        }
+        continue
+    try:
+        collection.validate()
+    except STACValidationError as error:
+        if isinstance(error.source, list):
+            message = [str(e) for e in error.source]
+        else:
+            message = str(error.source)
+        errors[path.name] = {
+            "type": "invalid",
+            "message": message,
+        }
+
+if errors:
+    json.dump(errors, sys.stdout, indent=2)
+    sys.exit(1)