Viirs orchestration

.github/workflows/dev_viirs_deployment.yml

@@ -0,0 +1,79 @@
+name: uah dev viirs deployment
+
+on:
+  push:
+    branches: [viirs_orchestration]
+  workflow_dispatch:
+
+env:
+  AWS_DEFAULT_REGION: us-west-2
+
+jobs:
+  unit-tests:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python: [3.8]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install Tox
+        run: pip install tox
+      - name: Run Tox test environment
+        # Run tox using the version of Python in `PATH`
+        run: tox -e py
+  dev-viirs-deployment:
+    strategy:
+      matrix:
+        python: [3.8]
+    runs-on: ubuntu-20.04
+    needs: [unit-tests]
+    environment:
+      name: dev-viirs
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install tox
+        run: pip install tox
+      - name: Configure awscli
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+      - name: Convert secrets to environment variables
+        env:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          SECRETS_JSON: ${{ toJson(secrets) }}
+        run: |
+          while read -rd $'' line; do
+            echo "$line" >> $GITHUB_ENV
+          done < <(
+            jq -r <<<"$SECRETS_JSON" 'to_entries|map("\(.key)=\(.value)\u0000")[]'
+          )
+      - name: Convert vars to environment variables
+        env:
+          VARS_JSON: ${{ toJson(vars) }}
+        run: |
+          while read -rd $'' line; do
+            echo "$line" >> $GITHUB_ENV
+          done < <(
+            jq -r <<<"$VARS_JSON" 'to_entries|map("\(.key)=\(.value)\u0000")[]'
+          )
+      - name: Deploy with tox
+        run: tox -v -e dev -r -- deploy --require-approval never
+      - name: Run DB setup
+        run: |
+          setupdb=$(aws cloudformation describe-stacks \
+            --stack-name $HLS_STACKNAME \
+            --query "Stacks[0].Outputs[?OutputKey=='setupdbexport'].OutputValue" \
+            --output=text
+          )
+          aws lambda invoke --function-name=$setupdb response.json

.github/workflows/mcp_dev_deployment.yml

@@ -1,4 +1,4 @@
-name: mcp dev deployment
+name: mcp development deployment
 
 on:
   push:
@@ -30,7 +30,7 @@ jobs:
       - name: Run Tox test environment
         # Run tox using the version of Python in `PATH`
         run: tox -e py
-  mcp-dev-deployment:
+  mcp-development-deployment:
     strategy:
       matrix:
         python: [3.8]

.github/workflows/mcp_dev_viirs_deployment.yml

@@ -0,0 +1,82 @@
+name: mcp dev viirs deployment
+
+on:
+  push:
+    branches: [viirs_orchestration]
+  workflow_dispatch:
+
+env:
+  AWS_DEFAULT_REGION: us-west-2
+
+# See https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-amazon-web-services#updating-your-github-actions-workflow
+permissions:
+  id-token: write # required for requesting the JWT
+  contents: read  # required for actions/checkout
+
+jobs:
+  unit-tests:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python: [3.8]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install Tox
+        run: pip install tox
+      - name: Run Tox test environment
+        # Run tox using the version of Python in `PATH`
+        run: tox -e py
+  mcp-dev-viirs-deployment:
+    strategy:
+      matrix:
+        python: [3.8]
+    runs-on: ubuntu-20.04
+    needs: [unit-tests]
+    environment:
+      name: mcp-dev-viirs
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install tox
+        run: pip install tox
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v3
+        with:
+          role-to-assume: ${{ vars.AWS_ROLE_TO_ASSUME_ARN }}
+          role-session-name: ${{ github.actor }}
+          aws-region: us-west-2
+      - name: Convert secrets to environment variables
+        env:
+          SECRETS_JSON: ${{ toJson(secrets) }}
+        run: |
+          while read -rd $'' line; do
+            echo "$line" >> $GITHUB_ENV
+          done < <(
+            jq -r <<<"$SECRETS_JSON" 'to_entries|map("\(.key)=\(.value)\u0000")[]'
+          )
+      - name: Convert vars to environment variables
+        env:
+          VARS_JSON: ${{ toJson(vars) }}
+        run: |
+          while read -rd $'' line; do
+            echo "$line" >> $GITHUB_ENV
+          done < <(
+            jq -r <<<"$VARS_JSON" 'to_entries|map("\(.key)=\(.value)\u0000")[]'
+          )
+      - name: Deploy with tox
+        run: tox -v -e dev -r -- deploy --require-approval never
+      - name: Run DB setup
+        run: |
+          setupdb=$(aws cloudformation describe-stacks \
+            --stack-name $HLS_STACKNAME \
+            --query "Stacks[0].Outputs[?OutputKey=='setupdbexport'].OutputValue" \
+            --output=text
+          )
+          aws lambda invoke --function-name=$setupdb response.json

.github/workflows/mcp_production_deployment.yml

@@ -0,0 +1,81 @@
+name: MCP production deployment
+
+on:
+  release:
+    types: [ published ]
+
+env:
+  AWS_DEFAULT_REGION: us-west-2
+
+# See https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-amazon-web-services#updating-your-github-actions-workflow
+permissions:
+  id-token: write # required for requesting the JWT
+  contents: read  # required for actions/checkout
+
+jobs:
+  unit-tests:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python: [3.8]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install Tox
+        run: pip install tox
+      - name: Run Tox test environment
+        # Run tox using the version of Python in `PATH`
+        run: tox -e py
+  mcp-production-deployment:
+    strategy:
+      matrix:
+        python: [3.8]
+    runs-on: ubuntu-20.04
+    needs: [unit-tests]
+    environment:
+      name: mcp-production
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install tox
+        run: pip install tox
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v3
+        with:
+          role-to-assume: ${{ vars.AWS_ROLE_TO_ASSUME_ARN }}
+          role-session-name: ${{ github.actor }}
+          aws-region: us-west-2
+      - name: Convert secrets to environment variables
+        env:
+          SECRETS_JSON: ${{ toJson(secrets) }}
+        run: |
+          while read -rd $'' line; do
+            echo "$line" >> $GITHUB_ENV
+          done < <(
+            jq -r <<<"$SECRETS_JSON" 'to_entries|map("\(.key)=\(.value)\u0000")[]'
+          )
+      - name: Convert vars to environment variables
+        env:
+          VARS_JSON: ${{ toJson(vars) }}
+        run: |
+          while read -rd $'' line; do
+            echo "$line" >> $GITHUB_ENV
+          done < <(
+            jq -r <<<"$VARS_JSON" 'to_entries|map("\(.key)=\(.value)\u0000")[]'
+          )
+      - name: Deploy with tox
+        run: tox -v -e dev -r -- deploy --require-approval never
+      - name: Run DB setup
+        run: |
+          setupdb=$(aws cloudformation describe-stacks \
+            --stack-name $HLS_STACKNAME \
+            --query "Stacks[0].Outputs[?OutputKey=='setupdbexport'].OutputValue" \
+            --output=text
+          )
+          aws lambda invoke --function-name=$setupdb response.json

lambda_functions/laads_available.py

@@ -17,22 +17,17 @@
 from botocore.errorfactory import ClientError
 
 s3 = boto3.client("s3")
-bucket = os.getenv("LAADS_BUCKET", None)
-print(bucket)
-if bucket is None:
-    raise Exception("No Bucket set")
 
 
-def key_exists(bucket: str, key: str):
+def key_pattern_exists(bucket: str, key_pattern: str):
     try:
-        s3.head_object(Bucket=bucket, Key=key)
+        response = s3.list_objects_v2(Bucket=bucket, Prefix=key_pattern)
+        if "Contents" in response:
+            return True
+        else:
+            return False
     except ClientError as e:
         print(e)
-        if e.response["Error"]["Code"] == "404":
-            return False
-        else:
-            raise Exception(e)
-    return True
 
 
 def getyyyydoy(date_str: str):
@@ -65,16 +60,25 @@ def handler(event: Dict, context: Dict):
     if date_str is None:
         raise Exception("Missing Date Parameter")
 
+    bucket = os.getenv("LAADS_BUCKET", None)
+    if bucket is None:
+        raise Exception("No Bucket set")
     ydoy, year = getyyyydoy(date_str)
-    key = f"lasrc_aux/LADS/{year}/L8ANC{ydoy}.hdf_fused"
-    print(f"------{bucket}    {key} ------")
-    exists = key_exists(bucket, key)
+    vj_pattern = f"lasrc_aux/LADS/{year}/VJ104ANC.A{ydoy}"
+    print(f"------{bucket}    {vj_pattern} ------")
+    vj_exists = key_pattern_exists(bucket, vj_pattern)
+    vnp_pattern = f"lasrc_aux/LADS/{year}/VNP04ANC.A{ydoy}"
+    vnp_exists = key_pattern_exists(bucket, vnp_pattern)
+    if vj_exists or vnp_exists:
+        exists = True
+    else:
+        exists = False
     output = {
         "granule": date_str,
         "year": year,
         "doy": ydoy,
         "bucket": bucket,
-        "key": key,
+        "pattern": f"{vj_pattern} {vnp_pattern}",
         "available": False,
     }
     if exists:

lambda_functions/laads_submit_climatology_job.py

@@ -0,0 +1,34 @@
+import datetime
+import json
+import os
+
+import boto3
+
+batch_client = boto3.client("batch")
+
+
+jobQueue = os.getenv("JOB_QUEUE")
+jobDefinition = os.getenv("JOB_DEFINITION")
+lasrc_aux = os.getenv("LASRC_AUX_DIR")
+laads_token = os.getenv("LAADS_TOKEN")
+laads_bucket = os.getenv("LAADS_BUCKET")
+
+
+def handler(event, context):
+    current_date = datetime.datetime.now()
+    response = batch_client.submit_job(
+        jobName="laads_climatology",
+        jobQueue=jobQueue,
+        jobDefinition=jobDefinition,
+        containerOverrides={
+            "environment": [
+                {"name": "LASRC_AUX_DIR", "value": lasrc_aux},
+                {"name": "LAADS_TOKEN", "value": laads_token},
+                {"name": "LAADS_BUCKET", "value": laads_bucket},
+                {"name": "CLIM_YEAR", "value": str(current_date.year)},
+                {"name": "CLIM_MONTH", "value": str(current_date.month)},
+            ],
+            "command": ["./usr/local/climatologies.sh"],
+        },
+    )
+    return response