Feature/forge py fargate (#47)

* update forge-py and update versioning in github actions * update version * update changelog * Update build.yml Updated name, added Build * Update CHANGELOG.md --------- Co-authored-by: James Wood <8115931+jamesfwood@users.noreply.github.com>
podaac · Jan 14, 2025 · e889817 · e889817
1 parent 3832efb
commit e889817
Show file tree

Hide file tree

Showing 10 changed files with 172 additions and 69 deletions.
diff --git a/.github/workflows/.release-created.yml b/.github/workflows/.release-created.yml
@@ -19,45 +19,24 @@ jobs:
       - uses: actions/checkout@v4
         with:
           ref: 'refs/heads/develop'
+
       - uses: actions/setup-python@v4
         with:
           python-version: '3.10'
+
       - name: Install Poetry
         uses: abatilo/actions-poetry@v3
         with:
           poetry-version: 1.8.5
+
       - name: Bump minor version
-        env:
-          COMMIT_VERSION: ${{ github.ref }}
         run: |
-
-          # only update the develop branch if were making #.#.0 release
-          # Get the branch name from the GITHUB_REF environment variable
-          branch_name=${GITHUB_REF#refs/heads/}
-
-          # Extract the last number in the branch name using a regular expression
-          if [[ $branch_name =~ /([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
-
-            first_number=${BASH_REMATCH[1]}
-            middle_number=${BASH_REMATCH[2]}
-            last_number=${BASH_REMATCH[3]}
-
-            # Increment the middle number by 1
-            incremented_middle_number=$((middle_number + 1))
-
-            # Check if the last number is '0'
-            if [ "$last_number" == "0" ]; then
-
-              update_version=$first_number.$incremented_middle_number.$last_number-alpha.1
-
-              poetry version $update_version
-              echo "software_version=$update_version" >> $GITHUB_ENV
-
-              git config --global user.name 'podaac-tig bot'
-              git config --global user.email 'podaac-tig@noreply.github.com'
-              git commit -am "/version ${{ env.software_version }}"
-              git push
-
-            fi
-
-          fi
+          poetry version ${GITHUB_REF#refs/heads/release/}
+          poetry version preminor
+          echo "software_version=$(poetry version | awk '{print $2}')" >> $GITHUB_ENV
+      - name: Commit Version Bump
+        run: |
+          git config --global user.name 'podaac-hitide-backfill-tool bot'
+          git config --global user.email 'podaac-hitide-backfill-tool@noreply.github.com'
+          git commit -am "/version ${{ env.software_version }}"
+          git push
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -18,6 +18,7 @@ env:
   PYTHON_VERSION: '3.10'
   POETRY_VERSION: '1.8.5'
   TERRAFORM_VERSION: '1.5.3'
+  CUMULUS_VERSION: 'v18.5.2'
 
 jobs:
   create-zips:
@@ -37,7 +38,12 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: nasa/cumulus
-          ref: v18.5.2
+          ref: ${{ env.CUMULUS_VERSION }}
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Setup Node.js
         uses: actions/setup-node@v4
@@ -53,10 +59,27 @@ jobs:
           FORMATTED_VERSION="nodejs${VERSION}.x"
           echo "cumulus_node_version=$FORMATTED_VERSION" >> $GITHUB_OUTPUT
 
-      - name: Setup Python
-        uses: actions/setup-python@v5
+      # Cache the node_modules
+      - name: Cache Node Modules
+        id: cache-node-modules
+        uses: actions/cache@v4
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          path: |
+            node_modules
+            */*/node_modules
+          key: node-modules-${{ env.CUMULUS_VERSION }}-${{ hashFiles('**/package-lock.json') }}
+
+      # Cache the build outputs
+      - name: Cache Build Outputs
+        id: cache-build
+        uses: actions/cache@v4
+        with:
+          path: |
+            tasks/post-to-cmr/dist
+            tasks/hyrax-metadata-updates/dist
+            tf-modules/cumulus
+          key: backfill-tool-cumulus-build-${{ env.CUMULUS_VERSION }}
+          retention-days: 150
 
       - name: Cache npm dependencies
         uses: actions/cache@v4
@@ -70,7 +93,8 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-npm-
 
-      - name: Install Dependencies
+      - name: Install Dependencies and Build
+        if: steps.cache-node-modules.outputs.cache-hit != 'true'
         run: |
           npm install -g lerna
           npm install
@@ -130,39 +154,32 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-poetry-
 
-      - name: Set Default Version and Environment
-        id: default-version
+      - name: Version Management
+        id: versioning
         run: |
-          echo "pyproject_name=$(poetry version | awk '{print $1}')" >> $GITHUB_ENV
-          VERSION=$(poetry version | awk '{print $2}')
+          current_version=$(poetry version -s)
+          base_version=$(echo "$current_version" | grep -oE '^[0-9]+\.[0-9]+\.[0-9]+')
           
-          # Set default environment based on branch
+          # Version calculation based on branch
           if [[ "${{ github.ref }}" =~ ^refs/heads/(issue|feature|dependabot)/ ]]; then
+            new_version="${base_version%%-*}+$(git rev-parse --short HEAD)"
             echo "TARGET_ENV_UPPERCASE=SIT" >> $GITHUB_ENV
-            NEW_VERSION="${VERSION}+$(git rev-parse --short HEAD)"
           elif [[ "${{ github.ref }}" == "refs/heads/develop" ]]; then
             echo "TARGET_ENV_UPPERCASE=SIT" >> $GITHUB_ENV
-            if [[ ${VERSION} =~ -alpha ]]; then
-              ALPHA_NUM=$(echo "${VERSION}" | grep -oP '(?<=-alpha.)\d+' || echo "0")
-              NEW_VERSION="${VERSION%-alpha.*}-alpha.$((ALPHA_NUM+1))"
-            else
-              NEW_VERSION="${VERSION}-alpha.1"
-            fi
+            new_version=$(poetry version prerelease -s)
           elif [[ "${{ github.ref }}" =~ ^refs/heads/release/ ]]; then
             echo "TARGET_ENV_UPPERCASE=UAT" >> $GITHUB_ENV
-            if [[ ${VERSION} =~ -rc ]]; then
-              RC_NUM=$(echo "${VERSION}" | grep -oP '(?<=-rc.)\d+' || echo "0")
-              NEW_VERSION="${VERSION%-rc.*}-rc.$((RC_NUM+1))"
+            if [[ ${current_version} =~ rc ]]; then
+              new_version=$(poetry version prerelease -s)
             else
-              NEW_VERSION="${VERSION}-rc.1"
+              new_version="${GITHUB_REF#refs/heads/release/}rc1"
             fi
           elif [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
             echo "TARGET_ENV_UPPERCASE=OPS" >> $GITHUB_ENV
-            NEW_VERSION="${VERSION%-rc*}"
-            NEW_VERSION="${NEW_VERSION%-alpha*}"
+            new_version=${base_version}
           fi
           
-          echo "new_version=${NEW_VERSION}" >> $GITHUB_ENV
+          echo "new_version=${new_version}" >> $GITHUB_ENV
 
       - name: Override Environment for Deploy Command
         if: contains(github.event.head_commit.message, '/deploy')
@@ -293,4 +310,4 @@ jobs:
           
           terraform init
           terraform plan -var-file=tfvars/"$TARGET_ENV".tfvars -var="app_version=${{ needs.build.outputs.new_version }}" -out=tfplan
-          terraform apply -auto-approve tfplan
+          terraform apply -auto-approve tfplan
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -21,11 +21,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   - List can be a list of GranuleURs or granule concept-IDs
 - Update db size from t2.micro to t3.micro
 - Made arguments *--cumulus-configurations* and *--default-message-config* optional in preview mode
-- Updated forge-py to 0.3.0
+- Updated forge-py to 0.4.0
 - Updated cumulus-postworkflow-normalizer to 0.4.1
 - Updated hitide-backfill-lambdas to 0.4.1
 - Update metadata aggregator to cumulus-metadata-aggregator-8.7.0-alpha.6-SNAPSHOT
 - Update forge-py memory to 2048
+- Added in forge-py fargate
+- Updated forge and tig workflow to fork based on granule size when determine lambda or fargate
+- Updated github actions workflow and versioning
 ### Deprecated
 ### Removed
 ### Fixed

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "podaac-hitide-backfill-tool"
-version = "0.10.0-rc.15"
+version = "0.10.0rc15"
 
 description = "Backfill footprints and images for HiTIDE collections"
 authors = ["PO.DAAC <podaac@jpl.nasa.gov>"]

diff --git a/terraform-deploy/forge_py.tf b/terraform-deploy/forge_py.tf
@@ -12,4 +12,18 @@ module "forge_py_module" {
     subnet_ids = data.aws_subnets.private.ids
     memory_size = 2048
     timeout = 900
+
+    # ECS Variables
+
+    # Fargate Variables
+    ecs_cluster_arn = aws_ecs_cluster.main.arn
+    forge_py_fargate = true
+    fargate_iam_role = aws_iam_role.fargate_execution.arn
+    ecs_cluster_name = aws_ecs_cluster.main.name
+    fargate_max_capacity = 100
+    fargate_cpu = 2048
+    fargate_memory = 16384
+    threshold_scale_down = 25
+    period_scale_down = 240
+
 }
diff --git a/terraform-deploy/forge_step.tf b/terraform-deploy/forge_step.tf
@@ -57,10 +57,10 @@ resource "aws_sfn_state_machine" "forge" {
             },
             {
               "Variable": "$.meta.collection.meta.workflowChoice.ecs_lambda",
-              "BooleanEquals": true
+              "StringEquals": "ecs"
             }
           ],
-          "Next": "FootprintProcessFargate"
+          "Next": "FootprintBranchingFargate"
         }
       ],
       "Default": "FootprintBranching"
@@ -100,6 +100,60 @@ resource "aws_sfn_state_machine" "forge" {
       ],
       "Next": "FootprintBranchChoice"
     },
+    "FootprintBranchingFargate": {
+      "Parameters": {
+        "cma": {
+          "event.$": "$",
+          "task_config": {
+            "execution_name": "{$.cumulus_meta.execution_name}",
+            "collection": "{$.meta.collection}",
+            "cumulus_message": {
+              "input": "{$.payload}"
+            }
+          }
+        }
+      },
+      "Type": "Task",
+      "Resource": "${module.forge_py_module.forge_branch_task_lambda_arn}",
+      "Catch": [
+        {
+          "ErrorEquals": [
+            "States.ALL"
+          ],
+          "ResultPath": "$.exception",
+          "Next": "WorkflowFailed"
+        }
+      ],
+      "Retry": [
+        {
+          "ErrorEquals": [
+            "States.ALL"
+          ],
+          "IntervalSeconds": 5,
+          "MaxAttempts": 2
+        }
+      ],
+      "Next": "FootprintFargateBranchChoice"
+    },
+    "FootprintFargateBranchChoice": {
+      "Type": "Choice",
+      "Choices": [
+        {
+          "And": [
+            {
+              "Variable": "$.meta.collection.meta.workflowChoice.forge_version",
+              "IsPresent": true
+            },
+            {
+              "Variable": "$.meta.collection.meta.workflowChoice.forge_version",
+              "StringEquals": "forge-py"
+            }
+          ],
+          "Next": "ForgePyProcessFargate"
+        }
+      ],
+      "Default": "FootprintProcessFargate"
+    },
     "FootprintBranchChoice": {
       "Type": "Choice",
       "Choices": [
@@ -154,6 +208,42 @@ resource "aws_sfn_state_machine" "forge" {
       ],
       "Next": "MetadataAggregator"
     },
+    "ForgePyProcessFargate": {
+      "Parameters": {
+        "cma": {
+          "event.$": "$",
+          "task_config": {
+            "execution_name": "{$.cumulus_meta.execution_name}",
+            "collection": "{$.meta.collection}",
+            "cumulus_message": {
+              "input": "{$.payload}"
+            }
+          }
+        }
+      },
+      "Type": "Task",
+      "Resource": "${module.forge_py_module.forge_py_ecs_task_id}",
+      "TimeoutSeconds": 10800,
+      "Catch": [
+        {
+          "ErrorEquals": [
+            "States.ALL"
+          ],
+          "ResultPath": "$.exception",
+          "Next": "WorkflowFailed"
+        }
+      ],
+      "Retry": [
+        {
+          "ErrorEquals": [
+            "States.ALL"
+          ],
+          "IntervalSeconds": 5,
+          "MaxAttempts": 2
+        }
+      ],
+      "Next": "MetadataAggregator"
+    },
     "ForgeProcess": {
       "Parameters": {
         "cma": {

diff --git a/terraform-deploy/terraform_env/ops.json b/terraform-deploy/terraform_env/ops.json
@@ -3,8 +3,8 @@
    "tig_image":"ghcr.io/podaac/tig:0.12.0",
    "forge_source":"https://github.com/podaac/forge/releases/download/0.11.0/forge-0.11.0.zip",
    "forge_image":"ghcr.io/podaac/forge:0.11.0",
-   "forge_py_source" : "https://github.com/podaac/forge-py/releases/download/0.3.0/forge-py-terraform-0.3.0.zip",
-   "forge_py_image": "ghcr.io/podaac/forge-py:0.3.0",
+   "forge_py_source" : "https://github.com/podaac/forge-py/releases/download/0.3.0/forge-py-terraform-0.4.0.zip",
+   "forge_py_image": "ghcr.io/podaac/forge-py:0.4.0",
    "backfill_lambdas_source": "https://github.com/podaac/hitide-backfill-lambdas/releases/download/0.4.1/hitide-backfill-lambdas-0.4.1.zip",
    "postworkflow_normalizer_source": "https://github.com/podaac/cumulus-postworkflow-normalizer/releases/download/0.4.1/postworkflow-normalizer-0.4.1.zip"
 }
diff --git a/terraform-deploy/terraform_env/sit.json b/terraform-deploy/terraform_env/sit.json
@@ -3,8 +3,8 @@
    "tig_image":"ghcr.io/podaac/tig:0.12.0-rc.2",
    "forge_source":"https://github.com/podaac/forge/releases/download/0.11.0-rc.2/forge-0.11.0-rc.2.zip",
    "forge_image":"ghcr.io/podaac/forge:0.11.0-rc.2",
-   "forge_py_source" : "https://github.com/podaac/forge-py/releases/download/0.3.0-rc.4/forge-py-terraform-0.3.0-rc.4.zip",
-   "forge_py_image": "ghcr.io/podaac/forge-py:0.3.0-rc.4",
+   "forge_py_source" : "https://github.com/podaac/forge-py/releases/download/0.4.0%2Bc2c0b46/forge-py-terraform-0.4.0+c2c0b46.zip",
+   "forge_py_image": "ghcr.io/podaac/forge-py:0.4.0-c2c0b46",
    "backfill_lambdas_source": "https://github.com/podaac/hitide-backfill-lambdas/releases/download/0.5.0-alpha.2/hitide-backfill-lambdas-0.5.0-alpha.2.zip",
    "postworkflow_normalizer_source": "https://github.com/podaac/cumulus-postworkflow-normalizer/releases/download/0.4.1-rc.1/postworkflow-normalizer-0.4.1-rc.1.zip"
 }
diff --git a/terraform-deploy/terraform_env/uat.json b/terraform-deploy/terraform_env/uat.json
@@ -3,8 +3,8 @@
    "tig_image":"ghcr.io/podaac/tig:0.12.0-rc.3",
    "forge_source":"https://github.com/podaac/forge/releases/download/0.11.0-rc.3/forge-0.11.0-rc.3.zip",
    "forge_image":"ghcr.io/podaac/forge:0.11.0-rc.3",
-   "forge_py_source" : "https://github.com/podaac/forge-py/releases/download/0.3.0-rc.4/forge-py-terraform-0.3.0-rc.4.zip",
-   "forge_py_image": "ghcr.io/podaac/forge-py:0.3.0-rc.4",
+   "forge_py_source" : "https://github.com/podaac/forge-py/releases/download/0.4.0rc3/forge-py-terraform-0.4.0rc3.zip",
+   "forge_py_image": "ghcr.io/podaac/forge-py:0.4.0rc3",
    "backfill_lambdas_source": "https://github.com/podaac/hitide-backfill-lambdas/releases/download/0.4.1-rc.2/hitide-backfill-lambdas-0.4.1-rc.2.zip",
    "postworkflow_normalizer_source": "https://github.com/podaac/cumulus-postworkflow-normalizer/releases/download/0.4.1-rc.1/postworkflow-normalizer-0.4.1-rc.1.zip"
 }
diff --git a/terraform-deploy/tig_step.tf b/terraform-deploy/tig_step.tf
@@ -57,7 +57,7 @@ resource "aws_sfn_state_machine" "tig" {
             },
             {
               "Variable": "$.meta.collection.meta.workflowChoice.ecs_lambda",
-              "BooleanEquals": true
+              "StringEquals": "ecs"
             }
           ],
           "Next": "ECSImageProcess"