[PROD] Fix seeding; sort by date not string; add code coverage; soft delete duplicate and orphaned objectives

.circleci/config.yml

@@ -26,6 +26,19 @@ executors:
     docker:
       - image: cimg/aws:2024.03
 commands:
+  install_cf_tools:
+    description: "Install Cloud Foundry CLI"
+    steps:
+      - run:
+          name: Install CF tools
+          command: |
+            # Install Cloud Foundry CLI
+            wget -q -O - https://packages.cloudfoundry.org/debian/cli.cloudfoundry.org.key | sudo apt-key add -
+            echo "deb https://packages.cloudfoundry.org/debian stable main" | sudo tee /etc/apt/sources.list.d/cloudfoundry-cli.list
+            sudo apt-get update
+            sudo apt-get install -y cf8-cli
+            # Install plugin needed for connect-to-service
+            cf install-plugin -f https://github.com/cloud-gov/cf-service-connect/releases/download/v1.1.3/cf-service-connect_linux_amd64
   sparse_checkout:
     description: "Checkout sparse directories from a specific branch."
     parameters:
@@ -353,6 +366,14 @@ commands:
               -p ${<< parameters.cloudgov_password >>} \
               -o << pipeline.parameters.cg_org >> \
               -s ${<< parameters.cloudgov_space >>}
+      - run:
+          name: Acquire Lock
+          command: |
+            chmod +x ./automation/ci/scripts/*-lock.sh
+            ./automation/ci/scripts/acquire-lock.sh \
+              "<< parameters.app_name >>" \
+              "<< parameters.build_branch >>" \
+              "<< pipeline.number >>"
       - run:
           name: Push application with deployment vars
           command: |
@@ -386,6 +407,15 @@ commands:
               --var BUILD_COMMIT=<< parameters.build_commit >> \
               --var BUILD_NUMBER=<< pipeline.number >> \
               --var BUILD_TIMESTAMP="$(date +"%Y-%m-%d %H:%M:%S")"
+      - run:
+          name: Release Lock
+          command: |
+            chmod +x ./automation/ci/scripts/*-lock.sh
+            ./automation/ci/scripts/release-lock.sh \
+              "<< parameters.app_name >>" \
+              "<< parameters.build_branch >>" \
+              "<< pipeline.number >>"
+          when: always
       # - run:
       #     name: Push maintenance application
       #     command: |
@@ -418,12 +448,28 @@ commands:
               -p ${<< parameters.cloudgov_password >>} \
               -o << pipeline.parameters.cg_org >> \
               -s ${<< parameters.cloudgov_space >>}
+      - run:
+          name: Acquire Lock
+          command: |
+            chmod +x ./automation/ci/scripts/*-lock.sh
+            ./automation/ci/scripts/acquire-lock.sh \
+              "tta-automation" \
+              "<< pipeline.git.branch >>" \
+              "<< pipeline.number >>"
       - run:
           name: Migrate database
           command: |
             cf run-task << parameters.app_name >> \
               --command "yarn db:migrate:prod" \
               --name "migrate"
+      - run:
+          name: Release Lock
+          command: |
+            chmod +x ./automation/ci/scripts/*-lock.sh
+            ./automation/ci/scripts/release-lock.sh \
+              "tta-automation" \
+              "<< pipeline.git.branch >>" \
+              "<< pipeline.number >>"
   cf_automation_task:
     description: "Login to Cloud Foundry space, run automation task, and send notification"
     parameters:
@@ -508,6 +554,14 @@ commands:
               -p ${<< parameters.cloudgov_password >>} \
               -o << pipeline.parameters.cg_org >> \
               -s ${<< parameters.cloudgov_space >>}
+      - run:
+          name: Acquire Lock
+          command: |
+            chmod +x ./automation/ci/scripts/*-lock.sh
+            ./automation/ci/scripts/acquire-lock.sh \
+              "tta-automation" \
+              "<< pipeline.git.branch >>" \
+              "<< pipeline.number >>"
       - run:
           name: Start Log Monitoring
           command: |
@@ -579,6 +633,14 @@ commands:
           slack_bot_token: $SLACK_BOT_TOKEN
           slack_channel: "acf-head-start-eng"
           message_text_file: "/tmp/message_file"
+      - run:
+          name: Release Lock
+          command: |
+            chmod +x ./automation/ci/scripts/*-lock.sh
+            ./automation/ci/scripts/release-lock.sh \
+              "tta-automation" \
+              "<< pipeline.git.branch >>" \
+              "<< pipeline.number >>"
       - run:
           name: Logout of service account
           command: |
@@ -696,7 +758,7 @@ parameters:
     type: string
   dev_git_branch: # change to feature branch to test deployment
     description: "Name of github branch that will deploy to dev"
-    default: "main"
+    default: "TTAHUB-3542/TTAHUB-3544/s3Queue-scanQueue-coverage"
     type: string
   sandbox_git_branch: # change to feature branch to test deployment
     default: "TTAHUB-3678/login"
@@ -716,6 +778,21 @@ parameters:
   manual-trigger:
     type: boolean
     default: false
+  env_list:
+    description: "List of environments to manage (start/stop)"
+    type: string
+    default: "DEV,SANDBOX"
+  space_list:
+    description: "List of Cloud Foundry spaces corresponding to each environment"
+    type: string
+    default: ""
+  env_state:
+    description: "State of the environment to change (start, stop, restart, restage)"
+    type: string
+    default: "none"
+  manual-manage-env:
+    type: boolean
+    default: false
   manual-restore:
     type: boolean
     default: false
@@ -1460,6 +1537,46 @@ jobs:
           rds_service_name: ttahub-prod
           s3_service_name: ttahub-db-backups
           backup_prefix: production
+  manage_env_apps:
+    executor: docker-executor
+    parameters:
+      env_list:
+        type: string
+        description: "Comma-separated list of environments to manage (both smarthub and similarity-api)"
+        default: "<< pipeline.parameters.env_list >>"
+      env_state:
+        type: string
+        description: "Action to perform on apps (start, stop, restart, restage)"
+        default: "<< pipeline.parameters.env_state >>"
+      check_activity:
+        type: boolean
+        description: "If true, only stop apps if inactive for more than activity_timeout minutes"
+        default: false
+      activity_timeout:
+        type: string
+        description: "number of minutes considered for inactivity"
+        default: "60"
+    steps:
+      - install_cf_tools
+      # Sparse checkout the automation scripts
+      - sparse_checkout:
+          directories: "automation/ci/scripts"
+          branch: "<< pipeline.git.branch >>"
+      # Perform the desired action on environments
+      - run:
+          name: Manage Apps
+          command: |
+            chmod +x ./automation/ci/scripts/*-lock.sh
+            chmod +x ./automation/ci/scripts/manage_apps.sh
+            ./automation/ci/scripts/manage_apps.sh \
+              --env_list "<< parameters.env_list >>" \
+              --env_state "<< parameters.env_state >>" \
+              --check_activity "<< parameters.check_activity >>" \
+              --activity_timeout << parameters.activity_timeout >> \
+              --cg_api "<< pipeline.parameters.cg_api >>" \
+              --cg_org "<< pipeline.parameters.cg_org >>" \
+              --branch "<< pipeline.git.branch >>" \
+              --build "<< pipeline.number >>"
   restore_production_for_processing:
     docker:
       - image: cimg/base:2024.05
@@ -1574,11 +1691,14 @@ workflows:
   build_test_deploy:
     when:
       and:
+        # Ensure the workflow is only triggered when `manual-trigger` is false
+        # and `env_state` is empty (i.e., it's not for starting/stopping environments)
         - equal: [false, << pipeline.parameters.manual-trigger >>]
         - equal: [false, << pipeline.parameters.manual-restore >>]
         - equal: [false, << pipeline.parameters.manual-process >>]
         - equal: [false, << pipeline.parameters.manual-backup >>]
         - equal: [false, << pipeline.parameters.manual-full-process >>]
+        - equal: [false, << pipeline.parameters.manual-manage-env >>]
         - equal: [false, << pipeline.parameters.manual-restore-staging >>]
         - equal: [false, << pipeline.parameters.manual-restore-sandbox >>]
         - equal: [false, << pipeline.parameters.manual-restore-dev >>]
@@ -1707,6 +1827,61 @@ workflows:
       equal: [true, << pipeline.parameters.manual-trigger >>]
     jobs:
       - backup_upload_production
+  stop_lower_env_workflow:
+    triggers:
+      - schedule:
+          cron: "0 1 * * 2-6"  # Runs at 6 PM PST M-F (1 AM UTC next day)
+          filters:
+            branches:
+              only:
+                - main
+    jobs:
+      - manage_env_apps:
+          env_state: "stop"
+          env_list: "<< pipeline.parameters.env_list >>"
+  start_lower_env_workflow:
+    triggers:
+      - schedule:
+          cron: "0 11 * * 1-5"  # Runs at 6 AM EST M-F(11 AM UTC)
+          filters:
+            branches:
+              only:
+                - main
+    jobs:
+      - manage_env_apps:
+          env_state: "start"
+          env_list: "<< pipeline.parameters.env_list >>"
+  manual_manage_env_workflow:
+    when:
+      equal: [true, << pipeline.parameters.manual-manage-env >>]
+    jobs:
+      - manage_env_apps:
+          env_state: "<< pipeline.parameters.env_state >>"
+          env_list: "<< pipeline.parameters.env_list >>"
+  monitor_and_shutdown_envs:
+    triggers:
+      # Every 15 minutes from 11 AM to 11:59 PM UTC (6 AM to 6:59 PM EST, 3 AM to 3:59 PM PST), Monday to Friday
+      - schedule:
+          cron: "0,15,30,45 11-23 * * 1-5"
+          filters:
+            branches:
+              only:
+                - main
+                - TTAHUB-3071/shutdown-unutilized-envs
+
+      # Every 15 minutes from 12 AM to 12:45 AM UTC (7 PM to 8:45 PM EST, 4 PM to 5:45 PM PST), Monday to Friday
+      - schedule:
+          cron: "0,15,30,45 0-3 * * 2-6"
+          filters:
+            branches:
+              only:
+                - main
+                - TTAHUB-3071/shutdown-unutilized-envs
+    jobs:
+      - manage_env_apps:
+          env_state: "stop"
+          env_list: "<< pipeline.parameters.env_list >>"
+          check_activity: true
   manual_restore_production:
     when:
       equal: [true, << pipeline.parameters.manual-restore >>]
@@ -1750,4 +1925,4 @@ workflows:
     when:
       equal: [true, << pipeline.parameters.manual-restore-dev >>]
     jobs:
-      - restore_processed_to_dev
+      - restore_processed_to_dev

automation/ci/scripts/acquire-lock.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# Convert environment to app name if necessary
+APP_NAME=$( [ "$1" == "DEV" ] && echo "tta-smarthub-dev" || ([ "$1" == "SANDBOX" ] && echo "tta-smarthub-sandbox") || echo "$1" )
+BRANCH=$2
+BUILD_ID=$3
+
+# Constants
+LOCK_TIMEOUT=7200  # 2 hours in seconds
+
+# Fetch environment variables
+LOCK_DATA=$(cf env "$APP_NAME" | grep -A 10 LOCK_APP | sed ':a;N;$!ba;s/\n/ /g' | grep -oP "[{][^}]+[}]")
+
+# Check if lock exists
+if [ -n "$LOCK_DATA" ]; then
+  LOCK_TIMESTAMP=$(echo "$LOCK_DATA" | jq -r '.timestamp')
+  LOCK_BRANCH=$(echo "$LOCK_DATA" | jq -r '.branch')
+  LOCK_BUILD_ID=$(echo "$LOCK_DATA" | jq -r '.build_id')
+
+  CURRENT_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+  TIME_DIFF=$(($(date -d "$CURRENT_TIME" +%s) - $(date -d "$LOCK_TIMESTAMP" +%s)))
+
+  if [ $TIME_DIFF -lt $LOCK_TIMEOUT ]; then
+    echo "App $APP_NAME is locked by branch $LOCK_BRANCH with build ID $LOCK_BUILD_ID."
+    exit 1
+  fi
+
+  echo "Lock is stale. Attempting to acquire lock..."
+fi
+
+# Check if app is restaging
+APP_STATE=$(cf apps | grep "$APP_NAME" | awk '{print $2}')
+if [ "$APP_STATE" != "started" ] && [ "$APP_STATE" != "stopped" ]; then
+  echo "App $APP_NAME is currently $APP_STATE. Cannot acquire lock."
+  exit 1
+fi
+
+# Acquire lock
+TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+LOCK_DATA_JSON=$(jq -n \
+  --arg branch "$BRANCH" \
+  --arg build_id "$BUILD_ID" \
+  --arg timestamp "$TIMESTAMP" \
+  '{branch: $branch, build_id: $build_id, timestamp: $timestamp}')
+
+cf set-env "$APP_NAME" LOCK_APP "$LOCK_DATA_JSON"
+
+# Validate the lock
+LOCK_DATA=$(cf env "$APP_NAME" | grep -A 10 LOCK_APP | sed ':a;N;$!ba;s/\n/ /g' | grep -oP "[{][^}]+[}]")
+VALID_BRANCH=$(echo "$LOCK_DATA" | jq -r '.branch')
+VALID_BUILD_ID=$(echo "$LOCK_DATA" | jq -r '.build_id')
+
+if [ "$VALID_BRANCH" == "$BRANCH" ] && [ "$VALID_BUILD_ID" == "$BUILD_ID" ]; then
+  echo "Lock successfully acquired for app $APP_NAME."
+  exit 0
+else
+  echo "Failed to acquire lock for app $APP_NAME."
+  exit 1
+fi