Fix buildkite pipeline for db-sync sync tests

IntersectMBO · Nov 15, 2024 · ead70be · ead70be
1 parent 8d5921f
commit ead70be
Show file tree

Hide file tree

Showing 8 changed files with 2,505 additions and 37 deletions.
diff --git a/.buildkite/db_sync_full_sync.yml b/.buildkite/db_sync_full_sync.yml
@@ -1,9 +1,10 @@
 steps:
   - label: ':drum_with_drumsticks: Full sync test :drum_with_drumsticks:'
     commands:
-      - nix develop --accept-flake-config ..#python --command python ./db_sync_tests/tests/full_sync_from_clean_state.py -npr "${node_pr}" -nbr "${node_branch}" -nv "${node_version}" -dbr "${db_sync_branch}" -dv "${db_sync_version}" -dsa "${db_sync_start_arguments}"  -e "${environment}"
-      - nix develop --accept-flake-config ..#python --command python ./db_sync_tests/tests/snapshot_creation.py -dpr "${db_sync_pr}" -dbr "${db_sync_branch}" -dv "${db_sync_version}" -e "${environment}" -rosc "${run_only_sync_test}"
-      - nix develop --accept-flake-config ..#python --command python ./db_sync_tests/tests/local_snapshot_restoration.py -npr "${node_pr}" -nbr "${node_branch}" -nv "${node_version}" -dbr "${db_sync_branch}" -dv "${db_sync_version}" -dsa "${db_sync_start_arguments}" -e "${environment}" -rosc "${run_only_sync_test}"
+      - ls -l
+      - nix develop --accept-flake-config .#python --command python ./db_sync_tests/tests/full_sync_from_clean_state.py -nv "${node_version}" -dv "${db_sync_version}" -dsa "${db_sync_start_arguments}"  -e "${environment}"
+      - nix develop --accept-flake-config .#python --command python ./db_sync_tests/tests/snapshot_creation.py -dv "${db_sync_version}" -e "${environment}" -rosc "${run_only_sync_test}"
+      - nix develop --accept-flake-config .#python --command python ./db_sync_tests/tests/local_snapshot_restoration.py -nv "${node_version}" -dv "${db_sync_version}" -dsa "${db_sync_start_arguments}" -e "${environment}" -rosc "${run_only_sync_test}"
     timeout_in_minutes: 43200
     agents:
       system: x86_64-linux

diff --git a/.github/workflows/db_sync_full_sync.yaml b/.github/workflows/db_sync_full_sync.yaml
@@ -3,38 +3,27 @@ name: db-sync - full cycle tests
 on:
   workflow_dispatch:
     inputs:
-      node_pr:
-        description: node pull request number
-        required: true
-        default: "4269"
-      node_branch:
-        description: node branch or tag
-        required: true
-        default: "1.35.3"
       node_version:
-        description: node version - 1.33.0-rc2 (tag number) or 1.33.0 (release number - for released versions) or 1.33.0_PR2124 (for not released and not tagged runs with a specific node PR/version)
-        required: true
-        default: "1.35.3"
-      db_sync_branch:
-        description: db-sync branch or tag
+        description: "cardano-node version"
         required: true
-        default: "tags/13.0.5"
-      db_sync_pr:
-        description: db-sync pr
-        required: true
-        default: "1208"
+        default: "tags/9.2.1"
       db_sync_version:
-        description: db-sync version - 12.0.0-rc2 (tag number) or 12.0.2 (release number - for released versions) or 12.0.2_PR2124 (for not released and not tagged runs with a specific db_sync PR/version)
+        description: "db-sync version"
         required: true
-        default: "13.0.5"
+        default: "tags/13.6.0.1"
       db_sync_start_arguments:
-        description: argument to be passed when starting the db-sync - none, disable-ledger, disable-epoch, disable-cache
+        description: "db-sync start arguments"
         required: false
         default: "none"
       environment:
-        description: environment on which to run the tests - shelley-qa, preview, preprod or mainnet
-        required: true
-        default: "preprod"
+        description: "Environment on which Buildkite agent will run tests"
+        type: choice
+        options:
+        - mainnet
+        - preprod
+        - preview
+        - shelley-qa
+        default: preprod
       run_only_sync_test:
         type: boolean
         default: true
@@ -48,19 +37,15 @@ jobs:
         uses: 'buildkite/trigger-pipeline-action@v1.5.0'
         env:
           BUILDKITE_API_ACCESS_TOKEN: ${{ secrets.BUILDKITE_API_ACCESS_TOKEN }}
-          PIPELINE: 'input-output-hk/db-sync_sync_tests'
+          PIPELINE: 'input-output-hk/db-sync-sync-tests'
           BRANCH: ${{ github.ref_name || 'main' }}
           MESSAGE: ':github: Triggered by GitHub Action'
           AWS_DB_USERNAME: ${{ secrets.AWS_DB_USERNAME }}
           AWS_DB_PASS: ${{ secrets.AWS_DB_PASS }}
           AWS_DB_NAME: ${{ secrets.AWS_DB_NAME }}
           AWS_DB_HOSTNAME: ${{ secrets.AWS_DB_HOSTNAME }}
           BUILD_ENV_VARS: '{
-          "node_pr":"${{ github.event.inputs.node_pr }}",
-          "node_branch":"${{ github.event.inputs.node_branch }}",
           "node_version":"${{ github.event.inputs.node_version }}",
-          "db_sync_pr":"${{ github.event.inputs.db_sync_pr }}",
-          "db_sync_branch":"${{ github.event.inputs.db_sync_branch }}",
           "db_sync_version":"${{ github.event.inputs.db_sync_version }}",
           "db_sync_start_arguments":"${{ github.event.inputs.db_sync_start_arguments }}",
           "environment":"${{ github.event.inputs.environment }}",

diff --git a/db_sync_tests/tests/__init__.py b/db_sync_tests/tests/__init__.py
diff --git a/db_sync_tests/tests/full_sync_from_clean_state.py b/db_sync_tests/tests/full_sync_from_clean_state.py
@@ -8,7 +8,7 @@
 
 sys.path.append(os.getcwd())
 
-from utils.utils import seconds_to_time, get_no_of_cpu_cores, get_current_date_time, \
+from db_sync_tests.utils.utils import seconds_to_time, get_no_of_cpu_cores, get_current_date_time, \
     get_os_type, get_total_ram_in_GB, upload_artifact, clone_repo, zip_file, execute_command, \
     print_file, stop_process, write_data_as_json_to_file, get_node_config_files, \
     get_node_version, get_db_sync_version, start_node_in_cwd, wait_for_db_to_sync, \
@@ -24,7 +24,7 @@
     NODE_ARCHIVE, DB_SYNC_ARCHIVE, SYNC_DATA_ARCHIVE, EXPECTED_DB_SCHEMA, EXPECTED_DB_INDEXES, \
     ENVIRONMENT \
 
-from utils.aws_db_utils import get_identifier_last_run_from_table, \
+from db_sync_tests.utils.aws_db_utils import get_identifier_last_run_from_table, \
     add_bulk_rows_into_db, add_single_row_into_db
 
 

diff --git a/db_sync_tests/utils/__init__.py b/db_sync_tests/utils/__init__.py
diff --git a/db_sync_tests/utils/utils.py b/db_sync_tests/utils/utils.py
@@ -29,11 +29,11 @@
 ROOT_TEST_PATH = Path.cwd()
 ENVIRONMENT = os.environ['environment']
 
-NODE_PR = os.environ['node_pr']
-NODE_BRANCH = os.environ['node_branch']
+NODE_PR = os.environ.get('node_pr', None)
+NODE_BRANCH = os.environ.get('node_branch', None)
 NODE_VERSION = os.environ['node_version']
 
-DB_SYNC_BRANCH = os.environ['db_sync_branch']
+DB_SYNC_BRANCH = os.environ.get('db_sync_branch', None)
 DB_SYNC_VERSION = os.environ['db_sync_version']
 
 POSTGRES_DIR = ROOT_TEST_PATH.parents[0]