Fix buildkite pipeline for db-sync sync tests

.buildkite/db_sync_full_sync.yml

@@ -1,9 +1,10 @@
 steps:
   - label: ':drum_with_drumsticks: Full sync test :drum_with_drumsticks:'
     commands:
-      - nix develop --accept-flake-config ..#python --command python ./db_sync_tests/tests/full_sync_from_clean_state.py -npr "${node_pr}" -nbr "${node_branch}" -nv "${node_version}" -dbr "${db_sync_branch}" -dv "${db_sync_version}" -dsa "${db_sync_start_arguments}"  -e "${environment}"
-      - nix develop --accept-flake-config ..#python --command python ./db_sync_tests/tests/snapshot_creation.py -dpr "${db_sync_pr}" -dbr "${db_sync_branch}" -dv "${db_sync_version}" -e "${environment}" -rosc "${run_only_sync_test}"
-      - nix develop --accept-flake-config ..#python --command python ./db_sync_tests/tests/local_snapshot_restoration.py -npr "${node_pr}" -nbr "${node_branch}" -nv "${node_version}" -dbr "${db_sync_branch}" -dv "${db_sync_version}" -dsa "${db_sync_start_arguments}" -e "${environment}" -rosc "${run_only_sync_test}"
+      - ls -l
+      - nix develop --accept-flake-config .#python --command python ./db_sync_tests/tests/full_sync_from_clean_state.py -nv "${node_version}" -dv "${db_sync_version}" -dsa "${db_sync_start_arguments}"  -e "${environment}"
+      - nix develop --accept-flake-config .#python --command python ./db_sync_tests/tests/snapshot_creation.py -dv "${db_sync_version}" -e "${environment}" -rosc "${run_only_sync_test}"
+      - nix develop --accept-flake-config .#python --command python ./db_sync_tests/tests/local_snapshot_restoration.py -nv "${node_version}" -dv "${db_sync_version}" -dsa "${db_sync_start_arguments}" -e "${environment}" -rosc "${run_only_sync_test}"
     timeout_in_minutes: 43200
     agents:
       system: x86_64-linux

.github/workflows/db_sync_full_sync.yaml

@@ -3,38 +3,27 @@ name: db-sync - full cycle tests
 on:
   workflow_dispatch:
     inputs:
-      node_pr:
-        description: node pull request number
-        required: true
-        default: "4269"
-      node_branch:
-        description: node branch or tag
-        required: true
-        default: "1.35.3"
       node_version:
-        description: node version - 1.33.0-rc2 (tag number) or 1.33.0 (release number - for released versions) or 1.33.0_PR2124 (for not released and not tagged runs with a specific node PR/version)
-        required: true
-        default: "1.35.3"
-      db_sync_branch:
-        description: db-sync branch or tag
+        description: "cardano-node version"
         required: true
-        default: "tags/13.0.5"
-      db_sync_pr:
-        description: db-sync pr
-        required: true
-        default: "1208"
+        default: "tags/10.1.2"
       db_sync_version:
-        description: db-sync version - 12.0.0-rc2 (tag number) or 12.0.2 (release number - for released versions) or 12.0.2_PR2124 (for not released and not tagged runs with a specific db_sync PR/version)
+        description: "db-sync version"
         required: true
-        default: "13.0.5"
+        default: "tags/13.6.0.1"
       db_sync_start_arguments:
-        description: argument to be passed when starting the db-sync - none, disable-ledger, disable-epoch, disable-cache
+        description: "db-sync start arguments"
         required: false
         default: "none"
       environment:
-        description: environment on which to run the tests - shelley-qa, preview, preprod or mainnet
-        required: true
-        default: "preprod"
+        description: "Environment on which Buildkite agent will run tests"
+        type: choice
+        options:
+        - mainnet
+        - preprod
+        - preview
+        - shelley-qa
+        default: preprod
       run_only_sync_test:
         type: boolean
         default: true
@@ -48,19 +37,15 @@ jobs:
         uses: 'buildkite/trigger-pipeline-action@v1.5.0'
         env:
           BUILDKITE_API_ACCESS_TOKEN: ${{ secrets.BUILDKITE_API_ACCESS_TOKEN }}
-          PIPELINE: 'input-output-hk/db-sync_sync_tests'
+          PIPELINE: 'input-output-hk/db-sync-sync-tests'
           BRANCH: ${{ github.ref_name || 'main' }}
           MESSAGE: ':github: Triggered by GitHub Action'
           AWS_DB_USERNAME: ${{ secrets.AWS_DB_USERNAME }}
           AWS_DB_PASS: ${{ secrets.AWS_DB_PASS }}
           AWS_DB_NAME: ${{ secrets.AWS_DB_NAME }}
           AWS_DB_HOSTNAME: ${{ secrets.AWS_DB_HOSTNAME }}
           BUILD_ENV_VARS: '{
-          "node_pr":"${{ github.event.inputs.node_pr }}",
-          "node_branch":"${{ github.event.inputs.node_branch }}",
           "node_version":"${{ github.event.inputs.node_version }}",
-          "db_sync_pr":"${{ github.event.inputs.db_sync_pr }}",
-          "db_sync_branch":"${{ github.event.inputs.db_sync_branch }}",
           "db_sync_version":"${{ github.event.inputs.db_sync_version }}",
           "db_sync_start_arguments":"${{ github.event.inputs.db_sync_start_arguments }}",
           "environment":"${{ github.event.inputs.environment }}",

db_sync_tests/tests/full_sync_from_clean_state.py

@@ -1,14 +1,15 @@
 import argparse
 import json
 import os
+import subprocess
 from collections import OrderedDict
 from pathlib import Path
 import sys
 import matplotlib.pyplot as plt
 
 sys.path.append(os.getcwd())
 
-from utils.utils import seconds_to_time, get_no_of_cpu_cores, get_current_date_time, \
+from db_sync_tests.utils.utils import seconds_to_time, get_no_of_cpu_cores, get_current_date_time, \
     get_os_type, get_total_ram_in_GB, upload_artifact, clone_repo, zip_file, execute_command, \
     print_file, stop_process, write_data_as_json_to_file, get_node_config_files, \
     get_node_version, get_db_sync_version, start_node_in_cwd, wait_for_db_to_sync, \
@@ -24,7 +25,7 @@
     NODE_ARCHIVE, DB_SYNC_ARCHIVE, SYNC_DATA_ARCHIVE, EXPECTED_DB_SCHEMA, EXPECTED_DB_INDEXES, \
     ENVIRONMENT \
 
-from utils.aws_db_utils import get_identifier_last_run_from_table, \
+from db_sync_tests.utils.aws_db_utils import get_identifier_last_run_from_table, \
     add_bulk_rows_into_db, add_single_row_into_db
 
 
@@ -168,10 +169,11 @@ def main():
     print(f"DB sync version: {db_sync_version_from_gh_action}")
 
     # cardano-node setup
-    NODE_DIR=clone_repo('cardano-node', node_branch)
+    NODE_DIR=clone_repo('cardano-node', node_version_from_gh_action)
     os.chdir(NODE_DIR)
     execute_command("nix build -v .#cardano-node -o cardano-node-bin")
     execute_command("nix build -v .#cardano-cli -o cardano-cli-bin")
+
     print("--- Node setup")
     copy_node_executables(build_method="nix")
     get_node_config_files(env)
@@ -184,7 +186,7 @@ def main():
 
     # cardano-db sync setup
     os.chdir(ROOT_TEST_PATH)
-    DB_SYNC_DIR = clone_repo('cardano-db-sync', db_branch)
+    DB_SYNC_DIR = clone_repo('cardano-db-sync', db_sync_version_from_gh_action.rstrip())
     os.chdir(DB_SYNC_DIR)
     print("--- Db sync setup")
     setup_postgres() # To login use: psql -h /path/to/postgres -p 5432 -e postgres

db_sync_tests/utils/utils.py

@@ -27,14 +27,14 @@
 
 ONE_MINUTE = 60
 ROOT_TEST_PATH = Path.cwd()
-ENVIRONMENT = os.environ['environment']
+ENVIRONMENT = os.environ.get('environment', None)
 
-NODE_PR = os.environ['node_pr']
-NODE_BRANCH = os.environ['node_branch']
-NODE_VERSION = os.environ['node_version']
+NODE_PR = os.environ.get('node_pr', None)
+NODE_BRANCH = os.environ.get('node_branch', None)
+NODE_VERSION = os.environ.get('node_version', None)
 
-DB_SYNC_BRANCH = os.environ['db_sync_branch']
-DB_SYNC_VERSION = os.environ['db_sync_version']
+DB_SYNC_BRANCH = os.environ.get('db_sync_branch', None)
+DB_SYNC_VERSION = os.environ.get('db_sync_version', None)
 
 POSTGRES_DIR = ROOT_TEST_PATH.parents[0]
 POSTGRES_USER = subprocess.run(['whoami'], stdout=subprocess.PIPE).stdout.decode('utf-8').strip()
@@ -493,7 +493,12 @@ def copy_node_executables(build_method="nix"):
     current_directory = os.getcwd()
     os.chdir(ROOT_TEST_PATH)
     node_dir = Path.cwd() / 'cardano-node'
+    node_bin_dir = node_dir / "cardano-node-bin/"
     os.chdir(node_dir)
+    print(f"current_directory: {os.getcwd()}")
+
+    result = subprocess.run(['nix', '--version'], stdout=subprocess.PIPE, text=True, check=True)
+    print(f"Nix version: {result.stdout.strip()}")
 
     if build_method == "nix":
         node_binary_location = "cardano-node-bin/bin/cardano-node"
@@ -1114,7 +1119,7 @@ def start_db_sync(env, start_args="", first_start="True"):
     export_env_var("LOG_FILEPATH", DB_SYNC_LOG)
 
     try:
-        cmd = "./scripts/db-sync-start.sh"
+        cmd = "./db_sync_tests/scripts/db-sync-start.sh"
         p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
         os.chdir(current_directory)
     except subprocess.CalledProcessError as e:
@@ -1185,7 +1190,7 @@ def setup_postgres(pg_dir=POSTGRES_DIR, pg_user=POSTGRES_USER, pg_port='5432'):
     export_env_var("PGPORT", pg_port)
 
     try:
-        cmd = ["./scripts/postgres-start.sh", f"{pg_dir}", "-k"]
+        cmd = ["./db_sync_tests/scripts/postgres-start.sh", f"{pg_dir}", "-k"]
         output = (
             subprocess.check_output(cmd, stderr=subprocess.STDOUT)
             .decode("utf-8")