sentry-smoke-test.yaml

name: Sentry Smoke Test

on:
  pull_request:
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true
jobs:
  sentry-smoke-test:
    timeout-minutes: 20
    runs-on: ubuntu-latest
    steps:
    - name: Checkout repository
      uses: actions/checkout@v2
    - name: Build xatu image
      run: |
        docker build -t ethpandaops/xatu:local .
        echo "Xatu image is built."
    - name: Install Kurtosis
      run: |
        echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
        sudo apt update
        sudo apt install kurtosis-cli
    - name: Create Kurtosis config file
      run: |
        cat <<EOF > network_params.yaml
          participants:
            - el_client_type: geth
              cl_client_type: teku
              cl_client_image: ethpandaops/teku:master-16c4354
            - el_client_type: nethermind
              cl_client_type: prysm
            - el_client_type: erigon
              cl_client_type: lighthouse
            - el_client_type: besu
              cl_client_type: lighthouse
            - el_client_type: reth
              cl_client_type: lodestar
            - el_client_type: ethereumjs
              cl_client_type: nimbus
          additional_services: []
          network_parans:
            genesis_delay: 180
          xatu_sentry_enabled: true
          xatu_sentry_params:
            xatu_server_addr: xatu-server:8080
            xatu_sentry_image: ethpandaops/xatu:local
            beacon_subscriptions:
            - attestation
            - block
            - chain_reorg
            - finalized_checkpoint
            - head
            - voluntary_exit
            - contribution_and_proof
        <<EOF

    - name: Start Ethereum network with Kurtosis and Run Xatu stack in parallel
      timeout-minutes: 20
      shell: bash
      run: |
        echo "Starting Kurtosis..."
        kurtosis run --enclave xatu github.com/kurtosis-tech/ethereum-package --args-file network_params.yaml & 
        KURTOSIS_PID=$!

        echo "Starting Xatu stack..."
        docker compose up --detach --quiet-pull &
        XATU_STACK_PID=$!

        echo "Waiting for Kurtosis to start..."
        wait $KURTOSIS_PID
        KURTOSIS_EXIT_CODE=$?

        if [ $KURTOSIS_EXIT_CODE -ne 0 ]; then
          echo "Kurtosis failed to start."
          exit $KURTOSIS_EXIT_CODE
        fi

        echo "Waiting for Xatu stack to start..."
        wait $XATU_STACK_PID
        XATU_STACK_EXIT_CODE=$?

        if [ $XATU_STACK_EXIT_CODE -ne 0 ]; then
          echo "Xatu stack failed to start."
          exit $XATU_STACK_EXIT_CODE
        fi

        echo "Kurtosis and Xatu stack have started successfully."
    - name: Add all xatu-sentry containers to the xatu network
      run: |
        for container in $(docker ps --filter name=xatu-sentry --format "{{.Names}}"); do docker network connect xatu_xatu-net $container; echo $container; docker restart $container; done
    - name: Verify Clickhouse has data from all sentries
      timeout-minutes: 10
      run: |
        echo "Checking Clickhouse for data from all sentries"

        all_sentries=($(kurtosis enclave inspect xatu | grep cl- | grep http | awk '{ print $2 }' | grep -v validator | sed 's/^cl-//'))

        tables=(
            "beacon_api_eth_v1_events_attestation"
            "beacon_api_eth_v1_events_block"
            "beacon_api_eth_v1_events_head"
        )

        # Define a function that prints the last 5 logs from all docker containers that have the argument in the name
        pretty_print() {
            local message=$1
            local color=$2
            local no_color='\033[0m'
            local green='\033[0;32m'
            local red='\033[0;31m'
            local bright_red='\033[1;31m'

            # Choose color based on the type of message
            if [ "$color" == "green" ]; then
                color=$green
            elif [ "$color" == "red" ]; then
                color=$red
            elif [ "$color" == "bright_red" ]; then
                color=$bright_red
            else
                color=$no_color
            fi

            echo -e "${color}######################${no_color}"
            echo -e "${color}      $message      ${no_color}"
            echo -e "${color}######################${no_color}"
        }

        print_logs() {
            for container in $(docker ps --filter name=$1 --format "{{.Names}}"); do
                echo "Logs for $container:\n\n"
                docker logs --tail 5 $container
            done
        }

        # Check for any data in the tables before digging in to the individual sentries
        for table in "${tables[@]}"; do
            pretty_print "Checking $table table..." "none"
            data_count=$(docker exec clickhouse-01 clickhouse-client --query "SELECT COUNT(*) FROM default.$table" || true)
            if [[ $data_count -gt 0 ]]; then
                pretty_print "$table table has $data_count entries" "green"
            else
                pretty_print "$table table has no entries." "bright_red"

                print_logs xatu-server
                print_logs vector
            fi
        done

        for table in "${tables[@]}"; do
            pretty_print "Checking $table table..." "none"
            for sentry in "${all_sentries[@]}"; do
                pretty_print "Checking $table table for $sentry..." "none"
                while true; do
                    data_count=$(docker exec clickhouse-01 clickhouse-client --query "SELECT COUNT(*) FROM default.$table WHERE meta_client_name = '$sentry'" || true)
                    if [[ $data_count -gt 0 ]]; then
                        pretty_print "$table has $data_count entries from $sentry" "green"
                        break
                    else
                        pretty_print "$table has no entries from $sentry." "bright_red"

                        print_logs $sentry

                        sleep 5
                    fi
                done
            done
        done

    - name: Collect docker logs on failure
      if: failure()
      uses: jwalton/gh-docker-logs@v2
      with:
        dest: './logs'
    - name: Tar logs
      if: failure()
      run: tar cvzf ./logs.tgz ./logs
    - name: Upload logs to GitHub
      if: failure()
      uses: actions/upload-artifact@master
      with:
        name: logs.tgz
        path: ./logs.tgz