Erigon3 upstream e4eb9fc

.github/workflows/kurtosis/config.yaml

@@ -0,0 +1,35 @@
+participants_matrix:
+ el:
+ - el_type: geth
+ - el_type: erigon
+ el_image: <<ERIGON_IMAGE_PLACEHOLDER>>
+ cl:
+ - cl_type: lighthouse
+ - cl_type: teku
+network_params:
+ electra_fork_epoch: 1
+ min_validator_withdrawability_delay: 1
+ shard_committee_period: 1
+additional_services:
+ - dora
+ - apache
+ - assertoor
+snooper_enabled: true
+assertoor_params:
+ run_stability_check: false
+ run_block_proposal_check: false
+ tests:
+ - https://raw.githubusercontent.com/ethpandaops/assertoor-test/master/assertoor-tests/all-opcodes-test.yaml
+ #- https://raw.githubusercontent.com/ethpandaops/assertoor-test/master/assertoor-tests/blob-transactions-test.yaml
+ #- https://raw.githubusercontent.com/ethpandaops/assertoor-test/master/assertoor-tests/dencun-opcodes-test.yaml
+ #- https://raw.githubusercontent.com/ethpandaops/assertoor-test/master/assertoor-tests/eoa-transactions-test.yaml
+ #- https://raw.githubusercontent.com/ethpandaops/assertoor-test/master/assertoor-tests/stability-check.yaml
+ - https://raw.githubusercontent.com/ethpandaops/assertoor-test/master/assertoor-tests/synchronized-check.yaml
+ #- https://raw.githubusercontent.com/ethpandaops/assertoor-test/master/assertoor-tests/validator-exit-test.yaml
+ # - https://raw.githubusercontent.com/ethpandaops/assertoor-test/master/assertoor-tests/validator-lifecycle-test-v2.yaml
+ # - https://raw.githubusercontent.com/ethpandaops/assertoor-test/master/assertoor-tests/validator-lifecycle-test.yaml
+ # - https://raw.githubusercontent.com/ethpandaops/assertoor-test/master/assertoor-tests/validator-proposer-slashing-test.yaml
+ # - https://raw.githubusercontent.com/ethpandaops/assertoor-test/master/assertoor-tests/validator-withdrawal-test.yaml
+ # - https://raw.githubusercontent.com/ethpandaops/assertoor-test/master/assertoor-tests/validator-withdrawal-test-v2.yaml
+ # - https://raw.githubusercontent.com/ethpandaops/assertoor-test/master/assertoor-tests/validator-slashing-test.yaml
+ # - https://raw.githubusercontent.com/ethpandaops/assertoor-test/master/assertoor-tests/block-proposal-check.yaml

.github/workflows/qa-snap-download.yml

@@ -2,17 +2,17 @@ name: QA - Snapshot Download
 
 on:
  schedule:
- - cron: '0 5 * * 1-6' # Run every day at 05:00 AM UTC except Sunday
+ - cron: '0 0 * * 1-6' # Run every night at 00:00 AM UTC except Sunday
  workflow_dispatch: # Run manually
 
 jobs:
  snap-download-test:
  runs-on: self-hosted
- timeout-minutes: 800
+ timeout-minutes: 500
  env:
  ERIGON_DATA_DIR: ${{ github.workspace }}/erigon_data
  ERIGON_QA_PATH: /home/qarunner/erigon-qa
- TOTAL_TIME_SECONDS: 43200 # 12 hours
+ TOTAL_TIME_SECONDS: 28800 # 8 hours
  CHAIN: mainnet
 
  steps:
@@ -64,15 +64,6 @@ jobs:
  echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT"
  fi
 
- - name: Clean up Erigon data directory
- if: always()
- run: |
- rm -rf $ERIGON_DATA_DIR
-
- - name: Resume the Erigon instance dedicated to db maintenance
- run: |
- python3 $ERIGON_QA_PATH/test_system/db-producer/resume_production.py || true
-
  - name: Save test results
  if: steps.test_step.outputs.test_executed == 'true'
  env:
@@ -93,7 +84,18 @@ jobs:
  uses: actions/upload-artifact@v4
  with:
  name: test-results
- path: ${{ github.workspace }}/result-${{ env.CHAIN }}.json
+ path: |
+ ${{ github.workspace }}/result-${{ env.CHAIN }}.json
+ ${{ github.workspace }}/erigon_data/logs/erigon.log
+
+ - name: Clean up Erigon data directory
+ if: always()
+ run: |
+ rm -rf $ERIGON_DATA_DIR
+
+ - name: Resume the Erigon instance dedicated to db maintenance
+ run: |
+ python3 $ERIGON_QA_PATH/test_system/db-producer/resume_production.py || true
 
  - name: Action for Success
  if: steps.test_step.outputs.TEST_RESULT == 'success'

.github/workflows/qa-sync-from-scratch-minimal-node.yml

@@ -2,18 +2,18 @@ name: QA - Sync from scratch (minimal node)
 
 on:
  schedule:
- - cron: '0 0 * * 6' # Run on Saturday at 00:00 AM UTC
+ - cron: '0 0 * * *' # Run every night at 00:00 AM UTC
  workflow_dispatch: # Run manually
 
 jobs:
  minimal-node-sync-from-scratch-test:
  runs-on: self-hosted
- timeout-minutes: 1500 # 25 hours
+ timeout-minutes: 360 # 6 hours
  env:
  ERIGON_DATA_DIR: ${{ github.workspace }}/erigon_data
  ERIGON_QA_PATH: /home/qarunner/erigon-qa
  TRACKING_TIME_SECONDS: 7200 # 2 hours
- TOTAL_TIME_SECONDS: 86400 # 24 hours
+ TOTAL_TIME_SECONDS: 18000 # 5 hours
  CHAIN: mainnet
 
  steps:
@@ -65,15 +65,6 @@ jobs:
  echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT"
  fi
 
- - name: Clean up Erigon data directory
- if: always()
- run: |
- rm -rf $ERIGON_DATA_DIR
-
- - name: Resume the Erigon instance dedicated to db maintenance
- run: |
- python3 $ERIGON_QA_PATH/test_system/db-producer/resume_production.py || true
-
  - name: Save test results
  if: steps.test_step.outputs.test_executed == 'true'
  env:
@@ -94,7 +85,18 @@ jobs:
  uses: actions/upload-artifact@v4
  with:
  name: test-results
- path: ${{ github.workspace }}/result-${{ env.CHAIN }}.json
+ path: |
+ ${{ github.workspace }}/result-${{ env.CHAIN }}.json
+ ${{ github.workspace }}/erigon_data/logs/erigon.log
+
+ - name: Clean up Erigon data directory
+ if: always()
+ run: |
+ rm -rf $ERIGON_DATA_DIR
+
+ - name: Resume the Erigon instance dedicated to db maintenance
+ run: |
+ python3 $ERIGON_QA_PATH/test_system/db-producer/resume_production.py || true
 
  - name: Action for Success
  if: steps.test_step.outputs.TEST_RESULT == 'success'

.github/workflows/qa-sync-from-scratch.yml

@@ -8,7 +8,7 @@ on:
 jobs:
  sync-from-scratch-test:
  runs-on: self-hosted
- timeout-minutes: 1100 # 18 hours plus 20 minutes
+ timeout-minutes: 740 # 12 hours plus 20 minutes
  strategy:
  fail-fast: false
  matrix:
@@ -17,7 +17,7 @@ jobs:
  ERIGON_DATA_DIR: ${{ github.workspace }}/erigon_data
  ERIGON_QA_PATH: /home/qarunner/erigon-qa
  TRACKING_TIME_SECONDS: 7200 # 2 hours
- TOTAL_TIME_SECONDS: 57600 # 16 hours
+ TOTAL_TIME_SECONDS: 43200 # 12 hours
  CHAIN: ${{ matrix.chain }}
 
  steps:
@@ -69,15 +69,6 @@ jobs:
  echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT"
  fi
 
- - name: Clean up Erigon data directory
- if: always()
- run: |
- rm -rf $ERIGON_DATA_DIR
-
- - name: Resume the Erigon instance dedicated to db maintenance
- run: |
- python3 $ERIGON_QA_PATH/test_system/db-producer/resume_production.py || true
-
  - name: Save test results
  if: steps.test_step.outputs.test_executed == 'true'
  env:
@@ -98,7 +89,18 @@ jobs:
  uses: actions/upload-artifact@v4
  with:
  name: test-results-${{ env.CHAIN }}
- path: ${{ github.workspace }}/result-${{ env.CHAIN }}.json
+ path: |
+ ${{ github.workspace }}/result-${{ env.CHAIN }}.json
+ ${{ github.workspace }}/erigon_data/logs/erigon.log
+
+ - name: Clean up Erigon data directory
+ if: always()
+ run: |
+ rm -rf $ERIGON_DATA_DIR
+
+ - name: Resume the Erigon instance dedicated to db maintenance
+ run: |
+ python3 $ERIGON_QA_PATH/test_system/db-producer/resume_production.py || true
 
  - name: Action for Success
  if: steps.test_step.outputs.TEST_RESULT == 'success'

.github/workflows/qa-tip-tracking.yml

@@ -2,7 +2,7 @@ name: QA - Tip tracking
 
 on:
  schedule:
- - cron: '0 0 * * 1-6' # Run every day at 00:00 AM UTC except Sunday
+ - cron: '0 0 * * 1-6' # Run every night at 00:00 AM UTC except Sunday
  workflow_dispatch: # Run manually
 
 jobs:
@@ -71,15 +71,6 @@ jobs:
  echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT"
  fi
 
- - name: Delete Erigon Testbed Data Directory
- if: always()
- run: |
- rm -rf $ERIGON_TESTBED_DATA_DIR
-
- - name: Resume the Erigon instance dedicated to db maintenance
- run: |
- python3 $ERIGON_QA_PATH/test_system/db-producer/resume_production.py || true
-
  - name: Save test results
  if: steps.test_step.outputs.test_executed == 'true'
  env:
@@ -106,7 +97,18 @@ jobs:
  uses: actions/upload-artifact@v4
  with:
  name: test-results
- path: ${{ github.workspace }}/result-${{ env.CHAIN }}.json
+ path: |
+ ${{ github.workspace }}/result-${{ env.CHAIN }}.json
+ ${{ env.ERIGON_TESTBED_DATA_DIR }}/logs/erigon.log
+
+ - name: Delete Erigon Testbed Data Directory
+ if: always()
+ run: |
+ rm -rf $ERIGON_TESTBED_DATA_DIR
+
+ - name: Resume the Erigon instance dedicated to db maintenance
+ run: |
+ python3 $ERIGON_QA_PATH/test_system/db-producer/resume_production.py || true
 
  - name: Action for Success
  if: steps.test_step.outputs.TEST_RESULT == 'success'

.github/workflows/test-kurtosis-assertoor.yml

@@ -0,0 +1,51 @@
+name: Kurtosis Assertoor GitHub Action
+
+on:
+ schedule:
+ - cron: "0 2,14 * * *" # runs at 2am and 2pm UTC
+ workflow_dispatch:
+ # pull_request:
+ # branches:
+ # - kurtosis_assertoor
+ # types:
+ # - opened
+ # - reopened
+ # - synchronize
+ # - ready_for_review
+ # push:
+ # branches:
+ # - kurtosis_assertoor
+
+
+
+jobs:
+ assertoor_test:
+ strategy:
+ matrix:
+ # list of os: https://github.com/actions/virtual-environments
+ os:
+ - { id: ubuntu-22.04, name: ubuntu } 
+ runs-on: ${{ matrix.os.id }}
+
+ steps:
+ - uses: actions/checkout@v4
+ - name: make docker (see dockerhub for image builds)
+ run: DOCKER_TAG=thorax/erigon:ci-$GITHUB_SHA DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) make docker
+
+
+ - name: Install dependencies on Linux
+ if: runner.os == 'Linux'
+ run: sudo apt update && sudo apt install build-essential
+
+ - name: download kurtosis config
+ run: |
+ wget -O kurtosis_config_with_p.yaml https://raw.githubusercontent.com/erigontech/erigon/kurtosis_assertoor/.github/workflows/kurtosis/config.yaml
+ sed 's/<<ERIGON_IMAGE_PLACEHOLDER>>/thorax\/erigon:ci-'$GITHUB_SHA'/' kurtosis_config_with_p.yaml > kurtosis_config.yaml
+
+ - name: Run Kurtosis + assertoor tests
+ uses: ethpandaops/kurtosis-assertoor-github-action@v1
+ with:
+ enclave_name: "kurtosis-run-${{ matrix.os.name }}-${{ github.run_id }}"
+ ethereum_package_args: "./kurtosis_config.yaml"
+ #kurtosis_extra_args: --verbosity detailed --cli-log-level trace
+ enclave_dump: false

.gitignore

@@ -16,7 +16,7 @@
 build/_vendor/pkg
 /*.a
 docs/readthedocs/build
-
+/.VSCodeCounter
 #*
 .#*
 *#

accounts/abi/bind/backends/simulated.go

@@ -647,7 +647,7 @@ func (b *SimulatedBackend) EstimateGas(ctx context.Context, call ethereum.CallMs
  }
  }
  gasCap = hi
- b.pendingState.SetTxContext(libcommon.Hash{}, libcommon.Hash{}, len(b.pendingBlock.Transactions()))
+ b.pendingState.SetTxContext(libcommon.Hash{}, len(b.pendingBlock.Transactions()))
 
  // Create a helper to check if a gas allowance results in an executable transaction
  executable := func(gas uint64) (bool, *evmtypes.ExecutionResult, error) {
@@ -756,7 +756,7 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, txn types.Transa
  return fmt.Errorf("invalid transaction nonce: got %d, want %d", txn.GetNonce(), nonce)
  }
 
- b.pendingState.SetTxContext(txn.Hash(), libcommon.Hash{}, len(b.pendingBlock.Transactions()))
+ b.pendingState.SetTxContext(txn.Hash(), len(b.pendingBlock.Transactions()))
  //fmt.Printf("==== Start producing block %d, header: %d\n", b.pendingBlock.NumberU64(), b.pendingHeader.Number.Uint64())
  if _, _, err := core.ApplyTransaction(
  b.m.ChainConfig, core.GetHashFn(b.pendingHeader, b.getHeader), b.m.Engine,

cl/antiquary/antiquary.go

@@ -40,7 +40,7 @@ import (
  "github.com/erigontech/erigon/turbo/snapshotsync/freezeblocks"
 )
 
-const safetyMargin = 2_000 // We retire snapshots 2k blocks after the finalized head
+const safetyMargin = 10_000 // We retire snapshots 10k blocks after the finalized head
 
 // Antiquary is where the snapshots go, aka old history, it is what keep track of the oldest records.
 type Antiquary struct {
@@ -360,6 +360,10 @@ func (a *Antiquary) antiquateBlobs() error {
  defer roTx.Rollback()
  // perform blob antiquation if it is time to.
  currentBlobsProgress := a.sn.FrozenBlobs()
+ // We should NEVER get ahead of the block snapshots.
+ if currentBlobsProgress >= a.sn.BlocksAvailable() {
+ return nil
+ }
  minimunBlobsProgress := ((a.cfg.DenebForkEpoch * a.cfg.SlotsPerEpoch) / snaptype.Erigon2MergeLimit) * snaptype.Erigon2MergeLimit
  currentBlobsProgress = max(currentBlobsProgress, minimunBlobsProgress)
  // read the finalized head

cl/antiquary/state_antiquary.go

@@ -440,8 +440,9 @@ func (s *Antiquary) initializeStateAntiquaryIfNeeded(ctx context.Context, tx kv.
  return err
  }
  // We want to backoff by some slots until we get a correct state from DB.
- // we start from 1 * clparams.SlotsPerDump.
- backoffStep := uint64(10)
+ // we start from 10 * clparams.SlotsPerDump.
+ backoffStrides := uint64(10)
+ backoffStep := backoffStrides
 
  historicalReader := historical_states_reader.NewHistoricalStatesReader(s.cfg, s.snReader, s.validatorsTable, s.genesisState)
 
@@ -464,6 +465,11 @@ func (s *Antiquary) initializeStateAntiquaryIfNeeded(ctx context.Context, tx kv.
  if err != nil {
  return fmt.Errorf("failed to read historical state at slot %d: %w", attempt, err)
  }
+ if s.currentState == nil {
+ log.Warn("historical state not found, backoff more and try again", "slot", attempt)
+ backoffStep += backoffStrides
+ continue
+ }
 
  computedBlockRoot, err := s.currentState.BlockRoot()
  if err != nil {
@@ -476,7 +482,7 @@ func (s *Antiquary) initializeStateAntiquaryIfNeeded(ctx context.Context, tx kv.
  if computedBlockRoot != expectedBlockRoot {
  log.Debug("Block root mismatch, trying again", "slot", attempt, "expected", expectedBlockRoot)
  // backoff more
- backoffStep += 10
+ backoffStep += backoffStrides
  continue
  }
  break