Eth2 showcase with log poller (#11214)

* move workflow to correct directory

* streamline on-demand values a bit

* get RPC urls and private keys from secrets

* download and run from inside the test folder

* checkout repo before running tests

* get inputs and mask them

* fix step ordering in workflow

* fix default image tag

* use latest pumba@CTF

* run one test

* fix directory name

* run on powerful runner

* show usage of ethereum env builder with eth2

* update usage of eth2

* adjust to latest

integration-tests/docker/test_env/test_env.go

@@ -40,15 +40,16 @@ type CLClusterTestEnv struct {
 	LogWatch *logwatch.LogWatch
 
 	/* components */
-	ClCluster        *ClCluster
-	Geth             *test_env.Geth          // for tests using --dev networks
-	PrivateChain     []test_env.PrivateChain // for tests using non-dev networks
-	MockAdapter      *test_env.Killgrave
-	EVMClient        blockchain.EVMClient
-	ContractDeployer contracts.ContractDeployer
-	ContractLoader   contracts.ContractLoader
-	l                zerolog.Logger
-	t                *testing.T
+	ClCluster             *ClCluster
+	PrivateChain          []test_env.PrivateChain // for tests using non-dev networks -- unify it with new approach
+	MockAdapter           *test_env.Killgrave
+	EVMClient             blockchain.EVMClient
+	ContractDeployer      contracts.ContractDeployer
+	ContractLoader        contracts.ContractLoader
+	RpcProvider           test_env.RpcProvider
+	PrivateEthereumConfig *test_env.EthereumNetwork // new approach to private chains, supporting eth1 and eth2
+	l                     zerolog.Logger
+	t                     *testing.T
 }
 
 func NewTestEnv() (*CLClusterTestEnv, error) {
@@ -59,27 +60,24 @@ func NewTestEnv() (*CLClusterTestEnv, error) {
 	}
 	n := []string{network.Name}
 	return &CLClusterTestEnv{
-		Geth:        test_env.NewGeth(n),
 		MockAdapter: test_env.NewKillgrave(n, ""),
 		Network:     network,
 		l:           log.Logger,
 	}, nil
 }
 
 // WithTestEnvConfig sets the test environment cfg.
-// Sets up the Geth and MockAdapter containers with the provided cfg.
+// Sets up private ethereum chain and MockAdapter containers with the provided cfg.
 func (te *CLClusterTestEnv) WithTestEnvConfig(cfg *TestEnvConfig) *CLClusterTestEnv {
 	te.Cfg = cfg
 	n := []string{te.Network.Name}
-	te.Geth = test_env.NewGeth(n, test_env.WithContainerName(te.Cfg.Geth.ContainerName))
 	te.MockAdapter = test_env.NewKillgrave(n, te.Cfg.MockAdapter.ImpostersPath, test_env.WithContainerName(te.Cfg.MockAdapter.ContainerName))
 	return te
 }
 
 func (te *CLClusterTestEnv) WithTestLogger(t *testing.T) *CLClusterTestEnv {
 	te.t = t
 	te.l = logging.GetTestLogger(t)
-	te.Geth.WithTestLogger(t)
 	te.MockAdapter.WithTestLogger(t)
 	return te
 }
@@ -128,8 +126,26 @@ func (te *CLClusterTestEnv) StartPrivateChain() error {
 	return nil
 }
 
-func (te *CLClusterTestEnv) StartGeth() (blockchain.EVMNetwork, test_env.InternalDockerUrls, error) {
-	return te.Geth.StartContainer()
+func (te *CLClusterTestEnv) StartEthereumNetwork(cfg *test_env.EthereumNetwork) (blockchain.EVMNetwork, test_env.RpcProvider, error) {
+	// if environment is being restored from a previous state, use the existing config
+	// this might fail terribly if temporary folders with chain data on the host machine were removed
+	if te.Cfg != nil && te.Cfg.EthereumNetwork != nil {
+		builder := test_env.NewEthereumNetworkBuilder()
+		c, err := builder.WithExistingConfig(*te.Cfg.EthereumNetwork).
+			WithTest(te.t).
+			Build()
+		if err != nil {
+			return blockchain.EVMNetwork{}, test_env.RpcProvider{}, err
+		}
+		cfg = &c
+	}
+	n, rpc, err := cfg.Start()
+
+	if err != nil {
+		return blockchain.EVMNetwork{}, test_env.RpcProvider{}, err
+	}
+
+	return n, rpc, nil
 }
 
 func (te *CLClusterTestEnv) StartMockAdapter() error {

integration-tests/docker/test_env/test_env_builder.go

@@ -31,8 +31,8 @@ const (
 )
 
 type CLTestEnvBuilder struct {
-	hasLogWatch            bool
-	hasGeth                bool
+	hasLogWatch bool
+	// hasGeth                bool
 	hasKillgrave           bool
 	hasForwarders          bool
 	clNodeConfig           *chainlink.Config
@@ -49,6 +49,7 @@ type CLTestEnvBuilder struct {
 	cleanUpCustomFn        func()
 	chainOptionsFn         []ChainOption
 	evmClientNetworkOption []EVMClientNetworkOption
+	ethereumNetwork        *test_env.EthereumNetwork
 
 	/* funding */
 	ETHFunds *big.Float
@@ -118,8 +119,27 @@ func (b *CLTestEnvBuilder) WithFunding(eth *big.Float) *CLTestEnvBuilder {
 	return b
 }
 
+// deprecated
+// left only for backward compatibility
 func (b *CLTestEnvBuilder) WithGeth() *CLTestEnvBuilder {
-	b.hasGeth = true
+	ethBuilder := test_env.NewEthereumNetworkBuilder()
+	cfg, err := ethBuilder.
+		WithConsensusType(test_env.ConsensusType_PoW).
+		WithExecutionLayer(test_env.ExecutionLayer_Geth).
+		WithTest(b.t).
+		Build()
+
+	if err != nil {
+		panic(err)
+	}
+
+	b.ethereumNetwork = &cfg
+
+	return b
+}
+
+func (b *CLTestEnvBuilder) WithPrivateEthereumNetwork(en test_env.EthereumNetwork) *CLTestEnvBuilder {
+	b.ethereumNetwork = &en
 	return b
 }
 
@@ -191,13 +211,6 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) {
 			return nil, err
 		}
 	}
-	b.l.Info().
-		Bool("hasGeth", b.hasGeth).
-		Bool("hasKillgrave", b.hasKillgrave).
-		Int("clNodesCount", b.clNodesCount).
-		Strs("customNodeCsaKeys", b.customNodeCsaKeys).
-		Strs("defaultNodeCsaKeys", b.defaultNodeCsaKeys).
-		Msg("Building CL cluster test environment..")
 
 	var err error
 	if b.t != nil {
@@ -259,13 +272,21 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) {
 		}
 		return b.te, nil
 	}
+
 	networkConfig := networks.MustGetSelectedNetworksFromEnv()[0]
-	var internalDockerUrls test_env.InternalDockerUrls
-	if b.hasGeth && networkConfig.Simulated {
-		networkConfig, internalDockerUrls, err = b.te.StartGeth()
+	var rpcProvider test_env.RpcProvider
+	if b.ethereumNetwork != nil && networkConfig.Simulated {
+		// TODO here we should save the ethereum network config to te.Cfg, but it doesn't exist at this point
+		// in general it seems we have no methods for saving config to file and we only load it from file
+		// but I don't know how that config file is to be created or whether anyone ever done that
+		var enCfg test_env.EthereumNetwork
+		b.ethereumNetwork.DockerNetworkNames = []string{b.te.Network.Name}
+		networkConfig, rpcProvider, err = b.te.StartEthereumNetwork(b.ethereumNetwork)
 		if err != nil {
 			return nil, err
 		}
+		b.te.RpcProvider = rpcProvider
+		b.te.PrivateEthereumConfig = &enCfg
 	}
 
 	if !b.isNonEVM {
@@ -311,8 +332,8 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) {
 			var httpUrls []string
 			var wsUrls []string
 			if networkConfig.Simulated {
-				httpUrls = []string{internalDockerUrls.HttpUrl}
-				wsUrls = []string{internalDockerUrls.WsUrl}
+				httpUrls = rpcProvider.PrivateHttpUrls()
+				wsUrls = rpcProvider.PrivateWsUrsl()
 			} else {
 				httpUrls = networkConfig.HTTPURLs
 				wsUrls = networkConfig.URLs
@@ -341,13 +362,28 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) {
 		b.defaultNodeCsaKeys = nodeCsaKeys
 	}
 
-	if b.hasGeth && b.clNodesCount > 0 && b.ETHFunds != nil {
+	if b.ethereumNetwork != nil && b.clNodesCount > 0 && b.ETHFunds != nil {
 		b.te.ParallelTransactions(true)
 		defer b.te.ParallelTransactions(false)
 		if err := b.te.FundChainlinkNodes(b.ETHFunds); err != nil {
 			return nil, err
 		}
 	}
 
+	var enDesc string
+	if b.te.PrivateEthereumConfig != nil {
+		enDesc = b.te.PrivateEthereumConfig.Describe()
+	} else {
+		enDesc = "none"
+	}
+
+	b.l.Info().
+		Str("privateEthereumNetwork", enDesc).
+		Bool("hasKillgrave", b.hasKillgrave).
+		Int("clNodesCount", b.clNodesCount).
+		Strs("customNodeCsaKeys", b.customNodeCsaKeys).
+		Strs("defaultNodeCsaKeys", b.defaultNodeCsaKeys).
+		Msg("Building CL cluster test environment..")
+
 	return b.te, nil
 }

integration-tests/docker/test_env/test_env_config.go

@@ -3,14 +3,16 @@ package test_env
 import (
 	"encoding/json"
 
+	cte "github.com/smartcontractkit/chainlink-testing-framework/docker/test_env"
 	env "github.com/smartcontractkit/chainlink/integration-tests/types/envcommon"
 )
 
 type TestEnvConfig struct {
-	Networks    []string          `json:"networks"`
-	Geth        GethConfig        `json:"geth"`
-	MockAdapter MockAdapterConfig `json:"mock_adapter"`
-	ClCluster   *ClCluster        `json:"clCluster"`
+	Networks        []string             `json:"networks"`
+	Geth            GethConfig           `json:"geth"`
+	MockAdapter     MockAdapterConfig    `json:"mock_adapter"`
+	ClCluster       *ClCluster           `json:"clCluster"`
+	EthereumNetwork *cte.EthereumNetwork `json:"private_ethereum_config"`
 }
 
 type MockAdapterConfig struct {

integration-tests/go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/scylladb/go-reflectx v1.0.1
 	github.com/segmentio/ksuid v1.0.4
 	github.com/slack-go/slack v0.12.2
-	github.com/smartcontractkit/chainlink-testing-framework v1.18.5
+	github.com/smartcontractkit/chainlink-testing-framework v1.18.6
 	github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000
 	github.com/smartcontractkit/libocr v0.0.0-20231107151413-13e0202ae8d7
 	github.com/smartcontractkit/ocr2keepers v0.7.28

integration-tests/go.sum

@@ -2375,8 +2375,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20231023133638-72f4e799ab0
 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20231023133638-72f4e799ab05/go.mod h1:o0Pn1pbaUluboaK6/yhf8xf7TiFCkyFl6WUOdwqamuU=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20231024133459-1ef3a11319eb h1:HiluOfEVGOQTM6BTDImOqYdMZZ7qq7fkZ3TJdmItNr8=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20231024133459-1ef3a11319eb/go.mod h1:/30flFG4L/iCYAFeA3DUzR0xuHSxAMONiWTzyzvsNwo=
-github.com/smartcontractkit/chainlink-testing-framework v1.18.5 h1:R0f13AUbon1ltHE/vudkyUnLRGaoeocIDVv+FsHZjno=
-github.com/smartcontractkit/chainlink-testing-framework v1.18.5/go.mod h1:zScXRqmvbyTFUooyLYrOp4+V/sFPUbFJNRc72YmnuIk=
+github.com/smartcontractkit/chainlink-testing-framework v1.18.6 h1:UL3DxsPflSRALP62rsg5v3NdOsa8RHGhHMUImoWDD6k=
+github.com/smartcontractkit/chainlink-testing-framework v1.18.6/go.mod h1:zScXRqmvbyTFUooyLYrOp4+V/sFPUbFJNRc72YmnuIk=
 github.com/smartcontractkit/go-plugin v0.0.0-20231003134350-e49dad63b306 h1:ko88+ZznniNJZbZPWAvHQU8SwKAdHngdDZ+pvVgB5ss=
 github.com/smartcontractkit/go-plugin v0.0.0-20231003134350-e49dad63b306/go.mod h1:w1sAEES3g3PuV/RzUrgow20W2uErMly84hhD3um1WL4=
 github.com/smartcontractkit/grpc-proxy v0.0.0-20230731113816-f1be6620749f h1:hgJif132UCdjo8u43i7iPN1/MFnu49hv7lFGFftCHKU=

integration-tests/smoke/automation_test.go

@@ -1070,8 +1070,8 @@ func setupAutomationTestDocker(
 		var httpUrls []string
 		var wsUrls []string
 		if network.Simulated {
-			httpUrls = []string{env.Geth.InternalHttpUrl}
-			wsUrls = []string{env.Geth.InternalWsUrl}
+			httpUrls = []string{env.RpcProvider.PrivateHttpUrls()[0]}
+			wsUrls = []string{env.RpcProvider.PrivateWsUrsl()[0]}
 		} else {
 			httpUrls = network.HTTPURLs
 			wsUrls = network.URLs