Merge branch 'develop' of github.com:smartcontractkit/chainlink into …

…develop

.github/workflows/integration-tests.yml

@@ -592,6 +592,7 @@ jobs:
       ]
     env:
       CONTRACT_ARTIFACTS_PATH: contracts/target/deploy
+      GOTOOLCHAIN: auto
     steps:
       - name: Checkout the repo
         if: (needs.changes.outputs.core_changes == 'true' || github.event_name == 'workflow_dispatch') && needs.solana-test-image-exists.outputs.exists == 'false'

README.md

@@ -32,7 +32,7 @@ regarding Chainlink social accounts, news, and networking.
 
 ## Build Chainlink
 
-1. [Install Go 1.22](https://golang.org/doc/install), and add your GOPATH's [bin directory to your PATH](https://golang.org/doc/code.html#GOPATH)
+1. [Install Go 1.23](https://golang.org/doc/install), and add your GOPATH's [bin directory to your PATH](https://golang.org/doc/code.html#GOPATH)
    - Example Path for macOS `export PATH=$GOPATH/bin:$PATH` & `export GOPATH=/Users/$USER/go`
 2. Install [NodeJS v20](https://nodejs.org/en/download/package-manager/) & [pnpm v9 via npm](https://pnpm.io/installation#using-npm).
    - It might be easier long term to use [nvm](https://nodejs.org/en/download/package-manager/#nvm) to switch between node versions for different projects. For example, assuming $NODE_VERSION was set to a valid version of NodeJS, you could run: `nvm install $NODE_VERSION && nvm use $NODE_VERSION`

core/cmd/shell.go

@@ -37,6 +37,7 @@ import (
 	"github.com/smartcontractkit/chainlink-common/pkg/loop"
 	"github.com/smartcontractkit/chainlink-common/pkg/sqlutil"
 	"github.com/smartcontractkit/chainlink-common/pkg/utils/mailbox"
+
 	"github.com/smartcontractkit/chainlink/v2/core/build"
 	"github.com/smartcontractkit/chainlink/v2/core/capabilities"
 	"github.com/smartcontractkit/chainlink/v2/core/chains/legacyevm"
@@ -269,6 +270,7 @@ func (n ChainlinkAppFactory) NewApplication(ctx context.Context, cfg chainlink.G
 		solanaCfg := chainlink.SolanaFactoryConfig{
 			Keystore:    keyStore.Solana(),
 			TOMLConfigs: cfg.SolanaConfigs(),
+			DS:          ds,
 		}
 		initOps = append(initOps, chainlink.InitSolana(ctx, relayerFactory, solanaCfg))
 	}

core/cmd/shell_test.go

@@ -18,8 +18,10 @@ import (
 	"github.com/urfave/cli"
 
 	commoncfg "github.com/smartcontractkit/chainlink-common/pkg/config"
+	"github.com/smartcontractkit/chainlink-common/pkg/sqlutil/sqltest"
 	solcfg "github.com/smartcontractkit/chainlink-solana/pkg/solana/config"
 	stkcfg "github.com/smartcontractkit/chainlink-starknet/relayer/pkg/chainlink/config"
+
 	"github.com/smartcontractkit/chainlink/v2/core/cmd"
 	"github.com/smartcontractkit/chainlink/v2/core/internal/cltest"
 	"github.com/smartcontractkit/chainlink/v2/core/internal/testutils"
@@ -353,6 +355,7 @@ func TestSetupSolanaRelayer(t *testing.T) {
 	lggr := logger.TestLogger(t)
 	reg := plugins.NewLoopRegistry(lggr, nil, nil, nil, "")
 	ks := mocks.NewSolana(t)
+	ds := sqltest.NewNoOpDataSource()
 
 	// config 3 chains but only enable 2 => should only be 2 relayer
 	nEnabledChains := 2
@@ -395,9 +398,14 @@ func TestSetupSolanaRelayer(t *testing.T) {
 		LoopRegistry: reg,
 	}
 
+	cfg := chainlink.SolanaFactoryConfig{
+		Keystore:    ks,
+		TOMLConfigs: tConfig.SolanaConfigs(),
+		DS:          ds}
+
 	// not parallel; shared state
 	t.Run("no plugin", func(t *testing.T) {
-		relayers, err := rf.NewSolana(ks, tConfig.SolanaConfigs())
+		relayers, err := rf.NewSolana(cfg)
 		require.NoError(t, err)
 		require.NotNil(t, relayers)
 		require.Len(t, relayers, nEnabledChains)
@@ -408,7 +416,7 @@ func TestSetupSolanaRelayer(t *testing.T) {
 	t.Run("plugin", func(t *testing.T) {
 		t.Setenv("CL_SOLANA_CMD", "phony_solana_cmd")
 
-		relayers, err := rf.NewSolana(ks, tConfig.SolanaConfigs())
+		relayers, err := rf.NewSolana(cfg)
 		require.NoError(t, err)
 		require.NotNil(t, relayers)
 		require.Len(t, relayers, nEnabledChains)
@@ -433,32 +441,41 @@ func TestSetupSolanaRelayer(t *testing.T) {
 			},
 		}
 	})
+	dupCfg := chainlink.SolanaFactoryConfig{
+		Keystore:    ks,
+		TOMLConfigs: duplicateConfig.SolanaConfigs(),
+		DS:          ds,
+	}
 
 	// not parallel; shared state
 	t.Run("no plugin, duplicate chains", func(t *testing.T) {
-		_, err := rf.NewSolana(ks, duplicateConfig.SolanaConfigs())
+		_, err := rf.NewSolana(dupCfg)
 		require.Error(t, err)
 	})
 
 	t.Run("plugin, duplicate chains", func(t *testing.T) {
 		t.Setenv("CL_SOLANA_CMD", "phony_solana_cmd")
-		_, err := rf.NewSolana(ks, duplicateConfig.SolanaConfigs())
+		_, err := rf.NewSolana(dupCfg)
 		require.Error(t, err)
 	})
 
 	t.Run("plugin env parsing fails", func(t *testing.T) {
 		t.Setenv("CL_SOLANA_CMD", "phony_solana_cmd")
 		t.Setenv("CL_SOLANA_ENV", "fake_path")
 
-		_, err := rf.NewSolana(ks, t2Config.SolanaConfigs())
+		_, err := rf.NewSolana(chainlink.SolanaFactoryConfig{
+			Keystore:    ks,
+			TOMLConfigs: t2Config.SolanaConfigs(),
+			DS:          ds,
+		})
 		require.Error(t, err)
 		require.Contains(t, err.Error(), "failed to parse Solana env file")
 	})
 
 	t.Run("plugin already registered", func(t *testing.T) {
 		t.Setenv("CL_SOLANA_CMD", "phony_solana_cmd")
 
-		_, err := rf.NewSolana(ks, tConfig.SolanaConfigs())
+		_, err := rf.NewSolana(cfg)
 		require.Error(t, err)
 		require.Contains(t, err.Error(), "failed to create Solana LOOP command")
 	})

core/internal/cltest/cltest.go

@@ -452,6 +452,7 @@ func NewApplicationWithConfig(t testing.TB, cfg chainlink.GeneralConfig, flagsAn
 		solanaCfg := chainlink.SolanaFactoryConfig{
 			Keystore:    keyStore.Solana(),
 			TOMLConfigs: cfg.SolanaConfigs(),
+			DS:          ds,
 		}
 		initOps = append(initOps, chainlink.InitSolana(ctx, relayerFactory, solanaCfg))
 	}

core/scripts/go.mod

@@ -303,7 +303,7 @@ require (
 	github.com/smartcontractkit/chainlink-feeds v0.1.1 // indirect
 	github.com/smartcontractkit/chainlink-protos/job-distributor v0.6.0 // indirect
 	github.com/smartcontractkit/chainlink-protos/orchestrator v0.3.0 // indirect
-	github.com/smartcontractkit/chainlink-solana v1.1.1-0.20241118190857-e2db20a6a969 // indirect
+	github.com/smartcontractkit/chainlink-solana v1.1.1-0.20241127201057-3c9282e39749 // indirect
 	github.com/smartcontractkit/chainlink-starknet/relayer v0.1.1-0.20241017135645-176a23722fd8 // indirect
 	github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7 // indirect
 	github.com/smartcontractkit/tdh2/go/ocr2/decryptionplugin v0.0.0-20241009055228-33d0c0bf38de // indirect

core/scripts/go.sum

@@ -1106,8 +1106,8 @@ github.com/smartcontractkit/chainlink-protos/job-distributor v0.6.0 h1:0ewLMbAz3
 github.com/smartcontractkit/chainlink-protos/job-distributor v0.6.0/go.mod h1:/dVVLXrsp+V0AbcYGJo3XMzKg3CkELsweA/TTopCsKE=
 github.com/smartcontractkit/chainlink-protos/orchestrator v0.3.0 h1:PBUaFfPLm+Efq7H9kdfGBivH+QhJ6vB5EZTR/sCZsxI=
 github.com/smartcontractkit/chainlink-protos/orchestrator v0.3.0/go.mod h1:m/A3lqD7ms/RsQ9BT5P2uceYY0QX5mIt4KQxT2G6qEo=
-github.com/smartcontractkit/chainlink-solana v1.1.1-0.20241118190857-e2db20a6a969 h1:M/SMFCY4URO0H1eB9r3pkRv0LS3Ofxk/GapSgGrLfFI=
-github.com/smartcontractkit/chainlink-solana v1.1.1-0.20241118190857-e2db20a6a969/go.mod h1:mGmRvlk54ufCufV4EBWizOGtXoXfePoFAuYEVC8EwdY=
+github.com/smartcontractkit/chainlink-solana v1.1.1-0.20241127201057-3c9282e39749 h1:gkrjGJAtbKMOliJPaZ73EyJmO8AyDVi80+PEJocRMn4=
+github.com/smartcontractkit/chainlink-solana v1.1.1-0.20241127201057-3c9282e39749/go.mod h1:nkIegLHodyrrZguxkYEHcNw2vAXv8H8xlCoLzwylcL0=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.1.1-0.20241017135645-176a23722fd8 h1:B4DFdk6MGcQnoCjjMBCx7Z+GWQpxRWJ4O8W/dVJyWGA=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.1.1-0.20241017135645-176a23722fd8/go.mod h1:WkBqgBo+g34Gm5vWkDDl8Fh3Mzd7bF5hXp7rryg0t5o=
 github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7 h1:12ijqMM9tvYVEm+nR826WsrNi6zCKpwBhuApq127wHs=

core/services/chainlink/relayer_chain_interoperators.go

@@ -11,6 +11,7 @@ import (
 	"github.com/smartcontractkit/chainlink-common/pkg/types"
 	"github.com/smartcontractkit/chainlink-cosmos/pkg/cosmos"
 	"github.com/smartcontractkit/chainlink-cosmos/pkg/cosmos/adapters"
+
 	"github.com/smartcontractkit/chainlink/v2/core/services/relay"
 
 	"github.com/smartcontractkit/chainlink/v2/core/chains"
@@ -154,7 +155,7 @@ func InitCosmos(ctx context.Context, factory RelayerFactory, config CosmosFactor
 // InitSolana is a option for instantiating Solana relayers
 func InitSolana(ctx context.Context, factory RelayerFactory, config SolanaFactoryConfig) CoreRelayerChainInitFunc {
 	return func(op *CoreRelayerChainInteroperators) error {
-		solRelayers, err := factory.NewSolana(config.Keystore, config.TOMLConfigs)
+		solRelayers, err := factory.NewSolana(config)
 		if err != nil {
 			return fmt.Errorf("failed to setup Solana relayer: %w", err)
 		}

core/services/chainlink/relayer_factory.go

@@ -107,9 +107,11 @@ func (r *RelayerFactory) NewEVM(ctx context.Context, config EVMFactoryConfig) (m
 type SolanaFactoryConfig struct {
 	Keystore keystore.Solana
 	solcfg.TOMLConfigs
+	DS sqlutil.DataSource
 }
 
-func (r *RelayerFactory) NewSolana(ks keystore.Solana, chainCfgs solcfg.TOMLConfigs) (map[types.RelayID]loop.Relayer, error) {
+func (r *RelayerFactory) NewSolana(config SolanaFactoryConfig) (map[types.RelayID]loop.Relayer, error) {
+	chainCfgs, ds, ks := config.TOMLConfigs, config.DS, config.Keystore
 	solanaRelayers := make(map[types.RelayID]loop.Relayer)
 	var (
 		solLggr = r.Logger.Named("Solana")
@@ -162,6 +164,7 @@ func (r *RelayerFactory) NewSolana(ks keystore.Solana, chainCfgs solcfg.TOMLConf
 			opts := solana.ChainOpts{
 				Logger:   lggr,
 				KeyStore: signer,
+				DS:       ds,
 			}
 
 			chain, err := solana.NewChain(chainCfg, opts)

core/services/relay/evm/capabilities/workflows/syncer/workflow_syncer_test.go

@@ -5,11 +5,13 @@ import (
 	"crypto/rand"
 	"encoding/hex"
 	"encoding/json"
+	"fmt"
 	"testing"
 	"time"
 
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/stretchr/testify/assert"
 
 	"github.com/smartcontractkit/chainlink-common/pkg/custmsg"
 	"github.com/smartcontractkit/chainlink-common/pkg/services/servicetest"
@@ -26,6 +28,111 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+type testEvtHandler struct {
+	events []syncer.Event
+}
+
+func (m *testEvtHandler) Handle(ctx context.Context, event syncer.Event) error {
+	m.events = append(m.events, event)
+	return nil
+}
+
+func newTestEvtHandler() *testEvtHandler {
+	return &testEvtHandler{
+		events: make([]syncer.Event, 0),
+	}
+}
+
+type testWorkflowRegistryContractLoader struct {
+}
+
+func (m *testWorkflowRegistryContractLoader) LoadWorkflows(ctx context.Context) (*types.Head, error) {
+	return &types.Head{
+		Height:    "0",
+		Hash:      nil,
+		Timestamp: 0,
+	}, nil
+}
+
+func Test_InitialStateSync(t *testing.T) {
+	ctx := coretestutils.Context(t)
+	lggr := logger.TestLogger(t)
+	backendTH := testutils.NewEVMBackendTH(t)
+	donID := uint32(1)
+
+	// Deploy a test workflow_registry
+	wfRegistryAddr, _, wfRegistryC, err := workflow_registry_wrapper.DeployWorkflowRegistry(backendTH.ContractsOwner, backendTH.Backend.Client())
+	backendTH.Backend.Commit()
+	require.NoError(t, err)
+
+	// Build the ContractReader config
+	contractReaderCfg := evmtypes.ChainReaderConfig{
+		Contracts: map[string]evmtypes.ChainContractReader{
+			syncer.WorkflowRegistryContractName: {
+				ContractABI: workflow_registry_wrapper.WorkflowRegistryABI,
+				Configs: map[string]*evmtypes.ChainReaderDefinition{
+					syncer.GetWorkflowMetadataListByDONMethodName: {
+						ChainSpecificName: syncer.GetWorkflowMetadataListByDONMethodName,
+					},
+				},
+			},
+		},
+	}
+
+	contractReaderCfgBytes, err := json.Marshal(contractReaderCfg)
+	require.NoError(t, err)
+
+	contractReader, err := backendTH.NewContractReader(ctx, t, contractReaderCfgBytes)
+	require.NoError(t, err)
+
+	err = contractReader.Bind(ctx, []types.BoundContract{{Name: syncer.WorkflowRegistryContractName, Address: wfRegistryAddr.Hex()}})
+	require.NoError(t, err)
+
+	// setup contract state to allow the secrets to be updated
+	updateAllowedDONs(t, backendTH, wfRegistryC, []uint32{donID}, true)
+	updateAuthorizedAddress(t, backendTH, wfRegistryC, []common.Address{backendTH.ContractsOwner.From}, true)
+
+	// The number of workflows should be greater than the workflow registry contracts pagination limit to ensure
+	// that the syncer will query the contract multiple times to get the full list of workflows
+	numberWorkflows := 250
+	for i := 0; i < numberWorkflows; i++ {
+		var workflowID [32]byte
+		_, err = rand.Read((workflowID)[:])
+		require.NoError(t, err)
+		workflow := RegisterWorkflowCMD{
+			Name:       fmt.Sprintf("test-wf-%d", i),
+			DonID:      donID,
+			Status:     uint8(1),
+			SecretsURL: "someurl",
+		}
+		workflow.ID = workflowID
+		registerWorkflow(t, backendTH, wfRegistryC, workflow)
+	}
+
+	testEventHandler := newTestEvtHandler()
+	loader := syncer.NewWorkflowRegistryContractLoader(wfRegistryAddr.Hex(), donID, contractReader, testEventHandler)
+
+	// Create the worker
+	worker := syncer.NewWorkflowRegistry(
+		lggr,
+		contractReader,
+		wfRegistryAddr.Hex(),
+		syncer.WorkflowEventPollerConfig{
+			QueryCount: 20,
+		},
+		testEventHandler,
+		loader,
+		syncer.WithTicker(make(chan time.Time)),
+	)
+
+	servicetest.Run(t, worker)
+
+	assert.Len(t, testEventHandler.events, numberWorkflows)
+	for _, event := range testEventHandler.events {
+		assert.Equal(t, syncer.WorkflowRegisteredEvent, event.GetEventType())
+	}
+}
+
 func Test_SecretsWorker(t *testing.T) {
 	var (
 		ctx       = coretestutils.Context(t)
@@ -49,7 +156,7 @@ func Test_SecretsWorker(t *testing.T) {
 		fetcherFn    = func(_ context.Context, _ string) ([]byte, error) {
 			return []byte(wantContents), nil
 		}
-		contractName            = syncer.ContractName
+		contractName            = syncer.WorkflowRegistryContractName
 		forceUpdateSecretsEvent = string(syncer.ForceUpdateSecretsEvent)
 	)
 
@@ -81,6 +188,9 @@ func Test_SecretsWorker(t *testing.T) {
 						ChainSpecificName: forceUpdateSecretsEvent,
 						ReadType:          evmtypes.Event,
 					},
+					syncer.GetWorkflowMetadataListByDONMethodName: {
+						ChainSpecificName: syncer.GetWorkflowMetadataListByDONMethodName,
+					},
 				},
 			},
 		},
@@ -112,26 +222,21 @@ func Test_SecretsWorker(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, contents, giveContents)
 
-	// Create the worker
-	worker := syncer.NewWorkflowRegistry(
-		lggr,
-		orm,
-		contractReader,
-		fetcherFn,
-		wfRegistryAddr.Hex(),
-		nil,
-		nil,
-		emitter,
-		syncer.WithTicker(giveTicker.C),
-	)
+	handler := syncer.NewEventHandler(lggr, orm, fetcherFn, nil, nil,
+		emitter, nil)
 
-	servicetest.Run(t, worker)
+	worker := syncer.NewWorkflowRegistry(lggr, contractReader, wfRegistryAddr.Hex(),
+		syncer.WorkflowEventPollerConfig{
+			QueryCount: 20,
+		}, handler, &testWorkflowRegistryContractLoader{}, syncer.WithTicker(giveTicker.C))
 
 	// setup contract state to allow the secrets to be updated
 	updateAllowedDONs(t, backendTH, wfRegistryC, []uint32{donID}, true)
 	updateAuthorizedAddress(t, backendTH, wfRegistryC, []common.Address{backendTH.ContractsOwner.From}, true)
 	registerWorkflow(t, backendTH, wfRegistryC, giveWorkflow)
 
+	servicetest.Run(t, worker)
+
 	// generate a log event
 	requestForceUpdateSecrets(t, backendTH, wfRegistryC, giveSecretsURL)