fix track config delay in ocr soak (#15144)

* fix track config delay in ocr soak

* Extracts link deployment

* Replace bespoke retry logic

* Update field name

* go mod tidy

integration-tests/actions/ocr2_helpers.go

@@ -4,12 +4,15 @@ import (
 	"crypto/ed25519"
 	"encoding/hex"
 	"fmt"
+	"net/http"
 	"strings"
 	"time"
 
+	"github.com/avast/retry-go"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/google/uuid"
 	"github.com/lib/pq"
+	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"golang.org/x/sync/errgroup"
 	"gopkg.in/guregu/null.v4"
@@ -192,6 +195,7 @@ func CreateOCRv2Jobs(
 	mockServerValue int, // Value to get from the mock server when querying the path
 	chainId int64, // EVM chain ID
 	forwardingAllowed bool,
+	l zerolog.Logger,
 ) error {
 	// Collect P2P ID
 	bootstrapP2PIds, err := bootstrapNode.MustReadP2PKeys()
@@ -218,6 +222,9 @@ func CreateOCRv2Jobs(
 		}
 	}
 
+	// Initialize map to store job IDs for each chainlink node
+	jobIDs := make(map[*nodeclient.ChainlinkK8sClient][]string)
+
 	for _, ocrInstance := range ocrInstances {
 		bootstrapSpec := &nodeclient.OCR2TaskJobSpec{
 			Name:    fmt.Sprintf("ocr2-bootstrap-%s", ocrInstance.Address()),
@@ -284,10 +291,46 @@ func CreateOCRv2Jobs(
 					P2PV2Bootstrappers:                pq.StringArray{p2pV2Bootstrapper},       // bootstrap node key and address <p2p-key>@bootstrap:6690
 				},
 			}
-			_, err = chainlinkNode.MustCreateJob(ocrSpec)
+			var ocrJob *nodeclient.Job
+			ocrJob, err = chainlinkNode.MustCreateJob(ocrSpec)
 			if err != nil {
 				return fmt.Errorf("creating OCR task job on OCR node have failed: %w", err)
 			}
+			jobIDs[chainlinkNode] = append(jobIDs[chainlinkNode], ocrJob.Data.ID) // Store each job ID per node
+		}
+	}
+	l.Info().Msg("Verify OCRv2 jobs have been created")
+	for chainlinkNode, ids := range jobIDs {
+		for _, jobID := range ids {
+			err := retry.Do(
+				func() error {
+					_, resp, err := chainlinkNode.ReadJob(jobID)
+					if err != nil {
+						return err
+					}
+					if resp.StatusCode != http.StatusOK {
+						return fmt.Errorf("unexpected response status: %d", resp.StatusCode)
+					}
+					l.Info().
+						Str("Node", chainlinkNode.PodName).
+						Str("Job ID", jobID).
+						Msg("OCRv2 job successfully created")
+					return nil
+				},
+				retry.Attempts(4),
+				retry.Delay(time.Second*2),
+				retry.OnRetry(func(n uint, err error) {
+					l.Debug().
+						Str("Node", chainlinkNode.PodName).
+						Str("Job ID", jobID).
+						Uint("Attempt", n+1).
+						Err(err).
+						Msg("Retrying job verification")
+				}),
+			)
+			if err != nil {
+				l.Error().Err(err).Str("Node", chainlinkNode.PodName).Str("JobID", jobID).Msg("Failed to verify OCRv2 job creation")
+			}
 		}
 	}
 	return nil

integration-tests/go.mod

@@ -11,6 +11,7 @@ require (
 	dario.cat/mergo v1.0.1
 	github.com/AlekSi/pointer v1.1.0
 	github.com/Masterminds/semver/v3 v3.3.0
+	github.com/avast/retry-go v3.0.0+incompatible
 	github.com/avast/retry-go/v4 v4.6.0
 	github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df
 	github.com/chaos-mesh/chaos-mesh/api v0.0.0-20240821051457-da69c6d9617a
@@ -102,7 +103,6 @@ require (
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/atombender/go-jsonschema v0.16.1-0.20240916205339-a74cd4e2851c // indirect
-	github.com/avast/retry-go v3.0.0+incompatible // indirect
 	github.com/awalterschulze/gographviz v2.0.3+incompatible // indirect
 	github.com/aws/aws-sdk-go v1.54.19 // indirect
 	github.com/aws/aws-sdk-go-v2 v1.32.2 // indirect