Merge branch 'develop' into chore/re-1882-action-version-bumps

.github/tracing/README.md

@@ -48,4 +48,55 @@ This folder contains the following config files:
 
 These config files are for an OTEL collector, grafana Tempo, and a grafana UI instance to run as containers on the same network.
 `otel-collector-dev.yaml` is the configuration for dev (i.e. your local machine) environments, and forwards traces from the otel collector to the grafana tempo instance on the same network. 
-`otel-collector-ci.yaml` is the configuration for the CI runs, and exports the trace data to the artifact from the github run. 
+`otel-collector-ci.yaml` is the configuration for the CI runs, and exports the trace data to the artifact from the github run.
+
+## Adding Traces to Plugins and to core
+
+Adding traces requires identifying an observability gap in a related group of code executions or a critical path in your application. This is intuitive for the developer:
+
+- "What's the flow of component interaction in this distributed system?"
+- "What's the behavior of the JobProcessorOne component when jobs with [x, y, z] attributes are processed?"
+- "Is this critical path workflow behaving the way we expect?"
+
+The developer will measure a flow of execution from end to end in one trace. Each logically separate measure of this flow is called a span. Spans have either one or no parent span and multiple children span. The relationship between parent and child spans in agreggate will form a directed acyclic graph. The trace begins at the root of this graph.
+
+The most trivial application of a span is measuring top level performance in one critical path. There is much more you can do, including creating human readable and timestamped events within a span (useful for monitoring concurrent access to resources), recording errors, linking parent and children spans through large parts of an application, and even extending a span beyond a single process.
+
+Spans are created by `tracers` and passed through go applications by `Context`s. A tracer must be initialized first. Both core and plugin developers will initialize a tracer from the globally registered trace provider:
+
+```
+tracer := otel.GetTracerProvider().Tracer("example.com/foo")
+```
+
+The globally registered tracer provider is available for plugins after they are initialized, and available in core after configuration is processed (`initGlobals`).
+
+Add spans by:
+```
+  func interestingFunc() {
+    // Assuming there is an appropriate parentContext
+	ctx, span := tracer.Start(parentContext, "hello-span")
+	defer span.End()
+
+	// do some work to track with hello-span
+  }
+```
+As implied by the example, `span` is a child of its parent span captured by `parentContext`.
+
+
+Note that in certain situations, there are 3rd party libraries that will setup spans. For instance:
+
+```
+import (
+	"github.com/gin-gonic/gin"
+	"go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin"
+)
+
+router := gin.Default()
+router.Use(otelgin.Middleware("service-name"))
+```
+
+The developer aligns with best practices when they:
+- Start with critical paths
+- Measure paths from end to end (Context is wired all the way through)
+- Emphasize broadness of measurement over depth
+- Use automatic instrumentation if possible

.github/workflows/integration-tests.yml

@@ -326,18 +326,12 @@ jobs:
             nodes: 1
             os: ubuntu-latest
             file: ocr
-            pyroscope_env: ci-smoke-ocr-evm-simulated                         
+            pyroscope_env: ci-smoke-ocr-evm-simulated
           - name: ocr2
             nodes: 1
             os: ubuntu-latest
             file: ocr2
-            pyroscope_env: ci-smoke-ocr2-evm-simulated                                                     
-          - name: ocr2
-            nodes: 1
-            os: ubuntu-latest
-            run: -run TestOCRv2Request
-            file: ocr2
-            pyroscope_env: ci-smoke-ocr2-evm-simulated                                                           
+            pyroscope_env: ci-smoke-ocr2-evm-simulated
           - name: ocr2
             nodes: 1
             os: ubuntu-latest
@@ -358,7 +352,7 @@ jobs:
           - name: vrfv2plus
             nodes: 1
             os: ubuntu-latest
-            pyroscope_env: ci-smoke-vrf2plus-evm-simulated                       
+            pyroscope_env: ci-smoke-vrf2plus-evm-simulated
           - name: forwarder_ocr
             nodes: 1
             os: ubuntu-latest

core/cmd/app.go

@@ -162,6 +162,17 @@ func NewApp(s *Shell) *cli.App {
 			Usage:       "Commands for the node's configuration",
 			Subcommands: initRemoteConfigSubCmds(s),
 		},
+		{
+			Name:   "health",
+			Usage:  "Prints a health report",
+			Action: s.Health,
+			Flags: []cli.Flag{
+				cli.BoolFlag{
+					Name:  "json, j",
+					Usage: "json output",
+				},
+			},
+		},
 		{
 			Name:        "jobs",
 			Usage:       "Commands for managing Jobs",

core/cmd/shell_local.go

@@ -51,6 +51,7 @@ import (
 	"github.com/smartcontractkit/chainlink/v2/core/utils"
 	"github.com/smartcontractkit/chainlink/v2/core/web"
 	webPresenters "github.com/smartcontractkit/chainlink/v2/core/web/presenters"
+	"github.com/smartcontractkit/chainlink/v2/internal/testdb"
 )
 
 var ErrProfileTooLong = errors.New("requested profile duration too large")
@@ -258,13 +259,6 @@ func initLocalSubCmds(s *Shell, safe bool) []cli.Command {
 // ownerPermsMask are the file permission bits reserved for owner.
 const ownerPermsMask = os.FileMode(0o700)
 
-// PristineDBName is a clean copy of test DB with migrations.
-// Used by heavyweight.FullTestDB* functions.
-const (
-	PristineDBName   = "chainlink_test_pristine"
-	TestDBNamePrefix = "chainlink_test_"
-)
-
 // RunNode starts the Chainlink core.
 func (s *Shell) RunNode(c *cli.Context) error {
 	if err := s.runNode(c); err != nil {
@@ -815,7 +809,7 @@ func dropDanglingTestDBs(lggr logger.Logger, db *sqlx.DB) (err error) {
 		}()
 	}
 	for _, dbname := range dbs {
-		if strings.HasPrefix(dbname, TestDBNamePrefix) && !strings.HasSuffix(dbname, "_pristine") {
+		if strings.HasPrefix(dbname, testdb.TestDBNamePrefix) && !strings.HasSuffix(dbname, "_pristine") {
 			ch <- dbname
 		}
 	}
@@ -1085,11 +1079,11 @@ func dropAndCreateDB(parsed url.URL) (err error) {
 }
 
 func dropAndCreatePristineDB(db *sqlx.DB, template string) (err error) {
-	_, err = db.Exec(fmt.Sprintf(`DROP DATABASE IF EXISTS "%s"`, PristineDBName))
+	_, err = db.Exec(fmt.Sprintf(`DROP DATABASE IF EXISTS "%s"`, testdb.PristineDBName))
 	if err != nil {
 		return fmt.Errorf("unable to drop postgres database: %v", err)
 	}
-	_, err = db.Exec(fmt.Sprintf(`CREATE DATABASE "%s" WITH TEMPLATE "%s"`, PristineDBName, template))
+	_, err = db.Exec(fmt.Sprintf(`CREATE DATABASE "%s" WITH TEMPLATE "%s"`, testdb.PristineDBName, template))
 	if err != nil {
 		return fmt.Errorf("unable to create postgres database: %v", err)
 	}

core/cmd/shell_remote.go

@@ -11,6 +11,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/gin-gonic/gin"
 	"github.com/manyminds/api2go/jsonapi"
 	"github.com/mitchellh/go-homedir"
 	"github.com/pelletier/go-toml"
@@ -511,6 +512,23 @@ func (s *Shell) checkRemoteBuildCompatibility(lggr logger.Logger, onlyWarn bool,
 	return nil
 }
 
+func (s *Shell) Health(c *cli.Context) error {
+	mime := gin.MIMEPlain
+	if c.Bool("json") {
+		mime = gin.MIMEJSON
+	}
+	resp, err := s.HTTP.Get("/health", map[string]string{"Accept": mime})
+	if err != nil {
+		return s.errorOut(err)
+	}
+	b, err := parseResponse(resp)
+	if err != nil {
+		return s.errorOut(err)
+	}
+	fmt.Println(string(b))
+	return nil
+}
+
 // ErrIncompatible is returned when the cli and remote versions are not compatible.
 type ErrIncompatible struct {
 	CLIVersion, CLISha       string

core/internal/cltest/heavyweight/orm.go

@@ -1,13 +1,8 @@
-package heavyweight
-
-// The heavyweight package contains cltest items that are costly and you should
+// Package heavyweight contains test helpers that are costly and you should
 // think **real carefully** before using in your tests.
+package heavyweight
 
 import (
-	"database/sql"
-	"errors"
-	"fmt"
-	"net/url"
 	"os"
 	"path"
 	"runtime"
@@ -20,41 +15,45 @@ import (
 
 	"github.com/jmoiron/sqlx"
 
-	"github.com/smartcontractkit/chainlink/v2/core/cmd"
 	"github.com/smartcontractkit/chainlink/v2/core/internal/testutils"
 	"github.com/smartcontractkit/chainlink/v2/core/internal/testutils/configtest"
 	"github.com/smartcontractkit/chainlink/v2/core/services/chainlink"
 	"github.com/smartcontractkit/chainlink/v2/core/services/pg"
 	"github.com/smartcontractkit/chainlink/v2/core/store/dialects"
 	"github.com/smartcontractkit/chainlink/v2/core/store/models"
+	"github.com/smartcontractkit/chainlink/v2/internal/testdb"
 )
 
 // FullTestDBV2 creates a pristine DB which runs in a separate database than the normal
 // unit tests, so you can do things like use other Postgres connection types with it.
 func FullTestDBV2(t testing.TB, overrideFn func(c *chainlink.Config, s *chainlink.Secrets)) (chainlink.GeneralConfig, *sqlx.DB) {
-	return prepareFullTestDBV2(t, false, true, overrideFn)
+	return KindFixtures.PrepareDB(t, overrideFn)
 }
 
 // FullTestDBNoFixturesV2 is the same as FullTestDB, but it does not load fixtures.
 func FullTestDBNoFixturesV2(t testing.TB, overrideFn func(c *chainlink.Config, s *chainlink.Secrets)) (chainlink.GeneralConfig, *sqlx.DB) {
-	return prepareFullTestDBV2(t, false, false, overrideFn)
+	return KindTemplate.PrepareDB(t, overrideFn)
 }
 
 // FullTestDBEmptyV2 creates an empty DB (without migrations).
 func FullTestDBEmptyV2(t testing.TB, overrideFn func(c *chainlink.Config, s *chainlink.Secrets)) (chainlink.GeneralConfig, *sqlx.DB) {
-	return prepareFullTestDBV2(t, true, false, overrideFn)
+	return KindEmpty.PrepareDB(t, overrideFn)
 }
 
 func generateName() string {
 	return strings.ReplaceAll(uuid.New().String(), "-", "")
 }
 
-func prepareFullTestDBV2(t testing.TB, empty bool, loadFixtures bool, overrideFn func(c *chainlink.Config, s *chainlink.Secrets)) (chainlink.GeneralConfig, *sqlx.DB) {
-	testutils.SkipShort(t, "FullTestDB")
+type Kind int
 
-	if empty && loadFixtures {
-		t.Fatal("could not load fixtures into an empty DB")
-	}
+const (
+	KindEmpty Kind = iota
+	KindTemplate
+	KindFixtures
+)
+
+func (c Kind) PrepareDB(t testing.TB, overrideFn func(c *chainlink.Config, s *chainlink.Secrets)) (chainlink.GeneralConfig, *sqlx.DB) {
+	testutils.SkipShort(t, "FullTestDB")
 
 	gcfg := configtest.NewGeneralConfigSimulated(t, func(c *chainlink.Config, s *chainlink.Secrets) {
 		c.Database.Dialect = dialects.Postgres
@@ -64,7 +63,7 @@ func prepareFullTestDBV2(t testing.TB, empty bool, loadFixtures bool, overrideFn
 	})
 
 	require.NoError(t, os.MkdirAll(gcfg.RootDir(), 0700))
-	migrationTestDBURL, err := dropAndCreateThrowawayTestDB(gcfg.Database().URL(), generateName(), empty)
+	migrationTestDBURL, err := testdb.CreateOrReplace(gcfg.Database().URL(), generateName(), c != KindEmpty)
 	require.NoError(t, err)
 	db, err := pg.NewConnection(migrationTestDBURL, dialects.Postgres, gcfg.Database())
 	require.NoError(t, err)
@@ -81,7 +80,7 @@ func prepareFullTestDBV2(t testing.TB, empty bool, loadFixtures bool, overrideFn
 		}
 	})
 
-	if loadFixtures {
+	if c == KindFixtures {
 		_, filename, _, ok := runtime.Caller(1)
 		if !ok {
 			t.Fatal("could not get runtime.Caller(1)")
@@ -95,39 +94,3 @@ func prepareFullTestDBV2(t testing.TB, empty bool, loadFixtures bool, overrideFn
 
 	return gcfg, db
 }
-
-func dropAndCreateThrowawayTestDB(parsed url.URL, postfix string, empty bool) (string, error) {
-	if parsed.Path == "" {
-		return "", errors.New("path missing from database URL")
-	}
-
-	// Match the naming schema that our dangling DB cleanup methods expect
-	dbname := cmd.TestDBNamePrefix + postfix
-	if l := len(dbname); l > 63 {
-		return "", fmt.Errorf("dbname %v too long (%d), max is 63 bytes. Try a shorter postfix", dbname, l)
-	}
-	// Cannot drop test database if we are connected to it, so we must connect
-	// to a different one. 'postgres' should be present on all postgres installations
-	parsed.Path = "/postgres"
-	db, err := sql.Open(string(dialects.Postgres), parsed.String())
-	if err != nil {
-		return "", fmt.Errorf("In order to drop the test database, we need to connect to a separate database"+
-			" called 'postgres'. But we are unable to open 'postgres' database: %+v\n", err)
-	}
-	defer db.Close()
-
-	_, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", dbname))
-	if err != nil {
-		return "", fmt.Errorf("unable to drop postgres migrations test database: %v", err)
-	}
-	if empty {
-		_, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", dbname))
-	} else {
-		_, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s WITH TEMPLATE %s", dbname, cmd.PristineDBName))
-	}
-	if err != nil {
-		return "", fmt.Errorf("unable to create postgres test database with name '%s': %v", dbname, err)
-	}
-	parsed.Path = fmt.Sprintf("/%s", dbname)
-	return parsed.String(), nil
-}

core/scripts/gateway/run_gateway.go

@@ -48,7 +48,7 @@ func main() {
 
 	lggr, _ := logger.NewLogger()
 
-	handlerFactory := gateway.NewHandlerFactory(nil, lggr)
+	handlerFactory := gateway.NewHandlerFactory(nil, nil, nil, lggr)
 	gw, err := gateway.NewGatewayFromConfig(&cfg, handlerFactory, lggr)
 	if err != nil {
 		fmt.Println("error creating Gateway object:", err)

core/services/chainlink/application.go

@@ -344,6 +344,8 @@ func NewApplication(opts ApplicationOpts) (Application, error) {
 			job.Gateway: gateway.NewDelegate(
 				legacyEVMChains,
 				keyStore.Eth(),
+				db,
+				cfg.Database(),
 				globalLogger),
 		}
 		webhookJobRunner = delegates[job.Webhook].(*webhook.Delegate).WebhookJobRunner()

core/services/gateway/delegate.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 
 	"github.com/google/uuid"
+	"github.com/jmoiron/sqlx"
 	"github.com/pelletier/go-toml"
 	"github.com/pkg/errors"
 
@@ -18,13 +19,21 @@ import (
 type Delegate struct {
 	legacyChains legacyevm.LegacyChainContainer
 	ks           keystore.Eth
+	db           *sqlx.DB
+	cfg          pg.QConfig
 	lggr         logger.Logger
 }
 
 var _ job.Delegate = (*Delegate)(nil)
 
-func NewDelegate(legacyChains legacyevm.LegacyChainContainer, ks keystore.Eth, lggr logger.Logger) *Delegate {
-	return &Delegate{legacyChains: legacyChains, ks: ks, lggr: lggr}
+func NewDelegate(legacyChains legacyevm.LegacyChainContainer, ks keystore.Eth, db *sqlx.DB, cfg pg.QConfig, lggr logger.Logger) *Delegate {
+	return &Delegate{
+		legacyChains: legacyChains,
+		ks:           ks,
+		db:           db,
+		cfg:          cfg,
+		lggr:         lggr,
+	}
 }
 
 func (d *Delegate) JobType() job.Type {
@@ -47,7 +56,7 @@ func (d *Delegate) ServicesForSpec(spec job.Job) (services []job.ServiceCtx, err
 	if err2 != nil {
 		return nil, errors.Wrap(err2, "unmarshal gateway config")
 	}
-	handlerFactory := NewHandlerFactory(d.legacyChains, d.lggr)
+	handlerFactory := NewHandlerFactory(d.legacyChains, d.db, d.cfg, d.lggr)
 	gateway, err := NewGatewayFromConfig(&gatewayConfig, handlerFactory, d.lggr)
 	if err != nil {
 		return nil, err