Move txdb and dialects to chainlink-common/pkg/pg (#15064)

* Move txdb.go and dialects.go to chainlink-common

Along with that:
- Replace dialects imports with pgcommon
- Replace testutils.Skip* with tests.Skip*
- Call RegisterTxDb from NewConnection
- Run goimports (somehow there were a few files on develop which were improperly formatted?)

* Fix merge conflict with copied txdb driver

While rebasing, I found that CCIP has copied the old txdb driver from
chainlink into chainlink/integrations-tests. The problem with duplicated
code is it makes it more difficult to update. Little did they know, it
had already been updated even before they copied it. So this removes the
copy and imports the newer version from chainlink-common.

The motivation for copying rather than import appears to have come from
a lint rule that enforces not being able to import anything from an
external package with "internal" in the directory name. This still
requires that the NewSqlxDB be copied, but that's just a wrapper
around the pgcommon version which reads the db url from the chainlik
toml config, which can't be accessed in chainlink-common.

* chainlink-common/pkg/pg -> chainlink-common/pkg/sqlutil/pg

* Fix lint introduced by rebase

* pg.NewSqlxDB -> pg.NewTestDB

* Update chainlink-common ref

* Fix merge conflict

GNUmakefile

@@ -133,13 +133,6 @@ testdb-force: ## Prepares the test database, drops any pesky user connections th
 testdb-user-only: ## Prepares the test database with user only.
 	go run . local db preparetest --user-only
 
-# Format for CI
-.PHONY: presubmit
-presubmit: ## Format go files and imports.
-	goimports -w .
-	gofmt -w .
-	go mod tidy
-
 .PHONY: gomods
 gomods: ## Install gomods
 	go install github.com/jmank88/gomods@v0.1.4

core/capabilities/remote/executable/client_test.go

@@ -12,7 +12,9 @@ import (
 	commoncap "github.com/smartcontractkit/chainlink-common/pkg/capabilities"
 	"github.com/smartcontractkit/chainlink-common/pkg/capabilities/pb"
 	"github.com/smartcontractkit/chainlink-common/pkg/services/servicetest"
+	"github.com/smartcontractkit/chainlink-common/pkg/utils/tests"
 	"github.com/smartcontractkit/chainlink-common/pkg/values"
+
 	"github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/executable"
 	remotetypes "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote/types"
 	"github.com/smartcontractkit/chainlink/v2/core/capabilities/transmission"
@@ -29,7 +31,7 @@ const (
 )
 
 func Test_Client_DonTopologies(t *testing.T) {
-	testutils.SkipFlakey(t, "https://smartcontract-it.atlassian.net/browse/CAPPL-363")
+	tests.SkipFlakey(t, "https://smartcontract-it.atlassian.net/browse/CAPPL-363")
 	ctx := testutils.Context(t)
 
 	transmissionSchedule, err := values.NewMap(map[string]any{
@@ -88,7 +90,7 @@ func Test_Client_DonTopologies(t *testing.T) {
 }
 
 func Test_Client_TransmissionSchedules(t *testing.T) {
-	testutils.SkipFlakey(t, "https://smartcontract-it.atlassian.net/browse/CAPPL-363")
+	tests.SkipFlakey(t, "https://smartcontract-it.atlassian.net/browse/CAPPL-363")
 	ctx := testutils.Context(t)
 
 	responseTest := func(t *testing.T, response commoncap.CapabilityResponse, responseError error) {

core/chains/evm/gas/models.go

@@ -9,6 +9,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/rpc"
 	pkgerrors "github.com/pkg/errors"
+
 	"github.com/smartcontractkit/chainlink-common/pkg/logger"
 	"github.com/smartcontractkit/chainlink-common/pkg/services"
 	bigmath "github.com/smartcontractkit/chainlink-common/pkg/utils/big_math"

core/chains/evm/gas/rollups/l1_oracle.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/rpc"
+
 	"github.com/smartcontractkit/chainlink-common/pkg/services"
 
 	"github.com/smartcontractkit/chainlink-common/pkg/logger"

core/cmd/shell_local.go

@@ -35,6 +35,8 @@ import (
 
 	cutils "github.com/smartcontractkit/chainlink-common/pkg/utils"
 
+	pgcommon "github.com/smartcontractkit/chainlink-common/pkg/sqlutil/pg"
+
 	"github.com/smartcontractkit/chainlink/v2/core/build"
 	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets"
 	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas"
@@ -47,7 +49,6 @@ import (
 	"github.com/smartcontractkit/chainlink/v2/core/sessions"
 	"github.com/smartcontractkit/chainlink/v2/core/shutdown"
 	"github.com/smartcontractkit/chainlink/v2/core/static"
-	"github.com/smartcontractkit/chainlink/v2/core/store/dialects"
 	"github.com/smartcontractkit/chainlink/v2/core/store/migrate"
 	"github.com/smartcontractkit/chainlink/v2/core/utils"
 	"github.com/smartcontractkit/chainlink/v2/core/web"
@@ -805,7 +806,7 @@ func (s *Shell) PrepareTestDatabase(c *cli.Context) error {
 
 	// Creating pristine DB copy to speed up FullTestDB
 	dbUrl := cfg.Database().URL()
-	db, err := sqlx.Open(string(dialects.Postgres), dbUrl.String())
+	db, err := sqlx.Open(string(pgcommon.Postgres), dbUrl.String())
 	if err != nil {
 		return s.errorOut(err)
 	}
@@ -1088,7 +1089,7 @@ type dbConfig interface {
 	MaxOpenConns() int
 	MaxIdleConns() int
 	URL() url.URL
-	Dialect() dialects.DialectName
+	Dialect() pgcommon.DialectName
 }
 
 func newConnection(ctx context.Context, cfg dbConfig) (*sqlx.DB, error) {
@@ -1104,7 +1105,7 @@ func dropAndCreateDB(parsed url.URL, force bool) (err error) {
 	// to a different one. template1 should be present on all postgres installations
 	dbname := parsed.Path[1:]
 	parsed.Path = "/template1"
-	db, err := sql.Open(string(dialects.Postgres), parsed.String())
+	db, err := sql.Open(string(pgcommon.Postgres), parsed.String())
 	if err != nil {
 		return fmt.Errorf("unable to open postgres database for creating test db: %+v", err)
 	}
@@ -1203,7 +1204,7 @@ func checkSchema(dbURL url.URL, prevSchema string) error {
 }
 
 func insertFixtures(dbURL url.URL, pathToFixtures string) (err error) {
-	db, err := sql.Open(string(dialects.Postgres), dbURL.String())
+	db, err := sql.Open(string(pgcommon.Postgres), dbURL.String())
 	if err != nil {
 		return fmt.Errorf("unable to open postgres database for creating test db: %+v", err)
 	}

core/cmd/shell_local_test.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	commonconfig "github.com/smartcontractkit/chainlink-common/pkg/config"
+	pgcommon "github.com/smartcontractkit/chainlink-common/pkg/sqlutil/pg"
 	"github.com/smartcontractkit/chainlink-common/pkg/utils/mailbox"
 
 	"github.com/smartcontractkit/chainlink/v2/common/client"
@@ -29,7 +30,6 @@ import (
 	chainlinkmocks "github.com/smartcontractkit/chainlink/v2/core/services/chainlink/mocks"
 	evmrelayer "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm"
 	"github.com/smartcontractkit/chainlink/v2/core/sessions/localauth"
-	"github.com/smartcontractkit/chainlink/v2/core/store/dialects"
 	"github.com/smartcontractkit/chainlink/v2/core/store/models"
 	"github.com/smartcontractkit/chainlink/v2/core/utils"
 	"github.com/smartcontractkit/chainlink/v2/core/utils/testutils/heavyweight"
@@ -283,7 +283,7 @@ func TestShell_RebroadcastTransactions_Txm(t *testing.T) {
 	// test multiple connections to the database, and changes made within
 	// the transaction cannot be seen from another connection.
 	config, sqlxDB := heavyweight.FullTestDBV2(t, func(c *chainlink.Config, s *chainlink.Secrets) {
-		c.Database.Dialect = dialects.Postgres
+		c.Database.Dialect = pgcommon.Postgres
 		// evm config is used in this test. but if set, it must be pass config validation.
 		// simplest to make it nil
 		c.EVM = nil
@@ -363,7 +363,7 @@ func TestShell_RebroadcastTransactions_OutsideRange_Txm(t *testing.T) {
 			// test multiple connections to the database, and changes made within
 			// the transaction cannot be seen from another connection.
 			config, sqlxDB := heavyweight.FullTestDBV2(t, func(c *chainlink.Config, s *chainlink.Secrets) {
-				c.Database.Dialect = dialects.Postgres
+				c.Database.Dialect = pgcommon.Postgres
 				// evm config is used in this test. but if set, it must be pass config validation.
 				// simplest to make it nil
 				c.EVM = nil
@@ -441,7 +441,7 @@ func TestShell_RebroadcastTransactions_AddressCheck(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			config, sqlxDB := heavyweight.FullTestDBV2(t, func(c *chainlink.Config, s *chainlink.Secrets) {
-				c.Database.Dialect = dialects.Postgres
+				c.Database.Dialect = pgcommon.Postgres
 
 				c.EVM = nil
 				// seems to be needed for config validate
@@ -499,7 +499,7 @@ func TestShell_RebroadcastTransactions_AddressCheck(t *testing.T) {
 func TestShell_CleanupChainTables(t *testing.T) {
 	// Just check if it doesn't error, command itself shouldn't be changed unless major schema changes were made.
 	// It would be really hard to write a test that accounts for schema changes, so this should be enough to alarm us that something broke.
-	config, _ := heavyweight.FullTestDBV2(t, func(c *chainlink.Config, s *chainlink.Secrets) { c.Database.Dialect = dialects.Postgres })
+	config, _ := heavyweight.FullTestDBV2(t, func(c *chainlink.Config, s *chainlink.Secrets) { c.Database.Dialect = pgcommon.Postgres })
 	client := cmd.Shell{
 		Config: config,
 		Logger: logger.TestLogger(t),

core/config/database_config.go

@@ -4,7 +4,7 @@ import (
 	"net/url"
 	"time"
 
-	"github.com/smartcontractkit/chainlink/v2/core/store/dialects"
+	pgcommon "github.com/smartcontractkit/chainlink-common/pkg/sqlutil/pg"
 )
 
 type Backup interface {
@@ -35,7 +35,7 @@ type Database interface {
 	DefaultIdleInTxSessionTimeout() time.Duration
 	DefaultLockTimeout() time.Duration
 	DefaultQueryTimeout() time.Duration
-	Dialect() dialects.DialectName
+	Dialect() pgcommon.DialectName
 	LogSQL() bool
 	MaxIdleConns() int
 	MaxOpenConns() int

core/config/docs/defaults.go

@@ -5,9 +5,10 @@ import (
 	"strings"
 
 	"github.com/smartcontractkit/chainlink-common/pkg/config"
+	pgcommon "github.com/smartcontractkit/chainlink-common/pkg/sqlutil/pg"
+
 	"github.com/smartcontractkit/chainlink/v2/core/config/toml"
 	"github.com/smartcontractkit/chainlink/v2/core/services/chainlink/cfgtest"
-	"github.com/smartcontractkit/chainlink/v2/core/store/dialects"
 )
 
 var (
@@ -22,7 +23,7 @@ func init() {
 
 func CoreDefaults() (c toml.Core) {
 	c.SetFrom(&defaults)
-	c.Database.Dialect = dialects.Postgres // not user visible - overridden for tests only
+	c.Database.Dialect = pgcommon.Postgres // not user visible - overridden for tests only
 	c.Tracing.Attributes = make(map[string]string)
 	return
 }

core/config/toml/types.go

@@ -16,17 +16,16 @@ import (
 	ocrcommontypes "github.com/smartcontractkit/libocr/commontypes"
 
 	commonconfig "github.com/smartcontractkit/chainlink-common/pkg/config"
+	pgcommon "github.com/smartcontractkit/chainlink-common/pkg/sqlutil/pg"
 
 	"github.com/smartcontractkit/chainlink/v2/core/build"
 	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/types"
 	"github.com/smartcontractkit/chainlink/v2/core/config"
 	"github.com/smartcontractkit/chainlink/v2/core/config/parse"
 	"github.com/smartcontractkit/chainlink/v2/core/services/keystore/keys/p2pkey"
 	"github.com/smartcontractkit/chainlink/v2/core/sessions"
-	"github.com/smartcontractkit/chainlink/v2/core/store/dialects"
 	"github.com/smartcontractkit/chainlink/v2/core/store/models"
 	"github.com/smartcontractkit/chainlink/v2/core/utils"
-
 	configutils "github.com/smartcontractkit/chainlink/v2/core/utils/config"
 )
 
@@ -339,7 +338,7 @@ type Database struct {
 	DefaultIdleInTxSessionTimeout *commonconfig.Duration
 	DefaultLockTimeout            *commonconfig.Duration
 	DefaultQueryTimeout           *commonconfig.Duration
-	Dialect                       dialects.DialectName `toml:"-"`
+	Dialect                       pgcommon.DialectName `toml:"-"`
 	LogQueries                    *bool
 	MaxIdleConns                  *int64
 	MaxOpenConns                  *int64

core/gethwrappers/go_generate_test.go

@@ -16,7 +16,8 @@ import (
 	"github.com/fatih/color"
 
 	cutils "github.com/smartcontractkit/chainlink-common/pkg/utils"
-	"github.com/smartcontractkit/chainlink/v2/core/internal/testutils"
+	"github.com/smartcontractkit/chainlink-common/pkg/utils/tests"
+
 	"github.com/smartcontractkit/chainlink/v2/core/utils"
 
 	"github.com/stretchr/testify/assert"
@@ -29,7 +30,7 @@ const compileCommand = "../../contracts/scripts/native_solc_compile_all"
 // contract artifacts in contracts/solc with the abi and bytecode stored in the
 // contract wrapper
 func TestCheckContractHashesFromLastGoGenerate(t *testing.T) {
-	testutils.SkipShort(t, "requires compiled artifacts")
+	tests.SkipShort(t, "requires compiled artifacts")
 	versions, err := ReadVersionsDB()
 	require.NoError(t, err)
 	require.NotEmpty(t, versions.GethVersion, `version DB should have a "GETH_VERSION:" line`)

core/internal/cltest/cltest.go

@@ -47,6 +47,7 @@ import (
 	"github.com/smartcontractkit/chainlink-common/pkg/loop"
 	"github.com/smartcontractkit/chainlink-common/pkg/sqlutil"
 	"github.com/smartcontractkit/chainlink-common/pkg/utils/mailbox"
+	"github.com/smartcontractkit/chainlink-common/pkg/utils/tests"
 
 	"github.com/smartcontractkit/chainlink/v2/common/client"
 	commonmocks "github.com/smartcontractkit/chainlink/v2/common/types/mocks"
@@ -531,7 +532,7 @@ func NewEthMocks(t testing.TB) *evmclimocks.Client {
 }
 
 func NewEthMocksWithStartupAssertions(t testing.TB) *evmclimocks.Client {
-	testutils.SkipShort(t, "long test")
+	tests.SkipShort(t, "long test")
 	c := NewEthMocks(t)
 	chHead := make(<-chan *evmtypes.Head)
 	c.On("Dial", mock.Anything).Maybe().Return(nil)
@@ -554,7 +555,7 @@ func NewEthMocksWithStartupAssertions(t testing.TB) *evmclimocks.Client {
 
 // NewEthMocksWithTransactionsOnBlocksAssertions sets an Eth mock with transactions on blocks
 func NewEthMocksWithTransactionsOnBlocksAssertions(t testing.TB) *evmclimocks.Client {
-	testutils.SkipShort(t, "long test")
+	tests.SkipShort(t, "long test")
 	c := NewEthMocks(t)
 	chHead := make(<-chan *evmtypes.Head)
 	c.On("Dial", mock.Anything).Maybe().Return(nil)

core/internal/features/features_test.go

@@ -40,6 +40,7 @@ import (
 
 	commonconfig "github.com/smartcontractkit/chainlink-common/pkg/config"
 	"github.com/smartcontractkit/chainlink-common/pkg/services/servicetest"
+	"github.com/smartcontractkit/chainlink-common/pkg/utils/tests"
 
 	"github.com/smartcontractkit/chainlink/v2/core/auth"
 	"github.com/smartcontractkit/chainlink/v2/core/bridges"
@@ -798,7 +799,7 @@ func setupForwarderEnabledNode(t *testing.T, owner *bind.TransactOpts, portV2 in
 
 func TestIntegration_OCR(t *testing.T) {
 	t.Skip("fails after geth upgrade https://github.com/smartcontractkit/chainlink/pull/11809; passes local but fails CI")
-	testutils.SkipShort(t, "long test")
+	tests.SkipShort(t, "long test")
 	t.Parallel()
 	tests := []struct {
 		id      int
@@ -1031,7 +1032,7 @@ observationSource = """
 
 func TestIntegration_OCR_ForwarderFlow(t *testing.T) {
 	t.Skip("fails after geth upgrade https://github.com/smartcontractkit/chainlink/pull/11809")
-	testutils.SkipShort(t, "long test")
+	tests.SkipShort(t, "long test")
 	t.Parallel()
 	numOracles := 4
 	t.Run("ocr_forwarder_flow", func(t *testing.T) {

core/internal/testutils/configtest/general_config.go

@@ -7,14 +7,16 @@ import (
 
 	"github.com/stretchr/testify/require"
 
+	pgcommon "github.com/smartcontractkit/chainlink-common/pkg/sqlutil/pg"
+
 	commonconfig "github.com/smartcontractkit/chainlink-common/pkg/config"
+
 	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/client"
 	evmclient "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client"
 	evmcfg "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config/toml"
 	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils/big"
 	"github.com/smartcontractkit/chainlink/v2/core/internal/testutils"
 	"github.com/smartcontractkit/chainlink/v2/core/services/chainlink"
-	"github.com/smartcontractkit/chainlink/v2/core/store/dialects"
 	"github.com/smartcontractkit/chainlink/v2/core/store/models"
 )
 
@@ -48,7 +50,7 @@ func overrides(c *chainlink.Config, s *chainlink.Secrets) {
 	c.InsecureFastScrypt = ptr(true)
 	c.ShutdownGracePeriod = commonconfig.MustNewDuration(testutils.DefaultWaitTimeout)
 
-	c.Database.Dialect = dialects.TransactionWrappedPostgres
+	c.Database.Dialect = pgcommon.TransactionWrappedPostgres
 	c.Database.Lock.Enabled = ptr(false)
 	c.Database.MaxIdleConns = ptr[int64](20)
 	c.Database.MaxOpenConns = ptr[int64](20)

core/internal/testutils/pgtest/pgtest.go

@@ -3,26 +3,25 @@ package pgtest
 import (
 	"testing"
 
-	"github.com/google/uuid"
 	"github.com/jmoiron/sqlx"
-	"github.com/scylladb/go-reflectx"
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/smartcontractkit/chainlink-common/pkg/sqlutil"
+	"github.com/smartcontractkit/chainlink-common/pkg/sqlutil/pg"
 	"github.com/smartcontractkit/chainlink-common/pkg/utils"
+
+	"github.com/smartcontractkit/chainlink/v2/core/config/env"
 	"github.com/smartcontractkit/chainlink/v2/core/internal/testutils"
-	"github.com/smartcontractkit/chainlink/v2/core/store/dialects"
 )
 
 func NewSqlxDB(t testing.TB) *sqlx.DB {
 	testutils.SkipShortDB(t)
-	db, err := sqlx.Open(string(dialects.TransactionWrappedPostgres), uuid.New().String())
-	require.NoError(t, err)
-	t.Cleanup(func() { assert.NoError(t, db.Close()) })
-	db.MapperFunc(reflectx.CamelToSnakeASCII)
-
-	return db
+	dbURL := string(env.DatabaseURL.Get())
+	if dbURL == "" {
+		t.Errorf("you must provide a CL_DATABASE_URL environment variable")
+		return nil
+	}
+	return pg.NewTestDB(t, dbURL)
 }
 
 func MustExec(t *testing.T, ds sqlutil.DataSource, stmt string, args ...interface{}) {