ethdb/pebble: cherry pick pebble backend from go-ethereum

cmd/ronin/chaincmd.go

@@ -49,6 +49,7 @@ var (
 		ArgsUsage: "<genesisPath>",
 		Flags: []cli.Flag{
 			utils.DataDirFlag,
+			utils.DBEngineFlag,
 			utils.ForceOverrideChainConfigFlag,
 		},
 		Category: "BLOCKCHAIN COMMANDS",
@@ -82,6 +83,7 @@ The dumpgenesis command dumps the genesis block configuration in JSON format to
 		ArgsUsage: "<filename> (<filename 2> ... <filename N>) ",
 		Flags: []cli.Flag{
 			utils.DataDirFlag,
+			utils.DBEngineFlag,
 			utils.CacheFlag,
 			utils.SyncModeFlag,
 			utils.GCModeFlag,
@@ -119,6 +121,7 @@ processing will proceed even if an individual RLP-file import failure occurs.`,
 		ArgsUsage: "<filename> [<blockNumFirst> <blockNumLast>]",
 		Flags: []cli.Flag{
 			utils.DataDirFlag,
+			utils.DBEngineFlag,
 			utils.CacheFlag,
 			utils.SyncModeFlag,
 		},
@@ -137,6 +140,7 @@ be gzipped.`,
 		ArgsUsage: "<datafile>",
 		Flags: []cli.Flag{
 			utils.DataDirFlag,
+			utils.DBEngineFlag,
 			utils.CacheFlag,
 			utils.SyncModeFlag,
 		},
@@ -153,6 +157,7 @@ It's deprecated, please use "geth db import" instead.
 		ArgsUsage: "<dumpfile>",
 		Flags: []cli.Flag{
 			utils.DataDirFlag,
+			utils.DBEngineFlag,
 			utils.CacheFlag,
 			utils.SyncModeFlag,
 		},
@@ -169,6 +174,7 @@ It's deprecated, please use "geth db export" instead.
 		ArgsUsage: "[? <blockHash> | <blockNum>]",
 		Flags: []cli.Flag{
 			utils.DataDirFlag,
+			utils.DBEngineFlag,
 			utils.CacheFlag,
 			utils.IterativeOutputFlag,
 			utils.ExcludeCodeFlag,

cmd/ronin/dao_test.go

@@ -123,7 +123,7 @@ func testDAOForkBlockNewChain(t *testing.T, test int, genesis string, expectBloc
 	}
 	// Retrieve the DAO config flag from the database
 	path := filepath.Join(datadir, "ronin", "chaindata")
-	db, err := rawdb.NewLevelDBDatabase(path, 0, 0, "", false)
+	db, err := rawdb.NewPebbleDBDatabase(path, 0, 0, "", false, true)
 	if err != nil {
 		t.Fatalf("test %d: failed to open test database: %v", test, err)
 	}

cmd/ronin/dbcmd.go

@@ -77,6 +77,7 @@ Remove blockchain and state databases`,
 		ArgsUsage: "<prefix> <start>",
 		Flags: []cli.Flag{
 			utils.DataDirFlag,
+			utils.DBEngineFlag,
 			utils.AncientFlag,
 			utils.SyncModeFlag,
 			utils.MainnetFlag,
@@ -94,6 +95,7 @@ Remove blockchain and state databases`,
 		Usage:  "Print leveldb statistics",
 		Flags: []cli.Flag{
 			utils.DataDirFlag,
+			utils.DBEngineFlag,
 			utils.SyncModeFlag,
 			utils.MainnetFlag,
 			utils.RopstenFlag,
@@ -108,6 +110,7 @@ Remove blockchain and state databases`,
 		Usage:  "Compact leveldb database. WARNING: May take a very long time",
 		Flags: []cli.Flag{
 			utils.DataDirFlag,
+			utils.DBEngineFlag,
 			utils.SyncModeFlag,
 			utils.MainnetFlag,
 			utils.RopstenFlag,
@@ -128,6 +131,7 @@ corruption if it is aborted during execution'!`,
 		ArgsUsage: "<hex-encoded key>",
 		Flags: []cli.Flag{
 			utils.DataDirFlag,
+			utils.DBEngineFlag,
 			utils.SyncModeFlag,
 			utils.MainnetFlag,
 			utils.RopstenFlag,
@@ -144,6 +148,7 @@ corruption if it is aborted during execution'!`,
 		ArgsUsage: "<hex-encoded key>",
 		Flags: []cli.Flag{
 			utils.DataDirFlag,
+			utils.DBEngineFlag,
 			utils.SyncModeFlag,
 			utils.MainnetFlag,
 			utils.RopstenFlag,
@@ -161,6 +166,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		ArgsUsage: "<hex-encoded key> <hex-encoded value>",
 		Flags: []cli.Flag{
 			utils.DataDirFlag,
+			utils.DBEngineFlag,
 			utils.SyncModeFlag,
 			utils.MainnetFlag,
 			utils.RopstenFlag,
@@ -178,6 +184,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		ArgsUsage: "<hex-encoded storage trie root> <hex-encoded start (optional)> <int max elements (optional)>",
 		Flags: []cli.Flag{
 			utils.DataDirFlag,
+			utils.DBEngineFlag,
 			utils.SyncModeFlag,
 			utils.MainnetFlag,
 			utils.RopstenFlag,
@@ -194,6 +201,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		ArgsUsage: "<type> <start (int)> <end (int)>",
 		Flags: []cli.Flag{
 			utils.DataDirFlag,
+			utils.DBEngineFlag,
 			utils.SyncModeFlag,
 			utils.MainnetFlag,
 			utils.RopstenFlag,
@@ -210,6 +218,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		ArgsUsage: "<dumpfile> <start (optional)",
 		Flags: []cli.Flag{
 			utils.DataDirFlag,
+			utils.DBEngineFlag,
 			utils.SyncModeFlag,
 			utils.MainnetFlag,
 			utils.RopstenFlag,
@@ -225,6 +234,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		ArgsUsage: "<type> <dumpfile>",
 		Flags: []cli.Flag{
 			utils.DataDirFlag,
+			utils.DBEngineFlag,
 			utils.SyncModeFlag,
 			utils.MainnetFlag,
 			utils.RopstenFlag,

cmd/ronin/genesis_test.go

@@ -17,9 +17,11 @@
 package main
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strconv"
 	"testing"
 )
 
@@ -71,6 +73,7 @@ var customGenesisTests = []struct {
 // Tests that initializing Geth with a custom genesis block and chain definitions
 // work properly.
 func TestCustomGenesis(t *testing.T) {
+	t.Parallel()
 	for i, tt := range customGenesisTests {
 		// Create a temporary data directory to use and inspect later
 		datadir := tmpdir(t)
@@ -92,3 +95,101 @@ func TestCustomGenesis(t *testing.T) {
 		geth.ExpectExit()
 	}
 }
+
+// TestCustomBackend that the backend selection and detection (leveldb vs pebble) works properly.
+func TestCustomBackend(t *testing.T) {
+	t.Parallel()
+	// Test pebble, but only on 64-bit platforms
+	if strconv.IntSize != 64 {
+		t.Skip("Custom backends are only available on 64-bit platform")
+	}
+	genesis := `{
+		"alloc"      : {},
+		"coinbase"   : "0x0000000000000000000000000000000000000000",
+			"difficulty" : "0x20000",
+			"extraData"  : "",
+			"gasLimit"   : "0x2fefd8",
+			"nonce"      : "0x0000000000001338",
+			"mixhash"    : "0x0000000000000000000000000000000000000000000000000000000000000000",
+			"parentHash" : "0x0000000000000000000000000000000000000000000000000000000000000000",
+			"timestamp"  : "0x00",
+			"config"     : {}
+	}`
+	type backendTest struct {
+		initArgs   []string
+		initExpect string
+		execArgs   []string
+		execExpect string
+	}
+	testfunc := func(t *testing.T, tt backendTest) error {
+		// Create a temporary data directory to use and inspect later
+		datadir := t.TempDir()
+
+		// Initialize the data directory with the custom genesis block
+		json := filepath.Join(datadir, "genesis.json")
+		if err := os.WriteFile(json, []byte(genesis), 0600); err != nil {
+			return fmt.Errorf("failed to write genesis file: %v", err)
+		}
+		{ // Init
+			args := append(tt.initArgs, "--datadir", datadir, "init", json)
+			geth := runGeth(t, args...)
+			geth.ExpectRegexp(tt.initExpect)
+			geth.ExpectExit()
+		}
+		{ // Exec + query
+			args := append(tt.execArgs, "--networkid", "1337", "--syncmode=full", "--cache", "16",
+				"--datadir", datadir, "--maxpeers", "0", "--port", "0",
+				"--nodiscover", "--nat", "none", "--ipcdisable",
+				"--exec", "eth.getBlock(0).nonce", "console")
+			geth := runGeth(t, args...)
+			geth.ExpectRegexp(tt.execExpect)
+			geth.ExpectExit()
+		}
+		return nil
+	}
+	for i, tt := range []backendTest{
+		{ // When not specified, it should default to pebble
+			execArgs:   []string{"--db.engine", "pebble"},
+			execExpect: "0x0000000000001338",
+		},
+		{ // Explicit leveldb
+			initArgs:   []string{"--db.engine", "leveldb"},
+			execArgs:   []string{"--db.engine", "leveldb"},
+			execExpect: "0x0000000000001338",
+		},
+		{ // Explicit leveldb first, then autodiscover
+			initArgs:   []string{"--db.engine", "leveldb"},
+			execExpect: "0x0000000000001338",
+		},
+		{ // Explicit pebble
+			initArgs:   []string{"--db.engine", "pebble"},
+			execArgs:   []string{"--db.engine", "pebble"},
+			execExpect: "0x0000000000001338",
+		},
+		{ // Explicit pebble, then auto-discover
+			initArgs:   []string{"--db.engine", "pebble"},
+			execExpect: "0x0000000000001338",
+		},
+		{ // Can't start pebble on top of leveldb
+			initArgs:   []string{"--db.engine", "leveldb"},
+			execArgs:   []string{"--db.engine", "pebble"},
+			execExpect: `Fatal: Failed to register the Ethereum service: db.engine choice was pebble but found pre-existing leveldb database in specified data directory`,
+		},
+		{ // Can't start leveldb on top of pebble
+			initArgs:   []string{"--db.engine", "pebble"},
+			execArgs:   []string{"--db.engine", "leveldb"},
+			execExpect: `Fatal: Failed to register the Ethereum service: db.engine choice was leveldb but found pre-existing pebble database in specified data directory`,
+		},
+		{ // Reject invalid backend choice
+			initArgs:   []string{"--db.engine", "mssql"},
+			initExpect: `Fatal: Invalid choice for db.engine 'mssql', allowed 'leveldb' or 'pebble'`,
+			// Since the init fails, this will return the (default) mainnet genesis
+			// block nonce
+			execExpect: `0x0000000000000042`,
+		},
+	} {
+		if err := testfunc(t, tt); err != nil {
+			t.Fatalf("test %d-leveldb: %v", i, err)
+		}
+	}
+}

cmd/ronin/main.go

@@ -175,6 +175,7 @@ var (
 		utils.BlsWalletPath,
 		utils.DisableRoninProtocol,
 		utils.AdditionalChainEventFlag,
+		utils.DBEngineFlag,
 	}
 
 	rpcFlags = []cli.Flag{

cmd/ronin/snapshot.go

@@ -60,6 +60,7 @@ var (
 				Category:  "MISCELLANEOUS COMMANDS",
 				Flags: []cli.Flag{
 					utils.DataDirFlag,
+					utils.DBEngineFlag,
 					utils.AncientFlag,
 					utils.RopstenFlag,
 					utils.SepoliaFlag,
@@ -91,6 +92,7 @@ the trie clean cache with default directory will be deleted.
 				Category:  "MISCELLANEOUS COMMANDS",
 				Flags: []cli.Flag{
 					utils.DataDirFlag,
+					utils.DBEngineFlag,
 					utils.AncientFlag,
 					utils.RopstenFlag,
 					utils.SepoliaFlag,
@@ -112,6 +114,7 @@ In other words, this command does the snapshot to trie conversion.
 				Category:  "MISCELLANEOUS COMMANDS",
 				Flags: []cli.Flag{
 					utils.DataDirFlag,
+					utils.DBEngineFlag,
 					utils.AncientFlag,
 					utils.RopstenFlag,
 					utils.SepoliaFlag,
@@ -135,6 +138,7 @@ It's also usable without snapshot enabled.
 				Category:  "MISCELLANEOUS COMMANDS",
 				Flags: []cli.Flag{
 					utils.DataDirFlag,
+					utils.DBEngineFlag,
 					utils.AncientFlag,
 					utils.RopstenFlag,
 					utils.SepoliaFlag,
@@ -159,6 +163,7 @@ It's also usable without snapshot enabled.
 				Category:  "MISCELLANEOUS COMMANDS",
 				Flags: []cli.Flag{
 					utils.DataDirFlag,
+					utils.DBEngineFlag,
 					utils.AncientFlag,
 					utils.RopstenFlag,
 					utils.SepoliaFlag,

cmd/ronin/usage.go

@@ -60,6 +60,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{
 			utils.StoreInternalTransactions,
 			utils.DisableRoninProtocol,
 			utils.AdditionalChainEventFlag,
+			utils.DBEngineFlag,
 		},
 	},
 	{

cmd/utils/flags.go

@@ -121,6 +121,11 @@ var (
 		Name:  "datadir.minfreedisk",
 		Usage: "Minimum free disk space in MB, once reached triggers auto shut down (default = --cache.gc converted to MB, 0 = disabled)",
 	}
+	DBEngineFlag = cli.StringFlag{
+		Name:  "db.engine",
+		Usage: "Backing database implementation to use ('leveldb' or 'pebble')",
+		Value: node.DefaultConfig.DBEngine,
+	}
 	KeyStoreDirFlag = DirectoryFlag{
 		Name:  "keystore",
 		Usage: "Directory for the keystore (default = inside the datadir)",
@@ -915,12 +920,12 @@ var (
 	}
 
 	MockValidatorsFlag = cli.StringFlag{
-		Name: "mock.validators",
+		Name:  "mock.validators",
 		Usage: "List of mock validators",
 	}
 
 	MockBlsPublicKeysFlag = cli.StringFlag{
-		Name: "mock.blspublickeys",
+		Name:  "mock.blspublickeys",
 		Usage: "List of mock bls public keys which are reflect 1:1 with mock.validators",
 	}
 )
@@ -1381,6 +1386,14 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) {
 	if ctx.GlobalIsSet(EnableSigningMethodsFlag.Name) {
 		cfg.EnableSigningMethods = ctx.GlobalBool(EnableSigningMethodsFlag.Name)
 	}
+	if ctx.GlobalIsSet(DBEngineFlag.Name) {
+		dbEngine := ctx.GlobalString(DBEngineFlag.Name)
+		if dbEngine != "leveldb" && dbEngine != "pebble" {
+			Fatalf("Invalid choice for db.engine '%s', allowed 'leveldb' or 'pebble'", dbEngine)
+		}
+		log.Info(fmt.Sprintf("Using %s as db engine", dbEngine))
+		cfg.DBEngine = dbEngine
+	}
 }
 
 func setFastFinality(ctx *cli.Context, cfg *node.Config) {