Merge pull request #281 from ethpandaops/feat/eth68

Feat/eth68
ethpandaops · Mar 6, 2024 · 97a1a33 · 97a1a33
2 parents 9781721 + 7928210
commit 97a1a33
Show file tree

Hide file tree

Showing 44 changed files with 1,691 additions and 1,057 deletions.
diff --git a/.github/workflows/alpha-releases.yaml b/.github/workflows/alpha-releases.yaml
@@ -9,7 +9,7 @@ jobs:
   tag-release:
     permissions:
       contents: write
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3

diff --git a/.github/workflows/golangci-lint.yaml b/.github/workflows/golangci-lint.yaml
@@ -17,13 +17,13 @@ jobs:
     steps:
       - uses: actions/setup-go@v4
         with:
-          go-version: '1.20'
+          go-version: '1.22'
       - uses: actions/checkout@v3
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
+        uses: golangci/golangci-lint-action@v4
         with:
           # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
-          version: v1.55
+          version: v1.56
 
           # Optional: working directory, useful for monorepos
           # working-directory: somedir

diff --git a/.github/workflows/goreleaser.yaml b/.github/workflows/goreleaser.yaml
@@ -40,7 +40,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: '1.20'
+          go-version: '1.22'
       - uses: actions/setup-node@v3
         name: Set up Node
         with:

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -10,9 +10,9 @@ jobs:
   full_ci:
     strategy:
       matrix:
-        go_version: [ 1.20.x ]
+        go_version: [ 1.22.x ]
 
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
 
     steps:
       - name: checkout

diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.20 AS builder
+FROM golang:1.22 AS builder
 WORKDIR /src
 COPY go.sum go.mod ./
 RUN go mod download

diff --git a/deploy/migrations/clickhouse/027_blob_txn.down.sql b/deploy/migrations/clickhouse/027_blob_txn.down.sql
@@ -0,0 +1,35 @@
+ALTER TABLE mempool_transaction_local on cluster '{cluster}'
+    DROP COLUMN gas_tip_cap,
+    DROP COLUMN gas_fee_cap,
+    DROP COLUMN blob_gas,
+    DROP COLUMN blob_gas_fee_cap,
+    DROP COLUMN blob_hashes,
+    DROP COLUMN blob_sidecars_size,
+    DROP COLUMN blob_sidecars_empty_size;
+
+ALTER TABLE mempool_transaction on cluster '{cluster}'
+    DROP COLUMN gas_tip_cap,
+    DROP COLUMN gas_fee_cap,
+    DROP COLUMN blob_gas,
+    DROP COLUMN blob_gas_fee_cap,
+    DROP COLUMN blob_hashes,
+    DROP COLUMN blob_sidecars_size,
+    DROP COLUMN blob_sidecars_empty_size;
+
+ALTER TABLE canonical_beacon_block_execution_transaction_local on cluster '{cluster}'
+    DROP COLUMN gas_tip_cap,
+    DROP COLUMN gas_fee_cap,
+    DROP COLUMN blob_gas,
+    DROP COLUMN blob_gas_fee_cap,
+    DROP COLUMN blob_hashes,
+    DROP COLUMN blob_sidecars_size,
+    DROP COLUMN blob_sidecars_empty_size;
+
+ALTER TABLE canonical_beacon_block_execution_transaction on cluster '{cluster}'
+    DROP COLUMN gas_tip_cap,
+    DROP COLUMN gas_fee_cap,
+    DROP COLUMN blob_gas,
+    DROP COLUMN blob_gas_fee_cap,
+    DROP COLUMN blob_hashes,
+    DROP COLUMN blob_sidecars_size,
+    DROP COLUMN blob_sidecars_empty_size;
diff --git a/deploy/migrations/clickhouse/027_blob_txn.up.sql b/deploy/migrations/clickhouse/027_blob_txn.up.sql
@@ -0,0 +1,63 @@
+ALTER TABLE mempool_transaction_local on cluster '{cluster}'
+ADD COLUMN gas_tip_cap Nullable(UInt128) Codec(ZSTD(1)) AFTER gas,
+ADD COLUMN gas_fee_cap Nullable(UInt128) Codec(ZSTD(1)) AFTER gas_tip_cap,
+ADD COLUMN blob_gas Nullable(UInt64) Codec(ZSTD(1)) AFTER call_data_size,
+ADD COLUMN blob_gas_fee_cap Nullable(UInt128) Codec(ZSTD(1)) AFTER blob_gas,
+ADD COLUMN blob_hashes Array(String) Codec(ZSTD(1)) AFTER blob_gas_fee_cap,
+ADD COLUMN blob_sidecars_size Nullable(UInt32) Codec(ZSTD(1)) AFTER blob_hashes,
+ADD COLUMN blob_sidecars_empty_size Nullable(UInt32) Codec(ZSTD(1)) AFTER blob_sidecars_size,
+COMMENT COLUMN gas_tip_cap 'The priority fee (tip) the user has set for the transaction',
+COMMENT COLUMN gas_fee_cap 'The max fee the user has set for the transaction',
+COMMENT COLUMN blob_gas 'The maximum gas provided for the blob transaction execution',
+COMMENT COLUMN blob_gas_fee_cap 'The max fee the user has set for the transaction',
+COMMENT COLUMN blob_hashes 'The hashes of the blob commitments for blob transactions',
+COMMENT COLUMN blob_sidecars_size 'The total size of the sidecars for blob transactions in bytes',
+COMMENT COLUMN blob_sidecars_empty_size 'The total empty size of the sidecars for blob transactions in bytes';
+
+ALTER TABLE mempool_transaction on cluster '{cluster}'
+ADD COLUMN gas_tip_cap Nullable(UInt128) Codec(ZSTD(1)) AFTER gas,
+ADD COLUMN gas_fee_cap Nullable(UInt128) Codec(ZSTD(1)) AFTER gas_tip_cap,
+ADD COLUMN blob_gas Nullable(UInt64) Codec(ZSTD(1)) AFTER call_data_size,
+ADD COLUMN blob_gas_fee_cap Nullable(UInt128) Codec(ZSTD(1)) AFTER blob_gas,
+ADD COLUMN blob_hashes Array(String) Codec(ZSTD(1)) AFTER blob_gas_fee_cap,
+ADD COLUMN blob_sidecars_size Nullable(UInt32) Codec(ZSTD(1)) AFTER blob_hashes,
+ADD COLUMN blob_sidecars_empty_size Nullable(UInt32) Codec(ZSTD(1)) AFTER blob_sidecars_size,
+COMMENT COLUMN gas_tip_cap 'The priority fee (tip) the user has set for the transaction',
+COMMENT COLUMN gas_fee_cap 'The max fee the user has set for the transaction',
+COMMENT COLUMN blob_gas 'The maximum gas provided for the blob transaction execution',
+COMMENT COLUMN blob_gas_fee_cap 'The max fee the user has set for the transaction',
+COMMENT COLUMN blob_hashes 'The hashes of the blob commitments for blob transactions',
+COMMENT COLUMN blob_sidecars_size 'The total size of the sidecars for blob transactions in bytes',
+COMMENT COLUMN blob_sidecars_empty_size 'The total empty size of the sidecars for blob transactions in bytes';
+
+ALTER TABLE canonical_beacon_block_execution_transaction_local on cluster '{cluster}'
+ADD COLUMN gas_tip_cap Nullable(UInt128) Codec(ZSTD(1)) AFTER gas,
+ADD COLUMN gas_fee_cap Nullable(UInt128) Codec(ZSTD(1)) AFTER gas_tip_cap,
+ADD COLUMN blob_gas Nullable(UInt64) Codec(ZSTD(1)) AFTER call_data_size,
+ADD COLUMN blob_gas_fee_cap Nullable(UInt128) Codec(ZSTD(1)) AFTER blob_gas,
+ADD COLUMN blob_hashes Array(String) Codec(ZSTD(1)) AFTER blob_gas_fee_cap,
+ADD COLUMN blob_sidecars_size Nullable(UInt32) Codec(ZSTD(1)) AFTER blob_hashes,
+ADD COLUMN blob_sidecars_empty_size Nullable(UInt32) Codec(ZSTD(1)) AFTER blob_sidecars_size,
+COMMENT COLUMN gas_tip_cap 'The priority fee (tip) the user has set for the transaction',
+COMMENT COLUMN gas_fee_cap 'The max fee the user has set for the transaction',
+COMMENT COLUMN blob_gas 'The maximum gas provided for the blob transaction execution',
+COMMENT COLUMN blob_gas_fee_cap 'The max fee the user has set for the transaction',
+COMMENT COLUMN blob_hashes 'The hashes of the blob commitments for blob transactions',
+COMMENT COLUMN blob_sidecars_size 'The total size of the sidecars for blob transactions in bytes',
+COMMENT COLUMN blob_sidecars_empty_size 'The total empty size of the sidecars for blob transactions in bytes';
+
+ALTER TABLE canonical_beacon_block_execution_transaction on cluster '{cluster}'
+ADD COLUMN gas_tip_cap Nullable(UInt128) Codec(ZSTD(1)) AFTER gas,
+ADD COLUMN gas_fee_cap Nullable(UInt128) Codec(ZSTD(1)) AFTER gas_tip_cap,
+ADD COLUMN blob_gas Nullable(UInt64) Codec(ZSTD(1)) AFTER call_data_size,
+ADD COLUMN blob_gas_fee_cap Nullable(UInt128) Codec(ZSTD(1)) AFTER blob_gas,
+ADD COLUMN blob_hashes Array(String) Codec(ZSTD(1)) AFTER blob_gas_fee_cap,
+ADD COLUMN blob_sidecars_size Nullable(UInt32) Codec(ZSTD(1)) AFTER blob_hashes,
+ADD COLUMN blob_sidecars_empty_size Nullable(UInt32) Codec(ZSTD(1)) AFTER blob_sidecars_size,
+COMMENT COLUMN gas_tip_cap 'The priority fee (tip) the user has set for the transaction',
+COMMENT COLUMN gas_fee_cap 'The max fee the user has set for the transaction',
+COMMENT COLUMN blob_gas 'The maximum gas provided for the blob transaction execution',
+COMMENT COLUMN blob_gas_fee_cap 'The max fee the user has set for the transaction',
+COMMENT COLUMN blob_hashes 'The hashes of the blob commitments for blob transactions',
+COMMENT COLUMN blob_sidecars_size 'The total size of the sidecars for blob transactions in bytes',
+COMMENT COLUMN blob_sidecars_empty_size 'The total empty size of the sidecars for blob transactions in bytes';
diff --git a/...x_replacing_tables_sharding_keys.down.sql → ...x_replacing_tables_sharding_keys.down.sql b/...x_replacing_tables_sharding_keys.down.sql → ...x_replacing_tables_sharding_keys.down.sql
diff --git a/...fix_replacing_tables_sharding_keys.up.sql → ...fix_replacing_tables_sharding_keys.up.sql b/...fix_replacing_tables_sharding_keys.up.sql → ...fix_replacing_tables_sharding_keys.up.sql
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -111,6 +111,9 @@ services:
   xatu-server:
     image: ethpandaops/xatu:latest
     command: server --config /etc/xatu-server/config.yaml
+    build:
+      context: .
+      dockerfile: Dockerfile
     ports:
       - "8080:8080"
       - "9096:9090"

diff --git a/go.mod b/go.mod
@@ -1,6 +1,6 @@
 module github.com/ethpandaops/xatu
 
-go 1.19
+go 1.22.0
 
 require (
 	github.com/IBM/sarama v1.41.2
@@ -10,9 +10,9 @@ require (
 	github.com/cenkalti/backoff/v4 v4.2.1
 	github.com/chuckpreslar/emission v0.0.0-20170206194824-a7ddd980baf9
 	github.com/creasty/defaults v1.7.0
-	github.com/ethereum/go-ethereum v1.12.0
+	github.com/ethereum/go-ethereum v1.13.14
 	github.com/ethpandaops/beacon v0.34.0
-	github.com/ethpandaops/ethcore v0.0.0-20230804013106-6453c36c8c30
+	github.com/ethpandaops/ethcore v0.0.0-20240306031202-16f9e1926c0c
 	github.com/ethpandaops/ethwallclock v0.3.0
 	github.com/go-co-op/gocron v1.27.1
 	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb
@@ -36,7 +36,7 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0
 	go.opentelemetry.io/otel/sdk v1.19.0
 	go.opentelemetry.io/otel/trace v1.19.0
-	golang.org/x/sync v0.4.0
+	golang.org/x/sync v0.5.0
 	google.golang.org/grpc v1.58.2
 	google.golang.org/protobuf v1.31.0
 	gopkg.in/cenkalti/backoff.v1 v1.1.0
@@ -45,29 +45,36 @@ require (
 
 require (
 	github.com/DataDog/zstd v1.5.5 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/VictoriaMetrics/fastcache v1.12.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bits-and-blooms/bitset v1.10.0 // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cockroachdb/errors v1.10.0 // indirect
 	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
-	github.com/cockroachdb/pebble v0.0.0-20230803185510-83c9361c3b82 // indirect
+	github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 // indirect
 	github.com/cockroachdb/redact v1.1.5 // indirect
-	github.com/cockroachdb/tokenbucket v0.0.0-20230613231145-182959a1fad6 // indirect
+	github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
+	github.com/consensys/bavard v0.1.13 // indirect
+	github.com/consensys/gnark-crypto v0.12.1 // indirect
+	github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 // indirect
+	github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/deckarep/golang-set/v2 v2.3.1 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/eapache/go-resiliency v1.4.0 // indirect
 	github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
+	github.com/ethereum/c-kzg-4844 v0.4.0 // indirect
 	github.com/fatih/color v1.16.0 // indirect
 	github.com/ferranbt/fastssz v0.1.3 // indirect
+	github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46 // indirect
 	github.com/getsentry/sentry-go v0.23.0 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
-	github.com/go-ole/go-ole v1.2.6 // indirect
-	github.com/go-stack/stack v1.8.1 // indirect
+	github.com/go-ole/go-ole v1.3.0 // indirect
 	github.com/goccy/go-yaml v1.9.8 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
@@ -101,6 +108,7 @@ require (
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/minio/sha256-simd v1.0.1 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/mmcloughlin/addchain v0.4.0 // indirect
 	github.com/mr-tron/base58 v1.2.0 // indirect
 	github.com/multiformats/go-base32 v0.1.0 // indirect
 	github.com/multiformats/go-base36 v0.2.0 // indirect
@@ -127,22 +135,25 @@ require (
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/supranational/blst v0.3.11 // indirect
 	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
-	github.com/tklauser/go-sysconf v0.3.11 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/yusufpapurcu/wmi v1.2.3 // indirect
 	go.opentelemetry.io/otel/metric v1.19.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	golang.org/x/crypto v0.18.0 // indirect
-	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
-	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
+	golang.org/x/mod v0.14.0 // indirect
+	golang.org/x/net v0.18.0 // indirect
 	golang.org/x/sys v0.16.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/tools v0.15.0 // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect
-	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	lukechampine.com/blake3 v1.2.1 // indirect
+	rsc.io/tmplfunc v0.0.3 // indirect
 )