diff --git a/.gitignore b/.gitignore index 76c9db65..c418707a 100644 --- a/.gitignore +++ b/.gitignore @@ -20,5 +20,7 @@ ssv-config/password.pass ssv-config/password ssv-config/encrypted_private_key.json ssv-config/config.yaml +ssv-config/config.yaml.original ssv-config/dkg-config.yaml +ssv-config/dkg-config.yaml.original .nada diff --git a/README.md b/README.md index a10f9c6c..f51d5c79 100644 --- a/README.md +++ b/README.md @@ -34,4 +34,4 @@ Eth Docker uses a "semver-ish" scheme. large. - Second through fourth digit, [semver](https://semver.org/). -This is Eth Docker v2.9.0.0 +This is Eth Docker v2.12.0.0 diff --git a/besu.yml b/besu.yml index d3437394..a7cd8956 100644 --- a/besu.yml +++ b/besu.yml @@ -79,6 +79,7 @@ services: - metrics.path=/metrics - metrics.port=6060 - metrics.instance=execution + - metrics.network=${NETWORK} set-prune-marker: profiles: ["tools"] diff --git a/besu/Dockerfile.binary b/besu/Dockerfile.binary index 1c8404b5..ff024867 100644 --- a/besu/Dockerfile.binary +++ b/besu/Dockerfile.binary @@ -1,5 +1,5 @@ -ARG DOCKER_TAG -ARG DOCKER_REPO +ARG DOCKER_TAG=latest +ARG DOCKER_REPO=hyperledger/besu FROM ${DOCKER_REPO}:${DOCKER_TAG} diff --git a/besu/Dockerfile.source b/besu/Dockerfile.source index 988445e5..054236d3 100644 --- a/besu/Dockerfile.source +++ b/besu/Dockerfile.source @@ -1,5 +1,5 @@ # Build Besu in a stock Ubuntu container -FROM eclipse-temurin:17-jdk-jammy as builder +FROM eclipse-temurin:21-jdk-jammy AS builder # This is here to avoid build-time complaints ARG DOCKER_TAG @@ -8,7 +8,7 @@ ARG DOCKER_REPO ARG BUILD_TARGET ARG SRC_REPO -RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates git +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates git libjemalloc-dev WORKDIR /usr/src RUN bash -c "git clone --recurse-submodules -j8 ${SRC_REPO} besu && cd besu && git config advice.detachedHead false && git fetch --all --tags && if [[ ${BUILD_TARGET} =~ pr-.+ ]]; then git fetch origin pull/$(echo ${BUILD_TARGET} | cut -d '-' -f 2)/head:besu-pr; git checkout besu-pr; else git checkout ${BUILD_TARGET}; fi && ./gradlew installDist" diff --git a/besu/docker-entrypoint.sh b/besu/docker-entrypoint.sh index ea5c8a5a..aa5ec239 100755 --- a/besu/docker-entrypoint.sh +++ b/besu/docker-entrypoint.sh @@ -55,7 +55,7 @@ if [ "${ARCHIVE_NODE}" = "true" ]; then echo "Besu archive node without pruning" __prune="--data-storage-format=FOREST --sync-mode=FULL" else - __prune="--data-storage-format=BONSAI --sync-mode=SNAP --Xbonsai-limit-trie-logs-enabled=true" + __prune="--data-storage-format=BONSAI --sync-mode=SNAP" fi __memtotal=$(awk '/MemTotal/ {printf "%d", int($2/1024/1024)}' /proc/meminfo) @@ -73,7 +73,7 @@ if [ -f /var/lib/besu/prune-marker ]; then fi # Word splitting is desired for the command line parameters # shellcheck disable=SC2086 - exec "$@" ${__network} ${__prune} ${EL_EXTRAS} storage x-trie-log prune + exec "$@" ${__network} ${__prune} ${EL_EXTRAS} storage trie-log prune else # Word splitting is desired for the command line parameters # shellcheck disable=SC2086 diff --git a/central-metrics.yml b/central-metrics.yml index 59636625..fee0bf0b 100644 --- a/central-metrics.yml +++ b/central-metrics.yml @@ -21,3 +21,4 @@ services: - metrics.path=/metrics - metrics.port=9090 - metrics.instance=ethereum-metrics-exporter + - metrics.network=${NETWORK} diff --git a/default.env b/default.env index efa75beb..d1653952 100644 --- a/default.env +++ b/default.env @@ -30,9 +30,11 @@ WEB3SIGNER=false # Whether to explicitly enable IPv6 P2P. See https://ethdocker.com/Support/ipv6 # Needs to be false or true for Eth Docker to work IPV6=false -# Directory to keep ancient data. In use for Geth, optional. +# Directory to keep ancient/static data. In use for Reth and Geth, optional. # If you use this, make sure to chown the directory to uid 10001 ANCIENT_DIR= +# Password for Lighthouse Siren +SIREN_PASSWORD= # Promtail logs label, something unique like the server name LOGS_LABEL=eth-docker @@ -86,8 +88,8 @@ PRYSM_UDP_PORT=9000 CL_QUIC_PORT=9001 # Local grafana dashboard port. Do not expose to Internet, it is insecure http GRAFANA_PORT=3000 -# Local Siren UI port. Do not expose to Internet, it is insecure http -SIREN_PORT=8080 +# Local Siren UI port +SIREN_PORT=2443 # Prometheus port used when exposing directly on host; used for federation PROMETHEUS_PORT=9090 # Local key manager port. Reachable only via localhost. Also doubles as Prysm web port @@ -202,8 +204,7 @@ TEKU_DOCKERFILE=Dockerfile.binary # SRC build target can be a tag, a branch, or a pr as "pr-ID" LH_SRC_BUILD_TARGET=stable LH_SRC_REPO=https://github.com/sigp/lighthouse -# If Lighthouse terminates with a SIGILL, make this "latest" -LH_DOCKER_TAG=latest-modern +LH_DOCKER_TAG=latest LH_DOCKER_REPO=sigp/lighthouse LH_DOCKERFILE=Dockerfile.binary @@ -235,7 +236,7 @@ LS_DOCKERFILE=Dockerfile.binary # SRC build target can be a tag, a branch, or a pr as "pr-ID" GRANDINE_SRC_BUILD_TARGET=master GRANDINE_SRC_REPO=https://github.com/grandinetech/grandine -GRANDINE_DOCKER_TAG=latest +GRANDINE_DOCKER_TAG=stable GRANDINE_DOCKER_REPO=sifrai/grandine GRANDINE_DOCKERFILE=Dockerfile.binary @@ -256,8 +257,7 @@ BESU_DOCKERFILE=Dockerfile.binary # SRC build target can be a tag, a branch, or a pr as "pr-ID" ERIGON_SRC_BUILD_TARGET='$(git describe --tags $(git rev-list --tags --max-count=1))' ERIGON_SRC_REPO=https://github.com/ledgerwatch/erigon -#ERIGON_DOCKER_TAG=stable # stable tag was abandoned by Erigon team, set a tag manually -ERIGON_DOCKER_TAG=v2.59.3 +ERIGON_DOCKER_TAG=v2.60.4 ERIGON_DOCKER_REPO=thorax/erigon ERIGON_DOCKERFILE=Dockerfile.binary @@ -308,4 +308,4 @@ DDNS_TAG=v2 NODE_EXPORTER_IGNORE_MOUNT_REGEX='^/(dev|proc|sys|run|var/lib/docker/.+)($|/)' # Used by ethd update - please do not adjust -ENV_VERSION=11 +ENV_VERSION=13 diff --git a/erigon.yml b/erigon.yml index 34afe415..0d80c001 100644 --- a/erigon.yml +++ b/erigon.yml @@ -101,6 +101,7 @@ services: - metrics.path=/debug/metrics/prometheus - metrics.port=6060 - metrics.instance=execution + - metrics.network=${NETWORK} volumes: erigon-el-data: diff --git a/erigon/Dockerfile.binary b/erigon/Dockerfile.binary index f318ee5d..13d7706f 100644 --- a/erigon/Dockerfile.binary +++ b/erigon/Dockerfile.binary @@ -1,5 +1,5 @@ -ARG DOCKER_TAG -ARG DOCKER_REPO +ARG DOCKER_TAG=stable +ARG DOCKER_REPO=thorax/erigon FROM ${DOCKER_REPO}:${DOCKER_TAG} diff --git a/erigon/Dockerfile.source b/erigon/Dockerfile.source index 13047ca0..87fbd92b 100644 --- a/erigon/Dockerfile.source +++ b/erigon/Dockerfile.source @@ -1,5 +1,5 @@ # Build Erigon in a stock Go build container -FROM golang:1.22-alpine as builder +FROM golang:1.22-alpine AS builder # Unused, this is here to avoid build time complaints ARG DOCKER_TAG diff --git a/ethd b/ethd index c9854a6c..f4549fc5 100755 --- a/ethd +++ b/ethd @@ -203,8 +203,10 @@ prep_conffiles() { ${__as_owner} cp ssv-config/dkg-config-sample.yaml ssv-config/dkg-config.yaml fi # Make sure local user owns the dkg output dir and everything in it - ${__auto_sudo} chown -R "${OWNER}:${OWNER_GROUP}" .eth/dkg_output - ${__auto_sudo} chmod -R 755 .eth/dkg_output + if find .eth/dkg_output \! -user "${OWNER}" -o \! -group "${OWNER_GROUP}" -o \! -perm 755 | grep -q .; then + ${__auto_sudo} chown -R "${OWNER}:${OWNER_GROUP}" .eth/dkg_output + ${__auto_sudo} chmod -R 755 .eth/dkg_output + fi # Create ext-network.yml if it doesn't exist if [ ! -f "ext-network.yml" ]; then ${__as_owner} cp ext-network.yml.sample ext-network.yml @@ -646,20 +648,31 @@ ssv_switch() { echo "Making changes to ssv-config/config.yaml" var="NETWORK" NETWORK=$(sed -n -e "s/^${var}=\(.*\)/\1/p" "${ENV_FILE}" || true) - sed -i'' 's/blox-ssv2.yml/ssv.yml/' "${ENV_FILE}".source + sed -i'.original' 's/blox-ssv2.yml/ssv.yml/' "${ENV_FILE}".source if ! grep -q "LogFilePath:" ssv-config/config.yaml; then - sed -i'' '/global:/a\ LogFilePath: /tmp/ssv/debug.log' ssv-config/config.yaml +# macOS-isms: Newline for sed add + sed -i'.original' '/global:/a\ + LogFilePath: /tmp/ssv/debug.log +' ssv-config/config.yaml fi if ! grep -q "MetricsAPIPort:" ssv-config/config.yaml; then - sed -i'' '$a\MetricsAPIPort: 15000' ssv-config/config.yaml + sed -i'.original' '$a\ +MetricsAPIPort: 15000 +' ssv-config/config.yaml fi if ! grep -q "ssv:" ssv-config/config.yaml; then - sed -i '/^ Network:/d' ssv-config/config.yaml # Remove old eth2 Network line if present - sed -i'' '$a\ssv:' ssv-config/config.yaml + sed -i'.original' '/^ Network:/d' ssv-config/config.yaml # Remove old eth2 Network line if present + sed -i'.original' '$a\ + ssv: + ' ssv-config/config.yaml if [ "${NETWORK}" = "holesky" ]; then - sed -i'' '$a\ Network: holesky' ssv-config/config.yaml + sed -i'.original' '$a\ + Network: holesky +' ssv-config/config.yaml elif [ "${NETWORK}" = "mainnet" ]; then - sed -i'' '$a\ Network: mainnet' ssv-config/config.yaml + sed -i'.original' '$a\ + Network: mainnet +' ssv-config/config.yaml else echo "${NETWORK} is not something that works with SSV." echo "Please fix this manually before running $__me update again." @@ -667,6 +680,7 @@ ssv_switch() { exit 1 fi fi + rm ssv-config/config.yaml.original } @@ -951,7 +965,7 @@ envmigrate() { PRYSM_UDP_PORT CL_QUIC_PORT GRAFANA_PORT SIREN_PORT PROMETHEUS_PORT KEY_API_PORT TRAEFIK_WEB_PORT \ TRAEFIK_WEB_HTTP_PORT CL_REST_PORT EL_RPC_PORT EL_WS_PORT EE_PORT ERIGON_TORRENT_PORT LOG_LEVEL JWT_SECRET \ EL_EXTRAS CL_EXTRAS VC_EXTRAS ARCHIVE_NODE SSV_P2P_PORT SSV_P2P_PORT_UDP ERIGON_P2P_PORT_2 \ - ERIGON_P2P_PORT_3 LODESTAR_HEAP SSV_DKG_PORT ) + ERIGON_P2P_PORT_3 LODESTAR_HEAP SSV_DKG_PORT SIREN_PASSWORD ) TARGET_VARS=( ETH_DOCKER_TAG NIM_SRC_BUILD_TARGET NIM_SRC_REPO NIM_DOCKER_TAG NIM_DOCKER_VC_TAG NIM_DOCKER_REPO \ NIM_DOCKER_VC_REPO NIM_DOCKERFILE TEKU_SRC_BUILD_TARGET TEKU_SRC_REPO TEKU_DOCKER_TAG TEKU_DOCKER_REPO \ TEKU_DOCKERFILE LH_SRC_BUILD_TARGET LH_SRC_REPO LH_DOCKER_TAG LH_DOCKER_REPO LH_DOCKERFILE \ @@ -971,6 +985,14 @@ envmigrate() { OLD_VARS=( ) NEW_VARS=( ) +# Always make sure we have a SIREN password + var="SIREN_PASSWORD" + SIREN_PASSWORD=$(sed -n -e "s/^${var}=\(.*\)/\1/p" "${ENV_FILE}" || true) + if [ -z "${SIREN_PASSWORD}" ]; then + SIREN_PASSWORD=$(head -c 8 /dev/urandom | od -A n -t u8 | tr -d '[:space:]' | sha256sum | head -c 32) + set_value_in_env + fi + var=ENV_VERSION __target_ver=$(sed -n -e "s/^${var}=\(.*\)/\1/p" "default.env" || true) __source_ver=$(sed -n -e "s/^${var}=\(.*\)/\1/p" "${ENV_FILE}" || true) @@ -998,7 +1020,7 @@ envmigrate() { # Literal match intended # shellcheck disable=SC2076 if [[ "${value}" =~ "blox-ssv2.yml" ]]; then - ssv_switch + ssv_switch fi # Migrate over user settings @@ -1015,6 +1037,11 @@ envmigrate() { value=$((value + 1)) echo "Adjusted CL_QUIC_PORT to ${value} so it does not conflict with CL_P2P_PORT" fi + __prysm_port=$(sed -n -e "s/^PRYSM_UDP_PORT=\(.*\)/\1/p" "${ENV_FILE}.source" || true) + if [ -n "${__prysm_port}" ] && [ "${__prysm_port}" = "${value}" ]; then # just in case this is one ahead + value=$((value + 1)) + echo "Adjusted CL_QUIC_PORT to ${value} so it does not conflict with PRYSM_UDP_PORT" + fi fi # Literal match intended # shellcheck disable=SC2076 @@ -1022,10 +1049,10 @@ envmigrate() { value="https://beaconstate.info" fi if [[ "${var}" = "HOST_IP" && "${value: -1}" = ":" ]]; then - value="${value%:}" # Undo Compose V1 accomodation + value="${value%:}" # Undo Compose V1 accommodation fi if [[ "${var}" = "SHARE_IP" && "${value: -1}" = ":" ]]; then - value="${value%:}" # Undo Compose V1 accomodation + value="${value%:}" # Undo Compose V1 accommodation fi # Handle & in GRAFFITI gracefully sed -i'.original' -e "s~^\(${var}\s*=\s*\).*\$~\1${value//&/\\&}~" "${ENV_FILE}" @@ -1046,6 +1073,12 @@ envmigrate() { if [[ "${var}" = "DDNS_TAG" && "${__source_ver}" -lt "8" ]]; then # Switch to ddns-updater value="v2" fi + if [[ "${var}" = "LH_DOCKER_TAG" && "${value}" = "latest-modern" ]]; then # LH 5.2 ditched latest-modern + value="latest" + fi + if [[ "${var}" = "ERIGON_DOCKER_TAG" && "${value}" = "stable" ]]; then # Erigon ditched stable + value="v2.60.1" + fi sed -i'.original' -e "s~^\(${var}\s*=\s*\).*$~\1${value}~" "${ENV_FILE}" fi done @@ -1524,6 +1557,99 @@ prune-besu() { } +prune-reth() { + __non_interactive=0 + while : + do + if [ -z "${1+x}" ]; then + break + fi + case "$1" in + --non-interactive) + __non_interactive=1 + shift + ;; + *) + echo "Error: Unknown option: $1" >&2 + exit 1 + ;; + esac + done + if [ "${ETHD_FRONTEND:-}" = "noninteractive" ]; then + __non_interactive=1 + fi + + if [ ! -f "${ENV_FILE}" ]; then + echo "${ENV_FILE} configuration file not found, aborting." + exit 1 + fi + + if ! grep -q '^COMPOSE_FILE=.*reth\.yml' "${ENV_FILE}" 2>/dev/null ; then + echo "You do not appear to be using Reth, aborting." + exit 1 + fi + + # Check for archive node + var="ARCHIVE_NODE" + value=$(sed -n -e "s/^${var}=\(.*\)/\1/p" "${ENV_FILE}" || true) + if [[ "${value}" = "true" ]]; then + echo "Reth is an archive node: Aborting." + exit 1 + fi + + rpc_line=$(grep '^EL_RPC_PORT=' "${ENV_FILE}") + regex='^EL_RPC_PORT=([0-9]+)' + if [[ ! "${rpc_line}" =~ ${regex} ]]; then + echo "Unable to determine EL_RPC_PORT, aborting." + exit 1 + else + rpc_port="${BASH_REMATCH[1]}" + fi + + set +e + sync_status=$(docompose exec -T execution wget -qO- "http://localhost:$rpc_port" \ + --header 'Content-Type: application/json' --post-data '{"jsonrpc":"2.0","method":"eth_syncing","params":[],"id":1}') + exitstatus=$? + set -e + if [ $exitstatus -ne 0 ]; then + echo "Unable to connect to Reth: Is it running?" + echo "Output: ${sync_status}" + echo "Aborting." + exit 1 + fi + + if [[ ! "${sync_status}" =~ "false" ]]; then + echo "Reth is not done syncing yet. Sync status:" + echo "${sync_status}" + echo + echo "Aborting." + exit 1 + fi + + if [ $__non_interactive = 0 ]; then + while true; do + read -rp "WARNING - this will stop Reth and prune its database. Do you wish to continue? (No/Yes) " yn + case $yn in + [Yy][Ee][Ss] ) break;; + * ) echo "Aborting, no changes made"; exit 130;; + esac + done + fi + + echo + echo "Starting Reth prune" + echo + docompose run --rm set-prune-marker "touch /var/lib/reth/prune-marker" + docompose stop execution && docompose rm -f execution + start + echo + echo "Prune is running, you can observe it with '$__me logs -f execution'" + echo + echo "When prune is done, Reth will automatically start again." + echo +} + + prune-nethermind() { __non_interactive=0 while : @@ -1783,7 +1909,7 @@ prep-keyimport() { exit 1 fi - __non_interactive=0 + __args="" while : do @@ -1831,7 +1957,11 @@ prep-keyimport() { echo "KEYSTORE_PASSWORD not set or empty, aborting" exit 1 fi - __non_interactive=1 + __args+="${__args:+ }--interactive" + shift + ;; + --debug) + __args+="${__args:+ }--debug" shift ;; *) @@ -2014,11 +2144,7 @@ keys() { #__i_haz_keys_service shift prep-keyimport "$@" - if [ ${__non_interactive} = 1 ]; then - docompose run --rm -e OWNER_UID="${__owner_uid}" validator-keys import --non-interactive - else - docompose run --rm -e OWNER_UID="${__owner_uid}" validator-keys import - fi + docompose run --rm -e OWNER_UID="${__owner_uid}" validator-keys import "${__args}" elif [ "${1:-}" = "create-prysm-wallet" ]; then var="COMPOSE_FILE" value=$(sed -n -e "s/^${var}=\(.*\)/\1/p" "${ENV_FILE}" || true) @@ -2355,26 +2481,40 @@ screen.\n\nCustom testnets only work with a URL to fetch their configuration fro query_deployment() { if [ "${NETWORK}" = "gnosis" ]; then + if uname -m | grep -q riscv64; then + echo "Gnosis network has no available client combos on RISC-V. Aborting." + exit 1 + fi __deployment=$(whiptail --notags --title "Select deployment type" --menu \ "What kind of deployment do you want to run?" 10 65 3 \ "node" "Ethereum node - consensus, execution and validator client" \ "rpc" "Ethereum RPC node - consensus and execution client" \ "validator" "Validator client only" 3>&1 1>&2 2>&3) - elif uname -a | grep -q aarch64; then + elif uname -m | grep -q aarch64 || uname -m | grep -q arm64; then __deployment=$(whiptail --notags --title "Select deployment type" --menu \ "What kind of deployment do you want to run?" 10 65 3 \ "node" "Ethereum node - consensus, execution and validator client" \ "rpc" "Ethereum RPC node - consensus and execution client" \ "rocket" "Validator client only - integrate with RocketPool" 3>&1 1>&2 2>&3) - else + elif uname -m | grep -q riscv64; then + __deployment=$(whiptail --notags --title "Select deployment type" --menu \ + "What kind of deployment do you want to run?" 10 65 3 \ + "node" "Ethereum node - consensus, execution and validator client" \ + "rpc" "Ethereum RPC node - consensus and execution client" \ + "rocket" "Validator client only - integrate with RocketPool" 3>&1 1>&2 2>&3) + elif uname -m | grep -q x86_64; then __deployment=$(whiptail --notags --title "Select deployment type" --menu \ "What kind of deployment do you want to run?" 11 65 4 \ "node" "Ethereum node - consensus, execution and validator client" \ "rpc" "Ethereum RPC node - consensus and execution client" \ "rocket" "Validator client only - integrate with RocketPool" \ "ssv" "SSV node - consensus, execution and ssv-node" 3>&1 1>&2 2>&3) + else + echo "Eth Docker does not recognize this CPU architecture. Aborting." + echo "Output of uname -m" + uname -m + exit 1 fi - echo "Your deployment choice is: ${__deployment}" } @@ -2388,7 +2528,7 @@ query_validator_client() { "lodestar-vc-only.yml" "Lodestar validator client" \ "nimbus-vc-only.yml" "Nimbus validator client" 3>&1 1>&2 2>&3) elif [ "${__deployment}" = "rocket" ]; then - if uname -a | grep -q aarch64; then + if uname -m | grep -q aarch64 || uname -m | grep -q arm64; then CONSENSUS_CLIENT=$(whiptail --notags --title "Select validator client" --menu \ "Which validator client do you want to run?" 11 65 4 \ "lighthouse-vc-only.yml" "Lighthouse validator client" \ @@ -2403,7 +2543,7 @@ query_validator_client() { "lodestar-vc-only.yml" "Lodestar validator client" \ "nimbus-vc-only.yml" "Nimbus validator client" 3>&1 1>&2 2>&3) fi - elif uname -a | grep -q aarch64; then + elif uname -m | grep -q aarch64 || uname -m | grep -q arm64; then CONSENSUS_CLIENT=$(whiptail --notags --title "Select validator client" --menu \ "Which validator client do you want to run?" 11 65 4 \ "lighthouse-vc-only.yml" "Lighthouse validator client" \ @@ -2432,7 +2572,7 @@ query_consensus_client() { "teku.yml" "Teku (Java) - consensus and validator client" \ "lodestar.yml" "Lodestar (Javascript) - consensus and validator client" \ "nimbus.yml" "Nimbus (Nim) - consensus and validator client" 3>&1 1>&2 2>&3) - elif uname -a | grep -q aarch64; then + elif uname -m | grep -q aarch64 || uname -m | grep -q arm64; then CONSENSUS_CLIENT=$(whiptail --notags --title "Select consensus client" --menu \ "Which consensus client do you want to run?" 11 65 4 \ "nimbus.yml" "Nimbus (Nim) - consensus and validator client" \ @@ -2440,6 +2580,10 @@ query_consensus_client() { "lodestar.yml" "Lodestar (Javascript) - consensus and validator client" \ "lighthouse.yml" "Lighthouse (Rust) - consensus and validator client" \ "teku.yml" "Teku (Java) - consensus and validator client" 3>&1 1>&2 2>&3) + elif uname -m | grep -q riscv64; then + CONSENSUS_CLIENT=$(whiptail --notags --title "Select consensus client" --menu \ + "Which consensus client do you want to run?" 11 65 4 \ + "nimbus.yml" "Nimbus (Nim) - consensus and validator client" 3>&1 1>&2 2>&3) else CONSENSUS_CLIENT=$(whiptail --notags --title "Select consensus client" --menu \ "Which consensus client do you want to run?" 13 65 6 \ @@ -2463,7 +2607,7 @@ query_consensus_only_client() { "teku-cl-only.yml" "Teku (Java) - consensus client" \ "lodestar-cl-only.yml" "Lodestar (Javascript) - consensus client" \ "nimbus-cl-only.yml" "Nimbus (Nim) - consensus client" 3>&1 1>&2 2>&3) - elif uname -a | grep -q aarch64; then + elif uname -m | grep -q aarch64 || uname -m | grep -q arm64; then CONSENSUS_CLIENT=$(whiptail --notags --title "Select consensus client" --menu \ "Which consensus client do you want to run?" 11 65 4 \ "nimbus-cl-only.yml" "Nimbus (Nim) - consensus client" \ @@ -2471,6 +2615,10 @@ query_consensus_only_client() { "lodestar-cl-only.yml" "Lodestar (Javascript) - consensus client" \ "lighthouse-cl-only.yml" "Lighthouse (Rust) - consensus client" \ "teku-cl-only.yml" "Teku (Java) - consensus client" 3>&1 1>&2 2>&3) + elif uname -m | grep -q riscv64; then + CONSENSUS_CLIENT=$(whiptail --notags --title "Select consensus client" --menu \ + "Which consensus client do you want to run?" 11 65 4 \ + "nimbus-cl-only.yml" "Nimbus (Nim) - consensus client" 3>&1 1>&2 2>&3) else CONSENSUS_CLIENT=$(whiptail --notags --title "Select consensus client" --menu \ "Which consensus client do you want to run?" 13 65 6 \ @@ -2518,7 +2666,7 @@ again or Cancel on the next screen." 10 65 query_execution_client() { if [ "${NETWORK}" = "gnosis" ]; then - if uname -a | grep -q aarch64; then + if uname -m | grep -q aarch64 || uname -m | grep -q arm64; then EXECUTION_CLIENT=$(whiptail --notags --title "Select execution client" --menu \ "Which execution client do you want to run?" 9 65 2 \ "nethermind.yml" "Nethermind (.NET)" \ @@ -2530,20 +2678,25 @@ query_execution_client() { "erigon.yml" "Erigon (Go)" \ "NONE" "Custom - Distributed" 3>&1 1>&2 2>&3) fi - elif uname -a | grep -q aarch64; then + elif uname -m | grep -q aarch64 || uname -m | grep -q arm64; then EXECUTION_CLIENT=$(whiptail --notags --title "Select execution client" --menu \ "Which execution client do you want to run?" 11 65 4 \ "besu.yml" "Besu (Java)" \ "nethermind.yml" "Nethermind (.NET)" \ "geth.yml" "Geth (Go)" \ "NONE" "Custom - Distributed" 3>&1 1>&2 2>&3) + elif uname -m | grep -q riscv64; then + EXECUTION_CLIENT=$(whiptail --notags --title "Select execution client" --menu \ + "Which execution client do you want to run?" 11 65 4 \ + "geth.yml" "Geth (Go)" \ + "NONE" "Custom - Distributed" 3>&1 1>&2 2>&3) else EXECUTION_CLIENT=$(whiptail --notags --title "Select execution client" --menu \ "Which execution client do you want to run?" 13 65 6 \ - "nethermind.yml" "Nethermind (.NET)" \ + "reth.yml" "Reth (Rust)" \ "besu.yml" "Besu (Java)" \ + "nethermind.yml" "Nethermind (.NET)" \ "geth.yml" "Geth (Go)" \ - "reth.yml" "Reth (Rust) - beta release" \ "erigon.yml" "Erigon (Go)" \ "NONE" "Custom - Distributed" 3>&1 1>&2 2>&3) fi @@ -2792,13 +2945,14 @@ query_dkg() { __ssv_operator_id=$(whiptail --title "Register SSV operator" --inputbox "\n1. Your SSV node public key:\n\n${__public_key}\n\n2. Register your operator in the SSV network with the public key\n\n3. Input your Operator ID \ (right-click to paste)" 22 85 3>&1 1>&2 2>&3) if [[ -n "${__ssv_operator_id}" && ! "${__ssv_operator_id}" = "-1" ]]; then - sed -i'' "s|operatorID: .*|operatorID: ${__ssv_operator_id}|" ./ssv-config/dkg-config.yaml + sed -i'.original' "s|operatorID: .*|operatorID: ${__ssv_operator_id}|" ./ssv-config/dkg-config.yaml echo "Your SSV Operator ID is: ${__ssv_operator_id}" else echo "Please manually edit \"./ssv-config/dkg-config.yaml\" with your SSV Operator ID" echo "and add \":ssv-dkg.yml\" to \"COMPOSE_FILE\" in \".env\" after registering your operator." fi fi + rm -f ssv-config/dkg-config.yaml.original } set_value_in_env() { @@ -2913,15 +3067,16 @@ config() { ;; "ssv") if [ "${NETWORK}" = "holesky" ]; then - sed -i'' 's/ Network: .*/ Network: holesky/' ssv-config/config.yaml + sed -i'.original' 's/ Network: .*/ Network: holesky/' ssv-config/config.yaml elif [ "${NETWORK}" = "mainnet" ]; then - sed -i'' 's/ Network: .*/ Network: mainnet/' ssv-config/config.yaml + sed -i'.original' 's/ Network: .*/ Network: mainnet/' ssv-config/config.yaml else echo "${NETWORK} is not something that works with SSV." echo "Please choose Holešovice or Mainnet when running $__me config again" echo "Aborting." exit 1 fi + rm ssv-config/config.yaml.original if [ ! -f "./ssv-config/password.pass" ]; then echo "Creating password file for encrypted SSV secret key" head -c 16 /dev/urandom | base64 | tr -d '[:space:]' >./ssv-config/password.pass @@ -3033,6 +3188,24 @@ config() { var=NIM_DOCKERFILE set_value_in_env fi + if uname -m | grep -q riscv64; then +# We are using the variable +# shellcheck disable=SC2034 + NIM_DOCKERFILE=Dockerfile.source + var=NIM_DOCKERFILE + set_value_in_env +# We are using the variable +# shellcheck disable=SC2034 + GETH_DOCKERFILE=Dockerfile.source + var=GETH_DOCKERFILE + set_value_in_env + fi + var="SIREN_PASSWORD" + SIREN_PASSWORD=$(sed -n -e "s/^${var}=\(.*\)/\1/p" "${ENV_FILE}" || true) + if [ -z "${SIREN_PASSWORD}" ]; then + SIREN_PASSWORD=$(head -c 8 /dev/urandom | od -A n -t u8 | tr -d '[:space:]' | sha256sum | head -c 32) + set_value_in_env + fi ${__as_owner} rm .env.original @@ -3206,6 +3379,8 @@ __full_help() { echo " restarts the Nethermind execution client and prunes its DB." echo " prune-besu [--non-interactive]" echo " stops the Besu execution client and prunes trie-logs." + echo " prune-reth [--non-interactive]" + echo " stops the Reth execution client and prunes its DB." echo " prune-lighthouse [--non-interactive]" echo " stops the Lighthouse consensus client and prunes state." echo " resync-execution" @@ -3249,7 +3424,7 @@ __during_migrate=0 __migrated=0 __command="" __me=$(basename "${BASH_SOURCE[0]}") -if ! grep -q "alias ethd" ~/.profile; then +if [ ! -f ~/.profile ] || ! grep -q "alias ethd" ~/.profile; then __me="./$__me" fi @@ -3328,7 +3503,7 @@ fi case "$__command" in help|config|keys|update|up|start|down|stop|restart|version|logs|cmd|terminate|prune-nethermind\ - |prune-besu|prune-lighthouse|resync-execution|resync-consensus|attach-geth|keyimport|space) + |prune-besu|prune-reth|prune-lighthouse|resync-execution|resync-consensus|attach-geth|keyimport|space) $__command "$@";; *) echo "Unrecognized command $__command" diff --git a/flashbots/Dockerfile.binary b/flashbots/Dockerfile.binary index 811ef6a7..5f06427b 100644 --- a/flashbots/Dockerfile.binary +++ b/flashbots/Dockerfile.binary @@ -1,5 +1,5 @@ -ARG DOCKER_TAG -ARG DOCKER_REPO +ARG DOCKER_TAG=latest +ARG DOCKER_REPO=flashbots/mev-boost FROM ${DOCKER_REPO}:${DOCKER_TAG} # Unused, this is here to avoid build time complaints diff --git a/flashbots/Dockerfile.source b/flashbots/Dockerfile.source index 31da8bdf..5bce47be 100644 --- a/flashbots/Dockerfile.source +++ b/flashbots/Dockerfile.source @@ -1,5 +1,5 @@ # Build in a stock Go build container -FROM golang:1.22-alpine as builder +FROM golang:1.22-alpine AS builder # Unused, this is here to avoid build time complaints ARG DOCKER_TAG diff --git a/geth.yml b/geth.yml index 82188fa4..638f8b1c 100644 --- a/geth.yml +++ b/geth.yml @@ -81,6 +81,7 @@ services: - metrics.path=/debug/metrics/prometheus - metrics.port=6060 - metrics.instance=execution + - metrics.network=${NETWORK} volumes: geth-eth1-data: diff --git a/geth/Dockerfile.binary b/geth/Dockerfile.binary index baa7652a..9d2263e2 100644 --- a/geth/Dockerfile.binary +++ b/geth/Dockerfile.binary @@ -1,5 +1,5 @@ -ARG DOCKER_TAG -ARG DOCKER_REPO +ARG DOCKER_TAG=stable +ARG DOCKER_REPO=ethereum/client-go FROM ${DOCKER_REPO}:${DOCKER_TAG} diff --git a/geth/Dockerfile.source b/geth/Dockerfile.source index 73055e39..a9ead3c3 100644 --- a/geth/Dockerfile.source +++ b/geth/Dockerfile.source @@ -1,5 +1,5 @@ # Build Geth in a stock Go build container -FROM golang:1.22-alpine as builder +FROM golang:1.22-alpine AS builder # Unused, this is here to avoid build time complaints ARG DOCKER_TAG diff --git a/grafana-cloud.yml b/grafana-cloud.yml index 24201bb1..97f78b45 100644 --- a/grafana-cloud.yml +++ b/grafana-cloud.yml @@ -39,6 +39,7 @@ services: - metrics.path=/metrics - metrics.port=9090 - metrics.instance=prometheus + - metrics.network=${NETWORK} ethereum-metrics-exporter: restart: "unless-stopped" @@ -77,6 +78,7 @@ services: - metrics.path=/metrics - metrics.port=9100 - metrics.instance=node-exporter + - metrics.network=${NETWORK} blackbox-exporter: restart: "unless-stopped" @@ -103,6 +105,7 @@ services: - metrics.path=/metrics - metrics.port=7979 - metrics.instance=json-exporter + - metrics.network=${NETWORK} cadvisor: restart: "unless-stopped" @@ -122,6 +125,7 @@ services: - metrics.path=/metrics - metrics.port=8080 - metrics.instance=cadvisor + - metrics.network=${NETWORK} promtail: image: grafana/promtail:latest @@ -142,6 +146,7 @@ services: - metrics.path=/metrics - metrics.port=9080 - metrics.instance=promtail + - metrics.network=${NETWORK} volumes: prom-data: diff --git a/grafana.yml b/grafana.yml index 22c08ea9..a7a8bab4 100644 --- a/grafana.yml +++ b/grafana.yml @@ -31,6 +31,7 @@ services: - metrics.path=/metrics - metrics.port=9090 - metrics.instance=prometheus + - metrics.network=${NETWORK} ethereum-metrics-exporter: restart: "unless-stopped" @@ -45,6 +46,7 @@ services: - metrics.path=/metrics - metrics.port=9090 - metrics.instance=ethereum-metrics-exporter + - metrics.network=${NETWORK} node-exporter: image: prom/node-exporter:latest @@ -69,6 +71,7 @@ services: - metrics.path=/metrics - metrics.port=9100 - metrics.instance=node-exporter + - metrics.network=${NETWORK} blackbox-exporter: restart: "unless-stopped" @@ -95,6 +98,7 @@ services: - metrics.path=/metrics - metrics.port=7979 - metrics.instance=json-exporter + - metrics.network=${NETWORK} cadvisor: restart: "unless-stopped" @@ -114,6 +118,7 @@ services: - metrics.path=/metrics - metrics.port=8080 - metrics.instance=cadvisor + - metrics.network=${NETWORK} promtail: image: grafana/promtail:latest @@ -136,6 +141,7 @@ services: - metrics.path=/metrics - metrics.port=9080 - metrics.instance=promtail + - metrics.network=${NETWORK} loki: image: grafana/loki:latest @@ -151,6 +157,7 @@ services: - metrics.path=/metrics - metrics.port=3100 - metrics.instance=loki + - metrics.network=${NETWORK} grafana: restart: "unless-stopped" @@ -179,6 +186,7 @@ services: - metrics.path=/metrics - metrics.port=3000 - metrics.instance=grafana + - metrics.network=${NETWORK} volumes: grafana-data: diff --git a/grafana/provision.sh b/grafana/provision.sh index 4938af08..65be2f84 100755 --- a/grafana/provision.sh +++ b/grafana/provision.sh @@ -31,11 +31,11 @@ case "$CLIENT" in # lighthouse_validator_client __url='https://raw.githubusercontent.com/sigp/lighthouse-metrics/master/dashboards/ValidatorClient.json' __file='/etc/grafana/provisioning/dashboards/lighthouse_validator_client.json' - wget -t 3 -T 10 -qcO - "${__url}" | jq '.title = "Lighthouse Validator Client"' >"${__file}" + wget -t 3 -T 10 -qcO - "${__url}" | jq '.title = "Lighthouse Validator Client"' | jq 'walk(if . == "${DS_PROMETHEUS}" then "Prometheus" else . end)' >"${__file}" # lighthouse_validator_monitor __url='https://raw.githubusercontent.com/sigp/lighthouse-metrics/master/dashboards/ValidatorMonitor.json' __file='/etc/grafana/provisioning/dashboards/lighthouse_validator_monitor.json' - wget -t 3 -T 10 -qcO - "${__url}" | jq '.title = "Lighthouse Validator Monitor"' >"${__file}" + wget -t 3 -T 10 -qcO - "${__url}" | jq '.title = "Lighthouse Validator Monitor"' | jq 'walk(if . == "${DS_PROMETHEUS}" then "Prometheus" else . end)' >"${__file}" ;;& *teku* ) # teku_overview diff --git a/grandine-allin1.yml b/grandine-allin1.yml index 438decfc..6ed7712c 100644 --- a/grandine-allin1.yml +++ b/grandine-allin1.yml @@ -105,6 +105,7 @@ services: - metrics.path=/metrics - metrics.port=8008 - metrics.instance=consensus + - metrics.network=${NETWORK} wipe-db: profiles: ["tools"] diff --git a/grandine-cl-only.yml b/grandine-cl-only.yml index a76e2040..57d73716 100644 --- a/grandine-cl-only.yml +++ b/grandine-cl-only.yml @@ -94,6 +94,7 @@ services: - metrics.path=/metrics - metrics.port=8008 - metrics.instance=consensus + - metrics.network=${NETWORK} wipe-db: profiles: ["tools"] diff --git a/grandine/Dockerfile.binary b/grandine/Dockerfile.binary index bc026f03..d261160f 100644 --- a/grandine/Dockerfile.binary +++ b/grandine/Dockerfile.binary @@ -1,5 +1,5 @@ -ARG DOCKER_TAG -ARG DOCKER_REPO +ARG DOCKER_TAG=stable +ARG DOCKER_REPO=sifrai/grandine FROM ${DOCKER_REPO}:${DOCKER_TAG} diff --git a/grandine/Dockerfile.source b/grandine/Dockerfile.source index a67f6e2f..ee98df78 100644 --- a/grandine/Dockerfile.source +++ b/grandine/Dockerfile.source @@ -1,5 +1,5 @@ # Build Grandine in a stock Rust build container -FROM rust:bookworm as builder +FROM rust:bookworm AS builder # Unused, this is here to avoid build time complaints ARG DOCKER_TAG diff --git a/grandine/docker-entrypoint.sh b/grandine/docker-entrypoint.sh index dc83241c..8aafac91 100755 --- a/grandine/docker-entrypoint.sh +++ b/grandine/docker-entrypoint.sh @@ -12,7 +12,7 @@ if [ -n "${JWT_SECRET}" ]; then fi if [[ -O "/var/lib/grandine/ee-secret" ]]; then - # In case someone specificies JWT_SECRET but it's not a distributed setup + # In case someone specifies JWT_SECRET but it's not a distributed setup chmod 777 /var/lib/grandine/ee-secret fi if [[ -O "/var/lib/grandine/ee-secret/jwtsecret" ]]; then @@ -52,7 +52,7 @@ if [ "${ARCHIVE_NODE}" = "true" ]; then echo "Grandine archive node without pruning" __prune="--back-sync" else - __prune="" + __prune="--prune-storage" fi # Check whether we should rapid sync @@ -73,7 +73,7 @@ fi # Check whether we should use MEV Boost if [ "${MEV_BOOST}" = "true" ]; then - __mev_boost="--builder-api-url ${MEV_NODE:-http://mev-boost:18550}" + __mev_boost="--builder-url ${MEV_NODE:-http://mev-boost:18550}" echo "MEV Boost enabled" else __mev_boost="" diff --git a/lighthouse-cl-only.yml b/lighthouse-cl-only.yml index b2c71d63..11dcd912 100644 --- a/lighthouse-cl-only.yml +++ b/lighthouse-cl-only.yml @@ -86,6 +86,7 @@ services: - metrics.path=/metrics - metrics.port=8008 - metrics.instance=consensus + - metrics.network=${NETWORK} set-cl-prune-marker: profiles: ["tools"] diff --git a/lighthouse-vc-only.yml b/lighthouse-vc-only.yml index 6e7e0b1a..172c0292 100644 --- a/lighthouse-vc-only.yml +++ b/lighthouse-vc-only.yml @@ -77,6 +77,7 @@ services: - metrics.path=/metrics - metrics.port=8009 - metrics.instance=validator + - metrics.network=${NETWORK} validator-exit: profiles: ["tools"] diff --git a/lighthouse.yml b/lighthouse.yml index 87696e12..cc8b7662 100644 --- a/lighthouse.yml +++ b/lighthouse.yml @@ -90,6 +90,7 @@ services: - metrics.path=/metrics - metrics.port=8008 - metrics.instance=consensus + - metrics.network=${NETWORK} validator: restart: "unless-stopped" @@ -154,6 +155,7 @@ services: - metrics.path=/metrics - metrics.port=8009 - metrics.instance=validator + - metrics.network=${NETWORK} validator-exit: profiles: ["tools"] diff --git a/lighthouse/Dockerfile.binary b/lighthouse/Dockerfile.binary index dd668e94..97cebb05 100644 --- a/lighthouse/Dockerfile.binary +++ b/lighthouse/Dockerfile.binary @@ -1,5 +1,5 @@ -ARG DOCKER_TAG -ARG DOCKER_REPO +ARG DOCKER_TAG=latest +ARG DOCKER_REPO=sigp/lighthouse FROM ${DOCKER_REPO}:${DOCKER_TAG} diff --git a/lighthouse/Dockerfile.source b/lighthouse/Dockerfile.source index d8f0eefd..69e1b0ac 100644 --- a/lighthouse/Dockerfile.source +++ b/lighthouse/Dockerfile.source @@ -1,5 +1,5 @@ # Build Lighthouse in a stock Rust build container -FROM rust:bookworm as builder +FROM rust:bookworm AS builder # Unused, this is here to avoid build time complaints ARG DOCKER_TAG diff --git a/lodestar-cl-only.yml b/lodestar-cl-only.yml index 7f27c2b7..9509fdeb 100644 --- a/lodestar-cl-only.yml +++ b/lodestar-cl-only.yml @@ -85,6 +85,7 @@ services: - metrics.path=/metrics - metrics.port=8008 - metrics.instance=consensus + - metrics.network=${NETWORK} volumes: lsconsensus-data: diff --git a/lodestar-vc-only.yml b/lodestar-vc-only.yml index e46b541a..53adb2f4 100644 --- a/lodestar-vc-only.yml +++ b/lodestar-vc-only.yml @@ -71,6 +71,7 @@ services: - metrics.path=/metrics - metrics.port=8009 - metrics.instance=validator + - metrics.network=${NETWORK} validator-exit: profiles: ["tools"] diff --git a/lodestar.yml b/lodestar.yml index c0336713..e24aa32f 100644 --- a/lodestar.yml +++ b/lodestar.yml @@ -61,8 +61,6 @@ services: - 0.0.0.0 - --rest.port - ${CL_REST_PORT:-5052} - - --rest.namespace - - "beacon,config,events,node,validator,lightclient,debug" - --port - ${CL_P2P_PORT:-9000} - --nat @@ -87,6 +85,7 @@ services: - metrics.path=/metrics - metrics.port=8008 - metrics.instance=consensus + - metrics.network=${NETWORK} validator: restart: "unless-stopped" @@ -125,6 +124,8 @@ services: - /var/lib/lodestar/validators - --beaconNodes - ${CL_NODE} + - --http.requestWireFormat + - "ssz" - --keymanager - --keymanager.address - 0.0.0.0 @@ -145,6 +146,7 @@ services: - metrics.path=/metrics - metrics.port=8009 - metrics.instance=validator + - metrics.network=${NETWORK} validator-exit: profiles: ["tools"] diff --git a/lodestar/Dockerfile.binary b/lodestar/Dockerfile.binary index b8da9b6d..2323f00a 100644 --- a/lodestar/Dockerfile.binary +++ b/lodestar/Dockerfile.binary @@ -1,5 +1,5 @@ -ARG DOCKER_TAG -ARG DOCKER_REPO +ARG DOCKER_TAG=latest +ARG DOCKER_REPO=chainsafe/lodestar FROM ${DOCKER_REPO}:${DOCKER_TAG} @@ -7,7 +7,7 @@ FROM ${DOCKER_REPO}:${DOCKER_TAG} ARG BUILD_TARGET ARG SRC_REPO -RUN apk update && apk add --no-cache ca-certificates tzdata bash su-exec git && rm -rf /var/cache/apk/* +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates tzdata bash gosu git && apt-get clean && rm -rf /var/lib/apt/lists/* ARG USER=lsconsensus ARG UID=10002 diff --git a/lodestar/Dockerfile.source b/lodestar/Dockerfile.source index 9f570f47..dc5a2f3f 100644 --- a/lodestar/Dockerfile.source +++ b/lodestar/Dockerfile.source @@ -1,4 +1,4 @@ -FROM node:20-alpine as build +FROM node:22.4-slim AS builder # Here only to avoid build-time errors ARG DOCKER_TAG @@ -7,16 +7,18 @@ ARG DOCKER_REPO ARG BUILD_TARGET ARG SRC_REPO -RUN apk update && apk add --no-cache git g++ make python3 bash && rm -rf /var/cache/apk/* -#RUN ln -s /usr/bin/python3 /usr/bin/python +RUN apt-get update && apt-get install -y --no-install-recommends git g++ make python3 python3-setuptools bash && apt-get clean && rm -rf /var/lib/apt/lists/* WORKDIR /usr/app -RUN bash -c "cd .. && rm -rf app && git clone ${SRC_REPO} app && cd app && git config advice.detachedHead false && git fetch --all --tags && if [[ ${BUILD_TARGET} =~ pr-.+ ]]; then git fetch origin pull/$(echo ${BUILD_TARGET} | cut -d '-' -f 2)/head:ls-pr; git checkout ls-pr; else git checkout ${BUILD_TARGET}; fi && yarn install --non-interactive --frozen-lockfile && yarn build" +RUN bash -c "cd .. && rm -rf app && git clone ${SRC_REPO} app && cd app && git config advice.detachedHead false \ + && git fetch --all --tags \ + && if [[ ${BUILD_TARGET} =~ pr-.+ ]]; then git fetch origin pull/$(echo ${BUILD_TARGET} | cut -d '-' -f 2)/head:ls-pr; git checkout ls-pr; else git checkout ${BUILD_TARGET}; fi \ + && yarn install --non-interactive --frozen-lockfile && yarn build" -FROM node:20-alpine +FROM node:22.4-slim -RUN apk update && apk add --no-cache ca-certificates tzdata bash su-exec git && rm -rf /var/cache/apk/* +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates tzdata bash gosu git && apt-get clean && rm -rf /var/lib/apt/lists/* ARG USER=lsconsensus ARG UID=10002 @@ -34,7 +36,7 @@ RUN adduser \ RUN mkdir -p /var/lib/lodestar/consensus/ee-secret && chown -R ${USER}:${USER} /var/lib/lodestar/consensus && chmod -R 700 /var/lib/lodestar/consensus && chmod 777 /var/lib/lodestar/consensus/ee-secret WORKDIR /usr/app -COPY --from=build --chown=${USER}:${USER} /usr/app . +COPY --from=builder --chown=${USER}:${USER} /usr/app . RUN ln -s ../../packages/cli/lib/index.js node_modules/.bin/lodestar # Cannot assume buildkit, hence no chmod diff --git a/lodestar/docker-entrypoint-vc.sh b/lodestar/docker-entrypoint-vc.sh index b204b340..4814e71b 100755 --- a/lodestar/docker-entrypoint-vc.sh +++ b/lodestar/docker-entrypoint-vc.sh @@ -3,7 +3,7 @@ set -Eeuo pipefail if [ "$(id -u)" = '0' ]; then chown -R lsvalidator:lsvalidator /var/lib/lodestar - exec su-exec lsvalidator docker-entrypoint.sh "$@" + exec gosu lsvalidator docker-entrypoint.sh "$@" fi if [[ "${NETWORK}" =~ ^https?:// ]]; then diff --git a/lodestar/docker-entrypoint.sh b/lodestar/docker-entrypoint.sh index 23834cb3..2b2b915f 100755 --- a/lodestar/docker-entrypoint.sh +++ b/lodestar/docker-entrypoint.sh @@ -2,7 +2,7 @@ if [ "$(id -u)" = '0' ]; then chown -R lsconsensus:lsconsensus /var/lib/lodestar - exec su-exec lsconsensus docker-entrypoint.sh "$@" + exec gosu lsconsensus docker-entrypoint.sh "$@" fi # Remove old low-entropy token, related to Sigma Prime security audit diff --git a/nethermind.yml b/nethermind.yml index ecc317a1..f85d92b7 100644 --- a/nethermind.yml +++ b/nethermind.yml @@ -89,6 +89,7 @@ services: - metrics.path=/metrics - metrics.port=6060 - metrics.instance=execution + - metrics.network=${NETWORK} volumes: nm-eth1-data: diff --git a/nethermind/Dockerfile.binary b/nethermind/Dockerfile.binary index 6bc6b979..24bad15d 100644 --- a/nethermind/Dockerfile.binary +++ b/nethermind/Dockerfile.binary @@ -1,5 +1,5 @@ -ARG DOCKER_TAG -ARG DOCKER_REPO +ARG DOCKER_TAG=latest +ARG DOCKER_REPO=nethermind/nethermind FROM ${DOCKER_REPO}:${DOCKER_TAG} @@ -12,6 +12,13 @@ ARG UID=10001 # GID 10002 is deliberate so it can exchange secret with CL ARG GID=10002 +RUN set -eux; \ + apt-get update; \ + DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y ca-certificates gosu tzdata wget git adduser; \ + rm -rf /var/lib/apt/lists/*; \ +# verify that the binary works + gosu nobody true + RUN addgroup \ --gid "${GID}" \ "${USER}" @@ -27,13 +34,6 @@ RUN adduser \ --ingroup "${USER}" \ "${USER}" -RUN set -eux; \ - apt-get update; \ - DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y ca-certificates gosu tzdata wget git; \ - rm -rf /var/lib/apt/lists/*; \ -# verify that the binary works - gosu nobody true - # This only goes so far. keystore, logs and nethermind_db are volumes and need to be chown'd in the entrypoint RUN chown -R ${USER}:${USER} /nethermind RUN mkdir -p /var/lib/nethermind/ee-secret && chown -R ${USER}:${USER} /var/lib/nethermind \ diff --git a/nethermind/Dockerfile.source b/nethermind/Dockerfile.source index 11e064e0..d86f92be 100644 --- a/nethermind/Dockerfile.source +++ b/nethermind/Dockerfile.source @@ -1,5 +1,5 @@ # Partially from Nethermind github -FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build +FROM mcr.microsoft.com/dotnet/sdk:8.0 AS builder # Unused, this is here to avoid build time complaints ARG DOCKER_TAG ARG DOCKER_REPO @@ -46,7 +46,7 @@ RUN adduser \ WORKDIR /nethermind # Cannot assume buildkit, hence no chmod -COPY --from=build --chown=${USER}:${USER} /nethermind/out . +COPY --from=builder --chown=${USER}:${USER} /nethermind/out . RUN chown -R ${USER}:${USER} /nethermind RUN mkdir -p /var/lib/nethermind/ee-secret && chown -R ${USER}:${USER} /var/lib/nethermind \ diff --git a/nethermind/docker-entrypoint.sh b/nethermind/docker-entrypoint.sh index bef68cca..a8bf6533 100755 --- a/nethermind/docker-entrypoint.sh +++ b/nethermind/docker-entrypoint.sh @@ -53,7 +53,7 @@ if [[ "${NETWORK}" =~ ^https?:// ]]; then __network="--config none.cfg --Init.ChainSpecPath=/var/lib/nethermind/testnet/${config_dir}/chainspec.json --Discovery.Bootnodes=${bootnodes} \ --JsonRpc.EnabledModules=Eth,Subscribe,Trace,TxPool,Web3,Personal,Proof,Net,Parity,Health,Rpc,Debug,Admin --Pruning.Mode=None --Init.IsMining=false" else - __network="--config ${NETWORK} --JsonRpc.EnabledModules Web3,Eth,Subscribe,Net,Health,Parity,Proof,Trace,TxPool" + __network="--config ${NETWORK}" fi __memtotal=$(awk '/MemTotal/ {printf "%d", int($2/1024/1024)}' /proc/meminfo) @@ -76,8 +76,7 @@ else fi if [ "${__memtotal}" -ge 30 ]; then # Investigating instability - #__prune="${__prune} --Pruning.CacheMb=4096 --Pruning.FullPruningMemoryBudgetMb=16384 --Init.StateDbKeyScheme=HalfPath" - __prune="${__prune} --Pruning.FullPruningMemoryBudgetMb=16384 --Init.StateDbKeyScheme=HalfPath" + __prune="${__prune} --Pruning.CacheMb=4096 --Pruning.FullPruningMemoryBudgetMb=16384 --Init.StateDbKeyScheme=HalfPath" fi echo "Using pruning parameters:" echo "${__prune}" diff --git a/nimbus-allin1.yml b/nimbus-allin1.yml index cbc6e73a..65ce944d 100644 --- a/nimbus-allin1.yml +++ b/nimbus-allin1.yml @@ -84,6 +84,7 @@ services: - metrics.path=/metrics - metrics.port=8008 - metrics.instance=consensus + - metrics.network=${NETWORK} wipe-db: profiles: ["tools"] diff --git a/nimbus-cl-only.yml b/nimbus-cl-only.yml index 556165f2..d17137ac 100644 --- a/nimbus-cl-only.yml +++ b/nimbus-cl-only.yml @@ -80,6 +80,7 @@ services: - metrics.path=/metrics - metrics.port=8008 - metrics.instance=consensus + - metrics.network=${NETWORK} # Uses keystore-m file and CL, so does belong here validator-exit: diff --git a/nimbus-el.yml b/nimbus-el.yml index 1a137490..49208d29 100644 --- a/nimbus-el.yml +++ b/nimbus-el.yml @@ -42,14 +42,11 @@ services: - docker-entrypoint.sh - nimbus - --rpc - - --rpc-address=0.0.0.0 - - --rpc-port=${EL_RPC_PORT:-8545} + - --http-address=0.0.0.0 + - --http-port=${EL_RPC_PORT:-8545} - --data-dir=/var/lib/nimbus - - --key-store=/var/lib/nimbus/nimbus/keystore - --tcp-port=${EL_P2P_PORT:-30303} - --ws - - --ws-address=0.0.0.0 - - --ws-port=${EL_WS_PORT:-8546} - --network=${NETWORK} - --metrics - --metrics-port=6060 @@ -58,6 +55,7 @@ services: - --engine-api - --engine-api-port=${EE_PORT:-8551} - --engine-api-address=0.0.0.0 + - --engine-api-ws - --max-peers=${EL_MAX_PEER_COUNT:-25} - --log-level=${LOG_LEVEL} labels: diff --git a/nimbus-el/Dockerfile.source b/nimbus-el/Dockerfile.source index ac89c735..d99351cd 100644 --- a/nimbus-el/Dockerfile.source +++ b/nimbus-el/Dockerfile.source @@ -1,5 +1,5 @@ # Build Nimbus in a stock debian container -FROM debian:bookworm-slim as builder +FROM debian:bookworm-slim AS builder # Included here to avoid build-time complaints ARG DOCKER_TAG @@ -8,7 +8,7 @@ ARG DOCKER_REPO ARG BUILD_TARGET ARG SRC_REPO -RUN apt-get update && apt-get install -y build-essential git ca-certificates librocksdb-dev +RUN apt-get update && apt-get install -y build-essential git ca-certificates librocksdb-dev curl WORKDIR /usr/src diff --git a/nimbus-vc-only.yml b/nimbus-vc-only.yml index 5dda319e..b5e12ca8 100644 --- a/nimbus-vc-only.yml +++ b/nimbus-vc-only.yml @@ -63,6 +63,7 @@ services: - metrics.path=/metrics - metrics.port=8009 - metrics.instance=validator + - metrics.network=${NETWORK} validator-exit: profiles: ["tools"] diff --git a/nimbus.yml b/nimbus.yml index f97241b7..dabbf36f 100644 --- a/nimbus.yml +++ b/nimbus.yml @@ -80,6 +80,7 @@ services: - metrics.path=/metrics - metrics.port=8008 - metrics.instance=consensus + - metrics.network=${NETWORK} validator: restart: "unless-stopped" @@ -126,6 +127,7 @@ services: - metrics.path=/metrics - metrics.port=8009 - metrics.instance=validator + - metrics.network=${NETWORK} validator-exit: profiles: ["tools"] diff --git a/nimbus/Dockerfile.binary b/nimbus/Dockerfile.binary index 6489f5d0..1432f1a8 100644 --- a/nimbus/Dockerfile.binary +++ b/nimbus/Dockerfile.binary @@ -1,9 +1,9 @@ -ARG DOCKER_TAG -ARG DOCKER_VC_TAG -ARG DOCKER_REPO -ARG DOCKER_VC_REPO +ARG DOCKER_TAG=multiarch-latest +ARG DOCKER_VC_TAG=multiarch-latest +ARG DOCKER_REPO=statusim/nimbus-eth2 +ARG DOCKER_VC_REPO=statusim/nimbus-validator-client -FROM ${DOCKER_REPO}:${DOCKER_TAG} as consensus +FROM ${DOCKER_REPO}:${DOCKER_TAG} AS consensus # Included here to avoid build-time complaints ARG BUILD_TARGET @@ -38,7 +38,7 @@ USER ${USER} ENTRYPOINT ["nimbus_beacon_node"] -FROM ${DOCKER_VC_REPO}:${DOCKER_VC_TAG} as validator +FROM ${DOCKER_VC_REPO}:${DOCKER_VC_TAG} AS validator # Included here to avoid build-time complaints ARG BUILD_TARGET diff --git a/nimbus/Dockerfile.source b/nimbus/Dockerfile.source index f61a907c..dbf8af7e 100644 --- a/nimbus/Dockerfile.source +++ b/nimbus/Dockerfile.source @@ -1,5 +1,5 @@ -# Build Nimbus in a stock debian container -FROM debian:bookworm-slim as builder +# Build Nimbus in a stock alpine container +FROM alpine:3 AS builder # Included here to avoid build-time complaints ARG DOCKER_TAG @@ -10,28 +10,19 @@ ARG DOCKER_VC_REPO ARG BUILD_TARGET ARG SRC_REPO -RUN apt-get update && apt-get install -y build-essential git ca-certificates +RUN apk update && apk add --no-cache make gcc musl-dev linux-headers git bash git-lfs nim WORKDIR /usr/src -RUN bash -c "git clone --recurse-submodules -j8 ${SRC_REPO} nimbus-eth2 && cd nimbus-eth2 && git config advice.detachedHead false && git fetch --all --tags && if [[ ${BUILD_TARGET} =~ pr-.+ ]]; then git fetch origin pull/$(echo ${BUILD_TARGET} | cut -d '-' -f 2)/head:nim-pr; git checkout nim-pr; else git checkout ${BUILD_TARGET}; fi && make -j$(nproc) update && make -j$(nproc) nimbus_beacon_node nimbus_validator_client" +RUN bash -c "git clone --recurse-submodules -j8 ${SRC_REPO} nimbus-eth2 && cd nimbus-eth2 && git config advice.detachedHead false && git fetch --all --tags && if [[ ${BUILD_TARGET} =~ pr-.+ ]]; then git fetch origin pull/$(echo ${BUILD_TARGET} | cut -d '-' -f 2)/head:nim-pr; git checkout nim-pr; else git checkout ${BUILD_TARGET}; fi && make USE_SYSTEM_NIM=1 -j$(nproc) update && make USE_SYSTEM_NIM=1 -j$(nproc) nimbus_beacon_node nimbus_validator_client" # Pull all binaries into a second stage deploy debian container -FROM debian:bookworm-slim as consensus +FROM alpine:3 AS consensus ARG USER=user ARG UID=10002 -RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y --no-install-recommends \ - ca-certificates bash tzdata git curl \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -RUN set -eux; \ - apt-get update; \ - apt-get install -y gosu; \ - rm -rf /var/lib/apt/lists/*; \ -# verify that the binary works - gosu nobody true +RUN apk update && apk add \ + ca-certificates bash tzdata git curl su-exec gcc # See https://stackoverflow.com/a/55757473/12429735RUN RUN adduser \ @@ -56,15 +47,13 @@ USER ${USER} ENTRYPOINT ["nimbus_beacon_node"] -FROM debian:bookworm-slim as validator +FROM alpine:3 AS validator ARG USER=user ARG UID=10000 -RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y --no-install-recommends \ - ca-certificates bash tzdata curl \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* +RUN apk update && apk add \ + ca-certificates bash tzdata curl su-exec gcc # See https://stackoverflow.com/a/55757473/12429735RUN RUN adduser \ diff --git a/nimbus/Dockerfile.sourcegnosis b/nimbus/Dockerfile.sourcegnosis index 92b2cec4..28db9057 100644 --- a/nimbus/Dockerfile.sourcegnosis +++ b/nimbus/Dockerfile.sourcegnosis @@ -1,5 +1,5 @@ # Build Nimbus in a stock debian container -FROM debian:bookworm-slim as builder +FROM debian:bookworm-slim AS builder # Included here to avoid build-time complaints ARG DOCKER_TAG @@ -16,7 +16,7 @@ WORKDIR /usr/src RUN bash -c "git clone --recurse-submodules -j8 ${SRC_REPO} nimbus-eth2 && cd nimbus-eth2 && git config advice.detachedHead false && git fetch --all --tags && if [[ ${BUILD_TARGET} =~ pr-.+ ]]; then git fetch origin pull/$(echo ${BUILD_TARGET} | cut -d '-' -f 2)/head:nim-pr; git checkout nim-pr; else git checkout ${BUILD_TARGET}; fi && make -j$(nproc) update && make -j$(nproc) gnosis-build nimbus_validator_client" # Pull all binaries into a second stage deploy debian container -FROM debian:bookworm-slim as consensus +FROM debian:bookworm-slim AS consensus ARG USER=user ARG UID=10002 @@ -55,7 +55,7 @@ USER ${USER} ENTRYPOINT ["nimbus_beacon_node"] -FROM debian:bookworm-slim as validator +FROM debian:bookworm-slim AS validator ARG USER=user ARG UID=10000 diff --git a/nimbus/Dockerfile.sourceslottime b/nimbus/Dockerfile.sourceslottime index 6cadbac2..09064f4e 100644 --- a/nimbus/Dockerfile.sourceslottime +++ b/nimbus/Dockerfile.sourceslottime @@ -1,5 +1,5 @@ # Build Nimbus in a stock debian container -FROM debian:bookworm-slim as builder +FROM debian:bookworm-slim AS builder # Included here to avoid build-time complaints ARG DOCKER_TAG diff --git a/nimbus/docker-entrypoint-vc.sh b/nimbus/docker-entrypoint-vc.sh index d9671712..d9f6ae5a 100755 --- a/nimbus/docker-entrypoint-vc.sh +++ b/nimbus/docker-entrypoint-vc.sh @@ -2,7 +2,7 @@ if [ "$(id -u)" = '0' ]; then chown -R user:user /var/lib/nimbus - exec gosu user docker-entrypoint-vc.sh "$@" + exec su-exec user docker-entrypoint-vc.sh "$@" fi # Remove old low-entropy token, related to Sigma Prime security audit diff --git a/nimbus/docker-entrypoint.sh b/nimbus/docker-entrypoint.sh index 2f9b6a02..8221b627 100755 --- a/nimbus/docker-entrypoint.sh +++ b/nimbus/docker-entrypoint.sh @@ -2,7 +2,12 @@ if [ "$(id -u)" = '0' ]; then chown -R user:user /var/lib/nimbus - exec gosu user docker-entrypoint.sh "$@" + if command -v gosu &>/dev/null; then + __as_user=gosu + else + __as_user=su-exec + fi + exec ${__as_user} user docker-entrypoint.sh "$@" fi # Remove old low-entropy token, related to Sigma Prime security audit diff --git a/prometheus/base-config.yml b/prometheus/base-config.yml index 02aaeb12..54a5ed8c 100644 --- a/prometheus/base-config.yml +++ b/prometheus/base-config.yml @@ -5,7 +5,7 @@ # # If you need to override the settings in this file, or add other top-level configuration # (e.g.) remote_write config, please use custom-prom.yml. -# The modifications in that file will supercede the settings here (or extend, in the case of lists) +# The modifications in that file will supersede the settings here (or extend, in the case of lists) global: scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. @@ -58,6 +58,8 @@ scrape_configs: source_labels: - __meta_docker_container_label_metrics_instance target_label: instance + - source_labels: [__meta_docker_container_label_metrics_network] + target_label: network scrape_config_files: - /etc/prometheus/conf.d/*.yml diff --git a/prometheus/rootless-base-config.yml b/prometheus/rootless-base-config.yml index 34ec0c2c..99be522e 100644 --- a/prometheus/rootless-base-config.yml +++ b/prometheus/rootless-base-config.yml @@ -5,7 +5,7 @@ # # If you need to override the settings in this file, or add other top-level configuration # (e.g.) remote_write config, please use custom-prom.yml. -# The modifications in that file will supercede the settings here (or extend, in the case of lists) +# The modifications in that file will supersede the settings here (or extend, in the case of lists) # This config differs from the regular one, as there is no access to the Docker socket itself # and the scrape configs get added manually diff --git a/prysm-cl-only.yml b/prysm-cl-only.yml index 7def716a..1d12996a 100644 --- a/prysm-cl-only.yml +++ b/prysm-cl-only.yml @@ -42,6 +42,7 @@ services: ports: - ${HOST_IP:-}:${PRYSM_PORT}:${PRYSM_PORT}/tcp - ${HOST_IP:-}:${PRYSM_UDP_PORT}:${PRYSM_UDP_PORT}/udp + - ${HOST_IP:-}:${CL_QUIC_PORT:-9001}:${CL_QUIC_PORT:-9001}/udp networks: default: aliases: @@ -72,6 +73,9 @@ services: - ${PRYSM_PORT} - --p2p-udp-port - ${PRYSM_UDP_PORT} + - --p2p-quic-port + - ${CL_QUIC_PORT} + - --enable-quic - --p2p-max-peers - ${CL_MAX_PEER_COUNT:-70} - --verbosity @@ -89,6 +93,7 @@ services: - metrics.path=/metrics - metrics.port=8008 - metrics.instance=consensus + - metrics.network=${NETWORK} volumes: prysmconsensus-data: diff --git a/prysm-vc-only.yml b/prysm-vc-only.yml index 2e5d7d95..e17fbcec 100644 --- a/prysm-vc-only.yml +++ b/prysm-vc-only.yml @@ -49,6 +49,8 @@ services: - /var/lib/prysm - --wallet-dir - /var/lib/prysm/ + - --keymanager-token-file + - /var/lib/prysm/auth-token - --beacon-rpc-provider - ${CL_NODE:-http://consensus:4000} - --verbosity @@ -80,6 +82,7 @@ services: - metrics.path=/metrics - metrics.port=8009 - metrics.instance=validator + - metrics.network=${NETWORK} create-wallet: profiles: ["tools"] diff --git a/prysm.yml b/prysm.yml index fffc2b43..01a8b3af 100644 --- a/prysm.yml +++ b/prysm.yml @@ -42,6 +42,7 @@ services: ports: - ${HOST_IP:-}:${PRYSM_PORT}:${PRYSM_PORT}/tcp - ${HOST_IP:-}:${PRYSM_UDP_PORT}:${PRYSM_UDP_PORT}/udp + - ${HOST_IP:-}:${CL_QUIC_PORT:-9001}:${CL_QUIC_PORT:-9001}/udp networks: default: aliases: @@ -73,6 +74,9 @@ services: - ${PRYSM_PORT} - --p2p-udp-port - ${PRYSM_UDP_PORT} + - --p2p-quic-port + - ${CL_QUIC_PORT} + - --enable-quic - --p2p-max-peers - ${CL_MAX_PEER_COUNT:-70} - --verbosity @@ -90,6 +94,7 @@ services: - metrics.path=/metrics - metrics.port=8008 - metrics.instance=consensus + - metrics.network=${NETWORK} validator: restart: "unless-stopped" @@ -122,6 +127,8 @@ services: - /var/lib/prysm - --wallet-dir - /var/lib/prysm/ + - --keymanager-token-file + - /var/lib/prysm/auth-token - --beacon-rpc-provider - consensus:4000 - --verbosity @@ -155,6 +162,7 @@ services: - metrics.path=/metrics - metrics.port=8009 - metrics.instance=validator + - metrics.network=${NETWORK} create-wallet: profiles: ["tools"] diff --git a/prysm/Dockerfile.binary b/prysm/Dockerfile.binary index d1f04885..371330c7 100644 --- a/prysm/Dockerfile.binary +++ b/prysm/Dockerfile.binary @@ -1,11 +1,11 @@ -ARG DOCKER_TAG -ARG DOCKER_VC_TAG -ARG DOCKER_REPO -ARG DOCKER_VC_REPO +ARG DOCKER_TAG=stable +ARG DOCKER_VC_TAG=stable +ARG DOCKER_REPO=gcr.io/prysmaticlabs/prysm/beacon-chain +ARG DOCKER_VC_REPO=gcr.io/prysmaticlabs/prysm/validator -FROM ${DOCKER_REPO}:${DOCKER_TAG} as clsource +FROM ${DOCKER_REPO}:${DOCKER_TAG} AS clsource -FROM debian:bookworm-slim as consensus +FROM debian:bookworm-slim AS consensus #Included here to avoid build-time complaints ARG BUILD_TARGET @@ -51,9 +51,9 @@ USER ${USER} ENTRYPOINT ["beacon-chain"] -FROM ${DOCKER_VC_REPO}:${DOCKER_VC_TAG} as vcsource +FROM ${DOCKER_VC_REPO}:${DOCKER_VC_TAG} AS vcsource -FROM consensus as validator +FROM consensus AS validator ARG USER=prysmvalidator ARG UID=10000 diff --git a/prysm/Dockerfile.source b/prysm/Dockerfile.source index 5885debb..b693481d 100644 --- a/prysm/Dockerfile.source +++ b/prysm/Dockerfile.source @@ -1,5 +1,5 @@ # Build Prysm in a stock Go build container -FROM golang:1.22-bookworm as builder +FROM golang:1.22-bookworm AS builder # Here only to avoid build-time errors ARG DOCKER_TAG @@ -19,7 +19,7 @@ if [[ ${BUILD_TARGET} =~ pr-.+ ]]; then git fetch origin pull/$(echo ${BUILD_TAR && bazel build --config=release //cmd/client-stats:client-stats" # Pull all binaries into a second stage deploy debian container -FROM debian:bookworm-slim as consensus +FROM debian:bookworm-slim AS consensus ARG USER=prysmconsensus ARG UID=10002 @@ -63,7 +63,7 @@ USER ${USER} ENTRYPOINT ["beacon-chain"] -FROM consensus as validator +FROM consensus AS validator ARG USER=prysmvalidator ARG UID=10000 diff --git a/prysm/docker-entrypoint.sh b/prysm/docker-entrypoint.sh index cef6342a..204993a5 100755 --- a/prysm/docker-entrypoint.sh +++ b/prysm/docker-entrypoint.sh @@ -68,21 +68,11 @@ else __prune="" fi -# Fetch genesis file as needed -if [[ "${NETWORK}" = "goerli" || "${NETWORK}" = "prater" ]]; then - GENESIS=/var/lib/prysm/genesis.ssz - if [ ! -f "$GENESIS" ]; then - echo "Fetching genesis file for Görli testnet" - curl -fsSL -o "$GENESIS" https://github.com/eth-clients/goerli/raw/main/prater/genesis.ssz - fi -# Word splitting is desired for the command line parameters -# shellcheck disable=SC2086 - exec "$@" "--genesis-state=$GENESIS" ${__network} ${__rapid_sync} ${__prune} ${__mev_boost} ${CL_EXTRAS} -elif [[ "${NETWORK}" = "sepolia" ]]; then +if [[ "${NETWORK}" = "sepolia" ]]; then GENESIS=/var/lib/prysm/genesis.ssz if [ ! -f "$GENESIS" ]; then echo "Fetching genesis file for Sepolia testnet" - curl -fsSL -o "$GENESIS" https://github.com/eth-clients/sepolia/raw/main/bepolia/genesis.ssz + curl -fsSL -o "$GENESIS" https://github.com/eth-clients/sepolia/raw/main/metadata/genesis.ssz fi # Word splitting is desired for the command line parameters # shellcheck disable=SC2086 @@ -91,7 +81,7 @@ elif [[ "${NETWORK}" = "holesky" ]]; then GENESIS=/var/lib/prysm/genesis.ssz if [ ! -f "$GENESIS" ]; then echo "Fetching genesis file for Holešky testnet" - curl -fsSL -o "$GENESIS" https://github.com/eth-clients/holesky/raw/main/custom_config_data/genesis.ssz + curl -fsSL -o "$GENESIS" https://github.com/eth-clients/holesky/raw/main/metadata/genesis.ssz fi # Word splitting is desired for the command line parameters # shellcheck disable=SC2086 diff --git a/reth.yml b/reth.yml index c96f6c79..5e2bd204 100644 --- a/reth.yml +++ b/reth.yml @@ -28,7 +28,9 @@ services: - LOG_LEVEL=${LOG_LEVEL:-info} - ARCHIVE_NODE=${ARCHIVE_NODE:-} - NETWORK=${NETWORK} + - STATIC_DIR=${ANCIENT_DIR} # Make this RUST_LOG=${LOG_LEVEL:-info},engine=trace when requiring deep debug + # RPC debug can be done with jsonrpsee=trace or jsonrpsee::target=trace for a specific target - RUST_LOG=${LOG_LEVEL:-info} - RUST_LOG_STYLE=never volumes: @@ -39,8 +41,7 @@ services: ports: - ${HOST_IP:-}:${EL_P2P_PORT:-30303}:${EL_P2P_PORT:-30303}/tcp - ${HOST_IP:-}:${EL_P2P_PORT:-30303}:${EL_P2P_PORT:-30303}/udp - - ${HOST_IP:-}:${ERIGON_P2P_PORT_2:-30304}:${ERIGON_P2P_PORT_2:-30303}/tcp - - ${HOST_IP:-}:${ERIGON_P2P_PORT_2:-30304}:${ERIGON_P2P_PORT_2:-30303}/udp + - ${HOST_IP:-}:${ERIGON_P2P_PORT_2:-30304}:${ERIGON_P2P_PORT_2:-30304}/udp networks: default: aliases: @@ -87,6 +88,17 @@ services: - metrics.path=/metrics - metrics.port=6060 - metrics.instance=execution + - metrics.network=${NETWORK} + + set-prune-marker: + profiles: ["tools"] + image: alpine:3 + user: "10001:10001" + restart: "no" + volumes: + - reth-el-data:/var/lib/reth + entrypoint: ["/bin/sh","-c"] + command: /bin/sh volumes: reth-el-data: diff --git a/reth/Dockerfile.binary b/reth/Dockerfile.binary index e67af8f9..8f336520 100644 --- a/reth/Dockerfile.binary +++ b/reth/Dockerfile.binary @@ -1,5 +1,5 @@ -ARG DOCKER_TAG -ARG DOCKER_REPO +ARG DOCKER_TAG=latest +ARG DOCKER_REPO=ghcr.io/paradigmxyz/reth FROM ${DOCKER_REPO}:${DOCKER_TAG} @@ -15,6 +15,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install tzdata \ gosu \ git \ + wget \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/reth/Dockerfile.source b/reth/Dockerfile.source index 11ed7a48..6d76d347 100644 --- a/reth/Dockerfile.source +++ b/reth/Dockerfile.source @@ -1,5 +1,5 @@ # Build reth in a stock rust container -FROM rust:bookworm as builder +FROM rust:bookworm AS builder # Unused, this is here to avoid build time complaints ARG DOCKER_TAG @@ -27,6 +27,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install tzdata \ gosu \ git \ + wget \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/reth/docker-entrypoint.sh b/reth/docker-entrypoint.sh index 2e2e37a2..a1cea933 100755 --- a/reth/docker-entrypoint.sh +++ b/reth/docker-entrypoint.sh @@ -82,12 +82,22 @@ else fi __static="" +if [ -n "${STATIC_DIR}" ] && [ ! "${STATIC_DIR}" = ".nada" ]; then + echo "Using separate static files directory at ${STATIC_DIR}." + __static="--datadir.static-files /var/lib/static" +fi -#if [ -n "${ANCIENT_DIR}" ] && [ ! "${ANCIENT_DIR}" = ".nada" ]; then -# echo "Using separate static files directory at ${ANCIENT_DIR}." -# __static="--datadir.static-files /var/lib/static" -#fi - +if [ -f /var/lib/reth/prune-marker ]; then + rm -f /var/lib/reth/prune-marker + if [ "${ARCHIVE_NODE}" = "true" ]; then + echo "Reth is an archive node. Not attempting to prune database: Aborting." + exit 1 + fi # Word splitting is desired for the command line parameters # shellcheck disable=SC2086 -exec "$@" ${__network} ${__verbosity} ${__prune} ${__static} ${EL_EXTRAS} + exec reth prune ${__network} --datadir /var/lib/reth ${__static} +else +# Word splitting is desired for the command line parameters +# shellcheck disable=SC2086 + exec "$@" ${__network} ${__verbosity} ${__prune} ${__static} ${EL_EXTRAS} +fi diff --git a/siren-shared.yml b/siren-shared.yml index c5f37da2..1bd7f1ab 100644 --- a/siren-shared.yml +++ b/siren-shared.yml @@ -2,4 +2,4 @@ services: siren: ports: - - ${SHARE_IP:-}:${SIREN_PORT}:80/tcp + - ${SHARE_IP:-}:${SIREN_PORT}:443/tcp diff --git a/siren.yml b/siren.yml index 689a6c20..90724f37 100644 --- a/siren.yml +++ b/siren.yml @@ -12,8 +12,19 @@ services: restart: "unless-stopped" image: ${SIREN_DOCKER_REPO}:${SIREN_DOCKER_TAG} volumes: + - lhvalidator-data:/var/lib/lighthouse + - ./siren/docker-entrypoint.sh:/usr/local/bin/docker-entrypoint.sh - /etc/localtime:/etc/localtime:ro + environment: + - BEACON_URL=${CL_NODE} + - VALIDATOR_URL=http://vc:7500 + - SESSION_PASSWORD=${SIREN_PASSWORD} + - SSL_ENABLED=true + - DEBUG=false + - SIREN_PORT=${SIREN_PORT} <<: *logging + entrypoint: + - docker-entrypoint.sh labels: - traefik.enable=true - traefik.http.routers.$(SIREN_HOST:-siren}.service=${SIREN_HOST:-siren} diff --git a/siren/docker-entrypoint.sh b/siren/docker-entrypoint.sh new file mode 100755 index 00000000..cc775d8c --- /dev/null +++ b/siren/docker-entrypoint.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +if [ ! -f /var/lib/lighthouse/validators/api-token.txt ]; then + echo "Validator client API token not found. Waiting 30s before restarting." + sleep 30 + exit 1 +fi + +API_TOKEN=$(cat /var/lib/lighthouse/validators/api-token.txt) +export API_TOKEN + +echo "Log into Siren at https://my-node-ip/${SIREN_PORT} with password ${SESSION_PASSWORD}" +exec /app/docker-assets/docker-entrypoint.sh diff --git a/ssv.yml b/ssv.yml index d65b0ad2..0df77f1c 100644 --- a/ssv.yml +++ b/ssv.yml @@ -28,6 +28,7 @@ services: - metrics.path=/metrics - metrics.port=15000 - metrics.instance=ssv-node + - metrics.network=${NETWORK} volumes: ssv-data: diff --git a/staking-deposit-cli/Dockerfile b/staking-deposit-cli/Dockerfile index ae858e4d..52b249d1 100644 --- a/staking-deposit-cli/Dockerfile +++ b/staking-deposit-cli/Dockerfile @@ -1,5 +1,5 @@ -from python:3.10-alpine as builder -#from python:3.12-alpine as builder +FROM python:3.10-alpine AS builder +#FROM python:3.12-alpine AS builder ARG BUILD_TARGET ARG SRC_REPO diff --git a/teku-allin1.yml b/teku-allin1.yml index 0ee601ce..3db9fd66 100644 --- a/teku-allin1.yml +++ b/teku-allin1.yml @@ -91,6 +91,7 @@ services: - metrics.path=/metrics - metrics.port=8008 - metrics.instance=consensus + - metrics.network=${NETWORK} wipe-db: profiles: ["tools"] diff --git a/teku-cl-only.yml b/teku-cl-only.yml index a99a3f4b..f62386d9 100644 --- a/teku-cl-only.yml +++ b/teku-cl-only.yml @@ -45,6 +45,8 @@ services: - DEFAULT_GRAFFITI=true - WEB3SIGNER=false - NETWORK=${NETWORK} + - IPV6=${IPV6:-false} + - CL_P2P_PORT=${CL_P2P_PORT:-9000} ports: - ${HOST_IP:-}:${CL_P2P_PORT:-9000}:${CL_P2P_PORT:-9000}/tcp - ${HOST_IP:-}:${CL_P2P_PORT:-9000}:${CL_P2P_PORT:-9000}/udp @@ -82,6 +84,7 @@ services: - metrics.path=/metrics - metrics.port=8008 - metrics.instance=consensus + - metrics.network=${NETWORK} volumes: tekuconsensus-data: diff --git a/teku-vc-only.yml b/teku-vc-only.yml index 4ab24bce..34bc2569 100644 --- a/teku-vc-only.yml +++ b/teku-vc-only.yml @@ -67,6 +67,7 @@ services: - metrics.path=/metrics - metrics.port=8009 - metrics.instance=validator + - metrics.network=${NETWORK} validator-exit: profiles: ["tools"] diff --git a/teku.yml b/teku.yml index 8d1d356f..3682f501 100644 --- a/teku.yml +++ b/teku.yml @@ -45,6 +45,8 @@ services: - WEB3SIGNER=false - EMBEDDED_VC=false - NETWORK=${NETWORK} + - IPV6=${IPV6:-false} + - CL_P2P_PORT=${CL_P2P_PORT:-9000} ports: - ${HOST_IP:-}:${CL_P2P_PORT:-9000}:${CL_P2P_PORT:-9000}/tcp - ${HOST_IP:-}:${CL_P2P_PORT:-9000}:${CL_P2P_PORT:-9000}/udp @@ -82,6 +84,7 @@ services: - metrics.path=/metrics - metrics.port=8008 - metrics.instance=consensus + - metrics.network=${NETWORK} validator: restart: "unless-stopped" @@ -134,6 +137,7 @@ services: - metrics.path=/metrics - metrics.port=8009 - metrics.instance=validator + - metrics.network=${NETWORK} validator-exit: profiles: ["tools"] diff --git a/teku/Dockerfile.binary b/teku/Dockerfile.binary index 047f28c3..3acea823 100644 --- a/teku/Dockerfile.binary +++ b/teku/Dockerfile.binary @@ -1,5 +1,5 @@ -ARG DOCKER_TAG -ARG DOCKER_REPO +ARG DOCKER_TAG=latest +ARG DOCKER_REPO=consensys/teku FROM ${DOCKER_REPO}:${DOCKER_TAG} diff --git a/teku/Dockerfile.source b/teku/Dockerfile.source index 67428306..073105b3 100644 --- a/teku/Dockerfile.source +++ b/teku/Dockerfile.source @@ -1,5 +1,5 @@ # Build Teku in a stock Ubuntu container -FROM eclipse-temurin:21-jdk-jammy as builder +FROM eclipse-temurin:21-jdk-jammy AS builder # This is here to avoid build-time complaints ARG DOCKER_TAG diff --git a/teku/docker-entrypoint.sh b/teku/docker-entrypoint.sh index 08354679..3c0488bf 100755 --- a/teku/docker-entrypoint.sh +++ b/teku/docker-entrypoint.sh @@ -131,12 +131,19 @@ else __w3s_url="" fi +if [ "${IPV6}" = "true" ]; then + echo "Configuring Teku to listen on IPv6 ports" + __ipv6="--p2p-interface 0.0.0.0,:: --p2p-port-ipv6 ${CL_P2P_PORT:-9000}" +else + __ipv6="" +fi + if [ "${DEFAULT_GRAFFITI}" = "true" ]; then # Word splitting is desired for the command line parameters # shellcheck disable=SC2086 - exec "$@" ${__network} ${__w3s_url} ${__mev_boost} ${__rapid_sync} ${__prune} ${__beacon_stats} ${__doppel} ${CL_EXTRAS} ${VC_EXTRAS} + exec "$@" ${__network} ${__w3s_url} ${__mev_boost} ${__rapid_sync} ${__prune} ${__beacon_stats} ${__doppel} ${__ipv6} ${CL_EXTRAS} ${VC_EXTRAS} else # Word splitting is desired for the command line parameters # shellcheck disable=SC2086 - exec "$@" ${__network} "--validators-graffiti=${GRAFFITI}" ${__w3s_url} ${__mev_boost} ${__rapid_sync} ${__prune} ${__beacon_stats} ${__doppel} ${CL_EXTRAS} ${VC_EXTRAS} + exec "$@" ${__network} "--validators-graffiti=${GRAFFITI}" ${__w3s_url} ${__mev_boost} ${__rapid_sync} ${__prune} ${__beacon_stats} ${__doppel} ${__ipv6} ${CL_EXTRAS} ${VC_EXTRAS} fi diff --git a/traefik-aws.yml b/traefik-aws.yml index 2288b634..39cf51b5 100644 --- a/traefik-aws.yml +++ b/traefik-aws.yml @@ -51,6 +51,7 @@ services: - metrics.path=/metrics - metrics.port=8080 - metrics.instance=traefik + - metrics.network=${NETWORK} <<: *logging volumes: diff --git a/traefik-cf.yml b/traefik-cf.yml index d3786a5c..e4066e56 100644 --- a/traefik-cf.yml +++ b/traefik-cf.yml @@ -45,6 +45,7 @@ services: - metrics.path=/metrics - metrics.port=8080 - metrics.instance=traefik + - metrics.network=${NETWORK} <<: *logging cf-ddns: diff --git a/traefik/Dockerfile b/traefik/Dockerfile index 77d2ef05..3a4a698d 100644 --- a/traefik/Dockerfile +++ b/traefik/Dockerfile @@ -1,5 +1,5 @@ # Add AWS CLI to traefik image -ARG DOCKER_TAG +ARG DOCKER_TAG=latest FROM traefik:${DOCKER_TAG} diff --git a/vc-utils/keymanager.sh b/vc-utils/keymanager.sh index ced14839..d8a62482 100755 --- a/vc-utils/keymanager.sh +++ b/vc-utils/keymanager.sh @@ -27,6 +27,7 @@ call_api() { else echo "This was a call without data" fi + echo "The token was ${__token} from ${__token_file}" echo "The return code was ${__code} and if we had result data, here it is." if [ -f /tmp/result.txt ]; then cat /tmp/result.txt @@ -76,11 +77,7 @@ call_cl_api() { get-token() { set +e - if [ -z "${PRYSM:+x}" ]; then - __token=$(< "${__token_file}") - else - __token=$(sed -n 2p "${__token_file}") - fi + __token=$(< "${__token_file}") __return=$? if [ $__return -ne 0 ]; then echo "Error encountered while trying to get the keymanager API token." @@ -338,8 +335,20 @@ exit-sign() { fi __pubkeys=() __api_path=eth/v1/keystores - get-token if [ "${__pubkey}" = "all" ]; then + if [ "${WEB3SIGNER}" = "true" ]; then + __token=NIL + __vc_api_container=${__api_container} + __api_container=web3signer + __vc_service=${__service} + __service=web3signer + __vc_api_port=${__api_port} + __api_port=9000 + __vc_api_tls=${__api_tls} + __api_tls=false + else + get-token + fi __validator-list-call if [ "$(echo "$__result" | jq '.data | length')" -eq 0 ]; then echo "No keys loaded, cannot sign anything" @@ -349,11 +358,18 @@ exit-sign() { # Word splitting is desired for the array # shellcheck disable=SC2206 __pubkeys+=( ${__keys_to_array} ) + if [ "${WEB3SIGNER}" = "true" ]; then + __api_container=${__vc_api_container} + __api_port=${__vc_api_port} + __api_tls=${__vc_api_tls} + __service=${__vc_service} + fi fi else __pubkeys+=( "${__pubkey}" ) fi + get-token for __pubkey in "${__pubkeys[@]}"; do __api_data="" __http_method=POST @@ -792,6 +808,14 @@ and secrets directories into .eth/validator_keys instead." fi echo "$__protect_json" > /tmp/protect.json + if [ "${__debug}" -eq 1 ]; then + echo "The keystore reads as $__keystore_json" + echo "And your password is $__password" + set +e + echo "Testing jq on these" + jq --arg keystore_value "$__keystore_json" --arg password_value "$__password" '. | .keystores += [$keystore_value] | .passwords += [$password_value]' <<< '{}' + set -e + fi if [ "$__do_a_protec" -eq 0 ]; then jq --arg keystore_value "$__keystore_json" --arg password_value "$__password" '. | .keystores += [$keystore_value] | .passwords += [$password_value]' <<< '{}' >/tmp/apidata.txt else @@ -816,7 +840,13 @@ and secrets directories into .eth/validator_keys instead." call_api case $__code in 200) ;; - 400) echo "The pubkey was formatted wrong. Error: $(echo "$__result" | jq -r '.message')"; exit 1;; + 400) + if [ -z "${PRYSM:+x}" ]; then + echo "The pubkey was formatted wrong. Error: $(echo "$__result" | jq -r '.message')"; exit 1 + else + echo "Bad format. Error: $__result"; exit 1 + fi + ;; 401) echo "No authorization token found. This is a bug. Error: $(echo "$__result" | jq -r '.message')"; exit 70;; 403) echo "The authorization token is invalid. Error: $(echo "$__result" | jq -r '.message')"; exit 1;; 500) echo "Internal server error. Error: $(echo "$__result" | jq -r '.message')"; exit 1;; diff --git a/web3signer.yml b/web3signer.yml index 080334cc..50d3db40 100644 --- a/web3signer.yml +++ b/web3signer.yml @@ -45,6 +45,7 @@ services: - metrics.path=/metrics - metrics.port=9001 - metrics.instance=web3signer + - metrics.network=${NETWORK} postgres: restart: "unless-stopped" diff --git a/web3signer/Dockerfile.binary b/web3signer/Dockerfile.binary index 15610965..7e521b80 100644 --- a/web3signer/Dockerfile.binary +++ b/web3signer/Dockerfile.binary @@ -1,7 +1,7 @@ -ARG DOCKER_TAG -ARG DOCKER_REPO +ARG DOCKER_TAG=latest +ARG DOCKER_REPO=consensys/web3signer -FROM flyway/flyway:latest as flyway +FROM flyway/flyway:latest AS flyway FROM ${DOCKER_REPO}:${DOCKER_TAG}