From 2325d91028790be6e4fa4fd41b771ed8e89aee2a Mon Sep 17 00:00:00 2001 From: Yorick Date: Tue, 18 Jun 2024 16:44:07 +0000 Subject: [PATCH] Nethermind path state --- ethd | 19 +++++++++++---- nethermind/docker-entrypoint.sh | 41 +++++++++++++++++++-------------- 2 files changed, 38 insertions(+), 22 deletions(-) diff --git a/ethd b/ethd index c53e36be..6ec767a3 100755 --- a/ethd +++ b/ethd @@ -454,7 +454,8 @@ check_disk_space() { echo "It should currently be auto-pruning, check logs with \"$__me logs -f --tail 500 execution | grep \ Full\". Free space:" else - echo "If the below reads above ${__safe_prune} GiB free, prune it with \"$__me prune-nethermind\"" + echo "You might be able to resync Nethermind for path storage if you haven't done so already, with \"$__me resync-execution\"." + echo "Also verify the amount of space your consensus layer client takes, and resync it as needed." fi echo __display_docker_dir @@ -462,7 +463,8 @@ Full\". Free space:" elif [[ "${value}" =~ "geth.yml" ]] && [[ "${__free_space}" -lt 104857600 ]]; then echo echo "You are running Geth and have less than 100 GiB of free disk space." - echo "You may resync from scratch to use PBSS and slow on-disk DB growth, with \"$__me resync-execution\"." + echo "You may resync from scratch to use PBSS and slow on-disk DB growth, if you haven't done so already, with \"$__me resync-execution\"." + echo "Also verify the amount of space your consensus layer client takes, and resync it as needed." echo __display_docker_dir __display_docker_volumes @@ -758,13 +760,13 @@ delete_erigon() { return 0 fi - if [ -z "$(dodocker volume ls -q -f "name=erigon-el-data")" ]; then # No Erigon volume + if [ -z "$(dodocker volume ls -q -f "name=$(basename "$(realpath .)")[_-]erigon-el-data")" ]; then # No Erigon volume return 0 fi # https://discord.com/channels/687972960811745322/886550095749873666/1068994749480054866 # Detect Erigon v3 by files in snapshots/history - if [ -n "$(dodocker run --rm -v "$(dodocker volume ls -q -f "name=erigon-el-data")":"/var/lib/erigon" \ + if [ -n "$(dodocker run --rm -v "$(dodocker volume ls -q -f "name=$(basename "$(realpath .)")[_-]name=erigon-el-data")":"/var/lib/erigon" \ alpine:3 ls /var/lib/erigon/snapshots/history)" ]; then return 0 # Files in snapshots/history means it's v3 fi @@ -781,7 +783,7 @@ delete_erigon() { echo "Stopping Erigon container" docompose stop execution && docompose rm -f execution - dodocker volume rm "$(dodocker volume ls -q -f "name=erigon-el-data")" + dodocker volume rm "$(dodocker volume ls -q -f "name=name=$(basename "$(realpath .)")[_-]erigon-el-data")" echo echo "Erigon stopped and database deleted." echo @@ -1589,6 +1591,12 @@ prune-nethermind() { var="NETWORK" NETWORK=$(sed -n -e "s/^${var}=\(.*\)/\1/p" "${ENV_FILE}" || true) + if [ -n "$(dodocker run --rm -v "$(dodocker volume ls -q -f "name=$(basename "$(realpath .)")[_-]nm-eth1-data")":"/var/lib/nethermind" \ + alpine:3 ls "/var/lib/nethermind/nethermind_db/${NETWORK}/pathState")" ]; then + echo "This Nethermind uses path-based storage. Manual pruning is not required." + return 0 + fi + if [ "${NETWORK}" = "mainnet" ] || [ "${NETWORK}" = "gnosis" ]; then __min_free=262144000 __min_gib=250 @@ -1649,6 +1657,7 @@ prune-nethermind() { echo "Nethermind should auto-prune below ${threshold} GiB free. Check logs with \"$__me logs -f --tail 500 \ execution | grep Full\" to see whether it is." fi + echo "Instead of pruning, consider resyncing Nethermind to use path-based storage, with \"$__me resync-execution\"." read -rp "WARNING - this will prune Nethermind's database in the background. Do you wish to continue? (No/Yes) " yn case $yn in [Yy][Ee][Ss] ) break;; diff --git a/nethermind/docker-entrypoint.sh b/nethermind/docker-entrypoint.sh index bef68cca..e5351d81 100755 --- a/nethermind/docker-entrypoint.sh +++ b/nethermind/docker-entrypoint.sh @@ -61,26 +61,33 @@ if [ "${ARCHIVE_NODE}" = "true" ]; then echo "Nethermind archive node without pruning" __prune="--Sync.DownloadBodiesInFastSync=false --Sync.DownloadReceiptsInFastSync=false --Sync.FastSync=false --Sync.SnapSync=false --Sync.FastBlocks=false --Pruning.Mode=None --Sync.PivotNumber=0" else - __parallel=$(($(nproc)/4)) - if [ "${__parallel}" -lt 2 ]; then - __parallel=2 - fi - __prune="--Pruning.FullPruningMaxDegreeOfParallelism=${__parallel}" - if [ "${AUTOPRUNE_NM}" = true ]; then - __prune="${__prune} --Pruning.FullPruningTrigger=VolumeFreeSpace" - if [ "${NETWORK}" = "mainnet" ] || [ "${NETWORK}" = "gnosis" ]; then - __prune="${__prune} --Pruning.FullPruningThresholdMb=375810" - else - __prune="${__prune} --Pruning.FullPruningThresholdMb=51200" + # Fresh DB or already path based storage + if [[ ! -d "/var/lib/nethermind/nethermind_db/${NETWORK}" || -d "/var/lib/nethermind/nethermind_db/${NETWORK}/pathState" ]]; then + __prune="" + else # Hash, set pruning parameters + __parallel=$(($(nproc)/4)) + if [ "${__parallel}" -lt 2 ]; then + __parallel=2 fi - fi - if [ "${__memtotal}" -ge 30 ]; then + __prune="--Pruning.FullPruningMaxDegreeOfParallelism=${__parallel}" + if [ "${AUTOPRUNE_NM}" = true ]; then + __prune="${__prune} --Pruning.FullPruningTrigger=VolumeFreeSpace" + if [ "${NETWORK}" = "mainnet" ] || [ "${NETWORK}" = "gnosis" ]; then + __prune="${__prune} --Pruning.FullPruningThresholdMb=375810" + else + __prune="${__prune} --Pruning.FullPruningThresholdMb=51200" + fi + fi + if [ "${__memtotal}" -ge 30 ]; then # Investigating instability - #__prune="${__prune} --Pruning.CacheMb=4096 --Pruning.FullPruningMemoryBudgetMb=16384 --Init.StateDbKeyScheme=HalfPath" - __prune="${__prune} --Pruning.FullPruningMemoryBudgetMb=16384 --Init.StateDbKeyScheme=HalfPath" + #__prune="${__prune} --Pruning.CacheMb=4096 --Pruning.FullPruningMemoryBudgetMb=16384" + __prune="${__prune} --Pruning.FullPruningMemoryBudgetMb=16384" + fi + fi + if [ -n "${__prune}" ]; then + echo "Using pruning parameters:" + echo "${__prune}" fi - echo "Using pruning parameters:" - echo "${__prune}" fi # Word splitting is desired for the command line parameters