From 41321554f6fdd5d8ff77214530837d728d7c9810 Mon Sep 17 00:00:00 2001 From: yorickdowne <71337066+yorickdowne@users.noreply.github.com> Date: Tue, 14 May 2024 06:34:44 -0400 Subject: [PATCH] Migrate to new volume names (#1831) --- ethd | 6 ++---- lighthouse.yml | 6 ++++-- lighthouse/Dockerfile.binary | 4 ++++ lighthouse/Dockerfile.source | 4 ++++ lighthouse/docker-entrypoint.sh | 10 ++++++++++ prysm.yml | 4 +++- prysm/Dockerfile.binary | 4 ++++ prysm/Dockerfile.source | 4 ++++ prysm/docker-entrypoint.sh | 10 ++++++++++ 9 files changed, 45 insertions(+), 7 deletions(-) diff --git a/ethd b/ethd index 1bf8961e..c9854a6c 100755 --- a/ethd +++ b/ethd @@ -1368,15 +1368,13 @@ resync-consensus() { value=$(sed -n -e "s/^${var}=\(.*\)/\1/p" "${ENV_FILE}" || true) case "${value}" in - *lighthouse.yml* ) __cl_volume='lhbeacon-data'; __cl_client="lighthouse";; - *lighthouse-cl-only.yml* ) __cl_volume='lhconsensus-data'; __cl_client="lighthouse";; + *lighthouse.yml* | *lighthouse-cl-only.yml* ) __cl_volume='lhconsensus-data'; __cl_client="lighthouse";; *teku-allin1.yml* ) __cl_volume='wipe-db'; __cl_client="teku";; *teku.yml* | *teku-cl-only.yml* ) __cl_volume='tekuconsensus-data'; __cl_client="teku";; *nimbus-allin1.yml* ) __cl_volume='wipe-db'; __cl_client="nimbus";; *nimbus.yml* | *nimbus-cl-only.yml* ) __cl_volume='nimbus-consensus-data'; __cl_client="nimbus";; *lodestar.yml* | *lodestar-cl-only.yml* ) __cl_volume='lsconsensus-data'; __cl_client="lodestar";; - *prysm.yml* ) __cl_volume='prysmbeacon-data'; __cl_client="prysm";; - *prysm-cl-only.yml* ) __cl_volume='prysmconsensus-data'; __cl_client="prysm";; + *prysm.yml* | *prysm-cl-only.yml* ) __cl_volume='prysmconsensus-data'; __cl_client="prysm";; *grandine-allin1.yml* ) __cl_volume='wipe-db'; __cl_client="grandine";; *grandine.yml* | *grandine-cl-only.yml* ) __cl_volume='grandineconsensus-data'; __cl_client="grandine";; * ) echo "You do not appear to be running a consensus layer client. Nothing to do."; return;; diff --git a/lighthouse.yml b/lighthouse.yml index 13fed5d2..88654d25 100644 --- a/lighthouse.yml +++ b/lighthouse.yml @@ -26,7 +26,8 @@ services: user: lhconsensus stop_grace_period: 1m volumes: - - lhbeacon-data:/var/lib/lighthouse + - lhbeacon-data:/var/lib/lighthouse-og + - lhconsensus-data:/var/lib/lighthouse - /etc/localtime:/etc/localtime:ro - jwtsecret:/var/lib/lighthouse/beacon/ee-secret environment: @@ -221,12 +222,13 @@ services: user: "10002:10002" restart: "no" volumes: - - lhbeacon-data:/var/lib/lighthouse + - lhconsensus-data:/var/lib/lighthouse entrypoint: ["/bin/sh","-c"] command: /bin/sh volumes: lhbeacon-data: + lhconsensus-data: lhvalidator-data: jwtsecret: diff --git a/lighthouse/Dockerfile.binary b/lighthouse/Dockerfile.binary index 8684bc3b..95040722 100644 --- a/lighthouse/Dockerfile.binary +++ b/lighthouse/Dockerfile.binary @@ -20,6 +20,8 @@ RUN adduser \ --uid "${UID}" \ "${USER}" +# Create mount point for old volume +RUN mkdir -p /var/lib/lighthouse-og && chown -R ${USER}:${USER} /var/lib/lighthouse-og # Create data mount point with permissions RUN mkdir -p /var/lib/lighthouse/beacon/ee-secret && chown -R ${USER}:${USER} /var/lib/lighthouse/beacon && chmod -R 700 /var/lib/lighthouse/beacon && chmod 777 /var/lib/lighthouse/beacon/ee-secret # Cannot assume buildkit, hence no chmod @@ -49,6 +51,8 @@ RUN set -eux; \ rm -rf /var/lib/apt/lists/*; \ # verify that the binary works gosu nobody true +# rsync for migration +RUN apt-get update && apt-get install -y --no-install-recommends rsync # Cannot assume buildkit, hence no chmod COPY --chown=${USER}:${USER} ./validator-exit.sh /usr/local/bin/ diff --git a/lighthouse/Dockerfile.source b/lighthouse/Dockerfile.source index 471ed2cd..ad28f33a 100644 --- a/lighthouse/Dockerfile.source +++ b/lighthouse/Dockerfile.source @@ -33,6 +33,8 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install wget \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* +# rsync for migration +RUN apt-get update && apt-get install -y --no-install-recommends rsync ARG USER=lhconsensus ARG UID=10002 @@ -47,6 +49,8 @@ RUN adduser \ --uid "${UID}" \ "${USER}" +# Create mount point for old volume +RUN mkdir -p /var/lib/lighthouse-og && chown -R ${USER}:${USER} /var/lib/lighthouse-og # Create data mount point with permissions RUN mkdir -p /var/lib/lighthouse/beacon/ee-secret && chown -R ${USER}:${USER} /var/lib/lighthouse/beacon && chmod -R 700 /var/lib/lighthouse/beacon && chmod 777 /var/lib/lighthouse/beacon/ee-secret # Cannot assume buildkit, hence no chmod diff --git a/lighthouse/docker-entrypoint.sh b/lighthouse/docker-entrypoint.sh index f186f2a7..c0b0aa0c 100755 --- a/lighthouse/docker-entrypoint.sh +++ b/lighthouse/docker-entrypoint.sh @@ -6,6 +6,16 @@ if [ "$(id -u)" = '0' ]; then exec gosu lhconsensus docker-entrypoint.sh "$@" fi +# Migrate from old to new volume +if [[ -d /var/lib/lighthouse-og/beacon && ! -f /var/lib/lighthouse-og/beacon/migrationdone \ + && $(ls -A /var/lib/lighthouse-og/beacon/) ]]; then + echo "Migrating from old Lighthouse volume to new one" + echo "This may take 10 minutes on a fast drive. Please be patient" + rsync -a --remove-source-files --exclude='ee-secret' /var/lib/lighthouse-og/beacon/ /var/lib/lighthouse/beacon/ + touch /var/lib/lighthouse-og/beacon/migrationdone + echo "Migration completed, data is now in volume \"lhconsensus-data\"" +fi + if [ -n "${JWT_SECRET}" ]; then echo -n "${JWT_SECRET}" > /var/lib/lighthouse/beacon/ee-secret/jwtsecret echo "JWT secret was supplied in .env" diff --git a/prysm.yml b/prysm.yml index ba6570ad..dfc8f579 100644 --- a/prysm.yml +++ b/prysm.yml @@ -28,7 +28,8 @@ services: user: prysmconsensus stop_grace_period: 1m volumes: - - prysmbeacon-data:/var/lib/prysm + - prysmbeacon-data:/var/lib/prysm-og + - prysmconsensus-data:/var/lib/prysm - /etc/localtime:/etc/localtime:ro - jwtsecret:/var/lib/prysm/ee-secret environment: @@ -248,6 +249,7 @@ services: volumes: prysmbeacon-data: + prysmconsensus-data: prysmvalidator-data: jwtsecret: diff --git a/prysm/Dockerfile.binary b/prysm/Dockerfile.binary index 479827ff..44200217 100644 --- a/prysm/Dockerfile.binary +++ b/prysm/Dockerfile.binary @@ -18,6 +18,8 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install ca-certificates curl bash tzdata git \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* +# rsync for migration +RUN apt-get update && apt-get install -y --no-install-recommends rsync RUN set -eux; \ apt-get update; \ @@ -36,6 +38,8 @@ RUN adduser \ --uid "${UID}" \ "${USER}" +# Create mount point for old volume +RUN mkdir -p /var/lib/prysm-og && chown -R ${USER}:${USER} /var/lib/prysm-og # Create data mount point with permissions RUN mkdir -p /var/lib/prysm/ee-secret && chown -R ${USER}:${USER} /var/lib/prysm && chmod -R 700 /var/lib/prysm && chmod 777 /var/lib/prysm/ee-secret diff --git a/prysm/Dockerfile.source b/prysm/Dockerfile.source index a79c231e..7b7ae0f2 100644 --- a/prysm/Dockerfile.source +++ b/prysm/Dockerfile.source @@ -28,6 +28,8 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install ca-certificates curl bash tzdata git \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* +# rsync for migration +RUN apt-get update && apt-get install -y --no-install-recommends rsync RUN set -eux; \ apt-get update; \ @@ -46,6 +48,8 @@ RUN adduser \ --uid "${UID}" \ "${USER}" +# Create mount point for old volume +RUN mkdir -p /var/lib/prysm-og && chown -R ${USER}:${USER} /var/lib/prysm-og # Create data mount point with permissions RUN mkdir -p /var/lib/prysm/ee-secret && chown -R ${USER}:${USER} /var/lib/prysm && chmod -R 700 /var/lib/prysm && chmod 777 /var/lib/prysm/ee-secret diff --git a/prysm/docker-entrypoint.sh b/prysm/docker-entrypoint.sh index cef6342a..47c6cbc5 100755 --- a/prysm/docker-entrypoint.sh +++ b/prysm/docker-entrypoint.sh @@ -6,6 +6,16 @@ if [ "$(id -u)" = '0' ]; then exec gosu prysmconsensus docker-entrypoint.sh "$@" fi +# Migrate from old to new volume +if [[ -d /var/lib/prysm-og && ! -f /var/lib/prysm-og/migrationdone \ + && $(ls -A /var/lib/prysm-og/) ]]; then + echo "Migrating from old Prysm volume to new one" + echo "This may take 10 minutes on a fast drive. Please be patient" + rsync -a --remove-source-files --exclude='ee-secret' /var/lib/prysm-og/ /var/lib/prysm/ + touch /var/lib/prysm-og/migrationdone + echo "Migration completed, data is now in volume \"prysmconsensus-data\"" +fi + if [ -n "${JWT_SECRET}" ]; then echo -n "${JWT_SECRET}" > /var/lib/prysm/ee-secret/jwtsecret echo "JWT secret was supplied in .env"