From 13bea6f559ddbd4409f69f0f941420f67966e5d2 Mon Sep 17 00:00:00 2001 From: Julian Rubino Date: Thu, 7 Nov 2024 23:32:58 +0900 Subject: [PATCH 1/6] Install lz4 in Docker image Install lz4 in Docker image to support lz4 snapshots. --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7aada35..2051000 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ FROM debian:bookworm AS base ENV PATH=/root/.zetacored/cosmovisor/current/bin/:${PATH} RUN apt update && \ - apt install -y ca-certificates curl jq && \ + apt install -y ca-certificates curl jq lz4 && \ rm -rf /var/lib/apt/lists/* COPY --from=base-build /go/bin/cosmovisor /go/bin/go-getter /go/bin/dl-pipe /usr/local/bin @@ -28,4 +28,4 @@ RUN apt update && \ RUN ARCH=$( [ "$TARGETARCH" = "amd64" ] && echo "x86_64" || echo "$TARGETARCH" ) && \ curl -L https://github.com/zeta-chain/cosmprund/releases/download/v0.2.0-zeta/cosmprund_Linux_${ARCH}.tar.gz | tar xz -C /usr/local/bin/ cosmprund &&\ - chmod +x /usr/local/bin/cosmprund \ No newline at end of file + chmod +x /usr/local/bin/cosmprund From a719d48ae1f9cb5d03b70ba4db3aea509fb5870f Mon Sep 17 00:00:00 2001 From: Julian Rubino Date: Thu, 7 Nov 2024 12:49:18 -0300 Subject: [PATCH 2/6] Extract new snapshots using lz4 --- init.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/init.sh b/init.sh index ff9988e..7ec436b 100755 --- a/init.sh +++ b/init.sh @@ -30,7 +30,7 @@ elif [[ "$ZETACHAIN_NETWORK" == "testnet" || "$ZETACHAIN_NETWORK" == "athens3" ] ZETACHAIN_INIT_API_URL=${ZETACHAIN_INIT_API_URL:-"https://zetachain-athens.g.allthatnode.com/archive/rest"} ZETACHAIN_SNAPSHOT_METADATA_URL=${ZETACHAIN_SNAPSHOT_METADATA_URL:-"https://snapshots.rpc.zetachain.com/testnet/${ZETACHAIN_SNAPSHOT_TYPE}/latest.json"} ZETACHAIN_NETWORK_CONFIG_URL_BASE=${ZETACHAIN_NETWORK_CONFIG_URL_BASE:-"https://raw.githubusercontent.com/zeta-chain/network-config/main/athens3"} -else +else echo "Invalid network" exit 1 fi @@ -81,7 +81,7 @@ install_genesis_zetacored() { .proposals[] | select(.status == "PROPOSAL_STATUS_PASSED") | .messages[] | - select(."@type" == "/cosmos.upgrade.v1beta1.MsgSoftwareUpgrade" and (.plan.height | + select(."@type" == "/cosmos.upgrade.v1beta1.MsgSoftwareUpgrade" and (.plan.height | tonumber < $max_height))' | jq -s '.[0]' | tee /tmp/init-upgrade-plan.json ZETACORED_BINARY_URL=$(jq -r '.plan.info' /tmp/init-upgrade-plan.json | jq -r ".binaries[\"linux/$GOARCH\"]") @@ -100,7 +100,7 @@ restore_snapshot() { snapshot_md5=$(echo "$snapshot" | jq -r '.checksums.md5') echo "Restoring snapshot from ${snapshot_link}" # https://github.com/zeta-chain/dl-pipe - dl-pipe -hash "md5:${snapshot_md5}" "$snapshot_link" | tar x -C $HOME/.zetacored + dl-pipe -hash "md5:${snapshot_md5}" "$snapshot_link" | tar -I lz4 -x -C $HOME/.zetacored } cd $HOME @@ -117,9 +117,9 @@ if [[ ! -f /root/init_completed ]]; then install_genesis_zetacored restore_snapshot touch /root/init_completed -else +else echo "Initialization already completed" fi # always set IP address as it may change after restart -sed -i -e "s/^external_address = .*/external_address = \"${MY_IP}:26656\"/" .zetacored/config/config.toml \ No newline at end of file +sed -i -e "s/^external_address = .*/external_address = \"${MY_IP}:26656\"/" .zetacored/config/config.toml From 6bcd980a133e0659751cb8968960fe5e78e9115c Mon Sep 17 00:00:00 2001 From: Julian Rubino Date: Thu, 7 Nov 2024 13:03:29 -0300 Subject: [PATCH 3/6] Add lz4 to snapshotter --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2051000..8afa40a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ FROM base AS snapshotter ARG TARGETARCH RUN apt update && \ - apt install -y rclone procps && \ + apt install -y rclone procps lz4 && \ rm -rf /var/lib/apt/lists/* RUN ARCH=$( [ "$TARGETARCH" = "amd64" ] && echo "x86_64" || echo "$TARGETARCH" ) && \ From 15eab76986a285dc99fd05500886a55715bb2120 Mon Sep 17 00:00:00 2001 From: Julian Rubino Date: Fri, 8 Nov 2024 01:24:46 +0900 Subject: [PATCH 4/6] Support decompressing both tar and lz4 snapshots Co-authored-by: Alex Gartner --- init.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/init.sh b/init.sh index 7ec436b..43bd0ad 100755 --- a/init.sh +++ b/init.sh @@ -100,7 +100,11 @@ restore_snapshot() { snapshot_md5=$(echo "$snapshot" | jq -r '.checksums.md5') echo "Restoring snapshot from ${snapshot_link}" # https://github.com/zeta-chain/dl-pipe - dl-pipe -hash "md5:${snapshot_md5}" "$snapshot_link" | tar -I lz4 -x -C $HOME/.zetacored +decompress_args="" + if [[ "$snapshot_link" == *"lz4"* ]]; then + decompress_args="-I lz4" + fi + dl-pipe -hash "md5:${snapshot_md5}" "$snapshot_link" | tar $decompress_args -x -C $HOME/.zetacored } cd $HOME From 0ffbaef1c0a454917c7ffbc5144d3d03e3685506 Mon Sep 17 00:00:00 2001 From: Julian Rubino Date: Thu, 7 Nov 2024 14:02:11 -0300 Subject: [PATCH 5/6] fix indentation --- init.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/init.sh b/init.sh index 43bd0ad..fb8e371 100755 --- a/init.sh +++ b/init.sh @@ -100,11 +100,11 @@ restore_snapshot() { snapshot_md5=$(echo "$snapshot" | jq -r '.checksums.md5') echo "Restoring snapshot from ${snapshot_link}" # https://github.com/zeta-chain/dl-pipe -decompress_args="" - if [[ "$snapshot_link" == *"lz4"* ]]; then - decompress_args="-I lz4" - fi - dl-pipe -hash "md5:${snapshot_md5}" "$snapshot_link" | tar $decompress_args -x -C $HOME/.zetacored + decompress_args="" + if [[ "$snapshot_link" == *"lz4"* ]]; then + decompress_args="-I lz4" + fi + dl-pipe -hash "md5:${snapshot_md5}" "$snapshot_link" | tar $decompress_args -x -C $HOME/.zetacored } cd $HOME From 8590e9d3da6202117d8b767ee7e16781809319d7 Mon Sep 17 00:00:00 2001 From: Julian Rubino Date: Thu, 7 Nov 2024 14:52:07 -0300 Subject: [PATCH 6/6] fix indentation --- init.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/init.sh b/init.sh index fb8e371..38781ea 100755 --- a/init.sh +++ b/init.sh @@ -101,10 +101,10 @@ restore_snapshot() { echo "Restoring snapshot from ${snapshot_link}" # https://github.com/zeta-chain/dl-pipe decompress_args="" - if [[ "$snapshot_link" == *"lz4"* ]]; then - decompress_args="-I lz4" - fi - dl-pipe -hash "md5:${snapshot_md5}" "$snapshot_link" | tar $decompress_args -x -C $HOME/.zetacored + if [[ "$snapshot_link" == *"lz4"* ]]; then + decompress_args="-I lz4" + fi + dl-pipe -hash "md5:${snapshot_md5}" "$snapshot_link" | tar $decompress_args -x -C $HOME/.zetacored } cd $HOME