From 1a414d52c80306711755fe394d58c4eaa0bfbf29 Mon Sep 17 00:00:00 2001 From: jumao Date: Mon, 18 Sep 2023 23:42:02 -0400 Subject: [PATCH] https://github.com/sonic-net/sonic-buildimage/pull/16466 fix based on review comments: MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Install rdbtools into syncd docker and add saidump.sh into host. * [saidump] • Saidump for DNX-SAI https://github.com/sonic-net/sonic-buildimage/issues/13561 Solution and modification: To use the Redis-db SAVE option to save the snapshot of DB each time and recover later, instead of looping through each entry in the table and saving it. (1) Updated platform/broadcom/docker-syncd-brcm-dnx/Dockerfile.j2, install Python library rdbtools into the syncd containter. (2) Updated sonic-buildimage/src/sonic-sairedis/saidump/saidump.cpp, add a new option -r, which updates the rdbtools's output-JSON files' format. (3) Updated sonic-buildimage/build_debian.sh, to add a new script file: files/scripts/saidump.sh into the host. This shell file does the below steps: For each ASIC0, such as ASIC0, 1. Save the Redis data. sudo sonic-db-cli -n asic$1 SAVE > /dev/null 2. Move dump files to /var/run/redisX/ docker exec database$1 sh -c "mv /var/lib/redis/dump.rdb /var/run/redis$1/" 3. Run rdb command to convert the dump files into JSON files docker exec syncd$1 sh -c "rdb --command json /var/run/redis$1/dump.rdb | tee /var/run/redis$1/dump.json > /dev/null" 4. Run saidump -r to update the JSON files' format as same as the saidump before. Then we can get the saidump result in standard output. docker exec syncd$1 sh -c "saidump -r /var/run/redis$1/dump.json -m 100" 5. clear sudo rm -f /var/run/redis$1/dump.rdb sudo rm -f /var/run/redis$1/dump.json (4) Update sonic-buildimage/src/sonic-utilities/scripts/generate_dump, to check the asic db size and if it is larger than xxx entries, then do with REDIS SAVE, otherwise, to do with old method: looping through each entry of Redis DB. --- build_debian.sh | 14 +--------- files/scripts/saidump.sh | 27 ++++++++++++------- .../docker-syncd-brcm-dnx/Dockerfile.j2 | 12 +++++++++ 3 files changed, 30 insertions(+), 23 deletions(-) diff --git a/build_debian.sh b/build_debian.sh index 3c8519b6288c..3fbc9065484f 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -729,22 +729,10 @@ SONIC_VERSION_CACHE=${SONIC_VERSION_CACHE} \ DBGOPT="${DBGOPT}" \ scripts/collect_host_image_version_files.sh $CONFIGURED_ARCH $IMAGE_DISTRO $TARGET_PATH $FILESYSTEM_ROOT -# Install libc6-dev and python3-dev for compiling python-lzf -sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install libc6-dev python3-dev - -# Install python-lzf -sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install 'python-lzf==0.2.4' - -# Install rdbtools -sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install 'rdbtools==0.1.15' - -# Uninstall libc6-dev and python3-dev for compiling python-lzf -sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y remove libc6-dev -sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y remove python3-dev - ## Copy saidump.sh sudo cp ./files/scripts/saidump.sh $FILESYSTEM_ROOT/usr/local/bin/saidump.sh sudo chmod +x $FILESYSTEM_ROOT/usr/local/bin/saidump.sh + # Remove GCC sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y remove gcc diff --git a/files/scripts/saidump.sh b/files/scripts/saidump.sh index 4deb2d0c3646..2d5f37a4929b 100755 --- a/files/scripts/saidump.sh +++ b/files/scripts/saidump.sh @@ -7,27 +7,34 @@ function debug() } save_saidump_by_rdb() { - debug "saidump.sh: [1] sonic-db-cli -n asic$1 SAVE." - sudo sonic-db-cli -n asic$1 SAVE > /dev/null - debug "saidump.sh: [2] Move dump.rdb to /var/run/redis$1/ in container database$1." - docker exec database$1 sh -c "mv /var/lib/redis/dump.rdb /var/run/redis$1/" + DEV=$1 # ASIC index to operate on + + if [ "$DEV" ]; then + debug "saidump.sh: [1] sonic-db-cli -n asic$DEV SAVE." + sonic-db-cli -n asic$DEV SAVE > /dev/null + else + debug "saidump.sh: [1] sonic-db-cli SAVE." + sonic-db-cli SAVE > /dev/null + fi + + debug "saidump.sh: [2] Move dump.rdb to /var/run/redis$DEV/ in container database$DEV." + docker exec database$DEV sh -c "mv /var/lib/redis/dump.rdb /var/run/redis$DEV/" debug "saidump.sh: [3] Run rdb command to convert the dump files into JSON files." - sudo python /usr/local/bin/rdb --command json /var/run/redis$1/dump.rdb | sudo tee /var/run/redis$1/dump.json > /dev/null + docker exec syncd$DEV sh -c "rdb --command json /var/run/redis$DEV/dump.rdb | tee /var/run/redis$DEV/dump.json > /dev/null" debug "saidump.sh: [4] Run saidump -r to update the JSON files' format as same as the saidump before. Then we can get the saidump's result in standard output." - docker exec syncd$1 sh -c "saidump -r /var/run/redis$1/dump.json" + docker exec syncd$DEV sh -c "saidump -r /var/run/redis$DEV/dump.json -m 100" debug "saidump.sh: [5] Clear temporary files." - sudo rm -f /var/run/redis$1/dump.rdb - sudo rm -f /var/run/redis$1/dump.json + sudo rm -f /var/run/redis$DEV/dump.rdb + sudo rm -f /var/run/redis$DEV/dump.json } NUM_ASICS=`python -c 'from sonic_py_common.multi_asic import get_num_asics; print(get_num_asics())'` - if (( $# == 0 && $NUM_ASICS == 1 )); then save_saidump_by_rdb #validate if the argument is an integer elif [[ "$1" =~ [0-9]+ ]] && (( $# == 1 && $1 >= 0 && $1 < $NUM_ASICS )) ; then - save_saidump_by_rdb $1 + save_saidump_by_rdb $1 else echo "The number of ASICS is $NUM_ASICS." echo "Usage:" diff --git a/platform/broadcom/docker-syncd-brcm-dnx/Dockerfile.j2 b/platform/broadcom/docker-syncd-brcm-dnx/Dockerfile.j2 index 8bf6a4e50289..49565bc3502e 100755 --- a/platform/broadcom/docker-syncd-brcm-dnx/Dockerfile.j2 +++ b/platform/broadcom/docker-syncd-brcm-dnx/Dockerfile.j2 @@ -23,6 +23,18 @@ RUN apt-get install -yf kmod ## BRCM uses ethtool to set host interface speed RUN apt-get install -y ethtool +## Install gcc, libc6-dev and python3-dev for compiling python-lzf +RUN apt-get -y install build-essential libc6-dev python3-dev + +## Install python-lzf +RUN pip3 install 'python-lzf==0.2.4' + +## Install rdbtools +RUN pip3 install 'rdbtools==0.1.15' + +## Uninstall gcc, libc6-dev and python3-dev for compiling python-lzf +RUN apt-get -y remove build-essential libc6-dev python3-dev + COPY ["files/dsserve", "files/bcmcmd", "start.sh", "start_led.sh", "bcmsh", "/usr/bin/"] RUN chmod +x /usr/bin/dsserve /usr/bin/bcmcmd