From c470b7dfd15a6b819dd15105f60b2e90c55b731f Mon Sep 17 00:00:00 2001 From: lixiaoyuner <35456895+lixiaoyuner@users.noreply.github.com> Date: Tue, 11 Jul 2023 13:16:29 +0800 Subject: [PATCH] Add health check probe for k8s upgrade containers. (#15223) #### Why I did it After k8s upgrade a container, k8s can only know the container is running, don't know the service's status inside container. So we need a probe inside container, k8s will call the probe to check whether the container is really ready. ##### Work item tracking - Microsoft ADO **(number only)**: 22453004 #### How I did it Add a health check probe inside config engine container, the probe will check whether the start service exit normally or not if the start service exists and call the python script to do container self-related specific checks if the script is there. The python script should be implemented by feature owner if it's needed. more details: [design doc](https://github.com/sonic-net/SONiC/blob/master/doc/kubernetes/health-check.md) #### How to verify it Check path /usr/bin/readiness_probe.sh inside container. #### Which release branch to backport (provide reason below if selected) - [ ] 201811 - [ ] 201911 - [ ] 202006 - [ ] 202012 - [ ] 202106 - [ ] 202111 - [x] 202205 - [x] 202211 #### Tested branch (Please provide the tested image version) - [x] 20220531.28 --- .../Dockerfile.j2 | 1 + .../docker-config-engine-buster/Dockerfile.j2 | 1 + rules/docker-config-engine-bullseye.mk | 1 + rules/docker-config-engine-buster.mk | 1 + rules/sonic-ctrmgrd.mk | 4 +++ src/sonic-ctrmgrd/ctrmgr/readiness_probe.sh | 35 +++++++++++++++++++ 6 files changed, 43 insertions(+) create mode 100644 src/sonic-ctrmgrd/ctrmgr/readiness_probe.sh diff --git a/dockers/docker-config-engine-bullseye/Dockerfile.j2 b/dockers/docker-config-engine-bullseye/Dockerfile.j2 index 57631df95c3e..f6804de903be 100644 --- a/dockers/docker-config-engine-bullseye/Dockerfile.j2 +++ b/dockers/docker-config-engine-bullseye/Dockerfile.j2 @@ -44,6 +44,7 @@ RUN pip3 install redis==4.5.4 # Copy files COPY ["files/swss_vars.j2", "/usr/share/sonic/templates/"] +COPY ["files/readiness_probe.sh", "/usr/bin/"] COPY ["files/container_startup.py", "/usr/share/sonic/scripts/"] ## Clean up diff --git a/dockers/docker-config-engine-buster/Dockerfile.j2 b/dockers/docker-config-engine-buster/Dockerfile.j2 index ccadb6e43957..084956eed5f5 100644 --- a/dockers/docker-config-engine-buster/Dockerfile.j2 +++ b/dockers/docker-config-engine-buster/Dockerfile.j2 @@ -44,6 +44,7 @@ RUN pip3 install redis==4.5.4 # Copy files COPY ["files/swss_vars.j2", "/usr/share/sonic/templates/"] +COPY ["files/readiness_probe.sh", "/usr/bin/"] COPY ["files/container_startup.py", "/usr/share/sonic/scripts/"] ## Clean up diff --git a/rules/docker-config-engine-bullseye.mk b/rules/docker-config-engine-bullseye.mk index 808905496e8f..084b4b82dc76 100644 --- a/rules/docker-config-engine-bullseye.mk +++ b/rules/docker-config-engine-bullseye.mk @@ -19,6 +19,7 @@ $(DOCKER_CONFIG_ENGINE_BULLSEYE)_LOAD_DOCKERS += $(DOCKER_BASE_BULLSEYE) $(DOCKER_CONFIG_ENGINE_BULLSEYE)_FILES += $(SWSS_VARS_TEMPLATE) $(DOCKER_CONFIG_ENGINE_BULLSEYE)_FILES += $(RSYSLOG_PLUGIN_CONF_J2) $(DOCKER_CONFIG_ENGINE_BULLSEYE)_FILES += $($(SONIC_CTRMGRD)_CONTAINER_SCRIPT) +$(DOCKER_CONFIG_ENGINE_BULLSEYE)_FILES += $($(SONIC_CTRMGRD)_HEALTH_PROBE) $(DOCKER_CONFIG_ENGINE_BULLSEYE)_FILES += $($(SONIC_CTRMGRD)_STARTUP_SCRIPT) $(DOCKER_CONFIG_ENGINE_BULLSEYE)_DBG_DEPENDS = $($(DOCKER_BASE_BULLSEYE)_DBG_DEPENDS) \ diff --git a/rules/docker-config-engine-buster.mk b/rules/docker-config-engine-buster.mk index 474617bbbd94..4d1e7e0a4b92 100644 --- a/rules/docker-config-engine-buster.mk +++ b/rules/docker-config-engine-buster.mk @@ -18,6 +18,7 @@ $(DOCKER_CONFIG_ENGINE_BUSTER)_LOAD_DOCKERS += $(DOCKER_BASE_BUSTER) $(DOCKER_CONFIG_ENGINE_BUSTER)_FILES += $(SWSS_VARS_TEMPLATE) $(DOCKER_CONFIG_ENGINE_BUSTER)_FILES += $(RSYSLOG_PLUGIN_CONF_J2) $(DOCKER_CONFIG_ENGINE_BUSTER)_FILES += $($(SONIC_CTRMGRD)_CONTAINER_SCRIPT) +$(DOCKER_CONFIG_ENGINE_BUSTER)_FILES += $($(SONIC_CTRMGRD)_HEALTH_PROBE) $(DOCKER_CONFIG_ENGINE_BUSTER)_FILES += $($(SONIC_CTRMGRD)_STARTUP_SCRIPT) $(DOCKER_CONFIG_ENGINE_BUSTER)_DBG_DEPENDS = $($(DOCKER_BASE_BUSTER)_DBG_DEPENDS) \ diff --git a/rules/sonic-ctrmgrd.mk b/rules/sonic-ctrmgrd.mk index 659a2cf4ace1..167d78c43c88 100644 --- a/rules/sonic-ctrmgrd.mk +++ b/rules/sonic-ctrmgrd.mk @@ -20,12 +20,16 @@ $($(SONIC_CTRMGRD)_CFG_JSON)_PATH = $($(SONIC_CTRMGRD)_FILES_PATH) $(SONIC_CTRMGRD)_SERVICE = ctrmgrd.service $($(SONIC_CTRMGRD)_SERVICE)_PATH = $($(SONIC_CTRMGRD)_FILES_PATH) +$(SONIC_CTRMGRD)_HEALTH_PROBE = readiness_probe.sh +$($(SONIC_CTRMGRD)_HEALTH_PROBE)_PATH = $($(SONIC_CTRMGRD)_FILES_PATH) + SONIC_PYTHON_WHEELS += $(SONIC_CTRMGRD) $(SONIC_CTRMGRD)_FILES = $($(SONIC_CTRMGRD)_CONTAINER_SCRIPT) $(SONIC_CTRMGRD)_FILES += $($(SONIC_CTRMGRD)_STARTUP_SCRIPT) $(SONIC_CTRMGRD)_FILES += $($(SONIC_CTRMGRD)_CFG_JSON) $(SONIC_CTRMGRD)_FILES += $($(SONIC_CTRMGRD)_SERVICE) +$(SONIC_CTRMGRD)_FILES += $($(SONIC_CTRMGRD)_HEALTH_PROBE) SONIC_COPY_FILES += $($(SONIC_CTRMGRD)_FILES) diff --git a/src/sonic-ctrmgrd/ctrmgr/readiness_probe.sh b/src/sonic-ctrmgrd/ctrmgr/readiness_probe.sh new file mode 100644 index 000000000000..9e796ca03816 --- /dev/null +++ b/src/sonic-ctrmgrd/ctrmgr/readiness_probe.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# This script is used by k8s to check the readiness of containers +# Check if the container is readiness or not, exit code 0 means readiness, others mean not readiness + +#### exit code contract, k8s only cares zero or not none-zero, but we want to use none-zero code to indicate different error +# 0: readiness +# 1: if the hook script is python code, the default crash exit code is 1 +# 2: supervisor start service doesn't exit normally +# other exit code: returned by post_check_script, define in the post_check_script, should not include 1,2 + +# check if the start service exists +# if the start service doesn't exist, do nothing +# if the start service exists, check if it exits normally +# if the start service doesn't exit normally, exit with code 2 +pre_check_service_name="start" +no_process_string="ERROR (no such process)" +service_status=$(supervisorctl status $pre_check_service_name) +if [[ $service_status != *"$no_process_string"* ]] && [[ $(echo $service_status |awk '{print $2}') != 'EXITED' ]]; then + exit 2 +fi + +# feature owner can add their own readiness check script +# check if the post_check_script exists +# if the post_check_script exists, run it +# if the post_check_script exits with non-zero code, exit with the code +post_check_script="/usr/bin/readiness_probe_hook" +if [ -x $post_check_script ]; then + $post_check_script + post_check_result=$? + if [ $post_check_result != 0 ]; then + exit $post_check_result + fi +fi + +exit 0