From 2a48dceb020a992c4274f20c5f97fc950c87de54 Mon Sep 17 00:00:00 2001 From: Tuomo Tanskanen Date: Tue, 19 Dec 2023 15:04:12 +0200 Subject: [PATCH 1/2] cleanup 02 script Cleanup the syntax so the fixes in the next commit stand out. Signed-off-by: Tuomo Tanskanen --- 02_configure_host.sh | 471 ++++++++++++++++++++++--------------------- 1 file changed, 240 insertions(+), 231 deletions(-) diff --git a/02_configure_host.sh b/02_configure_host.sh index 26c5c1907..b6a8db790 100755 --- a/02_configure_host.sh +++ b/02_configure_host.sh @@ -17,25 +17,27 @@ source lib/image_prepull.sh # kernel crashes with NULL pointer dereference # note(elfosardo): run this only if we have kvm support if grep -q vmx /proc/cpuinfo; then - sudo modprobe -r -a kvm_intel kvm - sudo modprobe kvm tdp_mmu=0 - sudo modprobe -a kvm kvm_intel + sudo modprobe -r -a kvm_intel kvm + sudo modprobe kvm tdp_mmu=0 + sudo modprobe -a kvm kvm_intel elif grep -q svm /proc/cpuinfo; then - sudo modprobe -r -a kvm_amd kvm - sudo modprobe kvm tdp_mmu=0 - sudo modprobe -a kvm kvm_amd + sudo modprobe -r -a kvm_amd kvm + sudo modprobe kvm tdp_mmu=0 + sudo modprobe -a kvm kvm_amd fi + # Clean, copy and extract local IPA -if [[ "${USE_LOCAL_IPA}" = "true" ]]; then - sudo rm -f "${IRONIC_DATA_DIR}/html/images/ironic-python-agent*" - sudo cp "${LOCAL_IPA_PATH}/ironic-python-agent.tar" "${IRONIC_DATA_DIR}/html/images" - sudo tar --extract --file "${IRONIC_DATA_DIR}/html/images/ironic-python-agent.tar" \ - --directory "${IRONIC_DATA_DIR}/html/images" - # avoid duplicating the same process in BMO run_local script - export USE_LOCAL_IPA="false" +if [[ "${USE_LOCAL_IPA}" == "true" ]]; then + sudo rm -f "${IRONIC_DATA_DIR}/html/images/ironic-python-agent*" + sudo cp "${LOCAL_IPA_PATH}/ironic-python-agent.tar" "${IRONIC_DATA_DIR}/html/images" + sudo tar --extract --file "${IRONIC_DATA_DIR}/html/images/ironic-python-agent.tar" \ + --directory "${IRONIC_DATA_DIR}/html/images" + # avoid duplicating the same process in BMO run_local script + export USE_LOCAL_IPA="false" fi -configure_minikube() { +configure_minikube() +{ minikube config set driver kvm2 minikube config set memory 4096 } @@ -43,30 +45,31 @@ configure_minikube() { # # Create Minikube VM and add correct interfaces # -init_minikube() { +init_minikube() +{ #If the vm exists, it has already been initialized if [[ ! "$(sudo virsh list --name --all)" =~ .*(minikube).* ]]; then - # Loop to ignore minikube issues - while /bin/true; do - minikube_error=0 - # This method, defined in lib/common.sh, will either ensure sockets are up'n'running - # for CS9 and RHEL9, or restart the libvirtd.service for other DISTRO - manage_libvirtd - configure_minikube - #NOTE(elfosardo): workaround for https://bugzilla.redhat.com/show_bug.cgi?id=2057769 - sudo mkdir -p "/etc/qemu/firmware" - sudo touch "/etc/qemu/firmware/50-edk2-ovmf-amdsev.json" - sudo su -l -c "minikube start --insecure-registry ${REGISTRY}" "${USER}" || minikube_error=1 - if [[ ${minikube_error} -eq 0 ]]; then - break - fi - sudo su -l -c 'minikube delete --all --purge' "${USER}" - # NOTE (Mohammed): workaround for https://github.com/kubernetes/minikube/issues/9878 - if ip link show virbr0 > /dev/null 2>&1; then - sudo ip link delete virbr0 - fi - done - sudo su -l -c "minikube stop" "${USER}" + # Loop to ignore minikube issues + while /bin/true; do + minikube_error=0 + # This method, defined in lib/common.sh, will either ensure sockets are up'n'running + # for CS9 and RHEL9, or restart the libvirtd.service for other DISTRO + manage_libvirtd + configure_minikube + #NOTE(elfosardo): workaround for https://bugzilla.redhat.com/show_bug.cgi?id=2057769 + sudo mkdir -p "/etc/qemu/firmware" + sudo touch "/etc/qemu/firmware/50-edk2-ovmf-amdsev.json" + sudo su -l -c "minikube start --insecure-registry ${REGISTRY}" "${USER}" || minikube_error=1 + if [[ ${minikube_error} -eq 0 ]]; then + break + fi + sudo su -l -c 'minikube delete --all --purge' "${USER}" + # NOTE (Mohammed): workaround for https://github.com/kubernetes/minikube/issues/9878 + if ip link show virbr0 > /dev/null 2>&1; then + sudo ip link delete virbr0 + fi + done + sudo su -l -c "minikube stop" "${USER}" fi MINIKUBE_IFACES="$(sudo virsh domiflist minikube)" @@ -75,44 +78,44 @@ init_minikube() { # so just attach it before next boot. As long as the # 02_configure_host.sh script does not run, the provisioning network does # not exist. Attempting to start Minikube will fail until it is created. - if ! echo "${MINIKUBE_IFACES}" | grep -w -q provisioning ; then - sudo virsh attach-interface --domain minikube \ - --model virtio --source provisioning \ - --type network --config + if ! echo "${MINIKUBE_IFACES}" | grep -w -q provisioning; then + sudo virsh attach-interface --domain minikube \ + --model virtio --source provisioning \ + --type network --config fi - if ! echo "${MINIKUBE_IFACES}" | grep -w -q external ; then - sudo virsh attach-interface --domain minikube \ - --model virtio --source external \ - --type network --config + if ! echo "${MINIKUBE_IFACES}" | grep -w -q external; then + sudo virsh attach-interface --domain minikube \ + --model virtio --source external \ + --type network --config fi } -if [[ "${EPHEMERAL_CLUSTER}" = "minikube" ]]; then - init_minikube +if [[ "${EPHEMERAL_CLUSTER}" == "minikube" ]]; then + init_minikube fi # Root needs a private key to talk to libvirt # See tripleo-quickstart-config/roles/virtbmc/tasks/configure-vbmc.yml -if sudo [ ! -f /root/.ssh/id_rsa_virt_power ]; then - sudo ssh-keygen -f /root/.ssh/id_rsa_virt_power -P "" - sudo cat /root/.ssh/id_rsa_virt_power.pub | sudo tee -a /root/.ssh/authorized_keys +if ! sudo test -f /root/.ssh/id_rsa_virt_power; then + sudo ssh-keygen -f /root/.ssh/id_rsa_virt_power -P "" + sudo cat /root/.ssh/id_rsa_virt_power.pub | sudo tee -a /root/.ssh/authorized_keys fi ANSIBLE_FORCE_COLOR=true ansible-playbook \ - -e "working_dir=$WORKING_DIR" \ - -e "num_nodes=$NUM_NODES" \ - -e "extradisks=$VM_EXTRADISKS" \ - -e "virthost=$HOSTNAME" \ - -e "platform=$NODES_PLATFORM" \ - -e "libvirt_firmware=$LIBVIRT_FIRMWARE" \ - -e "libvirt_secure_boot=$LIBVIRT_SECURE_BOOT" \ - -e "libvirt_domain_type=$LIBVIRT_DOMAIN_TYPE" \ - -e "default_memory=$TARGET_NODE_MEMORY" \ - -e "manage_external=$MANAGE_EXT_BRIDGE" \ - -e "provisioning_url_host=$BARE_METAL_PROVISIONER_URL_HOST" \ - -e "nodes_file=$NODES_FILE" \ - -e "node_hostname_format=$NODE_HOSTNAME_FORMAT" \ + -e "working_dir=${WORKING_DIR}" \ + -e "num_nodes=${NUM_NODES}" \ + -e "extradisks=${VM_EXTRADISKS}" \ + -e "virthost=${HOSTNAME}" \ + -e "platform=${NODES_PLATFORM}" \ + -e "libvirt_firmware=${LIBVIRT_FIRMWARE}" \ + -e "libvirt_secure_boot=${LIBVIRT_SECURE_BOOT}" \ + -e "libvirt_domain_type=${LIBVIRT_DOMAIN_TYPE}" \ + -e "default_memory=${TARGET_NODE_MEMORY}" \ + -e "manage_external=${MANAGE_EXT_BRIDGE}" \ + -e "provisioning_url_host=${BARE_METAL_PROVISIONER_URL_HOST}" \ + -e "nodes_file=${NODES_FILE}" \ + -e "node_hostname_format=${NODE_HOSTNAME_FORMAT}" \ -i vm-setup/inventory.ini \ -b vm-setup/setup-playbook.yml @@ -120,30 +123,30 @@ ANSIBLE_FORCE_COLOR=true ansible-playbook \ if ! sudo virsh pool-uuid default > /dev/null 2>&1 ; then sudo virsh pool-define /dev/stdin < - default - - /var/lib/libvirt/images - + default + + /var/lib/libvirt/images + EOF sudo virsh pool-start default sudo virsh pool-autostart default fi -if [[ $OS == ubuntu ]]; then - # source ubuntu_bridge_network_configuration.sh - # shellcheck disable=SC1091 - source ubuntu_bridge_network_configuration.sh - # shellcheck disable=SC1091 - source disable_apparmor_driver_libvirtd.sh +if [[ "${OS}" == "ubuntu" ]]; then + # source ubuntu_bridge_network_configuration.sh + # shellcheck disable=SC1091 + source ubuntu_bridge_network_configuration.sh + # shellcheck disable=SC1091 + source disable_apparmor_driver_libvirtd.sh else - if [ "$MANAGE_PRO_BRIDGE" == "y" ]; then - # Adding an IP address in the libvirt definition for this network results in - # dnsmasq being run, we don't want that as we have our own dnsmasq, so set - # the IP address here - if [ ! -e /etc/NetworkManager/system-connections/provisioning.nmconnection ] ; then - if [[ "${BARE_METAL_PROVISIONER_SUBNET_IPV6_ONLY}" = "true" ]]; then - sudo tee -a /etc/NetworkManager/system-connections/provisioning.nmconnection < Date: Wed, 20 Dec 2023 15:07:03 +0200 Subject: [PATCH 2/2] pass locally built images via config to tests Environment between "make" and "make test" is lost, so we are actually not using the locally built image in the pivoting tests. This has lead to false test results and breaking of integration. - Add locally built images into config, so they are passed to make test and used in pivoting tests, where Ironic is moved to another cluster - Add `set -u` to catch logic errors - Change EXT/INT/PRO_IF checks from running the variable content to checking the variable content - Rename some variables more logically, and also handle the LOCAL_IMAGE variables that have trailing / properly - In case of ironic and cluster-api, don't use regex to match image name, as they end up building wrong images (like cluster-api-provider-metal3, or ironic-client) Signed-off-by: Tuomo Tanskanen --- 02_configure_host.sh | 46 ++++++++++++++++++++++++++++---------------- lib/common.sh | 4 ++++ tests/run.sh | 10 ++++++++-- 3 files changed, 41 insertions(+), 19 deletions(-) diff --git a/02_configure_host.sh b/02_configure_host.sh index b6a8db790..9a9a7983a 100755 --- a/02_configure_host.sh +++ b/02_configure_host.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -xe +set -eux # shellcheck disable=SC1091 source lib/logging.sh @@ -13,6 +13,9 @@ source lib/releases.sh # shellcheck disable=SC1091 source lib/image_prepull.sh +# cleanup ci config file if it exists from earlier run +rm -f "${CI_CONFIG_FILE}" + # (workaround) disable tdp_mmu to avoid # kernel crashes with NULL pointer dereference # note(elfosardo): run this only if we have kvm support @@ -188,7 +191,7 @@ EOF sudo nmcli con up provisioning # Need to pass the provision interface for bare metal - if [[ "${PRO_IF}" ]]; then + if [[ -n "${PRO_IF}" ]]; then sudo tee -a /etc/NetworkManager/system-connections/"${PRO_IF}".nmconnection <>"${CI_CONFIG_FILE}" +export ${IMAGE_VAR/_LOCAL_IMAGE/_IMAGE}="${IMAGE_URL}" +EOF done # unset all *_IMAGE env vars that have a *_LOCAL_IMAGE counterpart to avoid @@ -392,6 +403,7 @@ done # consequently unset, it has to be redefined for local use if [[ "${BUILD_IRONIC_IMAGE_LOCALLY:-}" == "true" ]] || [[ -n "${IRONIC_LOCAL_IMAGE:-}" ]]; then IRONIC_IMAGE="${REGISTRY}/localimages/$(basename "${IRONIC_LOCAL_IMAGE}")" + export IRONIC_IMAGE fi VBMC_IMAGE=${VBMC_LOCAL_IMAGE:-${VBMC_IMAGE}} SUSHY_TOOLS_IMAGE=${SUSHY_TOOLS_LOCAL_IMAGE:-${SUSHY_TOOLS_IMAGE}} diff --git a/lib/common.sh b/lib/common.sh index 98b5569e7..eb1d5e0ff 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -30,6 +30,10 @@ fi # shellcheck disable=SC1090 source "${CONFIG}" +# CI config file for passing variables between make and make test +# used in 02 script and in run.sh +export CI_CONFIG_FILE="${TMP_DIR:-/tmp}/config_ci.sh" + # Set variables export MARIADB_HOST="mariaDB" export MARIADB_HOST_IP="127.0.0.1" diff --git a/tests/run.sh b/tests/run.sh index 2ad5c4a07..a6c958684 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -1,5 +1,6 @@ -#!/bin/bash -set -xe +#!/usr/bin/env bash + +set -eux METAL3_DIR="$(dirname "$(readlink -f "${0}")")/.." @@ -20,6 +21,11 @@ source "${METAL3_DIR}/lib/ironic_tls_setup.sh" # shellcheck disable=SC1090,SC1091 source "${METAL3_DIR}/lib/ironic_basic_auth.sh" +if [[ -r "${CI_CONFIG_FILE}" ]]; then + # shellcheck disable=SC1090,SC1091 + . "${CI_CONFIG_FILE}" +fi + # Disable SSH strong authentication export ANSIBLE_HOST_KEY_CHECKING=False