From f71b64088e99179e672014888bc4740c3de74519 Mon Sep 17 00:00:00 2001 From: Glenn Marcy Date: Tue, 30 Jul 2024 16:29:13 -0400 Subject: [PATCH] chore: updates for k8s distro to work with all guest os Signed-off-by: Glenn Marcy --- kubeinit/galaxy.yml | 10 +- kubeinit/group_vars/kubeinit_defaults.yml | 2 +- kubeinit/requirements.yml | 10 +- kubeinit/roles/kubeinit_k8s/tasks/main.yml | 2 +- .../kubeinit_k8s/tasks/prepare_cluster.yml | 117 ++---------------- .../tasks/cleanup_hypervisors.yml | 1 + .../tasks/deploy_centos_guest.yml | 2 +- .../tasks/deploy_debian_guest.yml | 69 +++++++++-- .../tasks/deploy_ubuntu_guest.yml | 68 ++++++++-- .../tasks/download_cloud_images.yml | 6 +- 10 files changed, 147 insertions(+), 140 deletions(-) diff --git a/kubeinit/galaxy.yml b/kubeinit/galaxy.yml index c1ea6ea70..ae0e27f0a 100644 --- a/kubeinit/galaxy.yml +++ b/kubeinit/galaxy.yml @@ -17,11 +17,11 @@ tags: - origin dependencies: ansible.posix: '==1.5.4' - ansible.utils: '==2.10.3' - community.crypto: '==2.13.1' - community.general: '==7.0.1' - community.libvirt: '==1.2.0' - containers.podman: '==1.10.2' + ansible.utils: '==5.0.0' + community.crypto: '==2.21.1' + community.general: '==9.2.0' + community.libvirt: '==1.3.0' + containers.podman: '==1.15.4' openvswitch.openvswitch: '==2.1.1' repository: 'https://github.com/kubeinit/kubeinit' diff --git a/kubeinit/group_vars/kubeinit_defaults.yml b/kubeinit/group_vars/kubeinit_defaults.yml index 3c7b32817..ba8dd5bff 100644 --- a/kubeinit/group_vars/kubeinit_defaults.yml +++ b/kubeinit/group_vars/kubeinit_defaults.yml @@ -17,7 +17,7 @@ cluster_node_configurations_docsplaceholder: 'we should have a cluster_node_conf cluster_node_default_distro: k8s: - os: centos + os: debian ocp: os: coreos okd: diff --git a/kubeinit/requirements.yml b/kubeinit/requirements.yml index b3f2e62e1..bcb5fa389 100644 --- a/kubeinit/requirements.yml +++ b/kubeinit/requirements.yml @@ -7,14 +7,14 @@ collections: - name: ansible.posix version: '1.5.4' - name: ansible.utils - version: '2.10.3' + version: '5.0.0' - name: community.crypto - version: '2.13.1' + version: '2.21.1' - name: community.general - version: '7.0.1' + version: '9.2.0' - name: community.libvirt - version: '1.2.0' + version: '1.3.0' - name: containers.podman - version: '1.10.2' + version: '1.15.4' - name: openvswitch.openvswitch version: '2.1.1' diff --git a/kubeinit/roles/kubeinit_k8s/tasks/main.yml b/kubeinit/roles/kubeinit_k8s/tasks/main.yml index 865cca329..bf0ad7425 100644 --- a/kubeinit/roles/kubeinit_k8s/tasks/main.yml +++ b/kubeinit/roles/kubeinit_k8s/tasks/main.yml @@ -187,7 +187,7 @@ - name: Label compute nodes ansible.builtin.command: | - kubectl label node {{ hostvars[compute_node].fqdn }} node-role.kubernetes.io/worker= + kubectl label node {{ compute_node }} node-role.kubernetes.io/worker= register: _result changed_when: "_result.rc == 0" loop: "{{ groups['all_compute_nodes'] | default([]) }}" diff --git a/kubeinit/roles/kubeinit_k8s/tasks/prepare_cluster.yml b/kubeinit/roles/kubeinit_k8s/tasks/prepare_cluster.yml index 66af7a71d..6853a4d22 100644 --- a/kubeinit/roles/kubeinit_k8s/tasks/prepare_cluster.yml +++ b/kubeinit/roles/kubeinit_k8s/tasks/prepare_cluster.yml @@ -58,130 +58,39 @@ register: _result changed_when: "_result.rc == 0" -#### Debian-based distributions -### -#### These instructions are for Kubernetes v1.30. -### -#### Update the apt package index and install packages needed to use the Kubernetes apt repository: -### -### sudo apt-get update -### # apt-transport-https may be a dummy package; if so, you can skip that package -### sudo apt-get install -y apt-transport-https ca-certificates curl gpg -### -#### Download the public signing key for the Kubernetes package repositories. The same signing key is used for all repositories so you can disregard the version in the URL: -### -### # If the directory `/etc/apt/keyrings` does not exist, it should be created before the curl command, read the note below. -### # sudo mkdir -p -m 755 /etc/apt/keyrings -### curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.30/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg -### -#### Note: -#### In releases older than Debian 12 and Ubuntu 22.04, directory /etc/apt/keyrings does not exist by default, and it should be created before the curl command. -### -#### Add the appropriate Kubernetes apt repository. Please note that this repository have packages only for Kubernetes 1.30; for other Kubernetes minor versions, you need to change the Kubernetes minor version in the URL to match your desired minor version (you should also check that you are reading the documentation for the version of Kubernetes that you plan to install). -### -### # This overwrites any existing configuration in /etc/apt/sources.list.d/kubernetes.list -### echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.30/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list -### -#### Update the apt package index, install kubelet, kubeadm and kubectl, and pin their version: -### -### sudo apt-get update -### sudo apt-get install -y kubelet kubeadm kubectl -### sudo apt-mark hold kubelet kubeadm kubectl -### -#### (Optional) Enable the kubelet service before running kubeadm: -### -### sudo systemctl enable --now kubelet -### -#### -#### END -#### -#### Distributions using deb packages -### -#### Install the dependencies for adding repositories -### apt-get update -### apt-get install -y software-properties-common curl -### -#### Add the Kubernetes repository -### curl -fsSL https://pkgs.k8s.io/core:/stable:/$KUBERNETES_VERSION/deb/Release.key | -### gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg -### -### echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$KUBERNETES_VERSION/deb/ /" | -### tee /etc/apt/sources.list.d/kubernetes.list -### -#### Add the CRI-O repository -### curl -fsSL https://pkgs.k8s.io/addons:/cri-o:/stable:/$CRIO_VERSION/deb/Release.key | -### gpg --dearmor -o /etc/apt/keyrings/cri-o-apt-keyring.gpg -### -### echo "deb [signed-by=/etc/apt/keyrings/cri-o-apt-keyring.gpg] https://pkgs.k8s.io/addons:/cri-o:/stable:/$CRIO_VERSION/deb/ /" | -### tee /etc/apt/sources.list.d/cri-o.list -### -#### Install the packages -### apt-get update -### apt-get install -y cri-o kubelet kubeadm kubectl -### -#### Start CRI-O -### systemctl start crio.service -### -#### Bootstrap a cluster -### swapoff -a -### modprobe br_netfilter -### sysctl -w net.ipv4.ip_forward=1 -### -### kubeadm init -#### -#### END -#### - - when: hostvars[kubeinit_provision_service_node].os != 'centos' + - when: hostvars[kubeinit_provision_service_node].os == 'debian' block: - name: Add kubernetes repo for latest kubectl (Debian) ansible.builtin.shell: | set -eo pipefail + apt-get install -y apt-transport-https ca-certificates curl gpg + mkdir -p /etc/apt/keyrings + curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.30/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.30/deb/ /' | tee /etc/apt/sources.list.d/kubernetes.list apt-get update - apt-get install -y software-properties-common curl - - curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.28/deb/Release.key | - gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg - echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.28/deb/ /" | - tee /etc/apt/sources.list.d/kubernetes.list - - curl -fsSL https://pkgs.k8s.io/addons:/cri-o:/prerelease:/main/deb/Release.key | - gpg --dearmor -o /etc/apt/keyrings/cri-o-apt-keyring.gpg - echo "deb [signed-by=/etc/apt/keyrings/cri-o-apt-keyring.gpg] https://pkgs.k8s.io/addons:/cri-o:/prerelease:/main/deb/ /" | - tee /etc/apt/sources.list.d/cri-o.list - - apt-get update - apt-get install -y cri-o kubelet kubeadm kubectl - - systemctl start crio.service + apt-get install -y kubectl + apt-mark hold kubectl args: executable: /bin/bash register: _result changed_when: "_result.rc == 0" - ### - # The Project's prerelease:/main prefix at the CRI-O's package path, can be replaced with stable:/v1.28, stable:/v1.29, prerelease:/v1.28 or prerelease:/v1.29 if another stream package is used. - ### + - when: hostvars[kubeinit_provision_service_node].os == 'ubuntu' + block: - name: Add kubernetes repo for latest kubectl (Ubuntu) ansible.builtin.shell: | set -eo pipefail - apt-get install -y apt-transport-https ca-certificates curl gnupg + apt-get install -y apt-transport-https ca-certificates curl gpg curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.30/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg - chmod 644 /etc/apt/keyrings/kubernetes-apt-keyring.gpg # allow unprivileged APT programs to read this keyring echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.30/deb/ /' | tee /etc/apt/sources.list.d/kubernetes.list - chmod 644 /etc/apt/sources.list.d/kubernetes.list # helps tools such as command-not-found to work correctly - apt-get update --allow-insecure-repositories + apt-get update + apt-get install -y kubectl + apt-mark hold kubectl args: executable: /bin/bash register: _result changed_when: "_result.rc == 0" - - name: Install services requirements - ansible.builtin.package: - name: - - kubectl - state: present - use: apt - delegate_to: "{{ kubeinit_provision_service_node }}" diff --git a/kubeinit/roles/kubeinit_libvirt/tasks/cleanup_hypervisors.yml b/kubeinit/roles/kubeinit_libvirt/tasks/cleanup_hypervisors.yml index 5703c3e17..75cc84f67 100644 --- a/kubeinit/roles/kubeinit_libvirt/tasks/cleanup_hypervisors.yml +++ b/kubeinit/roles/kubeinit_libvirt/tasks/cleanup_hypervisors.yml @@ -198,6 +198,7 @@ kubeinit_deployment_node_name: "{{ item[0] }}" service_name: "{{ kubeinit_cluster_name }}-{{ item[1] }}" delegate_to: "{{ kubeinit_deployment_node_name }}" + when: false - name: Remove any previous services podman pods containers.podman.podman_pod: diff --git a/kubeinit/roles/kubeinit_libvirt/tasks/deploy_centos_guest.yml b/kubeinit/roles/kubeinit_libvirt/tasks/deploy_centos_guest.yml index 52ab3ba72..09d694ef0 100644 --- a/kubeinit/roles/kubeinit_libvirt/tasks/deploy_centos_guest.yml +++ b/kubeinit/roles/kubeinit_libvirt/tasks/deploy_centos_guest.yml @@ -203,7 +203,7 @@ state: touch mode: '0644' - - name: Adding repository details in Kubernetes repo file. + - name: Adding repository details in CRI-O repo file. ansible.builtin.blockinfile: path: /etc/yum.repos.d/cri-o.repo block: | diff --git a/kubeinit/roles/kubeinit_libvirt/tasks/deploy_debian_guest.yml b/kubeinit/roles/kubeinit_libvirt/tasks/deploy_debian_guest.yml index b37ae5feb..9351c71ad 100644 --- a/kubeinit/roles/kubeinit_libvirt/tasks/deploy_debian_guest.yml +++ b/kubeinit/roles/kubeinit_libvirt/tasks/deploy_debian_guest.yml @@ -117,6 +117,15 @@ - name: Configure common requirements in Debian guests block: + - name: Update packages + ansible.builtin.command: apt update + + - name: Install resolvconf + ansible.builtin.package: + name: resolvconf + state: present + use: apt + - name: Make sure base file exists ansible.builtin.copy: content: "" @@ -172,18 +181,11 @@ - name: Force apt-get update ansible.builtin.shell: | apt-get update - apt-get install -y gnupg args: executable: /bin/bash register: _result changed_when: "_result.rc == 0" - - name: Update packages - ansible.builtin.package: - name: "*" - state: latest - use: apt - - name: Disable SWAP ansible.builtin.shell: | swapoff -a @@ -211,6 +213,59 @@ register: _result changed_when: "_result.rc == 0" + - name: Enable kernel modules and IP forward + ansible.builtin.shell: | + modprobe br_netfilter + echo br_netfilter > /etc/modules-load.d/br_netfilter.conf + #modprobe overlay + #echo overlay > /etc/modules-load.d/overlay.conf + sysctl -w net.ipv4.ip_forward=1 + #sysctl -w net.bridge.bridge-nf-call-iptables=1 + args: + executable: /bin/bash + register: _result + changed_when: "_result.rc == 0" + + - name: Add kubernetes repo for latest kubectl (Ubuntu) + ansible.builtin.shell: | + set -eo pipefail + export DEBIAN_FRONTEND=noninteractive + apt-get install -y apt-transport-https ca-certificates curl gpg + mkdir -p /etc/apt/keyrings + curl -fsSL https://pkgs.k8s.io/core:/stable:/v{{ kubeinit_k8s_kubernetes_version }}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v{{ kubeinit_k8s_kubernetes_version }}/deb/ /' | tee /etc/apt/sources.list.d/kubernetes.list + curl -fsSL https://pkgs.k8s.io/addons:/cri-o:/stable:/v{{ kubeinit_k8s_kubernetes_version }}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/cri-o-apt-keyring.gpg + echo "deb [signed-by=/etc/apt/keyrings/cri-o-apt-keyring.gpg] https://pkgs.k8s.io/addons:/cri-o:/stable:/v{{ kubeinit_k8s_kubernetes_version }}/deb/ /" | tee /etc/apt/sources.list.d/cri-o.list + apt-get update + args: + executable: /bin/bash + register: _result + changed_when: "_result.rc == 0" + + - name: Install requirements + ansible.builtin.shell: | + apt-get install -y cri-o={{ kubeinit_k8s_kubernetes_version }}.* kubelet={{ kubeinit_k8s_kubernetes_version }}.* kubeadm={{ kubeinit_k8s_kubernetes_version }}.* kubectl={{ kubeinit_k8s_kubernetes_version }}.* + apt-mark hold cri-o kubelet kubeadm kubectl + args: + executable: /bin/bash + register: _result + changed_when: "_result.rc == 0" + + - name: Enable/start/status cri-o + ansible.builtin.shell: | + systemctl enable crio + systemctl start crio + systemctl status crio + args: + executable: /bin/bash + register: _result + changed_when: "_result.rc == 0" + + - name: Enable kubelet + ansible.builtin.systemd: + name: kubelet + enabled: yes + - name: Update packages ansible.builtin.package: name: "*" diff --git a/kubeinit/roles/kubeinit_libvirt/tasks/deploy_ubuntu_guest.yml b/kubeinit/roles/kubeinit_libvirt/tasks/deploy_ubuntu_guest.yml index b7837ae79..3aa52cdf9 100644 --- a/kubeinit/roles/kubeinit_libvirt/tasks/deploy_ubuntu_guest.yml +++ b/kubeinit/roles/kubeinit_libvirt/tasks/deploy_ubuntu_guest.yml @@ -116,19 +116,8 @@ - name: Configure common requirements in Ubuntu guests block: - - name: Add kubernetes repo for latest kubectl (Ubuntu) - ansible.builtin.shell: | - set -eo pipefail - apt-get install -y apt-transport-https ca-certificates curl gnupg - curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.30/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg - chmod 644 /etc/apt/keyrings/kubernetes-apt-keyring.gpg # allow unprivileged APT programs to read this keyring - echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.30/deb/ /' | tee /etc/apt/sources.list.d/kubernetes.list - chmod 644 /etc/apt/sources.list.d/kubernetes.list # helps tools such as command-not-found to work correctly - apt-get update --allow-insecure-repositories - args: - executable: /bin/bash - register: _result - changed_when: "_result.rc == 0" + - name: Update packages + ansible.builtin.command: apt update - name: Install resolvconf ansible.builtin.package: @@ -223,6 +212,59 @@ register: _result changed_when: "_result.rc == 0" + - name: Enable kernel modules and IP forward + ansible.builtin.shell: | + modprobe br_netfilter + echo br_netfilter > /etc/modules-load.d/br_netfilter.conf + #modprobe overlay + #echo overlay > /etc/modules-load.d/overlay.conf + sysctl -w net.ipv4.ip_forward=1 + #sysctl -w net.bridge.bridge-nf-call-iptables=1 + args: + executable: /bin/bash + register: _result + changed_when: "_result.rc == 0" + + - name: Add kubernetes repo for latest kubectl (Ubuntu) + ansible.builtin.shell: | + set -eo pipefail + export DEBIAN_FRONTEND=noninteractive + apt-get install -y apt-transport-https ca-certificates curl gpg + mkdir -p /etc/apt/keyrings + curl -fsSL https://pkgs.k8s.io/core:/stable:/v{{ kubeinit_k8s_kubernetes_version }}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v{{ kubeinit_k8s_kubernetes_version }}/deb/ /' | tee /etc/apt/sources.list.d/kubernetes.list + curl -fsSL https://pkgs.k8s.io/addons:/cri-o:/stable:/v{{ kubeinit_k8s_kubernetes_version }}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/cri-o-apt-keyring.gpg + echo "deb [signed-by=/etc/apt/keyrings/cri-o-apt-keyring.gpg] https://pkgs.k8s.io/addons:/cri-o:/stable:/v{{ kubeinit_k8s_kubernetes_version }}/deb/ /" | tee /etc/apt/sources.list.d/cri-o.list + apt-get update + args: + executable: /bin/bash + register: _result + changed_when: "_result.rc == 0" + + - name: Install requirements + ansible.builtin.shell: | + apt-get install -y cri-o={{ kubeinit_k8s_kubernetes_version }}.* kubelet={{ kubeinit_k8s_kubernetes_version }}.* kubeadm={{ kubeinit_k8s_kubernetes_version }}.* kubectl={{ kubeinit_k8s_kubernetes_version }}.* + apt-mark hold cri-o kubelet kubeadm kubectl + args: + executable: /bin/bash + register: _result + changed_when: "_result.rc == 0" + + - name: Enable/start/status cri-o + ansible.builtin.shell: | + systemctl enable crio + systemctl start crio + systemctl status crio + args: + executable: /bin/bash + register: _result + changed_when: "_result.rc == 0" + + - name: Enable kubelet + ansible.builtin.systemd: + name: kubelet + enabled: yes + - name: Update packages ansible.builtin.package: name: "*" diff --git a/kubeinit/roles/kubeinit_libvirt/tasks/download_cloud_images.yml b/kubeinit/roles/kubeinit_libvirt/tasks/download_cloud_images.yml index be0d16f35..86c87a173 100644 --- a/kubeinit/roles/kubeinit_libvirt/tasks/download_cloud_images.yml +++ b/kubeinit/roles/kubeinit_libvirt/tasks/download_cloud_images.yml @@ -61,12 +61,12 @@ - name: Create tuples for hosts and cloud images ansible.builtin.set_fact: - all_hosts_cloud_images: "{{ (all_hosts_cloud_images | default([])) + ([hypervisor] | product([kubeinit_libvirt_cloud_images[_cluster_node_os]])) }}" + all_hosts_cloud_images: "{{ (all_hosts_cloud_images | default([])) + ([hypervisor] | product([kubeinit_libvirt_cloud_images[guest_os]])) }}" loop: "{{ all_hosts_guest_vms }}" vars: hypervisor: "{{ item[0] }}" - _cluster_node_os: "{{ hostvars['kubeinit-defaults'].cluster_node_default_distro[kubeinit_cluster_distro].os }}" - when: kubeinit_libvirt_cloud_images[_cluster_node_os] is defined + guest_os: "{{ hostvars[item[1]].os }}" + when: kubeinit_libvirt_cloud_images[guest_os] is defined - name: Remove duplicates ansible.builtin.set_fact: