diff --git a/LICENSE b/LICENSE index c6af185..f33fb03 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License -Copyright (c) 2020 Deavon McCaffery, Tiffany Wang, and Contributors +Copyright (c) 2020-2021 Deavon McCaffery, Tiffany Wang, and Contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/bridge.sh b/bridge.sh index b1214d7..8c5c383 100755 --- a/bridge.sh +++ b/bridge.sh @@ -1,4 +1,4 @@ -#! /usr/bin/env bash +#! /usr/bin/env sh set -eu # install all prerequisites @@ -9,38 +9,42 @@ sudo apt-get install qemu-kvm libvirt-daemon-system \ # make sure libvirtd is enabled sudo systemctl enable libvirtd -# get a list of active ethernet devices (deal with whitespace) -OLDIFS=$IFS -IFS=$'\n' -ACTIVE_CONNECTIONS=( $(nmcli -get-values NAME,DEVICE,TYPE connection show --active | grep ethernet | grep -v bridge | cut -d ':' -f 1,2) ) -IFS=$OLDIFS - -if [ -z "${ACTIVE_CONNECTIONS:-}" ]; then - echo - echo "no active connections to bridge..." - exit 1 -fi - # delete and recreate the bridge sudo nmcli connection delete br0 || true sudo nmcli con add ifname br0 type bridge con-name br0 || true sudo nmcli con modify br0 bridge.stp no # disable netfilter for the bridge network to avoid issues with docker network configs -sudo cat << EOF > /etc/sysctl.d/bridge.conf +cat << EOF | sudo tee /etc/sysctl.d/bridge.conf net.bridge.bridge-nf-call-ip6tables = 0 net.bridge.bridge-nf-call-iptables = 0 net.bridge.bridge-nf-call-arptables = 0 +net.ipv4.ip_forward = 1 EOF # activate the netfilter configuration for the bridge device -sudo cat << EOF > /etc/udev/rules.d/99-bridge.rules +cat << EOF | sudo tee /etc/udev/rules.d/99-bridge.rules ACTION=="add", SUBSYSTEM=="module", KERNEL=="br_netfilter", RUN+="/sbin/sysctl -p /etc/sysctl.d/bridge.conf" EOF # update network configuration sudo netplan apply +# wait for connections +sleep 15 + +# get a list of active ethernet devices (deal with whitespace) +OLDIFS=$IFS +IFS=$'\n' +ACTIVE_CONNECTIONS=( $(nmcli -get-values NAME,DEVICE,TYPE connection show --active | grep ethernet | grep -v bridge | cut -d ':' -f 1,2) ) +IFS=$OLDIFS + +if [ -z "${ACTIVE_CONNECTIONS:-}" ]; then + echo + echo "no active connections to bridge..." + exit 1 +fi + # iterate over each device for connection in "${ACTIVE_CONNECTIONS[@]}"; do diff --git a/cloud_init.cfg.template b/cloud_init.cfg.template index 170cd43..bff6226 100644 --- a/cloud_init.cfg.template +++ b/cloud_init.cfg.template @@ -35,6 +35,3 @@ bootcmd: runcmd: - mkdir -p /home/kube-admin/.ssh || true - - curl https://github.com/GH_USER.keys | tee -a /home/kube-admin/.ssh/authorized_keys - - chmod -R u=rwX,g=rX,o= /home/kube-admin/.ssh - - chown -R kube-admin:kube-admin /home/kube-admin/.ssh diff --git a/kvm.sh b/kvm.sh index f6526d6..2f30e93 100755 --- a/kvm.sh +++ b/kvm.sh @@ -1,6 +1,73 @@ -#! /usr/bin/env bash +#! /usr/bin/env sh +# shellcheck disable=SC2155 -set -e +set -eu + +FORMAT_CLEAR=$(tput sgr0) # CLEAR ALL FORMAT +FORMAT_BOLD=$(tput bold) # SET BRIGH + +CLR_RED=$(tput setaf 1) # ANSI RED +CLR_GREEN=$(tput setaf 2) # ANSI GREEN +CLR_YELLOW=$(tput setaf 3) # ANSI + +CLR_BRIGHT_RED="$FORMAT_BOLD$CLR_RED" # BRIGHT RED +CLR_BRIGHT_GREEN="$FORMAT_BOLD$CLR_GREEN" # BRIGHT GREEN +CLR_BRIGHT_YELLOW="$FORMAT_BOLD$CLR_YELLOW" # BRIGHT + +CLR_SUCCESS=$CLR_BRIGHT_GREEN +CLR_WARN=$CLR_BRIGHT_YELLOW +CLR_FAIL=$CLR_BRIGHT_RED + +print_success() +{ + printf "${CLR_SUCCESS}%s${FORMAT_CLEAR}\n" "$@" +} + +print_warn() +{ + printf "${CLR_WARN}%s${FORMAT_CLEAR}\n" "$@" +} + +print_fail() +{ + printf "${CLR_FAIL}%s${FORMAT_CLEAR}\n" "$@" >&2 +} + +help() { + print_success \ + "" \ + "KVM Configuration for Kubernetes" \ + "" \ + "Usage: ./kvm.sh [OPTIONS]" + print_warn \ + "" \ + "Options:" \ + "" \ + "--prefix DEFAULT: $(whoami), the prefix to use for kube node names" \ + "--masters DEFAULT: 1, the number of master nodes to create (min: 1, max: 50)" \ + "--workers DEFAULT: 1, the number of worker nodes to create (min: 1, max: 50)" \ + "--os-version DEFAULT: 7, the version of CentOS to use" \ + "" \ + "--cpu DEFAULT: 2, the number of CPUs for each node" \ + "--memory DEFAULT: 4096, the amount of memory to allocate for each node" \ + "" \ + "--network DEFAULT: bridge, the network for bridging VMs within qemu" \ + "--domain DEFAULT: local, the dns domain used to register IPs" \ + "" \ + "NOTE: ONLY THE LAST OCTET OF THE IP ADDRESSES ARE INCREMENTED, PLEASE ENSURE THERE IS ENOUGH IPS AVAILABLE" \ + "" \ + "--gh-user DEFAULT: tiffanywang3, the github username used to retrieve SSH keys." \ + " This can be specified multiple times" + print_fail \ + "" \ + "Example:" \ + "" \ + "./kvm.sh --masters 3 --workers 3 --cpu 4 --memory 8192 --gh-user somebody" +} + +PREFIX=$(whoami)- +NETWORK="bridge" +DOMAIN="local" MASTER_NODE_COUNT=1 WORKER_NODE_COUNT=1 @@ -9,192 +76,238 @@ CENTOS_VERSION=7 CPU_LIMIT=2 MEMORY_LIMIT=4096 -GH_USER=tiffanywang3 - -__kvm_help() { - echo - echo "KVM Configuration for Kubernetes" - echo - echo "Usage: ./kvm.sh [OPTIONS]" - echo - echo "Options:" - echo - echo -e "--masters\tDEFAULT: 1, the number of master nodes to create" - echo -e "--workers\tDEFAULT: 1, the number of worker nodes to create" - echo -e "--os-version\tDEFAULT: 7, the version of CentOS to use" - echo -e "--cpu\t\tDEFAULT: 2, the number of CPUs for each node" - echo -e "--memory\tDEFAULT: 4096, the amount of memory to allocate for each node" - echo -e "--gh-user\tDEFAULT: tiffanywang3, the github username used to retrieve SSH keys" - echo - echo "Example:" - echo - echo "./kvm.sh --masters 3 --workers 3 --cpu 4 --memory 8192 --gh-user somebody" -} - -while [[ $# > 0 ]]; do - case $1 in - --masters) - MASTER_NODE_COUNT=$2 - shift - ;; - --workers) - WORKER_NODE_COUNT=$2 - shift - ;; - --os-version) - CENTOS_VERSION=$2 - shift - ;; - --cpu) - CPU_LIMIT=$2 - shift - ;; - --memory) - MEMORY_LIMIT=$2 - shift - ;; - --gh-user) - GH_USER=$2 - shift - ;; - --help) - __kvm_help - exit 0 - ;; - *) - echo "unknown option: $1" - echo - __kvm_help - exit 1 - ;; - esac - shift +GH_USER= + +while [ $# -gt 0 ]; do + case $1 in + --prefix) + PREFIX="$2"- + shift + ;; + --masters) + MASTER_NODE_COUNT=$2 + shift + ;; + --workers) + WORKER_NODE_COUNT=$2 + shift + ;; + --os-version) + CENTOS_VERSION=$2 + shift + ;; + --cpu) + CPU_LIMIT=$2 + shift + ;; + --memory) + MEMORY_LIMIT=$2 + shift + ;; + --network) + NETWORK=$2 + shift + ;; + --domain) + DOMAIN=$2 + shift + ;; + --gh-user) + GH_USER="$2 $GH_USER" + shift + ;; + --help) + help + exit 0 + ;; + *) + printf "unknown option: %s\n\n" "$1" + help + exit 1 + ;; + esac + shift done -OS_VARIANT="centos${CENTOS_VERSION}.0" -BASE_IMAGE=CentOS-${CENTOS_VERSION}-x86_64-GenericCloud.qcow2 - -if [ ! -f $BASE_IMAGE ]; then - echo "Downloading $BASE_IMAGE...." - wget http://cloud.centos.org/centos/7/images/$BASE_IMAGE -O $BASE_IMAGE +# default the gh user if unspecified +if [ -z "$GH_USER" ]; then + GH_USER=tiffanywang3 fi -VM_NETWORKS="" +OS_VARIANT="centos$CENTOS_VERSION.0" +BASE_IMAGE=CentOS-$CENTOS_VERSION-x86_64-GenericCloud.qcow2 -# setup networks -for network in $(sudo virsh net-list | grep active | tr -s ' ' | cut -d ' ' -f 2); do - VM_NETWORKS="${VM_NETWORKS} --network network:${network}" -done +if [ ! -f "$BASE_IMAGE" ]; then + print_success "Downloading $BASE_IMAGE...." + wget http://cloud.centos.org/centos/7/images/"$BASE_IMAGE" -O "$BASE_IMAGE" +fi # install all prerequisites sudo apt update sudo apt-get install qemu-kvm libvirt-daemon-system \ - libvirt-clients libnss-libvirt virtinst + libvirt-clients libnss-libvirt virtinst # make sure libvirtd is enabled sudo systemctl enable libvirtd -# add user to libvirt group -sudo usermod -a -G libvirt ${USER} +# add user to libvirt and kvm groups +sudo usermod -a -G libvirt "$USER" +sudo usermod -a -G kvm "$USER" # delete existing nodes -for node in $(sudo virsh list --all --name | grep "kube-"); do - sudo virsh shutdown $node - sudo virsh destroy $node - sudo virsh undefine $node - - rm -f $node.qcow2 - rm -f $node-init.img - rm -f $node-metadata +for node in $(sudo virsh list --all --name | grep "kube-${PREFIX}"); do + sudo virsh shutdown "$node" + sudo virsh destroy "$node" + sudo virsh undefine "$node" done +# remove existing configs +rm -f kube-"$PREFIX"* + # ensure base image is accessible -sudo chmod u=rw,go=r $BASE_IMAGE +sudo chmod u=rw,go=r "$BASE_IMAGE" -# create cloud-init config -sed "s/GH_USER/${GH_USER}/g" cloud_init.cfg.template > cloud_init.cfg +# create the cloud-init config +cat cloud_init.cfg.template > cloud_init.cfg -# create the haproxy config -cat haproxy.cfg.template > haproxy.cfg +# emit every user key into authorized keys +for user in $GH_USER; do + printf " - curl https://github.com/%s.keys >> /home/kube-admin/.ssh/authorized_keys\n" "$user" >> cloud_init.cfg +done + +printf " %s\n" \ + "- chmod -R u=rwX,g=rX,o= /home/kube-admin/.ssh" \ + "- chown -R kube-admin:kube-admin /home/kube-admin/.ssh" >> cloud_init.cfg # create a virtual machine create_vm() { - hostname=$1 - snapshot=$hostname.qcow2 - init=$hostname-init.img - - # create snapshot and increase size to 30GB - qemu-img create -b $BASE_IMAGE -f qcow2 -F qcow2 $snapshot 30G - qemu-img info $snapshot - - # insert metadata into init image - echo "instance-id: $(uuidgen || echo i-abcdefg)" > $hostname-metadata - echo "local-hostname: $hostname.local" >> $hostname-metadata - - cloud-localds -v --network-config=network.cfg $init cloud_init.cfg $hostname-metadata - - # ensure file permissions belong to kvm group - sudo chmod ug=rw,o= $snapshot - sudo chown $USER:kvm $snapshot $init - - # create the vm - sudo virt-install --name $hostname \ - --virt-type kvm --memory ${MEMORY_LIMIT} --vcpus ${CPU_LIMIT} \ - --boot hd,menu=on \ - --disk path=$init,device=cdrom \ - --disk path=$snapshot,device=disk \ - --graphics vnc \ - --os-type Linux --os-variant ${OS_VARIANT} \ - ${VM_NETWORKS} \ - --autostart \ - --noautoconsole - - # set the timeout - sudo virsh guest-agent-timeout $hostname --timeout 60 + hostname=$1 + + snapshot=$hostname.qcow2 + init=$hostname-init.img + metadata=$hostname-metadata + cloud_cfg=$hostname-cloud_init.cfg + network_cfg=$hostname-network.cfg + + # copy the cloud_init.cfg + cp cloud_init.cfg "$cloud_cfg" + + if [ "$hostname" = "kube-${PREFIX}proxy" ]; then + # modify the cloud_init to include the haproxy.cfg + cat <<- EOF >> "$cloud_cfg" + - systemctl enable haproxy.service + - systemctl start haproxy.service + + write_files: + - path: /etc/haproxy/haproxy.cfg + encoding: base64 + content: $(base64 -w 0 < kube-"${PREFIX}"proxy-haproxy.cfg) + EOF + fi + + # create snapshot and increase size to 30GB + qemu-img create -b "$BASE_IMAGE" -f qcow2 -F qcow2 "$snapshot" 30G + qemu-img info "$snapshot" + + # insert metadata into init image + printf "instance-id: %s\n" "$(uuidgen || printf i-abcdefg)" > "$metadata" + printf "local-hostname: %s\n" "$hostname" >> "$metadata" + printf "hostname: %s.%s\n" "$hostname" "$DOMAIN" >> "$metadata" + + # create the network config + cp network.cfg.template "$network_cfg" + + # setup the cloud-init metadata + cloud-localds -v --network-config="$network_cfg" "$init" "$cloud_cfg" "$metadata" + + # ensure file permissions belong to kvm group + sudo chmod ug=rw,o= "$snapshot" + sudo chown "$USER":kvm "$snapshot" "$init" + + # create the vm + sudo virt-install --name "$hostname" \ + --virt-type kvm --memory "$MEMORY_LIMIT" --vcpus "$CPU_LIMIT" \ + --boot hd,menu=on \ + --disk path="$init",device=cdrom \ + --disk path="$snapshot",device=disk \ + --graphics vnc \ + --os-type Linux --os-variant "$OS_VARIANT" \ + --network network:default \ + --network network:"$NETWORK" \ + --autostart \ + --noautoconsole + + # set the timeout + sudo virsh guest-agent-timeout "$hostname" --timeout 60 } -# iterate over each node -for (( i=1; i<=$MASTER_NODE_COUNT; i++ )); do - # create the vm for the node - create_vm kube-master-$(printf "%02d" $i) +: $((i=1)) +while [ $((i<=MASTER_NODE_COUNT)) -ne 0 ]; do + create_vm kube-"${PREFIX}"master-"$(printf "%02d" "$i")" + : $((i=i+1)) done -# iterate over each node -for (( i=1; i<=$WORKER_NODE_COUNT; i++ )); do - # create the vm for the node - create_vm kube-worker-$(printf "%02d" $i) +: $((i=1)) +while [ $((i<=WORKER_NODE_COUNT)) -ne 0 ]; do + create_vm kube-"${PREFIX}"worker-"$(printf "%02d" "$i")" + : $((i=i+1)) done -# wait for nodes to come up -sleep 60 +# create the haproxy config +cp haproxy.cfg.template kube-"${PREFIX}"proxy-haproxy.cfg -# iterate over each node -for (( i=1; i<=$MASTER_NODE_COUNT; i++ )); do - # get the name of the master - name=kube-master-$(printf "%02d" $i) - ip=$(sudo virsh domifaddr --domain ${name} --source agent | grep -w eth1 | egrep -o '([[:digit:]]{1,3}\.){3}[[:digit:]]{1,3}') +# iterate over each master node +: $((i=1)) +while [ $((i<=MASTER_NODE_COUNT)) -ne 0 ]; do + name=kube-"$PREFIX"master-$(printf "%02d" "$i") - # add the ip to the haproxy config - echo " server $name $ip:6443 check" >> haproxy.cfg -done + until sudo virsh domifaddr --domain "$name" --source agent 2>/dev/null | grep -w eth0 | grep -w ipv4 1>/dev/null; do + print_warn "waiting for $name : eth0..." + sleep 5 + done -# modify the cloud_init to include the haproxy.cfg -cat << EOF >> cloud_init.cfg - - systemctl enable haproxy.service - - systemctl start haproxy.service + vip=$(sudo virsh domifaddr --domain "$name" --source agent | grep -w eth0 | grep -E -o '([[:digit:]]{1,3}\.){3}[[:digit:]]{1,3}') -write_files: - - path: /etc/haproxy/haproxy.cfg - encoding: base64 - content: $(cat haproxy.cfg | base64 -w 0) -EOF + # add the ip to the haproxy config + printf " server %s %s:6443 check\n" "$name" "$vip" >> kube-"${PREFIX}"proxy-haproxy.cfg + : $((i=i+1)) +done # create the ha proxy node -create_vm kube-proxy - -# print out the private ips and HAProxy stats -echo -echo PRIVATE IPs -sudo virsh net-dhcp-leases --network default -echo -echo HAProxy Stats: http://kube-proxy:8404/stats +create_vm kube-"${PREFIX}"proxy + +print_machines() { + role=$1 + count=$2 + + : $((i=1)) + while [ $((i<=count)) -ne 0 ]; do + name=kube-"$PREFIX""$role"-$(printf "%02d" "$i") + + until sudo virsh domifaddr --domain "$name" --source agent 2>/dev/null | grep -w eth0 | grep -w ipv4 1>/dev/null; do + sleep 5 + done + + until sudo virsh domifaddr --domain "$name" --source agent 2>/dev/null | grep -w eth1 | grep -w ipv4 1>/dev/null; do + sleep 5 + done + + private_ip=$(sudo virsh domifaddr --domain "$name" --source agent | grep -w eth0 | grep -E -o '([[:digit:]]{1,3}\.){3}[[:digit:]]{1,3}') + public_ip=$(sudo virsh domifaddr --domain "$name" --source agent | grep -w eth1 | grep -E -o '([[:digit:]]{1,3}\.){3}[[:digit:]]{1,3}') + + printf " - name: %s\n role: %s\n privateAddress: %s\n publicAddress: %s\n" \ + "$name" \ + "$role" \ + "$private_ip" \ + "$public_ip" + + : $((i=i+1)) + done +} + +print_success "The following nodes have been configured:" "" + +printf "machines:\n" +print_machines master "$MASTER_NODE_COUNT" +printf "\n" +print_machines worker "$WORKER_NODE_COUNT" diff --git a/network.cfg b/network.cfg.template similarity index 52% rename from network.cfg rename to network.cfg.template index 2c17424..8eb00f1 100644 --- a/network.cfg +++ b/network.cfg.template @@ -1,6 +1,6 @@ version: 2 ethernets: eth0: - dhcp4: true + dhcp4: true eth1: - dhcp4: true + dhcp4: true