From 1b9feb9cb84640847dd7006e0e10635ddb811cac Mon Sep 17 00:00:00 2001
From: "Deavon M. McCaffery" <dmccaffery@users.noreply.github.com>
Date: Mon, 11 Jan 2021 07:15:36 +0000
Subject: [PATCH 1/5] feat: support static ips and mulitple users

---
 LICENSE                 |   2 +-
 bridge.sh               |  30 +--
 cloud_init.cfg.template |   1 -
 kvm.sh                  | 490 +++++++++++++++++++++++++++-------------
 network.cfg             |   6 -
 network.cfg.template    |   6 +
 6 files changed, 361 insertions(+), 174 deletions(-)
 delete mode 100644 network.cfg
 create mode 100644 network.cfg.template

diff --git a/LICENSE b/LICENSE
index c6af185..f33fb03 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,6 +1,6 @@
 The MIT License
 
-Copyright (c) 2020 Deavon McCaffery, Tiffany Wang, and Contributors
+Copyright (c) 2020-2021 Deavon McCaffery, Tiffany Wang, and Contributors
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/bridge.sh b/bridge.sh
index b1214d7..f07ed25 100755
--- a/bridge.sh
+++ b/bridge.sh
@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#! /usr/bin/env sh
 set -eu
 
 # install all prerequisites
@@ -9,18 +9,6 @@ sudo apt-get install qemu-kvm libvirt-daemon-system \
 # make sure libvirtd is enabled
 sudo systemctl enable libvirtd
 
-# get a list of active ethernet devices (deal with whitespace)
-OLDIFS=$IFS
-IFS=$'\n'
-ACTIVE_CONNECTIONS=( $(nmcli -get-values NAME,DEVICE,TYPE connection show --active | grep ethernet | grep -v bridge | cut -d ':' -f 1,2) )
-IFS=$OLDIFS
-
-if [ -z "${ACTIVE_CONNECTIONS:-}" ]; then
-    echo
-    echo "no active connections to bridge..."
-    exit 1
-fi
-
 # delete and recreate the bridge
 sudo nmcli connection delete br0 || true
 sudo nmcli con add ifname br0 type bridge con-name br0 || true
@@ -31,6 +19,7 @@ sudo cat << EOF > /etc/sysctl.d/bridge.conf
 net.bridge.bridge-nf-call-ip6tables = 0
 net.bridge.bridge-nf-call-iptables = 0
 net.bridge.bridge-nf-call-arptables = 0
+net.ipv4.ip_forward = 1
 EOF
 
 # activate the netfilter configuration for the bridge device
@@ -41,6 +30,21 @@ EOF
 # update network configuration
 sudo netplan apply
 
+# wait for connections
+sleep 15
+
+# get a list of active ethernet devices (deal with whitespace)
+OLDIFS=$IFS
+IFS=$'\n'
+ACTIVE_CONNECTIONS=( $(nmcli -get-values NAME,DEVICE,TYPE connection show --active | grep ethernet | grep -v bridge | cut -d ':' -f 1,2) )
+IFS=$OLDIFS
+
+if [ -z "${ACTIVE_CONNECTIONS:-}" ]; then
+    echo
+    echo "no active connections to bridge..."
+    exit 1
+fi
+
 # iterate over each device
 for connection in "${ACTIVE_CONNECTIONS[@]}"; do
 
diff --git a/cloud_init.cfg.template b/cloud_init.cfg.template
index 170cd43..c20b00f 100644
--- a/cloud_init.cfg.template
+++ b/cloud_init.cfg.template
@@ -35,6 +35,5 @@ bootcmd:
 
 runcmd:
   - mkdir -p /home/kube-admin/.ssh || true
-  - curl https://github.com/GH_USER.keys | tee -a /home/kube-admin/.ssh/authorized_keys
   - chmod -R u=rwX,g=rX,o= /home/kube-admin/.ssh
   - chown -R kube-admin:kube-admin /home/kube-admin/.ssh
diff --git a/kvm.sh b/kvm.sh
index f6526d6..8de3e29 100755
--- a/kvm.sh
+++ b/kvm.sh
@@ -1,200 +1,384 @@
-#! /usr/bin/env bash
+#! /usr/bin/env sh
+# shellcheck disable=SC2155
 
 set -e
 
+FORMAT_CLEAR=$(tput sgr0)	# CLEAR ALL FORMAT
+FORMAT_BOLD=$(tput bold)	# SET BRIGH
+
+CLR_RED=$(tput setaf 1)		# ANSI RED
+CLR_GREEN=$(tput setaf 2)	# ANSI GREEN
+CLR_YELLOW=$(tput setaf 3)	# ANSI
+
+CLR_BRIGHT_RED="$FORMAT_BOLD$CLR_RED"		# BRIGHT RED
+CLR_BRIGHT_GREEN="$FORMAT_BOLD$CLR_GREEN"	# BRIGHT GREEN
+CLR_BRIGHT_YELLOW="$FORMAT_BOLD$CLR_YELLOW"	# BRIGHT
+
+CLR_SUCCESS=$CLR_BRIGHT_GREEN
+CLR_WARN=$CLR_BRIGHT_YELLOW
+CLR_FAIL=$CLR_BRIGHT_RED
+
+print_success()
+{
+	printf "${CLR_SUCCESS}%s${FORMAT_CLEAR}\n" "$@"
+}
+
+print_warn()
+{
+	printf "${CLR_WARN}%s${FORMAT_CLEAR}\n" "$@"
+}
+
+print_fail()
+{
+	printf "${CLR_FAIL}%s${FORMAT_CLEAR}\n" "$@" >&2
+}
+
+help() {
+	print_success \
+		"" \
+		"KVM Configuration for Kubernetes" \
+		"" \
+		"Usage: ./kvm.sh [OPTIONS]"
+	print_warn \
+		"" \
+		"Options:" \
+		"" \
+		"--masters            DEFAULT: 1, the number of master nodes to create (min: 1, max: 50)" \
+		"--workers            DEFAULT: 1, the number of worker nodes to create (min: 1, max: 50)" \
+		"--os-version         DEFAULT: 7, the version of CentOS to use" \
+		"" \
+		"--cpu                DEFAULT: 2, the number of CPUs for each node" \
+		"--memory             DEFAULT: 4096, the amount of memory to allocate for each node" \
+		"" \
+		"--network            DEFAULT: bridge, the network for bridging VMs within qemu" \
+		"" \
+		"--enable-static-ips  DEFAULT: false, a value indicating whether or not to use static ips" \
+		"--proxy-ip           DEFAULT: 192.168.50.100/24, the ip to use for the master load balancer" \
+		"--master-ip          DEFAULT: 192.168.50.101/24, the first ip to use for masters (incremented)" \
+		"--worker-ip          DEFAULT: 192.168.50.151/24, the first ip to use for workers (incremented)" \
+		"" \
+		"--proxy-mac-address  DEFAULT: 01:01:01:01:FF, the mac address used for bridge reservations (incremented)" \
+		"--master-mac-address DEFAULT: 01:01:01:01:01, the mac address used for bridge reservations (incremented)" \
+		"--worker-mac-address DEFAULT: 01:01:01:02:01, the mac address used for bridge reservations (incremented)" \
+		"" \
+		"--virtual-proxy-ip   DEFAULT: 192.168.60.100/24, the ip to use for the master load balancer" \
+		"--virtual-master-ip  DEFAULT: 192.168.60.101/24, the first ip to use for masters (incremented)" \
+		"--virtual-worker-ip  DEFAULT: 192.168.60.151/24, the first ip to use for workers (incremented)" \
+		"" \
+		"NOTE: ONLY THE LAST OCTET OF THE IP ADDRESSES ARE INCREMENTED, PLEASE ENSURE THERE IS ENOUGH IPS AVAILABLE" \
+		"" \
+		"--gh-user            DEFAULT: tiffanywang3, the github username used to retrieve SSH keys." \
+		"                     This can be specified multiple times"
+	print_fail \
+		"" \
+		"Example:" \
+		"" \
+		"./kvm.sh --masters 3 --workers 3 --cpu 4 --memory 8192 --enable-static-ips --gh-user somebody"
+}
+
+PROXY_IP="192.168.50.100/24"
+PROXY_MAC="02:01:01:01:01:FF"
+PROXY_VIRTUAL_IP="192.168.60.100/24"
+
 MASTER_NODE_COUNT=1
+MASTER_IP="192.168.50.101/24"
+MASTER_MAC="02:01:01:01:01:01"
+MASTER_VIRTUAL_IP="192.168.60.101/24"
+
 WORKER_NODE_COUNT=1
+WORKER_IP="192.168.50.151/24"
+WORKER_MAC="02:01:01:01:02:01"
+WORKER_VIRTUAL_IP="192.168.60.151/24"
+
+NETWORK="bridge"
 
 CENTOS_VERSION=7
 CPU_LIMIT=2
 MEMORY_LIMIT=4096
 
-GH_USER=tiffanywang3
-
-__kvm_help() {
-    echo
-    echo "KVM Configuration for Kubernetes"
-    echo
-    echo "Usage: ./kvm.sh [OPTIONS]"
-    echo
-    echo "Options:"
-    echo
-    echo -e "--masters\tDEFAULT: 1, the number of master nodes to create"
-    echo -e "--workers\tDEFAULT: 1, the number of worker nodes to create"
-    echo -e "--os-version\tDEFAULT: 7, the version of CentOS to use"
-    echo -e "--cpu\t\tDEFAULT: 2, the number of CPUs for each node"
-    echo -e "--memory\tDEFAULT: 4096, the amount of memory to allocate for each node"
-    echo -e "--gh-user\tDEFAULT: tiffanywang3, the github username used to retrieve SSH keys"
-    echo
-    echo "Example:"
-    echo
-    echo "./kvm.sh --masters 3 --workers 3 --cpu 4 --memory 8192 --gh-user somebody"
-}
-
-while [[ $# > 0 ]]; do
-    case $1 in
-        --masters)
-            MASTER_NODE_COUNT=$2
-            shift
-            ;;
-        --workers)
-            WORKER_NODE_COUNT=$2
-            shift
-            ;;
-        --os-version)
-            CENTOS_VERSION=$2
-            shift
-            ;;
-        --cpu)
-            CPU_LIMIT=$2
-            shift
-            ;;
-        --memory)
-            MEMORY_LIMIT=$2
-            shift
-            ;;
-        --gh-user)
-            GH_USER=$2
-            shift
-            ;;
-        --help)
-            __kvm_help
-            exit 0
-            ;;
-        *)
-            echo "unknown option: $1"
-            echo
-            __kvm_help
-            exit 1
-            ;;
-    esac
-    shift
+ENABLE_STATIC_IPS=
+ENABLE_HOST_DHCP=false
+
+while [ $# -gt 0 ]; do
+	case $1 in
+		--masters)
+			MASTER_NODE_COUNT=$2
+			shift
+			;;
+		--workers)
+			WORKER_NODE_COUNT=$2
+			shift
+			;;
+		--os-version)
+			CENTOS_VERSION=$2
+			shift
+			;;
+		--cpu)
+			CPU_LIMIT=$2
+			shift
+			;;
+		--memory)
+			MEMORY_LIMIT=$2
+			shift
+			;;
+		--network)
+			NETWORK=$2
+			shift
+			;;
+		--gh-user)
+			GH_USER="$2 $GH_USER"
+			shift
+			;;
+		--proxy-mac-address)
+			PROXY_MAC=$2
+			shift
+			;;
+		--proxy-ip)
+			PROXY_IP=$2
+			shift
+			;;
+		--proxy-virtual-ip)
+			PROXY_VIRTUAL_IP=$2
+			shift
+			;;
+		--master-mac-address)
+			MASTER_MAC=$2
+			shift
+			;;
+		--master-ip)
+			MASTER_IP=$2
+			shift
+			;;
+		--master-virtual-ip)
+			MASTER_VIRTUAL_IP=$2
+			shift
+			;;
+		--worker-mac-address)
+			WORKER_MAC=$2
+			shift
+			;;
+		--worker-ip)
+			WORKER_IP=$2
+			shift
+			;;
+		--worker-virtual-ip)
+			WORKER_VIRTUAL_IP=$2
+			shift
+			;;
+		--enable-static-ips)
+			ENABLE_STATIC_IPS=1
+			;;
+		--help)
+			help
+			exit 0
+			;;
+		*)
+			printf "unknown option: %s\n\n" "$1"
+			help
+			exit 1
+			;;
+	esac
+	shift
 done
 
-OS_VARIANT="centos${CENTOS_VERSION}.0"
-BASE_IMAGE=CentOS-${CENTOS_VERSION}-x86_64-GenericCloud.qcow2
+# default the gh user if unspecified
+if [ -z "$GH_USER" ]; then
+	GH_USER=tiffanywang3
+fi
+
+# if static ips are not enabled
+if [ -z "$ENABLE_STATIC_IPS" ]; then
+	# turn on host dhcp
+	ENABLE_HOST_DHCP=true
 
-if [ ! -f $BASE_IMAGE ]; then
-  echo "Downloading $BASE_IMAGE...."
-  wget http://cloud.centos.org/centos/7/images/$BASE_IMAGE -O $BASE_IMAGE
+	# clear the ips
+	PROXY_IP=
+	MASTER_IP=
+	WORKER_IP=
 fi
 
-VM_NETWORKS=""
+increment_ip() {
+	value="${1:-}"
 
-# setup networks
-for network in $(sudo virsh net-list | grep active |  tr -s ' ' | cut -d ' ' -f 2); do
-    VM_NETWORKS="${VM_NETWORKS} --network network:${network}"
-done
+	if [ -z "$value" ]; then
+		return
+	fi
+
+	prefix=${value%.*}
+	value=${value##*.}
+
+	octet=${value%/*}
+	subnet=${value##*/}
+
+	octet=$((octet+1))
+
+	printf "%s.%s/%s" "$prefix" "$octet" "$subnet"
+}
+
+increment_mac() {
+	value="${1:-}"
+
+	prefix=${value%:*}
+	value=${value##*:}
+
+	value=$((value+1))
+
+	printf "%s:%02d" "$prefix" "$value"
+}
+
+OS_VARIANT="centos$CENTOS_VERSION.0"
+BASE_IMAGE=CentOS-$CENTOS_VERSION-x86_64-GenericCloud.qcow2
+
+if [ ! -f "$BASE_IMAGE" ]; then
+	print_success "Downloading $BASE_IMAGE...."
+	wget http://cloud.centos.org/centos/7/images/"$BASE_IMAGE" -O "$BASE_IMAGE"
+fi
 
 # install all prerequisites
 sudo apt update
 sudo apt-get install qemu-kvm libvirt-daemon-system \
-   libvirt-clients libnss-libvirt virtinst
+	libvirt-clients libnss-libvirt virtinst
 
 # make sure libvirtd is enabled
 sudo systemctl enable libvirtd
 
-# add user to libvirt group
-sudo usermod -a -G libvirt ${USER}
+# add user to libvirt and kvm groups
+sudo usermod -a -G libvirt "$USER"
+sudo usermod -a -G kvm "$USER"
 
 # delete existing nodes
 for node in $(sudo virsh list --all --name | grep "kube-"); do
-  sudo virsh shutdown $node
-  sudo virsh destroy $node
-  sudo virsh undefine $node
-
-  rm -f $node.qcow2
-  rm -f $node-init.img
-  rm -f $node-metadata
+	sudo virsh shutdown "$node"
+	sudo virsh destroy "$node"
+	sudo virsh undefine "$node"
 done
 
+# remove existing configs
+rm -f kube-*
+
 # ensure base image is accessible
-sudo chmod u=rw,go=r $BASE_IMAGE
+sudo chmod u=rw,go=r "$BASE_IMAGE"
 
-# create cloud-init config
-sed "s/GH_USER/${GH_USER}/g" cloud_init.cfg.template > cloud_init.cfg
+# create the cloud-init config
+cat cloud_init.cfg.template > cloud_init.cfg
 
-# create the haproxy config
-cat haproxy.cfg.template > haproxy.cfg
+# emit every user key into authorized keys
+for user in $GH_USER; do
+	printf "  - curl https://github.com/%s.keys | tee -a /home/kube-admin/.ssh/authorized_keys\n" "$user" >> cloud_init.cfg
+done
 
 # create a virtual machine
 create_vm() {
-    hostname=$1
-    snapshot=$hostname.qcow2
-    init=$hostname-init.img
-
-    # create snapshot and increase size to 30GB
-    qemu-img create -b $BASE_IMAGE -f qcow2 -F qcow2 $snapshot 30G
-    qemu-img info $snapshot
-
-    # insert metadata into init image
-    echo "instance-id: $(uuidgen || echo i-abcdefg)" > $hostname-metadata
-    echo "local-hostname: $hostname.local" >> $hostname-metadata
-
-    cloud-localds -v --network-config=network.cfg $init cloud_init.cfg $hostname-metadata
-
-    # ensure file permissions belong to kvm group
-    sudo chmod ug=rw,o= $snapshot
-    sudo chown $USER:kvm $snapshot $init
-
-    # create the vm
-    sudo virt-install --name $hostname \
-        --virt-type kvm --memory ${MEMORY_LIMIT} --vcpus ${CPU_LIMIT} \
-        --boot hd,menu=on \
-        --disk path=$init,device=cdrom \
-        --disk path=$snapshot,device=disk \
-        --graphics vnc \
-        --os-type Linux --os-variant ${OS_VARIANT} \
-        ${VM_NETWORKS} \
-        --autostart \
-        --noautoconsole
-
-    # set the timeout
-    sudo virsh guest-agent-timeout $hostname --timeout 60
+	hostname=$1
+	virtual_ip=$2
+	host_mac=$3
+	host_ip=${4:-}
+
+	snapshot=$hostname.qcow2
+	init=$hostname-init.img
+	metadata=$hostname-metadata
+	cloud_cfg=$hostname-cloud_init.cfg
+	network_cfg=$hostname-network.cfg
+
+	# copy the cloud_init.cfg
+	cp cloud_init.cfg "$cloud_cfg"
+
+	if [ "$hostname" = "kube-proxy" ]; then
+		# modify the cloud_init to include the haproxy.cfg
+		cat <<- EOF >> "$cloud_cfg"
+		  - systemctl enable haproxy.service
+		  - systemctl start haproxy.service
+
+		write_files:
+		- path: /etc/haproxy/haproxy.cfg
+		  encoding: base64
+		  content: $(base64 -w 0 < kube-proxy-haproxy.cfg)
+		EOF
+	fi
+
+	# create snapshot and increase size to 30GB
+	qemu-img create -b "$BASE_IMAGE" -f qcow2 -F qcow2 "$snapshot" 30G
+	qemu-img info "$snapshot"
+
+	# insert metadata into init image
+	printf "instance-id: %s\n" "$(uuidgen || printf i-abcdefg)" > "$metadata"
+	printf "local-hostname: %s.local\n" "$hostname" >> "$metadata"
+
+	# create the network config
+	sed "s@VIRTUAL_IP@${virtual_ip}@g" network.cfg.template \
+		| sed "s@ENABLE_HOST_DHCP@${ENABLE_HOST_DHCP}@g" \
+		| sed "s@HOST_IP@${host_ip}@g" > "$network_cfg"
+
+	# setup the cloud-init metadata
+	cloud-localds -v --network-config="$network_cfg" "$init" "$cloud_cfg" "$metadata"
+
+	# ensure file permissions belong to kvm group
+	sudo chmod ug=rw,o= "$snapshot"
+	sudo chown "$USER":kvm "$snapshot" "$init"
+
+	# create the vm
+	sudo virt-install --name "$hostname" \
+		--virt-type kvm --memory "$MEMORY_LIMIT" --vcpus "$CPU_LIMIT" \
+		--boot hd,menu=on \
+		--disk path="$init",device=cdrom \
+		--disk path="$snapshot",device=disk \
+		--graphics vnc \
+		--os-type Linux --os-variant "$OS_VARIANT" \
+		--network default \
+		--network "$NETWORK",mac="$host_mac" \
+		--autostart \
+		--noautoconsole
+
+	# set the timeout
+	sudo virsh guest-agent-timeout "$hostname" --timeout 60
 }
 
-# iterate over each node
-for (( i=1; i<=$MASTER_NODE_COUNT; i++ )); do
-  # create the vm for the node
-  create_vm kube-master-$(printf "%02d" $i)
+: $((i=1))
+while [ $((i<=MASTER_NODE_COUNT)) -ne 0 ]; do
+	create_vm \
+		kube-master-"$(printf "%02d" "$i")" \
+		"$MASTER_VIRTUAL_IP" \
+		"$MASTER_MAC" \
+		"$MASTER_IP"
+
+	# increment the mac and ip
+	MASTER_VIRTUAL_IP=$(increment_ip "$MASTER_VIRTUAL_IP")
+	MASTER_MAC=$(increment_mac "$MASTER_MAC")
+	MASTER_IP=$(increment_ip "$MASTER_IP")
+	: $((i=i+1))
 done
 
-# iterate over each node
-for (( i=1; i<=$WORKER_NODE_COUNT; i++ )); do
-  # create the vm for the node
-  create_vm kube-worker-$(printf "%02d" $i)
+: $((i=1))
+while [ $((i<=WORKER_NODE_COUNT)) -ne 0 ]; do
+	create_vm \
+		kube-worker-"$(printf "%02d" "$i")" \
+		"$WORKER_VIRTUAL_IP" \
+		"$WORKER_MAC" \
+		"$WORKER_IP"
+
+	# increment the mac and ip
+	WORKER_VIRTUAL_IP=$(increment_ip "$WORKER_VIRTUAL_IP")
+	WORKER_MAC=$(increment_mac "$WORKER_MAC")
+	WORKER_IP=$(increment_ip "$WORKER_IP")
+	: $((i=i+1))
 done
 
-# wait for nodes to come up
+# wait for nodes to come up; # todo: replace with until / wait
 sleep 60
 
-# iterate over each node
-for (( i=1; i<=$MASTER_NODE_COUNT; i++ )); do
-  # get the name of the master
-  name=kube-master-$(printf "%02d" $i)
-  ip=$(sudo virsh domifaddr --domain ${name} --source agent | grep -w eth1 | egrep -o '([[:digit:]]{1,3}\.){3}[[:digit:]]{1,3}')
-
-  # add the ip to the haproxy config
-  echo "    server $name $ip:6443 check" >> haproxy.cfg
-done
+# create the haproxy config
+cat haproxy.cfg.template > kube-proxy-haproxy.cfg
 
-# modify the cloud_init to include the haproxy.cfg
-cat << EOF >> cloud_init.cfg
-  - systemctl enable haproxy.service
-  - systemctl start haproxy.service
+# iterate over each master node
+: $((i=1))
+while [ $((i<=MASTER_NODE_COUNT)) -ne 0 ]; do
+	name=kube-master-$(printf "%02d" "$i")
+	ip=$(sudo virsh domifaddr --domain "$name" --source agent | grep -w eth1 | grep -E -o '([[:digit:]]{1,3}\.){3}[[:digit:]]{1,3}')
 
-write_files:
-  - path: /etc/haproxy/haproxy.cfg
-    encoding: base64
-    content: $(cat haproxy.cfg | base64 -w 0)
-EOF
+	# add the ip to the haproxy config
+	printf "    server %s %s:6443 check\n" "$name" "$ip" >> kube-proxy-haproxy.cfg
+	: $((i=i+1))
+done
 
 # create the ha proxy node
-create_vm kube-proxy
-
-# print out the private ips and HAProxy stats
-echo
-echo PRIVATE IPs
-sudo virsh net-dhcp-leases --network default
-echo
-echo HAProxy Stats: http://kube-proxy:8404/stats
+create_vm kube-proxy "$PROXY_VIRTUAL_IP" "$PROXY_MAC" "$PROXY_IP"
diff --git a/network.cfg b/network.cfg
deleted file mode 100644
index 2c17424..0000000
--- a/network.cfg
+++ /dev/null
@@ -1,6 +0,0 @@
-version: 2
-ethernets:
-  eth0:
-     dhcp4: true
-  eth1:
-     dhcp4: true
diff --git a/network.cfg.template b/network.cfg.template
new file mode 100644
index 0000000..59f856c
--- /dev/null
+++ b/network.cfg.template
@@ -0,0 +1,6 @@
+version: 2
+ethernets:
+  eth0:
+     addresses: ["VIRTUAL_IP"]
+  eth1:
+     addresses: ["HOST_IP"]

From 9933384c153a71ebb53d9ab176735931ea3e57cd Mon Sep 17 00:00:00 2001
From: "Deavon M. McCaffery" <dmccaffery@users.noreply.github.com>
Date: Mon, 11 Jan 2021 08:01:50 +0000
Subject: [PATCH 2/5] feat: support gateway and domain

---
 kvm.sh               | 25 ++++++++++++++++++++-----
 network.cfg.template | 13 +++++++++++--
 2 files changed, 31 insertions(+), 7 deletions(-)

diff --git a/kvm.sh b/kvm.sh
index 8de3e29..21e2373 100755
--- a/kvm.sh
+++ b/kvm.sh
@@ -51,6 +51,8 @@ help() {
 		"--memory             DEFAULT: 4096, the amount of memory to allocate for each node" \
 		"" \
 		"--network            DEFAULT: bridge, the network for bridging VMs within qemu" \
+		"--gateway            DEFAULT: 192.168.50.1, the IP address of the gateway for the network subnet" \
+		"--domain             DEFAULT: local, the dns domain used to register IPs" \
 		"" \
 		"--enable-static-ips  DEFAULT: false, a value indicating whether or not to use static ips" \
 		"--proxy-ip           DEFAULT: 192.168.50.100/24, the ip to use for the master load balancer" \
@@ -76,6 +78,10 @@ help() {
 		"./kvm.sh --masters 3 --workers 3 --cpu 4 --memory 8192 --enable-static-ips --gh-user somebody"
 }
 
+NETWORK="bridge"
+GATEWAY="192.168.50.1"
+DOMAIN="local"
+
 PROXY_IP="192.168.50.100/24"
 PROXY_MAC="02:01:01:01:01:FF"
 PROXY_VIRTUAL_IP="192.168.60.100/24"
@@ -90,8 +96,6 @@ WORKER_IP="192.168.50.151/24"
 WORKER_MAC="02:01:01:01:02:01"
 WORKER_VIRTUAL_IP="192.168.60.151/24"
 
-NETWORK="bridge"
-
 CENTOS_VERSION=7
 CPU_LIMIT=2
 MEMORY_LIMIT=4096
@@ -125,6 +129,14 @@ while [ $# -gt 0 ]; do
 			NETWORK=$2
 			shift
 			;;
+		--gateway)
+			GATEWAY=$2
+			shift
+			;;
+		--domain)
+			DOMAIN=$2
+			shift
+			;;
 		--gh-user)
 			GH_USER="$2 $GH_USER"
 			shift
@@ -305,9 +317,12 @@ create_vm() {
 	printf "local-hostname: %s.local\n" "$hostname" >> "$metadata"
 
 	# create the network config
-	sed "s@VIRTUAL_IP@${virtual_ip}@g" network.cfg.template \
-		| sed "s@ENABLE_HOST_DHCP@${ENABLE_HOST_DHCP}@g" \
-		| sed "s@HOST_IP@${host_ip}@g" > "$network_cfg"
+	sed "s@VIRTUAL_IP@$virtual_ip@g" network.cfg.template \
+		| sed "s@ENABLE_HOST_DHCP@$ENABLE_HOST_DHCP@g" \
+		| sed "s@GATEWAY@$GATEWAY@g" \
+		| sed "s@DOMAIN@$DOMAIN@g" \
+		| sed "s@HOST_MAC$host_mac@g" \
+		| sed "s@HOST_IP@$host_ip@g" > "$network_cfg"
 
 	# setup the cloud-init metadata
 	cloud-localds -v --network-config="$network_cfg" "$init" "$cloud_cfg" "$metadata"
diff --git a/network.cfg.template b/network.cfg.template
index 59f856c..ab51057 100644
--- a/network.cfg.template
+++ b/network.cfg.template
@@ -1,6 +1,15 @@
 version: 2
 ethernets:
   eth0:
-     addresses: ["VIRTUAL_IP"]
+    addresses: ["VIRTUAL_IP"]
+    gateway4: ["GATEWAY"]
+    nameservers:
+      search: ["DOMAIN"]
+
   eth1:
-     addresses: ["HOST_IP"]
+    match:
+      macaddress: "HOST_MAC"
+    addresses: ["HOST_IP"]
+    gateway4: ["GATEWAY"]
+    nameservers:
+      search: ["DOMAIN"]

From 234411ee673b2f9c6c8ca53e18af3dcf7af15b94 Mon Sep 17 00:00:00 2001
From: "Deavon M. McCaffery" <dmccaffery@users.noreply.github.com>
Date: Mon, 11 Jan 2021 08:03:21 +0000
Subject: [PATCH 3/5] fix: do not tee authorized keys

---
 kvm.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kvm.sh b/kvm.sh
index 21e2373..e7d6a25 100755
--- a/kvm.sh
+++ b/kvm.sh
@@ -276,7 +276,7 @@ cat cloud_init.cfg.template > cloud_init.cfg
 
 # emit every user key into authorized keys
 for user in $GH_USER; do
-	printf "  - curl https://github.com/%s.keys | tee -a /home/kube-admin/.ssh/authorized_keys\n" "$user" >> cloud_init.cfg
+	printf "  - curl https://github.com/%s.keys >> /home/kube-admin/.ssh/authorized_keys\n" "$user" >> cloud_init.cfg
 done
 
 # create a virtual machine

From 7ae998a1d345d318dbc7d9248cb9063f0707b7c1 Mon Sep 17 00:00:00 2001
From: "Deavon M. McCaffery" <dmccaffery@users.noreply.github.com>
Date: Mon, 11 Jan 2021 10:53:29 +0000
Subject: [PATCH 4/5] feat: generate machine list

---
 bridge.sh               |   4 +-
 cloud_init.cfg.template |   2 -
 kvm.sh                  | 225 ++++++++++++----------------------------
 network.cfg.template    |  13 +--
 4 files changed, 72 insertions(+), 172 deletions(-)

diff --git a/bridge.sh b/bridge.sh
index f07ed25..8c5c383 100755
--- a/bridge.sh
+++ b/bridge.sh
@@ -15,7 +15,7 @@ sudo nmcli con add ifname br0 type bridge con-name br0 || true
 sudo nmcli con modify br0 bridge.stp no
 
 # disable netfilter for the bridge network to avoid issues with docker network configs
-sudo cat << EOF > /etc/sysctl.d/bridge.conf
+cat << EOF | sudo tee /etc/sysctl.d/bridge.conf
 net.bridge.bridge-nf-call-ip6tables = 0
 net.bridge.bridge-nf-call-iptables = 0
 net.bridge.bridge-nf-call-arptables = 0
@@ -23,7 +23,7 @@ net.ipv4.ip_forward = 1
 EOF
 
 # activate the netfilter configuration for the bridge device
-sudo cat << EOF > /etc/udev/rules.d/99-bridge.rules
+cat << EOF | sudo tee /etc/udev/rules.d/99-bridge.rules
 ACTION=="add", SUBSYSTEM=="module", KERNEL=="br_netfilter", RUN+="/sbin/sysctl -p /etc/sysctl.d/bridge.conf"
 EOF
 
diff --git a/cloud_init.cfg.template b/cloud_init.cfg.template
index c20b00f..bff6226 100644
--- a/cloud_init.cfg.template
+++ b/cloud_init.cfg.template
@@ -35,5 +35,3 @@ bootcmd:
 
 runcmd:
   - mkdir -p /home/kube-admin/.ssh || true
-  - chmod -R u=rwX,g=rX,o= /home/kube-admin/.ssh
-  - chown -R kube-admin:kube-admin /home/kube-admin/.ssh
diff --git a/kvm.sh b/kvm.sh
index e7d6a25..995677c 100755
--- a/kvm.sh
+++ b/kvm.sh
@@ -43,6 +43,7 @@ help() {
 		"" \
 		"Options:" \
 		"" \
+		"--prefix             DEFAULT: $(whoami), the prefix to use for kube node names" \
 		"--masters            DEFAULT: 1, the number of master nodes to create (min: 1, max: 50)" \
 		"--workers            DEFAULT: 1, the number of worker nodes to create (min: 1, max: 50)" \
 		"--os-version         DEFAULT: 7, the version of CentOS to use" \
@@ -51,22 +52,8 @@ help() {
 		"--memory             DEFAULT: 4096, the amount of memory to allocate for each node" \
 		"" \
 		"--network            DEFAULT: bridge, the network for bridging VMs within qemu" \
-		"--gateway            DEFAULT: 192.168.50.1, the IP address of the gateway for the network subnet" \
 		"--domain             DEFAULT: local, the dns domain used to register IPs" \
 		"" \
-		"--enable-static-ips  DEFAULT: false, a value indicating whether or not to use static ips" \
-		"--proxy-ip           DEFAULT: 192.168.50.100/24, the ip to use for the master load balancer" \
-		"--master-ip          DEFAULT: 192.168.50.101/24, the first ip to use for masters (incremented)" \
-		"--worker-ip          DEFAULT: 192.168.50.151/24, the first ip to use for workers (incremented)" \
-		"" \
-		"--proxy-mac-address  DEFAULT: 01:01:01:01:FF, the mac address used for bridge reservations (incremented)" \
-		"--master-mac-address DEFAULT: 01:01:01:01:01, the mac address used for bridge reservations (incremented)" \
-		"--worker-mac-address DEFAULT: 01:01:01:02:01, the mac address used for bridge reservations (incremented)" \
-		"" \
-		"--virtual-proxy-ip   DEFAULT: 192.168.60.100/24, the ip to use for the master load balancer" \
-		"--virtual-master-ip  DEFAULT: 192.168.60.101/24, the first ip to use for masters (incremented)" \
-		"--virtual-worker-ip  DEFAULT: 192.168.60.151/24, the first ip to use for workers (incremented)" \
-		"" \
 		"NOTE: ONLY THE LAST OCTET OF THE IP ADDRESSES ARE INCREMENTED, PLEASE ENSURE THERE IS ENOUGH IPS AVAILABLE" \
 		"" \
 		"--gh-user            DEFAULT: tiffanywang3, the github username used to retrieve SSH keys." \
@@ -75,36 +62,25 @@ help() {
 		"" \
 		"Example:" \
 		"" \
-		"./kvm.sh --masters 3 --workers 3 --cpu 4 --memory 8192 --enable-static-ips --gh-user somebody"
+		"./kvm.sh --masters 3 --workers 3 --cpu 4 --memory 8192 --gh-user somebody"
 }
 
+PREFIX=$(whoami)-
 NETWORK="bridge"
-GATEWAY="192.168.50.1"
-DOMAIN="local"
-
-PROXY_IP="192.168.50.100/24"
-PROXY_MAC="02:01:01:01:01:FF"
-PROXY_VIRTUAL_IP="192.168.60.100/24"
 
 MASTER_NODE_COUNT=1
-MASTER_IP="192.168.50.101/24"
-MASTER_MAC="02:01:01:01:01:01"
-MASTER_VIRTUAL_IP="192.168.60.101/24"
-
 WORKER_NODE_COUNT=1
-WORKER_IP="192.168.50.151/24"
-WORKER_MAC="02:01:01:01:02:01"
-WORKER_VIRTUAL_IP="192.168.60.151/24"
 
 CENTOS_VERSION=7
 CPU_LIMIT=2
 MEMORY_LIMIT=4096
 
-ENABLE_STATIC_IPS=
-ENABLE_HOST_DHCP=false
-
 while [ $# -gt 0 ]; do
 	case $1 in
+		--prefix)
+			PREFIX="$2"-
+			shift
+			;;
 		--masters)
 			MASTER_NODE_COUNT=$2
 			shift
@@ -129,10 +105,6 @@ while [ $# -gt 0 ]; do
 			NETWORK=$2
 			shift
 			;;
-		--gateway)
-			GATEWAY=$2
-			shift
-			;;
 		--domain)
 			DOMAIN=$2
 			shift
@@ -141,45 +113,6 @@ while [ $# -gt 0 ]; do
 			GH_USER="$2 $GH_USER"
 			shift
 			;;
-		--proxy-mac-address)
-			PROXY_MAC=$2
-			shift
-			;;
-		--proxy-ip)
-			PROXY_IP=$2
-			shift
-			;;
-		--proxy-virtual-ip)
-			PROXY_VIRTUAL_IP=$2
-			shift
-			;;
-		--master-mac-address)
-			MASTER_MAC=$2
-			shift
-			;;
-		--master-ip)
-			MASTER_IP=$2
-			shift
-			;;
-		--master-virtual-ip)
-			MASTER_VIRTUAL_IP=$2
-			shift
-			;;
-		--worker-mac-address)
-			WORKER_MAC=$2
-			shift
-			;;
-		--worker-ip)
-			WORKER_IP=$2
-			shift
-			;;
-		--worker-virtual-ip)
-			WORKER_VIRTUAL_IP=$2
-			shift
-			;;
-		--enable-static-ips)
-			ENABLE_STATIC_IPS=1
-			;;
 		--help)
 			help
 			exit 0
@@ -198,46 +131,6 @@ if [ -z "$GH_USER" ]; then
 	GH_USER=tiffanywang3
 fi
 
-# if static ips are not enabled
-if [ -z "$ENABLE_STATIC_IPS" ]; then
-	# turn on host dhcp
-	ENABLE_HOST_DHCP=true
-
-	# clear the ips
-	PROXY_IP=
-	MASTER_IP=
-	WORKER_IP=
-fi
-
-increment_ip() {
-	value="${1:-}"
-
-	if [ -z "$value" ]; then
-		return
-	fi
-
-	prefix=${value%.*}
-	value=${value##*.}
-
-	octet=${value%/*}
-	subnet=${value##*/}
-
-	octet=$((octet+1))
-
-	printf "%s.%s/%s" "$prefix" "$octet" "$subnet"
-}
-
-increment_mac() {
-	value="${1:-}"
-
-	prefix=${value%:*}
-	value=${value##*:}
-
-	value=$((value+1))
-
-	printf "%s:%02d" "$prefix" "$value"
-}
-
 OS_VARIANT="centos$CENTOS_VERSION.0"
 BASE_IMAGE=CentOS-$CENTOS_VERSION-x86_64-GenericCloud.qcow2
 
@@ -259,7 +152,7 @@ sudo usermod -a -G libvirt "$USER"
 sudo usermod -a -G kvm "$USER"
 
 # delete existing nodes
-for node in $(sudo virsh list --all --name | grep "kube-"); do
+for node in $(sudo virsh list --all --name | grep "kube-${PREFIX}"); do
 	sudo virsh shutdown "$node"
 	sudo virsh destroy "$node"
 	sudo virsh undefine "$node"
@@ -279,12 +172,13 @@ for user in $GH_USER; do
 	printf "  - curl https://github.com/%s.keys >> /home/kube-admin/.ssh/authorized_keys\n" "$user" >> cloud_init.cfg
 done
 
+printf "  %s\n"  \
+	"- chmod -R u=rwX,g=rX,o= /home/kube-admin/.ssh" \
+	"- chown -R kube-admin:kube-admin /home/kube-admin/.ssh" >> cloud_init.cfg
+
 # create a virtual machine
 create_vm() {
 	hostname=$1
-	virtual_ip=$2
-	host_mac=$3
-	host_ip=${4:-}
 
 	snapshot=$hostname.qcow2
 	init=$hostname-init.img
@@ -295,7 +189,7 @@ create_vm() {
 	# copy the cloud_init.cfg
 	cp cloud_init.cfg "$cloud_cfg"
 
-	if [ "$hostname" = "kube-proxy" ]; then
+	if [ "$hostname" = "kube-${PREFIX}proxy" ]; then
 		# modify the cloud_init to include the haproxy.cfg
 		cat <<- EOF >> "$cloud_cfg"
 		  - systemctl enable haproxy.service
@@ -304,7 +198,7 @@ create_vm() {
 		write_files:
 		- path: /etc/haproxy/haproxy.cfg
 		  encoding: base64
-		  content: $(base64 -w 0 < kube-proxy-haproxy.cfg)
+		  content: $(base64 -w 0 < kube-"${PREFIX}"proxy-haproxy.cfg)
 		EOF
 	fi
 
@@ -314,15 +208,11 @@ create_vm() {
 
 	# insert metadata into init image
 	printf "instance-id: %s\n" "$(uuidgen || printf i-abcdefg)" > "$metadata"
-	printf "local-hostname: %s.local\n" "$hostname" >> "$metadata"
+	printf "local-hostname: %s\n" "$hostname" >> "$metadata"
+	printf "hostname: %s.%s\n" "$hostname" "$DOMAIN" >> "$metadata"
 
 	# create the network config
-	sed "s@VIRTUAL_IP@$virtual_ip@g" network.cfg.template \
-		| sed "s@ENABLE_HOST_DHCP@$ENABLE_HOST_DHCP@g" \
-		| sed "s@GATEWAY@$GATEWAY@g" \
-		| sed "s@DOMAIN@$DOMAIN@g" \
-		| sed "s@HOST_MAC$host_mac@g" \
-		| sed "s@HOST_IP@$host_ip@g" > "$network_cfg"
+	cp network.cfg.template "$network_cfg"
 
 	# setup the cloud-init metadata
 	cloud-localds -v --network-config="$network_cfg" "$init" "$cloud_cfg" "$metadata"
@@ -339,8 +229,8 @@ create_vm() {
 		--disk path="$snapshot",device=disk \
 		--graphics vnc \
 		--os-type Linux --os-variant "$OS_VARIANT" \
-		--network default \
-		--network "$NETWORK",mac="$host_mac" \
+		--network network:default \
+		--network network:"$NETWORK" \
 		--autostart \
 		--noautoconsole
 
@@ -350,50 +240,71 @@ create_vm() {
 
 : $((i=1))
 while [ $((i<=MASTER_NODE_COUNT)) -ne 0 ]; do
-	create_vm \
-		kube-master-"$(printf "%02d" "$i")" \
-		"$MASTER_VIRTUAL_IP" \
-		"$MASTER_MAC" \
-		"$MASTER_IP"
-
-	# increment the mac and ip
-	MASTER_VIRTUAL_IP=$(increment_ip "$MASTER_VIRTUAL_IP")
-	MASTER_MAC=$(increment_mac "$MASTER_MAC")
-	MASTER_IP=$(increment_ip "$MASTER_IP")
+	create_vm kube-"${PREFIX}"master-"$(printf "%02d" "$i")"
 	: $((i=i+1))
 done
 
 : $((i=1))
 while [ $((i<=WORKER_NODE_COUNT)) -ne 0 ]; do
-	create_vm \
-		kube-worker-"$(printf "%02d" "$i")" \
-		"$WORKER_VIRTUAL_IP" \
-		"$WORKER_MAC" \
-		"$WORKER_IP"
-
-	# increment the mac and ip
-	WORKER_VIRTUAL_IP=$(increment_ip "$WORKER_VIRTUAL_IP")
-	WORKER_MAC=$(increment_mac "$WORKER_MAC")
-	WORKER_IP=$(increment_ip "$WORKER_IP")
+	create_vm kube-"${PREFIX}"worker-"$(printf "%02d" "$i")"
 	: $((i=i+1))
 done
 
-# wait for nodes to come up; # todo: replace with until / wait
-sleep 60
-
 # create the haproxy config
-cat haproxy.cfg.template > kube-proxy-haproxy.cfg
+cp haproxy.cfg.template kube-"${PREFIX}"proxy-haproxy.cfg
 
 # iterate over each master node
 : $((i=1))
 while [ $((i<=MASTER_NODE_COUNT)) -ne 0 ]; do
-	name=kube-master-$(printf "%02d" "$i")
-	ip=$(sudo virsh domifaddr --domain "$name" --source agent | grep -w eth1 | grep -E -o '([[:digit:]]{1,3}\.){3}[[:digit:]]{1,3}')
+	name=kube-"$PREFIX"master-$(printf "%02d" "$i")
+
+	until sudo virsh domifaddr --domain "$name" --source agent 2>/dev/null | grep -w eth0 | grep -w ipv4 1>/dev/null; do
+		print_warn "waiting for $name : eth0..."
+		sleep 5
+	done
+
+	vip=$(sudo virsh domifaddr --domain "$name" --source agent | grep -w eth0 | grep -E -o '([[:digit:]]{1,3}\.){3}[[:digit:]]{1,3}')
 
 	# add the ip to the haproxy config
-	printf "    server %s %s:6443 check\n" "$name" "$ip" >> kube-proxy-haproxy.cfg
+	printf "    server %s %s:6443 check\n" "$name" "$vip" >> kube-"${PREFIX}"proxy-haproxy.cfg
 	: $((i=i+1))
 done
 
 # create the ha proxy node
-create_vm kube-proxy "$PROXY_VIRTUAL_IP" "$PROXY_MAC" "$PROXY_IP"
+create_vm kube-"${PREFIX}"proxy "$PROXY_VIRTUAL_IP" "$PROXY_MAC" "$PROXY_IP"
+
+print_machines() {
+	role=$1
+	count=$2
+
+	: $((i=1))
+	while [ $((i<=count)) -ne 0 ]; do
+		name=kube-"$PREFIX""$role"-$(printf "%02d" "$i")
+
+		until sudo virsh domifaddr --domain "$name" --source agent 2>/dev/null | grep -w eth0 | grep -w ipv4 1>/dev/null; do
+			sleep 5
+		done
+
+		until sudo virsh domifaddr --domain "$name" --source agent 2>/dev/null | grep -w eth1 | grep -w ipv4 1>/dev/null; do
+			sleep 5
+		done
+
+		private_ip=$(sudo virsh domifaddr --domain "$name" --source agent | grep -w eth0 | grep -E -o '([[:digit:]]{1,3}\.){3}[[:digit:]]{1,3}')
+		public_ip=$(sudo virsh domifaddr --domain "$name" --source agent | grep -w eth1 | grep -E -o '([[:digit:]]{1,3}\.){3}[[:digit:]]{1,3}')
+
+		printf "  - name: %s\n    role: %s\n    privateAddress: %s\n    publicAddress: %s\n" \
+			"$name" \
+			"$role" \
+			"$private_ip" \
+			"$public_ip"
+
+		: $((i=i+1))
+	done
+}
+
+print_success "The following nodes have been configured:" ""
+
+printf "machines:\n"
+print_machines master "$MASTER_NODE_COUNT"
+printf "\n"
+print_machines worker "$WORKER_NODE_COUNT"
diff --git a/network.cfg.template b/network.cfg.template
index ab51057..8eb00f1 100644
--- a/network.cfg.template
+++ b/network.cfg.template
@@ -1,15 +1,6 @@
 version: 2
 ethernets:
   eth0:
-    addresses: ["VIRTUAL_IP"]
-    gateway4: ["GATEWAY"]
-    nameservers:
-      search: ["DOMAIN"]
-
+    dhcp4: true
   eth1:
-    match:
-      macaddress: "HOST_MAC"
-    addresses: ["HOST_IP"]
-    gateway4: ["GATEWAY"]
-    nameservers:
-      search: ["DOMAIN"]
+    dhcp4: true

From 123ec5f154c49078b1544a36cf12e309924db1ec Mon Sep 17 00:00:00 2001
From: "Deavon M. McCaffery" <dmccaffery@users.noreply.github.com>
Date: Mon, 11 Jan 2021 20:02:41 +0000
Subject: [PATCH 5/5] fix: add default domain and other cleanup

---
 kvm.sh | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/kvm.sh b/kvm.sh
index 995677c..2f30e93 100755
--- a/kvm.sh
+++ b/kvm.sh
@@ -1,7 +1,7 @@
 #! /usr/bin/env sh
 # shellcheck disable=SC2155
 
-set -e
+set -eu
 
 FORMAT_CLEAR=$(tput sgr0)	# CLEAR ALL FORMAT
 FORMAT_BOLD=$(tput bold)	# SET BRIGH
@@ -67,6 +67,7 @@ help() {
 
 PREFIX=$(whoami)-
 NETWORK="bridge"
+DOMAIN="local"
 
 MASTER_NODE_COUNT=1
 WORKER_NODE_COUNT=1
@@ -75,6 +76,8 @@ CENTOS_VERSION=7
 CPU_LIMIT=2
 MEMORY_LIMIT=4096
 
+GH_USER=
+
 while [ $# -gt 0 ]; do
 	case $1 in
 		--prefix)
@@ -159,7 +162,7 @@ for node in $(sudo virsh list --all --name | grep "kube-${PREFIX}"); do
 done
 
 # remove existing configs
-rm -f kube-*
+rm -f kube-"$PREFIX"*
 
 # ensure base image is accessible
 sudo chmod u=rw,go=r "$BASE_IMAGE"
@@ -271,7 +274,7 @@ while [ $((i<=MASTER_NODE_COUNT)) -ne 0 ]; do
 done
 
 # create the ha proxy node
-create_vm kube-"${PREFIX}"proxy "$PROXY_VIRTUAL_IP" "$PROXY_MAC" "$PROXY_IP"
+create_vm kube-"${PREFIX}"proxy
 
 print_machines() {
 	role=$1