kubevip enhancements

dev/testing.sh

@@ -27,20 +27,13 @@ kc() {
   sed -i /"$1 =/c\\$1 = $2" $CFG
 }
 
-# minimal cluster
-kc masters 1 ; kc workers 0
-$KCU
-kc masters 1 ; kc workers 1
-$KCU
-kc masters 1 ; kc workers 0
-$KCU
-
 # get pods
 get_pods="$KC kubectl get pods -A"
 
 # recreate 1 node
 #./kopsrox.py cluster destroy && ./kopsrox.py cluster create
-#$KCD ; $KCC
+kc workers 0 ; kc masters 1
+$KCD ; $KCC
 
 
 # add a worker and delete it

kopsrox.py

@@ -49,6 +49,9 @@
     "k3s-uninstall" : 'hostname',
     "rejoin-slave" : 'hostname',
   },
+  "kubevip": {
+    "reinstall": '',
+  }
 }
 
 # create list of verbs

lib/kopsrox_k3s.py

@@ -4,7 +4,7 @@
 from kopsrox_config import masterid, k3s_version, masters, workers, cluster_name, vmnames, vmip, cluster_info, list_kopsrox_vm, network_ip
 
 # standard imports
-from kopsrox_proxmox import qaexec, destroy, internet_check, clone
+from kopsrox_proxmox import qaexec, prox_destroy, internet_check, clone
 from kopsrox_kmsg import kmsg
 
 # standard imports
@@ -130,7 +130,7 @@ def k3s_remove_node(vmid):
   kubectl('delete node ' + vmname)
 
   # destroy vm
-  destroy(vmid)
+  prox_destroy(vmid)
 
 # remove cluster - leave master if restore = true
 def k3s_rm_cluster(restore = False):
@@ -151,7 +151,7 @@ def k3s_rm_cluster(restore = False):
 
     # remove node from cluster and proxmox
     if vmname == f'{cluster_name}-m1':
-      destroy(vmid)
+      prox_destroy(vmid)
     else:
       k3s_remove_node(vmid)
 
@@ -238,7 +238,7 @@ def kubeconfig():
 
 # kubectl
 def kubectl(cmd):
-  k3s_cmd = f'/usr/local/bin/kubectl {cmd}'
+  k3s_cmd = f'/usr/local/bin/kubectl {cmd} 2>&1'
   kcmd = qaexec(masterid,k3s_cmd)
   return(kcmd)
 
@@ -289,41 +289,32 @@ def export_k3s_token():
 def install_kube_vip():
 
   # read default kube vip manifest and replace with network_ip
-  kv_manifest = open('./lib/kube-vip/kube-vip.yaml', "r").read().replace('KOPSROX_IP', network_ip)
-  kmsg('k3s_kube-vip', f'creating {network_ip} vip')
+  kv_manifest = open('./lib/kubevip/kubevip.yaml', "r").read().replace('KOPSROX_IP', network_ip).strip()
 
-  # apply the manifest
-  kv_install = qaexec(masterid, f'''cat <<EOF> /tmp/kube-vip.yaml
+  # create the manifest
+  kv_install_manifest = qaexec(masterid, f'''cat <<EOF> /tmp/kubevip.yaml
 {kv_manifest}
 EOF
+''')
+  kubevip_install = kubectl('replace --force -f /tmp/kubevip.yaml')
 
-kubectl create -f /tmp/kube-vip.yaml''')
-
-  # check it installed ok
-  if not re.search('daemonset.apps/kube-vip-ds created', kv_install):
-    kmsg('k3s_kube-vip', f'failed to install kube-vip', 'err')
-    print(kv_install)
+  if not re.search('daemonset.apps/kubevip', kubevip_install):
+    kmsg('k3s_kubevip', f'failed to install kube-vip\n{kubevip_install}', 'err')
     exit(0)
 
+  kmsg('k6s_kubevip', f'created {network_ip} vip')
+
 # return current vip master
 def get_kube_vip_master():
-  kubevip_q = f'get nodes --selector kube-vip.io/has-ip={network_ip} 2>&1'
+  kubevip_q = f'get nodes --selector kube-vip.io/has-ip={network_ip}'
   kubevip_o = kubectl(kubevip_q)
   try:
     kubevip_m = kubevip_o.split()[5]
   except:
-    kmsg('kubevip_check', 'no kubevip label found - reloading kubevip', 'err')
-    kubevip_r = kubectl('rollout restart daemonset kube-vip-ds  -n kube-system')
-    time.sleep(1)
-    kubevip_o = kubectl(kubevip_q)
-    #print(kubevip_o.split())
-    kubevip_m = kubevip_o.split()[5]
-#    kubevip_m = ''
+    kubevip_m = ''
   return(kubevip_m)
 
-# check kube vip is ok by checking for label
-#if get_kube_vip_master() == '':
-#  kmsg('kube-vip_check', 'vip label not found','err')
-#  kubevip_r = kubectl('rollout restart daemonset kube-vip-ds  -n kube-system')
-#  kmsg('kube-vip_check', kubevip_r,'warn')
-#  exit(0)
+def kubevip_reload():
+    reload = kubectl('rollout restart daemonset kubevip -n kube-system')
+    print(reload)
+    time.sleep(2)

lib/kopsrox_proxmox.py

@@ -90,7 +90,7 @@ def qaexec(vmid = masterid,cmd = 'uptime'):
     if (pid_check['err-data']):
 
       # print err data warning \
-      kmsg('qaexec-stderr', ( 'CMD: ' +cmd + '\n' + pid_check['err-data'].strip()), 'err')
+      kmsg('qaexec_stderr', ( 'CMD: ' +cmd + '\n' + pid_check['err-data'].strip()), 'err')
 
       # if there is output return that otherwise exit
       if (pid_check['err-data'] and pid_check['out-data']):
@@ -133,9 +133,9 @@ def get_node(vmid):
   exit(0)
 
 # stop and destroy vm
-def destroy(vmid):
+def prox_destroy(vmid):
 
-    kname = 'proxmox_destroy'
+    kname = 'prox_destroy-vm'
 
     # get node and vmname
     vmname = vmnames[vmid]
@@ -152,10 +152,8 @@ def destroy(vmid):
       task_status(prox.nodes(node).qemu(vmid).delete())
       kmsg(kname, vmname)
     except:
-      # is this image check still required?
-      if not cluster_id == vmid:
-        kmsg(kname, f'unable to destroy {vmid}', 'err')
-        exit()
+      kmsg(kname, f'unable to destroy {vmid}', 'err')
+      exit(0)
 
 # clone
 def clone(vmid):

lib/kube-vip/kube-vip.yaml → lib/kubevip/kubevip.yaml

@@ -1,15 +1,15 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: kube-vip
+  name: kubevip
   namespace: kube-system
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
   annotations:
     rbac.authorization.kubernetes.io/autoupdate: "true"
-  name: system:kube-vip-role
+  name: system:kubevip-role
 rules:
   - apiGroups: [""]
     resources: ["services/status"]
@@ -30,31 +30,29 @@ rules:
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: system:kube-vip-binding
+  name: system:kubevip-binding
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: system:kube-vip-role
+  name: system:kubevip-role
 subjects:
 - kind: ServiceAccount
-  name: kube-vip
+  name: kubevip
   namespace: kube-system
 ---
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
+  name: kubevip
   namespace: kube-system
 spec:
   selector:
     matchLabels:
-      name: kube-vip-ds
+      name: kubevip
   template:
     metadata:
-      creationTimestamp: null
       labels:
-        name: kube-vip-ds
+        name: kubevip
     spec:
       affinity:
         nodeAffinity:
@@ -75,13 +73,13 @@ spec:
         - name: port
           value: "6443"
         - name: vip_interface
-          value: eth0
+          value: 'eth0'
         - name: vip_cidr
           value: "32"
         - name: cp_enable
           value: "true"
         - name: cp_namespace
-          value: kube-system
+          value: 'kube-system'
         - name: vip_ddns
           value: "false"
         - name: svc_enable
@@ -98,21 +96,19 @@ spec:
           value: "true"
         - name: address
           value: KOPSROX_IP
-        image: ghcr.io/kube-vip/kube-vip:v0.7.2
+        image: ghcr.io/kube-vip/kube-vip:main
         imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
+        name: kubevip
         securityContext:
           capabilities:
             add:
             - NET_ADMIN
             - NET_RAW
             - SYS_TIME
       hostNetwork: true
-      serviceAccountName: kube-vip
+      serviceAccountName: kubevip
       tolerations:
       - effect: NoSchedule
         operator: Exists
       - effect: NoExecute
         operator: Exists
-  updateStrategy: {}

lib/verb_cluster.py

@@ -29,7 +29,7 @@
 
   # if masterid not found running 
   if not masterid in list_kopsrox_vm():
-    kmsg(kname,f'{cluster_name}/{cluster_id}', 'sys')
+    kmsg(kname,f'creating {cluster_name} cluster id {cluster_id}', 'sys')
     clone(masterid)
 
   # install k3s on master

lib/verb_image.py

@@ -13,7 +13,7 @@
 import urllib.parse
 
 # proxmox functions
-from kopsrox_proxmox import task_status, destroy
+from kopsrox_proxmox import task_status, prox_destroy
 
 # kmsg
 from kopsrox_kmsg import kmsg
@@ -65,7 +65,7 @@
 
   # destroy template if it exists
   try:
-    destroy(cluster_id)
+    prox_destroy(cluster_id)
   except:
     pass
 
@@ -114,4 +114,4 @@
 # destroy image
 if cmd == 'destroy':
    kmsg(kname, f'{kopsrox_img()}/{cloud_image_desc}', 'warn')
-   destroy(cluster_id)
+   prox_destroy(cluster_id)