diff --git a/dev/testing.sh b/dev/testing.sh index d25e205..dd2a55f 100644 --- a/dev/testing.sh +++ b/dev/testing.sh @@ -27,20 +27,13 @@ kc() { sed -i /"$1 =/c\\$1 = $2" $CFG } -# minimal cluster -kc masters 1 ; kc workers 0 -$KCU -kc masters 1 ; kc workers 1 -$KCU -kc masters 1 ; kc workers 0 -$KCU - # get pods get_pods="$KC kubectl get pods -A" # recreate 1 node #./kopsrox.py cluster destroy && ./kopsrox.py cluster create -#$KCD ; $KCC +kc workers 0 ; kc masters 1 +$KCD ; $KCC # add a worker and delete it diff --git a/kopsrox.py b/kopsrox.py index 55c38ed..3fcad1d 100755 --- a/kopsrox.py +++ b/kopsrox.py @@ -49,6 +49,9 @@ "k3s-uninstall" : 'hostname', "rejoin-slave" : 'hostname', }, + "kubevip": { + "reinstall": '', + } } # create list of verbs diff --git a/lib/kopsrox_k3s.py b/lib/kopsrox_k3s.py index f59bf3d..08c4825 100755 --- a/lib/kopsrox_k3s.py +++ b/lib/kopsrox_k3s.py @@ -4,7 +4,7 @@ from kopsrox_config import masterid, k3s_version, masters, workers, cluster_name, vmnames, vmip, cluster_info, list_kopsrox_vm, network_ip # standard imports -from kopsrox_proxmox import qaexec, destroy, internet_check, clone +from kopsrox_proxmox import qaexec, prox_destroy, internet_check, clone from kopsrox_kmsg import kmsg # standard imports @@ -130,7 +130,7 @@ def k3s_remove_node(vmid): kubectl('delete node ' + vmname) # destroy vm - destroy(vmid) + prox_destroy(vmid) # remove cluster - leave master if restore = true def k3s_rm_cluster(restore = False): @@ -151,7 +151,7 @@ def k3s_rm_cluster(restore = False): # remove node from cluster and proxmox if vmname == f'{cluster_name}-m1': - destroy(vmid) + prox_destroy(vmid) else: k3s_remove_node(vmid) @@ -238,7 +238,7 @@ def kubeconfig(): # kubectl def kubectl(cmd): - k3s_cmd = f'/usr/local/bin/kubectl {cmd}' + k3s_cmd = f'/usr/local/bin/kubectl {cmd} 2>&1' kcmd = qaexec(masterid,k3s_cmd) return(kcmd) @@ -289,41 +289,32 @@ def export_k3s_token(): def install_kube_vip(): # read default kube vip manifest and replace with network_ip - kv_manifest = open('./lib/kube-vip/kube-vip.yaml', "r").read().replace('KOPSROX_IP', network_ip) - kmsg('k3s_kube-vip', f'creating {network_ip} vip') + kv_manifest = open('./lib/kubevip/kubevip.yaml', "r").read().replace('KOPSROX_IP', network_ip).strip() - # apply the manifest - kv_install = qaexec(masterid, f'''cat < /tmp/kube-vip.yaml + # create the manifest + kv_install_manifest = qaexec(masterid, f'''cat < /tmp/kubevip.yaml {kv_manifest} EOF +''') + kubevip_install = kubectl('replace --force -f /tmp/kubevip.yaml') -kubectl create -f /tmp/kube-vip.yaml''') - - # check it installed ok - if not re.search('daemonset.apps/kube-vip-ds created', kv_install): - kmsg('k3s_kube-vip', f'failed to install kube-vip', 'err') - print(kv_install) + if not re.search('daemonset.apps/kubevip', kubevip_install): + kmsg('k3s_kubevip', f'failed to install kube-vip\n{kubevip_install}', 'err') exit(0) + kmsg('k6s_kubevip', f'created {network_ip} vip') + # return current vip master def get_kube_vip_master(): - kubevip_q = f'get nodes --selector kube-vip.io/has-ip={network_ip} 2>&1' + kubevip_q = f'get nodes --selector kube-vip.io/has-ip={network_ip}' kubevip_o = kubectl(kubevip_q) try: kubevip_m = kubevip_o.split()[5] except: - kmsg('kubevip_check', 'no kubevip label found - reloading kubevip', 'err') - kubevip_r = kubectl('rollout restart daemonset kube-vip-ds -n kube-system') - time.sleep(1) - kubevip_o = kubectl(kubevip_q) - #print(kubevip_o.split()) - kubevip_m = kubevip_o.split()[5] -# kubevip_m = '' + kubevip_m = '' return(kubevip_m) -# check kube vip is ok by checking for label -#if get_kube_vip_master() == '': -# kmsg('kube-vip_check', 'vip label not found','err') -# kubevip_r = kubectl('rollout restart daemonset kube-vip-ds -n kube-system') -# kmsg('kube-vip_check', kubevip_r,'warn') -# exit(0) +def kubevip_reload(): + reload = kubectl('rollout restart daemonset kubevip -n kube-system') + print(reload) + time.sleep(2) diff --git a/lib/kopsrox_proxmox.py b/lib/kopsrox_proxmox.py index d89cfa7..2378293 100755 --- a/lib/kopsrox_proxmox.py +++ b/lib/kopsrox_proxmox.py @@ -90,7 +90,7 @@ def qaexec(vmid = masterid,cmd = 'uptime'): if (pid_check['err-data']): # print err data warning \ - kmsg('qaexec-stderr', ( 'CMD: ' +cmd + '\n' + pid_check['err-data'].strip()), 'err') + kmsg('qaexec_stderr', ( 'CMD: ' +cmd + '\n' + pid_check['err-data'].strip()), 'err') # if there is output return that otherwise exit if (pid_check['err-data'] and pid_check['out-data']): @@ -133,9 +133,9 @@ def get_node(vmid): exit(0) # stop and destroy vm -def destroy(vmid): +def prox_destroy(vmid): - kname = 'proxmox_destroy' + kname = 'prox_destroy-vm' # get node and vmname vmname = vmnames[vmid] @@ -152,10 +152,8 @@ def destroy(vmid): task_status(prox.nodes(node).qemu(vmid).delete()) kmsg(kname, vmname) except: - # is this image check still required? - if not cluster_id == vmid: - kmsg(kname, f'unable to destroy {vmid}', 'err') - exit() + kmsg(kname, f'unable to destroy {vmid}', 'err') + exit(0) # clone def clone(vmid): diff --git a/lib/kube-vip/kube-vip.yaml b/lib/kubevip/kubevip.yaml similarity index 84% rename from lib/kube-vip/kube-vip.yaml rename to lib/kubevip/kubevip.yaml index ffe4d24..2553685 100644 --- a/lib/kube-vip/kube-vip.yaml +++ b/lib/kubevip/kubevip.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: kube-vip + name: kubevip namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 @@ -9,7 +9,7 @@ kind: ClusterRole metadata: annotations: rbac.authorization.kubernetes.io/autoupdate: "true" - name: system:kube-vip-role + name: system:kubevip-role rules: - apiGroups: [""] resources: ["services/status"] @@ -30,31 +30,29 @@ rules: kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: system:kube-vip-binding + name: system:kubevip-binding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: system:kube-vip-role + name: system:kubevip-role subjects: - kind: ServiceAccount - name: kube-vip + name: kubevip namespace: kube-system --- apiVersion: apps/v1 kind: DaemonSet metadata: - creationTimestamp: null - name: kube-vip-ds + name: kubevip namespace: kube-system spec: selector: matchLabels: - name: kube-vip-ds + name: kubevip template: metadata: - creationTimestamp: null labels: - name: kube-vip-ds + name: kubevip spec: affinity: nodeAffinity: @@ -75,13 +73,13 @@ spec: - name: port value: "6443" - name: vip_interface - value: eth0 + value: 'eth0' - name: vip_cidr value: "32" - name: cp_enable value: "true" - name: cp_namespace - value: kube-system + value: 'kube-system' - name: vip_ddns value: "false" - name: svc_enable @@ -98,10 +96,9 @@ spec: value: "true" - name: address value: KOPSROX_IP - image: ghcr.io/kube-vip/kube-vip:v0.7.2 + image: ghcr.io/kube-vip/kube-vip:main imagePullPolicy: Always - name: kube-vip - resources: {} + name: kubevip securityContext: capabilities: add: @@ -109,10 +106,9 @@ spec: - NET_RAW - SYS_TIME hostNetwork: true - serviceAccountName: kube-vip + serviceAccountName: kubevip tolerations: - effect: NoSchedule operator: Exists - effect: NoExecute operator: Exists - updateStrategy: {} diff --git a/lib/verb_cluster.py b/lib/verb_cluster.py index 6972b9d..2786ae7 100755 --- a/lib/verb_cluster.py +++ b/lib/verb_cluster.py @@ -29,7 +29,7 @@ # if masterid not found running if not masterid in list_kopsrox_vm(): - kmsg(kname,f'{cluster_name}/{cluster_id}', 'sys') + kmsg(kname,f'creating {cluster_name} cluster id {cluster_id}', 'sys') clone(masterid) # install k3s on master diff --git a/lib/verb_image.py b/lib/verb_image.py index 3c102a3..aac7838 100755 --- a/lib/verb_image.py +++ b/lib/verb_image.py @@ -13,7 +13,7 @@ import urllib.parse # proxmox functions -from kopsrox_proxmox import task_status, destroy +from kopsrox_proxmox import task_status, prox_destroy # kmsg from kopsrox_kmsg import kmsg @@ -65,7 +65,7 @@ # destroy template if it exists try: - destroy(cluster_id) + prox_destroy(cluster_id) except: pass @@ -114,4 +114,4 @@ # destroy image if cmd == 'destroy': kmsg(kname, f'{kopsrox_img()}/{cloud_image_desc}', 'warn') - destroy(cluster_id) + prox_destroy(cluster_id)