From de7ccf94632b58fd3973477a3cb6afed778bcf41 Mon Sep 17 00:00:00 2001 From: Sid Shukla Date: Thu, 21 Mar 2024 18:00:55 +0100 Subject: [PATCH] restructuring by removing env var usage in clusterclass templates (#392) * removed portion which is added by patches in clusterclass this data will be passed thru variables * added default names to reduce more env vars * added default name to cluster with topology * Revert "added default name to cluster with topology" This reverts commit 584dae2f63205e392889f3e2d00ffc2c3826a3e8. * removed namespace env var * make cluster-e2e-templates-no-kubeproxy * removed more env var usage also restructured base-root to more readable ccm folder * removed clustername env var from providerID as not needed * removed more env var usage from ccm yamls separated cluster with and without topology related ccm yamls for easy updates thru variables * now moved ccm/cpi related objects to cluster instead of clusterclass This is useful since ccm applies to workload cluster only. * reverted configmapgenerator changes as crs causing issues * ccm only has one pc-trusted-bundle so no need to have a cluster specific name * fixed the e2e clusterclass file name * fixed lint-yaml * made prismCentral and controlPlaneEndpoing as required fields in api * Revert "made prismCentral and controlPlaneEndpoing as required fields in api" This reverts commit 8e318915c270d02aad991a7b1aec9856dce934ea. * made prismCentral and controlPlaneEndpoing as required fields in api * Revert "made prismCentral and controlPlaneEndpoing as required fields in api" This reverts commit b7a035b5c0ab17fa0af07e1ee5446b0aaf1adfa6. * Revert makefile e2e clusterclass template name Change it back to clusterclass-e2e.yaml * Set default value for tls-cipher-suites in clusterclass templates * Remove clusterclass e2e manifest This should not have been checked-in. * added missing file ref in e2e. * now made all the ref to new cc name --------- Co-authored-by: Deepak Muley --- .gitignore | 1 + Makefile | 8 +- scripts/ccm_nutanix_update.sh | 2 +- templates/{base-root => base}/cm.yaml | 0 templates/base/kustomization.yaml | 4 +- templates/{base-root => base}/secret.yaml | 0 .../{base-root => ccm}/kustomization.yaml | 2 - .../{base-root => ccm}/nutanix-ccm-crs.yaml | 2 +- .../nutanix-ccm-secret.yaml | 0 templates/{base-root => ccm}/nutanix-ccm.yaml | 0 templates/cluster-template-clusterclass.yaml | 404 ++------- templates/cluster-template-csi.yaml | 2 +- templates/cluster-template-topology.yaml | 293 ++++++- templates/cluster-template.yaml | 2 +- templates/clusterclass/clusterclass.yaml | 59 +- templates/clusterclass/kcpt.yaml | 12 +- templates/clusterclass/kct.yaml | 4 +- templates/clusterclass/kustomization.yaml | 1 - templates/clusterclass/nct.yaml | 2 +- templates/clusterclass/nmt-cp.yaml | 20 +- templates/clusterclass/nmt-md.yaml | 20 +- templates/topology/cluster-with-topology.yaml | 6 +- templates/topology/cm.yaml | 7 + templates/topology/kustomization.yaml | 10 + templates/topology/nutanix-ccm-crs.yaml | 22 + templates/topology/nutanix-ccm-secret.yaml | 26 + templates/topology/nutanix-ccm.yaml | 211 +++++ templates/topology/secret.yaml | 18 + test/e2e/config/nutanix.yaml | 4 +- .../kustomization.yaml | 10 +- .../kustomization.yaml | 10 +- .../kustomization.yaml | 10 +- .../kustomization.yaml | 10 +- .../kustomization.yaml | 8 +- .../kustomization.yaml | 14 +- .../kustomization.yaml | 10 +- .../kustomization.yaml | 10 +- .../v1beta1/clusterclass-e2e.yaml | 797 ------------------ .../kustomization.yaml | 2 + .../no-kubeproxy-clusterclass.yaml | 8 +- 40 files changed, 766 insertions(+), 1265 deletions(-) rename templates/{base-root => base}/cm.yaml (100%) rename templates/{base-root => base}/secret.yaml (100%) rename templates/{base-root => ccm}/kustomization.yaml (87%) rename templates/{base-root => ccm}/nutanix-ccm-crs.yaml (88%) rename templates/{base-root => ccm}/nutanix-ccm-secret.yaml (100%) rename templates/{base-root => ccm}/nutanix-ccm.yaml (100%) create mode 100644 templates/topology/cm.yaml create mode 100644 templates/topology/nutanix-ccm-crs.yaml create mode 100644 templates/topology/nutanix-ccm-secret.yaml create mode 100644 templates/topology/nutanix-ccm.yaml create mode 100644 templates/topology/secret.yaml delete mode 100644 test/e2e/data/infrastructure-nutanix/v1beta1/clusterclass-e2e.yaml create mode 100644 test/e2e/data/infrastructure-nutanix/v1beta1/no-kubeproxy/cluster-template-topology/kustomization.yaml diff --git a/.gitignore b/.gitignore index 4810778abd..0c5135b5a9 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ testbin/* # auto-generated e2e cluster-template yaml files test/e2e/data/infrastructure-nutanix/*/cluster-template*.yaml +test/e2e/data/infrastructure-nutanix/*/clusterclass-*.yaml # Output of the go coverage tool, specifically when used with LiteIDE *.out diff --git a/Makefile b/Makefile index fa9682f1e6..9be7d5e1cc 100644 --- a/Makefile +++ b/Makefile @@ -185,7 +185,7 @@ run: manifests generate fmt vet ## Run a controller from your host. .PHONY: docker-build docker-build: ## Build docker image with the manager. echo "Git commit hash: ${GIT_COMMIT_HASH}" - KO_DOCKER_REPO=ko.local GOFLAGS="-ldflags=-X=main.gitCommitHash=${GIT_COMMIT_HASH}" ko build -B --platform=${PLATFORMS} -t ${IMG_TAG} -L . + KO_DOCKER_REPO=ko.local GOFLAGS="-ldflags=-X=main.gitCommitHash=${GIT_COMMIT_HASH}" ko build -B --platform=${PLATFORMS} -t ${IMG_TAG} . .PHONY: docker-push docker-push: ## Push docker image with the manager. @@ -193,7 +193,7 @@ docker-push: ## Push docker image with the manager. .PHONY: docker-push-kind docker-push-kind: ## Make docker image available to kind cluster. - GOOS=linux GOARCH=${shell go env GOARCH} KO_DOCKER_REPO=ko.local ko build -B -t ${IMG_TAG} -L . + GOOS=linux GOARCH=${shell go env GOARCH} KO_DOCKER_REPO=ko.local ko build -B -t ${IMG_TAG} . docker tag ko.local/cluster-api-provider-nutanix:${IMG_TAG} ${IMG} kind load docker-image --name ${KIND_CLUSTER_NAME} ${IMG} @@ -251,7 +251,7 @@ cluster-e2e-templates-v1beta1: ## Generate cluster templates for v1beta1 kustomize build $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-csi --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-csi.yaml kustomize build $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-failure-domains --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-failure-domains.yaml kustomize build $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-clusterclass --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-clusterclass.yaml - kustomize build $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-clusterclass --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/clusterclass-e2e.yaml + kustomize build $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-clusterclass --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/clusterclass-nutanix-quick-start.yaml kustomize build $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-topology --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-topology.yaml cluster-e2e-templates-no-kubeproxy: ##Generate cluster templates without kubeproxy @@ -272,7 +272,7 @@ cluster-e2e-templates-no-kubeproxy: ##Generate cluster templates without kubepro kustomize build $(NUTANIX_E2E_TEMPLATES)/v1beta1/no-kubeproxy/cluster-template-csi --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-csi.yaml kustomize build $(NUTANIX_E2E_TEMPLATES)/v1beta1/no-kubeproxy/cluster-template-failure-domains --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-failure-domains.yaml kustomize build $(NUTANIX_E2E_TEMPLATES)/v1beta1/no-kubeproxy/cluster-template-clusterclass --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-clusterclass.yaml - kustomize build $(NUTANIX_E2E_TEMPLATES)/v1beta1/no-kubeproxy/cluster-template-clusterclass --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/clusterclass-e2e.yaml + kustomize build $(NUTANIX_E2E_TEMPLATES)/v1beta1/no-kubeproxy/cluster-template-clusterclass --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/clusterclass-nutanix-quick-start.yaml kustomize build $(NUTANIX_E2E_TEMPLATES)/v1beta1/no-kubeproxy/cluster-template-topology --load-restrictor LoadRestrictionsNone > $(NUTANIX_E2E_TEMPLATES)/v1beta1/cluster-template-topology.yaml cluster-templates: ## Generate cluster templates for all flavors diff --git a/scripts/ccm_nutanix_update.sh b/scripts/ccm_nutanix_update.sh index 69030b343f..a732eac672 100755 --- a/scripts/ccm_nutanix_update.sh +++ b/scripts/ccm_nutanix_update.sh @@ -22,4 +22,4 @@ helm template -n kube-system nutanix-cloud-provider nutanix/nutanix-cloud-provid --set prismCentralEndPoint='${NUTANIX_ENDPOINT}',prismCentralPort='${NUTANIX_PORT=9440}',prismCentralInsecure='${NUTANIX_INSECURE=false}' \ --set image.repository="\${CCM_REPO=$NUTANIX_CCM_REPO}",image.tag="\${CCM_TAG=v$NUTANIX_CCM_VERSION}" \ --set createSecret=false \ - > templates/base-root/nutanix-ccm.yaml + > templates/ccm/nutanix-ccm.yaml diff --git a/templates/base-root/cm.yaml b/templates/base/cm.yaml similarity index 100% rename from templates/base-root/cm.yaml rename to templates/base/cm.yaml diff --git a/templates/base/kustomization.yaml b/templates/base/kustomization.yaml index 834ffeba8d..8f36d0a4ca 100644 --- a/templates/base/kustomization.yaml +++ b/templates/base/kustomization.yaml @@ -4,7 +4,9 @@ kind: Kustomization namespace: "${NAMESPACE}" bases: - - ../base-root + - ../ccm + - ./cm.yaml + - ./secret.yaml - ./nutanix-cluster.yaml - ./cluster-without-topology.yaml - ./kcp.yaml diff --git a/templates/base-root/secret.yaml b/templates/base/secret.yaml similarity index 100% rename from templates/base-root/secret.yaml rename to templates/base/secret.yaml diff --git a/templates/base-root/kustomization.yaml b/templates/ccm/kustomization.yaml similarity index 87% rename from templates/base-root/kustomization.yaml rename to templates/ccm/kustomization.yaml index 2b76ef6b2a..df0e7a69b2 100644 --- a/templates/base-root/kustomization.yaml +++ b/templates/ccm/kustomization.yaml @@ -8,7 +8,5 @@ configMapGenerator: - nutanix-ccm.yaml bases: - - ./secret.yaml - - ./cm.yaml - ./nutanix-ccm-crs.yaml - ./nutanix-ccm-secret.yaml diff --git a/templates/base-root/nutanix-ccm-crs.yaml b/templates/ccm/nutanix-ccm-crs.yaml similarity index 88% rename from templates/base-root/nutanix-ccm-crs.yaml rename to templates/ccm/nutanix-ccm-crs.yaml index ede23956fa..843631017f 100644 --- a/templates/base-root/nutanix-ccm-crs.yaml +++ b/templates/ccm/nutanix-ccm-crs.yaml @@ -12,7 +12,7 @@ spec: - kind: Secret name: nutanix-ccm-secret - kind: ConfigMap - name: ${CLUSTER_NAME}-pc-trusted-ca-bundle + name: nutanix-ccm-pc-trusted-ca-bundle strategy: ApplyOnce --- apiVersion: v1 diff --git a/templates/base-root/nutanix-ccm-secret.yaml b/templates/ccm/nutanix-ccm-secret.yaml similarity index 100% rename from templates/base-root/nutanix-ccm-secret.yaml rename to templates/ccm/nutanix-ccm-secret.yaml diff --git a/templates/base-root/nutanix-ccm.yaml b/templates/ccm/nutanix-ccm.yaml similarity index 100% rename from templates/base-root/nutanix-ccm.yaml rename to templates/ccm/nutanix-ccm.yaml diff --git a/templates/cluster-template-clusterclass.yaml b/templates/cluster-template-clusterclass.yaml index 7cdfc58c90..981dd8fff9 100644 --- a/templates/cluster-template-clusterclass.yaml +++ b/templates/cluster-template-clusterclass.yaml @@ -1,294 +1,7 @@ -apiVersion: v1 -binaryData: - ca.crt: ${NUTANIX_ADDITIONAL_TRUST_BUNDLE=""} -kind: ConfigMap -metadata: - name: ${CLUSTER_NAME}-pc-trusted-ca-bundle ---- -apiVersion: v1 -data: - nutanix-ccm.yaml: | - --- - apiVersion: v1 - kind: ConfigMap - metadata: - name: ${CLUSTER_NAME}-pc-trusted-ca-bundle - namespace: kube-system - binaryData: - ca.crt: ${NUTANIX_ADDITIONAL_TRUST_BUNDLE=""} - --- - # Source: nutanix-cloud-provider/templates/rbac.yaml - apiVersion: v1 - kind: ServiceAccount - metadata: - name: cloud-controller-manager - namespace: kube-system - --- - # Source: nutanix-cloud-provider/templates/cm.yaml - kind: ConfigMap - apiVersion: v1 - metadata: - name: nutanix-config - namespace: kube-system - data: - nutanix_config.json: |- - { - "prismCentral": { - "address": "${NUTANIX_ENDPOINT}", - "port": ${NUTANIX_PORT=9440}, - "insecure": ${NUTANIX_INSECURE=false}, - "credentialRef": { - "kind": "secret", - "name": "nutanix-creds", - "namespace": "kube-system" - }, - "additionalTrustBundle": { - "kind": "ConfigMap", - "name": "${CLUSTER_NAME}-pc-trusted-ca-bundle", - "namespace": "kube-system" - } - }, - "enableCustomLabeling": ${CCM_CUSTOM_LABEL=false}, - "topologyDiscovery": { - "type": "Prism" - } - } - --- - # Source: nutanix-cloud-provider/templates/rbac.yaml - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - name: system:cloud-controller-manager - rules: - - apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update - - apiGroups: - - "" - resources: - - nodes - verbs: - - "*" - - apiGroups: - - "" - resources: - - nodes/status - verbs: - - patch - - apiGroups: - - "" - resources: - - serviceaccounts - verbs: - - create - - apiGroups: - - "" - resources: - - endpoints - verbs: - - create - - get - - list - - watch - - update - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete - --- - # Source: nutanix-cloud-provider/templates/rbac.yaml - kind: ClusterRoleBinding - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - name: system:cloud-controller-manager - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:cloud-controller-manager - subjects: - - kind: ServiceAccount - name: cloud-controller-manager - namespace: kube-system - --- - # Source: nutanix-cloud-provider/templates/cloud-provider-nutanix-deployment.yaml - apiVersion: apps/v1 - kind: Deployment - metadata: - labels: - k8s-app: nutanix-cloud-controller-manager - name: nutanix-cloud-controller-manager - namespace: kube-system - spec: - replicas: 1 - selector: - matchLabels: - k8s-app: nutanix-cloud-controller-manager - strategy: - type: Recreate - template: - metadata: - labels: - k8s-app: nutanix-cloud-controller-manager - spec: - hostNetwork: true - priorityClassName: system-cluster-critical - nodeSelector: - node-role.kubernetes.io/control-plane: "" - serviceAccountName: cloud-controller-manager - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - k8s-app: nutanix-cloud-controller-manager - topologyKey: kubernetes.io/hostname - dnsPolicy: Default - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - operator: Exists - - effect: NoExecute - key: node.kubernetes.io/unreachable - operator: Exists - tolerationSeconds: 120 - - effect: NoExecute - key: node.kubernetes.io/not-ready - operator: Exists - tolerationSeconds: 120 - - effect: NoSchedule - key: node.cloudprovider.kubernetes.io/uninitialized - operator: Exists - - effect: NoSchedule - key: node.kubernetes.io/not-ready - operator: Exists - containers: - - image: "${CCM_REPO=ghcr.io/nutanix-cloud-native/cloud-provider-nutanix/controller}:${CCM_TAG=v0.3.2}" - imagePullPolicy: IfNotPresent - name: nutanix-cloud-controller-manager - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - args: - - "--leader-elect=true" - - "--cloud-config=/etc/cloud/nutanix_config.json" - resources: - requests: - cpu: 100m - memory: 50Mi - volumeMounts: - - mountPath: /etc/cloud - name: nutanix-config-volume - readOnly: true - volumes: - - name: nutanix-config-volume - configMap: - name: nutanix-config -kind: ConfigMap -metadata: - name: nutanix-ccm ---- -apiVersion: v1 -kind: Secret -metadata: - name: ${CLUSTER_NAME} -stringData: - credentials: | - [ - { - "type": "basic_auth", - "data": { - "prismCentral":{ - "username": "${NUTANIX_USER}", - "password": "${NUTANIX_PASSWORD}" - } - } - } - ] ---- -apiVersion: v1 -kind: Secret -metadata: - name: nutanix-ccm-secret -stringData: - nutanix-ccm-secret.yaml: | - apiVersion: v1 - kind: Secret - metadata: - name: nutanix-creds - namespace: kube-system - stringData: - credentials: | - [ - { - "type": "basic_auth", - "data": { - "prismCentral":{ - "username": "${NUTANIX_USER}", - "password": "${NUTANIX_PASSWORD}" - }, - "prismElements": null - } - } - ] -type: addons.cluster.x-k8s.io/resource-set ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: nutanix-ccm-crs -spec: - clusterSelector: - matchLabels: - ccm: nutanix - resources: - - kind: ConfigMap - name: nutanix-ccm - - kind: Secret - name: nutanix-ccm-secret - - kind: ConfigMap - name: ${CLUSTER_NAME}-pc-trusted-ca-bundle - strategy: ApplyOnce ---- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate metadata: - name: ${CLUSTER_CLASS_NAME}-kcfg-0 + name: nutanix-quick-start-kcfg-0 spec: template: spec: @@ -297,7 +10,7 @@ spec: kubeletExtraArgs: cloud-provider: external eviction-hard: nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<15%,memory.available<100Mi,imagefs.inodesFree<10% - tls-cipher-suites: ${TLS_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256} + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 postKubeadmCommands: - echo "after kubeadm call" > /var/log/postkubeadm.log preKubeadmCommands: @@ -308,7 +21,7 @@ spec: apiVersion: cluster.x-k8s.io/v1beta1 kind: ClusterClass metadata: - name: ${CLUSTER_CLASS_NAME} + name: nutanix-quick-start spec: controlPlane: machineHealthCheck: @@ -337,19 +50,16 @@ spec: ref: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: NutanixMachineTemplate - name: ${CLUSTER_CLASS_NAME}-cp-nmt - namespace: ${NAMESPACE} + name: nutanix-quick-start-cp-nmt ref: apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlaneTemplate - name: ${CLUSTER_CLASS_NAME}-kcpt - namespace: ${NAMESPACE} + name: nutanix-quick-start-kcpt infrastructure: ref: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: NutanixClusterTemplate - name: ${CLUSTER_CLASS_NAME}-nct - namespace: ${NAMESPACE} + name: nutanix-quick-start-nct patches: - definitions: - jsonPatches: @@ -383,7 +93,7 @@ spec: matchResources: machineDeploymentClass: names: - - ${CLUSTER_CLASS_NAME}-worker + - nutanix-quick-start-worker name: add-ssh-user - definitions: - jsonPatches: @@ -409,10 +119,10 @@ spec: port: {{ .prismCentralEndpoint.port }} insecure: {{ .prismCentralEndpoint.insecure }} credentialRef: - name: "${CLUSTER_NAME}" + name: '{{ .prismCentralEndpoint.credentialSecret }}' kind: Secret additionalTrustBundle: - name: ${CLUSTER_NAME}-pc-trusted-ca-bundle + name: '{{ .prismCentralEndpoint.additionalTrustBundle }}' kind: ConfigMap selector: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 @@ -422,31 +132,31 @@ spec: name: add-pc-endpoint-and-creds - definitions: - jsonPatches: - - op: replace + - op: add path: /spec/template/spec/bootType valueFrom: variable: controlPlaneMachineDetails.bootType - - op: replace + - op: add path: /spec/template/spec/vcpusPerSocket valueFrom: variable: controlPlaneMachineDetails.vcpusPerSocket - - op: replace + - op: add path: /spec/template/spec/memorySize valueFrom: variable: controlPlaneMachineDetails.memorySize - - op: replace + - op: add path: /spec/template/spec/systemDiskSize valueFrom: variable: controlPlaneMachineDetails.systemDiskSize - - op: replace + - op: add path: /spec/template/spec/image/name valueFrom: variable: controlPlaneMachineDetails.imageName - - op: replace + - op: add path: /spec/template/spec/cluster/name valueFrom: variable: controlPlaneMachineDetails.clusterName - - op: replace + - op: add path: /spec/template/spec/subnet valueFrom: template: | @@ -460,31 +170,31 @@ spec: name: update-control-plane-machine-template - definitions: - jsonPatches: - - op: replace + - op: add path: /spec/template/spec/bootType valueFrom: variable: workerMachineDetails.bootType - - op: replace + - op: add path: /spec/template/spec/vcpusPerSocket valueFrom: variable: workerMachineDetails.vcpusPerSocket - - op: replace + - op: add path: /spec/template/spec/memorySize valueFrom: variable: workerMachineDetails.memorySize - - op: replace + - op: add path: /spec/template/spec/systemDiskSize valueFrom: variable: workerMachineDetails.systemDiskSize - - op: replace + - op: add path: /spec/template/spec/image/name valueFrom: variable: workerMachineDetails.imageName - - op: replace + - op: add path: /spec/template/spec/cluster/name valueFrom: variable: workerMachineDetails.clusterName - - op: replace + - op: add path: /spec/template/spec/subnet valueFrom: template: | @@ -496,7 +206,7 @@ spec: matchResources: machineDeploymentClass: names: - - ${CLUSTER_CLASS_NAME}-worker + - nutanix-quick-start-worker name: update-worker-machine-template variables: - name: sshKey @@ -520,8 +230,12 @@ spec: schema: openAPIV3Schema: properties: + additionalTrustBundle: + type: string address: type: string + credentialSecret: + type: string insecure: type: boolean port: @@ -573,7 +287,7 @@ spec: type: object workers: machineDeployments: - - class: ${CLUSTER_CLASS_NAME}-worker + - class: nutanix-quick-start-worker machineHealthCheck: maxUnhealthy: 40% nodeStartupTimeout: 10m @@ -601,19 +315,17 @@ spec: ref: apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate - name: ${CLUSTER_CLASS_NAME}-kcfg-0 - namespace: ${NAMESPACE} + name: nutanix-quick-start-kcfg-0 infrastructure: ref: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: NutanixMachineTemplate - name: ${CLUSTER_CLASS_NAME}-md-nmt - namespace: ${NAMESPACE} + name: nutanix-quick-start-md-nmt --- apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlaneTemplate metadata: - name: ${CLUSTER_CLASS_NAME}-kcpt + name: nutanix-quick-start-kcpt spec: template: spec: @@ -626,14 +338,14 @@ spec: - 0.0.0.0 extraArgs: cloud-provider: external - tls-cipher-suites: ${TLS_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256} + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 controllerManager: extraArgs: cloud-provider: external - tls-cipher-suites: ${TLS_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256} + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 scheduler: extraArgs: - tls-cipher-suites: ${TLS_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256} + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 files: - content: | apiVersion: v1 @@ -705,13 +417,13 @@ spec: kubeletExtraArgs: cloud-provider: external eviction-hard: nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<15%,memory.available<100Mi,imagefs.inodesFree<10% - tls-cipher-suites: ${TLS_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256} + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 joinConfiguration: nodeRegistration: kubeletExtraArgs: cloud-provider: external eviction-hard: nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<15%,memory.available<100Mi,imagefs.inodesFree<10% - tls-cipher-suites: ${TLS_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256} + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 postKubeadmCommands: - echo export KUBECONFIG=/etc/kubernetes/admin.conf >> /root/.bashrc - | @@ -744,7 +456,7 @@ spec: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: NutanixClusterTemplate metadata: - name: ${CLUSTER_CLASS_NAME}-nct + name: nutanix-quick-start-nct spec: template: spec: @@ -753,45 +465,45 @@ spec: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: NutanixMachineTemplate metadata: - name: ${CLUSTER_CLASS_NAME}-cp-nmt + name: nutanix-quick-start-cp-nmt spec: template: spec: - bootType: ${NUTANIX_MACHINE_BOOT_TYPE=legacy} + bootType: legacy cluster: - name: ${NUTANIX_PRISM_ELEMENT_CLUSTER_NAME} + name: "" type: name image: - name: ${NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME} + name: "" type: name - memorySize: ${NUTANIX_MACHINE_MEMORY_SIZE=4Gi} - providerID: nutanix://${CLUSTER_NAME}-m1 + memorySize: 4Gi + providerID: nutanix://vm-uuid subnet: - - name: ${NUTANIX_SUBNET_NAME} + - name: "" type: name - systemDiskSize: ${NUTANIX_SYSTEMDISK_SIZE=40Gi} - vcpuSockets: ${NUTANIX_MACHINE_VCPU_SOCKET=2} - vcpusPerSocket: ${NUTANIX_MACHINE_VCPU_PER_SOCKET=1} + systemDiskSize: 40Gi + vcpuSockets: 2 + vcpusPerSocket: 1 --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: NutanixMachineTemplate metadata: - name: ${CLUSTER_CLASS_NAME}-md-nmt + name: nutanix-quick-start-md-nmt spec: template: spec: - bootType: ${NUTANIX_MACHINE_BOOT_TYPE=legacy} + bootType: legacy cluster: - name: ${NUTANIX_PRISM_ELEMENT_CLUSTER_NAME} + name: "" type: name image: - name: ${NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME} + name: "" type: name - memorySize: ${NUTANIX_MACHINE_MEMORY_SIZE=4Gi} - providerID: nutanix://${CLUSTER_NAME}-m1 + memorySize: 4Gi + providerID: nutanix://vm-uuid subnet: - - name: ${NUTANIX_SUBNET_NAME} + - name: "" type: name - systemDiskSize: ${NUTANIX_SYSTEMDISK_SIZE=40Gi} - vcpuSockets: ${NUTANIX_MACHINE_VCPU_SOCKET=2} - vcpusPerSocket: ${NUTANIX_MACHINE_VCPU_PER_SOCKET=1} + systemDiskSize: 40Gi + vcpuSockets: 2 + vcpusPerSocket: 1 diff --git a/templates/cluster-template-csi.yaml b/templates/cluster-template-csi.yaml index fb08b2b73a..0cf150f9f2 100644 --- a/templates/cluster-template-csi.yaml +++ b/templates/cluster-template-csi.yaml @@ -1905,7 +1905,7 @@ spec: - kind: Secret name: nutanix-ccm-secret - kind: ConfigMap - name: ${CLUSTER_NAME}-pc-trusted-ca-bundle + name: nutanix-ccm-pc-trusted-ca-bundle strategy: ApplyOnce --- apiVersion: addons.cluster.x-k8s.io/v1beta1 diff --git a/templates/cluster-template-topology.yaml b/templates/cluster-template-topology.yaml index c3ac8ab5b7..8d6de08e1a 100644 --- a/templates/cluster-template-topology.yaml +++ b/templates/cluster-template-topology.yaml @@ -1,3 +1,290 @@ +apiVersion: v1 +binaryData: + ca.crt: ${NUTANIX_ADDITIONAL_TRUST_BUNDLE=""} +kind: ConfigMap +metadata: + name: ${CLUSTER_NAME}-pc-trusted-ca-bundle +--- +apiVersion: v1 +data: + nutanix-ccm.yaml: | + --- + apiVersion: v1 + kind: ConfigMap + metadata: + name: nutanix-ccm-pc-trusted-ca-bundle + namespace: kube-system + binaryData: + ca.crt: ${NUTANIX_ADDITIONAL_TRUST_BUNDLE=""} + --- + # Source: nutanix-cloud-provider/templates/rbac.yaml + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + # Source: nutanix-cloud-provider/templates/cm.yaml + kind: ConfigMap + apiVersion: v1 + metadata: + name: nutanix-config + namespace: kube-system + data: + nutanix_config.json: |- + { + "prismCentral": { + "address": "${NUTANIX_ENDPOINT}", + "port": ${NUTANIX_PORT=9440}, + "insecure": ${NUTANIX_INSECURE=false}, + "credentialRef": { + "kind": "secret", + "name": "nutanix-creds", + "namespace": "kube-system" + }, + "additionalTrustBundle": { + "kind": "ConfigMap", + "name": "nutanix-ccm-pc-trusted-ca-bundle", + "namespace": "kube-system" + } + }, + "enableCustomLabeling": ${CCM_CUSTOM_LABEL=false}, + "topologyDiscovery": { + "type": "Prism" + } + } + --- + # Source: nutanix-cloud-provider/templates/rbac.yaml + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + --- + # Source: nutanix-cloud-provider/templates/rbac.yaml + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + # Source: nutanix-cloud-provider/templates/cloud-provider-nutanix-deployment.yaml + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + k8s-app: nutanix-cloud-controller-manager + name: nutanix-cloud-controller-manager + namespace: kube-system + spec: + replicas: 1 + selector: + matchLabels: + k8s-app: nutanix-cloud-controller-manager + strategy: + type: Recreate + template: + metadata: + labels: + k8s-app: nutanix-cloud-controller-manager + spec: + hostNetwork: true + priorityClassName: system-cluster-critical + nodeSelector: + node-role.kubernetes.io/control-plane: "" + serviceAccountName: cloud-controller-manager + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: nutanix-cloud-controller-manager + topologyKey: kubernetes.io/hostname + dnsPolicy: Default + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoExecute + key: node.kubernetes.io/unreachable + operator: Exists + tolerationSeconds: 120 + - effect: NoExecute + key: node.kubernetes.io/not-ready + operator: Exists + tolerationSeconds: 120 + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + containers: + - image: "${CCM_REPO=ghcr.io/nutanix-cloud-native/cloud-provider-nutanix/controller}:${CCM_TAG=v0.3.2}" + imagePullPolicy: IfNotPresent + name: nutanix-cloud-controller-manager + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + args: + - "--leader-elect=true" + - "--cloud-config=/etc/cloud/nutanix_config.json" + resources: + requests: + cpu: 100m + memory: 50Mi + volumeMounts: + - mountPath: /etc/cloud + name: nutanix-config-volume + readOnly: true + volumes: + - name: nutanix-config-volume + configMap: + name: nutanix-config +kind: ConfigMap +metadata: + name: nutanix-ccm +--- +apiVersion: v1 +kind: Secret +metadata: + name: ${CLUSTER_NAME}-pc-creds +stringData: + credentials: | + [ + { + "type": "basic_auth", + "data": { + "prismCentral":{ + "username": "${NUTANIX_USER}", + "password": "${NUTANIX_PASSWORD}" + } + } + } + ] +--- +apiVersion: v1 +kind: Secret +metadata: + name: nutanix-ccm-secret +stringData: + nutanix-ccm-secret.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: nutanix-creds + namespace: kube-system + stringData: + credentials: | + [ + { + "type": "basic_auth", + "data": { + "prismCentral":{ + "username": "${NUTANIX_USER}", + "password": "${NUTANIX_PASSWORD}" + }, + "prismElements": null + } + } + ] +type: addons.cluster.x-k8s.io/resource-set +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: nutanix-ccm-crs +spec: + clusterSelector: + matchLabels: + ccm: nutanix + resources: + - kind: ConfigMap + name: nutanix-ccm + - kind: Secret + name: nutanix-ccm-secret + - kind: ConfigMap + name: nutanix-ccm-pc-trusted-ca-bundle + strategy: ApplyOnce +--- apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: @@ -15,7 +302,7 @@ spec: cidrBlocks: - 172.19.0.0/16 topology: - class: ${CLUSTER_CLASS_NAME} + class: nutanix-quick-start controlPlane: metadata: {} replicas: ${CONTROL_PLANE_MACHINE_COUNT} @@ -28,7 +315,9 @@ spec: port: ${CONTROL_PLANE_ENDPOINT_PORT=6443} - name: prismCentralEndpoint value: + additionalTrustBundle: ${CLUSTER_NAME}-pc-trusted-ca-bundle address: ${NUTANIX_ENDPOINT} + credentialSecret: ${CLUSTER_NAME}-pc-creds insecure: ${NUTANIX_INSECURE=false} port: ${NUTANIX_PORT=9440} - name: controlPlaneMachineDetails @@ -54,7 +343,7 @@ spec: version: ${KUBERNETES_VERSION} workers: machineDeployments: - - class: ${CLUSTER_CLASS_NAME}-worker + - class: nutanix-quick-start-worker metadata: {} name: md-0 replicas: ${WORKER_MACHINE_COUNT} diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index f38035dc12..4f58bb6423 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -287,7 +287,7 @@ spec: - kind: Secret name: nutanix-ccm-secret - kind: ConfigMap - name: ${CLUSTER_NAME}-pc-trusted-ca-bundle + name: nutanix-ccm-pc-trusted-ca-bundle strategy: ApplyOnce --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 diff --git a/templates/clusterclass/clusterclass.yaml b/templates/clusterclass/clusterclass.yaml index c5858fae00..1a355b8a2b 100644 --- a/templates/clusterclass/clusterclass.yaml +++ b/templates/clusterclass/clusterclass.yaml @@ -1,20 +1,18 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: ClusterClass metadata: - name: "${CLUSTER_CLASS_NAME}" + name: "nutanix-quick-start" spec: controlPlane: ref: apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlaneTemplate - name: ${CLUSTER_CLASS_NAME}-kcpt - namespace: ${NAMESPACE} + name: nutanix-quick-start-kcpt machineInfrastructure: ref: kind: NutanixMachineTemplate apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - name: ${CLUSTER_CLASS_NAME}-cp-nmt - namespace: ${NAMESPACE} + name: nutanix-quick-start-cp-nmt machineHealthCheck: maxUnhealthy: 40% nodeStartupTimeout: 10m @@ -39,20 +37,18 @@ spec: timeout: 300s workers: machineDeployments: - - class: ${CLUSTER_CLASS_NAME}-worker + - class: nutanix-quick-start-worker template: bootstrap: ref: apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate - name: "${CLUSTER_CLASS_NAME}-kcfg-0" - namespace: ${NAMESPACE} + name: "nutanix-quick-start-kcfg-0" infrastructure: ref: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: NutanixMachineTemplate - name: ${CLUSTER_CLASS_NAME}-md-nmt - namespace: ${NAMESPACE} + name: nutanix-quick-start-md-nmt machineHealthCheck: maxUnhealthy: 40% nodeStartupTimeout: 10m @@ -79,8 +75,7 @@ spec: ref: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: NutanixClusterTemplate - name: ${CLUSTER_CLASS_NAME}-nct - namespace: ${NAMESPACE} + name: nutanix-quick-start-nct patches: - name: add-ssh-user definitions: @@ -105,7 +100,7 @@ spec: matchResources: machineDeploymentClass: names: - - ${CLUSTER_CLASS_NAME}-worker + - nutanix-quick-start-worker jsonPatches: - op: add path: /spec/template/spec/users @@ -146,10 +141,10 @@ spec: port: {{ .prismCentralEndpoint.port }} insecure: {{ .prismCentralEndpoint.insecure }} credentialRef: - name: "${CLUSTER_NAME}" + name: '{{ .prismCentralEndpoint.credentialSecret }}' kind: Secret additionalTrustBundle: - name: ${CLUSTER_NAME}-pc-trusted-ca-bundle + name: '{{ .prismCentralEndpoint.additionalTrustBundle }}' kind: ConfigMap - name: update-control-plane-machine-template definitions: @@ -159,31 +154,31 @@ spec: matchResources: controlPlane: true jsonPatches: - - op: replace + - op: add path: /spec/template/spec/bootType valueFrom: variable: controlPlaneMachineDetails.bootType - - op: replace + - op: add path: /spec/template/spec/vcpusPerSocket valueFrom: variable: controlPlaneMachineDetails.vcpusPerSocket - - op: replace + - op: add path: /spec/template/spec/memorySize valueFrom: variable: controlPlaneMachineDetails.memorySize - - op: replace + - op: add path: /spec/template/spec/systemDiskSize valueFrom: variable: controlPlaneMachineDetails.systemDiskSize - - op: replace + - op: add path: /spec/template/spec/image/name valueFrom: variable: controlPlaneMachineDetails.imageName - - op: replace + - op: add path: /spec/template/spec/cluster/name valueFrom: variable: controlPlaneMachineDetails.clusterName - - op: replace + - op: add path: /spec/template/spec/subnet valueFrom: template: | @@ -197,33 +192,33 @@ spec: matchResources: machineDeploymentClass: names: - - ${CLUSTER_CLASS_NAME}-worker + - nutanix-quick-start-worker jsonPatches: - - op: replace + - op: add path: /spec/template/spec/bootType valueFrom: variable: workerMachineDetails.bootType - - op: replace + - op: add path: /spec/template/spec/vcpusPerSocket valueFrom: variable: workerMachineDetails.vcpusPerSocket - - op: replace + - op: add path: /spec/template/spec/memorySize valueFrom: variable: workerMachineDetails.memorySize - - op: replace + - op: add path: /spec/template/spec/systemDiskSize valueFrom: variable: workerMachineDetails.systemDiskSize - - op: replace + - op: add path: /spec/template/spec/image/name valueFrom: variable: workerMachineDetails.imageName - - op: replace + - op: add path: /spec/template/spec/cluster/name valueFrom: variable: workerMachineDetails.clusterName - - op: replace + - op: add path: /spec/template/spec/subnet valueFrom: template: | @@ -257,6 +252,10 @@ spec: type: integer insecure: type: boolean + credentialSecret: + type: string + additionalTrustBundle: + type: string type: object - name: controlPlaneMachineDetails required: true diff --git a/templates/clusterclass/kcpt.yaml b/templates/clusterclass/kcpt.yaml index 945f554d62..46c9bd50b7 100644 --- a/templates/clusterclass/kcpt.yaml +++ b/templates/clusterclass/kcpt.yaml @@ -1,7 +1,7 @@ apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlaneTemplate metadata: - name: "${CLUSTER_CLASS_NAME}-kcpt" + name: "nutanix-quick-start-kcpt" spec: template: spec: @@ -14,14 +14,14 @@ spec: - 0.0.0.0 extraArgs: cloud-provider: external - tls-cipher-suites: ${TLS_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256} + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 controllerManager: extraArgs: cloud-provider: external - tls-cipher-suites: ${TLS_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256} + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 scheduler: extraArgs: - tls-cipher-suites: ${TLS_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256} + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 files: - content: | apiVersion: v1 @@ -93,13 +93,13 @@ spec: kubeletExtraArgs: cloud-provider: external eviction-hard: nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<15%,memory.available<100Mi,imagefs.inodesFree<10% - tls-cipher-suites: ${TLS_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256} + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 joinConfiguration: nodeRegistration: kubeletExtraArgs: cloud-provider: external eviction-hard: nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<15%,memory.available<100Mi,imagefs.inodesFree<10% - tls-cipher-suites: ${TLS_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256} + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 preKubeadmCommands: - echo "before kubeadm call" > /var/log/prekubeadm.log - hostnamectl set-hostname "{{ ds.meta_data.hostname }}" diff --git a/templates/clusterclass/kct.yaml b/templates/clusterclass/kct.yaml index ca1ae6d599..6d85e2a169 100644 --- a/templates/clusterclass/kct.yaml +++ b/templates/clusterclass/kct.yaml @@ -1,7 +1,7 @@ apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate metadata: - name: "${CLUSTER_CLASS_NAME}-kcfg-0" + name: "nutanix-quick-start-kcfg-0" spec: template: spec: @@ -10,7 +10,7 @@ spec: kubeletExtraArgs: cloud-provider: external eviction-hard: nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<15%,memory.available<100Mi,imagefs.inodesFree<10% - tls-cipher-suites: ${TLS_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256} + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 postKubeadmCommands: - echo "after kubeadm call" > /var/log/postkubeadm.log preKubeadmCommands: diff --git a/templates/clusterclass/kustomization.yaml b/templates/clusterclass/kustomization.yaml index fd361ca6ed..ba02b9f26a 100644 --- a/templates/clusterclass/kustomization.yaml +++ b/templates/clusterclass/kustomization.yaml @@ -2,7 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization bases: - - ../base-root - ./nct.yaml - ./clusterclass.yaml - ./nmt-cp.yaml diff --git a/templates/clusterclass/nct.yaml b/templates/clusterclass/nct.yaml index fb89a84274..350355cde9 100644 --- a/templates/clusterclass/nct.yaml +++ b/templates/clusterclass/nct.yaml @@ -1,7 +1,7 @@ apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: NutanixClusterTemplate metadata: - name: ${CLUSTER_CLASS_NAME}-nct + name: nutanix-quick-start-nct spec: template: spec: diff --git a/templates/clusterclass/nmt-cp.yaml b/templates/clusterclass/nmt-cp.yaml index 4bb83eb78b..f60220f853 100644 --- a/templates/clusterclass/nmt-cp.yaml +++ b/templates/clusterclass/nmt-cp.yaml @@ -1,27 +1,27 @@ apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: NutanixMachineTemplate metadata: - name: "${CLUSTER_CLASS_NAME}-cp-nmt" + name: "nutanix-quick-start-cp-nmt" spec: template: spec: - providerID: "nutanix://${CLUSTER_NAME}-m1" + providerID: "nutanix://vm-uuid" # Supported options for boot type: legacy and uefi # Defaults to legacy if not set - bootType: ${NUTANIX_MACHINE_BOOT_TYPE=legacy} - vcpusPerSocket: ${NUTANIX_MACHINE_VCPU_PER_SOCKET=1} - vcpuSockets: ${NUTANIX_MACHINE_VCPU_SOCKET=2} - memorySize: "${NUTANIX_MACHINE_MEMORY_SIZE=4Gi}" - systemDiskSize: "${NUTANIX_SYSTEMDISK_SIZE=40Gi}" + bootType: legacy + vcpusPerSocket: 1 + vcpuSockets: 2 + memorySize: 4Gi + systemDiskSize: 40Gi image: type: name - name: "${NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME}" + name: "" cluster: type: name - name: "${NUTANIX_PRISM_ELEMENT_CLUSTER_NAME}" + name: "" subnet: - type: name - name: "${NUTANIX_SUBNET_NAME}" + name: "" # Adds additional categories to the virtual machines. # Note: Categories must already be present in Prism Central # additionalCategories: diff --git a/templates/clusterclass/nmt-md.yaml b/templates/clusterclass/nmt-md.yaml index 3368a2aee1..e36e8c9608 100644 --- a/templates/clusterclass/nmt-md.yaml +++ b/templates/clusterclass/nmt-md.yaml @@ -1,27 +1,27 @@ apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: NutanixMachineTemplate metadata: - name: "${CLUSTER_CLASS_NAME}-md-nmt" + name: "nutanix-quick-start-md-nmt" spec: template: spec: - providerID: "nutanix://${CLUSTER_NAME}-m1" + providerID: "nutanix://vm-uuid" # Supported options for boot type: legacy and uefi # Defaults to legacy if not set - bootType: ${NUTANIX_MACHINE_BOOT_TYPE=legacy} - vcpusPerSocket: ${NUTANIX_MACHINE_VCPU_PER_SOCKET=1} - vcpuSockets: ${NUTANIX_MACHINE_VCPU_SOCKET=2} - memorySize: "${NUTANIX_MACHINE_MEMORY_SIZE=4Gi}" - systemDiskSize: "${NUTANIX_SYSTEMDISK_SIZE=40Gi}" + bootType: legacy + vcpusPerSocket: 1 + vcpuSockets: 2 + memorySize: 4Gi + systemDiskSize: 40Gi image: type: name - name: "${NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME}" + name: "" cluster: type: name - name: "${NUTANIX_PRISM_ELEMENT_CLUSTER_NAME}" + name: "" subnet: - type: name - name: "${NUTANIX_SUBNET_NAME}" + name: "" # Adds additional categories to the virtual machines. # Note: Categories must already be present in Prism Central # additionalCategories: diff --git a/templates/topology/cluster-with-topology.yaml b/templates/topology/cluster-with-topology.yaml index 24da2d6c22..c8388429bc 100644 --- a/templates/topology/cluster-with-topology.yaml +++ b/templates/topology/cluster-with-topology.yaml @@ -12,14 +12,14 @@ spec: cidrBlocks: ["172.20.0.0/16"] serviceDomain: "cluster.local" topology: - class: "${CLUSTER_CLASS_NAME}" + class: "nutanix-quick-start" version: ${KUBERNETES_VERSION} controlPlane: replicas: ${CONTROL_PLANE_MACHINE_COUNT} metadata: {} workers: machineDeployments: - - class: ${CLUSTER_CLASS_NAME}-worker + - class: nutanix-quick-start-worker name: md-0 replicas: ${WORKER_MACHINE_COUNT} metadata: {} @@ -35,6 +35,8 @@ spec: address: "${NUTANIX_ENDPOINT}" port: ${NUTANIX_PORT=9440} insecure: ${NUTANIX_INSECURE=false} + credentialSecret: ${CLUSTER_NAME}-pc-creds + additionalTrustBundle: ${CLUSTER_NAME}-pc-trusted-ca-bundle - name: controlPlaneMachineDetails value: bootType: ${NUTANIX_MACHINE_BOOT_TYPE=legacy} diff --git a/templates/topology/cm.yaml b/templates/topology/cm.yaml new file mode 100644 index 0000000000..f95f785633 --- /dev/null +++ b/templates/topology/cm.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ${CLUSTER_NAME}-pc-trusted-ca-bundle +binaryData: + ca.crt: ${NUTANIX_ADDITIONAL_TRUST_BUNDLE=""} diff --git a/templates/topology/kustomization.yaml b/templates/topology/kustomization.yaml index 5a117193cb..51f6980c31 100644 --- a/templates/topology/kustomization.yaml +++ b/templates/topology/kustomization.yaml @@ -1,8 +1,18 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +configMapGenerator: +- name: nutanix-ccm + behavior: merge + files: + - nutanix-ccm.yaml + bases: + - ./cm.yaml - ./cluster-with-topology.yaml + - ./secret.yaml + - ./nutanix-ccm-secret.yaml + - ./nutanix-ccm-crs.yaml patchesStrategicMerge: - ./ccm-patch.yaml diff --git a/templates/topology/nutanix-ccm-crs.yaml b/templates/topology/nutanix-ccm-crs.yaml new file mode 100644 index 0000000000..843631017f --- /dev/null +++ b/templates/topology/nutanix-ccm-crs.yaml @@ -0,0 +1,22 @@ +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: nutanix-ccm-crs +spec: + clusterSelector: + matchLabels: + ccm: nutanix + resources: + - kind: ConfigMap + name: nutanix-ccm + - kind: Secret + name: nutanix-ccm-secret + - kind: ConfigMap + name: nutanix-ccm-pc-trusted-ca-bundle + strategy: ApplyOnce +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nutanix-ccm +data: diff --git a/templates/topology/nutanix-ccm-secret.yaml b/templates/topology/nutanix-ccm-secret.yaml new file mode 100644 index 0000000000..2c5ddbe59b --- /dev/null +++ b/templates/topology/nutanix-ccm-secret.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: Secret +metadata: + name: nutanix-ccm-secret +type: addons.cluster.x-k8s.io/resource-set +stringData: + nutanix-ccm-secret.yaml: | + apiVersion: v1 + kind: Secret + metadata: + name: nutanix-creds + namespace: kube-system + stringData: + credentials: | + [ + { + "type": "basic_auth", + "data": { + "prismCentral":{ + "username": "${NUTANIX_USER}", + "password": "${NUTANIX_PASSWORD}" + }, + "prismElements": null + } + } + ] diff --git a/templates/topology/nutanix-ccm.yaml b/templates/topology/nutanix-ccm.yaml new file mode 100644 index 0000000000..8af3d9d319 --- /dev/null +++ b/templates/topology/nutanix-ccm.yaml @@ -0,0 +1,211 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nutanix-ccm-pc-trusted-ca-bundle + namespace: kube-system +binaryData: + ca.crt: ${NUTANIX_ADDITIONAL_TRUST_BUNDLE=""} +--- +# Source: nutanix-cloud-provider/templates/rbac.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cloud-controller-manager + namespace: kube-system +--- +# Source: nutanix-cloud-provider/templates/cm.yaml +kind: ConfigMap +apiVersion: v1 +metadata: + name: nutanix-config + namespace: kube-system +data: + nutanix_config.json: |- + { + "prismCentral": { + "address": "${NUTANIX_ENDPOINT}", + "port": ${NUTANIX_PORT=9440}, + "insecure": ${NUTANIX_INSECURE=false}, + "credentialRef": { + "kind": "secret", + "name": "nutanix-creds", + "namespace": "kube-system" + }, + "additionalTrustBundle": { + "kind": "ConfigMap", + "name": "nutanix-ccm-pc-trusted-ca-bundle", + "namespace": "kube-system" + } + }, + "enableCustomLabeling": ${CCM_CUSTOM_LABEL=false}, + "topologyDiscovery": { + "type": "Prism" + } + } +--- +# Source: nutanix-cloud-provider/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:cloud-controller-manager +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +--- +# Source: nutanix-cloud-provider/templates/rbac.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:cloud-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager +subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +--- +# Source: nutanix-cloud-provider/templates/cloud-provider-nutanix-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + k8s-app: nutanix-cloud-controller-manager + name: nutanix-cloud-controller-manager + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: nutanix-cloud-controller-manager + strategy: + type: Recreate + template: + metadata: + labels: + k8s-app: nutanix-cloud-controller-manager + spec: + hostNetwork: true + priorityClassName: system-cluster-critical + nodeSelector: + node-role.kubernetes.io/control-plane: "" + serviceAccountName: cloud-controller-manager + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: nutanix-cloud-controller-manager + topologyKey: kubernetes.io/hostname + dnsPolicy: Default + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoExecute + key: node.kubernetes.io/unreachable + operator: Exists + tolerationSeconds: 120 + - effect: NoExecute + key: node.kubernetes.io/not-ready + operator: Exists + tolerationSeconds: 120 + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + containers: + - image: "${CCM_REPO=ghcr.io/nutanix-cloud-native/cloud-provider-nutanix/controller}:${CCM_TAG=v0.3.2}" + imagePullPolicy: IfNotPresent + name: nutanix-cloud-controller-manager + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + args: + - "--leader-elect=true" + - "--cloud-config=/etc/cloud/nutanix_config.json" + resources: + requests: + cpu: 100m + memory: 50Mi + volumeMounts: + - mountPath: /etc/cloud + name: nutanix-config-volume + readOnly: true + volumes: + - name: nutanix-config-volume + configMap: + name: nutanix-config diff --git a/templates/topology/secret.yaml b/templates/topology/secret.yaml new file mode 100644 index 0000000000..f4ad784fd6 --- /dev/null +++ b/templates/topology/secret.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: "${CLUSTER_NAME}-pc-creds" +stringData: + credentials: | + [ + { + "type": "basic_auth", + "data": { + "prismCentral":{ + "username": "${NUTANIX_USER}", + "password": "${NUTANIX_PASSWORD}" + } + } + } + ] diff --git a/test/e2e/config/nutanix.yaml b/test/e2e/config/nutanix.yaml index f1f0874e77..0ae45e2e62 100644 --- a/test/e2e/config/nutanix.yaml +++ b/test/e2e/config/nutanix.yaml @@ -213,7 +213,7 @@ providers: - sourcePath: "../data/infrastructure-nutanix/v1beta1/cluster-template-failure-domains.yaml" - sourcePath: "../data/infrastructure-nutanix/ccm-update.yaml" - sourcePath: "../data/infrastructure-nutanix/v1beta1/cluster-template-clusterclass.yaml" - - sourcePath: "../data/infrastructure-nutanix/v1beta1/clusterclass-e2e.yaml" + - sourcePath: "../data/infrastructure-nutanix/v1beta1/clusterclass-nutanix-quick-start.yaml" - sourcePath: "../data/infrastructure-nutanix/v1beta1/cluster-template-topology.yaml" variables: @@ -225,7 +225,7 @@ variables: IP_FAMILY: "IPv4" CLUSTERCTL_LOG_LEVEL: 10 CLUSTER_TOPOLOGY: "true" - CLUSTER_CLASS_NAME: "e2e" + CLUSTER_CLASS_NAME: "nutanix-quick-start" NUTANIX_PROVIDER: "true" NUTANIX_ENDPOINT: "" NUTANIX_USER: "" diff --git a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-failure-domains/kustomization.yaml b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-failure-domains/kustomization.yaml index 90dd08d63f..332190555f 100644 --- a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-failure-domains/kustomization.yaml +++ b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-failure-domains/kustomization.yaml @@ -3,12 +3,12 @@ bases: - ../../../../../../templates/base/cluster-without-topology.yaml - ../../../../../../templates/base/kcp.yaml - ../../../../../../templates/base/kct.yaml - - ../../../../../../templates/base-root/secret.yaml - - ../../../../../../templates/base-root/cm.yaml + - ../../../../../../templates/base/secret.yaml + - ../../../../../../templates/base/cm.yaml - ../../../../../../templates/base/md.yaml - ../../../../../../templates/base/mhc.yaml - - ../../../../../../templates/base-root/nutanix-ccm-crs.yaml - - ../../../../../../templates/base-root/nutanix-ccm-secret.yaml + - ../../../../../../templates/ccm/nutanix-ccm-crs.yaml + - ../../../../../../templates/ccm/nutanix-ccm-secret.yaml - ../base/crs.yaml - failure-domain-nmt.yaml @@ -16,7 +16,7 @@ configMapGenerator: - name: nutanix-ccm behavior: merge files: - - ../../../../../../templates/base-root/nutanix-ccm.yaml + - ../../../../../../templates/ccm/nutanix-ccm.yaml patchesStrategicMerge: - ../base/cni-patch.yaml diff --git a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-kcp-remediation/kustomization.yaml b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-kcp-remediation/kustomization.yaml index c5ca9c48fd..361ec22293 100644 --- a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-kcp-remediation/kustomization.yaml +++ b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-kcp-remediation/kustomization.yaml @@ -3,12 +3,12 @@ bases: - ../../../../../../templates/base/cluster-without-topology.yaml - ../../../../../../templates/base/kcp.yaml - ../../../../../../templates/base/kct.yaml - - ../../../../../../templates/base-root/secret.yaml - - ../../../../../../templates/base-root/cm.yaml + - ../../../../../../templates/base/secret.yaml + - ../../../../../../templates/base/cm.yaml - ../../../../../../templates/base/nmt.yaml - ../../../../../../templates/base/md.yaml - - ../../../../../../templates/base-root/nutanix-ccm-crs.yaml - - ../../../../../../templates/base-root/nutanix-ccm-secret.yaml + - ../../../../../../templates/ccm/nutanix-ccm-crs.yaml + - ../../../../../../templates/ccm/nutanix-ccm-secret.yaml - ../base/crs.yaml - ./mhc.yaml @@ -16,7 +16,7 @@ configMapGenerator: - name: nutanix-ccm behavior: merge files: - - ../../../../../../templates/base-root/nutanix-ccm.yaml + - ../../../../../../templates/ccm/nutanix-ccm.yaml patchesStrategicMerge: - ../base/cni-patch.yaml diff --git a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-md-remediation/kustomization.yaml b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-md-remediation/kustomization.yaml index e860f5a55e..d322449704 100644 --- a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-md-remediation/kustomization.yaml +++ b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-md-remediation/kustomization.yaml @@ -3,12 +3,12 @@ bases: - ../../../../../../templates/base/cluster-without-topology.yaml - ../../../../../../templates/base/kcp.yaml - ../../../../../../templates/base/kct.yaml - - ../../../../../../templates/base-root/secret.yaml - - ../../../../../../templates/base-root/cm.yaml + - ../../../../../../templates/base/secret.yaml + - ../../../../../../templates/base/cm.yaml - ../../../../../../templates/base/nmt.yaml - ../../../../../../templates/base/md.yaml - - ../../../../../../templates/base-root/nutanix-ccm-crs.yaml - - ../../../../../../templates/base-root/nutanix-ccm-secret.yaml + - ../../../../../../templates/ccm/nutanix-ccm-crs.yaml + - ../../../../../../templates/ccm/nutanix-ccm-secret.yaml - ../base/crs.yaml - ./mhc.yaml @@ -16,7 +16,7 @@ configMapGenerator: - name: nutanix-ccm behavior: merge files: - - ../../../../../../templates/base-root/nutanix-ccm.yaml + - ../../../../../../templates/ccm/nutanix-ccm.yaml patchesStrategicMerge: - ../base/cni-patch.yaml diff --git a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-no-nmt/kustomization.yaml b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-no-nmt/kustomization.yaml index 0af1938185..08aa42f8ab 100644 --- a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-no-nmt/kustomization.yaml +++ b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-no-nmt/kustomization.yaml @@ -3,19 +3,19 @@ bases: - ../../../../../../templates/base/cluster-without-topology.yaml - ../../../../../../templates/base/kcp.yaml - ../../../../../../templates/base/kct.yaml - - ../../../../../../templates/base-root/secret.yaml - - ../../../../../../templates/base-root/cm.yaml + - ../../../../../../templates/base/secret.yaml + - ../../../../../../templates/base/cm.yaml - ../../../../../../templates/base/md.yaml - ../../../../../../templates/base/mhc.yaml - - ../../../../../../templates/base-root/nutanix-ccm-crs.yaml - - ../../../../../../templates/base-root/nutanix-ccm-secret.yaml + - ../../../../../../templates/ccm/nutanix-ccm-crs.yaml + - ../../../../../../templates/ccm/nutanix-ccm-secret.yaml - ../base/crs.yaml configMapGenerator: - name: nutanix-ccm behavior: merge files: - - ../../../../../../templates/base-root/nutanix-ccm.yaml + - ../../../../../../templates/ccm/nutanix-ccm.yaml patchesStrategicMerge: - ../base/cni-patch.yaml diff --git a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-no-nutanix-cluster/kustomization.yaml b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-no-nutanix-cluster/kustomization.yaml index 398e1b0d1e..b30edc4480 100644 --- a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-no-nutanix-cluster/kustomization.yaml +++ b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-no-nutanix-cluster/kustomization.yaml @@ -4,18 +4,18 @@ bases: - ../../../../../../templates/base/kcp.yaml - ../../../../../../templates/base/kct.yaml - ../../../../../../templates/base/nmt.yaml - - ../../../../../../templates/base-root/cm.yaml + - ../../../../../../templates/base/cm.yaml - ../../../../../../templates/base/md.yaml - ../../../../../../templates/base/mhc.yaml - - ../../../../../../templates/base-root/nutanix-ccm-crs.yaml - - ../../../../../../templates/base-root/nutanix-ccm-secret.yaml + - ../../../../../../templates/ccm/nutanix-ccm-crs.yaml + - ../../../../../../templates/ccm/nutanix-ccm-secret.yaml - ../base/crs.yaml configMapGenerator: - name: nutanix-ccm behavior: merge files: - - ../../../../../../templates/base-root/nutanix-ccm.yaml + - ../../../../../../templates/ccm/nutanix-ccm.yaml patchesStrategicMerge: - ../base/cni-patch.yaml diff --git a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-no-secret/kustomization.yaml b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-no-secret/kustomization.yaml index d920b084ca..0ecbb159eb 100644 --- a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-no-secret/kustomization.yaml +++ b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-no-secret/kustomization.yaml @@ -6,16 +6,16 @@ bases: - ../../../../../../templates/base/nmt.yaml - ../../../../../../templates/base/md.yaml - ../../../../../../templates/base/mhc.yaml - - ../../../../../../templates/base-root/nutanix-ccm-crs.yaml - - ../../../../../../templates/base-root/nutanix-ccm-secret.yaml - - ../../../../../../templates/base-root/cm.yaml + - ../../../../../../templates/ccm/nutanix-ccm-crs.yaml + - ../../../../../../templates/ccm/nutanix-ccm-secret.yaml + - ../../../../../../templates/base/cm.yaml - ../base/crs.yaml configMapGenerator: -- name: nutanix-ccm - behavior: merge - files: - - ../../../../../../templates/base-root/nutanix-ccm.yaml + - name: nutanix-ccm + behavior: merge + files: + - ../../../../../../templates/ccm/nutanix-ccm.yaml patchesStrategicMerge: - ../base/cni-patch.yaml diff --git a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-project/kustomization.yaml b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-project/kustomization.yaml index 16c9404d47..fc83018b19 100644 --- a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-project/kustomization.yaml +++ b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-project/kustomization.yaml @@ -3,20 +3,20 @@ bases: - ../../../../../../templates/base/cluster-without-topology.yaml - ../../../../../../templates/base/kcp.yaml - ../../../../../../templates/base/kct.yaml - - ../../../../../../templates/base-root/secret.yaml - - ../../../../../../templates/base-root/cm.yaml + - ../../../../../../templates/base/secret.yaml + - ../../../../../../templates/base/cm.yaml - ../../../../../../templates/base/nmt.yaml - ../../../../../../templates/base/md.yaml - ../../../../../../templates/base/mhc.yaml - - ../../../../../../templates/base-root/nutanix-ccm-crs.yaml - - ../../../../../../templates/base-root/nutanix-ccm-secret.yaml + - ../../../../../../templates/ccm/nutanix-ccm-crs.yaml + - ../../../../../../templates/ccm/nutanix-ccm-secret.yaml - ../base/crs.yaml configMapGenerator: - name: nutanix-ccm behavior: merge files: - - ../../../../../../templates/base-root/nutanix-ccm.yaml + - ../../../../../../templates/ccm/nutanix-ccm.yaml patchesStrategicMerge: - ../base/cni-patch.yaml diff --git a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-upgrades/kustomization.yaml b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-upgrades/kustomization.yaml index cb02c7b719..b9903104ca 100644 --- a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-upgrades/kustomization.yaml +++ b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-upgrades/kustomization.yaml @@ -3,12 +3,12 @@ bases: - ../../../../../../templates/base/cluster-without-topology.yaml - ../../../../../../templates/base/kcp.yaml - ../../../../../../templates/base/kct.yaml - - ../../../../../../templates/base-root/secret.yaml - - ../../../../../../templates/base-root/cm.yaml + - ../../../../../../templates/base/secret.yaml + - ../../../../../../templates/base/cm.yaml - ../../../../../../templates/base/md.yaml - ../../../../../../templates/base/mhc.yaml - - ../../../../../../templates/base-root/nutanix-ccm-crs.yaml - - ../../../../../../templates/base-root/nutanix-ccm-secret.yaml + - ../../../../../../templates/ccm/nutanix-ccm-crs.yaml + - ../../../../../../templates/ccm/nutanix-ccm-secret.yaml - ../base/crs.yaml - ./nmt.yaml @@ -16,7 +16,7 @@ configMapGenerator: - name: nutanix-ccm behavior: merge files: - - ../../../../../../templates/base-root/nutanix-ccm.yaml + - ../../../../../../templates/ccm/nutanix-ccm.yaml patchesStrategicMerge: - ../base/cni-patch.yaml diff --git a/test/e2e/data/infrastructure-nutanix/v1beta1/clusterclass-e2e.yaml b/test/e2e/data/infrastructure-nutanix/v1beta1/clusterclass-e2e.yaml deleted file mode 100644 index 7cdfc58c90..0000000000 --- a/test/e2e/data/infrastructure-nutanix/v1beta1/clusterclass-e2e.yaml +++ /dev/null @@ -1,797 +0,0 @@ -apiVersion: v1 -binaryData: - ca.crt: ${NUTANIX_ADDITIONAL_TRUST_BUNDLE=""} -kind: ConfigMap -metadata: - name: ${CLUSTER_NAME}-pc-trusted-ca-bundle ---- -apiVersion: v1 -data: - nutanix-ccm.yaml: | - --- - apiVersion: v1 - kind: ConfigMap - metadata: - name: ${CLUSTER_NAME}-pc-trusted-ca-bundle - namespace: kube-system - binaryData: - ca.crt: ${NUTANIX_ADDITIONAL_TRUST_BUNDLE=""} - --- - # Source: nutanix-cloud-provider/templates/rbac.yaml - apiVersion: v1 - kind: ServiceAccount - metadata: - name: cloud-controller-manager - namespace: kube-system - --- - # Source: nutanix-cloud-provider/templates/cm.yaml - kind: ConfigMap - apiVersion: v1 - metadata: - name: nutanix-config - namespace: kube-system - data: - nutanix_config.json: |- - { - "prismCentral": { - "address": "${NUTANIX_ENDPOINT}", - "port": ${NUTANIX_PORT=9440}, - "insecure": ${NUTANIX_INSECURE=false}, - "credentialRef": { - "kind": "secret", - "name": "nutanix-creds", - "namespace": "kube-system" - }, - "additionalTrustBundle": { - "kind": "ConfigMap", - "name": "${CLUSTER_NAME}-pc-trusted-ca-bundle", - "namespace": "kube-system" - } - }, - "enableCustomLabeling": ${CCM_CUSTOM_LABEL=false}, - "topologyDiscovery": { - "type": "Prism" - } - } - --- - # Source: nutanix-cloud-provider/templates/rbac.yaml - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - name: system:cloud-controller-manager - rules: - - apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update - - apiGroups: - - "" - resources: - - nodes - verbs: - - "*" - - apiGroups: - - "" - resources: - - nodes/status - verbs: - - patch - - apiGroups: - - "" - resources: - - serviceaccounts - verbs: - - create - - apiGroups: - - "" - resources: - - endpoints - verbs: - - create - - get - - list - - watch - - update - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete - --- - # Source: nutanix-cloud-provider/templates/rbac.yaml - kind: ClusterRoleBinding - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - name: system:cloud-controller-manager - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:cloud-controller-manager - subjects: - - kind: ServiceAccount - name: cloud-controller-manager - namespace: kube-system - --- - # Source: nutanix-cloud-provider/templates/cloud-provider-nutanix-deployment.yaml - apiVersion: apps/v1 - kind: Deployment - metadata: - labels: - k8s-app: nutanix-cloud-controller-manager - name: nutanix-cloud-controller-manager - namespace: kube-system - spec: - replicas: 1 - selector: - matchLabels: - k8s-app: nutanix-cloud-controller-manager - strategy: - type: Recreate - template: - metadata: - labels: - k8s-app: nutanix-cloud-controller-manager - spec: - hostNetwork: true - priorityClassName: system-cluster-critical - nodeSelector: - node-role.kubernetes.io/control-plane: "" - serviceAccountName: cloud-controller-manager - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - k8s-app: nutanix-cloud-controller-manager - topologyKey: kubernetes.io/hostname - dnsPolicy: Default - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - operator: Exists - - effect: NoExecute - key: node.kubernetes.io/unreachable - operator: Exists - tolerationSeconds: 120 - - effect: NoExecute - key: node.kubernetes.io/not-ready - operator: Exists - tolerationSeconds: 120 - - effect: NoSchedule - key: node.cloudprovider.kubernetes.io/uninitialized - operator: Exists - - effect: NoSchedule - key: node.kubernetes.io/not-ready - operator: Exists - containers: - - image: "${CCM_REPO=ghcr.io/nutanix-cloud-native/cloud-provider-nutanix/controller}:${CCM_TAG=v0.3.2}" - imagePullPolicy: IfNotPresent - name: nutanix-cloud-controller-manager - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - args: - - "--leader-elect=true" - - "--cloud-config=/etc/cloud/nutanix_config.json" - resources: - requests: - cpu: 100m - memory: 50Mi - volumeMounts: - - mountPath: /etc/cloud - name: nutanix-config-volume - readOnly: true - volumes: - - name: nutanix-config-volume - configMap: - name: nutanix-config -kind: ConfigMap -metadata: - name: nutanix-ccm ---- -apiVersion: v1 -kind: Secret -metadata: - name: ${CLUSTER_NAME} -stringData: - credentials: | - [ - { - "type": "basic_auth", - "data": { - "prismCentral":{ - "username": "${NUTANIX_USER}", - "password": "${NUTANIX_PASSWORD}" - } - } - } - ] ---- -apiVersion: v1 -kind: Secret -metadata: - name: nutanix-ccm-secret -stringData: - nutanix-ccm-secret.yaml: | - apiVersion: v1 - kind: Secret - metadata: - name: nutanix-creds - namespace: kube-system - stringData: - credentials: | - [ - { - "type": "basic_auth", - "data": { - "prismCentral":{ - "username": "${NUTANIX_USER}", - "password": "${NUTANIX_PASSWORD}" - }, - "prismElements": null - } - } - ] -type: addons.cluster.x-k8s.io/resource-set ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: nutanix-ccm-crs -spec: - clusterSelector: - matchLabels: - ccm: nutanix - resources: - - kind: ConfigMap - name: nutanix-ccm - - kind: Secret - name: nutanix-ccm-secret - - kind: ConfigMap - name: ${CLUSTER_NAME}-pc-trusted-ca-bundle - strategy: ApplyOnce ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_CLASS_NAME}-kcfg-0 -spec: - template: - spec: - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-provider: external - eviction-hard: nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<15%,memory.available<100Mi,imagefs.inodesFree<10% - tls-cipher-suites: ${TLS_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256} - postKubeadmCommands: - - echo "after kubeadm call" > /var/log/postkubeadm.log - preKubeadmCommands: - - echo "before kubeadm call" > /var/log/prekubeadm.log - - hostnamectl set-hostname "{{ ds.meta_data.hostname }}" - verbosity: 10 ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: ClusterClass -metadata: - name: ${CLUSTER_CLASS_NAME} -spec: - controlPlane: - machineHealthCheck: - maxUnhealthy: 40% - nodeStartupTimeout: 10m - unhealthyConditions: - - status: "False" - timeout: 300s - type: Ready - - status: Unknown - timeout: 300s - type: Ready - - status: "True" - timeout: 300s - type: MemoryPressure - - status: "True" - timeout: 300s - type: DiskPressure - - status: "True" - timeout: 300s - type: PIDPressure - - status: "True" - timeout: 300s - type: NetworkUnavailable - machineInfrastructure: - ref: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: NutanixMachineTemplate - name: ${CLUSTER_CLASS_NAME}-cp-nmt - namespace: ${NAMESPACE} - ref: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlaneTemplate - name: ${CLUSTER_CLASS_NAME}-kcpt - namespace: ${NAMESPACE} - infrastructure: - ref: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: NutanixClusterTemplate - name: ${CLUSTER_CLASS_NAME}-nct - namespace: ${NAMESPACE} - patches: - - definitions: - - jsonPatches: - - op: add - path: /spec/template/spec/kubeadmConfigSpec/users - valueFrom: - template: | - - name: capxuser - lockPassword: false - sudo: ALL=(ALL) NOPASSWD:ALL - sshAuthorizedKeys: - - '{{ .sshKey }}' - selector: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlaneTemplate - matchResources: - controlPlane: true - - jsonPatches: - - op: add - path: /spec/template/spec/users - valueFrom: - template: | - - name: capxuser - lockPassword: false - sudo: ALL=(ALL) NOPASSWD:ALL - sshAuthorizedKeys: - - '{{ .sshKey }}' - selector: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - matchResources: - machineDeploymentClass: - names: - - ${CLUSTER_CLASS_NAME}-worker - name: add-ssh-user - - definitions: - - jsonPatches: - - op: add - path: /spec/template/spec/controlPlaneEndpoint - valueFrom: - template: | - host: '{{ .controlPlaneEndpoint.IP }}' - port: {{ .controlPlaneEndpoint.port }} - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: NutanixClusterTemplate - matchResources: - infrastructureCluster: true - name: update-control-plane-endpoint - - definitions: - - jsonPatches: - - op: add - path: /spec/template/spec/prismCentral - valueFrom: - template: | - address: '{{ .prismCentralEndpoint.address }}' - port: {{ .prismCentralEndpoint.port }} - insecure: {{ .prismCentralEndpoint.insecure }} - credentialRef: - name: "${CLUSTER_NAME}" - kind: Secret - additionalTrustBundle: - name: ${CLUSTER_NAME}-pc-trusted-ca-bundle - kind: ConfigMap - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: NutanixClusterTemplate - matchResources: - infrastructureCluster: true - name: add-pc-endpoint-and-creds - - definitions: - - jsonPatches: - - op: replace - path: /spec/template/spec/bootType - valueFrom: - variable: controlPlaneMachineDetails.bootType - - op: replace - path: /spec/template/spec/vcpusPerSocket - valueFrom: - variable: controlPlaneMachineDetails.vcpusPerSocket - - op: replace - path: /spec/template/spec/memorySize - valueFrom: - variable: controlPlaneMachineDetails.memorySize - - op: replace - path: /spec/template/spec/systemDiskSize - valueFrom: - variable: controlPlaneMachineDetails.systemDiskSize - - op: replace - path: /spec/template/spec/image/name - valueFrom: - variable: controlPlaneMachineDetails.imageName - - op: replace - path: /spec/template/spec/cluster/name - valueFrom: - variable: controlPlaneMachineDetails.clusterName - - op: replace - path: /spec/template/spec/subnet - valueFrom: - template: | - - type: name - name: {{ .controlPlaneMachineDetails.subnetName }} - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: NutanixMachineTemplate - matchResources: - controlPlane: true - name: update-control-plane-machine-template - - definitions: - - jsonPatches: - - op: replace - path: /spec/template/spec/bootType - valueFrom: - variable: workerMachineDetails.bootType - - op: replace - path: /spec/template/spec/vcpusPerSocket - valueFrom: - variable: workerMachineDetails.vcpusPerSocket - - op: replace - path: /spec/template/spec/memorySize - valueFrom: - variable: workerMachineDetails.memorySize - - op: replace - path: /spec/template/spec/systemDiskSize - valueFrom: - variable: workerMachineDetails.systemDiskSize - - op: replace - path: /spec/template/spec/image/name - valueFrom: - variable: workerMachineDetails.imageName - - op: replace - path: /spec/template/spec/cluster/name - valueFrom: - variable: workerMachineDetails.clusterName - - op: replace - path: /spec/template/spec/subnet - valueFrom: - template: | - - type: name - name: {{ .controlPlaneMachineDetails.subnetName }} - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: NutanixMachineTemplate - matchResources: - machineDeploymentClass: - names: - - ${CLUSTER_CLASS_NAME}-worker - name: update-worker-machine-template - variables: - - name: sshKey - required: true - schema: - openAPIV3Schema: - description: Public key to SSH onto the cluster nodes. - type: string - - name: controlPlaneEndpoint - required: true - schema: - openAPIV3Schema: - properties: - IP: - type: string - port: - type: integer - type: object - - name: prismCentralEndpoint - required: true - schema: - openAPIV3Schema: - properties: - address: - type: string - insecure: - type: boolean - port: - type: integer - type: object - - name: controlPlaneMachineDetails - required: true - schema: - openAPIV3Schema: - properties: - bootType: - type: string - clusterName: - type: string - imageName: - type: string - memorySize: - type: string - subnetName: - type: string - systemDiskSize: - type: string - vcpuSockets: - type: integer - vcpusPerSocket: - type: integer - type: object - - name: workerMachineDetails - required: true - schema: - openAPIV3Schema: - properties: - bootType: - type: string - clusterName: - type: string - imageName: - type: string - memorySize: - type: string - subnetName: - type: string - systemDiskSize: - type: string - vcpuSockets: - type: integer - vcpusPerSocket: - type: integer - type: object - workers: - machineDeployments: - - class: ${CLUSTER_CLASS_NAME}-worker - machineHealthCheck: - maxUnhealthy: 40% - nodeStartupTimeout: 10m - unhealthyConditions: - - status: "False" - timeout: 300s - type: Ready - - status: Unknown - timeout: 300s - type: Ready - - status: "True" - timeout: 300s - type: MemoryPressure - - status: "True" - timeout: 300s - type: DiskPressure - - status: "True" - timeout: 300s - type: PIDPressure - - status: "True" - timeout: 300s - type: NetworkUnavailable - template: - bootstrap: - ref: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: ${CLUSTER_CLASS_NAME}-kcfg-0 - namespace: ${NAMESPACE} - infrastructure: - ref: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: NutanixMachineTemplate - name: ${CLUSTER_CLASS_NAME}-md-nmt - namespace: ${NAMESPACE} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlaneTemplate -metadata: - name: ${CLUSTER_CLASS_NAME}-kcpt -spec: - template: - spec: - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - certSANs: - - localhost - - 127.0.0.1 - - 0.0.0.0 - extraArgs: - cloud-provider: external - tls-cipher-suites: ${TLS_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256} - controllerManager: - extraArgs: - cloud-provider: external - tls-cipher-suites: ${TLS_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256} - scheduler: - extraArgs: - tls-cipher-suites: ${TLS_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256} - files: - - content: | - apiVersion: v1 - kind: Pod - metadata: - name: kube-vip - namespace: kube-system - spec: - containers: - - name: kube-vip - image: ghcr.io/kube-vip/kube-vip:v0.6.4 - imagePullPolicy: IfNotPresent - args: - - manager - env: - - name: vip_arp - value: "true" - - name: address - value: "${CONTROL_PLANE_ENDPOINT_IP}" - - name: port - value: "${CONTROL_PLANE_ENDPOINT_PORT=6443}" - - name: vip_cidr - value: "32" - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: vip_ddns - value: "false" - - name: vip_leaderelection - value: "true" - - name: vip_leaseduration - value: "15" - - name: vip_renewdeadline - value: "10" - - name: vip_retryperiod - value: "2" - - name: svc_enable - value: "${KUBEVIP_SVC_ENABLE=false}" - - name: lb_enable - value: "${KUBEVIP_LB_ENABLE=false}" - - name: enableServicesElection - value: "${KUBEVIP_SVC_ELECTION=false}" - securityContext: - capabilities: - add: - - NET_ADMIN - - SYS_TIME - - NET_RAW - volumeMounts: - - mountPath: /etc/kubernetes/admin.conf - name: kubeconfig - resources: {} - hostNetwork: true - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 - volumes: - - name: kubeconfig - hostPath: - type: FileOrCreate - path: /etc/kubernetes/admin.conf - status: {} - owner: root:root - path: /etc/kubernetes/manifests/kube-vip.yaml - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-provider: external - eviction-hard: nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<15%,memory.available<100Mi,imagefs.inodesFree<10% - tls-cipher-suites: ${TLS_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256} - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-provider: external - eviction-hard: nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<15%,memory.available<100Mi,imagefs.inodesFree<10% - tls-cipher-suites: ${TLS_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256} - postKubeadmCommands: - - echo export KUBECONFIG=/etc/kubernetes/admin.conf >> /root/.bashrc - - | - KUBERNETES_VERSION_NO_V=${KUBERNETES_VERSION#v} - VERSION_TO_COMPARE=1.29.0 - if [ "$(printf '%s\n' "$KUBERNETES_VERSION_NO_V" "$VERSION_TO_COMPARE" | sort -V | head -n1)" != "$KUBERNETES_VERSION_NO_V" ]; then - if [ -f /run/kubeadm/kubeadm.yaml ]; then - sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' /etc/kubernetes/manifests/kube-vip.yaml; - fi - fi - - echo "after kubeadm call" > /var/log/postkubeadm.log - preKubeadmCommands: - - echo "before kubeadm call" > /var/log/prekubeadm.log - - hostnamectl set-hostname "{{ ds.meta_data.hostname }}" - - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts - - echo "127.0.0.1 localhost" >>/etc/hosts - - echo "127.0.0.1 kubernetes" >>/etc/hosts - - echo "127.0.0.1 {{ ds.meta_data.hostname }}" >> /etc/hosts - - | - KUBERNETES_VERSION_NO_V=${KUBERNETES_VERSION#v} - VERSION_TO_COMPARE=1.29.0 - if [ "$(printf '%s\n' "$KUBERNETES_VERSION_NO_V" "$VERSION_TO_COMPARE" | sort -V | head -n1)" != "$KUBERNETES_VERSION_NO_V" ]; then - if [ -f /run/kubeadm/kubeadm.yaml ]; then - sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' /etc/kubernetes/manifests/kube-vip.yaml; - fi - fi - useExperimentalRetryJoin: true - verbosity: 10 ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: NutanixClusterTemplate -metadata: - name: ${CLUSTER_CLASS_NAME}-nct -spec: - template: - spec: - failureDomains: [] ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: NutanixMachineTemplate -metadata: - name: ${CLUSTER_CLASS_NAME}-cp-nmt -spec: - template: - spec: - bootType: ${NUTANIX_MACHINE_BOOT_TYPE=legacy} - cluster: - name: ${NUTANIX_PRISM_ELEMENT_CLUSTER_NAME} - type: name - image: - name: ${NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME} - type: name - memorySize: ${NUTANIX_MACHINE_MEMORY_SIZE=4Gi} - providerID: nutanix://${CLUSTER_NAME}-m1 - subnet: - - name: ${NUTANIX_SUBNET_NAME} - type: name - systemDiskSize: ${NUTANIX_SYSTEMDISK_SIZE=40Gi} - vcpuSockets: ${NUTANIX_MACHINE_VCPU_SOCKET=2} - vcpusPerSocket: ${NUTANIX_MACHINE_VCPU_PER_SOCKET=1} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: NutanixMachineTemplate -metadata: - name: ${CLUSTER_CLASS_NAME}-md-nmt -spec: - template: - spec: - bootType: ${NUTANIX_MACHINE_BOOT_TYPE=legacy} - cluster: - name: ${NUTANIX_PRISM_ELEMENT_CLUSTER_NAME} - type: name - image: - name: ${NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME} - type: name - memorySize: ${NUTANIX_MACHINE_MEMORY_SIZE=4Gi} - providerID: nutanix://${CLUSTER_NAME}-m1 - subnet: - - name: ${NUTANIX_SUBNET_NAME} - type: name - systemDiskSize: ${NUTANIX_SYSTEMDISK_SIZE=40Gi} - vcpuSockets: ${NUTANIX_MACHINE_VCPU_SOCKET=2} - vcpusPerSocket: ${NUTANIX_MACHINE_VCPU_PER_SOCKET=1} diff --git a/test/e2e/data/infrastructure-nutanix/v1beta1/no-kubeproxy/cluster-template-topology/kustomization.yaml b/test/e2e/data/infrastructure-nutanix/v1beta1/no-kubeproxy/cluster-template-topology/kustomization.yaml new file mode 100644 index 0000000000..b52aafbcd4 --- /dev/null +++ b/test/e2e/data/infrastructure-nutanix/v1beta1/no-kubeproxy/cluster-template-topology/kustomization.yaml @@ -0,0 +1,2 @@ +bases: + - ../../cluster-template-topology/ diff --git a/test/e2e/data/infrastructure-nutanix/v1beta1/no-kubeproxy/no-kubeproxy-clusterclass.yaml b/test/e2e/data/infrastructure-nutanix/v1beta1/no-kubeproxy/no-kubeproxy-clusterclass.yaml index 4c49b849b7..409ccea20c 100644 --- a/test/e2e/data/infrastructure-nutanix/v1beta1/no-kubeproxy/no-kubeproxy-clusterclass.yaml +++ b/test/e2e/data/infrastructure-nutanix/v1beta1/no-kubeproxy/no-kubeproxy-clusterclass.yaml @@ -1,22 +1,22 @@ apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlaneTemplate metadata: - name: "${CLUSTER_CLASS_NAME}-kcpt" + name: "nutanix-quick-start-kcpt" spec: template: spec: kubeadmConfigSpec: initConfiguration: skipPhases: - - addon/kube-proxy + - addon/kube-proxy --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate metadata: - name: "${CLUSTER_NAME}-kcfg-0" + name: "nutanix-quick-start-kcfg-0" spec: template: spec: initConfiguration: skipPhases: - - addon/kube-proxy + - addon/kube-proxy