diff --git a/charts/ocean-kubernetes-controller/temp/NOTES.txt b/charts/ocean-kubernetes-controller/temp/NOTES.txt deleted file mode 100644 index f515c84..0000000 --- a/charts/ocean-kubernetes-controller/temp/NOTES.txt +++ /dev/null @@ -1,22 +0,0 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "ocean-kubernetes-controller.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "ocean-kubernetes-controller.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "ocean-kubernetes-controller.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "ocean-kubernetes-controller.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} diff --git a/charts/ocean-kubernetes-controller/temp/hpa.yaml b/charts/ocean-kubernetes-controller/temp/hpa.yaml deleted file mode 100644 index 8648685..0000000 --- a/charts/ocean-kubernetes-controller/temp/hpa.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "ocean-kubernetes-controller.fullname" . }} - labels: - {{- include "ocean-kubernetes-controller.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "ocean-kubernetes-controller.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/charts/ocean-kubernetes-controller/temp/ingress.yaml b/charts/ocean-kubernetes-controller/temp/ingress.yaml deleted file mode 100644 index 9fb48c7..0000000 --- a/charts/ocean-kubernetes-controller/temp/ingress.yaml +++ /dev/null @@ -1,61 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "ocean-kubernetes-controller.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "ocean-kubernetes-controller.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/charts/ocean-kubernetes-controller/temp/service.yaml b/charts/ocean-kubernetes-controller/temp/service.yaml deleted file mode 100644 index 9095a93..0000000 --- a/charts/ocean-kubernetes-controller/temp/service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "ocean-kubernetes-controller.fullname" . }} - labels: - {{- include "ocean-kubernetes-controller.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: http - protocol: TCP - name: http - selector: - {{- include "ocean-kubernetes-controller.selectorLabels" . | nindent 4 }} diff --git a/charts/ocean-kubernetes-controller/templates/clusterrole.yaml b/charts/ocean-kubernetes-controller/templates/clusterrole.yaml new file mode 100644 index 0000000..3cfb62f --- /dev/null +++ b/charts/ocean-kubernetes-controller/templates/clusterrole.yaml @@ -0,0 +1,120 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "ocean-kubernetes-controller.fullname" . }} + labels: + {{- include "ocean-kubernetes-controller.labels" . | nindent 4 }} +rules: +# --------------------------------------------------------------------------- +# feature: ocean/readonly +# --------------------------------------------------------------------------- +- apiGroups: [ "" ] + resources: [ "pods", "nodes", "services", "namespaces", "replicationcontrollers", "limitranges", "events", "persistentvolumes", "persistentvolumeclaims" ] + verbs: [ "get", "list", "watch" ] +- apiGroups: [ "apps" ] + resources: [ "deployments", "daemonsets", "statefulsets", "replicasets" ] + verbs: [ "get", "list", "watch" ] +- apiGroups: [ "storage.k8s.io" ] + resources: [ "storageclasses" ] + verbs: [ "get", "list", "watch" ] +- apiGroups: [ "batch" ] + resources: [ "jobs", "cronjobs" ] + verbs: [ "get", "list", "watch" ] +- apiGroups: [ "policy" ] + resources: [ "poddisruptionbudgets" ] + verbs: [ "get", "list", "watch" ] +- apiGroups: [ "metrics.k8s.io" ] + resources: [ "pods" ] + verbs: [ "get", "list", "watch" ] +- apiGroups: [ "autoscaling" ] + resources: [ "horizontalpodautoscalers" ] + verbs: [ "get", "list", "watch" ] +- apiGroups: [ "apiextensions.k8s.io" ] + resources: [ "customresourcedefinitions" ] + verbs: [ "get", "list", "watch" ] +- apiGroups: [ "node.k8s.io" ] + resources: [ "runtimeclasses" ] + verbs: [ "get", "list", "watch" ] +- nonResourceURLs: [ "/version/", "/version" ] + verbs: [ "get" ] +# --------------------------------------------------------------------------- +# feature: ocean/draining +# --------------------------------------------------------------------------- +- apiGroups: [""] + resources: ["nodes"] + verbs: ["patch", "update"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["delete"] +- apiGroups: [""] + resources: ["pods/eviction"] + verbs: ["create"] +# --------------------------------------------------------------------------- +# feature: ocean/cleanup +# --------------------------------------------------------------------------- +- apiGroups: [""] + resources: ["nodes"] + verbs: ["delete"] +# --------------------------------------------------------------------------- +# feature: ocean/csr-approval +# --------------------------------------------------------------------------- +- apiGroups: ["certificates.k8s.io"] + resources: ["certificatesigningrequests"] + verbs: ["get", "list", "delete", "create"] +- apiGroups: ["certificates.k8s.io"] + resources: ["certificatesigningrequests/approval"] + verbs: ["patch", "update"] +- apiGroups: ["certificates.k8s.io"] + resources: ["signers"] + resourceNames: ["kubernetes.io/kubelet-serving", "kubernetes.io/kube-apiserver-client-kubelet"] + verbs: ["approve"] +# --------------------------------------------------------------------------- +# feature: ocean/auto-update +# --------------------------------------------------------------------------- +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["clusterroles"] + resourceNames: ["spotinst-kubernetes-cluster-controller"] + verbs: ["patch", "update", "escalate"] +- apiGroups: ["apps"] + resources: ["deployments"] + resourceNames: ["spotinst-kubernetes-cluster-controller"] + verbs: ["patch", "update"] +# --------------------------------------------------------------------------- +# feature: ocean/apply +# --------------------------------------------------------------------------- +- apiGroups: ["apps"] + resources: ["deployments", "daemonsets"] + verbs: ["get", "list", "patch", "update", "create", "delete"] +- apiGroups: ["extensions"] + resources: ["daemonsets"] + verbs: ["get", "list", "patch", "update", "create", "delete"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "patch", "update", "create", "delete"] +- apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "list", "patch", "update", "create", "delete"] +# --------------------------------------------------------------------------- +# feature: wave +# --------------------------------------------------------------------------- +- apiGroups: ["sparkoperator.k8s.io"] + resources: ["sparkapplications", "scheduledsparkapplications"] + verbs: ["get", "list", "patch", "update", "create", "delete"] +- apiGroups: ["wave.spot.io"] + resources: ["sparkapplications", "wavecomponents", "waveenvironments"] + verbs: ["get", "list"] +- apiGroups: ["bigdata.spot.io"] + resources: ["bigdataenvironments"] + verbs: ["get", "list", "patch", "update", "create", "delete"] +# --------------------------------------------------------------------------- +# feature: controller/leader-election (high-availability) +# --------------------------------------------------------------------------- +- apiGroups: [ "coordination.k8s.io" ] + resources: [ "leases" ] + verbs: [ "get","list","patch","update","create","delete" ] +# --------------------------------------------------------------------------- +# feature: controller/report-events +# --------------------------------------------------------------------------- +- apiGroups: [ "" ] + resources: [ "events" ] + verbs: [ "create" ] \ No newline at end of file diff --git a/charts/ocean-kubernetes-controller/templates/clusterrolebinding.yaml b/charts/ocean-kubernetes-controller/templates/clusterrolebinding.yaml new file mode 100644 index 0000000..a73fd74 --- /dev/null +++ b/charts/ocean-kubernetes-controller/templates/clusterrolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "ocean-kubernetes-controller.fullname" . }} + labels: + {{- include "ocean-kubernetes-controller.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "ocean-kubernetes-controller.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ include "ocean-kubernetes-controller.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} diff --git a/charts/ocean-kubernetes-controller/templates/configmap.yaml b/charts/ocean-kubernetes-controller/templates/configmap.yaml new file mode 100644 index 0000000..1c8cf09 --- /dev/null +++ b/charts/ocean-kubernetes-controller/templates/configmap.yaml @@ -0,0 +1,14 @@ +{{- if .Values.configMap.create }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "ocean-kubernetes-controller.configMapName" . }} + labels: + {{- include "ocean-kubernetes-controller.labels" . | nindent 4 }} +data: + spotinst.cluster-identifier: {{ required "`spotinst.clusterIdentifier` must be specified" .Values.spotinst.clusterIdentifier }} + base-url: {{ default "" .Values.spotinst.baseUrl | quote }} + proxy-url: {{ default "" .Values.spotinst.proxyUrl | quote }} + disable-auto-update: {{ default "false" .Values.spotinst.disableAutoUpdate | quote }} + enable-csr-approval: {{ default "false" .Values.spotinst.enableCsrApproval | quote }} +{{- end }} diff --git a/charts/ocean-kubernetes-controller/templates/deployment.yaml b/charts/ocean-kubernetes-controller/templates/deployment.yaml index 1fe17a2..62c788e 100644 --- a/charts/ocean-kubernetes-controller/templates/deployment.yaml +++ b/charts/ocean-kubernetes-controller/templates/deployment.yaml @@ -153,15 +153,57 @@ spec: nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} + {{- if kindIs "invalid" .Values.affinity }} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: NotIn + values: + - windows + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + {{- else }} {{- with .Values.affinity }} affinity: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.tolerations }} + {{- end }} + {{- if kindIs "invalid" .Values.tolerations }} tolerations: + - key: node.kubernetes.io/not-ready + effect: NoExecute + operator: Exists + tolerationSeconds: 150 + - key: node.kubernetes.io/unreachable + effect: NoExecute + operator: Exists + tolerationSeconds: 150 + - key: node-role.kubernetes.io/master + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + {{- else }} + {{- with .Values.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} {{- end }} + {{- end }} + {{- if kindIs "invalid" .Values.topologySpreadConstraints }} + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + labelSelector: + {{- include "ocean-kubernetes-controller.selectorLabels" . | nindent 6 }} + {{- else }} {{- with .Values.topologySpreadConstraints }} topologySpreadConstraints: {{- toYaml . | nindent 8 }} {{- end }} + {{- end }} diff --git a/charts/ocean-kubernetes-controller/templates/secret.yaml b/charts/ocean-kubernetes-controller/templates/secret.yaml new file mode 100644 index 0000000..38f01b0 --- /dev/null +++ b/charts/ocean-kubernetes-controller/templates/secret.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.secret.create }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "ocean-kubernetes-controller.secretName" . }} + labels: + {{- include "ocean-kubernetes-controller.labels" . | nindent 4 }} +type: Opaque +data: + token: {{ required "`spotinst.token` must be specified" .Values.spotinst.token | quote | b64enc }} + account: {{ required "`spotinst.account` must be specified" .Values.spotinst.account | quote | b64enc }} +{{- end }} diff --git a/charts/ocean-kubernetes-controller/temp/serviceaccount.yaml b/charts/ocean-kubernetes-controller/templates/serviceaccount.yaml similarity index 100% rename from charts/ocean-kubernetes-controller/temp/serviceaccount.yaml rename to charts/ocean-kubernetes-controller/templates/serviceaccount.yaml diff --git a/charts/ocean-kubernetes-controller/templates/tests/test-connection.yaml b/charts/ocean-kubernetes-controller/templates/tests/test-connection.yaml deleted file mode 100644 index a8b2204..0000000 --- a/charts/ocean-kubernetes-controller/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "ocean-kubernetes-controller.fullname" . }}-test-connection" - labels: - {{- include "ocean-kubernetes-controller.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['{{ include "ocean-kubernetes-controller.fullname" . }}:{{ .Values.service.port }}'] - restartPolicy: Never diff --git a/charts/ocean-kubernetes-controller/values.yaml b/charts/ocean-kubernetes-controller/values.yaml index a1af8f9..983336c 100644 --- a/charts/ocean-kubernetes-controller/values.yaml +++ b/charts/ocean-kubernetes-controller/values.yaml @@ -115,10 +115,6 @@ readinessProbe: initialDelaySeconds: 5 periodSeconds: 10 -service: - type: ClusterIP - port: 80 - # Controller pod resources. (Optional) resources: {} # requests: @@ -134,61 +130,41 @@ nodeSelector: {} # -- Tolerations for nodes that have taints on them. (Optional) # Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ tolerations: -- key: node.kubernetes.io/not-ready - effect: NoExecute - operator: Exists - tolerationSeconds: 150 -- key: node.kubernetes.io/unreachable - effect: NoExecute - operator: Exists - tolerationSeconds: 150 -- key: node-role.kubernetes.io/master - operator: Exists -- key: node-role.kubernetes.io/control-plane - operator: Exists +# - key: node.kubernetes.io/not-ready +# effect: NoExecute +# operator: Exists +# tolerationSeconds: 150 # Pod scheduling preferences. # Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity affinity: - # -- Node affinity. (Optional) - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/os - operator: NotIn - values: - - windows - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - preference: - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists - # -- Pod anti-affinity. (Optional) - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 50 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app.kubernetes.io/name - operator: In - values: - - spotinst-kubernetes-cluster-controller - topologyKey: kubernetes.io/hostname +# podAntiAffinity: +# preferredDuringSchedulingIgnoredDuringExecution: +# - weight: 50 +# podAffinityTerm: +# labelSelector: +# matchExpressions: +# - key: app.kubernetes.io/name +# operator: In +# values: +# - spotinst-kubernetes-cluster-controller +# topologyKey: kubernetes.io/hostname + +topologySpreadConstraints: +# - maxSkew: 1 +# topologyKey: kubernetes.io/hostname +# labelSelector: +# app: test extraVolumeMounts: [] extraVolumes: [] -topologySpreadConstraints: [] +schedulerName: "" # Annotations to add to the deployment deploymentAnnotations: {} -schedulerName: "" - # Metrics Server configuration. metrics-server: # -- Specifies whether the metrics-server chart should be deployed. (Optional)