diff --git a/CHANGELOG.md b/CHANGELOG.md index fa07917..aff2552 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +* split deployment manifests (#11) * Project Status: beta phase (#10) * Remove multizone and fix URL environment (#4) diff --git a/README.md b/README.md index 30bb1a5..d687a3b 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Exoscale Block Storage Container Storage Interface Driver. ## Deployment ``` -kubectl apply -f deployment/exoscale-csi.yaml +kubectl apply -k 'github.com/exoscale/exoscale-csi-driver/deployment?ref=main' ``` ## Using it diff --git a/deployment/controller-rbac.yaml b/deployment/controller-rbac.yaml new file mode 100644 index 0000000..f5de9c0 --- /dev/null +++ b/deployment/controller-rbac.yaml @@ -0,0 +1,138 @@ +kind: ServiceAccount +apiVersion: v1 +metadata: + name: exoscale-csi-controller + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: exoscale-csi-provisioner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments", "volumeattachments/status"] + verbs: ["get", "list", "watch", "update", "create", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots", "volumesnapshots/status", "volumesnapshotclasses", "volumesnapshotcontents", "volumesnapshotcontents/status"] + verbs: ["get", "list", "watch", "update", "create", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: exoscale-csi-controller +subjects: + - kind: ServiceAccount + name: exoscale-csi-controller + namespace: kube-system +roleRef: + kind: ClusterRole + name: exoscale-csi-provisioner + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: exoscale-csi-attacher +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "watch", "list", "update", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "watch", "list"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "watch", "list", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments", "volumeattachments/status"] + verbs: ["get", "watch", "list", "create", "update", "patch", "delete"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: exoscale-csi-attacher +subjects: + - kind: ServiceAccount + name: exoscale-csi-controller + namespace: kube-system +roleRef: + kind: ClusterRole + name: exoscale-csi-attacher + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: exoscale-csi-snapshotter +rules: + - apiGroups: [""] + resources: ["persistentvolumes", "persistentvolumeclaims", "events"] + verbs: ["get", "watch", "list", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "watch", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots", "volumesnapshots/status", "volumesnapshotclasses", "volumesnapshotcontents"] + verbs: ["get", "watch", "list", "create", "update", "patch", "delete"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions", "leases"] + verbs: ["get", "watch", "list", "create", "update", "patch", "delete"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: exoscale-csi-snapshotter +subjects: + - kind: ServiceAccount + name: exoscale-csi-controller + namespace: kube-system +roleRef: + kind: ClusterRole + name: exoscale-csi-snapshotter + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: external-resizer +rules: + - apiGroups: [""] + resources: ["persistentvolumes", "persistentvolumeclaims", "persistentvolumeclaims/status"] + verbs: ["get", "watch", "list", "create", "update", "patch"] + - apiGroups: [""] + resources: ["pods", "events"] + verbs: ["get", "watch", "list"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-resizer-role +subjects: + - kind: ServiceAccount + name: exoscale-csi-controller + namespace: kube-system +roleRef: + kind: ClusterRole + name: external-resizer + apiGroup: rbac.authorization.k8s.io diff --git a/deployment/controller.yaml b/deployment/controller.yaml new file mode 100644 index 0000000..c1b56f2 --- /dev/null +++ b/deployment/controller.yaml @@ -0,0 +1,172 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: exoscale-csi-controller + namespace: kube-system +spec: + selector: + matchLabels: + app: exoscale-csi-controller + replicas: 2 + template: + metadata: + labels: + app: exoscale-csi-controller + spec: + dnsPolicy: Default + priorityClassName: system-cluster-critical + serviceAccount: exoscale-csi-controller + containers: + - name: exoscale-csi-plugin + image: exoscale/csi-driver:latest + imagePullPolicy: Always + args : + - "--endpoint=$(CSI_ENDPOINT)" + - "--mode=controller" + - "--v=4" + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + envFrom: + - secretRef: + name: exoscale-credentials + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 2 + failureThreshold: 5 + resources: + limits: + cpu: 1000m + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + - name: csi-provisioner + image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.0 + args: + - "--v=5" + - "--csi-address=$(CSI_ADDRESS)" + - "--leader-election" + - "--feature-gates=Topology=true" + - "--default-fstype=ext4" + env: + - name: CSI_ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + resources: + limits: + cpu: 1000m + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + - name: csi-attacher + image: registry.k8s.io/sig-storage/csi-attacher:v4.4.1 + args: + - "--v=5" + - "--csi-address=$(CSI_ADDRESS)" + - "--leader-election" + env: + - name: CSI_ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + resources: + limits: + cpu: 1000m + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + - name: csi-snapshotter + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.3.0 + args: + - "--v=5" + - "--csi-address=$(CSI_ADDRESS)" + - "--leader-election" + env: + - name: CSI_ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + resources: + limits: + cpu: 400m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + - name: snapshot-controller + image: registry.k8s.io/sig-storage/snapshot-controller:v6.3.0 + args: + - "--v=5" + - "--leader-election" + resources: + limits: + cpu: 400m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + - name: csi-resizer + image: registry.k8s.io/sig-storage/csi-resizer:v1.9.0 + args: + - "--v=5" + - "--csi-address=$(CSI_ADDRESS)" + - "--leader-election" + env: + - name: CSI_ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + resources: + limits: + cpu: 1000m + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + - name: liveness-probe + image: registry.k8s.io/sig-storage/livenessprobe:v2.11.0 + args: + - "--csi-address=$(CSI_ADDRESS)" + env: + - name: CSI_ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + resources: + limits: + cpu: 200m + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + volumes: + - name: socket-dir + emptyDir: {} diff --git a/deployment/exoscale-csi.yaml b/deployment/crds.yaml similarity index 76% rename from deployment/exoscale-csi.yaml rename to deployment/crds.yaml index 611d409..e477402 100644 --- a/deployment/exoscale-csi.yaml +++ b/deployment/crds.yaml @@ -1,178 +1,3 @@ -apiVersion: storage.k8s.io/v1 -kind: CSIDriver -metadata: - name: csi.exoscale.com -spec: - attachRequired: true - podInfoOnMount: true ---- -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: exoscale-sbs - namespace: kube-system -provisioner: csi.exoscale.com -reclaimPolicy: Delete -allowVolumeExpansion: true ---- -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: exoscale-csi-node - namespace: kube-system - labels: -spec: - selector: - matchLabels: - app: exoscale-csi-node - template: - metadata: - labels: - app: exoscale-csi-node - role: csi - spec: - serviceAccount: exoscale-csi-node - nodeSelector: - kubernetes.io/os: linux - priorityClassName: system-node-critical - containers: - - name: exoscale-csi-plugin - image: exoscale/csi-driver:latest - imagePullPolicy: Always - args : - - "--endpoint=$(CSI_ENDPOINT)" - - "--v=4" - - "--mode=node" - env: - - name: CSI_ENDPOINT - value: unix:///csi/csi.sock - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - securityContext: - privileged: true - ports: - - name: healthz - containerPort: 9808 - protocol: TCP - livenessProbe: - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 10 - timeoutSeconds: 3 - periodSeconds: 2 - failureThreshold: 5 - volumeMounts: - - name: plugin-dir - mountPath: /csi - - name: kubelet-dir - mountPath: /var/lib/kubelet - mountPropagation: "Bidirectional" - - name: device-dir - mountPath: /dev - resources: - limits: - cpu: 400m - memory: 200Mi - requests: - cpu: 10m - memory: 20Mi - - name: csi-node-driver-registrar - image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.0 - args: - - "--v=2" - - "--csi-address=$(CSI_ADDRESS)" - - "--kubelet-registration-path=$(KUBELET_REGISTRATION_PATH)" - env: - - name: CSI_ADDRESS - value: /csi/csi.sock - - name: KUBELET_REGISTRATION_PATH - value: /var/lib/kubelet/plugins/csi.exoscale.com/csi.sock - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - volumeMounts: - - name: plugin-dir - mountPath: /csi/ - - name: registration-dir - mountPath: /registration/ - resources: - limits: - cpu: 200m - memory: 100Mi - requests: - cpu: 10m - memory: 20Mi - - name: liveness-probe - image: registry.k8s.io/sig-storage/livenessprobe:v2.11.0 - args: - - "--csi-address=$(CSI_ADDRESS)" - env: - - name: CSI_ADDRESS - value: /csi/csi.sock - volumeMounts: - - name: plugin-dir - mountPath: /csi - resources: - limits: - cpu: 200m - memory: 100Mi - requests: - cpu: 10m - memory: 20Mi - volumes: - - name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins_registry/ - type: DirectoryOrCreate - - name: plugin-dir - hostPath: - path: /var/lib/kubelet/plugins/csi.exoscale.com - type: DirectoryOrCreate - - name: kubelet-dir - hostPath: - path: /var/lib/kubelet - type: Directory - - name: device-dir - hostPath: - path: /dev ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: exoscale-csi-node - namespace: kube-system ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: exoscale-csi-node-driver - namespace: kube-system -rules: - - apiGroups: [""] - resources: ["pods", "nodes"] - verbs: ["get", "list", "watch"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: exoscale-csi-node-driver -subjects: - - kind: ServiceAccount - name: exoscale-csi-node - namespace: kube-system -roleRef: - kind: ClusterRole - name: exoscale-csi-node-driver - apiGroup: rbac.authorization.k8s.io ---- ## volumesnapshotclasses CRD - copied from https://github.com/kubernetes-csi/external-snapshotter/blob/master/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -284,7 +109,7 @@ status: conditions: [] storedVersions: [] --- -## volumesnapshotcontents CRD - copied from +## volumesnapshotcontents CRD - copied from apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -801,322 +626,3 @@ status: plural: "" conditions: [] storedVersions: [] ---- -apiVersion: snapshot.storage.k8s.io/v1 -kind: VolumeSnapshotClass -metadata: - name: exoscale-snapshot -driver: csi.exoscale.com -deletionPolicy: Delete ---- -kind: Deployment -apiVersion: apps/v1 -metadata: - name: exoscale-csi-controller - namespace: kube-system -spec: - selector: - matchLabels: - app: exoscale-csi-controller - replicas: 2 - template: - metadata: - labels: - app: exoscale-csi-controller - spec: - dnsPolicy: Default - priorityClassName: system-cluster-critical - serviceAccount: exoscale-csi-controller - containers: - - name: exoscale-csi-plugin - image: exoscale/csi-driver:latest - imagePullPolicy: Always - args : - - "--endpoint=$(CSI_ENDPOINT)" - - "--mode=controller" - - "--v=4" - env: - - name: CSI_ENDPOINT - value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - envFrom: - - secretRef: - name: exoscale-credentials - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - ports: - - name: healthz - containerPort: 9808 - protocol: TCP - livenessProbe: - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 10 - timeoutSeconds: 3 - periodSeconds: 2 - failureThreshold: 5 - resources: - limits: - cpu: 1000m - memory: 500Mi - requests: - cpu: 10m - memory: 20Mi - - name: csi-provisioner - image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.0 - args: - - "--v=5" - - "--csi-address=$(CSI_ADDRESS)" - - "--leader-election" - - "--feature-gates=Topology=true" - - "--default-fstype=ext4" - env: - - name: CSI_ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - resources: - limits: - cpu: 1000m - memory: 500Mi - requests: - cpu: 10m - memory: 20Mi - - name: csi-attacher - image: registry.k8s.io/sig-storage/csi-attacher:v4.4.1 - args: - - "--v=5" - - "--csi-address=$(CSI_ADDRESS)" - - "--leader-election" - env: - - name: CSI_ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - resources: - limits: - cpu: 1000m - memory: 500Mi - requests: - cpu: 10m - memory: 20Mi - - name: csi-snapshotter - image: registry.k8s.io/sig-storage/csi-snapshotter:v6.3.0 - args: - - "--v=5" - - "--csi-address=$(CSI_ADDRESS)" - - "--leader-election" - env: - - name: CSI_ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - resources: - limits: - cpu: 400m - memory: 200Mi - requests: - cpu: 10m - memory: 20Mi - - name: snapshot-controller - image: registry.k8s.io/sig-storage/snapshot-controller:v6.3.0 - args: - - "--v=5" - - "--leader-election" - resources: - limits: - cpu: 400m - memory: 200Mi - requests: - cpu: 10m - memory: 20Mi - - name: csi-resizer - image: registry.k8s.io/sig-storage/csi-resizer:v1.9.0 - args: - - "--v=5" - - "--csi-address=$(CSI_ADDRESS)" - - "--leader-election" - env: - - name: CSI_ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - resources: - limits: - cpu: 1000m - memory: 500Mi - requests: - cpu: 10m - memory: 20Mi - - name: liveness-probe - image: registry.k8s.io/sig-storage/livenessprobe:v2.11.0 - args: - - "--csi-address=$(CSI_ADDRESS)" - env: - - name: CSI_ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - resources: - limits: - cpu: 200m - memory: 100Mi - requests: - cpu: 10m - memory: 20Mi - volumes: - - name: socket-dir - emptyDir: {} ---- -kind: ServiceAccount -apiVersion: v1 -metadata: - name: exoscale-csi-controller - namespace: kube-system ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: exoscale-csi-provisioner -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments", "volumeattachments/status"] - verbs: ["get", "list", "watch", "update", "create", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots", "volumesnapshots/status", "volumesnapshotclasses", "volumesnapshotcontents", "volumesnapshotcontents/status"] - verbs: ["get", "list", "watch", "update", "create", "patch"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: exoscale-csi-controller -subjects: - - kind: ServiceAccount - name: exoscale-csi-controller - namespace: kube-system -roleRef: - kind: ClusterRole - name: exoscale-csi-provisioner - apiGroup: rbac.authorization.k8s.io ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: exoscale-csi-attacher -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "watch", "list", "update", "patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "watch", "list"] - - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "watch", "list", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments", "volumeattachments/status"] - verbs: ["get", "watch", "list", "create", "update", "patch", "delete"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: exoscale-csi-attacher -subjects: - - kind: ServiceAccount - name: exoscale-csi-controller - namespace: kube-system -roleRef: - kind: ClusterRole - name: exoscale-csi-attacher - apiGroup: rbac.authorization.k8s.io ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: exoscale-csi-snapshotter -rules: - - apiGroups: [""] - resources: ["persistentvolumes", "persistentvolumeclaims", "events"] - verbs: ["get", "watch", "list", "create", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "watch", "list"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots", "volumesnapshots/status", "volumesnapshotclasses", "volumesnapshotcontents"] - verbs: ["get", "watch", "list", "create", "update", "patch", "delete"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions", "leases"] - verbs: ["get", "watch", "list", "create", "update", "patch", "delete"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: exoscale-csi-snapshotter -subjects: - - kind: ServiceAccount - name: exoscale-csi-controller - namespace: kube-system -roleRef: - kind: ClusterRole - name: exoscale-csi-snapshotter - apiGroup: rbac.authorization.k8s.io ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: external-resizer -rules: - - apiGroups: [""] - resources: ["persistentvolumes", "persistentvolumeclaims", "persistentvolumeclaims/status"] - verbs: ["get", "watch", "list", "create", "update", "patch"] - - apiGroups: [""] - resources: ["pods", "events"] - verbs: ["get", "watch", "list"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-resizer-role -subjects: - - kind: ServiceAccount - name: exoscale-csi-controller - namespace: kube-system -roleRef: - kind: ClusterRole - name: external-resizer - apiGroup: rbac.authorization.k8s.io diff --git a/deployment/csi-driver.yaml b/deployment/csi-driver.yaml new file mode 100644 index 0000000..4395a67 --- /dev/null +++ b/deployment/csi-driver.yaml @@ -0,0 +1,7 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: csi.exoscale.com +spec: + attachRequired: true + podInfoOnMount: true diff --git a/deployment/kustomization.yaml b/deployment/kustomization.yaml new file mode 100644 index 0000000..c3bac17 --- /dev/null +++ b/deployment/kustomization.yaml @@ -0,0 +1,9 @@ +resources: + - crds.yaml + - controller-rbac.yaml + - controller.yaml + - csi-driver.yaml + - node-driver-rbac.yaml + - node-driver.yaml + - storage-class.yaml + - volume-snapshot-class.yaml diff --git a/deployment/node-driver-rbac.yaml b/deployment/node-driver-rbac.yaml new file mode 100644 index 0000000..b302549 --- /dev/null +++ b/deployment/node-driver-rbac.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: exoscale-csi-node + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: exoscale-csi-node-driver + namespace: kube-system +rules: + - apiGroups: [""] + resources: ["pods", "nodes"] + verbs: ["get", "list", "watch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: exoscale-csi-node-driver +subjects: + - kind: ServiceAccount + name: exoscale-csi-node + namespace: kube-system +roleRef: + kind: ClusterRole + name: exoscale-csi-node-driver + apiGroup: rbac.authorization.k8s.io diff --git a/deployment/node-driver.yaml b/deployment/node-driver.yaml new file mode 100644 index 0000000..aaf8b39 --- /dev/null +++ b/deployment/node-driver.yaml @@ -0,0 +1,128 @@ +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: exoscale-csi-node + namespace: kube-system + labels: +spec: + selector: + matchLabels: + app: exoscale-csi-node + template: + metadata: + labels: + app: exoscale-csi-node + role: csi + spec: + serviceAccount: exoscale-csi-node + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + containers: + - name: exoscale-csi-plugin + image: exoscale/csi-driver:latest + imagePullPolicy: Always + args : + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=4" + - "--mode=node" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + securityContext: + privileged: true + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 2 + failureThreshold: 5 + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + - name: device-dir + mountPath: /dev + resources: + limits: + cpu: 400m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + - name: csi-node-driver-registrar + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.0 + args: + - "--v=2" + - "--csi-address=$(CSI_ADDRESS)" + - "--kubelet-registration-path=$(KUBELET_REGISTRATION_PATH)" + env: + - name: CSI_ADDRESS + value: /csi/csi.sock + - name: KUBELET_REGISTRATION_PATH + value: /var/lib/kubelet/plugins/csi.exoscale.com/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: plugin-dir + mountPath: /csi/ + - name: registration-dir + mountPath: /registration/ + resources: + limits: + cpu: 200m + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + - name: liveness-probe + image: registry.k8s.io/sig-storage/livenessprobe:v2.11.0 + args: + - "--csi-address=$(CSI_ADDRESS)" + env: + - name: CSI_ADDRESS + value: /csi/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: /csi + resources: + limits: + cpu: 200m + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + volumes: + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/csi.exoscale.com + type: DirectoryOrCreate + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: device-dir + hostPath: + path: /dev diff --git a/deployment/storage-class.yaml b/deployment/storage-class.yaml new file mode 100644 index 0000000..eefa4ac --- /dev/null +++ b/deployment/storage-class.yaml @@ -0,0 +1,8 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: exoscale-sbs + namespace: kube-system +provisioner: csi.exoscale.com +reclaimPolicy: Delete +allowVolumeExpansion: true diff --git a/deployment/volume-snapshot-class.yaml b/deployment/volume-snapshot-class.yaml new file mode 100644 index 0000000..60cb1d8 --- /dev/null +++ b/deployment/volume-snapshot-class.yaml @@ -0,0 +1,6 @@ +apiVersion: snapshot.storage.k8s.io/v1 +kind: VolumeSnapshotClass +metadata: + name: exoscale-snapshot +driver: csi.exoscale.com +deletionPolicy: Delete diff --git a/internal/integ/cluster/teardown.go b/internal/integ/cluster/teardown.go index c4f05a2..237a722 100644 --- a/internal/integ/cluster/teardown.go +++ b/internal/integ/cluster/teardown.go @@ -2,6 +2,7 @@ package cluster import ( "fmt" + "log/slog" "github.com/exoscale/exoscale/csi-driver/internal/integ/flags" ) @@ -45,12 +46,18 @@ func (c *Cluster) TearDown() error { } func (c *Cluster) tearDownCSI() error { - err := c.K8s.DeleteManifest(c.exoV2Context, mainManifest) - if err != nil { - return err + var finalErr error = nil + + for _, manifestPath := range allManifests { + err := c.K8s.DeleteManifest(c.exoV2Context, manifestDir+manifestPath) + if err != nil { + slog.Error("failed to delete manifest", "manifest", manifestPath) + + finalErr = fmt.Errorf("errors while deleting manifests") + } } - return err + return finalErr // TODO (sauterp) reenable once we have a non-legacy key in GH // return c.deleteAPIKeyAndRole() diff --git a/internal/integ/cluster/utils.go b/internal/integ/cluster/utils.go index d0fc048..2149a79 100644 --- a/internal/integ/cluster/utils.go +++ b/internal/integ/cluster/utils.go @@ -27,7 +27,27 @@ const ( ) var ( - mainManifest = util.GetRepoRootDir() + "deployment/exoscale-csi.yaml" + manifestDir = util.GetRepoRootDir() + "deployment/" + + controllerRBACManifest = "controller-rbac.yaml" + controllerManifest = "controller.yaml" + crdsManifest = "crds.yaml" + csiDriverManifest = "csi-driver.yaml" + nodeDriverRBACManifest = "node-driver-rbac.yaml" + nodeDriverManifest = "node-driver.yaml" + storageClassManifest = "storage-class.yaml" + volumeSnapshotClassManifest = "volume-snapshot-class.yaml" + + allManifests = []string{ + crdsManifest, + controllerRBACManifest, + controllerManifest, + csiDriverManifest, + nodeDriverRBACManifest, + nodeDriverManifest, + storageClassManifest, + volumeSnapshotClassManifest, + } ) func ptr[T any](v T) *T { @@ -241,9 +261,15 @@ func (c *Cluster) applyCSI() error { } } - err := c.K8s.ApplyManifest(c.exoV2Context, mainManifest) - if err != nil { - return fmt.Errorf("error applying CSI manifests: %w", err) + if *flags.Image != "" { + slog.Info("testing CSI image", "path", *flags.Image) + } + + for _, manifestPath := range allManifests { + err := c.K8s.ApplyManifest(c.exoV2Context, manifestDir+manifestPath) + if err != nil { + return fmt.Errorf("error applying CSI manifest: %q %w", manifestPath, err) + } } // TODO(sauterp) this shouldn't be necessary anymore once the CSI addon is available. diff --git a/internal/integ/integ_test.go b/internal/integ/integ_test.go index 4375447..9acff47 100644 --- a/internal/integ/integ_test.go +++ b/internal/integ/integ_test.go @@ -235,9 +235,13 @@ func TestSnapshot(t *testing.T) { awaitExpectation(t, "Bound", func() interface{} { pvc, err := pvcClient.Get(ns.CTX, "my-snap-1-pvc", metav1.GetOptions{}) - assert.NoError(t, err) - return pvc.Status.Phase + if err != nil { + assert.NoError(t, err) + return "error" + } + + return string(pvc.Status.Phase) }) // delete snapshot @@ -246,7 +250,10 @@ func TestSnapshot(t *testing.T) { awaitExpectation(t, 0, func() interface{} { snapshots, err := snapshotClient.List(ns.CTX, v1.ListOptions{}) - assert.NoError(t, err) + if err != nil { + assert.NoError(t, err) + return 0 + } return len(snapshots.Items) }) diff --git a/internal/integ/k8s/k8s.go b/internal/integ/k8s/k8s.go index 9f14e8b..11cdfe9 100644 --- a/internal/integ/k8s/k8s.go +++ b/internal/integ/k8s/k8s.go @@ -101,8 +101,6 @@ func (k *K8S) ApplyManifest(ctx context.Context, file string) error { fileContentStr := string(fileContent) if *flags.Image != "" { - slog.Info("testing CSI image", "path", *flags.Image) - fileContentStr = strings.ReplaceAll(fileContentStr, `exoscale/csi-driver:latest`, *flags.Image) } @@ -211,6 +209,15 @@ func (k *K8S) PrintEvents(ctx context.Context, ns string) { } } +func (k *K8S) UpdateResourceList() { + _, newResourceList, err := k.DiscoveryClient.ServerGroupsAndResources() + if err != nil { + slog.Warn("failed to update resource list", "err", err) + } else { + k.ResourceList = newResourceList + } +} + func (k *K8S) applyManifest(ctx context.Context, manifest []byte) error { decoder := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme) obj := &unstructured.Unstructured{} @@ -219,20 +226,20 @@ func (k *K8S) applyManifest(ctx context.Context, manifest []byte) error { return err } - rrr := k.findResource(obj.GetKind()) - if rrr == nil { + res := k.findResource(obj.GetKind()) + if res == nil { return nil } - gvr := gvk.GroupVersion().WithResource(rrr.Name) + gvr := gvk.GroupVersion().WithResource(res.Name) namespace := "" - if rrr.Namespaced { + if res.Namespaced { namespace = obj.GetNamespace() go k.PrintEvents(ctx, namespace) } resourceInterface := k.DynamicClient.Resource(gvr).Namespace(namespace) - res, err := resourceInterface.Get(ctx, obj.GetName(), metav1.GetOptions{}) + resIf, err := resourceInterface.Get(ctx, obj.GetName(), metav1.GetOptions{}) if err != nil { slog.Info("creating", "resource", gvr, "name", obj.GetName()) @@ -244,7 +251,7 @@ func (k *K8S) applyManifest(ctx context.Context, manifest []byte) error { } else { slog.Info("updating", "resource", gvr, "name", obj.GetName()) - obj.SetResourceVersion(res.GetResourceVersion()) + obj.SetResourceVersion(resIf.GetResourceVersion()) // If it exists, update it retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { @@ -256,12 +263,9 @@ func (k *K8S) applyManifest(ctx context.Context, manifest []byte) error { } } - _, newResourceList, err := k.DiscoveryClient.ServerGroupsAndResources() - if err != nil { - slog.Warn("failed to update resource list", "err", err) - } else { - k.ResourceList = newResourceList - } + // The resources we just applied need to be fetched from the server + // so that the client can apply manifests that depend on these. + k.UpdateResourceList() return nil }