From dc222243d92f2a9f5c35f18e58e94f19e6422446 Mon Sep 17 00:00:00 2001 From: weizhichen Date: Thu, 20 Jun 2024 08:46:33 +0000 Subject: [PATCH 1/2] fix --- charts/README.md | 1 + .../templates/csi-azurefile-node.yaml | 41 +++++++++++++++++++ .../latest/azurefile-csi-driver/values.yaml | 7 ++++ pkg/azurefile/azurefile.go | 4 ++ pkg/azurefile/azurefile_options.go | 2 + pkg/azurefile/nodeserver.go | 3 ++ pkg/azurefileplugin/Dockerfile | 11 ++++- 7 files changed, 68 insertions(+), 1 deletion(-) diff --git a/charts/README.md b/charts/README.md index bc38a3c18f..29e47240fa 100644 --- a/charts/README.md +++ b/charts/README.md @@ -124,6 +124,7 @@ The following table lists the configurable parameters of the latest Azure File C | `node.maxUnavailable` | `maxUnavailable` value of driver node daemonset | `1` | `node.livenessProbe.healthPort ` | health check port for liveness probe | `29613` | | `node.logLevel` | node driver log level |`5` | +| `node.enableAznfsMount` | enable [AZNFS mount helper](https://github.com/Azure/AZNFS-mount/) for NFS protocol | true | `snapshot.enabled` | whether enable snapshot feature | `false` | | `snapshot.image.csiSnapshotter.repository` | csi-snapshotter docker image | `/oss/kubernetes-csi/csi-snapshotter` | | `snapshot.image.csiSnapshotter.tag` | csi-snapshotter docker image tag | `v6.3.3` | diff --git a/charts/latest/azurefile-csi-driver/templates/csi-azurefile-node.yaml b/charts/latest/azurefile-csi-driver/templates/csi-azurefile-node.yaml index b94b2c82fe..67d76e649e 100644 --- a/charts/latest/azurefile-csi-driver/templates/csi-azurefile-node.yaml +++ b/charts/latest/azurefile-csi-driver/templates/csi-azurefile-node.yaml @@ -39,6 +39,9 @@ spec: {{ toYaml . | indent 8 }} {{- end }} spec: +{{- if .Values.node.enableAznfsMount }} + hostPID: true +{{- end }} hostNetwork: true dnsPolicy: {{ .Values.linux.dnsPolicy }} serviceAccountName: {{ .Values.serviceAccount.node }} @@ -136,6 +139,7 @@ spec: - "--mount-permissions={{ .Values.linux.mountPermissions }}" - "--allow-inline-volume-key-access-with-identity={{ .Values.node.allowInlineVolumeKeyAccessWithIdentity }}" - "--metrics-address=0.0.0.0:{{ .Values.node.metricsPort }}" + - "--enable-aznfs-mount={{ .Values.node.enableAznfsMount }}" livenessProbe: failureThreshold: 5 httpGet: @@ -190,7 +194,34 @@ spec: mountPath: /etc/pki/ca-trust/extracted readOnly: true {{- end }} + {{- if .Values.node.enableAznfsMount }} + - mountPath: /opt/microsoft/aznfs/data + name: aznfs-data + - mountPath: /lib/modules + name: lib-modules + readOnly: true + {{- end }} resources: {{- toYaml .Values.linux.resources.azurefile | nindent 12 }} +{{- if .Values.node.enableAznfsMount }} + - name: aznfswatchdogv4 +{{- if hasPrefix "/" .Values.image.azurefile.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.azurefile.repository }}:{{ .Values.image.azurefile.tag }}" +{{- else }} + image: "{{ .Values.image.azurefile.repository }}:{{ .Values.image.azurefile.tag }}" +{{- end }} + command: + - "aznfswatchdogv4" + imagePullPolicy: {{ .Values.image.azurefile.pullPolicy }} + securityContext: + privileged: true + resources: {{- toYaml .Values.linux.resources.aznfswatchdog | nindent 12 }} + volumeMounts: + - mountPath: /opt/microsoft/aznfs/data + name: aznfs-data + - mountPath: {{ .Values.linux.kubelet }}/ + mountPropagation: Bidirectional + name: mountpoint-dir +{{- end }} volumes: - hostPath: path: {{ .Values.linux.kubelet }}/plugins/{{ .Values.driver.name }} @@ -220,4 +251,14 @@ spec: hostPath: path: /etc/pki/ca-trust/extracted {{- end }} + {{- if .Values.node.enableAznfsMount }} + - hostPath: + path: /opt/microsoft/aznfs-azurefile/data # host path `/opt/microsoft/aznfs-azurefile/data` for azurefile and host path `/opt/microsoft/aznfs/data` for blob + type: DirectoryOrCreate + name: aznfs-data + - name: lib-modules + hostPath: + path: /lib/modules + type: DirectoryOrCreate + {{- end }} {{- end -}} diff --git a/charts/latest/azurefile-csi-driver/values.yaml b/charts/latest/azurefile-csi-driver/values.yaml index e6d44c11ab..d42e380c62 100644 --- a/charts/latest/azurefile-csi-driver/values.yaml +++ b/charts/latest/azurefile-csi-driver/values.yaml @@ -117,6 +117,7 @@ node: livenessProbe: healthPort: 29613 logLevel: 5 + enableAznfsMount: true snapshot: enabled: false @@ -189,6 +190,12 @@ linux: requests: cpu: 10m memory: 20Mi + aznfswatchdog: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi tolerations: - operator: "Exists" nodeSelector: {} diff --git a/pkg/azurefile/azurefile.go b/pkg/azurefile/azurefile.go index c672c1a80e..90244de0e3 100644 --- a/pkg/azurefile/azurefile.go +++ b/pkg/azurefile/azurefile.go @@ -130,6 +130,7 @@ const ( cifs = "cifs" smb = "smb" nfs = "nfs" + aznfs = "aznfs" ext4 = "ext4" ext3 = "ext3" ext2 = "ext2" @@ -261,6 +262,8 @@ type Driver struct { waitForAzCopyTimeoutMinutes int // azcopy for provide exec mock for ut azcopy *fileutil.Azcopy + // enable aznfs mount + enableAznfsMount bool kubeconfig string endpoint string @@ -302,6 +305,7 @@ func NewDriver(options *DriverOptions) *Driver { driver.azcopy = &fileutil.Azcopy{} driver.kubeconfig = options.KubeConfig driver.endpoint = options.Endpoint + driver.enableAznfsMount = options.EnableAznfsMount var err error getter := func(key string) (interface{}, error) { return nil, nil } diff --git a/pkg/azurefile/azurefile_options.go b/pkg/azurefile/azurefile_options.go index a0ea5808f5..22afb8e112 100644 --- a/pkg/azurefile/azurefile_options.go +++ b/pkg/azurefile/azurefile_options.go @@ -49,6 +49,7 @@ type DriverOptions struct { WaitForAzCopyTimeoutMinutes int KubeConfig string Endpoint string + EnableAznfsMount bool } func (o *DriverOptions) AddFlags() *flag.FlagSet { @@ -85,6 +86,7 @@ func (o *DriverOptions) AddFlags() *flag.FlagSet { fs.IntVar(&o.WaitForAzCopyTimeoutMinutes, "wait-for-azcopy-timeout-minutes", 5, "timeout in minutes for waiting for azcopy to finish") fs.StringVar(&o.KubeConfig, "kubeconfig", "", "Absolute path to the kubeconfig file. Required only when running out of cluster.") fs.StringVar(&o.Endpoint, "endpoint", "unix://tmp/csi.sock", "CSI endpoint") + fs.BoolVar(&o.EnableAznfsMount, "enable-aznfs-mount", false, "replace nfs mount with aznfs mount") return fs } diff --git a/pkg/azurefile/nodeserver.go b/pkg/azurefile/nodeserver.go index 6699bee693..a079b787a2 100644 --- a/pkg/azurefile/nodeserver.go +++ b/pkg/azurefile/nodeserver.go @@ -335,6 +335,9 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe mountFsType := cifs if protocol == nfs { mountFsType = nfs + if d.enableAznfsMount { + mountFsType = aznfs + } } if err := prepareStagePath(cifsMountPath, d.mounter); err != nil { return nil, status.Errorf(codes.Internal, "prepare stage path failed for %s with error: %v", cifsMountPath, err) diff --git a/pkg/azurefileplugin/Dockerfile b/pkg/azurefileplugin/Dockerfile index e82c1e56f7..79359199ef 100644 --- a/pkg/azurefileplugin/Dockerfile +++ b/pkg/azurefileplugin/Dockerfile @@ -18,7 +18,16 @@ ARG ARCH=amd64 ARG binary=./_output/${ARCH}/azurefileplugin COPY ${binary} /azurefileplugin -RUN apt update && apt upgrade -y && apt-mark unhold libcap2 && clean-install ca-certificates cifs-utils util-linux e2fsprogs mount udev xfsprogs nfs-common netbase wget +# packages that are only needed by aznfs: procps conntrack iptables bind9-host iproute2 bash netcat sysvinit-utils kmod net-tools +RUN apt update && apt upgrade -y && apt-mark unhold libcap2 && clean-install ca-certificates cifs-utils util-linux e2fsprogs mount udev xfsprogs nfs-common netbase wget procps conntrack iptables bind9-host iproute2 bash netcat-traditional sysvinit-utils kmod net-tools + +# install aznfs +ARG aznfsVer=0.1.349 +RUN if [ "$ARCH" = "amd64" ] ; then \ + wget -O aznfs.tar.gz https://github.com/Azure/AZNFS-mount/releases/download/${aznfsVer}/aznfs-${aznfsVer}-1.x86_64.tar.gz; \ +else \ + wget -O aznfs.tar.gz https://github.com/Azure/AZNFS-mount/releases/download/${aznfsVer}/aznfs-${aznfsVer}-1.arm64.tar.gz;fi +RUN tar xvzf aznfs.tar.gz -C / --keep-directory-symlink && rm aznfs.tar.gz # install azcopy ARG azcopyURL=https://azcopyvnext.azureedge.net/releases/release-10.24.0-20240326/azcopy_linux_amd64_10.24.0.tar.gz From 781651dc6b55799660279bdd90349f655a99f144 Mon Sep 17 00:00:00 2001 From: weizhichen Date: Mon, 14 Oct 2024 02:42:31 +0000 Subject: [PATCH 2/2] support EiT --- .../templates/csi-azurefile-node.yaml | 16 ++++++++++++++++ pkg/azurefileplugin/Dockerfile | 18 ++++-------------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/charts/latest/azurefile-csi-driver/templates/csi-azurefile-node.yaml b/charts/latest/azurefile-csi-driver/templates/csi-azurefile-node.yaml index e64cb295be..0c7d5da561 100644 --- a/charts/latest/azurefile-csi-driver/templates/csi-azurefile-node.yaml +++ b/charts/latest/azurefile-csi-driver/templates/csi-azurefile-node.yaml @@ -203,6 +203,10 @@ spec: - mountPath: /lib/modules name: lib-modules readOnly: true + - mountPath: /etc/stunnel/microsoft/aznfs/nfsv4_fileShare + name: stunnel-config + - mountPath: /etc/stunnel/microsoft/aznfs/nfsv4_fileShare/logs + name: stunnel-logs {{- end }} resources: {{- toYaml .Values.linux.resources.azurefile | nindent 12 }} {{- if .Values.node.enableAznfsMount }} @@ -224,6 +228,10 @@ spec: - mountPath: {{ .Values.linux.kubelet }}/ mountPropagation: Bidirectional name: mountpoint-dir + - mountPath: /etc/stunnel/microsoft/aznfs/nfsv4_fileShare + name: stunnel-config + - mountPath: /etc/stunnel/microsoft/aznfs/nfsv4_fileShare/logs + name: stunnel-logs {{- end }} volumes: - hostPath: @@ -263,5 +271,13 @@ spec: hostPath: path: /lib/modules type: DirectoryOrCreate + - name: stunnel-config + hostPath: + path: /etc/stunnel/microsoft/aznfs/nfsv4_fileShare + type: DirectoryOrCreate + - name: stunnel-logs + hostPath: + path: /etc/stunnel/microsoft/aznfs/nfsv4_fileShare/logs + type: DirectoryOrCreate {{- end }} {{- end -}} diff --git a/pkg/azurefileplugin/Dockerfile b/pkg/azurefileplugin/Dockerfile index 59ad690fee..a33558f215 100644 --- a/pkg/azurefileplugin/Dockerfile +++ b/pkg/azurefileplugin/Dockerfile @@ -33,28 +33,18 @@ ARG binary=./_output/${ARCH}/azurefileplugin COPY ${binary} /azurefileplugin COPY --from=builder --chown=root:root /usr/local/bin/azcopy /usr/local/bin/azcopy -# packages that are only needed by aznfs: procps conntrack iptables bind9-host iproute2 bash netcat sysvinit-utils kmod net-tools -RUN apt update && apt upgrade -y && apt-mark unhold libcap2 && clean-install ca-certificates cifs-utils util-linux e2fsprogs mount udev xfsprogs nfs-common netbase procps conntrack iptables bind9-host iproute2 bash netcat-traditional sysvinit-utils kmod net-tools +# packages that are only needed by aznfs: procps conntrack iptables bind9-host iproute2 bash netcat sysvinit-utils kmod net-tools stunnel4 +RUN apt update && apt upgrade -y && apt-mark unhold libcap2 && clean-install ca-certificates cifs-utils util-linux e2fsprogs mount udev xfsprogs nfs-common netbase wget procps conntrack iptables bind9-host iproute2 bash netcat-traditional sysvinit-utils kmod net-tools stunnel4 # install aznfs -ARG aznfsVer=0.1.349 +# TODO: Dont use wget to download and install aznfs +ARG aznfsVer=0.1.394 RUN if [ "$ARCH" = "amd64" ] ; then \ wget -O aznfs.tar.gz https://github.com/Azure/AZNFS-mount/releases/download/${aznfsVer}/aznfs-${aznfsVer}-1.x86_64.tar.gz; \ else \ wget -O aznfs.tar.gz https://github.com/Azure/AZNFS-mount/releases/download/${aznfsVer}/aznfs-${aznfsVer}-1.arm64.tar.gz;fi RUN tar xvzf aznfs.tar.gz -C / --keep-directory-symlink && rm aznfs.tar.gz -# install azcopy -ARG azcopyURL=https://azcopyvnext.azureedge.net/releases/release-10.25.1-20240612/azcopy_linux_amd64_10.25.1.tar.gz -RUN if [ "$ARCH" == "arm64" ] ; then \ - azcopyURL=https://azcopyvnext.azureedge.net/releases/release-10.25.1-20240612/azcopy_linux_arm64_10.25.1.tar.gz; fi -RUN wget -O azcopy.tar.gz ${azcopyURL} && \ - tar xvzf azcopy.tar.gz -C . && rm azcopy.tar.gz && \ - mv ./azcopy_linux_$ARCH_*/azcopy /usr/local/bin/azcopy && \ - rm -rf ./azcopy_linux_$ARCH_* -RUN chmod +x /usr/local/bin/azcopy -RUN apt remove wget -y - LABEL maintainers="andyzhangx" LABEL description="AzureFile CSI Driver"