chore(nfs): cleanup RWX experiments (refs #57)

mborne · May 21, 2024 · ca9a655 · ca9a655
1 parent 7f0ab5e
commit ca9a655
Show file tree

Hide file tree

Showing 14 changed files with 239 additions and 22 deletions.
diff --git a/README.md b/README.md
@@ -69,10 +69,11 @@ This is my playground to learn and illustrate how to deploy application with [do
 
 | Name                                                                         | Description                                                                                         | Docker  |   K8S   |
 | ---------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------- | :-----: | :-----: |
-| [Longhorn](longhorn/README.md)                                               | **Distributed block storage** for Kubernetes providing `ReadWriteMany` volumes                      |   NA    | &#9745; |
-| [nfs-server-provisioner](nfs-server-provisioner/README.md)                   | Deploy a NFS server to provide `ReadWriteMany` volumes                                              |   NA    | &#9745; |
 | [nfs-subdir-external-provisioner](nfs-subdir-external-provisioner/README.md) | Use existing NFS server to provide `ReadWriteMany` volumes                                          |   NA    | &#9745; |
+| [nfs-demo](nfs-demo/README.md)                                               | Illustrates the use of a "nfs" storage class providing ReadWriteMany support                        |
 | [nfs-server](nfs-server/README.md)                                           | **NFS server** to test [nfs-subdir-external-provisioner](nfs-subdir-external-provisioner/README.md) | &#9745; | &#9745; |
+| [nfs-server-provisioner](nfs-server-provisioner/README.md)                   | Deploy a NFS server to provide `ReadWriteMany` volumes                                              |   NA    | &#9745; |
+| [Longhorn](longhorn/README.md)                                               | **Distributed block storage** for Kubernetes providing `ReadWriteMany` volumes                      |   NA    | &#9745; |
 
 ### Database
 

diff --git a/nfs-demo/README.md b/nfs-demo/README.md
@@ -0,0 +1,23 @@
+# nfs-demo
+
+Illustrates the use of a "nfs" storage class providing ReadWriteMany support.
+
+## How it works?
+
+* [manifest/pvc.yaml](manifest/pvc.yaml) claims a RWX PersistentVolume for nginx data
+* [manifest/deployment.yaml](manifest/deployment.yaml) creates a nginx server
+* [manifest/service.yaml](manifest/services.yaml) exposes nginx as services
+* [manifest/cronjob.yaml](manifest/cronjob.yaml) periodically updates index.html
+
+## Usage with Kubernetes
+
+```bash
+# Deploy nfs-demo
+bash nfs-demo/k8s-install.sh
+
+# Test service
+kubectl -n nfs-demo run --rm -ti busybox --image=busybox --restart=Never -- /bin/sh -c "wget -q -O- http://nfs-demo"
+```
+
+
+
diff --git a/nfs-demo/k8s-install.sh b/nfs-demo/k8s-install.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+DEVBOX_HOSTNAME=${DEVBOX_HOSTNAME:-dev.localhost}
+DEVBOX_INGRESS=${DEVBOX_INGRESS:-traefik}
+DEVBOX_ISSUER=${DEVBOX_ISSUER:-mkcert}
+
+# Create namespace nfs-demo if not exists
+kubectl create namespace nfs-demo --dry-run=client -o yaml | kubectl apply -f -
+
+# Deploy nfs-demo with Kustomize
+kubectl -n nfs-demo apply -k ${SCRIPT_DIR}/manifest
+
+# Wait for Pods to be ready
+kubectl -n nfs-demo wait \
+    --for=condition=ready pod \
+    --selector=component=server \
+    --timeout=60s
+
+# Create Ingress with dynamic hostname
+cat <<EOF | kubectl -n nfs-demo apply -f -
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: nfs-demo
+  annotations:
+    cert-manager.io/cluster-issuer: "${DEVBOX_ISSUER}"
+spec:
+  ingressClassName: ${DEVBOX_INGRESS}
+  rules:
+  - host: nfs-demo.$DEVBOX_HOSTNAME
+    http:
+      paths:
+      - pathType: ImplementationSpecific
+        path: "/"
+        backend:
+          service:
+            name: nfs-demo
+            port:
+              number: 80
+  tls:
+  - hosts:
+    - nfs-demo.$DEVBOX_HOSTNAME
+    secretName: nfs-demo-cert
+EOF
+
+# Display resources
+kubectl -n nfs-demo get pvc,pods,svc,ingress
+
diff --git a/nfs-demo/k8s-uninstall.sh b/nfs-demo/k8s-uninstall.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+# Uninstall nfs-demo
+kubectl -n nfs-demo delete -k ${SCRIPT_DIR}/manifest
+
+# Remove Ingress
+kubectl -n nfs-demo delete ingress nfs-demo
+
+# Remove namespace
+kubectl delete namespace nfs-demo
+
diff --git a/nfs-demo/manifest/cronjob.yaml b/nfs-demo/manifest/cronjob.yaml
@@ -0,0 +1,28 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: nfs-demo-update
+spec:
+  jobTemplate:
+    metadata:
+      name: nfs-demo-update
+    spec:
+      template:
+        metadata:
+          labels:
+            component: backend
+        spec:
+          containers:
+          - image: busybox
+            name: update
+            command: ["/bin/sh","-c","echo $(date) -- update index.html ... ; date > /usr/share/nginx/html/index.html"]
+            volumeMounts:
+            - name: nfs-demo-data
+              mountPath: /usr/share/nginx/html
+          volumes:
+            - name: nfs-demo-data
+              persistentVolumeClaim:
+                claimName: nfs-demo-pvc
+          restartPolicy: OnFailure
+  schedule: '* * * * *'
+  successfulJobsHistoryLimit: 2
diff --git a/nfs-demo/manifest/deployment.yaml b/nfs-demo/manifest/deployment.yaml
@@ -0,0 +1,27 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nfs-demo
+  name: nfs-demo
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: nfs-demo
+  template:
+    metadata:
+      labels:
+        app: nfs-demo
+        component: server
+    spec:
+      containers:
+      - image: nginx
+        name: nginx
+        volumeMounts:
+        - name: nfs-demo-data
+          mountPath: /usr/share/nginx/html
+      volumes:
+        - name: nfs-demo-data
+          persistentVolumeClaim:
+            claimName: nfs-demo-pvc
diff --git a/nfs-demo/manifest/kustomization.yaml b/nfs-demo/manifest/kustomization.yaml
@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - pvc.yaml
+  - deployment.yaml
+  - service.yaml
+  - cronjob.yaml
diff --git a/nfs-demo/manifest/pvc.yaml b/nfs-demo/manifest/pvc.yaml
@@ -0,0 +1,14 @@
+# Claims a PersistentVolume using "nfs" storage class.
+# As a network file system, it can be mounted on Pods 
+# running on multiple nodes (ReadWriteMany)
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: nfs-demo-pvc
+spec:
+  storageClassName: "nfs"
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 100Mi
diff --git a/nfs-demo/manifest/service.yaml b/nfs-demo/manifest/service.yaml
@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: nfs-demo
+  name: nfs-demo
+spec:
+  ports:
+  - port: 80
+  selector:
+    app: nfs-demo
diff --git a/nfs-server-provisioner/README.md b/nfs-server-provisioner/README.md
@@ -16,12 +16,7 @@ kubectl get storageclass
 
 ## Alternatives
 
-### External DNS
-
-See [kubernetes-sigs/nfs-subdir-external-provisioner](https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner#readme) :
-
-* [github.com/kubernetes-sigs - Kubernetes NFS Subdir External Provisioner](https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner#kubernetes-nfs-subdir-external-provisioner)
-* [docs.ovh.com - Configuring multi-attach persistent volumes with OVHcloud NAS-HA](https://docs.ovh.com/gb/en/kubernetes/configuring-multi-attach-persistent-volumes-with-ovhcloud-nas-ha/)
+See [nfs-subdir-external-provisioner](../nfs-subdir-external-provisioner/README.md) to use an external NFS instance.
 
 ## Ressources
 

diff --git a/nfs-server-provisioner/k8s-uninstall.sh b/nfs-server-provisioner/k8s-uninstall.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+helm -n nfs-system uninstall nfs-server
+
diff --git a/nfs-server/manifest/service.yaml b/nfs-server/manifest/service.yaml
@@ -3,12 +3,13 @@ apiVersion: v1
 metadata:
   name: nfs-server
 spec:
+  type: ClusterIP
   ports:
     - name: nfs
       port: 2049
-    - name: mountd
-      port: 20048
-    - name: rpcbind
-      port: 111
+    # - name: mountd
+    #   port: 20048
+    # - name: rpcbind
+    #   port: 111
   selector:
     role: nfs-server
diff --git a/nfs-subdir-external-provisioner/README.md b/nfs-subdir-external-provisioner/README.md
@@ -2,31 +2,71 @@
 
 Deploy [kubernetes-sigs/nfs-subdir-external-provisioner](https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner#readme) to provide a RWX storage class based on an existing NFS server.
 
+## Requirements
+
+* An available NFS server
+
+> See [nfs-server](../nfs-server/README.md) for test purpose or [www.digitalocean.com - How To Set Up an NFS Mount on Ubuntu 22.04](https://www.digitalocean.com/community/tutorials/how-to-set-up-an-nfs-mount-on-ubuntu-22-04) for classic installation with `nfs-kernel-server`
+
 ## Parameters
 
-| Name          | Description          | Default                                 |
-|---------------|----------------------|-----------------------------------------|
-| NFS_SERVER_IP | IP of the NFS server | nfs-server.nfs-system.svc.cluster.local |
-| NFS_PATH      | Path to the export   | `/exports`                              |
+| Name            | Description          | Default                                       |
+| --------------- | -------------------- | --------------------------------------------- |
+| `NFS_SERVER_IP` | IP of the NFS server | `nfs-server.nfs-system.svc.cluster.local` (1) |
+| `NFS_PATH`      | Path to the export   | `/` (2)                                       |
+
+> (1) See [Troubleshooting](#troubleshooting)
+> (2) Use export with `fsid=0` in `/etc/exports` on NFS server
 
 ## Usage with Kubernetes
 
 * Deploy [nfs-server](../nfs-server/README.md) or adapt params to use an existing NFS server :
 
 ```bash
-export NFS_SERVER_IP=nfs-server.nfs-system.svc.cluster.local
-export NFS_PATH=/exports
+# Ex with nfs-server on vagrantbox-1
+# (https://github.com/mborne/k3s-deploy)
+export NFS_SERVER_IP=192.168.50.201
+export NFS_PATH=/
 ```
 
 * Read [k8s-install.sh](k8s-install.sh) and run :
 
 ```bash
 bash k8s-install.sh
+
+# check nfs-subdir-external-provisioner deployment status (READY 1/1)
+kubectl -n nfs-system get deployment
+
+# ensure that storageclass is defined
 kubectl get storageclass
 ```
 
+* See [nfs-demo](../nfs-demo/README.md) for an example using the "nfs" StorageClass
+
+## Troubleshooting
+
+### DNS resolution from node
+
+With `NFS_SERVER_IP=nfs-server.nfs-system.svc.cluster.local`, you may face **name resolution issues especially with kind** :
+
+> Unable to attach or mount volumes: unmounted volumes=[nfs-subdir-external-provisioner-root], unattached volumes=[kube-api-access-p2vmw nfs-subdir-external-provisioner-root]: timed out waiting for the condition
+
+Use the following workaround if nodes are not configured to invoke embedded DNS to resolve `.cluster.local` IP's :
+
+```bash
+export NFS_SERVER_IP=$(kubectl -n nfs-system get svc/nfs-server -o jsonpath='{.spec.clusterIP}')
+export NFS_PATH=/
+bash nfs-subdir-external-provisioner/k8s-install.sh
+```
+
+### IP stability
+
+Note that `NFS_SERVER_IP` will be present in PersistentVolume definitions and that the corresponding IP is resolved at mount time if a domain name is provided.
+
+Using something like `NFS_SERVER_IP=dns-server.priv.example.com` and ensuring stability NFS server IP is probably a good idea to avoid problems.
+
+
 ## Resources
 
 * [github.com/kubernetes-sigs - Kubernetes NFS Subdir External Provisioner](https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner#kubernetes-nfs-subdir-external-provisioner)
 * [docs.ovh.com - Configuring multi-attach persistent volumes with OVHcloud NAS-HA](https://docs.ovh.com/gb/en/kubernetes/configuring-multi-attach-persistent-volumes-with-ovhcloud-nas-ha/)
-
diff --git a/nfs-subdir-external-provisioner/k8s-install.sh b/nfs-subdir-external-provisioner/k8s-install.sh
@@ -13,12 +13,14 @@ helm repo add nfs-subdir-external-provisioner https://kubernetes-sigs.github.io/
 helm repo update
 
 # Create namespace nfs if not exists
-kubectl create namespace nfs --dry-run=client -o yaml | kubectl apply -f -
+kubectl create namespace nfs-system --dry-run=client -o yaml | kubectl apply -f -
 
 # Deploy nfs-subdir-external-provisioner with helm
 # see https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner/blob/master/charts/nfs-subdir-external-provisioner/README.md#readme
-helm -n nfs upgrade --install nfs-subdir-external-provisioner nfs-subdir-external-provisioner/nfs-subdir-external-provisioner \
+helm -n nfs-system upgrade --install nfs-subdir-external-provisioner nfs-subdir-external-provisioner/nfs-subdir-external-provisioner \
     --set nfs.server=${NFS_SERVER_IP} \
     --set nfs.path=${NFS_PATH} \
     --set storageClass.accessModes=ReadWriteMany \
-    --set storageClass.name=nfs-client
+    --set storageClass.name=nfs
+
+