Hybrid EXT4-zvol vault for kubevirt

Re-use existing ZFS vault path
Split clustered-storage into etc and vault

Signed-off-by: Andrew Durbin <andrewd@zededa.com>

pkg/kube/cluster-init.sh

@@ -54,6 +54,28 @@ wait_for_default_route() {
   return 1
 }
 
+wait_for_vault() {
+        logmsg "Starting wait for Vault"
+        pillarRootfs=/hostfs/containers/services/pillar/rootfs
+        while ! LD_LIBRARY_PATH=${pillarRootfs}/usr/lib/ ${pillarRootfs}/opt/zededa/bin/vaultmgr waitUnsealed;
+        do
+               sleep 1
+        done
+        logmsg "Vault ready"
+}
+
+mount_etcd_vol() {
+        # NOTE: We only support zfs storage in production systems because data is persisted on zvol.
+        # This is formatted in vaultmgr
+        logmsg "Wait for persist/etcd-storage zvol"
+        while [ ! -b /dev/zvol/persist/etcd-storage ];
+        do
+                sleep 1
+        done
+        mount /dev/zvol/persist/etcd-storage /var/lib  ## This is where we persist the cluster components (etcd)
+        logmsg "persist/etcd-storage available"
+}
+
 #Prereqs
 setup_prereqs () {
         modprobe tun
@@ -68,6 +90,8 @@ setup_prereqs () {
         #Check network and default routes are up
         wait_for_default_route
         check_network_connection
+        wait_for_vault
+        mount_etcd_vol
 }
 
 check_start_containerd() {
@@ -106,15 +130,6 @@ trigger_k3s_selfextraction() {
         /usr/bin/k3s check-config >> $INSTALL_LOG 2>&1
 }
 
-# NOTE: We only support zfs storage in production systems because data is persisted on zvol.
-# If ZFS is not available we still go ahead and provide the service but the data is lost on reboot
-# because /var/lib will be on overlayfs. The only reason to allow that is to provide a quick debugging env for developers.
-if [ -b /dev/zvol/persist/clustered-storage ]; then
-        mount /dev/zvol/persist/clustered-storage /var/lib  ## This is where we persist the cluster components (k3s containers)
-        logmsg "Using ZFS persistent storage"
-else
-        logmsg "WARNING: Using overlayfs non-persistent storage"
-fi
 
 #Make sure all prereqs are set after /var/lib is mounted to get logging info
 setup_prereqs

pkg/kube/config.yaml

@@ -8,3 +8,5 @@ log: "/var/lib/rancher/k3s/k3s.log"
 debug: true
 etcd-expose-metrics: true
 container-runtime-endpoint: "/run/containerd-user/containerd.sock"
+etcd-arg:
+  - "quota-backend-bytes=8589934592"

pkg/mkimage-raw-efi/install

@@ -151,36 +151,12 @@ prepare_mounts_and_zfs_pool() {
   chroot /root zfs create -o refreservation="$(chroot /root zfs get -o value -Hp available persist | awk '{ print ($1/1024/1024)/5 }')"m persist/reserved
   chroot /root zfs set mountpoint="/persist" persist
   chroot /root zfs set primarycache=metadata persist
-  chroot /root zfs create -o mountpoint="/persist/containerd/io.containerd.snapshotter.v1.zfs" persist/snapshots
   # we need this mount to propagate persist from /root/persist after call to zfs command in changed root
   [ -e /persist ] || mkdir /persist && mount --rbind --make-rslave /root/persist /persist
 
-  # If clustered storage, then create volume of (available space - RESERVE_EVE_STORAGE_SIZEGB) and format it with ext4
-  if [ "$2" = true ]; then
-          logmsg "Creating clustered-storage"
-          # When k3s cluster components start they will mount this volume
-          # If zfs-udev module is available at this point we will use path to the volume
-          # else we will format the raw device /dev/zd0 which is guaranteed to exist since its the first
-          # zfs volume being created on this device.
-          availableGB="$(chroot /root zfs get -o value -Hp available persist | awk '{ print int($1/1024/1024/1024) }')"
-          reservedGB=$3
-          logmsg "AvailableGB : $availableGB  ReservedGB : $reservedGB"
-          SIZEGB=$((availableGB - reservedGB))
-          if [ $SIZEGB -gt 0 ]; then
-             SIZEGB=$SIZEGB"G"
-             logmsg "Creating clustered-storage volume of size $SIZEGB"
-             /sbin/modprobe  zfs-udev
-             chroot /root zfs create -V $SIZEGB -o volblocksize=16k persist/clustered-storage
-             if [ -b /dev/zvol/persist/clustered-storage ]; then
-                logmsg "Formatting clustered-storage to ext4"
-                /sbin/mkfs -t ext4 /dev/zvol/persist/clustered-storage
-             elif [ -b /dev/zd0 ]; then
-                logmsg "Formatting /dev/zd0 to ext4"
-                /sbin/mkfs -t ext4 /dev/zd0
-             else
-                logmsg "FATAL: clustered-storage volume creation failed."
-             fi
-          fi
+  if [ "$2" = false ]; then
+    # kubevirt persist/vault is an ext4 zvol
+    chroot /root zfs create -o mountpoint="/persist/containerd/io.containerd.snapshotter.v1.zfs" persist/snapshots
   fi
 }
 

pkg/pillar/cmd/vaultmgr/vaultmgr.go

@@ -188,7 +188,7 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar
 
 	// if any args defined, will run command inline and return
 	if len(ctx.args) > 0 {
-		return runInline(ctx.args[0], ctx.args[1:])
+		return runInline(ps, ctx.args[0], ctx.args[1:])
 	}
 
 	log.Functionf("Starting %s\n", agentName)
@@ -309,13 +309,18 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar
 	}
 }
 
-func runInline(command string, _ []string) int {
+func runInline(ps *pubsub.PubSub, command string, _ []string) int {
 	switch command {
 	case "setupDeprecatedVaults":
 		if err := handler.SetupDeprecatedVaults(); err != nil {
 			log.Error(err)
 			return 1
 		}
+	case "waitUnsealed":
+		if err := utils.WaitForVault(ps, log, agentName, warningTime, errorTime); err != nil {
+			log.Fatal(err)
+			return 1
+		}
 	default:
 		log.Errorf("Unknown command %s", command)
 		return 1

pkg/pillar/cmd/volumemgr/volumemgr.go

@@ -23,6 +23,7 @@ import (
 	"github.com/lf-edge/eve/pkg/pillar/utils"
 	"github.com/lf-edge/eve/pkg/pillar/vault"
 	"github.com/lf-edge/eve/pkg/pillar/worker"
+	"github.com/lf-edge/eve/pkg/pillar/zfs"
 	"github.com/sirupsen/logrus"
 )
 
@@ -237,12 +238,16 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar
 	log.Functionf("user containerd ready")
 
 	if ctx.persistType == types.PersistZFS {
-		// create datasets for volumes
-		initializeDatasets()
-		// Iterate over volume datasets and prepares map of
-		// volume's content format with the volume key
-		populateExistingVolumesFormatDatasets(&ctx, types.VolumeEncryptedZFSDataset)
-		populateExistingVolumesFormatDatasets(&ctx, types.VolumeClearZFSDataset)
+		if isZvol, _ := zfs.IsDatasetTypeZvol(types.SealedDataset); isZvol {
+			initializeDirs()
+		} else {
+			// create datasets for volumes
+			initializeDatasets()
+			// Iterate over volume datasets and prepares map of
+			// volume's content format with the volume key
+			populateExistingVolumesFormatDatasets(&ctx, types.VolumeEncryptedZFSDataset)
+			populateExistingVolumesFormatDatasets(&ctx, types.VolumeClearZFSDataset)
+		}
 	} else {
 		// create the directories
 		initializeDirs()

pkg/pillar/types/locationconsts.go

@@ -126,4 +126,6 @@ var (
 	VolumeClearZFSDataset = ClearDataset + "/volumes"
 	//VolumeEncryptedZFSDataset - dataset to create volumes with encryption
 	VolumeEncryptedZFSDataset = SealedDataset + "/volumes"
+	// EtcdZvol - zvol encrypted for etcd storage
+	EtcdZvol = PersistDataset + "/etcd-storage"
 )