Add a provision for handling image based bootstrap (#406) (#411)

* Add a provision for handling image based bootstrap

AHV has a limit of 32KB for cloud-init userdata. In Openshift, the
ignition can be rather large (a magnitude over the limit). In
order to support larger userdata files, we allow mounting the customization
as an image.

* Only set guestcustomization explicitly when bootstrap ref is secret

* Use lowercase for data_source_reference kind.

---------

Co-authored-by: Yanhua Li <yanhua.li@nutanix.com>

api/v1beta1/nutanixmachine_types.go

@@ -35,6 +35,16 @@ const (
 	// resources associated with NutanixMachine before removing it from the
 	// API Server.
 	NutanixMachineFinalizer = "nutanixmachine.infrastructure.cluster.x-k8s.io"
+
+	// NutanixMachineBootstrapRefKindSecret represents the Kind of Secret
+	// referenced by NutanixMachine's BootstrapRef.
+	NutanixMachineBootstrapRefKindSecret = "Secret"
+
+	// NutanixMachineBootstrapRefKindImage represents the Kind of Image
+	// referenced by NutanixMachine's BootstrapRef. If the BootstrapRef.Kind is set
+	// to Image, the NutanixMachine will be created with the image mounted
+	// as a CD-ROM.
+	NutanixMachineBootstrapRefKindImage = "Image"
 )
 
 // NutanixMachineSpec defines the desired state of NutanixMachine

controllers/nutanixmachine_controller.go

@@ -32,6 +32,7 @@ import (
 	apitypes "k8s.io/apimachinery/pkg/types"
 	kerrors "k8s.io/apimachinery/pkg/util/errors"
 	coreinformers "k8s.io/client-go/informers/core/v1"
+	"k8s.io/utils/ptr"
 	capiv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	capierrors "sigs.k8s.io/cluster-api/errors"
 	capiutil "sigs.k8s.io/cluster-api/util"
@@ -56,6 +57,9 @@ import (
 
 const (
 	projectKind = "project"
+
+	deviceTypeCDROM = "CDROM"
+	adapterTypeIDE  = "IDE"
 )
 
 var (
@@ -518,31 +522,6 @@ func (r *NutanixMachineReconciler) getOrCreateVM(rctx *nctx.MachineContext) (*nu
 		return nil, err
 	}
 
-	// Get Image UUID
-	imageUUID, err := GetImageUUID(ctx, nc, rctx.NutanixMachine.Spec.Image.Name, rctx.NutanixMachine.Spec.Image.UUID)
-	if err != nil {
-		errorMsg := fmt.Errorf("failed to get the image UUID to create the VM %s. %v", vmName, err)
-		rctx.SetFailureStatus(capierrors.CreateMachineError, errorMsg)
-		return nil, err
-	}
-
-	// Get the bootstrapData from the referenced secret
-	bootstrapData, err := r.getBootstrapData(rctx)
-	if err != nil {
-		log.Error(err, fmt.Sprintf("failed to get the bootstrap data to create the VM %s", vmName))
-		return nil, err
-	}
-	// Encode the bootstrapData by base64
-	bsdataEncoded := base64.StdEncoding.EncodeToString(bootstrapData)
-	log.V(1).Info(fmt.Sprintf("Retrieved the bootstrap data from secret %s (before encoding size: %d, encoded string size:%d)",
-		rctx.NutanixMachine.Spec.BootstrapRef.Name, len(bootstrapData), len(bsdataEncoded)))
-
-	// Generate metadata for the VM
-	vmUUID := uuid.New()
-	metadata := fmt.Sprintf("{\"hostname\": \"%s\", \"uuid\": \"%s\"}", rctx.Machine.Name, vmUUID)
-	// Encode the metadata by base64
-	metadataEncoded := base64.StdEncoding.EncodeToString([]byte(metadata))
-
 	vmInput := &nutanixClientV3.VMIntentInput{}
 	vmSpec := &nutanixClientV3.VM{Name: utils.StringPtr(vmName)}
 
@@ -556,19 +535,6 @@ func (r *NutanixMachineReconciler) getOrCreateVM(rctx *nctx.MachineContext) (*nu
 		}
 	}
 
-	// Create Disk Spec for systemdisk to be set later in VM Spec
-	diskSize := rctx.NutanixMachine.Spec.SystemDiskSize
-	diskSizeMib := GetMibValueOfQuantity(diskSize)
-	systemDisk, err := CreateSystemDiskSpec(imageUUID, diskSizeMib)
-	if err != nil {
-		errorMsg := fmt.Errorf("error occurred while creating system disk spec: %v", err)
-		rctx.SetFailureStatus(capierrors.CreateMachineError, errorMsg)
-		return nil, errorMsg
-	}
-	diskList := []*nutanixClientV3.VMDisk{
-		systemDisk,
-	}
-
 	// Set Categories to VM Sepc before creating VM
 	categories, err := GetCategoryVMSpec(ctx, nc, r.getMachineCategoryIdentifiers(rctx))
 	if err != nil {
@@ -598,8 +564,14 @@ func (r *NutanixMachineReconciler) getOrCreateVM(rctx *nctx.MachineContext) (*nu
 		return nil, err
 	}
 
-	memorySize := rctx.NutanixMachine.Spec.MemorySize
-	memorySizeMib := GetMibValueOfQuantity(memorySize)
+	diskList, err := getDiskList(rctx)
+	if err != nil {
+		errorMsg := fmt.Errorf("failed to get the disk list to create the VM %s. %v", vmName, err)
+		rctx.SetFailureStatus(capierrors.CreateMachineError, errorMsg)
+		return nil, err
+	}
+
+	memorySizeMib := GetMibValueOfQuantity(rctx.NutanixMachine.Spec.MemorySize)
 	vmSpec.Resources = &nutanixClientV3.VMResources{
 		PowerState:            utils.StringPtr("ON"),
 		HardwareClockTimezone: utils.StringPtr("UTC"),
@@ -609,19 +581,18 @@ func (r *NutanixMachineReconciler) getOrCreateVM(rctx *nctx.MachineContext) (*nu
 		NicList:               nicList,
 		DiskList:              diskList,
 		GpuList:               gpuList,
-		GuestCustomization: &nutanixClientV3.GuestCustomization{
-			IsOverridable: utils.BoolPtr(true),
-			CloudInit: &nutanixClientV3.GuestCustomizationCloudInit{
-				UserData: utils.StringPtr(bsdataEncoded),
-				MetaData: utils.StringPtr(metadataEncoded),
-			},
-		},
 	}
 	vmSpec.ClusterReference = &nutanixClientV3.Reference{
 		Kind: utils.StringPtr("cluster"),
 		UUID: utils.StringPtr(peUUID),
 	}
 
+	if err := r.addGuestCustomizationToVM(rctx, vmSpec); err != nil {
+		errorMsg := fmt.Errorf("error occurred while adding guest customization to vm spec: %v", err)
+		rctx.SetFailureStatus(capierrors.CreateMachineError, errorMsg)
+		return nil, err
+	}
+
 	// Set BootType in VM Spec before creating VM
 	err = r.addBootTypeToVM(rctx, vmSpec)
 	if err != nil {
@@ -685,6 +656,100 @@ func (r *NutanixMachineReconciler) getOrCreateVM(rctx *nctx.MachineContext) (*nu
 	return vm, nil
 }
 
+func (r *NutanixMachineReconciler) addGuestCustomizationToVM(rctx *nctx.MachineContext, vmSpec *nutanixClientV3.VM) error {
+	// Get the bootstrapData
+	bootstrapRef := rctx.NutanixMachine.Spec.BootstrapRef
+	if bootstrapRef.Kind == infrav1.NutanixMachineBootstrapRefKindSecret {
+		bootstrapData, err := r.getBootstrapData(rctx)
+		if err != nil {
+			return err
+		}
+
+		// Encode the bootstrapData by base64
+		bsdataEncoded := base64.StdEncoding.EncodeToString(bootstrapData)
+		metadata := fmt.Sprintf("{\"hostname\": \"%s\", \"uuid\": \"%s\"}", rctx.Machine.Name, uuid.New())
+		metadataEncoded := base64.StdEncoding.EncodeToString([]byte(metadata))
+
+		vmSpec.Resources.GuestCustomization = &nutanixClientV3.GuestCustomization{
+			IsOverridable: utils.BoolPtr(true),
+			CloudInit: &nutanixClientV3.GuestCustomizationCloudInit{
+				UserData: utils.StringPtr(bsdataEncoded),
+				MetaData: utils.StringPtr(metadataEncoded),
+			},
+		}
+	}
+
+	return nil
+}
+
+func getDiskList(rctx *nctx.MachineContext) ([]*nutanixClientV3.VMDisk, error) {
+	diskList := make([]*nutanixClientV3.VMDisk, 0)
+
+	systemDisk, err := getSystemDisk(rctx)
+	if err != nil {
+		return nil, err
+	}
+	diskList = append(diskList, systemDisk)
+
+	bootstrapRef := rctx.NutanixMachine.Spec.BootstrapRef
+	if bootstrapRef.Kind == infrav1.NutanixMachineBootstrapRefKindImage {
+		bootstrapDisk, err := getBootstrapDisk(rctx)
+		if err != nil {
+			return nil, err
+		}
+
+		diskList = append(diskList, bootstrapDisk)
+	}
+
+	return diskList, nil
+}
+
+func getSystemDisk(rctx *nctx.MachineContext) (*nutanixClientV3.VMDisk, error) {
+	nodeOSImageName := rctx.NutanixMachine.Spec.Image.Name
+	nodeOSImageUUID, err := GetImageUUID(rctx.Context, rctx.NutanixClient, nodeOSImageName, rctx.NutanixMachine.Spec.Image.UUID)
+	if err != nil {
+		errorMsg := fmt.Errorf("failed to get the image UUID for image named %q: %w", *nodeOSImageName, err)
+		rctx.SetFailureStatus(capierrors.CreateMachineError, errorMsg)
+		return nil, err
+	}
+
+	systemDiskSizeMib := GetMibValueOfQuantity(rctx.NutanixMachine.Spec.SystemDiskSize)
+	systemDisk, err := CreateSystemDiskSpec(nodeOSImageUUID, systemDiskSizeMib)
+	if err != nil {
+		errorMsg := fmt.Errorf("error occurred while creating system disk spec: %w", err)
+		rctx.SetFailureStatus(capierrors.CreateMachineError, errorMsg)
+		return nil, err
+	}
+
+	return systemDisk, nil
+}
+
+func getBootstrapDisk(rctx *nctx.MachineContext) (*nutanixClientV3.VMDisk, error) {
+	bootstrapImageName := rctx.NutanixMachine.Spec.BootstrapRef.Name
+	bootstrapImageUUID, err := GetImageUUID(rctx.Context, rctx.NutanixClient, &bootstrapImageName, nil)
+	if err != nil {
+		errorMsg := fmt.Errorf("failed to get the image UUID for image named %q: %w", bootstrapImageName, err)
+		rctx.SetFailureStatus(capierrors.CreateMachineError, errorMsg)
+		return nil, err
+	}
+
+	bootstrapDisk := &nutanixClientV3.VMDisk{
+		DeviceProperties: &nutanixClientV3.VMDiskDeviceProperties{
+			DeviceType: ptr.To(deviceTypeCDROM),
+			DiskAddress: &nutanixClientV3.DiskAddress{
+				AdapterType: ptr.To(adapterTypeIDE),
+				DeviceIndex: ptr.To(int64(0)),
+			},
+		},
+		DataSourceReference: &nutanixClientV3.Reference{
+			Kind: ptr.To(strings.ToLower(infrav1.NutanixMachineBootstrapRefKindImage)),
+			UUID: ptr.To(bootstrapImageUUID),
+		},
+	}
+
+	return bootstrapDisk, nil
+}
+
 // getBootstrapData returns the Bootstrap data from the ref secret
 func (r *NutanixMachineReconciler) getBootstrapData(rctx *nctx.MachineContext) ([]byte, error) {
 	if rctx.NutanixMachine.Spec.BootstrapRef == nil {

go.mod

@@ -4,6 +4,7 @@ go 1.21
 
 require (
 	github.com/blang/semver v3.5.1+incompatible
+	github.com/blang/semver/v4 v4.0.0
 	github.com/google/uuid v1.3.0
 	github.com/nutanix-cloud-native/prism-go-client v0.3.4
 	github.com/onsi/ginkgo/v2 v2.6.0
@@ -15,7 +16,7 @@ require (
 	k8s.io/apiextensions-apiserver v0.25.2
 	k8s.io/apimachinery v0.25.2
 	k8s.io/client-go v0.25.2
-	k8s.io/utils v0.0.0-20220823124924-e9cbc92d1a73
+	k8s.io/utils v0.0.0-20240310230437-4693a0247e57
 	sigs.k8s.io/cluster-api v1.3.5
 	sigs.k8s.io/cluster-api/test v1.3.5
 	sigs.k8s.io/controller-runtime v0.13.1
@@ -43,7 +44,6 @@ require (
 	github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220418222510-f25a4f6275ed // indirect
 	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/coredns/caddy v1.1.0 // indirect
 	github.com/coredns/corefile-migration v1.0.20 // indirect

go.sum

@@ -1357,8 +1357,8 @@ k8s.io/kube-openapi v0.0.0-20220803164354-a70c9af30aea h1:3QOH5+2fGsY8e1qf+GIFpg
 k8s.io/kube-openapi v0.0.0-20220803164354-a70c9af30aea/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU=
 k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-k8s.io/utils v0.0.0-20220823124924-e9cbc92d1a73 h1:H9TCJUUx+2VA0ZiD9lvtaX8fthFsMoD+Izn93E/hm8U=
-k8s.io/utils v0.0.0-20220823124924-e9cbc92d1a73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+k8s.io/utils v0.0.0-20240310230437-4693a0247e57 h1:gbqbevonBh57eILzModw6mrkbwM0gQBEuevE/AaBsHY=
+k8s.io/utils v0.0.0-20240310230437-4693a0247e57/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=