From 98239ba02cb2c9b1f9a4864b7f9eea3da59f9034 Mon Sep 17 00:00:00 2001 From: Sid Shukla Date: Tue, 16 Apr 2024 13:13:57 -0400 Subject: [PATCH] Add a provision for handling image based bootstrap (#406) (#411) * Add a provision for handling image based bootstrap AHV has a limit of 32KB for cloud-init userdata. In Openshift, the ignition can be rather large (a magnitude over the limit). In order to support larger userdata files, we allow mounting the customization as an image. * Only set guestcustomization explicitly when bootstrap ref is secret * Use lowercase for data_source_reference kind. --------- Co-authored-by: Yanhua Li --- api/v1beta1/nutanixmachine_types.go | 10 ++ controllers/nutanixmachine_controller.go | 159 ++++++++++++++++------- go.mod | 4 +- go.sum | 4 +- 4 files changed, 126 insertions(+), 51 deletions(-) diff --git a/api/v1beta1/nutanixmachine_types.go b/api/v1beta1/nutanixmachine_types.go index 170011f60d..97a5c7cd5a 100644 --- a/api/v1beta1/nutanixmachine_types.go +++ b/api/v1beta1/nutanixmachine_types.go @@ -35,6 +35,16 @@ const ( // resources associated with NutanixMachine before removing it from the // API Server. NutanixMachineFinalizer = "nutanixmachine.infrastructure.cluster.x-k8s.io" + + // NutanixMachineBootstrapRefKindSecret represents the Kind of Secret + // referenced by NutanixMachine's BootstrapRef. + NutanixMachineBootstrapRefKindSecret = "Secret" + + // NutanixMachineBootstrapRefKindImage represents the Kind of Image + // referenced by NutanixMachine's BootstrapRef. If the BootstrapRef.Kind is set + // to Image, the NutanixMachine will be created with the image mounted + // as a CD-ROM. + NutanixMachineBootstrapRefKindImage = "Image" ) // NutanixMachineSpec defines the desired state of NutanixMachine diff --git a/controllers/nutanixmachine_controller.go b/controllers/nutanixmachine_controller.go index 1b35b61204..f9db7dde56 100644 --- a/controllers/nutanixmachine_controller.go +++ b/controllers/nutanixmachine_controller.go @@ -32,6 +32,7 @@ import ( apitypes "k8s.io/apimachinery/pkg/types" kerrors "k8s.io/apimachinery/pkg/util/errors" coreinformers "k8s.io/client-go/informers/core/v1" + "k8s.io/utils/ptr" capiv1 "sigs.k8s.io/cluster-api/api/v1beta1" capierrors "sigs.k8s.io/cluster-api/errors" capiutil "sigs.k8s.io/cluster-api/util" @@ -56,6 +57,9 @@ import ( const ( projectKind = "project" + + deviceTypeCDROM = "CDROM" + adapterTypeIDE = "IDE" ) var ( @@ -518,31 +522,6 @@ func (r *NutanixMachineReconciler) getOrCreateVM(rctx *nctx.MachineContext) (*nu return nil, err } - // Get Image UUID - imageUUID, err := GetImageUUID(ctx, nc, rctx.NutanixMachine.Spec.Image.Name, rctx.NutanixMachine.Spec.Image.UUID) - if err != nil { - errorMsg := fmt.Errorf("failed to get the image UUID to create the VM %s. %v", vmName, err) - rctx.SetFailureStatus(capierrors.CreateMachineError, errorMsg) - return nil, err - } - - // Get the bootstrapData from the referenced secret - bootstrapData, err := r.getBootstrapData(rctx) - if err != nil { - log.Error(err, fmt.Sprintf("failed to get the bootstrap data to create the VM %s", vmName)) - return nil, err - } - // Encode the bootstrapData by base64 - bsdataEncoded := base64.StdEncoding.EncodeToString(bootstrapData) - log.V(1).Info(fmt.Sprintf("Retrieved the bootstrap data from secret %s (before encoding size: %d, encoded string size:%d)", - rctx.NutanixMachine.Spec.BootstrapRef.Name, len(bootstrapData), len(bsdataEncoded))) - - // Generate metadata for the VM - vmUUID := uuid.New() - metadata := fmt.Sprintf("{\"hostname\": \"%s\", \"uuid\": \"%s\"}", rctx.Machine.Name, vmUUID) - // Encode the metadata by base64 - metadataEncoded := base64.StdEncoding.EncodeToString([]byte(metadata)) - vmInput := &nutanixClientV3.VMIntentInput{} vmSpec := &nutanixClientV3.VM{Name: utils.StringPtr(vmName)} @@ -556,19 +535,6 @@ func (r *NutanixMachineReconciler) getOrCreateVM(rctx *nctx.MachineContext) (*nu } } - // Create Disk Spec for systemdisk to be set later in VM Spec - diskSize := rctx.NutanixMachine.Spec.SystemDiskSize - diskSizeMib := GetMibValueOfQuantity(diskSize) - systemDisk, err := CreateSystemDiskSpec(imageUUID, diskSizeMib) - if err != nil { - errorMsg := fmt.Errorf("error occurred while creating system disk spec: %v", err) - rctx.SetFailureStatus(capierrors.CreateMachineError, errorMsg) - return nil, errorMsg - } - diskList := []*nutanixClientV3.VMDisk{ - systemDisk, - } - // Set Categories to VM Sepc before creating VM categories, err := GetCategoryVMSpec(ctx, nc, r.getMachineCategoryIdentifiers(rctx)) if err != nil { @@ -598,8 +564,14 @@ func (r *NutanixMachineReconciler) getOrCreateVM(rctx *nctx.MachineContext) (*nu return nil, err } - memorySize := rctx.NutanixMachine.Spec.MemorySize - memorySizeMib := GetMibValueOfQuantity(memorySize) + diskList, err := getDiskList(rctx) + if err != nil { + errorMsg := fmt.Errorf("failed to get the disk list to create the VM %s. %v", vmName, err) + rctx.SetFailureStatus(capierrors.CreateMachineError, errorMsg) + return nil, err + } + + memorySizeMib := GetMibValueOfQuantity(rctx.NutanixMachine.Spec.MemorySize) vmSpec.Resources = &nutanixClientV3.VMResources{ PowerState: utils.StringPtr("ON"), HardwareClockTimezone: utils.StringPtr("UTC"), @@ -609,19 +581,18 @@ func (r *NutanixMachineReconciler) getOrCreateVM(rctx *nctx.MachineContext) (*nu NicList: nicList, DiskList: diskList, GpuList: gpuList, - GuestCustomization: &nutanixClientV3.GuestCustomization{ - IsOverridable: utils.BoolPtr(true), - CloudInit: &nutanixClientV3.GuestCustomizationCloudInit{ - UserData: utils.StringPtr(bsdataEncoded), - MetaData: utils.StringPtr(metadataEncoded), - }, - }, } vmSpec.ClusterReference = &nutanixClientV3.Reference{ Kind: utils.StringPtr("cluster"), UUID: utils.StringPtr(peUUID), } + if err := r.addGuestCustomizationToVM(rctx, vmSpec); err != nil { + errorMsg := fmt.Errorf("error occurred while adding guest customization to vm spec: %v", err) + rctx.SetFailureStatus(capierrors.CreateMachineError, errorMsg) + return nil, err + } + // Set BootType in VM Spec before creating VM err = r.addBootTypeToVM(rctx, vmSpec) if err != nil { @@ -685,6 +656,100 @@ func (r *NutanixMachineReconciler) getOrCreateVM(rctx *nctx.MachineContext) (*nu return vm, nil } +func (r *NutanixMachineReconciler) addGuestCustomizationToVM(rctx *nctx.MachineContext, vmSpec *nutanixClientV3.VM) error { + // Get the bootstrapData + bootstrapRef := rctx.NutanixMachine.Spec.BootstrapRef + if bootstrapRef.Kind == infrav1.NutanixMachineBootstrapRefKindSecret { + bootstrapData, err := r.getBootstrapData(rctx) + if err != nil { + return err + } + + // Encode the bootstrapData by base64 + bsdataEncoded := base64.StdEncoding.EncodeToString(bootstrapData) + metadata := fmt.Sprintf("{\"hostname\": \"%s\", \"uuid\": \"%s\"}", rctx.Machine.Name, uuid.New()) + metadataEncoded := base64.StdEncoding.EncodeToString([]byte(metadata)) + + vmSpec.Resources.GuestCustomization = &nutanixClientV3.GuestCustomization{ + IsOverridable: utils.BoolPtr(true), + CloudInit: &nutanixClientV3.GuestCustomizationCloudInit{ + UserData: utils.StringPtr(bsdataEncoded), + MetaData: utils.StringPtr(metadataEncoded), + }, + } + } + + return nil +} + +func getDiskList(rctx *nctx.MachineContext) ([]*nutanixClientV3.VMDisk, error) { + diskList := make([]*nutanixClientV3.VMDisk, 0) + + systemDisk, err := getSystemDisk(rctx) + if err != nil { + return nil, err + } + diskList = append(diskList, systemDisk) + + bootstrapRef := rctx.NutanixMachine.Spec.BootstrapRef + if bootstrapRef.Kind == infrav1.NutanixMachineBootstrapRefKindImage { + bootstrapDisk, err := getBootstrapDisk(rctx) + if err != nil { + return nil, err + } + + diskList = append(diskList, bootstrapDisk) + } + + return diskList, nil +} + +func getSystemDisk(rctx *nctx.MachineContext) (*nutanixClientV3.VMDisk, error) { + nodeOSImageName := rctx.NutanixMachine.Spec.Image.Name + nodeOSImageUUID, err := GetImageUUID(rctx.Context, rctx.NutanixClient, nodeOSImageName, rctx.NutanixMachine.Spec.Image.UUID) + if err != nil { + errorMsg := fmt.Errorf("failed to get the image UUID for image named %q: %w", *nodeOSImageName, err) + rctx.SetFailureStatus(capierrors.CreateMachineError, errorMsg) + return nil, err + } + + systemDiskSizeMib := GetMibValueOfQuantity(rctx.NutanixMachine.Spec.SystemDiskSize) + systemDisk, err := CreateSystemDiskSpec(nodeOSImageUUID, systemDiskSizeMib) + if err != nil { + errorMsg := fmt.Errorf("error occurred while creating system disk spec: %w", err) + rctx.SetFailureStatus(capierrors.CreateMachineError, errorMsg) + return nil, err + } + + return systemDisk, nil +} + +func getBootstrapDisk(rctx *nctx.MachineContext) (*nutanixClientV3.VMDisk, error) { + bootstrapImageName := rctx.NutanixMachine.Spec.BootstrapRef.Name + bootstrapImageUUID, err := GetImageUUID(rctx.Context, rctx.NutanixClient, &bootstrapImageName, nil) + if err != nil { + errorMsg := fmt.Errorf("failed to get the image UUID for image named %q: %w", bootstrapImageName, err) + rctx.SetFailureStatus(capierrors.CreateMachineError, errorMsg) + return nil, err + } + + bootstrapDisk := &nutanixClientV3.VMDisk{ + DeviceProperties: &nutanixClientV3.VMDiskDeviceProperties{ + DeviceType: ptr.To(deviceTypeCDROM), + DiskAddress: &nutanixClientV3.DiskAddress{ + AdapterType: ptr.To(adapterTypeIDE), + DeviceIndex: ptr.To(int64(0)), + }, + }, + DataSourceReference: &nutanixClientV3.Reference{ + Kind: ptr.To(strings.ToLower(infrav1.NutanixMachineBootstrapRefKindImage)), + UUID: ptr.To(bootstrapImageUUID), + }, + } + + return bootstrapDisk, nil +} + // getBootstrapData returns the Bootstrap data from the ref secret func (r *NutanixMachineReconciler) getBootstrapData(rctx *nctx.MachineContext) ([]byte, error) { if rctx.NutanixMachine.Spec.BootstrapRef == nil { diff --git a/go.mod b/go.mod index f834186f83..b310d0bff9 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.21 require ( github.com/blang/semver v3.5.1+incompatible + github.com/blang/semver/v4 v4.0.0 github.com/google/uuid v1.3.0 github.com/nutanix-cloud-native/prism-go-client v0.3.4 github.com/onsi/ginkgo/v2 v2.6.0 @@ -15,7 +16,7 @@ require ( k8s.io/apiextensions-apiserver v0.25.2 k8s.io/apimachinery v0.25.2 k8s.io/client-go v0.25.2 - k8s.io/utils v0.0.0-20220823124924-e9cbc92d1a73 + k8s.io/utils v0.0.0-20240310230437-4693a0247e57 sigs.k8s.io/cluster-api v1.3.5 sigs.k8s.io/cluster-api/test v1.3.5 sigs.k8s.io/controller-runtime v0.13.1 @@ -43,7 +44,6 @@ require ( github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220418222510-f25a4f6275ed // indirect github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/coredns/caddy v1.1.0 // indirect github.com/coredns/corefile-migration v1.0.20 // indirect diff --git a/go.sum b/go.sum index b6e4039122..fd1b6146c9 100644 --- a/go.sum +++ b/go.sum @@ -1357,8 +1357,8 @@ k8s.io/kube-openapi v0.0.0-20220803164354-a70c9af30aea h1:3QOH5+2fGsY8e1qf+GIFpg k8s.io/kube-openapi v0.0.0-20220803164354-a70c9af30aea/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20220823124924-e9cbc92d1a73 h1:H9TCJUUx+2VA0ZiD9lvtaX8fthFsMoD+Izn93E/hm8U= -k8s.io/utils v0.0.0-20220823124924-e9cbc92d1a73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20240310230437-4693a0247e57 h1:gbqbevonBh57eILzModw6mrkbwM0gQBEuevE/AaBsHY= +k8s.io/utils v0.0.0-20240310230437-4693a0247e57/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=