add aws capi cases

pkg/capi/aws.go

@@ -0,0 +1,246 @@
+package capi
+
+import (
+	"context"
+	"fmt"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	configv1 "github.com/openshift/api/config/v1"
+	mapiv1 "github.com/openshift/api/machine/v1beta1"
+	"github.com/openshift/cluster-api-actuator-pkg/pkg/framework"
+	"github.com/openshift/cluster-api-actuator-pkg/pkg/framework/gatherer"
+	capiinfrastructurev1beta2resourcebuilder "github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/cluster-api/infrastructure/v1beta2"
+	corev1 "k8s.io/api/core/v1"
+	awsv1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2"
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+
+	"sigs.k8s.io/controller-runtime/pkg/client/config"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	yaml "sigs.k8s.io/yaml"
+)
+
+const (
+	awsMachineTemplateName = "aws-machine-template"
+	infrastructureName     = "cluster"
+	infraAPIVersion        = "infrastructure.cluster.x-k8s.io/v1beta1"
+)
+
+var _ = Describe("Cluster API AWS MachineSet", framework.LabelCAPI, framework.LabelDisruptive, Ordered, func() {
+	var (
+		cl                      client.Client
+		ctx                     = context.Background()
+		platform                configv1.PlatformType
+		clusterName             string
+		oc                      *gatherer.CLI
+		awsMachineTemplate      *awsv1.AWSMachineTemplate
+		machineSetParams        framework.CAPIMachineSetParams
+		machineSet              *clusterv1.MachineSet
+		mapiDefaultProviderSpec *mapiv1.AWSMachineProviderConfig
+		err                     error
+	)
+
+	BeforeAll(func() {
+		cfg, err := config.GetConfig()
+		Expect(err).ToNot(HaveOccurred(), "Failed to GetConfig")
+
+		cl, err = client.New(cfg, client.Options{})
+		Expect(err).ToNot(HaveOccurred(), "Failed to create Kubernetes client for test")
+
+		infra := &configv1.Infrastructure{}
+		infraName := client.ObjectKey{
+			Name: infrastructureName,
+		}
+		Expect(cl.Get(ctx, infraName, infra)).To(Succeed(), "Failed to get cluster infrastructure object")
+		Expect(infra.Status.PlatformStatus).ToNot(BeNil(), "expected the infrastructure Status.PlatformStatus to not be nil")
+		clusterName = infra.Status.InfrastructureName
+		platform = infra.Status.PlatformStatus.Type
+		if platform != configv1.AWSPlatformType {
+			Skip("Skipping AWS E2E tests")
+		}
+		oc, err = framework.NewCLI()
+		Expect(err).ToNot(HaveOccurred(), "Failed to new CLI")
+		framework.SkipIfNotTechPreviewNoUpgrade(oc, cl)
+		_, mapiDefaultProviderSpec = getDefaultAWSMAPIProviderSpec(cl)
+		machineSetParams = framework.NewCAPIMachineSetParams(
+			"aws-machineset",
+			clusterName,
+			mapiDefaultProviderSpec.Placement.AvailabilityZone,
+			1,
+			corev1.ObjectReference{
+				Kind:       "AWSMachineTemplate",
+				APIVersion: infraAPIVersion,
+				Name:       awsMachineTemplateName,
+			},
+		)
+		framework.CreateCoreCluster(ctx, cl, clusterName, "AWSCluster")
+	})
+
+	AfterEach(func() {
+		framework.DeleteCAPIMachineSets(ctx, cl, machineSet)
+		framework.WaitForCAPIMachineSetsDeleted(ctx, cl, machineSet)
+		framework.DeleteObjects(ctx, cl, awsMachineTemplate)
+	})
+
+	//huliu-OCP-51071 - [CAPI] Create machineset with CAPI on aws
+	It("should be able to run a machine with a default provider spec", func() {
+		awsMachineTemplate = newAWSMachineTemplate(mapiDefaultProviderSpec)
+		if err = cl.Create(ctx, awsMachineTemplate); err != nil {
+			Expect(err).ToNot(HaveOccurred(), "Failed to create awsmachinetemplate")
+		}
+		machineSetParams = framework.UpdateCAPIMachineSetName("aws-machineset-51071", machineSetParams)
+		machineSet, err = framework.CreateCAPIMachineSet(ctx, cl, machineSetParams)
+		Expect(err).ToNot(HaveOccurred(), "Failed to create CAPI machineset")
+		framework.WaitForCAPIMachinesRunning(ctx, cl, machineSet.Name)
+	})
+
+	//huliu-OCP-75395 - [CAPI] AWS Placement group support.
+	It("should be able to run a machine with cluster placement group", func() {
+		awsClient := framework.NewAwsClient(framework.GetCredentialsFromCluster(oc))
+		placementGroupName := clusterName + "pgcluster"
+		placementGroupID, err := awsClient.CreatePlacementGroup(placementGroupName, "cluster")
+		Expect(err).ToNot(HaveOccurred(), "Failed to create placementgroup")
+		Expect(placementGroupID).ToNot(Equal(""), "expected the placementGroupID to not be empty string")
+		DeferCleanup(func() {
+			_, err = awsClient.DeletePlacementGroup(placementGroupName)
+			Expect(err).ToNot(HaveOccurred(), "Failed to delete placementgroup")
+		})
+
+		awsMachineTemplate = newAWSMachineTemplate(mapiDefaultProviderSpec)
+		awsMachineTemplate.Spec.Template.Spec.PlacementGroupName = placementGroupName
+		if err = cl.Create(ctx, awsMachineTemplate); err != nil {
+			Expect(err).ToNot(HaveOccurred(), "Failed to create awsmachinetemplate")
+		}
+		machineSetParams = framework.UpdateCAPIMachineSetName("aws-machineset-75395", machineSetParams)
+		machineSet, err = framework.CreateCAPIMachineSet(ctx, cl, machineSetParams)
+		Expect(err).ToNot(HaveOccurred(), "Failed to create CAPI machineset")
+		framework.WaitForCAPIMachinesRunning(ctx, cl, machineSet.Name)
+	})
+
+	//huliu-OCP-75396 - [CAPI] Creating machines using KMS keys from AWS.
+	It("should be able to run a machine using KMS keys", func() {
+		awsMachineTemplate = newAWSMachineTemplate(mapiDefaultProviderSpec)
+		region := mapiDefaultProviderSpec.Placement.Region
+		if region != "us-east-1" && region != "us-east-2" {
+			Skip("Region is " + region + ", skip this test scenario because we only created kms key in us-east-1/us-east-2 region")
+		}
+		var key string
+		switch region {
+		case "us-east-1":
+			key = "arn:aws:kms:us-east-1:301721915996:key/c471ec83-cfaf-41a2-9241-d9e99c4da344"
+		case "us-east-2":
+			key = "arn:aws:kms:us-east-2:301721915996:key/c228ef83-df2c-4151-84c4-d9f39f39a972"
+		}
+		awskmsClient := framework.NewAwsKmsClient(framework.GetCredentialsFromCluster(oc))
+		_, err = awskmsClient.DescribeKeyByID(key)
+		if err != nil {
+			Skip(fmt.Sprintf("Skip because cannot get the key %v", err))
+		}
+		encryptBool := true
+		awsMachineTemplate.Spec.Template.Spec.NonRootVolumes = []awsv1.Volume{
+			{
+				DeviceName:    "/dev/xvda",
+				Size:          140,
+				Type:          awsv1.VolumeTypeIO1,
+				IOPS:          5000,
+				Encrypted:     &encryptBool,
+				EncryptionKey: key,
+			},
+		}
+		Expect(cl.Create(ctx, awsMachineTemplate)).To(Succeed(), "Failed to create awsmachinetemplate")
+		machineSetParams = framework.UpdateCAPIMachineSetName("aws-machineset-75396", machineSetParams)
+		machineSet, err = framework.CreateCAPIMachineSet(ctx, cl, machineSetParams)
+		Expect(err).ToNot(HaveOccurred(), "Failed to create CAPI machineset")
+		framework.WaitForCAPIMachinesRunning(ctx, cl, machineSet.Name)
+	})
+})
+
+func getDefaultAWSMAPIProviderSpec(cl client.Client) (*mapiv1.MachineSet, *mapiv1.AWSMachineProviderConfig) {
+	machineSetList := &mapiv1.MachineSetList{}
+
+	Eventually(func() error {
+		return cl.List(framework.GetContext(), machineSetList, client.InNamespace(framework.MachineAPINamespace))
+	}, framework.WaitShort, framework.RetryShort).Should(Succeed(), "it should be able to list the MAPI machinesets")
+	Expect(machineSetList.Items).ToNot(HaveLen(0), "expected the MAPI machinesets to be present")
+
+	machineSet := &machineSetList.Items[0]
+	Expect(machineSet.Spec.Template.Spec.ProviderSpec.Value).ToNot(BeNil(), "expected the MAPI machinesets ProviderSpec value to not be nil")
+
+	providerSpec := &mapiv1.AWSMachineProviderConfig{}
+	Expect(yaml.Unmarshal(machineSet.Spec.Template.Spec.ProviderSpec.Value.Raw, providerSpec)).To(Succeed(), "it should be able to unmarshal the raw yaml into providerSpec")
+
+	return machineSet, providerSpec
+}
+
+func newAWSMachineTemplate(mapiProviderSpec *mapiv1.AWSMachineProviderConfig) *awsv1.AWSMachineTemplate {
+	By("Creating AWS machine template")
+
+	Expect(mapiProviderSpec).ToNot(BeNil(), "expected the mapi ProviderSpec to not be nil")
+	Expect(mapiProviderSpec.IAMInstanceProfile).ToNot(BeNil(), "expected the mapi IAMInstanceProfile to not be nil")
+	Expect(mapiProviderSpec.IAMInstanceProfile.ID).ToNot(BeNil(), "expected the mapi IAMInstanceProfile.ID to not be nil")
+	Expect(mapiProviderSpec.InstanceType).ToNot(BeEmpty(), "expected the mapi InstanceType to not be empty")
+	Expect(mapiProviderSpec.Placement.AvailabilityZone).ToNot(BeEmpty(), "expected the mapi Placement.AvailabilityZone to not be empty")
+	Expect(mapiProviderSpec.AMI.ID).ToNot(BeNil(), "expected the mapi AMI.ID to not be nil")
+	Expect(mapiProviderSpec.SecurityGroups).ToNot(HaveLen(0), "expected the mapi SecurityGroups to be present")
+	Expect(mapiProviderSpec.SecurityGroups[0].Filters).ToNot(HaveLen(0), "expected the mapi SecurityGroups[0].Filters to be present")
+	Expect(mapiProviderSpec.SecurityGroups[0].Filters[0].Values).ToNot(HaveLen(0), "expected the mapi SecurityGroups[0].Filters[0].Values to be present")
+
+	var subnet awsv1.AWSResourceReference
+
+	if len(mapiProviderSpec.Subnet.Filters) == 0 {
+		subnet = awsv1.AWSResourceReference{
+			ID: mapiProviderSpec.Subnet.ID,
+		}
+	} else {
+		subnet = awsv1.AWSResourceReference{
+			Filters: []awsv1.Filter{
+				{
+					Name:   "tag:Name",
+					Values: mapiProviderSpec.Subnet.Filters[0].Values,
+				},
+			},
+		}
+	}
+
+	uncompressedUserData := true
+	ami := awsv1.AMIReference{
+		ID: mapiProviderSpec.AMI.ID,
+	}
+	ignition := &awsv1.Ignition{
+		Version:     "3.4",
+		StorageType: awsv1.IgnitionStorageTypeOptionUnencryptedUserData,
+	}
+	additionalSecurityGroups := []awsv1.AWSResourceReference{
+		{
+			Filters: []awsv1.Filter{
+				{
+					Name:   "tag:Name",
+					Values: mapiProviderSpec.SecurityGroups[0].Filters[0].Values,
+				},
+			},
+		},
+		{
+			Filters: []awsv1.Filter{
+				{
+					Name:   "tag:Name",
+					Values: mapiProviderSpec.SecurityGroups[1].Filters[0].Values,
+				},
+			},
+		},
+	}
+	awsmt := capiinfrastructurev1beta2resourcebuilder.
+		AWSMachineTemplate().
+		WithUncompressedUserData(uncompressedUserData).
+		WithIAMInstanceProfile(*mapiProviderSpec.IAMInstanceProfile.ID).
+		WithInstanceType(mapiProviderSpec.InstanceType).
+		WithAMI(ami).
+		WithIgnition(ignition).
+		WithSubnet(&subnet).
+		WithAdditionalSecurityGroups(additionalSecurityGroups).
+		WithName(awsMachineTemplateName).
+		WithNamespace(framework.ClusterAPINamespace).
+		Build()
+
+	return awsmt
+}

pkg/e2e_test.go

@@ -14,6 +14,7 @@ import (
 	machinev1 "github.com/openshift/api/machine/v1beta1"
 	"github.com/openshift/cluster-api-actuator-pkg/pkg/framework"
 	caov1alpha1 "github.com/openshift/cluster-autoscaler-operator/pkg/apis"
+	awsv1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2"
 	azurev1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 
@@ -48,6 +49,10 @@ func init() {
 	if err := azurev1.AddToScheme(scheme.Scheme); err != nil {
 		klog.Fatal(err)
 	}
+
+	if err := awsv1.AddToScheme(scheme.Scheme); err != nil {
+		klog.Fatal(err)
+	}
 }
 
 func TestE2E(t *testing.T) {

pkg/framework/aws_client.go

@@ -4,7 +4,10 @@ import (
 	"fmt"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/kms"
 	"k8s.io/klog"
 	"k8s.io/utils/ptr"
 )
@@ -14,6 +17,45 @@ type AwsClient struct {
 	Svc *ec2.EC2
 }
 
+// Init the aws client.
+func NewAwsClient(accessKeyID []byte, secureKey []byte, clusterRegion string) *AwsClient {
+	awsSession := newAwsSession(accessKeyID, secureKey, clusterRegion)
+	aClient := &AwsClient{
+		Svc: ec2.New(awsSession),
+	}
+
+	return aClient
+}
+
+// AwsKmsClient struct.
+type AwsKmsClient struct {
+	kmssvc *kms.KMS
+}
+
+// Init the aws kms client.
+func NewAwsKmsClient(accessKeyID []byte, secureKey []byte, clusterRegion string) *AwsKmsClient {
+	awsSession := newAwsSession(accessKeyID, secureKey, clusterRegion)
+	kmsClient := &AwsKmsClient{
+		kmssvc: kms.New(awsSession),
+	}
+
+	return kmsClient
+}
+
+// Create aws backend session connection.
+func newAwsSession(accessKeyID []byte, secureKey []byte, clusterRegion string) *session.Session {
+	awsConfig := &aws.Config{
+		Region: aws.String(clusterRegion),
+		Credentials: credentials.NewStaticCredentials(
+			string(accessKeyID),
+			string(secureKey),
+			"",
+		),
+	}
+
+	return session.Must(session.NewSession(awsConfig))
+}
+
 // CreateCapacityReservation Create CapacityReservation.
 func (a *AwsClient) CreateCapacityReservation(instanceType string, instancePlatform string, availabilityZone string, instanceCount int64) (string, error) {
 	input := &ec2.CreateCapacityReservationInput{
@@ -45,3 +87,54 @@ func (a *AwsClient) CancelCapacityReservation(capacityReservationID string) (boo
 
 	return ptr.Deref(result.Return, false), err
 }
+
+// CreatePlacementGroup Create a PlacementGroup.
+func (a *AwsClient) CreatePlacementGroup(groupName string, strategy string, partitionCount ...int64) (string, error) {
+	var input *ec2.CreatePlacementGroupInput
+	if len(partitionCount) > 0 {
+		input = &ec2.CreatePlacementGroupInput{
+			GroupName:      aws.String(groupName),
+			PartitionCount: aws.Int64(partitionCount[0]),
+			Strategy:       aws.String(strategy),
+		}
+	} else {
+		input = &ec2.CreatePlacementGroupInput{
+			GroupName: aws.String(groupName),
+			Strategy:  aws.String(strategy),
+		}
+	}
+
+	result, err := a.Svc.CreatePlacementGroup(input)
+
+	if err != nil {
+		return "", fmt.Errorf("error creating placement group: %w", err)
+	}
+
+	placementGroupID := ptr.Deref(result.PlacementGroup.GroupId, "")
+	klog.Infof("The created placementGroupID is %s", placementGroupID)
+
+	return placementGroupID, nil
+}
+
+// DeletePlacementGroup Delete a PlacementGroup.
+func (a *AwsClient) DeletePlacementGroup(groupName string) (string, error) {
+	input := &ec2.DeletePlacementGroupInput{
+		GroupName: aws.String(groupName),
+	}
+	result, err := a.Svc.DeletePlacementGroup(input)
+
+	if err != nil {
+		return "", fmt.Errorf("could not delete placement group: %w", err)
+	}
+
+	return result.String(), nil
+}
+
+// Describes aws customer managed kms key info.
+func (akms *AwsKmsClient) DescribeKeyByID(kmsKeyID string) (describeResult *kms.DescribeKeyOutput, err error) {
+	input := &kms.DescribeKeyInput{
+		KeyId: aws.String(kmsKeyID),
+	}
+
+	return akms.kmssvc.DescribeKey(input)
+}

pkg/framework/capi_machinesets.go

@@ -40,6 +40,19 @@ func NewCAPIMachineSetParams(msName, clusterName, failureDomain string, replicas
 	}
 }
 
+// UpdateCAPIMachineSetName returns CAPIMachineSetParams object with the updated machineset name.
+func UpdateCAPIMachineSetName(msName string, params CAPIMachineSetParams) CAPIMachineSetParams {
+	Expect(msName).ToNot(BeEmpty(), "expected the capi msName to not be empty")
+
+	return CAPIMachineSetParams{
+		msName:            msName,
+		clusterName:       params.clusterName,
+		replicas:          params.replicas,
+		infrastructureRef: params.infrastructureRef,
+		failureDomain:     params.failureDomain,
+	}
+}
+
 // CreateCAPIMachineSet creates a new MachineSet resource.
 func CreateCAPIMachineSet(ctx context.Context, cl client.Client, params CAPIMachineSetParams) (*clusterv1.MachineSet, error) {
 	By(fmt.Sprintf("Creating MachineSet %q", params.msName))