add aws capi cases

pkg/capi/aws.go

@@ -0,0 +1,252 @@
+package capi
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	configv1 "github.com/openshift/api/config/v1"
+	mapiv1 "github.com/openshift/api/machine/v1beta1"
+	"github.com/openshift/cluster-api-actuator-pkg/pkg/framework"
+	"github.com/openshift/cluster-api-actuator-pkg/pkg/framework/gatherer"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	awsv1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2"
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+
+	"sigs.k8s.io/controller-runtime/pkg/client/config"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	yaml "sigs.k8s.io/yaml"
+)
+
+const (
+	awsMachineTemplateName = "aws-machine-template"
+	infrastructureName     = "cluster"
+	infraAPIVersion        = "infrastructure.cluster.x-k8s.io/v1beta1"
+)
+
+var (
+	cl          client.Client
+	ctx         = context.Background()
+	platform    configv1.PlatformType
+	clusterName string
+	oc          *gatherer.CLI
+)
+
+var _ = Describe("Cluster API AWS MachineSet", framework.LabelCAPI, framework.LabelDisruptive, Ordered, func() {
+	var (
+		awsMachineTemplate      *awsv1.AWSMachineTemplate
+		machineSetParams        framework.CAPIMachineSetParams
+		machineSet              *clusterv1.MachineSet
+		mapiDefaultProviderSpec *mapiv1.AWSMachineProviderConfig
+		deleted                 bool
+		err                     error
+	)
+
+	BeforeAll(func() {
+		cfg, err := config.GetConfig()
+		Expect(err).ToNot(HaveOccurred())
+
+		cl, err = client.New(cfg, client.Options{})
+		Expect(err).ToNot(HaveOccurred())
+
+		infra := &configv1.Infrastructure{}
+		infraName := client.ObjectKey{
+			Name: infrastructureName,
+		}
+		Expect(cl.Get(ctx, infraName, infra)).To(Succeed())
+		Expect(infra.Status.PlatformStatus).ToNot(BeNil())
+		clusterName = infra.Status.InfrastructureName
+		platform = infra.Status.PlatformStatus.Type
+		deleted = false
+		oc, _ = framework.NewCLI()
+		framework.SkipIfNotTechPreviewNoUpgrade(oc, cl)
+		_, mapiDefaultProviderSpec = getDefaultAWSMAPIProviderSpec(cl)
+		machineSetParams = framework.NewCAPIMachineSetParams(
+			"aws-machineset",
+			clusterName,
+			mapiDefaultProviderSpec.Placement.AvailabilityZone,
+			1,
+			corev1.ObjectReference{
+				Kind:       "AWSMachineTemplate",
+				APIVersion: infraAPIVersion,
+				Name:       awsMachineTemplateName,
+			},
+		)
+
+		if platform != configv1.AWSPlatformType {
+			Skip("Skipping AWS E2E tests")
+		}
+
+		framework.CreateCoreCluster(ctx, cl, clusterName, "AWSCluster")
+	})
+
+	BeforeEach(func() {
+		deleted = false
+	})
+
+	AfterEach(func() {
+		if platform != configv1.AWSPlatformType {
+			// Because AfterEach always runs, even when tests are skipped, we have to
+			// explicitly skip it here for other platforms.
+			Skip("Skipping AWS E2E tests")
+		}
+		if !deleted {
+			framework.DeleteCAPIMachineSets(ctx, cl, machineSet)
+			framework.WaitForCAPIMachineSetsDeleted(ctx, cl, machineSet)
+			framework.DeleteObjects(ctx, cl, awsMachineTemplate)
+		}
+	})
+
+	//huliu-OCP-51071 - [CAPI] Create machineset with CAPI on aws
+	It("should be able to run a machine with a default provider spec", func() {
+		awsMachineTemplate = newAWSMachineTemplate(mapiDefaultProviderSpec)
+		if err = cl.Create(ctx, awsMachineTemplate); err != nil {
+			Expect(err).ToNot(HaveOccurred())
+		}
+		machineSetParams = framework.UpdateCAPIMachineSetName("aws-machineset-51071", machineSetParams)
+		machineSet, err = framework.CreateCAPIMachineSet(ctx, cl, machineSetParams)
+		Expect(err).ToNot(HaveOccurred(), "Failed to create CAPI machineset")
+		framework.WaitForCAPIMachinesRunning(ctx, cl, machineSet.Name)
+	})
+
+	//huliu-OCP-75395 - [CAPI] AWS Placement group support.
+	It("should be able to run a machine with cluster placement group", func() {
+		awsClient := framework.CreateAWSClient(oc)
+		placementGroupName := clusterName + "pgcluster"
+		placementGroupID, err := awsClient.CreatePlacementGroup(placementGroupName, "cluster")
+		Expect(err).ToNot(HaveOccurred())
+		Expect(placementGroupID).ToNot(Equal(""))
+		defer func() {
+			framework.DeleteCAPIMachineSets(ctx, cl, machineSet)
+			framework.WaitForCAPIMachineSetsDeleted(ctx, cl, machineSet)
+			framework.DeleteObjects(ctx, cl, awsMachineTemplate)
+			deleted = true
+			_, err = awsClient.DeletePlacementGroup(placementGroupName)
+			Expect(err).ToNot(HaveOccurred())
+		}()
+
+		awsMachineTemplate = newAWSMachineTemplate(mapiDefaultProviderSpec)
+		awsMachineTemplate.Spec.Template.Spec.PlacementGroupName = placementGroupName
+		if err = cl.Create(ctx, awsMachineTemplate); err != nil {
+			Expect(err).ToNot(HaveOccurred())
+		}
+		machineSetParams = framework.UpdateCAPIMachineSetName("aws-machineset-75395", machineSetParams)
+		machineSet, err = framework.CreateCAPIMachineSet(ctx, cl, machineSetParams)
+		Expect(err).ToNot(HaveOccurred(), "Failed to create CAPI machineset")
+		framework.WaitForCAPIMachinesRunning(ctx, cl, machineSet.Name)
+	})
+
+	//huliu-OCP-75396 - [CAPI] Creating machines using KMS keys from AWS.
+	It("should be able to run a machine using KMS keys", func() {
+		awsMachineTemplate = newAWSMachineTemplate(mapiDefaultProviderSpec)
+		region := mapiDefaultProviderSpec.Placement.Region
+		if region != "us-east-1" && region != "us-east-2" {
+			Skip("Region is " + region + ", skip this test scenario because we only created kms key in us-east-1/us-east-2 region")
+		}
+		var key string
+		switch region {
+		case "us-east-1":
+			key = "arn:aws:kms:us-east-1:301721915996:key/c471ec83-cfaf-41a2-9241-d9e99c4da344"
+		case "us-east-2":
+			key = "arn:aws:kms:us-east-2:301721915996:key/c228ef83-df2c-4151-84c4-d9f39f39a972"
+		}
+
+		encryptBool := true
+		awsMachineTemplate.Spec.Template.Spec.NonRootVolumes = []awsv1.Volume{
+			{
+				DeviceName:    "/dev/xvda",
+				Size:          140,
+				Type:          awsv1.VolumeTypeIO1,
+				IOPS:          5000,
+				Encrypted:     &encryptBool,
+				EncryptionKey: key,
+			},
+		}
+		if err := cl.Create(ctx, awsMachineTemplate); err != nil {
+			Expect(err).ToNot(HaveOccurred())
+		}
+		machineSetParams = framework.UpdateCAPIMachineSetName("aws-machineset-75396", machineSetParams)
+		machineSet, err = framework.CreateCAPIMachineSet(ctx, cl, machineSetParams)
+		Expect(err).ToNot(HaveOccurred(), "Failed to create CAPI machineset")
+		framework.WaitForCAPIMachinesRunning(ctx, cl, machineSet.Name)
+	})
+})
+
+func getDefaultAWSMAPIProviderSpec(cl client.Client) (*mapiv1.MachineSet, *mapiv1.AWSMachineProviderConfig) {
+	machineSetList := &mapiv1.MachineSetList{}
+	Expect(cl.List(ctx, machineSetList, client.InNamespace(framework.MachineAPINamespace))).To(Succeed())
+
+	Expect(machineSetList.Items).ToNot(HaveLen(0))
+	machineSet := &machineSetList.Items[0]
+	Expect(machineSet.Spec.Template.Spec.ProviderSpec.Value).ToNot(BeNil())
+
+	providerSpec := &mapiv1.AWSMachineProviderConfig{}
+	Expect(yaml.Unmarshal(machineSet.Spec.Template.Spec.ProviderSpec.Value.Raw, providerSpec)).To(Succeed())
+
+	return machineSet, providerSpec
+}
+
+func newAWSMachineTemplate(mapiProviderSpec *mapiv1.AWSMachineProviderConfig) *awsv1.AWSMachineTemplate {
+	By("Creating AWS machine template")
+
+	Expect(mapiProviderSpec).ToNot(BeNil())
+	Expect(mapiProviderSpec.IAMInstanceProfile).ToNot(BeNil())
+	Expect(mapiProviderSpec.IAMInstanceProfile.ID).ToNot(BeNil())
+	Expect(mapiProviderSpec.InstanceType).ToNot(BeEmpty())
+	Expect(mapiProviderSpec.Placement.AvailabilityZone).ToNot(BeEmpty())
+	Expect(mapiProviderSpec.AMI.ID).ToNot(BeNil())
+	Expect(mapiProviderSpec.Subnet.Filters).ToNot(HaveLen(0))
+	Expect(mapiProviderSpec.Subnet.Filters[0].Values).ToNot(HaveLen(0))
+	Expect(mapiProviderSpec.SecurityGroups).ToNot(HaveLen(0))
+	Expect(mapiProviderSpec.SecurityGroups[0].Filters).ToNot(HaveLen(0))
+	Expect(mapiProviderSpec.SecurityGroups[0].Filters[0].Values).ToNot(HaveLen(0))
+
+	uncompressedUserData := true
+
+	awsMachineSpec := awsv1.AWSMachineSpec{
+		UncompressedUserData: &uncompressedUserData,
+		IAMInstanceProfile:   *mapiProviderSpec.IAMInstanceProfile.ID,
+		InstanceType:         mapiProviderSpec.InstanceType,
+		AMI: awsv1.AMIReference{
+			ID: mapiProviderSpec.AMI.ID,
+		},
+		Ignition: &awsv1.Ignition{
+			Version:     "3.4",
+			StorageType: awsv1.IgnitionStorageTypeOptionUnencryptedUserData,
+		},
+		Subnet: &awsv1.AWSResourceReference{
+			Filters: []awsv1.Filter{
+				{
+					Name:   "tag:Name",
+					Values: mapiProviderSpec.Subnet.Filters[0].Values,
+				},
+			},
+		},
+		AdditionalSecurityGroups: []awsv1.AWSResourceReference{
+			{
+				Filters: []awsv1.Filter{
+					{
+						Name:   "tag:Name",
+						Values: mapiProviderSpec.SecurityGroups[0].Filters[0].Values,
+					},
+				},
+			},
+		},
+	}
+
+	awsMachineTemplate := &awsv1.AWSMachineTemplate{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      awsMachineTemplateName,
+			Namespace: framework.ClusterAPINamespace,
+		},
+		Spec: awsv1.AWSMachineTemplateSpec{
+			Template: awsv1.AWSMachineTemplateResource{
+				Spec: awsMachineSpec,
+			},
+		},
+	}
+
+	return awsMachineTemplate
+}

pkg/e2e_test.go

@@ -14,6 +14,7 @@ import (
 	machinev1 "github.com/openshift/api/machine/v1beta1"
 	"github.com/openshift/cluster-api-actuator-pkg/pkg/framework"
 	caov1alpha1 "github.com/openshift/cluster-autoscaler-operator/pkg/apis"
+	awsv1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2"
 	azurev1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 
@@ -48,6 +49,10 @@ func init() {
 	if err := azurev1.AddToScheme(scheme.Scheme); err != nil {
 		klog.Fatal(err)
 	}
+
+	if err := awsv1.AddToScheme(scheme.Scheme); err != nil {
+		klog.Fatal(err)
+	}
 }
 
 func TestE2E(t *testing.T) {

pkg/framework/aws_client.go

@@ -45,3 +45,41 @@ func (a *AwsClient) CancelCapacityReservation(capacityReservationID string) (boo
 
 	return ptr.Deref(result.Return, false), err
 }
+
+// CreateCapacityReservation Create CapacityReservation.
+func (a *AwsClient) CreatePlacementGroup(groupName string, strategy string, partitionCount ...int64) (string, error) {
+	var input *ec2.CreatePlacementGroupInput
+	if len(partitionCount) > 0 {
+		input = &ec2.CreatePlacementGroupInput{
+			GroupName:      aws.String(groupName),
+			PartitionCount: aws.Int64(partitionCount[0]),
+			Strategy:       aws.String(strategy),
+		}
+	} else {
+		input = &ec2.CreatePlacementGroupInput{
+			GroupName: aws.String(groupName),
+			Strategy:  aws.String(strategy),
+		}
+	}
+
+	result, err := a.Svc.CreatePlacementGroup(input)
+
+	if err != nil {
+		return "", fmt.Errorf("error creating placement group: %w", err)
+	}
+
+	placementGroupID := ptr.Deref(result.PlacementGroup.GroupId, "")
+	klog.Infof("The created placementGroupID is %s", placementGroupID)
+
+	return placementGroupID, err
+}
+
+// DeletePlacementGroup Delete a PlacementGroup.
+func (a *AwsClient) DeletePlacementGroup(groupName string) (string, error) {
+	input := &ec2.DeletePlacementGroupInput{
+		GroupName: aws.String(groupName),
+	}
+	result, err := a.Svc.DeletePlacementGroup(input)
+
+	return result.String(), err
+}

pkg/framework/capi_machinesets.go

@@ -40,6 +40,19 @@ func NewCAPIMachineSetParams(msName, clusterName, failureDomain string, replicas
 	}
 }
 
+// UpdateCAPIMachineSetName returns CAPIMachineSetParams object with the updated machineset name.
+func UpdateCAPIMachineSetName(msName string, params CAPIMachineSetParams) CAPIMachineSetParams {
+	Expect(msName).ToNot(BeEmpty(), "expected the capi msName to not be empty")
+
+	return CAPIMachineSetParams{
+		msName:            msName,
+		clusterName:       params.clusterName,
+		replicas:          params.replicas,
+		infrastructureRef: params.infrastructureRef,
+		failureDomain:     params.failureDomain,
+	}
+}
+
 // CreateCAPIMachineSet creates a new MachineSet resource.
 func CreateCAPIMachineSet(ctx context.Context, cl client.Client, params CAPIMachineSetParams) (*clusterv1.MachineSet, error) {
 	By(fmt.Sprintf("Creating MachineSet %q", params.msName))

pkg/framework/framework.go

@@ -2,16 +2,22 @@ package framework
 
 import (
 	"context"
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
 	"time"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ec2"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	configv1 "github.com/openshift/api/config/v1"
 	cov1helpers "github.com/openshift/library-go/pkg/config/clusteroperator/v1helpers"
+	"github.com/tidwall/gjson"
 	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -292,3 +298,38 @@ func SkipIfNotTechPreviewNoUpgrade(oc *gatherer.CLI, cl runtimeclient.Client) {
 		Skip("FeatureSet is not TechPreviewNoUpgradec, skip it!")
 	}
 }
+
+// CreateAWSClient create AWS client.
+func CreateAWSClient(oc *gatherer.CLI) *AwsClient {
+	awscreds, err := oc.WithoutNamespace().Run("get").Args("secret/aws-creds", "-n", "kube-system", "-o", "json").Output()
+	if err != nil {
+		Skip("Unable to get AWS credentials secret, skipping the testing.")
+	}
+
+	accessKeyIDBase64, secureKeyBase64 := gjson.Get(awscreds, `data.aws_access_key_id`).String(), gjson.Get(awscreds, `data.aws_secret_access_key`).String()
+
+	accessKeyID, err := base64.StdEncoding.DecodeString(accessKeyIDBase64)
+	Expect(err).NotTo(HaveOccurred())
+	secureKey, err := base64.StdEncoding.DecodeString(secureKeyBase64)
+	Expect(err).NotTo(HaveOccurred())
+	clusterRegion, err := oc.WithoutNamespace().Run("get").Args("infrastructure", "cluster", "-o=jsonpath={.status.platformStatus.aws.region}").Output()
+	Expect(err).NotTo(HaveOccurred())
+
+	awsConfig := &aws.Config{
+		Region: aws.String(clusterRegion),
+		Credentials: credentials.NewStaticCredentials(
+			string(accessKeyID),
+			string(secureKey),
+			"",
+		),
+	}
+
+	sess, err := session.NewSession(awsConfig)
+	Expect(err).ToNot(HaveOccurred())
+
+	aClient := &AwsClient{
+		Svc: ec2.New(sess),
+	}
+
+	return aClient
+}