From b6329be5832d94e24d7f18138ec1f5120dd0662d Mon Sep 17 00:00:00 2001 From: huali9 Date: Thu, 21 Nov 2024 10:09:17 +0800 Subject: [PATCH] add aws capi cases --- pkg/capi/aws.go | 252 ++++++++++++++++++++++++++++++ pkg/e2e_test.go | 5 + pkg/framework/aws_client.go | 38 +++++ pkg/framework/capi_machinesets.go | 13 ++ pkg/framework/framework.go | 41 +++++ 5 files changed, 349 insertions(+) create mode 100644 pkg/capi/aws.go diff --git a/pkg/capi/aws.go b/pkg/capi/aws.go new file mode 100644 index 000000000..963ad0020 --- /dev/null +++ b/pkg/capi/aws.go @@ -0,0 +1,252 @@ +package capi + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + configv1 "github.com/openshift/api/config/v1" + mapiv1 "github.com/openshift/api/machine/v1beta1" + "github.com/openshift/cluster-api-actuator-pkg/pkg/framework" + "github.com/openshift/cluster-api-actuator-pkg/pkg/framework/gatherer" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + awsv1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + + "sigs.k8s.io/controller-runtime/pkg/client/config" + + "sigs.k8s.io/controller-runtime/pkg/client" + yaml "sigs.k8s.io/yaml" +) + +const ( + awsMachineTemplateName = "aws-machine-template" + infrastructureName = "cluster" + infraAPIVersion = "infrastructure.cluster.x-k8s.io/v1beta1" +) + +var ( + cl client.Client + ctx = context.Background() + platform configv1.PlatformType + clusterName string + oc *gatherer.CLI +) + +var _ = Describe("Cluster API AWS MachineSet", framework.LabelCAPI, framework.LabelDisruptive, Ordered, func() { + var ( + awsMachineTemplate *awsv1.AWSMachineTemplate + machineSetParams framework.CAPIMachineSetParams + machineSet *clusterv1.MachineSet + mapiDefaultProviderSpec *mapiv1.AWSMachineProviderConfig + deleted bool + err error + ) + + BeforeAll(func() { + cfg, err := config.GetConfig() + Expect(err).ToNot(HaveOccurred()) + + cl, err = client.New(cfg, client.Options{}) + Expect(err).ToNot(HaveOccurred()) + + infra := &configv1.Infrastructure{} + infraName := client.ObjectKey{ + Name: infrastructureName, + } + Expect(cl.Get(ctx, infraName, infra)).To(Succeed()) + Expect(infra.Status.PlatformStatus).ToNot(BeNil()) + clusterName = infra.Status.InfrastructureName + platform = infra.Status.PlatformStatus.Type + deleted = false + oc, _ = framework.NewCLI() + framework.SkipIfNotTechPreviewNoUpgrade(oc, cl) + _, mapiDefaultProviderSpec = getDefaultAWSMAPIProviderSpec(cl) + machineSetParams = framework.NewCAPIMachineSetParams( + "aws-machineset", + clusterName, + mapiDefaultProviderSpec.Placement.AvailabilityZone, + 1, + corev1.ObjectReference{ + Kind: "AWSMachineTemplate", + APIVersion: infraAPIVersion, + Name: awsMachineTemplateName, + }, + ) + + if platform != configv1.AWSPlatformType { + Skip("Skipping AWS E2E tests") + } + + framework.CreateCoreCluster(ctx, cl, clusterName, "AWSCluster") + }) + + BeforeEach(func() { + deleted = false + }) + + AfterEach(func() { + if platform != configv1.AWSPlatformType { + // Because AfterEach always runs, even when tests are skipped, we have to + // explicitly skip it here for other platforms. + Skip("Skipping AWS E2E tests") + } + if !deleted { + framework.DeleteCAPIMachineSets(ctx, cl, machineSet) + framework.WaitForCAPIMachineSetsDeleted(ctx, cl, machineSet) + framework.DeleteObjects(ctx, cl, awsMachineTemplate) + } + }) + + //huliu-OCP-51071 - [CAPI] Create machineset with CAPI on aws + It("should be able to run a machine with a default provider spec", func() { + awsMachineTemplate = newAWSMachineTemplate(mapiDefaultProviderSpec) + if err = cl.Create(ctx, awsMachineTemplate); err != nil { + Expect(err).ToNot(HaveOccurred()) + } + machineSetParams = framework.UpdateCAPIMachineSetName("aws-machineset-51071", machineSetParams) + machineSet, err = framework.CreateCAPIMachineSet(ctx, cl, machineSetParams) + Expect(err).ToNot(HaveOccurred(), "Failed to create CAPI machineset") + framework.WaitForCAPIMachinesRunning(ctx, cl, machineSet.Name) + }) + + //huliu-OCP-75395 - [CAPI] AWS Placement group support. + It("should be able to run a machine with cluster placement group", func() { + awsClient := framework.CreateAWSClient(oc) + placementGroupName := clusterName + "pgcluster" + placementGroupID, err := awsClient.CreatePlacementGroup(placementGroupName, "cluster") + Expect(err).ToNot(HaveOccurred()) + Expect(placementGroupID).ToNot(Equal("")) + defer func() { + framework.DeleteCAPIMachineSets(ctx, cl, machineSet) + framework.WaitForCAPIMachineSetsDeleted(ctx, cl, machineSet) + framework.DeleteObjects(ctx, cl, awsMachineTemplate) + deleted = true + _, err = awsClient.DeletePlacementGroup(placementGroupName) + Expect(err).ToNot(HaveOccurred()) + }() + + awsMachineTemplate = newAWSMachineTemplate(mapiDefaultProviderSpec) + awsMachineTemplate.Spec.Template.Spec.PlacementGroupName = placementGroupName + if err = cl.Create(ctx, awsMachineTemplate); err != nil { + Expect(err).ToNot(HaveOccurred()) + } + machineSetParams = framework.UpdateCAPIMachineSetName("aws-machineset-75395", machineSetParams) + machineSet, err = framework.CreateCAPIMachineSet(ctx, cl, machineSetParams) + Expect(err).ToNot(HaveOccurred(), "Failed to create CAPI machineset") + framework.WaitForCAPIMachinesRunning(ctx, cl, machineSet.Name) + }) + + //huliu-OCP-75396 - [CAPI] Creating machines using KMS keys from AWS. + It("should be able to run a machine using KMS keys", func() { + awsMachineTemplate = newAWSMachineTemplate(mapiDefaultProviderSpec) + region := mapiDefaultProviderSpec.Placement.Region + if region != "us-east-1" && region != "us-east-2" { + Skip("Region is " + region + ", skip this test scenario because we only created kms key in us-east-1/us-east-2 region") + } + var key string + switch region { + case "us-east-1": + key = "arn:aws:kms:us-east-1:301721915996:key/c471ec83-cfaf-41a2-9241-d9e99c4da344" + case "us-east-2": + key = "arn:aws:kms:us-east-2:301721915996:key/c228ef83-df2c-4151-84c4-d9f39f39a972" + } + + encryptBool := true + awsMachineTemplate.Spec.Template.Spec.NonRootVolumes = []awsv1.Volume{ + { + DeviceName: "/dev/xvda", + Size: 140, + Type: awsv1.VolumeTypeIO1, + IOPS: 5000, + Encrypted: &encryptBool, + EncryptionKey: key, + }, + } + if err := cl.Create(ctx, awsMachineTemplate); err != nil { + Expect(err).ToNot(HaveOccurred()) + } + machineSetParams = framework.UpdateCAPIMachineSetName("aws-machineset-75396", machineSetParams) + machineSet, err = framework.CreateCAPIMachineSet(ctx, cl, machineSetParams) + Expect(err).ToNot(HaveOccurred(), "Failed to create CAPI machineset") + framework.WaitForCAPIMachinesRunning(ctx, cl, machineSet.Name) + }) +}) + +func getDefaultAWSMAPIProviderSpec(cl client.Client) (*mapiv1.MachineSet, *mapiv1.AWSMachineProviderConfig) { + machineSetList := &mapiv1.MachineSetList{} + Expect(cl.List(ctx, machineSetList, client.InNamespace(framework.MachineAPINamespace))).To(Succeed()) + + Expect(machineSetList.Items).ToNot(HaveLen(0)) + machineSet := &machineSetList.Items[0] + Expect(machineSet.Spec.Template.Spec.ProviderSpec.Value).ToNot(BeNil()) + + providerSpec := &mapiv1.AWSMachineProviderConfig{} + Expect(yaml.Unmarshal(machineSet.Spec.Template.Spec.ProviderSpec.Value.Raw, providerSpec)).To(Succeed()) + + return machineSet, providerSpec +} + +func newAWSMachineTemplate(mapiProviderSpec *mapiv1.AWSMachineProviderConfig) *awsv1.AWSMachineTemplate { + By("Creating AWS machine template") + + Expect(mapiProviderSpec).ToNot(BeNil()) + Expect(mapiProviderSpec.IAMInstanceProfile).ToNot(BeNil()) + Expect(mapiProviderSpec.IAMInstanceProfile.ID).ToNot(BeNil()) + Expect(mapiProviderSpec.InstanceType).ToNot(BeEmpty()) + Expect(mapiProviderSpec.Placement.AvailabilityZone).ToNot(BeEmpty()) + Expect(mapiProviderSpec.AMI.ID).ToNot(BeNil()) + Expect(mapiProviderSpec.Subnet.Filters).ToNot(HaveLen(0)) + Expect(mapiProviderSpec.Subnet.Filters[0].Values).ToNot(HaveLen(0)) + Expect(mapiProviderSpec.SecurityGroups).ToNot(HaveLen(0)) + Expect(mapiProviderSpec.SecurityGroups[0].Filters).ToNot(HaveLen(0)) + Expect(mapiProviderSpec.SecurityGroups[0].Filters[0].Values).ToNot(HaveLen(0)) + + uncompressedUserData := true + + awsMachineSpec := awsv1.AWSMachineSpec{ + UncompressedUserData: &uncompressedUserData, + IAMInstanceProfile: *mapiProviderSpec.IAMInstanceProfile.ID, + InstanceType: mapiProviderSpec.InstanceType, + AMI: awsv1.AMIReference{ + ID: mapiProviderSpec.AMI.ID, + }, + Ignition: &awsv1.Ignition{ + Version: "3.4", + StorageType: awsv1.IgnitionStorageTypeOptionUnencryptedUserData, + }, + Subnet: &awsv1.AWSResourceReference{ + Filters: []awsv1.Filter{ + { + Name: "tag:Name", + Values: mapiProviderSpec.Subnet.Filters[0].Values, + }, + }, + }, + AdditionalSecurityGroups: []awsv1.AWSResourceReference{ + { + Filters: []awsv1.Filter{ + { + Name: "tag:Name", + Values: mapiProviderSpec.SecurityGroups[0].Filters[0].Values, + }, + }, + }, + }, + } + + awsMachineTemplate := &awsv1.AWSMachineTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: awsMachineTemplateName, + Namespace: framework.ClusterAPINamespace, + }, + Spec: awsv1.AWSMachineTemplateSpec{ + Template: awsv1.AWSMachineTemplateResource{ + Spec: awsMachineSpec, + }, + }, + } + + return awsMachineTemplate +} diff --git a/pkg/e2e_test.go b/pkg/e2e_test.go index b132bd7b3..d391f64dd 100644 --- a/pkg/e2e_test.go +++ b/pkg/e2e_test.go @@ -14,6 +14,7 @@ import ( machinev1 "github.com/openshift/api/machine/v1beta1" "github.com/openshift/cluster-api-actuator-pkg/pkg/framework" caov1alpha1 "github.com/openshift/cluster-autoscaler-operator/pkg/apis" + awsv1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" azurev1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" @@ -48,6 +49,10 @@ func init() { if err := azurev1.AddToScheme(scheme.Scheme); err != nil { klog.Fatal(err) } + + if err := awsv1.AddToScheme(scheme.Scheme); err != nil { + klog.Fatal(err) + } } func TestE2E(t *testing.T) { diff --git a/pkg/framework/aws_client.go b/pkg/framework/aws_client.go index 8f239ca65..2205b5015 100644 --- a/pkg/framework/aws_client.go +++ b/pkg/framework/aws_client.go @@ -45,3 +45,41 @@ func (a *AwsClient) CancelCapacityReservation(capacityReservationID string) (boo return ptr.Deref(result.Return, false), err } + +// CreateCapacityReservation Create CapacityReservation. +func (a *AwsClient) CreatePlacementGroup(groupName string, strategy string, partitionCount ...int64) (string, error) { + var input *ec2.CreatePlacementGroupInput + if len(partitionCount) > 0 { + input = &ec2.CreatePlacementGroupInput{ + GroupName: aws.String(groupName), + PartitionCount: aws.Int64(partitionCount[0]), + Strategy: aws.String(strategy), + } + } else { + input = &ec2.CreatePlacementGroupInput{ + GroupName: aws.String(groupName), + Strategy: aws.String(strategy), + } + } + + result, err := a.Svc.CreatePlacementGroup(input) + + if err != nil { + return "", fmt.Errorf("error creating placement group: %w", err) + } + + placementGroupID := ptr.Deref(result.PlacementGroup.GroupId, "") + klog.Infof("The created placementGroupID is %s", placementGroupID) + + return placementGroupID, err +} + +// DeletePlacementGroup Delete a PlacementGroup. +func (a *AwsClient) DeletePlacementGroup(groupName string) (string, error) { + input := &ec2.DeletePlacementGroupInput{ + GroupName: aws.String(groupName), + } + result, err := a.Svc.DeletePlacementGroup(input) + + return result.String(), err +} diff --git a/pkg/framework/capi_machinesets.go b/pkg/framework/capi_machinesets.go index 87dd5798e..c7866c977 100644 --- a/pkg/framework/capi_machinesets.go +++ b/pkg/framework/capi_machinesets.go @@ -40,6 +40,19 @@ func NewCAPIMachineSetParams(msName, clusterName, failureDomain string, replicas } } +// UpdateCAPIMachineSetName returns CAPIMachineSetParams object with the updated machineset name. +func UpdateCAPIMachineSetName(msName string, params CAPIMachineSetParams) CAPIMachineSetParams { + Expect(msName).ToNot(BeEmpty(), "expected the capi msName to not be empty") + + return CAPIMachineSetParams{ + msName: msName, + clusterName: params.clusterName, + replicas: params.replicas, + infrastructureRef: params.infrastructureRef, + failureDomain: params.failureDomain, + } +} + // CreateCAPIMachineSet creates a new MachineSet resource. func CreateCAPIMachineSet(ctx context.Context, cl client.Client, params CAPIMachineSetParams) (*clusterv1.MachineSet, error) { By(fmt.Sprintf("Creating MachineSet %q", params.msName)) diff --git a/pkg/framework/framework.go b/pkg/framework/framework.go index 9be416c01..c73e356eb 100644 --- a/pkg/framework/framework.go +++ b/pkg/framework/framework.go @@ -2,16 +2,22 @@ package framework import ( "context" + "encoding/base64" "errors" "fmt" "os" "path/filepath" "time" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" configv1 "github.com/openshift/api/config/v1" cov1helpers "github.com/openshift/library-go/pkg/config/clusteroperator/v1helpers" + "github.com/tidwall/gjson" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -292,3 +298,38 @@ func SkipIfNotTechPreviewNoUpgrade(oc *gatherer.CLI, cl runtimeclient.Client) { Skip("FeatureSet is not TechPreviewNoUpgradec, skip it!") } } + +// CreateAWSClient create AWS client. +func CreateAWSClient(oc *gatherer.CLI) *AwsClient { + awscreds, err := oc.WithoutNamespace().Run("get").Args("secret/aws-creds", "-n", "kube-system", "-o", "json").Output() + if err != nil { + Skip("Unable to get AWS credentials secret, skipping the testing.") + } + + accessKeyIDBase64, secureKeyBase64 := gjson.Get(awscreds, `data.aws_access_key_id`).String(), gjson.Get(awscreds, `data.aws_secret_access_key`).String() + + accessKeyID, err := base64.StdEncoding.DecodeString(accessKeyIDBase64) + Expect(err).NotTo(HaveOccurred()) + secureKey, err := base64.StdEncoding.DecodeString(secureKeyBase64) + Expect(err).NotTo(HaveOccurred()) + clusterRegion, err := oc.WithoutNamespace().Run("get").Args("infrastructure", "cluster", "-o=jsonpath={.status.platformStatus.aws.region}").Output() + Expect(err).NotTo(HaveOccurred()) + + awsConfig := &aws.Config{ + Region: aws.String(clusterRegion), + Credentials: credentials.NewStaticCredentials( + string(accessKeyID), + string(secureKey), + "", + ), + } + + sess, err := session.NewSession(awsConfig) + Expect(err).ToNot(HaveOccurred()) + + aClient := &AwsClient{ + Svc: ec2.New(sess), + } + + return aClient +}