diff --git a/pkg/capi/aws.go b/pkg/capi/aws.go new file mode 100644 index 000000000..50a531263 --- /dev/null +++ b/pkg/capi/aws.go @@ -0,0 +1,266 @@ +package capi + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + configv1 "github.com/openshift/api/config/v1" + mapiv1 "github.com/openshift/api/machine/v1beta1" + "github.com/openshift/cluster-api-actuator-pkg/pkg/framework" + "github.com/openshift/cluster-api-actuator-pkg/pkg/framework/gatherer" + capiinfrastructurev1beta2resourcebuilder "github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/cluster-api/infrastructure/v1beta2" + corev1 "k8s.io/api/core/v1" + awsv1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + + "sigs.k8s.io/controller-runtime/pkg/client/config" + + "sigs.k8s.io/controller-runtime/pkg/client" + yaml "sigs.k8s.io/yaml" +) + +const ( + awsMachineTemplateName = "aws-machine-template" + infrastructureName = "cluster" + infraAPIVersion = "infrastructure.cluster.x-k8s.io/v1beta1" +) + +var ( + cl client.Client + ctx = context.Background() + platform configv1.PlatformType + clusterName string + oc *gatherer.CLI +) + +var _ = Describe("Cluster API AWS MachineSet", framework.LabelCAPI, framework.LabelDisruptive, Ordered, func() { + var ( + awsMachineTemplate *awsv1.AWSMachineTemplate + machineSetParams framework.CAPIMachineSetParams + machineSet *clusterv1.MachineSet + mapiDefaultProviderSpec *mapiv1.AWSMachineProviderConfig + deleted bool + err error + ) + + BeforeAll(func() { + cfg, err := config.GetConfig() + Expect(err).ToNot(HaveOccurred(), "Failed to GetConfig") + + cl, err = client.New(cfg, client.Options{}) + Expect(err).ToNot(HaveOccurred(), "Failed to create Kubernetes client for test") + + infra := &configv1.Infrastructure{} + infraName := client.ObjectKey{ + Name: infrastructureName, + } + Expect(cl.Get(ctx, infraName, infra)).To(Succeed(), "Failed to get cluster infrastructure object") + Expect(infra.Status.PlatformStatus).ToNot(BeNil(), "expected the infrastructure Status.PlatformStatus to not be nil") + clusterName = infra.Status.InfrastructureName + platform = infra.Status.PlatformStatus.Type + if platform != configv1.AWSPlatformType { + Skip("Skipping AWS E2E tests") + } + oc, _ = framework.NewCLI() + framework.SkipIfNotTechPreviewNoUpgrade(oc, cl) + _, mapiDefaultProviderSpec = getDefaultAWSMAPIProviderSpec(cl) + machineSetParams = framework.NewCAPIMachineSetParams( + "aws-machineset", + clusterName, + mapiDefaultProviderSpec.Placement.AvailabilityZone, + 1, + corev1.ObjectReference{ + Kind: "AWSMachineTemplate", + APIVersion: infraAPIVersion, + Name: awsMachineTemplateName, + }, + ) + framework.CreateCoreCluster(ctx, cl, clusterName, "AWSCluster") + }) + + BeforeEach(func() { + deleted = false + }) + + AfterEach(func() { + if platform != configv1.AWSPlatformType { + // Because AfterEach always runs, even when tests are skipped, we have to + // explicitly skip it here for other platforms. + Skip("Skipping AWS E2E tests") + } + if !deleted { + framework.DeleteCAPIMachineSets(ctx, cl, machineSet) + framework.WaitForCAPIMachineSetsDeleted(ctx, cl, machineSet) + framework.DeleteObjects(ctx, cl, awsMachineTemplate) + } + }) + + //huliu-OCP-51071 - [CAPI] Create machineset with CAPI on aws + It("should be able to run a machine with a default provider spec", func() { + awsMachineTemplate = newAWSMachineTemplate(mapiDefaultProviderSpec) + if err = cl.Create(ctx, awsMachineTemplate); err != nil { + Expect(err).ToNot(HaveOccurred(), "Failed to create awsmachinetemplate") + } + machineSetParams = framework.UpdateCAPIMachineSetName("aws-machineset-51071", machineSetParams) + machineSet, err = framework.CreateCAPIMachineSet(ctx, cl, machineSetParams) + Expect(err).ToNot(HaveOccurred(), "Failed to create CAPI machineset") + framework.WaitForCAPIMachinesRunning(ctx, cl, machineSet.Name) + }) + + //huliu-OCP-75395 - [CAPI] AWS Placement group support. + It("should be able to run a machine with cluster placement group", func() { + awsClient := framework.NewAwsClient(framework.GetCredentialsFromCluster(oc)) + placementGroupName := clusterName + "pgcluster" + placementGroupID, err := awsClient.CreatePlacementGroup(placementGroupName, "cluster") + Expect(err).ToNot(HaveOccurred(), "Failed to create placementgroup") + Expect(placementGroupID).ToNot(Equal(""), "expected the placementGroupID to not be empty string") + defer func() { + framework.DeleteCAPIMachineSets(ctx, cl, machineSet) + framework.WaitForCAPIMachineSetsDeleted(ctx, cl, machineSet) + framework.DeleteObjects(ctx, cl, awsMachineTemplate) + deleted = true + _, err = awsClient.DeletePlacementGroup(placementGroupName) + Expect(err).ToNot(HaveOccurred(), "Failed to delete placementgroup") + }() + + awsMachineTemplate = newAWSMachineTemplate(mapiDefaultProviderSpec) + awsMachineTemplate.Spec.Template.Spec.PlacementGroupName = placementGroupName + if err = cl.Create(ctx, awsMachineTemplate); err != nil { + Expect(err).ToNot(HaveOccurred(), "Failed to create awsmachinetemplate") + } + machineSetParams = framework.UpdateCAPIMachineSetName("aws-machineset-75395", machineSetParams) + machineSet, err = framework.CreateCAPIMachineSet(ctx, cl, machineSetParams) + Expect(err).ToNot(HaveOccurred(), "Failed to create CAPI machineset") + framework.WaitForCAPIMachinesRunning(ctx, cl, machineSet.Name) + }) + + //huliu-OCP-75396 - [CAPI] Creating machines using KMS keys from AWS. + It("should be able to run a machine using KMS keys", func() { + awsMachineTemplate = newAWSMachineTemplate(mapiDefaultProviderSpec) + region := mapiDefaultProviderSpec.Placement.Region + if region != "us-east-1" && region != "us-east-2" { + Skip("Region is " + region + ", skip this test scenario because we only created kms key in us-east-1/us-east-2 region") + } + var key string + switch region { + case "us-east-1": + key = "arn:aws:kms:us-east-1:301721915996:key/c471ec83-cfaf-41a2-9241-d9e99c4da344" + case "us-east-2": + key = "arn:aws:kms:us-east-2:301721915996:key/c228ef83-df2c-4151-84c4-d9f39f39a972" + } + awskmsClient := framework.NewAwsKmsClient(framework.GetCredentialsFromCluster(oc)) + _, err = awskmsClient.DescribeKeyByID(key) + if err != nil { + Skip(fmt.Sprintf("Skip because cannot get the key %v", err)) + } + encryptBool := true + awsMachineTemplate.Spec.Template.Spec.NonRootVolumes = []awsv1.Volume{ + { + DeviceName: "/dev/xvda", + Size: 140, + Type: awsv1.VolumeTypeIO1, + IOPS: 5000, + Encrypted: &encryptBool, + EncryptionKey: key, + }, + } + if err := cl.Create(ctx, awsMachineTemplate); err != nil { + Expect(err).ToNot(HaveOccurred(), "Failed to create awsmachinetemplate") + } + machineSetParams = framework.UpdateCAPIMachineSetName("aws-machineset-75396", machineSetParams) + machineSet, err = framework.CreateCAPIMachineSet(ctx, cl, machineSetParams) + Expect(err).ToNot(HaveOccurred(), "Failed to create CAPI machineset") + framework.WaitForCAPIMachinesRunning(ctx, cl, machineSet.Name) + }) +}) + +func getDefaultAWSMAPIProviderSpec(cl client.Client) (*mapiv1.MachineSet, *mapiv1.AWSMachineProviderConfig) { + machineSetList := &mapiv1.MachineSetList{} + + Eventually(func() error { + return cl.List(ctx, machineSetList, client.InNamespace(framework.MachineAPINamespace)) + }, framework.WaitShort, framework.RetryShort).Should(Succeed(), "it should be able to list the MAPI machinesets") + + Expect(machineSetList.Items).ToNot(HaveLen(0), "expected the MAPI machinesets to be present") + machineSet := &machineSetList.Items[0] + Expect(machineSet.Spec.Template.Spec.ProviderSpec.Value).ToNot(BeNil(), "expected the MAPI machinesets ProviderSpec value to not be nil") + + providerSpec := &mapiv1.AWSMachineProviderConfig{} + Expect(yaml.Unmarshal(machineSet.Spec.Template.Spec.ProviderSpec.Value.Raw, providerSpec)).To(Succeed(), "it should be able to unmarshal the raw yaml into providerSpec") + + return machineSet, providerSpec +} + +func newAWSMachineTemplate(mapiProviderSpec *mapiv1.AWSMachineProviderConfig) *awsv1.AWSMachineTemplate { + By("Creating AWS machine template") + + Expect(mapiProviderSpec).ToNot(BeNil(), "expected the mapi ProviderSpec to not be nil") + Expect(mapiProviderSpec.IAMInstanceProfile).ToNot(BeNil(), "expected the mapi IAMInstanceProfile to not be nil") + Expect(mapiProviderSpec.IAMInstanceProfile.ID).ToNot(BeNil(), "expected the mapi IAMInstanceProfile.ID to not be nil") + Expect(mapiProviderSpec.InstanceType).ToNot(BeEmpty(), "expected the mapi InstanceType to not be empty") + Expect(mapiProviderSpec.Placement.AvailabilityZone).ToNot(BeEmpty(), "expected the mapi Placement.AvailabilityZone to not be empty") + Expect(mapiProviderSpec.AMI.ID).ToNot(BeNil(), "expected the mapi AMI.ID to not be nil") + Expect(mapiProviderSpec.SecurityGroups).ToNot(HaveLen(0), "expected the mapi SecurityGroups to be present") + Expect(mapiProviderSpec.SecurityGroups[0].Filters).ToNot(HaveLen(0), "expected the mapi SecurityGroups[0].Filters to be present") + Expect(mapiProviderSpec.SecurityGroups[0].Filters[0].Values).ToNot(HaveLen(0), "expected the mapi SecurityGroups[0].Filters[0].Values to be present") + + var subnet awsv1.AWSResourceReference + + if len(mapiProviderSpec.Subnet.Filters) == 0 { + subnet = awsv1.AWSResourceReference{ + ID: mapiProviderSpec.Subnet.ID, + } + } else { + subnet = awsv1.AWSResourceReference{ + Filters: []awsv1.Filter{ + { + Name: "tag:Name", + Values: mapiProviderSpec.Subnet.Filters[0].Values, + }, + }, + } + } + + uncompressedUserData := true + ami := awsv1.AMIReference{ + ID: mapiProviderSpec.AMI.ID, + } + ignition := &awsv1.Ignition{ + Version: "3.4", + StorageType: awsv1.IgnitionStorageTypeOptionUnencryptedUserData, + } + additionalSecurityGroups := []awsv1.AWSResourceReference{ + { + Filters: []awsv1.Filter{ + { + Name: "tag:Name", + Values: mapiProviderSpec.SecurityGroups[0].Filters[0].Values, + }, + }, + }, + { + Filters: []awsv1.Filter{ + { + Name: "tag:Name", + Values: mapiProviderSpec.SecurityGroups[1].Filters[0].Values, + }, + }, + }, + } + awsmt := capiinfrastructurev1beta2resourcebuilder. + AWSMachineTemplate(). + WithUncompressedUserData(uncompressedUserData). + WithIAMInstanceProfile(*mapiProviderSpec.IAMInstanceProfile.ID). + WithInstanceType(mapiProviderSpec.InstanceType). + WithAMI(ami). + WithIgnition(ignition). + WithSubnet(&subnet). + WithAdditionalSecurityGroups(additionalSecurityGroups). + WithName(awsMachineTemplateName). + WithNamespace(framework.ClusterAPINamespace). + Build() + + return awsmt +} diff --git a/pkg/e2e_test.go b/pkg/e2e_test.go index b132bd7b3..d391f64dd 100644 --- a/pkg/e2e_test.go +++ b/pkg/e2e_test.go @@ -14,6 +14,7 @@ import ( machinev1 "github.com/openshift/api/machine/v1beta1" "github.com/openshift/cluster-api-actuator-pkg/pkg/framework" caov1alpha1 "github.com/openshift/cluster-autoscaler-operator/pkg/apis" + awsv1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" azurev1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" @@ -48,6 +49,10 @@ func init() { if err := azurev1.AddToScheme(scheme.Scheme); err != nil { klog.Fatal(err) } + + if err := awsv1.AddToScheme(scheme.Scheme); err != nil { + klog.Fatal(err) + } } func TestE2E(t *testing.T) { diff --git a/pkg/framework/aws_client.go b/pkg/framework/aws_client.go index 8f239ca65..cc41af881 100644 --- a/pkg/framework/aws_client.go +++ b/pkg/framework/aws_client.go @@ -4,14 +4,56 @@ import ( "fmt" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/kms" "k8s.io/klog" "k8s.io/utils/ptr" ) // AwsClient struct. type AwsClient struct { - Svc *ec2.EC2 + svc *ec2.EC2 +} + +// Init the aws client. +func NewAwsClient(accessKeyID []byte, secureKey []byte, clusterRegion string) *AwsClient { + awsSession := newAwsSession(accessKeyID, secureKey, clusterRegion) + aClient := &AwsClient{ + svc: ec2.New(awsSession), + } + + return aClient +} + +// AwsKmsClient struct. +type AwsKmsClient struct { + kmssvc *kms.KMS +} + +// Init the aws kms client. +func NewAwsKmsClient(accessKeyID []byte, secureKey []byte, clusterRegion string) *AwsKmsClient { + awsSession := newAwsSession(accessKeyID, secureKey, clusterRegion) + kmsClient := &AwsKmsClient{ + kmssvc: kms.New(awsSession), + } + + return kmsClient +} + +// Create aws backend session connection. +func newAwsSession(accessKeyID []byte, secureKey []byte, clusterRegion string) *session.Session { + awsConfig := &aws.Config{ + Region: aws.String(clusterRegion), + Credentials: credentials.NewStaticCredentials( + string(accessKeyID), + string(secureKey), + "", + ), + } + + return session.Must(session.NewSession(awsConfig)) } // CreateCapacityReservation Create CapacityReservation. @@ -24,7 +66,7 @@ func (a *AwsClient) CreateCapacityReservation(instanceType string, instancePlatf InstanceMatchCriteria: aws.String("targeted"), EndDateType: aws.String("unlimited"), } - result, err := a.Svc.CreateCapacityReservation(input) + result, err := a.svc.CreateCapacityReservation(input) if err != nil { return "", fmt.Errorf("error creating capacity reservation: %w", err) @@ -41,7 +83,54 @@ func (a *AwsClient) CancelCapacityReservation(capacityReservationID string) (boo input := &ec2.CancelCapacityReservationInput{ CapacityReservationId: aws.String(capacityReservationID), } - result, err := a.Svc.CancelCapacityReservation(input) + result, err := a.svc.CancelCapacityReservation(input) return ptr.Deref(result.Return, false), err } + +// CreatePlacementGroup Create a PlacementGroup. +func (a *AwsClient) CreatePlacementGroup(groupName string, strategy string, partitionCount ...int64) (string, error) { + var input *ec2.CreatePlacementGroupInput + if len(partitionCount) > 0 { + input = &ec2.CreatePlacementGroupInput{ + GroupName: aws.String(groupName), + PartitionCount: aws.Int64(partitionCount[0]), + Strategy: aws.String(strategy), + } + } else { + input = &ec2.CreatePlacementGroupInput{ + GroupName: aws.String(groupName), + Strategy: aws.String(strategy), + } + } + + result, err := a.svc.CreatePlacementGroup(input) + + if err != nil { + return "", fmt.Errorf("error creating placement group: %w", err) + } + + placementGroupID := ptr.Deref(result.PlacementGroup.GroupId, "") + klog.Infof("The created placementGroupID is %s", placementGroupID) + + return placementGroupID, err +} + +// DeletePlacementGroup Delete a PlacementGroup. +func (a *AwsClient) DeletePlacementGroup(groupName string) (string, error) { + input := &ec2.DeletePlacementGroupInput{ + GroupName: aws.String(groupName), + } + result, err := a.svc.DeletePlacementGroup(input) + + return result.String(), err +} + +// Describes aws customer managed kms key info. +func (akms *AwsKmsClient) DescribeKeyByID(kmsKeyID string) (describeResult *kms.DescribeKeyOutput, err error) { + input := &kms.DescribeKeyInput{ + KeyId: aws.String(kmsKeyID), + } + + return akms.kmssvc.DescribeKey(input) +} diff --git a/pkg/framework/capi_machinesets.go b/pkg/framework/capi_machinesets.go index 87dd5798e..c7866c977 100644 --- a/pkg/framework/capi_machinesets.go +++ b/pkg/framework/capi_machinesets.go @@ -40,6 +40,19 @@ func NewCAPIMachineSetParams(msName, clusterName, failureDomain string, replicas } } +// UpdateCAPIMachineSetName returns CAPIMachineSetParams object with the updated machineset name. +func UpdateCAPIMachineSetName(msName string, params CAPIMachineSetParams) CAPIMachineSetParams { + Expect(msName).ToNot(BeEmpty(), "expected the capi msName to not be empty") + + return CAPIMachineSetParams{ + msName: msName, + clusterName: params.clusterName, + replicas: params.replicas, + infrastructureRef: params.infrastructureRef, + failureDomain: params.failureDomain, + } +} + // CreateCAPIMachineSet creates a new MachineSet resource. func CreateCAPIMachineSet(ctx context.Context, cl client.Client, params CAPIMachineSetParams) (*clusterv1.MachineSet, error) { By(fmt.Sprintf("Creating MachineSet %q", params.msName)) diff --git a/pkg/framework/framework.go b/pkg/framework/framework.go index 9be416c01..f6e40b6ed 100644 --- a/pkg/framework/framework.go +++ b/pkg/framework/framework.go @@ -2,6 +2,7 @@ package framework import ( "context" + "encoding/base64" "errors" "fmt" "os" @@ -12,6 +13,7 @@ import ( . "github.com/onsi/gomega" configv1 "github.com/openshift/api/config/v1" cov1helpers "github.com/openshift/library-go/pkg/config/clusteroperator/v1helpers" + "github.com/tidwall/gjson" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -292,3 +294,22 @@ func SkipIfNotTechPreviewNoUpgrade(oc *gatherer.CLI, cl runtimeclient.Client) { Skip("FeatureSet is not TechPreviewNoUpgradec, skip it!") } } + +// GetCredentialsFromCluster get credentials from cluster. +func GetCredentialsFromCluster(oc *gatherer.CLI) ([]byte, []byte, string) { + awscreds, err := oc.WithoutNamespace().Run("get").Args("secret/aws-creds", "-n", "kube-system", "-o", "json").Output() + if err != nil { + Skip("Unable to get AWS credentials secret, skipping the testing.") + } + + accessKeyIDBase64, secureKeyBase64 := gjson.Get(awscreds, `data.aws_access_key_id`).String(), gjson.Get(awscreds, `data.aws_secret_access_key`).String() + + accessKeyID, err := base64.StdEncoding.DecodeString(accessKeyIDBase64) + Expect(err).NotTo(HaveOccurred(), "Failed to decode accessKeyID") + secureKey, err := base64.StdEncoding.DecodeString(secureKeyBase64) + Expect(err).NotTo(HaveOccurred(), "Failed to decode secureKey") + clusterRegion, err := oc.WithoutNamespace().Run("get").Args("infrastructure", "cluster", "-o=jsonpath={.status.platformStatus.aws.region}").Output() + Expect(err).NotTo(HaveOccurred(), "Failed to get clusterRegion") + + return accessKeyID, secureKey, clusterRegion +} diff --git a/pkg/providers/aws.go b/pkg/providers/aws.go index fd0a50328..0574e7b9a 100644 --- a/pkg/providers/aws.go +++ b/pkg/providers/aws.go @@ -2,16 +2,11 @@ package providers import ( "context" - "encoding/base64" "encoding/json" "fmt" "k8s.io/client-go/kubernetes" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/ec2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -22,48 +17,12 @@ import ( "github.com/openshift/cluster-api-actuator-pkg/pkg/framework" "github.com/openshift/cluster-api-actuator-pkg/pkg/framework/gatherer" - "github.com/tidwall/gjson" ) const ( amiIDMetadataEndpoint = "http://169.254.169.254/latest/meta-data/ami-id" ) -// createAWSClient create AWS client. -func createAWSClient(oc *gatherer.CLI) *framework.AwsClient { - awscreds, err := oc.WithoutNamespace().Run("get").Args("secret/aws-creds", "-n", "kube-system", "-o", "json").Output() - if err != nil { - Skip("Unable to get AWS credentials secret, skipping the testing.") - } - - accessKeyIDBase64, secureKeyBase64 := gjson.Get(awscreds, `data.aws_access_key_id`).String(), gjson.Get(awscreds, `data.aws_secret_access_key`).String() - - accessKeyID, err := base64.StdEncoding.DecodeString(accessKeyIDBase64) - Expect(err).NotTo(HaveOccurred()) - secureKey, err := base64.StdEncoding.DecodeString(secureKeyBase64) - Expect(err).NotTo(HaveOccurred()) - clusterRegion, err := oc.WithoutNamespace().Run("get").Args("infrastructure", "cluster", "-o=jsonpath={.status.platformStatus.aws.region}").Output() - Expect(err).NotTo(HaveOccurred()) - - awsConfig := &aws.Config{ - Region: aws.String(clusterRegion), - Credentials: credentials.NewStaticCredentials( - string(accessKeyID), - string(secureKey), - "", - ), - } - - sess, err := session.NewSession(awsConfig) - Expect(err).ToNot(HaveOccurred()) - - aClient := &framework.AwsClient{ - Svc: ec2.New(sess), - } - - return aClient -} - var _ = Describe("MetadataServiceOptions", framework.LabelCloudProviderSpecific, framework.LabelProviderAWS, func() { var client runtimeclient.Client var clientset *kubernetes.Clientset @@ -278,9 +237,11 @@ var _ = Describe("CapacityReservationID", framework.LabelCloudProviderSpecific, By("Access AWS to create CapacityReservation") oc, _ := framework.NewCLI() - awsClient := createAWSClient(oc) + awsClient := framework.NewAwsClient(framework.GetCredentialsFromCluster(oc)) capacityReservationID, err := awsClient.CreateCapacityReservation(awsProviderConfig.InstanceType, "Linux/UNIX", awsProviderConfig.Placement.AvailabilityZone, 1) - Expect(err).ToNot(HaveOccurred()) + if err != nil { + Skip(fmt.Sprintf("Skip because CreateCapacityReservation failed %v", err)) + } Expect(capacityReservationID).ToNot(Equal("")) defer func() {