Fetch latest E2E test AMI ID dynamically

cmd/integration_test/build/script/create_infra_config.sh

@@ -21,7 +21,6 @@ cat << EOF > ${INTEGRATION_TEST_INFRA_CONFIG}
 ---
 
 ec2:
-  amiId: ${INTEGRATION_TEST_AL2_AMI_ID}
   subnetId: ${INTEGRATION_TEST_SUBNET_ID}
 
 vSphere:

internal/pkg/ec2/create.go

@@ -4,6 +4,7 @@ import (
 	"encoding/base64"
 	"fmt"
 	"math/rand"
+	"reflect"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -15,6 +16,13 @@ import (
 	"github.com/aws/eks-anywhere/pkg/retrier"
 )
 
+const (
+	e2eTestAMINameFilter        = "eksa-integration-test-AL2-*"
+	e2eTestAMIDescriptionFilter = "*Kernel version 5.X*"
+	e2eTestAMIOwner             = "857151390494"
+	timeLayout                  = time.RFC3339
+)
+
 var dockerLogsUserData = `
 #!/bin/bash
 cat <<'EOF' >> /etc/docker/daemon.json
@@ -26,7 +34,10 @@ EOF
 systemctl restart docker --no-block
 `
 
-func CreateInstance(session *session.Session, amiId, key, tag, instanceProfileName, subnetId, name string) (string, error) {
+// CreateInstance launches a test runner EC2 instance in the given subnet and
+// with the given properties. The AMI ID used for the instance is dynamically
+// obtained.
+func CreateInstance(session *session.Session, key, tag, instanceProfileName, subnetID, name string) (string, error) {
 	r := retrier.New(180*time.Minute, retrier.WithBackoffFactor(1.5), retrier.WithRetryPolicy(func(totalRetries int, err error) (retry bool, wait time.Duration) {
 		// EC2 Request token bucket has a refill rate of 2 request tokens
 		// per second, so waiting between 5 and 10 seconds per retry with a backoff factor of 1.5 should be sufficient
@@ -41,12 +52,47 @@ func CreateInstance(session *session.Session, amiId, key, tag, instanceProfileNa
 	}))
 
 	service := ec2.New(session)
-	var result *ec2.Reservation
 
+	var describeImagesResult *ec2.DescribeImagesOutput
 	err := r.Retry(func() error {
+		var err error
+		describeImagesResult, err = service.DescribeImages(&ec2.DescribeImagesInput{
+			Filters: []*ec2.Filter{
+				{
+					Name: aws.String("name"),
+					Values: []*string{
+						aws.String(e2eTestAMINameFilter),
+					},
+				},
+				{
+					Name: aws.String("description"),
+					Values: []*string{
+						aws.String(e2eTestAMIDescriptionFilter),
+					},
+				},
+			},
+			Owners: []*string{
+				aws.String(e2eTestAMIOwner),
+			},
+		})
+
+		return err
+	})
+	if err != nil {
+		return "", fmt.Errorf("retries exhausted when trying to describe images: %v", err)
+	}
+
+	amiID, err := getLatestAMIID(describeImagesResult.Images)
+	if err != nil {
+		return "", fmt.Errorf("getting latest AMI ID by creation time: %v", err)
+	}
+	fmt.Printf("Found latest E2E test AMI: %v\n", amiID)
+
+	var result *ec2.Reservation
+	err = r.Retry(func() error {
 		var err error
 		result, err = service.RunInstances(&ec2.RunInstancesInput{
-			ImageId:      aws.String(amiId),
+			ImageId:      aws.String(amiID),
 			InstanceType: aws.String("t3.2xlarge"),
 			MinCount:     aws.Int64(1),
 			MaxCount:     aws.Int64(1),
@@ -61,7 +107,7 @@ func CreateInstance(session *session.Session, amiId, key, tag, instanceProfileNa
 			IamInstanceProfile: &ec2.IamInstanceProfileSpecification{
 				Name: aws.String(instanceProfileName),
 			},
-			SubnetId: aws.String(subnetId),
+			SubnetId: aws.String(subnetID),
 			TagSpecifications: []*ec2.TagSpecification{
 				{
 					ResourceType: aws.String("instance"),
@@ -111,3 +157,30 @@ func isThrottleError(err error) bool {
 
 	return false
 }
+
+func getLatestAMIID(images []*ec2.Image) (string, error) {
+	latestImage := &ec2.Image{}
+	latestImage.CreationDate = aws.String(time.Time{}.Format(timeLayout))
+	for _, image := range images {
+		if image.CreationDate != nil {
+			imageCreationTime, err := time.Parse(timeLayout, *image.CreationDate)
+			if err != nil {
+				return "", fmt.Errorf("parsing creation timestamp for image: %v", err)
+			}
+
+			latestImageCreationTime, err := time.Parse(timeLayout, *latestImage.CreationDate)
+			if err != nil {
+				return "", fmt.Errorf("parsing creation timestamp for current latest image: %v", err)
+			}
+			if latestImageCreationTime.Before(imageCreationTime) {
+				latestImage = image
+			}
+		}
+	}
+
+	if reflect.DeepEqual(latestImage, &ec2.Image{CreationDate: aws.String(time.Time{}.Format(timeLayout))}) {
+		return "", fmt.Errorf("no matching AMIs found")
+	}
+
+	return *latestImage.ImageId, nil
+}

internal/pkg/ec2/create_test.go

@@ -0,0 +1,72 @@
+package ec2
+
+import (
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+)
+
+const incorrectTimeLayout = time.RFC1123
+
+func TestGetLatestAMIID(t *testing.T) {
+	tests := []struct {
+		name      string
+		images    []*ec2.Image
+		wantAMIID string
+		wantErr   bool
+	}{
+		{
+			name: "Get latest image ID success",
+			images: []*ec2.Image{
+				{
+					ImageId:      aws.String("ami-01234"),
+					CreationDate: aws.String(time.Now().Format(timeLayout)),
+				},
+				{
+					ImageId:      aws.String("ami-56789"),
+					CreationDate: aws.String(time.Time{}.Format(timeLayout)),
+				},
+			},
+			wantAMIID: "ami-01234",
+			wantErr:   false,
+		},
+		{
+			name: "Get latest image ID failure - incorrect time layout",
+			images: []*ec2.Image{
+				{
+					ImageId:      aws.String("ami-01234"),
+					CreationDate: aws.String(time.Now().Format(timeLayout)),
+				},
+				{
+					ImageId:      aws.String("ami-56789"),
+					CreationDate: aws.String(time.Time{}.Format(incorrectTimeLayout)),
+				},
+			},
+			wantAMIID: "",
+			wantErr:   true,
+		},
+		{
+			name:      "Get latest image ID failure - empty images list",
+			images:    []*ec2.Image{},
+			wantAMIID: "",
+			wantErr:   true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotImageID, err := getLatestAMIID(tt.images)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("getLatestAMIID() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if err == nil {
+				if gotImageID != tt.wantAMIID {
+					t.Errorf("getLatestAMIID() got = %v, want %v", gotImageID, tt.wantAMIID)
+				}
+			}
+		})
+	}
+}

internal/test/e2e/testRunner.go

@@ -91,7 +91,6 @@ type testRunner struct {
 
 type Ec2TestRunner struct {
 	testRunner
-	AmiID    string `yaml:"amiId"`
 	SubnetID string `yaml:"subnetId"`
 }
 
@@ -187,7 +186,7 @@ func (v *VSphereTestRunner) createInstance(c instanceRunConf) (string, error) {
 func (e *Ec2TestRunner) createInstance(c instanceRunConf) (string, error) {
 	name := getTestRunnerName(e.logger, c.JobID)
 	e.logger.V(1).Info("Creating ec2 Test Runner instance", "name", name)
-	instanceID, err := ec2.CreateInstance(c.Session, e.AmiID, key, tag, c.InstanceProfileName, e.SubnetID, name)
+	instanceID, err := ec2.CreateInstance(c.Session, key, tag, c.InstanceProfileName, e.SubnetID, name)
 	if err != nil {
 		return "", fmt.Errorf("creating instance for e2e tests: %v", err)
 	}

scripts/e2e_test_docker.sh

@@ -59,7 +59,6 @@ cat << EOF > ${INTEGRATION_TEST_INFRA_CONFIG}
 ---
 
 ec2:
-  amiId: ${INTEGRATION_TEST_AL2_AMI_ID}
   subnetId:
 
 EOF