diff --git a/pkg/controllers/nodeclass/status/ami_test.go b/pkg/controllers/nodeclass/status/ami_test.go index 8ff07296c27f..4eb0cd94a012 100644 --- a/pkg/controllers/nodeclass/status/ami_test.go +++ b/pkg/controllers/nodeclass/status/ami_test.go @@ -33,7 +33,9 @@ import ( ) var _ = Describe("NodeClass AMI Status Controller", func() { + var k8sVersion string BeforeEach(func() { + k8sVersion = lo.Must(awsEnv.VersionProvider.Get(ctx)) nodeClass = test.EC2NodeClass(v1.EC2NodeClass{ Spec: v1.EC2NodeClassSpec{ SubnetSelectorTerms: []v1.SubnetSelectorTerm{ @@ -57,205 +59,435 @@ var _ = Describe("NodeClass AMI Status Controller", func() { awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{ Images: []*ec2.Image{ { - Name: aws.String("test-ami-1"), - ImageId: aws.String("ami-test1"), + Name: aws.String("amd64-standard"), + ImageId: aws.String("ami-amd64-standard"), CreationDate: aws.String(time.Now().Format(time.RFC3339)), Architecture: aws.String("x86_64"), Tags: []*ec2.Tag{ - {Key: aws.String("Name"), Value: aws.String("test-ami-1")}, + {Key: aws.String("Name"), Value: aws.String("amd64-standard")}, {Key: aws.String("foo"), Value: aws.String("bar")}, }, }, { - Name: aws.String("test-ami-2"), - ImageId: aws.String("ami-test2"), + Name: aws.String("amd64-standard-new"), + ImageId: aws.String("ami-amd64-standard-new"), CreationDate: aws.String(time.Now().Add(time.Minute).Format(time.RFC3339)), Architecture: aws.String("x86_64"), Tags: []*ec2.Tag{ - {Key: aws.String("Name"), Value: aws.String("test-ami-2")}, + {Key: aws.String("Name"), Value: aws.String("amd64-standard")}, {Key: aws.String("foo"), Value: aws.String("bar")}, }, }, { - Name: aws.String("test-ami-3"), - ImageId: aws.String("ami-test3"), - CreationDate: aws.String(time.Now().Add(2 * time.Minute).Format(time.RFC3339)), + Name: aws.String("amd64-nvidia"), + ImageId: aws.String("ami-amd64-nvidia"), + CreationDate: aws.String(time.Now().Format(time.RFC3339)), Architecture: aws.String("x86_64"), Tags: []*ec2.Tag{ - {Key: aws.String("Name"), Value: aws.String("test-ami-3")}, + {Key: aws.String("Name"), Value: aws.String("amd64-nvidia")}, {Key: aws.String("foo"), Value: aws.String("bar")}, }, }, - }, - }) - }) - It("should fail to resolve AMIs if the nodeclass has ubuntu incompatible annotation", func() { - nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2) - nodeClass.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{v1.AnnotationUbuntuCompatibilityKey: v1.AnnotationUbuntuCompatibilityIncompatible}) - ExpectApplied(ctx, env.Client, nodeClass) - ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) - nodeClass = ExpectExists(ctx, env.Client, nodeClass) - cond := nodeClass.StatusConditions().Get(v1.ConditionTypeAMIsReady) - Expect(cond.IsTrue()).To(BeFalse()) - Expect(cond.Message).To(Equal("Ubuntu AMI discovery is not supported at v1, refer to the upgrade guide (https://karpenter.sh/docs/upgrading/upgrade-guide/#upgrading-to-100)")) - Expect(cond.Reason).To(Equal("AMINotFound")) - - }) - It("should resolve amiSelector AMIs and requirements into status", func() { - version := lo.Must(awsEnv.VersionProvider.Get(ctx)) - - awsEnv.SSMAPI.Parameters = map[string]string{ - fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", version): "ami-id-123", - fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-gpu/recommended/image_id", version): "ami-id-456", - fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2%s/recommended/image_id", version, fmt.Sprintf("-%s", karpv1.ArchitectureArm64)): "ami-id-789", - } - - awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{ - Images: []*ec2.Image{ { - Name: aws.String("test-ami-1"), - ImageId: aws.String("ami-id-123"), + Name: aws.String("amd64-neuron"), + ImageId: aws.String("ami-amd64-neuron"), CreationDate: aws.String(time.Now().Format(time.RFC3339)), Architecture: aws.String("x86_64"), Tags: []*ec2.Tag{ - {Key: aws.String("Name"), Value: aws.String("test-ami-1")}, + {Key: aws.String("Name"), Value: aws.String("amd64-neuron")}, {Key: aws.String("foo"), Value: aws.String("bar")}, }, }, { - Name: aws.String("test-ami-2"), - ImageId: aws.String("ami-id-456"), - CreationDate: aws.String(time.Now().Add(time.Minute).Format(time.RFC3339)), - Architecture: aws.String("x86_64"), + Name: aws.String("arm64-standard"), + ImageId: aws.String("ami-arm64-standard"), + CreationDate: aws.String(time.Now().Format(time.RFC3339)), + Architecture: aws.String("arm64"), Tags: []*ec2.Tag{ - {Key: aws.String("Name"), Value: aws.String("test-ami-2")}, + {Key: aws.String("Name"), Value: aws.String("arm64-standard")}, {Key: aws.String("foo"), Value: aws.String("bar")}, }, }, { - Name: aws.String("test-ami-3"), - ImageId: aws.String("ami-id-789"), - CreationDate: aws.String(time.Now().Add(2 * time.Minute).Format(time.RFC3339)), + Name: aws.String("arm64-nvidia"), + ImageId: aws.String("ami-arm64-nvidia"), + CreationDate: aws.String(time.Now().Format(time.RFC3339)), Architecture: aws.String("arm64"), Tags: []*ec2.Tag{ - {Key: aws.String("Name"), Value: aws.String("test-ami-3")}, + {Key: aws.String("Name"), Value: aws.String("arm64-nvidia")}, {Key: aws.String("foo"), Value: aws.String("bar")}, }, }, }, }) - nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2) - nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "al2@latest"}} - ExpectApplied(ctx, env.Client, nodeClass) - ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) - nodeClass = ExpectExists(ctx, env.Client, nodeClass) - Expect(len(nodeClass.Status.AMIs)).To(Equal(4)) - Expect(nodeClass.Status.AMIs).To(ContainElements([]v1.AMI{ - { - Name: "test-ami-3", - ID: "ami-id-789", - Requirements: []corev1.NodeSelectorRequirement{ - { - Key: corev1.LabelArchStable, - Operator: corev1.NodeSelectorOpIn, - Values: []string{karpv1.ArchitectureArm64}, + }) + Context("Aliases", func() { + It("Should resolve all AMIs with correct requirements for AL2023", func() { + awsEnv.SSMAPI.Parameters = map[string]string{ + fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", k8sVersion): "ami-amd64-standard", + fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/nvidia/recommended/image_id", k8sVersion): "ami-amd64-nvidia", + fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/neuron/recommended/image_id", k8sVersion): "ami-amd64-neuron", + fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/arm64/standard/recommended/image_id", k8sVersion): "ami-arm64-standard", + } + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "al2023@latest"}} + ExpectApplied(ctx, env.Client, nodeClass) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) + nodeClass = ExpectExists(ctx, env.Client, nodeClass) + + Expect(len(nodeClass.Status.AMIs)).To(Equal(4)) + Expect(nodeClass.Status.AMIs).To(ContainElements([]v1.AMI{ + { + Name: "amd64-standard", + ID: "ami-amd64-standard", + Requirements: []corev1.NodeSelectorRequirement{ + { + Key: corev1.LabelArchStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.ArchitectureAmd64}, + }, + { + Key: v1.LabelInstanceGPUCount, + Operator: corev1.NodeSelectorOpDoesNotExist, + }, + { + Key: v1.LabelInstanceAcceleratorCount, + Operator: corev1.NodeSelectorOpDoesNotExist, + }, }, - { - Key: v1.LabelInstanceGPUCount, - Operator: corev1.NodeSelectorOpDoesNotExist, + }, + { + Name: "amd64-nvidia", + ID: "ami-amd64-nvidia", + Requirements: []corev1.NodeSelectorRequirement{ + { + Key: corev1.LabelArchStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.ArchitectureAmd64}, + }, + { + Key: v1.LabelInstanceGPUCount, + Operator: corev1.NodeSelectorOpExists, + }, }, - { - Key: v1.LabelInstanceAcceleratorCount, - Operator: corev1.NodeSelectorOpDoesNotExist, + }, + { + Name: "amd64-neuron", + ID: "ami-amd64-neuron", + Requirements: []corev1.NodeSelectorRequirement{ + { + Key: corev1.LabelArchStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.ArchitectureAmd64}, + }, + { + Key: v1.LabelInstanceAcceleratorCount, + Operator: corev1.NodeSelectorOpExists, + }, }, }, - }, - { - Name: "test-ami-2", - ID: "ami-id-456", - Requirements: []corev1.NodeSelectorRequirement{ - { - Key: corev1.LabelArchStable, - Operator: corev1.NodeSelectorOpIn, - Values: []string{karpv1.ArchitectureAmd64}, + { + Name: "arm64-standard", + ID: "ami-arm64-standard", + Requirements: []corev1.NodeSelectorRequirement{ + { + Key: corev1.LabelArchStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.ArchitectureArm64}, + }, + { + Key: v1.LabelInstanceGPUCount, + Operator: corev1.NodeSelectorOpDoesNotExist, + }, + { + Key: v1.LabelInstanceAcceleratorCount, + Operator: corev1.NodeSelectorOpDoesNotExist, + }, }, - { - Key: v1.LabelInstanceGPUCount, - Operator: corev1.NodeSelectorOpExists, + }, + })) + Expect(nodeClass.StatusConditions().IsTrue(v1.ConditionTypeAMIsReady)).To(BeTrue()) + }) + It("Should resolve all AMIs with correct requirements for AL2", func() { + awsEnv.SSMAPI.Parameters = map[string]string{ + fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", k8sVersion): "ami-amd64-standard", + fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-gpu/recommended/image_id", k8sVersion): "ami-amd64-nvidia", + fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-arm64/recommended/image_id", k8sVersion): "ami-arm64-standard", + } + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "al2@latest"}} + ExpectApplied(ctx, env.Client, nodeClass) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) + nodeClass = ExpectExists(ctx, env.Client, nodeClass) + + Expect(len(nodeClass.Status.AMIs)).To(Equal(4)) + Expect(nodeClass.Status.AMIs).To(ContainElements([]v1.AMI{ + { + Name: "amd64-standard", + ID: "ami-amd64-standard", + Requirements: []corev1.NodeSelectorRequirement{ + { + Key: corev1.LabelArchStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.ArchitectureAmd64}, + }, + { + Key: v1.LabelInstanceGPUCount, + Operator: corev1.NodeSelectorOpDoesNotExist, + }, + { + Key: v1.LabelInstanceAcceleratorCount, + Operator: corev1.NodeSelectorOpDoesNotExist, + }, }, }, - }, - { - Name: "test-ami-2", - ID: "ami-id-456", - Requirements: []corev1.NodeSelectorRequirement{ - { - Key: corev1.LabelArchStable, - Operator: corev1.NodeSelectorOpIn, - Values: []string{karpv1.ArchitectureAmd64}, + { + Name: "amd64-nvidia", + ID: "ami-amd64-nvidia", + Requirements: []corev1.NodeSelectorRequirement{ + { + Key: corev1.LabelArchStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.ArchitectureAmd64}, + }, + { + Key: v1.LabelInstanceGPUCount, + Operator: corev1.NodeSelectorOpExists, + }, }, - { - Key: v1.LabelInstanceAcceleratorCount, - Operator: corev1.NodeSelectorOpExists, + }, + // Note: AL2 uses the same AMI for nvidia and neuron, we use the nvidia AMI here + { + Name: "amd64-nvidia", + ID: "ami-amd64-nvidia", + Requirements: []corev1.NodeSelectorRequirement{ + { + Key: corev1.LabelArchStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.ArchitectureAmd64}, + }, + { + Key: v1.LabelInstanceAcceleratorCount, + Operator: corev1.NodeSelectorOpExists, + }, }, }, - }, - { - Name: "test-ami-1", - ID: "ami-id-123", - Requirements: []corev1.NodeSelectorRequirement{ - { - Key: corev1.LabelArchStable, - Operator: corev1.NodeSelectorOpIn, - Values: []string{karpv1.ArchitectureAmd64}, + { + Name: "arm64-standard", + ID: "ami-arm64-standard", + Requirements: []corev1.NodeSelectorRequirement{ + { + Key: corev1.LabelArchStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.ArchitectureArm64}, + }, + { + Key: v1.LabelInstanceGPUCount, + Operator: corev1.NodeSelectorOpDoesNotExist, + }, + { + Key: v1.LabelInstanceAcceleratorCount, + Operator: corev1.NodeSelectorOpDoesNotExist, + }, }, - { - Key: v1.LabelInstanceGPUCount, - Operator: corev1.NodeSelectorOpDoesNotExist, + }, + })) + Expect(nodeClass.StatusConditions().IsTrue(v1.ConditionTypeAMIsReady)).To(BeTrue()) + }) + It("Should resolve all AMIs with correct requirements for Bottlerocket", func() { + awsEnv.SSMAPI.Parameters = map[string]string{ + fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/x86_64/latest/image_id", k8sVersion): "ami-amd64-standard", + fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/arm64/latest/image_id", k8sVersion): "ami-arm64-standard", + fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s-nvidia/x86_64/latest/image_id", k8sVersion): "ami-amd64-nvidia", + fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s-nvidia/arm64/latest/image_id", k8sVersion): "ami-arm64-nvidia", + } + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "bottlerocket@latest"}} + ExpectApplied(ctx, env.Client, nodeClass) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) + nodeClass = ExpectExists(ctx, env.Client, nodeClass) + + Expect(len(nodeClass.Status.AMIs)).To(Equal(6)) + Expect(nodeClass.Status.AMIs).To(ContainElements([]v1.AMI{ + { + Name: "amd64-standard", + ID: "ami-amd64-standard", + Requirements: []corev1.NodeSelectorRequirement{ + { + Key: corev1.LabelArchStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.ArchitectureAmd64}, + }, + { + Key: v1.LabelInstanceGPUCount, + Operator: corev1.NodeSelectorOpDoesNotExist, + }, + { + Key: v1.LabelInstanceAcceleratorCount, + Operator: corev1.NodeSelectorOpDoesNotExist, + }, }, - { - Key: v1.LabelInstanceAcceleratorCount, - Operator: corev1.NodeSelectorOpDoesNotExist, + }, + { + Name: "arm64-standard", + ID: "ami-arm64-standard", + Requirements: []corev1.NodeSelectorRequirement{ + { + Key: corev1.LabelArchStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.ArchitectureArm64}, + }, + { + Key: v1.LabelInstanceGPUCount, + Operator: corev1.NodeSelectorOpDoesNotExist, + }, + { + Key: v1.LabelInstanceAcceleratorCount, + Operator: corev1.NodeSelectorOpDoesNotExist, + }, }, }, - }, - })) - Expect(nodeClass.StatusConditions().IsTrue(v1.ConditionTypeAMIsReady)).To(BeTrue()) - }) - It("should resolve amiSelector AMis and requirements into status when all SSM aliases don't resolve", func() { - version := lo.Must(awsEnv.VersionProvider.Get(ctx)) - // This parameter set doesn't include any of the Nvidia AMIs - awsEnv.SSMAPI.Parameters = map[string]string{ - fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/x86_64/latest/image_id", version): "ami-id-123", - fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/arm64/latest/image_id", version): "ami-id-456", - } - nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ - Alias: "bottlerocket@latest", - }} - awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{ - Images: []*ec2.Image{ { - Name: aws.String("test-ami-1"), - ImageId: aws.String("ami-id-123"), - CreationDate: aws.String(time.Now().Format(time.RFC3339)), - Architecture: aws.String("x86_64"), - Tags: []*ec2.Tag{ - {Key: aws.String("Name"), Value: aws.String("test-ami-1")}, - {Key: aws.String("foo"), Value: aws.String("bar")}, + Name: "amd64-nvidia", + ID: "ami-amd64-nvidia", + Requirements: []corev1.NodeSelectorRequirement{ + { + Key: corev1.LabelArchStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.ArchitectureAmd64}, + }, + { + Key: v1.LabelInstanceGPUCount, + Operator: corev1.NodeSelectorOpExists, + }, }, }, { - Name: aws.String("test-ami-2"), - ImageId: aws.String("ami-id-456"), - CreationDate: aws.String(time.Now().Add(time.Minute).Format(time.RFC3339)), - Architecture: aws.String("arm64"), - Tags: []*ec2.Tag{ - {Key: aws.String("Name"), Value: aws.String("test-ami-2")}, - {Key: aws.String("foo"), Value: aws.String("bar")}, + Name: "arm64-nvidia", + ID: "ami-arm64-nvidia", + Requirements: []corev1.NodeSelectorRequirement{ + { + Key: corev1.LabelArchStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.ArchitectureArm64}, + }, + { + Key: v1.LabelInstanceGPUCount, + Operator: corev1.NodeSelectorOpExists, + }, }, }, - }, + // Note: Bottlerocket uses the same AMI for nvidia and neuron, we use the nvidia AMI here + { + Name: "amd64-nvidia", + ID: "ami-amd64-nvidia", + Requirements: []corev1.NodeSelectorRequirement{ + { + Key: corev1.LabelArchStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.ArchitectureAmd64}, + }, + { + Key: v1.LabelInstanceAcceleratorCount, + Operator: corev1.NodeSelectorOpExists, + }, + }, + }, + { + Name: "arm64-nvidia", + ID: "ami-arm64-nvidia", + Requirements: []corev1.NodeSelectorRequirement{ + { + Key: corev1.LabelArchStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.ArchitectureArm64}, + }, + { + Key: v1.LabelInstanceAcceleratorCount, + Operator: corev1.NodeSelectorOpExists, + }, + }, + }, + })) + Expect(nodeClass.StatusConditions().IsTrue(v1.ConditionTypeAMIsReady)).To(BeTrue()) + }) + It("Should resolve all AMIs with correct requirements for Windows2019", func() { + awsEnv.SSMAPI.Parameters = map[string]string{ + fmt.Sprintf("/aws/service/ami-windows-latest/Windows_Server-2019-English-Core-EKS_Optimized-%s/image_id", k8sVersion): "ami-amd64-standard", + } + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "windows2019@latest"}} + ExpectApplied(ctx, env.Client, nodeClass) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) + nodeClass = ExpectExists(ctx, env.Client, nodeClass) + + Expect(len(nodeClass.Status.AMIs)).To(Equal(1)) + Expect(nodeClass.Status.AMIs).To(ContainElements([]v1.AMI{ + { + Name: "amd64-standard", + ID: "ami-amd64-standard", + Requirements: []corev1.NodeSelectorRequirement{ + { + Key: corev1.LabelOSStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{string(corev1.Windows)}, + }, + { + Key: corev1.LabelArchStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.ArchitectureAmd64}, + }, + { + Key: corev1.LabelWindowsBuild, + Operator: corev1.NodeSelectorOpIn, + Values: []string{v1.Windows2019Build}, + }, + }, + }, + })) + Expect(nodeClass.StatusConditions().IsTrue(v1.ConditionTypeAMIsReady)).To(BeTrue()) + }) + It("Should resolve all AMIs with correct requirements for Windows2022", func() { + awsEnv.SSMAPI.Parameters = map[string]string{ + fmt.Sprintf("/aws/service/ami-windows-latest/Windows_Server-2022-English-Core-EKS_Optimized-%s/image_id", k8sVersion): "ami-amd64-standard", + } + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Alias: "windows2022@latest"}} + ExpectApplied(ctx, env.Client, nodeClass) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) + nodeClass = ExpectExists(ctx, env.Client, nodeClass) + + Expect(len(nodeClass.Status.AMIs)).To(Equal(1)) + Expect(nodeClass.Status.AMIs).To(ContainElements([]v1.AMI{ + { + Name: "amd64-standard", + ID: "ami-amd64-standard", + Requirements: []corev1.NodeSelectorRequirement{ + { + Key: corev1.LabelOSStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{string(corev1.Windows)}, + }, + { + Key: corev1.LabelArchStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.ArchitectureAmd64}, + }, + { + Key: corev1.LabelWindowsBuild, + Operator: corev1.NodeSelectorOpIn, + Values: []string{v1.Windows2022Build}, + }, + }, + }, + })) + Expect(nodeClass.StatusConditions().IsTrue(v1.ConditionTypeAMIsReady)).To(BeTrue()) }) + }) + It("should resolve amiSelector AMIs and requirements into status when all SSM parameters don't resolve", func() { + // This parameter set doesn't include any of the Nvidia AMIs + awsEnv.SSMAPI.Parameters = map[string]string{ + fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/x86_64/latest/image_id", k8sVersion): "ami-amd64-standard", + fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/arm64/latest/image_id", k8sVersion): "ami-arm64-standard", + } + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ + Alias: "bottlerocket@latest", + }} ExpectApplied(ctx, env.Client, nodeClass) ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) @@ -263,8 +495,8 @@ var _ = Describe("NodeClass AMI Status Controller", func() { Expect(len(nodeClass.Status.AMIs)).To(Equal(2)) Expect(nodeClass.Status.AMIs).To(ContainElements([]v1.AMI{ { - Name: "test-ami-2", - ID: "ami-id-456", + Name: "arm64-standard", + ID: "ami-arm64-standard", Requirements: []corev1.NodeSelectorRequirement{ { Key: corev1.LabelArchStable, @@ -282,8 +514,8 @@ var _ = Describe("NodeClass AMI Status Controller", func() { }, }, { - Name: "test-ami-1", - ID: "ami-id-123", + Name: "amd64-standard", + ID: "ami-amd64-standard", Requirements: []corev1.NodeSelectorRequirement{ { Key: corev1.LabelArchStable, @@ -303,21 +535,23 @@ var _ = Describe("NodeClass AMI Status Controller", func() { })) Expect(nodeClass.StatusConditions().IsTrue(v1.ConditionTypeAMIsReady)).To(BeTrue()) }) - It("Should resolve a valid AMI selector", func() { + It("should resolve a valid AMI selector", func() { + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ + Tags: map[string]string{"Name": "amd64-standard"}, + }} ExpectApplied(ctx, env.Client, nodeClass) ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) nodeClass = ExpectExists(ctx, env.Client, nodeClass) Expect(nodeClass.Status.AMIs).To(Equal( []v1.AMI{ { - Name: "test-ami-3", - ID: "ami-test3", + Name: "amd64-standard-new", + ID: "ami-amd64-standard-new", Requirements: []corev1.NodeSelectorRequirement{{ Key: corev1.LabelArchStable, Operator: corev1.NodeSelectorOpIn, Values: []string{karpv1.ArchitectureAmd64}, - }, - }, + }}, }, }, )) @@ -330,4 +564,15 @@ var _ = Describe("NodeClass AMI Status Controller", func() { nodeClass = ExpectExists(ctx, env.Client, nodeClass) Expect(nodeClass.StatusConditions().IsTrue(v1.ConditionTypeAMIsReady)).To(BeFalse()) }) + It("should fail to resolve AMIs if the nodeclass has ubuntu incompatible annotation", func() { + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2) + nodeClass.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{v1.AnnotationUbuntuCompatibilityKey: v1.AnnotationUbuntuCompatibilityIncompatible}) + ExpectApplied(ctx, env.Client, nodeClass) + ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass) + nodeClass = ExpectExists(ctx, env.Client, nodeClass) + cond := nodeClass.StatusConditions().Get(v1.ConditionTypeAMIsReady) + Expect(cond.IsTrue()).To(BeFalse()) + Expect(cond.Message).To(Equal("Ubuntu AMI discovery is not supported at v1, refer to the upgrade guide (https://karpenter.sh/docs/upgrading/upgrade-guide/#upgrading-to-100)")) + Expect(cond.Reason).To(Equal("AMINotFound")) + }) }) diff --git a/pkg/providers/amifamily/types.go b/pkg/providers/amifamily/types.go index ac37d20b06b0..5dae3150a1cc 100644 --- a/pkg/providers/amifamily/types.go +++ b/pkg/providers/amifamily/types.go @@ -82,9 +82,9 @@ func (v Variant) Requirements() scheduling.Requirements { scheduling.NewRequirement(v1.LabelInstanceGPUCount, corev1.NodeSelectorOpDoesNotExist), ) case VariantNvidia: - return scheduling.NewRequirements(scheduling.NewRequirement(v1.LabelInstanceAcceleratorCount, corev1.NodeSelectorOpExists)) - case VariantNeuron: return scheduling.NewRequirements(scheduling.NewRequirement(v1.LabelInstanceGPUCount, corev1.NodeSelectorOpExists)) + case VariantNeuron: + return scheduling.NewRequirements(scheduling.NewRequirement(v1.LabelInstanceAcceleratorCount, corev1.NodeSelectorOpExists)) } return nil }