Skip to content

Commit

Permalink
provisioner: add support for configuring AWS IMDSv2 access method (ky…
Browse files Browse the repository at this point in the history 
…ma-project#3429)

* provisioner: add support for configuring AWS IMDSv2 access method

* address comments
  • Loading branch information
@ebensom
ebensom authored Apr 29, 2024
1 parent 9dbba9d commit ea75326
Show file tree
Hide file tree
Showing 13 changed files with 239 additions and 42 deletions.
3 changes: 2 additions & 1 deletion components/provisioner/cmd/init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,11 +45,12 @@ func newProvisioningService(
shootUpgradeQueue queue.OperationQueue,
defaultEnableKubernetesVersionAutoUpdate,
defaultEnableMachineImageVersionAutoUpdate bool,
defaultEnableIMDSv2 bool,
runtimeRegistrationEnabled bool,
dynamicKubeconfigProvider DynamicKubeconfigProvider) provisioning.Service {

uuidGenerator := uuid.NewUUIDGenerator()
inputConverter := provisioning.NewInputConverter(uuidGenerator, gardenerProject, defaultEnableKubernetesVersionAutoUpdate, defaultEnableMachineImageVersionAutoUpdate)
inputConverter := provisioning.NewInputConverter(uuidGenerator, gardenerProject, defaultEnableKubernetesVersionAutoUpdate, defaultEnableMachineImageVersionAutoUpdate, defaultEnableIMDSv2)
graphQLConverter := provisioning.NewGraphQLConverter()

return provisioning.NewProvisioningService(
Expand Down
6 changes: 4 additions & 2 deletions components/provisioner/cmd/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ type config struct {
ClusterCleanupResourceSelector string `envconfig:"default=https://service-manager."`
DefaultEnableKubernetesVersionAutoUpdate bool `envconfig:"default=false"`
DefaultEnableMachineImageVersionAutoUpdate bool `envconfig:"default=false"`
DefaultEnableIMDSv2 bool `envconfig:"default=false"`
}

LatestDownloadedReleases int `envconfig:"default=5"`
Expand All @@ -99,7 +100,7 @@ func (c *config) String() string {
"DeprovisioningNoInstallTimeoutClusterDeletion: %s, DeprovisioningNoInstallTimeoutWaitingForClusterDeletion: %s "+
"ShootUpgradeTimeout: %s, "+
"OperatorRoleBindingL2SubjectName: %s, OperatorRoleBindingL3SubjectName: %s, OperatorRoleBindingCreatingForAdmin: %t "+
"GardenerProject: %s, GardenerKubeconfigPath: %s, GardenerAuditLogsPolicyConfigMap: %s, AuditLogsTenantConfigPath: %s, "+
"GardenerProject: %s, GardenerKubeconfigPath: %s, GardenerAuditLogsPolicyConfigMap: %s, AuditLogsTenantConfigPath: %s, DefaultEnableIMDSv2: %v"+
"LatestDownloadedReleases: %d, DownloadPreReleases: %v, "+
"EnqueueInProgressOperations: %v"+
"LogLevel: %s",
Expand All @@ -114,7 +115,7 @@ func (c *config) String() string {
c.DeprovisioningTimeout.ClusterDeletion.String(), c.DeprovisioningTimeout.WaitingForClusterDeletion.String(),
c.ProvisioningTimeout.ShootUpgrade.String(),
c.OperatorRoleBinding.L2SubjectName, c.OperatorRoleBinding.L3SubjectName, c.OperatorRoleBinding.CreatingForAdmin,
c.Gardener.Project, c.Gardener.KubeconfigPath, c.Gardener.AuditLogsPolicyConfigMap, c.Gardener.AuditLogsTenantConfigPath,
c.Gardener.Project, c.Gardener.KubeconfigPath, c.Gardener.AuditLogsPolicyConfigMap, c.Gardener.AuditLogsTenantConfigPath, c.Gardener.DefaultEnableIMDSv2,
c.LatestDownloadedReleases, c.DownloadPreReleases,
c.EnqueueInProgressOperations,
c.LogLevel)
Expand Down Expand Up @@ -232,6 +233,7 @@ func main() {
shootUpgradeQueue,
cfg.Gardener.DefaultEnableKubernetesVersionAutoUpdate,
cfg.Gardener.DefaultEnableMachineImageVersionAutoUpdate,
cfg.Gardener.DefaultEnableIMDSv2,
cfg.RuntimeRegistrationEnabled,
kubeconfigProvider,
)
Expand Down
3 changes: 2 additions & 1 deletion components/provisioner/internal/api/resolver_integration_with_gardener_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ const (

defaultEnableKubernetesVersionAutoUpdate = false
defaultEnableMachineImageVersionAutoUpdate = false
defaultEnableIMDSv2 = true

mockedKubeconfig = `apiVersion: v1
clusters:
Expand Down Expand Up @@ -203,7 +204,7 @@ func TestProvisioning_ProvisionRuntimeWithDatabase(t *testing.T) {
uuidGenerator := uuid.NewUUIDGenerator()
provisioner := gardener.NewProvisioner(namespace, shootInterface, dbsFactory, auditLogPolicyCMName, maintenanceWindowConfigPath)

inputConverter := provisioning.NewInputConverter(uuidGenerator, "Project", defaultEnableKubernetesVersionAutoUpdate, defaultEnableMachineImageVersionAutoUpdate)
inputConverter := provisioning.NewInputConverter(uuidGenerator, "Project", defaultEnableKubernetesVersionAutoUpdate, defaultEnableMachineImageVersionAutoUpdate, defaultEnableIMDSv2)
graphQLConverter := provisioning.NewGraphQLConverter()

runtimeRegistrationEnabled := true
Expand Down
51 changes: 48 additions & 3 deletions components/provisioner/internal/model/gardener_config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ const (
)

var networkingType = "calico"
var awsIMDSv2HTTPPutResponseHopLimit int64 = 2

type OIDCConfig struct {
ClientID string `json:"clientID"`
Expand Down Expand Up @@ -553,8 +554,9 @@ func (c AWSGardenerConfig) AsProviderSpecificConfig() gqlschema.ProviderSpecific
}

return gqlschema.AWSProviderConfig{
AwsZones: zones,
VpcCidr: &c.input.VpcCidr,
AwsZones: zones,
VpcCidr: &c.input.VpcCidr,
EnableIMDSv2: c.input.EnableIMDSv2,
}
}

Expand Down Expand Up @@ -590,7 +592,41 @@ func (c AWSGardenerConfig) ValidateShootConfigChange(shoot *gardener_types.Shoot
}

func (c AWSGardenerConfig) EditShootConfig(gardenerConfig GardenerConfig, shoot *gardener_types.Shoot) apperrors.AppError {
return updateShootConfig(gardenerConfig, shoot)
err := updateShootConfig(gardenerConfig, shoot)
if err != nil {
return err
}

if c.input.EnableIMDSv2 != nil && *c.input.EnableIMDSv2 {
var (
workerConfig *aws.WorkerConfig
)
if shoot.Spec.Provider.Workers[0].ProviderConfig == nil {
workerConfig = NewAWSWorkerConfig(awsIMDSv2HTTPPutResponseHopLimit)
} else {
workerConfig = &aws.WorkerConfig{}
err := json.Unmarshal(shoot.Spec.Provider.Workers[0].ProviderConfig.Raw, &workerConfig)
if err != nil {
return apperrors.Internal("error decoding aws worker config: %s", err.Error())
}
if workerConfig.InstanceMetadataOptions == nil {
workerConfig.InstanceMetadataOptions = &aws.InstanceMetadataOptions{}
}
if workerConfig.InstanceMetadataOptions.HTTPTokens == nil || *workerConfig.InstanceMetadataOptions.HTTPTokens != aws.HTTPTokensRequired {
workerConfig.InstanceMetadataOptions.HTTPTokens = &aws.HTTPTokensRequired
}
if workerConfig.InstanceMetadataOptions.HTTPPutResponseHopLimit == nil || *workerConfig.InstanceMetadataOptions.HTTPPutResponseHopLimit != awsIMDSv2HTTPPutResponseHopLimit {
workerConfig.InstanceMetadataOptions.HTTPPutResponseHopLimit = &awsIMDSv2HTTPPutResponseHopLimit
}
}
jsonWCData, err := json.Marshal(workerConfig)
if err != nil {
return apperrors.Internal("error encoding aws worker config: %s", err.Error())
}
shoot.Spec.Provider.Workers[0].ProviderConfig = &apimachineryRuntime.RawExtension{Raw: jsonWCData}
}

return nil
}

func (c AWSGardenerConfig) ExtendShootConfig(gardenerConfig GardenerConfig, shoot *gardener_types.Shoot) apperrors.AppError {
Expand All @@ -612,6 +648,15 @@ func (c AWSGardenerConfig) ExtendShootConfig(gardenerConfig GardenerConfig, shoo
return apperrors.Internal("error encoding control plane config: %s", err.Error())
}

if c.input.EnableIMDSv2 != nil && *c.input.EnableIMDSv2 {
awsWorkerConfig := NewAWSWorkerConfig(awsIMDSv2HTTPPutResponseHopLimit)
jsonWCData, err := json.Marshal(awsWorkerConfig)
if err != nil {
return apperrors.Internal("error encoding aws worker config: %s", err.Error())
}
workers[0].ProviderConfig = &apimachineryRuntime.RawExtension{Raw: jsonWCData}
}

shoot.Spec.Provider = gardener_types.Provider{
Type: "aws",
ControlPlaneConfig: &apimachineryRuntime.RawExtension{Raw: jsonCPData},
Expand Down
52 changes: 36 additions & 16 deletions components/provisioner/internal/model/gardener_config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ func Test_NewGardenerConfigFromJSON(t *testing.T) {
azureConfigJSON := `{"vnetCidr":"10.10.11.11/255", "zones":["fix-az-zone-1", "fix-az-zone-2"], "enableNatGateway":true, "idleConnectionTimeoutMinutes":4}`
azureNoZonesConfigJSON := `{"vnetCidr":"10.10.11.11/255"}`
azureZoneSubnetsConfigJSON := `{"vnetCidr":"10.10.11.11/255", "azureZones":[{"name":1,"cidr":"10.10.11.12/255"}, {"name":2,"cidr":"10.10.11.13/255"}], "enableNatGateway":true, "idleConnectionTimeoutMinutes":4}`
awsConfigJSON := `{"vpcCidr":"10.10.11.11/255","awsZones":[{"name":"zone","publicCidr":"10.10.11.12/255","internalCidr":"10.10.11.13/255","workerCidr":"10.10.11.11/255"}]}
awsConfigJSON := `{"vpcCidr":"10.10.11.11/255","awsZones":[{"name":"zone","publicCidr":"10.10.11.12/255","internalCidr":"10.10.11.13/255","workerCidr":"10.10.11.11/255"}], "enableIMDSv2": true}
`

for _, testCase := range []struct {
Expand Down Expand Up @@ -109,7 +109,8 @@ func Test_NewGardenerConfigFromJSON(t *testing.T) {
WorkerCidr: "10.10.11.11/255",
},
},
VpcCidr: "10.10.11.11/255",
VpcCidr: "10.10.11.11/255",
EnableIMDSv2: util.PtrTo(true),
},
},
expectedProviderSpecificConfig: gqlschema.AWSProviderConfig{
Expand All @@ -121,7 +122,8 @@ func Test_NewGardenerConfigFromJSON(t *testing.T) {
WorkerCidr: util.PtrTo("10.10.11.11/255"),
},
},
VpcCidr: util.PtrTo("10.10.11.11/255"),
VpcCidr: util.PtrTo("10.10.11.11/255"),
EnableIMDSv2: util.PtrTo(true),
},
},
} {
Expand Down Expand Up @@ -158,7 +160,7 @@ func TestGardenerConfig_ToShootTemplate(t *testing.T) {
azureZoneSubnetsGardenerProvider, err := NewAzureGardenerConfig(fixAzureZoneSubnetsInput(true))
require.NoError(t, err)

awsGardenerProvider, err := NewAWSGardenerConfig(fixAWSGardenerInput())
awsGardenerProvider, err := NewAWSGardenerConfig(fixAWSGardenerInput(true))
require.NoError(t, err)

for _, testCase := range []struct {
Expand Down Expand Up @@ -202,7 +204,7 @@ func TestGardenerConfig_ToShootTemplate(t *testing.T) {
Raw: []byte(`{"kind":"InfrastructureConfig","apiVersion":"gcp.provider.extensions.gardener.cloud/v1alpha1","networks":{"worker":"10.10.10.10/255","workers":"10.10.10.10/255"}}`),
},
Workers: []gardener_types.Worker{
fixWorker([]string{"fix-zone-1", "fix-zone-2"}),
fixWorker([]string{"fix-zone-1", "fix-zone-2"}, nil),
},
},
Purpose: &purpose,
Expand Down Expand Up @@ -284,7 +286,7 @@ func TestGardenerConfig_ToShootTemplate(t *testing.T) {
Raw: []byte(`{"kind":"InfrastructureConfig","apiVersion":"azure.provider.extensions.gardener.cloud/v1alpha1","networks":{"vnet":{"cidr":"10.10.11.11/255"},"workers":"10.10.10.10/255","natGateway":{"enabled":true,"idleConnectionTimeoutMinutes":4}},"zoned":true}`),
},
Workers: []gardener_types.Worker{
fixWorker([]string{"fix-zone-1", "fix-zone-2"}),
fixWorker([]string{"fix-zone-1", "fix-zone-2"}, nil),
},
},
Purpose: &purpose,
Expand Down Expand Up @@ -366,7 +368,7 @@ func TestGardenerConfig_ToShootTemplate(t *testing.T) {
Raw: []byte(`{"kind":"InfrastructureConfig","apiVersion":"azure.provider.extensions.gardener.cloud/v1alpha1","networks":{"vnet":{"cidr":"10.10.11.11/255"},"workers":"10.10.10.10/255"},"zoned":false}`),
},
Workers: []gardener_types.Worker{
fixWorker(nil),
fixWorker(nil, nil),
},
},
Purpose: &purpose,
Expand Down Expand Up @@ -448,7 +450,7 @@ func TestGardenerConfig_ToShootTemplate(t *testing.T) {
Raw: []byte(`{"kind":"InfrastructureConfig","apiVersion":"azure.provider.extensions.gardener.cloud/v1alpha1","networks":{"vnet":{"cidr":"10.10.11.11/255"},"zones":[{"name":1,"cidr":"10.10.11.12/255","natGateway":{"enabled":true,"idleConnectionTimeoutMinutes":4}},{"name":2,"cidr":"10.10.11.13/255","natGateway":{"enabled":true,"idleConnectionTimeoutMinutes":4}}]},"zoned":true}`),
},
Workers: []gardener_types.Worker{
fixWorker([]string{"1", "2"}),
fixWorker([]string{"1", "2"}, nil),
},
},
Purpose: &purpose,
Expand Down Expand Up @@ -530,7 +532,9 @@ func TestGardenerConfig_ToShootTemplate(t *testing.T) {
Raw: []byte(`{"kind":"InfrastructureConfig","apiVersion":"aws.provider.extensions.gardener.cloud/v1alpha1","networks":{"vpc":{"cidr":"10.10.11.11/255"},"zones":[{"name":"zone","internal":"10.10.11.13/255","public":"10.10.11.12/255","workers":"10.10.11.12/255"}]}}`),
},
Workers: []gardener_types.Worker{
fixWorker([]string{"zone"}),
fixWorker([]string{"zone"}, &apimachineryRuntime.RawExtension{
Raw: []byte(`{"kind":"WorkerConfig","apiVersion":"aws.provider.extensions.gardener.cloud/v1alpha1","instanceMetadataOptions":{"httpTokens":"required","httpPutResponseHopLimit":2}}`),
}),
},
},
Purpose: &purpose,
Expand Down Expand Up @@ -645,7 +649,10 @@ func TestEditShootConfig(t *testing.T) {
ToWorker()).
ToShoot()

awsProviderConfig, err := NewAWSGardenerConfig(fixAWSGardenerInput())
awsProviderConfig, err := NewAWSGardenerConfig(fixAWSGardenerInput(false))
require.NoError(t, err)

awsProviderConfigWithEnableIMDSv2, err := NewAWSGardenerConfig(fixAWSGardenerInput(true))
require.NoError(t, err)

azureProviderConfig, err := NewAzureGardenerConfig(fixAzureGardenerInput(zones, nil))
Expand All @@ -665,6 +672,11 @@ func TestEditShootConfig(t *testing.T) {
Raw: []byte(azureProviderConfig.RawJSON()),
}

expectedShootConfigWithIMDSv2Enabled := expectedShoot.DeepCopy()
expectedShootConfigWithIMDSv2Enabled.Spec.Provider.Workers[0].ProviderConfig = &apimachineryRuntime.RawExtension{
Raw: []byte(`{"kind":"WorkerConfig","apiVersion":"aws.provider.extensions.gardener.cloud/v1alpha1","instanceMetadataOptions":{"httpTokens":"required","httpPutResponseHopLimit":2}}`),
}

expectedShootWithNATEnabled := expectedShoot.DeepCopy()
expectedShootWithNATEnabled.Spec.Provider.InfrastructureConfig = &apimachineryRuntime.RawExtension{
Raw: []byte(`{"networks":{"vnet":{"cidr":"10.10.11.11/255"},"natGateway":{"enabled":true,"idleConnectionTimeoutMinutes":4}},"zoned":false}`),
Expand Down Expand Up @@ -693,6 +705,12 @@ func TestEditShootConfig(t *testing.T) {
initialShoot: initialShoot.DeepCopy(),
expectedShoot: expectedShoot.DeepCopy(),
},
{description: "should edit AWS shoot template with IMDSv2 enabled",
provider: "aws",
upgradeConfig: fixGardenerConfig("aws", awsProviderConfigWithEnableIMDSv2),
initialShoot: initialShoot.DeepCopy(),
expectedShoot: expectedShootConfigWithIMDSv2Enabled.DeepCopy(),
},
{description: "should edit Azure shoot template",
provider: "az",
upgradeConfig: fixGardenerConfig("az", azureProviderConfig),
Expand Down Expand Up @@ -806,7 +824,7 @@ func fixGardenerConfig(provider string, providerCfg GardenerProviderConfig) Gard
}
}

func fixAWSGardenerInput() *gqlschema.AWSProviderConfigInput {
func fixAWSGardenerInput(enableIMDSv2 bool) *gqlschema.AWSProviderConfigInput {
return &gqlschema.AWSProviderConfigInput{
AwsZones: []*gqlschema.AWSZoneInput{
{
Expand All @@ -816,7 +834,8 @@ func fixAWSGardenerInput() *gqlschema.AWSProviderConfigInput {
WorkerCidr: "10.10.11.12/255",
},
},
VpcCidr: "10.10.11.11/255",
VpcCidr: "10.10.11.11/255",
EnableIMDSv2: &enableIMDSv2,
}
}

Expand Down Expand Up @@ -846,7 +865,7 @@ func fixAzureZoneSubnetsInput(enableNAT bool) *gqlschema.AzureProviderConfigInpu
}
}

func fixWorker(zones []string) gardener_types.Worker {
func fixWorker(zones []string, providerConfig *apimachineryRuntime.RawExtension) gardener_types.Worker {
return gardener_types.Worker{
Name: "cpu-worker-0",
MaxSurge: util.PtrTo(intstr.FromInt(30)),
Expand All @@ -862,9 +881,10 @@ func fixWorker(zones []string) gardener_types.Worker {
Type: util.PtrTo("SSD"),
VolumeSize: "30Gi",
},
Maximum: 3,
Minimum: 1,
Zones: zones,
Maximum: 3,
Minimum: 1,
Zones: zones,
ProviderConfig: providerConfig,
}
}

Expand Down
15 changes: 15 additions & 0 deletions components/provisioner/internal/model/infrastructure.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
const (
infrastructureConfigKind = "InfrastructureConfig"
controlPlaneConfigKind = "ControlPlaneConfig"
workerConfigKind = "WorkerConfig"

gcpAPIVersion = "gcp.provider.extensions.gardener.cloud/v1alpha1"
azureAPIVersion = "azure.provider.extensions.gardener.cloud/v1alpha1"
Expand Down Expand Up @@ -166,3 +167,17 @@ func NewOpenStackControlPlane(loadBalancerProvider string) *openstack.ControlPla
LoadBalancerProvider: loadBalancerProvider,
}
}

func NewAWSWorkerConfig(httpPutResponseHopLimit int64) *aws.WorkerConfig {

return &aws.WorkerConfig{
TypeMeta: v1.TypeMeta{
APIVersion: awsAPIVersion,
Kind: workerConfigKind,
},
InstanceMetadataOptions: &aws.InstanceMetadataOptions{
HTTPTokens: &aws.HTTPTokensRequired,
HTTPPutResponseHopLimit: util.PtrTo(httpPutResponseHopLimit),
},
}
}
34 changes: 34 additions & 0 deletions components/provisioner/internal/model/infrastructure/aws/worker.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
package aws

import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// WorkerConfig contains configuration settings for the worker nodes.
type WorkerConfig struct {
metav1.TypeMeta `json:",inline"`

// InstanceMetadataOptions contains configuration for controlling access to the metadata API.
InstanceMetadataOptions *InstanceMetadataOptions `json:"instanceMetadataOptions,omitempty"`
}

// HTTPTokensValue is a constant for HTTPTokens values.
type HTTPTokensValue string

var (
// HTTPTokensRequired is a constant for requiring the use of tokens to access IMDS. Effectively disables access via
// the IMDSv1 endpoints.
HTTPTokensRequired HTTPTokensValue = "required"
// HTTPTokensOptional that makes the use of tokens for IMDS optional. Effectively allows access via both IMDSv1 and
// IMDSv2 endpoints.
HTTPTokensOptional HTTPTokensValue = "optional"
)

// InstanceMetadataOptions contains configuration for controlling access to the metadata API.
type InstanceMetadataOptions struct {
// HTTPTokens enforces the use of metadata v2 API.
HTTPTokens *HTTPTokensValue `json:"httpTokens,omitempty"`
// HTTPPutResponseHopLimit is the response hop limit for instance metadata requests.
// Valid values are between 1 and 64.
HTTPPutResponseHopLimit *int64 `json:"httpPutResponseHopLimit,omitempty"`
}
Loading

0 comments on commit ea75326

Please sign in to comment.