From 8c67ffe912a758795a407939e47f9d5ff0ea69d6 Mon Sep 17 00:00:00 2001 From: Jenna Goldstrich Date: Thu, 24 Aug 2023 12:24:08 -0700 Subject: [PATCH] Add WIP Shallow Replication, need to fix unit test with new publish function --- builder/azure/arm/builder.go | 1 + builder/azure/arm/config.go | 2 + builder/azure/arm/config.hcl2spec.go | 18 ++--- .../step_publish_to_shared_image_gallery.go | 69 ++++++++++++++----- builder/azure/common/constants/stateBag.go | 1 + ...edImageGalleryDestination-not-required.mdx | 2 + 6 files changed, 67 insertions(+), 26 deletions(-) diff --git a/builder/azure/arm/builder.go b/builder/azure/arm/builder.go index 61b61da7..c9d7826c 100644 --- a/builder/azure/arm/builder.go +++ b/builder/azure/arm/builder.go @@ -528,6 +528,7 @@ func (b *Builder) configureStateBag(stateBag multistep.StateBag) { stateBag.Put(constants.ArmManagedImageSharedGalleryImageVersion, b.config.SharedGalleryDestination.SigDestinationImageVersion) stateBag.Put(constants.ArmManagedImageSharedGalleryImageVersionStorageAccountType, b.config.SharedGalleryDestination.SigDestinationStorageAccountType) stateBag.Put(constants.ArmSharedImageGalleryDestinationSpecialized, b.config.SharedGalleryDestination.SigDestinationSpecialized) + stateBag.Put(constants.ArmSharedImageGalleryDestinationShallowReplication, b.config.SharedGalleryDestination.SigDestinationShallowReplicationMode) stateBag.Put(constants.ArmManagedImageSubscription, b.config.ClientConfig.SubscriptionID) stateBag.Put(constants.ArmManagedImageSharedGalleryImageVersionEndOfLifeDate, b.config.SharedGalleryImageVersionEndOfLifeDate) stateBag.Put(constants.ArmManagedImageSharedGalleryImageVersionReplicaCount, b.config.SharedGalleryImageVersionReplicaCount) diff --git a/builder/azure/arm/config.go b/builder/azure/arm/config.go index 1bcafdb0..f13ff0f9 100644 --- a/builder/azure/arm/config.go +++ b/builder/azure/arm/config.go @@ -108,6 +108,8 @@ type SharedImageGalleryDestination struct { SigDestinationStorageAccountType string `mapstructure:"storage_account_type"` // Set to true if publishing to a Specialized Gallery, this skips a call to set the build VM's OS state as Generalized SigDestinationSpecialized bool `mapstructure:"specialized"` + // Set to true if you want to publish using Shallow Replication, this is faster but skips copying the full image to the gallery image region, read more about Shallow Replication [here](https://learn.microsoft.com/en-us/azure/virtual-machines/shared-image-galleries?tabs=azure-cli#shallow-replication) + SigDestinationShallowReplicationMode bool `mapstructure:"shallow_replication"` } type Spot struct { diff --git a/builder/azure/arm/config.hcl2spec.go b/builder/azure/arm/config.hcl2spec.go index b576e440..ce473b0f 100644 --- a/builder/azure/arm/config.hcl2spec.go +++ b/builder/azure/arm/config.hcl2spec.go @@ -359,14 +359,15 @@ func (*FlatSharedImageGallery) HCL2Spec() map[string]hcldec.Spec { // FlatSharedImageGalleryDestination is an auto-generated flat version of SharedImageGalleryDestination. // Where the contents of a field with a `mapstructure:,squash` tag are bubbled up. type FlatSharedImageGalleryDestination struct { - SigDestinationSubscription *string `mapstructure:"subscription" cty:"subscription" hcl:"subscription"` - SigDestinationResourceGroup *string `mapstructure:"resource_group" cty:"resource_group" hcl:"resource_group"` - SigDestinationGalleryName *string `mapstructure:"gallery_name" cty:"gallery_name" hcl:"gallery_name"` - SigDestinationImageName *string `mapstructure:"image_name" cty:"image_name" hcl:"image_name"` - SigDestinationImageVersion *string `mapstructure:"image_version" cty:"image_version" hcl:"image_version"` - SigDestinationReplicationRegions []string `mapstructure:"replication_regions" cty:"replication_regions" hcl:"replication_regions"` - SigDestinationStorageAccountType *string `mapstructure:"storage_account_type" cty:"storage_account_type" hcl:"storage_account_type"` - SigDestinationSpecialized *bool `mapstructure:"specialized" cty:"specialized" hcl:"specialized"` + SigDestinationSubscription *string `mapstructure:"subscription" cty:"subscription" hcl:"subscription"` + SigDestinationResourceGroup *string `mapstructure:"resource_group" cty:"resource_group" hcl:"resource_group"` + SigDestinationGalleryName *string `mapstructure:"gallery_name" cty:"gallery_name" hcl:"gallery_name"` + SigDestinationImageName *string `mapstructure:"image_name" cty:"image_name" hcl:"image_name"` + SigDestinationImageVersion *string `mapstructure:"image_version" cty:"image_version" hcl:"image_version"` + SigDestinationReplicationRegions []string `mapstructure:"replication_regions" cty:"replication_regions" hcl:"replication_regions"` + SigDestinationStorageAccountType *string `mapstructure:"storage_account_type" cty:"storage_account_type" hcl:"storage_account_type"` + SigDestinationSpecialized *bool `mapstructure:"specialized" cty:"specialized" hcl:"specialized"` + SigDestinationShallowReplicationMode *bool `mapstructure:"shallow_replication" cty:"shallow_replication" hcl:"shallow_replication"` } // FlatMapstructure returns a new FlatSharedImageGalleryDestination. @@ -389,6 +390,7 @@ func (*FlatSharedImageGalleryDestination) HCL2Spec() map[string]hcldec.Spec { "replication_regions": &hcldec.AttrSpec{Name: "replication_regions", Type: cty.List(cty.String), Required: false}, "storage_account_type": &hcldec.AttrSpec{Name: "storage_account_type", Type: cty.String, Required: false}, "specialized": &hcldec.AttrSpec{Name: "specialized", Type: cty.Bool, Required: false}, + "shallow_replication": &hcldec.AttrSpec{Name: "shallow_replication", Type: cty.Bool, Required: false}, } return s } diff --git a/builder/azure/arm/step_publish_to_shared_image_gallery.go b/builder/azure/arm/step_publish_to_shared_image_gallery.go index 776988b4..84041c83 100644 --- a/builder/azure/arm/step_publish_to_shared_image_gallery.go +++ b/builder/azure/arm/step_publish_to_shared_image_gallery.go @@ -16,12 +16,26 @@ import ( type StepPublishToSharedImageGallery struct { client *AzureClient - publish func(ctx context.Context, subscriptionID string, sourceID string, sharedImageGallery SharedImageGalleryDestination, miSGImageVersionEndOfLifeDate string, miSGImageVersionExcludeFromLatest bool, miSigReplicaCount int64, location string, diskEncryptionSetId string, tags map[string]string) (string, error) + publish func(publishArgs PublishArgs) (string, error) say func(message string) error func(e error) toSIG func() bool } +type PublishArgs struct { + ctx context.Context + subscriptionID string + sourceID string + sharedImageGallery SharedImageGalleryDestination + miSGImageVersionEndOfLifeDate string + miSGImageVersionExcludeFromLatest bool + miSigReplicaCount int64 + location string + diskEncryptionSetId string + shallowReplicationMode bool + tags map[string]string +} + func NewStepPublishToSharedImageGallery(client *AzureClient, ui packersdk.Ui, config *Config) *StepPublishToSharedImageGallery { var step = &StepPublishToSharedImageGallery{ client: client, @@ -72,58 +86,63 @@ func getSigDestination(state multistep.StateBag) SharedImageGalleryDestination { } } -func (s *StepPublishToSharedImageGallery) publishToSig(ctx context.Context, subscriptionID string, sourceID string, sharedImageGallery SharedImageGalleryDestination, miSGImageVersionEndOfLifeDate string, miSGImageVersionExcludeFromLatest bool, miSigReplicaCount int64, location string, diskEncryptionSetId string, tags map[string]string) (string, error) { - replicationRegions := make([]galleryimageversions.TargetRegion, len(sharedImageGallery.SigDestinationReplicationRegions)) - for i, v := range sharedImageGallery.SigDestinationReplicationRegions { +func (s *StepPublishToSharedImageGallery) publishToSig(publishArgs PublishArgs) (string, error) { + replicationRegions := make([]galleryimageversions.TargetRegion, len(publishArgs.sharedImageGallery.SigDestinationReplicationRegions)) + for i, v := range publishArgs.sharedImageGallery.SigDestinationReplicationRegions { regionName := v replicationRegions[i] = galleryimageversions.TargetRegion{Name: regionName} } - storageAccountType, err := getSigDestinationStorageAccountType(sharedImageGallery.SigDestinationStorageAccountType) + storageAccountType, err := getSigDestinationStorageAccountType(publishArgs.sharedImageGallery.SigDestinationStorageAccountType) if err != nil { s.error(err) return "", err } - if diskEncryptionSetId != "" { + if publishArgs.diskEncryptionSetId != "" { for index, targetRegion := range replicationRegions { targetRegion.Encryption = &galleryimageversions.EncryptionImages{ OsDiskImage: &galleryimageversions.OSDiskImageEncryption{ - DiskEncryptionSetId: &diskEncryptionSetId, + DiskEncryptionSetId: &publishArgs.diskEncryptionSetId, }, } replicationRegions[index] = targetRegion } } + replicationMode := galleryimageversions.ReplicationModeFull + if true { + replicationMode = galleryimageversions.ReplicationModeShallow + } galleryImageVersion := galleryimageversions.GalleryImageVersion{ - Location: location, - Tags: &tags, + Location: publishArgs.location, + Tags: &publishArgs.tags, Properties: &galleryimageversions.GalleryImageVersionProperties{ StorageProfile: galleryimageversions.GalleryImageVersionStorageProfile{ Source: &galleryimageversions.GalleryArtifactVersionFullSource{ - Id: &sourceID, + Id: &publishArgs.sourceID, }, }, PublishingProfile: &galleryimageversions.GalleryArtifactPublishingProfileBase{ TargetRegions: &replicationRegions, - EndOfLifeDate: &miSGImageVersionEndOfLifeDate, - ExcludeFromLatest: &miSGImageVersionExcludeFromLatest, - ReplicaCount: &miSigReplicaCount, + EndOfLifeDate: &publishArgs.miSGImageVersionEndOfLifeDate, + ExcludeFromLatest: &publishArgs.miSGImageVersionExcludeFromLatest, + ReplicaCount: &publishArgs.miSigReplicaCount, + ReplicationMode: &replicationMode, StorageAccountType: &storageAccountType, }, }, } - pollingContext, cancel := context.WithTimeout(ctx, s.client.SharedGalleryTimeout) + pollingContext, cancel := context.WithTimeout(publishArgs.ctx, s.client.SharedGalleryTimeout) defer cancel() - galleryImageVersionId := galleryimageversions.NewImageVersionID(subscriptionID, sharedImageGallery.SigDestinationResourceGroup, sharedImageGallery.SigDestinationGalleryName, sharedImageGallery.SigDestinationImageName, sharedImageGallery.SigDestinationImageVersion) + galleryImageVersionId := galleryimageversions.NewImageVersionID(publishArgs.subscriptionID, publishArgs.sharedImageGallery.SigDestinationResourceGroup, publishArgs.sharedImageGallery.SigDestinationGalleryName, publishArgs.sharedImageGallery.SigDestinationImageName, publishArgs.sharedImageGallery.SigDestinationImageVersion) err = s.client.GalleryImageVersionsClient.CreateOrUpdateThenPoll(pollingContext, galleryImageVersionId, galleryImageVersion) if err != nil { s.say(s.client.LastError.Error()) return "", err } - createdSGImageVersion, err := s.client.GalleryImageVersionsClient.Get(ctx, galleryImageVersionId, galleryimageversions.DefaultGetOperationOptions()) + createdSGImageVersion, err := s.client.GalleryImageVersionsClient.Get(publishArgs.ctx, galleryImageVersionId, galleryimageversions.DefaultGetOperationOptions()) if err != nil { s.say(s.client.LastError.Error()) @@ -187,9 +206,23 @@ func (s *StepPublishToSharedImageGallery) Run(ctx context.Context, stateBag mult s.say(fmt.Sprintf(" -> SIG image version endoflife date : '%s'", miSGImageVersionEndOfLifeDate)) s.say(fmt.Sprintf(" -> SIG image version exclude from latest : '%t'", miSGImageVersionExcludeFromLatest)) s.say(fmt.Sprintf(" -> SIG replica count [1, 100] : '%d'", miSigReplicaCount)) - + shallowReplicationMode := stateBag.Get(constants.ArmSharedImageGalleryDestinationShallowReplication).(bool) subscriptionID := stateBag.Get(constants.ArmSharedImageGalleryDestinationSubscription).(string) - createdGalleryImageVersionID, err := s.publish(ctx, subscriptionID, sourceID, sharedImageGallery, miSGImageVersionEndOfLifeDate, miSGImageVersionExcludeFromLatest, miSigReplicaCount, location, diskEncryptionSetId, tags) + createdGalleryImageVersionID, err := s.publish( + PublishArgs{ + ctx: ctx, + subscriptionID: subscriptionID, + sourceID: sourceID, + sharedImageGallery: sharedImageGallery, + miSGImageVersionEndOfLifeDate: miSGImageVersionEndOfLifeDate, + miSGImageVersionExcludeFromLatest: miSGImageVersionExcludeFromLatest, + miSigReplicaCount: miSigReplicaCount, + location: location, + diskEncryptionSetId: diskEncryptionSetId, + shallowReplicationMode: shallowReplicationMode, + tags: tags, + }, + ) if err != nil { stateBag.Put(constants.Error, err) diff --git a/builder/azure/common/constants/stateBag.go b/builder/azure/common/constants/stateBag.go index 4670bdb0..07baba27 100644 --- a/builder/azure/common/constants/stateBag.go +++ b/builder/azure/common/constants/stateBag.go @@ -55,6 +55,7 @@ const ( ArmManagedImageSharedGalleryImageVersionStorageAccountType string = "arm.ArmManagedImageSharedGalleryImageVersionStorageAccountType" ArmSharedImageGalleryDestinationSubscription string = "arm.ArmSharedImageGalleryDestinationSubscription" ArmSharedImageGalleryDestinationSpecialized string = "arm.ArmSharedImageGalleryDestinationSpecialized" + ArmSharedImageGalleryDestinationShallowReplication string = "arm.ArmSharedImageGalleryDestinationShallowReplication" ArmManagedImageSubscription string = "arm.ArmManagedImageSubscription" ArmAsyncResourceGroupDelete string = "arm.AsyncResourceGroupDelete" ArmManagedImageOSDiskSnapshotName string = "arm.ManagedImageOSDiskSnapshotName" diff --git a/docs-partials/builder/azure/arm/SharedImageGalleryDestination-not-required.mdx b/docs-partials/builder/azure/arm/SharedImageGalleryDestination-not-required.mdx index b20654c9..a858ad40 100644 --- a/docs-partials/builder/azure/arm/SharedImageGalleryDestination-not-required.mdx +++ b/docs-partials/builder/azure/arm/SharedImageGalleryDestination-not-required.mdx @@ -17,4 +17,6 @@ - `specialized` (bool) - Set to true if publishing to a Specialized Gallery, this skips a call to set the build VM's OS state as Generalized +- `shallow_replication` (bool) - Set to true if you want to publish using Shallow Replication, this is faster but skips copying the full image to the gallery image region, read more about Shallow Replication [here](https://learn.microsoft.com/en-us/azure/virtual-machines/shared-image-galleries?tabs=azure-cli#shallow-replication) +