diff --git a/cmd/drone-autoscaler/main.go b/cmd/drone-autoscaler/main.go index 4876949..6df0313 100644 --- a/cmd/drone-autoscaler/main.go +++ b/cmd/drone-autoscaler/main.go @@ -312,7 +312,7 @@ func setupProvider(c config.Config) (autoscaler.Provider, error) { amazon.WithSecurityGroup(c.Amazon.SecurityGroup...), amazon.WithSize(c.Amazon.Instance), amazon.WithSizeAlt(c.Amazon.InstanceAlt), - amazon.WithSubnet(c.Amazon.SubnetID), + amazon.WithSubnets(append([]string{c.Amazon.SubnetID}, c.Amazon.SubnetIDsAlt...)), amazon.WithTags(c.Amazon.Tags), amazon.WithUserData(c.Amazon.UserData), amazon.WithUserDataFile(c.Amazon.UserDataFile), diff --git a/config/config.go b/config/config.go index 3b36185..427e4da 100644 --- a/config/config.go +++ b/config/config.go @@ -132,6 +132,7 @@ type ( Retries int SSHKey string SubnetID string `split_words:"true"` + SubnetIDsAlt []string `envconfig:"DRONE_AMAZON_SUBNET_IDS_ALT"` // In the same manner as InstanceAlt, allows fallback to other subnets if provisioning in the main one fails SecurityGroup []string `split_words:"true"` Tags map[string]string UserData string `envconfig:"DRONE_AMAZON_USERDATA"` diff --git a/config/load_test.go b/config/load_test.go index 49b44d0..e9aeb5c 100644 --- a/config/load_test.go +++ b/config/load_test.go @@ -125,6 +125,7 @@ func TestLoad(t *testing.T) { "DRONE_AMAZON_REGION": "us-east-2", "DRONE_AMAZON_SSHKEY": "id_rsa", "DRONE_AMAZON_SUBNET_ID": "subnet-0b32177f", + "DRONE_AMAZON_SUBNET_IDS_ALT": "subnet-abcd,subnet-efgh", "DRONE_AMAZON_SECURITY_GROUP": "sg-770eabe1", "DRONE_AMAZON_TAGS": "os:linux,arch:amd64", "DRONE_AMAZON_USERDATA": "#cloud-init", @@ -264,6 +265,10 @@ var jsonConfig = []byte(`{ "Region": "us-east-2", "SSHKey": "id_rsa", "SubnetID": "subnet-0b32177f", + "SubnetIDsAlt": [ + "subnet-abcd", + "subnet-efgh" + ], "SecurityGroup": [ "sg-770eabe1" ], diff --git a/drivers/amazon/create.go b/drivers/amazon/create.go index 7d999ac..a6be16e 100644 --- a/drivers/amazon/create.go +++ b/drivers/amazon/create.go @@ -8,6 +8,7 @@ import ( "bytes" "context" "encoding/base64" + "fmt" "time" "github.com/drone/autoscaler" @@ -17,16 +18,41 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" ) +type attemptOverrides struct { + attempt int + size string + subnet string +} + func (p *provider) Create(ctx context.Context, opts autoscaler.InstanceCreateOpts) (*autoscaler.Instance, error) { - instance, err := p.create(ctx, opts, false) - // if the instance was successfully provisioned, - // return the instance. - if err == nil { - return instance, err + attemptOverrides := attemptOverrides{ + attempt: 1, + size: p.size, } - // if the instance was provisioned with errors, - // return the instance and the error + tryCreateInAllSubnets := func() (*autoscaler.Instance, error) { + var ( + instance *autoscaler.Instance + err error + ) + for _, subnet := range p.subnets { + attemptOverrides.subnet = subnet + + instance, err = p.create(ctx, opts, attemptOverrides) + // if the instance was provisioned (with or without errors), return the instance. + if instance != nil { + return instance, err + } + + attemptOverrides.attempt++ + } + + return nil, fmt.Errorf("failed to create instance in all subnets: %w", err) + } + + instance, err := tryCreateInAllSubnets() + + // if the instance was provisioned (with or without errors), return the instance. if instance != nil { return instance, err } @@ -34,14 +60,15 @@ func (p *provider) Create(ctx context.Context, opts autoscaler.InstanceCreateOpt // if the instance was not provisioned, and fallback // parameters were provided, retry using the fallback if p.sizeAlt != "" { - instance, err = p.create(ctx, opts, true) + attemptOverrides.size = p.sizeAlt + instance, err = tryCreateInAllSubnets() } // if there is no fallback logic do not retry return instance, err } -func (p *provider) create(ctx context.Context, opts autoscaler.InstanceCreateOpts, retry bool) (*autoscaler.Instance, error) { +func (p *provider) create(ctx context.Context, opts autoscaler.InstanceCreateOpts, overrides attemptOverrides) (*autoscaler.Instance, error) { p.init.Do(func() { p.setup(ctx) }) @@ -76,7 +103,7 @@ func (p *provider) create(ctx context.Context, opts autoscaler.InstanceCreateOpt in := &ec2.RunInstancesInput{ KeyName: aws.String(p.key), ImageId: aws.String(p.image), - InstanceType: aws.String(p.size), + InstanceType: aws.String(overrides.size), MinCount: aws.Int64(1), MaxCount: aws.Int64(1), InstanceMarketOptions: marketOptions, @@ -86,7 +113,7 @@ func (p *provider) create(ctx context.Context, opts autoscaler.InstanceCreateOpt { AssociatePublicIpAddress: aws.Bool(!p.privateIP), DeviceIndex: aws.Int64(0), - SubnetId: aws.String(p.subnet), + SubnetId: aws.String(overrides.subnet), Groups: aws.StringSlice(p.groups), }, }, @@ -125,28 +152,12 @@ func (p *provider) create(ctx context.Context, opts autoscaler.InstanceCreateOpt } logger := logger.FromContext(ctx). - WithField("attempt", 1). + WithField("attempt", overrides.attempt). + WithField("size", overrides.size). + WithField("subnet", overrides.subnet). WithField("region", p.region). WithField("image", p.image). - WithField("size", p.size). WithField("name", opts.Name) - - // TODO(bradyrdzewski) instead of passing a re-try flag - // and then setting parameters, we should instead accept - // an struct that specifies the size, image and any other - // alternate values that one may want to try - - // if this is our second attempt to create the instance, - // re-create using the alternate instance size. - if retry { - in.InstanceType = aws.String(p.sizeAlt) - - // update the logger to reflect this is a retry. - logger = logger. - WithField("size", p.sizeAlt). - WithField("attempt", 2) - } - logger.Debug("instance create") results, err := client.RunInstances(in) diff --git a/drivers/amazon/option.go b/drivers/amazon/option.go index 15aba88..6684dfe 100644 --- a/drivers/amazon/option.go +++ b/drivers/amazon/option.go @@ -78,10 +78,10 @@ func WithSSHKey(key string) Option { } } -// WithSubnet returns an option to set the subnet id. -func WithSubnet(id string) Option { +// WithSubnets returns an option to set the subnet ids. +func WithSubnets(ids []string) Option { return func(p *provider) { - p.subnet = id + p.subnets = ids } } diff --git a/drivers/amazon/option_test.go b/drivers/amazon/option_test.go index a3c4d6a..d154023 100644 --- a/drivers/amazon/option_test.go +++ b/drivers/amazon/option_test.go @@ -16,7 +16,7 @@ func TestOptions(t *testing.T) { WithSecurityGroup("sg-770eabe1"), WithSize("t3.2xlarge"), WithSSHKey("id_rsa"), - WithSubnet("subnet-0b32177f"), + WithSubnets([]string{"subnet-0b32177f"}), WithTags(map[string]string{"foo": "bar", "baz": "qux"}), WithVolumeSize(64), WithVolumeType("io1"), @@ -40,7 +40,7 @@ func TestOptions(t *testing.T) { if got, want := p.groups[0], "sg-770eabe1"; got != want { t.Errorf("Want security groups %q, got %q", want, got) } - if got, want := p.subnet, "subnet-0b32177f"; got != want { + if got, want := p.subnets, []string{"subnet-0b32177f"}; len(got) != 1 || got[0] != want[0] { t.Errorf("Want subnet %q, got %q", want, got) } if got, want := p.retries, 10; got != want { diff --git a/drivers/amazon/provider.go b/drivers/amazon/provider.go index 8b78724..62234a8 100644 --- a/drivers/amazon/provider.go +++ b/drivers/amazon/provider.go @@ -32,7 +32,7 @@ type provider struct { userdata *template.Template size string sizeAlt string - subnet string + subnets []string groups []string tags map[string]string iamProfileArn string diff --git a/drivers/amazon/setup.go b/drivers/amazon/setup.go index c93b064..4651bd2 100644 --- a/drivers/amazon/setup.go +++ b/drivers/amazon/setup.go @@ -21,7 +21,7 @@ func (p *provider) setup(ctx context.Context) error { return p.setupKeypair(ctx) }) } - if p.subnet == "" { + if len(p.subnets) == 0 { // TODO: find or create subnet } if len(p.groups) == 0 {