Skip to content

Commit

Permalink
feat: allow target replacements on all types
Browse files Browse the repository at this point in the history
  • Loading branch information
@joshiste
joshiste committed Jul 10, 2024
1 parent 916d2d2 commit 4856510
Show file tree
Hide file tree
Showing 6 changed files with 131 additions and 76 deletions.
6 changes: 3 additions & 3 deletions charts/steadybit-extension-loadtest/templates/statefulset.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,9 +90,9 @@ spec:
- name: STEADYBIT_EXTENSION_ATTRIBUTE_UPDATES
value: {{ .Values.loadtest.attributeUpdates | toJson | quote }}
{{ end }}
{{ if .Values.loadtest.containerTargetCreationsAndDeletions -}}
- name: STEADYBIT_EXTENSION_CONTAINER_TARGET_CREATIONS_AND_DELETIONS
value: {{ .Values.loadtest.containerTargetCreationsAndDeletions | toJson | quote }}
{{ if .Values.loadtest.targetReplacements -}}
- name: STEADYBIT_EXTENSION_TARGET_REPLACEMENTS
value: {{ .Values.loadtest.targetReplacements | toJson | quote }}
{{ end }}
{{ if .Values.discovery.attributes.excludes.container -}}
- name: STEADYBIT_EXTENSION_DISCOVERY_ATTRIBUTES_EXCLUDES_CONTAINER
Expand Down
6 changes: 3 additions & 3 deletions charts/steadybit-extension-loadtest/tests/__snapshot__/statefulset_test.yaml.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,9 @@ manifest should match snapshot:
- name: STEADYBIT_EXTENSION_PODS_PER_DEPLOYMENT
value: "4"
- name: STEADYBIT_EXTENSION_ATTRIBUTE_UPDATES
value: '[{"attributeName":"aws-ec2.label.change-ts","interval":600,"rate":0.2,"type":"com.steadybit.extension_aws.ec2-instance"},{"attributeName":"container.label.change-ts","interval":180,"rate":0.2,"type":"com.steadybit.extension_container.container"},{"attributeName":"k8s.label.change-ts","interval":180,"rate":0.2,"type":"com.steadybit.extension_kubernetes.kubernetes-container"},{"attributeName":"k8s.label.change-ts","interval":180,"rate":0.2,"type":"com.steadybit.extension_kubernetes.kubernetes-deployment"}]'
- name: STEADYBIT_EXTENSION_CONTAINER_TARGET_CREATIONS_AND_DELETIONS
value: '{"count":10,"interval":180}'
value: '[{"attributeName":"aws-ec2.label.change-ts","interval":600,"rate":0.05,"type":"com.steadybit.extension_aws.ec2-instance"},{"attributeName":"container.label.change-ts","interval":180,"rate":0.05,"type":"com.steadybit.extension_container.container"},{"attributeName":"k8s.label.change-ts","interval":180,"rate":0.05,"type":"com.steadybit.extension_kubernetes.kubernetes-container"},{"attributeName":"k8s.label.change-ts","interval":180,"rate":0.05,"type":"com.steadybit.extension_kubernetes.kubernetes-deployment"}]'
- name: STEADYBIT_EXTENSION_TARGET_REPLACEMENTS
value: '[{"count":1,"interval":600,"type":"com.steadybit.extension_aws.ec2-instance"},{"count":2,"interval":180,"type":"com.steadybit.extension_container.container"},{"count":2,"interval":180,"type":"com.steadybit.extension_kubernetes.kubernetes-container"},{"count":2,"interval":180,"type":"com.steadybit.extension_kubernetes.kubernetes-deployment"}]'
- name: STEADYBIT_EXTENSION_SERVICES_ENABLED
value: "false"
- name: STEADYBIT_EXTENSION_DISABLE_AWS_DISCOVERY
Expand Down
24 changes: 17 additions & 7 deletions charts/steadybit-extension-loadtest/values.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,23 +32,33 @@ loadtest:
attributeUpdates:
- type: "com.steadybit.extension_aws.ec2-instance"
attributeName: "aws-ec2.label.change-ts"
rate: 0.20
rate: 0.05
interval: 600
- type: "com.steadybit.extension_container.container"
attributeName: "container.label.change-ts"
rate: 0.20
rate: 0.05
interval: 180
- type: "com.steadybit.extension_kubernetes.kubernetes-container"
attributeName: "k8s.label.change-ts"
rate: 0.20
rate: 0.05
interval: 180
- type: "com.steadybit.extension_kubernetes.kubernetes-deployment"
attributeName: "k8s.label.change-ts"
rate: 0.20
rate: 0.05
interval: 180
targetReplacements:
- type: "com.steadybit.extension_aws.ec2-instance"
count: 1
interval: 600
- type: "com.steadybit.extension_container.container"
count: 2
interval: 180
- type: "com.steadybit.extension_kubernetes.kubernetes-container"
count: 2
interval: 180
- type: "com.steadybit.extension_kubernetes.kubernetes-deployment"
count: 2
interval: 180
containerTargetCreationsAndDeletions:
count: 10
interval: 180


tls:
Expand Down
33 changes: 21 additions & 12 deletions config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,8 @@ type Specification struct {
PodsPerDeployment int `json:"podsPerDeployment" split_words:"true" required:"false" default:"2"`
ContainerPerPod int `json:"containerPerPod" split_words:"true" required:"false" default:"2"`

AttributeUpdates AttributeUpdateSpecifications `split_words:"true" required:"false" default:"[{\"type\": \"com.steadybit.extension_aws.ec2-instance\", \"attributeName\": \"aws-ec2.label.change-ts\", \"rate\": 0.20, \"interval\": 600},{\"type\": \"com.steadybit.extension_container.container\", \"attributeName\": \"container.label.change-ts\", \"rate\": 0.20, \"interval\": 180},{\"type\": \"com.steadybit.extension_kubernetes.kubernetes-container\", \"attributeName\": \"k8s.label.change-ts\", \"rate\": 0.20, \"interval\": 180},{\"type\": \"com.steadybit.extension_kubernetes.kubernetes-deployment\", \"attributeName\": \"k8s.label.change-ts\", \"rate\": 0.20, \"interval\": 180}]"`

//Simulate created and deleted com.steadybit.extension_container.container targets every 180 seconds with a randomized count between 0 and the given value.
ContainerTargetCreationsAndDeletions ContainerTargetCreationsAndDeletionsSpecification `split_words:"true" required:"false" default:"{\"count\": 10, \"interval\": 60}"`
AttributeUpdates AttributeUpdateSpecifications `split_words:"true" required:"false" default:"[]"`
TargetReplacements TargetReplacementsSpecifications `split_words:"true" required:"false" default:""`

DiscoveryAttributesExcludesContainer []string `json:"discoveryAttributesExcludesContainer" split_words:"true" required:"false"`
DiscoveryAttributesExcludesEc2 []string `json:"discoveryAttributesExcludesEc2" split_words:"true" required:"false"`
Expand Down Expand Up @@ -66,6 +64,7 @@ func IsPodZero() bool {
}

type AttributeUpdateSpecifications []AttributeUpdateSpecification
type TargetReplacementsSpecifications []TargetReplacementsSpecification

func (s *AttributeUpdateSpecifications) Decode(value string) error {
var specs []AttributeUpdateSpecification
Expand All @@ -84,8 +83,8 @@ type AttributeUpdateSpecification struct {
Interval int `json:"interval" split_words:"true"`
}

func (s *ContainerTargetCreationsAndDeletionsSpecification) Decode(value string) error {
var spec ContainerTargetCreationsAndDeletionsSpecification
func (s *TargetReplacementsSpecification) Decode(value string) error {
var spec TargetReplacementsSpecification
err := json.Unmarshal([]byte(value), &spec)
if err != nil {
return fmt.Errorf("invalid json: %w", err)
Expand All @@ -94,9 +93,10 @@ func (s *ContainerTargetCreationsAndDeletionsSpecification) Decode(value string)
return nil
}

type ContainerTargetCreationsAndDeletionsSpecification struct {
Count int `json:"count" split_words:"true"`
Interval int `json:"interval" split_words:"true"`
type TargetReplacementsSpecification struct {
Type string `json:"type" split_words:"true"`
Count int `json:"count" split_words:"true"`
Interval int `json:"interval" split_words:"true"`
}

var (
Expand All @@ -114,9 +114,18 @@ func ValidateConfiguration() {
}

func (s *Specification) FindAttributeUpdate(t string) *AttributeUpdateSpecification {
for _, attributeUpdate := range s.AttributeUpdates {
if attributeUpdate.Type == t {
return &attributeUpdate
for _, update := range s.AttributeUpdates {
if update.Type == t {
return &update
}
}
return nil
}

func (s *Specification) FindTargetReplacementsSpecification(t string) *TargetReplacementsSpecification {
for _, replacement := range s.TargetReplacements {
if replacement.Type == t {
return &replacement
}
}
return nil
Expand Down
50 changes: 35 additions & 15 deletions extloadtest/discovery.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,26 @@ import (
)

type TargetData struct {
hosts []discovery_kit_api.Target
ec2Instances []discovery_kit_api.Target
azureInstances []discovery_kit_api.Target
gcpInstances []discovery_kit_api.Target
kubernetesClusters []discovery_kit_api.Target
kubernetesDeployments []discovery_kit_api.Target
kubernetesPods []discovery_kit_api.Target
kubernetesContainers []discovery_kit_api.EnrichmentData
kubernetesNodes []discovery_kit_api.Target
containers []discovery_kit_api.Target
containersBackup []discovery_kit_api.Target
hosts []discovery_kit_api.Target
hostsBackup []discovery_kit_api.Target
ec2Instances []discovery_kit_api.Target
ec2InstancesBackup []discovery_kit_api.Target
azureInstances []discovery_kit_api.Target
azureInstancesBackup []discovery_kit_api.Target
gcpInstances []discovery_kit_api.Target
gcpInstancesBackup []discovery_kit_api.Target
kubernetesClusters []discovery_kit_api.Target
kubernetesClustersBackup []discovery_kit_api.Target
kubernetesDeployments []discovery_kit_api.Target
kubernetesDeploymentsBackup []discovery_kit_api.Target
kubernetesPods []discovery_kit_api.Target
kubernetesPodsBackup []discovery_kit_api.Target
kubernetesContainers []discovery_kit_api.EnrichmentData
kubernetesContainersBackup []discovery_kit_api.EnrichmentData
kubernetesNodes []discovery_kit_api.Target
kubernetesNodesBackup []discovery_kit_api.Target
containers []discovery_kit_api.Target
containersBackup []discovery_kit_api.Target
}

func NewTargetData() *TargetData {
Expand Down Expand Up @@ -91,7 +100,7 @@ func NewTargetData() *TargetData {
log.Info().Msgf("Generated %d kubernetes nodes", len(ec2KubernetesNodes)+len(gcpKubernetesNodes)+len(azureKubernetesNodes))
log.Info().Msgf("Generated %d containers", len(ec2Containers)+len(gcpContainers)+len(azureContainers))

var targetsAvailable = 0;
var targetsAvailable = 0
if !config.Config.DisableHostDiscovery {
targetsAvailable += len(ec2Hosts) + len(gcpHosts) + len(azureHosts)
}
Expand Down Expand Up @@ -225,29 +234,40 @@ func (t *TargetData) RegisterDiscoveries() {
func (t *TargetData) ScheduleUpdates() {
if !config.Config.DisableHostDiscovery {
scheduleTargetAttributeUpdateIfNecessary(t.hosts, "com.steadybit.extension_host.host")
scheduleTargetReplacementIfNecessary(&t.hosts, &t.hostsBackup, "com.steadybit.extension_host.host")
}
if !config.Config.DisableAWSDiscovery {
scheduleTargetAttributeUpdateIfNecessary(t.ec2Instances, "com.steadybit.extension_aws.ec2-instance")
scheduleTargetReplacementIfNecessary(&t.ec2Instances, &t.ec2InstancesBackup, "com.steadybit.extension_aws.ec2-instance")

}
if !config.Config.DisableGCPDiscovery {
scheduleTargetAttributeUpdateIfNecessary(t.gcpInstances, "com.steadybit.extension_gcp.vm")
scheduleTargetReplacementIfNecessary(&t.gcpInstances, &t.gcpInstancesBackup, "com.steadybit.extension_gcp.vm")

}
if !config.Config.DisableAzureDiscovery {
scheduleTargetAttributeUpdateIfNecessary(t.azureInstances, "com.steadybit.extension_azure.scale_set.instance")
scheduleTargetReplacementIfNecessary(&t.azureInstances, &t.azureInstancesBackup, "com.steadybit.extension_azure.scale_set.instance")
}

if !config.Config.DisableKubernetesDiscovery {
scheduleTargetAttributeUpdateIfNecessary(t.kubernetesClusters, "com.steadybit.extension_kubernetes.kubernetes-cluster")
scheduleTargetReplacementIfNecessary(&t.kubernetesClusters, &t.kubernetesClustersBackup, "com.steadybit.extension_kubernetes.kubernetes-cluster")

scheduleTargetAttributeUpdateIfNecessary(t.kubernetesDeployments, "com.steadybit.extension_kubernetes.kubernetes-deployment")
scheduleTargetReplacementIfNecessary(&t.kubernetesDeployments, &t.kubernetesDeploymentsBackup, "com.steadybit.extension_kubernetes.kubernetes-deployment")

scheduleTargetAttributeUpdateIfNecessary(t.kubernetesPods, "com.steadybit.extension_kubernetes.kubernetes-pod")
scheduleTargetReplacementIfNecessary(&t.kubernetesPods, &t.kubernetesPodsBackup, "com.steadybit.extension_kubernetes.kubernetes-pod")
}
if !config.Config.DisableContainerDiscovery {
scheduleTargetAttributeUpdateIfNecessary(t.containers, "com.steadybit.extension_container.container")
scheduleTargetReplacementIfNecessary(&t.containers, &t.containersBackup, "com.steadybit.extension_container.container")
}
if !config.Config.DisableKubernetesDiscovery {
scheduleEnrichmentDataAttributeUpdateIfNecessary(t.kubernetesContainers, "com.steadybit.extension_kubernetes.kubernetes-container")
}
if !config.Config.DisableContainerDiscovery {
scheduleContainerTargetChanges(&t.containers, &t.containersBackup)
scheduleEnrichmentDataReplacementIfNecessary(&t.kubernetesContainers, &t.kubernetesContainersBackup, "com.steadybit.extension_kubernetes.kubernetes-container")
}
}

Expand Down
88 changes: 52 additions & 36 deletions extloadtest/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,42 +30,6 @@ func scheduleEnrichmentDataAttributeUpdateIfNecessary(items []discovery_kit_api.
})
}

func scheduleContainerTargetChanges(containers *[]discovery_kit_api.Target, backup *[]discovery_kit_api.Target) {
if config.Config.ContainerTargetCreationsAndDeletions.Count > 0 {
interval := config.Config.ContainerTargetCreationsAndDeletions.Interval
delay := time.Duration(interval) * time.Second
_, err := scheduler.ScheduleWithFixedDelay(func(ctx context.Context) {
//restore previously deleted containers
restoredCount := len(*backup)
*containers = append(*containers, *backup...)
if restoredCount > 0 {
for _, t := range *backup {
log.Debug().Str("id", t.Id).Msg("Restored container")
}
}
*backup = []discovery_kit_api.Target{}

//delete random containers
deletedCount := rand.Intn(config.Config.ContainerTargetCreationsAndDeletions.Count)
log.Debug().Int("count", deletedCount).Msgf("Deleted containers")
for i := 0; i < deletedCount; i++ {
index := rand.Intn(len(*containers))
*backup = append(*backup, (*containers)[index])
log.Debug().Str("id", (*containers)[index].Id).Msg("Deleted container")
*containers = append((*containers)[:index], (*containers)[index+1:]...)
}
log.Info().Msgf("Deleted %d containers and (re-)added %d containers. Total count is now %d", deletedCount, restoredCount, len(*containers))
}, delay, chrono.WithTime(time.Now().Add(delay)))
if err != nil {
log.Fatal().Msgf("Failed to schedule container target changes: %s", err.Error())
}
log.Info().
Int("interval", interval).
Int("maxCount", config.Config.ContainerTargetCreationsAndDeletions.Count).
Msg("scheduled container target creation/deletion")
}
}

func scheduleAttributeUpdateIfNecessary[T any](items []T, typeId string, attributeMapAccessor func(item T) map[string][]string) {
spec := config.Config.FindAttributeUpdate(typeId)
if spec == nil || spec.Rate <= 0.00000000001 {
Expand All @@ -90,6 +54,58 @@ func scheduleAttributeUpdateIfNecessary[T any](items []T, typeId string, attribu
}
}

func scheduleTargetReplacementIfNecessary(targets, backup *[]discovery_kit_api.Target, typeId string) {
scheduleReplacementIfNecessary(targets, backup, typeId, func(t discovery_kit_api.Target) string {
return t.Id
})
}

func scheduleEnrichmentDataReplacementIfNecessary(targets, backup *[]discovery_kit_api.EnrichmentData, typeId string) {
scheduleReplacementIfNecessary(targets, backup, typeId, func(t discovery_kit_api.EnrichmentData) string {
return t.Id
})
}

func scheduleReplacementIfNecessary[T any](targets, backup *[]T, typeId string, id func(T) string) {
spec := config.Config.FindTargetReplacementsSpecification(typeId)
if spec == nil || spec.Count <= 0 {
return
}

interval := spec.Interval
delay := time.Duration(interval) * time.Second
_, err := scheduler.ScheduleWithFixedDelay(func(ctx context.Context) {
//restore previously deleted containers
restoredCount := len(*backup)
*targets = append(*targets, *backup...)
if restoredCount > 0 {
for _, t := range *backup {
log.Debug().Str("id", id(t)).Msg("Restored container")
}
}

*backup = []T{}
//delete random targets
deletedCount := rand.Intn(spec.Count)
log.Debug().Int("count", deletedCount).Msgf("Deleted targets")
for i := 0; i < deletedCount; i++ {
index := rand.Intn(len(*targets))
*backup = append(*backup, (*targets)[index])
log.Debug().Str("id", id((*targets)[index])).Msg("Deleted target")
*targets = append((*targets)[:index], (*targets)[index+1:]...)
}
log.Info().Msgf("Deleted %d targets and (re-)added %d targets. Total count is now %d", deletedCount, restoredCount, len(*targets))
}, delay, chrono.WithTime(time.Now().Add(delay)))

if err != nil {
log.Fatal().Msgf("Failed to schedule %s changes: %s", typeId, err.Error())
}
log.Info().
Int("interval", interval).
Int("maxCount", spec.Count).
Msgf("scheduled container %s creation/deletion", typeId)
}

func initAttributes[T any](items []T, spec *config.AttributeUpdateSpecification, getAttributeMap func(item T) map[string][]string) {
for _, item := range items {
getAttributeMap(item)[spec.AttributeName] = []string{time.Now().String()}
Expand Down

0 comments on commit 4856510

Please sign in to comment.