Skip to content

Commit

Permalink
test responses from scaledobjects and hpa
Browse files Browse the repository at this point in the history
  • Loading branch information
@Richard87
Richard87 committed Jun 13, 2024
1 parent aaae0cd commit d387e15
Show file tree
Hide file tree
Showing 2 changed files with 152 additions and 73 deletions.
129 changes: 112 additions & 17 deletions api/deployments/component_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"github.com/equinor/radix-api/api/utils/labelselector"
radixhttp "github.com/equinor/radix-common/net/http"
radixutils "github.com/equinor/radix-common/utils"
"github.com/equinor/radix-common/utils/pointers"
"github.com/equinor/radix-common/utils/slice"
"github.com/equinor/radix-operator/pkg/apis/kube"
v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1"
Expand All @@ -23,6 +24,7 @@ import (
"github.com/stretchr/testify/require"
v2 "k8s.io/api/autoscaling/v2"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)
Expand Down Expand Up @@ -563,17 +565,21 @@ func TestGetComponents_WithHorizontalScaling(t *testing.T) {
// Setup

testScenarios := []struct {
name string
deploymentName string
minReplicas int32
maxReplicas int32
targetCpu *int32
targetMemory *int32
name string
deploymentName string
minReplicas int32
maxReplicas int32
targetCpu *int32
targetMemory *int32
targetCron *int32
targetAzureServiceBus *int32
}{
{"targetCpu and targetMemory are nil", "dep1", 2, 6, nil, nil},
{"targetCpu is nil, targetMemory is non-nil", "dep2", 2, 6, nil, numbers.Int32Ptr(75)},
{"targetCpu is non-nil, targetMemory is nil", "dep3", 2, 6, numbers.Int32Ptr(60), nil},
{"targetCpu and targetMemory are non-nil", "dep4", 2, 6, numbers.Int32Ptr(62), numbers.Int32Ptr(79)},
{"targetCpu and targetMemory are nil", "dep1", 2, 6, nil, nil, nil, nil},
{"targetCpu is nil, targetMemory is non-nil", "dep2", 2, 6, nil, pointers.Ptr[int32](75), nil, nil},
{"targetCpu is non-nil, targetMemory is nil", "dep3", 2, 6, pointers.Ptr[int32](60), nil, nil, nil},
{"targetCpu and targetMemory are non-nil", "dep4", 2, 6, pointers.Ptr[int32](62), pointers.Ptr[int32](79), nil, nil},
{"Test CRON trigger is found", "dep5", 2, 6, nil, nil, pointers.Ptr[int32](5), nil},
{"Test Azure trigger is found", "dep6", 2, 6, nil, nil, nil, pointers.Ptr[int32](15)},
}

for _, scenario := range testScenarios {
Expand All @@ -592,7 +598,7 @@ func TestGetComponents_WithHorizontalScaling(t *testing.T) {
require.NoError(t, err)

ns := operatorUtils.GetEnvironmentNamespace(anyAppName, "prod")
scaler, hpa := createHorizontalScalingObjects("frontend", numbers.Int32Ptr(scenario.minReplicas), scenario.maxReplicas, scenario.targetCpu, scenario.targetMemory)
scaler, hpa := createHorizontalScalingObjects("frontend", numbers.Int32Ptr(scenario.minReplicas), scenario.maxReplicas, scenario.targetCpu, scenario.targetMemory, scenario.targetCron, scenario.targetAzureServiceBus)
_, err = kedaClient.KedaV1alpha1().ScaledObjects(ns).Create(context.Background(), &scaler, metav1.CreateOptions{})
require.NoError(t, err)
_, err = client.AutoscalingV2().HorizontalPodAutoscalers(ns).Create(context.Background(), &hpa, metav1.CreateOptions{})
Expand Down Expand Up @@ -624,7 +630,7 @@ func TestGetComponents_WithHorizontalScaling(t *testing.T) {
assert.False(t, ok)
} else {
require.True(t, ok)
assert.Equal(t, string(*scenario.targetMemory), memoryTrigger.TargetUtilization)
assert.Equal(t, fmt.Sprintf("%d", *scenario.targetMemory), memoryTrigger.TargetUtilization)
assert.Empty(t, memoryTrigger.CurrentUtilization)
assert.Empty(t, memoryTrigger.Error)
assert.Equal(t, "memory", memoryTrigger.Type)
Expand All @@ -637,29 +643,56 @@ func TestGetComponents_WithHorizontalScaling(t *testing.T) {
assert.False(t, ok)
} else {
require.True(t, ok)
assert.Equal(t, string(*scenario.targetCpu), cpuTrigger.TargetUtilization)
assert.Equal(t, fmt.Sprintf("%d", *scenario.targetCpu), cpuTrigger.TargetUtilization)
assert.Empty(t, cpuTrigger.CurrentUtilization)
assert.Empty(t, cpuTrigger.Error)
assert.Equal(t, "cpu", cpuTrigger.Type)
}

// todo: test CRON trigger and AzureServiceBus
cronTrigger, ok := slice.FindFirst(components[0].HorizontalScalingSummary.Triggers, func(s deploymentModels.HorizontalScalingSummaryTriggerStatus) bool {
return s.Name == "cron"
})
if scenario.targetCron == nil {
assert.False(t, ok)
} else {
require.True(t, ok)
assert.Equal(t, fmt.Sprintf("%d", *scenario.targetCron), cronTrigger.TargetUtilization)
assert.Equal(t, fmt.Sprintf("%d", *scenario.targetCron), cronTrigger.CurrentUtilization)
assert.Empty(t, cronTrigger.Error)
assert.Equal(t, "cron", cronTrigger.Type)
}

azureTrigger, ok := slice.FindFirst(components[0].HorizontalScalingSummary.Triggers, func(s deploymentModels.HorizontalScalingSummaryTriggerStatus) bool {
return s.Name == "azure-servicebus"
})
if scenario.targetAzureServiceBus == nil {
assert.False(t, ok)
} else {
require.True(t, ok)
assert.Equal(t, fmt.Sprintf("%d", *scenario.targetAzureServiceBus), azureTrigger.TargetUtilization)
assert.Equal(t, fmt.Sprintf("%d", *scenario.targetAzureServiceBus), azureTrigger.CurrentUtilization)
assert.Empty(t, azureTrigger.Error)
assert.Equal(t, "azure-servicebus", azureTrigger.Type)
}
})
}
}

func createHorizontalScalingObjects(name string, minReplicas *int32, maxReplicas int32, targetCpu *int32, targetMemory *int32) (v1alpha1.ScaledObject, v2.HorizontalPodAutoscaler) {
func createHorizontalScalingObjects(name string, minReplicas *int32, maxReplicas int32, targetCpu *int32, targetMemory *int32, targetCron *int32, targetAzureServiceBus *int32) (v1alpha1.ScaledObject, v2.HorizontalPodAutoscaler) {
var triggers []v1alpha1.ScaleTriggers
var metrics []v2.MetricSpec
resourceMetricNames := []string{}
externalMetricNames := []string{}
health := map[string]v1alpha1.HealthStatus{}
metricStatus := []v2.MetricStatus{}

if targetCpu != nil {
resourceMetricNames = append(resourceMetricNames, "cpu")
triggers = append(triggers, v1alpha1.ScaleTriggers{
Type: "cpu",
Name: "cpu",
Metadata: map[string]string{
"value": string(*targetCpu),
"value": fmt.Sprintf("%d", *targetCpu),
},
AuthenticationRef: nil,
MetricType: "Utilization",
Expand All @@ -681,7 +714,7 @@ func createHorizontalScalingObjects(name string, minReplicas *int32, maxReplicas
Type: "memory",
Name: "memory",
Metadata: map[string]string{
"value": string(*targetMemory),
"value": fmt.Sprintf("%d", *targetMemory),
},
MetricType: "Utilization",
})
Expand All @@ -696,6 +729,63 @@ func createHorizontalScalingObjects(name string, minReplicas *int32, maxReplicas
})
}

if targetCron != nil {
externalMetricName := fmt.Sprintf("s%d-cron-Europe-Oslo-08xx1-5-016xx1-5", len(triggers))
externalMetricNames = append(externalMetricNames, externalMetricName)
triggers = append(triggers, v1alpha1.ScaleTriggers{
Type: "cron",
Name: "cron",
Metadata: map[string]string{
"end": "0 16 * * 1-5",
"start": "0 8 * * 1-5",
"timezone": "Europe/Oslo",
"desiredReplicas": fmt.Sprintf("%d", *targetCron),
},
})
health[externalMetricName] = v1alpha1.HealthStatus{
NumberOfFailures: pointers.Ptr[int32](0),
Status: "Happy",
}
metricStatus = append(metricStatus, v2.MetricStatus{
Type: "External",
External: &v2.ExternalMetricStatus{
Current: v2.MetricValueStatus{
AverageValue: resource.NewQuantity(int64(*targetCron), resource.DecimalSI),
},
Metric: v2.MetricIdentifier{
Name: externalMetricName,
},
},
})
}

if targetAzureServiceBus != nil {
externalMetricName := fmt.Sprintf("s%d-azure-servicebus-orders", len(triggers))
externalMetricNames = append(externalMetricNames, externalMetricName)
triggers = append(triggers, v1alpha1.ScaleTriggers{
Type: "azure-servicebus",
Name: "azure-servicebus",
Metadata: map[string]string{
"messageCount": fmt.Sprintf("%d", *targetAzureServiceBus),
},
})
health[externalMetricName] = v1alpha1.HealthStatus{
NumberOfFailures: pointers.Ptr[int32](0),
Status: "Happy",
}
metricStatus = append(metricStatus, v2.MetricStatus{
Type: "External",
External: &v2.ExternalMetricStatus{
Current: v2.MetricValueStatus{
AverageValue: resource.NewQuantity(int64(*targetAzureServiceBus), resource.DecimalSI),
},
Metric: v2.MetricIdentifier{
Name: externalMetricName,
},
},
})
}

scaler := v1alpha1.ScaledObject{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Expand All @@ -708,7 +798,9 @@ func createHorizontalScalingObjects(name string, minReplicas *int32, maxReplicas
},
Status: v1alpha1.ScaledObjectStatus{
HpaName: fmt.Sprintf("hpa-%s", name),
Health: health,
ResourceMetricNames: resourceMetricNames,
ExternalMetricNames: externalMetricNames,
},
}

Expand All @@ -722,6 +814,9 @@ func createHorizontalScalingObjects(name string, minReplicas *int32, maxReplicas
MaxReplicas: maxReplicas,
Metrics: metrics,
},
Status: v2.HorizontalPodAutoscalerStatus{
CurrentMetrics: metricStatus,
},
}

return scaler, hpa
Expand Down
96 changes: 40 additions & 56 deletions api/models/horizontal_scaling_summary.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import (
corev1 "k8s.io/api/core/v1"
)

var triggerIndexRegex = regexp.MustCompile(`/^s(\d+)-/`)
var triggerIndexRegex = regexp.MustCompile(`^s(\d+)-`)

func GetHpaSummary(appName, componentName string, hpaList []autoscalingv2.HorizontalPodAutoscaler, scalerList []v1alpha1.ScaledObject) *deploymentModels.HorizontalScalingSummary {
scaler, ok := slice.FindFirst(scalerList, predicate.IsScaledObjectForComponent(appName, componentName))
Expand Down Expand Up @@ -49,40 +49,29 @@ func GetHpaSummary(appName, componentName string, hpaList []autoscalingv2.Horizo

// ResourceMetricNames lists resource types, not metric names
for _, resourceType := range scaler.Status.ResourceMetricNames {
trigger, ok := slice.FindFirst(scaler.Spec.Triggers, func(t v1alpha1.ScaleTriggers) bool {
var trigger v1alpha1.ScaleTriggers

if trigger, ok = slice.FindFirst(scaler.Spec.Triggers, func(t v1alpha1.ScaleTriggers) bool {
return t.Type == resourceType
})
if !ok {
}); !ok {
continue
}

triggers = append(triggers, getResourceStatus(corev1.ResourceName(trigger.Type), trigger, &hpa))
triggers = append(triggers, getResourceMetricStatus(hpa, trigger))
}

for _, triggerName := range scaler.Status.ExternalMetricNames {
index, err := strconv.Atoi(triggerIndexRegex.FindString(triggerName))
if err != nil {
continue
}

trigger := scaler.Spec.Triggers[index]
metricStatus, ok := slice.FindFirst(hpa.Status.CurrentMetrics, func(s autoscalingv2.MetricStatus) bool {
return s.External != nil && s.External.Metric.Name == triggerName
})
if !ok {
match := triggerIndexRegex.FindStringSubmatch(triggerName)
if len(match) != 2 {
continue
}
health, ok := scaler.Status.Health[triggerName]
if !ok {
index, err := strconv.Atoi(match[1])
if err != nil {
continue
}

switch trigger.Type {
case "cron":
triggers = append(triggers, getCronStatus(trigger, metricStatus, health))
case "azure-servicebus":
triggers = append(triggers, getAzureServiceBusStatus(trigger, metricStatus, health))
}
trigger := scaler.Spec.Triggers[index]
triggers = append(triggers, getExternalMetricStatus(hpa, triggerName, scaler, trigger))
}

hpaSummary := deploymentModels.HorizontalScalingSummary{
Expand All @@ -99,57 +88,52 @@ func GetHpaSummary(appName, componentName string, hpaList []autoscalingv2.Horizo
return &hpaSummary
}

func getResourceStatus(triggerType corev1.ResourceName, trigger v1alpha1.ScaleTriggers, hpa *autoscalingv2.HorizontalPodAutoscaler) deploymentModels.HorizontalScalingSummaryTriggerStatus {
var current, target string

if c := getHpaCurrentMetric(hpa, triggerType); c != nil {
current = strconv.Itoa(int(*c))
}
if t := horizontalscaling.GetHpaMetric(hpa, triggerType); t != nil {
target = trigger.Metadata["value"]
func getResourceMetricStatus(hpa autoscalingv2.HorizontalPodAutoscaler, trigger v1alpha1.ScaleTriggers) deploymentModels.HorizontalScalingSummaryTriggerStatus {
var current string
if metricStatus, ok := slice.FindFirst(hpa.Status.CurrentMetrics, func(s autoscalingv2.MetricStatus) bool {
return s.Resource != nil && s.Resource.Name.String() == trigger.Type
}); ok && metricStatus.Resource != nil {
current = fmt.Sprintf("%d", *metricStatus.Resource.Current.AverageUtilization)
}

return deploymentModels.HorizontalScalingSummaryTriggerStatus{
status := deploymentModels.HorizontalScalingSummaryTriggerStatus{
Name: trigger.Name,
CurrentUtilization: current,
TargetUtilization: target,
Type: string(triggerType),
TargetUtilization: trigger.Metadata["value"],
Type: trigger.Type,
Error: "",
}
return status
}
func getCronStatus(trigger v1alpha1.ScaleTriggers, metricStatus autoscalingv2.MetricStatus, health v1alpha1.HealthStatus) deploymentModels.HorizontalScalingSummaryTriggerStatus {
var err string

current := metricStatus.External.Current.AverageValue.String()
target := trigger.Metadata["desiredReplicas"]
func getExternalMetricStatus(hpa autoscalingv2.HorizontalPodAutoscaler, triggerName string, scaler v1alpha1.ScaledObject, trigger v1alpha1.ScaleTriggers) deploymentModels.HorizontalScalingSummaryTriggerStatus {
var current, target, errStr string

if health.Status != "Happy" {
err = fmt.Sprintf("%s: number of failurs: %d", health.Status, health.NumberOfFailures)
}
return deploymentModels.HorizontalScalingSummaryTriggerStatus{
Name: trigger.Name,
CurrentUtilization: current,
TargetUtilization: target,
Type: trigger.Type,
Error: err,
if metricStatus, ok := slice.FindFirst(hpa.Status.CurrentMetrics, func(s autoscalingv2.MetricStatus) bool {
return s.External != nil && s.External.Metric.Name == triggerName
}); ok && metricStatus.External != nil {
current = metricStatus.External.Current.AverageValue.String()
}
}
func getAzureServiceBusStatus(trigger v1alpha1.ScaleTriggers, metricStatus autoscalingv2.MetricStatus, health v1alpha1.HealthStatus) deploymentModels.HorizontalScalingSummaryTriggerStatus {
var err string

current := metricStatus.External.Current.AverageValue.String()
target := trigger.Metadata["messageCount"]
if health, ok := scaler.Status.Health[triggerName]; ok && health.Status != "Happy" {
errStr = fmt.Sprintf("%s: number of failurs: %d", health.Status, *health.NumberOfFailures)
}

if health.Status != "Happy" {
err = fmt.Sprintf("%s: number of failurs: %d", health.Status, health.NumberOfFailures)
switch trigger.Type {
case "cron":
target = trigger.Metadata["desiredReplicas"]
case "azure-servicebus":
target = trigger.Metadata["messageCount"]
}
return deploymentModels.HorizontalScalingSummaryTriggerStatus{

status := deploymentModels.HorizontalScalingSummaryTriggerStatus{
Name: trigger.Name,
CurrentUtilization: current,
TargetUtilization: target,
Type: trigger.Type,
Error: err,
Error: errStr,
}
return status
}

func getHpaMetrics(hpa *autoscalingv2.HorizontalPodAutoscaler, resourceName corev1.ResourceName) (*int32, *int32) {
Expand Down

0 comments on commit d387e15

Please sign in to comment.