Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

e2e: Add backend health check e2e case via active http #3677

Merged
merged 12 commits into from
Jul 3, 2024
Merged

e2e: Add backend health check e2e case via active http #3677

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
3509d8b
Add backend health check e2e case via active http
aoledk Jun 26, 2024
85079f7
Merge branch 'main' into backend-http-active-health-check-e2e
aoledk Jun 27, 2024
db1e92a
Use membership_healthy stats to check backend health check
aoledk Jun 28, 2024
c0f0c79
Merge branch 'main' into backend-http-active-health-check-e2e
aoledk Jun 28, 2024
5486deb
Use non-zero cluster health check stats to verify backend health check
aoledk Jun 29, 2024
62f8088
Merge branch 'main' into backend-http-active-health-check-e2e
aoledk Jun 29, 2024
14e09dd
Merge branch 'main' into backend-http-active-health-check-e2e
aoledk Jul 1, 2024
d7e66aa
Merge branch 'main' into backend-http-active-health-check-e2e
aoledk Jul 2, 2024
344a380
Merge branch 'main' into backend-http-active-health-check-e2e
aoledk Jul 2, 2024
1fd4c33
Merge branch 'main' into backend-http-active-health-check-e2e
aoledk Jul 3, 2024
fb9b7a1
Merge branch 'main' into backend-http-active-health-check-e2e
zirain Jul 3, 2024
bf9cabf
Merge branch 'main' into backend-http-active-health-check-e2e
aoledk Jul 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 67 additions & 3 deletions test/e2e/tests/backend_health_check.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,11 @@
package tests

import (
"fmt"
"testing"
"time"

"github.com/stretchr/testify/require"
"k8s.io/apimachinery/pkg/types"
gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
Expand All @@ -19,6 +22,7 @@ import (
"sigs.k8s.io/gateway-api/conformance/utils/suite"

"github.com/envoyproxy/gateway/internal/gatewayapi"
"github.com/envoyproxy/gateway/test/e2e/utils/prometheus"
)

func init() {
Expand All @@ -32,10 +36,10 @@ var BackendHealthCheckActiveHTTPTest = suite.ConformanceTest{
Test: func(t *testing.T, suite *suite.ConformanceTestSuite) {
t.Run("active http", func(t *testing.T) {
ns := "gateway-conformance-infra"
route1NN := types.NamespacedName{Name: "http-with-health-check-active-http-pass", Namespace: ns}
route2NN := types.NamespacedName{Name: "http-with-health-check-active-http-fail", Namespace: ns}
passRouteNN := types.NamespacedName{Name: "http-with-health-check-active-http-pass", Namespace: ns}
failRouteNN := types.NamespacedName{Name: "http-with-health-check-active-http-fail", Namespace: ns}
gwNN := types.NamespacedName{Name: "same-namespace", Namespace: ns}
gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), route1NN, route2NN)
gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), passRouteNN, failRouteNN)

ancestorRef := gwapiv1a2.ParentReference{
Group: gatewayapi.GroupPtr(gwapiv1.GroupName),
Expand All @@ -46,6 +50,66 @@ var BackendHealthCheckActiveHTTPTest = suite.ConformanceTest{
BackendTrafficPolicyMustBeAccepted(t, suite.Client, types.NamespacedName{Name: "health-check-active-http-pass-btp", Namespace: ns}, suite.ControllerName, ancestorRef)
BackendTrafficPolicyMustBeAccepted(t, suite.Client, types.NamespacedName{Name: "health-check-active-http-fail-btp", Namespace: ns}, suite.ControllerName, ancestorRef)

promAddr, err := prometheus.Address(suite.Client,
types.NamespacedName{Name: "prometheus", Namespace: "monitoring"},
)
require.NoError(t, err)

passClusterName := fmt.Sprintf("httproute/%s/%s/rule/0", ns, passRouteNN.Name)
failClusterName := fmt.Sprintf("httproute/%s/%s/rule/0", ns, failRouteNN.Name)
gtwName := "same-namespace"

// health check requests will be distributed to the cluster with configured path.
// we can use membership_healthy stats to check whether health check works as expected.
passPromQL := fmt.Sprintf(`envoy_cluster_membership_healthy{envoy_cluster_name="%s",gateway_envoyproxy_io_owning_gateway_name="%s"}`, passClusterName, gtwName)
failPromQL := fmt.Sprintf(`envoy_cluster_membership_healthy{envoy_cluster_name="%s",gateway_envoyproxy_io_owning_gateway_name="%s"}`, failClusterName, gtwName)

http.AwaitConvergence(
t,
suite.TimeoutConfig.RequiredConsecutiveSuccesses,
suite.TimeoutConfig.MaxTimeToConsistency,
func(_ time.Duration) bool {
// check membership_healthy stats from Prometheus
v, err := prometheus.QuerySum(promAddr, passPromQL)
if err != nil {
// wait until Prometheus sync stats
return false
}
t.Logf("cluster pass health check: membership_healthy stats query count: %v", v)

if v == 0 {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

shouldn't this be non zero ?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you prob need to wait until non zero is reached

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've turned to use non-zero cluster health check stats to verify whether backend health check works, for both pass and fail cases. And undo changes in e2e/utils/prometheus.sum to restore the behavior of only return nil when non-zero value is queried.

# TYPE envoy_cluster_health_check_attempt counter
envoy_cluster_health_check_attempt{envoy_cluster_name="httproute/gateway-conformance-infra/http-with-health-check-active-http-fail/rule/0"} 2
envoy_cluster_health_check_attempt{envoy_cluster_name="httproute/gateway-conformance-infra/http-with-health-check-active-http-pass/rule/0"} 2

# TYPE envoy_cluster_health_check_success counter
envoy_cluster_health_check_success{envoy_cluster_name="httproute/gateway-conformance-infra/http-with-health-check-active-http-fail/rule/0"} 0
envoy_cluster_health_check_success{envoy_cluster_name="httproute/gateway-conformance-infra/http-with-health-check-active-http-pass/rule/0"} 2

# TYPE envoy_cluster_health_check_failure counter
envoy_cluster_health_check_failure{envoy_cluster_name="httproute/gateway-conformance-infra/http-with-health-check-active-http-fail/rule/0"} 2
envoy_cluster_health_check_failure{envoy_cluster_name="httproute/gateway-conformance-infra/http-with-health-check-active-http-pass/rule/0"} 0

t.Error("healthy membership is not the same as expected")
} else {
t.Log("healthy membership is the same as expected")
}

return true
},
)

http.AwaitConvergence(
t,
suite.TimeoutConfig.RequiredConsecutiveSuccesses,
suite.TimeoutConfig.MaxTimeToConsistency,
func(_ time.Duration) bool {
// check membership_healthy stats from Prometheus
v, err := prometheus.QuerySum(promAddr, failPromQL)
if err != nil {
// wait until Prometheus sync stats
return false
}
t.Logf("cluster fail health check: membership_healthy stats query count: %v", v)

if v == 0 {
t.Log("healthy membership is same as expected")
} else {
t.Error("healthy membership is not same as expected")
}

return true
},
)

t.Run("health check pass", func(t *testing.T) {
expectedResponse := http.ExpectedResponse{
Request: http.Request{
Expand Down
6 changes: 2 additions & 4 deletions test/e2e/utils/prometheus/prometheus.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,6 @@ func sum(val model.Value) (float64, error) {
valueCount += float64(sample.Value)
}

if valueCount > 0.0 {
return valueCount, nil
}
return 0, fmt.Errorf("value not found")
// the actual sum maybe 0.0, it's not error
return valueCount, nil
}
Loading