From d2824e240a6de0665d18af13a15201618b6ead9d Mon Sep 17 00:00:00 2001 From: Alex Volchok Date: Tue, 4 Jun 2024 01:09:41 +0200 Subject: [PATCH] fix client backend mtls secrets updates (#3526) * ensure that whenver client backend mtls secrets are updated, the latest secrets are used Signed-off-by: Alexander Volchok * updating Signed-off-by: Alexander Volchok --------- Signed-off-by: Alexander Volchok --- internal/gatewayapi/backendtlspolicy.go | 28 +-- internal/provider/kubernetes/controller.go | 2 +- internal/provider/kubernetes/indexers.go | 22 ++ internal/provider/kubernetes/predicates.go | 32 +++ .../provider/kubernetes/predicates_test.go | 31 +++ ...end-tls-settings-client-cert-rotation.yaml | 20 ++ ...eam-tls.yaml => backend-tls-settings.yaml} | 0 ...-service-tls-settings-new-mtls-secret.yaml | 28 +++ ...ice-tls-settings-updated-tls-settings.yaml | 28 +++ ...ls_settings.go => backend_tls_settings.go} | 216 ++++++++++-------- 10 files changed, 298 insertions(+), 109 deletions(-) create mode 100644 test/e2e/testdata/backend-tls-settings-client-cert-rotation.yaml rename test/e2e/testdata/{upstream-tls.yaml => backend-tls-settings.yaml} (100%) create mode 100644 test/e2e/testdata/expect/echo-service-tls-settings-new-mtls-secret.yaml create mode 100644 test/e2e/testdata/expect/echo-service-tls-settings-updated-tls-settings.yaml rename test/e2e/tests/{uptream_tls_settings.go => backend_tls_settings.go} (55%) diff --git a/internal/gatewayapi/backendtlspolicy.go b/internal/gatewayapi/backendtlspolicy.go index 3c8b5451a9c..95954217131 100644 --- a/internal/gatewayapi/backendtlspolicy.go +++ b/internal/gatewayapi/backendtlspolicy.go @@ -82,32 +82,32 @@ func (t *Translator) processBackendTLSPolicy( return tlsBundle, policy } -func (t *Translator) applyEnvoyProxyBackendTLSSetting(policy *gwapiv1a3.BackendTLSPolicy, tlsBundle *ir.TLSUpstreamConfig, resources *Resources, parent gwapiv1a2.ParentReference) *ir.TLSUpstreamConfig { +func (t *Translator) applyEnvoyProxyBackendTLSSetting(policy *gwapiv1a3.BackendTLSPolicy, tlsConfig *ir.TLSUpstreamConfig, resources *Resources, parent gwapiv1a2.ParentReference) *ir.TLSUpstreamConfig { ep := resources.EnvoyProxy - if ep == nil || ep.Spec.BackendTLS == nil { - return tlsBundle + if ep == nil || ep.Spec.BackendTLS == nil || tlsConfig == nil { + return tlsConfig } if len(ep.Spec.BackendTLS.Ciphers) > 0 { - tlsBundle.Ciphers = ep.Spec.BackendTLS.Ciphers + tlsConfig.Ciphers = ep.Spec.BackendTLS.Ciphers } if len(ep.Spec.BackendTLS.ECDHCurves) > 0 { - tlsBundle.ECDHCurves = ep.Spec.BackendTLS.ECDHCurves + tlsConfig.ECDHCurves = ep.Spec.BackendTLS.ECDHCurves } if len(ep.Spec.BackendTLS.SignatureAlgorithms) > 0 { - tlsBundle.SignatureAlgorithms = ep.Spec.BackendTLS.SignatureAlgorithms + tlsConfig.SignatureAlgorithms = ep.Spec.BackendTLS.SignatureAlgorithms } if ep.Spec.BackendTLS.MinVersion != nil { - tlsBundle.MinVersion = ptr.To(ir.TLSVersion(*ep.Spec.BackendTLS.MinVersion)) + tlsConfig.MinVersion = ptr.To(ir.TLSVersion(*ep.Spec.BackendTLS.MinVersion)) } if ep.Spec.BackendTLS.MaxVersion != nil { - tlsBundle.MaxVersion = ptr.To(ir.TLSVersion(*ep.Spec.BackendTLS.MaxVersion)) + tlsConfig.MaxVersion = ptr.To(ir.TLSVersion(*ep.Spec.BackendTLS.MaxVersion)) } if len(ep.Spec.BackendTLS.ALPNProtocols) > 0 { - tlsBundle.ALPNProtocols = make([]string, len(ep.Spec.BackendTLS.ALPNProtocols)) + tlsConfig.ALPNProtocols = make([]string, len(ep.Spec.BackendTLS.ALPNProtocols)) for i := range ep.Spec.BackendTLS.ALPNProtocols { - tlsBundle.ALPNProtocols[i] = string(ep.Spec.BackendTLS.ALPNProtocols[i]) + tlsConfig.ALPNProtocols[i] = string(ep.Spec.BackendTLS.ALPNProtocols[i]) } } if ep.Spec.BackendTLS != nil && ep.Spec.BackendTLS.ClientCertificateRef != nil { @@ -121,7 +121,7 @@ func (t *Translator) applyEnvoyProxyBackendTLSSetting(policy *gwapiv1a3.BackendT t.GatewayControllerName, policy.Generation, status.Error2ConditionMsg(fmt.Errorf("client authentication TLS secret is not located in the same namespace as Envoyproxy. Secret namespace: %s does not match Envoyproxy namespace: %s", ns, ep.Namespace))) - return tlsBundle + return tlsConfig } secret := resources.GetSecret(ns, string(ep.Spec.BackendTLS.ClientCertificateRef.Name)) if secret == nil { @@ -131,12 +131,12 @@ func (t *Translator) applyEnvoyProxyBackendTLSSetting(policy *gwapiv1a3.BackendT policy.Generation, status.Error2ConditionMsg(fmt.Errorf("failed to locate TLS secret for client auth: %s in namespace: %s", ep.Spec.BackendTLS.ClientCertificateRef.Name, ns)), ) - return tlsBundle + return tlsConfig } tlsConf := irTLSConfigs(secret) - tlsBundle.ClientCertificates = tlsConf.Certificates + tlsConfig.ClientCertificates = tlsConf.Certificates } - return tlsBundle + return tlsConfig } func backendTLSTargetMatched(policy gwapiv1a3.BackendTLSPolicy, target gwapiv1a2.LocalPolicyTargetReferenceWithSectionName, backendNamespace string) bool { diff --git a/internal/provider/kubernetes/controller.go b/internal/provider/kubernetes/controller.go index 40421d54e31..202a5ea1f2e 100644 --- a/internal/provider/kubernetes/controller.go +++ b/internal/provider/kubernetes/controller.go @@ -325,7 +325,7 @@ func (r *gatewayAPIReconciler) processEnvoyProxySecretRef(ctx context.Context, g gatewayapi.KindEnvoyProxy, *certRef); err != nil { r.log.Error(err, - "failed to process TLS SecretRef for gateway", + "failed to process TLS SecretRef for EnvoyProxy", "gateway", "issue", "secretRef", certRef) } } diff --git a/internal/provider/kubernetes/indexers.go b/internal/provider/kubernetes/indexers.go index d2abc6a25bc..80426ed1747 100644 --- a/internal/provider/kubernetes/indexers.go +++ b/internal/provider/kubernetes/indexers.go @@ -42,6 +42,7 @@ const ( configMapBtlsIndex = "configMapBtlsIndex" backendEnvoyExtensionPolicyIndex = "backendEnvoyExtensionPolicyIndex" backendEnvoyProxyTelemetryIndex = "backendEnvoyProxyTelemetryIndex" + secretEnvoyProxyIndex = "secretEnvoyProxyIndex" ) func addReferenceGrantIndexers(ctx context.Context, mgr manager.Manager) error { @@ -111,11 +112,32 @@ func backendHTTPRouteIndexFunc(rawObj client.Object) []string { return backendRefs } +func secretEnvoyProxyIndexFunc(rawObj client.Object) []string { + ep := rawObj.(*v1alpha1.EnvoyProxy) + var secretReferences []string + if ep.Spec.BackendTLS != nil { + if ep.Spec.BackendTLS.ClientCertificateRef != nil { + if *ep.Spec.BackendTLS.ClientCertificateRef.Kind == gatewayapi.KindSecret { + secretReferences = append(secretReferences, + types.NamespacedName{ + Namespace: gatewayapi.NamespaceDerefOr(ep.Spec.BackendTLS.ClientCertificateRef.Namespace, ep.Namespace), + Name: string(ep.Spec.BackendTLS.ClientCertificateRef.Name), + }.String()) + } + } + } + return secretReferences +} + func addEnvoyProxyIndexers(ctx context.Context, mgr manager.Manager) error { if err := mgr.GetFieldIndexer().IndexField(ctx, &v1alpha1.EnvoyProxy{}, backendEnvoyProxyTelemetryIndex, backendEnvoyProxyTelemetryIndexFunc); err != nil { return err } + if err := mgr.GetFieldIndexer().IndexField(ctx, &v1alpha1.EnvoyProxy{}, secretEnvoyProxyIndex, secretEnvoyProxyIndexFunc); err != nil { + return err + } + return nil } diff --git a/internal/provider/kubernetes/predicates.go b/internal/provider/kubernetes/predicates.go index 9b6eb4a8724..97714bd3a40 100644 --- a/internal/provider/kubernetes/predicates.go +++ b/internal/provider/kubernetes/predicates.go @@ -154,6 +154,38 @@ func (r *gatewayAPIReconciler) validateSecretForReconcile(obj client.Object) boo return true } + if r.isEnvoyProxyReferencingSecret(&nsName) { + return true + } + + return false +} + +func (r *gatewayAPIReconciler) isEnvoyProxyReferencingSecret(nsName *types.NamespacedName) bool { + epList := &egv1a1.EnvoyProxyList{} + if err := r.client.List(context.Background(), epList, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(secretEnvoyProxyIndex, nsName.String()), + }); err != nil { + r.log.Error(err, "unable to find associated Gateways") + return false + } + + if len(epList.Items) == 0 { + return false + } + + for _, ep := range epList.Items { + if ep.Spec.BackendTLS != nil { + if ep.Spec.BackendTLS.ClientCertificateRef != nil { + certRef := ep.Spec.BackendTLS.ClientCertificateRef + ns := gatewayapi.NamespaceDerefOr(certRef.Namespace, ep.Namespace) + if nsName.Name == string(certRef.Name) && nsName.Namespace == ns { + return true + } + continue + } + } + } return false } diff --git a/internal/provider/kubernetes/predicates_test.go b/internal/provider/kubernetes/predicates_test.go index c7b6054e504..431d9638272 100644 --- a/internal/provider/kubernetes/predicates_test.go +++ b/internal/provider/kubernetes/predicates_test.go @@ -183,12 +183,42 @@ func TestValidateGatewayForReconcile(t *testing.T) { // TestValidateSecretForReconcile tests the validateSecretForReconcile // predicate function. func TestValidateSecretForReconcile(t *testing.T) { + mtlsEnabledEnvoyProxyConfig := &v1alpha1.EnvoyProxy{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "mtls-settings", + }, + Spec: v1alpha1.EnvoyProxySpec{ + BackendTLS: &v1alpha1.BackendTLSConfig{ + ClientCertificateRef: &gwapiv1.SecretObjectReference{ + Kind: gatewayapi.KindPtr("Secret"), + Name: "client-tls-certificate", + }, + TLSSettings: v1alpha1.TLSSettings{}, + }, + }, + } testCases := []struct { name string configs []client.Object secret client.Object expect bool }{ + { + name: "envoy proxy references a secret", + configs: []client.Object{ + test.GetGatewayClass("test-secret-ref", v1alpha1.GatewayControllerName, &test.GroupKindNamespacedName{ + Group: gwapiv1.Group(mtlsEnabledEnvoyProxyConfig.GroupVersionKind().Group), + Kind: gwapiv1.Kind(mtlsEnabledEnvoyProxyConfig.Kind), + Namespace: gwapiv1.Namespace(mtlsEnabledEnvoyProxyConfig.Namespace), + Name: gwapiv1.ObjectName(mtlsEnabledEnvoyProxyConfig.Name), + }), + test.GetSecret(types.NamespacedName{Namespace: mtlsEnabledEnvoyProxyConfig.Namespace, Name: "client-tls-certificate"}), + mtlsEnabledEnvoyProxyConfig, + }, + secret: test.GetSecret(types.NamespacedName{Namespace: mtlsEnabledEnvoyProxyConfig.Namespace, Name: "client-tls-certificate"}), + expect: true, + }, { name: "references valid gateway", configs: []client.Object{ @@ -298,6 +328,7 @@ func TestValidateSecretForReconcile(t *testing.T) { WithObjects(tc.configs...). WithIndex(&gwapiv1.Gateway{}, secretGatewayIndex, secretGatewayIndexFunc). WithIndex(&v1alpha1.SecurityPolicy{}, secretSecurityPolicyIndex, secretSecurityPolicyIndexFunc). + WithIndex(&v1alpha1.EnvoyProxy{}, secretEnvoyProxyIndex, secretEnvoyProxyIndexFunc). Build() t.Run(tc.name, func(t *testing.T) { res := r.validateSecretForReconcile(tc.secret) diff --git a/test/e2e/testdata/backend-tls-settings-client-cert-rotation.yaml b/test/e2e/testdata/backend-tls-settings-client-cert-rotation.yaml new file mode 100644 index 00000000000..03ac6f32db2 --- /dev/null +++ b/test/e2e/testdata/backend-tls-settings-client-cert-rotation.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: v1 +data: + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURFVENDQWZrQ0ZGZ1A5ckEyNFA2NnhXcDRQSnEzY05KdHJYRXBNQTBHQ1NxR1NJYjNEUUVCQ3dVQU1FVXgKQ3pBSkJnTlZCQVlUQWtGVk1STXdFUVlEVlFRSURBcFRiMjFsTFZOMFlYUmxNU0V3SHdZRFZRUUtEQmhKYm5SbApjbTVsZENCWGFXUm5hWFJ6SUZCMGVTQk1kR1F3SGhjTk1qUXdOakF5TVRnMU5ESXlXaGNOTWpVd05qQXlNVGcxCk5ESXlXakJGTVFzd0NRWURWUVFHRXdKQlZURVRNQkVHQTFVRUNBd0tVMjl0WlMxVGRHRjBaVEVoTUI4R0ExVUUKQ2d3WVNXNTBaWEp1WlhRZ1YybGtaMmwwY3lCUWRIa2dUSFJrTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQwpBUThBTUlJQkNnS0NBUUVBblpYeHQ0Tk5TQVYxT2YyWDBDY1E2clk1eUlKQjRVWlkzVytPSlpQZmZtT0xZSFE5Cld6ZEQyUStCdHpocHNnVmhuZ25XZjVCeDlzbFExakdmSWY3eGFJbDNHcE13S3dsbk1mRlB6d0lxcTc1MHV4bzIKcU9QV1VwTHhoWXl4eVVHU0xXeTZuZ1RHOTBnRjM2MUNsWkJqVWxML2g3M2VHSmMydERVWG85T1k1SFhJc0hnbwp0WlJTWXdJV1kwbjgyMmFTT0tPTCtBeDc0eVV3ODMwSnRxK1RmbDlCbjFZMWNZVjgrNDZqVnhodzBVWHlac1diCk5CdllzQk1jZmFVcmdyWjlSVXYzYlJuNS9kUVFlanMvRzJVOGlmNW9qR0NaSG5nUEFLV0NyS0MxejU1RmVYbEgKSDR1cVJ5ZGRKL1IyYUVsYnkyeUhrSmVEaFRjQmhxUXFIT2dMU1FJREFRQUJNQTBHQ1NxR1NJYjNEUUVCQ3dVQQpBNElCQVFBTU1ldzg2dmVrM1hpdGNUd25lcnlNOUU0MjVCOVYrd24ya2swYTFNMDhlSGhNU2xHREQ4bG5mb1hKClNhZHEzbStBSlgyNHByZTBJQkMzRlhXajBrdCt5ZXN1dCtQR01jaHR1YXJYOU1PakpPRVk1NFFCRUdBNXJ0TzgKRWM0bWdqNkkzSGxGa2RTbUVkZTJjVTFPMDFlU3JZZG9vWUpmc2RLN1drUWEzNElRb1lPREhPbUFnMnluaVhZbQppSXZaUHkzUHdYSnN5VFNpYi9BMXQ3WEpoS3RIdUxvWGxPTHpVSUFMVkZsRzRNQjRreHcrMlVueFZHdjI0NUg0CjFsTnVaRk1BSXhWMXdZYkplbTFUT1ZLcFdMVmVqWE85SWZvcks1VmdqVzV2TkM1ejU3Um41dDljSGwxUHVhVmQKYk01U2ozU2pMWW52cXNQZ2JReXAxdWc3UEJBUwotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2QUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktZd2dnU2lBZ0VBQW9JQkFRQ2RsZkczZzAxSUJYVTUKL1pmUUp4RHF0am5JZ2tIaFJsamRiNDRsazk5K1k0dGdkRDFiTjBQWkQ0RzNPR215QldHZUNkWi9rSEgyeVZEVwpNWjhoL3ZGb2lYY2FrekFyQ1djeDhVL1BBaXFydm5TN0dqYW80OVpTa3ZHRmpMSEpRWkl0YkxxZUJNYjNTQVhmCnJVS1ZrR05TVXYrSHZkNFlsemEwTlJlajA1amtkY2l3ZUNpMWxGSmpBaFpqU2Z6YlpwSTRvNHY0REh2akpURHoKZlFtMnI1TitYMEdmVmpWeGhYejdqcU5YR0hEUlJmSm14WnMwRzlpd0V4eDlwU3VDdG4xRlMvZHRHZm45MUJCNgpPejhiWlR5Si9taU1ZSmtlZUE4QXBZS3NvTFhQbmtWNWVVY2ZpNnBISjEwbjlIWm9TVnZMYkllUWw0T0ZOd0dHCnBDb2M2QXRKQWdNQkFBRUNnZ0VBVVN0ejl4K2x4ZXd4eHE0TjdZVEIzUFFCN1hyU3JsUEs0RHhlenBKTmMwK1kKOGhUTnFkZGNsa2k3ZXdHT3g4aW4wSHlteE14Rk4yWFl3b3VLdVRzVlk5QXl6amgwZmlaNHhpdlMvUURUcytVRgpRWDZ1R3U1SWJmNE1jOHJWcEtHbXhza1RrdU5KRFFSSEtrQ1haR3ljREVKK0pwQXcveCtjYU5SVUdlbWEva215CklrZ0FRMUpxQ1lFNTJKUG1PWGRhL3VjMEJtMGYyQkFZUmh6V3JmRGV4SnlzelloMDdhNHNWVW9FRXlxbjlFV2EKbVZOQkV2SzQwVm1KVFcyV1ZuN3lZUnk2OUgvbGl2cGZWdG1RbE9SaVBkYXRaK1RUaXA2VHF2SFIwNHZyam1pMApQUXdhbkJmN3Z6aE9SU3Z1VDhOUEtWV1pkeHAxQm1JaWtrUmpSYjRyMFFLQmdRRE0yaWk5QXowZDVSbnZzV24zClJrWlNZbG1CaHhROGFKajBqY1l0cHBycEQ3ZFgvbGh4dDRHNFBSVWx5QTcwSmFMOVBrTVhZcklhNlpJMUQyRC8Kb2ZSbFoybE9HMUtnVVVTUW42bFFiaWo3NkdyaFhlOXhvRDE5eEdEai9HWTFSRDB1VGI5OWhtU1lyVzJySnREQwpMWG0vWUV1TjJGc2N4Q3QzOHhBVWl1bzZiUUtCZ1FERTdwV1laa3RzeXcyTkJhRmo1NXZJYkcyVjhkUk9vdzFsCnY4cWxsMllHWXpHWEJzNlBodzlGMUNzcjl4dm9HNmhPR1hFNmxOL25RZGtxTEtONEJqM25lNXZJMGUxKzI0TG0Ka0lZK0ltTWo1U2w4QzFuUzR3MVFVTlREYWIyakhackFXd0hJQlBUdFYzbDdMY3cvQXh5NnRjcVJ2alpoQjVnZAplL1N3RStzS3pRS0JnQm5wbHJHaUVUOFExZUVPRGh0clZrWGhqdlRsZzFmSTIyQkQ5c2ViaFlqcHBnV0pkT0tkCmxka2FVT3lBaS9PeU54WFZwR0wyNXhTa2F3d3ZMOVBtUnFYMUdNcjZoYzhsdUlpYXlhNFU0VFpNUmdqUCt4UGkKY3lUUGpIb0tXVnR2a0ZXbEhBM2l6Q0xML1UxakVaRWVjNElUeWpyZEhWbGNMeXR6SVp1WG50MVpBb0dBQUxsago0WENXM0dxT3ZUTUZHZW56SDdTT1hwdktEUlA2YTZKZDYyZjRIeFBrVGNyZm5aV0FqK0FzM0hlSEtiNVlTeGs2CjJsMUx5WHpyZ0lVemdMQjlMOG03ayt4NXRCcTRpNEtDaTkzeWdXSkpXY1JzNnlLY25Pdi9MRXpLUHJ4UUlsN3oKVGJuKzhKUit4TjR4UHhZUzEvanRLc1lKU3lnTS9pYkRpcFk0S2cwQ2dZQmExL0RYQlVIVWI3UlFWcVBrdVJsYQpHenhVeTd4c3dXQkNVbWtkNkdyUG4vM0pGRW5udTRCdUovUHBRNTNQY1hLTTNWM041Uis2dkVHdHRteWtxRDZZCkp5NjZrQkxMdk1aLzZvUHJzZWxNUUVZZjRwa21CSnlHdHNxOElkR3RZT1dVdXpwbmhuTUdWRG8vODVrUVIvaEwKaUhxQW9TMEJXMDU1eEdsVzVFa2VGZz09Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K +kind: Secret +metadata: + name: client-tls-certificate + namespace: envoy-gateway-system +type: kubernetes.io/tls +--- +apiVersion: v1 +data: + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURFVENDQWZrQ0ZGZ1A5ckEyNFA2NnhXcDRQSnEzY05KdHJYRXBNQTBHQ1NxR1NJYjNEUUVCQ3dVQU1FVXgKQ3pBSkJnTlZCQVlUQWtGVk1STXdFUVlEVlFRSURBcFRiMjFsTFZOMFlYUmxNU0V3SHdZRFZRUUtEQmhKYm5SbApjbTVsZENCWGFXUm5hWFJ6SUZCMGVTQk1kR1F3SGhjTk1qUXdOakF5TVRnMU5ESXlXaGNOTWpVd05qQXlNVGcxCk5ESXlXakJGTVFzd0NRWURWUVFHRXdKQlZURVRNQkVHQTFVRUNBd0tVMjl0WlMxVGRHRjBaVEVoTUI4R0ExVUUKQ2d3WVNXNTBaWEp1WlhRZ1YybGtaMmwwY3lCUWRIa2dUSFJrTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQwpBUThBTUlJQkNnS0NBUUVBblpYeHQ0Tk5TQVYxT2YyWDBDY1E2clk1eUlKQjRVWlkzVytPSlpQZmZtT0xZSFE5Cld6ZEQyUStCdHpocHNnVmhuZ25XZjVCeDlzbFExakdmSWY3eGFJbDNHcE13S3dsbk1mRlB6d0lxcTc1MHV4bzIKcU9QV1VwTHhoWXl4eVVHU0xXeTZuZ1RHOTBnRjM2MUNsWkJqVWxML2g3M2VHSmMydERVWG85T1k1SFhJc0hnbwp0WlJTWXdJV1kwbjgyMmFTT0tPTCtBeDc0eVV3ODMwSnRxK1RmbDlCbjFZMWNZVjgrNDZqVnhodzBVWHlac1diCk5CdllzQk1jZmFVcmdyWjlSVXYzYlJuNS9kUVFlanMvRzJVOGlmNW9qR0NaSG5nUEFLV0NyS0MxejU1RmVYbEgKSDR1cVJ5ZGRKL1IyYUVsYnkyeUhrSmVEaFRjQmhxUXFIT2dMU1FJREFRQUJNQTBHQ1NxR1NJYjNEUUVCQ3dVQQpBNElCQVFBTU1ldzg2dmVrM1hpdGNUd25lcnlNOUU0MjVCOVYrd24ya2swYTFNMDhlSGhNU2xHREQ4bG5mb1hKClNhZHEzbStBSlgyNHByZTBJQkMzRlhXajBrdCt5ZXN1dCtQR01jaHR1YXJYOU1PakpPRVk1NFFCRUdBNXJ0TzgKRWM0bWdqNkkzSGxGa2RTbUVkZTJjVTFPMDFlU3JZZG9vWUpmc2RLN1drUWEzNElRb1lPREhPbUFnMnluaVhZbQppSXZaUHkzUHdYSnN5VFNpYi9BMXQ3WEpoS3RIdUxvWGxPTHpVSUFMVkZsRzRNQjRreHcrMlVueFZHdjI0NUg0CjFsTnVaRk1BSXhWMXdZYkplbTFUT1ZLcFdMVmVqWE85SWZvcks1VmdqVzV2TkM1ejU3Um41dDljSGwxUHVhVmQKYk01U2ozU2pMWW52cXNQZ2JReXAxdWc3UEJBUwotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2QUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktZd2dnU2lBZ0VBQW9JQkFRQ2RsZkczZzAxSUJYVTUKL1pmUUp4RHF0am5JZ2tIaFJsamRiNDRsazk5K1k0dGdkRDFiTjBQWkQ0RzNPR215QldHZUNkWi9rSEgyeVZEVwpNWjhoL3ZGb2lYY2FrekFyQ1djeDhVL1BBaXFydm5TN0dqYW80OVpTa3ZHRmpMSEpRWkl0YkxxZUJNYjNTQVhmCnJVS1ZrR05TVXYrSHZkNFlsemEwTlJlajA1amtkY2l3ZUNpMWxGSmpBaFpqU2Z6YlpwSTRvNHY0REh2akpURHoKZlFtMnI1TitYMEdmVmpWeGhYejdqcU5YR0hEUlJmSm14WnMwRzlpd0V4eDlwU3VDdG4xRlMvZHRHZm45MUJCNgpPejhiWlR5Si9taU1ZSmtlZUE4QXBZS3NvTFhQbmtWNWVVY2ZpNnBISjEwbjlIWm9TVnZMYkllUWw0T0ZOd0dHCnBDb2M2QXRKQWdNQkFBRUNnZ0VBVVN0ejl4K2x4ZXd4eHE0TjdZVEIzUFFCN1hyU3JsUEs0RHhlenBKTmMwK1kKOGhUTnFkZGNsa2k3ZXdHT3g4aW4wSHlteE14Rk4yWFl3b3VLdVRzVlk5QXl6amgwZmlaNHhpdlMvUURUcytVRgpRWDZ1R3U1SWJmNE1jOHJWcEtHbXhza1RrdU5KRFFSSEtrQ1haR3ljREVKK0pwQXcveCtjYU5SVUdlbWEva215CklrZ0FRMUpxQ1lFNTJKUG1PWGRhL3VjMEJtMGYyQkFZUmh6V3JmRGV4SnlzelloMDdhNHNWVW9FRXlxbjlFV2EKbVZOQkV2SzQwVm1KVFcyV1ZuN3lZUnk2OUgvbGl2cGZWdG1RbE9SaVBkYXRaK1RUaXA2VHF2SFIwNHZyam1pMApQUXdhbkJmN3Z6aE9SU3Z1VDhOUEtWV1pkeHAxQm1JaWtrUmpSYjRyMFFLQmdRRE0yaWk5QXowZDVSbnZzV24zClJrWlNZbG1CaHhROGFKajBqY1l0cHBycEQ3ZFgvbGh4dDRHNFBSVWx5QTcwSmFMOVBrTVhZcklhNlpJMUQyRC8Kb2ZSbFoybE9HMUtnVVVTUW42bFFiaWo3NkdyaFhlOXhvRDE5eEdEai9HWTFSRDB1VGI5OWhtU1lyVzJySnREQwpMWG0vWUV1TjJGc2N4Q3QzOHhBVWl1bzZiUUtCZ1FERTdwV1laa3RzeXcyTkJhRmo1NXZJYkcyVjhkUk9vdzFsCnY4cWxsMllHWXpHWEJzNlBodzlGMUNzcjl4dm9HNmhPR1hFNmxOL25RZGtxTEtONEJqM25lNXZJMGUxKzI0TG0Ka0lZK0ltTWo1U2w4QzFuUzR3MVFVTlREYWIyakhackFXd0hJQlBUdFYzbDdMY3cvQXh5NnRjcVJ2alpoQjVnZAplL1N3RStzS3pRS0JnQm5wbHJHaUVUOFExZUVPRGh0clZrWGhqdlRsZzFmSTIyQkQ5c2ViaFlqcHBnV0pkT0tkCmxka2FVT3lBaS9PeU54WFZwR0wyNXhTa2F3d3ZMOVBtUnFYMUdNcjZoYzhsdUlpYXlhNFU0VFpNUmdqUCt4UGkKY3lUUGpIb0tXVnR2a0ZXbEhBM2l6Q0xML1UxakVaRWVjNElUeWpyZEhWbGNMeXR6SVp1WG50MVpBb0dBQUxsago0WENXM0dxT3ZUTUZHZW56SDdTT1hwdktEUlA2YTZKZDYyZjRIeFBrVGNyZm5aV0FqK0FzM0hlSEtiNVlTeGs2CjJsMUx5WHpyZ0lVemdMQjlMOG03ayt4NXRCcTRpNEtDaTkzeWdXSkpXY1JzNnlLY25Pdi9MRXpLUHJ4UUlsN3oKVGJuKzhKUit4TjR4UHhZUzEvanRLc1lKU3lnTS9pYkRpcFk0S2cwQ2dZQmExL0RYQlVIVWI3UlFWcVBrdVJsYQpHenhVeTd4c3dXQkNVbWtkNkdyUG4vM0pGRW5udTRCdUovUHBRNTNQY1hLTTNWM041Uis2dkVHdHRteWtxRDZZCkp5NjZrQkxMdk1aLzZvUHJzZWxNUUVZZjRwa21CSnlHdHNxOElkR3RZT1dVdXpwbmhuTUdWRG8vODVrUVIvaEwKaUhxQW9TMEJXMDU1eEdsVzVFa2VGZz09Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K +kind: Secret +metadata: + name: client-tls-validation + namespace: gateway-conformance-infra +type: kubernetes.io/tls diff --git a/test/e2e/testdata/upstream-tls.yaml b/test/e2e/testdata/backend-tls-settings.yaml similarity index 100% rename from test/e2e/testdata/upstream-tls.yaml rename to test/e2e/testdata/backend-tls-settings.yaml diff --git a/test/e2e/testdata/expect/echo-service-tls-settings-new-mtls-secret.yaml b/test/e2e/testdata/expect/echo-service-tls-settings-new-mtls-secret.yaml new file mode 100644 index 00000000000..9a11a7cc1f4 --- /dev/null +++ b/test/e2e/testdata/expect/echo-service-tls-settings-new-mtls-secret.yaml @@ -0,0 +1,28 @@ +--- +tls: + cipherSuite: "TLS_AES_128_GCM_SHA256" + negotiatedProtocol: "http/1.1" + peerCertificates: + - | + -----BEGIN CERTIFICATE----- + MIIDETCCAfkCFFgP9rA24P66xWp4PJq3cNJtrXEpMA0GCSqGSIb3DQEBCwUAMEUx + CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl + cm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjQwNjAyMTg1NDIyWhcNMjUwNjAyMTg1 + NDIyWjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE + CgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC + AQ8AMIIBCgKCAQEAnZXxt4NNSAV1Of2X0CcQ6rY5yIJB4UZY3W+OJZPffmOLYHQ9 + WzdD2Q+BtzhpsgVhngnWf5Bx9slQ1jGfIf7xaIl3GpMwKwlnMfFPzwIqq750uxo2 + qOPWUpLxhYyxyUGSLWy6ngTG90gF361ClZBjUlL/h73eGJc2tDUXo9OY5HXIsHgo + tZRSYwIWY0n822aSOKOL+Ax74yUw830Jtq+Tfl9Bn1Y1cYV8+46jVxhw0UXyZsWb + NBvYsBMcfaUrgrZ9RUv3bRn5/dQQejs/G2U8if5ojGCZHngPAKWCrKC1z55FeXlH + H4uqRyddJ/R2aElby2yHkJeDhTcBhqQqHOgLSQIDAQABMA0GCSqGSIb3DQEBCwUA + A4IBAQAMMew86vek3XitcTwneryM9E425B9V+wn2kk0a1M08eHhMSlGDD8lnfoXJ + Sadq3m+AJX24pre0IBC3FXWj0kt+yesut+PGMchtuarX9MOjJOEY54QBEGA5rtO8 + Ec4mgj6I3HlFkdSmEde2cU1O01eSrYdooYJfsdK7WkQa34IQoYODHOmAg2yniXYm + iIvZPy3PwXJsyTSib/A1t7XJhKtHuLoXlOLzUIALVFlG4MB4kxw+2UnxVGv245H4 + 1lNuZFMAIxV1wYbJem1TOVKpWLVejXO9IforK5VgjW5vNC5z57Rn5t9cHl1PuaVd + bM5Sj3SjLYnvqsPgbQyp1ug7PBAS + -----END CERTIFICATE----- + + serverName: "example.com" + version: "TLSv1.3" diff --git a/test/e2e/testdata/expect/echo-service-tls-settings-updated-tls-settings.yaml b/test/e2e/testdata/expect/echo-service-tls-settings-updated-tls-settings.yaml new file mode 100644 index 00000000000..369950be307 --- /dev/null +++ b/test/e2e/testdata/expect/echo-service-tls-settings-updated-tls-settings.yaml @@ -0,0 +1,28 @@ +--- +tls: + cipherSuite: "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384" + negotiatedProtocol: "http/1.1" + peerCertificates: + - | + -----BEGIN CERTIFICATE----- + MIIDETCCAfkCFFgP9rA24P66xWp4PJq3cNJtrXEpMA0GCSqGSIb3DQEBCwUAMEUx + CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl + cm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjQwNjAyMTg1NDIyWhcNMjUwNjAyMTg1 + NDIyWjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE + CgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC + AQ8AMIIBCgKCAQEAnZXxt4NNSAV1Of2X0CcQ6rY5yIJB4UZY3W+OJZPffmOLYHQ9 + WzdD2Q+BtzhpsgVhngnWf5Bx9slQ1jGfIf7xaIl3GpMwKwlnMfFPzwIqq750uxo2 + qOPWUpLxhYyxyUGSLWy6ngTG90gF361ClZBjUlL/h73eGJc2tDUXo9OY5HXIsHgo + tZRSYwIWY0n822aSOKOL+Ax74yUw830Jtq+Tfl9Bn1Y1cYV8+46jVxhw0UXyZsWb + NBvYsBMcfaUrgrZ9RUv3bRn5/dQQejs/G2U8if5ojGCZHngPAKWCrKC1z55FeXlH + H4uqRyddJ/R2aElby2yHkJeDhTcBhqQqHOgLSQIDAQABMA0GCSqGSIb3DQEBCwUA + A4IBAQAMMew86vek3XitcTwneryM9E425B9V+wn2kk0a1M08eHhMSlGDD8lnfoXJ + Sadq3m+AJX24pre0IBC3FXWj0kt+yesut+PGMchtuarX9MOjJOEY54QBEGA5rtO8 + Ec4mgj6I3HlFkdSmEde2cU1O01eSrYdooYJfsdK7WkQa34IQoYODHOmAg2yniXYm + iIvZPy3PwXJsyTSib/A1t7XJhKtHuLoXlOLzUIALVFlG4MB4kxw+2UnxVGv245H4 + 1lNuZFMAIxV1wYbJem1TOVKpWLVejXO9IforK5VgjW5vNC5z57Rn5t9cHl1PuaVd + bM5Sj3SjLYnvqsPgbQyp1ug7PBAS + -----END CERTIFICATE----- + + serverName: "example.com" + version: "TLSv1.2" diff --git a/test/e2e/tests/uptream_tls_settings.go b/test/e2e/tests/backend_tls_settings.go similarity index 55% rename from test/e2e/tests/uptream_tls_settings.go rename to test/e2e/tests/backend_tls_settings.go index a6f19a583e6..14205b6e0b2 100644 --- a/test/e2e/tests/uptream_tls_settings.go +++ b/test/e2e/tests/backend_tls_settings.go @@ -16,15 +16,21 @@ import ( nethttp "net/http" "os" "path" - "reflect" "testing" "time" + "github.com/google/go-cmp/cmp" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" "sigs.k8s.io/gateway-api/apis/v1alpha3" + "sigs.k8s.io/gateway-api/conformance/utils/config" "sigs.k8s.io/gateway-api/conformance/utils/http" "sigs.k8s.io/gateway-api/conformance/utils/kubernetes" "sigs.k8s.io/gateway-api/conformance/utils/suite" @@ -34,18 +40,18 @@ import ( "github.com/envoyproxy/gateway/internal/gatewayapi" ) -const UpstreamTLSChangesMaxTimeout = 30 * time.Second +const BackendTLSChangesMaxTimeout = 30 * time.Second func init() { - ConformanceTests = append(ConformanceTests, UpstreamTLSSettingsTest) + ConformanceTests = append(ConformanceTests, BackendTLSSettingsTest) } -var UpstreamTLSSettingsTest = suite.ConformanceTest{ - ShortName: "Upstream tls settings", - Description: "Use envoy proxy tls settings with upstream", - Manifests: []string{"testdata/upstream-tls.yaml"}, +var BackendTLSSettingsTest = suite.ConformanceTest{ + ShortName: "Backend tls settings", + Description: "Use envoy proxy tls settings with backend", + Manifests: []string{"testdata/backend-tls-settings.yaml"}, Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { - t.Run("Apply custom TLS settings when making upstream requests.", func(t *testing.T) { + t.Run("Apply custom TLS settings when making backend requests.", func(t *testing.T) { depNS := "envoy-gateway-system" ns := "gateway-conformance-infra" routeNN := types.NamespacedName{Name: "http-with-backend-tls", Namespace: ns} @@ -75,11 +81,6 @@ var UpstreamTLSSettingsTest = suite.ConformanceTest{ if err != nil { t.Error(err) } - transport := &nethttp.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, //nolint:gosec - }, - } expectedRes, err := asExpectedResponse("echo-service-tls-settings-res") if err != nil { @@ -94,45 +95,66 @@ var UpstreamTLSSettingsTest = suite.ConformanceTest{ }, Namespace: ns, } - confirmEchoBackendRes := func(httpRes *http.ExpectedResponse, expectedResBody *Response) error { - req := http.MakeRequest(t, httpRes, gwAddr, "HTTPS", "https") - res, err := casePreservingRoundTrip(req, transport, suite) - if err != nil { - t.Log(err) - } - err = expectNewEchoBackendResponse(res, expectedResBody) - if err != nil { - return err - } - return nil + + // Reconfigure backend tls settings + err = WaitUntil(func(httpRes *http.ExpectedResponse, expectedResBody *Response) error { + return confirmEchoBackendRes(httpRes, expectedResBody, gwAddr, t, suite) + }, BackendTLSChangesMaxTimeout, &expectOkResp, expectedRes) + if err != nil { + t.Error(err) + } + + // rotate the client mTLS secret to ensure that a new secret is used. + suite.Applier.MustApplyWithCleanup(t, suite.Client, suite.TimeoutConfig, "testdata/backend-tls-settings-client-cert-rotation.yaml", false) + + err = restartDeploymentAndWaitForRollout(t, suite.TimeoutConfig, suite.Client, &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tls-backend", + Namespace: "gateway-conformance-infra", + }, + }) + if err != nil { + t.Error(err) + } + + // confirm new mtls client cert is used when connecting to backend + expectedResNewMTLSSecret, err := asExpectedResponse("echo-service-tls-settings-new-mtls-secret") + if err != nil { + t.Error(err) } - // Reconfigure upstream tls settings + err = WaitUntil(func(httpRes *http.ExpectedResponse, expectedResBody *Response) error { - return confirmEchoBackendRes(httpRes, expectedResBody) - }, UpstreamTLSChangesMaxTimeout, &expectOkResp, expectedRes) + return confirmEchoBackendRes(httpRes, expectedResBody, gwAddr, t, suite) + }, BackendTLSChangesMaxTimeout, &expectOkResp, expectedResNewMTLSSecret) if err != nil { t.Error(err) } - // Ensure that changes to envoy proxy re-configure the upstream tls settings. + config.TLSSettings = v1alpha1.TLSSettings{ MinVersion: ptr.To(v1alpha1.TLSv12), MaxVersion: ptr.To(v1alpha1.TLSv12), Ciphers: []string{"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384"}, } + err = UpdateProxyConfig(suite.Client, proxyNN, config) if err != nil { t.Error(err) } - expectedRes.TLS.Version = "TLSv1.2" - expectedRes.TLS.CipherSuite = "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384" + + // confirm tls settings can be updated + expectedUpdatedTLSSettings, err := asExpectedResponse("echo-service-tls-settings-updated-tls-settings") + if err != nil { + t.Error(err) + } + err = WaitUntil(func(httpRes *http.ExpectedResponse, expectedResBody *Response) error { - return confirmEchoBackendRes(httpRes, expectedResBody) - }, UpstreamTLSChangesMaxTimeout, &expectOkResp, expectedRes) + return confirmEchoBackendRes(httpRes, expectedResBody, gwAddr, t, suite) + }, BackendTLSChangesMaxTimeout, &expectOkResp, expectedUpdatedTLSSettings) if err != nil { t.Error(err) } - // Cleanup upstream tls settings. + // Cleanup backend tls settings. err = UpdateProxyConfig(suite.Client, proxyNN, &v1alpha1.BackendTLSConfig{ ClientCertificateRef: nil, TLSSettings: v1alpha1.TLSSettings{}, @@ -144,6 +166,24 @@ var UpstreamTLSSettingsTest = suite.ConformanceTest{ }, } +func confirmEchoBackendRes(httpRes *http.ExpectedResponse, expectedResBody *Response, gwAddr string, t *testing.T, suite *suite.ConformanceTestSuite) error { + transport := &nethttp.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, //nolint:gosec + }, + } + req := http.MakeRequest(t, httpRes, gwAddr, "HTTPS", "https") + res, err := casePreservingRoundTrip(req, transport, suite) + if err != nil { + return err + } + err = expectNewEchoBackendResponse(res, expectedResBody) + if err != nil { + return err + } + return nil +} + // UpdateProxyConfig updates the proxy configuration with BackendTLS settings. func UpdateProxyConfig(client client.Client, proxyNN types.NamespacedName, config *v1alpha1.BackendTLSConfig) error { proxyConfig := &v1alpha1.EnvoyProxy{} @@ -185,10 +225,10 @@ func expectNewEchoBackendResponse(respBody interface{}, expect *Response) error return err } - if ok, err := hasAllFieldsAndValues(res.TLS, expect.TLS); !ok { - return err + if cmp.Equal(res.TLS, expect.TLS) { + return nil } - return nil + return fmt.Errorf("mismatch found between returned and expected response. Difference: %s", cmp.Diff(res.TLS, expect.TLS)) } func asExpectedResponse(fileName string) (*Response, error) { @@ -207,76 +247,64 @@ func asExpectedResponse(fileName string) (*Response, error) { return &res, nil } -// Function to check if obj1 has all the fields of obj2, including nested fields, and matching values -func hasAllFieldsAndValues(obj1, obj2 interface{}) (bool, error) { - return hasAllFieldsAndValuesRecursive(reflect.ValueOf(obj1), reflect.ValueOf(obj2)) +// Response defines echo server response +type Response struct { + TLS TLSInfo `json:"tls"` } -func hasAllFieldsAndValuesRecursive(v1, v2 reflect.Value) (bool, error) { - if v1.Kind() == reflect.Ptr { - v1 = v1.Elem() - } - if v2.Kind() == reflect.Ptr { - v2 = v2.Elem() - } +type TLSInfo struct { + Version string `json:"version"` + PeerCertificates []string `json:"peerCertificates"` + ServerName string `json:"serverName"` + NegotiatedProtocol string `json:"negotiatedProtocol"` + CipherSuite string `json:"cipherSuite"` +} - if v1.Kind() != reflect.Struct || v2.Kind() != reflect.Struct { - return false, fmt.Errorf("both parameters must be structs") +func restartDeploymentAndWaitForRollout(t *testing.T, timeoutConfig config.TimeoutConfig, c client.Client, dp *appsv1.Deployment) error { + t.Helper() + const restartAnnotation = "kubectl.kubernetes.io/restartedAt" + restartTime := time.Now().Format(time.RFC3339) + ctx := context.Background() + + if err := c.Get(context.Background(), types.NamespacedName{Name: dp.Name, Namespace: dp.Namespace}, dp); err != nil { + return err } - t1 := v1.Type() - t2 := v2.Type() + // Update an annotation to trigger a rolling update + if dp.Spec.Template.Annotations == nil { + dp.Spec.Template.Annotations = make(map[string]string) + } + dp.Spec.Template.Annotations["kubectl.kubernetes.io/restartedAt"] = restartTime - for i := 0; i < t2.NumField(); i++ { - field2 := t2.Field(i) - field1, found := t1.FieldByName(field2.Name) + if err := c.Update(ctx, dp); err != nil { + return err + } - if !found { - fmt.Printf("Field %s is missing in obj1\n", field2.Name) - return false, nil + return wait.PollUntilContextTimeout(ctx, 1*time.Second, timeoutConfig.CreateTimeout, true, func(ctx context.Context) (bool, error) { + // wait for replicaset with the same annotation to reach ready status + podList := &v1.PodList{} + listOpts := []client.ListOption{ + client.InNamespace(dp.Namespace), + client.MatchingLabelsSelector{Selector: labels.SelectorFromSet(dp.Spec.Selector.MatchLabels)}, } - value1 := v1.FieldByName(field2.Name) - value2 := v2.Field(i) + err := c.List(ctx, podList, listOpts...) + if err != nil { + return false, err + } - // Recursively check nested fields - if field2.Type.Kind() == reflect.Struct { - hasFields, err := hasAllFieldsAndValuesRecursive(value1, value2) - if err != nil || !hasFields { - return hasFields, err - } - } else { - // Check if the field types and values are the same - if field1.Type != field2.Type { - return false, fmt.Errorf("field %s has different type in obj1: %v (expected %v)", field2.Name, field1.Type, field2.Type) - } - if !reflect.DeepEqual(value1.Interface(), value2.Interface()) { - return false, fmt.Errorf("field %s has different value in obj1: %v (expected %v)", field2.Name, value1.Interface(), value2.Interface()) + rolled := int32(0) + for _, rs := range podList.Items { + if rs.Annotations[restartAnnotation] == restartTime { + rolled++ } } - } - return true, nil -} - -// Response defines echo server response -type Response struct { - Path string `json:"path"` - Host string `json:"host"` - Method string `json:"method"` - Proto string `json:"proto"` - Headers map[string][]string `json:"headers"` - Namespace string `json:"namespace"` - Ingress string `json:"ingress"` - Service string `json:"service"` - Pod string `json:"pod"` - TLS TLSInfo `json:"tls"` -} + // all pods are rolled + if rolled == int32(len(podList.Items)) && rolled >= *dp.Spec.Replicas { + return true, nil + } -type TLSInfo struct { - Version string `json:"version"` - PeerCertificates []string `json:"peerCertificates"` - ServerName string `json:"serverName"` - NegotiatedProtocol string `json:"negotiatedProtocol"` - CipherSuite string `json:"cipherSuite"` + return false, nil + }) }