feat: support merging user and default bootstrap configurations (#1791)

Signed-off-by: huabing zhao <zhaohuabing@gmail.com>
Co-authored-by: zirain <zirain2009@gmail.com>

api/config/v1alpha1/envoyproxy_types.go

@@ -56,7 +56,7 @@ type EnvoyProxySpec struct {
 	// Bootstrap configuration used. You can edit this configuration, and rerun `egctl x translate` to ensure there are no validation errors.
 	//
 	// +optional
-	Bootstrap *string `json:"bootstrap,omitempty"`
+	Bootstrap *ProxyBootstrap `json:"bootstrap,omitempty"`
 
 	// Concurrency defines the number of worker threads to run. If unset, it defaults to
 	// the number of cpuset threads on the platform.
@@ -161,6 +161,32 @@ const (
 	LogComponentRuntime LogComponent = "runtime"
 )
 
+// ProxyBootstrap defines Envoy Bootstrap configuration.
+type ProxyBootstrap struct {
+	// Type is the type of the bootstrap configuration, it should be either Replace or Merge.
+	// If unspecified, it defaults to Replace.
+	// +optional
+	// +kubebuilder:default=Replace
+	Type *BootstrapType `json:"type"`
+
+	// Value is a YAML string of the bootstrap.
+	Value string `json:"value"`
+}
+
+// BootstrapType defines the types of bootstrap supported by Envoy Gateway.
+// +kubebuilder:validation:Enum=Merge;Replace
+type BootstrapType string
+
+const (
+	// Merge merges the provided bootstrap with the default one. The provided bootstrap can add or override a value
+	// within a map, or add a new value to a list.
+	// Please note that the provided bootstrap can't override a value within a list.
+	BootstrapTypeMerge BootstrapType = "Merge"
+
+	// Replace replaces the default bootstrap with the provided one.
+	BootstrapTypeReplace BootstrapType = "Replace"
+)
+
 // EnvoyProxyStatus defines the observed state of EnvoyProxy. This type is not implemented
 // until https://github.com/envoyproxy/gateway/issues/1007 is fixed.
 type EnvoyProxyStatus struct {

api/config/v1alpha1/validation/testdata/merge-user-bootstrap.yaml

@@ -0,0 +1,19 @@
+admin:
+  address:
+    socket_address:
+      port_value: 8080
+static_resources:
+  clusters:
+  - name: prometheus_stats
+    connect_timeout: 0.250s
+    type: STATIC
+    lb_policy: ROUND_ROBIN
+    load_assignment:
+      cluster_name: prometheus_stats
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: 127.0.0.1
+                port_value: 19000

api/config/v1alpha1/validation/validate.go

@@ -87,13 +87,25 @@ func validateServiceType(spec *egcfgv1a1.EnvoyProxySpec) []error {
 	return errs
 }
 
-func validateBootstrap(boostrapConfig *string) error {
-	userBootstrap := &bootstrapv3.Bootstrap{}
-	jsonData, err := yaml.YAMLToJSON([]byte(*boostrapConfig))
+func validateBootstrap(boostrapConfig *egcfgv1a1.ProxyBootstrap) error {
+	defaultBootstrap := &bootstrapv3.Bootstrap{}
+	// TODO: need validate when enable prometheus?
+	defaultBootstrapStr, err := bootstrap.GetRenderedBootstrapConfig(nil)
+	if err != nil {
+		return err
+	}
+
+	userBootstrapStr, err := bootstrap.ApplyBootstrapConfig(boostrapConfig, defaultBootstrapStr)
+	if err != nil {
+		return err
+	}
+
+	jsonData, err := yaml.YAMLToJSON([]byte(userBootstrapStr))
 	if err != nil {
 		return fmt.Errorf("unable to convert user bootstrap to json: %w", err)
 	}
 
+	userBootstrap := &bootstrapv3.Bootstrap{}
 	if err := protojson.Unmarshal(jsonData, userBootstrap); err != nil {
 		return fmt.Errorf("unable to unmarshal user bootstrap: %w", err)
 	}
@@ -102,12 +114,6 @@ func validateBootstrap(boostrapConfig *string) error {
 	if err := userBootstrap.Validate(); err != nil {
 		return fmt.Errorf("validation failed for user bootstrap: %w", err)
 	}
-	defaultBootstrap := &bootstrapv3.Bootstrap{}
-	// TODO: need validate when enable prometheus?
-	defaultBootstrapStr, err := bootstrap.GetRenderedBootstrapConfig(nil)
-	if err != nil {
-		return err
-	}
 
 	jsonData, err = yaml.YAMLToJSON([]byte(defaultBootstrapStr))
 	if err != nil {

api/config/v1alpha1/validation/validate_test.go

@@ -15,13 +15,16 @@ import (
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/pointer"
 
 	egcfgv1a1 "github.com/envoyproxy/gateway/api/config/v1alpha1"
 )
 
 var (
 	//go:embed testdata/valid-user-bootstrap.yaml
 	validUserBootstrap string
+	//go:embed testdata/merge-user-bootstrap.yaml
+	mergeUserBootstrap string
 	//go:embed testdata/missing-admin-address-user-bootstrap.yaml
 	missingAdminAddressUserBootstrap string
 	//go:embed testdata/different-dynamic-resources-user-bootstrap.yaml
@@ -168,14 +171,32 @@ func TestValidateEnvoyProxy(t *testing.T) {
 			expected: true,
 		},
 		{
-			name: "valid user bootstrap",
+			name: "valid user bootstrap replace type",
 			proxy: &egcfgv1a1.EnvoyProxy{
 				ObjectMeta: metav1.ObjectMeta{
 					Namespace: "test",
 					Name:      "test",
 				},
 				Spec: egcfgv1a1.EnvoyProxySpec{
-					Bootstrap: &validUserBootstrap,
+					Bootstrap: &egcfgv1a1.ProxyBootstrap{
+						Value: validUserBootstrap,
+					},
+				},
+			},
+			expected: true,
+		},
+		{
+			name: "valid user bootstrap merge type",
+			proxy: &egcfgv1a1.EnvoyProxy{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "test",
+					Name:      "test",
+				},
+				Spec: egcfgv1a1.EnvoyProxySpec{
+					Bootstrap: &egcfgv1a1.ProxyBootstrap{
+						Type:  (*egcfgv1a1.BootstrapType)(pointer.String(string(egcfgv1a1.BootstrapTypeMerge))),
+						Value: mergeUserBootstrap,
+					},
 				},
 			},
 			expected: true,
@@ -188,7 +209,9 @@ func TestValidateEnvoyProxy(t *testing.T) {
 					Name:      "test",
 				},
 				Spec: egcfgv1a1.EnvoyProxySpec{
-					Bootstrap: &missingAdminAddressUserBootstrap,
+					Bootstrap: &egcfgv1a1.ProxyBootstrap{
+						Value: missingAdminAddressUserBootstrap,
+					},
 				},
 			},
 			expected: false,
@@ -201,7 +224,9 @@ func TestValidateEnvoyProxy(t *testing.T) {
 					Name:      "test",
 				},
 				Spec: egcfgv1a1.EnvoyProxySpec{
-					Bootstrap: &differentDynamicResourcesUserBootstrap,
+					Bootstrap: &egcfgv1a1.ProxyBootstrap{
+						Value: differentDynamicResourcesUserBootstrap,
+					},
 				},
 			},
 			expected: false,
@@ -214,7 +239,9 @@ func TestValidateEnvoyProxy(t *testing.T) {
 					Name:      "test",
 				},
 				Spec: egcfgv1a1.EnvoyProxySpec{
-					Bootstrap: &differentXdsClusterAddressBootstrap,
+					Bootstrap: &egcfgv1a1.ProxyBootstrap{
+						Value: differentXdsClusterAddressBootstrap,
+					},
 				},
 			},
 			expected: false,

charts/gateway-helm/crds/generated/config.gateway.envoyproxy.io_envoyproxies.yaml

@@ -49,7 +49,22 @@ spec:
                   `Bootstrap` field set to the default Bootstrap configuration used.
                   You can edit this configuration, and rerun `egctl x translate` to
                   ensure there are no validation errors.
-                type: string
+                properties:
+                  type:
+                    default: Replace
+                    description: Type is the type of the bootstrap configuration,
+                      it should be either Replace or Merge. If unspecified, it defaults
+                      to Replace.
+                    enum:
+                    - Merge
+                    - Replace
+                    type: string
+                  value:
+                    description: Value is a YAML string of the bootstrap.
+                    type: string
+                required:
+                - value
+                type: object
               concurrency:
                 description: Concurrency defines the number of worker threads to run.
                   If unset, it defaults to the number of cpuset threads on the platform.

docs/latest/api/config_types.md

@@ -16,6 +16,17 @@ API group.
 
 
 
+## BootstrapType
+
+_Underlying type:_ `string`
+
+BootstrapType defines the types of bootstrap supported by Envoy Gateway.
+
+_Appears in:_
+- [ProxyBootstrap](#proxybootstrap)
+
+
+
 ## CustomTag
 
 
@@ -321,7 +332,7 @@ _Appears in:_
 | `provider` _[EnvoyProxyProvider](#envoyproxyprovider)_ | Provider defines the desired resource provider and provider-specific configuration. If unspecified, the "Kubernetes" resource provider is used with default configuration parameters. |
 | `logging` _[ProxyLogging](#proxylogging)_ | Logging defines logging parameters for managed proxies. |
 | `telemetry` _[ProxyTelemetry](#proxytelemetry)_ | Telemetry defines telemetry parameters for managed proxies. |
-| `bootstrap` _string_ | Bootstrap defines the Envoy Bootstrap as a YAML string. Visit https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/bootstrap/v3/bootstrap.proto#envoy-v3-api-msg-config-bootstrap-v3-bootstrap to learn more about the syntax. If set, this is the Bootstrap configuration used for the managed Envoy Proxy fleet instead of the default Bootstrap configuration set by Envoy Gateway. Some fields within the Bootstrap that are required to communicate with the xDS Server (Envoy Gateway) and receive xDS resources from it are not configurable and will result in the `EnvoyProxy` resource being rejected. Backward compatibility across minor versions is not guaranteed. We strongly recommend using `egctl x translate` to generate a `EnvoyProxy` resource with the `Bootstrap` field set to the default Bootstrap configuration used. You can edit this configuration, and rerun `egctl x translate` to ensure there are no validation errors. |
+| `bootstrap` _[ProxyBootstrap](#proxybootstrap)_ | Bootstrap defines the Envoy Bootstrap as a YAML string. Visit https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/bootstrap/v3/bootstrap.proto#envoy-v3-api-msg-config-bootstrap-v3-bootstrap to learn more about the syntax. If set, this is the Bootstrap configuration used for the managed Envoy Proxy fleet instead of the default Bootstrap configuration set by Envoy Gateway. Some fields within the Bootstrap that are required to communicate with the xDS Server (Envoy Gateway) and receive xDS resources from it are not configurable and will result in the `EnvoyProxy` resource being rejected. Backward compatibility across minor versions is not guaranteed. We strongly recommend using `egctl x translate` to generate a `EnvoyProxy` resource with the `Bootstrap` field set to the default Bootstrap configuration used. You can edit this configuration, and rerun `egctl x translate` to ensure there are no validation errors. |
 | `concurrency` _integer_ | Concurrency defines the number of worker threads to run. If unset, it defaults to the number of cpuset threads on the platform. |
 
 
@@ -756,6 +767,21 @@ _Appears in:_
 
 
 
+## ProxyBootstrap
+
+
+
+ProxyBootstrap defines Envoy Bootstrap configuration.
+
+_Appears in:_
+- [EnvoyProxySpec](#envoyproxyspec)
+
+| Field | Description |
+| --- | --- |
+| `type` _[BootstrapType](#bootstraptype)_ | Type is the type of the bootstrap configuration, it should be either Replace or Merge. If unspecified, it defaults to Replace. |
+| `value` _string_ | Value is a YAML string of the bootstrap. |
+
+
 ## ProxyLogging