From 17e932c7bd7ed62d3fa91d4f1afd176a69bd83eb Mon Sep 17 00:00:00 2001 From: zirain Date: Thu, 28 Nov 2024 11:35:06 +0800 Subject: [PATCH] xds: use `::1` if IPFamily is IPv6 on admin server (#4801) xds: use if IPFamily is IPv6 Signed-off-by: zirain --- internal/infrastructure/common/proxy_args.go | 13 + .../proxy/resource_provider_test.go | 33 +- .../proxy/testdata/deployments/ipv6.yaml | 375 ++++++++++++++++++ internal/xds/bootstrap/bootstrap.go | 10 +- internal/xds/bootstrap/bootstrap_test.go | 6 + .../xds/bootstrap/testdata/render/ipv6.yaml | 169 ++++++++ 6 files changed, 595 insertions(+), 11 deletions(-) create mode 100644 internal/infrastructure/kubernetes/proxy/testdata/deployments/ipv6.yaml create mode 100644 internal/xds/bootstrap/testdata/render/ipv6.yaml diff --git a/internal/infrastructure/common/proxy_args.go b/internal/infrastructure/common/proxy_args.go index 0ffaa36c64e..165d004e5fe 100644 --- a/internal/infrastructure/common/proxy_args.go +++ b/internal/infrastructure/common/proxy_args.go @@ -13,6 +13,14 @@ import ( "github.com/envoyproxy/gateway/internal/xds/bootstrap" ) +func getIPFamily(infra *ir.ProxyInfra) *egv1a1.IPFamily { + if infra == nil || infra.Config == nil { + return nil + } + + return infra.Config.Spec.IPFamily +} + // BuildProxyArgs builds command arguments for proxy infrastructure. func BuildProxyArgs( infra *ir.ProxyInfra, @@ -20,6 +28,11 @@ func BuildProxyArgs( bootstrapConfigOptions *bootstrap.RenderBootstrapConfigOptions, serviceNode string, ) ([]string, error) { + // If IPFamily is not set, try to determine it from the infrastructure. + if bootstrapConfigOptions != nil && bootstrapConfigOptions.IPFamily == nil { + bootstrapConfigOptions.IPFamily = getIPFamily(infra) + } + bootstrapConfigurations, err := bootstrap.GetRenderedBootstrapConfig(bootstrapConfigOptions) if err != nil { return nil, err diff --git a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go index 0cf54a40427..ca3e45d1523 100644 --- a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go +++ b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go @@ -44,6 +44,16 @@ func newTestInfra() *ir.Infra { return newTestInfraWithAnnotations(nil) } +func newTestIPv6Infra() *ir.Infra { + i := newTestInfra() + i.Proxy.Config = &egv1a1.EnvoyProxy{ + Spec: egv1a1.EnvoyProxySpec{ + IPFamily: ptr.To(egv1a1.IPv6), + }, + } + return i +} + func newTestInfraWithAnnotations(annotations map[string]string) *ir.Infra { return newTestInfraWithAnnotationsAndLabels(annotations, nil) } @@ -200,6 +210,11 @@ func TestDeployment(t *testing.T) { deploy: nil, bootstrap: `test bootstrap config`, }, + { + caseName: "ipv6", + infra: newTestIPv6Infra(), + deploy: nil, + }, { caseName: "extension-env", infra: newTestInfra(), @@ -568,15 +583,6 @@ func TestDeployment(t *testing.T) { dp, err := r.Deployment() require.NoError(t, err) - expected, err := loadDeployment(tc.caseName) - require.NoError(t, err) - - sortEnv := func(env []corev1.EnvVar) { - sort.Slice(env, func(i, j int) bool { - return env[i].Name > env[j].Name - }) - } - if *overrideTestData { deploymentYAML, err := yaml.Marshal(dp) require.NoError(t, err) @@ -586,8 +592,17 @@ func TestDeployment(t *testing.T) { return } + expected, err := loadDeployment(tc.caseName) + require.NoError(t, err) + + sortEnv := func(env []corev1.EnvVar) { + sort.Slice(env, func(i, j int) bool { + return env[i].Name > env[j].Name + }) + } sortEnv(dp.Spec.Template.Spec.Containers[0].Env) sortEnv(expected.Spec.Template.Spec.Containers[0].Env) + assert.Equal(t, expected, dp) }) } diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/ipv6.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/ipv6.yaml new file mode 100644 index 00000000000..da324336017 --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/ipv6.yaml @@ -0,0 +1,375 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + name: envoy-default-37a8eec1 + namespace: envoy-gateway-system +spec: + progressDeadlineSeconds: 600 + revisionHistoryLimit: 10 + selector: + matchLabels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + strategy: + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/path: /stats/prometheus + prometheus.io/port: "19001" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + spec: + automountServiceAccountToken: false + containers: + - args: + - --service-cluster default + - --service-node $(ENVOY_POD_NAME) + - | + --config-yaml admin: + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + address: + socket_address: + address: ::1 + port_value: 19000 + layered_runtime: + layers: + - name: global_config + static_layer: + envoy.restart_features.use_eds_cache_for_ads: true + re2.max_program_size.error_level: 4294967295 + re2.max_program_size.warn_level: 1000 + dynamic_resources: + ads_config: + api_type: DELTA_GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + set_node_on_first_message_only: true + lds_config: + ads: {} + resource_api_version: V3 + cds_config: + ads: {} + resource_api_version: V3 + static_resources: + listeners: + - name: envoy-gateway-proxy-ready-::-19001 + address: + socket_address: + address: '::' + port_value: 19001 + protocol: TCP + ipv4_compat: true + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: eg-ready-http + route_config: + name: local_route + virtual_hosts: + - name: prometheus_stats + domains: + - "*" + routes: + - match: + prefix: /stats/prometheus + route: + cluster: prometheus_stats + http_filters: + - name: envoy.filters.http.health_check + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.health_check.v3.HealthCheck + pass_through_mode: false + headers: + - name: ":path" + string_match: + exact: /ready + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - name: prometheus_stats + connect_timeout: 0.250s + type: STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: prometheus_stats + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: ::1 + port_value: 19000 + - connect_timeout: 10s + load_assignment: + cluster_name: xds_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway.envoy-gateway-system.svc.cluster.local + port_value: 18000 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: + connection_keepalive: + interval: 30s + timeout: 5s + name: xds_cluster + type: STRICT_DNS + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: /sds/xds-certificate.json + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: /sds/xds-trusted-ca.json + resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: /sds/xds-certificate.json + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: /sds/xds-trusted-ca.json + resource_api_version: V3 + overload_manager: + refresh_interval: 0.25s + resource_monitors: + - name: "envoy.resource_monitors.global_downstream_max_connections" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig + max_active_downstream_connections: 50000 + - --log-level warn + - --cpuset-threads + - --drain-strategy immediate + - --drain-time-s 60 + command: + - envoy + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: docker.io/envoyproxy/envoy:distroless-dev + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + httpGet: + path: /shutdown/ready + port: 19002 + scheme: HTTP + name: envoy + ports: + - containerPort: 19001 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 1 + httpGet: + path: /ready + port: 19001 + scheme: HTTP + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 100m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + seccompProfile: + type: RuntimeDefault + startupProbe: + failureThreshold: 30 + httpGet: + path: /ready + port: 19001 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /certs + name: certs + readOnly: true + - mountPath: /sds + name: sds + - args: + - envoy + - shutdown-manager + command: + - envoy-gateway + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: docker.io/envoyproxy/gateway-dev:latest + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - envoy-gateway + - envoy + - shutdown + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: shutdown-manager + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 10m + memory: 32Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + seccompProfile: + type: RuntimeDefault + startupProbe: + failureThreshold: 30 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + serviceAccountName: envoy-default-37a8eec1 + terminationGracePeriodSeconds: 360 + volumes: + - name: certs + secret: + defaultMode: 420 + secretName: envoy + - configMap: + defaultMode: 420 + items: + - key: xds-trusted-ca.json + path: xds-trusted-ca.json + - key: xds-certificate.json + path: xds-certificate.json + name: envoy-default-37a8eec1 + optional: false + name: sds +status: {} diff --git a/internal/xds/bootstrap/bootstrap.go b/internal/xds/bootstrap/bootstrap.go index b1af89d60ac..45847d4a6bf 100644 --- a/internal/xds/bootstrap/bootstrap.go +++ b/internal/xds/bootstrap/bootstrap.go @@ -27,8 +27,9 @@ const ( // envoyGatewayXdsServerHost is the DNS name of the Xds Server within Envoy Gateway. // It defaults to the Envoy Gateway Kubernetes service. envoyGatewayXdsServerHost = "envoy-gateway" - // EnvoyAdminAddress is the listening address of the envoy admin interface. - EnvoyAdminAddress = "127.0.0.1" + // EnvoyAdminAddress is the listening v4 address of the envoy admin interface. + EnvoyAdminAddress = "127.0.0.1" + EnvoyAdminAddressV6 = "::1" // EnvoyAdminPort is the port used to expose admin interface. EnvoyAdminPort = 19000 // envoyAdminAccessLogPath is the path used to expose admin access log. @@ -140,6 +141,7 @@ type overloadManagerParameters struct { } type RenderBootstrapConfigOptions struct { + IPFamily *egv1a1.IPFamily ProxyMetrics *egv1a1.ProxyMetrics SdsConfig SdsConfigPath XdsServerHost *string @@ -301,6 +303,10 @@ func GetRenderedBootstrapConfig(opts *RenderBootstrapConfigOptions) (string, err cfg.parameters.WasmServer.Port = *opts.WasmServerPort } + if opts.IPFamily != nil && *opts.IPFamily == egv1a1.IPv6 { + cfg.parameters.AdminServer.Address = EnvoyAdminAddressV6 + } + cfg.parameters.OverloadManager.MaxHeapSizeBytes = opts.MaxHeapSizeBytes } diff --git a/internal/xds/bootstrap/bootstrap_test.go b/internal/xds/bootstrap/bootstrap_test.go index 3c334eeaeb5..d72220a0141 100644 --- a/internal/xds/bootstrap/bootstrap_test.go +++ b/internal/xds/bootstrap/bootstrap_test.go @@ -161,6 +161,12 @@ func TestGetRenderedBootstrapConfig(t *testing.T) { SdsConfig: sds, }, }, + { + name: "ipv6", + opts: &RenderBootstrapConfigOptions{ + IPFamily: ptr.To(egv1a1.IPv6), + }, + }, } for _, tc := range cases { diff --git a/internal/xds/bootstrap/testdata/render/ipv6.yaml b/internal/xds/bootstrap/testdata/render/ipv6.yaml new file mode 100644 index 00000000000..ab63a3e7439 --- /dev/null +++ b/internal/xds/bootstrap/testdata/render/ipv6.yaml @@ -0,0 +1,169 @@ +admin: + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + address: + socket_address: + address: ::1 + port_value: 19000 +layered_runtime: + layers: + - name: global_config + static_layer: + envoy.restart_features.use_eds_cache_for_ads: true + re2.max_program_size.error_level: 4294967295 + re2.max_program_size.warn_level: 1000 +dynamic_resources: + ads_config: + api_type: DELTA_GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + set_node_on_first_message_only: true + lds_config: + ads: {} + resource_api_version: V3 + cds_config: + ads: {} + resource_api_version: V3 +static_resources: + listeners: + - name: envoy-gateway-proxy-ready-::-19001 + address: + socket_address: + address: '::' + port_value: 19001 + protocol: TCP + ipv4_compat: true + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: eg-ready-http + route_config: + name: local_route + virtual_hosts: + - name: prometheus_stats + domains: + - "*" + routes: + - match: + prefix: /stats/prometheus + route: + cluster: prometheus_stats + http_filters: + - name: envoy.filters.http.health_check + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.health_check.v3.HealthCheck + pass_through_mode: false + headers: + - name: ":path" + string_match: + exact: /ready + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - name: prometheus_stats + connect_timeout: 0.250s + type: STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: prometheus_stats + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: ::1 + port_value: 19000 + - connect_timeout: 10s + load_assignment: + cluster_name: xds_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18000 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: + connection_keepalive: + interval: 30s + timeout: 5s + name: xds_cluster + type: STRICT_DNS + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: /sds/xds-certificate.json + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: /sds/xds-trusted-ca.json + resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: /sds/xds-certificate.json + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: /sds/xds-trusted-ca.json + resource_api_version: V3 +overload_manager: + refresh_interval: 0.25s + resource_monitors: + - name: "envoy.resource_monitors.global_downstream_max_connections" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig + max_active_downstream_connections: 50000