From ae350beb1315455ba9f641cea66c4983ca4d004c Mon Sep 17 00:00:00 2001 From: shahar-h Date: Thu, 28 Dec 2023 03:39:25 +0200 Subject: [PATCH] feat: add NodeSelector to PodSpec (#2361) * feat: add NodeSelector to PodSpec Signed-off-by: Shahar Harari * add RL test Signed-off-by: Shahar Harari --------- Signed-off-by: Shahar Harari --- api/v1alpha1/shared_types.go | 7 + api/v1alpha1/zz_generated.deepcopy.go | 7 + .../gateway.envoyproxy.io_envoyproxies.yaml | 8 + .../kubernetes/proxy/resource_provider.go | 1 + .../proxy/resource_provider_test.go | 12 ++ .../deployments/with-node-selector.yaml | 202 ++++++++++++++++++ .../kubernetes/ratelimit/resource_provider.go | 1 + .../ratelimit/resource_provider_test.go | 12 ++ .../deployments/with-node-selector.yaml | 116 ++++++++++ site/content/en/latest/api/extension_types.md | 1 + 10 files changed, 367 insertions(+) create mode 100644 internal/infrastructure/kubernetes/proxy/testdata/deployments/with-node-selector.yaml create mode 100644 internal/infrastructure/kubernetes/ratelimit/testdata/deployments/with-node-selector.yaml diff --git a/api/v1alpha1/shared_types.go b/api/v1alpha1/shared_types.go index 3cf1b1ba951..addf6af69c4 100644 --- a/api/v1alpha1/shared_types.go +++ b/api/v1alpha1/shared_types.go @@ -125,6 +125,13 @@ type KubernetesPodSpec struct { // // +optional ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"` + + // NodeSelector is a selector which must be true for the pod to fit on a node. + // Selector which must match a node's labels for the pod to be scheduled on that node. + // More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + // + // +optional + NodeSelector map[string]string `json:"nodeSelector,omitempty"` } // KubernetesContainerSpec defines the desired state of the Kubernetes container resource. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 7dd724536bd..eec9e73cc01 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1672,6 +1672,13 @@ func (in *KubernetesPodSpec) DeepCopyInto(out *KubernetesPodSpec) { *out = make([]corev1.LocalObjectReference, len(*in)) copy(*out, *in) } + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesPodSpec. diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml index 073dfcf4dfb..32abae9f7ee 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -3288,6 +3288,14 @@ spec: should be tagged to the pods. By default, no additional pod labels are tagged. type: object + nodeSelector: + additionalProperties: + type: string + description: 'NodeSelector is a selector which must + be true for the pod to fit on a node. Selector which + must match a node''s labels for the pod to be scheduled + on that node. More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/' + type: object securityContext: description: 'SecurityContext holds pod-level security attributes and common container settings. Optional: diff --git a/internal/infrastructure/kubernetes/proxy/resource_provider.go b/internal/infrastructure/kubernetes/proxy/resource_provider.go index 6e9cfda8d09..73c62c68b49 100644 --- a/internal/infrastructure/kubernetes/proxy/resource_provider.go +++ b/internal/infrastructure/kubernetes/proxy/resource_provider.go @@ -240,6 +240,7 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { Tolerations: deploymentConfig.Pod.Tolerations, Volumes: expectedDeploymentVolumes(r.infra.Name, deploymentConfig), ImagePullSecrets: deploymentConfig.Pod.ImagePullSecrets, + NodeSelector: deploymentConfig.Pod.NodeSelector, }, }, RevisionHistoryLimit: ptr.To[int32](10), diff --git a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go index e23db7fe1b6..0c9f531c603 100644 --- a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go +++ b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go @@ -391,6 +391,18 @@ func TestDeployment(t *testing.T) { }, }, }, + { + caseName: "with-node-selector", + infra: newTestInfra(), + deploy: &egv1a1.KubernetesDeploymentSpec{ + Pod: &egv1a1.KubernetesPodSpec{ + NodeSelector: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + }, + }, + }, } for _, tc := range cases { t.Run(tc.caseName, func(t *testing.T) { diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-node-selector.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-node-selector.yaml new file mode 100644 index 00000000000..9763b207395 --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-node-selector.yaml @@ -0,0 +1,202 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: envoy + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + name: envoy-default-37a8eec1 + namespace: envoy-gateway-system +spec: + replicas: 1 + strategy: + type: RollingUpdate + selector: + matchLabels: + app.kubernetes.io/name: envoy + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + template: + metadata: + labels: + app.kubernetes.io/name: envoy + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + spec: + automountServiceAccountToken: false + containers: + - args: + - --service-cluster default + - --service-node $(ENVOY_POD_NAME) + - | + --config-yaml admin: + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + dynamic_resources: + ads_config: + api_type: DELTA_GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + set_node_on_first_message_only: true + lds_config: + ads: {} + resource_api_version: V3 + cds_config: + ads: {} + resource_api_version: V3 + static_resources: + listeners: + - name: envoy-gateway-proxy-ready-0.0.0.0-19001 + address: + socket_address: + address: 0.0.0.0 + port_value: 19001 + protocol: TCP + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: eg-ready-http + route_config: + name: local_route + http_filters: + - name: envoy.filters.http.health_check + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.health_check.v3.HealthCheck + pass_through_mode: false + headers: + - name: ":path" + string_match: + exact: /ready + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - connect_timeout: 10s + load_assignment: + cluster_name: xds_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18000 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: + connection_keepalive: + interval: 30s + timeout: 5s + name: xds_cluster + type: STRICT_DNS + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 + - --log-level warn + - --cpuset-threads + command: + - envoy + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/envoy-dev:latest + imagePullPolicy: IfNotPresent + name: envoy + ports: + - containerPort: 8080 + name: EnvoyH-d76a15e2 + protocol: TCP + - containerPort: 8443 + name: EnvoyH-6658f727 + protocol: TCP + resources: + requests: + cpu: 100m + memory: 512Mi + readinessProbe: + httpGet: + path: /ready + port: 19001 + scheme: HTTP + timeoutSeconds: 1 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /certs + name: certs + readOnly: true + - mountPath: /sds + name: sds + dnsPolicy: ClusterFirst + nodeSelector: + key1: value1 + key2: value2 + restartPolicy: Always + schedulerName: default-scheduler + serviceAccountName: envoy-default-37a8eec1 + terminationGracePeriodSeconds: 300 + volumes: + - name: certs + secret: + secretName: envoy + defaultMode: 420 + - configMap: + defaultMode: 420 + items: + - key: xds-trusted-ca.json + path: xds-trusted-ca.json + - key: xds-certificate.json + path: xds-certificate.json + name: envoy-default-37a8eec1 + optional: false + name: sds + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go index ad9d572c491..355a4293074 100644 --- a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go +++ b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go @@ -183,6 +183,7 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { Affinity: r.rateLimitDeployment.Pod.Affinity, Tolerations: r.rateLimitDeployment.Pod.Tolerations, ImagePullSecrets: r.rateLimitDeployment.Pod.ImagePullSecrets, + NodeSelector: r.rateLimitDeployment.Pod.NodeSelector, }, }, RevisionHistoryLimit: ptr.To[int32](10), diff --git a/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go b/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go index d3cb1f8bc2c..5612c2052aa 100644 --- a/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go +++ b/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go @@ -481,6 +481,18 @@ func TestDeployment(t *testing.T) { }, }, }, + { + caseName: "with-node-selector", + rateLimit: rateLimit, + deploy: &egv1a1.KubernetesDeploymentSpec{ + Pod: &egv1a1.KubernetesPodSpec{ + NodeSelector: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + }, + }, + }, } for _, tc := range cases { t.Run(tc.caseName, func(t *testing.T) { diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/with-node-selector.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/with-node-selector.yaml new file mode 100644 index 00000000000..55864cd9995 --- /dev/null +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/with-node-selector.yaml @@ -0,0 +1,116 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: envoy-ratelimit + app.kubernetes.io/component: ratelimit + app.kubernetes.io/managed-by: envoy-gateway + name: envoy-ratelimit + namespace: envoy-gateway-system + ownerReferences: + - apiVersion: apps/v1 + kind: Deployment + name: envoy-gateway + uid: test-owner-reference-uid-for-deployment +spec: + replicas: 1 + strategy: + type: RollingUpdate + selector: + matchLabels: + app.kubernetes.io/name: envoy-ratelimit + app.kubernetes.io/component: ratelimit + app.kubernetes.io/managed-by: envoy-gateway + template: + metadata: + labels: + app.kubernetes.io/name: envoy-ratelimit + app.kubernetes.io/component: ratelimit + app.kubernetes.io/managed-by: envoy-gateway + spec: + automountServiceAccountToken: false + containers: + - command: + - /bin/ratelimit + env: + - name: REDIS_SOCKET_TYPE + value: tcp + - name: REDIS_URL + value: redis.redis.svc:6379 + - name: RUNTIME_ROOT + value: /data + - name: RUNTIME_SUBDIRECTORY + value: ratelimit + - name: RUNTIME_IGNOREDOTFILES + value: "true" + - name: RUNTIME_WATCH_ROOT + value: "false" + - name: LOG_LEVEL + value: info + - name: USE_STATSD + value: "false" + - name: CONFIG_TYPE + value: GRPC_XDS_SOTW + - name: CONFIG_GRPC_XDS_SERVER_URL + value: envoy-gateway:18001 + - name: CONFIG_GRPC_XDS_NODE_ID + value: envoy-ratelimit + - name: GRPC_SERVER_USE_TLS + value: "true" + - name: GRPC_SERVER_TLS_CERT + value: "/certs/tls.crt" + - name: GRPC_SERVER_TLS_KEY + value: "/certs/tls.key" + - name: GRPC_SERVER_TLS_CA_CERT + value: "/certs/ca.crt" + - name: CONFIG_GRPC_XDS_SERVER_USE_TLS + value: "true" + - name: CONFIG_GRPC_XDS_CLIENT_TLS_CERT + value: "/certs/tls.crt" + - name: CONFIG_GRPC_XDS_CLIENT_TLS_KEY + value: "/certs/tls.key" + - name: CONFIG_GRPC_XDS_SERVER_TLS_CACERT + value: "/certs/ca.crt" + - name: FORCE_START_WITHOUT_INITIAL_CONFIG + value: "true" + image: envoyproxy/ratelimit:master + imagePullPolicy: IfNotPresent + name: envoy-ratelimit + ports: + - containerPort: 8081 + name: grpc + protocol: TCP + resources: + requests: + cpu: 100m + memory: 512Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /certs + name: certs + readOnly: true + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthcheck + port: 8080 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + dnsPolicy: ClusterFirst + nodeSelector: + key1: value1 + key2: value2 + restartPolicy: Always + schedulerName: default-scheduler + serviceAccountName: envoy-ratelimit + terminationGracePeriodSeconds: 300 + volumes: + - name: certs + secret: + secretName: envoy-rate-limit + defaultMode: 420 + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index da5c6ec4761..e5ec70e86e4 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -1103,6 +1103,7 @@ _Appears in:_ | `volumes` _[Volume](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#volume-v1-core) array_ | Volumes that can be mounted by containers belonging to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes | | `hostNetwork` _boolean_ | HostNetwork, If this is set to true, the pod will use host's network namespace. | | `imagePullSecrets` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#localobjectreference-v1-core) array_ | ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. If specified, these secrets will be passed to individual puller implementations for them to use. More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod | +| `nodeSelector` _object (keys:string, values:string)_ | NodeSelector is a selector which must be true for the pod to fit on a node. Selector which must match a node's labels for the pod to be scheduled on that node. More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ | #### KubernetesServiceSpec