diff --git a/api/v1alpha1/envoyproxy_helpers.go b/api/v1alpha1/envoyproxy_helpers.go index c34611271531..2bcc086cbba9 100644 --- a/api/v1alpha1/envoyproxy_helpers.go +++ b/api/v1alpha1/envoyproxy_helpers.go @@ -70,11 +70,20 @@ func (r *EnvoyProxyProvider) GetEnvoyProxyKubeProvider() *EnvoyProxyKubernetesPr return r.Kubernetes } - if r.Kubernetes.EnvoyDeployment == nil { + // if EnvoyDeployment and EnvoyDaemonSet are both nil, use EnvoyDeployment + if r.Kubernetes.EnvoyDeployment == nil && r.Kubernetes.EnvoyDaemonSet == nil { r.Kubernetes.EnvoyDeployment = DefaultKubernetesDeployment(DefaultEnvoyProxyImage) } - r.Kubernetes.EnvoyDeployment.defaultKubernetesDeploymentSpec(DefaultEnvoyProxyImage) + // if use EnvoyDeployment, set default values + if r.Kubernetes.EnvoyDeployment != nil { + r.Kubernetes.EnvoyDeployment.defaultKubernetesDeploymentSpec(DefaultEnvoyProxyImage) + } + + // if use EnvoyDaemonSet, set default values + if r.Kubernetes.EnvoyDaemonSet != nil { + r.Kubernetes.EnvoyDaemonSet.defaultKubernetesDaemonSetSpec(DefaultEnvoyProxyImage) + } if r.Kubernetes.EnvoyService == nil { r.Kubernetes.EnvoyService = DefaultKubernetesService() diff --git a/api/v1alpha1/envoyproxy_types.go b/api/v1alpha1/envoyproxy_types.go index a8bb15057175..05fc9927da12 100644 --- a/api/v1alpha1/envoyproxy_types.go +++ b/api/v1alpha1/envoyproxy_types.go @@ -134,6 +134,9 @@ type ShutdownConfig struct { MinDrainDuration *metav1.Duration `json:"minDrainDuration,omitempty"` } +// +kubebuilder:validation:XValidation:rule="((has(self.envoyDeployment) && !has(self.envoyDaemonSet)) || (!has(self.envoyDeployment) && has(self.envoyDaemonSet))) || (!has(self.envoyDeployment) && !has(self.envoyDaemonSet))",message="only one of envoyDeployment or envoyDaemonSet can be specified" +// +kubebuilder:validation:XValidation:rule="((has(self.envoyHpa) && !has(self.envoyDaemonSet)) || (!has(self.envoyHpa) && has(self.envoyDaemonSet))) || (!has(self.envoyHpa) && !has(self.envoyDaemonSet))",message="cannot use envoyHpa if envoyDaemonSet is used" +// // EnvoyProxyKubernetesProvider defines configuration for the Kubernetes resource // provider. type EnvoyProxyKubernetesProvider struct { @@ -144,6 +147,12 @@ type EnvoyProxyKubernetesProvider struct { // +optional EnvoyDeployment *KubernetesDeploymentSpec `json:"envoyDeployment,omitempty"` + // EnvoyDaemonSet defines the desired state of the Envoy daemonset resource. + // Disabled by default, a deployment resource is used instead to provision the Envoy Proxy fleet + // + // +optional + EnvoyDaemonSet *KubernetesDaemonSetSpec `json:"envoyDaemonSet,omitempty"` + // EnvoyService defines the desired state of the Envoy service resource. // If unspecified, default settings for the managed Envoy service resource // are applied. diff --git a/api/v1alpha1/kubernetes_helpers.go b/api/v1alpha1/kubernetes_helpers.go index 957fabe72b05..f22f51822f4a 100644 --- a/api/v1alpha1/kubernetes_helpers.go +++ b/api/v1alpha1/kubernetes_helpers.go @@ -24,6 +24,13 @@ func DefaultKubernetesDeploymentStrategy() *appv1.DeploymentStrategy { } } +// DefaultKubernetesDaemonSetStrategy returns the default daemonset strategy settings. +func DefaultKubernetesDaemonSetStrategy() *appv1.DaemonSetUpdateStrategy { + return &appv1.DaemonSetUpdateStrategy{ + Type: appv1.RollingUpdateDaemonSetStrategyType, + } +} + // DefaultKubernetesContainerImage returns the default envoyproxy image. func DefaultKubernetesContainerImage(image string) *string { return ptr.To(image) @@ -38,6 +45,15 @@ func DefaultKubernetesDeployment(image string) *KubernetesDeploymentSpec { } } +// DefaultKubernetesDaemonSet returns a new DefaultKubernetesDaemonSet with default settings. +func DefaultKubernetesDaemonSet(image string) *KubernetesDaemonSetSpec { + return &KubernetesDaemonSetSpec{ + Strategy: DefaultKubernetesDaemonSetStrategy(), + Pod: DefaultKubernetesPod(), + Container: DefaultKubernetesContainer(image), + } +} + // DefaultKubernetesPod returns a new KubernetesPodSpec with default settings. func DefaultKubernetesPod() *KubernetesPodSpec { return &KubernetesPodSpec{} @@ -110,6 +126,29 @@ func (deployment *KubernetesDeploymentSpec) defaultKubernetesDeploymentSpec(imag } } +// defaultKubernetesDaemonSetSpec fill a default KubernetesDaemonSetSpec if unspecified. +func (daemonset *KubernetesDaemonSetSpec) defaultKubernetesDaemonSetSpec(image string) { + if daemonset.Strategy == nil { + daemonset.Strategy = DefaultKubernetesDaemonSetStrategy() + } + + if daemonset.Pod == nil { + daemonset.Pod = DefaultKubernetesPod() + } + + if daemonset.Container == nil { + daemonset.Container = DefaultKubernetesContainer(image) + } + + if daemonset.Container.Resources == nil { + daemonset.Container.Resources = DefaultResourceRequirements() + } + + if daemonset.Container.Image == nil { + daemonset.Container.Image = DefaultKubernetesContainerImage(image) + } +} + // setDefault fill a default HorizontalPodAutoscalerSpec if unspecified func (hpa *KubernetesHorizontalPodAutoscalerSpec) setDefault() { if len(hpa.Metrics) == 0 { @@ -153,6 +192,42 @@ func (deployment *KubernetesDeploymentSpec) ApplyMergePatch(old *appv1.Deploymen return &patchedDeployment, nil } +// ApplyMergePatch applies a merge patch to a daemonset based on the merge type +func (daemonset *KubernetesDaemonSetSpec) ApplyMergePatch(old *appv1.DaemonSet) (*appv1.DaemonSet, error) { + if daemonset.Patch == nil { + return old, nil + } + + var patchedJSON []byte + var err error + + // Serialize the current daemonset to JSON + originalJSON, err := json.Marshal(old) + if err != nil { + return nil, fmt.Errorf("error marshaling original daemonset: %w", err) + } + + switch { + case daemonset.Patch.Type == nil || *daemonset.Patch.Type == StrategicMerge: + patchedJSON, err = strategicpatch.StrategicMergePatch(originalJSON, daemonset.Patch.Value.Raw, appv1.DaemonSet{}) + case *daemonset.Patch.Type == JSONMerge: + patchedJSON, err = jsonpatch.MergePatch(originalJSON, daemonset.Patch.Value.Raw) + default: + return nil, fmt.Errorf("unsupported merge type: %s", *daemonset.Patch.Type) + } + if err != nil { + return nil, fmt.Errorf("error applying merge patch: %w", err) + } + + // Deserialize the patched JSON into a new daemonset object + var patchedDaemonSet appv1.DaemonSet + if err := json.Unmarshal(patchedJSON, &patchedDaemonSet); err != nil { + return nil, fmt.Errorf("error unmarshaling patched daemonset: %w", err) + } + + return &patchedDaemonSet, nil +} + // ApplyMergePatch applies a merge patch to a service based on the merge type func (service *KubernetesServiceSpec) ApplyMergePatch(old *corev1.Service) (*corev1.Service, error) { if service.Patch == nil { diff --git a/api/v1alpha1/shared_types.go b/api/v1alpha1/shared_types.go index 7d8128df3c61..de39b929cbf6 100644 --- a/api/v1alpha1/shared_types.go +++ b/api/v1alpha1/shared_types.go @@ -93,6 +93,28 @@ type KubernetesDeploymentSpec struct { // TODO: Expose config as use cases are better understood, e.g. labels. } +// KubernetesDaemonsetSpec defines the desired state of the Kubernetes daemonset resource. +type KubernetesDaemonSetSpec struct { + // Patch defines how to perform the patch operation to daemonset + // + // +optional + Patch *KubernetesPatchSpec `json:"patch,omitempty"` + + // The daemonset strategy to use to replace existing pods with new ones. + // +optional + Strategy *appv1.DaemonSetUpdateStrategy `json:"strategy,omitempty"` + + // Pod defines the desired specification of pod. + // + // +optional + Pod *KubernetesPodSpec `json:"pod,omitempty"` + + // Container defines the desired specification of main container. + // + // +optional + Container *KubernetesContainerSpec `json:"container,omitempty"` +} + // KubernetesPodSpec defines the desired state of the Kubernetes pod resource. type KubernetesPodSpec struct { // Annotations are the annotations that should be appended to the pods. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 0055d7669f17..29a52ebab900 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1394,6 +1394,11 @@ func (in *EnvoyProxyKubernetesProvider) DeepCopyInto(out *EnvoyProxyKubernetesPr *out = new(KubernetesDeploymentSpec) (*in).DeepCopyInto(*out) } + if in.EnvoyDaemonSet != nil { + in, out := &in.EnvoyDaemonSet, &out.EnvoyDaemonSet + *out = new(KubernetesDaemonSetSpec) + (*in).DeepCopyInto(*out) + } if in.EnvoyService != nil { in, out := &in.EnvoyService, &out.EnvoyService *out = new(KubernetesServiceSpec) @@ -2353,6 +2358,41 @@ func (in *KubernetesContainerSpec) DeepCopy() *KubernetesContainerSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubernetesDaemonSetSpec) DeepCopyInto(out *KubernetesDaemonSetSpec) { + *out = *in + if in.Patch != nil { + in, out := &in.Patch, &out.Patch + *out = new(KubernetesPatchSpec) + (*in).DeepCopyInto(*out) + } + if in.Strategy != nil { + in, out := &in.Strategy, &out.Strategy + *out = new(appsv1.DaemonSetUpdateStrategy) + (*in).DeepCopyInto(*out) + } + if in.Pod != nil { + in, out := &in.Pod, &out.Pod + *out = new(KubernetesPodSpec) + (*in).DeepCopyInto(*out) + } + if in.Container != nil { + in, out := &in.Container, &out.Container + *out = new(KubernetesContainerSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesDaemonSetSpec. +func (in *KubernetesDaemonSetSpec) DeepCopy() *KubernetesDaemonSetSpec { + if in == nil { + return nil + } + out := new(KubernetesDaemonSetSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubernetesDeployMode) DeepCopyInto(out *KubernetesDeployMode) { *out = *in diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml index 76d96eccaeab..d1436a473e73 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -128,6 +128,3603 @@ spec: e.g. Envoy proxy. If unspecified and type is "Kubernetes", default settings for managed Kubernetes resources are applied. properties: + envoyDaemonSet: + description: |- + EnvoyDaemonSet defines the desired state of the Envoy daemonset resource. + Disabled by default, a deployment resource is used instead to provision the Envoy Proxy fleet + properties: + container: + description: Container defines the desired specification + of main container. + properties: + env: + description: List of environment variables to set + in the container. + items: + description: EnvVar represents an environment variable + present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the + FieldPath is written in terms of, + defaults to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults + to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in + the pod's namespace + properties: + key: + description: The key of the secret to + select from. Must be a valid secret + key. + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + image: + description: Image specifies the EnvoyProxy container + image to be used, instead of the default image. + type: string + resources: + description: |- + Resources required by this container. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry + in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default is DefaultProcMount which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the + name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + volumeMounts: + description: |- + VolumeMounts are volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting of + a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + type: object + patch: + description: Patch defines how to perform the patch operation + to daemonset + properties: + type: + description: |- + Type is the type of merge operation to perform + + + By default, StrategicMerge is used as the patch type. + type: string + value: + description: Object contains the raw configuration + for merged object + x-kubernetes-preserve-unknown-fields: true + required: + - value + type: object + pod: + description: Pod defines the desired specification of + pod. + properties: + affinity: + description: If specified, the pod's scheduling constraints. + properties: + nodeAffinity: + description: Describes node affinity scheduling + rules for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated + with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key + that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key + that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with + matching the corresponding nodeSelectorTerm, + in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node + selector terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key + that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key + that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling + rules (e.g. co-locate this pod in the same node, + zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added + per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling + rules (e.g. avoid putting this pod in the same + node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added + per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + Also, MatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. + Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + annotations: + additionalProperties: + type: string + description: |- + Annotations are the annotations that should be appended to the pods. + By default, no pod annotations are appended. + type: object + imagePullSecrets: + description: |- + ImagePullSecrets is an optional list of references to secrets + in the same namespace to use for pulling any of the images used by this PodSpec. + If specified, these secrets will be passed to individual puller implementations for them to use. + More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + type: array + labels: + additionalProperties: + type: string + description: |- + Labels are the additional labels that should be tagged to the pods. + By default, no additional pod labels are tagged. + type: object + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the pod to fit on a node. + Selector which must match a node's labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + type: object + securityContext: + description: |- + SecurityContext holds pod-level security attributes and common container settings. + Optional: Defaults to empty. See type description for default values of each field. + properties: + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in addition + to the container's primary GID, the fsGroup (if specified), and group memberships + defined in the container image for the uid of the container process. If unspecified, + no additional groups are added to any container. Note that group memberships + defined in the container image for the uid of the container process are still effective, + even if they are not included in this list. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter + to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the + name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + tolerations: + description: If specified, the pod's tolerations. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how a group of pods ought to spread across topology + domains. Scheduler will schedule pods in a way which abides by the constraints. + All topologySpreadConstraints are ANDed. + items: + description: TopologySpreadConstraint specifies + how to spread matching pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + + + This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default). + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + + If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + + If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + volumes: + description: |- + Volumes that can be mounted by containers belonging to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes + items: + description: Volume represents a named volume in + a pod that may be accessed by any container in + the pod. + properties: + awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + TODO: how do we prevent errors in the filesystem from compromising the machine + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + format: int32 + type: integer + readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: boolean + volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + required: + - volumeID + type: object + azureDisk: + description: azureDisk represents an Azure Data + Disk mount on the host and bind mount to the + pod. + properties: + cachingMode: + description: 'cachingMode is the Host Caching + mode: None, Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the + data disk in the blob storage + type: string + diskURI: + description: diskURI is the URI of data + disk in the blob storage + type: string + fsType: + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: + multiple blob disks per storage account Dedicated: + single blob disk per storage account Managed: + azure managed data disk (only in managed + availability set). defaults to shared' + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure File + Service mount on the host and bind mount to + the pod. + properties: + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of + secret that contains Azure Storage Account + Name and Key + type: string + shareName: + description: shareName is the azure share + Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: cephFS represents a Ceph FS mount + on the host that shares a pod's lifetime + properties: + monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + items: + type: string + type: array + path: + description: 'path is Optional: Used as + the mounted root, rather than the full + Ceph tree, default is /' + type: string + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: boolean + secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + required: + - monitors + type: object + cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: boolean + secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap + that should populate this volume + properties: + defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: optional specify whether the + ConfigMap or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) + represents ephemeral storage that is handled + by certain external CSI drivers (Beta feature). + properties: + driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. + type: string + fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward + API about the pod that should populate this + volume + properties: + defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: Items is a list of downward + API volume file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a + field of the pod: only annotations, + labels, name and namespace are supported.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in + terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field + to select in the specified API + version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the + relative path name of the file to + be created. Must not be absolute + or contain the ''..'' path. Must + be utf-8 encoded. The first item + of the relative path must not start + with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: + required for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. + properties: + volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + + Required, must not be nil. + properties: + metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. + type: object + spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type + of resource being referenced + type: string + name: + description: Name is the name + of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type + of resource being referenced + type: string + name: + description: Name is the name + of resource being referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label + query over volumes to consider + for binding. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#volumeattributesclass + (Alpha) Using this field requires the VolumeAttributesClass feature gate to be enabled. + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding + reference to the PersistentVolume + backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource + that is attached to a kubelet's host machine + and then exposed to the pod. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + TODO: how do we prevent errors in the filesystem from compromising the machine + type: string + lun: + description: 'lun is Optional: FC target + lun number' + format: int32 + type: integer + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC + target worldwide names (WWNs)' + items: + type: string + type: array + wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. + items: + type: string + type: array + type: object + flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + properties: + driver: + description: driver is the name of the driver + to use for this volume. + type: string + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this + field holds extra command options if any.' + type: object + readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: flocker represents a Flocker volume + attached to a kubelet's host machine. This + depends on the Flocker control service being + running + properties: + datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of + the dataset. This is unique identifier + of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + properties: + fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + TODO: how do we prevent errors in the filesystem from compromising the machine + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + format: int32 + type: integer + pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: boolean + required: + - pdName + type: object + gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. + properties: + directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash + for the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/glusterfs/README.md + properties: + endpoints: + description: |- + endpoints is the endpoint name that details Glusterfs topology. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + --- + TODO(jonesdl) We need to restrict who can use host directory mounts and who can/can not + mount host directories as read/write. + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://examples.k8s.io/volumes/iscsi/README.md + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether + support iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether + support iSCSI Session CHAP authentication + type: boolean + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + TODO: how do we prevent errors in the filesystem from compromising the machine + type: string + initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified + Name. + type: string + iscsiInterface: + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target + Lun number. + format: int32 + type: integer + portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret + for iSCSI target and initiator authentication + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + properties: + path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: boolean + server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: photonPersistentDisk represents + a PhotonController persistent disk attached + and mounted on kubelets host machine + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies + Photon Controller persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents a portworx + volume attached and mounted on kubelets host + machine + properties: + fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies + a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one + resources secrets, configmaps, and downward + API + properties: + defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: sources is the list of volume + projections + items: + description: Projection that may be projected + along with other supported volume types + properties: + clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from + the volume root to write the + bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + configMap: + description: configMap information + about the configMap data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key + to a path within a volume. + properties: + key: + description: key is the + key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: optional specify + whether the ConfigMap or its + keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information + about the downwardAPI data to project + properties: + items: + description: Items is a list of + DownwardAPIVolume file + items: + description: DownwardAPIVolumeFile + represents information to + create the file containing + the pod field + properties: + fieldRef: + description: 'Required: + Selects a field of the + pod: only annotations, + labels, name and namespace + are supported.' + properties: + apiVersion: + description: Version + of the schema the + FieldPath is written + in terms of, defaults + to "v1". + type: string + fieldPath: + description: Path of + the field to select + in the specified API + version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: + Path is the relative + path name of the file + to be created. Must not + be absolute or contain + the ''..'' path. Must + be utf-8 encoded. The + first item of the relative + path must not start with + ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container + name: required for + volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies + the output format + of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: + resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + secret: + description: secret information about + the secret data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key + to a path within a volume. + properties: + key: + description: key is the + key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: optional field specify + whether the Secret or its key + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken is + information about the serviceAccountToken + data to project + properties: + audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. + type: string + expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: |- + path is the path relative to the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + description: quobyte represents a Quobyte mount + on the host that shares a pod's lifetime + properties: + group: + description: |- + group to map volume access to + Default is no group + type: string + readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes + type: string + tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin + type: string + user: + description: |- + user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that references + an already created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/rbd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + TODO: how do we prevent errors in the filesystem from compromising the machine + type: string + image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + keyring: + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + items: + type: string + type: array + pool: + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: boolean + secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO persistent + volume attached and mounted on Kubernetes + nodes. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". + type: string + gateway: + description: gateway is the host address + of the ScaleIO API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name + of the ScaleIO Protection Domain for the + configured storage. + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable + SSL communication with Gateway, default + false + type: boolean + storageMode: + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO + Storage Pool associated with the protection + domain. + type: string + system: + description: system is the name of the storage + system as configured in ScaleIO. + type: string + volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + optional: + description: optional field specify whether + the Secret or its keys must be defined + type: boolean + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + type: object + storageos: + description: storageOS represents a StorageOS + volume attached and mounted on Kubernetes + nodes. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. + type: string + volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: vsphereVolume represents a vSphere + volume attached and mounted on kubelets host + machine + properties: + fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage + Policy Based Management (SPBM) profile + ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage + Policy Based Management (SPBM) profile + name. + type: string + volumePath: + description: volumePath is the path that + identifies vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + type: object + strategy: + description: The daemonset strategy to use to replace + existing pods with new ones. + properties: + rollingUpdate: + description: |- + Rolling update config params. Present only if type = "RollingUpdate". + --- + TODO: Update this to follow our convention for oneOf, whatever we decide it + to be. Same as Deployment `strategy.rollingUpdate`. + See https://github.com/kubernetes/kubernetes/issues/35345 + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of nodes with an existing available DaemonSet pod that + can have an updated DaemonSet pod during during an update. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + This can not be 0 if MaxUnavailable is 0. + Absolute number is calculated from percentage by rounding up to a minimum of 1. + Default value is 0. + Example: when this is set to 30%, at most 30% of the total number of nodes + that should be running the daemon pod (i.e. status.desiredNumberScheduled) + can have their a new pod created before the old pod is marked as deleted. + The update starts by launching new pods on 30% of nodes. Once an updated + pod is available (Ready for at least minReadySeconds) the old DaemonSet pod + on that node is marked deleted. If the old pod becomes unavailable for any + reason (Ready transitions to false, is evicted, or is drained) an updated + pod is immediatedly created on that node without considering surge limits. + Allowing surge implies the possibility that the resources consumed by the + daemonset on any given node can double if the readiness check fails, and + so resource intensive daemonsets should take into account that they may + cause evictions during disruption. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of DaemonSet pods that can be unavailable during the + update. Value can be an absolute number (ex: 5) or a percentage of total + number of DaemonSet pods at the start of the update (ex: 10%). Absolute + number is calculated from percentage by rounding up. + This cannot be 0 if MaxSurge is 0 + Default value is 1. + Example: when this is set to 30%, at most 30% of the total number of nodes + that should be running the daemon pod (i.e. status.desiredNumberScheduled) + can have their pods stopped for an update at any given time. The update + starts by stopping at most 30% of those DaemonSet pods and then brings + up new DaemonSet pods in their place. Once the new pods are available, + it then proceeds onto other DaemonSet pods, thus ensuring that at least + 70% of original number of DaemonSet pods are available at all times during + the update. + x-kubernetes-int-or-string: true + type: object + type: + description: Type of daemon set update. Can be "RollingUpdate" + or "OnDelete". Default is RollingUpdate. + type: string + type: object + type: object envoyDeployment: description: |- EnvoyDeployment defines the desired state of the Envoy deployment resource. @@ -5802,6 +9399,16 @@ spec: type rule: '!has(self.loadBalancerIP) || self.type == ''LoadBalancer''' type: object + x-kubernetes-validations: + - message: only one of envoyDeployment or envoyDaemonSet can be + specified + rule: ((has(self.envoyDeployment) && !has(self.envoyDaemonSet)) + || (!has(self.envoyDeployment) && has(self.envoyDaemonSet))) + || (!has(self.envoyDeployment) && !has(self.envoyDaemonSet)) + - message: cannot use envoyHpa if envoyDaemonSet is used + rule: ((has(self.envoyHpa) && !has(self.envoyDaemonSet)) || + (!has(self.envoyHpa) && has(self.envoyDaemonSet))) || (!has(self.envoyHpa) + && !has(self.envoyDaemonSet)) type: description: |- Type is the type of resource provider to use. A resource provider provides diff --git a/internal/infrastructure/kubernetes/infra.go b/internal/infrastructure/kubernetes/infra.go index 882d29ef8b4a..03e8e412caaa 100644 --- a/internal/infrastructure/kubernetes/infra.go +++ b/internal/infrastructure/kubernetes/infra.go @@ -26,6 +26,7 @@ type ResourceRender interface { Service() (*corev1.Service, error) ConfigMap() (*corev1.ConfigMap, error) Deployment() (*appsv1.Deployment, error) + DaemonSet() (*appsv1.DaemonSet, error) HorizontalPodAutoscaler() (*autoscalingv2.HorizontalPodAutoscaler, error) } @@ -66,6 +67,10 @@ func (i *Infra) createOrUpdate(ctx context.Context, r ResourceRender) error { return fmt.Errorf("failed to create or update deployment %s/%s: %w", i.Namespace, r.Name(), err) } + if err := i.createOrUpdateDaemonSet(ctx, r); err != nil { + return fmt.Errorf("failed to create or update daemonset %s/%s: %w", i.Namespace, r.Name(), err) + } + if err := i.createOrUpdateService(ctx, r); err != nil { return fmt.Errorf("failed to create or update service %s/%s: %w", i.Namespace, r.Name(), err) } @@ -91,6 +96,10 @@ func (i *Infra) delete(ctx context.Context, r ResourceRender) error { return fmt.Errorf("failed to delete deployment %s/%s: %w", i.Namespace, r.Name(), err) } + if err := i.deleteDaemonSet(ctx, r); err != nil { + return fmt.Errorf("failed to delete daemonset %s/%s: %w", i.Namespace, r.Name(), err) + } + if err := i.deleteService(ctx, r); err != nil { return fmt.Errorf("failed to delete service %s/%s: %w", i.Namespace, r.Name(), err) } diff --git a/internal/infrastructure/kubernetes/infra_resource.go b/internal/infrastructure/kubernetes/infra_resource.go index 20f5ea9929c8..91b880d5d54d 100644 --- a/internal/infrastructure/kubernetes/infra_resource.go +++ b/internal/infrastructure/kubernetes/infra_resource.go @@ -48,9 +48,34 @@ func (i *Infra) createOrUpdateDeployment(ctx context.Context, r ResourceRender) return err } + // delete the deployment and return early + // this handles the case where a daemonset has been + // congured + if deployment == nil { + return i.deleteDeployment(ctx, r) + } + return i.Client.ServerSideApply(ctx, deployment) } +// createOrUpdateDaemonSet creates a DaemonSet in the kube api server based on the provided +// ResourceRender, if it doesn't exist and updates it if it does. +func (i *Infra) createOrUpdateDaemonSet(ctx context.Context, r ResourceRender) error { + daemonSet, err := r.DaemonSet() + if err != nil { + return err + } + + // delete the daemonset and return early + // this handles the case where a deployment has been + // congured + if daemonSet == nil { + return i.deleteDaemonSet(ctx, r) + } + + return i.Client.ServerSideApply(ctx, daemonSet) +} + // createOrUpdateHPA creates HorizontalPodAutoscaler object in the kube api server based on // the provided ResourceRender, if it doesn't exist and updates it if it does, // and delete hpa if not set. @@ -104,6 +129,18 @@ func (i *Infra) deleteDeployment(ctx context.Context, r ResourceRender) error { return i.Client.Delete(ctx, deployment) } +// deleteDaemonSet deletes the Envoy DaemonSet in the kube api server, if it exists. +func (i *Infra) deleteDaemonSet(ctx context.Context, r ResourceRender) error { + daemonSet := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: i.Namespace, + Name: r.Name(), + }, + } + + return i.Client.Delete(ctx, daemonSet) +} + // deleteConfigMap deletes the ConfigMap in the kube api server, if it exists. func (i *Infra) deleteConfigMap(ctx context.Context, r ResourceRender) error { cm := &corev1.ConfigMap{ diff --git a/internal/infrastructure/kubernetes/proxy/resource.go b/internal/infrastructure/kubernetes/proxy/resource.go index 0e5f40191da8..e006ac2dc035 100644 --- a/internal/infrastructure/kubernetes/proxy/resource.go +++ b/internal/infrastructure/kubernetes/proxy/resource.go @@ -101,7 +101,7 @@ func enablePrometheus(infra *ir.ProxyInfra) bool { // expectedProxyContainers returns expected proxy containers. func expectedProxyContainers(infra *ir.ProxyInfra, - deploymentConfig *egv1a1.KubernetesDeploymentSpec, + containerSpec *egv1a1.KubernetesContainerSpec, shutdownConfig *egv1a1.ShutdownConfig) ([]corev1.Container, error) { // Define slice to hold container ports var ports []corev1.ContainerPort @@ -144,7 +144,7 @@ func expectedProxyContainers(infra *ir.ProxyInfra, proxyMetrics = infra.Config.Spec.Telemetry.Metrics } - maxHeapSizeBytes := caclulateMaxHeapSizeBytes(deploymentConfig.Container.Resources) + maxHeapSizeBytes := caclulateMaxHeapSizeBytes(containerSpec.Resources) // Get the default Bootstrap bootstrapConfigurations, err := bootstrap.GetRenderedBootstrapConfig(&bootstrap.RenderBootsrapConfigOptions{ @@ -194,15 +194,15 @@ func expectedProxyContainers(infra *ir.ProxyInfra, containers := []corev1.Container{ { Name: envoyContainerName, - Image: *deploymentConfig.Container.Image, + Image: *containerSpec.Image, ImagePullPolicy: corev1.PullIfNotPresent, Command: []string{"envoy"}, Args: args, - Env: expectedContainerEnv(deploymentConfig.Container), - Resources: *deploymentConfig.Container.Resources, - SecurityContext: deploymentConfig.Container.SecurityContext, + Env: expectedContainerEnv(containerSpec), + Resources: *containerSpec.Resources, + SecurityContext: containerSpec.SecurityContext, Ports: ports, - VolumeMounts: expectedContainerVolumeMounts(deploymentConfig.Container), + VolumeMounts: expectedContainerVolumeMounts(containerSpec), TerminationMessagePolicy: corev1.TerminationMessageReadFile, TerminationMessagePath: "/dev/termination-log", ReadinessProbe: &corev1.Probe{ @@ -327,8 +327,8 @@ func expectedContainerVolumeMounts(containerSpec *egv1a1.KubernetesContainerSpec return resource.ExpectedContainerVolumeMounts(containerSpec, volumeMounts) } -// expectedDeploymentVolumes returns expected proxy deployment volumes. -func expectedDeploymentVolumes(name string, deploymentSpec *egv1a1.KubernetesDeploymentSpec) []corev1.Volume { +// expectedVolumes returns expected proxy deployment volumes. +func expectedVolumes(name string, pod *egv1a1.KubernetesPodSpec) []corev1.Volume { volumes := []corev1.Volume{ { Name: "certs", @@ -363,7 +363,7 @@ func expectedDeploymentVolumes(name string, deploymentSpec *egv1a1.KubernetesDep }, } - return resource.ExpectedDeploymentVolumes(deploymentSpec.Pod, volumes) + return resource.ExpectedVolumes(pod, volumes) } // expectedContainerEnv returns expected proxy container envs. diff --git a/internal/infrastructure/kubernetes/proxy/resource_provider.go b/internal/infrastructure/kubernetes/proxy/resource_provider.go index 2a11f7317d36..b46fe84e6f5f 100644 --- a/internal/infrastructure/kubernetes/proxy/resource_provider.go +++ b/internal/infrastructure/kubernetes/proxy/resource_provider.go @@ -193,36 +193,27 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { } deploymentConfig := provider.GetEnvoyProxyKubeProvider().EnvoyDeployment + // If deployment config is nil, it's not Deployment installation. + if deploymentConfig == nil { + return nil, nil + } + // Get expected bootstrap configurations rendered ProxyContainers - containers, err := expectedProxyContainers(r.infra, deploymentConfig, proxyConfig.Spec.Shutdown) + containers, err := expectedProxyContainers(r.infra, deploymentConfig.Container, proxyConfig.Spec.Shutdown) if err != nil { return nil, err } - // Set the labels based on the owning gateway name. dpAnnotations := r.infra.GetProxyMetadata().Annotations - labels := r.infra.GetProxyMetadata().Labels - dpLabels := envoyLabels(labels) - if OwningGatewayLabelsAbsent(dpLabels) { - return nil, fmt.Errorf("missing owning gateway labels") - } + podAnnotations := r.getPodAnnotations(dpAnnotations, deploymentConfig.Pod) - maps.Copy(labels, deploymentConfig.Pod.Labels) - podLabels := envoyLabels(labels) - selector := resource.GetSelector(podLabels) - - // Get annotations - podAnnotations := map[string]string{} - maps.Copy(podAnnotations, dpAnnotations) - maps.Copy(podAnnotations, deploymentConfig.Pod.Annotations) - if enablePrometheus(r.infra) { - podAnnotations["prometheus.io/path"] = "/stats/prometheus" // TODO: make this configurable - podAnnotations["prometheus.io/scrape"] = "true" - podAnnotations["prometheus.io/port"] = strconv.Itoa(bootstrap.EnvoyReadinessPort) - } - if len(podAnnotations) == 0 { - podAnnotations = nil + // Set the labels based on the owning gateway name. + dpLabels, err := r.getLabels() + if err != nil { + return nil, err } + podLabels := r.getPodLabels(deploymentConfig.Pod) + selector := resource.GetSelector(podLabels) deployment := &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{ @@ -256,7 +247,7 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { SecurityContext: deploymentConfig.Pod.SecurityContext, Affinity: deploymentConfig.Pod.Affinity, Tolerations: deploymentConfig.Pod.Tolerations, - Volumes: expectedDeploymentVolumes(r.infra.Name, deploymentConfig), + Volumes: expectedVolumes(r.infra.Name, deploymentConfig.Pod), ImagePullSecrets: deploymentConfig.Pod.ImagePullSecrets, NodeSelector: deploymentConfig.Pod.NodeSelector, TopologySpreadConstraints: deploymentConfig.Pod.TopologySpreadConstraints, @@ -280,12 +271,69 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { return deployment, nil } -func expectedTerminationGracePeriodSeconds(cfg *egv1a1.ShutdownConfig) *int64 { - s := 900 // default - if cfg != nil && cfg.DrainTimeout != nil { - s = int(cfg.DrainTimeout.Seconds() + 300) // 5 minutes longer than drain timeout +func (r *ResourceRender) DaemonSet() (*appsv1.DaemonSet, error) { + proxyConfig := r.infra.GetProxyConfig() + + // Get the EnvoyProxy config to configure the daemonset. + provider := proxyConfig.GetEnvoyProxyProvider() + if provider.Type != egv1a1.ProviderTypeKubernetes { + return nil, fmt.Errorf("invalid provider type %v for Kubernetes infra manager", provider.Type) } - return ptr.To(int64(s)) + + daemonSetConfig := provider.GetEnvoyProxyKubeProvider().EnvoyDaemonSet + + // If daemonset config is nil, it's not DaemonSet installation. + if daemonSetConfig == nil { + return nil, nil + } + + // Get expected bootstrap configurations rendered ProxyContainers + containers, err := expectedProxyContainers(r.infra, daemonSetConfig.Container, proxyConfig.Spec.Shutdown) + if err != nil { + return nil, err + } + + dsAnnotations := r.infra.GetProxyMetadata().Annotations + podAnnotations := r.getPodAnnotations(dsAnnotations, daemonSetConfig.Pod) + + // Set the labels based on the owning gateway name. + dsLabels, err := r.getLabels() + if err != nil { + return nil, err + } + podLabels := r.getPodLabels(daemonSetConfig.Pod) + selector := resource.GetSelector(podLabels) + + daemonSet := &appsv1.DaemonSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "DaemonSet", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: r.Namespace, + Name: r.Name(), + Labels: dsLabels, + Annotations: dsAnnotations, + }, + Spec: appsv1.DaemonSetSpec{ + Selector: selector, + UpdateStrategy: *daemonSetConfig.Strategy, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: selector.MatchLabels, + Annotations: podAnnotations, + }, + Spec: r.getPodSpec(containers, nil, daemonSetConfig.Pod, proxyConfig), + }, + }, + } + + // apply merge patch to daemonset + if daemonSet, err = daemonSetConfig.ApplyMergePatch(daemonSet); err != nil { + return nil, err + } + + return daemonSet, nil } func (r *ResourceRender) HorizontalPodAutoscaler() (*autoscalingv2.HorizontalPodAutoscaler, error) { @@ -326,6 +374,73 @@ func (r *ResourceRender) HorizontalPodAutoscaler() (*autoscalingv2.HorizontalPod return hpa, nil } +func expectedTerminationGracePeriodSeconds(cfg *egv1a1.ShutdownConfig) *int64 { + s := 900 // default + if cfg != nil && cfg.DrainTimeout != nil { + s = int(cfg.DrainTimeout.Seconds() + 300) // 5 minutes longer than drain timeout + } + return ptr.To(int64(s)) +} + +func (r *ResourceRender) getPodSpec( + containers, initContainers []corev1.Container, + pod *egv1a1.KubernetesPodSpec, + proxyConfig *egv1a1.EnvoyProxy, +) corev1.PodSpec { + return corev1.PodSpec{ + Containers: containers, + InitContainers: initContainers, + ServiceAccountName: ExpectedResourceHashedName(r.infra.Name), + AutomountServiceAccountToken: ptr.To(false), + TerminationGracePeriodSeconds: expectedTerminationGracePeriodSeconds(proxyConfig.Spec.Shutdown), + DNSPolicy: corev1.DNSClusterFirst, + RestartPolicy: corev1.RestartPolicyAlways, + SchedulerName: "default-scheduler", + SecurityContext: pod.SecurityContext, + Affinity: pod.Affinity, + Tolerations: pod.Tolerations, + Volumes: expectedVolumes(r.infra.Name, pod), + ImagePullSecrets: pod.ImagePullSecrets, + NodeSelector: pod.NodeSelector, + TopologySpreadConstraints: pod.TopologySpreadConstraints, + } +} + +func (r *ResourceRender) getPodAnnotations(resourceAnnotation map[string]string, pod *egv1a1.KubernetesPodSpec) map[string]string { + podAnnotations := map[string]string{} + maps.Copy(podAnnotations, resourceAnnotation) + maps.Copy(podAnnotations, pod.Annotations) + + if enablePrometheus(r.infra) { + podAnnotations["prometheus.io/path"] = "/stats/prometheus" // TODO: make this configurable + podAnnotations["prometheus.io/scrape"] = "true" + podAnnotations["prometheus.io/port"] = strconv.Itoa(bootstrap.EnvoyReadinessPort) + } + + if len(podAnnotations) == 0 { + podAnnotations = nil + } + + return podAnnotations +} + +func (r *ResourceRender) getLabels() (map[string]string, error) { + // Set the labels based on the owning gateway name. + resourceLabels := envoyLabels(r.infra.GetProxyMetadata().Labels) + if OwningGatewayLabelsAbsent(resourceLabels) { + return nil, fmt.Errorf("missing owning gateway labels") + } + + return resourceLabels, nil +} + +func (r *ResourceRender) getPodLabels(pod *egv1a1.KubernetesPodSpec) map[string]string { + labels := r.infra.GetProxyMetadata().Labels + maps.Copy(labels, pod.Labels) + + return envoyLabels(labels) +} + // OwningGatewayLabelsAbsent Check if labels are missing some OwningGatewayLabels func OwningGatewayLabelsAbsent(labels map[string]string) bool { return (len(labels[gatewayapi.OwningGatewayNameLabel]) == 0 || diff --git a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go index 3b105cc96c3d..6a9548817a9d 100644 --- a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go +++ b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go @@ -587,6 +587,427 @@ func loadDeployment(caseName string) (*appsv1.Deployment, error) { return deployment, nil } +func TestDaemonSet(t *testing.T) { + cfg, err := config.New() + require.NoError(t, err) + + cases := []struct { + caseName string + infra *ir.Infra + daemonset *egv1a1.KubernetesDaemonSetSpec + shutdown *egv1a1.ShutdownConfig + proxyLogging map[egv1a1.ProxyLogComponent]egv1a1.LogLevel + bootstrap string + telemetry *egv1a1.ProxyTelemetry + concurrency *int32 + extraArgs []string + }{ + { + caseName: "default", + infra: newTestInfra(), + daemonset: nil, + }, + { + caseName: "custom", + infra: newTestInfra(), + daemonset: &egv1a1.KubernetesDaemonSetSpec{ + Strategy: egv1a1.DefaultKubernetesDaemonSetStrategy(), + Pod: &egv1a1.KubernetesPodSpec{ + Annotations: map[string]string{ + "prometheus.io/scrape": "true", + }, + Labels: map[string]string{ + "foo.bar": "custom-label", + }, + SecurityContext: &corev1.PodSecurityContext{ + RunAsUser: ptr.To[int64](1000), + }, + }, + Container: &egv1a1.KubernetesContainerSpec{ + Image: ptr.To("envoyproxy/envoy:v1.2.3"), + Resources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("400m"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("200m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(true), + }, + }, + }, + }, + { + caseName: "patch-daemonset", + infra: newTestInfra(), + daemonset: &egv1a1.KubernetesDaemonSetSpec{ + Patch: &egv1a1.KubernetesPatchSpec{ + Type: ptr.To(egv1a1.StrategicMerge), + Value: v1.JSON{ + Raw: []byte("{\"spec\":{\"template\":{\"spec\":{\"hostNetwork\":true,\"dnsPolicy\":\"ClusterFirstWithHostNet\"}}}}"), + }, + }, + }, + }, + { + caseName: "shutdown-manager", + infra: newTestInfra(), + daemonset: &egv1a1.KubernetesDaemonSetSpec{ + Patch: &egv1a1.KubernetesPatchSpec{ + Type: ptr.To(egv1a1.StrategicMerge), + Value: v1.JSON{ + Raw: []byte(`{ + "spec":{ + "template":{ + "spec":{ + "containers":[{ + "name":"shutdown-manager", + "resources":{ + "requests":{"cpu":"100m","memory":"64Mi"}, + "limits":{"cpu":"200m","memory":"96Mi"} + }, + "securityContext":{"runAsUser":1234}, + "env":[ + {"name":"env_a","value":"env_a_value"}, + {"name":"env_b","value":"env_b_value"} + ], + "image":"envoyproxy/gateway-dev:v1.2.3" + }] + } + } + } + }`), + }, + }, + }, + shutdown: &egv1a1.ShutdownConfig{ + DrainTimeout: &metav1.Duration{ + Duration: 30 * time.Second, + }, + MinDrainDuration: &metav1.Duration{ + Duration: 15 * time.Second, + }, + }, + }, + { + caseName: "extension-env", + infra: newTestInfra(), + daemonset: &egv1a1.KubernetesDaemonSetSpec{ + Strategy: egv1a1.DefaultKubernetesDaemonSetStrategy(), + Pod: &egv1a1.KubernetesPodSpec{ + Annotations: map[string]string{ + "prometheus.io/scrape": "true", + }, + SecurityContext: &corev1.PodSecurityContext{ + RunAsUser: ptr.To[int64](1000), + }, + }, + Container: &egv1a1.KubernetesContainerSpec{ + Env: []corev1.EnvVar{ + { + Name: "env_a", + Value: "env_a_value", + }, + { + Name: "env_b", + Value: "env_b_value", + }, + }, + Image: ptr.To("envoyproxy/envoy:v1.2.3"), + Resources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("400m"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("200m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(true), + }, + }, + }, + }, + { + caseName: "default-env", + infra: newTestInfra(), + daemonset: &egv1a1.KubernetesDaemonSetSpec{ + Strategy: egv1a1.DefaultKubernetesDaemonSetStrategy(), + Pod: &egv1a1.KubernetesPodSpec{ + Annotations: map[string]string{ + "prometheus.io/scrape": "true", + }, + SecurityContext: &corev1.PodSecurityContext{ + RunAsUser: ptr.To[int64](1000), + }, + }, + Container: &egv1a1.KubernetesContainerSpec{ + Env: nil, + Image: ptr.To("envoyproxy/envoy:v1.2.3"), + Resources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("400m"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("200m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(true), + }, + }, + }, + }, + { + caseName: "volumes", + infra: newTestInfra(), + daemonset: &egv1a1.KubernetesDaemonSetSpec{ + Strategy: egv1a1.DefaultKubernetesDaemonSetStrategy(), + Pod: &egv1a1.KubernetesPodSpec{ + Annotations: map[string]string{ + "prometheus.io/scrape": "true", + }, + SecurityContext: &corev1.PodSecurityContext{ + RunAsUser: ptr.To[int64](1000), + }, + Volumes: []corev1.Volume{ + { + Name: "certs", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "custom-envoy-cert", + DefaultMode: ptr.To[int32](420), + }, + }, + }, + }, + }, + Container: &egv1a1.KubernetesContainerSpec{ + Env: []corev1.EnvVar{ + { + Name: "env_a", + Value: "env_a_value", + }, + { + Name: "env_b", + Value: "env_b_value", + }, + }, + Image: ptr.To("envoyproxy/envoy:v1.2.3"), + Resources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("400m"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("200m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(true), + }, + }, + }, + }, + { + caseName: "component-level", + infra: newTestInfra(), + daemonset: nil, + proxyLogging: map[egv1a1.ProxyLogComponent]egv1a1.LogLevel{ + egv1a1.LogComponentDefault: egv1a1.LogLevelError, + egv1a1.LogComponentFilter: egv1a1.LogLevelInfo, + }, + bootstrap: `test bootstrap config`, + }, + { + caseName: "disable-prometheus", + infra: newTestInfra(), + telemetry: &egv1a1.ProxyTelemetry{ + Metrics: &egv1a1.ProxyMetrics{ + Prometheus: &egv1a1.ProxyPrometheusProvider{ + Disable: true, + }, + }, + }, + }, + { + caseName: "with-concurrency", + infra: newTestInfra(), + daemonset: nil, + concurrency: ptr.To[int32](4), + bootstrap: `test bootstrap config`, + }, + { + caseName: "with-annotations", + infra: newTestInfraWithAnnotations(map[string]string{ + "anno1": "value1", + "anno2": "value2", + }), + daemonset: nil, + }, + { + caseName: "override-labels-and-annotations", + infra: newTestInfraWithAnnotationsAndLabels(map[string]string{ + "anno1": "value1", + "anno2": "value2", + }, map[string]string{ + "label1": "value1", + "label2": "value2", + }), + daemonset: &egv1a1.KubernetesDaemonSetSpec{ + Pod: &egv1a1.KubernetesPodSpec{ + Annotations: map[string]string{ + "anno1": "value1-override", + }, + Labels: map[string]string{ + "label1": "value1-override", + }, + }, + }, + }, + { + caseName: "with-image-pull-secrets", + infra: newTestInfra(), + daemonset: &egv1a1.KubernetesDaemonSetSpec{ + Pod: &egv1a1.KubernetesPodSpec{ + ImagePullSecrets: []corev1.LocalObjectReference{ + { + Name: "aaa", + }, + { + Name: "bbb", + }, + }, + }, + }, + }, + { + caseName: "with-node-selector", + infra: newTestInfra(), + daemonset: &egv1a1.KubernetesDaemonSetSpec{ + Pod: &egv1a1.KubernetesPodSpec{ + NodeSelector: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + }, + }, + }, + { + caseName: "with-topology-spread-constraints", + infra: newTestInfra(), + daemonset: &egv1a1.KubernetesDaemonSetSpec{ + Pod: &egv1a1.KubernetesPodSpec{ + TopologySpreadConstraints: []corev1.TopologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: "kubernetes.io/hostname", + WhenUnsatisfiable: corev1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "foo"}, + }, + MatchLabelKeys: []string{"pod-template-hash"}, + }, + }, + }, + }, + }, + { + caseName: "with-extra-args", + infra: newTestInfra(), + extraArgs: []string{"--key1 val1", "--key2 val2"}, + }, + } + for _, tc := range cases { + t.Run(tc.caseName, func(t *testing.T) { + kube := tc.infra.GetProxyInfra().GetProxyConfig().GetEnvoyProxyProvider().GetEnvoyProxyKubeProvider() + + // fill deploument, use daemonset + kube.EnvoyDeployment = nil + kube.EnvoyDaemonSet = egv1a1.DefaultKubernetesDaemonSet(egv1a1.DefaultEnvoyProxyImage) + + if tc.daemonset != nil { + kube.EnvoyDaemonSet = tc.daemonset + } + + replace := egv1a1.BootstrapTypeReplace + if tc.bootstrap != "" { + tc.infra.Proxy.Config.Spec.Bootstrap = &egv1a1.ProxyBootstrap{ + Type: &replace, + Value: tc.bootstrap, + } + } + + if tc.telemetry != nil { + tc.infra.Proxy.Config.Spec.Telemetry = tc.telemetry + } + + if len(tc.proxyLogging) > 0 { + tc.infra.Proxy.Config.Spec.Logging = egv1a1.ProxyLogging{ + Level: tc.proxyLogging, + } + } + + if tc.concurrency != nil { + tc.infra.Proxy.Config.Spec.Concurrency = tc.concurrency + } + + if tc.shutdown != nil { + tc.infra.Proxy.Config.Spec.Shutdown = tc.shutdown + } + + if len(tc.extraArgs) > 0 { + tc.infra.Proxy.Config.Spec.ExtraArgs = tc.extraArgs + } + + r := NewResourceRender(cfg.Namespace, tc.infra.GetProxyInfra()) + ds, err := r.DaemonSet() + require.NoError(t, err) + + expected, err := loadDaemonSet(tc.caseName) + require.NoError(t, err) + + sortEnv := func(env []corev1.EnvVar) { + sort.Slice(env, func(i, j int) bool { + return env[i].Name > env[j].Name + }) + } + + if *overrideTestData { + deploymentYAML, err := yaml.Marshal(ds) + require.NoError(t, err) + // nolint: gosec + err = os.WriteFile(fmt.Sprintf("testdata/daemonsets/%s.yaml", tc.caseName), deploymentYAML, 0644) + require.NoError(t, err) + return + } + + sortEnv(ds.Spec.Template.Spec.Containers[0].Env) + sortEnv(expected.Spec.Template.Spec.Containers[0].Env) + assert.Equal(t, expected, ds) + }) + } +} + +func loadDaemonSet(caseName string) (*appsv1.DaemonSet, error) { + daemonsetYAML, err := os.ReadFile(fmt.Sprintf("testdata/daemonsets/%s.yaml", caseName)) + if err != nil { + return nil, err + } + daemonset := &appsv1.DaemonSet{} + _ = yaml.Unmarshal(daemonsetYAML, daemonset) + return daemonset, nil +} + func TestService(t *testing.T) { cfg, err := config.New() require.NoError(t, err) diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/component-level.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/component-level.yaml new file mode 100644 index 000000000000..fba6404dcba5 --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/component-level.yaml @@ -0,0 +1,173 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + name: envoy-default-37a8eec1 + namespace: envoy-gateway-system +spec: + selector: + matchLabels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + template: + metadata: + annotations: + prometheus.io/path: /stats/prometheus + prometheus.io/port: "19001" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + spec: + automountServiceAccountToken: false + containers: + - args: + - --service-cluster default + - --service-node $(ENVOY_POD_NAME) + - --config-yaml test bootstrap config + - --log-level error + - --cpuset-threads + - --component-log-level filter:info + command: + - envoy + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/envoy:distroless-dev + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + httpGet: + path: /shutdown/ready + port: 19002 + scheme: HTTP + name: envoy + ports: + - containerPort: 8080 + name: EnvoyH-d76a15e2 + protocol: TCP + - containerPort: 8443 + name: EnvoyH-6658f727 + protocol: TCP + - containerPort: 19001 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /ready + port: 19001 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 100m + memory: 512Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /certs + name: certs + readOnly: true + - mountPath: /sds + name: sds + - args: + - envoy + - shutdown-manager + command: + - envoy-gateway + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/gateway-dev:latest + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - envoy-gateway + - envoy + - shutdown + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: shutdown-manager + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 10m + memory: 32Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + serviceAccountName: envoy-default-37a8eec1 + terminationGracePeriodSeconds: 900 + volumes: + - name: certs + secret: + defaultMode: 420 + secretName: envoy + - configMap: + defaultMode: 420 + items: + - key: xds-trusted-ca.json + path: xds-trusted-ca.json + - key: xds-certificate.json + path: xds-certificate.json + name: envoy-default-37a8eec1 + optional: false + name: sds + updateStrategy: + type: RollingUpdate +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/custom.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/custom.yaml new file mode 100644 index 000000000000..2f42db21b757 --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/custom.yaml @@ -0,0 +1,326 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + name: envoy-default-37a8eec1 + namespace: envoy-gateway-system +spec: + selector: + matchLabels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + foo.bar: custom-label + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + template: + metadata: + annotations: + prometheus.io/path: /stats/prometheus + prometheus.io/port: "19001" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + foo.bar: custom-label + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + spec: + automountServiceAccountToken: false + containers: + - args: + - --service-cluster default + - --service-node $(ENVOY_POD_NAME) + - | + --config-yaml admin: + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + layered_runtime: + layers: + - name: global_config + static_layer: + envoy.restart_features.use_eds_cache_for_ads: true + re2.max_program_size.error_level: 4294967295 + re2.max_program_size.warn_level: 1000 + dynamic_resources: + ads_config: + api_type: DELTA_GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + set_node_on_first_message_only: true + lds_config: + ads: {} + resource_api_version: V3 + cds_config: + ads: {} + resource_api_version: V3 + static_resources: + listeners: + - name: envoy-gateway-proxy-ready-0.0.0.0-19001 + address: + socket_address: + address: 0.0.0.0 + port_value: 19001 + protocol: TCP + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: eg-ready-http + route_config: + name: local_route + virtual_hosts: + - name: prometheus_stats + domains: + - "*" + routes: + - match: + prefix: /stats/prometheus + route: + cluster: prometheus_stats + http_filters: + - name: envoy.filters.http.health_check + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.health_check.v3.HealthCheck + pass_through_mode: false + headers: + - name: ":path" + string_match: + exact: /ready + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - name: prometheus_stats + connect_timeout: 0.250s + type: STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: prometheus_stats + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + - connect_timeout: 10s + load_assignment: + cluster_name: xds_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18000 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: + connection_keepalive: + interval: 30s + timeout: 5s + name: xds_cluster + type: STRICT_DNS + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 + overload_manager: + refresh_interval: 0.25s + resource_monitors: + - name: "envoy.resource_monitors.global_downstream_max_connections" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig + max_active_downstream_connections: 50000 + - name: "envoy.resource_monitors.fixed_heap" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.fixed_heap.v3.FixedHeapConfig + max_heap_size_bytes: 1717986918 + actions: + - name: "envoy.overload_actions.shrink_heap" + triggers: + - name: "envoy.resource_monitors.fixed_heap" + threshold: + value: 0.95 + - name: "envoy.overload_actions.stop_accepting_requests" + triggers: + - name: "envoy.resource_monitors.fixed_heap" + threshold: + value: 0.98 + - --log-level warn + - --cpuset-threads + command: + - envoy + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/envoy:v1.2.3 + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + httpGet: + path: /shutdown/ready + port: 19002 + scheme: HTTP + name: envoy + ports: + - containerPort: 8080 + name: EnvoyH-d76a15e2 + protocol: TCP + - containerPort: 8443 + name: EnvoyH-6658f727 + protocol: TCP + - containerPort: 19001 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /ready + port: 19001 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + cpu: 400m + memory: 2Gi + requests: + cpu: 200m + memory: 1Gi + securityContext: + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /certs + name: certs + readOnly: true + - mountPath: /sds + name: sds + - args: + - envoy + - shutdown-manager + command: + - envoy-gateway + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/gateway-dev:latest + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - envoy-gateway + - envoy + - shutdown + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: shutdown-manager + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 10m + memory: 32Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + runAsUser: 1000 + serviceAccountName: envoy-default-37a8eec1 + terminationGracePeriodSeconds: 900 + volumes: + - name: certs + secret: + defaultMode: 420 + secretName: envoy + - configMap: + defaultMode: 420 + items: + - key: xds-trusted-ca.json + path: xds-trusted-ca.json + - key: xds-certificate.json + path: xds-certificate.json + name: envoy-default-37a8eec1 + optional: false + name: sds + updateStrategy: + type: RollingUpdate +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/default-env.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/default-env.yaml new file mode 100644 index 000000000000..0f2d8ff4c45d --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/default-env.yaml @@ -0,0 +1,324 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + name: envoy-default-37a8eec1 + namespace: envoy-gateway-system +spec: + selector: + matchLabels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + template: + metadata: + annotations: + prometheus.io/path: /stats/prometheus + prometheus.io/port: "19001" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + spec: + automountServiceAccountToken: false + containers: + - args: + - --service-cluster default + - --service-node $(ENVOY_POD_NAME) + - | + --config-yaml admin: + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + layered_runtime: + layers: + - name: global_config + static_layer: + envoy.restart_features.use_eds_cache_for_ads: true + re2.max_program_size.error_level: 4294967295 + re2.max_program_size.warn_level: 1000 + dynamic_resources: + ads_config: + api_type: DELTA_GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + set_node_on_first_message_only: true + lds_config: + ads: {} + resource_api_version: V3 + cds_config: + ads: {} + resource_api_version: V3 + static_resources: + listeners: + - name: envoy-gateway-proxy-ready-0.0.0.0-19001 + address: + socket_address: + address: 0.0.0.0 + port_value: 19001 + protocol: TCP + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: eg-ready-http + route_config: + name: local_route + virtual_hosts: + - name: prometheus_stats + domains: + - "*" + routes: + - match: + prefix: /stats/prometheus + route: + cluster: prometheus_stats + http_filters: + - name: envoy.filters.http.health_check + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.health_check.v3.HealthCheck + pass_through_mode: false + headers: + - name: ":path" + string_match: + exact: /ready + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - name: prometheus_stats + connect_timeout: 0.250s + type: STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: prometheus_stats + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + - connect_timeout: 10s + load_assignment: + cluster_name: xds_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18000 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: + connection_keepalive: + interval: 30s + timeout: 5s + name: xds_cluster + type: STRICT_DNS + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 + overload_manager: + refresh_interval: 0.25s + resource_monitors: + - name: "envoy.resource_monitors.global_downstream_max_connections" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig + max_active_downstream_connections: 50000 + - name: "envoy.resource_monitors.fixed_heap" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.fixed_heap.v3.FixedHeapConfig + max_heap_size_bytes: 1717986918 + actions: + - name: "envoy.overload_actions.shrink_heap" + triggers: + - name: "envoy.resource_monitors.fixed_heap" + threshold: + value: 0.95 + - name: "envoy.overload_actions.stop_accepting_requests" + triggers: + - name: "envoy.resource_monitors.fixed_heap" + threshold: + value: 0.98 + - --log-level warn + - --cpuset-threads + command: + - envoy + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/envoy:v1.2.3 + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + httpGet: + path: /shutdown/ready + port: 19002 + scheme: HTTP + name: envoy + ports: + - containerPort: 8080 + name: EnvoyH-d76a15e2 + protocol: TCP + - containerPort: 8443 + name: EnvoyH-6658f727 + protocol: TCP + - containerPort: 19001 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /ready + port: 19001 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + cpu: 400m + memory: 2Gi + requests: + cpu: 200m + memory: 1Gi + securityContext: + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /certs + name: certs + readOnly: true + - mountPath: /sds + name: sds + - args: + - envoy + - shutdown-manager + command: + - envoy-gateway + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/gateway-dev:latest + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - envoy-gateway + - envoy + - shutdown + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: shutdown-manager + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 10m + memory: 32Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + runAsUser: 1000 + serviceAccountName: envoy-default-37a8eec1 + terminationGracePeriodSeconds: 900 + volumes: + - name: certs + secret: + defaultMode: 420 + secretName: envoy + - configMap: + defaultMode: 420 + items: + - key: xds-trusted-ca.json + path: xds-trusted-ca.json + - key: xds-certificate.json + path: xds-certificate.json + name: envoy-default-37a8eec1 + optional: false + name: sds + updateStrategy: + type: RollingUpdate +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/default.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/default.yaml new file mode 100644 index 000000000000..e3c1af008cc1 --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/default.yaml @@ -0,0 +1,302 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + name: envoy-default-37a8eec1 + namespace: envoy-gateway-system +spec: + selector: + matchLabels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + template: + metadata: + annotations: + prometheus.io/path: /stats/prometheus + prometheus.io/port: "19001" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + spec: + automountServiceAccountToken: false + containers: + - args: + - --service-cluster default + - --service-node $(ENVOY_POD_NAME) + - | + --config-yaml admin: + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + layered_runtime: + layers: + - name: global_config + static_layer: + envoy.restart_features.use_eds_cache_for_ads: true + re2.max_program_size.error_level: 4294967295 + re2.max_program_size.warn_level: 1000 + dynamic_resources: + ads_config: + api_type: DELTA_GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + set_node_on_first_message_only: true + lds_config: + ads: {} + resource_api_version: V3 + cds_config: + ads: {} + resource_api_version: V3 + static_resources: + listeners: + - name: envoy-gateway-proxy-ready-0.0.0.0-19001 + address: + socket_address: + address: 0.0.0.0 + port_value: 19001 + protocol: TCP + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: eg-ready-http + route_config: + name: local_route + virtual_hosts: + - name: prometheus_stats + domains: + - "*" + routes: + - match: + prefix: /stats/prometheus + route: + cluster: prometheus_stats + http_filters: + - name: envoy.filters.http.health_check + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.health_check.v3.HealthCheck + pass_through_mode: false + headers: + - name: ":path" + string_match: + exact: /ready + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - name: prometheus_stats + connect_timeout: 0.250s + type: STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: prometheus_stats + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + - connect_timeout: 10s + load_assignment: + cluster_name: xds_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18000 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: + connection_keepalive: + interval: 30s + timeout: 5s + name: xds_cluster + type: STRICT_DNS + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 + overload_manager: + refresh_interval: 0.25s + resource_monitors: + - name: "envoy.resource_monitors.global_downstream_max_connections" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig + max_active_downstream_connections: 50000 + - --log-level warn + - --cpuset-threads + command: + - envoy + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/envoy:distroless-dev + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + httpGet: + path: /shutdown/ready + port: 19002 + scheme: HTTP + name: envoy + ports: + - containerPort: 8080 + name: EnvoyH-d76a15e2 + protocol: TCP + - containerPort: 8443 + name: EnvoyH-6658f727 + protocol: TCP + - containerPort: 19001 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /ready + port: 19001 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 100m + memory: 512Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /certs + name: certs + readOnly: true + - mountPath: /sds + name: sds + - args: + - envoy + - shutdown-manager + command: + - envoy-gateway + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/gateway-dev:latest + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - envoy-gateway + - envoy + - shutdown + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: shutdown-manager + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 10m + memory: 32Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + serviceAccountName: envoy-default-37a8eec1 + terminationGracePeriodSeconds: 900 + volumes: + - name: certs + secret: + defaultMode: 420 + secretName: envoy + - configMap: + defaultMode: 420 + items: + - key: xds-trusted-ca.json + path: xds-trusted-ca.json + - key: xds-certificate.json + path: xds-certificate.json + name: envoy-default-37a8eec1 + optional: false + name: sds + updateStrategy: + type: RollingUpdate +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/disable-prometheus.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/disable-prometheus.yaml new file mode 100644 index 000000000000..78f4290aa36d --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/disable-prometheus.yaml @@ -0,0 +1,273 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + name: envoy-default-37a8eec1 + namespace: envoy-gateway-system +spec: + selector: + matchLabels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + template: + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + spec: + automountServiceAccountToken: false + containers: + - args: + - --service-cluster default + - --service-node $(ENVOY_POD_NAME) + - | + --config-yaml admin: + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + layered_runtime: + layers: + - name: global_config + static_layer: + envoy.restart_features.use_eds_cache_for_ads: true + re2.max_program_size.error_level: 4294967295 + re2.max_program_size.warn_level: 1000 + dynamic_resources: + ads_config: + api_type: DELTA_GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + set_node_on_first_message_only: true + lds_config: + ads: {} + resource_api_version: V3 + cds_config: + ads: {} + resource_api_version: V3 + static_resources: + listeners: + - name: envoy-gateway-proxy-ready-0.0.0.0-19001 + address: + socket_address: + address: 0.0.0.0 + port_value: 19001 + protocol: TCP + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: eg-ready-http + route_config: + name: local_route + http_filters: + - name: envoy.filters.http.health_check + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.health_check.v3.HealthCheck + pass_through_mode: false + headers: + - name: ":path" + string_match: + exact: /ready + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - connect_timeout: 10s + load_assignment: + cluster_name: xds_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18000 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: + connection_keepalive: + interval: 30s + timeout: 5s + name: xds_cluster + type: STRICT_DNS + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 + overload_manager: + refresh_interval: 0.25s + resource_monitors: + - name: "envoy.resource_monitors.global_downstream_max_connections" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig + max_active_downstream_connections: 50000 + - --log-level warn + - --cpuset-threads + command: + - envoy + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/envoy:distroless-dev + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + httpGet: + path: /shutdown/ready + port: 19002 + scheme: HTTP + name: envoy + ports: + - containerPort: 8080 + name: EnvoyH-d76a15e2 + protocol: TCP + - containerPort: 8443 + name: EnvoyH-6658f727 + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /ready + port: 19001 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 100m + memory: 512Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /certs + name: certs + readOnly: true + - mountPath: /sds + name: sds + - args: + - envoy + - shutdown-manager + command: + - envoy-gateway + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/gateway-dev:latest + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - envoy-gateway + - envoy + - shutdown + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: shutdown-manager + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 10m + memory: 32Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + serviceAccountName: envoy-default-37a8eec1 + terminationGracePeriodSeconds: 900 + volumes: + - name: certs + secret: + defaultMode: 420 + secretName: envoy + - configMap: + defaultMode: 420 + items: + - key: xds-trusted-ca.json + path: xds-trusted-ca.json + - key: xds-certificate.json + path: xds-certificate.json + name: envoy-default-37a8eec1 + optional: false + name: sds + updateStrategy: + type: RollingUpdate +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/extension-env.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/extension-env.yaml new file mode 100644 index 000000000000..c5deef21f0e3 --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/extension-env.yaml @@ -0,0 +1,328 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + name: envoy-default-37a8eec1 + namespace: envoy-gateway-system +spec: + selector: + matchLabels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + template: + metadata: + annotations: + prometheus.io/path: /stats/prometheus + prometheus.io/port: "19001" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + spec: + automountServiceAccountToken: false + containers: + - args: + - --service-cluster default + - --service-node $(ENVOY_POD_NAME) + - | + --config-yaml admin: + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + layered_runtime: + layers: + - name: global_config + static_layer: + envoy.restart_features.use_eds_cache_for_ads: true + re2.max_program_size.error_level: 4294967295 + re2.max_program_size.warn_level: 1000 + dynamic_resources: + ads_config: + api_type: DELTA_GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + set_node_on_first_message_only: true + lds_config: + ads: {} + resource_api_version: V3 + cds_config: + ads: {} + resource_api_version: V3 + static_resources: + listeners: + - name: envoy-gateway-proxy-ready-0.0.0.0-19001 + address: + socket_address: + address: 0.0.0.0 + port_value: 19001 + protocol: TCP + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: eg-ready-http + route_config: + name: local_route + virtual_hosts: + - name: prometheus_stats + domains: + - "*" + routes: + - match: + prefix: /stats/prometheus + route: + cluster: prometheus_stats + http_filters: + - name: envoy.filters.http.health_check + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.health_check.v3.HealthCheck + pass_through_mode: false + headers: + - name: ":path" + string_match: + exact: /ready + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - name: prometheus_stats + connect_timeout: 0.250s + type: STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: prometheus_stats + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + - connect_timeout: 10s + load_assignment: + cluster_name: xds_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18000 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: + connection_keepalive: + interval: 30s + timeout: 5s + name: xds_cluster + type: STRICT_DNS + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 + overload_manager: + refresh_interval: 0.25s + resource_monitors: + - name: "envoy.resource_monitors.global_downstream_max_connections" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig + max_active_downstream_connections: 50000 + - name: "envoy.resource_monitors.fixed_heap" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.fixed_heap.v3.FixedHeapConfig + max_heap_size_bytes: 1717986918 + actions: + - name: "envoy.overload_actions.shrink_heap" + triggers: + - name: "envoy.resource_monitors.fixed_heap" + threshold: + value: 0.95 + - name: "envoy.overload_actions.stop_accepting_requests" + triggers: + - name: "envoy.resource_monitors.fixed_heap" + threshold: + value: 0.98 + - --log-level warn + - --cpuset-threads + command: + - envoy + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: env_a + value: env_a_value + - name: env_b + value: env_b_value + image: envoyproxy/envoy:v1.2.3 + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + httpGet: + path: /shutdown/ready + port: 19002 + scheme: HTTP + name: envoy + ports: + - containerPort: 8080 + name: EnvoyH-d76a15e2 + protocol: TCP + - containerPort: 8443 + name: EnvoyH-6658f727 + protocol: TCP + - containerPort: 19001 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /ready + port: 19001 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + cpu: 400m + memory: 2Gi + requests: + cpu: 200m + memory: 1Gi + securityContext: + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /certs + name: certs + readOnly: true + - mountPath: /sds + name: sds + - args: + - envoy + - shutdown-manager + command: + - envoy-gateway + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/gateway-dev:latest + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - envoy-gateway + - envoy + - shutdown + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: shutdown-manager + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 10m + memory: 32Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + runAsUser: 1000 + serviceAccountName: envoy-default-37a8eec1 + terminationGracePeriodSeconds: 900 + volumes: + - name: certs + secret: + defaultMode: 420 + secretName: envoy + - configMap: + defaultMode: 420 + items: + - key: xds-trusted-ca.json + path: xds-trusted-ca.json + - key: xds-certificate.json + path: xds-certificate.json + name: envoy-default-37a8eec1 + optional: false + name: sds + updateStrategy: + type: RollingUpdate +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/override-labels-and-annotations.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/override-labels-and-annotations.yaml new file mode 100644 index 000000000000..2d35370a80bc --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/override-labels-and-annotations.yaml @@ -0,0 +1,313 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + annotations: + anno1: value1 + anno2: value2 + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + label1: value1 + label2: value2 + name: envoy-default-37a8eec1 + namespace: envoy-gateway-system +spec: + selector: + matchLabels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + label1: value1-override + label2: value2 + template: + metadata: + annotations: + anno1: value1-override + anno2: value2 + prometheus.io/path: /stats/prometheus + prometheus.io/port: "19001" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + label1: value1-override + label2: value2 + spec: + automountServiceAccountToken: false + containers: + - args: + - --service-cluster default + - --service-node $(ENVOY_POD_NAME) + - | + --config-yaml admin: + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + layered_runtime: + layers: + - name: global_config + static_layer: + envoy.restart_features.use_eds_cache_for_ads: true + re2.max_program_size.error_level: 4294967295 + re2.max_program_size.warn_level: 1000 + dynamic_resources: + ads_config: + api_type: DELTA_GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + set_node_on_first_message_only: true + lds_config: + ads: {} + resource_api_version: V3 + cds_config: + ads: {} + resource_api_version: V3 + static_resources: + listeners: + - name: envoy-gateway-proxy-ready-0.0.0.0-19001 + address: + socket_address: + address: 0.0.0.0 + port_value: 19001 + protocol: TCP + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: eg-ready-http + route_config: + name: local_route + virtual_hosts: + - name: prometheus_stats + domains: + - "*" + routes: + - match: + prefix: /stats/prometheus + route: + cluster: prometheus_stats + http_filters: + - name: envoy.filters.http.health_check + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.health_check.v3.HealthCheck + pass_through_mode: false + headers: + - name: ":path" + string_match: + exact: /ready + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - name: prometheus_stats + connect_timeout: 0.250s + type: STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: prometheus_stats + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + - connect_timeout: 10s + load_assignment: + cluster_name: xds_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18000 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: + connection_keepalive: + interval: 30s + timeout: 5s + name: xds_cluster + type: STRICT_DNS + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 + overload_manager: + refresh_interval: 0.25s + resource_monitors: + - name: "envoy.resource_monitors.global_downstream_max_connections" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig + max_active_downstream_connections: 50000 + - --log-level warn + - --cpuset-threads + command: + - envoy + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/envoy:distroless-dev + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + httpGet: + path: /shutdown/ready + port: 19002 + scheme: HTTP + name: envoy + ports: + - containerPort: 8080 + name: EnvoyH-d76a15e2 + protocol: TCP + - containerPort: 8443 + name: EnvoyH-6658f727 + protocol: TCP + - containerPort: 19001 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /ready + port: 19001 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 100m + memory: 512Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /certs + name: certs + readOnly: true + - mountPath: /sds + name: sds + - args: + - envoy + - shutdown-manager + command: + - envoy-gateway + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/gateway-dev:latest + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - envoy-gateway + - envoy + - shutdown + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: shutdown-manager + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 10m + memory: 32Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + serviceAccountName: envoy-default-37a8eec1 + terminationGracePeriodSeconds: 900 + volumes: + - name: certs + secret: + defaultMode: 420 + secretName: envoy + - configMap: + defaultMode: 420 + items: + - key: xds-trusted-ca.json + path: xds-trusted-ca.json + - key: xds-certificate.json + path: xds-certificate.json + name: envoy-default-37a8eec1 + optional: false + name: sds + updateStrategy: + type: RollingUpdate +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/patch-daemonset.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/patch-daemonset.yaml new file mode 100644 index 000000000000..a74320a1de8f --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/patch-daemonset.yaml @@ -0,0 +1,303 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + name: envoy-default-37a8eec1 + namespace: envoy-gateway-system +spec: + selector: + matchLabels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + template: + metadata: + annotations: + prometheus.io/path: /stats/prometheus + prometheus.io/port: "19001" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + spec: + automountServiceAccountToken: false + containers: + - args: + - --service-cluster default + - --service-node $(ENVOY_POD_NAME) + - | + --config-yaml admin: + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + layered_runtime: + layers: + - name: global_config + static_layer: + envoy.restart_features.use_eds_cache_for_ads: true + re2.max_program_size.error_level: 4294967295 + re2.max_program_size.warn_level: 1000 + dynamic_resources: + ads_config: + api_type: DELTA_GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + set_node_on_first_message_only: true + lds_config: + ads: {} + resource_api_version: V3 + cds_config: + ads: {} + resource_api_version: V3 + static_resources: + listeners: + - name: envoy-gateway-proxy-ready-0.0.0.0-19001 + address: + socket_address: + address: 0.0.0.0 + port_value: 19001 + protocol: TCP + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: eg-ready-http + route_config: + name: local_route + virtual_hosts: + - name: prometheus_stats + domains: + - "*" + routes: + - match: + prefix: /stats/prometheus + route: + cluster: prometheus_stats + http_filters: + - name: envoy.filters.http.health_check + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.health_check.v3.HealthCheck + pass_through_mode: false + headers: + - name: ":path" + string_match: + exact: /ready + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - name: prometheus_stats + connect_timeout: 0.250s + type: STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: prometheus_stats + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + - connect_timeout: 10s + load_assignment: + cluster_name: xds_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18000 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: + connection_keepalive: + interval: 30s + timeout: 5s + name: xds_cluster + type: STRICT_DNS + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 + overload_manager: + refresh_interval: 0.25s + resource_monitors: + - name: "envoy.resource_monitors.global_downstream_max_connections" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig + max_active_downstream_connections: 50000 + - --log-level warn + - --cpuset-threads + command: + - envoy + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/envoy:distroless-dev + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + httpGet: + path: /shutdown/ready + port: 19002 + scheme: HTTP + name: envoy + ports: + - containerPort: 8080 + name: EnvoyH-d76a15e2 + protocol: TCP + - containerPort: 8443 + name: EnvoyH-6658f727 + protocol: TCP + - containerPort: 19001 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /ready + port: 19001 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 100m + memory: 512Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /certs + name: certs + readOnly: true + - mountPath: /sds + name: sds + - args: + - envoy + - shutdown-manager + command: + - envoy-gateway + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/gateway-dev:latest + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - envoy-gateway + - envoy + - shutdown + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: shutdown-manager + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 10m + memory: 32Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + restartPolicy: Always + schedulerName: default-scheduler + serviceAccountName: envoy-default-37a8eec1 + terminationGracePeriodSeconds: 900 + volumes: + - name: certs + secret: + defaultMode: 420 + secretName: envoy + - configMap: + defaultMode: 420 + items: + - key: xds-trusted-ca.json + path: xds-trusted-ca.json + - key: xds-certificate.json + path: xds-certificate.json + name: envoy-default-37a8eec1 + optional: false + name: sds + updateStrategy: + type: RollingUpdate +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/shutdown-manager.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/shutdown-manager.yaml new file mode 100644 index 000000000000..c00c3c159365 --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/shutdown-manager.yaml @@ -0,0 +1,315 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + name: envoy-default-37a8eec1 + namespace: envoy-gateway-system +spec: + selector: + matchLabels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + template: + metadata: + annotations: + prometheus.io/path: /stats/prometheus + prometheus.io/port: "19001" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + spec: + automountServiceAccountToken: false + containers: + - args: + - --service-cluster default + - --service-node $(ENVOY_POD_NAME) + - | + --config-yaml admin: + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + layered_runtime: + layers: + - name: global_config + static_layer: + envoy.restart_features.use_eds_cache_for_ads: true + re2.max_program_size.error_level: 4294967295 + re2.max_program_size.warn_level: 1000 + dynamic_resources: + ads_config: + api_type: DELTA_GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + set_node_on_first_message_only: true + lds_config: + ads: {} + resource_api_version: V3 + cds_config: + ads: {} + resource_api_version: V3 + static_resources: + listeners: + - name: envoy-gateway-proxy-ready-0.0.0.0-19001 + address: + socket_address: + address: 0.0.0.0 + port_value: 19001 + protocol: TCP + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: eg-ready-http + route_config: + name: local_route + virtual_hosts: + - name: prometheus_stats + domains: + - "*" + routes: + - match: + prefix: /stats/prometheus + route: + cluster: prometheus_stats + http_filters: + - name: envoy.filters.http.health_check + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.health_check.v3.HealthCheck + pass_through_mode: false + headers: + - name: ":path" + string_match: + exact: /ready + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - name: prometheus_stats + connect_timeout: 0.250s + type: STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: prometheus_stats + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + - connect_timeout: 10s + load_assignment: + cluster_name: xds_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18000 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: + connection_keepalive: + interval: 30s + timeout: 5s + name: xds_cluster + type: STRICT_DNS + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 + overload_manager: + refresh_interval: 0.25s + resource_monitors: + - name: "envoy.resource_monitors.global_downstream_max_connections" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig + max_active_downstream_connections: 50000 + - --log-level warn + - --cpuset-threads + - --drain-time-s 30 + command: + - envoy + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/envoy:distroless-dev + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + httpGet: + path: /shutdown/ready + port: 19002 + scheme: HTTP + name: envoy + ports: + - containerPort: 8080 + name: EnvoyH-d76a15e2 + protocol: TCP + - containerPort: 8443 + name: EnvoyH-6658f727 + protocol: TCP + - containerPort: 19001 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /ready + port: 19001 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 100m + memory: 512Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /certs + name: certs + readOnly: true + - mountPath: /sds + name: sds + - args: + - envoy + - shutdown-manager + - --ready-timeout=40s + command: + - envoy-gateway + env: + - name: env_a + value: env_a_value + - name: env_b + value: env_b_value + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/gateway-dev:v1.2.3 + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - envoy-gateway + - envoy + - shutdown + - --drain-timeout=30s + - --min-drain-duration=15s + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: shutdown-manager + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + cpu: 200m + memory: 96Mi + requests: + cpu: 100m + memory: 64Mi + securityContext: + runAsUser: 1234 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + serviceAccountName: envoy-default-37a8eec1 + terminationGracePeriodSeconds: 330 + volumes: + - name: certs + secret: + defaultMode: 420 + secretName: envoy + - configMap: + defaultMode: 420 + items: + - key: xds-trusted-ca.json + path: xds-trusted-ca.json + - key: xds-certificate.json + path: xds-certificate.json + name: envoy-default-37a8eec1 + optional: false + name: sds + updateStrategy: + type: RollingUpdate +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/volumes.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/volumes.yaml new file mode 100644 index 000000000000..e59817967d9a --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/volumes.yaml @@ -0,0 +1,328 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + name: envoy-default-37a8eec1 + namespace: envoy-gateway-system +spec: + selector: + matchLabels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + template: + metadata: + annotations: + prometheus.io/path: /stats/prometheus + prometheus.io/port: "19001" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + spec: + automountServiceAccountToken: false + containers: + - args: + - --service-cluster default + - --service-node $(ENVOY_POD_NAME) + - | + --config-yaml admin: + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + layered_runtime: + layers: + - name: global_config + static_layer: + envoy.restart_features.use_eds_cache_for_ads: true + re2.max_program_size.error_level: 4294967295 + re2.max_program_size.warn_level: 1000 + dynamic_resources: + ads_config: + api_type: DELTA_GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + set_node_on_first_message_only: true + lds_config: + ads: {} + resource_api_version: V3 + cds_config: + ads: {} + resource_api_version: V3 + static_resources: + listeners: + - name: envoy-gateway-proxy-ready-0.0.0.0-19001 + address: + socket_address: + address: 0.0.0.0 + port_value: 19001 + protocol: TCP + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: eg-ready-http + route_config: + name: local_route + virtual_hosts: + - name: prometheus_stats + domains: + - "*" + routes: + - match: + prefix: /stats/prometheus + route: + cluster: prometheus_stats + http_filters: + - name: envoy.filters.http.health_check + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.health_check.v3.HealthCheck + pass_through_mode: false + headers: + - name: ":path" + string_match: + exact: /ready + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - name: prometheus_stats + connect_timeout: 0.250s + type: STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: prometheus_stats + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + - connect_timeout: 10s + load_assignment: + cluster_name: xds_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18000 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: + connection_keepalive: + interval: 30s + timeout: 5s + name: xds_cluster + type: STRICT_DNS + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 + overload_manager: + refresh_interval: 0.25s + resource_monitors: + - name: "envoy.resource_monitors.global_downstream_max_connections" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig + max_active_downstream_connections: 50000 + - name: "envoy.resource_monitors.fixed_heap" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.fixed_heap.v3.FixedHeapConfig + max_heap_size_bytes: 1717986918 + actions: + - name: "envoy.overload_actions.shrink_heap" + triggers: + - name: "envoy.resource_monitors.fixed_heap" + threshold: + value: 0.95 + - name: "envoy.overload_actions.stop_accepting_requests" + triggers: + - name: "envoy.resource_monitors.fixed_heap" + threshold: + value: 0.98 + - --log-level warn + - --cpuset-threads + command: + - envoy + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: env_a + value: env_a_value + - name: env_b + value: env_b_value + image: envoyproxy/envoy:v1.2.3 + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + httpGet: + path: /shutdown/ready + port: 19002 + scheme: HTTP + name: envoy + ports: + - containerPort: 8080 + name: EnvoyH-d76a15e2 + protocol: TCP + - containerPort: 8443 + name: EnvoyH-6658f727 + protocol: TCP + - containerPort: 19001 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /ready + port: 19001 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + cpu: 400m + memory: 2Gi + requests: + cpu: 200m + memory: 1Gi + securityContext: + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /certs + name: certs + readOnly: true + - mountPath: /sds + name: sds + - args: + - envoy + - shutdown-manager + command: + - envoy-gateway + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/gateway-dev:latest + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - envoy-gateway + - envoy + - shutdown + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: shutdown-manager + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 10m + memory: 32Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + runAsUser: 1000 + serviceAccountName: envoy-default-37a8eec1 + terminationGracePeriodSeconds: 900 + volumes: + - name: certs + secret: + defaultMode: 420 + secretName: custom-envoy-cert + - configMap: + defaultMode: 420 + items: + - key: xds-trusted-ca.json + path: xds-trusted-ca.json + - key: xds-certificate.json + path: xds-certificate.json + name: envoy-default-37a8eec1 + optional: false + name: sds + updateStrategy: + type: RollingUpdate +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-annotations.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-annotations.yaml new file mode 100644 index 000000000000..0a3b8bedd57f --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-annotations.yaml @@ -0,0 +1,307 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + annotations: + anno1: value1 + anno2: value2 + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + name: envoy-default-37a8eec1 + namespace: envoy-gateway-system +spec: + selector: + matchLabels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + template: + metadata: + annotations: + anno1: value1 + anno2: value2 + prometheus.io/path: /stats/prometheus + prometheus.io/port: "19001" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + spec: + automountServiceAccountToken: false + containers: + - args: + - --service-cluster default + - --service-node $(ENVOY_POD_NAME) + - | + --config-yaml admin: + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + layered_runtime: + layers: + - name: global_config + static_layer: + envoy.restart_features.use_eds_cache_for_ads: true + re2.max_program_size.error_level: 4294967295 + re2.max_program_size.warn_level: 1000 + dynamic_resources: + ads_config: + api_type: DELTA_GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + set_node_on_first_message_only: true + lds_config: + ads: {} + resource_api_version: V3 + cds_config: + ads: {} + resource_api_version: V3 + static_resources: + listeners: + - name: envoy-gateway-proxy-ready-0.0.0.0-19001 + address: + socket_address: + address: 0.0.0.0 + port_value: 19001 + protocol: TCP + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: eg-ready-http + route_config: + name: local_route + virtual_hosts: + - name: prometheus_stats + domains: + - "*" + routes: + - match: + prefix: /stats/prometheus + route: + cluster: prometheus_stats + http_filters: + - name: envoy.filters.http.health_check + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.health_check.v3.HealthCheck + pass_through_mode: false + headers: + - name: ":path" + string_match: + exact: /ready + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - name: prometheus_stats + connect_timeout: 0.250s + type: STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: prometheus_stats + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + - connect_timeout: 10s + load_assignment: + cluster_name: xds_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18000 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: + connection_keepalive: + interval: 30s + timeout: 5s + name: xds_cluster + type: STRICT_DNS + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 + overload_manager: + refresh_interval: 0.25s + resource_monitors: + - name: "envoy.resource_monitors.global_downstream_max_connections" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig + max_active_downstream_connections: 50000 + - --log-level warn + - --cpuset-threads + command: + - envoy + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/envoy:distroless-dev + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + httpGet: + path: /shutdown/ready + port: 19002 + scheme: HTTP + name: envoy + ports: + - containerPort: 8080 + name: EnvoyH-d76a15e2 + protocol: TCP + - containerPort: 8443 + name: EnvoyH-6658f727 + protocol: TCP + - containerPort: 19001 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /ready + port: 19001 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 100m + memory: 512Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /certs + name: certs + readOnly: true + - mountPath: /sds + name: sds + - args: + - envoy + - shutdown-manager + command: + - envoy-gateway + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/gateway-dev:latest + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - envoy-gateway + - envoy + - shutdown + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: shutdown-manager + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 10m + memory: 32Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + serviceAccountName: envoy-default-37a8eec1 + terminationGracePeriodSeconds: 900 + volumes: + - name: certs + secret: + defaultMode: 420 + secretName: envoy + - configMap: + defaultMode: 420 + items: + - key: xds-trusted-ca.json + path: xds-trusted-ca.json + - key: xds-certificate.json + path: xds-certificate.json + name: envoy-default-37a8eec1 + optional: false + name: sds + updateStrategy: + type: RollingUpdate +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-concurrency.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-concurrency.yaml new file mode 100644 index 000000000000..b957a0e4aca5 --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-concurrency.yaml @@ -0,0 +1,173 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + name: envoy-default-37a8eec1 + namespace: envoy-gateway-system +spec: + selector: + matchLabels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + template: + metadata: + annotations: + prometheus.io/path: /stats/prometheus + prometheus.io/port: "19001" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + spec: + automountServiceAccountToken: false + containers: + - args: + - --service-cluster default + - --service-node $(ENVOY_POD_NAME) + - --config-yaml test bootstrap config + - --log-level warn + - --cpuset-threads + - --concurrency 4 + command: + - envoy + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/envoy:distroless-dev + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + httpGet: + path: /shutdown/ready + port: 19002 + scheme: HTTP + name: envoy + ports: + - containerPort: 8080 + name: EnvoyH-d76a15e2 + protocol: TCP + - containerPort: 8443 + name: EnvoyH-6658f727 + protocol: TCP + - containerPort: 19001 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /ready + port: 19001 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 100m + memory: 512Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /certs + name: certs + readOnly: true + - mountPath: /sds + name: sds + - args: + - envoy + - shutdown-manager + command: + - envoy-gateway + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/gateway-dev:latest + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - envoy-gateway + - envoy + - shutdown + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: shutdown-manager + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 10m + memory: 32Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + serviceAccountName: envoy-default-37a8eec1 + terminationGracePeriodSeconds: 900 + volumes: + - name: certs + secret: + defaultMode: 420 + secretName: envoy + - configMap: + defaultMode: 420 + items: + - key: xds-trusted-ca.json + path: xds-trusted-ca.json + - key: xds-certificate.json + path: xds-certificate.json + name: envoy-default-37a8eec1 + optional: false + name: sds + updateStrategy: + type: RollingUpdate +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-extra-args.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-extra-args.yaml new file mode 100644 index 000000000000..bdd312109c46 --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-extra-args.yaml @@ -0,0 +1,304 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + name: envoy-default-37a8eec1 + namespace: envoy-gateway-system +spec: + selector: + matchLabels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + template: + metadata: + annotations: + prometheus.io/path: /stats/prometheus + prometheus.io/port: "19001" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + spec: + automountServiceAccountToken: false + containers: + - args: + - --service-cluster default + - --service-node $(ENVOY_POD_NAME) + - | + --config-yaml admin: + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + layered_runtime: + layers: + - name: global_config + static_layer: + envoy.restart_features.use_eds_cache_for_ads: true + re2.max_program_size.error_level: 4294967295 + re2.max_program_size.warn_level: 1000 + dynamic_resources: + ads_config: + api_type: DELTA_GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + set_node_on_first_message_only: true + lds_config: + ads: {} + resource_api_version: V3 + cds_config: + ads: {} + resource_api_version: V3 + static_resources: + listeners: + - name: envoy-gateway-proxy-ready-0.0.0.0-19001 + address: + socket_address: + address: 0.0.0.0 + port_value: 19001 + protocol: TCP + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: eg-ready-http + route_config: + name: local_route + virtual_hosts: + - name: prometheus_stats + domains: + - "*" + routes: + - match: + prefix: /stats/prometheus + route: + cluster: prometheus_stats + http_filters: + - name: envoy.filters.http.health_check + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.health_check.v3.HealthCheck + pass_through_mode: false + headers: + - name: ":path" + string_match: + exact: /ready + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - name: prometheus_stats + connect_timeout: 0.250s + type: STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: prometheus_stats + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + - connect_timeout: 10s + load_assignment: + cluster_name: xds_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18000 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: + connection_keepalive: + interval: 30s + timeout: 5s + name: xds_cluster + type: STRICT_DNS + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 + overload_manager: + refresh_interval: 0.25s + resource_monitors: + - name: "envoy.resource_monitors.global_downstream_max_connections" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig + max_active_downstream_connections: 50000 + - --log-level warn + - --cpuset-threads + - --key1 val1 + - --key2 val2 + command: + - envoy + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/envoy:distroless-dev + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + httpGet: + path: /shutdown/ready + port: 19002 + scheme: HTTP + name: envoy + ports: + - containerPort: 8080 + name: EnvoyH-d76a15e2 + protocol: TCP + - containerPort: 8443 + name: EnvoyH-6658f727 + protocol: TCP + - containerPort: 19001 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /ready + port: 19001 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 100m + memory: 512Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /certs + name: certs + readOnly: true + - mountPath: /sds + name: sds + - args: + - envoy + - shutdown-manager + command: + - envoy-gateway + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/gateway-dev:latest + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - envoy-gateway + - envoy + - shutdown + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: shutdown-manager + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 10m + memory: 32Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + serviceAccountName: envoy-default-37a8eec1 + terminationGracePeriodSeconds: 900 + volumes: + - name: certs + secret: + defaultMode: 420 + secretName: envoy + - configMap: + defaultMode: 420 + items: + - key: xds-trusted-ca.json + path: xds-trusted-ca.json + - key: xds-certificate.json + path: xds-certificate.json + name: envoy-default-37a8eec1 + optional: false + name: sds + updateStrategy: + type: RollingUpdate +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-image-pull-secrets.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-image-pull-secrets.yaml new file mode 100644 index 000000000000..9148a694c468 --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-image-pull-secrets.yaml @@ -0,0 +1,305 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + name: envoy-default-37a8eec1 + namespace: envoy-gateway-system +spec: + selector: + matchLabels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + template: + metadata: + annotations: + prometheus.io/path: /stats/prometheus + prometheus.io/port: "19001" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + spec: + automountServiceAccountToken: false + containers: + - args: + - --service-cluster default + - --service-node $(ENVOY_POD_NAME) + - | + --config-yaml admin: + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + layered_runtime: + layers: + - name: global_config + static_layer: + envoy.restart_features.use_eds_cache_for_ads: true + re2.max_program_size.error_level: 4294967295 + re2.max_program_size.warn_level: 1000 + dynamic_resources: + ads_config: + api_type: DELTA_GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + set_node_on_first_message_only: true + lds_config: + ads: {} + resource_api_version: V3 + cds_config: + ads: {} + resource_api_version: V3 + static_resources: + listeners: + - name: envoy-gateway-proxy-ready-0.0.0.0-19001 + address: + socket_address: + address: 0.0.0.0 + port_value: 19001 + protocol: TCP + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: eg-ready-http + route_config: + name: local_route + virtual_hosts: + - name: prometheus_stats + domains: + - "*" + routes: + - match: + prefix: /stats/prometheus + route: + cluster: prometheus_stats + http_filters: + - name: envoy.filters.http.health_check + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.health_check.v3.HealthCheck + pass_through_mode: false + headers: + - name: ":path" + string_match: + exact: /ready + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - name: prometheus_stats + connect_timeout: 0.250s + type: STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: prometheus_stats + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + - connect_timeout: 10s + load_assignment: + cluster_name: xds_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18000 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: + connection_keepalive: + interval: 30s + timeout: 5s + name: xds_cluster + type: STRICT_DNS + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 + overload_manager: + refresh_interval: 0.25s + resource_monitors: + - name: "envoy.resource_monitors.global_downstream_max_connections" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig + max_active_downstream_connections: 50000 + - --log-level warn + - --cpuset-threads + command: + - envoy + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/envoy:distroless-dev + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + httpGet: + path: /shutdown/ready + port: 19002 + scheme: HTTP + name: envoy + ports: + - containerPort: 8080 + name: EnvoyH-d76a15e2 + protocol: TCP + - containerPort: 8443 + name: EnvoyH-6658f727 + protocol: TCP + - containerPort: 19001 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /ready + port: 19001 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 100m + memory: 512Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /certs + name: certs + readOnly: true + - mountPath: /sds + name: sds + - args: + - envoy + - shutdown-manager + command: + - envoy-gateway + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/gateway-dev:latest + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - envoy-gateway + - envoy + - shutdown + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: shutdown-manager + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 10m + memory: 32Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + imagePullSecrets: + - name: aaa + - name: bbb + restartPolicy: Always + schedulerName: default-scheduler + serviceAccountName: envoy-default-37a8eec1 + terminationGracePeriodSeconds: 900 + volumes: + - name: certs + secret: + defaultMode: 420 + secretName: envoy + - configMap: + defaultMode: 420 + items: + - key: xds-trusted-ca.json + path: xds-trusted-ca.json + - key: xds-certificate.json + path: xds-certificate.json + name: envoy-default-37a8eec1 + optional: false + name: sds + updateStrategy: + type: RollingUpdate +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-node-selector.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-node-selector.yaml new file mode 100644 index 000000000000..0d2f89660deb --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-node-selector.yaml @@ -0,0 +1,305 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + name: envoy-default-37a8eec1 + namespace: envoy-gateway-system +spec: + selector: + matchLabels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + template: + metadata: + annotations: + prometheus.io/path: /stats/prometheus + prometheus.io/port: "19001" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + spec: + automountServiceAccountToken: false + containers: + - args: + - --service-cluster default + - --service-node $(ENVOY_POD_NAME) + - | + --config-yaml admin: + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + layered_runtime: + layers: + - name: global_config + static_layer: + envoy.restart_features.use_eds_cache_for_ads: true + re2.max_program_size.error_level: 4294967295 + re2.max_program_size.warn_level: 1000 + dynamic_resources: + ads_config: + api_type: DELTA_GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + set_node_on_first_message_only: true + lds_config: + ads: {} + resource_api_version: V3 + cds_config: + ads: {} + resource_api_version: V3 + static_resources: + listeners: + - name: envoy-gateway-proxy-ready-0.0.0.0-19001 + address: + socket_address: + address: 0.0.0.0 + port_value: 19001 + protocol: TCP + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: eg-ready-http + route_config: + name: local_route + virtual_hosts: + - name: prometheus_stats + domains: + - "*" + routes: + - match: + prefix: /stats/prometheus + route: + cluster: prometheus_stats + http_filters: + - name: envoy.filters.http.health_check + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.health_check.v3.HealthCheck + pass_through_mode: false + headers: + - name: ":path" + string_match: + exact: /ready + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - name: prometheus_stats + connect_timeout: 0.250s + type: STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: prometheus_stats + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + - connect_timeout: 10s + load_assignment: + cluster_name: xds_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18000 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: + connection_keepalive: + interval: 30s + timeout: 5s + name: xds_cluster + type: STRICT_DNS + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 + overload_manager: + refresh_interval: 0.25s + resource_monitors: + - name: "envoy.resource_monitors.global_downstream_max_connections" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig + max_active_downstream_connections: 50000 + - --log-level warn + - --cpuset-threads + command: + - envoy + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/envoy:distroless-dev + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + httpGet: + path: /shutdown/ready + port: 19002 + scheme: HTTP + name: envoy + ports: + - containerPort: 8080 + name: EnvoyH-d76a15e2 + protocol: TCP + - containerPort: 8443 + name: EnvoyH-6658f727 + protocol: TCP + - containerPort: 19001 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /ready + port: 19001 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 100m + memory: 512Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /certs + name: certs + readOnly: true + - mountPath: /sds + name: sds + - args: + - envoy + - shutdown-manager + command: + - envoy-gateway + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/gateway-dev:latest + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - envoy-gateway + - envoy + - shutdown + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: shutdown-manager + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 10m + memory: 32Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + nodeSelector: + key1: value1 + key2: value2 + restartPolicy: Always + schedulerName: default-scheduler + serviceAccountName: envoy-default-37a8eec1 + terminationGracePeriodSeconds: 900 + volumes: + - name: certs + secret: + defaultMode: 420 + secretName: envoy + - configMap: + defaultMode: 420 + items: + - key: xds-trusted-ca.json + path: xds-trusted-ca.json + - key: xds-certificate.json + path: xds-certificate.json + name: envoy-default-37a8eec1 + optional: false + name: sds + updateStrategy: + type: RollingUpdate +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-topology-spread-constraints.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-topology-spread-constraints.yaml new file mode 100644 index 000000000000..c6d6eee9df20 --- /dev/null +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-topology-spread-constraints.yaml @@ -0,0 +1,311 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + name: envoy-default-37a8eec1 + namespace: envoy-gateway-system +spec: + selector: + matchLabels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + template: + metadata: + annotations: + prometheus.io/path: /stats/prometheus + prometheus.io/port: "19001" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app.kubernetes.io/component: proxy + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy + gateway.envoyproxy.io/owning-gateway-name: default + gateway.envoyproxy.io/owning-gateway-namespace: default + spec: + automountServiceAccountToken: false + containers: + - args: + - --service-cluster default + - --service-node $(ENVOY_POD_NAME) + - | + --config-yaml admin: + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + layered_runtime: + layers: + - name: global_config + static_layer: + envoy.restart_features.use_eds_cache_for_ads: true + re2.max_program_size.error_level: 4294967295 + re2.max_program_size.warn_level: 1000 + dynamic_resources: + ads_config: + api_type: DELTA_GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + set_node_on_first_message_only: true + lds_config: + ads: {} + resource_api_version: V3 + cds_config: + ads: {} + resource_api_version: V3 + static_resources: + listeners: + - name: envoy-gateway-proxy-ready-0.0.0.0-19001 + address: + socket_address: + address: 0.0.0.0 + port_value: 19001 + protocol: TCP + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: eg-ready-http + route_config: + name: local_route + virtual_hosts: + - name: prometheus_stats + domains: + - "*" + routes: + - match: + prefix: /stats/prometheus + route: + cluster: prometheus_stats + http_filters: + - name: envoy.filters.http.health_check + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.health_check.v3.HealthCheck + pass_through_mode: false + headers: + - name: ":path" + string_match: + exact: /ready + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - name: prometheus_stats + connect_timeout: 0.250s + type: STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: prometheus_stats + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + - connect_timeout: 10s + load_assignment: + cluster_name: xds_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18000 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: + connection_keepalive: + interval: 30s + timeout: 5s + name: xds_cluster + type: STRICT_DNS + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 + overload_manager: + refresh_interval: 0.25s + resource_monitors: + - name: "envoy.resource_monitors.global_downstream_max_connections" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig + max_active_downstream_connections: 50000 + - --log-level warn + - --cpuset-threads + command: + - envoy + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/envoy:distroless-dev + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + httpGet: + path: /shutdown/ready + port: 19002 + scheme: HTTP + name: envoy + ports: + - containerPort: 8080 + name: EnvoyH-d76a15e2 + protocol: TCP + - containerPort: 8443 + name: EnvoyH-6658f727 + protocol: TCP + - containerPort: 19001 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /ready + port: 19001 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 100m + memory: 512Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /certs + name: certs + readOnly: true + - mountPath: /sds + name: sds + - args: + - envoy + - shutdown-manager + command: + - envoy-gateway + env: + - name: ENVOY_GATEWAY_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENVOY_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: envoyproxy/gateway-dev:latest + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - envoy-gateway + - envoy + - shutdown + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: shutdown-manager + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 19002 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 10m + memory: 32Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + serviceAccountName: envoy-default-37a8eec1 + terminationGracePeriodSeconds: 900 + topologySpreadConstraints: + - labelSelector: + matchLabels: + app: foo + matchLabelKeys: + - pod-template-hash + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + volumes: + - name: certs + secret: + defaultMode: 420 + secretName: envoy + - configMap: + defaultMode: 420 + items: + - key: xds-trusted-ca.json + path: xds-trusted-ca.json + - key: xds-certificate.json + path: xds-certificate.json + name: envoy-default-37a8eec1 + optional: false + name: sds + updateStrategy: + type: RollingUpdate +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 diff --git a/internal/infrastructure/kubernetes/ratelimit/resource.go b/internal/infrastructure/kubernetes/ratelimit/resource.go index 7e7a9d3722d2..9f6303247cda 100644 --- a/internal/infrastructure/kubernetes/ratelimit/resource.go +++ b/internal/infrastructure/kubernetes/ratelimit/resource.go @@ -284,7 +284,7 @@ func expectedDeploymentVolumes(rateLimit *egv1a1.RateLimit, rateLimitDeployment }) } - return resource.ExpectedDeploymentVolumes(rateLimitDeployment.Pod, volumes) + return resource.ExpectedVolumes(rateLimitDeployment.Pod, volumes) } // expectedRateLimitContainerEnv returns expected rateLimit container envs. diff --git a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go index 885cb4ddca65..62d8e3df4ce3 100644 --- a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go +++ b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go @@ -262,6 +262,11 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { return deployment, nil } +// TODO: implement this method +func (r *ResourceRender) DaemonSet() (*appsv1.DaemonSet, error) { + return nil, nil +} + func (r *ResourceRender) HorizontalPodAutoscaler() (*autoscalingv2.HorizontalPodAutoscaler, error) { return nil, nil } diff --git a/internal/infrastructure/kubernetes/resource/resource.go b/internal/infrastructure/kubernetes/resource/resource.go index bec8257071d5..b949a77cb8da 100644 --- a/internal/infrastructure/kubernetes/resource/resource.go +++ b/internal/infrastructure/kubernetes/resource/resource.go @@ -73,8 +73,8 @@ func ExpectedContainerEnv(container *egv1a1.KubernetesContainerSpec, env []corev return env } -// ExpectedDeploymentVolumes returns expected deployment volumes. -func ExpectedDeploymentVolumes(pod *egv1a1.KubernetesPodSpec, volumes []corev1.Volume) []corev1.Volume { +// ExpectedVolumes returns expected deployment volumes. +func ExpectedVolumes(pod *egv1a1.KubernetesPodSpec, volumes []corev1.Volume) []corev1.Volume { amendFunc := func(volume corev1.Volume) { for index, e := range volumes { if e.Name == volume.Name { diff --git a/internal/infrastructure/kubernetes/resource/resource_test.go b/internal/infrastructure/kubernetes/resource/resource_test.go index 9eb71ce4cedd..667644b12213 100644 --- a/internal/infrastructure/kubernetes/resource/resource_test.go +++ b/internal/infrastructure/kubernetes/resource/resource_test.go @@ -474,7 +474,7 @@ func TestExpectedDeploymentVolumes(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.Equalf(t, tt.want, ExpectedDeploymentVolumes(tt.args.pod, tt.args.volumes), "ExpectedDeploymentVolumes(%v, %v)", tt.args.pod, tt.args.volumes) + assert.Equalf(t, tt.want, ExpectedVolumes(tt.args.pod, tt.args.volumes), "ExpectedVolumes(%v, %v)", tt.args.pod, tt.args.volumes) }) } } diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index a9847ac06451..090ec1ae5d3f 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -1051,6 +1051,7 @@ _Appears in:_ | Field | Type | Required | Description | | --- | --- | --- | --- | | `envoyDeployment` | _[KubernetesDeploymentSpec](#kubernetesdeploymentspec)_ | false | EnvoyDeployment defines the desired state of the Envoy deployment resource.
If unspecified, default settings for the managed Envoy deployment resource
are applied. | +| `envoyDaemonSet` | _[KubernetesDaemonSetSpec](#kubernetesdaemonsetspec)_ | false | EnvoyDaemonSet defines the desired state of the Envoy daemonset resource.
Disabled by default, a deployment resource is used instead to provision the Envoy Proxy fleet | | `envoyService` | _[KubernetesServiceSpec](#kubernetesservicespec)_ | false | EnvoyService defines the desired state of the Envoy service resource.
If unspecified, default settings for the managed Envoy service resource
are applied. | | `envoyHpa` | _[KubernetesHorizontalPodAutoscalerSpec](#kuberneteshorizontalpodautoscalerspec)_ | false | EnvoyHpa defines the Horizontal Pod Autoscaler settings for Envoy Proxy Deployment.
Once the HPA is being set, Replicas field from EnvoyDeployment will be ignored. | @@ -1689,6 +1690,7 @@ _Appears in:_ KubernetesContainerSpec defines the desired state of the Kubernetes container resource. _Appears in:_ +- [KubernetesDaemonSetSpec](#kubernetesdaemonsetspec) - [KubernetesDeploymentSpec](#kubernetesdeploymentspec) | Field | Type | Required | Description | @@ -1700,6 +1702,23 @@ _Appears in:_ | `volumeMounts` | _[VolumeMount](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#volumemount-v1-core) array_ | false | VolumeMounts are volumes to mount into the container's filesystem.
Cannot be updated. | +#### KubernetesDaemonSetSpec + + + +KubernetesDaemonsetSpec defines the desired state of the Kubernetes daemonset resource. + +_Appears in:_ +- [EnvoyProxyKubernetesProvider](#envoyproxykubernetesprovider) + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `patch` | _[KubernetesPatchSpec](#kubernetespatchspec)_ | false | Patch defines how to perform the patch operation to daemonset | +| `strategy` | _[DaemonSetUpdateStrategy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#daemonsetupdatestrategy-v1-apps)_ | false | The daemonset strategy to use to replace existing pods with new ones. | +| `pod` | _[KubernetesPodSpec](#kubernetespodspec)_ | false | Pod defines the desired specification of pod. | +| `container` | _[KubernetesContainerSpec](#kubernetescontainerspec)_ | false | Container defines the desired specification of main container. | + + #### KubernetesDeployMode @@ -1759,6 +1778,7 @@ _Appears in:_ KubernetesPatchSpec defines how to perform the patch operation _Appears in:_ +- [KubernetesDaemonSetSpec](#kubernetesdaemonsetspec) - [KubernetesDeploymentSpec](#kubernetesdeploymentspec) - [KubernetesServiceSpec](#kubernetesservicespec) @@ -1775,6 +1795,7 @@ _Appears in:_ KubernetesPodSpec defines the desired state of the Kubernetes pod resource. _Appears in:_ +- [KubernetesDaemonSetSpec](#kubernetesdaemonsetspec) - [KubernetesDeploymentSpec](#kubernetesdeploymentspec) | Field | Type | Required | Description | diff --git a/test/cel-validation/envoyproxy_test.go b/test/cel-validation/envoyproxy_test.go index 3b2785f713da..33b798048134 100644 --- a/test/cel-validation/envoyproxy_test.go +++ b/test/cel-validation/envoyproxy_test.go @@ -1219,6 +1219,39 @@ func TestEnvoyProxyProvider(t *testing.T) { } }, }, + { + desc: "EnvoyDeployment-and-EnvoyDaemonSet-both-used", + mutate: func(envoy *egv1a1.EnvoyProxy) { + envoy.Spec = egv1a1.EnvoyProxySpec{ + Provider: &egv1a1.EnvoyProxyProvider{ + Type: egv1a1.ProviderTypeKubernetes, + Kubernetes: &egv1a1.EnvoyProxyKubernetesProvider{ + EnvoyDeployment: &egv1a1.KubernetesDeploymentSpec{}, + EnvoyDaemonSet: &egv1a1.KubernetesDaemonSetSpec{}, + }, + }, + } + }, + wantErrors: []string{"only one of envoyDeployment or envoyDaemonSet can be specified"}, + }, + { + desc: "EnvoyHpa-and-EnvoyDaemonSet-both-used", + mutate: func(envoy *egv1a1.EnvoyProxy) { + envoy.Spec = egv1a1.EnvoyProxySpec{ + Provider: &egv1a1.EnvoyProxyProvider{ + Type: egv1a1.ProviderTypeKubernetes, + Kubernetes: &egv1a1.EnvoyProxyKubernetesProvider{ + EnvoyDaemonSet: &egv1a1.KubernetesDaemonSetSpec{}, + EnvoyHpa: &egv1a1.KubernetesHorizontalPodAutoscalerSpec{ + MinReplicas: ptr.To[int32](5), + MaxReplicas: ptr.To[int32](10), + }, + }, + }, + } + }, + wantErrors: []string{"cannot use envoyHpa if envoyDaemonSet is used"}, + }, } for _, tc := range cases {