From 258eecdab4e62e1a7bc015f66ff3395d0cd5d16b Mon Sep 17 00:00:00 2001 From: sh2 Date: Sat, 22 Jun 2024 17:33:18 +0800 Subject: [PATCH 01/23] follow-up: update docs and ci pipeline to utilize gateway-addons-helm (#3610) * update opentelemetry-collector version Signed-off-by: shawnh2 * update makefile cmd to utilize gateway-addons-helm Signed-off-by: shawnh2 * update docs according to gateway-addons-helm Signed-off-by: shawnh2 * correct prometheus server name config Signed-off-by: shawnh2 * fix e2e test labels Signed-off-by: shawnh2 * correct grafana datasource url Signed-off-by: shawnh2 * add template doc Signed-off-by: shawnh2 * specify helm-generate Signed-off-by: shawnh2 * remove tags Signed-off-by: shawnh2 * fix gen-check Signed-off-by: shawnh2 --------- Signed-off-by: shawnh2 --- charts/gateway-addons-helm/Chart.lock | 6 +- charts/gateway-addons-helm/Chart.yaml | 14 +- charts/gateway-addons-helm/README.md | 9 +- charts/gateway-addons-helm/values.yaml | 10 +- examples/fluent-bit/README.md | 8 - examples/fluent-bit/helm-values.yaml | 61 ---- examples/grafana/helm-values.yaml | 12 - examples/loki/loki.yaml | 282 ------------------ examples/otel-collector/helm-values.yaml | 55 ---- examples/prometheus/helm-values.yaml | 24 -- examples/tempo/helm-values.yaml | 2 - .../latest/install/gateway-addons-helm-api.md | 9 +- .../observability/gateway-observability.md | 12 +- .../observability/grafana-integration.md | 20 +- .../observability/proxy-observability.md | 32 +- .../observability/rate-limit-observability.md | 13 +- .../content/en/template/o11y_prerequisites.md | 12 + .../latest/install/gateway-addons-helm-api.md | 9 +- test/e2e/testdata/metric.yaml | 2 +- tools/make/kube.mk | 54 +--- 20 files changed, 42 insertions(+), 604 deletions(-) delete mode 100644 examples/fluent-bit/README.md delete mode 100644 examples/fluent-bit/helm-values.yaml delete mode 100644 examples/grafana/helm-values.yaml delete mode 100644 examples/loki/loki.yaml delete mode 100644 examples/otel-collector/helm-values.yaml delete mode 100644 examples/prometheus/helm-values.yaml delete mode 100644 examples/tempo/helm-values.yaml create mode 100644 site/content/en/template/o11y_prerequisites.md diff --git a/charts/gateway-addons-helm/Chart.lock b/charts/gateway-addons-helm/Chart.lock index 297e11c9610..4b6f92ac77c 100644 --- a/charts/gateway-addons-helm/Chart.lock +++ b/charts/gateway-addons-helm/Chart.lock @@ -16,6 +16,6 @@ dependencies: version: 1.3.1 - name: opentelemetry-collector repository: https://open-telemetry.github.io/opentelemetry-helm-charts - version: 0.60.0 -digest: sha256:52aabfaf2c568f7b77da5cf5a771e936e052df2c66afdfd53a2c892452f06d6b -generated: "2024-06-11T22:19:54.569241+08:00" + version: 0.73.1 +digest: sha256:4c16df8d7efc27aff566fa5dfd2eba6527adbf3fc8e94e7e3ccfc0cee7836f1c +generated: "2024-06-20T11:46:59.148579+08:00" diff --git a/charts/gateway-addons-helm/Chart.yaml b/charts/gateway-addons-helm/Chart.yaml index 3d40fbf7d2d..84ac6228f62 100644 --- a/charts/gateway-addons-helm/Chart.yaml +++ b/charts/gateway-addons-helm/Chart.yaml @@ -29,35 +29,23 @@ dependencies: version: 25.21.0 repository: https://prometheus-community.github.io/helm-charts condition: prometheus.enabled - tags: - - metrics - name: grafana repository: https://grafana.github.io/helm-charts version: 8.0.0 condition: grafana.enabled - tags: - - metrics - name: fluent-bit repository: https://fluent.github.io/helm-charts version: 0.30.4 condition: fluent-bit.enabled - tags: - - logging - name: loki version: 4.8.0 repository: https://grafana.github.io/helm-charts condition: loki.enabled - tags: - - logging - name: tempo repository: https://grafana.github.io/helm-charts version: 1.3.1 condition: tempo.enabled - tags: - - tracing - name: opentelemetry-collector repository: https://open-telemetry.github.io/opentelemetry-helm-charts - version: 0.60.0 + version: 0.73.1 condition: opentelemetry-collector.enabled - tags: - - metrics diff --git a/charts/gateway-addons-helm/README.md b/charts/gateway-addons-helm/README.md index b5d9388d9dc..7d545d0b90f 100644 --- a/charts/gateway-addons-helm/README.md +++ b/charts/gateway-addons-helm/README.md @@ -25,7 +25,7 @@ An Add-ons Helm chart for Envoy Gateway | https://grafana.github.io/helm-charts | grafana | 8.0.0 | | https://grafana.github.io/helm-charts | loki | 4.8.0 | | https://grafana.github.io/helm-charts | tempo | 1.3.1 | -| https://open-telemetry.github.io/opentelemetry-helm-charts | opentelemetry-collector | 0.60.0 | +| https://open-telemetry.github.io/opentelemetry-helm-charts | opentelemetry-collector | 0.73.1 | | https://prometheus-community.github.io/helm-charts | prometheus | 25.21.0 | ## Usage @@ -79,7 +79,7 @@ To uninstall the chart: | grafana.datasources."datasources.yaml".apiVersion | int | `1` | | | grafana.datasources."datasources.yaml".datasources[0].name | string | `"Prometheus"` | | | grafana.datasources."datasources.yaml".datasources[0].type | string | `"prometheus"` | | -| grafana.datasources."datasources.yaml".datasources[0].url | string | `"http://prometheus-server"` | | +| grafana.datasources."datasources.yaml".datasources[0].url | string | `"http://prometheus"` | | | grafana.enabled | bool | `true` | | | grafana.fullnameOverride | string | `"grafana"` | | | grafana.service.type | string | `"LoadBalancer"` | | @@ -125,16 +125,13 @@ To uninstall the chart: | prometheus.kube-state-metrics.enabled | bool | `false` | | | prometheus.prometheus-node-exporter.enabled | bool | `false` | | | prometheus.prometheus-pushgateway.enabled | bool | `false` | | -| prometheus.server.fullnameOverride | string | `"prometheus-server"` | | +| prometheus.server.fullnameOverride | string | `"prometheus"` | | | prometheus.server.global.scrape_interval | string | `"15s"` | | | prometheus.server.image.repository | string | `"prom/prometheus"` | | | prometheus.server.persistentVolume.enabled | bool | `false` | | | prometheus.server.readinessProbeInitialDelay | int | `0` | | | prometheus.server.securityContext | object | `{}` | | | prometheus.server.service.type | string | `"LoadBalancer"` | | -| tags.logging | bool | `false` | | -| tags.metrics | bool | `true` | | -| tags.tracing | bool | `false` | | | tempo.enabled | bool | `true` | | | tempo.fullnameOverride | string | `"tempo"` | | | tempo.service.type | string | `"LoadBalancer"` | | diff --git a/charts/gateway-addons-helm/values.yaml b/charts/gateway-addons-helm/values.yaml index e810eaf0a03..e63e0bdee72 100644 --- a/charts/gateway-addons-helm/values.yaml +++ b/charts/gateway-addons-helm/values.yaml @@ -1,9 +1,3 @@ -tags: - metrics: true - logging: false - tracing: false - - # Values for Grafana dependency grafana: enabled: true @@ -14,7 +8,7 @@ grafana: datasources: - name: Prometheus type: prometheus - url: http://prometheus-server + url: http://prometheus adminPassword: admin service: type: LoadBalancer @@ -47,7 +41,7 @@ prometheus: prometheus-node-exporter: enabled: false server: - fullnameOverride: prometheus-server + fullnameOverride: prometheus persistentVolume: enabled: false readinessProbeInitialDelay: 0 diff --git a/examples/fluent-bit/README.md b/examples/fluent-bit/README.md deleted file mode 100644 index e6221742d6d..00000000000 --- a/examples/fluent-bit/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# Fluent-bit - - -``` -helm repo add fluent https://fluent.github.io/helm-charts -helm repo update -helm upgrade --install fluent-bit fluent/fluent-bit -f examples/fluent-bit/helm-values.yaml -n monitoring --create-namespace --version 0.30.4 -``` \ No newline at end of file diff --git a/examples/fluent-bit/helm-values.yaml b/examples/fluent-bit/helm-values.yaml deleted file mode 100644 index 8e6edf6c50e..00000000000 --- a/examples/fluent-bit/helm-values.yaml +++ /dev/null @@ -1,61 +0,0 @@ -testFramework: - enabled: false -podAnnotations: - prometheus.io/scrape: "true" - prometheus.io/port: "2020" - prometheus.io/path: /api/v1/metrics/prometheus - fluentbit.io/exclude: "true" -## https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/classic-mode/configuration-file -config: - service: | - [SERVICE] - Daemon Off - Flush {{ .Values.flush }} - Log_Level {{ .Values.logLevel }} - Parsers_File parsers.conf - Parsers_File custom_parsers.conf - HTTP_Server On - HTTP_Listen 0.0.0.0 - HTTP_Port {{ .Values.metricsPort }} - Health_Check On - - ## https://docs.fluentbit.io/manual/pipeline/inputs - inputs: | - [INPUT] - Name tail - Path /var/log/containers/*.log - multiline.parser docker, cri - Tag kube.* - Mem_Buf_Limit 5MB - Skip_Long_Lines On - - ## https://docs.fluentbit.io/manual/pipeline/filters - filters: | - [FILTER] - Name kubernetes - Match kube.* - Merge_Log On - Keep_Log Off - K8S-Logging.Parser On - K8S-Logging.Exclude On - - [FILTER] - Name grep - Match kube.* - Regex $kubernetes['container_name'] ^envoy$ - - [FILTER] - Name parser - Match kube.* - Key_Name log - Parser envoy - Reserve_Data True - - ## https://docs.fluentbit.io/manual/pipeline/outputs - outputs: | - [OUTPUT] - Name loki - Match kube.* - Host loki.monitoring.svc.cluster.local - Port 3100 - Labels job=fluentbit, app=$kubernetes['labels']['app'], k8s_namespace_name=$kubernetes['namespace_name'], k8s_pod_name=$kubernetes['pod_name'], k8s_container_name=$kubernetes['container_name'] diff --git a/examples/grafana/helm-values.yaml b/examples/grafana/helm-values.yaml deleted file mode 100644 index 49000b76798..00000000000 --- a/examples/grafana/helm-values.yaml +++ /dev/null @@ -1,12 +0,0 @@ -datasources: - datasources.yaml: - apiVersion: 1 - datasources: - - name: Prometheus - type: prometheus - url: http://prometheus-server - -adminPassword: admin - -service: - type: LoadBalancer diff --git a/examples/loki/loki.yaml b/examples/loki/loki.yaml deleted file mode 100644 index 3f15a75b45e..00000000000 --- a/examples/loki/loki.yaml +++ /dev/null @@ -1,282 +0,0 @@ ---- -# Source: loki/templates/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: loki - labels: - helm.sh/chart: loki-4.8.0 - app.kubernetes.io/name: loki - app.kubernetes.io/instance: loki - app.kubernetes.io/version: "2.7.3" - app.kubernetes.io/managed-by: Helm -automountServiceAccountToken: true ---- -# Source: loki/templates/configmap.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: loki - labels: - helm.sh/chart: loki-4.8.0 - app.kubernetes.io/name: loki - app.kubernetes.io/instance: loki - app.kubernetes.io/version: "2.7.3" - app.kubernetes.io/managed-by: Helm -data: - config.yaml: | - auth_enabled: false - common: - compactor_address: 'loki' - path_prefix: /var/loki - replication_factor: 1 - storage: - filesystem: - chunks_directory: /var/loki/chunks - rules_directory: /var/loki/rules - limits_config: - enforce_metric_name: false - max_cache_freshness_per_query: 10m - reject_old_samples: true - reject_old_samples_max_age: 168h - split_queries_by_interval: 15m - memberlist: - join_members: - - loki-memberlist - query_range: - align_queries_with_step: true - ruler: - storage: - type: local - runtime_config: - file: /etc/loki/runtime-config/runtime-config.yaml - schema_config: - configs: - - from: "2022-01-11" - index: - period: 24h - prefix: loki_index_ - object_store: filesystem - schema: v12 - store: boltdb-shipper - server: - grpc_listen_port: 9095 - http_listen_port: 3100 - storage_config: - hedging: - at: 250ms - max_per_second: 20 - up_to: 3 - table_manager: - retention_deletes_enabled: false - retention_period: 0 ---- -# Source: loki/templates/runtime-configmap.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: loki-runtime - labels: - helm.sh/chart: loki-4.8.0 - app.kubernetes.io/name: loki - app.kubernetes.io/instance: loki - app.kubernetes.io/version: "2.7.3" - app.kubernetes.io/managed-by: Helm -data: - runtime-config.yaml: | - - {} ---- -# Source: loki/templates/service-memberlist.yaml -apiVersion: v1 -kind: Service -metadata: - name: loki-memberlist - labels: - helm.sh/chart: loki-4.8.0 - app.kubernetes.io/name: loki - app.kubernetes.io/instance: loki - app.kubernetes.io/version: "2.7.3" - app.kubernetes.io/managed-by: Helm -spec: - type: ClusterIP - clusterIP: None - ports: - - name: tcp - port: 7946 - targetPort: http-memberlist - protocol: TCP - selector: - app.kubernetes.io/name: loki - app.kubernetes.io/instance: loki - app.kubernetes.io/part-of: memberlist ---- -# Source: loki/templates/single-binary/service-headless.yaml -apiVersion: v1 -kind: Service -metadata: - name: loki-headless - labels: - helm.sh/chart: loki-4.8.0 - app.kubernetes.io/name: loki - app.kubernetes.io/instance: loki - app.kubernetes.io/version: "2.7.3" - app.kubernetes.io/managed-by: Helm - variant: headless - prometheus.io/service-monitor: "false" -spec: - clusterIP: None - ports: - - name: http-metrics - port: 3100 - targetPort: http-metrics - protocol: TCP - selector: - app.kubernetes.io/name: loki - app.kubernetes.io/instance: loki ---- -# Source: loki/templates/single-binary/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: loki - labels: - helm.sh/chart: loki-4.8.0 - app.kubernetes.io/name: loki - app.kubernetes.io/instance: loki - app.kubernetes.io/version: "2.7.3" - app.kubernetes.io/managed-by: Helm -spec: - type: LoadBalancer - ports: - - name: http-metrics - port: 3100 - targetPort: http-metrics - protocol: TCP - - name: grpc - port: 9095 - targetPort: grpc - protocol: TCP - selector: - app.kubernetes.io/name: loki - app.kubernetes.io/instance: loki - app.kubernetes.io/component: single-binary ---- -# Source: loki/templates/single-binary/statefulset.yaml -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: loki - labels: - helm.sh/chart: loki-4.8.0 - app.kubernetes.io/name: loki - app.kubernetes.io/instance: loki - app.kubernetes.io/version: "2.7.3" - app.kubernetes.io/managed-by: Helm - app.kubernetes.io/component: single-binary - app.kubernetes.io/part-of: memberlist -spec: - replicas: 1 - podManagementPolicy: Parallel - updateStrategy: - rollingUpdate: - partition: 0 - serviceName: loki-headless - revisionHistoryLimit: 10 - - persistentVolumeClaimRetentionPolicy: - whenDeleted: Delete - whenScaled: Delete - selector: - matchLabels: - app.kubernetes.io/name: loki - app.kubernetes.io/instance: loki - app.kubernetes.io/component: single-binary - template: - metadata: - annotations: - checksum/config: a9239b6352e34bbfc748669ed46cb24211fc3491ee7f2c6381af805f8f08fe29 - labels: - app.kubernetes.io/name: loki - app.kubernetes.io/instance: loki - app.kubernetes.io/component: single-binary - app.kubernetes.io/part-of: memberlist - spec: - serviceAccountName: loki - automountServiceAccountToken: true - enableServiceLinks: true - - securityContext: - fsGroup: 10001 - runAsGroup: 10001 - runAsNonRoot: true - runAsUser: 10001 - terminationGracePeriodSeconds: 30 - containers: - - name: loki - image: docker.io/grafana/loki:2.7.3 - imagePullPolicy: IfNotPresent - args: - - -config.file=/etc/loki/config/config.yaml - - -target=all - ports: - - name: http-metrics - containerPort: 3100 - protocol: TCP - - name: grpc - containerPort: 9095 - protocol: TCP - - name: http-memberlist - containerPort: 7946 - protocol: TCP - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - readinessProbe: - httpGet: - path: /ready - port: http-metrics - initialDelaySeconds: 30 - timeoutSeconds: 1 - volumeMounts: - - name: tmp - mountPath: /tmp - - name: config - mountPath: /etc/loki/config - - name: runtime-config - mountPath: /etc/loki/runtime-config - - name: storage - mountPath: /var/loki - resources: - {} - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - app.kubernetes.io/name: loki - app.kubernetes.io/instance: loki - app.kubernetes.io/component: single-binary - topologyKey: kubernetes.io/hostname - - volumes: - - name: tmp - emptyDir: {} - - name: config - configMap: - name: loki - - name: runtime-config - configMap: - name: loki-runtime - volumeClaimTemplates: - - metadata: - name: storage - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: "10Gi" diff --git a/examples/otel-collector/helm-values.yaml b/examples/otel-collector/helm-values.yaml deleted file mode 100644 index 465005bdedf..00000000000 --- a/examples/otel-collector/helm-values.yaml +++ /dev/null @@ -1,55 +0,0 @@ -fullnameOverride: otel-collector -mode: deployment -config: - exporters: - prometheus: - endpoint: 0.0.0.0:19001 - logging: - verbosity: detailed - loki: - endpoint: "http://loki.monitoring.svc:3100/loki/api/v1/push" - otlp: - endpoint: tempo.monitoring.svc:4317 - tls: - insecure: true - extensions: - # The health_check extension is mandatory for this chart. - # Without the health_check extension the collector will fail the readiness and liveliness probes. - # The health_check extension can be modified, but should never be removed. - health_check: {} - processors: - attributes: - actions: - - action: insert - key: loki.attribute.labels - # k8s.pod.name is OpenTelemetry format for Kubernetes Pod name, - # Loki will convert this to k8s_pod_name label. - value: k8s.pod.name, k8s.namespace.name - receivers: - otlp: - protocols: - grpc: - endpoint: ${env:MY_POD_IP}:4317 - http: - endpoint: ${env:MY_POD_IP}:4318 - service: - extensions: - - health_check - pipelines: - metrics: - exporters: - - prometheus - receivers: - - otlp - logs: - exporters: - - loki - processors: - - attributes - receivers: - - otlp - traces: - exporters: - - otlp - receivers: - - otlp diff --git a/examples/prometheus/helm-values.yaml b/examples/prometheus/helm-values.yaml deleted file mode 100644 index 7cee4e7bcee..00000000000 --- a/examples/prometheus/helm-values.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# To simplify the deployment, disable non-essential components -alertmanager: - enabled: false -prometheus-pushgateway: - enabled: false -kube-state-metrics: - enabled: false -prometheus-node-exporter: - enabled: false -server: - fullnameOverride: prometheus - persistentVolume: - enabled: false - readinessProbeInitialDelay: 0 - global: - # Speed up scraping a bit from the default - scrape_interval: 15s - service: - # use LoadBalancer to expose prometheus - type: LoadBalancer - # use dockerhub - image: - repository: prom/prometheus - securityContext: null diff --git a/examples/tempo/helm-values.yaml b/examples/tempo/helm-values.yaml deleted file mode 100644 index 42838b76e43..00000000000 --- a/examples/tempo/helm-values.yaml +++ /dev/null @@ -1,2 +0,0 @@ -service: - type: LoadBalancer diff --git a/site/content/en/latest/install/gateway-addons-helm-api.md b/site/content/en/latest/install/gateway-addons-helm-api.md index 4401cc03efe..5401fbe7f83 100644 --- a/site/content/en/latest/install/gateway-addons-helm-api.md +++ b/site/content/en/latest/install/gateway-addons-helm-api.md @@ -27,7 +27,7 @@ An Add-ons Helm chart for Envoy Gateway | https://grafana.github.io/helm-charts | grafana | 8.0.0 | | https://grafana.github.io/helm-charts | loki | 4.8.0 | | https://grafana.github.io/helm-charts | tempo | 1.3.1 | -| https://open-telemetry.github.io/opentelemetry-helm-charts | opentelemetry-collector | 0.60.0 | +| https://open-telemetry.github.io/opentelemetry-helm-charts | opentelemetry-collector | 0.73.1 | | https://prometheus-community.github.io/helm-charts | prometheus | 25.21.0 | ## Values @@ -58,7 +58,7 @@ An Add-ons Helm chart for Envoy Gateway | grafana.datasources."datasources.yaml".apiVersion | int | `1` | | | grafana.datasources."datasources.yaml".datasources[0].name | string | `"Prometheus"` | | | grafana.datasources."datasources.yaml".datasources[0].type | string | `"prometheus"` | | -| grafana.datasources."datasources.yaml".datasources[0].url | string | `"http://prometheus-server"` | | +| grafana.datasources."datasources.yaml".datasources[0].url | string | `"http://prometheus"` | | | grafana.enabled | bool | `true` | | | grafana.fullnameOverride | string | `"grafana"` | | | grafana.service.type | string | `"LoadBalancer"` | | @@ -104,16 +104,13 @@ An Add-ons Helm chart for Envoy Gateway | prometheus.kube-state-metrics.enabled | bool | `false` | | | prometheus.prometheus-node-exporter.enabled | bool | `false` | | | prometheus.prometheus-pushgateway.enabled | bool | `false` | | -| prometheus.server.fullnameOverride | string | `"prometheus-server"` | | +| prometheus.server.fullnameOverride | string | `"prometheus"` | | | prometheus.server.global.scrape_interval | string | `"15s"` | | | prometheus.server.image.repository | string | `"prom/prometheus"` | | | prometheus.server.persistentVolume.enabled | bool | `false` | | | prometheus.server.readinessProbeInitialDelay | int | `0` | | | prometheus.server.securityContext | object | `{}` | | | prometheus.server.service.type | string | `"LoadBalancer"` | | -| tags.logging | bool | `false` | | -| tags.metrics | bool | `true` | | -| tags.tracing | bool | `false` | | | tempo.enabled | bool | `true` | | | tempo.fullnameOverride | string | `"tempo"` | | | tempo.service.type | string | `"LoadBalancer"` | | diff --git a/site/content/en/latest/tasks/observability/gateway-observability.md b/site/content/en/latest/tasks/observability/gateway-observability.md index 745821e3ddd..93ed8114bcc 100644 --- a/site/content/en/latest/tasks/observability/gateway-observability.md +++ b/site/content/en/latest/tasks/observability/gateway-observability.md @@ -7,17 +7,7 @@ This task show you how to config gateway control-plane observability, includes m ## Prerequisites -Follow the steps from the [Quickstart](../quickstart) to install Envoy Gateway and the example manifest. -Before proceeding, you should be able to query the example backend using HTTP. - -[OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) offers a vendor-agnostic implementation of how to receive, process and export telemetry data. -Install OTel-Collector: - -```shell -helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts -helm repo update -helm upgrade --install otel-collector open-telemetry/opentelemetry-collector -f https://raw.githubusercontent.com/envoyproxy/gateway/latest/examples/otel-collector/helm-values.yaml -n monitoring --create-namespace --version 0.60.0 -``` +{{% readfile "../../../template/o11y_prerequisites.md" %}} ## Metrics diff --git a/site/content/en/latest/tasks/observability/grafana-integration.md b/site/content/en/latest/tasks/observability/grafana-integration.md index f86b382278c..e8cbe7e2942 100644 --- a/site/content/en/latest/tasks/observability/grafana-integration.md +++ b/site/content/en/latest/tasks/observability/grafana-integration.md @@ -7,29 +7,11 @@ This task shows you how to visualise the metrics exposed to Prometheus using Gra ## Prerequisites -Follow the steps from the [Quickstart](../../quickstart) to install Envoy Gateway and the example manifest. -Before proceeding, you should be able to query the example backend using HTTP. +{{% readfile "../../../template/o11y_prerequisites.md" %}} Follow the steps from the [Gateway Observability](../gateway-observability) and [Proxy Observability](../proxy-observability#metrics) to enable Prometheus metrics for both Envoy Gateway (Control Plane) and Envoy Proxy (Data Plane). -[Prometheus](https://prometheus.io) is used to scrape metrics from the Envoy Gateway and Envoy Proxy instances. Install Prometheus: - -```shell -helm repo add prometheus-community https://prometheus-community.github.io/helm-charts -helm repo update -helm upgrade --install prometheus prometheus-community/prometheus -n monitoring --create-namespace -``` - -[Grafana](https://grafana.com/grafana/) is used to visualise the metrics exposed by the Envoy Gateway and Envoy Proxy instances. -Install Grafana: - -```shell -helm repo add grafana https://grafana.github.io/helm-charts -helm repo update -helm upgrade --install grafana grafana/grafana -f https://raw.githubusercontent.com/envoyproxy/gateway/latest/examples/grafana/helm-values.yaml -n monitoring --create-namespace -``` - Expose endpoints: ```shell diff --git a/site/content/en/latest/tasks/observability/proxy-observability.md b/site/content/en/latest/tasks/observability/proxy-observability.md index 90cab919325..5f24a76eaa9 100644 --- a/site/content/en/latest/tasks/observability/proxy-observability.md +++ b/site/content/en/latest/tasks/observability/proxy-observability.md @@ -7,38 +7,12 @@ This task show you how to config proxy observability, includes metrics, logs, an ## Prerequisites -Follow the steps from the [Quickstart](../quickstart) to install Envoy Gateway and the example manifest. -Before proceeding, you should be able to query the example backend using HTTP. +{{% readfile "../../../template/o11y_prerequisites.md" %}} -[FluentBit](https://fluentbit.io/) is used to collect logs from the EnvoyProxy instances and forward them to Loki. Install FluentBit: +By default, the Service type of `loki` is ClusterIP, you can change it to LoadBalancer type for further usage: ```shell -helm repo add fluent https://fluent.github.io/helm-charts -helm repo update -helm upgrade --install fluent-bit fluent/fluent-bit -f https://raw.githubusercontent.com/envoyproxy/gateway/latest/examples/fluent-bit/helm-values.yaml -n monitoring --create-namespace --version 0.30.4 -``` - -[Loki](https://grafana.com/oss/loki/) is used to store logs. Install Loki: - -```shell -kubectl apply -f https://raw.githubusercontent.com/envoyproxy/gateway/latest/examples/loki/loki.yaml -n monitoring -``` - -[Tempo](https://grafana.com/oss/tempo/) is used to store traces. Install Tempo: - -```shell -helm repo add grafana https://grafana.github.io/helm-charts -helm repo update -helm upgrade --install tempo grafana/tempo -f https://raw.githubusercontent.com/envoyproxy/gateway/latest/examples/tempo/helm-values.yaml -n monitoring --create-namespace --version 1.3.1 -``` - -[OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) offers a vendor-agnostic implementation of how to receive, process and export telemetry data. -Install OTel-Collector: - -```shell -helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts -helm repo update -helm upgrade --install otel-collector open-telemetry/opentelemetry-collector -f https://raw.githubusercontent.com/envoyproxy/gateway/latest/examples/otel-collector/helm-values.yaml -n monitoring --create-namespace --version 0.60.0 +kubectl patch service loki -n monitoring -p '{"spec": {"type": "LoadBalancer"}}' ``` Expose endpoints: diff --git a/site/content/en/latest/tasks/observability/rate-limit-observability.md b/site/content/en/latest/tasks/observability/rate-limit-observability.md index 478b85859b9..2cad2369d68 100644 --- a/site/content/en/latest/tasks/observability/rate-limit-observability.md +++ b/site/content/en/latest/tasks/observability/rate-limit-observability.md @@ -7,18 +7,9 @@ This guide show you how to config RateLimit observability, includes traces. ## Prerequisites -Follow the steps from the [Quickstart Guide](../quickstart) to install Envoy Gateway and the HTTPRoute example manifest. -Before proceeding, you should be able to query the example backend using HTTP. Follow the steps from the [Global Rate Limit](../traffic/global-rate-limit) to install RateLimit. +{{% readfile "../../../template/o11y_prerequisites.md" %}} -[OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) offers a vendor-agnostic implementation of how to receive, process and export telemetry data. - -Install OTel-Collector: - -```shell -helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts -helm repo update -helm upgrade --install otel-collector open-telemetry/opentelemetry-collector -f https://raw.githubusercontent.com/envoyproxy/gateway/latest/examples/otel-collector/helm-values.yaml -n monitoring --create-namespace --version 0.60.0 -``` +Follow the steps from the [Global Rate Limit](../traffic/global-rate-limit) to install RateLimit. ## Traces diff --git a/site/content/en/template/o11y_prerequisites.md b/site/content/en/template/o11y_prerequisites.md new file mode 100644 index 00000000000..74c547a946c --- /dev/null +++ b/site/content/en/template/o11y_prerequisites.md @@ -0,0 +1,12 @@ + +Follow the steps from the [Quickstart](../quickstart) to install Envoy Gateway and the example manifest. +Before proceeding, you should be able to query the example backend using HTTP. + +Envoy Gateway provides an add-ons Helm Chart, which includes all the needing components for observability. +By default, the [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) is disabled. + +Install the add-ons Helm Chart: + +```shell +helm install eg-addons oci://docker.io/envoyproxy/gateway-addons-helm --version v0.0.0-latest --set opentelemetry-collector.enabled=true -n monitoring --create-namespace +``` diff --git a/site/content/zh/latest/install/gateway-addons-helm-api.md b/site/content/zh/latest/install/gateway-addons-helm-api.md index 4401cc03efe..5401fbe7f83 100644 --- a/site/content/zh/latest/install/gateway-addons-helm-api.md +++ b/site/content/zh/latest/install/gateway-addons-helm-api.md @@ -27,7 +27,7 @@ An Add-ons Helm chart for Envoy Gateway | https://grafana.github.io/helm-charts | grafana | 8.0.0 | | https://grafana.github.io/helm-charts | loki | 4.8.0 | | https://grafana.github.io/helm-charts | tempo | 1.3.1 | -| https://open-telemetry.github.io/opentelemetry-helm-charts | opentelemetry-collector | 0.60.0 | +| https://open-telemetry.github.io/opentelemetry-helm-charts | opentelemetry-collector | 0.73.1 | | https://prometheus-community.github.io/helm-charts | prometheus | 25.21.0 | ## Values @@ -58,7 +58,7 @@ An Add-ons Helm chart for Envoy Gateway | grafana.datasources."datasources.yaml".apiVersion | int | `1` | | | grafana.datasources."datasources.yaml".datasources[0].name | string | `"Prometheus"` | | | grafana.datasources."datasources.yaml".datasources[0].type | string | `"prometheus"` | | -| grafana.datasources."datasources.yaml".datasources[0].url | string | `"http://prometheus-server"` | | +| grafana.datasources."datasources.yaml".datasources[0].url | string | `"http://prometheus"` | | | grafana.enabled | bool | `true` | | | grafana.fullnameOverride | string | `"grafana"` | | | grafana.service.type | string | `"LoadBalancer"` | | @@ -104,16 +104,13 @@ An Add-ons Helm chart for Envoy Gateway | prometheus.kube-state-metrics.enabled | bool | `false` | | | prometheus.prometheus-node-exporter.enabled | bool | `false` | | | prometheus.prometheus-pushgateway.enabled | bool | `false` | | -| prometheus.server.fullnameOverride | string | `"prometheus-server"` | | +| prometheus.server.fullnameOverride | string | `"prometheus"` | | | prometheus.server.global.scrape_interval | string | `"15s"` | | | prometheus.server.image.repository | string | `"prom/prometheus"` | | | prometheus.server.persistentVolume.enabled | bool | `false` | | | prometheus.server.readinessProbeInitialDelay | int | `0` | | | prometheus.server.securityContext | object | `{}` | | | prometheus.server.service.type | string | `"LoadBalancer"` | | -| tags.logging | bool | `false` | | -| tags.metrics | bool | `true` | | -| tags.tracing | bool | `false` | | | tempo.enabled | bool | `true` | | | tempo.fullnameOverride | string | `"tempo"` | | | tempo.service.type | string | `"LoadBalancer"` | | diff --git a/test/e2e/testdata/metric.yaml b/test/e2e/testdata/metric.yaml index 2d2c26311dc..e31c72fdab3 100644 --- a/test/e2e/testdata/metric.yaml +++ b/test/e2e/testdata/metric.yaml @@ -41,7 +41,7 @@ metadata: namespace: monitoring spec: selector: - app.kubernetes.io/instance: otel-collector + app.kubernetes.io/instance: eg-addons app.kubernetes.io/name: opentelemetry-collector component: standalone-collector ports: diff --git a/tools/make/kube.mk b/tools/make/kube.mk index 81d63e9cc25..31ea2c4c6d7 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -10,9 +10,6 @@ GATEWAY_RELEASE_URL ?= https://github.com/kubernetes-sigs/gateway-api/releases/d WAIT_TIMEOUT ?= 15m -FLUENT_BIT_CHART_VERSION ?= 0.30.4 -OTEL_COLLECTOR_CHART_VERSION ?= 0.73.1 -TEMPO_CHART_VERSION ?= 1.3.1 E2E_RUN_TEST ?= E2E_RUN_EG_UPGRADE_TESTS ?= false E2E_CLEANUP ?= true @@ -34,9 +31,9 @@ CONTROLLERGEN_OBJECT_FLAGS := object:headerFile="$(ROOT_DIR)/tools/boilerplate/ .PHONY: manifests manifests: $(tools/controller-gen) generate-gwapi-manifests ## Generate WebhookConfiguration and CustomResourceDefinition objects. - @$(LOG_TARGET) $(tools/controller-gen) crd:allowDangerousTypes=true paths="./..." output:crd:artifacts:config=charts/gateway-helm/crds/generated + .PHONY: generate-gwapi-manifests generate-gwapi-manifests: generate-gwapi-manifests: ## Generate GWAPI manifests and make it consistent with the go mod version. @@ -63,7 +60,7 @@ ifndef ignore-not-found endif .PHONY: kube-deploy -kube-deploy: manifests helm-generate ## Install Envoy Gateway into the Kubernetes cluster specified in ~/.kube/config. +kube-deploy: manifests helm-generate.gateway-helm ## Install Envoy Gateway into the Kubernetes cluster specified in ~/.kube/config. @$(LOG_TARGET) helm install eg charts/gateway-helm --set deployment.envoyGateway.imagePullPolicy=$(IMAGE_PULL_POLICY) -n envoy-gateway-system --create-namespace --debug --timeout='$(WAIT_TIMEOUT)' --wait --wait-for-jobs @@ -142,54 +139,17 @@ endif endif .PHONY: install-e2e-telemetry -install-e2e-telemetry: prepare-helm-repo install-fluent-bit install-loki install-tempo install-otel-collector install-prometheus +install-e2e-telemetry: helm-generate.gateway-addons-helm @$(LOG_TARGET) - kubectl rollout status daemonset fluent-bit -n monitoring --timeout 5m - kubectl rollout status statefulset loki -n monitoring --timeout 5m - kubectl rollout status statefulset tempo -n monitoring --timeout 5m - kubectl rollout status deployment otel-collector -n monitoring --timeout 5m - kubectl rollout status deployment prometheus -n monitoring --timeout 5m + helm install eg-addons charts/gateway-addons-helm --set grafana.enabled=false,opentelemetry-collector.enabled=true -n monitoring --create-namespace --timeout='$(WAIT_TIMEOUT)' --wait --wait-for-jobs + # Change loki service type from ClusterIP to LoadBalancer + kubectl patch service loki -n monitoring -p '{"spec": {"type": "LoadBalancer"}}' .PHONY: uninstall-e2e-telemetry uninstall-e2e-telemetry: @$(LOG_TARGET) - kubectl delete -f examples/loki/loki.yaml -n monitoring --ignore-not-found helm delete $(shell helm list -n monitoring -q) -n monitoring -.PHONY: prepare-helm-repo -prepare-helm-repo: - @$(LOG_TARGET) - helm repo add fluent https://fluent.github.io/helm-charts - helm repo add grafana https://grafana.github.io/helm-charts - helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts - helm repo add prometheus-community https://prometheus-community.github.io/helm-charts - helm repo update - -.PHONY: install-fluent-bit -install-fluent-bit: - @$(LOG_TARGET) - helm upgrade --install fluent-bit fluent/fluent-bit -f examples/fluent-bit/helm-values.yaml -n monitoring --create-namespace --version $(FLUENT_BIT_CHART_VERSION) - -.PHONY: install-loki -install-loki: - @$(LOG_TARGET) - kubectl apply -f examples/loki/loki.yaml -n monitoring - -.PHONY: install-tempo -install-tempo: - @$(LOG_TARGET) - helm upgrade --install tempo grafana/tempo -f examples/tempo/helm-values.yaml -n monitoring --create-namespace --version $(TEMPO_CHART_VERSION) - -.PHONY: install-prometheus -install-prometheus: - @$(LOG_TARGET) - helm upgrade --install prometheus prometheus-community/prometheus -f examples/prometheus/helm-values.yaml -n monitoring --create-namespace - -.PHONY: install-otel-collector -install-otel-collector: - @$(LOG_TARGET) - helm upgrade --install otel-collector open-telemetry/opentelemetry-collector -f examples/otel-collector/helm-values.yaml -n monitoring --create-namespace --version $(OTEL_COLLECTOR_CHART_VERSION) - .PHONY: create-cluster create-cluster: $(tools/kind) ## Create a kind cluster suitable for running Gateway API conformance. @$(LOG_TARGET) @@ -222,7 +182,7 @@ delete-cluster: $(tools/kind) ## Delete kind cluster. $(tools/kind) delete cluster --name envoy-gateway .PHONY: generate-manifests -generate-manifests: helm-generate ## Generate Kubernetes release manifests. +generate-manifests: helm-generate.gateway-helm ## Generate Kubernetes release manifests. @$(LOG_TARGET) @$(call log, "Generating kubernetes manifests") mkdir -p $(OUTPUT_DIR)/ From 9830c4d50fd0297b953f18a0a05e2fda41ecf06a Mon Sep 17 00:00:00 2001 From: Guy Daich Date: Sun, 23 Jun 2024 21:11:11 -0500 Subject: [PATCH 02/23] fix: add retries to ext-proc tests (#3641) use converging assertion Signed-off-by: Guy Daich --- test/e2e/tests/ext_proc.go | 80 ++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 46 deletions(-) diff --git a/test/e2e/tests/ext_proc.go b/test/e2e/tests/ext_proc.go index 45cda23c0c3..b4bd96d453e 100644 --- a/test/e2e/tests/ext_proc.go +++ b/test/e2e/tests/ext_proc.go @@ -56,9 +56,17 @@ var ExtProcTest = suite.ConformanceTest{ Host: "www.example.com", Path: "/processor", Headers: map[string]string{ - "x-request-ext-processed": "true", // header added by ext-processor to backend-bound request - "x-request-client-header-received": "original", // this is the original client header preserved by ext-proc in a new header - "x-request-client-header": "mutated", // this is the mutated value expected to reach upstream + "x-request-client-header": "original", // add a request header that will be mutated by ext-proc + }, + }, + ExpectedRequest: &http.ExpectedRequest{ + Request: http.Request{ + Path: "/processor", + Headers: map[string]string{ + "x-request-ext-processed": "true", // header added by ext-processor to backend-bound request + "x-request-client-header-received": "original", // this is the original client header preserved by ext-proc in a new header + "x-request-client-header": "mutated", // this is the mutated value expected to reach upstream + }, }, }, Response: http.Response{ @@ -70,19 +78,7 @@ var ExtProcTest = suite.ConformanceTest{ Namespace: ns, } - req := http.MakeRequest(t, &expectedResponse, gwAddr, "HTTP", "http") - - // add a request header that will be mutated by ext-proc - req.Headers["x-request-client-header"] = []string{"original"} - - cReq, cResp, err := suite.RoundTripper.CaptureRoundTrip(req) - if err != nil { - t.Errorf("failed to get expected response: %v", err) - } - - if err := http.CompareRequest(t, &req, cReq, cResp, expectedResponse); err != nil { - t.Errorf("failed to compare request and response: %v", err) - } + http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, expectedResponse) }) t.Run("http route without proc mode", func(t *testing.T) { @@ -109,7 +105,15 @@ var ExtProcTest = suite.ConformanceTest{ Host: "www.example.com", Path: "/no-processor", Headers: map[string]string{ - "x-request-client-header": "original", + "x-request-client-header": "original", // add a request header that will be mutated by ext-proc + }, + }, + ExpectedRequest: &http.ExpectedRequest{ + Request: http.Request{ + Path: "/no-processor", + Headers: map[string]string{ + "x-request-client-header": "original", // this is the original value expected to reach upstream + }, }, }, Response: http.Response{ @@ -119,19 +123,7 @@ var ExtProcTest = suite.ConformanceTest{ Namespace: ns, } - req := http.MakeRequest(t, &expectedResponse, gwAddr, "HTTP", "http") - - // add a request header that will be mutated by ext-proc if the request headers are sent - req.Headers["x-request-client-header"] = []string{"original"} - - cReq, cResp, err := suite.RoundTripper.CaptureRoundTrip(req) - if err != nil { - t.Errorf("failed to get expected response: %v", err) - } - - if err := http.CompareRequest(t, &req, cReq, cResp, expectedResponse); err != nil { - t.Errorf("failed to compare request and response: %v", err) - } + http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, expectedResponse) }) t.Run("http route with uds ext proc", func(t *testing.T) { @@ -158,9 +150,17 @@ var ExtProcTest = suite.ConformanceTest{ Host: "www.example.com", Path: "/uds-processor", Headers: map[string]string{ - "x-request-ext-processed": "true", // header added by ext-processor to backend-bound request - "x-request-client-header-received": "original", // this is the original client header preserved by ext-proc in a new header - "x-request-client-header": "mutated", // this is the mutated value expected to reach upstream + "x-request-client-header": "original", // add a request header that will be mutated by ext-proc + }, + }, + ExpectedRequest: &http.ExpectedRequest{ + Request: http.Request{ + Path: "/uds-processor", + Headers: map[string]string{ + "x-request-ext-processed": "true", // header added by ext-processor to backend-bound request + "x-request-client-header-received": "original", // this is the original client header preserved by ext-proc in a new header + "x-request-client-header": "mutated", // this is the mutated value expected to reach upstream + }, }, }, Response: http.Response{ @@ -172,19 +172,7 @@ var ExtProcTest = suite.ConformanceTest{ Namespace: ns, } - req := http.MakeRequest(t, &expectedResponse, gwAddr, "HTTP", "http") - - // add a request header that will be mutated by ext-proc - req.Headers["x-request-client-header"] = []string{"original"} - - cReq, cResp, err := suite.RoundTripper.CaptureRoundTrip(req) - if err != nil { - t.Errorf("failed to get expected response: %v", err) - } - - if err := http.CompareRequest(t, &req, cReq, cResp, expectedResponse); err != nil { - t.Errorf("failed to compare request and response: %v", err) - } + http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, expectedResponse) }) }, } From 51196b46379d1a03c619bc8fe3cb347d5e24cc27 Mon Sep 17 00:00:00 2001 From: Alex Marston Date: Mon, 24 Jun 2024 17:51:08 +0100 Subject: [PATCH 03/23] api: Adding Zipkin Tracing support (#3630) * api: Adding Zipkin Tracing support Signed-off-by: Alex Marston * rollback /internal changes Signed-off-by: Alex Marston * nest zipkin configuration under TracingProvider struct Signed-off-by: Alex Marston * rollback internal testdata changes Signed-off-by: Alex Marston * rollback site changes Signed-off-by: Alex Marston * Change ZipkinConfiguration to a pointer. Co-authored-by: zirain Signed-off-by: Alex Marston * update deepcopy Signed-off-by: Alex Marston * update description for TraceId128Bit field Signed-off-by: Alex Marston * update site docs Signed-off-by: Alex Marston * rename Zipkin config struct Signed-off-by: Alex Marston * update api Signed-off-by: Alex Marston * pointers for optional fields Signed-off-by: Alex Marston * remove CollectorHostname Signed-off-by: Alex Marston * TraceID128Bit -> Enable128BitTraceID Signed-off-by: Alex Marston --------- Signed-off-by: Alex Marston Signed-off-by: Alex Marston Co-authored-by: zirain --- api/v1alpha1/tracing_types.go | 21 +++++++++++-- api/v1alpha1/zz_generated.deepcopy.go | 30 +++++++++++++++++++ .../gateway.envoyproxy.io_envoyproxies.yaml | 25 ++++++++++++---- site/content/en/latest/api/extension_types.md | 21 +++++++++++-- site/content/zh/latest/api/extension_types.md | 21 +++++++++++-- 5 files changed, 105 insertions(+), 13 deletions(-) diff --git a/api/v1alpha1/tracing_types.go b/api/v1alpha1/tracing_types.go index 1b8b55edc47..b7be478de15 100644 --- a/api/v1alpha1/tracing_types.go +++ b/api/v1alpha1/tracing_types.go @@ -18,7 +18,6 @@ type ProxyTracing struct { // If provider is kubernetes, pod name and namespace are added by default. CustomTags map[string]CustomTag `json:"customTags,omitempty"` // Provider defines the tracing provider. - // Only OpenTelemetry is supported currently. Provider TracingProvider `json:"provider"` } @@ -26,6 +25,7 @@ type TracingProviderType string const ( TracingProviderTypeOpenTelemetry TracingProviderType = "OpenTelemetry" + TracingProviderTypeZipkin TracingProviderType = "Zipkin" ) // TracingProvider defines the tracing provider configuration. @@ -33,8 +33,7 @@ const ( // +kubebuilder:validation:XValidation:message="host or backendRefs needs to be set",rule="has(self.host) || self.backendRefs.size() > 0" type TracingProvider struct { // Type defines the tracing provider type. - // EG currently only supports OpenTelemetry. - // +kubebuilder:validation:Enum=OpenTelemetry + // +kubebuilder:validation:Enum=OpenTelemetry;Zipkin // +kubebuilder:default=OpenTelemetry Type TracingProviderType `json:"type"` // Host define the provider service hostname. @@ -58,6 +57,9 @@ type TracingProvider struct { // +kubebuilder:validation:XValidation:message="only support Service kind.",rule="self.all(f, f.kind == 'Service')" // +kubebuilder:validation:XValidation:message="BackendRefs only supports Core group.",rule="self.all(f, f.group == '')" BackendRefs []BackendRef `json:"backendRefs,omitempty"` + // Zipkin defines the Zipkin tracing provider configuration + // +optional + Zipkin *ZipkinTracingProvider `json:"zipkin,omitempty"` } type CustomTagType string @@ -114,3 +116,16 @@ type RequestHeaderCustomTag struct { // +optional DefaultValue *string `json:"defaultValue,omitempty"` } + +// ZipkinTracingProvider defines the Zipkin tracing provider configuration. +type ZipkinTracingProvider struct { + // Enable128BitTraceID determines whether a 128bit trace id will be used + // when creating a new trace instance. If set to false, a 64bit trace + // id will be used. + // +optional + Enable128BitTraceID *bool `json:"enable128BitTraceId,omitempty"` + // DisableSharedSpanContext determines whether the default Envoy behaviour of + // client and server spans sharing the same span context should be disabled. + // +optional + DisableSharedSpanContext *bool `json:"disableSharedSpanContext,omitempty"` +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 1c3ae25c430..e1442dd636a 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -4725,6 +4725,11 @@ func (in *TracingProvider) DeepCopyInto(out *TracingProvider) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Zipkin != nil { + in, out := &in.Zipkin, &out.Zipkin + *out = new(ZipkinTracingProvider) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TracingProvider. @@ -4877,3 +4882,28 @@ func (in *XForwardedForSettings) DeepCopy() *XForwardedForSettings { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ZipkinTracingProvider) DeepCopyInto(out *ZipkinTracingProvider) { + *out = *in + if in.Enable128BitTraceID != nil { + in, out := &in.Enable128BitTraceID, &out.Enable128BitTraceID + *out = new(bool) + **out = **in + } + if in.DisableSharedSpanContext != nil { + in, out := &in.DisableSharedSpanContext, &out.DisableSharedSpanContext + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ZipkinTracingProvider. +func (in *ZipkinTracingProvider) DeepCopy() *ZipkinTracingProvider { + if in == nil { + return nil + } + out := new(ZipkinTracingProvider) + in.DeepCopyInto(out) + return out +} diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml index 2da9760560f..f40ff9b0e96 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -10864,9 +10864,7 @@ spec: If provider is kubernetes, pod name and namespace are added by default. type: object provider: - description: |- - Provider defines the tracing provider. - Only OpenTelemetry is supported currently. + description: Provider defines the tracing provider. properties: backendRefs: description: |- @@ -10972,12 +10970,27 @@ spec: type: integer type: default: OpenTelemetry - description: |- - Type defines the tracing provider type. - EG currently only supports OpenTelemetry. + description: Type defines the tracing provider type. enum: - OpenTelemetry + - Zipkin type: string + zipkin: + description: Zipkin defines the Zipkin tracing provider + configuration + properties: + disableSharedSpanContext: + description: |- + DisableSharedSpanContext determines whether the default Envoy behaviour of + client and server spans sharing the same span context should be disabled. + type: boolean + enable128BitTraceId: + description: |- + Enable128BitTraceID determines whether a 128bit trace id will be used + when creating a new trace instance. If set to false, a 64bit trace + id will be used. + type: boolean + type: object required: - type type: object diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index 57b24c2b5b9..258d99f27fe 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -2910,7 +2910,7 @@ _Appears in:_ | --- | --- | --- | --- | | `samplingRate` | _integer_ | false | SamplingRate controls the rate at which traffic will be
selected for tracing if no prior sampling decision has been made.
Defaults to 100, valid values [0-100]. 100 indicates 100% sampling. | | `customTags` | _object (keys:string, values:[CustomTag](#customtag))_ | true | CustomTags defines the custom tags to add to each span.
If provider is kubernetes, pod name and namespace are added by default. | -| `provider` | _[TracingProvider](#tracingprovider)_ | true | Provider defines the tracing provider.
Only OpenTelemetry is supported currently. | +| `provider` | _[TracingProvider](#tracingprovider)_ | true | Provider defines the tracing provider. | #### RateLimit @@ -3564,10 +3564,11 @@ _Appears in:_ | Field | Type | Required | Description | | --- | --- | --- | --- | -| `type` | _[TracingProviderType](#tracingprovidertype)_ | true | Type defines the tracing provider type.
EG currently only supports OpenTelemetry. | +| `type` | _[TracingProviderType](#tracingprovidertype)_ | true | Type defines the tracing provider type. | | `host` | _string_ | false | Host define the provider service hostname.
Deprecated: Use BackendRefs instead. | | `port` | _integer_ | false | Port defines the port the provider service is exposed on.
Deprecated: Use BackendRefs instead. | | `backendRefs` | _[BackendRef](#backendref) array_ | false | BackendRefs references a Kubernetes object that represents the
backend server to which the trace will be sent.
Only Service kind is supported for now. | +| `zipkin` | _[ZipkinTracingProvider](#zipkintracingprovider)_ | false | Zipkin defines the Zipkin tracing provider configuration | #### TracingProviderType @@ -3583,6 +3584,7 @@ _Appears in:_ | ----- | ----------- | | `OpenTelemetry` | | | `OpenTelemetry` | | +| `Zipkin` | | #### TriggerEnum @@ -3795,3 +3797,18 @@ _Appears in:_ | `numTrustedHops` | _integer_ | false | NumTrustedHops controls the number of additional ingress proxy hops from the right side of XFF HTTP
headers to trust when determining the origin client's IP address.
Refer to https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers#x-forwarded-for
for more details. | +#### ZipkinTracingProvider + + + +ZipkinTracingProvider defines the Zipkin tracing provider configuration. + +_Appears in:_ +- [TracingProvider](#tracingprovider) + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `enable128BitTraceId` | _boolean_ | false | Enable128BitTraceID determines whether a 128bit trace id will be used
when creating a new trace instance. If set to false, a 64bit trace
id will be used. | +| `disableSharedSpanContext` | _boolean_ | false | DisableSharedSpanContext determines whether the default Envoy behaviour of
client and server spans sharing the same span context should be disabled. | + + diff --git a/site/content/zh/latest/api/extension_types.md b/site/content/zh/latest/api/extension_types.md index 57b24c2b5b9..258d99f27fe 100644 --- a/site/content/zh/latest/api/extension_types.md +++ b/site/content/zh/latest/api/extension_types.md @@ -2910,7 +2910,7 @@ _Appears in:_ | --- | --- | --- | --- | | `samplingRate` | _integer_ | false | SamplingRate controls the rate at which traffic will be
selected for tracing if no prior sampling decision has been made.
Defaults to 100, valid values [0-100]. 100 indicates 100% sampling. | | `customTags` | _object (keys:string, values:[CustomTag](#customtag))_ | true | CustomTags defines the custom tags to add to each span.
If provider is kubernetes, pod name and namespace are added by default. | -| `provider` | _[TracingProvider](#tracingprovider)_ | true | Provider defines the tracing provider.
Only OpenTelemetry is supported currently. | +| `provider` | _[TracingProvider](#tracingprovider)_ | true | Provider defines the tracing provider. | #### RateLimit @@ -3564,10 +3564,11 @@ _Appears in:_ | Field | Type | Required | Description | | --- | --- | --- | --- | -| `type` | _[TracingProviderType](#tracingprovidertype)_ | true | Type defines the tracing provider type.
EG currently only supports OpenTelemetry. | +| `type` | _[TracingProviderType](#tracingprovidertype)_ | true | Type defines the tracing provider type. | | `host` | _string_ | false | Host define the provider service hostname.
Deprecated: Use BackendRefs instead. | | `port` | _integer_ | false | Port defines the port the provider service is exposed on.
Deprecated: Use BackendRefs instead. | | `backendRefs` | _[BackendRef](#backendref) array_ | false | BackendRefs references a Kubernetes object that represents the
backend server to which the trace will be sent.
Only Service kind is supported for now. | +| `zipkin` | _[ZipkinTracingProvider](#zipkintracingprovider)_ | false | Zipkin defines the Zipkin tracing provider configuration | #### TracingProviderType @@ -3583,6 +3584,7 @@ _Appears in:_ | ----- | ----------- | | `OpenTelemetry` | | | `OpenTelemetry` | | +| `Zipkin` | | #### TriggerEnum @@ -3795,3 +3797,18 @@ _Appears in:_ | `numTrustedHops` | _integer_ | false | NumTrustedHops controls the number of additional ingress proxy hops from the right side of XFF HTTP
headers to trust when determining the origin client's IP address.
Refer to https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers#x-forwarded-for
for more details. | +#### ZipkinTracingProvider + + + +ZipkinTracingProvider defines the Zipkin tracing provider configuration. + +_Appears in:_ +- [TracingProvider](#tracingprovider) + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `enable128BitTraceId` | _boolean_ | false | Enable128BitTraceID determines whether a 128bit trace id will be used
when creating a new trace instance. If set to false, a 64bit trace
id will be used. | +| `disableSharedSpanContext` | _boolean_ | false | DisableSharedSpanContext determines whether the default Envoy behaviour of
client and server spans sharing the same span context should be disabled. | + + From abe71c9d8c292dcc2672ef907a4e7746a4a7d1a1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Jun 2024 12:54:48 -0700 Subject: [PATCH 04/23] build(deps): bump fortio.org/fortio from 1.63.10 to 1.65.0 (#3658) Bumps [fortio.org/fortio](https://github.com/fortio/fortio) from 1.63.10 to 1.65.0. - [Release notes](https://github.com/fortio/fortio/releases) - [Commits](https://github.com/fortio/fortio/compare/v1.63.10...v1.65.0) --- updated-dependencies: - dependency-name: fortio.org/fortio dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 13 +++++++------ go.sum | 26 ++++++++++++++------------ 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 6244179d866..84de1ce156c 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/envoyproxy/gateway go 1.22.4 require ( - fortio.org/fortio v1.63.10 + fortio.org/fortio v1.65.0 fortio.org/log v1.12.2 github.com/Masterminds/semver/v3 v3.2.1 github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b @@ -37,7 +37,7 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.27.0 go.opentelemetry.io/proto/otlp v1.3.1 go.uber.org/zap v1.27.0 - golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 + golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 golang.org/x/sys v0.21.0 google.golang.org/grpc v1.64.0 google.golang.org/protobuf v1.34.2 @@ -58,11 +58,11 @@ require ( require ( cel.dev/expr v0.15.0 // indirect - fortio.org/cli v1.5.2 // indirect + fortio.org/cli v1.6.0 // indirect fortio.org/dflag v1.7.2 // indirect - fortio.org/scli v1.14.3 // indirect + fortio.org/scli v1.15.0 // indirect fortio.org/sets v1.1.1 // indirect - fortio.org/struct2env v0.4.0 // indirect + fortio.org/struct2env v0.4.1 // indirect fortio.org/version v1.0.4 // indirect github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect github.com/BurntSushi/toml v1.3.2 // indirect @@ -113,6 +113,7 @@ require ( github.com/xeipuuv/gojsonschema v1.2.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect golang.org/x/crypto v0.24.0 // indirect + golang.org/x/crypto/x509roots/fallback v0.0.0-20240604170348-d4e7c9cb6cb8 // indirect k8s.io/apiserver v0.30.2 // indirect oras.land/oras-go v1.2.4 // indirect ) @@ -184,7 +185,7 @@ require ( golang.org/x/tools v0.22.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/component-base v0.30.2 // indirect diff --git a/go.sum b/go.sum index 3fac7625e21..4381300c98f 100644 --- a/go.sum +++ b/go.sum @@ -5,20 +5,20 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= fortio.org/assert v1.2.1 h1:48I39urpeDj65RP1KguF7akCjILNeu6vICiYMEysR7Q= fortio.org/assert v1.2.1/go.mod h1:039mG+/iYDPO8Ibx8TrNuJCm2T2SuhwRI3uL9nHTTls= -fortio.org/cli v1.5.2 h1:MfEcHAhYyIkwG04/K1YJL946Y8/jyAjmF9WeR5ZG/5E= -fortio.org/cli v1.5.2/go.mod h1:SdQufh5PLd6oX2EtvtzLFw++gw8zVoejD1WlwGXAvYw= +fortio.org/cli v1.6.0 h1:EX9zf+BLzgE+yrq2a3XFZz2F8CK1g9ecJj9ZXVOfoww= +fortio.org/cli v1.6.0/go.mod h1:QSCd+8OD3MrFKo2XwAHVJJu5gz/U0Jg1Vhse/4nHn3I= fortio.org/dflag v1.7.2 h1:lUhXFvDlw4CJj/q7hPv/TC+n/wVoQylzQO6bUg5GQa0= fortio.org/dflag v1.7.2/go.mod h1:6yO/NIgrWfQH195WbHJ3Y45SCx11ffivQjfx2C/FS1U= -fortio.org/fortio v1.63.10 h1:8LoN24Dr2ktNbvuUjvCBNwWGBJpeR3gzOsq4NXnQX4c= -fortio.org/fortio v1.63.10/go.mod h1:ruHDZPCFdh8+Q8y8VCVl30LYNADDwq6IIa69/ISx62I= +fortio.org/fortio v1.65.0 h1:HQVyJrxYT4GXmwQe9vHJN9evmbqVPqKxhQac7qWQ+QA= +fortio.org/fortio v1.65.0/go.mod h1:R9jL6u4zKkQHO8HvuJxdWGmDg02w+2kouXKza/R3eaU= fortio.org/log v1.12.2 h1:JwLDFvEUKGfqA09fcf+mOn8kxsvwhjXV92xghxNnnwA= fortio.org/log v1.12.2/go.mod h1:1tMBG/Elr6YqjmJCWiejJp2FPvXg7/9UAN0Rfpkyt1o= -fortio.org/scli v1.14.3 h1:iYEmpr2wcRFLmWd4DsN/8HQr3YVlR4VeEDf1mLDIIXE= -fortio.org/scli v1.14.3/go.mod h1:yS8UTD/JXrDY0GfXerp5LFZrMUuNwN5iFR28b6oVXVc= +fortio.org/scli v1.15.0 h1:2LSnphdc3NGLHRD236/yINgrTTqvaPRmVhaK55V7wKo= +fortio.org/scli v1.15.0/go.mod h1:dFpj7h+mpMmsMYm9Bf/buVDGkpDysugl9gVemLDEsAo= fortio.org/sets v1.1.1 h1:Q7Z1Ft2lpUc1N7bfI8HofIK0QskrOflfYRyKT2LzBng= fortio.org/sets v1.1.1/go.mod h1:J2BwIxNOLWsSU7IMZUg541kh3Au4JEKHrghVwXs68tE= -fortio.org/struct2env v0.4.0 h1:k5alSOTf3YHiB3MuacjDHQ3YhVWvNZ95ZP/a6MqvyLo= -fortio.org/struct2env v0.4.0/go.mod h1:lENUe70UwA1zDUCX+8AsO663QCFqYaprk5lnPhjD410= +fortio.org/struct2env v0.4.1 h1:rJludAMO5eBvpWplWEQNqoVDFZr4RWMQX7RUapgZyc0= +fortio.org/struct2env v0.4.1/go.mod h1:lENUe70UwA1zDUCX+8AsO663QCFqYaprk5lnPhjD410= fortio.org/version v1.0.4 h1:FWUMpJ+hVTNc4RhvvOJzb0xesrlRmG/a+D6bjbQ4+5U= fortio.org/version v1.0.4/go.mod h1:2JQp9Ax+tm6QKiGuzR5nJY63kFeANcgrZ0osoQFDVm0= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= @@ -726,9 +726,11 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto/x509roots/fallback v0.0.0-20240604170348-d4e7c9cb6cb8 h1:+kWDWI3Eb5cPIOr4cP+R2RLDwK3/dXppL+7XmSOh2LA= +golang.org/x/crypto/x509roots/fallback v0.0.0-20240604170348-d4e7c9cb6cb8/go.mod h1:kNa9WdvYnzFwC79zRpLRMJbdEFlhyM5RPFBBZp/wWH8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 h1:LoYXNGAShUG3m/ehNk4iFctuhGX/+R1ZpfJ4/ia80JM= -golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -872,8 +874,8 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 h1:P8OJ/WCl/Xo4E4zoe4/bifHpSmmKwARqyqE4nW6J2GQ= google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5/go.mod h1:RGnPtTG7r4i8sPlNyDeikXF99hMM+hN6QMm4ooG9g2g= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e h1:Elxv5MwEkCI9f5SkoL6afed6NTdxaGoAo39eANBwHL8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 h1:Di6ANFilr+S60a4S61ZM00vLdw0IrQOSMS2/6mrnOU0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= From 7ea294248f6e8edd66b042f26331db1ef571828d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Jun 2024 12:56:22 -0700 Subject: [PATCH 05/23] build(deps): bump softprops/action-gh-release from 2.0.5 to 2.0.6 (#3656) Bumps [softprops/action-gh-release](https://github.com/softprops/action-gh-release) from 2.0.5 to 2.0.6. - [Release notes](https://github.com/softprops/action-gh-release/releases) - [Changelog](https://github.com/softprops/action-gh-release/blob/master/CHANGELOG.md) - [Commits](https://github.com/softprops/action-gh-release/compare/69320dbe05506a9a39fc8ae11030b214ec2d1f87...a74c6b72af54cfa997e81df42d94703d6313a2d0) --- updated-dependencies: - dependency-name: softprops/action-gh-release dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/latest_release.yaml | 2 +- .github/workflows/release.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/latest_release.yaml b/.github/workflows/latest_release.yaml index e6e45463a2b..66ebc5def70 100644 --- a/.github/workflows/latest_release.yaml +++ b/.github/workflows/latest_release.yaml @@ -62,7 +62,7 @@ jobs: GITHUB_REPOSITORY: ${{ github.repository_owner }}/${{ github.event.repository.name }} - name: Recreate the Latest Release and Tag - uses: softprops/action-gh-release@69320dbe05506a9a39fc8ae11030b214ec2d1f87 # v0.1.15 + uses: softprops/action-gh-release@a74c6b72af54cfa997e81df42d94703d6313a2d0 # v0.1.15 with: draft: false prerelease: true diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 088bd2a614f..679c6184e39 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -40,7 +40,7 @@ jobs: run: IMAGE_PULL_POLICY=IfNotPresent OCI_REGISTRY=oci://docker.io/envoyproxy CHART_VERSION=${{ env.release_tag }} IMAGE=docker.io/envoyproxy/gateway TAG=${{ env.release_tag }} make helm-package helm-push - name: Upload Release Manifests - uses: softprops/action-gh-release@69320dbe05506a9a39fc8ae11030b214ec2d1f87 # v0.1.15 + uses: softprops/action-gh-release@a74c6b72af54cfa997e81df42d94703d6313a2d0 # v0.1.15 with: files: | release-artifacts/install.yaml From 9a8149287cb757e8c0ebfd5c91f845a4f0141f1f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Jun 2024 12:59:08 -0700 Subject: [PATCH 06/23] build(deps): bump helm.sh/helm/v3 from 3.15.1 to 3.15.2 (#3657) Bumps [helm.sh/helm/v3](https://github.com/helm/helm) from 3.15.1 to 3.15.2. - [Release notes](https://github.com/helm/helm/releases) - [Commits](https://github.com/helm/helm/compare/v3.15.1...v3.15.2) --- updated-dependencies: - dependency-name: helm.sh/helm/v3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 17 ++++++++--------- go.sum | 34 ++++++++++++++++------------------ 2 files changed, 24 insertions(+), 27 deletions(-) diff --git a/go.mod b/go.mod index 84de1ce156c..ce96653d303 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( google.golang.org/grpc v1.64.0 google.golang.org/protobuf v1.34.2 gopkg.in/yaml.v3 v3.0.1 - helm.sh/helm/v3 v3.15.1 + helm.sh/helm/v3 v3.15.2 k8s.io/api v0.30.2 k8s.io/apiextensions-apiserver v0.30.2 k8s.io/apimachinery v0.30.2 @@ -74,13 +74,13 @@ require ( github.com/containerd/containerd v1.7.12 // indirect github.com/containerd/log v0.1.0 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect - github.com/docker/cli v24.0.6+incompatible // indirect - github.com/docker/distribution v2.8.2+incompatible // indirect - github.com/docker/docker v24.0.9+incompatible // indirect + github.com/distribution/reference v0.5.0 // indirect + github.com/docker/cli v25.0.1+incompatible // indirect + github.com/docker/distribution v2.8.3+incompatible // indirect + github.com/docker/docker v25.0.5+incompatible // indirect github.com/docker/docker-credential-helpers v0.7.0 // indirect - github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-metrics v0.0.1 // indirect - github.com/docker/go-units v0.5.0 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect github.com/gobwas/glob v0.2.3 // indirect @@ -100,9 +100,8 @@ require ( github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/locker v1.0.1 // indirect - github.com/morikuni/aec v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0-rc5 // indirect + github.com/opencontainers/image-spec v1.1.0-rc6 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/rubenv/sql-migrate v1.5.2 // indirect @@ -115,7 +114,7 @@ require ( golang.org/x/crypto v0.24.0 // indirect golang.org/x/crypto/x509roots/fallback v0.0.0-20240604170348-d4e7c9cb6cb8 // indirect k8s.io/apiserver v0.30.2 // indirect - oras.land/oras-go v1.2.4 // indirect + oras.land/oras-go v1.2.5 // indirect ) require ( diff --git a/go.sum b/go.sum index 4381300c98f..ef11b990ea7 100644 --- a/go.sum +++ b/go.sum @@ -141,25 +141,25 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2 h1:aBfCb7iqHmDEIp6fBvC/hQUddQfg+3qdYjwzaiP9Hnc= github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2/go.mod h1:WHNsWjnIn2V1LYOrME7e8KxSeKunYHsxEm4am0BUtcI= -github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY= -github.com/docker/cli v24.0.6+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= -github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= +github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/cli v25.0.1+incompatible h1:mFpqnrS6Hsm3v1k7Wa/BO23oz0k121MTbTO1lpcGSkU= +github.com/docker/cli v25.0.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= +github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v24.0.9+incompatible h1:HPGzNmwfLZWdxHqK9/II92pyi1EpYKsAqcl4G0Of9v0= -github.com/docker/docker v24.0.9+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v25.0.5+incompatible h1:UmQydMduGkrD5nQde1mecF/YnSbTOaPeFIeP5C4W+DE= +github.com/docker/docker v25.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= -github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 h1:ZClxb8laGDf5arXfYcAtECDFgAgHklGI8CxgjHnXKJ4= github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= @@ -503,8 +503,6 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= @@ -533,8 +531,8 @@ github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk= github.com/onsi/gomega v1.32.0/go.mod h1:a4x4gW6Pz2yK1MAmvluYme5lvYTn61afQ2ETw/8n4Lg= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI= -github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= +github.com/opencontainers/image-spec v1.1.0-rc6 h1:XDqvyKsJEbRtATzkgItUqBA7QHk58yxX1Ov9HERHNqU= +github.com/opencontainers/image-spec v1.1.0-rc6/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= @@ -926,8 +924,8 @@ gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= -helm.sh/helm/v3 v3.15.1 h1:22ztacHz4gMqhXNqCQ9NAg6BFWoRUryNLvnkz6OVyw0= -helm.sh/helm/v3 v3.15.1/go.mod h1:fvfoRcB8UKRUV5jrIfOTaN/pG1TPhuqSb56fjYdTKXg= +helm.sh/helm/v3 v3.15.2 h1:/3XINUFinJOBjQplGnjw92eLGpgXXp1L8chWPkCkDuw= +helm.sh/helm/v3 v3.15.2/go.mod h1:FzSIP8jDQaa6WAVg9F+OkKz7J0ZmAga4MABtTbsb9WQ= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -977,8 +975,8 @@ k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl k8s.io/utils v0.0.0-20200603063816-c1c6865ac451/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20240423183400-0849a56e8f22 h1:ao5hUqGhsqdm+bYbjH/pRkCs0unBGe9UyDahzs9zQzQ= k8s.io/utils v0.0.0-20240423183400-0849a56e8f22/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -oras.land/oras-go v1.2.4 h1:djpBY2/2Cs1PV87GSJlxv4voajVOMZxqqtq9AB8YNvY= -oras.land/oras-go v1.2.4/go.mod h1:DYcGfb3YF1nKjcezfX2SNlDAeQFKSXmf+qrFmrh4324= +oras.land/oras-go v1.2.5 h1:XpYuAwAb0DfQsunIyMfeET92emK8km3W4yEzZvUbsTo= +oras.land/oras-go v1.2.5/go.mod h1:PuAwRShRZCsZb7g8Ar3jKKQR/2A/qN+pkYxIOd/FAoo= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0= sigs.k8s.io/controller-runtime v0.6.1/go.mod h1:XRYBPdbf5XJu9kpS84VJiZ7h/u1hF3gEORz0efEja7A= sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw= From 9d79cb8d889169fd07673dcc52cd2ccaa7c4fbd9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Jun 2024 13:00:44 -0700 Subject: [PATCH 07/23] build(deps): bump github.com/bufbuild/buf from 1.33.0 to 1.34.0 in /tools/src/buf (#3659) build(deps): bump github.com/bufbuild/buf in /tools/src/buf Bumps [github.com/bufbuild/buf](https://github.com/bufbuild/buf) from 1.33.0 to 1.34.0. - [Release notes](https://github.com/bufbuild/buf/releases) - [Changelog](https://github.com/bufbuild/buf/blob/main/CHANGELOG.md) - [Commits](https://github.com/bufbuild/buf/compare/v1.33.0...v1.34.0) --- updated-dependencies: - dependency-name: github.com/bufbuild/buf dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/src/buf/go.mod | 27 +++++++++---------- tools/src/buf/go.sum | 62 ++++++++++++++++++++------------------------ 2 files changed, 42 insertions(+), 47 deletions(-) diff --git a/tools/src/buf/go.mod b/tools/src/buf/go.mod index 4c56f149bce..bdca3173c2c 100644 --- a/tools/src/buf/go.mod +++ b/tools/src/buf/go.mod @@ -2,12 +2,12 @@ module local go 1.22.4 -require github.com/bufbuild/buf v1.33.0 +require github.com/bufbuild/buf v1.34.0 require ( - buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.1-20240508200655-46a4cf4ba109.1 // indirect - buf.build/gen/go/bufbuild/registry/connectrpc/go v1.16.2-20240606161333-696c2cfeae8c.1 // indirect - buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.34.1-20240606161333-696c2cfeae8c.1 // indirect + buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240508200655-46a4cf4ba109.2 // indirect + buf.build/gen/go/bufbuild/registry/connectrpc/go v1.16.2-20240610164129-660609bc46d3.1 // indirect + buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.34.2-20240610164129-660609bc46d3.2 // indirect connectrpc.com/connect v1.16.2 // indirect connectrpc.com/otelconnect v0.7.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect @@ -22,23 +22,24 @@ require ( github.com/distribution/reference v0.6.0 // indirect github.com/docker/cli v26.1.4+incompatible // indirect github.com/docker/distribution v2.8.3+incompatible // indirect - github.com/docker/docker v26.1.4+incompatible // indirect + github.com/docker/docker v27.0.0+incompatible // indirect github.com/docker/docker-credential-helpers v0.8.2 // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/felixge/fgprof v0.9.4 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/go-chi/chi/v5 v5.0.12 // indirect + github.com/go-chi/chi/v5 v5.0.13 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gofrs/uuid/v5 v5.2.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/google/cel-go v0.20.1 // indirect - github.com/google/go-containerregistry v0.19.1 // indirect - github.com/google/pprof v0.0.0-20240528025155-186aa0362fba // indirect + github.com/google/go-containerregistry v0.19.2 // indirect + github.com/google/pprof v0.0.0-20240618054019-d3b898a103f8 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jdx/go-netrc v1.0.0 // indirect - github.com/klauspost/compress v1.17.8 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect @@ -52,7 +53,7 @@ require ( github.com/rs/cors v1.11.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect - github.com/spf13/cobra v1.8.0 // indirect + github.com/spf13/cobra v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stoewer/go-strcase v1.3.0 // indirect github.com/vbatts/tar-split v0.11.5 // indirect @@ -66,15 +67,15 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/crypto v0.24.0 // indirect - golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 // indirect + golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect golang.org/x/mod v0.18.0 // indirect golang.org/x/net v0.26.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.21.0 // indirect golang.org/x/term v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/tools/src/buf/go.sum b/tools/src/buf/go.sum index 624ef7c6905..6b67bac010b 100644 --- a/tools/src/buf/go.sum +++ b/tools/src/buf/go.sum @@ -1,10 +1,9 @@ -buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.1-20240401165935-b983156c5e99.1/go.mod h1:XF+P8+RmfdufmIYpGUC+6bF7S+IlmHDEnCrO3OXaUAQ= -buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.1-20240508200655-46a4cf4ba109.1 h1:LEXWFH/xZ5oOWrC3oOtHbUyBdzRWMCPpAQmKC9v05mA= -buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.1-20240508200655-46a4cf4ba109.1/go.mod h1:XF+P8+RmfdufmIYpGUC+6bF7S+IlmHDEnCrO3OXaUAQ= -buf.build/gen/go/bufbuild/registry/connectrpc/go v1.16.2-20240606161333-696c2cfeae8c.1 h1:J/IxoC5LijfcHUh87Am1XHp+eyqd9TAYKzimDSuN9p4= -buf.build/gen/go/bufbuild/registry/connectrpc/go v1.16.2-20240606161333-696c2cfeae8c.1/go.mod h1:kAi136n1j61b2WcTc9HewyA3cNmxAIEy1+cnTWWxL30= -buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.34.1-20240606161333-696c2cfeae8c.1 h1:1Kbs41Eas72MI6pCx931SctvL/jS7KXw3tesOrJfbAM= -buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.34.1-20240606161333-696c2cfeae8c.1/go.mod h1:8ONhsyCTLQ9kBslWnMgPrXTcxzCkKlxZqN9ewUveui8= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240508200655-46a4cf4ba109.2 h1:cFrEG/pJch6t62+jqndcPXeTNkYcztS4tBRgNkR+drw= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.2-20240508200655-46a4cf4ba109.2/go.mod h1:ylS4c28ACSI59oJrOdW4pHS4n0Hw4TgSPHn8rpHl4Yw= +buf.build/gen/go/bufbuild/registry/connectrpc/go v1.16.2-20240610164129-660609bc46d3.1 h1:PmSlGbLLyhKIAm46ROmzdGVaaYgDdFsQNA+VftjuCLs= +buf.build/gen/go/bufbuild/registry/connectrpc/go v1.16.2-20240610164129-660609bc46d3.1/go.mod h1:4ptL49VoWyYwajT6j4zu5vmQ/k/om4tGMB9atY2FhEo= +buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.34.2-20240610164129-660609bc46d3.2 h1:y1+UxFIWzj/eF2RCPqt9egR7Rt9vgQkXNUzSdmR6iEU= +buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.34.2-20240610164129-660609bc46d3.2/go.mod h1:psseUmlKRo9v5LZJtR/aTpdTLuyp9o3X7rnLT87SZEo= connectrpc.com/connect v1.16.2 h1:ybd6y+ls7GOlb7Bh5C8+ghA6SvCBajHwxssO2CGFjqE= connectrpc.com/connect v1.16.2/go.mod h1:n2kgwskMHXC+lVqb18wngEpF95ldBHXjZYJussz5FRc= connectrpc.com/otelconnect v0.7.0 h1:ZH55ZZtcJOTKWWLy3qmL4Pam4RzRWBJFOqTPyAqCXkY= @@ -15,8 +14,8 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= -github.com/bufbuild/buf v1.33.0 h1:LVjW7eGlYohGehEUA1vX/YOL051wkYJus5c5P2es+9o= -github.com/bufbuild/buf v1.33.0/go.mod h1:1KPS3Cwbfxc+rPP6k+a2zhfQT2Qlak3x1K28R582p6U= +github.com/bufbuild/buf v1.34.0 h1:rZSVfYS5SakOe6ds9PDjbHVwOc+vBGVWNW9Ei+Rg/+c= +github.com/bufbuild/buf v1.34.0/go.mod h1:Fj+KBmY2ODYD2Ld02w4LH9Y3WiRH2203IjGJbKYK5Hc= github.com/bufbuild/protocompile v0.14.0 h1:z3DW4IvXE5G/uTOnSQn+qwQQxvhckkTWLS/0No/o7KU= github.com/bufbuild/protocompile v0.14.0/go.mod h1:N6J1NYzkspJo3ZwyL4Xjvli86XOj1xq4qAasUFxGups= github.com/bufbuild/protoplugin v0.0.0-20240323223605-e2735f6c31ee h1:E6ET8YUcYJ1lAe6ctR3as7yqzW2BNItDFnaB5zQq/8M= @@ -40,7 +39,6 @@ github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/stargz-snapshotter/estargz v0.15.1 h1:eXJjw9RbkLFgioVaTG+G/ZW/0kEe2oEKCdS/ZxIyoCU= github.com/containerd/stargz-snapshotter/estargz v0.15.1/go.mod h1:gr2RNwukQ/S9Nv33Lt6UC7xEx58C+LHRdoqbEKjz1Kk= -github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= @@ -54,8 +52,8 @@ github.com/docker/cli v26.1.4+incompatible h1:I8PHdc0MtxEADqYJZvhBrW9bo8gawKwwen github.com/docker/cli v26.1.4+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v26.1.4+incompatible h1:vuTpXDuoga+Z38m1OZHzl7NKisKWaWlhjQk7IDPSLsU= -github.com/docker/docker v26.1.4+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v27.0.0+incompatible h1:JRugTYuelmWlW0M3jakcIadDx2HUoUO6+Tf2C5jVfwA= +github.com/docker/docker v27.0.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.8.2 h1:bX3YxiGzFP5sOXWc3bTPEXdEaZSeVMrFgOr3T+zrFAo= github.com/docker/docker-credential-helpers v0.8.2/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= @@ -69,8 +67,8 @@ github.com/felixge/fgprof v0.9.4 h1:ocDNwMFlnA0NU0zSB3I52xkO4sFXk80VK9lXjLClu88= github.com/felixge/fgprof v0.9.4/go.mod h1:yKl+ERSa++RYOs32d8K6WEXCB4uXdLls4ZaZPpayhMM= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= -github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-chi/chi/v5 v5.0.13 h1:JlH2F2M8qnwl0N1+JFFzlX9TlKJYas3aPXdiuTmJL+w= +github.com/go-chi/chi/v5 v5.0.13/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -85,20 +83,18 @@ github.com/gofrs/uuid/v5 v5.2.0 h1:qw1GMx6/y8vhVsx626ImfKMuS5CvJmhIKKtuyvfajMM= github.com/gofrs/uuid/v5 v5.2.0/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/cel-go v0.20.1 h1:nDx9r8S3L4pE61eDdt8igGj8rf5kjYR3ILxWIpWNi84= github.com/google/cel-go v0.20.1/go.mod h1:kWcIzTsPX0zmQ+H3TirHstLLf9ep5QTsZBN9u4dOYLg= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-containerregistry v0.19.1 h1:yMQ62Al6/V0Z7CqIrrS1iYoA5/oQCm88DeNujc7C1KY= -github.com/google/go-containerregistry v0.19.1/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI= +github.com/google/go-containerregistry v0.19.2 h1:TannFKE1QSajsP6hPWb5oJNgKe1IKjHukIKDUmvsV6w= +github.com/google/go-containerregistry v0.19.2/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI= github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= -github.com/google/pprof v0.0.0-20240528025155-186aa0362fba h1:ql1qNgCyOB7iAEk8JTNM+zJrgIbnyCKX/wdlyPufP5g= -github.com/google/pprof v0.0.0-20240528025155-186aa0362fba/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= +github.com/google/pprof v0.0.0-20240618054019-d3b898a103f8 h1:ASJ/LAqdCHOyMYI+dwNxn7Rd8FscNkMyTr1KZU1JI/M= +github.com/google/pprof v0.0.0-20240618054019-d3b898a103f8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= @@ -112,8 +108,8 @@ github.com/jhump/protoreflect v1.16.0/go.mod h1:oYPd7nPvcBw/5wlDfm/AVmU9zH9BgqGC github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= -github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -151,8 +147,8 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= -github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= @@ -186,8 +182,8 @@ go.opentelemetry.io/otel/sdk/metric v1.19.0 h1:EJoTO5qysMsYCa+w4UghwFV/ptQgqSL/8 go.opentelemetry.io/otel/sdk/metric v1.19.0/go.mod h1:XjG0jQyFJrv2PbMvwND7LwCEhsJzCzV5210euduKcKY= go.opentelemetry.io/otel/trace v1.25.0 h1:tqukZGLwQYRIFtSQM2u2+yfMVTgGVeqRLPUYx1Dq6RM= go.opentelemetry.io/otel/trace v1.25.0/go.mod h1:hCCs70XM/ljO+BeQkyFnbK28SBIJ/Emuha+ccrCRT7I= -go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= -go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -201,8 +197,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= -golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 h1:LoYXNGAShUG3m/ehNk4iFctuhGX/+R1ZpfJ4/ia80JM= -golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= @@ -247,14 +243,12 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 h1:+rdxYoE3E5htTEWIe15GlN6IfvbURM//Jt0mmkmm6ZU= -google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117/go.mod h1:OimBR/bc1wPO9iV4NC2bpyjy3VnAwZh5EBPQdtaE5oo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 h1:MuYw1wJzT+ZkybKfaOXKp5hJiZDn2iHaXRw0mRYdHSc= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4/go.mod h1:px9SlOOZBg1wM1zdnr8jEL4CNGUBZ+ZKYtNPApNQc4c= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 h1:Di6ANFilr+S60a4S61ZM00vLdw0IrQOSMS2/6mrnOU0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From e7048f25bc500c047d60799f712dd4b7f2471ae9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Jun 2024 04:17:39 +0800 Subject: [PATCH 08/23] build(deps): bump aquasecurity/trivy-action from 0.22.0 to 0.23.0 (#3655) Bumps [aquasecurity/trivy-action](https://github.com/aquasecurity/trivy-action) from 0.22.0 to 0.23.0. - [Release notes](https://github.com/aquasecurity/trivy-action/releases) - [Commits](https://github.com/aquasecurity/trivy-action/compare/595be6a0f6560a0a8fc419ddf630567fc623531d...7c2007bcb556501da015201bcba5aa14069b74e2) --- updated-dependencies: - dependency-name: aquasecurity/trivy-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/trivy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index 11775e85eda..e86341238ac 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -25,7 +25,7 @@ jobs: IMAGE=envoy-proxy/gateway-dev TAG=${{ github.sha }} make image - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@595be6a0f6560a0a8fc419ddf630567fc623531d # v0.22.0 + uses: aquasecurity/trivy-action@7c2007bcb556501da015201bcba5aa14069b74e2 # v0.23.0 with: image-ref: envoy-proxy/gateway-dev:${{ github.sha }} exit-code: '1' From 798132c4a4c40eb52bb899015e460dfe785883c1 Mon Sep 17 00:00:00 2001 From: zirain Date: Tue, 25 Jun 2024 05:35:04 +0800 Subject: [PATCH 09/23] docs: introduce shortcode boilerplate (#3652) Signed-off-by: zirain --- site/content/en/boilerplates/index.md | 5 +++++ .../o11y_prerequisites.md | 3 ++- .../observability/gateway-observability.md | 2 +- .../observability/grafana-integration.md | 2 +- .../observability/proxy-observability.md | 2 +- .../observability/rate-limit-observability.md | 2 +- site/go.mod | 7 ++++--- site/go.sum | 8 +++++++ site/layouts/shortcodes/boilerplate.html | 21 +++++++++++++++++++ 9 files changed, 44 insertions(+), 8 deletions(-) create mode 100644 site/content/en/boilerplates/index.md rename site/content/en/{template => boilerplates}/o11y_prerequisites.md (98%) create mode 100644 site/layouts/shortcodes/boilerplate.html diff --git a/site/content/en/boilerplates/index.md b/site/content/en/boilerplates/index.md new file mode 100644 index 00000000000..dda80adbcbf --- /dev/null +++ b/site/content/en/boilerplates/index.md @@ -0,0 +1,5 @@ +--- +headless: true +--- + +This file tells Hugo that the files in this directory tree shouldn't be rendered as normal pages on the site. diff --git a/site/content/en/template/o11y_prerequisites.md b/site/content/en/boilerplates/o11y_prerequisites.md similarity index 98% rename from site/content/en/template/o11y_prerequisites.md rename to site/content/en/boilerplates/o11y_prerequisites.md index 74c547a946c..fa20e77c43b 100644 --- a/site/content/en/template/o11y_prerequisites.md +++ b/site/content/en/boilerplates/o11y_prerequisites.md @@ -1,4 +1,5 @@ - +--- +--- Follow the steps from the [Quickstart](../quickstart) to install Envoy Gateway and the example manifest. Before proceeding, you should be able to query the example backend using HTTP. diff --git a/site/content/en/latest/tasks/observability/gateway-observability.md b/site/content/en/latest/tasks/observability/gateway-observability.md index 93ed8114bcc..6e0040b4f5d 100644 --- a/site/content/en/latest/tasks/observability/gateway-observability.md +++ b/site/content/en/latest/tasks/observability/gateway-observability.md @@ -7,7 +7,7 @@ This task show you how to config gateway control-plane observability, includes m ## Prerequisites -{{% readfile "../../../template/o11y_prerequisites.md" %}} +{{< boilerplate o11y_prerequisites >}} ## Metrics diff --git a/site/content/en/latest/tasks/observability/grafana-integration.md b/site/content/en/latest/tasks/observability/grafana-integration.md index e8cbe7e2942..600a7f5d550 100644 --- a/site/content/en/latest/tasks/observability/grafana-integration.md +++ b/site/content/en/latest/tasks/observability/grafana-integration.md @@ -7,7 +7,7 @@ This task shows you how to visualise the metrics exposed to Prometheus using Gra ## Prerequisites -{{% readfile "../../../template/o11y_prerequisites.md" %}} +{{< boilerplate o11y_prerequisites >}} Follow the steps from the [Gateway Observability](../gateway-observability) and [Proxy Observability](../proxy-observability#metrics) to enable Prometheus metrics for both Envoy Gateway (Control Plane) and Envoy Proxy (Data Plane). diff --git a/site/content/en/latest/tasks/observability/proxy-observability.md b/site/content/en/latest/tasks/observability/proxy-observability.md index 5f24a76eaa9..ea7ba8fdc8f 100644 --- a/site/content/en/latest/tasks/observability/proxy-observability.md +++ b/site/content/en/latest/tasks/observability/proxy-observability.md @@ -7,7 +7,7 @@ This task show you how to config proxy observability, includes metrics, logs, an ## Prerequisites -{{% readfile "../../../template/o11y_prerequisites.md" %}} +{{< boilerplate o11y_prerequisites >}} By default, the Service type of `loki` is ClusterIP, you can change it to LoadBalancer type for further usage: diff --git a/site/content/en/latest/tasks/observability/rate-limit-observability.md b/site/content/en/latest/tasks/observability/rate-limit-observability.md index 2cad2369d68..a0e523d6c8a 100644 --- a/site/content/en/latest/tasks/observability/rate-limit-observability.md +++ b/site/content/en/latest/tasks/observability/rate-limit-observability.md @@ -7,7 +7,7 @@ This guide show you how to config RateLimit observability, includes traces. ## Prerequisites -{{% readfile "../../../template/o11y_prerequisites.md" %}} +{{< boilerplate o11y_prerequisites >}} Follow the steps from the [Global Rate Limit](../traffic/global-rate-limit) to install RateLimit. diff --git a/site/go.mod b/site/go.mod index 6b54e56af72..85d2a6a4a0b 100644 --- a/site/go.mod +++ b/site/go.mod @@ -3,7 +3,8 @@ module github.com/google/docsy-example go 1.22.4 require ( - github.com/FortAwesome/Font-Awesome v0.0.0-20230327165841-0698449d50f2 // indirect - github.com/google/docsy v0.7.1 // indirect - github.com/twbs/bootstrap v5.2.3+incompatible // indirect + github.com/FortAwesome/Font-Awesome v0.0.0-20240402185447-c0f460dca7f7 // indirect + github.com/google/docsy v0.10.0 // indirect + github.com/google/docsy/dependencies v0.7.2 // indirect + github.com/twbs/bootstrap v5.3.3+incompatible // indirect ) diff --git a/site/go.sum b/site/go.sum index e1d4ad4df70..0db3acceffe 100644 --- a/site/go.sum +++ b/site/go.sum @@ -4,6 +4,8 @@ github.com/FortAwesome/Font-Awesome v0.0.0-20221115183454-96cafbd73ec4 h1:xfr9Si github.com/FortAwesome/Font-Awesome v0.0.0-20221115183454-96cafbd73ec4/go.mod h1:IUgezN/MFpCDIlFezw3L8j83oeiIuYoj28Miwr/KUYo= github.com/FortAwesome/Font-Awesome v0.0.0-20230327165841-0698449d50f2 h1:Uv1z5EqCfmiK4IHUwT0m3h/u/WCk+kpRfxvAZhpC7Gc= github.com/FortAwesome/Font-Awesome v0.0.0-20230327165841-0698449d50f2/go.mod h1:IUgezN/MFpCDIlFezw3L8j83oeiIuYoj28Miwr/KUYo= +github.com/FortAwesome/Font-Awesome v0.0.0-20240402185447-c0f460dca7f7 h1:2aWEKCRLqQ9nPyXaz4/IYtRrDr3PzEiX0DUSUr2/EDs= +github.com/FortAwesome/Font-Awesome v0.0.0-20240402185447-c0f460dca7f7/go.mod h1:IUgezN/MFpCDIlFezw3L8j83oeiIuYoj28Miwr/KUYo= github.com/google/docsy v0.5.1 h1:D/ZdFKiE29xM/gwPwQzmkyXhcbQGkReRS6aGrF7lnYk= github.com/google/docsy v0.5.1/go.mod h1:maoUAQU5H/d+FrZIB4xg1EVWAx7RyFMGSDJyWghm31E= github.com/google/docsy v0.6.0 h1:43bVF18t2JihAamelQjjGzx1vO2ljCilVrBgetCA8oI= @@ -12,11 +14,17 @@ github.com/google/docsy v0.7.0 h1:JaeZ0/KufX/BJ3SyATb/fmZa1DFI7o5d9KU+i6+lLJY= github.com/google/docsy v0.7.0/go.mod h1:5WhIFchr5BfH6agjcInhpLRz7U7map0bcmKSpcrg6BE= github.com/google/docsy v0.7.1 h1:DUriA7Nr3lJjNi9Ulev1SfiG1sUYmvyDeU4nTp7uDxY= github.com/google/docsy v0.7.1/go.mod h1:JCmE+c+izhE0Rvzv3y+AzHhz1KdwlA9Oj5YBMklJcfc= +github.com/google/docsy v0.10.0 h1:6tMDacPwAyRWNCfvsn/9qGOZDQ8b0aRzjRZvnZPY5dg= +github.com/google/docsy v0.10.0/go.mod h1:c0nIAqmRTOuJ01F85U/wJPQtc3Zj9N58Kea9bOT2AJc= github.com/google/docsy/dependencies v0.5.1/go.mod h1:EDGc2znMbGUw0RW5kWwy2oGgLt0iVXBmoq4UOqstuNE= github.com/google/docsy/dependencies v0.6.0/go.mod h1:EDGc2znMbGUw0RW5kWwy2oGgLt0iVXBmoq4UOqstuNE= github.com/google/docsy/dependencies v0.7.0/go.mod h1:gihhs5gmgeO+wuoay4FwOzob+jYJVyQbNaQOh788lD4= github.com/google/docsy/dependencies v0.7.1/go.mod h1:gihhs5gmgeO+wuoay4FwOzob+jYJVyQbNaQOh788lD4= +github.com/google/docsy/dependencies v0.7.2 h1:+t5ufoADQAj4XneFphz4A+UU0ICAxmNaRHVWtMYXPSI= +github.com/google/docsy/dependencies v0.7.2/go.mod h1:gihhs5gmgeO+wuoay4FwOzob+jYJVyQbNaQOh788lD4= github.com/twbs/bootstrap v4.6.2+incompatible h1:TDa+R51BTiy1wEHSYjmqDb8LxNl/zaEjAOpRE9Hwh/o= github.com/twbs/bootstrap v4.6.2+incompatible/go.mod h1:fZTSrkpSf0/HkL0IIJzvVspTt1r9zuf7XlZau8kpcY0= github.com/twbs/bootstrap v5.2.3+incompatible h1:lOmsJx587qfF7/gE7Vv4FxEofegyJlEACeVV+Mt7cgc= github.com/twbs/bootstrap v5.2.3+incompatible/go.mod h1:fZTSrkpSf0/HkL0IIJzvVspTt1r9zuf7XlZau8kpcY0= +github.com/twbs/bootstrap v5.3.3+incompatible h1:goFoqinzdHfkeegpFP7pvhbd0g+A3O2hbU3XCjuNrEQ= +github.com/twbs/bootstrap v5.3.3+incompatible/go.mod h1:fZTSrkpSf0/HkL0IIJzvVspTt1r9zuf7XlZau8kpcY0= diff --git a/site/layouts/shortcodes/boilerplate.html b/site/layouts/shortcodes/boilerplate.html new file mode 100644 index 00000000000..96b939e9fc2 --- /dev/null +++ b/site/layouts/shortcodes/boilerplate.html @@ -0,0 +1,21 @@ +{{- /* This will try to find a resource in the "boilerplates" top-level content directory */ -}} +{{- $name := .Get 0 -}} +{{- $position := .Position }} + +{{- if $name -}} + {{- $bundle := .Page.GetPage "/boilerplates" -}} + {{- with $bundle -}} + {{- $name := printf "%s.md" $name -}} + {{- $pattern := printf "%s*" $name -}} + {{- $resource := $bundle.Resources.GetMatch $pattern -}} + {{- with $resource -}} + {{- .Content | markdownify -}} + {{- else -}} + {{- errorf "Could not find boilerplate '%s' (%s)" $name $position -}} + {{- end -}} + {{- else -}} + {{- errorf "'boilerplates' directory was not found (%s)" $position -}} + {{- end -}} +{{- else -}} + {{- errorf "Missing name in boilerplate (%s)" $position -}} +{{- end -}} From 0ebfae84cf90fa7c56e85577097602c3f659bd08 Mon Sep 17 00:00:00 2001 From: Shyunn <1147212064@qq.com> Date: Tue, 25 Jun 2024 05:40:08 +0800 Subject: [PATCH 10/23] refactor: refactor client/backend connection (#3650) * refactor: refactor client/backend connection Signed-off-by: ShyunnY <1147212064@qq.com> * fix: fix lint Signed-off-by: ShyunnY <1147212064@qq.com> --------- Signed-off-by: ShyunnY <1147212064@qq.com> --- api/v1alpha1/backendtrafficpolicy_types.go | 2 +- api/v1alpha1/clienttrafficpolicy_types.go | 2 +- api/v1alpha1/connection_types.go | 16 +++- api/v1alpha1/zz_generated.deepcopy.go | 74 ++++++++++++------- internal/gatewayapi/backendtrafficpolicy.go | 2 +- internal/gatewayapi/clienttrafficpolicy.go | 6 +- internal/ir/xds.go | 8 +- internal/ir/zz_generated.deepcopy.go | 54 +++++++------- internal/xds/translator/listener.go | 13 ++-- site/content/en/latest/api/extension_types.md | 62 ++++++++-------- site/content/zh/latest/api/extension_types.md | 62 ++++++++-------- .../backendtrafficpolicy_test.go | 4 +- .../clienttrafficpolicy_test.go | 2 +- 13 files changed, 173 insertions(+), 134 deletions(-) diff --git a/api/v1alpha1/backendtrafficpolicy_types.go b/api/v1alpha1/backendtrafficpolicy_types.go index 38773cf70a1..8c23b133231 100644 --- a/api/v1alpha1/backendtrafficpolicy_types.go +++ b/api/v1alpha1/backendtrafficpolicy_types.go @@ -109,7 +109,7 @@ type BackendTrafficPolicySpec struct { // Connection includes backend connection settings. // // +optional - Connection *BackendTrafficPolicyConnection `json:"connection,omitempty"` + Connection *BackendConnection `json:"connection,omitempty"` } // +kubebuilder:object:root=true diff --git a/api/v1alpha1/clienttrafficpolicy_types.go b/api/v1alpha1/clienttrafficpolicy_types.go index 740e03e0e2b..20953b1960b 100644 --- a/api/v1alpha1/clienttrafficpolicy_types.go +++ b/api/v1alpha1/clienttrafficpolicy_types.go @@ -81,7 +81,7 @@ type ClientTrafficPolicySpec struct { // Connection includes client connection settings. // // +optional - Connection *Connection `json:"connection,omitempty"` + Connection *ClientConnection `json:"connection,omitempty"` // HTTP1 provides HTTP/1 configuration on the listener. // // +optional diff --git a/api/v1alpha1/connection_types.go b/api/v1alpha1/connection_types.go index 999cfcc4144..758a22fddc7 100644 --- a/api/v1alpha1/connection_types.go +++ b/api/v1alpha1/connection_types.go @@ -10,8 +10,8 @@ import ( gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" ) -// Connection allows users to configure connection-level settings -type Connection struct { +// ClientConnection allows users to configure connection-level settings of client +type ClientConnection struct { // ConnectionLimit defines limits related to connections // // +optional @@ -26,6 +26,18 @@ type Connection struct { BufferLimit *resource.Quantity `json:"bufferLimit,omitempty"` } +// BackendConnection allows users to configure connection-level settings of backend +type BackendConnection struct { + // BufferLimit Soft limit on size of the cluster’s connections read and write buffers. + // If unspecified, an implementation defined default is applied (32768 bytes). + // For example, 20Mi, 1Gi, 256Ki etc. + // Note: that when the suffix is not provided, the value is interpreted as bytes. + // + // +kubebuilder:validation:XValidation:rule="type(self) == string ? self.matches(r\"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\") : type(self) == int",message="BufferLimit must be of the format \"^[1-9]+[0-9]*([EPTGMK]i|[EPTGMk])?$\"" + // +optional + BufferLimit *resource.Quantity `json:"bufferLimit,omitempty"` +} + type ConnectionLimit struct { // Value of the maximum concurrent connections limit. // When the limit is reached, incoming connections will be closed after the CloseDelay duration. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index e1442dd636a..ac4a3602528 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -252,6 +252,26 @@ func (in *Backend) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackendConnection) DeepCopyInto(out *BackendConnection) { + *out = *in + if in.BufferLimit != nil { + in, out := &in.BufferLimit, &out.BufferLimit + x := (*in).DeepCopy() + *out = &x + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackendConnection. +func (in *BackendConnection) DeepCopy() *BackendConnection { + if in == nil { + return nil + } + out := new(BackendConnection) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackendEndpoint) DeepCopyInto(out *BackendEndpoint) { *out = *in @@ -546,7 +566,7 @@ func (in *BackendTrafficPolicySpec) DeepCopyInto(out *BackendTrafficPolicySpec) } if in.Connection != nil { in, out := &in.Connection, &out.Connection - *out = new(BackendTrafficPolicyConnection) + *out = new(BackendConnection) (*in).DeepCopyInto(*out) } } @@ -677,6 +697,31 @@ func (in *ClaimToHeader) DeepCopy() *ClaimToHeader { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClientConnection) DeepCopyInto(out *ClientConnection) { + *out = *in + if in.ConnectionLimit != nil { + in, out := &in.ConnectionLimit, &out.ConnectionLimit + *out = new(ConnectionLimit) + (*in).DeepCopyInto(*out) + } + if in.BufferLimit != nil { + in, out := &in.BufferLimit, &out.BufferLimit + x := (*in).DeepCopy() + *out = &x + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClientConnection. +func (in *ClientConnection) DeepCopy() *ClientConnection { + if in == nil { + return nil + } + out := new(ClientConnection) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClientIPDetectionSettings) DeepCopyInto(out *ClientIPDetectionSettings) { *out = *in @@ -848,7 +893,7 @@ func (in *ClientTrafficPolicySpec) DeepCopyInto(out *ClientTrafficPolicySpec) { } if in.Connection != nil { in, out := &in.Connection, &out.Connection - *out = new(Connection) + *out = new(ClientConnection) (*in).DeepCopyInto(*out) } if in.HTTP1 != nil { @@ -925,31 +970,6 @@ func (in *Compression) DeepCopy() *Compression { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Connection) DeepCopyInto(out *Connection) { - *out = *in - if in.ConnectionLimit != nil { - in, out := &in.ConnectionLimit, &out.ConnectionLimit - *out = new(ConnectionLimit) - (*in).DeepCopyInto(*out) - } - if in.BufferLimit != nil { - in, out := &in.BufferLimit, &out.BufferLimit - x := (*in).DeepCopy() - *out = &x - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Connection. -func (in *Connection) DeepCopy() *Connection { - if in == nil { - return nil - } - out := new(Connection) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ConnectionLimit) DeepCopyInto(out *ConnectionLimit) { *out = *in diff --git a/internal/gatewayapi/backendtrafficpolicy.go b/internal/gatewayapi/backendtrafficpolicy.go index bd536b8443f..bcc6a5507c2 100644 --- a/internal/gatewayapi/backendtrafficpolicy.go +++ b/internal/gatewayapi/backendtrafficpolicy.go @@ -1164,7 +1164,7 @@ func int64ToUint32(in int64) (uint32, bool) { func (t *Translator) buildBackendConnection(policy *egv1a1.BackendTrafficPolicy) (*ir.BackendConnection, error) { var ( bcIR = &ir.BackendConnection{} - bc = &egv1a1.BackendTrafficPolicyConnection{} + bc = &egv1a1.BackendConnection{} ) if policy.Spec.Connection != nil { diff --git a/internal/gatewayapi/clienttrafficpolicy.go b/internal/gatewayapi/clienttrafficpolicy.go index b8faaef5bc9..cde236dbc96 100644 --- a/internal/gatewayapi/clienttrafficpolicy.go +++ b/internal/gatewayapi/clienttrafficpolicy.go @@ -406,7 +406,7 @@ func (t *Translator) translateClientTrafficPolicyForListener(policy *egv1a1.Clie // HTTP and TCP listeners can both be configured by common fields below. var ( keepalive *ir.TCPKeepalive - connection *ir.Connection + connection *ir.ClientConnection tlsConfig *ir.TLSConfig enableProxyProtocol bool timeout *ir.ClientTimeout @@ -852,12 +852,12 @@ func (t *Translator) buildListenerTLSParameters(policy *egv1a1.ClientTrafficPoli return irTLSConfig, nil } -func buildConnection(connection *egv1a1.Connection) (*ir.Connection, error) { +func buildConnection(connection *egv1a1.ClientConnection) (*ir.ClientConnection, error) { if connection == nil { return nil, nil } - irConnection := &ir.Connection{} + irConnection := &ir.ClientConnection{} if connection.ConnectionLimit != nil { irConnectionLimit := &ir.ConnectionLimit{} diff --git a/internal/ir/xds.go b/internal/ir/xds.go index 9de48f29b33..cab235895cf 100644 --- a/internal/ir/xds.go +++ b/internal/ir/xds.go @@ -267,7 +267,7 @@ type HTTPListener struct { // ClientTimeout sets the timeout configuration for downstream connections Timeout *ClientTimeout `json:"timeout,omitempty" yaml:"clientTimeout,omitempty"` // Connection settings - Connection *Connection `json:"connection,omitempty" yaml:"connection,omitempty"` + Connection *ClientConnection `json:"connection,omitempty" yaml:"connection,omitempty"` } // Validate the fields within the HTTPListener structure @@ -1351,7 +1351,7 @@ type TCPListener struct { // ClientTimeout sets the timeout configuration for downstream connections. Timeout *ClientTimeout `json:"timeout,omitempty" yaml:"clientTimeout,omitempty"` // Connection settings for clients - Connection *Connection `json:"connection,omitempty" yaml:"connection,omitempty"` + Connection *ClientConnection `json:"connection,omitempty" yaml:"connection,omitempty"` // Routes associated with TCP traffic to the listener. Routes []*TCPRoute `json:"routes,omitempty" yaml:"routes,omitempty"` } @@ -2136,9 +2136,9 @@ type BackendConnection struct { BufferLimitBytes *uint32 `json:"bufferLimit,omitempty" yaml:"bufferLimit,omitempty"` } -// Connection settings for downstream connections +// ClientConnection settings for downstream connections // +k8s:deepcopy-gen=true -type Connection struct { +type ClientConnection struct { // ConnectionLimit is the limit of number of connections ConnectionLimit *ConnectionLimit `json:"limit,omitempty" yaml:"limit,omitempty"` // BufferLimitBytes is the maximum number of bytes that can be buffered for a connection. diff --git a/internal/ir/zz_generated.deepcopy.go b/internal/ir/zz_generated.deepcopy.go index 69e10d9dc94..459c21ef10d 100644 --- a/internal/ir/zz_generated.deepcopy.go +++ b/internal/ir/zz_generated.deepcopy.go @@ -317,6 +317,31 @@ func (in *CircuitBreaker) DeepCopy() *CircuitBreaker { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClientConnection) DeepCopyInto(out *ClientConnection) { + *out = *in + if in.ConnectionLimit != nil { + in, out := &in.ConnectionLimit, &out.ConnectionLimit + *out = new(ConnectionLimit) + (*in).DeepCopyInto(*out) + } + if in.BufferLimitBytes != nil { + in, out := &in.BufferLimitBytes, &out.BufferLimitBytes + *out = new(uint32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClientConnection. +func (in *ClientConnection) DeepCopy() *ClientConnection { + if in == nil { + return nil + } + out := new(ClientConnection) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClientIPDetectionSettings) DeepCopyInto(out *ClientIPDetectionSettings) { *out = *in @@ -367,31 +392,6 @@ func (in *ClientTimeout) DeepCopy() *ClientTimeout { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Connection) DeepCopyInto(out *Connection) { - *out = *in - if in.ConnectionLimit != nil { - in, out := &in.ConnectionLimit, &out.ConnectionLimit - *out = new(ConnectionLimit) - (*in).DeepCopyInto(*out) - } - if in.BufferLimitBytes != nil { - in, out := &in.BufferLimitBytes, &out.BufferLimitBytes - *out = new(uint32) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Connection. -func (in *Connection) DeepCopy() *Connection { - if in == nil { - return nil - } - out := new(Connection) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ConnectionLimit) DeepCopyInto(out *ConnectionLimit) { *out = *in @@ -1048,7 +1048,7 @@ func (in *HTTPListener) DeepCopyInto(out *HTTPListener) { } if in.Connection != nil { in, out := &in.Connection, &out.Connection - *out = new(Connection) + *out = new(ClientConnection) (*in).DeepCopyInto(*out) } } @@ -2294,7 +2294,7 @@ func (in *TCPListener) DeepCopyInto(out *TCPListener) { } if in.Connection != nil { in, out := &in.Connection, &out.Connection - *out = new(Connection) + *out = new(ClientConnection) (*in).DeepCopyInto(*out) } if in.Routes != nil { diff --git a/internal/xds/translator/listener.go b/internal/xds/translator/listener.go index e72cf797d4c..ee1f5c7d133 100644 --- a/internal/xds/translator/listener.go +++ b/internal/xds/translator/listener.go @@ -137,7 +137,7 @@ func originalIPDetectionExtensions(clientIPDetection *ir.ClientIPDetectionSettin // buildXdsTCPListener creates a xds Listener resource // TODO: Improve function parameters -func buildXdsTCPListener(name, address string, port uint32, keepalive *ir.TCPKeepalive, connection *ir.Connection, accesslog *ir.AccessLog) *listenerv3.Listener { +func buildXdsTCPListener(name, address string, port uint32, keepalive *ir.TCPKeepalive, connection *ir.ClientConnection, accesslog *ir.AccessLog) *listenerv3.Listener { socketOptions := buildTCPSocketOptions(keepalive) al := buildXdsAccessLog(accesslog, true) bufferLimitBytes := buildPerConnectionBufferLimitBytes(connection) @@ -163,7 +163,7 @@ func buildXdsTCPListener(name, address string, port uint32, keepalive *ir.TCPKee } } -func buildPerConnectionBufferLimitBytes(connection *ir.Connection) *wrapperspb.UInt32Value { +func buildPerConnectionBufferLimitBytes(connection *ir.ClientConnection) *wrapperspb.UInt32Value { if connection != nil && connection.BufferLimitBytes != nil { return wrapperspb.UInt32(*connection.BufferLimitBytes) } @@ -209,7 +209,7 @@ func buildXdsQuicListener(name, address string, port uint32, accesslog *ir.Acces // The newly created TCP filter chain is configured with a filter chain match to // match the server names(SNI) based on the listener's hostnames. func (t *Translator) addHCMToXDSListener(xdsListener *listenerv3.Listener, irListener *ir.HTTPListener, - accesslog *ir.AccessLog, tracing *ir.Tracing, http3Listener bool, connection *ir.Connection, + accesslog *ir.AccessLog, tracing *ir.Tracing, http3Listener bool, connection *ir.ClientConnection, ) error { al := buildXdsAccessLog(accesslog, false) @@ -391,7 +391,10 @@ func findXdsHTTPRouteConfigName(xdsListener *listenerv3.Listener) string { return "" } -func addXdsTCPFilterChain(xdsListener *listenerv3.Listener, irRoute *ir.TCPRoute, clusterName string, accesslog *ir.AccessLog, timeout *ir.ClientTimeout, connection *ir.Connection) error { +func addXdsTCPFilterChain(xdsListener *listenerv3.Listener, irRoute *ir.TCPRoute, + clusterName string, accesslog *ir.AccessLog, timeout *ir.ClientTimeout, + connection *ir.ClientConnection, +) error { if irRoute == nil { return errors.New("tcp listener is nil") } @@ -470,7 +473,7 @@ func addXdsTCPFilterChain(xdsListener *listenerv3.Listener, irRoute *ir.TCPRoute return nil } -func buildConnectionLimitFilter(statPrefix string, connection *ir.Connection) *connection_limitv3.ConnectionLimit { +func buildConnectionLimitFilter(statPrefix string, connection *ir.ClientConnection) *connection_limitv3.ConnectionLimit { cl := &connection_limitv3.ConnectionLimit{ StatPrefix: statPrefix, MaxConnections: wrapperspb.UInt64(*connection.ConnectionLimit.Value), diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index 258d99f27fe..953a5d29783 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -277,6 +277,20 @@ _Appears in:_ +#### BackendConnection + + + +BackendConnection allows users to configure connection-level settings of backend + +_Appears in:_ +- [BackendTrafficPolicySpec](#backendtrafficpolicyspec) + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `bufferLimit` | _[Quantity](#quantity)_ | false | BufferLimit Soft limit on size of the cluster’s connections read and write buffers.
If unspecified, an implementation defined default is applied (32768 bytes).
For example, 20Mi, 1Gi, 256Ki etc.
Note: that when the suffix is not provided, the value is interpreted as bytes. | + + #### BackendEndpoint @@ -403,18 +417,6 @@ _Appears in:_ | `status` | _[PolicyStatus](https://gateway-api.sigs.k8s.io/reference/spec/#gateway.networking.k8s.io/v1alpha2.PolicyStatus)_ | true | status defines the current status of BackendTrafficPolicy. | -#### BackendTrafficPolicyConnection - - - -BackendTrafficPolicyConnection allows users to configure connection-level settings of backend - -_Appears in:_ -- [BackendTrafficPolicySpec](#backendtrafficpolicyspec) - -| Field | Type | Required | Description | -| --- | --- | --- | --- | -| `bufferLimit` | _[Quantity](#quantity)_ | false | BufferLimit Soft limit on size of the cluster’s connections read and write buffers.
If unspecified, an implementation defined default is applied (32768 bytes).
For example, 20Mi, 1Gi, 256Ki etc.
Note: that when the suffix is not provided, the value is interpreted as bytes. | #### BackendTrafficPolicyList @@ -456,7 +458,7 @@ _Appears in:_ | `retry` | _[Retry](#retry)_ | false | Retry provides more advanced usage, allowing users to customize the number of retries, retry fallback strategy, and retry triggering conditions.
If not set, retry will be disabled. | | `useClientProtocol` | _boolean_ | false | UseClientProtocol configures Envoy to prefer sending requests to backends using
the same HTTP protocol that the incoming request used. Defaults to false, which means
that Envoy will use the protocol indicated by the attached BackendRef. | | `timeout` | _[Timeout](#timeout)_ | false | Timeout settings for the backend connections. | -| `connection` | _[BackendTrafficPolicyConnection](#backendtrafficpolicyconnection)_ | false | Connection includes backend connection settings. | +| `connection` | _[BackendConnection](#backendconnection)_ | false | Connection includes backend connection settings. | #### BasicAuth @@ -552,6 +554,21 @@ _Appears in:_ | `claim` | _string_ | true | Claim is the JWT Claim that should be saved into the header : it can be a nested claim of type
(eg. "claim.nested.key", "sub"). The nested claim name must use dot "."
to separate the JSON name path. | +#### ClientConnection + + + +ClientConnection allows users to configure connection-level settings of client + +_Appears in:_ +- [ClientTrafficPolicySpec](#clienttrafficpolicyspec) + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `connectionLimit` | _[ConnectionLimit](#connectionlimit)_ | false | ConnectionLimit defines limits related to connections | +| `bufferLimit` | _[Quantity](#quantity)_ | false | BufferLimit provides configuration for the maximum buffer size in bytes for each incoming connection.
For example, 20Mi, 1Gi, 256Ki etc.
Note that when the suffix is not provided, the value is interpreted as bytes.
Default: 32768 bytes. | + + #### ClientIPDetectionSettings @@ -657,7 +674,7 @@ _Appears in:_ | `path` | _[PathSettings](#pathsettings)_ | false | Path enables managing how the incoming path set by clients can be normalized. | | `headers` | _[HeaderSettings](#headersettings)_ | false | HeaderSettings provides configuration for header management. | | `timeout` | _[ClientTimeout](#clienttimeout)_ | false | Timeout settings for the client connections. | -| `connection` | _[Connection](#connection)_ | false | Connection includes client connection settings. | +| `connection` | _[ClientConnection](#clientconnection)_ | false | Connection includes client connection settings. | | `http1` | _[HTTP1Settings](#http1settings)_ | false | HTTP1 provides HTTP/1 configuration on the listener. | | `http2` | _[HTTP2Settings](#http2settings)_ | false | HTTP2 provides HTTP/2 configuration on the listener. | | `http3` | _[HTTP3Settings](#http3settings)_ | false | HTTP3 provides HTTP/3 configuration on the listener. | @@ -709,21 +726,6 @@ _Appears in:_ -#### Connection - - - -Connection allows users to configure connection-level settings - -_Appears in:_ -- [ClientTrafficPolicySpec](#clienttrafficpolicyspec) - -| Field | Type | Required | Description | -| --- | --- | --- | --- | -| `connectionLimit` | _[ConnectionLimit](#connectionlimit)_ | false | ConnectionLimit defines limits related to connections | -| `bufferLimit` | _[Quantity](#quantity)_ | false | BufferLimit provides configuration for the maximum buffer size in bytes for each incoming connection.
For example, 20Mi, 1Gi, 256Ki etc.
Note that when the suffix is not provided, the value is interpreted as bytes.
Default: 32768 bytes. | - - #### ConnectionLimit @@ -731,7 +733,7 @@ _Appears in:_ _Appears in:_ -- [Connection](#connection) +- [ClientConnection](#clientconnection) | Field | Type | Required | Description | | --- | --- | --- | --- | diff --git a/site/content/zh/latest/api/extension_types.md b/site/content/zh/latest/api/extension_types.md index 258d99f27fe..953a5d29783 100644 --- a/site/content/zh/latest/api/extension_types.md +++ b/site/content/zh/latest/api/extension_types.md @@ -277,6 +277,20 @@ _Appears in:_ +#### BackendConnection + + + +BackendConnection allows users to configure connection-level settings of backend + +_Appears in:_ +- [BackendTrafficPolicySpec](#backendtrafficpolicyspec) + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `bufferLimit` | _[Quantity](#quantity)_ | false | BufferLimit Soft limit on size of the cluster’s connections read and write buffers.
If unspecified, an implementation defined default is applied (32768 bytes).
For example, 20Mi, 1Gi, 256Ki etc.
Note: that when the suffix is not provided, the value is interpreted as bytes. | + + #### BackendEndpoint @@ -403,18 +417,6 @@ _Appears in:_ | `status` | _[PolicyStatus](https://gateway-api.sigs.k8s.io/reference/spec/#gateway.networking.k8s.io/v1alpha2.PolicyStatus)_ | true | status defines the current status of BackendTrafficPolicy. | -#### BackendTrafficPolicyConnection - - - -BackendTrafficPolicyConnection allows users to configure connection-level settings of backend - -_Appears in:_ -- [BackendTrafficPolicySpec](#backendtrafficpolicyspec) - -| Field | Type | Required | Description | -| --- | --- | --- | --- | -| `bufferLimit` | _[Quantity](#quantity)_ | false | BufferLimit Soft limit on size of the cluster’s connections read and write buffers.
If unspecified, an implementation defined default is applied (32768 bytes).
For example, 20Mi, 1Gi, 256Ki etc.
Note: that when the suffix is not provided, the value is interpreted as bytes. | #### BackendTrafficPolicyList @@ -456,7 +458,7 @@ _Appears in:_ | `retry` | _[Retry](#retry)_ | false | Retry provides more advanced usage, allowing users to customize the number of retries, retry fallback strategy, and retry triggering conditions.
If not set, retry will be disabled. | | `useClientProtocol` | _boolean_ | false | UseClientProtocol configures Envoy to prefer sending requests to backends using
the same HTTP protocol that the incoming request used. Defaults to false, which means
that Envoy will use the protocol indicated by the attached BackendRef. | | `timeout` | _[Timeout](#timeout)_ | false | Timeout settings for the backend connections. | -| `connection` | _[BackendTrafficPolicyConnection](#backendtrafficpolicyconnection)_ | false | Connection includes backend connection settings. | +| `connection` | _[BackendConnection](#backendconnection)_ | false | Connection includes backend connection settings. | #### BasicAuth @@ -552,6 +554,21 @@ _Appears in:_ | `claim` | _string_ | true | Claim is the JWT Claim that should be saved into the header : it can be a nested claim of type
(eg. "claim.nested.key", "sub"). The nested claim name must use dot "."
to separate the JSON name path. | +#### ClientConnection + + + +ClientConnection allows users to configure connection-level settings of client + +_Appears in:_ +- [ClientTrafficPolicySpec](#clienttrafficpolicyspec) + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `connectionLimit` | _[ConnectionLimit](#connectionlimit)_ | false | ConnectionLimit defines limits related to connections | +| `bufferLimit` | _[Quantity](#quantity)_ | false | BufferLimit provides configuration for the maximum buffer size in bytes for each incoming connection.
For example, 20Mi, 1Gi, 256Ki etc.
Note that when the suffix is not provided, the value is interpreted as bytes.
Default: 32768 bytes. | + + #### ClientIPDetectionSettings @@ -657,7 +674,7 @@ _Appears in:_ | `path` | _[PathSettings](#pathsettings)_ | false | Path enables managing how the incoming path set by clients can be normalized. | | `headers` | _[HeaderSettings](#headersettings)_ | false | HeaderSettings provides configuration for header management. | | `timeout` | _[ClientTimeout](#clienttimeout)_ | false | Timeout settings for the client connections. | -| `connection` | _[Connection](#connection)_ | false | Connection includes client connection settings. | +| `connection` | _[ClientConnection](#clientconnection)_ | false | Connection includes client connection settings. | | `http1` | _[HTTP1Settings](#http1settings)_ | false | HTTP1 provides HTTP/1 configuration on the listener. | | `http2` | _[HTTP2Settings](#http2settings)_ | false | HTTP2 provides HTTP/2 configuration on the listener. | | `http3` | _[HTTP3Settings](#http3settings)_ | false | HTTP3 provides HTTP/3 configuration on the listener. | @@ -709,21 +726,6 @@ _Appears in:_ -#### Connection - - - -Connection allows users to configure connection-level settings - -_Appears in:_ -- [ClientTrafficPolicySpec](#clienttrafficpolicyspec) - -| Field | Type | Required | Description | -| --- | --- | --- | --- | -| `connectionLimit` | _[ConnectionLimit](#connectionlimit)_ | false | ConnectionLimit defines limits related to connections | -| `bufferLimit` | _[Quantity](#quantity)_ | false | BufferLimit provides configuration for the maximum buffer size in bytes for each incoming connection.
For example, 20Mi, 1Gi, 256Ki etc.
Note that when the suffix is not provided, the value is interpreted as bytes.
Default: 32768 bytes. | - - #### ConnectionLimit @@ -731,7 +733,7 @@ _Appears in:_ _Appears in:_ -- [Connection](#connection) +- [ClientConnection](#clientconnection) | Field | Type | Required | Description | | --- | --- | --- | --- | diff --git a/test/cel-validation/backendtrafficpolicy_test.go b/test/cel-validation/backendtrafficpolicy_test.go index 2e5ed776e71..e8418790d31 100644 --- a/test/cel-validation/backendtrafficpolicy_test.go +++ b/test/cel-validation/backendtrafficpolicy_test.go @@ -1078,7 +1078,7 @@ func TestBackendTrafficPolicyTarget(t *testing.T) { }, }, }, - Connection: &egv1a1.BackendTrafficPolicyConnection{ + Connection: &egv1a1.BackendConnection{ BufferLimit: ptr.To(resource.MustParse("1Mi")), }, } @@ -1097,7 +1097,7 @@ func TestBackendTrafficPolicyTarget(t *testing.T) { }, }, }, - Connection: &egv1a1.BackendTrafficPolicyConnection{ + Connection: &egv1a1.BackendConnection{ BufferLimit: ptr.To(resource.MustParse("1m")), }, } diff --git a/test/cel-validation/clienttrafficpolicy_test.go b/test/cel-validation/clienttrafficpolicy_test.go index db5d3aa65e6..e28ad6c83bf 100644 --- a/test/cel-validation/clienttrafficpolicy_test.go +++ b/test/cel-validation/clienttrafficpolicy_test.go @@ -306,7 +306,7 @@ func TestClientTrafficPolicyTarget(t *testing.T) { }, }, }, - Connection: &egv1a1.Connection{ + Connection: &egv1a1.ClientConnection{ BufferLimit: ptr.To(resource.MustParse("15m")), }, } From 95d0fc7c0cc524a470d5d217096b604d3f7f79fc Mon Sep 17 00:00:00 2001 From: zirain Date: Tue, 25 Jun 2024 11:13:44 +0800 Subject: [PATCH 11/23] docs: remove docs-api-headings (#3653) Signed-off-by: zirain --- tools/crd-ref-docs/templates/gv_list.tpl | 5 ++++- tools/hack/docs-headings.sh | 21 --------------------- tools/make/docs.mk | 10 ++-------- 3 files changed, 6 insertions(+), 30 deletions(-) delete mode 100755 tools/hack/docs-headings.sh diff --git a/tools/crd-ref-docs/templates/gv_list.tpl b/tools/crd-ref-docs/templates/gv_list.tpl index a4d3dadf18c..84a0f75a9d2 100644 --- a/tools/crd-ref-docs/templates/gv_list.tpl +++ b/tools/crd-ref-docs/templates/gv_list.tpl @@ -1,7 +1,10 @@ {{- define "gvList" -}} {{- $groupVersions := . -}} -# API Reference ++++ +title = "API Reference" ++++ + ## Packages {{- range $groupVersions }} diff --git a/tools/hack/docs-headings.sh b/tools/hack/docs-headings.sh deleted file mode 100755 index 4f61928065b..00000000000 --- a/tools/hack/docs-headings.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -if [ "$#" -ne 1 ]; then - echo "Usage: $0 " - exit 1 -fi - -input_file=$1 - -temp_file=$(mktemp) - -sed -n ' -/^# / { - s/^# \(.*\)/+++\ntitle = "\1"\n+++\n/ - p - d -} -p -' "$input_file" > "$temp_file" - -mv "$temp_file" "$input_file" diff --git a/tools/make/docs.mk b/tools/make/docs.mk index 8c50a5b9878..a8c59c29999 100644 --- a/tools/make/docs.mk +++ b/tools/make/docs.mk @@ -4,7 +4,7 @@ RELEASE_VERSIONS ?= $(foreach v,$(wildcard ${ROOT_DIR}/docs/*),$(notdir ${v})) ##@ Docs .PHONY: docs -docs: docs.clean helm-readme-gen docs-api docs-api-headings ## Generate Envoy Gateway Docs Sources +docs: docs.clean helm-readme-gen docs-api ## Generate Envoy Gateway Docs Sources @$(LOG_TARGET) cd $(ROOT_DIR)/site && npm install cd $(ROOT_DIR)/site && npm run build:production @@ -32,7 +32,7 @@ docs.clean: rm -f site/.hugo_build.lock .PHONY: docs-api -docs-api: docs-api-gen helm-readme-gen docs-api-headings +docs-api: docs-api-gen helm-readme-gen .PHONY: helm-readme-gen helm-readme-gen: @@ -77,12 +77,6 @@ docs-api-gen: $(tools/crd-ref-docs) # below line copy command for sync English api doc into Chinese cp site/content/en/latest/api/extension_types.md site/content/zh/latest/api/extension_types.md -.PHONY: docs-api-headings # Required since sphinx mst does not link to h4 headings. -docs-api-headings: - @$(LOG_TARGET) - tools/hack/docs-headings.sh site/content/en/latest/api/extension_types.md - tools/hack/docs-headings.sh site/content/zh/latest/api/extension_types.md - .PHONY: docs-release-prepare docs-release-prepare: @$(LOG_TARGET) From 8e3436616efe12e9bb37b95650a94fe25bab5d1a Mon Sep 17 00:00:00 2001 From: sh2 Date: Tue, 25 Jun 2024 11:20:58 +0800 Subject: [PATCH 12/23] chore: update PR template to highlight the api changes (#3637) * update PR template to highlight the api changes Signed-off-by: shawnh2 * add title example Signed-off-by: shawnh2 --------- Signed-off-by: shawnh2 --- .github/PULL_REQUEST_TEMPLATE.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 6300aebd8d4..8dc586ce114 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,12 +1,18 @@ **What type of PR is this?** + + **What this PR does / why we need it**: From abe33cc82dbbb4016f14a2c1ba93b00fd23dd850 Mon Sep 17 00:00:00 2001 From: zirain Date: Wed, 26 Jun 2024 02:00:54 +0800 Subject: [PATCH 13/23] chore: update gateway-addons-helm (#3649) * update gateway-addons-helm Signed-off-by: zirain * disabled selfMonitoring Signed-off-by: zirain * update test Signed-off-by: zirain * cleanup helm makefile Signed-off-by: zirain * fix lint Signed-off-by: zirain * template addon in monitoring namespace Signed-off-by: zirain * add more test Signed-off-by: zirain --------- Signed-off-by: zirain --- charts/gateway-addons-helm/README.md | 5 +- charts/gateway-addons-helm/values.yaml | 7 +- .../latest/install/gateway-addons-helm-api.md | 5 +- .../latest/install/gateway-addons-helm-api.md | 5 +- test/helm/gateway-addons-helm/default.in.yaml | 0 .../helm/gateway-addons-helm/default.out.yaml | 10169 ++++++++++++++++ test/helm/gateway-addons-helm/e2e.in.yaml | 4 + test/helm/gateway-addons-helm/e2e.out.yaml | 10057 +++++++++++++++ .../control-plane-with-pdb.in.yaml | 0 .../control-plane-with-pdb.out.yaml | 64 +- .../{ => gateway-helm}/default-config.in.yaml | 0 .../default-config.out.yaml | 62 +- .../deployment-custom-topology.in.yaml | 0 .../deployment-custom-topology.out.yaml | 62 +- .../deployment-images-config.in.yaml | 0 .../deployment-images-config.out.yaml | 62 +- .../envoy-gateway-config.in.yaml | 0 .../envoy-gateway-config.out.yaml | 62 +- .../global-images-config.in.yaml | 0 .../global-images-config.out.yaml | 62 +- tools/linter/yamllint/.yamllint | 1 + tools/make/common.mk | 2 +- tools/make/helm.mk | 31 +- tools/make/kube.mk | 2 +- 24 files changed, 20446 insertions(+), 216 deletions(-) create mode 100644 test/helm/gateway-addons-helm/default.in.yaml create mode 100644 test/helm/gateway-addons-helm/default.out.yaml create mode 100644 test/helm/gateway-addons-helm/e2e.in.yaml create mode 100644 test/helm/gateway-addons-helm/e2e.out.yaml rename test/helm/{ => gateway-helm}/control-plane-with-pdb.in.yaml (100%) rename test/helm/{ => gateway-helm}/control-plane-with-pdb.out.yaml (89%) rename test/helm/{ => gateway-helm}/default-config.in.yaml (100%) rename test/helm/{ => gateway-helm}/default-config.out.yaml (90%) rename test/helm/{ => gateway-helm}/deployment-custom-topology.in.yaml (100%) rename test/helm/{ => gateway-helm}/deployment-custom-topology.out.yaml (90%) rename test/helm/{ => gateway-helm}/deployment-images-config.in.yaml (100%) rename test/helm/{ => gateway-helm}/deployment-images-config.out.yaml (90%) rename test/helm/{ => gateway-helm}/envoy-gateway-config.in.yaml (100%) rename test/helm/{ => gateway-helm}/envoy-gateway-config.out.yaml (90%) rename test/helm/{ => gateway-helm}/global-images-config.in.yaml (100%) rename test/helm/{ => gateway-helm}/global-images-config.out.yaml (90%) diff --git a/charts/gateway-addons-helm/README.md b/charts/gateway-addons-helm/README.md index 7d545d0b90f..ff47a1e4797 100644 --- a/charts/gateway-addons-helm/README.md +++ b/charts/gateway-addons-helm/README.md @@ -94,9 +94,12 @@ To uninstall the chart: | loki.loki.memberlist | string | `"loki-memberlist"` | | | loki.loki.rulerConfig.storage.type | string | `"local"` | | | loki.loki.storage.type | string | `"filesystem"` | | -| loki.monitoring.selfMonitoring.grafanaAgent.installOperator | bool | `true` | | +| loki.monitoring.lokiCanary.enabled | bool | `false` | | +| loki.monitoring.selfMonitoring.enabled | bool | `false` | | +| loki.monitoring.selfMonitoring.grafanaAgent.installOperator | bool | `false` | | | loki.read.replicas | int | `0` | | | loki.singleBinary.replicas | int | `1` | | +| loki.test.enabled | bool | `false` | | | loki.write.replicas | int | `0` | | | opentelemetry-collector.config.exporters.logging.verbosity | string | `"detailed"` | | | opentelemetry-collector.config.exporters.loki.endpoint | string | `"http://loki.monitoring.svc:3100/loki/api/v1/push"` | | diff --git a/charts/gateway-addons-helm/values.yaml b/charts/gateway-addons-helm/values.yaml index e63e0bdee72..cc17ef52398 100644 --- a/charts/gateway-addons-helm/values.yaml +++ b/charts/gateway-addons-helm/values.yaml @@ -142,6 +142,8 @@ loki: rulerConfig: storage: type: "local" + test: + enabled: false singleBinary: replicas: 1 read: @@ -151,9 +153,12 @@ loki: write: replicas: 0 monitoring: + lokiCanary: + enabled: false selfMonitoring: + enabled: false grafanaAgent: - installOperator: true + installOperator: false # Disable gateway. gateway: enabled: false diff --git a/site/content/en/latest/install/gateway-addons-helm-api.md b/site/content/en/latest/install/gateway-addons-helm-api.md index 5401fbe7f83..222121967a6 100644 --- a/site/content/en/latest/install/gateway-addons-helm-api.md +++ b/site/content/en/latest/install/gateway-addons-helm-api.md @@ -73,9 +73,12 @@ An Add-ons Helm chart for Envoy Gateway | loki.loki.memberlist | string | `"loki-memberlist"` | | | loki.loki.rulerConfig.storage.type | string | `"local"` | | | loki.loki.storage.type | string | `"filesystem"` | | -| loki.monitoring.selfMonitoring.grafanaAgent.installOperator | bool | `true` | | +| loki.monitoring.lokiCanary.enabled | bool | `false` | | +| loki.monitoring.selfMonitoring.enabled | bool | `false` | | +| loki.monitoring.selfMonitoring.grafanaAgent.installOperator | bool | `false` | | | loki.read.replicas | int | `0` | | | loki.singleBinary.replicas | int | `1` | | +| loki.test.enabled | bool | `false` | | | loki.write.replicas | int | `0` | | | opentelemetry-collector.config.exporters.logging.verbosity | string | `"detailed"` | | | opentelemetry-collector.config.exporters.loki.endpoint | string | `"http://loki.monitoring.svc:3100/loki/api/v1/push"` | | diff --git a/site/content/zh/latest/install/gateway-addons-helm-api.md b/site/content/zh/latest/install/gateway-addons-helm-api.md index 5401fbe7f83..222121967a6 100644 --- a/site/content/zh/latest/install/gateway-addons-helm-api.md +++ b/site/content/zh/latest/install/gateway-addons-helm-api.md @@ -73,9 +73,12 @@ An Add-ons Helm chart for Envoy Gateway | loki.loki.memberlist | string | `"loki-memberlist"` | | | loki.loki.rulerConfig.storage.type | string | `"local"` | | | loki.loki.storage.type | string | `"filesystem"` | | -| loki.monitoring.selfMonitoring.grafanaAgent.installOperator | bool | `true` | | +| loki.monitoring.lokiCanary.enabled | bool | `false` | | +| loki.monitoring.selfMonitoring.enabled | bool | `false` | | +| loki.monitoring.selfMonitoring.grafanaAgent.installOperator | bool | `false` | | | loki.read.replicas | int | `0` | | | loki.singleBinary.replicas | int | `1` | | +| loki.test.enabled | bool | `false` | | | loki.write.replicas | int | `0` | | | opentelemetry-collector.config.exporters.logging.verbosity | string | `"detailed"` | | | opentelemetry-collector.config.exporters.loki.endpoint | string | `"http://loki.monitoring.svc:3100/loki/api/v1/push"` | | diff --git a/test/helm/gateway-addons-helm/default.in.yaml b/test/helm/gateway-addons-helm/default.in.yaml new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/helm/gateway-addons-helm/default.out.yaml b/test/helm/gateway-addons-helm/default.out.yaml new file mode 100644 index 00000000000..45cd9480326 --- /dev/null +++ b/test/helm/gateway-addons-helm/default.out.yaml @@ -0,0 +1,10169 @@ +--- +# Source: gateway-addons-helm/charts/fluent-bit/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fluent-bit + namespace: monitoring + labels: + helm.sh/chart: fluent-bit-0.30.4 + app.kubernetes.io/name: fluent-bit + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.1.4" + app.kubernetes.io/managed-by: Helm +--- +# Source: gateway-addons-helm/charts/grafana/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: false +metadata: + labels: + helm.sh/chart: grafana-8.0.0 + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "11.0.0" + app.kubernetes.io/managed-by: Helm + name: grafana + namespace: monitoring +--- +# Source: gateway-addons-helm/charts/loki/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: loki + labels: + helm.sh/chart: loki-4.8.0 + app.kubernetes.io/name: loki + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.7.3" + app.kubernetes.io/managed-by: Helm +automountServiceAccountToken: true +--- +# Source: gateway-addons-helm/charts/prometheus/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: v2.52.0 + helm.sh/chart: prometheus-25.21.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: prometheus + name: prometheus + namespace: monitoring + annotations: + {} +--- +# Source: gateway-addons-helm/charts/tempo/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: tempo + namespace: monitoring + labels: + helm.sh/chart: tempo-1.3.1 + app.kubernetes.io/name: tempo + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.1.1" + app.kubernetes.io/managed-by: Helm +automountServiceAccountToken: true +--- +# Source: gateway-addons-helm/charts/grafana/templates/secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: grafana + namespace: monitoring + labels: + helm.sh/chart: grafana-8.0.0 + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "11.0.0" + app.kubernetes.io/managed-by: Helm +type: Opaque +data: + + admin-user: "YWRtaW4=" + admin-password: "YWRtaW4=" + ldap-toml: "" +--- +# Source: gateway-addons-helm/charts/fluent-bit/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: fluent-bit + namespace: monitoring + labels: + helm.sh/chart: fluent-bit-0.30.4 + app.kubernetes.io/name: fluent-bit + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.1.4" + app.kubernetes.io/managed-by: Helm +data: + custom_parsers.conf: | + [PARSER] + Name docker_no_time + Format json + Time_Keep Off + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L + + fluent-bit.conf: | + [SERVICE] + Daemon Off + Flush 1 + Log_Level info + Parsers_File parsers.conf + Parsers_File custom_parsers.conf + HTTP_Server On + HTTP_Listen 0.0.0.0 + HTTP_Port 2020 + Health_Check On + + [INPUT] + Name tail + Path /var/log/containers/*.log + multiline.parser docker, cri + Tag kube.* + Mem_Buf_Limit 5MB + Skip_Long_Lines On + + [FILTER] + Name kubernetes + Match kube.* + Merge_Log On + Keep_Log Off + K8S-Logging.Parser On + K8S-Logging.Exclude On + + [FILTER] + Name grep + Match kube.* + Regex $kubernetes['container_name'] ^envoy$ + + [FILTER] + Name parser + Match kube.* + Key_Name log + Parser envoy + Reserve_Data True + + [OUTPUT] + Name loki + Match kube.* + Host loki.monitoring.svc.cluster.local + Port 3100 + Labels job=fluentbit, app=$kubernetes['labels']['app'], k8s_namespace_name=$kubernetes['namespace_name'], k8s_pod_name=$kubernetes['pod_name'], k8s_container_name=$kubernetes['container_name'] +--- +# Source: gateway-addons-helm/charts/grafana/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana + namespace: monitoring + labels: + helm.sh/chart: grafana-8.0.0 + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "11.0.0" + app.kubernetes.io/managed-by: Helm +data: + + grafana.ini: | + [analytics] + check_for_updates = true + [grafana_net] + url = https://grafana.net + [log] + mode = console + [paths] + data = /var/lib/grafana/ + logs = /var/log/grafana + plugins = /var/lib/grafana/plugins + provisioning = /etc/grafana/provisioning + [server] + domain = '' + datasources.yaml: | + apiVersion: 1 + datasources: + - name: Prometheus + type: prometheus + url: http://prometheus + dashboardproviders.yaml: | + apiVersion: 1 + providers: + - disableDeletion: false + editable: true + folder: envoy-gateway + name: envoy-gateway + options: + path: /var/lib/grafana/dashboards/envoy-gateway + orgId: 1 + type: file +--- +# Source: gateway-addons-helm/charts/loki/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: loki + labels: + helm.sh/chart: loki-4.8.0 + app.kubernetes.io/name: loki + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.7.3" + app.kubernetes.io/managed-by: Helm +data: + config.yaml: | + auth_enabled: false + common: + compactor_address: 'loki' + path_prefix: /var/loki + replication_factor: 1 + storage: + filesystem: + chunks_directory: /var/loki/chunks + rules_directory: /var/loki/rules + limits_config: + enforce_metric_name: false + max_cache_freshness_per_query: 10m + reject_old_samples: true + reject_old_samples_max_age: 168h + split_queries_by_interval: 15m + memberlist: + join_members: + - loki-memberlist + query_range: + align_queries_with_step: true + ruler: + storage: + type: local + runtime_config: + file: /etc/loki/runtime-config/runtime-config.yaml + schema_config: + configs: + - from: "2022-01-11" + index: + period: 24h + prefix: loki_index_ + object_store: filesystem + schema: v12 + store: boltdb-shipper + server: + grpc_listen_port: 9095 + http_listen_port: 3100 + storage_config: + hedging: + at: 250ms + max_per_second: 20 + up_to: 3 + table_manager: + retention_deletes_enabled: false + retention_period: 0 +--- +# Source: gateway-addons-helm/charts/loki/templates/runtime-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: loki-runtime + labels: + helm.sh/chart: loki-4.8.0 + app.kubernetes.io/name: loki + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.7.3" + app.kubernetes.io/managed-by: Helm +data: + runtime-config.yaml: | + + {} +--- +# Source: gateway-addons-helm/charts/prometheus/templates/cm.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: v2.52.0 + helm.sh/chart: prometheus-25.21.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: prometheus + name: prometheus + namespace: monitoring +data: + allow-snippet-annotations: "false" + alerting_rules.yml: | + {} + alerts: | + {} + prometheus.yml: | + global: + evaluation_interval: 1m + scrape_interval: 15s + scrape_timeout: 10s + rule_files: + - /etc/config/recording_rules.yml + - /etc/config/alerting_rules.yml + - /etc/config/rules + - /etc/config/alerts + scrape_configs: + - job_name: prometheus + static_configs: + - targets: + - localhost:9090 + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-apiservers + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: default;kubernetes;https + source_labels: + - __meta_kubernetes_namespace + - __meta_kubernetes_service_name + - __meta_kubernetes_endpoint_port_name + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-nodes + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - replacement: kubernetes.default.svc:443 + target_label: __address__ + - regex: (.+) + replacement: /api/v1/nodes/$1/proxy/metrics + source_labels: + - __meta_kubernetes_node_name + target_label: __metrics_path__ + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-nodes-cadvisor + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - replacement: kubernetes.default.svc:443 + target_label: __address__ + - regex: (.+) + replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor + source_labels: + - __meta_kubernetes_node_name + target_label: __metrics_path__ + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + - honor_labels: true + job_name: kubernetes-service-endpoints + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape + - action: drop + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (.+?)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_service_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_service_name + target_label: service + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + - honor_labels: true + job_name: kubernetes-service-endpoints-slow + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (.+?)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_service_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_service_name + target_label: service + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + scrape_interval: 5m + scrape_timeout: 30s + - honor_labels: true + job_name: prometheus-pushgateway + kubernetes_sd_configs: + - role: service + relabel_configs: + - action: keep + regex: pushgateway + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_probe + - honor_labels: true + job_name: kubernetes-services + kubernetes_sd_configs: + - role: service + metrics_path: /probe + params: + module: + - http_2xx + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_probe + - source_labels: + - __address__ + target_label: __param_target + - replacement: blackbox + target_label: __address__ + - source_labels: + - __param_target + target_label: instance + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_service_name + target_label: service + - honor_labels: true + job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape + - action: drop + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4}) + replacement: '[$2]:$1' + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: replace + regex: (\d+);((([0-9]+?)(\.|$)){4}) + replacement: $2:$1 + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - action: drop + regex: Pending|Succeeded|Failed|Completed + source_labels: + - __meta_kubernetes_pod_phase + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + - honor_labels: true + job_name: kubernetes-pods-slow + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4}) + replacement: '[$2]:$1' + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: replace + regex: (\d+);((([0-9]+?)(\.|$)){4}) + replacement: $2:$1 + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - action: drop + regex: Pending|Succeeded|Failed|Completed + source_labels: + - __meta_kubernetes_pod_phase + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + scrape_interval: 5m + scrape_timeout: 30s + recording_rules.yml: | + {} + rules: | + {} +--- +# Source: gateway-addons-helm/charts/tempo/templates/configmap-tempo.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: tempo + namespace: monitoring + labels: + helm.sh/chart: tempo-1.3.1 + app.kubernetes.io/name: tempo + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.1.1" + app.kubernetes.io/managed-by: Helm +data: + overrides.yaml: | + overrides: + {} + tempo.yaml: | + multitenancy_enabled: false + usage_report: + reporting_enabled: true + compactor: + compaction: + block_retention: 24h + distributor: + receivers: + jaeger: + protocols: + grpc: + endpoint: 0.0.0.0:14250 + thrift_binary: + endpoint: 0.0.0.0:6832 + thrift_compact: + endpoint: 0.0.0.0:6831 + thrift_http: + endpoint: 0.0.0.0:14268 + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + ingester: + {} + server: + http_listen_port: 3100 + storage: + trace: + backend: local + local: + path: /var/tempo/traces + wal: + path: /var/tempo/wal + querier: + {} + query_frontend: + {} + overrides: + per_tenant_override_config: /conf/overrides.yaml +--- +# Source: gateway-addons-helm/templates/dashboards_config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboards + namespace: 'monitoring' +data: + envoy-clusters.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Envoy proxy monitoring Dashboard with cluster and service level templates. ", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 11021, + "graphTooltip": 0, + "id": 2, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 3, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "#299c46", + "value": 2 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 0, + "y": 0 + }, + "id": 9, + "maxDataPoints": 100, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(envoy_server_live{})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Live servers", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 5, + "y": 0 + }, + "id": 12, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "avg(envoy_server_uptime)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Avg uptime per node", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 9, + "y": 0 + }, + "id": 11, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "builder", + "expr": "SUM(envoy_server_memory_allocated{})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Allocated Memory", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 14, + "y": 0 + }, + "id": 13, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "SUM(envoy_server_memory_heap_size)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Heap Size", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 18, + "y": 0 + }, + "id": 19, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "(sum(envoy_cluster_membership_healthy{envoy_cluster_name=~\"$cluster\"}) - sum(envoy_cluster_membership_total{envoy_cluster_name=~\"$cluster\"}))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Unhealthy Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "text": "NOT WELL" + }, + "1": { + "text": "OK" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 0 + }, + { + "color": "#299c46", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 21, + "y": 0 + }, + "id": 20, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "(sum(envoy_cluster_membership_total{envoy_cluster_name=~\"$cluster\"})-sum(envoy_cluster_membership_healthy{envoy_cluster_name=~\"$cluster\"})) == bool 0", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Cluster State", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(envoy_cluster_upstream_cx_active{envoy_cluster_name=~\"$cluster\"}) by (envoy_cluster_name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{envoy_cluster_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Total active connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "httproute/default/backend/rule/0" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(irate(envoy_cluster_upstream_rq_total{envoy_cluster_name=~\"$cluster\"}[5m])) by (envoy_cluster_name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{envoy_cluster_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Total requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(irate(envoy_cluster_upstream_cx_rx_bytes_total{envoy_cluster_name=~\"$cluster\"}[5m])) by (envoy_cluster_name)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{envoy_cluster_name}} - in", + "range": true, + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(irate(envoy_cluster_upstream_cx_tx_bytes_total{envoy_cluster_name=~\"$cluster\"}[5m])) by (envoy_cluster_name)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{envoy_cluster_name}} - out", + "range": true, + "refId": "B" + } + ], + "title": "Upstream Network Traffic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "builder", + "expr": "sum(irate(envoy_http_downstream_cx_rx_bytes_total{envoy_http_conn_manager_prefix=~\"http\"}[5m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{service}} - in", + "range": true, + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "builder", + "expr": "sum(irate(envoy_http_downstream_cx_tx_bytes_total{envoy_http_conn_manager_prefix=~\"http\"}[5m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{service}} - out", + "range": true, + "refId": "B" + } + ], + "title": "Downstream Network Traffic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~\"$cluster\"}[5m])) by (le, envoy_cluster_name))", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} 99%", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~\"$cluster\"}[5m])) by (le, envoy_cluster_name))", + "hide": false, + "instant": false, + "legendFormat": "{{envoy_cluster_name}} 90%", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~\"$cluster\"}[5m])) by (le, envoy_cluster_name))", + "hide": false, + "instant": false, + "legendFormat": "{{envoy_cluster_name}} 50%", + "range": true, + "refId": "C" + } + ], + "title": "Upstream Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 19 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_upstream_rq_xx{envoy_response_code_class=~\"2\", envoy_cluster_name=~\"$cluster\"}[5m])) by (envoy_cluster_name)", + "instant": false, + "legendFormat": "{{envoy_cluster_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Upstream 2xx Responses", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 19 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_upstream_rq_xx{envoy_cluster_name=~\"$cluster\",envoy_response_code_class=~\"4\"}[1m])) by (envoy_cluster_name)", + "instant": false, + "legendFormat": "{{envoy_cluster_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Upstream 4xx Responses", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(envoy_cluster_membership_healthy{envoy_cluster_name=~\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "healthy", + "range": true, + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(envoy_cluster_membership_total{envoy_cluster_name=~\"$cluster\",service=~\"$service\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "total", + "refId": "B" + } + ], + "title": "Downstream members", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 27 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_upstream_rq_xx{envoy_cluster_name=~\"$cluster\",envoy_response_code_class=~\"5\"}[5m])) by (envoy_cluster_name)", + "instant": false, + "legendFormat": "{{envoy_cluster_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Upstream 5xx Responses", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 27 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_upstream_rq_xx{envoy_cluster_name=~\"$cluster\",envoy_response_code_class=~\"3\"}[5m])) by (envoy_cluster_name)", + "instant": false, + "legendFormat": "{{envoy_cluster_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Upstream 3xx Responses", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "Data Plane" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "httproute/default/backend/rule/0", + "value": "httproute/default/backend/rule/0" + }, + "datasource": { + "uid": "$datasource" + }, + "definition": "label_values(envoy_cluster_name)", + "hide": 0, + "includeAll": true, + "label": "Cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": { + "query": "label_values(envoy_cluster_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Envoy Clusters", + "uid": "8WkEOMnANKE6PW5hhpVv", + "version": 1, + "weekStart": "" + } + envoy-gateway-global.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Envoy Gateway monitoring Dashboard with exported metrics.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [], + "title": "Watching Components", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "How long in seconds a subscribed watchable is handled.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 7, + "x": 0, + "y": 1 + }, + "id": 1, + "maxPerRow": 3, + "options": { + "displayMode": "basic", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.0.0", + "repeat": "Runner", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(le) (watchable_subscribed_duration_seconds_bucket{runner=~\"$Runner\", namespace=\"$Namespace\"})", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{le}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Duration Bucket: $Runner", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 7, + "y": 1 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": false + }, + "pluginVersion": "11.0.0", + "repeat": "Runner", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "avg by(runner) (watchable_subscribed_duration_seconds_sum{runner=~\"$Runner\", namespace=\"$Namespace\"})", + "format": "time_series", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Avg", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "max by(runner) (watchable_subscribed_duration_seconds_sum{runner=~\"$Runner\", namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Max", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Duration Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Current depth of watchable map.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "super-light-blue", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 10, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": false + }, + "pluginVersion": "11.0.0", + "repeat": "Runner", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(runner) (watchable_depth{runner=~\"$Runner\", namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Depth", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Success" + }, + "properties": [ + { + "id": "displayName", + "value": "Success" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 7, + "x": 12, + "y": 1 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "repeat": "Runner", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(runner) (watchable_subscribed_total{runner=~\"$Runner\", namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Total", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(runner) (watchable_subscribed_errors_total{runner=~\"$Runner\", namespace=\"$Namespace\"}) OR vector(0)", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Errors", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "$A-$B", + "hide": false, + "refId": "Success", + "type": "math" + } + ], + "title": "Statistics", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-red", + "value": null + }, + { + "color": "#EAB839", + "value": 30 + }, + { + "color": "semi-dark-green", + "value": 70 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 19, + "y": 1 + }, + "id": 23, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.0.0", + "repeat": "Runner", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(runner) (watchable_subscribed_total{runner=~\"$Runner\", namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "hide": true, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(runner) (watchable_subscribed_errors_total{runner=~\"$Runner\", namespace=\"$Namespace\"}) OR vector(0)", + "fullMetaSearch": false, + "hide": true, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "(($A-$B) / $A) * 100", + "hide": false, + "refId": "C", + "type": "math" + } + ], + "title": "Success Rate", + "type": "gauge" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 35, + "panels": [], + "title": "Status Updater", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "How long a status update takes to finish for all Kind.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "super-light-blue", + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 0, + "y": 37 + }, + "id": 61, + "options": { + "displayMode": "basic", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(le) (status_update_duration_seconds_bucket{namespace=\"$Namespace\"})", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{le}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total Duration Bucket", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "#EAB839", + "value": 0.2 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 6, + "y": 37 + }, + "id": 82, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "status_update_duration_seconds_sum{namespace=\"$Namespace\"}", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Avg Duration", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "#EAB839", + "value": 0.1 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 12, + "y": 37 + }, + "id": 83, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "status_update_duration_seconds_sum{namespace=\"$Namespace\"}", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Max Duration", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 0.01 + }, + { + "color": "red", + "value": 0.1 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 18, + "y": 37 + }, + "id": 84, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "logmin" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "status_update_duration_seconds_sum{namespace=\"$Namespace\"}", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Min Duration", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of status updates by object kind.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 0, + "y": 46 + }, + "id": 56, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "status_update_total{namespace=\"$Namespace\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "#EAB839", + "value": 50 + }, + { + "color": "green", + "value": 85 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 14, + "x": 10, + "y": 46 + }, + "id": 105, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(kind) (status_update_success_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "hide": true, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(kind) (status_update_success_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "hide": true, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "($B / $A) * 100", + "hide": false, + "refId": "Rate:", + "type": "math" + } + ], + "title": "Success Rate", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Number of status updates that succeeded by object kind.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 54 + }, + "id": 57, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "status_update_success_total{namespace=\"$Namespace\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Success", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Number of status updates that are no-ops by object kind. This is a subset of successful status updates.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 54 + }, + "id": 59, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "status_update_noop_total{namespace=\"$Namespace\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "No-ops", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Number of status updates that failed by object kind.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 54 + }, + "id": 58, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "status_update_failed_total{namespace=\"$Namespace\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Fail", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Number of status update conflicts encountered by object kind.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 54 + }, + "id": 60, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "status_update_conflict_total{namespace=\"$Namespace\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Conflict", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 62 + }, + "id": 126, + "panels": [], + "title": "xDS Server", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "super-light-green", + "mode": "shades" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 0, + "y": 63 + }, + "id": 127, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": false + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "xds_snapshot_creation_total{namespace=\"$Namespace\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Total", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "xds_snapshot_creation_success{namespace=\"$Namespace\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Success", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "xds_snapshot_creation_failed{namespace=\"$Namespace\"} OR on() vector(0)", + "hide": false, + "instant": false, + "legendFormat": "Fail", + "range": true, + "refId": "C" + } + ], + "title": "Creation Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "green", + "value": 85 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 10, + "y": 63 + }, + "id": 148, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum(xds_snapshot_creation_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "hide": true, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum(xds_snapshot_creation_success{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "hide": true, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "($B / $A) * 100", + "hide": false, + "refId": "C", + "type": "math" + } + ], + "title": "Creation Success Rate", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 15, + "y": 63 + }, + "id": 149, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "max(xds_delta_stream_duration_seconds_bucket{namespace=\"$Namespace\"})", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Finished Stream", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Maximum duration seconds for finished xDS delta stream connection.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 63 + }, + "id": 150, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "xds_delta_stream_duration_seconds_sum{namespace=\"$Namespace\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Max Duration", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Minimum duration seconds for finished xDS delta stream connection.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 67 + }, + "id": 151, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "min" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "xds_delta_stream_duration_seconds_sum{namespace=\"$Namespace\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Min Duration", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of xds snapshot cache updates by node id.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 10, + "x": 0, + "y": 71 + }, + "id": 152, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "xds_snapshot_update_total{namespace=\"$Namespace\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{nodeID}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Update Total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "green", + "value": 85 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 5, + "x": 10, + "y": 71 + }, + "id": 155, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum(xds_snapshot_update_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "hide": true, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum(xds_snapshot_update_success{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "hide": true, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "($B / $A) * 100", + "hide": false, + "refId": "C", + "type": "math" + } + ], + "title": "Update Success Rate", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Number of xds snapshot cache updates that succeed by node id.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 9, + "x": 15, + "y": 71 + }, + "id": 153, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "xds_snapshot_update_success{namespace=\"$Namespace\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{nodeID}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Update Success", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Number of xds snapshot cache updates that failed by node id.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 9, + "x": 15, + "y": 76 + }, + "id": 154, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "xds_snapshot_update_failed{namespace=\"$Namespace\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{nodeID}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Update Fail", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 81 + }, + "id": 156, + "panels": [], + "title": "Infrastructure Manager", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 82 + }, + "id": 199, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(le) (resource_apply_duration_seconds_bucket{namespace=\"$Namespace\"})", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{le}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total Apply Duration Bucket", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 0.3 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 82 + }, + "id": 220, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(kind) (resource_apply_duration_seconds_sum{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Avg Apply Duration", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 0.3 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 82 + }, + "id": 221, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(kind) (resource_apply_duration_seconds_sum{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Max Apply Duration", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 0.3 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 82 + }, + "id": 222, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "logmin" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(kind) (resource_apply_duration_seconds_sum{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Min Apply Duration", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of applied resources sumed by kind (include No-ops).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 90 + }, + "id": 157, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(kind) (resource_apply_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total Applied Resources", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of applied resources sumed by infra name.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 90 + }, + "id": 178, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(name) (resource_apply_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total Applied Infrastructures", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of applied resources that succeed sumed by kind (include No-ops).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 10, + "x": 0, + "y": 97 + }, + "id": 229, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(kind) (resource_apply_success{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Success Applied Resources", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of applied resources that failed sumed by kind (include No-ops).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 10, + "x": 10, + "y": 97 + }, + "id": 230, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(kind) (resource_apply_failed{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Fail Applied Resources", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 104 + }, + "id": 223, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (resource_delete_duration_seconds_bucket{namespace=\"$Namespace\"})", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{le}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total Delete Duration Bucket", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 0.1 + }, + { + "color": "red", + "value": 0.3 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 104 + }, + "id": 224, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(kind) (resource_delete_duration_seconds_sum{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Avg Delete Duration", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 0.1 + }, + { + "color": "red", + "value": 0.3 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 104 + }, + "id": 225, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(kind) (resource_delete_duration_seconds_sum{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Max Delete Duration", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 0.1 + }, + { + "color": "red", + "value": 0.3 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 104 + }, + "id": 226, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "logmin" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(kind) (resource_delete_duration_seconds_sum{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Min Delete Duration", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of deleted resources sumed by kind (include No-ops).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 112 + }, + "id": 227, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(kind) (resource_delete_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total Deleted Resources", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of deleted resources sumed by infra name.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 112 + }, + "id": 228, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(name) (resource_delete_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total Deleted Infrastructures", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of deleted resources that succeed sumed by kind (include No-ops).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 10, + "x": 0, + "y": 119 + }, + "id": 232, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(kind) (resource_delete_success{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Success Deleted Resources", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of deleted resources that failed sumed by kind (include No-ops).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 10, + "x": 10, + "y": 119 + }, + "id": 233, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(kind) (resource_delete_failed{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Fail Deleted Resources", + "type": "stat" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [ + "Control Plane" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "envoy-gateway-system", + "value": "envoy-gateway-system" + }, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "definition": "label_values(watchable_depth,namespace)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "Namespace", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(watchable_depth,namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "definition": "label_values(watchable_depth,runner)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "Runner", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(watchable_depth,runner)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": {}, + "timezone": "browser", + "title": "Envoy Gateway Global", + "uid": "bdn8lriao7myoa", + "version": 1, + "weekStart": "" + } + envoy-global.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Envoy proxy monitoring Dashboard with service level templates.", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 11022, + "graphTooltip": 0, + "id": 3, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 3, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "#299c46", + "value": 2 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 0, + "y": 0 + }, + "id": 37, + "maxDataPoints": 100, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "builder", + "expr": "sum(envoy_server_live)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Live servers", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 5, + "y": 0 + }, + "id": 39, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "builder", + "expr": "avg by(pod) (envoy_server_uptime{namespace=~\"$Namespace\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Avg uptime per node", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 10, + "y": 0 + }, + "id": 43, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by(pod) (envoy_server_memory_heap_size{namespace=~\"$Namespace\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Heap Size", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 15, + "y": 0 + }, + "id": 41, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "builder", + "expr": "sum by(pod) (envoy_server_memory_allocated{namespace=~\"$Namespace\"})", + "hide": false, + "instant": false, + "range": true, + "refId": "B" + } + ], + "title": "Allocated Memory", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 24, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "DownStream", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_http_downstream_rq_total[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Envoy HTTP Downstream Rq total", + "range": true, + "refId": "A" + } + ], + "title": "Downstream RPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 8 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "sum by(namespace) (rate(envoy_http_downstream_cx_total{namespace=~\"$Namespace\"}[5m]))", + "instant": false, + "legendFormat": "{{namespace}}", + "range": true, + "refId": "A" + } + ], + "title": "Downstream CPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 8 + }, + "id": 16, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by(le) (rate(envoy_http_downstream_rq_time_bucket[5m])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{service}} 90%", + "range": true, + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "builder", + "expr": "histogram_quantile(0.5, sum by(le) (rate(envoy_http_downstream_rq_time_bucket[5m])))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{service}} 50% ", + "range": true, + "refId": "B" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "builder", + "expr": "histogram_quantile(0.99, sum by(le) (rate(envoy_http_downstream_rq_time_bucket[5m])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{service}} 99%", + "range": true, + "refId": "C" + } + ], + "title": "Downstream Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by(namespace) (envoy_listener_downstream_cx_active{namespace=~\"$Namespace\"})", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "title": "Downstream Total Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 16 + }, + "id": 32, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "sum by(namespace) (rate(envoy_tcp_downstream_cx_total{namespace=~\"$Namespace\"}[5m]))", + "instant": false, + "legendFormat": "{{namespace}}", + "range": true, + "refId": "A" + } + ], + "title": "TCP Downstream CPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 16 + }, + "id": 31, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "sum by(namespace) (rate(envoy_tcp_downstream_cx_rx_bytes_total{namespace=~\"$Namespace\"}[5m]))", + "instant": false, + "legendFormat": "{{namespace}}", + "range": true, + "refId": "A" + } + ], + "title": "TCP Downstream Bytes Rx/second", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 16 + }, + "id": 33, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "sum by(namespace) (rate(envoy_tcp_downstream_cx_tx_bytes_total{namespace=~\"$Namespace\"}[5m]))", + "instant": false, + "legendFormat": "{{namespace}}", + "range": true, + "refId": "A" + } + ], + "title": "TCP Downstream Bytes Tx/Second", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 22, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "UpStream", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Displays the number of Requests per Second being performed against each Upstream.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 25 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "builder", + "expr": "sum by(namespace) (rate(envoy_cluster_upstream_rq_total{namespace=~\"$Namespace\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Upstream RPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 25 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_upstream_cx_total{namespace=~\"$Namespace\",}[5m])) by (namespace)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Upstream CPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 25 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{namespace=~\"$Namespace\"}[5m])) by (le, namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{namespace}} 99%", + "range": true, + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum(rate(envoy_cluster_upstream_rq_time_bucket{namespace=~\"$Namespace\"}[5m])) by (le, namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{namespace}} 90%", + "range": true, + "refId": "C" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(envoy_cluster_upstream_rq_time_bucket{namespace=~\"$Namespace\"}[5m])) by (le, namespace))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{namespace}} 50% ", + "range": true, + "refId": "B" + } + ], + "title": "Upstream Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 25 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(envoy_cluster_upstream_cx_active{namespace=~\"$Namespace\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{namespace}}", + "range": true, + "refId": "A" + } + ], + "title": "Upstream Total Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 33 + }, + "id": 34, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "sum by(namespace) (rate(envoy_cluster_upstream_cx_rx_bytes_total{namespace=~\"$Namespace\"}[5m]))", + "instant": false, + "legendFormat": "{{namespace}}", + "range": true, + "refId": "A" + } + ], + "title": "Upstream Bytes Rx/Second", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 33 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "sum by(namespace) (rate(envoy_cluster_upstream_cx_rx_bytes_total{namespace=~\"$Namespace\"}[5m]))", + "instant": false, + "legendFormat": "{{namespace}}", + "range": true, + "refId": "A" + } + ], + "title": "Upstream Bytes Tx/Second", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 28, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Upstream Response Codes", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 42 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "sum(rate(envoy_cluster_upstream_rq_xx{envoy_response_code_class=~\"2\"}[5m]))", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Value", + "range": true, + "refId": "A" + } + ], + "title": "Upstream 2xx Responses", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 42 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "builder", + "expr": "sum(rate(envoy_cluster_upstream_rq_xx{envoy_response_code_class=~\"3\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "value", + "range": true, + "refId": "A" + } + ], + "title": "Upstream 3xx Responses", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 42 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_upstream_rq_xx{envoy_response_code_class=\"4\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Value", + "range": true, + "refId": "A" + } + ], + "title": "Upstream 4xx Responses", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 42 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_upstream_rq_xx{envoy_response_code_class=\"5\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Value", + "range": true, + "refId": "A" + } + ], + "title": "Upstream 5xx Responses", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 26, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Total", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 51 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "avg(envoy_cluster_membership_healthy{namespace=~\"$Namespace\"}) by (namespace) / avg(envoy_cluster_membership_total{namespace=~\"$Namespace\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "range": true, + "refId": "A" + } + ], + "title": "Endpoint Percentage Health", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 51 + }, + "id": 19, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "builder", + "expr": "sum by(namespace) (envoy_cluster_membership_total{namespace=~\"$Namespace\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "range": true, + "refId": "A" + } + ], + "title": "Total Endpoints", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 51 + }, + "id": 18, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "builder", + "expr": "sum by(namespace) (envoy_cluster_membership_healthy{namespace=~\"$Namespace\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "range": true, + "refId": "A" + } + ], + "title": "Healthy Endpoints", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 51 + }, + "id": 20, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by(namespace) (envoy_cluster_membership_total{namespace=~\"$Namespace\"}) - sum by(namespace) (envoy_cluster_membership_healthy{namespace=~\"$Namespace\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "range": true, + "refId": "A" + } + ], + "title": "Unhealthy Endpoints", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": [ + "Data Plane" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "uid": "$datasource" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "Namespace", + "options": [], + "query": "label_values(envoy_cluster_upstream_rq_time_bucket,namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Envoy Global", + "uid": "heHhNSFf6Na8vIZWRs8H", + "version": 1, + "weekStart": "" + } + envoy-pod-resource.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Envoy Pod Memory and CPU Usage", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 4, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by(pod) (container_memory_working_set_bytes{container=~\"envoy\"}/1000000)", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Memory Working Set Envoy Pods(mb)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{container=\"envoy\"}[5m]))", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage Envoy Pods", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "Data Plane" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": {}, + "timezone": "", + "title": "Envoy Pod Resources", + "uid": "f2279235-80b7-4c85-84f4-f25a3bf3eac0", + "version": 1, + "weekStart": "" + } +--- +# Source: gateway-addons-helm/charts/fluent-bit/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: fluent-bit + labels: + helm.sh/chart: fluent-bit-0.30.4 + app.kubernetes.io/name: fluent-bit + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.1.4" + app.kubernetes.io/managed-by: Helm +rules: + - apiGroups: + - "" + resources: + - namespaces + - pods + verbs: + - get + - list + - watch +--- +# Source: gateway-addons-helm/charts/grafana/templates/clusterrole.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + labels: + helm.sh/chart: grafana-8.0.0 + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "11.0.0" + app.kubernetes.io/managed-by: Helm + name: grafana-clusterrole +rules: [] +--- +# Source: gateway-addons-helm/charts/prometheus/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: v2.52.0 + helm.sh/chart: prometheus-25.21.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: prometheus + name: prometheus +rules: + - apiGroups: + - "" + resources: + - nodes + - nodes/proxy + - nodes/metrics + - services + - endpoints + - pods + - ingresses + - configmaps + verbs: + - get + - list + - watch + - apiGroups: + - "extensions" + - "networking.k8s.io" + resources: + - ingresses/status + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "discovery.k8s.io" + resources: + - endpointslices + verbs: + - get + - list + - watch + - nonResourceURLs: + - "/metrics" + verbs: + - get +--- +# Source: gateway-addons-helm/charts/fluent-bit/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: fluent-bit + labels: + helm.sh/chart: fluent-bit-0.30.4 + app.kubernetes.io/name: fluent-bit + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.1.4" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: fluent-bit +subjects: + - kind: ServiceAccount + name: fluent-bit + namespace: monitoring +--- +# Source: gateway-addons-helm/charts/grafana/templates/clusterrolebinding.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: grafana-clusterrolebinding + labels: + helm.sh/chart: grafana-8.0.0 + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "11.0.0" + app.kubernetes.io/managed-by: Helm +subjects: + - kind: ServiceAccount + name: grafana + namespace: monitoring +roleRef: + kind: ClusterRole + name: grafana-clusterrole + apiGroup: rbac.authorization.k8s.io +--- +# Source: gateway-addons-helm/charts/prometheus/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: v2.52.0 + helm.sh/chart: prometheus-25.21.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: prometheus + name: prometheus +subjects: + - kind: ServiceAccount + name: prometheus + namespace: monitoring +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +--- +# Source: gateway-addons-helm/charts/grafana/templates/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: grafana + namespace: monitoring + labels: + helm.sh/chart: grafana-8.0.0 + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "11.0.0" + app.kubernetes.io/managed-by: Helm +rules: [] +--- +# Source: gateway-addons-helm/charts/grafana/templates/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: grafana + namespace: monitoring + labels: + helm.sh/chart: grafana-8.0.0 + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "11.0.0" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: grafana +subjects: +- kind: ServiceAccount + name: grafana + namespace: monitoring +--- +# Source: gateway-addons-helm/charts/fluent-bit/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: fluent-bit + namespace: monitoring + labels: + helm.sh/chart: fluent-bit-0.30.4 + app.kubernetes.io/name: fluent-bit + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.1.4" + app.kubernetes.io/managed-by: Helm +spec: + type: ClusterIP + ports: + - port: 2020 + targetPort: http + protocol: TCP + name: http + selector: + app.kubernetes.io/name: fluent-bit + app.kubernetes.io/instance: gateway-addons-helm +--- +# Source: gateway-addons-helm/charts/grafana/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: grafana + namespace: monitoring + labels: + helm.sh/chart: grafana-8.0.0 + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "11.0.0" + app.kubernetes.io/managed-by: Helm +spec: + type: LoadBalancer + ports: + - name: service + port: 80 + protocol: TCP + targetPort: 3000 + selector: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: gateway-addons-helm +--- +# Source: gateway-addons-helm/charts/loki/templates/service-memberlist.yaml +apiVersion: v1 +kind: Service +metadata: + name: loki-memberlist + labels: + helm.sh/chart: loki-4.8.0 + app.kubernetes.io/name: loki + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.7.3" + app.kubernetes.io/managed-by: Helm +spec: + type: ClusterIP + clusterIP: None + ports: + - name: tcp + port: 7946 + targetPort: http-memberlist + protocol: TCP + selector: + app.kubernetes.io/name: loki + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/part-of: memberlist +--- +# Source: gateway-addons-helm/charts/loki/templates/single-binary/service-headless.yaml +apiVersion: v1 +kind: Service +metadata: + name: loki-headless + namespace: monitoring + labels: + helm.sh/chart: loki-4.8.0 + app.kubernetes.io/name: loki + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.7.3" + app.kubernetes.io/managed-by: Helm + variant: headless + prometheus.io/service-monitor: "false" +spec: + clusterIP: None + ports: + - name: http-metrics + port: 3100 + targetPort: http-metrics + protocol: TCP + selector: + app.kubernetes.io/name: loki + app.kubernetes.io/instance: gateway-addons-helm +--- +# Source: gateway-addons-helm/charts/loki/templates/single-binary/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: loki + labels: + helm.sh/chart: loki-4.8.0 + app.kubernetes.io/name: loki + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.7.3" + app.kubernetes.io/managed-by: Helm +spec: + type: ClusterIP + ports: + - name: http-metrics + port: 3100 + targetPort: http-metrics + protocol: TCP + - name: grpc + port: 9095 + targetPort: grpc + protocol: TCP + selector: + app.kubernetes.io/name: loki + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/component: single-binary +--- +# Source: gateway-addons-helm/charts/prometheus/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: v2.52.0 + helm.sh/chart: prometheus-25.21.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: prometheus + name: prometheus + namespace: monitoring +spec: + ports: + - name: http + port: 80 + protocol: TCP + targetPort: 9090 + selector: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: gateway-addons-helm + sessionAffinity: None + type: "LoadBalancer" +--- +# Source: gateway-addons-helm/charts/tempo/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: tempo + namespace: monitoring + labels: + helm.sh/chart: tempo-1.3.1 + app.kubernetes.io/name: tempo + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.1.1" + app.kubernetes.io/managed-by: Helm +spec: + type: LoadBalancer + ports: + - name: tempo-prom-metrics + port: 3100 + targetPort: 3100 + - name: tempo-jaeger-thrift-compact + port: 6831 + protocol: UDP + targetPort: 6831 + - name: tempo-jaeger-thrift-binary + port: 6832 + protocol: UDP + targetPort: 6832 + - name: tempo-jaeger-thrift-http + port: 14268 + protocol: TCP + targetPort: 14268 + - name: grpc-tempo-jaeger + port: 14250 + protocol: TCP + targetPort: 14250 + - name: tempo-zipkin + port: 9411 + protocol: TCP + targetPort: 9411 + - name: tempo-otlp-legacy + port: 55680 + protocol: TCP + targetPort: 55680 + - name: tempo-otlp-http-legacy + port: 55681 + protocol: TCP + targetPort: 4318 + - name: grpc-tempo-otlp + port: 4317 + protocol: TCP + targetPort: 4317 + - name: tempo-otlp-http + port: 4318 + protocol: TCP + targetPort: 4318 + - name: tempo-opencensus + port: 55678 + protocol: TCP + targetPort: 55678 + selector: + app.kubernetes.io/name: tempo + app.kubernetes.io/instance: gateway-addons-helm +--- +# Source: gateway-addons-helm/charts/fluent-bit/templates/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: fluent-bit + namespace: monitoring + labels: + helm.sh/chart: fluent-bit-0.30.4 + app.kubernetes.io/name: fluent-bit + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.1.4" + app.kubernetes.io/managed-by: Helm +spec: + selector: + matchLabels: + app.kubernetes.io/name: fluent-bit + app.kubernetes.io/instance: gateway-addons-helm + template: + metadata: + annotations: + checksum/config: 03d122555879033ccf6443369f73463490b100f195550b1483d337f497c749e3 + checksum/luascripts: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 + fluentbit.io/exclude: "true" + prometheus.io/path: /api/v1/metrics/prometheus + prometheus.io/port: "2020" + prometheus.io/scrape: "true" + labels: + app.kubernetes.io/name: fluent-bit + app.kubernetes.io/instance: gateway-addons-helm + spec: + serviceAccountName: fluent-bit + hostNetwork: false + dnsPolicy: ClusterFirst + containers: + - name: fluent-bit + image: "cr.fluentbit.io/fluent/fluent-bit:2.1.4" + imagePullPolicy: Always + ports: + - name: http + containerPort: 2020 + protocol: TCP + livenessProbe: + httpGet: + path: / + port: http + readinessProbe: + httpGet: + path: /api/v1/health + port: http + volumeMounts: + - mountPath: /fluent-bit/etc/fluent-bit.conf + name: config + subPath: fluent-bit.conf + - mountPath: /fluent-bit/etc/custom_parsers.conf + name: config + subPath: custom_parsers.conf + - mountPath: /var/log + name: varlog + - mountPath: /var/lib/docker/containers + name: varlibdockercontainers + readOnly: true + - mountPath: /etc/machine-id + name: etcmachineid + readOnly: true + volumes: + - name: config + configMap: + name: fluent-bit + - hostPath: + path: /var/log + name: varlog + - hostPath: + path: /var/lib/docker/containers + name: varlibdockercontainers + - hostPath: + path: /etc/machine-id + type: File + name: etcmachineid +--- +# Source: gateway-addons-helm/charts/grafana/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + namespace: monitoring + labels: + helm.sh/chart: grafana-8.0.0 + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "11.0.0" + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: gateway-addons-helm + strategy: + type: RollingUpdate + template: + metadata: + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: gateway-addons-helm + annotations: + checksum/config: fb83d9a834484a53eeac493c30f6d1f333707950c42350afddcbc340c63abaf9 + checksum/sc-dashboard-provider-config: 593c0a8778b83f11fe80ccb21dfb20bc46705e2be3178df1dc4c89d164c8cd9c + checksum/secret: bed677784356b2af7fb0d87455db21f077853059b594101a4f6532bfbd962a7f + kubectl.kubernetes.io/default-container: grafana + spec: + + serviceAccountName: grafana + automountServiceAccountToken: true + securityContext: + fsGroup: 472 + runAsGroup: 472 + runAsNonRoot: true + runAsUser: 472 + enableServiceLinks: true + containers: + - name: grafana + image: "docker.io/grafana/grafana:11.0.0" + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: config + mountPath: "/etc/grafana/grafana.ini" + subPath: grafana.ini + - name: storage + mountPath: "/var/lib/grafana" + - name: dashboards-envoy-gateway + mountPath: "/var/lib/grafana/dashboards/envoy-gateway" + - name: config + mountPath: "/etc/grafana/provisioning/datasources/datasources.yaml" + subPath: "datasources.yaml" + - name: config + mountPath: "/etc/grafana/provisioning/dashboards/dashboardproviders.yaml" + subPath: "dashboardproviders.yaml" + ports: + - name: grafana + containerPort: 3000 + protocol: TCP + - name: gossip-tcp + containerPort: 9094 + protocol: TCP + - name: gossip-udp + containerPort: 9094 + protocol: UDP + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: GF_SECURITY_ADMIN_USER + valueFrom: + secretKeyRef: + name: grafana + key: admin-user + - name: GF_SECURITY_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: grafana + key: admin-password + - name: GF_PATHS_DATA + value: /var/lib/grafana/ + - name: GF_PATHS_LOGS + value: /var/log/grafana + - name: GF_PATHS_PLUGINS + value: /var/lib/grafana/plugins + - name: GF_PATHS_PROVISIONING + value: /etc/grafana/provisioning + livenessProbe: + failureThreshold: 10 + httpGet: + path: /api/health + port: 3000 + initialDelaySeconds: 60 + timeoutSeconds: 30 + readinessProbe: + httpGet: + path: /api/health + port: 3000 + volumes: + - name: config + configMap: + name: grafana + - name: dashboards-envoy-gateway + configMap: + name: grafana-dashboards + - name: storage + emptyDir: {} +--- +# Source: gateway-addons-helm/charts/prometheus/templates/deploy.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: v2.52.0 + helm.sh/chart: prometheus-25.21.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: prometheus + name: prometheus + namespace: monitoring +spec: + selector: + matchLabels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: gateway-addons-helm + replicas: 1 + revisionHistoryLimit: 10 + strategy: + type: Recreate + rollingUpdate: null + template: + metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: v2.52.0 + helm.sh/chart: prometheus-25.21.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: prometheus + spec: + enableServiceLinks: true + serviceAccountName: prometheus + containers: + - name: prometheus-server-configmap-reload + image: "quay.io/prometheus-operator/prometheus-config-reloader:v0.73.2" + imagePullPolicy: "IfNotPresent" + args: + - --watched-dir=/etc/config + - --reload-url=http://127.0.0.1:9090/-/reload + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + + - name: prometheus-server + image: "prom/prometheus:v2.52.0" + imagePullPolicy: "IfNotPresent" + args: + - --storage.tsdb.retention.time=15d + - --config.file=/etc/config/prometheus.yml + - --storage.tsdb.path=/data + - --web.console.libraries=/etc/prometheus/console_libraries + - --web.console.templates=/etc/prometheus/consoles + - --web.enable-lifecycle + ports: + - containerPort: 9090 + readinessProbe: + httpGet: + path: /-/ready + port: 9090 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 4 + failureThreshold: 3 + successThreshold: 1 + livenessProbe: + httpGet: + path: /-/healthy + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 15 + timeoutSeconds: 10 + failureThreshold: 3 + successThreshold: 1 + volumeMounts: + - name: config-volume + mountPath: /etc/config + - name: storage-volume + mountPath: /data + subPath: "" + dnsPolicy: ClusterFirst + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + terminationGracePeriodSeconds: 300 + volumes: + - name: config-volume + configMap: + name: prometheus + - name: storage-volume + emptyDir: + {} +--- +# Source: gateway-addons-helm/charts/loki/templates/single-binary/statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: loki + labels: + helm.sh/chart: loki-4.8.0 + app.kubernetes.io/name: loki + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.7.3" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: single-binary + app.kubernetes.io/part-of: memberlist +spec: + replicas: 1 + podManagementPolicy: Parallel + updateStrategy: + rollingUpdate: + partition: 0 + serviceName: loki-headless + revisionHistoryLimit: 10 + + persistentVolumeClaimRetentionPolicy: + whenDeleted: Delete + whenScaled: Delete + selector: + matchLabels: + app.kubernetes.io/name: loki + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/component: single-binary + template: + metadata: + annotations: + checksum/config: 39a9cea617408d4add363b9ca660a8889e48b866eba2e8c8e4bfc10870b29162 + labels: + app.kubernetes.io/name: loki + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/component: single-binary + app.kubernetes.io/part-of: memberlist + spec: + serviceAccountName: loki + automountServiceAccountToken: true + enableServiceLinks: true + + securityContext: + fsGroup: 10001 + runAsGroup: 10001 + runAsNonRoot: true + runAsUser: 10001 + terminationGracePeriodSeconds: 30 + containers: + - name: loki + image: docker.io/grafana/loki:2.7.3 + imagePullPolicy: IfNotPresent + args: + - -config.file=/etc/loki/config/config.yaml + - -target=all + ports: + - name: http-metrics + containerPort: 3100 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: http-memberlist + containerPort: 7946 + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 30 + timeoutSeconds: 1 + volumeMounts: + - name: tmp + mountPath: /tmp + - name: config + mountPath: /etc/loki/config + - name: runtime-config + mountPath: /etc/loki/runtime-config + - name: storage + mountPath: /var/loki + resources: + {} + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/name: loki + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/component: single-binary + topologyKey: kubernetes.io/hostname + + volumes: + - name: tmp + emptyDir: {} + - name: config + configMap: + name: loki + - name: runtime-config + configMap: + name: loki-runtime + volumeClaimTemplates: + - metadata: + name: storage + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "10Gi" +--- +# Source: gateway-addons-helm/charts/tempo/templates/statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: tempo + namespace: monitoring + labels: + helm.sh/chart: tempo-1.3.1 + app.kubernetes.io/name: tempo + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.1.1" + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: tempo + app.kubernetes.io/instance: gateway-addons-helm + serviceName: tempo-headless + template: + metadata: + labels: + app.kubernetes.io/name: tempo + app.kubernetes.io/instance: gateway-addons-helm + annotations: + checksum/config: 0898f7ca87563d700c35d7ea1824cd042cf93fb2f05e254d12a854aa97a5c5e5 + spec: + serviceAccountName: tempo + automountServiceAccountToken: true + containers: + - args: + - -config.file=/conf/tempo.yaml + - -mem-ballast-size-mbs=1024 + image: grafana/tempo:2.1.1 + imagePullPolicy: IfNotPresent + name: tempo + ports: + - containerPort: 3100 + name: prom-metrics + - containerPort: 6831 + name: jaeger-thrift-c + protocol: UDP + - containerPort: 6832 + name: jaeger-thrift-b + protocol: UDP + - containerPort: 14268 + name: jaeger-thrift-h + - containerPort: 14250 + name: jaeger-grpc + - containerPort: 9411 + name: zipkin + - containerPort: 55680 + name: otlp-legacy + - containerPort: 4317 + name: otlp-grpc + - containerPort: 55681 + name: otlp-httplegacy + - containerPort: 4318 + name: otlp-http + - containerPort: 55678 + name: opencensus + resources: + {} + env: + volumeMounts: + - mountPath: /conf + name: tempo-conf + volumes: + - configMap: + name: tempo + name: tempo-conf + updateStrategy: + type: + RollingUpdate +--- +# Source: gateway-addons-helm/charts/grafana/templates/tests/test-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + helm.sh/chart: grafana-8.0.0 + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "11.0.0" + app.kubernetes.io/managed-by: Helm + name: grafana-test + namespace: monitoring + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" +--- +# Source: gateway-addons-helm/charts/grafana/templates/tests/test-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-test + namespace: monitoring + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + labels: + helm.sh/chart: grafana-8.0.0 + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "11.0.0" + app.kubernetes.io/managed-by: Helm +data: + run.sh: |- + @test "Test Health" { + url="http://grafana/api/health" + + code=$(wget --server-response --spider --timeout 90 --tries 10 ${url} 2>&1 | awk '/^ HTTP/{print $2}') + [ "$code" == "200" ] + } +--- +# Source: gateway-addons-helm/charts/grafana/templates/tests/test.yaml +apiVersion: v1 +kind: Pod +metadata: + name: grafana-test + labels: + helm.sh/chart: grafana-8.0.0 + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "11.0.0" + app.kubernetes.io/managed-by: Helm + annotations: + "helm.sh/hook": test-success + "helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded" + namespace: monitoring +spec: + serviceAccountName: grafana-test + containers: + - name: gateway-addons-helm-test + image: "docker.io/bats/bats:v1.4.1" + imagePullPolicy: "IfNotPresent" + command: ["/opt/bats/bin/bats", "-t", "/tests/run.sh"] + volumeMounts: + - mountPath: /tests + name: tests + readOnly: true + volumes: + - name: tests + configMap: + name: grafana-test + restartPolicy: Never diff --git a/test/helm/gateway-addons-helm/e2e.in.yaml b/test/helm/gateway-addons-helm/e2e.in.yaml new file mode 100644 index 00000000000..93ce0d8d622 --- /dev/null +++ b/test/helm/gateway-addons-helm/e2e.in.yaml @@ -0,0 +1,4 @@ +grafana: + enabled: false +opentelemetry-collector: + enabled: true diff --git a/test/helm/gateway-addons-helm/e2e.out.yaml b/test/helm/gateway-addons-helm/e2e.out.yaml new file mode 100644 index 00000000000..1440dd9fba3 --- /dev/null +++ b/test/helm/gateway-addons-helm/e2e.out.yaml @@ -0,0 +1,10057 @@ +--- +# Source: gateway-addons-helm/charts/fluent-bit/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fluent-bit + namespace: monitoring + labels: + helm.sh/chart: fluent-bit-0.30.4 + app.kubernetes.io/name: fluent-bit + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.1.4" + app.kubernetes.io/managed-by: Helm +--- +# Source: gateway-addons-helm/charts/loki/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: loki + labels: + helm.sh/chart: loki-4.8.0 + app.kubernetes.io/name: loki + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.7.3" + app.kubernetes.io/managed-by: Helm +automountServiceAccountToken: true +--- +# Source: gateway-addons-helm/charts/opentelemetry-collector/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: otel-collector + namespace: monitoring + labels: + helm.sh/chart: opentelemetry-collector-0.73.1 + app.kubernetes.io/name: opentelemetry-collector + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "0.88.0" + app.kubernetes.io/managed-by: Helm +--- +# Source: gateway-addons-helm/charts/prometheus/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: v2.52.0 + helm.sh/chart: prometheus-25.21.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: prometheus + name: prometheus + namespace: monitoring + annotations: + {} +--- +# Source: gateway-addons-helm/charts/tempo/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: tempo + namespace: monitoring + labels: + helm.sh/chart: tempo-1.3.1 + app.kubernetes.io/name: tempo + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.1.1" + app.kubernetes.io/managed-by: Helm +automountServiceAccountToken: true +--- +# Source: gateway-addons-helm/charts/fluent-bit/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: fluent-bit + namespace: monitoring + labels: + helm.sh/chart: fluent-bit-0.30.4 + app.kubernetes.io/name: fluent-bit + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.1.4" + app.kubernetes.io/managed-by: Helm +data: + custom_parsers.conf: | + [PARSER] + Name docker_no_time + Format json + Time_Keep Off + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L + + fluent-bit.conf: | + [SERVICE] + Daemon Off + Flush 1 + Log_Level info + Parsers_File parsers.conf + Parsers_File custom_parsers.conf + HTTP_Server On + HTTP_Listen 0.0.0.0 + HTTP_Port 2020 + Health_Check On + + [INPUT] + Name tail + Path /var/log/containers/*.log + multiline.parser docker, cri + Tag kube.* + Mem_Buf_Limit 5MB + Skip_Long_Lines On + + [FILTER] + Name kubernetes + Match kube.* + Merge_Log On + Keep_Log Off + K8S-Logging.Parser On + K8S-Logging.Exclude On + + [FILTER] + Name grep + Match kube.* + Regex $kubernetes['container_name'] ^envoy$ + + [FILTER] + Name parser + Match kube.* + Key_Name log + Parser envoy + Reserve_Data True + + [OUTPUT] + Name loki + Match kube.* + Host loki.monitoring.svc.cluster.local + Port 3100 + Labels job=fluentbit, app=$kubernetes['labels']['app'], k8s_namespace_name=$kubernetes['namespace_name'], k8s_pod_name=$kubernetes['pod_name'], k8s_container_name=$kubernetes['container_name'] +--- +# Source: gateway-addons-helm/charts/loki/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: loki + labels: + helm.sh/chart: loki-4.8.0 + app.kubernetes.io/name: loki + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.7.3" + app.kubernetes.io/managed-by: Helm +data: + config.yaml: | + auth_enabled: false + common: + compactor_address: 'loki' + path_prefix: /var/loki + replication_factor: 1 + storage: + filesystem: + chunks_directory: /var/loki/chunks + rules_directory: /var/loki/rules + limits_config: + enforce_metric_name: false + max_cache_freshness_per_query: 10m + reject_old_samples: true + reject_old_samples_max_age: 168h + split_queries_by_interval: 15m + memberlist: + join_members: + - loki-memberlist + query_range: + align_queries_with_step: true + ruler: + storage: + type: local + runtime_config: + file: /etc/loki/runtime-config/runtime-config.yaml + schema_config: + configs: + - from: "2022-01-11" + index: + period: 24h + prefix: loki_index_ + object_store: filesystem + schema: v12 + store: boltdb-shipper + server: + grpc_listen_port: 9095 + http_listen_port: 3100 + storage_config: + hedging: + at: 250ms + max_per_second: 20 + up_to: 3 + table_manager: + retention_deletes_enabled: false + retention_period: 0 +--- +# Source: gateway-addons-helm/charts/loki/templates/runtime-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: loki-runtime + labels: + helm.sh/chart: loki-4.8.0 + app.kubernetes.io/name: loki + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.7.3" + app.kubernetes.io/managed-by: Helm +data: + runtime-config.yaml: | + + {} +--- +# Source: gateway-addons-helm/charts/opentelemetry-collector/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: otel-collector + namespace: monitoring + labels: + helm.sh/chart: opentelemetry-collector-0.73.1 + app.kubernetes.io/name: opentelemetry-collector + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "0.88.0" + app.kubernetes.io/managed-by: Helm +data: + relay: | + exporters: + debug: {} + logging: + verbosity: detailed + loki: + endpoint: http://loki.monitoring.svc:3100/loki/api/v1/push + otlp: + endpoint: tempo.monitoring.svc:4317 + tls: + insecure: true + prometheus: + endpoint: 0.0.0.0:19001 + extensions: + health_check: {} + memory_ballast: + size_in_percentage: 40 + processors: + attributes: + actions: + - action: insert + key: loki.attribute.labels + value: k8s.pod.name, k8s.namespace.name + batch: {} + memory_limiter: + check_interval: 5s + limit_percentage: 80 + spike_limit_percentage: 25 + receivers: + jaeger: + protocols: + grpc: + endpoint: ${env:MY_POD_IP}:14250 + thrift_compact: + endpoint: ${env:MY_POD_IP}:6831 + thrift_http: + endpoint: ${env:MY_POD_IP}:14268 + otlp: + protocols: + grpc: + endpoint: ${env:MY_POD_IP}:4317 + http: + endpoint: ${env:MY_POD_IP}:4318 + prometheus: + config: + scrape_configs: + - job_name: opentelemetry-collector + scrape_interval: 10s + static_configs: + - targets: + - ${env:MY_POD_IP}:8888 + zipkin: + endpoint: ${env:MY_POD_IP}:9411 + service: + extensions: + - health_check + pipelines: + logs: + exporters: + - loki + processors: + - attributes + receivers: + - otlp + metrics: + exporters: + - prometheus + processors: + - memory_limiter + - batch + receivers: + - otlp + traces: + exporters: + - otlp + processors: + - memory_limiter + - batch + receivers: + - otlp + telemetry: + metrics: + address: ${env:MY_POD_IP}:8888 +--- +# Source: gateway-addons-helm/charts/prometheus/templates/cm.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: v2.52.0 + helm.sh/chart: prometheus-25.21.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: prometheus + name: prometheus + namespace: monitoring +data: + allow-snippet-annotations: "false" + alerting_rules.yml: | + {} + alerts: | + {} + prometheus.yml: | + global: + evaluation_interval: 1m + scrape_interval: 15s + scrape_timeout: 10s + rule_files: + - /etc/config/recording_rules.yml + - /etc/config/alerting_rules.yml + - /etc/config/rules + - /etc/config/alerts + scrape_configs: + - job_name: prometheus + static_configs: + - targets: + - localhost:9090 + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-apiservers + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: default;kubernetes;https + source_labels: + - __meta_kubernetes_namespace + - __meta_kubernetes_service_name + - __meta_kubernetes_endpoint_port_name + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-nodes + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - replacement: kubernetes.default.svc:443 + target_label: __address__ + - regex: (.+) + replacement: /api/v1/nodes/$1/proxy/metrics + source_labels: + - __meta_kubernetes_node_name + target_label: __metrics_path__ + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-nodes-cadvisor + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - replacement: kubernetes.default.svc:443 + target_label: __address__ + - regex: (.+) + replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor + source_labels: + - __meta_kubernetes_node_name + target_label: __metrics_path__ + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + - honor_labels: true + job_name: kubernetes-service-endpoints + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape + - action: drop + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (.+?)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_service_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_service_name + target_label: service + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + - honor_labels: true + job_name: kubernetes-service-endpoints-slow + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (.+?)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_service_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_service_name + target_label: service + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + scrape_interval: 5m + scrape_timeout: 30s + - honor_labels: true + job_name: prometheus-pushgateway + kubernetes_sd_configs: + - role: service + relabel_configs: + - action: keep + regex: pushgateway + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_probe + - honor_labels: true + job_name: kubernetes-services + kubernetes_sd_configs: + - role: service + metrics_path: /probe + params: + module: + - http_2xx + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_probe + - source_labels: + - __address__ + target_label: __param_target + - replacement: blackbox + target_label: __address__ + - source_labels: + - __param_target + target_label: instance + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_service_name + target_label: service + - honor_labels: true + job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape + - action: drop + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4}) + replacement: '[$2]:$1' + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: replace + regex: (\d+);((([0-9]+?)(\.|$)){4}) + replacement: $2:$1 + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - action: drop + regex: Pending|Succeeded|Failed|Completed + source_labels: + - __meta_kubernetes_pod_phase + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + - honor_labels: true + job_name: kubernetes-pods-slow + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4}) + replacement: '[$2]:$1' + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: replace + regex: (\d+);((([0-9]+?)(\.|$)){4}) + replacement: $2:$1 + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - action: drop + regex: Pending|Succeeded|Failed|Completed + source_labels: + - __meta_kubernetes_pod_phase + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + scrape_interval: 5m + scrape_timeout: 30s + recording_rules.yml: | + {} + rules: | + {} +--- +# Source: gateway-addons-helm/charts/tempo/templates/configmap-tempo.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: tempo + namespace: monitoring + labels: + helm.sh/chart: tempo-1.3.1 + app.kubernetes.io/name: tempo + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.1.1" + app.kubernetes.io/managed-by: Helm +data: + overrides.yaml: | + overrides: + {} + tempo.yaml: | + multitenancy_enabled: false + usage_report: + reporting_enabled: true + compactor: + compaction: + block_retention: 24h + distributor: + receivers: + jaeger: + protocols: + grpc: + endpoint: 0.0.0.0:14250 + thrift_binary: + endpoint: 0.0.0.0:6832 + thrift_compact: + endpoint: 0.0.0.0:6831 + thrift_http: + endpoint: 0.0.0.0:14268 + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + ingester: + {} + server: + http_listen_port: 3100 + storage: + trace: + backend: local + local: + path: /var/tempo/traces + wal: + path: /var/tempo/wal + querier: + {} + query_frontend: + {} + overrides: + per_tenant_override_config: /conf/overrides.yaml +--- +# Source: gateway-addons-helm/templates/dashboards_config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboards + namespace: 'monitoring' +data: + envoy-clusters.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Envoy proxy monitoring Dashboard with cluster and service level templates. ", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 11021, + "graphTooltip": 0, + "id": 2, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 3, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "#299c46", + "value": 2 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 0, + "y": 0 + }, + "id": 9, + "maxDataPoints": 100, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(envoy_server_live{})", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Live servers", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 5, + "y": 0 + }, + "id": 12, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "avg(envoy_server_uptime)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Avg uptime per node", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 9, + "y": 0 + }, + "id": 11, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "builder", + "expr": "SUM(envoy_server_memory_allocated{})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Allocated Memory", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 14, + "y": 0 + }, + "id": 13, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "SUM(envoy_server_memory_heap_size)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Heap Size", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 18, + "y": 0 + }, + "id": 19, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "(sum(envoy_cluster_membership_healthy{envoy_cluster_name=~\"$cluster\"}) - sum(envoy_cluster_membership_total{envoy_cluster_name=~\"$cluster\"}))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Unhealthy Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "text": "NOT WELL" + }, + "1": { + "text": "OK" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 0 + }, + { + "color": "#299c46", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 21, + "y": 0 + }, + "id": 20, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "(sum(envoy_cluster_membership_total{envoy_cluster_name=~\"$cluster\"})-sum(envoy_cluster_membership_healthy{envoy_cluster_name=~\"$cluster\"})) == bool 0", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Cluster State", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(envoy_cluster_upstream_cx_active{envoy_cluster_name=~\"$cluster\"}) by (envoy_cluster_name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{envoy_cluster_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Total active connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "httproute/default/backend/rule/0" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(irate(envoy_cluster_upstream_rq_total{envoy_cluster_name=~\"$cluster\"}[5m])) by (envoy_cluster_name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{envoy_cluster_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Total requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(irate(envoy_cluster_upstream_cx_rx_bytes_total{envoy_cluster_name=~\"$cluster\"}[5m])) by (envoy_cluster_name)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{envoy_cluster_name}} - in", + "range": true, + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(irate(envoy_cluster_upstream_cx_tx_bytes_total{envoy_cluster_name=~\"$cluster\"}[5m])) by (envoy_cluster_name)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{envoy_cluster_name}} - out", + "range": true, + "refId": "B" + } + ], + "title": "Upstream Network Traffic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "builder", + "expr": "sum(irate(envoy_http_downstream_cx_rx_bytes_total{envoy_http_conn_manager_prefix=~\"http\"}[5m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{service}} - in", + "range": true, + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "builder", + "expr": "sum(irate(envoy_http_downstream_cx_tx_bytes_total{envoy_http_conn_manager_prefix=~\"http\"}[5m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{service}} - out", + "range": true, + "refId": "B" + } + ], + "title": "Downstream Network Traffic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~\"$cluster\"}[5m])) by (le, envoy_cluster_name))", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} 99%", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~\"$cluster\"}[5m])) by (le, envoy_cluster_name))", + "hide": false, + "instant": false, + "legendFormat": "{{envoy_cluster_name}} 90%", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~\"$cluster\"}[5m])) by (le, envoy_cluster_name))", + "hide": false, + "instant": false, + "legendFormat": "{{envoy_cluster_name}} 50%", + "range": true, + "refId": "C" + } + ], + "title": "Upstream Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 19 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_upstream_rq_xx{envoy_response_code_class=~\"2\", envoy_cluster_name=~\"$cluster\"}[5m])) by (envoy_cluster_name)", + "instant": false, + "legendFormat": "{{envoy_cluster_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Upstream 2xx Responses", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 19 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_upstream_rq_xx{envoy_cluster_name=~\"$cluster\",envoy_response_code_class=~\"4\"}[1m])) by (envoy_cluster_name)", + "instant": false, + "legendFormat": "{{envoy_cluster_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Upstream 4xx Responses", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(envoy_cluster_membership_healthy{envoy_cluster_name=~\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "healthy", + "range": true, + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(envoy_cluster_membership_total{envoy_cluster_name=~\"$cluster\",service=~\"$service\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "total", + "refId": "B" + } + ], + "title": "Downstream members", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 27 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_upstream_rq_xx{envoy_cluster_name=~\"$cluster\",envoy_response_code_class=~\"5\"}[5m])) by (envoy_cluster_name)", + "instant": false, + "legendFormat": "{{envoy_cluster_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Upstream 5xx Responses", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 27 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_upstream_rq_xx{envoy_cluster_name=~\"$cluster\",envoy_response_code_class=~\"3\"}[5m])) by (envoy_cluster_name)", + "instant": false, + "legendFormat": "{{envoy_cluster_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Upstream 3xx Responses", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "Data Plane" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "httproute/default/backend/rule/0", + "value": "httproute/default/backend/rule/0" + }, + "datasource": { + "uid": "$datasource" + }, + "definition": "label_values(envoy_cluster_name)", + "hide": 0, + "includeAll": true, + "label": "Cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": { + "query": "label_values(envoy_cluster_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Envoy Clusters", + "uid": "8WkEOMnANKE6PW5hhpVv", + "version": 1, + "weekStart": "" + } + envoy-gateway-global.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Envoy Gateway monitoring Dashboard with exported metrics.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [], + "title": "Watching Components", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "How long in seconds a subscribed watchable is handled.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 7, + "x": 0, + "y": 1 + }, + "id": 1, + "maxPerRow": 3, + "options": { + "displayMode": "basic", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.0.0", + "repeat": "Runner", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(le) (watchable_subscribed_duration_seconds_bucket{runner=~\"$Runner\", namespace=\"$Namespace\"})", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{le}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Duration Bucket: $Runner", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 7, + "y": 1 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": false + }, + "pluginVersion": "11.0.0", + "repeat": "Runner", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "avg by(runner) (watchable_subscribed_duration_seconds_sum{runner=~\"$Runner\", namespace=\"$Namespace\"})", + "format": "time_series", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Avg", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "max by(runner) (watchable_subscribed_duration_seconds_sum{runner=~\"$Runner\", namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Max", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Duration Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Current depth of watchable map.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "super-light-blue", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 10, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": false + }, + "pluginVersion": "11.0.0", + "repeat": "Runner", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(runner) (watchable_depth{runner=~\"$Runner\", namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Depth", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Success" + }, + "properties": [ + { + "id": "displayName", + "value": "Success" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 7, + "x": 12, + "y": 1 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "repeat": "Runner", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(runner) (watchable_subscribed_total{runner=~\"$Runner\", namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Total", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(runner) (watchable_subscribed_errors_total{runner=~\"$Runner\", namespace=\"$Namespace\"}) OR vector(0)", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Errors", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "$A-$B", + "hide": false, + "refId": "Success", + "type": "math" + } + ], + "title": "Statistics", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-red", + "value": null + }, + { + "color": "#EAB839", + "value": 30 + }, + { + "color": "semi-dark-green", + "value": 70 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 19, + "y": 1 + }, + "id": 23, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.0.0", + "repeat": "Runner", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(runner) (watchable_subscribed_total{runner=~\"$Runner\", namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "hide": true, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(runner) (watchable_subscribed_errors_total{runner=~\"$Runner\", namespace=\"$Namespace\"}) OR vector(0)", + "fullMetaSearch": false, + "hide": true, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "(($A-$B) / $A) * 100", + "hide": false, + "refId": "C", + "type": "math" + } + ], + "title": "Success Rate", + "type": "gauge" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 35, + "panels": [], + "title": "Status Updater", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "How long a status update takes to finish for all Kind.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "super-light-blue", + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 0, + "y": 37 + }, + "id": 61, + "options": { + "displayMode": "basic", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(le) (status_update_duration_seconds_bucket{namespace=\"$Namespace\"})", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{le}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total Duration Bucket", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "#EAB839", + "value": 0.2 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 6, + "y": 37 + }, + "id": 82, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "status_update_duration_seconds_sum{namespace=\"$Namespace\"}", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Avg Duration", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "#EAB839", + "value": 0.1 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 12, + "y": 37 + }, + "id": 83, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "status_update_duration_seconds_sum{namespace=\"$Namespace\"}", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Max Duration", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 0.01 + }, + { + "color": "red", + "value": 0.1 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 18, + "y": 37 + }, + "id": 84, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "logmin" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "status_update_duration_seconds_sum{namespace=\"$Namespace\"}", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Min Duration", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of status updates by object kind.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 0, + "y": 46 + }, + "id": 56, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "status_update_total{namespace=\"$Namespace\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "#EAB839", + "value": 50 + }, + { + "color": "green", + "value": 85 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 14, + "x": 10, + "y": 46 + }, + "id": 105, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(kind) (status_update_success_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "hide": true, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(kind) (status_update_success_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "hide": true, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "($B / $A) * 100", + "hide": false, + "refId": "Rate:", + "type": "math" + } + ], + "title": "Success Rate", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Number of status updates that succeeded by object kind.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 54 + }, + "id": 57, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "status_update_success_total{namespace=\"$Namespace\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Success", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Number of status updates that are no-ops by object kind. This is a subset of successful status updates.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 54 + }, + "id": 59, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "status_update_noop_total{namespace=\"$Namespace\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "No-ops", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Number of status updates that failed by object kind.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 54 + }, + "id": 58, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "status_update_failed_total{namespace=\"$Namespace\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Fail", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Number of status update conflicts encountered by object kind.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 54 + }, + "id": 60, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "status_update_conflict_total{namespace=\"$Namespace\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Conflict", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 62 + }, + "id": 126, + "panels": [], + "title": "xDS Server", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "super-light-green", + "mode": "shades" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 0, + "y": 63 + }, + "id": 127, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": false + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "xds_snapshot_creation_total{namespace=\"$Namespace\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Total", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "xds_snapshot_creation_success{namespace=\"$Namespace\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Success", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "xds_snapshot_creation_failed{namespace=\"$Namespace\"} OR on() vector(0)", + "hide": false, + "instant": false, + "legendFormat": "Fail", + "range": true, + "refId": "C" + } + ], + "title": "Creation Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "green", + "value": 85 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 10, + "y": 63 + }, + "id": 148, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum(xds_snapshot_creation_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "hide": true, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum(xds_snapshot_creation_success{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "hide": true, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "($B / $A) * 100", + "hide": false, + "refId": "C", + "type": "math" + } + ], + "title": "Creation Success Rate", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 15, + "y": 63 + }, + "id": 149, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "max(xds_delta_stream_duration_seconds_bucket{namespace=\"$Namespace\"})", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Finished Stream", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Maximum duration seconds for finished xDS delta stream connection.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 63 + }, + "id": 150, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "xds_delta_stream_duration_seconds_sum{namespace=\"$Namespace\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Max Duration", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Minimum duration seconds for finished xDS delta stream connection.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 67 + }, + "id": 151, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "min" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "xds_delta_stream_duration_seconds_sum{namespace=\"$Namespace\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Min Duration", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of xds snapshot cache updates by node id.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 10, + "x": 0, + "y": 71 + }, + "id": 152, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "xds_snapshot_update_total{namespace=\"$Namespace\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{nodeID}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Update Total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "green", + "value": 85 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 5, + "x": 10, + "y": 71 + }, + "id": 155, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum(xds_snapshot_update_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "hide": true, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum(xds_snapshot_update_success{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "hide": true, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "($B / $A) * 100", + "hide": false, + "refId": "C", + "type": "math" + } + ], + "title": "Update Success Rate", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Number of xds snapshot cache updates that succeed by node id.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 9, + "x": 15, + "y": 71 + }, + "id": 153, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "xds_snapshot_update_success{namespace=\"$Namespace\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{nodeID}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Update Success", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Number of xds snapshot cache updates that failed by node id.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 9, + "x": 15, + "y": 76 + }, + "id": 154, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "xds_snapshot_update_failed{namespace=\"$Namespace\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{nodeID}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Update Fail", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 81 + }, + "id": 156, + "panels": [], + "title": "Infrastructure Manager", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 82 + }, + "id": 199, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(le) (resource_apply_duration_seconds_bucket{namespace=\"$Namespace\"})", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{le}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total Apply Duration Bucket", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 0.3 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 82 + }, + "id": 220, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(kind) (resource_apply_duration_seconds_sum{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Avg Apply Duration", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 0.3 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 82 + }, + "id": 221, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(kind) (resource_apply_duration_seconds_sum{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Max Apply Duration", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 0.3 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 82 + }, + "id": 222, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "logmin" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(kind) (resource_apply_duration_seconds_sum{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Min Apply Duration", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of applied resources sumed by kind (include No-ops).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 90 + }, + "id": 157, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(kind) (resource_apply_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total Applied Resources", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of applied resources sumed by infra name.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 90 + }, + "id": 178, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(name) (resource_apply_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total Applied Infrastructures", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of applied resources that succeed sumed by kind (include No-ops).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 10, + "x": 0, + "y": 97 + }, + "id": 229, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(kind) (resource_apply_success{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Success Applied Resources", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of applied resources that failed sumed by kind (include No-ops).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 10, + "x": 10, + "y": 97 + }, + "id": 230, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(kind) (resource_apply_failed{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Fail Applied Resources", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 104 + }, + "id": 223, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (resource_delete_duration_seconds_bucket{namespace=\"$Namespace\"})", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{le}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total Delete Duration Bucket", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 0.1 + }, + { + "color": "red", + "value": 0.3 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 104 + }, + "id": 224, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(kind) (resource_delete_duration_seconds_sum{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Avg Delete Duration", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 0.1 + }, + { + "color": "red", + "value": 0.3 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 104 + }, + "id": 225, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(kind) (resource_delete_duration_seconds_sum{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Max Delete Duration", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 0.1 + }, + { + "color": "red", + "value": 0.3 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 104 + }, + "id": 226, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "logmin" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(kind) (resource_delete_duration_seconds_sum{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Min Delete Duration", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of deleted resources sumed by kind (include No-ops).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 112 + }, + "id": 227, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(kind) (resource_delete_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total Deleted Resources", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of deleted resources sumed by infra name.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 112 + }, + "id": 228, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(name) (resource_delete_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total Deleted Infrastructures", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of deleted resources that succeed sumed by kind (include No-ops).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 10, + "x": 0, + "y": 119 + }, + "id": 232, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(kind) (resource_delete_success{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Success Deleted Resources", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of deleted resources that failed sumed by kind (include No-ops).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 10, + "x": 10, + "y": 119 + }, + "id": 233, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(kind) (resource_delete_failed{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Fail Deleted Resources", + "type": "stat" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [ + "Control Plane" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "envoy-gateway-system", + "value": "envoy-gateway-system" + }, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "definition": "label_values(watchable_depth,namespace)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "Namespace", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(watchable_depth,namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "definition": "label_values(watchable_depth,runner)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "Runner", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(watchable_depth,runner)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": {}, + "timezone": "browser", + "title": "Envoy Gateway Global", + "uid": "bdn8lriao7myoa", + "version": 1, + "weekStart": "" + } + envoy-global.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Envoy proxy monitoring Dashboard with service level templates.", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 11022, + "graphTooltip": 0, + "id": 3, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 3, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "#299c46", + "value": 2 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 0, + "y": 0 + }, + "id": 37, + "maxDataPoints": 100, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "builder", + "expr": "sum(envoy_server_live)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Live servers", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 5, + "y": 0 + }, + "id": 39, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "builder", + "expr": "avg by(pod) (envoy_server_uptime{namespace=~\"$Namespace\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Avg uptime per node", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 10, + "y": 0 + }, + "id": 43, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by(pod) (envoy_server_memory_heap_size{namespace=~\"$Namespace\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Heap Size", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 15, + "y": 0 + }, + "id": 41, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "builder", + "expr": "sum by(pod) (envoy_server_memory_allocated{namespace=~\"$Namespace\"})", + "hide": false, + "instant": false, + "range": true, + "refId": "B" + } + ], + "title": "Allocated Memory", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 24, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "DownStream", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_http_downstream_rq_total[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Envoy HTTP Downstream Rq total", + "range": true, + "refId": "A" + } + ], + "title": "Downstream RPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 8 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "sum by(namespace) (rate(envoy_http_downstream_cx_total{namespace=~\"$Namespace\"}[5m]))", + "instant": false, + "legendFormat": "{{namespace}}", + "range": true, + "refId": "A" + } + ], + "title": "Downstream CPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 8 + }, + "id": 16, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by(le) (rate(envoy_http_downstream_rq_time_bucket[5m])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{service}} 90%", + "range": true, + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "builder", + "expr": "histogram_quantile(0.5, sum by(le) (rate(envoy_http_downstream_rq_time_bucket[5m])))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{service}} 50% ", + "range": true, + "refId": "B" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "builder", + "expr": "histogram_quantile(0.99, sum by(le) (rate(envoy_http_downstream_rq_time_bucket[5m])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{service}} 99%", + "range": true, + "refId": "C" + } + ], + "title": "Downstream Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by(namespace) (envoy_listener_downstream_cx_active{namespace=~\"$Namespace\"})", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "title": "Downstream Total Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 16 + }, + "id": 32, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "sum by(namespace) (rate(envoy_tcp_downstream_cx_total{namespace=~\"$Namespace\"}[5m]))", + "instant": false, + "legendFormat": "{{namespace}}", + "range": true, + "refId": "A" + } + ], + "title": "TCP Downstream CPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 16 + }, + "id": 31, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "sum by(namespace) (rate(envoy_tcp_downstream_cx_rx_bytes_total{namespace=~\"$Namespace\"}[5m]))", + "instant": false, + "legendFormat": "{{namespace}}", + "range": true, + "refId": "A" + } + ], + "title": "TCP Downstream Bytes Rx/second", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 16 + }, + "id": 33, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "sum by(namespace) (rate(envoy_tcp_downstream_cx_tx_bytes_total{namespace=~\"$Namespace\"}[5m]))", + "instant": false, + "legendFormat": "{{namespace}}", + "range": true, + "refId": "A" + } + ], + "title": "TCP Downstream Bytes Tx/Second", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 22, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "UpStream", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Displays the number of Requests per Second being performed against each Upstream.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 25 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "builder", + "expr": "sum by(namespace) (rate(envoy_cluster_upstream_rq_total{namespace=~\"$Namespace\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Upstream RPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 25 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_upstream_cx_total{namespace=~\"$Namespace\",}[5m])) by (namespace)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Upstream CPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 25 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{namespace=~\"$Namespace\"}[5m])) by (le, namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{namespace}} 99%", + "range": true, + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum(rate(envoy_cluster_upstream_rq_time_bucket{namespace=~\"$Namespace\"}[5m])) by (le, namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{namespace}} 90%", + "range": true, + "refId": "C" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(envoy_cluster_upstream_rq_time_bucket{namespace=~\"$Namespace\"}[5m])) by (le, namespace))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{namespace}} 50% ", + "range": true, + "refId": "B" + } + ], + "title": "Upstream Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 25 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(envoy_cluster_upstream_cx_active{namespace=~\"$Namespace\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{namespace}}", + "range": true, + "refId": "A" + } + ], + "title": "Upstream Total Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 33 + }, + "id": 34, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "sum by(namespace) (rate(envoy_cluster_upstream_cx_rx_bytes_total{namespace=~\"$Namespace\"}[5m]))", + "instant": false, + "legendFormat": "{{namespace}}", + "range": true, + "refId": "A" + } + ], + "title": "Upstream Bytes Rx/Second", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 33 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "sum by(namespace) (rate(envoy_cluster_upstream_cx_rx_bytes_total{namespace=~\"$Namespace\"}[5m]))", + "instant": false, + "legendFormat": "{{namespace}}", + "range": true, + "refId": "A" + } + ], + "title": "Upstream Bytes Tx/Second", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 28, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Upstream Response Codes", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 42 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "sum(rate(envoy_cluster_upstream_rq_xx{envoy_response_code_class=~\"2\"}[5m]))", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Value", + "range": true, + "refId": "A" + } + ], + "title": "Upstream 2xx Responses", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 42 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "builder", + "expr": "sum(rate(envoy_cluster_upstream_rq_xx{envoy_response_code_class=~\"3\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "value", + "range": true, + "refId": "A" + } + ], + "title": "Upstream 3xx Responses", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 42 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_upstream_rq_xx{envoy_response_code_class=\"4\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Value", + "range": true, + "refId": "A" + } + ], + "title": "Upstream 4xx Responses", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 42 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_upstream_rq_xx{envoy_response_code_class=\"5\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Value", + "range": true, + "refId": "A" + } + ], + "title": "Upstream 5xx Responses", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 26, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Total", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 51 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "avg(envoy_cluster_membership_healthy{namespace=~\"$Namespace\"}) by (namespace) / avg(envoy_cluster_membership_total{namespace=~\"$Namespace\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "range": true, + "refId": "A" + } + ], + "title": "Endpoint Percentage Health", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 51 + }, + "id": 19, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "builder", + "expr": "sum by(namespace) (envoy_cluster_membership_total{namespace=~\"$Namespace\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "range": true, + "refId": "A" + } + ], + "title": "Total Endpoints", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 51 + }, + "id": 18, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "builder", + "expr": "sum by(namespace) (envoy_cluster_membership_healthy{namespace=~\"$Namespace\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "range": true, + "refId": "A" + } + ], + "title": "Healthy Endpoints", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 51 + }, + "id": 20, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by(namespace) (envoy_cluster_membership_total{namespace=~\"$Namespace\"}) - sum by(namespace) (envoy_cluster_membership_healthy{namespace=~\"$Namespace\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "range": true, + "refId": "A" + } + ], + "title": "Unhealthy Endpoints", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": [ + "Data Plane" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "uid": "$datasource" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "Namespace", + "options": [], + "query": "label_values(envoy_cluster_upstream_rq_time_bucket,namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Envoy Global", + "uid": "heHhNSFf6Na8vIZWRs8H", + "version": 1, + "weekStart": "" + } + envoy-pod-resource.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Envoy Pod Memory and CPU Usage", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 4, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by(pod) (container_memory_working_set_bytes{container=~\"envoy\"}/1000000)", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Memory Working Set Envoy Pods(mb)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{container=\"envoy\"}[5m]))", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage Envoy Pods", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "Data Plane" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": {}, + "timezone": "", + "title": "Envoy Pod Resources", + "uid": "f2279235-80b7-4c85-84f4-f25a3bf3eac0", + "version": 1, + "weekStart": "" + } +--- +# Source: gateway-addons-helm/charts/fluent-bit/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: fluent-bit + labels: + helm.sh/chart: fluent-bit-0.30.4 + app.kubernetes.io/name: fluent-bit + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.1.4" + app.kubernetes.io/managed-by: Helm +rules: + - apiGroups: + - "" + resources: + - namespaces + - pods + verbs: + - get + - list + - watch +--- +# Source: gateway-addons-helm/charts/prometheus/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: v2.52.0 + helm.sh/chart: prometheus-25.21.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: prometheus + name: prometheus +rules: + - apiGroups: + - "" + resources: + - nodes + - nodes/proxy + - nodes/metrics + - services + - endpoints + - pods + - ingresses + - configmaps + verbs: + - get + - list + - watch + - apiGroups: + - "extensions" + - "networking.k8s.io" + resources: + - ingresses/status + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "discovery.k8s.io" + resources: + - endpointslices + verbs: + - get + - list + - watch + - nonResourceURLs: + - "/metrics" + verbs: + - get +--- +# Source: gateway-addons-helm/charts/fluent-bit/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: fluent-bit + labels: + helm.sh/chart: fluent-bit-0.30.4 + app.kubernetes.io/name: fluent-bit + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.1.4" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: fluent-bit +subjects: + - kind: ServiceAccount + name: fluent-bit + namespace: monitoring +--- +# Source: gateway-addons-helm/charts/prometheus/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: v2.52.0 + helm.sh/chart: prometheus-25.21.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: prometheus + name: prometheus +subjects: + - kind: ServiceAccount + name: prometheus + namespace: monitoring +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +--- +# Source: gateway-addons-helm/charts/fluent-bit/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: fluent-bit + namespace: monitoring + labels: + helm.sh/chart: fluent-bit-0.30.4 + app.kubernetes.io/name: fluent-bit + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.1.4" + app.kubernetes.io/managed-by: Helm +spec: + type: ClusterIP + ports: + - port: 2020 + targetPort: http + protocol: TCP + name: http + selector: + app.kubernetes.io/name: fluent-bit + app.kubernetes.io/instance: gateway-addons-helm +--- +# Source: gateway-addons-helm/charts/loki/templates/service-memberlist.yaml +apiVersion: v1 +kind: Service +metadata: + name: loki-memberlist + labels: + helm.sh/chart: loki-4.8.0 + app.kubernetes.io/name: loki + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.7.3" + app.kubernetes.io/managed-by: Helm +spec: + type: ClusterIP + clusterIP: None + ports: + - name: tcp + port: 7946 + targetPort: http-memberlist + protocol: TCP + selector: + app.kubernetes.io/name: loki + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/part-of: memberlist +--- +# Source: gateway-addons-helm/charts/loki/templates/single-binary/service-headless.yaml +apiVersion: v1 +kind: Service +metadata: + name: loki-headless + namespace: monitoring + labels: + helm.sh/chart: loki-4.8.0 + app.kubernetes.io/name: loki + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.7.3" + app.kubernetes.io/managed-by: Helm + variant: headless + prometheus.io/service-monitor: "false" +spec: + clusterIP: None + ports: + - name: http-metrics + port: 3100 + targetPort: http-metrics + protocol: TCP + selector: + app.kubernetes.io/name: loki + app.kubernetes.io/instance: gateway-addons-helm +--- +# Source: gateway-addons-helm/charts/loki/templates/single-binary/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: loki + labels: + helm.sh/chart: loki-4.8.0 + app.kubernetes.io/name: loki + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.7.3" + app.kubernetes.io/managed-by: Helm +spec: + type: ClusterIP + ports: + - name: http-metrics + port: 3100 + targetPort: http-metrics + protocol: TCP + - name: grpc + port: 9095 + targetPort: grpc + protocol: TCP + selector: + app.kubernetes.io/name: loki + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/component: single-binary +--- +# Source: gateway-addons-helm/charts/opentelemetry-collector/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: otel-collector + namespace: monitoring + labels: + helm.sh/chart: opentelemetry-collector-0.73.1 + app.kubernetes.io/name: opentelemetry-collector + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "0.88.0" + app.kubernetes.io/managed-by: Helm + component: standalone-collector +spec: + type: ClusterIP + ports: + + - name: jaeger-compact + port: 6831 + targetPort: 6831 + protocol: UDP + - name: jaeger-grpc + port: 14250 + targetPort: 14250 + protocol: TCP + - name: jaeger-thrift + port: 14268 + targetPort: 14268 + protocol: TCP + - name: otlp + port: 4317 + targetPort: 4317 + protocol: TCP + appProtocol: grpc + - name: otlp-http + port: 4318 + targetPort: 4318 + protocol: TCP + - name: zipkin + port: 9411 + targetPort: 9411 + protocol: TCP + selector: + app.kubernetes.io/name: opentelemetry-collector + app.kubernetes.io/instance: gateway-addons-helm + component: standalone-collector + internalTrafficPolicy: Cluster +--- +# Source: gateway-addons-helm/charts/prometheus/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: v2.52.0 + helm.sh/chart: prometheus-25.21.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: prometheus + name: prometheus + namespace: monitoring +spec: + ports: + - name: http + port: 80 + protocol: TCP + targetPort: 9090 + selector: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: gateway-addons-helm + sessionAffinity: None + type: "LoadBalancer" +--- +# Source: gateway-addons-helm/charts/tempo/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: tempo + namespace: monitoring + labels: + helm.sh/chart: tempo-1.3.1 + app.kubernetes.io/name: tempo + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.1.1" + app.kubernetes.io/managed-by: Helm +spec: + type: LoadBalancer + ports: + - name: tempo-prom-metrics + port: 3100 + targetPort: 3100 + - name: tempo-jaeger-thrift-compact + port: 6831 + protocol: UDP + targetPort: 6831 + - name: tempo-jaeger-thrift-binary + port: 6832 + protocol: UDP + targetPort: 6832 + - name: tempo-jaeger-thrift-http + port: 14268 + protocol: TCP + targetPort: 14268 + - name: grpc-tempo-jaeger + port: 14250 + protocol: TCP + targetPort: 14250 + - name: tempo-zipkin + port: 9411 + protocol: TCP + targetPort: 9411 + - name: tempo-otlp-legacy + port: 55680 + protocol: TCP + targetPort: 55680 + - name: tempo-otlp-http-legacy + port: 55681 + protocol: TCP + targetPort: 4318 + - name: grpc-tempo-otlp + port: 4317 + protocol: TCP + targetPort: 4317 + - name: tempo-otlp-http + port: 4318 + protocol: TCP + targetPort: 4318 + - name: tempo-opencensus + port: 55678 + protocol: TCP + targetPort: 55678 + selector: + app.kubernetes.io/name: tempo + app.kubernetes.io/instance: gateway-addons-helm +--- +# Source: gateway-addons-helm/charts/fluent-bit/templates/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: fluent-bit + namespace: monitoring + labels: + helm.sh/chart: fluent-bit-0.30.4 + app.kubernetes.io/name: fluent-bit + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.1.4" + app.kubernetes.io/managed-by: Helm +spec: + selector: + matchLabels: + app.kubernetes.io/name: fluent-bit + app.kubernetes.io/instance: gateway-addons-helm + template: + metadata: + annotations: + checksum/config: 03d122555879033ccf6443369f73463490b100f195550b1483d337f497c749e3 + checksum/luascripts: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 + fluentbit.io/exclude: "true" + prometheus.io/path: /api/v1/metrics/prometheus + prometheus.io/port: "2020" + prometheus.io/scrape: "true" + labels: + app.kubernetes.io/name: fluent-bit + app.kubernetes.io/instance: gateway-addons-helm + spec: + serviceAccountName: fluent-bit + hostNetwork: false + dnsPolicy: ClusterFirst + containers: + - name: fluent-bit + image: "cr.fluentbit.io/fluent/fluent-bit:2.1.4" + imagePullPolicy: Always + ports: + - name: http + containerPort: 2020 + protocol: TCP + livenessProbe: + httpGet: + path: / + port: http + readinessProbe: + httpGet: + path: /api/v1/health + port: http + volumeMounts: + - mountPath: /fluent-bit/etc/fluent-bit.conf + name: config + subPath: fluent-bit.conf + - mountPath: /fluent-bit/etc/custom_parsers.conf + name: config + subPath: custom_parsers.conf + - mountPath: /var/log + name: varlog + - mountPath: /var/lib/docker/containers + name: varlibdockercontainers + readOnly: true + - mountPath: /etc/machine-id + name: etcmachineid + readOnly: true + volumes: + - name: config + configMap: + name: fluent-bit + - hostPath: + path: /var/log + name: varlog + - hostPath: + path: /var/lib/docker/containers + name: varlibdockercontainers + - hostPath: + path: /etc/machine-id + type: File + name: etcmachineid +--- +# Source: gateway-addons-helm/charts/opentelemetry-collector/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: otel-collector + namespace: monitoring + labels: + helm.sh/chart: opentelemetry-collector-0.73.1 + app.kubernetes.io/name: opentelemetry-collector + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "0.88.0" + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app.kubernetes.io/name: opentelemetry-collector + app.kubernetes.io/instance: gateway-addons-helm + component: standalone-collector + strategy: + type: RollingUpdate + template: + metadata: + annotations: + checksum/config: 077be33cb293f9c37d065397da8156bf9f33a42ad56a3d5876f39de72c874023 + + labels: + app.kubernetes.io/name: opentelemetry-collector + app.kubernetes.io/instance: gateway-addons-helm + component: standalone-collector + + spec: + + serviceAccountName: otel-collector + securityContext: + {} + containers: + - name: opentelemetry-collector + command: + - /otelcol-contrib + - --config=/conf/relay.yaml + securityContext: + {} + image: "otel/opentelemetry-collector-contrib:0.88.0" + imagePullPolicy: IfNotPresent + ports: + + - name: jaeger-compact + containerPort: 6831 + protocol: UDP + - name: jaeger-grpc + containerPort: 14250 + protocol: TCP + - name: jaeger-thrift + containerPort: 14268 + protocol: TCP + - name: otlp + containerPort: 4317 + protocol: TCP + - name: otlp-http + containerPort: 4318 + protocol: TCP + - name: zipkin + containerPort: 9411 + protocol: TCP + env: + - name: MY_POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + livenessProbe: + httpGet: + path: / + port: 13133 + readinessProbe: + httpGet: + path: / + port: 13133 + volumeMounts: + - mountPath: /conf + name: opentelemetry-collector-configmap + volumes: + - name: opentelemetry-collector-configmap + configMap: + name: otel-collector + items: + - key: relay + path: relay.yaml + hostNetwork: false +--- +# Source: gateway-addons-helm/charts/prometheus/templates/deploy.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: v2.52.0 + helm.sh/chart: prometheus-25.21.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: prometheus + name: prometheus + namespace: monitoring +spec: + selector: + matchLabels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: gateway-addons-helm + replicas: 1 + revisionHistoryLimit: 10 + strategy: + type: Recreate + rollingUpdate: null + template: + metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: v2.52.0 + helm.sh/chart: prometheus-25.21.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: prometheus + spec: + enableServiceLinks: true + serviceAccountName: prometheus + containers: + - name: prometheus-server-configmap-reload + image: "quay.io/prometheus-operator/prometheus-config-reloader:v0.73.2" + imagePullPolicy: "IfNotPresent" + args: + - --watched-dir=/etc/config + - --reload-url=http://127.0.0.1:9090/-/reload + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + + - name: prometheus-server + image: "prom/prometheus:v2.52.0" + imagePullPolicy: "IfNotPresent" + args: + - --storage.tsdb.retention.time=15d + - --config.file=/etc/config/prometheus.yml + - --storage.tsdb.path=/data + - --web.console.libraries=/etc/prometheus/console_libraries + - --web.console.templates=/etc/prometheus/consoles + - --web.enable-lifecycle + ports: + - containerPort: 9090 + readinessProbe: + httpGet: + path: /-/ready + port: 9090 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 4 + failureThreshold: 3 + successThreshold: 1 + livenessProbe: + httpGet: + path: /-/healthy + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 15 + timeoutSeconds: 10 + failureThreshold: 3 + successThreshold: 1 + volumeMounts: + - name: config-volume + mountPath: /etc/config + - name: storage-volume + mountPath: /data + subPath: "" + dnsPolicy: ClusterFirst + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + terminationGracePeriodSeconds: 300 + volumes: + - name: config-volume + configMap: + name: prometheus + - name: storage-volume + emptyDir: + {} +--- +# Source: gateway-addons-helm/charts/loki/templates/single-binary/statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: loki + labels: + helm.sh/chart: loki-4.8.0 + app.kubernetes.io/name: loki + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.7.3" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: single-binary + app.kubernetes.io/part-of: memberlist +spec: + replicas: 1 + podManagementPolicy: Parallel + updateStrategy: + rollingUpdate: + partition: 0 + serviceName: loki-headless + revisionHistoryLimit: 10 + + persistentVolumeClaimRetentionPolicy: + whenDeleted: Delete + whenScaled: Delete + selector: + matchLabels: + app.kubernetes.io/name: loki + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/component: single-binary + template: + metadata: + annotations: + checksum/config: 39a9cea617408d4add363b9ca660a8889e48b866eba2e8c8e4bfc10870b29162 + labels: + app.kubernetes.io/name: loki + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/component: single-binary + app.kubernetes.io/part-of: memberlist + spec: + serviceAccountName: loki + automountServiceAccountToken: true + enableServiceLinks: true + + securityContext: + fsGroup: 10001 + runAsGroup: 10001 + runAsNonRoot: true + runAsUser: 10001 + terminationGracePeriodSeconds: 30 + containers: + - name: loki + image: docker.io/grafana/loki:2.7.3 + imagePullPolicy: IfNotPresent + args: + - -config.file=/etc/loki/config/config.yaml + - -target=all + ports: + - name: http-metrics + containerPort: 3100 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: http-memberlist + containerPort: 7946 + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 30 + timeoutSeconds: 1 + volumeMounts: + - name: tmp + mountPath: /tmp + - name: config + mountPath: /etc/loki/config + - name: runtime-config + mountPath: /etc/loki/runtime-config + - name: storage + mountPath: /var/loki + resources: + {} + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/name: loki + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/component: single-binary + topologyKey: kubernetes.io/hostname + + volumes: + - name: tmp + emptyDir: {} + - name: config + configMap: + name: loki + - name: runtime-config + configMap: + name: loki-runtime + volumeClaimTemplates: + - metadata: + name: storage + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "10Gi" +--- +# Source: gateway-addons-helm/charts/tempo/templates/statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: tempo + namespace: monitoring + labels: + helm.sh/chart: tempo-1.3.1 + app.kubernetes.io/name: tempo + app.kubernetes.io/instance: gateway-addons-helm + app.kubernetes.io/version: "2.1.1" + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: tempo + app.kubernetes.io/instance: gateway-addons-helm + serviceName: tempo-headless + template: + metadata: + labels: + app.kubernetes.io/name: tempo + app.kubernetes.io/instance: gateway-addons-helm + annotations: + checksum/config: 0898f7ca87563d700c35d7ea1824cd042cf93fb2f05e254d12a854aa97a5c5e5 + spec: + serviceAccountName: tempo + automountServiceAccountToken: true + containers: + - args: + - -config.file=/conf/tempo.yaml + - -mem-ballast-size-mbs=1024 + image: grafana/tempo:2.1.1 + imagePullPolicy: IfNotPresent + name: tempo + ports: + - containerPort: 3100 + name: prom-metrics + - containerPort: 6831 + name: jaeger-thrift-c + protocol: UDP + - containerPort: 6832 + name: jaeger-thrift-b + protocol: UDP + - containerPort: 14268 + name: jaeger-thrift-h + - containerPort: 14250 + name: jaeger-grpc + - containerPort: 9411 + name: zipkin + - containerPort: 55680 + name: otlp-legacy + - containerPort: 4317 + name: otlp-grpc + - containerPort: 55681 + name: otlp-httplegacy + - containerPort: 4318 + name: otlp-http + - containerPort: 55678 + name: opencensus + resources: + {} + env: + volumeMounts: + - mountPath: /conf + name: tempo-conf + volumes: + - configMap: + name: tempo + name: tempo-conf + updateStrategy: + type: + RollingUpdate diff --git a/test/helm/control-plane-with-pdb.in.yaml b/test/helm/gateway-helm/control-plane-with-pdb.in.yaml similarity index 100% rename from test/helm/control-plane-with-pdb.in.yaml rename to test/helm/gateway-helm/control-plane-with-pdb.in.yaml diff --git a/test/helm/control-plane-with-pdb.out.yaml b/test/helm/gateway-helm/control-plane-with-pdb.out.yaml similarity index 89% rename from test/helm/control-plane-with-pdb.out.yaml rename to test/helm/gateway-helm/control-plane-with-pdb.out.yaml index 806bdfa4df8..d555ce7e6b6 100644 --- a/test/helm/control-plane-with-pdb.out.yaml +++ b/test/helm/gateway-helm/control-plane-with-pdb.out.yaml @@ -11,7 +11,7 @@ spec: matchLabels: control-plane: envoy-gateway app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm --- # Source: gateway-helm/templates/envoy-gateway-deployment.yaml apiVersion: v1 @@ -22,7 +22,7 @@ metadata: labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm --- @@ -35,7 +35,7 @@ metadata: labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm data: @@ -70,7 +70,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: creationTimestamp: null - name: eg-gateway-helm-envoy-gateway-role + name: gateway-helm-envoy-gateway-role rules: - apiGroups: - "" @@ -188,11 +188,11 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: eg-gateway-helm-envoy-gateway-rolebinding + name: gateway-helm-envoy-gateway-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: eg-gateway-helm-envoy-gateway-role + name: gateway-helm-envoy-gateway-role subjects: - kind: ServiceAccount name: 'envoy-gateway' @@ -202,12 +202,12 @@ subjects: apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: eg-gateway-helm-infra-manager + name: gateway-helm-infra-manager namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm rules: @@ -247,12 +247,12 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: eg-gateway-helm-leader-election-role + name: gateway-helm-leader-election-role namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm rules: @@ -292,18 +292,18 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: eg-gateway-helm-infra-manager + name: gateway-helm-infra-manager namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: 'eg-gateway-helm-infra-manager' + name: 'gateway-helm-infra-manager' subjects: - kind: ServiceAccount name: 'envoy-gateway' @@ -313,18 +313,18 @@ subjects: apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: eg-gateway-helm-leader-election-rolebinding + name: gateway-helm-leader-election-rolebinding namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: 'eg-gateway-helm-leader-election-role' + name: 'gateway-helm-leader-election-role' subjects: - kind: ServiceAccount name: 'envoy-gateway' @@ -340,14 +340,14 @@ metadata: control-plane: envoy-gateway helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm spec: selector: control-plane: envoy-gateway app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm ports: - name: grpc port: 18000 @@ -369,7 +369,7 @@ metadata: control-plane: envoy-gateway helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm spec: @@ -378,7 +378,7 @@ spec: matchLabels: control-plane: envoy-gateway app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm template: metadata: annotations: @@ -387,7 +387,7 @@ spec: labels: control-plane: envoy-gateway app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm spec: containers: - args: @@ -457,12 +457,12 @@ spec: apiVersion: v1 kind: ServiceAccount metadata: - name: eg-gateway-helm-certgen + name: gateway-helm-certgen namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm annotations: @@ -472,12 +472,12 @@ metadata: apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: eg-gateway-helm-certgen + name: gateway-helm-certgen namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm annotations: @@ -496,12 +496,12 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: eg-gateway-helm-certgen + name: gateway-helm-certgen namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm annotations: @@ -509,22 +509,22 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: 'eg-gateway-helm-certgen' + name: 'gateway-helm-certgen' subjects: - kind: ServiceAccount - name: 'eg-gateway-helm-certgen' + name: 'gateway-helm-certgen' namespace: 'envoy-gateway-system' --- # Source: gateway-helm/templates/certgen.yaml apiVersion: batch/v1 kind: Job metadata: - name: eg-gateway-helm-certgen + name: gateway-helm-certgen namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm annotations: @@ -559,5 +559,5 @@ spec: runAsGroup: 65534 runAsNonRoot: true runAsUser: 65534 - serviceAccountName: eg-gateway-helm-certgen + serviceAccountName: gateway-helm-certgen ttlSecondsAfterFinished: 30 diff --git a/test/helm/default-config.in.yaml b/test/helm/gateway-helm/default-config.in.yaml similarity index 100% rename from test/helm/default-config.in.yaml rename to test/helm/gateway-helm/default-config.in.yaml diff --git a/test/helm/default-config.out.yaml b/test/helm/gateway-helm/default-config.out.yaml similarity index 90% rename from test/helm/default-config.out.yaml rename to test/helm/gateway-helm/default-config.out.yaml index b1181bbdc87..2169dee5233 100644 --- a/test/helm/default-config.out.yaml +++ b/test/helm/gateway-helm/default-config.out.yaml @@ -8,7 +8,7 @@ metadata: labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm --- @@ -21,7 +21,7 @@ metadata: labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm data: @@ -56,7 +56,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: creationTimestamp: null - name: eg-gateway-helm-envoy-gateway-role + name: gateway-helm-envoy-gateway-role rules: - apiGroups: - "" @@ -174,11 +174,11 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: eg-gateway-helm-envoy-gateway-rolebinding + name: gateway-helm-envoy-gateway-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: eg-gateway-helm-envoy-gateway-role + name: gateway-helm-envoy-gateway-role subjects: - kind: ServiceAccount name: 'envoy-gateway' @@ -188,12 +188,12 @@ subjects: apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: eg-gateway-helm-infra-manager + name: gateway-helm-infra-manager namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm rules: @@ -233,12 +233,12 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: eg-gateway-helm-leader-election-role + name: gateway-helm-leader-election-role namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm rules: @@ -278,18 +278,18 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: eg-gateway-helm-infra-manager + name: gateway-helm-infra-manager namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: 'eg-gateway-helm-infra-manager' + name: 'gateway-helm-infra-manager' subjects: - kind: ServiceAccount name: 'envoy-gateway' @@ -299,18 +299,18 @@ subjects: apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: eg-gateway-helm-leader-election-rolebinding + name: gateway-helm-leader-election-rolebinding namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: 'eg-gateway-helm-leader-election-role' + name: 'gateway-helm-leader-election-role' subjects: - kind: ServiceAccount name: 'envoy-gateway' @@ -326,14 +326,14 @@ metadata: control-plane: envoy-gateway helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm spec: selector: control-plane: envoy-gateway app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm ports: - name: grpc port: 18000 @@ -355,7 +355,7 @@ metadata: control-plane: envoy-gateway helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm spec: @@ -364,7 +364,7 @@ spec: matchLabels: control-plane: envoy-gateway app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm template: metadata: annotations: @@ -373,7 +373,7 @@ spec: labels: control-plane: envoy-gateway app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm spec: containers: - args: @@ -443,12 +443,12 @@ spec: apiVersion: v1 kind: ServiceAccount metadata: - name: eg-gateway-helm-certgen + name: gateway-helm-certgen namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm annotations: @@ -458,12 +458,12 @@ metadata: apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: eg-gateway-helm-certgen + name: gateway-helm-certgen namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm annotations: @@ -482,12 +482,12 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: eg-gateway-helm-certgen + name: gateway-helm-certgen namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm annotations: @@ -495,22 +495,22 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: 'eg-gateway-helm-certgen' + name: 'gateway-helm-certgen' subjects: - kind: ServiceAccount - name: 'eg-gateway-helm-certgen' + name: 'gateway-helm-certgen' namespace: 'envoy-gateway-system' --- # Source: gateway-helm/templates/certgen.yaml apiVersion: batch/v1 kind: Job metadata: - name: eg-gateway-helm-certgen + name: gateway-helm-certgen namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm annotations: @@ -545,5 +545,5 @@ spec: runAsGroup: 65534 runAsNonRoot: true runAsUser: 65534 - serviceAccountName: eg-gateway-helm-certgen + serviceAccountName: gateway-helm-certgen ttlSecondsAfterFinished: 30 diff --git a/test/helm/deployment-custom-topology.in.yaml b/test/helm/gateway-helm/deployment-custom-topology.in.yaml similarity index 100% rename from test/helm/deployment-custom-topology.in.yaml rename to test/helm/gateway-helm/deployment-custom-topology.in.yaml diff --git a/test/helm/deployment-custom-topology.out.yaml b/test/helm/gateway-helm/deployment-custom-topology.out.yaml similarity index 90% rename from test/helm/deployment-custom-topology.out.yaml rename to test/helm/gateway-helm/deployment-custom-topology.out.yaml index 6cf0c0f154b..40f18f8f849 100644 --- a/test/helm/deployment-custom-topology.out.yaml +++ b/test/helm/gateway-helm/deployment-custom-topology.out.yaml @@ -8,7 +8,7 @@ metadata: labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm --- @@ -21,7 +21,7 @@ metadata: labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm data: @@ -56,7 +56,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: creationTimestamp: null - name: eg-gateway-helm-envoy-gateway-role + name: gateway-helm-envoy-gateway-role rules: - apiGroups: - "" @@ -174,11 +174,11 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: eg-gateway-helm-envoy-gateway-rolebinding + name: gateway-helm-envoy-gateway-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: eg-gateway-helm-envoy-gateway-role + name: gateway-helm-envoy-gateway-role subjects: - kind: ServiceAccount name: 'envoy-gateway' @@ -188,12 +188,12 @@ subjects: apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: eg-gateway-helm-infra-manager + name: gateway-helm-infra-manager namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm rules: @@ -233,12 +233,12 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: eg-gateway-helm-leader-election-role + name: gateway-helm-leader-election-role namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm rules: @@ -278,18 +278,18 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: eg-gateway-helm-infra-manager + name: gateway-helm-infra-manager namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: 'eg-gateway-helm-infra-manager' + name: 'gateway-helm-infra-manager' subjects: - kind: ServiceAccount name: 'envoy-gateway' @@ -299,18 +299,18 @@ subjects: apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: eg-gateway-helm-leader-election-rolebinding + name: gateway-helm-leader-election-rolebinding namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: 'eg-gateway-helm-leader-election-role' + name: 'gateway-helm-leader-election-role' subjects: - kind: ServiceAccount name: 'envoy-gateway' @@ -326,14 +326,14 @@ metadata: control-plane: envoy-gateway helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm spec: selector: control-plane: envoy-gateway app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm ports: - name: grpc port: 18000 @@ -355,7 +355,7 @@ metadata: control-plane: envoy-gateway helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm spec: @@ -364,7 +364,7 @@ spec: matchLabels: control-plane: envoy-gateway app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm template: metadata: annotations: @@ -373,7 +373,7 @@ spec: labels: control-plane: envoy-gateway app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm spec: affinity: nodeAffinity: @@ -471,12 +471,12 @@ spec: apiVersion: v1 kind: ServiceAccount metadata: - name: eg-gateway-helm-certgen + name: gateway-helm-certgen namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm annotations: @@ -486,12 +486,12 @@ metadata: apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: eg-gateway-helm-certgen + name: gateway-helm-certgen namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm annotations: @@ -510,12 +510,12 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: eg-gateway-helm-certgen + name: gateway-helm-certgen namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm annotations: @@ -523,22 +523,22 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: 'eg-gateway-helm-certgen' + name: 'gateway-helm-certgen' subjects: - kind: ServiceAccount - name: 'eg-gateway-helm-certgen' + name: 'gateway-helm-certgen' namespace: 'envoy-gateway-system' --- # Source: gateway-helm/templates/certgen.yaml apiVersion: batch/v1 kind: Job metadata: - name: eg-gateway-helm-certgen + name: gateway-helm-certgen namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm annotations: @@ -573,5 +573,5 @@ spec: runAsGroup: 65534 runAsNonRoot: true runAsUser: 65534 - serviceAccountName: eg-gateway-helm-certgen + serviceAccountName: gateway-helm-certgen ttlSecondsAfterFinished: 30 diff --git a/test/helm/deployment-images-config.in.yaml b/test/helm/gateway-helm/deployment-images-config.in.yaml similarity index 100% rename from test/helm/deployment-images-config.in.yaml rename to test/helm/gateway-helm/deployment-images-config.in.yaml diff --git a/test/helm/deployment-images-config.out.yaml b/test/helm/gateway-helm/deployment-images-config.out.yaml similarity index 90% rename from test/helm/deployment-images-config.out.yaml rename to test/helm/gateway-helm/deployment-images-config.out.yaml index 415b9508ece..261544dba3b 100644 --- a/test/helm/deployment-images-config.out.yaml +++ b/test/helm/gateway-helm/deployment-images-config.out.yaml @@ -8,7 +8,7 @@ metadata: labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm --- @@ -21,7 +21,7 @@ metadata: labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm data: @@ -56,7 +56,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: creationTimestamp: null - name: eg-gateway-helm-envoy-gateway-role + name: gateway-helm-envoy-gateway-role rules: - apiGroups: - "" @@ -174,11 +174,11 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: eg-gateway-helm-envoy-gateway-rolebinding + name: gateway-helm-envoy-gateway-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: eg-gateway-helm-envoy-gateway-role + name: gateway-helm-envoy-gateway-role subjects: - kind: ServiceAccount name: 'envoy-gateway' @@ -188,12 +188,12 @@ subjects: apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: eg-gateway-helm-infra-manager + name: gateway-helm-infra-manager namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm rules: @@ -233,12 +233,12 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: eg-gateway-helm-leader-election-role + name: gateway-helm-leader-election-role namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm rules: @@ -278,18 +278,18 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: eg-gateway-helm-infra-manager + name: gateway-helm-infra-manager namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: 'eg-gateway-helm-infra-manager' + name: 'gateway-helm-infra-manager' subjects: - kind: ServiceAccount name: 'envoy-gateway' @@ -299,18 +299,18 @@ subjects: apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: eg-gateway-helm-leader-election-rolebinding + name: gateway-helm-leader-election-rolebinding namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: 'eg-gateway-helm-leader-election-role' + name: 'gateway-helm-leader-election-role' subjects: - kind: ServiceAccount name: 'envoy-gateway' @@ -326,14 +326,14 @@ metadata: control-plane: envoy-gateway helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm spec: selector: control-plane: envoy-gateway app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm ports: - name: grpc port: 18000 @@ -355,7 +355,7 @@ metadata: control-plane: envoy-gateway helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm spec: @@ -364,7 +364,7 @@ spec: matchLabels: control-plane: envoy-gateway app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm template: metadata: annotations: @@ -373,7 +373,7 @@ spec: labels: control-plane: envoy-gateway app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm spec: containers: - args: @@ -445,12 +445,12 @@ spec: apiVersion: v1 kind: ServiceAccount metadata: - name: eg-gateway-helm-certgen + name: gateway-helm-certgen namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm annotations: @@ -460,12 +460,12 @@ metadata: apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: eg-gateway-helm-certgen + name: gateway-helm-certgen namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm annotations: @@ -484,12 +484,12 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: eg-gateway-helm-certgen + name: gateway-helm-certgen namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm annotations: @@ -497,22 +497,22 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: 'eg-gateway-helm-certgen' + name: 'gateway-helm-certgen' subjects: - kind: ServiceAccount - name: 'eg-gateway-helm-certgen' + name: 'gateway-helm-certgen' namespace: 'envoy-gateway-system' --- # Source: gateway-helm/templates/certgen.yaml apiVersion: batch/v1 kind: Job metadata: - name: eg-gateway-helm-certgen + name: gateway-helm-certgen namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm annotations: @@ -549,5 +549,5 @@ spec: runAsGroup: 65534 runAsNonRoot: true runAsUser: 65534 - serviceAccountName: eg-gateway-helm-certgen + serviceAccountName: gateway-helm-certgen ttlSecondsAfterFinished: 30 diff --git a/test/helm/envoy-gateway-config.in.yaml b/test/helm/gateway-helm/envoy-gateway-config.in.yaml similarity index 100% rename from test/helm/envoy-gateway-config.in.yaml rename to test/helm/gateway-helm/envoy-gateway-config.in.yaml diff --git a/test/helm/envoy-gateway-config.out.yaml b/test/helm/gateway-helm/envoy-gateway-config.out.yaml similarity index 90% rename from test/helm/envoy-gateway-config.out.yaml rename to test/helm/gateway-helm/envoy-gateway-config.out.yaml index 46f33d794da..bdb60c8f131 100644 --- a/test/helm/envoy-gateway-config.out.yaml +++ b/test/helm/gateway-helm/envoy-gateway-config.out.yaml @@ -8,7 +8,7 @@ metadata: labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm --- @@ -21,7 +21,7 @@ metadata: labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm data: @@ -58,7 +58,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: creationTimestamp: null - name: eg-gateway-helm-envoy-gateway-role + name: gateway-helm-envoy-gateway-role rules: - apiGroups: - "" @@ -176,11 +176,11 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: eg-gateway-helm-envoy-gateway-rolebinding + name: gateway-helm-envoy-gateway-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: eg-gateway-helm-envoy-gateway-role + name: gateway-helm-envoy-gateway-role subjects: - kind: ServiceAccount name: 'envoy-gateway' @@ -190,12 +190,12 @@ subjects: apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: eg-gateway-helm-infra-manager + name: gateway-helm-infra-manager namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm rules: @@ -235,12 +235,12 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: eg-gateway-helm-leader-election-role + name: gateway-helm-leader-election-role namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm rules: @@ -280,18 +280,18 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: eg-gateway-helm-infra-manager + name: gateway-helm-infra-manager namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: 'eg-gateway-helm-infra-manager' + name: 'gateway-helm-infra-manager' subjects: - kind: ServiceAccount name: 'envoy-gateway' @@ -301,18 +301,18 @@ subjects: apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: eg-gateway-helm-leader-election-rolebinding + name: gateway-helm-leader-election-rolebinding namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: 'eg-gateway-helm-leader-election-role' + name: 'gateway-helm-leader-election-role' subjects: - kind: ServiceAccount name: 'envoy-gateway' @@ -328,14 +328,14 @@ metadata: control-plane: envoy-gateway helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm spec: selector: control-plane: envoy-gateway app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm ports: - name: grpc port: 18000 @@ -357,7 +357,7 @@ metadata: control-plane: envoy-gateway helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm spec: @@ -366,7 +366,7 @@ spec: matchLabels: control-plane: envoy-gateway app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm template: metadata: annotations: @@ -375,7 +375,7 @@ spec: labels: control-plane: envoy-gateway app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm spec: containers: - args: @@ -445,12 +445,12 @@ spec: apiVersion: v1 kind: ServiceAccount metadata: - name: eg-gateway-helm-certgen + name: gateway-helm-certgen namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm annotations: @@ -460,12 +460,12 @@ metadata: apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: eg-gateway-helm-certgen + name: gateway-helm-certgen namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm annotations: @@ -484,12 +484,12 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: eg-gateway-helm-certgen + name: gateway-helm-certgen namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm annotations: @@ -497,22 +497,22 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: 'eg-gateway-helm-certgen' + name: 'gateway-helm-certgen' subjects: - kind: ServiceAccount - name: 'eg-gateway-helm-certgen' + name: 'gateway-helm-certgen' namespace: 'envoy-gateway-system' --- # Source: gateway-helm/templates/certgen.yaml apiVersion: batch/v1 kind: Job metadata: - name: eg-gateway-helm-certgen + name: gateway-helm-certgen namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm annotations: @@ -547,5 +547,5 @@ spec: runAsGroup: 65534 runAsNonRoot: true runAsUser: 65534 - serviceAccountName: eg-gateway-helm-certgen + serviceAccountName: gateway-helm-certgen ttlSecondsAfterFinished: 30 diff --git a/test/helm/global-images-config.in.yaml b/test/helm/gateway-helm/global-images-config.in.yaml similarity index 100% rename from test/helm/global-images-config.in.yaml rename to test/helm/gateway-helm/global-images-config.in.yaml diff --git a/test/helm/global-images-config.out.yaml b/test/helm/gateway-helm/global-images-config.out.yaml similarity index 90% rename from test/helm/global-images-config.out.yaml rename to test/helm/gateway-helm/global-images-config.out.yaml index 5ed2ebb7537..4f1490d32e7 100644 --- a/test/helm/global-images-config.out.yaml +++ b/test/helm/gateway-helm/global-images-config.out.yaml @@ -8,7 +8,7 @@ metadata: labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm --- @@ -21,7 +21,7 @@ metadata: labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm data: @@ -60,7 +60,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: creationTimestamp: null - name: eg-gateway-helm-envoy-gateway-role + name: gateway-helm-envoy-gateway-role rules: - apiGroups: - "" @@ -178,11 +178,11 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: eg-gateway-helm-envoy-gateway-rolebinding + name: gateway-helm-envoy-gateway-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: eg-gateway-helm-envoy-gateway-role + name: gateway-helm-envoy-gateway-role subjects: - kind: ServiceAccount name: 'envoy-gateway' @@ -192,12 +192,12 @@ subjects: apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: eg-gateway-helm-infra-manager + name: gateway-helm-infra-manager namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm rules: @@ -237,12 +237,12 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: eg-gateway-helm-leader-election-role + name: gateway-helm-leader-election-role namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm rules: @@ -282,18 +282,18 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: eg-gateway-helm-infra-manager + name: gateway-helm-infra-manager namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: 'eg-gateway-helm-infra-manager' + name: 'gateway-helm-infra-manager' subjects: - kind: ServiceAccount name: 'envoy-gateway' @@ -303,18 +303,18 @@ subjects: apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: eg-gateway-helm-leader-election-rolebinding + name: gateway-helm-leader-election-rolebinding namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: 'eg-gateway-helm-leader-election-role' + name: 'gateway-helm-leader-election-role' subjects: - kind: ServiceAccount name: 'envoy-gateway' @@ -330,14 +330,14 @@ metadata: control-plane: envoy-gateway helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm spec: selector: control-plane: envoy-gateway app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm ports: - name: grpc port: 18000 @@ -359,7 +359,7 @@ metadata: control-plane: envoy-gateway helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm spec: @@ -368,7 +368,7 @@ spec: matchLabels: control-plane: envoy-gateway app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm template: metadata: annotations: @@ -377,7 +377,7 @@ spec: labels: control-plane: envoy-gateway app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm spec: containers: - args: @@ -449,12 +449,12 @@ spec: apiVersion: v1 kind: ServiceAccount metadata: - name: eg-gateway-helm-certgen + name: gateway-helm-certgen namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm annotations: @@ -464,12 +464,12 @@ metadata: apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: eg-gateway-helm-certgen + name: gateway-helm-certgen namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm annotations: @@ -488,12 +488,12 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: eg-gateway-helm-certgen + name: gateway-helm-certgen namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm annotations: @@ -501,22 +501,22 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: 'eg-gateway-helm-certgen' + name: 'gateway-helm-certgen' subjects: - kind: ServiceAccount - name: 'eg-gateway-helm-certgen' + name: 'gateway-helm-certgen' namespace: 'envoy-gateway-system' --- # Source: gateway-helm/templates/certgen.yaml apiVersion: batch/v1 kind: Job metadata: - name: eg-gateway-helm-certgen + name: gateway-helm-certgen namespace: 'envoy-gateway-system' labels: helm.sh/chart: gateway-helm-v0.0.0-latest app.kubernetes.io/name: gateway-helm - app.kubernetes.io/instance: eg + app.kubernetes.io/instance: gateway-helm app.kubernetes.io/version: "latest" app.kubernetes.io/managed-by: Helm annotations: @@ -553,5 +553,5 @@ spec: runAsGroup: 65534 runAsNonRoot: true runAsUser: 65534 - serviceAccountName: eg-gateway-helm-certgen + serviceAccountName: gateway-helm-certgen ttlSecondsAfterFinished: 30 diff --git a/tools/linter/yamllint/.yamllint b/tools/linter/yamllint/.yamllint index 45705c71c68..4a9e282a263 100644 --- a/tools/linter/yamllint/.yamllint +++ b/tools/linter/yamllint/.yamllint @@ -8,6 +8,7 @@ ignore: | charts/gateway-helm/ charts/gateway-addons-helm/ bin/install.yaml + test/helm/ rules: braces: diff --git a/tools/make/common.mk b/tools/make/common.mk index 84865267e6c..3dd383e7ee4 100644 --- a/tools/make/common.mk +++ b/tools/make/common.mk @@ -119,7 +119,7 @@ export USAGE_OPTIONS .PHONY: generate generate: ## Generate go code from templates and tags -generate: kube-generate docs-api helm-generate helm-template go.generate +generate: kube-generate docs-api helm-generate go.generate ## help: Show this help info. .PHONY: help diff --git a/tools/make/helm.mk b/tools/make/helm.mk index d23305a6743..94d2b9c3f93 100644 --- a/tools/make/helm.mk +++ b/tools/make/helm.mk @@ -10,7 +10,6 @@ IMAGE_PULL_POLICY ?= IfNotPresent OCI_REGISTRY ?= oci://docker.io/envoyproxy CHART_NAME ?= gateway-helm CHART_VERSION ?= ${RELEASE_VERSION} -RELEASE_NAMESPACE ?= envoy-gateway-system ##@ Helm .PHONY: helm-package @@ -41,20 +40,6 @@ helm-push.%: helm-package.% $(eval CHART_NAME := $(COMMAND)) helm push ${OUTPUT_DIR}/charts/${CHART_NAME}-${CHART_VERSION}.tgz ${OCI_REGISTRY} -.PHONY: helm-install -helm-install: ## Install envoy gateway relevant helm charts from OCI registry. -helm-install: - @for chart in $(CHARTS); do \ - $(LOG_TARGET); \ - $(MAKE) $(addprefix helm-install., $$(basename $${chart})); \ - done - -.PHONY: helm-install.% -helm-install.%: helm-generate.% - $(eval COMMAND := $(word 1,$(subst ., ,$*))) - $(eval CHART_NAME := $(COMMAND)) - helm install eg ${OCI_REGISTRY}/${CHART_NAME} --version ${CHART_VERSION} -n ${RELEASE_NAMESPACE} --create-namespace - .PHONY: helm-generate helm-generate: @for chart in $(CHARTS); do \ @@ -70,15 +55,15 @@ helm-generate.%: GatewayImage=${IMAGE}:${TAG} GatewayImagePullPolicy=${IMAGE_PULL_POLICY} \ envsubst < charts/${CHART_NAME}/values.tmpl.yaml > ./charts/${CHART_NAME}/values.yaml; \ fi - helm dependency update charts/${CHART_NAME} # Update dependencies for add-ons chart. + helm dependency update charts/${CHART_NAME} helm lint charts/${CHART_NAME} - -HELM_VALUES := $(wildcard test/helm/*.in.yaml) - -helm-template: ## Template envoy gateway helm chart.z - @$(LOG_TARGET) - @for file in $(HELM_VALUES); do \ + $(call log, "Run helm template for chart: ${CHART_NAME}!"); + @for file in $(wildcard test/helm/${CHART_NAME}/*.in.yaml); do \ filename=$$(basename $${file}); \ output="$${filename%.in.*}.out.yaml"; \ - helm template eg charts/${CHART_NAME} -f $${file} > test/helm/$$output --namespace=${RELEASE_NAMESPACE}; \ + if [ ${CHART_NAME} == "gateway-addons-helm" ]; then \ + helm template ${CHART_NAME} charts/${CHART_NAME} -f $${file} > test/helm/${CHART_NAME}/$$output --namespace=monitoring; \ + else \ + helm template ${CHART_NAME} charts/${CHART_NAME} -f $${file} > test/helm/${CHART_NAME}/$$output --namespace=envoy-gateway-system; \ + fi; \ done diff --git a/tools/make/kube.mk b/tools/make/kube.mk index 31ea2c4c6d7..67d54ad8347 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -141,7 +141,7 @@ endif .PHONY: install-e2e-telemetry install-e2e-telemetry: helm-generate.gateway-addons-helm @$(LOG_TARGET) - helm install eg-addons charts/gateway-addons-helm --set grafana.enabled=false,opentelemetry-collector.enabled=true -n monitoring --create-namespace --timeout='$(WAIT_TIMEOUT)' --wait --wait-for-jobs + helm upgrade -i eg-addons charts/gateway-addons-helm --set grafana.enabled=false,opentelemetry-collector.enabled=true -n monitoring --create-namespace --timeout='$(WAIT_TIMEOUT)' --wait --wait-for-jobs # Change loki service type from ClusterIP to LoadBalancer kubectl patch service loki -n monitoring -p '{"spec": {"type": "LoadBalancer"}}' From 68463d2a376a342bdda3f4245f5ae82a405d90f6 Mon Sep 17 00:00:00 2001 From: Arko Dasgupta Date: Tue, 25 Jun 2024 12:09:03 -0700 Subject: [PATCH 14/23] docs: rm backend redirect docs (#3591) docs: update redirect tasks * dont use `port: 443` in the redirect example * dont specify a backendRefs when the filter is a redirect filter Fixes: https://github.com/envoyproxy/gateway/issues/3589 Signed-off-by: Arko Dasgupta --- .../en/latest/tasks/traffic/http-redirect.md | 8 - .../en/v1.0.2/tasks/traffic/http-redirect.md | 163 ++++++++++++++++-- 2 files changed, 153 insertions(+), 18 deletions(-) diff --git a/site/content/en/latest/tasks/traffic/http-redirect.md b/site/content/en/latest/tasks/traffic/http-redirect.md index 2a41777f80b..b3177e89263 100644 --- a/site/content/en/latest/tasks/traffic/http-redirect.md +++ b/site/content/en/latest/tasks/traffic/http-redirect.md @@ -40,7 +40,6 @@ spec: scheme: https statusCode: 301 hostname: www.example.com - port: 443 EOF ``` @@ -66,7 +65,6 @@ spec: scheme: https statusCode: 301 hostname: www.example.com - port: 443 ``` {{% /tab %}} @@ -342,9 +340,6 @@ spec: type: ReplaceFullPath replaceFullPath: /status/200 statusCode: 302 - backendRefs: - - name: backend - port: 3000 EOF ``` @@ -375,9 +370,6 @@ spec: type: ReplaceFullPath replaceFullPath: /status/200 statusCode: 302 - backendRefs: - - name: backend - port: 3000 ``` {{% /tab %}} diff --git a/site/content/en/v1.0.2/tasks/traffic/http-redirect.md b/site/content/en/v1.0.2/tasks/traffic/http-redirect.md index aeb4db8a5a2..b3177e89263 100644 --- a/site/content/en/v1.0.2/tasks/traffic/http-redirect.md +++ b/site/content/en/v1.0.2/tasks/traffic/http-redirect.md @@ -19,6 +19,9 @@ Redirects return HTTP 3XX responses to a client, instructing it to retrieve a di For example, to issue a permanent redirect (301) from HTTP to HTTPS, configure `requestRedirect.statusCode=301` and `requestRedirect.scheme="https"`: +{{< tabpane text=true >}} +{{% tab header="Apply from stdin" %}} + ```shell cat <}} + __Note:__ `301` (default) and `302` are the only supported statusCodes. The HTTPRoute status should indicate that it has been accepted and is bound to the example Gateway. @@ -69,7 +95,7 @@ $ curl -L -vvv --header "Host: redirect.example" "http://${GATEWAY_HOST}/get" ... ``` -If you followed the steps in the [Secure Gateways](../security/secure-gateways) guide, you should be able to curl the redirect +If you followed the steps in the [Secure Gateways](../security/secure-gateways) task, you should be able to curl the redirect location. ## HTTP --> HTTPS @@ -107,8 +133,11 @@ kubectl create secret tls example-com --key=tls.key --cert=tls.crt Define a https listener on the existing gateway +{{< tabpane text=true >}} +{{% tab header="Apply from stdin" %}} + ```shell -cat <}} + Check for any TLS certificate issues on the gateway. ```bash @@ -140,9 +200,11 @@ kubectl -n default describe gateway eg Create two HTTPRoutes and attach them to the HTTP and HTTPS listeners using the [sectionName][] field. +{{< tabpane text=true >}} +{{% tab header="Apply from stdin" %}} ```shell -cat <}} + Curl the example app through http listener: ```bash @@ -203,6 +314,9 @@ curl -v -H 'Host:www.example.com' --resolve "www.example.com:443:$GATEWAY_HOST" Path redirects use an HTTP Path Modifier to replace either entire paths or path prefixes. For example, the HTTPRoute below will issue a 302 redirect to all `path.redirect.example` requests whose path begins with `/get` to `/status/200`. +{{< tabpane text=true >}} +{{% tab header="Apply from stdin" %}} + ```shell cat <}} + The HTTPRoute status should indicate that it has been accepted and is bound to the example Gateway. ```shell From 77f956ca0199d522ff583b4394ca9bda9c5115a4 Mon Sep 17 00:00:00 2001 From: Alex Marston Date: Wed, 26 Jun 2024 00:25:02 +0100 Subject: [PATCH 15/23] feat: implement zipkin tracing (#3668) * feat: initial zipkin tracing support Signed-off-by: Alex Marston * add zipkin tracing example Signed-off-by: Alex Marston * update testdata Signed-off-by: Alex Marston * update host in testdata Signed-off-by: Alex Marston * update host in testdata Signed-off-by: Alex Marston * lint Signed-off-by: Alex Marston * handle if pointer is null Signed-off-by: Alex Marston --------- Signed-off-by: Alex Marston --- examples/kubernetes/tracing/zipkin.yaml | 47 ++++++++++++++ internal/gatewayapi/listener.go | 1 + .../envoyproxy-tracing-backend.out.yaml | 6 ++ ...with-infrastructure-parametersref.out.yaml | 4 ++ ...astructure-parametersref-fallback.out.yaml | 4 ++ .../tracing-merged-multiple-routes.out.yaml | 4 ++ .../testdata/tracing-multiple-routes.out.yaml | 8 +++ internal/ir/xds.go | 1 + internal/ir/zz_generated.deepcopy.go | 1 + .../in/xds-ir/tracing-endpoint-stats.yaml | 4 ++ .../xds-ir/tracing-unknown-provider-type.yaml | 51 +++++++++++++++ .../testdata/in/xds-ir/tracing-zipkin.yaml | 55 ++++++++++++++++ .../testdata/in/xds-ir/tracing.yaml | 4 ++ .../out/xds-ir/tracing-zipkin.clusters.yaml | 49 ++++++++++++++ .../out/xds-ir/tracing-zipkin.endpoints.yaml | 12 ++++ .../out/xds-ir/tracing-zipkin.listeners.yaml | 64 +++++++++++++++++++ .../out/xds-ir/tracing-zipkin.routes.yaml | 14 ++++ internal/xds/translator/tracing.go | 61 ++++++++++++++---- internal/xds/translator/translator_test.go | 3 + 19 files changed, 381 insertions(+), 12 deletions(-) create mode 100644 examples/kubernetes/tracing/zipkin.yaml create mode 100644 internal/xds/translator/testdata/in/xds-ir/tracing-unknown-provider-type.yaml create mode 100644 internal/xds/translator/testdata/in/xds-ir/tracing-zipkin.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/tracing-zipkin.clusters.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/tracing-zipkin.endpoints.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/tracing-zipkin.listeners.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/tracing-zipkin.routes.yaml diff --git a/examples/kubernetes/tracing/zipkin.yaml b/examples/kubernetes/tracing/zipkin.yaml new file mode 100644 index 00000000000..6a2b99f637d --- /dev/null +++ b/examples/kubernetes/tracing/zipkin.yaml @@ -0,0 +1,47 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: eg +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parametersRef: + group: gateway.envoyproxy.io + kind: EnvoyProxy + name: zipkin-tracing + namespace: envoy-gateway-system +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: zipkin-tracing + namespace: envoy-gateway-system +spec: + telemetry: + tracing: + # sample 100% of requests + samplingRate: 100 + provider: + host: otel-collector.monitoring.svc.cluster.local + port: 9411 + type: Zipkin + # zipkin specific configuration + zipkin: + enable128BitTraceId: true + customTags: + # This is an example of using a literal as a tag value + key1: + type: Literal + literal: + value: "val1" + # This is an example of using an environment variable as a tag value + env1: + type: Environment + environment: + name: ENV1 + defaultValue: "-" + # This is an example of using a header value as a tag value + header1: + type: RequestHeader + requestHeader: + name: X-Header-1 + defaultValue: "-" diff --git a/internal/gatewayapi/listener.go b/internal/gatewayapi/listener.go index 8957874cbae..6978d806ad6 100644 --- a/internal/gatewayapi/listener.go +++ b/internal/gatewayapi/listener.go @@ -353,6 +353,7 @@ func (t *Translator) processTracing(gw *gwapiv1.Gateway, envoyproxy *egv1a1.Envo Name: "tracing", // TODO: rename this, so that we can share backend with accesslog? Settings: ds, }, + Provider: tracing.Provider, }, nil } diff --git a/internal/gatewayapi/testdata/envoyproxy-tracing-backend.out.yaml b/internal/gatewayapi/testdata/envoyproxy-tracing-backend.out.yaml index 8393d9c4ff8..f413abe8655 100644 --- a/internal/gatewayapi/testdata/envoyproxy-tracing-backend.out.yaml +++ b/internal/gatewayapi/testdata/envoyproxy-tracing-backend.out.yaml @@ -146,5 +146,11 @@ xdsIR: - host: 8.7.6.5 port: 4317 protocol: GRPC + provider: + backendRefs: + - name: otel-collector + namespace: monitoring + port: 4317 + type: OpenTelemetry samplingRate: 100 serviceName: gateway-1.envoy-gateway diff --git a/internal/gatewayapi/testdata/gateway-with-infrastructure-parametersref.out.yaml b/internal/gatewayapi/testdata/gateway-with-infrastructure-parametersref.out.yaml index 1e47494ffed..56d7485639d 100644 --- a/internal/gatewayapi/testdata/gateway-with-infrastructure-parametersref.out.yaml +++ b/internal/gatewayapi/testdata/gateway-with-infrastructure-parametersref.out.yaml @@ -159,5 +159,9 @@ xdsIR: port: 4317 protocol: GRPC weight: 1 + provider: + host: otel-collector.monitoring.svc.cluster.local + port: 4317 + type: OpenTelemetry samplingRate: 100 serviceName: gateway-1.envoy-gateway diff --git a/internal/gatewayapi/testdata/gateway-with-invalid-infrastructure-parametersref-fallback.out.yaml b/internal/gatewayapi/testdata/gateway-with-invalid-infrastructure-parametersref-fallback.out.yaml index 8a477792a8f..554b6189380 100644 --- a/internal/gatewayapi/testdata/gateway-with-invalid-infrastructure-parametersref-fallback.out.yaml +++ b/internal/gatewayapi/testdata/gateway-with-invalid-infrastructure-parametersref-fallback.out.yaml @@ -159,5 +159,9 @@ xdsIR: port: 4317 protocol: GRPC weight: 1 + provider: + host: otel-collector.monitoring.svc.cluster.local + port: 4317 + type: OpenTelemetry samplingRate: 100 serviceName: gateway-1.envoy-gateway diff --git a/internal/gatewayapi/testdata/tracing-merged-multiple-routes.out.yaml b/internal/gatewayapi/testdata/tracing-merged-multiple-routes.out.yaml index 9fd0cc12b2d..0a230d17d1c 100644 --- a/internal/gatewayapi/testdata/tracing-merged-multiple-routes.out.yaml +++ b/internal/gatewayapi/testdata/tracing-merged-multiple-routes.out.yaml @@ -301,5 +301,9 @@ xdsIR: port: 4317 protocol: GRPC weight: 1 + provider: + host: otel-collector.monitoring.svc.cluster.local + port: 4317 + type: OpenTelemetry samplingRate: 100 serviceName: envoy-gateway-class diff --git a/internal/gatewayapi/testdata/tracing-multiple-routes.out.yaml b/internal/gatewayapi/testdata/tracing-multiple-routes.out.yaml index 90f8dc6a27d..6e848009bb5 100644 --- a/internal/gatewayapi/testdata/tracing-multiple-routes.out.yaml +++ b/internal/gatewayapi/testdata/tracing-multiple-routes.out.yaml @@ -291,6 +291,10 @@ xdsIR: port: 4317 protocol: GRPC weight: 1 + provider: + host: otel-collector.monitoring.svc.cluster.local + port: 4317 + type: OpenTelemetry samplingRate: 100 serviceName: gateway-1.envoy-gateway envoy-gateway/gateway-2: @@ -343,5 +347,9 @@ xdsIR: port: 4317 protocol: GRPC weight: 1 + provider: + host: otel-collector.monitoring.svc.cluster.local + port: 4317 + type: OpenTelemetry samplingRate: 100 serviceName: gateway-2.envoy-gateway diff --git a/internal/ir/xds.go b/internal/ir/xds.go index cab235895cf..d3eb48d853d 100644 --- a/internal/ir/xds.go +++ b/internal/ir/xds.go @@ -1684,6 +1684,7 @@ type Tracing struct { SamplingRate float64 `json:"samplingRate,omitempty"` CustomTags map[string]egv1a1.CustomTag `json:"customTags,omitempty"` Destination RouteDestination `json:"destination,omitempty"` + Provider egv1a1.TracingProvider `json:"provider"` } // Metrics defines the configuration for metrics generated by Envoy diff --git a/internal/ir/zz_generated.deepcopy.go b/internal/ir/zz_generated.deepcopy.go index 459c21ef10d..f4882032c03 100644 --- a/internal/ir/zz_generated.deepcopy.go +++ b/internal/ir/zz_generated.deepcopy.go @@ -2631,6 +2631,7 @@ func (in *Tracing) DeepCopyInto(out *Tracing) { } } in.Destination.DeepCopyInto(&out.Destination) + in.Provider.DeepCopyInto(&out.Provider) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Tracing. diff --git a/internal/xds/translator/testdata/in/xds-ir/tracing-endpoint-stats.yaml b/internal/xds/translator/testdata/in/xds-ir/tracing-endpoint-stats.yaml index f57949f2ce4..b5ee8b57dd9 100644 --- a/internal/xds/translator/testdata/in/xds-ir/tracing-endpoint-stats.yaml +++ b/internal/xds/translator/testdata/in/xds-ir/tracing-endpoint-stats.yaml @@ -27,6 +27,10 @@ tracing: - host: "otel-collector.default.svc.cluster.local" port: 4317 protocol: "GRPC" + provider: + host: otel-collector.monitoring.svc.cluster.local + port: 4317 + type: OpenTelemetry http: - name: "first-listener" address: "0.0.0.0" diff --git a/internal/xds/translator/testdata/in/xds-ir/tracing-unknown-provider-type.yaml b/internal/xds/translator/testdata/in/xds-ir/tracing-unknown-provider-type.yaml new file mode 100644 index 00000000000..45f669ef643 --- /dev/null +++ b/internal/xds/translator/testdata/in/xds-ir/tracing-unknown-provider-type.yaml @@ -0,0 +1,51 @@ +name: "tracing" +tracing: + serviceName: "fake-name.fake-ns" + samplingRate: 90 + customTags: + "literal1": + type: Literal + literal: + value: "value1" + "env1": + type: Environment + environment: + name: "env1" + defaultValue: "-" + "req1": + type: RequestHeader + requestHeader: + name: "X-Request-Id" + defaultValue: "-" + authority: "datadog-agent.default.svc.cluster.local" + destination: + name: "tracing-0" + settings: + - endpoints: + - host: "datadog-agent.default.svc.cluster.local" + port: 8126 + provider: + host: datadog-agent.monitoring.svc.cluster.local + port: 8126 + type: Datadog +http: + - name: "first-listener" + address: "0.0.0.0" + port: 10080 + hostnames: + - "*" + path: + mergeSlashes: true + escapedSlashesAction: UnescapeAndRedirect + routes: + - name: "direct-route" + hostname: "*" + destination: + name: "direct-route-dest" + settings: + - endpoints: + - host: "1.2.3.4" + port: 50000 + directResponse: + body: "Unknown custom filter type: UnsupportedType" + statusCode: 500 diff --git a/internal/xds/translator/testdata/in/xds-ir/tracing-zipkin.yaml b/internal/xds/translator/testdata/in/xds-ir/tracing-zipkin.yaml new file mode 100644 index 00000000000..a60183dd268 --- /dev/null +++ b/internal/xds/translator/testdata/in/xds-ir/tracing-zipkin.yaml @@ -0,0 +1,55 @@ +name: "tracing" +tracing: + serviceName: "fake-name.fake-ns" + samplingRate: 90 + customTags: + "literal1": + type: Literal + literal: + value: "value1" + "env1": + type: Environment + environment: + name: "env1" + defaultValue: "-" + "req1": + type: RequestHeader + requestHeader: + name: "X-Request-Id" + defaultValue: "-" + authority: "zipkin.default.svc.cluster.local" + destination: + name: "tracing-0" + settings: + - endpoints: + - host: "zipkin.default.svc.cluster.local" + port: 9411 + protocol: "GRPC" + provider: + host: zipkin.default.svc.cluster.local + port: 9411 + type: Zipkin + zipkin: + enable128BitTraceId: true + disableSharedSpanContext: true +http: +- name: "first-listener" + address: "0.0.0.0" + port: 10080 + hostnames: + - "*" + path: + mergeSlashes: true + escapedSlashesAction: UnescapeAndRedirect + routes: + - name: "direct-route" + hostname: "*" + destination: + name: "direct-route-dest" + settings: + - endpoints: + - host: "1.2.3.4" + port: 50000 + directResponse: + body: "Unknown custom filter type: UnsupportedType" + statusCode: 500 diff --git a/internal/xds/translator/testdata/in/xds-ir/tracing.yaml b/internal/xds/translator/testdata/in/xds-ir/tracing.yaml index fd5a29672dd..0f3555524ff 100644 --- a/internal/xds/translator/testdata/in/xds-ir/tracing.yaml +++ b/internal/xds/translator/testdata/in/xds-ir/tracing.yaml @@ -25,6 +25,10 @@ tracing: - host: "otel-collector.default.svc.cluster.local" port: 4317 protocol: "GRPC" + provider: + host: otel-collector.monitoring.svc.cluster.local + port: 4317 + type: OpenTelemetry http: - name: "first-listener" address: "0.0.0.0" diff --git a/internal/xds/translator/testdata/out/xds-ir/tracing-zipkin.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/tracing-zipkin.clusters.yaml new file mode 100644 index 00000000000..b8a1ac3df39 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/tracing-zipkin.clusters.yaml @@ -0,0 +1,49 @@ +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_ONLY + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: direct-route-dest + lbPolicy: LEAST_REQUEST + name: direct-route-dest + outlierDetection: {} + perConnectionBufferLimitBytes: 32768 + type: EDS +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_ONLY + dnsRefreshRate: 30s + lbPolicy: LEAST_REQUEST + loadAssignment: + clusterName: tracing-0 + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: zipkin.default.svc.cluster.local + portValue: 9411 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: tracing-0/backend/0 + name: tracing-0 + outlierDetection: {} + perConnectionBufferLimitBytes: 32768 + respectDnsTtl: true + type: STRICT_DNS + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicitHttpConfig: + http2ProtocolOptions: {} diff --git a/internal/xds/translator/testdata/out/xds-ir/tracing-zipkin.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/tracing-zipkin.endpoints.yaml new file mode 100644 index 00000000000..20c80b3aaaa --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/tracing-zipkin.endpoints.yaml @@ -0,0 +1,12 @@ +- clusterName: direct-route-dest + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.2.3.4 + portValue: 50000 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: direct-route-dest/backend/0 diff --git a/internal/xds/translator/testdata/out/xds-ir/tracing-zipkin.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/tracing-zipkin.listeners.yaml new file mode 100644 index 00000000000..25b3e9e4d40 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/tracing-zipkin.listeners.yaml @@ -0,0 +1,64 @@ +- address: + socketAddress: + address: 0.0.0.0 + portValue: 10080 + defaultFilterChain: + filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + commonHttpProtocolOptions: + headersWithUnderscoresAction: REJECT_REQUEST + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 + maxConcurrentStreams: 100 + httpFilters: + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + suppressEnvoyHeaders: true + mergeSlashes: true + normalizePath: true + pathWithEscapedSlashesAction: UNESCAPE_AND_REDIRECT + rds: + configSource: + ads: {} + resourceApiVersion: V3 + routeConfigName: first-listener + serverHeaderTransformation: PASS_THROUGH + statPrefix: http + tracing: + clientSampling: + value: 100 + customTags: + - environment: + defaultValue: '-' + name: env1 + tag: env1 + - literal: + value: value1 + tag: literal1 + - requestHeader: + defaultValue: '-' + name: X-Request-Id + tag: req1 + overallSampling: + value: 100 + provider: + name: envoy.traces.zipkin + typedConfig: + '@type': type.googleapis.com/envoy.config.trace.v3.ZipkinConfig + collectorCluster: tracing-0 + collectorEndpoint: /api/v2/spans + collectorEndpointVersion: HTTP_JSON + sharedSpanContext: false + traceId128bit: true + randomSampling: + value: 90 + spawnUpstreamSpan: true + useRemoteAddress: true + name: first-listener + drainType: MODIFY_ONLY + name: first-listener + perConnectionBufferLimitBytes: 32768 diff --git a/internal/xds/translator/testdata/out/xds-ir/tracing-zipkin.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/tracing-zipkin.routes.yaml new file mode 100644 index 00000000000..d4a7fa5ae20 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/tracing-zipkin.routes.yaml @@ -0,0 +1,14 @@ +- ignorePortInHostMatching: true + name: first-listener + virtualHosts: + - domains: + - '*' + name: first-listener/* + routes: + - directResponse: + body: + inlineString: 'Unknown custom filter type: UnsupportedType' + status: 500 + match: + prefix: / + name: direct-route diff --git a/internal/xds/translator/tracing.go b/internal/xds/translator/tracing.go index c31d25b7290..b2a52ec6a18 100644 --- a/internal/xds/translator/tracing.go +++ b/internal/xds/translator/tracing.go @@ -15,7 +15,9 @@ import ( hcm "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" tracingtype "github.com/envoyproxy/go-control-plane/envoy/type/tracing/v3" xdstype "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "google.golang.org/protobuf/types/known/anypb" "google.golang.org/protobuf/types/known/wrapperspb" + "k8s.io/utils/ptr" egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/ir" @@ -23,26 +25,61 @@ import ( "github.com/envoyproxy/gateway/internal/xds/types" ) +const ( + envoyOpenTelemetry = "envoy.tracers.opentelemetry" + envoyZipkin = "envoy.traces.zipkin" +) + +type typConfigGen func() (*anypb.Any, error) + func buildHCMTracing(tracing *ir.Tracing) (*hcm.HttpConnectionManager_Tracing, error) { if tracing == nil { return nil, nil } - oc := &tracecfg.OpenTelemetryConfig{ - GrpcService: &corev3.GrpcService{ - TargetSpecifier: &corev3.GrpcService_EnvoyGrpc_{ - EnvoyGrpc: &corev3.GrpcService_EnvoyGrpc{ - ClusterName: tracing.Destination.Name, - Authority: tracing.Authority, + var providerName string + var providerConfig typConfigGen + + switch tracing.Provider.Type { + case egv1a1.TracingProviderTypeOpenTelemetry: + providerName = envoyOpenTelemetry + + providerConfig = func() (*anypb.Any, error) { + config := &tracecfg.OpenTelemetryConfig{ + GrpcService: &corev3.GrpcService{ + TargetSpecifier: &corev3.GrpcService_EnvoyGrpc_{ + EnvoyGrpc: &corev3.GrpcService_EnvoyGrpc{ + ClusterName: tracing.Destination.Name, + Authority: tracing.Authority, + }, + }, }, - }, - }, - ServiceName: tracing.ServiceName, + ServiceName: tracing.ServiceName, + } + + return protocov.ToAnyWithError(config) + } + case egv1a1.TracingProviderTypeZipkin: + providerName = envoyZipkin + + providerConfig = func() (*anypb.Any, error) { + config := &tracecfg.ZipkinConfig{ + CollectorCluster: tracing.Destination.Name, + CollectorEndpoint: "/api/v2/spans", + TraceId_128Bit: ptr.Deref(tracing.Provider.Zipkin.Enable128BitTraceID, false), + SharedSpanContext: wrapperspb.Bool(!ptr.Deref(tracing.Provider.Zipkin.DisableSharedSpanContext, false)), + CollectorEndpointVersion: tracecfg.ZipkinConfig_HTTP_JSON, + } + + return protocov.ToAnyWithError(config) + } + default: + return nil, fmt.Errorf("unknown tracing provider type: %s", tracing.Provider.Type) } - ocAny, err := protocov.ToAnyWithError(oc) + ocAny, err := providerConfig() if err != nil { - return nil, fmt.Errorf("failed to marshal OpenTelemetryConfig: %w", err) + return nil, fmt.Errorf("failed to marshal tracing configuration: %w", err) } tags := make([]*tracingtype.CustomTag, 0, len(tracing.CustomTags)) @@ -108,7 +145,7 @@ func buildHCMTracing(tracing *ir.Tracing) (*hcm.HttpConnectionManager_Tracing, e Value: tracing.SamplingRate, }, Provider: &tracecfg.Tracing_Http{ - Name: "envoy.tracers.opentelemetry", + Name: providerName, ConfigType: &tracecfg.Tracing_Http_TypedConfig{ TypedConfig: ocAny, }, diff --git a/internal/xds/translator/translator_test.go b/internal/xds/translator/translator_test.go index 4e559dda22b..81563abf050 100644 --- a/internal/xds/translator/translator_test.go +++ b/internal/xds/translator/translator_test.go @@ -94,6 +94,9 @@ func TestTranslateXds(t *testing.T) { "tracing-invalid": { errMsg: "validation failed for xds resource", }, + "tracing-unknown-provider-type": { + errMsg: "unknown tracing provider type: Datadog", + }, } inputFiles, err := filepath.Glob(filepath.Join("testdata", "in", "xds-ir", "*.yaml")) From fa5ccfb4d31cc618737ffb64ba2a51feb10982c1 Mon Sep 17 00:00:00 2001 From: Arko Dasgupta Date: Tue, 25 Jun 2024 17:37:27 -0700 Subject: [PATCH 16/23] docs: rm active development alert (#3674) Envoy Gateway is GA and stable, and the alert is no longer applicable Signed-off-by: Arko Dasgupta --- site/content/en/latest/_index.md | 6 ------ site/content/en/v1.0.2/_index.md | 6 ------ 2 files changed, 12 deletions(-) diff --git a/site/content/en/latest/_index.md b/site/content/en/latest/_index.md index ea08d244d31..92ae8586885 100644 --- a/site/content/en/latest/_index.md +++ b/site/content/en/latest/_index.md @@ -7,12 +7,6 @@ description = "Envoy Gateway Documents" type = "docs" +++ -{{% alert title="Note" color="primary" %}} - -This project is under **active** development. Many features are not complete. We would love for you to [Get Involved](/contributions)! - -{{% /alert %}} - Envoy Gateway is an open source project for managing [Envoy Proxy](https://www.envoyproxy.io/) as a standalone or Kubernetes-based application gateway. [Gateway API](https://gateway-api.sigs.k8s.io/) resources are used to dynamically provision and configure the managed Envoy Proxies. diff --git a/site/content/en/v1.0.2/_index.md b/site/content/en/v1.0.2/_index.md index ea08d244d31..92ae8586885 100644 --- a/site/content/en/v1.0.2/_index.md +++ b/site/content/en/v1.0.2/_index.md @@ -7,12 +7,6 @@ description = "Envoy Gateway Documents" type = "docs" +++ -{{% alert title="Note" color="primary" %}} - -This project is under **active** development. Many features are not complete. We would love for you to [Get Involved](/contributions)! - -{{% /alert %}} - Envoy Gateway is an open source project for managing [Envoy Proxy](https://www.envoyproxy.io/) as a standalone or Kubernetes-based application gateway. [Gateway API](https://gateway-api.sigs.k8s.io/) resources are used to dynamically provision and configure the managed Envoy Proxies. From d784c3237765aa258c3526b32765dd9c3abf6415 Mon Sep 17 00:00:00 2001 From: Arko Dasgupta Date: Tue, 25 Jun 2024 18:32:41 -0700 Subject: [PATCH 17/23] docs: fix GATEWAY_HOST address for v1.0.2 and latest docs (#3676) Signed-off-by: Arko Dasgupta --- site/content/en/latest/tasks/quickstart.md | 2 +- site/content/en/v1.0.2/tasks/quickstart.md | 39 +++++++++++++--------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/site/content/en/latest/tasks/quickstart.md b/site/content/en/latest/tasks/quickstart.md index 4f345fa289a..c48fec6f83f 100644 --- a/site/content/en/latest/tasks/quickstart.md +++ b/site/content/en/latest/tasks/quickstart.md @@ -54,7 +54,7 @@ You can also test the same functionality by sending traffic to the External IP. Envoy service, run: ```shell -export GATEWAY_HOST=$(kubectl get svc/${ENVOY_SERVICE} -n envoy-gateway-system -o jsonpath='{.status.loadBalancer.ingress[0].ip}') +export GATEWAY_HOST=$(kubectl get gateway/eg -o jsonpath='{.status.addresses[0].value}') ``` In certain environments, the load balancer may be exposed using a hostname, instead of an IP address. If so, replace diff --git a/site/content/en/v1.0.2/tasks/quickstart.md b/site/content/en/v1.0.2/tasks/quickstart.md index 5cf407d2e39..b898b5e989b 100644 --- a/site/content/en/v1.0.2/tasks/quickstart.md +++ b/site/content/en/v1.0.2/tasks/quickstart.md @@ -4,7 +4,7 @@ weight: 1 description: Get started with Envoy Gateway in a few simple steps. --- -This guide will help you get started with Envoy Gateway in a few simple steps. +This "quick start" will help you get started with Envoy Gateway in a few simple steps. ## Prerequisites @@ -47,42 +47,49 @@ consideration when debugging. ## Testing the Configuration -Get the name of the Envoy service created the by the example Gateway: +{{< tabpane text=true >}} +{{% tab header="With External LoadBalancer Support" %}} -```shell -export ENVOY_SERVICE=$(kubectl get svc -n envoy-gateway-system --selector=gateway.envoyproxy.io/owning-gateway-namespace=default,gateway.envoyproxy.io/owning-gateway-name=eg -o jsonpath='{.items[0].metadata.name}') -``` - -Port forward to the Envoy service: +You can also test the same functionality by sending traffic to the External IP. To get the external IP of the +Envoy service, run: ```shell -kubectl -n envoy-gateway-system port-forward service/${ENVOY_SERVICE} 8888:80 & +export GATEWAY_HOST=$(kubectl get gateway/eg -o jsonpath='{.status.addresses[0].value}') ``` +In certain environments, the load balancer may be exposed using a hostname, instead of an IP address. If so, replace +`ip` in the above command with `hostname`. + Curl the example app through Envoy proxy: ```shell -curl --verbose --header "Host: www.example.com" http://localhost:8888/get +curl --verbose --header "Host: www.example.com" http://$GATEWAY_HOST/get ``` -### External LoadBalancer Support +{{% /tab %}} +{{% tab header="Without LoadBalancer Support" %}} -You can also test the same functionality by sending traffic to the External IP. To get the external IP of the -Envoy service, run: +Get the name of the Envoy service created the by the example Gateway: ```shell -export GATEWAY_HOST=$(kubectl get svc/${ENVOY_SERVICE} -n envoy-gateway-system -o jsonpath='{.status.loadBalancer.ingress[0].ip}') +export ENVOY_SERVICE=$(kubectl get svc -n envoy-gateway-system --selector=gateway.envoyproxy.io/owning-gateway-namespace=default,gateway.envoyproxy.io/owning-gateway-name=eg -o jsonpath='{.items[0].metadata.name}') ``` -In certain environments, the load balancer may be exposed using a hostname, instead of an IP address. If so, replace -`ip` in the above command with `hostname`. +Port forward to the Envoy service: + +```shell +kubectl -n envoy-gateway-system port-forward service/${ENVOY_SERVICE} 8888:80 & +``` Curl the example app through Envoy proxy: ```shell -curl --verbose --header "Host: www.example.com" http://$GATEWAY_HOST/get +curl --verbose --header "Host: www.example.com" http://localhost:8888/get ``` +{{% /tab %}} +{{< /tabpane >}} + ## What to explore next? In this quickstart, you have: From 22984d79da1ebd1dece47838ed5a964bc6335ede Mon Sep 17 00:00:00 2001 From: zirain Date: Thu, 27 Jun 2024 09:26:34 +0800 Subject: [PATCH 18/23] api: support AccessLog filter (#3669) * api: support AccessLog filter Signed-off-by: zirain * notImplementedHide Signed-off-by: zirain * rename matches Signed-off-by: zirain * gen Signed-off-by: zirain * CEL link Signed-off-by: zirain * address arko's comment Signed-off-by: zirain --------- Signed-off-by: zirain --- api/v1alpha1/accesslogging_types.go | 5 +++++ api/v1alpha1/zz_generated.deepcopy.go | 5 +++++ .../generated/gateway.envoyproxy.io_envoyproxies.yaml | 8 ++++++++ 3 files changed, 18 insertions(+) diff --git a/api/v1alpha1/accesslogging_types.go b/api/v1alpha1/accesslogging_types.go index d1f60203138..51ecf0be811 100644 --- a/api/v1alpha1/accesslogging_types.go +++ b/api/v1alpha1/accesslogging_types.go @@ -19,6 +19,11 @@ type ProxyAccessLog struct { type ProxyAccessLogSetting struct { // Format defines the format of accesslog. Format ProxyAccessLogFormat `json:"format"` + // Matches defines the match conditions for accesslog in CEL expression. + // An accesslog will be emitted only when one or more match conditions are evaluated to true. + // Invalid [CEL](https://www.envoyproxy.io/docs/envoy/latest/xds/type/v3/cel.proto.html#common-expression-language-cel-proto) expressions will be ignored. + // +notImplementedHide + Matches []string `json:"matches,omitempty"` // Sinks defines the sinks of accesslog. // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=50 diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index ac4a3602528..6d5db4529ba 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -3719,6 +3719,11 @@ func (in *ProxyAccessLogFormat) DeepCopy() *ProxyAccessLogFormat { func (in *ProxyAccessLogSetting) DeepCopyInto(out *ProxyAccessLogSetting) { *out = *in in.Format.DeepCopyInto(&out.Format) + if in.Matches != nil { + in, out := &in.Matches, &out.Matches + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.Sinks != nil { in, out := &in.Sinks, &out.Sinks *out = make([]ProxyAccessLogSink, len(*in)) diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml index f40ff9b0e96..31f954ac8ec 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -10274,6 +10274,14 @@ spec: - message: If AccessLogFormat type is JSON, json field needs to be set. rule: 'self.type == ''JSON'' ? has(self.json) : !has(self.json)' + matches: + description: |- + Matches defines the match conditions for accesslog in CEL expression. + An accesslog will be emitted only when one or more match conditions are evaluated to true. + Invalid [CEL](https://www.envoyproxy.io/docs/envoy/latest/xds/type/v3/cel.proto.html#common-expression-language-cel-proto) expressions will be ignored. + items: + type: string + type: array sinks: description: Sinks defines the sinks of accesslog. items: From 8abf1efef4176092fe4db5ad9239e6d433ca57f2 Mon Sep 17 00:00:00 2001 From: sh2 Date: Thu, 27 Jun 2024 16:37:23 +0800 Subject: [PATCH 19/23] chore: cleanup and upgrade some api to v1 (#3644) * upgrade some api from v1a2 to v1 and cleanup Signed-off-by: shawnh2 * rm unused parentRef check Signed-off-by: shawnh2 --------- Signed-off-by: shawnh2 Co-authored-by: zirain --- internal/gatewayapi/contexts.go | 36 ++--- internal/gatewayapi/helpers_v1alpha2.go | 151 +-------------------- internal/gatewayapi/route.go | 4 +- internal/provider/kubernetes/controller.go | 16 +-- internal/provider/kubernetes/indexers.go | 12 +- internal/provider/kubernetes/routes.go | 18 +-- 6 files changed, 37 insertions(+), 200 deletions(-) diff --git a/internal/gatewayapi/contexts.go b/internal/gatewayapi/contexts.go index 414386c973d..6ecb2cf7318 100644 --- a/internal/gatewayapi/contexts.go +++ b/internal/gatewayapi/contexts.go @@ -206,9 +206,6 @@ func GetRouteType(route RouteContext) gwapiv1.Kind { return gwapiv1.Kind(rv.FieldByName("Kind").String()) } -// TODO: [v1alpha2-gwapiv1] This should not be required once all Route -// objects being implemented are of type gwapiv1. - // GetHostnames returns the hosts targeted by the Route object. func GetHostnames(route RouteContext) []string { rv := reflect.ValueOf(route).Elem() @@ -225,8 +222,6 @@ func GetHostnames(route RouteContext) []string { return hostnames } -// TODO: [v1alpha2-gwapiv1] This should not be required once all Route -// objects being implemented are of type gwapiv1. // GetParentReferences returns the ParentReference of the Route object. func GetParentReferences(route RouteContext) []gwapiv1.ParentReference { rv := reflect.ValueOf(route).Elem() @@ -256,11 +251,6 @@ func GetRouteParentContext(route RouteContext, forParentRef gwapiv1.ParentRefere return ctx } - isHTTPRoute := false - if rv.FieldByName("Kind").String() == KindHTTPRoute { - isHTTPRoute = true - } - var parentRef *gwapiv1.ParentReference specParentRefs := rv.FieldByName("Spec").FieldByName("ParentRefs") for i := 0; i < specParentRefs.Len(); i++ { @@ -275,32 +265,28 @@ func GetRouteParentContext(route RouteContext, forParentRef gwapiv1.ParentRefere } routeParentStatusIdx := -1 + defaultNamespace := gwapiv1.Namespace(metav1.NamespaceDefault) statusParents := rv.FieldByName("Status").FieldByName("Parents") for i := 0; i < statusParents.Len(); i++ { p := statusParents.Index(i).FieldByName("ParentRef").Interface().(gwapiv1.ParentReference) - if !isHTTPRoute { - p = UpgradeParentReference(p) - defaultNamespace := gwapiv1.Namespace(metav1.NamespaceDefault) - if forParentRef.Namespace == nil { - forParentRef.Namespace = &defaultNamespace - } - if p.Namespace == nil { - p.Namespace = &defaultNamespace - } + // For those non-v1 routes, their underlying type of `ParentReference` is v1 as well. + // So we can skip upgrading these routes for simplicity. + if forParentRef.Namespace == nil { + forParentRef.Namespace = &defaultNamespace + } + if p.Namespace == nil { + p.Namespace = &defaultNamespace } if reflect.DeepEqual(p, forParentRef) { routeParentStatusIdx = i break } } + if routeParentStatusIdx == -1 { - tmpPR := forParentRef - if !isHTTPRoute { - tmpPR = DowngradeParentReference(tmpPR) - } rParentStatus := gwapiv1a2.RouteParentStatus{ ControllerName: gwapiv1a2.GatewayController(rv.FieldByName("GatewayControllerName").String()), - ParentRef: tmpPR, + ParentRef: forParentRef, } statusParents.Set(reflect.Append(statusParents, reflect.ValueOf(rParentStatus))) routeParentStatusIdx = statusParents.Len() - 1 @@ -372,8 +358,8 @@ func GetBackendRef(b BackendRefContext) *gwapiv1.BackendRef { if br.IsValid() { backendRef := br.Interface().(gwapiv1.BackendRef) return &backendRef - } + backendRef := b.(gwapiv1.BackendRef) return &backendRef } diff --git a/internal/gatewayapi/helpers_v1alpha2.go b/internal/gatewayapi/helpers_v1alpha2.go index afa80413175..3b1dffde66f 100644 --- a/internal/gatewayapi/helpers_v1alpha2.go +++ b/internal/gatewayapi/helpers_v1alpha2.go @@ -17,125 +17,7 @@ import ( ) // TODO: [gwapiv1a2-gwapiv1] -// This file can be removed once TLSRoute graduates to gwapiv1. - -func GroupPtrV1Alpha2(group string) *gwapiv1a2.Group { - gwGroup := gwapiv1a2.Group(group) - return &gwGroup -} - -func KindPtrV1Alpha2(kind string) *gwapiv1a2.Kind { - gwKind := gwapiv1a2.Kind(kind) - return &gwKind -} - -func NamespacePtrV1Alpha2(namespace string) *gwapiv1a2.Namespace { - gwNamespace := gwapiv1a2.Namespace(namespace) - return &gwNamespace -} - -func SectionNamePtrV1Alpha2(sectionName string) *gwapiv1a2.SectionName { - gwSectionName := gwapiv1a2.SectionName(sectionName) - return &gwSectionName -} - -func PortNumPtrV1Alpha2(port int) *gwapiv1a2.PortNumber { - pn := gwapiv1a2.PortNumber(port) - return &pn -} - -func UpgradeParentReferences(old []gwapiv1a2.ParentReference) []gwapiv1.ParentReference { - newParentReferences := make([]gwapiv1.ParentReference, len(old)) - for i, o := range old { - newParentReferences[i] = UpgradeParentReference(o) - } - return newParentReferences -} - -// UpgradeParentReference converts gwapiv1a2.ParentReference to gwapiv1.ParentReference -func UpgradeParentReference(old gwapiv1a2.ParentReference) gwapiv1.ParentReference { - upgraded := gwapiv1.ParentReference{} - - if old.Group != nil { - upgraded.Group = GroupPtr(string(*old.Group)) - } - - if old.Kind != nil { - upgraded.Kind = KindPtr(string(*old.Kind)) - } - - if old.Namespace != nil { - upgraded.Namespace = NamespacePtr(string(*old.Namespace)) - } - - upgraded.Name = old.Name - - if old.SectionName != nil { - upgraded.SectionName = SectionNamePtr(string(*old.SectionName)) - } - - if old.Port != nil { - upgraded.Port = PortNumPtr(int32(*old.Port)) - } - - return upgraded -} - -func DowngradeParentReference(old gwapiv1.ParentReference) gwapiv1a2.ParentReference { - downgraded := gwapiv1a2.ParentReference{} - - if old.Group != nil { - downgraded.Group = GroupPtrV1Alpha2(string(*old.Group)) - } - - if old.Kind != nil { - downgraded.Kind = KindPtrV1Alpha2(string(*old.Kind)) - } - - if old.Namespace != nil { - downgraded.Namespace = NamespacePtrV1Alpha2(string(*old.Namespace)) - } - - downgraded.Name = old.Name - - if old.SectionName != nil { - downgraded.SectionName = SectionNamePtrV1Alpha2(string(*old.SectionName)) - } - - if old.Port != nil { - downgraded.Port = PortNumPtrV1Alpha2(int(*old.Port)) - } - - return downgraded -} - -func UpgradeRouteParentStatuses(routeParentStatuses []gwapiv1a2.RouteParentStatus) []gwapiv1.RouteParentStatus { - var res []gwapiv1.RouteParentStatus - - for _, rps := range routeParentStatuses { - res = append(res, gwapiv1.RouteParentStatus{ - ParentRef: UpgradeParentReference(rps.ParentRef), - ControllerName: rps.ControllerName, - Conditions: rps.Conditions, - }) - } - - return res -} - -func DowngradeRouteParentStatuses(routeParentStatuses []gwapiv1.RouteParentStatus) []gwapiv1a2.RouteParentStatus { - var res []gwapiv1a2.RouteParentStatus - - for _, rps := range routeParentStatuses { - res = append(res, gwapiv1a2.RouteParentStatus{ - ParentRef: DowngradeParentReference(rps.ParentRef), - ControllerName: rps.ControllerName, - Conditions: rps.Conditions, - }) - } - - return res -} +// This file can be removed once all routes graduates to gwapiv1. // UpgradeBackendRef converts gwapiv1a2.BackendRef to gwapiv1.BackendRef func UpgradeBackendRef(old gwapiv1a2.BackendRef) gwapiv1.BackendRef { @@ -161,34 +43,3 @@ func UpgradeBackendRef(old gwapiv1a2.BackendRef) gwapiv1.BackendRef { return upgraded } - -func DowngradeBackendRef(old gwapiv1.BackendRef) gwapiv1a2.BackendRef { - downgraded := gwapiv1a2.BackendRef{} - - if old.Group != nil { - downgraded.Group = GroupPtrV1Alpha2(string(*old.Group)) - } - - if old.Kind != nil { - downgraded.Kind = KindPtrV1Alpha2(string(*old.Kind)) - } - - if old.Namespace != nil { - downgraded.Namespace = NamespacePtrV1Alpha2(string(*old.Namespace)) - } - - downgraded.Name = old.Name - - if old.Port != nil { - downgraded.Port = PortNumPtrV1Alpha2(int(*old.Port)) - } - - return downgraded -} - -func NamespaceDerefOrAlpha(namespace *gwapiv1a2.Namespace, defaultNamespace string) string { - if namespace != nil && *namespace != "" { - return string(*namespace) - } - return defaultNamespace -} diff --git a/internal/gatewayapi/route.go b/internal/gatewayapi/route.go index db59bf551e2..ebebab2356d 100644 --- a/internal/gatewayapi/route.go +++ b/internal/gatewayapi/route.go @@ -228,7 +228,7 @@ func (t *Translator) processHTTPRouteRules(httpRoute *HTTPRouteContext, parentRe httpRoute.GetGeneration(), gwapiv1.RouteConditionResolvedRefs, metav1.ConditionFalse, - gwapiv1a2.RouteReasonResolvedRefs, + gwapiv1.RouteReasonResolvedRefs, "Mixed endpointslice address type between backendRefs is not supported") } @@ -1236,7 +1236,7 @@ func (t *Translator) processDestination(backendRefContext BackendRefContext, route.GetGeneration(), gwapiv1.RouteConditionResolvedRefs, metav1.ConditionFalse, - gwapiv1a2.RouteReasonResolvedRefs, + gwapiv1.RouteReasonResolvedRefs, err.Error()) } diff --git a/internal/provider/kubernetes/controller.go b/internal/provider/kubernetes/controller.go index 27a985456a4..343d4edc9a7 100644 --- a/internal/provider/kubernetes/controller.go +++ b/internal/provider/kubernetes/controller.go @@ -519,7 +519,7 @@ func (r *gatewayAPIReconciler) processSecurityPolicyObjectRefs( resourceMap.allAssociatedBackendRefs.Insert(gwapiv1.BackendObjectReference{ Group: backendRef.Group, Kind: backendRef.Kind, - Namespace: gatewayapi.NamespacePtrV1Alpha2(backendNamespace), + Namespace: gatewayapi.NamespacePtr(backendNamespace), Name: backendRef.Name, }) @@ -1641,11 +1641,11 @@ func (r *gatewayAPIReconciler) processEnvoyProxy(ep *egv1a1.EnvoyProxy, resource continue } for _, backendRef := range sink.OpenTelemetry.BackendRefs { - backendNamespace := gatewayapi.NamespaceDerefOrAlpha(backendRef.Namespace, ep.Namespace) + backendNamespace := gatewayapi.NamespaceDerefOr(backendRef.Namespace, ep.Namespace) resourceMap.allAssociatedBackendRefs.Insert(gwapiv1.BackendObjectReference{ Group: backendRef.BackendObjectReference.Group, Kind: backendRef.BackendObjectReference.Kind, - Namespace: gatewayapi.NamespacePtrV1Alpha2(backendNamespace), + Namespace: gatewayapi.NamespacePtr(backendNamespace), Name: backendRef.Name, }) } @@ -1659,11 +1659,11 @@ func (r *gatewayAPIReconciler) processEnvoyProxy(ep *egv1a1.EnvoyProxy, resource continue } for _, backendRef := range sink.OpenTelemetry.BackendRefs { - backendNamespace := gatewayapi.NamespaceDerefOrAlpha(backendRef.Namespace, ep.Namespace) + backendNamespace := gatewayapi.NamespaceDerefOr(backendRef.Namespace, ep.Namespace) resourceMap.allAssociatedBackendRefs.Insert(gwapiv1.BackendObjectReference{ Group: backendRef.BackendObjectReference.Group, Kind: backendRef.BackendObjectReference.Kind, - Namespace: gatewayapi.NamespacePtrV1Alpha2(backendNamespace), + Namespace: gatewayapi.NamespacePtr(backendNamespace), Name: backendRef.Name, }) } @@ -1672,11 +1672,11 @@ func (r *gatewayAPIReconciler) processEnvoyProxy(ep *egv1a1.EnvoyProxy, resource if telemetry.Tracing != nil { for _, backendRef := range telemetry.Tracing.Provider.BackendRefs { - backendNamespace := gatewayapi.NamespaceDerefOrAlpha(backendRef.Namespace, ep.Namespace) + backendNamespace := gatewayapi.NamespaceDerefOr(backendRef.Namespace, ep.Namespace) resourceMap.allAssociatedBackendRefs.Insert(gwapiv1.BackendObjectReference{ Group: backendRef.BackendObjectReference.Group, Kind: backendRef.BackendObjectReference.Kind, - Namespace: gatewayapi.NamespacePtrV1Alpha2(backendNamespace), + Namespace: gatewayapi.NamespacePtr(backendNamespace), Name: backendRef.Name, }) @@ -1851,7 +1851,7 @@ func (r *gatewayAPIReconciler) processEnvoyExtensionPolicyObjectRefs( resourceMap.allAssociatedBackendRefs.Insert(gwapiv1.BackendObjectReference{ Group: backendRef.Group, Kind: backendRef.Kind, - Namespace: gatewayapi.NamespacePtrV1Alpha2(backendNamespace), + Namespace: gatewayapi.NamespacePtr(backendNamespace), Name: backendRef.Name, }) diff --git a/internal/provider/kubernetes/indexers.go b/internal/provider/kubernetes/indexers.go index 9b4a3f17a1f..ba5bfd51db2 100644 --- a/internal/provider/kubernetes/indexers.go +++ b/internal/provider/kubernetes/indexers.go @@ -292,7 +292,7 @@ func addTLSRouteIndexers(ctx context.Context, mgr manager.Manager) error { // lookup the provided Gateway Name. gateways = append(gateways, types.NamespacedName{ - Namespace: gatewayapi.NamespaceDerefOrAlpha(parent.Namespace, tlsRoute.Namespace), + Namespace: gatewayapi.NamespaceDerefOr(parent.Namespace, tlsRoute.Namespace), Name: string(parent.Name), }.String(), ) @@ -319,7 +319,7 @@ func backendTLSRouteIndexFunc(rawObj client.Object) []string { // lookup the provided Gateway Name. backendRefs = append(backendRefs, types.NamespacedName{ - Namespace: gatewayapi.NamespaceDerefOrAlpha(backend.Namespace, tlsroute.Namespace), + Namespace: gatewayapi.NamespaceDerefOr(backend.Namespace, tlsroute.Namespace), Name: string(backend.Name), }.String(), ) @@ -342,7 +342,7 @@ func addTCPRouteIndexers(ctx context.Context, mgr manager.Manager) error { // lookup the provided Gateway Name. gateways = append(gateways, types.NamespacedName{ - Namespace: gatewayapi.NamespaceDerefOrAlpha(parent.Namespace, tcpRoute.Namespace), + Namespace: gatewayapi.NamespaceDerefOr(parent.Namespace, tcpRoute.Namespace), Name: string(parent.Name), }.String(), ) @@ -369,7 +369,7 @@ func backendTCPRouteIndexFunc(rawObj client.Object) []string { // lookup the provided Gateway Name. backendRefs = append(backendRefs, types.NamespacedName{ - Namespace: gatewayapi.NamespaceDerefOrAlpha(backend.Namespace, tcpRoute.Namespace), + Namespace: gatewayapi.NamespaceDerefOr(backend.Namespace, tcpRoute.Namespace), Name: string(backend.Name), }.String(), ) @@ -394,7 +394,7 @@ func addUDPRouteIndexers(ctx context.Context, mgr manager.Manager) error { // lookup the provided Gateway Name. gateways = append(gateways, types.NamespacedName{ - Namespace: gatewayapi.NamespaceDerefOrAlpha(parent.Namespace, udpRoute.Namespace), + Namespace: gatewayapi.NamespaceDerefOr(parent.Namespace, udpRoute.Namespace), Name: string(parent.Name), }.String(), ) @@ -421,7 +421,7 @@ func backendUDPRouteIndexFunc(rawObj client.Object) []string { // lookup the provided Gateway Name. backendRefs = append(backendRefs, types.NamespacedName{ - Namespace: gatewayapi.NamespaceDerefOrAlpha(backend.Namespace, udproute.Namespace), + Namespace: gatewayapi.NamespaceDerefOr(backend.Namespace, udproute.Namespace), Name: string(backend.Name), }.String(), ) diff --git a/internal/provider/kubernetes/routes.go b/internal/provider/kubernetes/routes.go index 32e2f470382..ad2638684cd 100644 --- a/internal/provider/kubernetes/routes.go +++ b/internal/provider/kubernetes/routes.go @@ -61,11 +61,11 @@ func (r *gatewayAPIReconciler) processTLSRoutes(ctx context.Context, gatewayName continue } - backendNamespace := gatewayapi.NamespaceDerefOrAlpha(backendRef.Namespace, tlsRoute.Namespace) + backendNamespace := gatewayapi.NamespaceDerefOr(backendRef.Namespace, tlsRoute.Namespace) resourceMap.allAssociatedBackendRefs.Insert(gwapiv1.BackendObjectReference{ Group: backendRef.BackendObjectReference.Group, Kind: backendRef.BackendObjectReference.Kind, - Namespace: gatewayapi.NamespacePtrV1Alpha2(backendNamespace), + Namespace: gatewayapi.NamespacePtr(backendNamespace), Name: backendRef.Name, }) @@ -144,7 +144,7 @@ func (r *gatewayAPIReconciler) processGRPCRoutes(ctx context.Context, gatewayNam resourceMap.allAssociatedBackendRefs.Insert(gwapiv1.BackendObjectReference{ Group: backendRef.BackendObjectReference.Group, Kind: backendRef.BackendObjectReference.Kind, - Namespace: gatewayapi.NamespacePtrV1Alpha2(backendNamespace), + Namespace: gatewayapi.NamespacePtr(backendNamespace), Name: backendRef.Name, }) @@ -279,7 +279,7 @@ func (r *gatewayAPIReconciler) processHTTPRoutes(ctx context.Context, gatewayNam resourceMap.allAssociatedBackendRefs.Insert(gwapiv1.BackendObjectReference{ Group: backendRef.BackendObjectReference.Group, Kind: backendRef.BackendObjectReference.Kind, - Namespace: gatewayapi.NamespacePtrV1Alpha2(backendNamespace), + Namespace: gatewayapi.NamespacePtr(backendNamespace), Name: backendRef.Name, }) @@ -346,7 +346,7 @@ func (r *gatewayAPIReconciler) processHTTPRoutes(ctx context.Context, gatewayNam resourceMap.allAssociatedBackendRefs.Insert(gwapiv1.BackendObjectReference{ Group: mirrorBackendRef.BackendObjectReference.Group, Kind: mirrorBackendRef.BackendObjectReference.Kind, - Namespace: gatewayapi.NamespacePtrV1Alpha2(backendNamespace), + Namespace: gatewayapi.NamespacePtr(backendNamespace), Name: mirrorBackendRef.Name, }) @@ -454,11 +454,11 @@ func (r *gatewayAPIReconciler) processTCPRoutes(ctx context.Context, gatewayName continue } - backendNamespace := gatewayapi.NamespaceDerefOrAlpha(backendRef.Namespace, tcpRoute.Namespace) + backendNamespace := gatewayapi.NamespaceDerefOr(backendRef.Namespace, tcpRoute.Namespace) resourceMap.allAssociatedBackendRefs.Insert(gwapiv1.BackendObjectReference{ Group: backendRef.BackendObjectReference.Group, Kind: backendRef.BackendObjectReference.Kind, - Namespace: gatewayapi.NamespacePtrV1Alpha2(backendNamespace), + Namespace: gatewayapi.NamespacePtr(backendNamespace), Name: backendRef.Name, }) @@ -533,11 +533,11 @@ func (r *gatewayAPIReconciler) processUDPRoutes(ctx context.Context, gatewayName continue } - backendNamespace := gatewayapi.NamespaceDerefOrAlpha(backendRef.Namespace, udpRoute.Namespace) + backendNamespace := gatewayapi.NamespaceDerefOr(backendRef.Namespace, udpRoute.Namespace) resourceMap.allAssociatedBackendRefs.Insert(gwapiv1.BackendObjectReference{ Group: backendRef.BackendObjectReference.Group, Kind: backendRef.BackendObjectReference.Kind, - Namespace: gatewayapi.NamespacePtrV1Alpha2(backendNamespace), + Namespace: gatewayapi.NamespacePtr(backendNamespace), Name: backendRef.Name, }) From 2a869970e33f02855c38b42a8d19c47d2c43cad9 Mon Sep 17 00:00:00 2001 From: sh2 Date: Fri, 28 Jun 2024 02:06:08 +0800 Subject: [PATCH 20/23] Add benchmark testing framework (#3599) * initial implementation of benchmark test suite Signed-off-by: shawnh2 * implement benchmark test run Signed-off-by: shawnh2 * add benchmark in ci and spawn a job for benchmark test run Signed-off-by: shawnh2 * fix lint Signed-off-by: shawnh2 * fix ci config Signed-off-by: shawnh2 * save benchmark test result into report Signed-off-by: shawnh2 * add control-plane metrics to benchmark test report Signed-off-by: shawnh2 * change httproutes scale number to perform the benchmark test Signed-off-by: shawnh2 * increase poll timeout Signed-off-by: shawnh2 * add longer timeout for go-test, collect reports in suite and change scale to scale-up Signed-off-by: shawnh2 * update resource limits for both envoyproxy and envoygateway pod Signed-off-by: shawnh2 * fix github action unit problem Signed-off-by: shawnh2 * add scale-down test case support Signed-off-by: shawnh2 * fix according to benchmark-result: - add uninstall commands and properly cleanup resources - increase timeout config - change local port of port forwarder to random assign Signed-off-by: shawnh2 * increase memory to 2GiB to see the difference Signed-off-by: shawnh2 * return report for every benchmark test run Signed-off-by: shawnh2 * right memory setup for github ci Signed-off-by: shawnh2 * update report util methods Signed-off-by: shawnh2 * add export benchmark report support Signed-off-by: shawnh2 * view benchmark report in github comment Signed-off-by: shawnh2 * correct github comment ci Signed-off-by: shawnh2 * fix env setting table Signed-off-by: shawnh2 * correct ci args Signed-off-by: shawnh2 * add benchmark report Signed-off-by: shawnh2 * simplify the report Signed-off-by: shawnh2 * add pull_request_target to benchmark report commenter Signed-off-by: shawnh2 * add envoyproxy metrics report and render support Signed-off-by: shawnh2 * grant benchmark-test job with write access Signed-off-by: shawnh2 * upload latest benchmark report and remove commenter ci Signed-off-by: shawnh2 * fix lint and address comments Signed-off-by: shawnh2 --------- Signed-off-by: shawnh2 --- .github/workflows/benchmark.yaml | 56 ++ go.mod | 2 +- test/benchmark/benchmark_report.md | 925 ++++++++++++++++++ test/benchmark/benchmark_test.go | 56 ++ test/benchmark/config/gateway.yaml | 14 + test/benchmark/config/gatewayclass.yaml | 82 ++ test/benchmark/config/httproute.yaml | 22 + test/benchmark/config/nighthawk-client.yaml | 18 + .../config/nighthawk-test-server-config.yaml | 44 + .../config/nighthawk-test-server.yaml | 53 + test/benchmark/suite/client.go | 32 + test/benchmark/suite/flags.go | 19 + test/benchmark/suite/render.go | 329 +++++++ test/benchmark/suite/report.go | 219 +++++ test/benchmark/suite/suite.go | 355 +++++++ test/benchmark/suite/test.go | 34 + test/benchmark/tests/scale_httproutes.go | 115 +++ test/benchmark/tests/tests.go | 13 + tools/make/kube.mk | 42 + 19 files changed, 2429 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/benchmark.yaml create mode 100644 test/benchmark/benchmark_report.md create mode 100644 test/benchmark/benchmark_test.go create mode 100644 test/benchmark/config/gateway.yaml create mode 100644 test/benchmark/config/gatewayclass.yaml create mode 100644 test/benchmark/config/httproute.yaml create mode 100644 test/benchmark/config/nighthawk-client.yaml create mode 100644 test/benchmark/config/nighthawk-test-server-config.yaml create mode 100644 test/benchmark/config/nighthawk-test-server.yaml create mode 100644 test/benchmark/suite/client.go create mode 100644 test/benchmark/suite/flags.go create mode 100644 test/benchmark/suite/render.go create mode 100644 test/benchmark/suite/report.go create mode 100644 test/benchmark/suite/suite.go create mode 100644 test/benchmark/suite/test.go create mode 100644 test/benchmark/tests/scale_httproutes.go create mode 100644 test/benchmark/tests/tests.go diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml new file mode 100644 index 00000000000..9f87ca4cbc3 --- /dev/null +++ b/.github/workflows/benchmark.yaml @@ -0,0 +1,56 @@ +name: Benchmarking Tests at Scale +on: + pull_request: + branches: + - "main" + - "release/v*" + workflow_dispatch: + inputs: + rps: + description: "The target requests-per-second rate. Default: 10000" + default: '10000' + type: string + required: false + connections: + description: "The maximum allowed number of concurrent connections per event loop. HTTP/1 only. Default: 100." + default: '100' + type: string + required: false + duration: + description: "The number of seconds that the test should run. Default: 90." + default: '90' + type: string + required: false + cpu_limits: + description: "The CPU resource limits for the envoy gateway, in unit 'm'. Default: 1000." + default: '1000' + type: string + required: false + memory_limits: + description: "The memory resource limits for the envoy gateway, in unit 'Mi'. Default: 1024." + default: '1024' + type: string + required: false + +jobs: + benchmark-test: + name: Benchmark Test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - uses: ./tools/github-actions/setup-deps + + - name: Run Benchmark tests + env: + KIND_NODE_TAG: v1.28.0 + IMAGE_PULL_POLICY: IfNotPresent + BENCHMARK_RPS: ${{ github.event.inputs.rps || 10000 }} + BENCHMARK_CONNECTIONS: ${{ github.event.inputs.connections || 100 }} + BENCHMARK_DURATION: ${{ github.event.inputs.duration || 90 }} + BENCHMARK_CPU_LIMITS: ${{ github.event.inputs.cpu_limits || 1000 }} + BENCHMARK_MEMORY_LIMITS: ${{ github.event.inputs.memory_limits || 2048 }} + run: make benchmark + + - name: Read Benchmark report + run: cat test/benchmark/benchmark_report.md diff --git a/go.mod b/go.mod index ce96653d303..4154134d6b0 100644 --- a/go.mod +++ b/go.mod @@ -164,7 +164,7 @@ require ( github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/client_model v0.6.1 github.com/prometheus/procfs v0.15.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect diff --git a/test/benchmark/benchmark_report.md b/test/benchmark/benchmark_report.md new file mode 100644 index 00000000000..4d6d57da1a0 --- /dev/null +++ b/test/benchmark/benchmark_report.md @@ -0,0 +1,925 @@ +# Benchmark Report + +Benchmark test settings: + +|RPS |Connections|Duration (Seconds)|CPU Limits (m)|Memory Limits (MiB)| +|- |- |- |- |- | +|1000|100 |90 |1000 |2048 | + +## Test: ScaleHTTPRoute + +Fixed one Gateway and different scales of HTTPRoutes. + + +### Results + +Click to see the full results. + + +
+scale-up-httproutes-10 + +```plaintext +[2024-06-25 14:23:36.545][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. +[14:23:36.545677][1][I] Detected 4 (v)CPUs with affinity.. +[14:23:36.545689][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. +[14:23:36.545691][1][I] Global targets: 400 connections and 4000 calls per second. +[14:23:36.545692][1][I] (Per-worker targets: 100 connections and 1000 calls per second) +[14:25:07.247563][19][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9998444444686 per second.) +[14:25:07.248019][20][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9975333394178 per second.) +[14:25:07.248086][22][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.999577777956 per second.) +[14:25:07.248380][24][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9990666675377 per second.) +[14:25:07.818186][1][I] Done. +Nighthawk - A layer 7 protocol benchmarking tool. + +benchmark_http_client.latency_2xx (359802 samples) + min: 0s 000ms 290us | mean: 0s 000ms 544us | max: 0s 062ms 072us | pstdev: 0s 000ms 863us + + Percentile Count Value + 0.5 179926 0s 000ms 448us + 0.75 269854 0s 000ms 530us + 0.8 287861 0s 000ms 552us + 0.9 323828 0s 000ms 684us + 0.95 341812 0s 000ms 793us + 0.990625 356429 0s 001ms 863us + 0.99902344 359451 0s 009ms 407us + +Queueing and connection setup latency (359802 samples) + min: 0s 000ms 002us | mean: 0s 000ms 011us | max: 0s 023ms 417us | pstdev: 0s 000ms 100us + + Percentile Count Value + 0.5 179902 0s 000ms 010us + 0.75 269909 0s 000ms 010us + 0.8 288003 0s 000ms 011us + 0.9 323850 0s 000ms 011us + 0.95 341822 0s 000ms 012us + 0.990625 356429 0s 000ms 029us + 0.99902344 359451 0s 000ms 170us + +Request start to response end (359802 samples) + min: 0s 000ms 289us | mean: 0s 000ms 543us | max: 0s 062ms 072us | pstdev: 0s 000ms 863us + + Percentile Count Value + 0.5 179915 0s 000ms 448us + 0.75 269861 0s 000ms 530us + 0.8 287848 0s 000ms 552us + 0.9 323824 0s 000ms 683us + 0.95 341813 0s 000ms 792us + 0.990625 356429 0s 001ms 862us + 0.99902344 359451 0s 009ms 406us + +Response body size in bytes (359802 samples) + min: 10 | mean: 10 | max: 10 | pstdev: 0 + +Response header size in bytes (359802 samples) + min: 110 | mean: 110 | max: 110 | pstdev: 0 + +Blocking. Results are skewed when significant numbers are reported here. (2 samples) + min: 0s 000ms 901us | mean: 0s 001ms 551us | max: 0s 002ms 202us | pstdev: 0s 000ms 650us + + Percentile Count Value + 0.5 1 0s 000ms 901us + +Initiation to completion (360000 samples) + min: 0s 000ms 006us | mean: 0s 000ms 562us | max: 0s 062ms 310us | pstdev: 0s 000ms 883us + + Percentile Count Value + 0.5 180016 0s 000ms 465us + 0.75 270003 0s 000ms 547us + 0.8 288024 0s 000ms 570us + 0.9 324002 0s 000ms 702us + 0.95 342002 0s 000ms 815us + 0.990625 356625 0s 001ms 912us + 0.99902344 359649 0s 009ms 617us + +Counter Value Per second +benchmark.http_2xx 359802 3997.80 +benchmark.pool_overflow 198 2.20 +cluster_manager.cluster_added 4 0.04 +default.total_match_count 4 0.04 +membership_change 4 0.04 +runtime.load_success 1 0.01 +runtime.override_dir_not_exists 1 0.01 +upstream_cx_http1_total 108 1.20 +upstream_cx_rx_bytes_total 56488914 627653.97 +upstream_cx_total 108 1.20 +upstream_cx_tx_bytes_total 15471486 171905.23 +upstream_rq_pending_overflow 198 2.20 +upstream_rq_pending_total 108 1.20 +upstream_rq_total 359802 3997.80 + + +``` + +
+ +
+scale-up-httproutes-50 + +```plaintext +[2024-06-25 14:25:18.533][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. +[14:25:18.533824][1][I] Detected 4 (v)CPUs with affinity.. +[14:25:18.533833][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. +[14:25:18.533836][1][I] Global targets: 400 connections and 4000 calls per second. +[14:25:18.533837][1][I] (Per-worker targets: 100 connections and 1000 calls per second) +[14:26:49.235731][18][I] Stopping after 89999 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000.0000222222228 per second.) +[14:26:49.235977][19][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000 per second.) +[14:26:49.236240][21][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89999. (Completion rate was 999.9888111119815 per second.) +[14:26:49.236565][23][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89999. (Completion rate was 999.9879333448638 per second.) +[14:26:54.772502][21][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +Nighthawk - A layer 7 protocol benchmarking tool. + +benchmark_http_client.latency_2xx (359912 samples) + min: 0s 000ms 303us | mean: 0s 000ms 508us | max: 0s 031ms 259us | pstdev: 0s 000ms 449us + + Percentile Count Value + 0.5 179967 0s 000ms 437us + 0.75 269938 0s 000ms 509us + 0.8 287950 0s 000ms 533us + 0.9 323925 0s 000ms 643us + 0.95 341919 0s 000ms 735us + 0.990625 356538 0s 001ms 526us + 0.99902344 359561 0s 006ms 589us + +Queueing and connection setup latency (359914 samples) + min: 0s 000ms 001us | mean: 0s 000ms 011us | max: 0s 023ms 449us | pstdev: 0s 000ms 080us + + Percentile Count Value + 0.5 180198 0s 000ms 010us + 0.75 270795 0s 000ms 011us + 0.8 288165 0s 000ms 011us + 0.9 324019 0s 000ms 011us + 0.95 341989 0s 000ms 011us + 0.990625 356540 0s 000ms 025us + 0.99902344 359564 0s 000ms 159us + +Request start to response end (359912 samples) + min: 0s 000ms 302us | mean: 0s 000ms 508us | max: 0s 031ms 258us | pstdev: 0s 000ms 448us + + Percentile Count Value + 0.5 179970 0s 000ms 437us + 0.75 269934 0s 000ms 509us + 0.8 287943 0s 000ms 533us + 0.9 323921 0s 000ms 643us + 0.95 341917 0s 000ms 735us + 0.990625 356538 0s 001ms 525us + 0.99902344 359561 0s 006ms 589us + +Response body size in bytes (359912 samples) + min: 10 | mean: 10 | max: 10 | pstdev: 0 + +Response header size in bytes (359912 samples) + min: 110 | mean: 110 | max: 110 | pstdev: 0 + +Initiation to completion (359998 samples) + min: 0s 000ms 006us | mean: 0s 000ms 525us | max: 0s 032ms 263us | pstdev: 0s 000ms 461us + + Percentile Count Value + 0.5 180024 0s 000ms 453us + 0.75 270017 0s 000ms 526us + 0.8 288004 0s 000ms 550us + 0.9 323999 0s 000ms 663us + 0.95 341999 0s 000ms 753us + 0.990625 356624 0s 001ms 571us + 0.99902344 359647 0s 006ms 655us + +Counter Value Per second +benchmark.http_2xx 359912 3999.02 +benchmark.pool_overflow 86 0.96 +cluster_manager.cluster_added 4 0.04 +default.total_match_count 4 0.04 +membership_change 4 0.04 +runtime.load_success 1 0.01 +runtime.override_dir_not_exists 1 0.01 +upstream_cx_http1_total 61 0.68 +upstream_cx_rx_bytes_total 56506184 627846.33 +upstream_cx_total 61 0.68 +upstream_cx_tx_bytes_total 15476302 171958.87 +upstream_rq_pending_overflow 86 0.96 +upstream_rq_pending_total 61 0.68 +upstream_rq_total 359914 3999.04 + +[14:26:59.773726][23][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +[14:26:59.775576][1][I] Done. + +``` + +
+ +
+scale-up-httproutes-100 + +```plaintext +[2024-06-25 14:27:18.024][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. +[14:27:18.024969][1][I] Detected 4 (v)CPUs with affinity.. +[14:27:18.024981][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. +[14:27:18.024984][1][I] Global targets: 400 connections and 4000 calls per second. +[14:27:18.024985][1][I] (Per-worker targets: 100 connections and 1000 calls per second) +[14:28:48.726723][18][I] Stopping after 89999 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000.0000111111112 per second.) +[14:28:48.726970][19][I] Stopping after 89999 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000.0000222222228 per second.) +[14:28:48.727250][21][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89999. (Completion rate was 999.9885555593705 per second.) +[14:28:48.727579][23][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89999. (Completion rate was 999.9878444571402 per second.) +[14:28:54.260999][21][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +Nighthawk - A layer 7 protocol benchmarking tool. + +benchmark_http_client.latency_2xx (359869 samples) + min: 0s 000ms 301us | mean: 0s 000ms 514us | max: 0s 029ms 035us | pstdev: 0s 000ms 431us + + Percentile Count Value + 0.5 179965 0s 000ms 444us + 0.75 269904 0s 000ms 518us + 0.8 287910 0s 000ms 540us + 0.9 323888 0s 000ms 641us + 0.95 341879 0s 000ms 744us + 0.990625 356497 0s 001ms 522us + 0.99902344 359518 0s 006ms 553us + +Queueing and connection setup latency (359871 samples) + min: 0s 000ms 001us | mean: 0s 000ms 011us | max: 0s 027ms 366us | pstdev: 0s 000ms 058us + + Percentile Count Value + 0.5 179998 0s 000ms 010us + 0.75 271128 0s 000ms 011us + 0.8 288699 0s 000ms 011us + 0.9 324073 0s 000ms 011us + 0.95 341895 0s 000ms 012us + 0.990625 356498 0s 000ms 029us + 0.99902344 359520 0s 000ms 166us + +Request start to response end (359869 samples) + min: 0s 000ms 301us | mean: 0s 000ms 513us | max: 0s 029ms 035us | pstdev: 0s 000ms 431us + + Percentile Count Value + 0.5 179935 0s 000ms 444us + 0.75 269906 0s 000ms 517us + 0.8 287896 0s 000ms 539us + 0.9 323883 0s 000ms 641us + 0.95 341876 0s 000ms 743us + 0.990625 356496 0s 001ms 521us + 0.99902344 359518 0s 006ms 552us + +Response body size in bytes (359869 samples) + min: 10 | mean: 10 | max: 10 | pstdev: 0 + +Response header size in bytes (359869 samples) + min: 110 | mean: 110 | max: 110 | pstdev: 0 + +Initiation to completion (359998 samples) + min: 0s 000ms 007us | mean: 0s 000ms 532us | max: 0s 029ms 055us | pstdev: 0s 000ms 451us + + Percentile Count Value + 0.5 180023 0s 000ms 461us + 0.75 270008 0s 000ms 535us + 0.8 288030 0s 000ms 557us + 0.9 324004 0s 000ms 661us + 0.95 341999 0s 000ms 763us + 0.990625 356624 0s 001ms 570us + 0.99902344 359647 0s 007ms 023us + +Counter Value Per second +benchmark.http_2xx 359869 3998.54 +benchmark.pool_overflow 129 1.43 +cluster_manager.cluster_added 4 0.04 +default.total_match_count 4 0.04 +membership_change 4 0.04 +runtime.load_success 1 0.01 +runtime.override_dir_not_exists 1 0.01 +upstream_cx_http1_total 63 0.70 +upstream_cx_rx_bytes_total 56499433 627771.26 +upstream_cx_total 63 0.70 +upstream_cx_tx_bytes_total 15474453 171938.31 +upstream_rq_pending_overflow 129 1.43 +upstream_rq_pending_total 63 0.70 +upstream_rq_total 359871 3998.57 + +[14:28:59.262084][23][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +[14:28:59.263795][1][I] Done. + +``` + +
+ +
+scale-up-httproutes-300 + +```plaintext +[2024-06-25 14:32:23.491][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. +[14:32:23.491963][1][I] Detected 4 (v)CPUs with affinity.. +[14:32:23.491978][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. +[14:32:23.491980][1][I] Global targets: 400 connections and 4000 calls per second. +[14:32:23.491981][1][I] (Per-worker targets: 100 connections and 1000 calls per second) +[14:33:54.193887][18][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9999888888891 per second.) +[14:33:54.194133][19][I] Stopping after 89999 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000.0000222222228 per second.) +[14:33:54.194387][21][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9999888888891 per second.) +[14:33:54.194633][23][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000 per second.) +[14:33:54.712898][1][I] Done. +Nighthawk - A layer 7 protocol benchmarking tool. + +benchmark_http_client.latency_2xx (359845 samples) + min: 0s 000ms 302us | mean: 0s 000ms 507us | max: 0s 019ms 987us | pstdev: 0s 000ms 380us + + Percentile Count Value + 0.5 179931 0s 000ms 440us + 0.75 269886 0s 000ms 512us + 0.8 287876 0s 000ms 533us + 0.9 323864 0s 000ms 637us + 0.95 341854 0s 000ms 734us + 0.990625 356472 0s 001ms 469us + 0.99902344 359494 0s 006ms 259us + +Queueing and connection setup latency (359845 samples) + min: 0s 000ms 001us | mean: 0s 000ms 011us | max: 0s 017ms 634us | pstdev: 0s 000ms 043us + + Percentile Count Value + 0.5 180160 0s 000ms 010us + 0.75 270791 0s 000ms 010us + 0.8 287918 0s 000ms 011us + 0.9 324365 0s 000ms 011us + 0.95 341940 0s 000ms 011us + 0.990625 356473 0s 000ms 027us + 0.99902344 359494 0s 000ms 167us + +Request start to response end (359845 samples) + min: 0s 000ms 302us | mean: 0s 000ms 506us | max: 0s 019ms 987us | pstdev: 0s 000ms 380us + + Percentile Count Value + 0.5 179923 0s 000ms 440us + 0.75 269893 0s 000ms 511us + 0.8 287905 0s 000ms 533us + 0.9 323861 0s 000ms 636us + 0.95 341854 0s 000ms 733us + 0.990625 356472 0s 001ms 468us + 0.99902344 359494 0s 006ms 259us + +Response body size in bytes (359845 samples) + min: 10 | mean: 10 | max: 10 | pstdev: 0 + +Response header size in bytes (359845 samples) + min: 110 | mean: 110 | max: 110 | pstdev: 0 + +Initiation to completion (360000 samples) + min: 0s 000ms 007us | mean: 0s 000ms 525us | max: 0s 020ms 035us | pstdev: 0s 000ms 409us + + Percentile Count Value + 0.5 180011 0s 000ms 456us + 0.75 270028 0s 000ms 529us + 0.8 288004 0s 000ms 550us + 0.9 324006 0s 000ms 656us + 0.95 342005 0s 000ms 751us + 0.990625 356625 0s 001ms 525us + 0.99902344 359649 0s 006ms 569us + +Counter Value Per second +benchmark.http_2xx 359845 3998.28 +benchmark.pool_overflow 155 1.72 +cluster_manager.cluster_added 4 0.04 +default.total_match_count 4 0.04 +membership_change 4 0.04 +runtime.load_success 1 0.01 +runtime.override_dir_not_exists 1 0.01 +upstream_cx_http1_total 62 0.69 +upstream_cx_rx_bytes_total 56495665 627729.61 +upstream_cx_total 62 0.69 +upstream_cx_tx_bytes_total 15473335 171925.94 +upstream_rq_pending_overflow 155 1.72 +upstream_rq_pending_total 62 0.69 +upstream_rq_total 359845 3998.28 + + +``` + +
+ +
+scale-up-httproutes-500 + +```plaintext +[2024-06-25 14:38:43.691][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. +[14:38:43.691938][1][I] Detected 4 (v)CPUs with affinity.. +[14:38:43.691951][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. +[14:38:43.691953][1][I] Global targets: 400 connections and 4000 calls per second. +[14:38:43.691954][1][I] (Per-worker targets: 100 connections and 1000 calls per second) +[14:40:14.393764][18][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89999. (Completion rate was 999.9885666703507 per second.) +[14:40:14.393981][19][I] Stopping after 89999 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000.0000555555587 per second.) +[14:40:14.394312][21][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9992666672044 per second.) +[14:40:14.394484][23][I] Stopping after 89999 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000.0000333333344 per second.) +[14:40:19.954533][18][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +Nighthawk - A layer 7 protocol benchmarking tool. + +benchmark_http_client.latency_2xx (359743 samples) + min: 0s 000ms 311us | mean: 0s 000ms 566us | max: 0s 064ms 673us | pstdev: 0s 001ms 003us + + Percentile Count Value + 0.5 179890 0s 000ms 441us + 0.75 269813 0s 000ms 519us + 0.8 287802 0s 000ms 544us + 0.9 323771 0s 000ms 684us + 0.95 341757 0s 000ms 790us + 0.990625 356372 0s 002ms 465us + 0.99902344 359392 0s 016ms 404us + +Queueing and connection setup latency (359744 samples) + min: 0s 000ms 001us | mean: 0s 000ms 011us | max: 0s 040ms 116us | pstdev: 0s 000ms 093us + + Percentile Count Value + 0.5 180015 0s 000ms 010us + 0.75 269817 0s 000ms 011us + 0.8 288758 0s 000ms 011us + 0.9 324218 0s 000ms 011us + 0.95 341791 0s 000ms 012us + 0.990625 356372 0s 000ms 029us + 0.99902344 359393 0s 000ms 172us + +Request start to response end (359743 samples) + min: 0s 000ms 311us | mean: 0s 000ms 566us | max: 0s 064ms 673us | pstdev: 0s 001ms 003us + + Percentile Count Value + 0.5 179874 0s 000ms 441us + 0.75 269820 0s 000ms 519us + 0.8 287800 0s 000ms 544us + 0.9 323772 0s 000ms 684us + 0.95 341758 0s 000ms 790us + 0.990625 356372 0s 002ms 464us + 0.99902344 359392 0s 016ms 404us + +Response body size in bytes (359743 samples) + min: 10 | mean: 10 | max: 10 | pstdev: 0 + +Response header size in bytes (359743 samples) + min: 110 | mean: 110 | max: 110 | pstdev: 0 + +Blocking. Results are skewed when significant numbers are reported here. (1 samples) + min: 0s 001ms 331us | mean: 0s 001ms 331us | max: 0s 001ms 331us | pstdev: 0s 000ms 000us + +Initiation to completion (359999 samples) + min: 0s 000ms 005us | mean: 0s 000ms 593us | max: 0s 064ms 710us | pstdev: 0s 001ms 145us + + Percentile Count Value + 0.5 180029 0s 000ms 458us + 0.75 270014 0s 000ms 537us + 0.8 288011 0s 000ms 562us + 0.9 324008 0s 000ms 703us + 0.95 342000 0s 000ms 813us + 0.990625 356625 0s 002ms 640us + 0.99902344 359648 0s 018ms 319us + +Counter Value Per second +benchmark.http_2xx 359743 3997.14 +benchmark.pool_overflow 256 2.84 +cluster_manager.cluster_added 4 0.04 +default.total_match_count 4 0.04 +membership_change 4 0.04 +runtime.load_success 1 0.01 +runtime.override_dir_not_exists 1 0.01 +upstream_cx_http1_total 96 1.07 +upstream_cx_rx_bytes_total 56479651 627551.52 +upstream_cx_total 96 1.07 +upstream_cx_tx_bytes_total 15468992 171877.65 +upstream_rq_pending_overflow 256 2.84 +upstream_rq_pending_total 96 1.07 +upstream_rq_total 359744 3997.15 + +[14:40:19.962562][1][I] Done. + +``` + +
+ +
+scale-down-httproutes-300 + +```plaintext +[2024-06-25 14:40:47.629][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. +[14:40:47.629688][1][I] Detected 4 (v)CPUs with affinity.. +[14:40:47.629700][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. +[14:40:47.629703][1][I] Global targets: 400 connections and 4000 calls per second. +[14:40:47.629704][1][I] (Per-worker targets: 100 connections and 1000 calls per second) +[14:42:18.331523][18][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9990222231783 per second.) +[14:42:18.331681][19][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9999777777783 per second.) +[14:42:18.331935][21][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89999. (Completion rate was 999.988855555927 per second.) +[14:42:18.332255][23][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.999166667361 per second.) +[14:42:23.961708][21][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +Nighthawk - A layer 7 protocol benchmarking tool. + +benchmark_http_client.latency_2xx (359722 samples) + min: 0s 000ms 303us | mean: 0s 001ms 203us | max: 0s 121ms 311us | pstdev: 0s 006ms 029us + + Percentile Count Value + 0.5 179885 0s 000ms 456us + 0.75 269810 0s 000ms 543us + 0.8 287781 0s 000ms 570us + 0.9 323750 0s 000ms 749us + 0.95 341736 0s 001ms 344us + 0.990625 356350 0s 016ms 743us + 0.99902344 359371 0s 083ms 783us + +Queueing and connection setup latency (359723 samples) + min: 0s 000ms 001us | mean: 0s 000ms 012us | max: 0s 041ms 897us | pstdev: 0s 000ms 131us + + Percentile Count Value + 0.5 179921 0s 000ms 010us + 0.75 270953 0s 000ms 011us + 0.8 288381 0s 000ms 011us + 0.9 323867 0s 000ms 011us + 0.95 341759 0s 000ms 012us + 0.990625 356351 0s 000ms 032us + 0.99902344 359372 0s 000ms 206us + +Request start to response end (359722 samples) + min: 0s 000ms 302us | mean: 0s 001ms 202us | max: 0s 121ms 311us | pstdev: 0s 006ms 029us + + Percentile Count Value + 0.5 179877 0s 000ms 456us + 0.75 269810 0s 000ms 542us + 0.8 287783 0s 000ms 570us + 0.9 323752 0s 000ms 748us + 0.95 341736 0s 001ms 344us + 0.990625 356350 0s 016ms 742us + 0.99902344 359371 0s 083ms 783us + +Response body size in bytes (359722 samples) + min: 10 | mean: 10 | max: 10 | pstdev: 0 + +Response header size in bytes (359722 samples) + min: 110 | mean: 110 | max: 110 | pstdev: 0 + +Blocking. Results are skewed when significant numbers are reported here. (1022 samples) + min: 0s 000ms 045us | mean: 0s 006ms 403us | max: 0s 099ms 049us | pstdev: 0s 015ms 130us + + Percentile Count Value + 0.5 511 0s 001ms 012us + 0.75 767 0s 003ms 262us + 0.8 818 0s 004ms 782us + 0.9 920 0s 015ms 482us + 0.95 971 0s 047ms 316us + 0.990625 1013 0s 075ms 657us + 0.99902344 1022 0s 099ms 049us + +Initiation to completion (359999 samples) + min: 0s 000ms 002us | mean: 0s 001ms 224us | max: 0s 121ms 405us | pstdev: 0s 006ms 041us + + Percentile Count Value + 0.5 180002 0s 000ms 473us + 0.75 270009 0s 000ms 560us + 0.8 288002 0s 000ms 589us + 0.9 324002 0s 000ms 770us + 0.95 342001 0s 001ms 390us + 0.990625 356625 0s 017ms 278us + 0.99902344 359648 0s 083ms 849us + +Counter Value Per second +benchmark.http_2xx 359722 3996.91 +benchmark.pool_overflow 277 3.08 +cluster_manager.cluster_added 4 0.04 +default.total_match_count 4 0.04 +membership_change 4 0.04 +runtime.load_success 1 0.01 +runtime.override_dir_not_exists 1 0.01 +upstream_cx_http1_total 123 1.37 +upstream_cx_rx_bytes_total 56476354 627514.75 +upstream_cx_total 123 1.37 +upstream_cx_tx_bytes_total 15468089 171867.57 +upstream_rq_pending_overflow 277 3.08 +upstream_rq_pending_total 123 1.37 +upstream_rq_total 359723 3996.92 + +[14:42:23.965353][1][I] Done. + +``` + +
+ +
+scale-down-httproutes-100 + +```plaintext +[2024-06-25 14:42:41.429][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. +[14:42:41.430321][1][I] Detected 4 (v)CPUs with affinity.. +[14:42:41.430334][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. +[14:42:41.430336][1][I] Global targets: 400 connections and 4000 calls per second. +[14:42:41.430338][1][I] (Per-worker targets: 100 connections and 1000 calls per second) +[14:44:12.132884][18][I] Stopping after 89999 ms. Initiated: 90000 / Completed: 89996. (Completion rate was 999.9555777767906 per second.) +[14:44:12.133405][21][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.999777777827 per second.) +[14:44:12.133681][23][I] Stopping after 90000 ms. Initiated: 89995 / Completed: 89995. (Completion rate was 999.9439111410252 per second.) +[14:44:12.138437][19][I] Stopping after 90005 ms. Initiated: 89979 / Completed: 89958. (Completion rate was 999.474453182769 per second.) +[14:44:17.976650][18][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +Nighthawk - A layer 7 protocol benchmarking tool. + +benchmark_http_client.latency_2xx (359649 samples) + min: 0s 000ms 297us | mean: 0s 009ms 493us | max: 0s 163ms 553us | pstdev: 0s 017ms 560us + + Percentile Count Value + 0.5 179826 0s 002ms 042us + 0.75 269740 0s 008ms 322us + 0.8 287721 0s 011ms 606us + 0.9 323685 0s 030ms 779us + 0.95 341667 0s 054ms 808us + 0.990625 356278 0s 079ms 679us + 0.99902344 359299 0s 106ms 098us + +Queueing and connection setup latency (359674 samples) + min: 0s 000ms 001us | mean: 0s 000ms 015us | max: 0s 033ms 947us | pstdev: 0s 000ms 269us + + Percentile Count Value + 0.5 179931 0s 000ms 008us + 0.75 269789 0s 000ms 010us + 0.8 287838 0s 000ms 011us + 0.9 323819 0s 000ms 011us + 0.95 341697 0s 000ms 015us + 0.990625 356303 0s 000ms 059us + 0.99902344 359323 0s 001ms 303us + +Request start to response end (359649 samples) + min: 0s 000ms 296us | mean: 0s 009ms 493us | max: 0s 163ms 553us | pstdev: 0s 017ms 560us + + Percentile Count Value + 0.5 179829 0s 002ms 041us + 0.75 269740 0s 008ms 321us + 0.8 287722 0s 011ms 606us + 0.9 323687 0s 030ms 779us + 0.95 341667 0s 054ms 808us + 0.990625 356278 0s 079ms 679us + 0.99902344 359299 0s 106ms 098us + +Response body size in bytes (359649 samples) + min: 10 | mean: 10 | max: 10 | pstdev: 0 + +Response header size in bytes (359649 samples) + min: 110 | mean: 110 | max: 110 | pstdev: 0 + +Blocking. Results are skewed when significant numbers are reported here. (19052 samples) + min: 0s 000ms 039us | mean: 0s 006ms 142us | max: 0s 146ms 767us | pstdev: 0s 012ms 746us + + Percentile Count Value + 0.5 9526 0s 001ms 273us + 0.75 14289 0s 004ms 540us + 0.8 15242 0s 006ms 354us + 0.9 17147 0s 018ms 240us + 0.95 18100 0s 036ms 243us + 0.990625 18874 0s 063ms 096us + 0.99902344 19034 0s 087ms 937us + +Initiation to completion (359949 samples) + min: 0s 000ms 006us | mean: 0s 009ms 592us | max: 0s 163ms 594us | pstdev: 0s 017ms 643us + + Percentile Count Value + 0.5 179976 0s 002ms 093us + 0.75 269962 0s 008ms 477us + 0.8 287962 0s 011ms 824us + 0.9 323955 0s 030ms 991us + 0.95 341957 0s 055ms 142us + 0.990625 356578 0s 080ms 228us + 0.99902344 359598 0s 106ms 164us + +Counter Value Per second +benchmark.http_2xx 359649 3996.04 +benchmark.pool_overflow 300 3.33 +cluster_manager.cluster_added 4 0.04 +default.total_match_count 4 0.04 +membership_change 4 0.04 +runtime.load_success 1 0.01 +runtime.override_dir_not_exists 1 0.01 +upstream_cx_http1_total 100 1.11 +upstream_cx_rx_bytes_total 56464893 627378.34 +upstream_cx_total 100 1.11 +upstream_cx_tx_bytes_total 15465939 171841.20 +upstream_rq_pending_overflow 300 3.33 +upstream_rq_pending_total 100 1.11 +upstream_rq_total 359674 3996.32 + +[14:44:22.981221][19][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +[14:44:22.985214][1][I] Done. + +``` + +
+ +
+scale-down-httproutes-50 + +```plaintext +[2024-06-25 14:44:35.399][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. +[14:44:35.400594][1][I] Detected 4 (v)CPUs with affinity.. +[14:44:35.400604][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. +[14:44:35.400607][1][I] Global targets: 400 connections and 4000 calls per second. +[14:44:35.400609][1][I] (Per-worker targets: 100 connections and 1000 calls per second) +[14:45:50.873708][19][E] Exiting due to failing termination predicate +[14:45:50.873747][19][I] Stopping after 74767 ms. Initiated: 74767 / Completed: 74746. (Completion rate was 999.716774109537 per second.) +[14:45:50.874918][18][E] Exiting due to failing termination predicate +[14:45:50.875120][18][I] Stopping after 74769 ms. Initiated: 74770 / Completed: 74749. (Completion rate was 999.7209036132042 per second.) +[14:45:50.878184][20][E] Exiting due to failing termination predicate +[14:45:50.878203][20][I] Stopping after 74772 ms. Initiated: 74772 / Completed: 74750. (Completion rate was 999.6972288218956 per second.) +[14:45:50.880264][21][E] Exiting due to failing termination predicate +[14:45:50.880286][21][I] Stopping after 74773 ms. Initiated: 74774 / Completed: 74747. (Completion rate was 999.6424011911454 per second.) +[14:45:51.840758][1][E] Terminated early because of a failure predicate. +[14:45:51.840780][1][I] Check the output for problematic counter values. The default Nighthawk failure predicates report failure if (1) Nighthawk could not connect to the target (see 'benchmark.pool_connection_failure' counter; check the address and port number, and try explicitly setting --address-family v4 or v6, especially when using DNS; instead of localhost try 127.0.0.1 or ::1 explicitly), (2) the protocol was not supported by the target (see 'benchmark.stream_resets' counter; check http/https in the URI, --h2), (3) the target returned a 4xx or 5xx HTTP response code (see 'benchmark.http_4xx' and 'benchmark.http_5xx' counters; check the URI path and the server config), or (4) a custom gRPC RequestSource failed. --failure-predicate can be used to relax expectations. +[14:45:56.841540][18][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +Nighthawk - A layer 7 protocol benchmarking tool. + +benchmark_http_client.latency_2xx (298738 samples) + min: 0s 000ms 307us | mean: 0s 011ms 515us | max: 0s 187ms 269us | pstdev: 0s 019ms 783us + + Percentile Count Value + 0.5 149369 0s 002ms 416us + 0.75 224054 0s 011ms 230us + 0.8 238992 0s 016ms 571us + 0.9 268865 0s 041ms 666us + 0.95 283803 0s 059ms 402us + 0.990625 295939 0s 085ms 483us + 0.99902344 298447 0s 120ms 631us + +benchmark_http_client.latency_5xx (7 samples) + min: 0s 000ms 210us | mean: 0s 000ms 884us | max: 0s 001ms 337us | pstdev: 0s 000ms 404us + + Percentile Count Value + 0.5 4 0s 000ms 683us + 0.75 6 0s 001ms 309us + 0.8 6 0s 001ms 309us + +Queueing and connection setup latency (298836 samples) + min: 0s 000ms 001us | mean: 0s 000ms 016us | max: 0s 075ms 845us | pstdev: 0s 000ms 315us + + Percentile Count Value + 0.5 149531 0s 000ms 008us + 0.75 224310 0s 000ms 010us + 0.8 239171 0s 000ms 010us + 0.9 268973 0s 000ms 011us + 0.95 283896 0s 000ms 016us + 0.990625 296035 0s 000ms 061us + 0.99902344 298545 0s 001ms 244us + +Request start to response end (298745 samples) + min: 0s 000ms 210us | mean: 0s 011ms 514us | max: 0s 187ms 269us | pstdev: 0s 019ms 783us + + Percentile Count Value + 0.5 149373 0s 002ms 415us + 0.75 224059 0s 011ms 229us + 0.8 238996 0s 016ms 569us + 0.9 268871 0s 041ms 664us + 0.95 283810 0s 059ms 402us + 0.990625 295946 0s 085ms 483us + 0.99902344 298454 0s 120ms 631us + +Response body size in bytes (298745 samples) + min: 0 | mean: 9.999765686455003 | max: 10 | pstdev: 0.048405377254271256 + +Response header size in bytes (298745 samples) + min: 58 | mean: 109.99878156956602 | max: 110 | pstdev: 0.25170796172221055 + +Blocking. Results are skewed when significant numbers are reported here. (8534 samples) + min: 0s 000ms 040us | mean: 0s 006ms 742us | max: 0s 101ms 433us | pstdev: 0s 012ms 626us + + Percentile Count Value + 0.5 4267 0s 001ms 369us + 0.75 6401 0s 005ms 750us + 0.8 6828 0s 008ms 406us + 0.9 7681 0s 022ms 739us + 0.95 8108 0s 037ms 554us + 0.990625 8454 0s 058ms 765us + 0.99902344 8526 0s 078ms 221us + +Initiation to completion (298992 samples) + min: 0s 000ms 005us | mean: 0s 011ms 618us | max: 0s 187ms 277us | pstdev: 0s 019ms 857us + + Percentile Count Value + 0.5 149496 0s 002ms 465us + 0.75 224244 0s 011ms 490us + 0.8 239195 0s 016ms 867us + 0.9 269093 0s 041ms 924us + 0.95 284043 0s 059ms 586us + 0.990625 296189 0s 085ms 766us + 0.99902344 298701 0s 120ms 954us + +Counter Value Per second +benchmark.http_2xx 298738 3995.38 +benchmark.http_5xx 7 0.09 +benchmark.pool_overflow 247 3.30 +cluster_manager.cluster_added 4 0.05 +default.total_match_count 4 0.05 +membership_change 4 0.05 +runtime.load_success 1 0.01 +runtime.override_dir_not_exists 1 0.01 +sequencer.failed_terminations 4 0.05 +upstream_cx_http1_total 153 2.05 +upstream_cx_rx_bytes_total 46902510 627283.31 +upstream_cx_total 153 2.05 +upstream_cx_tx_bytes_total 12849948 171857.71 +upstream_rq_pending_overflow 247 3.30 +upstream_rq_pending_total 153 2.05 +upstream_rq_total 298836 3996.69 + +[14:46:01.842997][19][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +[14:46:06.845068][20][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +[14:46:11.847277][21][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +[14:46:11.850206][1][E] An error occurred. + +``` + +
+ +
+scale-down-httproutes-10 + +```plaintext +[2024-06-25 14:48:06.510][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. +[14:48:06.510881][1][I] Detected 4 (v)CPUs with affinity.. +[14:48:06.510893][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. +[14:48:06.510895][1][I] Global targets: 400 connections and 4000 calls per second. +[14:48:06.510897][1][I] (Per-worker targets: 100 connections and 1000 calls per second) +[14:49:37.212970][19][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9998000000401 per second.) +[14:49:37.213444][21][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9973000072899 per second.) +[14:49:37.213541][23][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9989777788228 per second.) +[14:49:37.213971][18][I] Stopping after 90001 ms. Initiated: 89999 / Completed: 89996. (Completion rate was 999.9414452707166 per second.) +Nighthawk - A layer 7 protocol benchmarking tool. + +benchmark_http_client.latency_2xx (359809 samples) + min: 0s 000ms 297us | mean: 0s 000ms 525us | max: 0s 048ms 130us | pstdev: 0s 000ms 466us + +[14:49:42.728878][18][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. + Percentile Count Value + 0.5 179925 0s 000ms 450us + 0.75 269871 0s 000ms 525us + 0.8 287855 0s 000ms 549us + 0.9 323830 0s 000ms 661us + 0.95 341819 0s 000ms 762us + 0.990625 356436 0s 001ms 585us + 0.99902344 359458 0s 007ms 084us + +Queueing and connection setup latency (359812 samples) + min: 0s 000ms 002us | mean: 0s 000ms 012us | max: 0s 034ms 091us | pstdev: 0s 000ms 067us + + Percentile Count Value + 0.5 180343 0s 000ms 010us + 0.75 269955 0s 000ms 011us + 0.8 288513 0s 000ms 011us + 0.9 323857 0s 000ms 012us + 0.95 341824 0s 000ms 023us + 0.990625 356439 0s 000ms 050us + 0.99902344 359461 0s 000ms 189us + +Request start to response end (359809 samples) + min: 0s 000ms 297us | mean: 0s 000ms 524us | max: 0s 048ms 130us | pstdev: 0s 000ms 466us + + Percentile Count Value + 0.5 179906 0s 000ms 450us + 0.75 269861 0s 000ms 524us + 0.8 287864 0s 000ms 549us + 0.9 323829 0s 000ms 661us + 0.95 341820 0s 000ms 761us + 0.990625 356436 0s 001ms 585us + 0.99902344 359458 0s 007ms 083us + +Response body size in bytes (359809 samples) + min: 10 | mean: 10 | max: 10 | pstdev: 0 + +Response header size in bytes (359809 samples) + min: 110 | mean: 110 | max: 110 | pstdev: 0 + +Initiation to completion (359996 samples) + min: 0s 000ms 005us | mean: 0s 000ms 543us | max: 0s 048ms 146us | pstdev: 0s 000ms 476us + + Percentile Count Value + 0.5 179999 0s 000ms 468us + 0.75 269997 0s 000ms 543us + 0.8 288002 0s 000ms 568us + 0.9 324002 0s 000ms 683us + 0.95 341998 0s 000ms 783us + 0.990625 356622 0s 001ms 632us + 0.99902344 359645 0s 007ms 247us + +Counter Value Per second +benchmark.http_2xx 359809 3997.86 +benchmark.pool_overflow 187 2.08 +cluster_manager.cluster_added 4 0.04 +default.total_match_count 4 0.04 +membership_change 4 0.04 +runtime.load_success 1 0.01 +runtime.override_dir_not_exists 1 0.01 +upstream_cx_http1_total 66 0.73 +upstream_cx_rx_bytes_total 56490013 627663.98 +upstream_cx_total 66 0.73 +upstream_cx_tx_bytes_total 15471916 171909.40 +upstream_rq_pending_overflow 187 2.08 +upstream_rq_pending_total 66 0.73 +upstream_rq_total 359812 3997.89 + +[14:49:42.732798][1][I] Done. + +``` + +
+ +### Metrics + +|Benchmark Name |Envoy Gateway Memory (MiB)|Envoy Gateway Total CPU (Seconds)|Envoy Proxy Memory: k5s2s[1] (MiB)| +|- |- |- |- | +|scale-up-httproutes-10 |76 |0.34 |7 | +|scale-up-httproutes-50 |114 |1.94 |12 | +|scale-up-httproutes-100 |195 |10.85 |20 | +|scale-up-httproutes-300 |1124 |176.36 |81 | +|scale-up-httproutes-500 |1588 |16.69 |190 | +|scale-down-httproutes-300|661 |6.04 |87 | +|scale-down-httproutes-100|679 |104.73 |95 | +|scale-down-httproutes-50 |143 |172.84 |53 | +|scale-down-httproutes-10 |118 |174.16 |18 | +1. envoy-gateway-system/envoy-benchmark-test-benchmark-0520098c-7668c94dd5-k5s2s diff --git a/test/benchmark/benchmark_test.go b/test/benchmark/benchmark_test.go new file mode 100644 index 00000000000..7edbd215cec --- /dev/null +++ b/test/benchmark/benchmark_test.go @@ -0,0 +1,56 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build benchmark +// +build benchmark + +package benchmark + +import ( + "flag" + "testing" + + "github.com/stretchr/testify/require" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" + + "github.com/envoyproxy/gateway/test/benchmark/suite" + "github.com/envoyproxy/gateway/test/benchmark/tests" +) + +func TestBenchmark(t *testing.T) { + cfg, err := config.GetConfig() + require.NoError(t, err) + + cli, err := client.New(cfg, client.Options{}) + require.NoError(t, err) + + // Install all the scheme for kubernetes client. + suite.CheckInstallScheme(t, cli) + + // Parse benchmark options. + flag.Parse() + options := suite.NewBenchmarkOptions( + *suite.RPS, + *suite.Connections, + *suite.Duration, + *suite.Concurrency, + ) + + bSuite, err := suite.NewBenchmarkTestSuite( + cli, + options, + "config/gateway.yaml", + "config/httproute.yaml", + "config/nighthawk-client.yaml", + *suite.ReportSavePath, + ) + if err != nil { + t.Fatalf("Failed to create BenchmarkTestSuite: %v", err) + } + + t.Logf("Running %d benchmark tests", len(tests.BenchmarkTests)) + bSuite.Run(t, tests.BenchmarkTests) +} diff --git a/test/benchmark/config/gateway.yaml b/test/benchmark/config/gateway.yaml new file mode 100644 index 00000000000..d7863e86ac5 --- /dev/null +++ b/test/benchmark/config/gateway.yaml @@ -0,0 +1,14 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: "{GATEWAY_NAME}" + namespace: benchmark-test +spec: + gatewayClassName: envoy-gateway + listeners: + - name: http + port: 8081 + protocol: HTTP + allowedRoutes: + namespaces: + from: Same diff --git a/test/benchmark/config/gatewayclass.yaml b/test/benchmark/config/gatewayclass.yaml new file mode 100644 index 00000000000..fbecf76332d --- /dev/null +++ b/test/benchmark/config/gatewayclass.yaml @@ -0,0 +1,82 @@ +kind: GatewayClass +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: envoy-gateway +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parametersRef: + group: gateway.envoyproxy.io + kind: EnvoyProxy + name: proxy-config + namespace: envoy-gateway-system +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: proxy-config + namespace: envoy-gateway-system +spec: + provider: + type: Kubernetes + kubernetes: + envoyDeployment: + container: + resources: + limits: + memory: "1024Mi" + cpu: "1000m" + requests: + memory: "256Mi" + cpu: "500m" + telemetry: + metrics: + prometheus: {} + sinks: + - type: OpenTelemetry + openTelemetry: + backendRefs: + - name: otel-collector + namespace: monitoring + port: 4317 + accessLog: + settings: + - format: + type: Text + text: | + [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" + sinks: + - type: File + file: + path: /dev/stdout + - type: OpenTelemetry + openTelemetry: + backendRefs: + - name: otel-collector + namespace: monitoring + port: 4317 + resources: + k8s.cluster.name: "envoy-gateway" + tracing: + provider: + backendRefs: + - name: otel-collector + namespace: monitoring + port: 4317 + customTags: + "k8s.cluster.name": + type: Literal + literal: + value: "envoy-gateway" + "k8s.pod.name": + type: Environment + environment: + name: ENVOY_POD_NAME + defaultValue: "-" + "k8s.namespace.name": + type: Environment + environment: + name: ENVOY_GATEWAY_NAMESPACE + defaultValue: "envoy-gateway-system" + shutdown: + drainTimeout: 5s + minDrainDuration: 1s diff --git a/test/benchmark/config/httproute.yaml b/test/benchmark/config/httproute.yaml new file mode 100644 index 00000000000..b422d4f2d85 --- /dev/null +++ b/test/benchmark/config/httproute.yaml @@ -0,0 +1,22 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: "{HTTPROUTE_NAME}" + namespace: benchmark-test +spec: + parentRefs: + - name: "{REF_GATEWAY_NAME}" + hostnames: + - "www.benchmark.com" + rules: + - backendRefs: + - group: "" + kind: Service + name: nighthawk-test-server + namespace: benchmark-test + port: 8080 + weight: 1 + matches: + - path: + type: PathPrefix + value: / diff --git a/test/benchmark/config/nighthawk-client.yaml b/test/benchmark/config/nighthawk-client.yaml new file mode 100644 index 00000000000..90375d8c05c --- /dev/null +++ b/test/benchmark/config/nighthawk-client.yaml @@ -0,0 +1,18 @@ +### Nighthawk test client job template +apiVersion: batch/v1 +kind: Job +metadata: + name: "{NIGHTHAWK_CLIENT_NAME}" + namespace: benchmark-test + labels: + benchmark-test/client: "true" +spec: + template: + spec: + containers: + - name: nighthawk-client + image: envoyproxy/nighthawk-dev:latest + imagePullPolicy: IfNotPresent + args: ["nighthawk_client"] # Fill-up args at runtime + restartPolicy: Never + backoffLimit: 3 diff --git a/test/benchmark/config/nighthawk-test-server-config.yaml b/test/benchmark/config/nighthawk-test-server-config.yaml new file mode 100644 index 00000000000..f8e69f6f1cb --- /dev/null +++ b/test/benchmark/config/nighthawk-test-server-config.yaml @@ -0,0 +1,44 @@ +static_resources: + listeners: + # define an origin server on :10000 that always returns "lorem ipsum..." + - address: + socket_address: + address: 0.0.0.0 + port_value: 8080 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + generate_request_id: false + codec_type: AUTO + stat_prefix: ingress_http + route_config: + name: local_route + virtual_hosts: + - name: service + domains: + - "*" + http_filters: + - name: dynamic-delay + typed_config: + "@type": type.googleapis.com/nighthawk.server.DynamicDelayConfiguration + static_delay: 0s + - name: test-server # before envoy.router because order matters! + typed_config: + "@type": type.googleapis.com/nighthawk.server.ResponseOptions + response_body_size: 10 + v3_response_headers: + - {header: {key: "foo", value: "bar"}} + - {header: {key: "foo", value: "bar2"}, append: true} + - {header: {key: "x-nh", value: "1"}} + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + dynamic_stats: false +admin: + access_log_path: /tmp/envoy.log + address: + socket_address: + address: 0.0.0.0 + port_value: 8081 diff --git a/test/benchmark/config/nighthawk-test-server.yaml b/test/benchmark/config/nighthawk-test-server.yaml new file mode 100644 index 00000000000..dfd91aae464 --- /dev/null +++ b/test/benchmark/config/nighthawk-test-server.yaml @@ -0,0 +1,53 @@ +### Nighthawk test server deployment & service +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nighthawk-test-server + namespace: benchmark-test +spec: + replicas: 1 + selector: + matchLabels: + app: nighthawk-test-server + template: + metadata: + labels: + app: nighthawk-test-server + spec: + serviceAccountName: default + containers: + - name: nighthawk-server + image: envoyproxy/nighthawk-dev:latest + imagePullPolicy: IfNotPresent + args: ["nighthawk_test_server", "-c", "/etc/test-server-config/nighthawk-test-server-config.yaml"] + ports: + - containerPort: 8080 + volumeMounts: + - name: test-server-config + mountPath: "/etc/test-server-config" + env: + - name: PORT + value: "8080" + resources: + requests: + cpu: "2" + limits: + cpu: "2" + volumes: + - name: test-server-config + configMap: + name: test-server-config # Created directly from file +--- +apiVersion: v1 +kind: Service +metadata: + name: nighthawk-test-server + namespace: benchmark-test +spec: + type: ClusterIP + selector: + app: nighthawk-test-server + ports: + - name: http + port: 8080 + targetPort: 8080 diff --git a/test/benchmark/suite/client.go b/test/benchmark/suite/client.go new file mode 100644 index 00000000000..3db4c16c52d --- /dev/null +++ b/test/benchmark/suite/client.go @@ -0,0 +1,32 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build benchmark +// +build benchmark + +package suite + +import ( + "testing" + + "github.com/stretchr/testify/require" + batchv1 "k8s.io/api/batch/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gwapiv1a3 "sigs.k8s.io/gateway-api/apis/v1alpha3" + gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" + + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" +) + +func CheckInstallScheme(t *testing.T, c client.Client) { + require.NoError(t, gwapiv1a3.Install(c.Scheme())) + require.NoError(t, gwapiv1a2.Install(c.Scheme())) + require.NoError(t, gwapiv1b1.Install(c.Scheme())) + require.NoError(t, gwapiv1.Install(c.Scheme())) + require.NoError(t, egv1a1.AddToScheme(c.Scheme())) + require.NoError(t, batchv1.AddToScheme(c.Scheme())) +} diff --git a/test/benchmark/suite/flags.go b/test/benchmark/suite/flags.go new file mode 100644 index 00000000000..a07d2d4b010 --- /dev/null +++ b/test/benchmark/suite/flags.go @@ -0,0 +1,19 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build benchmark +// +build benchmark + +package suite + +import "flag" + +var ( + RPS = flag.String("rps", "1000", "The target requests-per-second rate.") + Connections = flag.String("connections", "10", "The maximum allowed number of concurrent connections per event loop. HTTP/1 only.") + Duration = flag.String("duration", "60", "The number of seconds that the test should run.") + Concurrency = flag.String("concurrency", "auto", "The number of concurrent event loops that should be used.") + ReportSavePath = flag.String("report-save-path", "", "The path where to save the benchmark test report.") +) diff --git a/test/benchmark/suite/render.go b/test/benchmark/suite/render.go new file mode 100644 index 00000000000..e3c059685f1 --- /dev/null +++ b/test/benchmark/suite/render.go @@ -0,0 +1,329 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build benchmark +// +build benchmark + +package suite + +import ( + "bytes" + "fmt" + "io" + "math" + "os" + "strconv" + "strings" + "text/tabwriter" + + prom "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" + "k8s.io/apimachinery/pkg/util/sets" +) + +const ( + omitEmptyValue = "-" + benchmarkEnvPrefix = "BENCHMARK_" + + // Supported metric type. + metricTypeGauge = "gauge" + metricTypeCounter = "counter" + + // Supported metric unit. + metricUnitMiB = "MiB" + metricUnitSeconds = "Seconds" + metricUnitMilliCPU = "m" +) + +type ReportTableHeader struct { + Name string + Metric *MetricEntry + + // Underlying name of one envoy-proxy, used by data-plane metrics. + ProxyName string +} + +type MetricEntry struct { + Name string + Type string + FromControlPlane bool + DisplayUnit string + ConvertUnit func(float64) float64 +} + +// RenderReport renders a report out of given list of benchmark report in Markdown format. +func RenderReport(writer io.Writer, name, description string, reports []*BenchmarkReport, titleLevel int) error { + headerSettings := []ReportTableHeader{ + { + Name: "Benchmark Name", + }, + { + Name: "Envoy Gateway Memory", + Metric: &MetricEntry{ + Name: "process_resident_memory_bytes", + Type: metricTypeGauge, + DisplayUnit: metricUnitMiB, + FromControlPlane: true, + ConvertUnit: byteToMiB, + }, + }, + { + Name: "Envoy Gateway Total CPU", + Metric: &MetricEntry{ + Name: "process_cpu_seconds_total", + Type: metricTypeCounter, + DisplayUnit: metricUnitSeconds, + FromControlPlane: true, + }, + }, + { + Name: "Envoy Proxy Memory", + Metric: &MetricEntry{ + Name: "envoy_server_memory_allocated", + Type: metricTypeGauge, + DisplayUnit: metricUnitMiB, + FromControlPlane: false, + ConvertUnit: byteToMiB, + }, + }, + } + + writeSection(writer, name, titleLevel, description) + + writeSection(writer, "Results", titleLevel+1, "Click to see the full results.") + renderResultsTable(writer, reports) + + writeSection(writer, "Metrics", titleLevel+1, "") + err := renderMetricsTable(writer, headerSettings, reports) + if err != nil { + return err + } + + return nil +} + +// newMarkdownStyleTableWriter returns a tabwriter that write table in Markdown style. +func newMarkdownStyleTableWriter(writer io.Writer) *tabwriter.Writer { + return tabwriter.NewWriter(writer, 0, 0, 0, ' ', tabwriter.Debug) +} + +func renderEnvSettingsTable(writer io.Writer) { + _, _ = fmt.Fprintln(writer, "Benchmark test settings:", "\n") + + table := newMarkdownStyleTableWriter(writer) + + headers := []ReportTableHeader{ + { + Name: "RPS", + }, + { + Name: "Connections", + }, + { + Name: "Duration", + Metric: &MetricEntry{ + DisplayUnit: metricUnitSeconds, + }, + }, + { + Name: "CPU Limits", + Metric: &MetricEntry{ + DisplayUnit: metricUnitMilliCPU, + }, + }, + { + Name: "Memory Limits", + Metric: &MetricEntry{ + DisplayUnit: metricUnitMiB, + }, + }, + } + + renderMetricsTableHeader(table, headers) + + writeTableRow(table, headers, func(_ int, h ReportTableHeader) string { + env := strings.Replace(strings.ToUpper(h.Name), " ", "_", -1) + if v, ok := os.LookupEnv(benchmarkEnvPrefix + env); ok { + return v + } + return omitEmptyValue + }) + + _ = table.Flush() +} + +func renderResultsTable(writer io.Writer, reports []*BenchmarkReport) { + // TODO: better processing these benchmark results. + for _, report := range reports { + writeCollapsibleSection(writer, report.Name, report.RawResult) + } +} + +func renderMetricsTable(writer io.Writer, headerSettings []ReportTableHeader, reports []*BenchmarkReport) error { + table := newMarkdownStyleTableWriter(writer) + + // Preprocess the table header for metrics table. + var headers []ReportTableHeader + // 1. Collect all the possible proxy names. + proxyNames := sets.NewString() + for _, report := range reports { + for name := range report.RawDPMetrics { + proxyNames.Insert(name) + } + } + // 2. Generate header names for data-plane proxies. + for _, hs := range headerSettings { + if hs.Metric != nil && !hs.Metric.FromControlPlane { + for i, proxyName := range proxyNames.List() { + names := strings.Split(proxyName, "-") + headers = append(headers, ReportTableHeader{ + Name: fmt.Sprintf("%s: %s[%d]", hs.Name, names[len(names)-1], i+1), + Metric: hs.Metric, + ProxyName: proxyName, + }) + } + } else { + // For control-plane metrics or plain header. + headers = append(headers, hs) + } + } + + renderMetricsTableHeader(table, headers) + + for _, report := range reports { + mfCP, err := parseMetrics(report.RawCPMetrics) + if err != nil { + return err + } + + mfDPs := make(map[string]map[string]*prom.MetricFamily, len(report.RawDPMetrics)) + for dpName, dpMetrics := range report.RawDPMetrics { + mfDP, err := parseMetrics(dpMetrics) + if err != nil { + return err + } + mfDPs[dpName] = mfDP + } + + writeTableRow(table, headers, func(_ int, h ReportTableHeader) string { + if h.Metric == nil { + return report.Name + } + + if h.Metric.FromControlPlane { + return processMetricValue(mfCP, h) + } else { + if mfDP, ok := mfDPs[h.ProxyName]; ok { + return processMetricValue(mfDP, h) + } + } + + return omitEmptyValue + }) + } + + _ = table.Flush() + + // Generate footnotes for envoy-proxy headers. + for i, proxyName := range proxyNames.List() { + _, _ = fmt.Fprintln(writer, fmt.Sprintf("%d.", i+1), proxyName) + } + + return nil +} + +func renderMetricsTableHeader(table *tabwriter.Writer, headers []ReportTableHeader) { + writeTableRow(table, headers, func(_ int, h ReportTableHeader) string { + if h.Metric != nil && len(h.Metric.DisplayUnit) > 0 { + return fmt.Sprintf("%s (%s)", h.Name, h.Metric.DisplayUnit) + } + return h.Name + }) + + writeTableDelimiter(table, len(headers)) +} + +func byteToMiB(x float64) float64 { + return math.Round(x / (1024 * 1024)) +} + +// writeSection writes one section in Markdown style, content is optional. +func writeSection(writer io.Writer, title string, level int, content string) { + md := fmt.Sprintf("\n%s %s\n", strings.Repeat("#", level), title) + if len(content) > 0 { + md += fmt.Sprintf("\n%s\n", content) + } + _, _ = fmt.Fprintln(writer, md) +} + +// writeCollapsibleSection writes one collapsible section in Markdown style. +func writeCollapsibleSection(writer io.Writer, title string, content []byte) { + _, _ = fmt.Fprintln(writer, fmt.Sprintf(` +
+%s + +%s + +
`, title, fmt.Sprintf("```plaintext\n%s\n```", content))) +} + +// writeTableRow writes one row in Markdown table style. +func writeTableRow[T any](table *tabwriter.Writer, values []T, get func(int, T) string) { + row := "|" + for i, v := range values { + row += get(i, v) + "\t" + } + + _, _ = fmt.Fprintln(table, row) +} + +// writeTableDelimiter writes table delimiter in Markdown table style. +func writeTableDelimiter(table *tabwriter.Writer, n int) { + sep := "|" + for i := 0; i < n; i++ { + sep += "-\t" + } + + _, _ = fmt.Fprintln(table, sep) +} + +// parseMetrics parses input metrics that in Prometheus format. +func parseMetrics(metrics []byte) (map[string]*prom.MetricFamily, error) { + var ( + reader = bytes.NewReader(metrics) + parser expfmt.TextParser + ) + + mf, err := parser.TextToMetricFamilies(reader) + if err != nil { + return nil, err + } + + return mf, nil +} + +// processMetricValue process one metric value according to the given header and metric families. +func processMetricValue(metricFamilies map[string]*prom.MetricFamily, header ReportTableHeader) string { + if mf, ok := metricFamilies[header.Metric.Name]; ok { + var value float64 + + switch header.Metric.Type { + case metricTypeGauge: + value = *mf.Metric[0].Gauge.Value + case metricTypeCounter: + value = *mf.Metric[0].Counter.Value + default: + return omitEmptyValue + } + + if header.Metric.ConvertUnit != nil { + value = header.Metric.ConvertUnit(value) + } + + return strconv.FormatFloat(value, 'f', -1, 64) + } + + return omitEmptyValue +} diff --git a/test/benchmark/suite/report.go b/test/benchmark/suite/report.go new file mode 100644 index 00000000000..949fb5fa8d2 --- /dev/null +++ b/test/benchmark/suite/report.go @@ -0,0 +1,219 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build benchmark +// +build benchmark + +package suite + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/envoyproxy/gateway/internal/cmd/options" + kube "github.com/envoyproxy/gateway/internal/kubernetes" +) + +const ( + localMetricsPort = 0 + controlPlaneMetricsPort = 19001 +) + +type BenchmarkReport struct { + Name string + RawResult []byte + RawCPMetrics []byte + RawDPMetrics map[string][]byte + + kubeClient kube.CLIClient +} + +func NewBenchmarkReport(name string) (*BenchmarkReport, error) { + kubeClient, err := kube.NewCLIClient(options.DefaultConfigFlags.ToRawKubeConfigLoader()) + if err != nil { + return nil, err + } + + return &BenchmarkReport{ + Name: name, + RawDPMetrics: make(map[string][]byte), + kubeClient: kubeClient, + }, nil +} + +// Print prints the raw report of one benchmark test. +func (r *BenchmarkReport) Print(t *testing.T, name string) { + t.Logf("The raw report of benchmark test: %s", name) + + t.Logf("=== Benchmark Result: \n\n %s \n\n", r.RawResult) + t.Logf("=== Control-Plane Metrics: \n\n %s \n\n", r.RawCPMetrics) + + for dpName, dpMetrics := range r.RawDPMetrics { + t.Logf("=== Data-Plane Metrics for %s: \n\n %s \n\n", dpName, dpMetrics) + } +} + +func (r *BenchmarkReport) Collect(t *testing.T, ctx context.Context, job *types.NamespacedName) error { + if err := r.GetBenchmarkResult(t, ctx, job); err != nil { + return err + } + + if err := r.GetControlPlaneMetrics(t, ctx); err != nil { + return err + } + + if err := r.GetDataPlaneMetrics(t, ctx); err != nil { + return err + } + + return nil +} + +func (r *BenchmarkReport) GetBenchmarkResult(t *testing.T, ctx context.Context, job *types.NamespacedName) error { + pods, err := r.kubeClient.Kube().CoreV1().Pods(job.Namespace).List(ctx, metav1.ListOptions{LabelSelector: "job-name=" + job.Name}) + + if len(pods.Items) < 1 { + return fmt.Errorf("failed to get any pods for job %s", job.String()) + } + + if len(pods.Items) > 1 { + t.Logf("Got %d pod(s) associated job %s, should be 1 pod, could be pod err and job backoff then restart, please check your pod(s) status", + len(pods.Items), job.Name) + } + + pod := &pods.Items[0] + logs, err := r.getLogsFromPod( + ctx, &types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, + ) + if err != nil { + return err + } + + r.RawResult = logs + + return nil +} + +func (r *BenchmarkReport) GetControlPlaneMetrics(t *testing.T, ctx context.Context) error { + egPods, err := r.kubeClient.Kube().CoreV1().Pods("envoy-gateway-system"). + List(ctx, metav1.ListOptions{LabelSelector: "control-plane=envoy-gateway"}) + if err != nil { + return err + } + + if len(egPods.Items) < 1 { + return fmt.Errorf("failed to get any pods for envoy-gateway") + } + + if len(egPods.Items) > 1 { + t.Logf("Got %d pod(s), using the first one as default envoy-gateway pod", len(egPods.Items)) + } + + egPod := &egPods.Items[0] + metrics, err := r.getMetricsFromPortForwarder( + t, &types.NamespacedName{Name: egPod.Name, Namespace: egPod.Namespace}, "/metrics", + ) + if err != nil { + return err + } + + r.RawCPMetrics = metrics + + return nil +} + +func (r *BenchmarkReport) GetDataPlaneMetrics(t *testing.T, ctx context.Context) error { + epPods, err := r.kubeClient.Kube().CoreV1().Pods("envoy-gateway-system"). + List(ctx, metav1.ListOptions{LabelSelector: "gateway.envoyproxy.io/owning-gateway-namespace=benchmark-test,gateway.envoyproxy.io/owning-gateway-name=benchmark"}) + if err != nil { + return err + } + + if len(epPods.Items) < 1 { + return fmt.Errorf("failed to get any pods for envoy-proxies") + } + + t.Logf("Got %d pod(s) from data-plane", len(epPods.Items)) + + for _, epPod := range epPods.Items { + podNN := &types.NamespacedName{Name: epPod.Name, Namespace: epPod.Namespace} + metrics, err := r.getMetricsFromPortForwarder(t, podNN, "/stats/prometheus") + if err != nil { + return err + } + + r.RawDPMetrics[podNN.String()] = metrics + } + + return nil +} + +// getLogsFromPod scrapes the logs directly from the pod (default container). +func (r *BenchmarkReport) getLogsFromPod(ctx context.Context, pod *types.NamespacedName) ([]byte, error) { + podLogOpts := corev1.PodLogOptions{} + + req := r.kubeClient.Kube().CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &podLogOpts) + podLogs, err := req.Stream(ctx) + if err != nil { + return nil, err + } + + defer podLogs.Close() + + buf := new(bytes.Buffer) + _, err = io.Copy(buf, podLogs) + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +// getMetricsFromPortForwarder retrieves metrics from pod by request url, like `/metrics`. +func (r *BenchmarkReport) getMetricsFromPortForwarder(t *testing.T, pod *types.NamespacedName, url string) ([]byte, error) { + fw, err := kube.NewLocalPortForwarder(r.kubeClient, *pod, localMetricsPort, controlPlaneMetricsPort) + if err != nil { + return nil, fmt.Errorf("failed to build port forwarder for pod %s: %v", pod.String(), err) + } + + if err = fw.Start(); err != nil { + fw.Stop() + + return nil, fmt.Errorf("failed to start port forwarder for pod %s: %v", pod.String(), err) + } + + var out []byte + // Retrieving metrics from Pod. + go func() { + defer fw.Stop() + + url := fmt.Sprintf("http://%s%s", fw.Address(), url) + resp, err := http.Get(url) + if err != nil { + t.Errorf("failed to request %s: %v", url, err) + return + } + + metrics, err := io.ReadAll(resp.Body) + if err != nil { + t.Errorf("failed to read metrics: %v", err) + return + } + + out = metrics + }() + + fw.WaitForStop() + + return out, nil +} diff --git a/test/benchmark/suite/suite.go b/test/benchmark/suite/suite.go new file mode 100644 index 00000000000..919e019e63a --- /dev/null +++ b/test/benchmark/suite/suite.go @@ -0,0 +1,355 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build benchmark +// +build benchmark + +package suite + +import ( + "bytes" + "context" + "fmt" + "os" + "strconv" + "testing" + "time" + + batchv1 "k8s.io/api/batch/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/client" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + "sigs.k8s.io/gateway-api/conformance/utils/config" + "sigs.k8s.io/yaml" +) + +const ( + BenchmarkTestScaledKey = "benchmark-test/scaled" + BenchmarkTestClientKey = "benchmark-test/client" + DefaultControllerName = "gateway.envoyproxy.io/gatewayclass-controller" +) + +type BenchmarkTestSuite struct { + Client client.Client + TimeoutConfig config.TimeoutConfig + ControllerName string + Options BenchmarkOptions + ReportSavePath string + + // Resources template for supported benchmark targets. + GatewayTemplate *gwapiv1.Gateway + HTTPRouteTemplate *gwapiv1.HTTPRoute + BenchmarkClientJob *batchv1.Job + + // Indicates which resources are scaled. + scaledLabel map[string]string +} + +func NewBenchmarkTestSuite(client client.Client, options BenchmarkOptions, + gatewayManifest, httpRouteManifest, benchmarkClientManifest, reportPath string) (*BenchmarkTestSuite, error) { + var ( + gateway = new(gwapiv1.Gateway) + httproute = new(gwapiv1.HTTPRoute) + benchmarkClient = new(batchv1.Job) + timeoutConfig = config.TimeoutConfig{} + ) + + data, err := os.ReadFile(gatewayManifest) + if err != nil { + return nil, err + } + if err = yaml.Unmarshal(data, gateway); err != nil { + return nil, err + } + + data, err = os.ReadFile(httpRouteManifest) + if err != nil { + return nil, err + } + if err = yaml.Unmarshal(data, httproute); err != nil { + return nil, err + } + + data, err = os.ReadFile(benchmarkClientManifest) + if err != nil { + return nil, err + } + if err = yaml.Unmarshal(data, benchmarkClient); err != nil { + return nil, err + } + + // Reset some timeout config for the benchmark test. + config.SetupTimeoutConfig(&timeoutConfig) + timeoutConfig.RouteMustHaveParents = 180 * time.Second + + // Prepare static options for benchmark client. + staticArgs := prepareBenchmarkClientStaticArgs(options) + container := &benchmarkClient.Spec.Template.Spec.Containers[0] + container.Args = append(container.Args, staticArgs...) + + return &BenchmarkTestSuite{ + Client: client, + Options: options, + TimeoutConfig: timeoutConfig, + ControllerName: DefaultControllerName, + ReportSavePath: reportPath, + GatewayTemplate: gateway, + HTTPRouteTemplate: httproute, + BenchmarkClientJob: benchmarkClient, + scaledLabel: map[string]string{ + BenchmarkTestScaledKey: "true", + }, + }, nil +} + +func (b *BenchmarkTestSuite) Run(t *testing.T, tests []BenchmarkTest) { + t.Logf("Running %d benchmark test", len(tests)) + + buf := make([]byte, 0) + writer := bytes.NewBuffer(buf) + + writeSection(writer, "Benchmark Report", 1, "") + renderEnvSettingsTable(writer) + + for _, test := range tests { + t.Logf("Running benchmark test: %s", test.ShortName) + + reports := test.Test(t, b) + if len(reports) == 0 { + continue + } + + // Generate a human-readable benchmark report for each test. + t.Logf("Got %d reports for test: %s", len(reports), test.ShortName) + + if err := RenderReport(writer, "Test: "+test.ShortName, test.Description, reports, 2); err != nil { + t.Errorf("Error generating report for %s: %v", test.ShortName, err) + } + } + + if len(b.ReportSavePath) > 0 { + if err := os.WriteFile(b.ReportSavePath, writer.Bytes(), 0644); err != nil { + t.Errorf("Error writing report to path '%s': %v", b.ReportSavePath, err) + } else { + t.Logf("Writing report to path '%s' successfully", b.ReportSavePath) + } + } else { + t.Log(fmt.Sprintf("%s", writer.Bytes())) + } +} + +// Benchmark runs benchmark test as a Kubernetes Job, and return the benchmark result. +// +// TODO: currently running benchmark test via nighthawk_client, +// consider switching to gRPC nighthawk-service for benchmark test. +// ref: https://github.com/envoyproxy/nighthawk/blob/main/api/client/service.proto +func (b *BenchmarkTestSuite) Benchmark(t *testing.T, ctx context.Context, name, gatewayHostPort string, requestHeaders ...string) (*BenchmarkReport, error) { + t.Logf("Running benchmark test: %s", name) + + jobNN, err := b.createBenchmarkClientJob(ctx, name, gatewayHostPort, requestHeaders...) + if err != nil { + return nil, err + } + + duration, err := strconv.ParseInt(b.Options.Duration, 10, 64) + if err != nil { + return nil, err + } + + // Wait from benchmark test job to complete. + if err = wait.PollUntilContextTimeout(ctx, 10*time.Second, time.Duration(duration*10)*time.Second, true, func(ctx context.Context) (bool, error) { + job := new(batchv1.Job) + if err = b.Client.Get(ctx, *jobNN, job); err != nil { + return false, err + } + + for _, condition := range job.Status.Conditions { + if condition.Type == batchv1.JobComplete && condition.Status == "True" { + return true, nil + } + + // Early return if job already failed. + if condition.Type == batchv1.JobFailed && condition.Status == "True" && + condition.Reason == batchv1.JobReasonBackoffLimitExceeded { + return false, fmt.Errorf("job already failed") + } + } + + t.Logf("Job %s still not complete", name) + + return false, nil + }); err != nil { + t.Errorf("Failed to run benchmark test: %v", err) + + return nil, err + } + + t.Logf("Running benchmark test: %s successfully", name) + + report, err := NewBenchmarkReport(name) + if err != nil { + return nil, err + } + + // Get all the reports from this benchmark test run. + if err = report.Collect(t, ctx, jobNN); err != nil { + return nil, err + } + + report.Print(t, name) + + return report, nil +} + +func (b *BenchmarkTestSuite) createBenchmarkClientJob(ctx context.Context, name, gatewayHostPort string, requestHeaders ...string) (*types.NamespacedName, error) { + job := b.BenchmarkClientJob.DeepCopy() + job.SetName(name) + + runtimeArgs := prepareBenchmarkClientRuntimeArgs(gatewayHostPort, requestHeaders...) + container := &job.Spec.Template.Spec.Containers[0] + container.Args = append(container.Args, runtimeArgs...) + + if err := b.CreateResource(ctx, job); err != nil { + return nil, err + } + + return &types.NamespacedName{Name: job.Name, Namespace: job.Namespace}, nil +} + +func prepareBenchmarkClientStaticArgs(options BenchmarkOptions) []string { + staticArgs := []string{ + "--rps", options.RPS, + "--connections", options.Connections, + "--duration", options.Duration, + "--concurrency", options.Concurrency, + } + return staticArgs +} + +func prepareBenchmarkClientRuntimeArgs(gatewayHostPort string, requestHeaders ...string) []string { + args := make([]string, 0, len(requestHeaders)*2+1) + + for _, reqHeader := range requestHeaders { + args = append(args, "--request-header", reqHeader) + } + args = append(args, "http://"+gatewayHostPort) + + return args +} + +// ScaleUpHTTPRoutes scales up HTTPRoutes that are all referenced to one Gateway according to +// the scale range: (a, b], which scales up from a to b with a <= b. +// +// The `afterCreation` is a callback function that only runs every time after one HTTPRoutes +// has been created successfully. +// +// All created scaled resources will be labeled with BenchmarkTestScaledKey. +func (b *BenchmarkTestSuite) ScaleUpHTTPRoutes(ctx context.Context, scaleRange [2]uint16, routeNameFormat, refGateway string, afterCreation func(*gwapiv1.HTTPRoute)) error { + var i, begin, end uint16 + begin, end = scaleRange[0], scaleRange[1] + + if begin > end { + return fmt.Errorf("got wrong scale range, %d is not greater than %d", end, begin) + } + + for i = begin + 1; i <= end; i++ { + routeName := fmt.Sprintf(routeNameFormat, i) + newRoute := b.HTTPRouteTemplate.DeepCopy() + newRoute.SetName(routeName) + newRoute.SetLabels(b.scaledLabel) + newRoute.Spec.ParentRefs[0].Name = gwapiv1.ObjectName(refGateway) + + if err := b.CreateResource(ctx, newRoute); err != nil { + return err + } + + if afterCreation != nil { + afterCreation(newRoute) + } + } + + return nil +} + +// ScaleDownHTTPRoutes scales down HTTPRoutes that are all referenced to one Gateway according to +// the scale range: [a, b), which scales down from a to b with a > b. +// +// The `afterDeletion` is a callback function that only runs every time after one HTTPRoutes has +// been deleted successfully. +func (b *BenchmarkTestSuite) ScaleDownHTTPRoutes(ctx context.Context, scaleRange [2]uint16, routeNameFormat, refGateway string, afterDeletion func(*gwapiv1.HTTPRoute)) error { + var i, begin, end uint16 + begin, end = scaleRange[0], scaleRange[1] + + if begin <= end { + return fmt.Errorf("got wrong scale range, %d is not less than %d", end, begin) + } + + if end == 0 { + return fmt.Errorf("cannot scale routes down to zero") + } + + for i = begin; i > end; i-- { + routeName := fmt.Sprintf(routeNameFormat, i) + oldRoute := b.HTTPRouteTemplate.DeepCopy() + oldRoute.SetName(routeName) + oldRoute.SetLabels(b.scaledLabel) + oldRoute.Spec.ParentRefs[0].Name = gwapiv1.ObjectName(refGateway) + + if err := b.DeleteResource(ctx, oldRoute); err != nil { + return err + } + + if afterDeletion != nil { + afterDeletion(oldRoute) + } + } + + return nil +} + +func (b *BenchmarkTestSuite) CreateResource(ctx context.Context, object client.Object) error { + if err := b.Client.Create(ctx, object); err != nil { + if !kerrors.IsAlreadyExists(err) { + return err + } else { + return nil + } + } + return nil +} + +func (b *BenchmarkTestSuite) DeleteResource(ctx context.Context, object client.Object) error { + if err := b.Client.Delete(ctx, object); err != nil { + if !kerrors.IsNotFound(err) { + return err + } else { + return nil + } + } + return nil +} + +// DeleteScaledResources only cleanups all the resources under benchmark-test namespace. +func (b *BenchmarkTestSuite) DeleteScaledResources(ctx context.Context, object client.Object) error { + if err := b.Client.DeleteAllOf(ctx, object, + client.MatchingLabels{BenchmarkTestScaledKey: "true"}, client.InNamespace("benchmark-test")); err != nil { + return err + } + return nil +} + +// RegisterCleanup registers cleanup functions for all benchmark test resources. +func (b *BenchmarkTestSuite) RegisterCleanup(t *testing.T, ctx context.Context, object, scaledObject client.Object) { + t.Cleanup(func() { + t.Logf("Start to cleanup benchmark test resources") + + _ = b.DeleteResource(ctx, object) + _ = b.DeleteScaledResources(ctx, scaledObject) + + t.Logf("Clean up complete!") + }) +} diff --git a/test/benchmark/suite/test.go b/test/benchmark/suite/test.go new file mode 100644 index 00000000000..17138dfad25 --- /dev/null +++ b/test/benchmark/suite/test.go @@ -0,0 +1,34 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build benchmark +// +build benchmark + +package suite + +import "testing" + +type BenchmarkTest struct { + ShortName string + Description string + Test func(*testing.T, *BenchmarkTestSuite) []*BenchmarkReport +} + +// BenchmarkOptions for nighthawk-client. +type BenchmarkOptions struct { + RPS string + Connections string + Duration string + Concurrency string +} + +func NewBenchmarkOptions(rps, connections, duration, concurrency string) BenchmarkOptions { + return BenchmarkOptions{ + RPS: rps, + Connections: connections, + Duration: duration, + Concurrency: concurrency, + } +} diff --git a/test/benchmark/tests/scale_httproutes.go b/test/benchmark/tests/scale_httproutes.go new file mode 100644 index 00000000000..ca9c13f1828 --- /dev/null +++ b/test/benchmark/tests/scale_httproutes.go @@ -0,0 +1,115 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build benchmark +// +build benchmark + +package tests + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/types" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + "sigs.k8s.io/gateway-api/conformance/utils/kubernetes" + + "github.com/envoyproxy/gateway/test/benchmark/suite" +) + +func init() { + BenchmarkTests = append(BenchmarkTests, ScaleHTTPRoutes) +} + +var ScaleHTTPRoutes = suite.BenchmarkTest{ + ShortName: "ScaleHTTPRoute", + Description: "Fixed one Gateway and different scales of HTTPRoutes.", + Test: func(t *testing.T, bSuite *suite.BenchmarkTestSuite) (reports []*suite.BenchmarkReport) { + var ( + ctx = context.Background() + ns = "benchmark-test" + err error + requestHeaders = []string{ + "Host: www.benchmark.com", + } + ) + + gatewayNN := types.NamespacedName{Name: "benchmark", Namespace: ns} + gateway := bSuite.GatewayTemplate.DeepCopy() + gateway.SetName(gatewayNN.Name) + err = bSuite.CreateResource(ctx, gateway) + require.NoError(t, err) + + routeNameFormat := "benchmark-route-%d" + routeScales := []uint16{10, 50, 100, 300, 500} + routeScalesN := len(routeScales) + routeNNs := make([]types.NamespacedName, 0, routeScales[routeScalesN-1]) + + bSuite.RegisterCleanup(t, ctx, gateway, &gwapiv1.HTTPRoute{}) + + t.Run("scaling up httproutes", func(t *testing.T) { + var start uint16 = 0 + for _, scale := range routeScales { + t.Run(fmt.Sprintf("scaling up httproutes to %d", scale), func(t *testing.T) { + err = bSuite.ScaleUpHTTPRoutes(ctx, [2]uint16{start, scale}, routeNameFormat, gatewayNN.Name, func(route *gwapiv1.HTTPRoute) { + routeNN := types.NamespacedName{Name: route.Name, Namespace: route.Namespace} + routeNNs = append(routeNNs, routeNN) + + t.Logf("Create HTTPRoute: %s", routeNN.String()) + }) + require.NoError(t, err) + start = scale + + gatewayAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, bSuite.Client, bSuite.TimeoutConfig, + bSuite.ControllerName, kubernetes.NewGatewayRef(gatewayNN), routeNNs...) + + // Run benchmark test at different scale. + name := fmt.Sprintf("scale-up-httproutes-%d", scale) + report, err := bSuite.Benchmark(t, ctx, name, gatewayAddr, requestHeaders...) + require.NoError(t, err) + + reports = append(reports, report) + }) + } + }) + + t.Run("scaling down httproutes", func(t *testing.T) { + var start = routeScales[routeScalesN-1] + + for i := routeScalesN - 2; i >= 0; i-- { + scale := routeScales[i] + + t.Run(fmt.Sprintf("scaling down httproutes to %d", scale), func(t *testing.T) { + err = bSuite.ScaleDownHTTPRoutes(ctx, [2]uint16{start, scale}, routeNameFormat, gatewayNN.Name, func(route *gwapiv1.HTTPRoute) { + routeNN := routeNNs[len(routeNNs)-1] + routeNNs = routeNNs[:len(routeNNs)-1] + + // Making sure we are deleting the right one route. + require.Equal(t, routeNN, + types.NamespacedName{Name: route.Name, Namespace: route.Namespace}) + + t.Logf("Delete HTTPRoute: %s", routeNN.String()) + }) + require.NoError(t, err) + start = scale + + gatewayAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, bSuite.Client, bSuite.TimeoutConfig, + bSuite.ControllerName, kubernetes.NewGatewayRef(gatewayNN), routeNNs...) + + // Run benchmark test at different scale. + name := fmt.Sprintf("scale-down-httproutes-%d", scale) + report, err := bSuite.Benchmark(t, ctx, name, gatewayAddr, requestHeaders...) + require.NoError(t, err) + + reports = append(reports, report) + }) + } + }) + + return + }, +} diff --git a/test/benchmark/tests/tests.go b/test/benchmark/tests/tests.go new file mode 100644 index 00000000000..0aa49a13bb1 --- /dev/null +++ b/test/benchmark/tests/tests.go @@ -0,0 +1,13 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build benchmark +// +build benchmark + +package tests + +import "github.com/envoyproxy/gateway/test/benchmark/suite" + +var BenchmarkTests []suite.BenchmarkTest diff --git a/tools/make/kube.mk b/tools/make/kube.mk index 67d54ad8347..ddbe3fdd5d1 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -10,6 +10,13 @@ GATEWAY_RELEASE_URL ?= https://github.com/kubernetes-sigs/gateway-api/releases/d WAIT_TIMEOUT ?= 15m +BENCHMARK_TIMEOUT ?= 60m +BENCHMARK_CPU_LIMITS ?= 1000 # unit: 'm' +BENCHMARK_MEMORY_LIMITS ?= 1024 # unit: 'Mi' +BENCHMARK_RPS ?= 10000 +BENCHMARK_CONNECTIONS ?= 100 +BENCHMARK_DURATION ?= 60 + E2E_RUN_TEST ?= E2E_RUN_EG_UPGRADE_TESTS ?= false E2E_CLEANUP ?= true @@ -64,6 +71,14 @@ kube-deploy: manifests helm-generate.gateway-helm ## Install Envoy Gateway into @$(LOG_TARGET) helm install eg charts/gateway-helm --set deployment.envoyGateway.imagePullPolicy=$(IMAGE_PULL_POLICY) -n envoy-gateway-system --create-namespace --debug --timeout='$(WAIT_TIMEOUT)' --wait --wait-for-jobs +.PHONY: kube-deploy-for-benchmark-test +kube-deploy-for-benchmark-test: manifests helm-generate ## Install Envoy Gateway for benchmark test purpose only. + @$(LOG_TARGET) + helm install eg charts/gateway-helm --set deployment.envoyGateway.imagePullPolicy=$(IMAGE_PULL_POLICY) \ + --set deployment.envoyGateway.resources.limits.cpu=$(BENCHMARK_CPU_LIMITS)m \ + --set deployment.envoyGateway.resources.limits.memory=$(BENCHMARK_MEMORY_LIMITS)Mi \ + -n envoy-gateway-system --create-namespace --debug --timeout='$(WAIT_TIMEOUT)' --wait --wait-for-jobs + .PHONY: kube-undeploy kube-undeploy: manifests ## Uninstall the Envoy Gateway into the Kubernetes cluster specified in ~/.kube/config. @$(LOG_TARGET) @@ -100,6 +115,9 @@ conformance: create-cluster kube-install-image kube-deploy run-conformance delet .PHONY: experimental-conformance ## Create a kind cluster, deploy EG into it, run Gateway API experimental conformance, and clean up. experimental-conformance: create-cluster kube-install-image kube-deploy run-experimental-conformance delete-cluster ## Create a kind cluster, deploy EG into it, run Gateway API conformance, and clean up. +.PHONY: benchmark +benchmark: create-cluster kube-install-image kube-deploy-for-benchmark-test run-benchmark delete-cluster ## Create a kind cluster, deploy EG into it, run Envoy Gateway benchmark test, and clean up. + .PHONY: e2e e2e: create-cluster kube-install-image kube-deploy install-ratelimit run-e2e delete-cluster @@ -138,6 +156,30 @@ else endif endif +.PHONY: run-benchmark +run-benchmark: install-benchmark-server ## Run benchmark tests + @$(LOG_TARGET) + mkdir -p $(OUTPUT_DIR)/benchmark + kubectl wait --timeout=$(WAIT_TIMEOUT) -n benchmark-test deployment/nighthawk-test-server --for=condition=Available + kubectl wait --timeout=$(WAIT_TIMEOUT) -n envoy-gateway-system deployment/envoy-gateway --for=condition=Available + kubectl apply -f test/benchmark/config/gatewayclass.yaml + go test -v -tags benchmark -timeout $(BENCHMARK_TIMEOUT) ./test/benchmark --rps=$(BENCHMARK_RPS) --connections=$(BENCHMARK_CONNECTIONS) --duration=$(BENCHMARK_DURATION) --report-save-path=benchmark_report.md + +.PHONY: install-benchmark-server +install-benchmark-server: ## Install nighthawk server for benchmark test + @$(LOG_TARGET) + kubectl create namespace benchmark-test + kubectl -n benchmark-test create configmap test-server-config --from-file=test/benchmark/config/nighthawk-test-server-config.yaml -o yaml + kubectl apply -f test/benchmark/config/nighthawk-test-server.yaml + +.PHONY: uninstall-benchmark-server +uninstall-benchmark-server: ## Uninstall nighthawk server for benchmark test + @$(LOG_TARGET) + kubectl delete job -n benchmark-test -l benchmark-test/client=true + kubectl delete -f test/benchmark/config/nighthawk-test-server.yaml + kubectl delete configmap test-server-config -n benchmark-test + kubectl delete namespace benchmark-test + .PHONY: install-e2e-telemetry install-e2e-telemetry: helm-generate.gateway-addons-helm @$(LOG_TARGET) From 9ebcfac2e000fcb31144d5534b99f771d5f5435a Mon Sep 17 00:00:00 2001 From: Huabing Zhao Date: Fri, 28 Jun 2024 15:29:56 -0700 Subject: [PATCH 21/23] feat: Wasm OCI image (#3564) * support Wasm OCI image Signed-off-by: Huabing Zhao * set up test registry Signed-off-by: Huabing Zhao * add test for registry authn Signed-off-by: Huabing Zhao * fix lint Signed-off-by: Huabing Zhao * fix e2e Signed-off-by: Huabing Zhao * fix e2e Signed-off-by: Huabing Zhao * add test for unauthed private image Signed-off-by: Huabing Zhao * fix e2e Signed-off-by: Huabing Zhao * fix e2e Signed-off-by: Huabing Zhao * fix lint Signed-off-by: Huabing Zhao * refactor Signed-off-by: Huabing Zhao * add max failed attempts limit Signed-off-by: Huabing Zhao * remove retries Signed-off-by: Huabing Zhao * clean up e2e tests Signed-off-by: Huabing Zhao * add e2e test for wrong password Signed-off-by: Huabing Zhao * Update api/v1alpha1/authorization_types.go Co-authored-by: Arko Dasgupta Signed-off-by: Huabing Zhao * Update api/v1alpha1/wasm_types.go Co-authored-by: Arko Dasgupta Signed-off-by: Huabing Zhao * remove unnecessary replace Signed-off-by: Huabing Zhao * remove set package Signed-off-by: Huabing Zhao * fix gen check Signed-off-by: Huabing Zhao * add test for failed attempts Signed-off-by: Huabing Zhao * address comments Signed-off-by: Huabing Zhao * address comments Signed-off-by: Huabing Zhao * minor wording Signed-off-by: Huabing Zhao * move sha256 inside code source Signed-off-by: Huabing Zhao * address comments Signed-off-by: Huabing Zhao * fix e2e Signed-off-by: Huabing Zhao * fix flaky test Signed-off-by: Huabing Zhao * change comments Signed-off-by: Huabing Zhao * address comments Signed-off-by: Huabing Zhao * address comments Signed-off-by: Huabing Zhao * fail the eep translation if the wasm cache failed to start Signed-off-by: Huabing Zhao --------- Signed-off-by: Huabing Zhao Co-authored-by: Arko Dasgupta --- api/v1alpha1/authorization_types.go | 3 +- api/v1alpha1/wasm_types.go | 97 +- api/v1alpha1/zz_generated.deepcopy.go | 28 +- charts/gateway-helm/README.md | 9 +- ....envoyproxy.io_envoyextensionpolicies.yaml | 74 +- ...ateway.envoyproxy.io_securitypolicies.yaml | 5 +- charts/gateway-helm/values.tmpl.yaml | 5 +- go.mod | 89 +- go.sum | 225 ++-- .../translate/out/default-resources.all.yaml | 76 ++ .../out/from-gateway-api-to-xds.all.json | 65 ++ .../out/from-gateway-api-to-xds.all.yaml | 38 + .../from-gateway-api-to-xds.bootstrap.yaml | 38 + ...-single-route-single-match-to-xds.all.json | 65 ++ ...-single-route-single-match-to-xds.all.yaml | 38 + ...e-route-single-match-to-xds.bootstrap.yaml | 38 + internal/gatewayapi/envoyextensionpolicy.go | 205 +++- .../gatewayapi/envoyextensionpolicy_test.go | 66 ++ internal/gatewayapi/helpers.go | 8 + internal/gatewayapi/runner/runner.go | 124 ++- internal/gatewayapi/securitypolicy.go | 7 - .../backend-invalid-feature-disabled.out.yaml | 2 +- .../testdata/custom-filter-order.in.yaml | 4 +- .../testdata/custom-filter-order.out.yaml | 14 +- ...-extproc-invalid-no-matching-port.out.yaml | 2 +- ...licy-with-extproc-invalid-no-port.out.yaml | 2 +- ...xtproc-invalid-no-reference-grant.out.yaml | 4 +- ...y-with-extproc-invalid-no-service.out.yaml | 2 +- ...tensionpolicy-with-wasm-targetrefs.in.yaml | 5 +- ...ensionpolicy-with-wasm-targetrefs.out.yaml | 24 +- .../envoyextensionpolicy-with-wasm.in.yaml | 31 +- .../envoyextensionpolicy-with-wasm.out.yaml | 54 +- internal/gatewayapi/translator.go | 4 + internal/gatewayapi/translator_test.go | 24 + internal/gatewayapi/validate.go | 2 + .../proxy/testdata/daemonsets/custom.yaml | 38 + .../testdata/daemonsets/default-env.yaml | 38 + .../proxy/testdata/daemonsets/default.yaml | 38 + .../daemonsets/disable-prometheus.yaml | 38 + .../testdata/daemonsets/extension-env.yaml | 38 + .../override-labels-and-annotations.yaml | 38 + .../testdata/daemonsets/patch-daemonset.yaml | 38 + .../testdata/daemonsets/shutdown-manager.yaml | 38 + .../proxy/testdata/daemonsets/volumes.yaml | 38 + .../testdata/daemonsets/with-annotations.yaml | 38 + .../testdata/daemonsets/with-extra-args.yaml | 38 + .../daemonsets/with-image-pull-secrets.yaml | 38 + .../proxy/testdata/daemonsets/with-name.yaml | 38 + .../daemonsets/with-node-selector.yaml | 38 + .../with-topology-spread-constraints.yaml | 38 + .../proxy/testdata/deployments/custom.yaml | 38 + .../custom_with_initcontainers.yaml | 38 + .../testdata/deployments/default-env.yaml | 38 + .../proxy/testdata/deployments/default.yaml | 38 + .../deployments/disable-prometheus.yaml | 38 + .../testdata/deployments/extension-env.yaml | 38 + .../override-labels-and-annotations.yaml | 38 + .../deployments/patch-deployment.yaml | 38 + .../deployments/shutdown-manager.yaml | 38 + .../proxy/testdata/deployments/volumes.yaml | 38 + .../deployments/with-annotations.yaml | 38 + .../deployments/with-empty-memory-limits.yaml | 38 + .../testdata/deployments/with-extra-args.yaml | 38 + .../deployments/with-image-pull-secrets.yaml | 38 + .../proxy/testdata/deployments/with-name.yaml | 38 + .../deployments/with-node-selector.yaml | 38 + .../with-topology-spread-constraints.yaml | 38 + internal/ir/xds.go | 21 +- internal/ir/zz_generated.deepcopy.go | 19 +- internal/provider/kubernetes/controller.go | 19 + internal/provider/kubernetes/indexers.go | 26 + internal/provider/kubernetes/predicates.go | 17 + .../provider/kubernetes/predicates_test.go | 42 + internal/wasm/cache.go | 495 +++++++++ internal/wasm/cache_test.go | 993 ++++++++++++++++++ internal/wasm/httpfetcher.go | 224 ++++ internal/wasm/httpfetcher_test.go | 277 +++++ internal/wasm/httpserver.go | 247 +++++ internal/wasm/httpserver_test.go | 368 +++++++ internal/wasm/imagefetcher.go | 377 +++++++ internal/wasm/imagefetcher_test.go | 636 +++++++++++ internal/wasm/metrics.go | 54 + internal/wasm/options.go | 108 ++ internal/xds/bootstrap/bootstrap.go | 16 +- internal/xds/bootstrap/bootstrap.yaml.tpl | 38 + .../bootstrap/testdata/merge/default.out.yaml | 38 + .../testdata/merge/stats_sinks.out.yaml | 38 + .../testdata/render/custom-stats-matcher.yaml | 38 + .../testdata/render/disable-prometheus.yaml | 38 + .../enable-prometheus-gzip-compression.yaml | 38 + .../testdata/render/enable-prometheus.yaml | 38 + .../render/otel-metrics-backendref.yaml | 38 + .../testdata/render/otel-metrics.yaml | 38 + .../render/with-max-heap-size-bytes.yaml | 38 + .../in/xds-ir/custom-filter-order.yaml | 4 +- .../translator/testdata/in/xds-ir/wasm.yaml | 34 +- .../xds-ir/custom-filter-order.clusters.yaml | 36 - .../xds-ir/custom-filter-order.listeners.yaml | 8 +- .../testdata/out/xds-ir/wasm.clusters.yaml | 72 -- .../testdata/out/xds-ir/wasm.listeners.yaml | 49 +- .../testdata/out/xds-ir/wasm.routes.yaml | 9 +- internal/xds/translator/translator_test.go | 1 + internal/xds/translator/wasm.go | 43 +- site/content/en/latest/api/extension_types.md | 53 +- .../en/latest/install/gateway-helm-api.md | 9 +- site/content/zh/latest/api/extension_types.md | 53 +- .../zh/latest/install/gateway-helm-api.md | 9 +- .../testdata/{wasm.yaml => wasm-http.yaml} | 2 +- .../wasm-oci-registry-test-server.yaml | 57 + test/e2e/testdata/wasm-oci.yaml | 39 + test/e2e/testdata/wasm/Dockerfile | 3 + test/e2e/testdata/wasm/plugin.wasm | Bin 0 -> 59641 bytes test/e2e/tests/utils.go | 67 ++ test/e2e/tests/{wasm.go => wasm_http.go} | 10 +- test/e2e/tests/wasm_oci.go | 427 ++++++++ .../control-plane-with-pdb.out.yaml | 5 + .../helm/gateway-helm/default-config.out.yaml | 5 + .../deployment-custom-topology.out.yaml | 5 + .../deployment-images-config.out.yaml | 5 + .../envoy-gateway-config.out.yaml | 5 + .../global-images-config.out.yaml | 5 + tools/docker/envoy-gateway/Dockerfile | 5 + 122 files changed, 7500 insertions(+), 532 deletions(-) create mode 100644 internal/gatewayapi/envoyextensionpolicy_test.go create mode 100644 internal/wasm/cache.go create mode 100644 internal/wasm/cache_test.go create mode 100644 internal/wasm/httpfetcher.go create mode 100644 internal/wasm/httpfetcher_test.go create mode 100644 internal/wasm/httpserver.go create mode 100644 internal/wasm/httpserver_test.go create mode 100644 internal/wasm/imagefetcher.go create mode 100644 internal/wasm/imagefetcher_test.go create mode 100644 internal/wasm/metrics.go create mode 100644 internal/wasm/options.go rename test/e2e/testdata/{wasm.yaml => wasm-http.yaml} (93%) create mode 100644 test/e2e/testdata/wasm-oci-registry-test-server.yaml create mode 100644 test/e2e/testdata/wasm-oci.yaml create mode 100644 test/e2e/testdata/wasm/Dockerfile create mode 100644 test/e2e/testdata/wasm/plugin.wasm rename test/e2e/tests/{wasm.go => wasm_http.go} (93%) create mode 100644 test/e2e/tests/wasm_oci.go diff --git a/api/v1alpha1/authorization_types.go b/api/v1alpha1/authorization_types.go index e02ba4dafed..3a589daef9f 100644 --- a/api/v1alpha1/authorization_types.go +++ b/api/v1alpha1/authorization_types.go @@ -27,7 +27,8 @@ type Authorization struct { // AuthorizationRule defines a single authorization rule. type AuthorizationRule struct { - // Name is a user-friendly name for the rule. It's just for display purposes. + // Name is a user-friendly name for the rule. + // If not specified, Envoy Gateway will generate a unique name for the rule.n // +optional Name *string `json:"name,omitempty"` diff --git a/api/v1alpha1/wasm_types.go b/api/v1alpha1/wasm_types.go index 425c8e45892..1c41513f941 100644 --- a/api/v1alpha1/wasm_types.go +++ b/api/v1alpha1/wasm_types.go @@ -10,7 +10,7 @@ import ( gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" ) -// Wasm defines a wasm extension. +// Wasm defines a Wasm extension. // // Note: at the moment, Envoy Gateway does not support configuring Wasm runtime. // v8 is used as the VM runtime for the Wasm extensions. @@ -18,23 +18,19 @@ type Wasm struct { // Name is a unique name for this Wasm extension. It is used to identify the // Wasm extension if multiple extensions are handled by the same vm_id and root_id. // It's also used for logging/debugging. - Name string `json:"name"` - - // VMID is an ID that will be used along with a hash of the wasm code to - // determine which VM will be used to load the Wasm extension. All extensions - // that have the same vm_id and code will use the same VM. + // If not specified, EG will generate a unique name for the Wasm extension. // - // Note that sharing a VM between plugins can reduce memory utilization and - // make sharing of data easier, but it may have security implications. - // VMID *string `json:"vmID,omitempty"` + // +optional + Name *string `json:"name,omitempty"` // RootID is a unique ID for a set of extensions in a VM which will share a // RootContext and Contexts if applicable (e.g., an Wasm HttpFilter and an Wasm AccessLog). // If left blank, all extensions with a blank root_id with the same vm_id will share Context(s). - // RootID must match the root_id parameter used to register the Context in the Wasm code. + // + // Note: RootID must match the root_id parameter used to register the Context in the Wasm code. RootID *string `json:"rootID,omitempty"` - // Code is the wasm code for the extension. + // Code is the Wasm code for the extension. Code WasmCodeSource `json:"code"` // Config is the configuration for the Wasm extension. @@ -58,73 +54,100 @@ type Wasm struct { // Priority *uint32 `json:"priority,omitempty"` } -// WasmCodeSource defines the source of the wasm code. +// WasmCodeSource defines the source of the Wasm code. +// +union +// +// +kubebuilder:validation:XValidation:rule="self.type == 'HTTP' ? has(self.http) : !has(self.http)",message="If type is HTTP, http field needs to be set." +// +kubebuilder:validation:XValidation:rule="self.type == 'Image' ? has(self.image) : !has(self.image)",message="If type is Image, image field needs to be set." type WasmCodeSource struct { - // Type is the type of the source of the wasm code. + // Type is the type of the source of the Wasm code. // Valid WasmCodeSourceType values are "HTTP" or "Image". // // +kubebuilder:validation:Enum=HTTP;Image;ConfigMap // +unionDiscriminator Type WasmCodeSourceType `json:"type"` - // HTTP is the HTTP URL containing the wasm code. + // HTTP is the HTTP URL containing the Wasm code. // // Note that the HTTP server must be accessible from the Envoy proxy. // +optional HTTP *HTTPWasmCodeSource `json:"http,omitempty"` - // Image is the OCI image containing the wasm code. + // Image is the OCI image containing the Wasm code. // // Note that the image must be accessible from the Envoy Gateway. // +optional Image *ImageWasmCodeSource `json:"image,omitempty"` - // SHA256 checksum that will be used to verify the wasm code. + // PullPolicy is the policy to use when pulling the Wasm module by either the HTTP or Image source. + // This field is only applicable when the SHA256 field is not set. // - // kubebuilder:validation:Pattern=`^[a-f0-9]{64}$` - SHA256 string `json:"sha256"` + // If not specified, the default policy is IfNotPresent except for OCI images whose tag is latest. + // + // Note: EG does not update the Wasm module every time an Envoy proxy requests + // the Wasm module even if the pull policy is set to Always. + // It only updates the Wasm module when the EnvoyExtension resource version changes. + // +optional + PullPolicy *ImagePullPolicy `json:"pullPolicy,omitempty"` } -// WasmCodeSourceType specifies the types of sources for the wasm code. +// WasmCodeSourceType specifies the types of sources for the Wasm code. // +kubebuilder:validation:Enum=HTTP;Image type WasmCodeSourceType string const ( - // HTTPWasmCodeSourceType allows the user to specify the wasm code in an HTTP URL. + // HTTPWasmCodeSourceType allows the user to specify the Wasm code in an HTTP URL. HTTPWasmCodeSourceType WasmCodeSourceType = "HTTP" - // ImageWasmCodeSourceType allows the user to specify the wasm code in an OCI image. + // ImageWasmCodeSourceType allows the user to specify the Wasm code in an OCI image. ImageWasmCodeSourceType WasmCodeSourceType = "Image" ) -// HTTPWasmCodeSource defines the HTTP URL containing the wasm code. +// HTTPWasmCodeSource defines the HTTP URL containing the Wasm code. type HTTPWasmCodeSource struct { - // URL is the URL containing the wasm code. + // URL is the URL containing the Wasm code. + // +kubebuilder:validation:Pattern=`^((https?:)(\/\/\/?)([\w]*(?::[\w]*)?@)?([\d\w\.-]+)(?::(\d+))?)?([\/\\\w\.()-]*)?(?:([?][^#]*)?(#.*)?)*` URL string `json:"url"` + + // SHA256 checksum that will be used to verify the Wasm code. + // + // If not specified, Envoy Gateway will not verify the downloaded Wasm code. + // kubebuilder:validation:Pattern=`^[a-f0-9]{64}$` + // +optional + SHA256 *string `json:"sha256"` } -// ImageWasmCodeSource defines the OCI image containing the wasm code. +// ImageWasmCodeSource defines the OCI image containing the Wasm code. type ImageWasmCodeSource struct { // URL is the URL of the OCI image. + // URL can be in the format of `registry/image:tag` or `registry/image@sha256:digest`. URL string `json:"url"` - // PullSecretRef is a reference to the secret containing the credentials to pull the image. - PullSecretRef gwapiv1b1.SecretObjectReference `json:"pullSecret"` + // SHA256 checksum that will be used to verify the OCI image. + // + // It must match the digest of the OCI image. + // + // If not specified, Envoy Gateway will not verify the downloaded OCI image. + // kubebuilder:validation:Pattern=`^[a-f0-9]{64}$` + // +optional + SHA256 *string `json:"sha256"` - // PullPolicy is the policy to use when pulling the image. - // If not specified, the default policy is IfNotPresent for images whose tag is not latest, - // and Always for images whose tag is latest. + // PullSecretRef is a reference to the secret containing the credentials to pull the image. + // Only support Kubernetes Secret resource from the same namespace. + // +kubebuilder:validation:XValidation:message="only support Secret kind.",rule="self.kind == 'Secret'" // +optional - // PullPolicy *PullPolicy `json:"pullPolicy,omitempty"` + PullSecretRef *gwapiv1b1.SecretObjectReference `json:"pullSecretRef,omitempty"` } -// PullPolicy defines the policy to use when pulling an OIC image. -/* type PullPolicy string +// ImagePullPolicy defines the policy to use when pulling an OIC image. +// +kubebuilder:validation:Enum=IfNotPresent;Always +type ImagePullPolicy string const ( - // PullPolicyIfNotPresent will only pull the image if it does not already exist. - PullPolicyIfNotPresent PullPolicy = "IfNotPresent" + // ImagePullPolicyIfNotPresent will only pull the image if it does not already exist in the EG cache. + ImagePullPolicyIfNotPresent ImagePullPolicy = "IfNotPresent" - // PullPolicyAlways will always pull the image. - PullPolicyAlways PullPolicy = "Always" -)*/ + // ImagePullPolicyAlways will pull the image when the EnvoyExtension resource version changes. + // Note: EG does not update the Wasm module every time an Envoy proxy requests the Wasm module. + ImagePullPolicyAlways ImagePullPolicy = "Always" +) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 6d5db4529ba..56a090358ab 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -2565,6 +2565,11 @@ func (in *HTTPTimeout) DeepCopy() *HTTPTimeout { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HTTPWasmCodeSource) DeepCopyInto(out *HTTPWasmCodeSource) { *out = *in + if in.SHA256 != nil { + in, out := &in.SHA256, &out.SHA256 + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPWasmCodeSource. @@ -2715,7 +2720,16 @@ func (in *IPEndpoint) DeepCopy() *IPEndpoint { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ImageWasmCodeSource) DeepCopyInto(out *ImageWasmCodeSource) { *out = *in - in.PullSecretRef.DeepCopyInto(&out.PullSecretRef) + if in.SHA256 != nil { + in, out := &in.SHA256, &out.SHA256 + *out = new(string) + **out = **in + } + if in.PullSecretRef != nil { + in, out := &in.PullSecretRef, &out.PullSecretRef + *out = new(apisv1.SecretObjectReference) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageWasmCodeSource. @@ -4785,6 +4799,11 @@ func (in *UnixSocket) DeepCopy() *UnixSocket { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Wasm) DeepCopyInto(out *Wasm) { *out = *in + if in.Name != nil { + in, out := &in.Name, &out.Name + *out = new(string) + **out = **in + } if in.RootID != nil { in, out := &in.RootID, &out.RootID *out = new(string) @@ -4819,13 +4838,18 @@ func (in *WasmCodeSource) DeepCopyInto(out *WasmCodeSource) { if in.HTTP != nil { in, out := &in.HTTP, &out.HTTP *out = new(HTTPWasmCodeSource) - **out = **in + (*in).DeepCopyInto(*out) } if in.Image != nil { in, out := &in.Image, &out.Image *out = new(ImageWasmCodeSource) (*in).DeepCopyInto(*out) } + if in.PullPolicy != nil { + in, out := &in.PullPolicy, &out.PullPolicy + *out = new(ImagePullPolicy) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WasmCodeSource. diff --git a/charts/gateway-helm/README.md b/charts/gateway-helm/README.md index 3e034e83519..da3f7572bd1 100644 --- a/charts/gateway-helm/README.md +++ b/charts/gateway-helm/README.md @@ -84,9 +84,12 @@ To uninstall the chart: | deployment.ports[1].name | string | `"ratelimit"` | | | deployment.ports[1].port | int | `18001` | | | deployment.ports[1].targetPort | int | `18001` | | -| deployment.ports[2].name | string | `"metrics"` | | -| deployment.ports[2].port | int | `19001` | | -| deployment.ports[2].targetPort | int | `19001` | | +| deployment.ports[2].name | string | `"wasm"` | | +| deployment.ports[2].port | int | `18002` | | +| deployment.ports[2].targetPort | int | `18002` | | +| deployment.ports[3].name | string | `"metrics"` | | +| deployment.ports[3].port | int | `19001` | | +| deployment.ports[3].targetPort | int | `19001` | | | deployment.replicas | int | `1` | | | global.images.envoyGateway.image | string | `nil` | | | global.images.envoyGateway.pullPolicy | string | `nil` | | diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyextensionpolicies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyextensionpolicies.yaml index 8c5d032d496..92a0c9defba 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyextensionpolicies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyextensionpolicies.yaml @@ -308,38 +308,48 @@ spec: defined in this list. items: description: |- - Wasm defines a wasm extension. + Wasm defines a Wasm extension. Note: at the moment, Envoy Gateway does not support configuring Wasm runtime. v8 is used as the VM runtime for the Wasm extensions. properties: code: - description: Code is the wasm code for the extension. + description: Code is the Wasm code for the extension. properties: http: description: |- - HTTP is the HTTP URL containing the wasm code. + HTTP is the HTTP URL containing the Wasm code. Note that the HTTP server must be accessible from the Envoy proxy. properties: + sha256: + description: |- + SHA256 checksum that will be used to verify the Wasm code. + + + If not specified, Envoy Gateway will not verify the downloaded Wasm code. + kubebuilder:validation:Pattern=`^[a-f0-9]{64}$` + type: string url: - description: URL is the URL containing the wasm code. + description: URL is the URL containing the Wasm code. + pattern: ^((https?:)(\/\/\/?)([\w]*(?::[\w]*)?@)?([\d\w\.-]+)(?::(\d+))?)?([\/\\\w\.()-]*)?(?:([?][^#]*)?(#.*)?)* type: string required: - url type: object image: description: |- - Image is the OCI image containing the wasm code. + Image is the OCI image containing the Wasm code. Note that the image must be accessible from the Envoy Gateway. properties: - pullSecret: - description: PullSecretRef is a reference to the secret - containing the credentials to pull the image. + pullSecretRef: + description: |- + PullSecretRef is a reference to the secret containing the credentials to pull the image. + Only support Kubernetes Secret resource from the same namespace. properties: group: default: "" @@ -382,19 +392,43 @@ spec: required: - name type: object + x-kubernetes-validations: + - message: only support Secret kind. + rule: self.kind == 'Secret' + sha256: + description: |- + SHA256 checksum that will be used to verify the OCI image. + + + It must match the digest of the OCI image. + + + If not specified, Envoy Gateway will not verify the downloaded OCI image. + kubebuilder:validation:Pattern=`^[a-f0-9]{64}$` + type: string url: - description: URL is the URL of the OCI image. + description: |- + URL is the URL of the OCI image. + URL can be in the format of `registry/image:tag` or `registry/image@sha256:digest`. type: string required: - - pullSecret - url type: object - sha256: + pullPolicy: description: |- - SHA256 checksum that will be used to verify the wasm code. + PullPolicy is the policy to use when pulling the Wasm module by either the HTTP or Image source. + This field is only applicable when the SHA256 field is not set. + + If not specified, the default policy is IfNotPresent except for OCI images whose tag is latest. - kubebuilder:validation:Pattern=`^[a-f0-9]{64}$` + + Note: EG does not update the Wasm module every time an Envoy proxy requests + the Wasm module even if the pull policy is set to Always. + It only updates the Wasm module when the EnvoyExtension resource version changes. + enum: + - IfNotPresent + - Always type: string type: allOf: @@ -406,13 +440,17 @@ spec: - Image - ConfigMap description: |- - Type is the type of the source of the wasm code. + Type is the type of the source of the Wasm code. Valid WasmCodeSourceType values are "HTTP" or "Image". type: string required: - - sha256 - type type: object + x-kubernetes-validations: + - message: If type is HTTP, http field needs to be set. + rule: 'self.type == ''HTTP'' ? has(self.http) : !has(self.http)' + - message: If type is Image, image field needs to be set. + rule: 'self.type == ''Image'' ? has(self.image) : !has(self.image)' config: description: |- Config is the configuration for the Wasm extension. @@ -433,17 +471,19 @@ spec: Name is a unique name for this Wasm extension. It is used to identify the Wasm extension if multiple extensions are handled by the same vm_id and root_id. It's also used for logging/debugging. + If not specified, EG will generate a unique name for the Wasm extension. type: string rootID: description: |- RootID is a unique ID for a set of extensions in a VM which will share a RootContext and Contexts if applicable (e.g., an Wasm HttpFilter and an Wasm AccessLog). If left blank, all extensions with a blank root_id with the same vm_id will share Context(s). - RootID must match the root_id parameter used to register the Context in the Wasm code. + + + Note: RootID must match the root_id parameter used to register the Context in the Wasm code. type: string required: - code - - name type: object maxItems: 16 type: array diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_securitypolicies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_securitypolicies.yaml index 28bb861f56c..29c2cb1352a 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_securitypolicies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_securitypolicies.yaml @@ -81,8 +81,9 @@ spec: - Deny type: string name: - description: Name is a user-friendly name for the rule. - It's just for display purposes. + description: |- + Name is a user-friendly name for the rule. + If not specified, Envoy Gateway will generate a unique name for the rule.n type: string principal: description: Principal specifies the client identity of diff --git a/charts/gateway-helm/values.tmpl.yaml b/charts/gateway-helm/values.tmpl.yaml index 497c136a80b..9240c4c2c13 100644 --- a/charts/gateway-helm/values.tmpl.yaml +++ b/charts/gateway-helm/values.tmpl.yaml @@ -41,6 +41,9 @@ deployment: - name: ratelimit port: 18001 targetPort: 18001 + - name: wasm + port: 18002 + targetPort: 18002 - name: metrics port: 19001 targetPort: 19001 @@ -76,4 +79,4 @@ certgen: ttlSecondsAfterFinished: 30 rbac: annotations: {} - labels: {} \ No newline at end of file + labels: {} diff --git a/go.mod b/go.mod index 4154134d6b0..9d4f56f40d1 100644 --- a/go.mod +++ b/go.mod @@ -2,12 +2,15 @@ module github.com/envoyproxy/gateway go 1.22.4 +replace github.com/imdario/mergo => github.com/imdario/mergo v0.3.16 + require ( fortio.org/fortio v1.65.0 fortio.org/log v1.12.2 github.com/Masterminds/semver/v3 v3.2.1 github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b - github.com/davecgh/go-spew v1.1.1 + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc + github.com/docker/cli v26.1.3+incompatible github.com/dominikbraun/graph v0.23.0 github.com/envoyproxy/go-control-plane v0.12.1-0.20240612043845-c54ec4ce422d github.com/envoyproxy/ratelimit v1.4.1-0.20230427142404-e2a87f41d3a7 @@ -19,7 +22,9 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/golang/protobuf v1.5.4 github.com/google/go-cmp v0.6.0 + github.com/google/go-containerregistry v0.19.1 github.com/grafana/tempo v1.5.0 + github.com/hashicorp/go-multierror v1.1.1 github.com/miekg/dns v1.1.61 github.com/prometheus/client_golang v1.19.1 github.com/prometheus/common v0.54.0 @@ -49,13 +54,15 @@ require ( k8s.io/cli-runtime v0.30.2 k8s.io/client-go v0.30.2 k8s.io/kubectl v0.30.2 - k8s.io/utils v0.0.0-20240423183400-0849a56e8f22 + k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 sigs.k8s.io/controller-runtime v0.18.4 sigs.k8s.io/gateway-api v1.1.0 sigs.k8s.io/mcs-api v0.1.0 sigs.k8s.io/yaml v1.4.0 ) +require github.com/docker/docker v26.1.3+incompatible + require ( cel.dev/expr v0.15.0 // indirect fortio.org/cli v1.6.0 // indirect @@ -65,32 +72,33 @@ require ( fortio.org/struct2env v0.4.1 // indirect fortio.org/version v1.0.4 // indirect github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect - github.com/BurntSushi/toml v1.3.2 // indirect + github.com/BurntSushi/toml v1.4.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/sprig/v3 v3.2.3 // indirect github.com/Masterminds/squirrel v1.5.4 // indirect - github.com/Microsoft/hcsshim v0.11.4 // indirect - github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect - github.com/containerd/containerd v1.7.12 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/Microsoft/hcsshim v0.12.3 // indirect + github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/containerd/containerd v1.7.17 // indirect github.com/containerd/log v0.1.0 // indirect - github.com/cyphar/filepath-securejoin v0.2.4 // indirect - github.com/distribution/reference v0.5.0 // indirect - github.com/docker/cli v25.0.1+incompatible // indirect + github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect + github.com/cyphar/filepath-securejoin v0.2.5 // indirect + github.com/distribution/reference v0.6.0 // indirect github.com/docker/distribution v2.8.3+incompatible // indirect - github.com/docker/docker v25.0.5+incompatible // indirect - github.com/docker/docker-credential-helpers v0.7.0 // indirect + github.com/docker/docker-credential-helpers v0.8.2 // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-metrics v0.0.1 // indirect - github.com/felixge/httpsnoop v1.0.3 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/gosuri/uitable v0.0.4 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/huandu/xstrings v1.4.0 // indirect github.com/jmoiron/sqlx v1.3.5 // indirect - github.com/klauspost/compress v1.16.0 // indirect + github.com/klauspost/compress v1.17.2 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect github.com/lib/pq v1.10.9 // indirect @@ -98,60 +106,67 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/locker v1.0.1 // indirect + github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/sys/sequential v0.5.0 // indirect + github.com/moby/sys/user v0.1.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0-rc6 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect - github.com/rivo/uniseg v0.2.0 // indirect - github.com/rubenv/sql-migrate v1.5.2 // indirect - github.com/shopspring/decimal v1.3.1 // indirect - github.com/spf13/cast v1.5.0 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/rubenv/sql-migrate v1.6.1 // indirect + github.com/shopspring/decimal v1.4.0 // indirect + github.com/spf13/cast v1.6.0 // indirect + github.com/vbatts/tar-split v0.11.5 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect golang.org/x/crypto v0.24.0 // indirect golang.org/x/crypto/x509roots/fallback v0.0.0-20240604170348-d4e7c9cb6cb8 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect k8s.io/apiserver v0.30.2 // indirect oras.land/oras-go v1.2.5 // indirect ) require ( - github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/chai2010/gettext-go v1.0.2 // indirect - github.com/emicklei/go-restful/v3 v3.12.0 // indirect + github.com/chai2010/gettext-go v1.0.3 // indirect + github.com/emicklei/go-restful/v3 v3.12.1 // indirect github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect github.com/evanphx/json-patch v5.9.0+incompatible - github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect + github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect - github.com/go-errors/errors v1.4.2 // indirect + github.com/go-errors/errors v1.5.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/google/btree v1.0.1 // indirect + github.com/google/btree v1.1.2 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.1 // indirect - github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect + github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect - github.com/imdario/mergo v0.3.16 // indirect + github.com/imdario/mergo v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kelseyhightower/envconfig v1.4.0 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect - github.com/lyft/gostats v0.4.1 // indirect + github.com/lyft/gostats v0.4.14 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/moby/spdystream v0.2.0 // indirect @@ -163,35 +178,35 @@ require ( github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pkg/errors v0.9.1 - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.1 - github.com/prometheus/procfs v0.15.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/tsaarni/x500dn v1.0.0 // indirect github.com/xlab/treeprint v1.2.0 // indirect go.opentelemetry.io/otel/sdk v1.27.0 go.opentelemetry.io/otel/trace v1.27.0 // indirect - go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect + go.starlark.net v0.0.0-20240520160348-046347dcd104 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/mod v0.18.0 // indirect golang.org/x/net v0.26.0 - golang.org/x/oauth2 v0.20.0 // indirect + golang.org/x/oauth2 v0.21.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/term v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.22.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/component-base v0.30.2 // indirect k8s.io/klog/v2 v2.120.1 // indirect - k8s.io/kube-openapi v0.0.0-20240423202451-8948a665c108 // indirect + k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect - sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect + sigs.k8s.io/kustomize/api v0.17.2 // indirect + sigs.k8s.io/kustomize/kyaml v0.17.1 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) diff --git a/go.sum b/go.sum index ef11b990ea7..83196a0ef50 100644 --- a/go.sum +++ b/go.sum @@ -24,8 +24,8 @@ fortio.org/version v1.0.4/go.mod h1:2JQp9Ax+tm6QKiGuzR5nJY63kFeANcgrZ0osoQFDVm0= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= @@ -34,8 +34,8 @@ github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxB github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= -github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= +github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= @@ -49,10 +49,10 @@ github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM= github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= -github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= -github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8= -github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/Microsoft/hcsshim v0.12.3 h1:LS9NXqXhMoqNCplK1ApmVSfB4UnVLRDWRapB6EIlxE0= +github.com/Microsoft/hcsshim v0.12.3/go.mod h1:Iyl1WVpZzr+UkzjekHZbV8o5Z9ZkxNGx6CtY2Qg/JVQ= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -72,14 +72,16 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY= -github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bshuster-repo/logrus-logstash-hook v1.0.0 h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70= github.com/bshuster-repo/logrus-logstash-hook v1.0.0/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd h1:rFt+Y/IK1aEZkEHchZRSq9OQbsSzIT/OrI8YFFmRIng= @@ -96,23 +98,25 @@ github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= -github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/chai2010/gettext-go v1.0.3 h1:9liNh8t+u26xl5ddmWLmsOsdNLwkdRTg5AG+JnTiM80= +github.com/chai2010/gettext-go v1.0.3/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b h1:ga8SEFjZ60pxLcmhnThWgvH2wg8376yUJmPhEH4H3kw= github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= -github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= -github.com/containerd/containerd v1.7.12 h1:+KQsnv4VnzyxWcfO9mlxxELaoztsDEjOuCMPAuPqgU0= -github.com/containerd/containerd v1.7.12/go.mod h1:/5OMpE1p0ylxtEUGY8kuCYkDRzJm9NO1TFMWjUpdevk= +github.com/containerd/cgroups/v3 v3.0.2 h1:f5WFqIVSgo5IZmtTT3qVBo6TzI1ON6sycSBKkymb9L0= +github.com/containerd/cgroups/v3 v3.0.2/go.mod h1:JUgITrzdFqp42uI2ryGA+ge0ap/nxzYgkGmIcetmErE= +github.com/containerd/containerd v1.7.17 h1:KjNnn0+tAVQHAoaWRjmdak9WlvnFR/8rU1CHHy8Rm2A= +github.com/containerd/containerd v1.7.17/go.mod h1:vK+hhT4TIv2uejlcDlbVIc8+h/BqtKLIyNrtCZol8lI= github.com/containerd/continuity v0.4.2 h1:v3y/4Yz5jwnvqPKJJ+7Wf93fyWoCB3F5EclWG023MDM= github.com/containerd/continuity v0.4.2/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= +github.com/containerd/errdefs v0.1.0 h1:m0wCRBiu1WJT/Fr+iOoQHMQS/eP5myQ8lCv4Dz5ZURM= +github.com/containerd/errdefs v0.1.0/go.mod h1:YgWiiHtLmSeBrvpw+UfPijzbLaB77mEG1WwJTDETIV0= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/stargz-snapshotter/estargz v0.15.1 h1:eXJjw9RbkLFgioVaTG+G/ZW/0kEe2oEKCdS/ZxIyoCU= +github.com/containerd/stargz-snapshotter/estargz v0.15.1/go.mod h1:gr2RNwukQ/S9Nv33Lt6UC7xEx58C+LHRdoqbEKjz1Kk= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= @@ -130,28 +134,29 @@ github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= -github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= -github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= +github.com/cyphar/filepath-securejoin v0.2.5 h1:6iR5tXJ/e6tJZzzdMc1km3Sa7RRIVBKAK32O2s7AYfo= +github.com/cyphar/filepath-securejoin v0.2.5/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/datawire/dlib v1.3.0 h1:KkmyXU1kwm3oPBk1ypR70YbcOlEXWzEbx5RE0iRXTGk= github.com/datawire/dlib v1.3.0/go.mod h1:NiGDmetmbkBvtznpWSx6C0vA0s0LK9aHna3LJDqjruk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2 h1:aBfCb7iqHmDEIp6fBvC/hQUddQfg+3qdYjwzaiP9Hnc= github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2/go.mod h1:WHNsWjnIn2V1LYOrME7e8KxSeKunYHsxEm4am0BUtcI= -github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= -github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/cli v25.0.1+incompatible h1:mFpqnrS6Hsm3v1k7Wa/BO23oz0k121MTbTO1lpcGSkU= -github.com/docker/cli v25.0.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/cli v26.1.3+incompatible h1:bUpXT/N0kDE3VUHI2r5VMsYQgi38kYuoC0oL9yt3lqc= +github.com/docker/cli v26.1.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v25.0.5+incompatible h1:UmQydMduGkrD5nQde1mecF/YnSbTOaPeFIeP5C4W+DE= -github.com/docker/docker v25.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= -github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= +github.com/docker/docker v26.1.3+incompatible h1:lLCzRbrVZrljpVNobJu1J2FHk8V0s4BawoZippkc+xo= +github.com/docker/docker v26.1.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker-credential-helpers v0.8.2 h1:bX3YxiGzFP5sOXWc3bTPEXdEaZSeVMrFgOr3T+zrFAo= +github.com/docker/docker-credential-helpers v0.8.2/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= @@ -160,6 +165,8 @@ github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQ github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 h1:ZClxb8laGDf5arXfYcAtECDFgAgHklGI8CxgjHnXKJ4= github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= @@ -171,8 +178,8 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful/v3 v3.12.0 h1:y2DdzBAURM29NFF94q6RaY4vjIH1rtwDapwQtU84iWk= -github.com/emicklei/go-restful/v3 v3.12.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= +github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.12.1-0.20240612043845-c54ec4ce422d h1:RopQsG28t61pLLZRkwzwBsi60yDsOP8RvW47A3eAcGo= github.com/envoyproxy/go-control-plane v0.12.1-0.20240612043845-c54ec4ce422d/go.mod h1:5Wkq+JduFtdAXihLmeTJf+tRYIT4KBc2vPXDhwVo1pA= @@ -188,17 +195,17 @@ github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/evanphx/json-patch/v5 v5.0.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= -github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= -github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= +github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4= +github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= -github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= -github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6FI= github.com/foxcpp/go-mockdns v1.0.0/go.mod h1:lgRN6+KxQBawyIghpnl5CezHFGS9VLzvtVlwxvzXTQ4= -github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= -github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= @@ -209,8 +216,8 @@ github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD50WnA= github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= -github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= -github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk= +github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs= github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpjH8TixEw= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -280,14 +287,9 @@ github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfC github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gobuffalo/flect v0.2.0/go.mod h1:W3K3X9ksuZfir8f/LrfVtWmCDQFfayuylOJ7sz/Fj80= -github.com/gobuffalo/logger v1.0.6 h1:nnZNpxYo0zx+Aj9RfMPBm+x9zAU2OayFh/xrAWi34HU= -github.com/gobuffalo/logger v1.0.6/go.mod h1:J31TBEHR1QLV2683OXTAItYIg8pv2JMHnF/quuAbMjs= -github.com/gobuffalo/packd v1.0.1 h1:U2wXfRr4E9DH8IdsDLlRFwTZTK7hLfq9qT/QHXGVe/0= -github.com/gobuffalo/packd v1.0.1/go.mod h1:PP2POP3p3RXGz7Jh6eYEf93S7vA2za6xM7QT85L4+VY= -github.com/gobuffalo/packr/v2 v2.8.3 h1:xE1yzvnO56cUC0sTpKR3DIbxZgB54AftTFMhB2XEWlY= -github.com/gobuffalo/packr/v2 v2.8.3/go.mod h1:0SahksCVcx4IMnigTjiFuyldmTrdTctXsOdiU5KwbKc= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -311,7 +313,6 @@ github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:x github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= @@ -319,27 +320,27 @@ github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k github.com/gomodule/redigo v1.8.2/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= +github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-containerregistry v0.19.1 h1:yMQ62Al6/V0Z7CqIrrS1iYoA5/oQCm88DeNujc7C1KY= +github.com/google/go-containerregistry v0.19.1/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -364,8 +365,9 @@ github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= github.com/grafana/tempo v1.5.0 h1:JSwulLVtXvUw2MyuUPcvRg3MJiwTUs5XWnbG6fOKatc= github.com/grafana/tempo v1.5.0/go.mod h1:IB52YU6zkGL+3t0eNrY8kAExx0lLa4LH20wGu3c4wD8= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= +github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= @@ -387,9 +389,6 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU= github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= @@ -411,16 +410,14 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= -github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= -github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= +github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -440,8 +437,8 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= -github.com/lyft/gostats v0.4.1 h1:oR6p4HRCGxt0nUntmZIWmYMgyothBi3eZH2A71vRjsc= -github.com/lyft/gostats v0.4.1/go.mod h1:Tpx2xRzz4t+T2Tx0xdVgIoBdR2UMVz+dKnE3X01XSd8= +github.com/lyft/gostats v0.4.14 h1:xmP4yMfDvEKtlNZEcS2sYz0cvnps1ri337ZEEbw3ab8= +github.com/lyft/gostats v0.4.14/go.mod h1:cJWqEVL8JIewIJz/olUIios2F1q06Nc51hXejPQmBH0= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -451,12 +448,6 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/markbates/errx v1.1.0 h1:QDFeR+UP95dO12JgW+tgi2UVfo0V8YBHiUIOaeBPiEI= -github.com/markbates/errx v1.1.0/go.mod h1:PLa46Oex9KNbVDZhKel8v1OT7hD5JZ2eI7AHhA0wswc= -github.com/markbates/oncer v1.0.0 h1:E83IaVAHygyndzPimgUYJjbshhDTALZyXxvk9FOlQRY= -github.com/markbates/oncer v1.0.0/go.mod h1:Z59JA581E9GP6w96jai+TGqafHPW+cPfRxz2aSZ0mcI= -github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI= -github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -471,14 +462,15 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= -github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= -github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI= +github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.1.61 h1:nLxbwF3XxhwVSm8g9Dghm9MHPaUZuqhPiGL+675ZmEs= github.com/miekg/dns v1.1.61/go.mod h1:mnAarhS3nWaW+NVP2wTkYVIZyHNJ098SJZUki3eykwQ= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= @@ -486,12 +478,20 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= +github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= +github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= +github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= +github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -503,6 +503,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= @@ -520,19 +522,19 @@ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo/v2 v2.17.1 h1:V++EzdbhI4ZV4ev0UTIj0PzhzOcReJFyJaLjtSF55M8= -github.com/onsi/ginkgo/v2 v2.17.1/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= +github.com/onsi/ginkgo/v2 v2.17.2 h1:7eMhcy3GimbsA3hEnVKdw/PQM9XN9krpKVXsZdph0/g= +github.com/onsi/ginkgo/v2 v2.17.2/go.mod h1:nP2DPOQoNsQmsVyv5rDA8JkXQoCs6goXIvr/PRJ1eCc= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk= -github.com/onsi/gomega v1.32.0/go.mod h1:a4x4gW6Pz2yK1MAmvluYme5lvYTn61afQ2ETw/8n4Lg= +github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= +github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0-rc6 h1:XDqvyKsJEbRtATzkgItUqBA7QHk58yxX1Ov9HERHNqU= -github.com/opencontainers/image-spec v1.1.0-rc6/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= @@ -546,8 +548,9 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY= github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= @@ -574,26 +577,27 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.15.0 h1:A82kmvXJq2jTu5YUhSGNlYoxh85zLnKgPz4bMZgI5Ek= -github.com/prometheus/procfs v0.15.0/go.mod h1:Y0RJ/Y5g5wJpkTisOtqwDSo4HwhGmLB4VQSw2sQJLHk= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/rubenv/sql-migrate v1.5.2 h1:bMDqOnrJVV/6JQgQ/MxOpU+AdO8uzYYA/TxFUBzFtS0= -github.com/rubenv/sql-migrate v1.5.2/go.mod h1:H38GW8Vqf8F0Su5XignRyaRcbXbJunSWxs+kmzlg0Is= +github.com/rubenv/sql-migrate v1.6.1 h1:bo6/sjsan9HaXAsNxYP/jCEDUGibHp8JmOBw7NTGRos= +github.com/rubenv/sql-migrate v1.6.1/go.mod h1:tPzespupJS0jacLfhbwto/UjSX+8h2FdWB7ar+QlHa0= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= +github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= -github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -605,8 +609,8 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= -github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= @@ -629,6 +633,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= @@ -646,6 +651,8 @@ github.com/tsaarni/x500dn v1.0.0/go.mod h1:QaHa3EcUKC4dfCAZmj8+ZRGLKukWgpGv9H3oO github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/vbatts/tar-split v0.11.5 h1:3bHCTIheBm1qFTcgh9oPu+nNBtX+XJIupG/vacinCts= +github.com/vbatts/tar-split v0.11.5/go.mod h1:yZbwRsSeGjusneWgA781EKej9HF8vme8okylkAeNKLk= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= @@ -684,6 +691,10 @@ go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.27.0 h1:bFg go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.27.0/go.mod h1:xJntEd2KL6Qdg5lwp97HMLQDVeAhrYxmzFseAMDPQ8I= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.27.0 h1:CIHWikMsN3wO+wq1Tp5VGdVRTcON+DmOJSfDjXypKOc= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.27.0/go.mod h1:TNupZ6cxqyFEpLXAZW7On+mLFL0/g0TE3unIYL91xWc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= go.opentelemetry.io/otel/exporters/prometheus v0.49.0 h1:Er5I1g/YhfYv9Affk9nJLfH/+qCCVVg1f2R9AbJfqDQ= go.opentelemetry.io/otel/exporters/prometheus v0.49.0/go.mod h1:KfQ1wpjf3zsHjzP149P4LyAwWRupc6c7t1ZJ9eXpKQM= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.27.0 h1:/jlt1Y8gXWiHG9FBx6cJaIC5hYx5Fe64nC8w5Cylt/0= @@ -698,8 +709,8 @@ go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5 go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= -go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY= -go.starlark.net v0.0.0-20230525235612-a134d8f9ddca/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= +go.starlark.net v0.0.0-20240520160348-046347dcd104 h1:3qhteRISupnJvaWshOmeqEUs2y9oc/+/ePPvDh3Eygg= +go.starlark.net v0.0.0-20240520160348-046347dcd104/go.mod h1:YKMCv9b1WrfWmeqdV5MAuEHWsu5iC+fe6kYl2sQjdI8= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -767,8 +778,8 @@ golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= -golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -817,7 +828,6 @@ golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= @@ -869,9 +879,8 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 h1:P8OJ/WCl/Xo4E4zoe4/bifHpSmmKwARqyqE4nW6J2GQ= -google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5/go.mod h1:RGnPtTG7r4i8sPlNyDeikXF99hMM+hN6QMm4ooG9g2g= +google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw= +google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU= google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 h1:Di6ANFilr+S60a4S61ZM00vLdw0IrQOSMS2/6mrnOU0= google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -879,7 +888,6 @@ google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -887,10 +895,7 @@ google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -900,6 +905,8 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= @@ -967,14 +974,14 @@ k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= -k8s.io/kube-openapi v0.0.0-20240423202451-8948a665c108 h1:Q8Z7VlGhcJgBHJHYugJ/K/7iB8a2eSxCyxdVjJp+lLY= -k8s.io/kube-openapi v0.0.0-20240423202451-8948a665c108/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a h1:zD1uj3Jf+mD4zmA7W+goE5TxDkI7OGJjBNBzq5fJtLA= +k8s.io/kube-openapi v0.0.0-20240521193020-835d969ad83a/go.mod h1:UxDHUPsUwTOOxSU+oXURfFBcAS6JwiRXTYqYwfuGowc= k8s.io/kubectl v0.30.2 h1:cgKNIvsOiufgcs4yjvgkK0+aPCfa8pUwzXdJtkbhsH8= k8s.io/kubectl v0.30.2/go.mod h1:rz7GHXaxwnigrqob0lJsiA07Df8RE3n1TSaC2CTeuB4= k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20200603063816-c1c6865ac451/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20240423183400-0849a56e8f22 h1:ao5hUqGhsqdm+bYbjH/pRkCs0unBGe9UyDahzs9zQzQ= -k8s.io/utils v0.0.0-20240423183400-0849a56e8f22/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak= +k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= oras.land/oras-go v1.2.5 h1:XpYuAwAb0DfQsunIyMfeET92emK8km3W4yEzZvUbsTo= oras.land/oras-go v1.2.5/go.mod h1:PuAwRShRZCsZb7g8Ar3jKKQR/2A/qN+pkYxIOd/FAoo= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0= @@ -987,10 +994,10 @@ sigs.k8s.io/gateway-api v1.1.0/go.mod h1:ZH4lHrL2sDi0FHZ9jjneb8kKnGzFWyrTya35sWU sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/kind v0.8.1/go.mod h1:oNKTxUVPYkV9lWzY6CVMNluVq8cBsyq+UgPJdvA3uu4= -sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 h1:XX3Ajgzov2RKUdc5jW3t5jwY7Bo7dcRm+tFxT+NfgY0= -sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3/go.mod h1:9n16EZKMhXBNSiUC5kSdFQJkdH3zbxS/JoO619G1VAY= -sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 h1:W6cLQc5pnqM7vh3b7HvGNfXrJ/xL6BDMS0v1V/HHg5U= -sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3/go.mod h1:JWP1Fj0VWGHyw3YUPjXSQnRnrwezrZSrApfX5S0nIag= +sigs.k8s.io/kustomize/api v0.17.2 h1:E7/Fjk7V5fboiuijoZHgs4aHuexi5Y2loXlVOAVAG5g= +sigs.k8s.io/kustomize/api v0.17.2/go.mod h1:UWTz9Ct+MvoeQsHcJ5e+vziRRkwimm3HytpZgIYqye0= +sigs.k8s.io/kustomize/kyaml v0.17.1 h1:TnxYQxFXzbmNG6gOINgGWQt09GghzgTP6mIurOgrLCQ= +sigs.k8s.io/kustomize/kyaml v0.17.1/go.mod h1:9V0mCjIEYjlXuCdYsSXvyoy2BTsLESH7TlGV81S282U= sigs.k8s.io/mcs-api v0.1.0 h1:edDbg0oRGfXw8TmZjKYep06LcJLv/qcYLidejnUp0PM= sigs.k8s.io/mcs-api v0.1.0/go.mod h1:gGiAryeFNB4GBsq2LBmVqSgKoobLxt+p7ii/WG5QYYw= sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= diff --git a/internal/cmd/egctl/testdata/translate/out/default-resources.all.yaml b/internal/cmd/egctl/testdata/translate/out/default-resources.all.yaml index 1ef7ecd9163..f1bc3b8eb15 100644 --- a/internal/cmd/egctl/testdata/translate/out/default-resources.all.yaml +++ b/internal/cmd/egctl/testdata/translate/out/default-resources.all.yaml @@ -130,6 +130,44 @@ envoyProxyForGatewayClass: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: @@ -600,6 +638,44 @@ xds: connectionKeepalive: interval: 30s timeout: 5s + - connectTimeout: 10s + loadAssignment: + clusterName: wasm_cluster + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: envoy-gateway + portValue: 18002 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + name: wasm_cluster + transportSocket: + name: envoy.transport_sockets.tls + typedConfig: + '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + commonTlsContext: + tlsCertificateSdsSecretConfigs: + - name: xds_certificate + sdsConfig: + pathConfigSource: + path: /sds/xds-certificate.json + resourceApiVersion: V3 + tlsParams: + tlsMaximumProtocolVersion: TLSv1_3 + validationContextSdsSecretConfig: + name: xds_trusted_ca + sdsConfig: + pathConfigSource: + path: /sds/xds-trusted-ca.json + resourceApiVersion: V3 + type: STRICT_DNS + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicitHttpConfig: + http2ProtocolOptions: {} listeners: - address: socketAddress: diff --git a/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.json b/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.json index 34264fa6150..a68fdc77509 100644 --- a/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.json +++ b/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.json @@ -163,6 +163,71 @@ } } } + }, + { + "connectTimeout": "10s", + "loadAssignment": { + "clusterName": "wasm_cluster", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "envoy-gateway", + "portValue": 18002 + } + } + }, + "loadBalancingWeight": 1 + } + ], + "loadBalancingWeight": 1 + } + ] + }, + "name": "wasm_cluster", + "transportSocket": { + "name": "envoy.transport_sockets.tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsCertificateSdsSecretConfigs": [ + { + "name": "xds_certificate", + "sdsConfig": { + "pathConfigSource": { + "path": "/sds/xds-certificate.json" + }, + "resourceApiVersion": "V3" + } + } + ], + "tlsParams": { + "tlsMaximumProtocolVersion": "TLSv1_3" + }, + "validationContextSdsSecretConfig": { + "name": "xds_trusted_ca", + "sdsConfig": { + "pathConfigSource": { + "path": "/sds/xds-trusted-ca.json" + }, + "resourceApiVersion": "V3" + } + } + } + } + }, + "type": "STRICT_DNS", + "typedExtensionProtocolOptions": { + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", + "explicitHttpConfig": { + "http2ProtocolOptions": {} + } + } + } } ], "listeners": [ diff --git a/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.yaml b/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.yaml index 5ac91ed8e45..48d1bfad14a 100644 --- a/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.yaml +++ b/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.yaml @@ -96,6 +96,44 @@ xds: connectionKeepalive: interval: 30s timeout: 5s + - connectTimeout: 10s + loadAssignment: + clusterName: wasm_cluster + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: envoy-gateway + portValue: 18002 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + name: wasm_cluster + transportSocket: + name: envoy.transport_sockets.tls + typedConfig: + '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + commonTlsContext: + tlsCertificateSdsSecretConfigs: + - name: xds_certificate + sdsConfig: + pathConfigSource: + path: /sds/xds-certificate.json + resourceApiVersion: V3 + tlsParams: + tlsMaximumProtocolVersion: TLSv1_3 + validationContextSdsSecretConfig: + name: xds_trusted_ca + sdsConfig: + pathConfigSource: + path: /sds/xds-trusted-ca.json + resourceApiVersion: V3 + type: STRICT_DNS + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicitHttpConfig: + http2ProtocolOptions: {} listeners: - address: socketAddress: diff --git a/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.bootstrap.yaml b/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.bootstrap.yaml index 61174308058..b378fdc17b5 100644 --- a/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.bootstrap.yaml +++ b/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.bootstrap.yaml @@ -95,6 +95,44 @@ xds: connectionKeepalive: interval: 30s timeout: 5s + - connectTimeout: 10s + loadAssignment: + clusterName: wasm_cluster + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: envoy-gateway + portValue: 18002 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + name: wasm_cluster + transportSocket: + name: envoy.transport_sockets.tls + typedConfig: + '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + commonTlsContext: + tlsCertificateSdsSecretConfigs: + - name: xds_certificate + sdsConfig: + pathConfigSource: + path: /sds/xds-certificate.json + resourceApiVersion: V3 + tlsParams: + tlsMaximumProtocolVersion: TLSv1_3 + validationContextSdsSecretConfig: + name: xds_trusted_ca + sdsConfig: + pathConfigSource: + path: /sds/xds-trusted-ca.json + resourceApiVersion: V3 + type: STRICT_DNS + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicitHttpConfig: + http2ProtocolOptions: {} listeners: - address: socketAddress: diff --git a/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.all.json b/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.all.json index 640bc9977b7..dd447633e32 100644 --- a/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.all.json +++ b/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.all.json @@ -163,6 +163,71 @@ } } } + }, + { + "connectTimeout": "10s", + "loadAssignment": { + "clusterName": "wasm_cluster", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "socketAddress": { + "address": "envoy-gateway", + "portValue": 18002 + } + } + }, + "loadBalancingWeight": 1 + } + ], + "loadBalancingWeight": 1 + } + ] + }, + "name": "wasm_cluster", + "transportSocket": { + "name": "envoy.transport_sockets.tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", + "commonTlsContext": { + "tlsCertificateSdsSecretConfigs": [ + { + "name": "xds_certificate", + "sdsConfig": { + "pathConfigSource": { + "path": "/sds/xds-certificate.json" + }, + "resourceApiVersion": "V3" + } + } + ], + "tlsParams": { + "tlsMaximumProtocolVersion": "TLSv1_3" + }, + "validationContextSdsSecretConfig": { + "name": "xds_trusted_ca", + "sdsConfig": { + "pathConfigSource": { + "path": "/sds/xds-trusted-ca.json" + }, + "resourceApiVersion": "V3" + } + } + } + } + }, + "type": "STRICT_DNS", + "typedExtensionProtocolOptions": { + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", + "explicitHttpConfig": { + "http2ProtocolOptions": {} + } + } + } } ], "listeners": [ diff --git a/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.all.yaml b/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.all.yaml index 0786edce3ea..545eba1cf63 100644 --- a/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.all.yaml +++ b/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.all.yaml @@ -96,6 +96,44 @@ xds: connectionKeepalive: interval: 30s timeout: 5s + - connectTimeout: 10s + loadAssignment: + clusterName: wasm_cluster + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: envoy-gateway + portValue: 18002 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + name: wasm_cluster + transportSocket: + name: envoy.transport_sockets.tls + typedConfig: + '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + commonTlsContext: + tlsCertificateSdsSecretConfigs: + - name: xds_certificate + sdsConfig: + pathConfigSource: + path: /sds/xds-certificate.json + resourceApiVersion: V3 + tlsParams: + tlsMaximumProtocolVersion: TLSv1_3 + validationContextSdsSecretConfig: + name: xds_trusted_ca + sdsConfig: + pathConfigSource: + path: /sds/xds-trusted-ca.json + resourceApiVersion: V3 + type: STRICT_DNS + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicitHttpConfig: + http2ProtocolOptions: {} listeners: - address: socketAddress: diff --git a/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.bootstrap.yaml b/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.bootstrap.yaml index 98710133fc5..f2838d95bc9 100644 --- a/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.bootstrap.yaml +++ b/internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.bootstrap.yaml @@ -95,6 +95,44 @@ xds: connectionKeepalive: interval: 30s timeout: 5s + - connectTimeout: 10s + loadAssignment: + clusterName: wasm_cluster + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: envoy-gateway + portValue: 18002 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + name: wasm_cluster + transportSocket: + name: envoy.transport_sockets.tls + typedConfig: + '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + commonTlsContext: + tlsCertificateSdsSecretConfigs: + - name: xds_certificate + sdsConfig: + pathConfigSource: + path: /sds/xds-certificate.json + resourceApiVersion: V3 + tlsParams: + tlsMaximumProtocolVersion: TLSv1_3 + validationContextSdsSecretConfig: + name: xds_trusted_ca + sdsConfig: + pathConfigSource: + path: /sds/xds-trusted-ca.json + resourceApiVersion: V3 + type: STRICT_DNS + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicitHttpConfig: + http2ProtocolOptions: {} listeners: - address: socketAddress: diff --git a/internal/gatewayapi/envoyextensionpolicy.go b/internal/gatewayapi/envoyextensionpolicy.go index 69dd7b10887..af7904ec3bf 100644 --- a/internal/gatewayapi/envoyextensionpolicy.go +++ b/internal/gatewayapi/envoyextensionpolicy.go @@ -9,22 +9,29 @@ import ( "errors" "fmt" "sort" + "strconv" "strings" "time" perr "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/gatewayapi/status" "github.com/envoyproxy/gateway/internal/ir" "github.com/envoyproxy/gateway/internal/utils" + "github.com/envoyproxy/gateway/internal/wasm" ) +// oci URL prefix +const ociURLPrefix = "oci://" + func (t *Translator) ProcessEnvoyExtensionPolicies(envoyExtensionPolicies []*egv1a1.EnvoyExtensionPolicy, gateways []*GatewayContext, routes []RouteContext, @@ -302,16 +309,19 @@ func resolveEEPolicyRouteTargetRef(policy *egv1a1.EnvoyExtensionPolicy, target g return route.RouteContext, nil } -func (t *Translator) translateEnvoyExtensionPolicyForRoute(policy *egv1a1.EnvoyExtensionPolicy, route RouteContext, - xdsIR XdsIRMap, resources *Resources, +func (t *Translator) translateEnvoyExtensionPolicyForRoute( + policy *egv1a1.EnvoyExtensionPolicy, + route RouteContext, + xdsIR XdsIRMap, + resources *Resources, ) error { var ( wasms []ir.Wasm err, errs error ) - if wasms, err = t.buildWasms(policy); err != nil { - err = perr.WithMessage(err, "WASMs") + if wasms, err = t.buildWasms(policy, resources); err != nil { + err = perr.WithMessage(err, "WASM") errs = errors.Join(errs, err) } @@ -332,7 +342,7 @@ func (t *Translator) translateEnvoyExtensionPolicyForRoute(policy *egv1a1.EnvoyE var extProcs []ir.ExtProc if extProcs, err = t.buildExtProcs(policy, resources, gtwCtx.envoyProxy); err != nil { - err = perr.WithMessage(err, "ExtProcs") + err = perr.WithMessage(err, "ExtProc") errs = errors.Join(errs, err) } irKey := t.getIRKey(gtwCtx.Gateway) @@ -366,11 +376,11 @@ func (t *Translator) translateEnvoyExtensionPolicyForGateway( ) if extProcs, err = t.buildExtProcs(policy, resources, gateway.envoyProxy); err != nil { - err = perr.WithMessage(err, "ExtProcs") + err = perr.WithMessage(err, "ExtProc") errs = errors.Join(errs, err) } - if wasms, err = t.buildWasms(policy); err != nil { - err = perr.WithMessage(err, "WASMs") + if wasms, err = t.buildWasms(policy, resources); err != nil { + err = perr.WithMessage(err, "WASM") errs = errors.Join(errs, err) } @@ -532,7 +542,14 @@ func irConfigNameForEEP(policy *egv1a1.EnvoyExtensionPolicy, idx int) string { idx) } -func (t *Translator) buildWasms(policy *egv1a1.EnvoyExtensionPolicy) ([]ir.Wasm, error) { +func (t *Translator) buildWasms( + policy *egv1a1.EnvoyExtensionPolicy, + resources *Resources, +) ([]ir.Wasm, error) { + if t.WasmCache == nil { + return nil, fmt.Errorf("wasm cache is not initialized") + } + var wasmIRList []ir.Wasm if policy == nil { @@ -540,8 +557,8 @@ func (t *Translator) buildWasms(policy *egv1a1.EnvoyExtensionPolicy) ([]ir.Wasm, } for idx, wasm := range policy.Spec.Wasm { - name := irConfigNameForEEP(policy, idx) - wasmIR, err := t.buildWasm(name, wasm) + name := irConfigNameForWasm(policy, idx) + wasmIR, err := t.buildWasm(name, wasm, policy, idx, resources) if err != nil { return nil, err } @@ -550,37 +567,169 @@ func (t *Translator) buildWasms(policy *egv1a1.EnvoyExtensionPolicy) ([]ir.Wasm, return wasmIRList, nil } -func (t *Translator) buildWasm(name string, wasm egv1a1.Wasm) (*ir.Wasm, error) { +func (t *Translator) buildWasm( + name string, + config egv1a1.Wasm, + policy *egv1a1.EnvoyExtensionPolicy, + idx int, + resources *Resources, +) (*ir.Wasm, error) { var ( - failOpen = false - httpWasmCode *ir.HTTPWasmCode + failOpen = false + code *ir.HTTPWasmCode + pullPolicy wasm.PullPolicy + // the checksum provided by the user, it's used to validate the wasm module + // downloaded from the original HTTP server or the OCI registry + originalChecksum string + servingURL string // the wasm module download URL from the EG HTTP server + err error ) - if wasm.FailOpen != nil { - failOpen = *wasm.FailOpen + if config.FailOpen != nil { + failOpen = *config.FailOpen } - switch wasm.Code.Type { + if config.Code.PullPolicy != nil { + switch *config.Code.PullPolicy { + case egv1a1.ImagePullPolicyAlways: + pullPolicy = wasm.Always + case egv1a1.ImagePullPolicyIfNotPresent: + pullPolicy = wasm.IfNotPresent + default: + pullPolicy = wasm.Unspecified + } + } + + switch config.Code.Type { case egv1a1.HTTPWasmCodeSourceType: - httpWasmCode = &ir.HTTPWasmCode{ - URL: wasm.Code.HTTP.URL, - SHA256: wasm.Code.SHA256, + // This is a sanity check, the validation should have caught this + if config.Code.HTTP == nil { + return nil, fmt.Errorf("missing HTTP field in Wasm code source") + } + + if config.Code.HTTP.SHA256 != nil { + originalChecksum = *config.Code.HTTP.SHA256 + } + + http := config.Code.HTTP + + if servingURL, _, err = t.WasmCache.Get(http.URL, wasm.GetOptions{ + Checksum: originalChecksum, + PullPolicy: pullPolicy, + ResourceName: irConfigNameForWasm(policy, idx), + ResourceVersion: policy.ResourceVersion, + }); err != nil { + return nil, err } + + code = &ir.HTTPWasmCode{ + ServingURL: servingURL, + OriginalURL: http.URL, + SHA256: originalChecksum, + } + case egv1a1.ImageWasmCodeSourceType: - return nil, fmt.Errorf("OCI image Wasm code source is not supported yet") + var ( + image = config.Code.Image + secret *corev1.Secret + pullSecret []byte + // the checksum of the wasm module extracted from the OCI image + // it's different from the checksum for the OCI image + checksum string + ) + + // This is a sanity check, the validation should have caught this + if image == nil { + return nil, fmt.Errorf("missing Image field in Wasm code source") + } + + if image.PullSecretRef != nil { + from := crossNamespaceFrom{ + group: egv1a1.GroupName, + kind: KindEnvoyExtensionPolicy, + namespace: policy.Namespace, + } + + if secret, err = t.validateSecretRef( + false, from, *image.PullSecretRef, resources); err != nil { + return nil, err + } + + if data, ok := secret.Data[corev1.DockerConfigJsonKey]; ok { + pullSecret = data + } else { + return nil, fmt.Errorf("missing %s key in secret %s/%s", corev1.DockerConfigJsonKey, secret.Namespace, secret.Name) + } + } + + // Wasm Cache requires the URL to be in the format "scheme://" + imageURL := image.URL + if !strings.HasPrefix(image.URL, ociURLPrefix) { + imageURL = fmt.Sprintf("%s%s", ociURLPrefix, image.URL) + } + + // If the url is an OCI image, and neither digest nor tag is provided, use the latest tag. + if !hasDigest(imageURL) && !hasTag(imageURL) { + imageURL += ":latest" + } + + if config.Code.Image.SHA256 != nil { + originalChecksum = *config.Code.Image.SHA256 + } + + // The wasm checksum is different from the OCI image digest. + // The original checksum in the EEP is used to match the digest of OCI image. + // The returned checksum from the cache is the checksum of the wasm file + // extracted from the OCI image, which is used by the envoy to verify the wasm file. + if servingURL, checksum, err = t.WasmCache.Get(imageURL, wasm.GetOptions{ + Checksum: originalChecksum, + PullSecret: pullSecret, + PullPolicy: pullPolicy, + ResourceName: irConfigNameForWasm(policy, idx), + ResourceVersion: policy.ResourceVersion, + }); err != nil { + return nil, err + } + + code = &ir.HTTPWasmCode{ + ServingURL: servingURL, + SHA256: checksum, + OriginalURL: imageURL, + } default: // should never happen because of kubebuilder validation, just a sanity check - return nil, fmt.Errorf("unsupported Wasm code source type %q", wasm.Code.Type) + return nil, fmt.Errorf("unsupported Wasm code source type %q", config.Code.Type) } + wasmName := name + if config.Name != nil { + wasmName = *config.Name + } wasmIR := &ir.Wasm{ - Name: name, - RootID: wasm.RootID, - WasmName: wasm.Name, - Config: wasm.Config, - FailOpen: failOpen, - HTTPWasmCode: httpWasmCode, + Name: name, + RootID: config.RootID, + WasmName: wasmName, + Config: config.Config, + FailOpen: failOpen, + Code: code, } return wasmIR, nil } + +func hasDigest(imageURL string) bool { + return strings.Contains(imageURL, "@") +} + +func hasTag(imageURL string) bool { + parts := strings.Split(imageURL[len(ociURLPrefix):], ":") + // Verify that we aren't confusing a tag for a hostname with port. + return len(parts) > 1 && !strings.Contains(parts[len(parts)-1], "/") +} + +func irConfigNameForWasm(policy client.Object, index int) string { + return fmt.Sprintf( + "%s/wasm/%s", + irConfigName(policy), + strconv.Itoa(index)) +} diff --git a/internal/gatewayapi/envoyextensionpolicy_test.go b/internal/gatewayapi/envoyextensionpolicy_test.go new file mode 100644 index 00000000000..5c611163fca --- /dev/null +++ b/internal/gatewayapi/envoyextensionpolicy_test.go @@ -0,0 +1,66 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package gatewayapi + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_hasTag(t *testing.T) { + tests := []struct { + name string + imageURL string + want bool + }{ + { + name: "image with scheme and tag", + imageURL: "oci://www.example.com/wasm:v1.0.0", + want: true, + }, + { + name: "image with scheme, host port and tag", + imageURL: "oci://www.example.com:8080/wasm:v1.0.0", + want: true, + }, + { + name: "image with scheme without tag", + imageURL: "oci://www.example.com/wasm", + want: false, + }, + { + name: "image with scheme, host port without tag", + imageURL: "oci://www.example.com:8080/wasm", + want: false, + }, + { + name: "image without scheme with tag", + imageURL: "www.example.com/wasm:v1.0.0", + want: true, + }, + { + name: "image without scheme with host port and tag", + imageURL: "www.example.com:8080/wasm:v1.0.0", + want: true, + }, + { + name: "image without scheme without tag", + imageURL: "www.example.com/wasm", + want: false, + }, + { + name: "image without scheme with host port without tag", + imageURL: "www.example.com:8080/wasm", + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, hasTag(tt.imageURL), "hasTag(%v)", tt.imageURL) + }) + } +} diff --git a/internal/gatewayapi/helpers.go b/internal/gatewayapi/helpers.go index ed35961ca0f..408a40ab0a4 100644 --- a/internal/gatewayapi/helpers.go +++ b/internal/gatewayapi/helpers.go @@ -15,6 +15,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" @@ -487,3 +488,10 @@ func parseCIDR(cidr string) (*ir.CIDRMatch, error) { IsIPv6: ip.To4() == nil, }, nil } + +func irConfigName(policy client.Object) string { + return fmt.Sprintf( + "%s/%s", + strings.ToLower(policy.GetObjectKind().GroupVersionKind().Kind), + utils.NamespacedName(policy).String()) +} diff --git a/internal/gatewayapi/runner/runner.go b/internal/gatewayapi/runner/runner.go index b4419aff63b..8b9b57fc839 100644 --- a/internal/gatewayapi/runner/runner.go +++ b/internal/gatewayapi/runner/runner.go @@ -7,12 +7,22 @@ package runner import ( "context" + "crypto/tls" + "crypto/x509" "encoding/json" + "errors" + "fmt" + "os" "reflect" + "github.com/docker/docker/pkg/fileutils" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/kubernetes" + ctrl "sigs.k8s.io/controller-runtime" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" @@ -22,6 +32,14 @@ import ( "github.com/envoyproxy/gateway/internal/gatewayapi" "github.com/envoyproxy/gateway/internal/message" "github.com/envoyproxy/gateway/internal/utils" + "github.com/envoyproxy/gateway/internal/wasm" +) + +const ( + wasmCacheDir = "/var/lib/eg/wasm" + serveTLSCertFilename = "/certs/tls.crt" + serveTLSKeyFilename = "/certs/tls.key" + serveTLSCaFilename = "/certs/ca.crt" ) type Config struct { @@ -34,12 +52,21 @@ type Config struct { type Runner struct { Config + wasmCache wasm.Cache } func New(cfg *Config) *Runner { - return &Runner{Config: *cfg} + return &Runner{ + Config: *cfg, + } } +const ( + // nolint: gosec + hmacSecretName = "envoy-oidc-hmac" + hmacSecretKey = "hmac-secret" +) + func (r *Runner) Name() string { return string(egv1a1.LogComponentGatewayAPIRunner) } @@ -47,11 +74,45 @@ func (r *Runner) Name() string { // Start starts the gateway-api translator runner func (r *Runner) Start(ctx context.Context) (err error) { r.Logger = r.Logger.WithName(r.Name()).WithValues("runner", r.Name()) + + go r.startWasmCache(ctx) go r.subscribeAndTranslate(ctx) r.Logger.Info("started") return } +func (r *Runner) startWasmCache(ctx context.Context) { + // Start the wasm cache server + // EG reuse the OIDC HMAC secret as a hash salt to generate an unguessable + // downloading path for the Wasm module. + salt, err := hmac(ctx, r.Namespace) + if err != nil { + r.Logger.Error(err, "failed to get hmac secret") + return + } + tlsConfig, err := r.tlsConfig() + if err != nil { + r.Logger.Error(err, "failed to create tls config") + return + } + // Create the file directory if it does not exist. + if err = fileutils.CreateIfNotExists(wasmCacheDir, true); err != nil { + r.Logger.Error(err, "Failed to create Wasm cache directory") + return + } + r.wasmCache = wasm.NewHTTPServerWithFileCache( + // HTTP server options + wasm.SeverOptions{ + Salt: salt, + TLSConfig: tlsConfig, + }, + // Wasm cache options + wasm.CacheOptions{ + CacheDir: wasmCacheDir, + }, r.Logger) + r.wasmCache.Start(ctx) +} + func (r *Runner) subscribeAndTranslate(ctx context.Context) { message.HandleSubscription(message.Metadata{Runner: string(egv1a1.LogComponentGatewayAPIRunner), Message: "provider-resources"}, r.ProviderResources.GatewayAPIResources.Subscribe(ctx), func(update message.Update[string, *gatewayapi.ControllerResources], errChan chan error) { @@ -88,6 +149,7 @@ func (r *Runner) subscribeAndTranslate(ctx context.Context) { BackendEnabled: r.EnvoyGateway.ExtensionAPIs != nil && r.EnvoyGateway.ExtensionAPIs.EnableBackend, Namespace: r.Namespace, MergeGateways: gatewayapi.IsMergeGatewaysEnabled(resources), + WasmCache: r.wasmCache, } // If an extension is loaded, pass its supported groups/kinds to the translator @@ -427,3 +489,63 @@ func getIRKeysToDelete(curKeys, newKeys []string) []string { return delSet.List() } + +// hmac returns the HMAC secret generated by the CertGen job. +// hmac will be used as a hash salt to generate unguessable downloading paths for Wasm modules. +func hmac(ctx context.Context, namespace string) (hmac []byte, err error) { + // Get the HMAC secret. + // HMAC secret is generated by the CertGen job and stored in a secret + cfg, err := ctrl.GetConfig() + if err != nil { + return nil, err + } + client, err := kubernetes.NewForConfig(cfg) + if err != nil { + return nil, err + } + secret, err := client.CoreV1().Secrets(namespace).Get(ctx, hmacSecretName, metav1.GetOptions{}) + if err != nil { + if kerrors.IsNotFound(err) { + return nil, fmt.Errorf("HMAC secret %s/%s not found", namespace, hmacSecretName) + } + return nil, err + } + hmac, ok := secret.Data[hmacSecretKey] + if !ok || len(hmac) == 0 { + return nil, fmt.Errorf( + "HMAC secret not found in secret %s/%s", namespace, hmacSecretName) + } + return +} + +func (r *Runner) tlsConfig() (*tls.Config, error) { + var ( + serverCert tls.Certificate // server's certificate and private key + caCert []byte // the CA certificate for client verification + caCertPool *x509.CertPool + err error + ) + + // Load server's certificate and private key + if serverCert, err = tls.LoadX509KeyPair(serveTLSCertFilename, serveTLSKeyFilename); err != nil { + return nil, err + } + + // Load client's CA certificate + if caCert, err = os.ReadFile(serveTLSCaFilename); err != nil { + return nil, err + } + + caCertPool = x509.NewCertPool() + if !caCertPool.AppendCertsFromPEM(caCert) { + return nil, errors.New("failed to parse CA certificate") + } + + // Configure the server to require client certificates + return &tls.Config{ + Certificates: []tls.Certificate{serverCert}, + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: caCertPool, + MinVersion: tls.VersionTLS13, + }, nil +} diff --git a/internal/gatewayapi/securitypolicy.go b/internal/gatewayapi/securitypolicy.go index 34e2410ae55..470113b60e0 100644 --- a/internal/gatewayapi/securitypolicy.go +++ b/internal/gatewayapi/securitypolicy.go @@ -914,13 +914,6 @@ func irExtServiceDestinationName(policy *egv1a1.SecurityPolicy, backendRef *gwap nn.String())) } -func irConfigName(policy *egv1a1.SecurityPolicy) string { - return fmt.Sprintf( - "%s/%s", - strings.ToLower(KindSecurityPolicy), - utils.NamespacedName(policy).String()) -} - func (t *Translator) buildAuthorization(policy *egv1a1.SecurityPolicy) (*ir.Authorization, error) { var ( authorization = policy.Spec.Authorization diff --git a/internal/gatewayapi/testdata/backend-invalid-feature-disabled.out.yaml b/internal/gatewayapi/testdata/backend-invalid-feature-disabled.out.yaml index 6b95feb0a07..a825b0a6e2b 100644 --- a/internal/gatewayapi/testdata/backend-invalid-feature-disabled.out.yaml +++ b/internal/gatewayapi/testdata/backend-invalid-feature-disabled.out.yaml @@ -45,7 +45,7 @@ envoyExtensionPolicies: sectionName: http conditions: - lastTransitionTime: null - message: 'ExtProcs: resource backend-ip of type Backend cannot be used since + message: 'ExtProc: resource backend-ip of type Backend cannot be used since Backend is disabled in Envoy Gateway configuration.' reason: Invalid status: "False" diff --git a/internal/gatewayapi/testdata/custom-filter-order.in.yaml b/internal/gatewayapi/testdata/custom-filter-order.in.yaml index d3e931aed52..99b46e6de82 100644 --- a/internal/gatewayapi/testdata/custom-filter-order.in.yaml +++ b/internal/gatewayapi/testdata/custom-filter-order.in.yaml @@ -111,7 +111,7 @@ envoyextensionpolicies: type: HTTP http: url: https://www.example.com/wasm-filter-1.wasm - sha256: 746df05c8f3a0b07a46c0967cfbc5cbe5b9d48d0f79b6177eeedf8be6c8b34b5 + sha256: 746df05c8f3a0b07a46c0967cfbc5cbe5b9d48d0f79b6177eeedf8be6c8b34b5 config: parameter1: key1: value1 @@ -122,7 +122,7 @@ envoyextensionpolicies: type: HTTP http: url: https://www.example.com/wasm-filter-2.wasm - sha256: a1efca12ea51069abb123bf9c77889fcc2a31cc5483fc14d115e44fdf07c7980 + sha256: a1efca12ea51069abb123bf9c77889fcc2a31cc5483fc14d115e44fdf07c7980 config: parameter1: value1 parameter2: value2 diff --git a/internal/gatewayapi/testdata/custom-filter-order.out.yaml b/internal/gatewayapi/testdata/custom-filter-order.out.yaml index 5a520a9a9da..be53fc5c2ec 100644 --- a/internal/gatewayapi/testdata/custom-filter-order.out.yaml +++ b/internal/gatewayapi/testdata/custom-filter-order.out.yaml @@ -13,8 +13,8 @@ envoyExtensionPolicies: wasm: - code: http: + sha256: 746df05c8f3a0b07a46c0967cfbc5cbe5b9d48d0f79b6177eeedf8be6c8b34b5 url: https://www.example.com/wasm-filter-1.wasm - sha256: 746df05c8f3a0b07a46c0967cfbc5cbe5b9d48d0f79b6177eeedf8be6c8b34b5 type: HTTP config: parameter1: @@ -24,8 +24,8 @@ envoyExtensionPolicies: name: wasm-filter-1 - code: http: + sha256: a1efca12ea51069abb123bf9c77889fcc2a31cc5483fc14d115e44fdf07c7980 url: https://www.example.com/wasm-filter-2.wasm - sha256: a1efca12ea51069abb123bf9c77889fcc2a31cc5483fc14d115e44fdf07c7980 type: HTTP config: parameter1: value1 @@ -291,16 +291,18 @@ xdsIR: parameter2: value3 failOpen: false httpWasmCode: + originalDownloadingURL: https://www.example.com/wasm-filter-1.wasm + servingURL: https://envoy-gateway:18002/5c90b9a82642ce00a7753923fabead306b9d9a54a7c0bd2463a1af3efcfb110b.wasm sha256: 746df05c8f3a0b07a46c0967cfbc5cbe5b9d48d0f79b6177eeedf8be6c8b34b5 - url: https://www.example.com/wasm-filter-1.wasm - name: envoyextensionpolicy/envoy-gateway/policy-for-gateway/0 + name: envoyextensionpolicy/envoy-gateway/policy-for-gateway/wasm/0 wasmName: wasm-filter-1 - config: parameter1: value1 parameter2: value2 failOpen: false httpWasmCode: + originalDownloadingURL: https://www.example.com/wasm-filter-2.wasm + servingURL: https://envoy-gateway:18002/593e4cc60a7e0fa4d4f86531a5e20e785213a52000f056a7a8b5c5afcb908052.wasm sha256: a1efca12ea51069abb123bf9c77889fcc2a31cc5483fc14d115e44fdf07c7980 - url: https://www.example.com/wasm-filter-2.wasm - name: envoyextensionpolicy/envoy-gateway/policy-for-gateway/1 + name: envoyextensionpolicy/envoy-gateway/policy-for-gateway/wasm/1 wasmName: wasm-filter-2 diff --git a/internal/gatewayapi/testdata/envoyextensionpolicy-with-extproc-invalid-no-matching-port.out.yaml b/internal/gatewayapi/testdata/envoyextensionpolicy-with-extproc-invalid-no-matching-port.out.yaml index 76ce25c20f6..835a92ba92b 100644 --- a/internal/gatewayapi/testdata/envoyextensionpolicy-with-extproc-invalid-no-matching-port.out.yaml +++ b/internal/gatewayapi/testdata/envoyextensionpolicy-with-extproc-invalid-no-matching-port.out.yaml @@ -23,7 +23,7 @@ envoyExtensionPolicies: namespace: default conditions: - lastTransitionTime: null - message: 'ExtProcs: TCP Port 4000 not found on service default/grpc-backend.' + message: 'ExtProc: TCP Port 4000 not found on service default/grpc-backend.' reason: Invalid status: "False" type: Accepted diff --git a/internal/gatewayapi/testdata/envoyextensionpolicy-with-extproc-invalid-no-port.out.yaml b/internal/gatewayapi/testdata/envoyextensionpolicy-with-extproc-invalid-no-port.out.yaml index c8d26813c85..80c04f0c71e 100644 --- a/internal/gatewayapi/testdata/envoyextensionpolicy-with-extproc-invalid-no-port.out.yaml +++ b/internal/gatewayapi/testdata/envoyextensionpolicy-with-extproc-invalid-no-port.out.yaml @@ -22,7 +22,7 @@ envoyExtensionPolicies: namespace: default conditions: - lastTransitionTime: null - message: 'ExtProcs: a valid port number corresponding to a port on the Service + message: 'ExtProc: a valid port number corresponding to a port on the Service must be specified.' reason: Invalid status: "False" diff --git a/internal/gatewayapi/testdata/envoyextensionpolicy-with-extproc-invalid-no-reference-grant.out.yaml b/internal/gatewayapi/testdata/envoyextensionpolicy-with-extproc-invalid-no-reference-grant.out.yaml index 50148ea73d0..75e70c1cc01 100644 --- a/internal/gatewayapi/testdata/envoyextensionpolicy-with-extproc-invalid-no-reference-grant.out.yaml +++ b/internal/gatewayapi/testdata/envoyextensionpolicy-with-extproc-invalid-no-reference-grant.out.yaml @@ -24,8 +24,8 @@ envoyExtensionPolicies: namespace: default conditions: - lastTransitionTime: null - message: 'ExtProcs: backend ref to Service envoy-gateway/grpc-backend not - permitted by any ReferenceGrant.' + message: 'ExtProc: backend ref to Service envoy-gateway/grpc-backend not permitted + by any ReferenceGrant.' reason: Invalid status: "False" type: Accepted diff --git a/internal/gatewayapi/testdata/envoyextensionpolicy-with-extproc-invalid-no-service.out.yaml b/internal/gatewayapi/testdata/envoyextensionpolicy-with-extproc-invalid-no-service.out.yaml index 7e5e5aba4b5..1f90e0a26fb 100644 --- a/internal/gatewayapi/testdata/envoyextensionpolicy-with-extproc-invalid-no-service.out.yaml +++ b/internal/gatewayapi/testdata/envoyextensionpolicy-with-extproc-invalid-no-service.out.yaml @@ -24,7 +24,7 @@ envoyExtensionPolicies: namespace: default conditions: - lastTransitionTime: null - message: 'ExtProcs: service envoy-gateway/grpc-backend not found.' + message: 'ExtProc: service envoy-gateway/grpc-backend not found.' reason: Invalid status: "False" type: Accepted diff --git a/internal/gatewayapi/testdata/envoyextensionpolicy-with-wasm-targetrefs.in.yaml b/internal/gatewayapi/testdata/envoyextensionpolicy-with-wasm-targetrefs.in.yaml index 2ff19ac2a6e..106267da645 100644 --- a/internal/gatewayapi/testdata/envoyextensionpolicy-with-wasm-targetrefs.in.yaml +++ b/internal/gatewayapi/testdata/envoyextensionpolicy-with-wasm-targetrefs.in.yaml @@ -72,7 +72,7 @@ envoyextensionpolicies: type: HTTP http: url: https://www.example.com/wasm-filter-1.wasm - sha256: 746df05c8f3a0b07a46c0967cfbc5cbe5b9d48d0f79b6177eeedf8be6c8b34b5 + sha256: 746df05c8f3a0b07a46c0967cfbc5cbe5b9d48d0f79b6177eeedf8be6c8b34b5 config: parameter1: key1: value1 @@ -83,8 +83,7 @@ envoyextensionpolicies: type: HTTP http: url: https://www.example.com/wasm-filter-2.wasm - sha256: a1efca12ea51069abb123bf9c77889fcc2a31cc5483fc14d115e44fdf07c7980 + sha256: a1efca12ea51069abb123bf9c77889fcc2a31cc5483fc14d115e44fdf07c7980 config: parameter1: value1 parameter2: value2 - diff --git a/internal/gatewayapi/testdata/envoyextensionpolicy-with-wasm-targetrefs.out.yaml b/internal/gatewayapi/testdata/envoyextensionpolicy-with-wasm-targetrefs.out.yaml index 463f604c21a..47ab4e6d799 100644 --- a/internal/gatewayapi/testdata/envoyextensionpolicy-with-wasm-targetrefs.out.yaml +++ b/internal/gatewayapi/testdata/envoyextensionpolicy-with-wasm-targetrefs.out.yaml @@ -16,8 +16,8 @@ envoyExtensionPolicies: wasm: - code: http: + sha256: 746df05c8f3a0b07a46c0967cfbc5cbe5b9d48d0f79b6177eeedf8be6c8b34b5 url: https://www.example.com/wasm-filter-1.wasm - sha256: 746df05c8f3a0b07a46c0967cfbc5cbe5b9d48d0f79b6177eeedf8be6c8b34b5 type: HTTP config: parameter1: @@ -27,8 +27,8 @@ envoyExtensionPolicies: name: wasm-filter-1 - code: http: + sha256: a1efca12ea51069abb123bf9c77889fcc2a31cc5483fc14d115e44fdf07c7980 url: https://www.example.com/wasm-filter-2.wasm - sha256: a1efca12ea51069abb123bf9c77889fcc2a31cc5483fc14d115e44fdf07c7980 type: HTTP config: parameter1: value1 @@ -238,18 +238,20 @@ xdsIR: parameter2: value3 failOpen: false httpWasmCode: + originalDownloadingURL: https://www.example.com/wasm-filter-1.wasm + servingURL: https://envoy-gateway:18002/5c90b9a82642ce00a7753923fabead306b9d9a54a7c0bd2463a1af3efcfb110b.wasm sha256: 746df05c8f3a0b07a46c0967cfbc5cbe5b9d48d0f79b6177eeedf8be6c8b34b5 - url: https://www.example.com/wasm-filter-1.wasm - name: envoyextensionpolicy/envoy-gateway/policy-for-gateway/0 + name: envoyextensionpolicy/envoy-gateway/policy-for-gateway/wasm/0 wasmName: wasm-filter-1 - config: parameter1: value1 parameter2: value2 failOpen: false httpWasmCode: + originalDownloadingURL: https://www.example.com/wasm-filter-2.wasm + servingURL: https://envoy-gateway:18002/593e4cc60a7e0fa4d4f86531a5e20e785213a52000f056a7a8b5c5afcb908052.wasm sha256: a1efca12ea51069abb123bf9c77889fcc2a31cc5483fc14d115e44fdf07c7980 - url: https://www.example.com/wasm-filter-2.wasm - name: envoyextensionpolicy/envoy-gateway/policy-for-gateway/1 + name: envoyextensionpolicy/envoy-gateway/policy-for-gateway/wasm/1 wasmName: wasm-filter-2 - destination: name: httproute/envoy-gateway/httproute-2/rule/0 @@ -272,16 +274,18 @@ xdsIR: parameter2: value3 failOpen: false httpWasmCode: + originalDownloadingURL: https://www.example.com/wasm-filter-1.wasm + servingURL: https://envoy-gateway:18002/5c90b9a82642ce00a7753923fabead306b9d9a54a7c0bd2463a1af3efcfb110b.wasm sha256: 746df05c8f3a0b07a46c0967cfbc5cbe5b9d48d0f79b6177eeedf8be6c8b34b5 - url: https://www.example.com/wasm-filter-1.wasm - name: envoyextensionpolicy/envoy-gateway/policy-for-gateway/0 + name: envoyextensionpolicy/envoy-gateway/policy-for-gateway/wasm/0 wasmName: wasm-filter-1 - config: parameter1: value1 parameter2: value2 failOpen: false httpWasmCode: + originalDownloadingURL: https://www.example.com/wasm-filter-2.wasm + servingURL: https://envoy-gateway:18002/593e4cc60a7e0fa4d4f86531a5e20e785213a52000f056a7a8b5c5afcb908052.wasm sha256: a1efca12ea51069abb123bf9c77889fcc2a31cc5483fc14d115e44fdf07c7980 - url: https://www.example.com/wasm-filter-2.wasm - name: envoyextensionpolicy/envoy-gateway/policy-for-gateway/1 + name: envoyextensionpolicy/envoy-gateway/policy-for-gateway/wasm/1 wasmName: wasm-filter-2 diff --git a/internal/gatewayapi/testdata/envoyextensionpolicy-with-wasm.in.yaml b/internal/gatewayapi/testdata/envoyextensionpolicy-with-wasm.in.yaml index 4d3e39cf48a..5cb2b192553 100644 --- a/internal/gatewayapi/testdata/envoyextensionpolicy-with-wasm.in.yaml +++ b/internal/gatewayapi/testdata/envoyextensionpolicy-with-wasm.in.yaml @@ -1,3 +1,11 @@ +secrets: +- apiVersion: v1 + kind: Secret + metadata: + namespace: envoy-gateway + name: my-pull-secret + data: + .dockerconfigjson: VGhpc0lzTm90QVJlYWxEb2NrZXJDb25maWdKc29u gateways: - apiVersion: gateway.networking.k8s.io/v1 kind: Gateway @@ -69,21 +77,28 @@ envoyextensionpolicies: type: HTTP http: url: https://www.example.com/wasm-filter-1.wasm - sha256: 746df05c8f3a0b07a46c0967cfbc5cbe5b9d48d0f79b6177eeedf8be6c8b34b5 + sha256: 746df05c8f3a0b07a46c0967cfbc5cbe5b9d48d0f79b6177eeedf8be6c8b34b5 config: parameter1: key1: value1 key2: value2 parameter2: value3 - name: wasm-filter-2 + rootID: "my-root-id" code: - type: HTTP - http: - url: https://www.example.com/wasm-filter-2.wasm - sha256: a1efca12ea51069abb123bf9c77889fcc2a31cc5483fc14d115e44fdf07c7980 + type: Image + image: + url: oci://www.example.com/wasm-filter-2:v1.0.0 + pullSecretRef: + name: my-pull-secret + sha256: a1efca12ea51069abb123bf9c77889fcc2a31cc5483fc14d115e44fdf07c7980 config: parameter1: value1 parameter2: value2 + - code: + type: Image + image: + url: www.example.com:8080/wasm-filter-3 - apiVersion: gateway.envoyproxy.io/v1alpha1 kind: EnvoyExtensionPolicy metadata: @@ -95,12 +110,12 @@ envoyextensionpolicies: kind: HTTPRoute name: httproute-1 wasm: - - name: wasm-filter-3 + - name: wasm-filter-4 code: type: HTTP http: - url: https://www.test.com/wasm-filter-3.wasm - sha256: a1f0b78b8c1320690327800e3a5de10e7dbba7b6c752e702193a395a52c727b6 + url: https://www.test.com/wasm-filter-4.wasm + sha256: a1f0b78b8c1320690327800e3a5de10e7dbba7b6c752e702193a395a52c727b6 config: parameter1: key1: value1 diff --git a/internal/gatewayapi/testdata/envoyextensionpolicy-with-wasm.out.yaml b/internal/gatewayapi/testdata/envoyextensionpolicy-with-wasm.out.yaml index 30bda358c18..f4e00c241bc 100644 --- a/internal/gatewayapi/testdata/envoyextensionpolicy-with-wasm.out.yaml +++ b/internal/gatewayapi/testdata/envoyextensionpolicy-with-wasm.out.yaml @@ -13,8 +13,8 @@ envoyExtensionPolicies: wasm: - code: http: - url: https://www.test.com/wasm-filter-3.wasm - sha256: a1f0b78b8c1320690327800e3a5de10e7dbba7b6c752e702193a395a52c727b6 + sha256: a1f0b78b8c1320690327800e3a5de10e7dbba7b6c752e702193a395a52c727b6 + url: https://www.test.com/wasm-filter-4.wasm type: HTTP config: parameter1: @@ -23,7 +23,7 @@ envoyExtensionPolicies: key2: key3: value3 failOpen: true - name: wasm-filter-3 + name: wasm-filter-4 status: ancestors: - ancestorRef: @@ -53,8 +53,8 @@ envoyExtensionPolicies: wasm: - code: http: + sha256: 746df05c8f3a0b07a46c0967cfbc5cbe5b9d48d0f79b6177eeedf8be6c8b34b5 url: https://www.example.com/wasm-filter-1.wasm - sha256: 746df05c8f3a0b07a46c0967cfbc5cbe5b9d48d0f79b6177eeedf8be6c8b34b5 type: HTTP config: parameter1: @@ -63,14 +63,24 @@ envoyExtensionPolicies: parameter2: value3 name: wasm-filter-1 - code: - http: - url: https://www.example.com/wasm-filter-2.wasm - sha256: a1efca12ea51069abb123bf9c77889fcc2a31cc5483fc14d115e44fdf07c7980 - type: HTTP + image: + pullSecretRef: + group: null + kind: null + name: my-pull-secret + sha256: a1efca12ea51069abb123bf9c77889fcc2a31cc5483fc14d115e44fdf07c7980 + url: oci://www.example.com/wasm-filter-2:v1.0.0 + type: Image config: parameter1: value1 parameter2: value2 name: wasm-filter-2 + rootID: my-root-id + - code: + image: + sha256: null + url: www.example.com:8080/wasm-filter-3 + type: Image status: ancestors: - ancestorRef: @@ -266,10 +276,11 @@ xdsIR: key3: value3 failOpen: true httpWasmCode: + originalDownloadingURL: https://www.test.com/wasm-filter-4.wasm + servingURL: https://envoy-gateway:18002/fe571e7b1ef5dc626ceb2c2c86782a134a92989a2643485238951696ae4334c3.wasm sha256: a1f0b78b8c1320690327800e3a5de10e7dbba7b6c752e702193a395a52c727b6 - url: https://www.test.com/wasm-filter-3.wasm - name: envoyextensionpolicy/default/policy-for-http-route/0 - wasmName: wasm-filter-3 + name: envoyextensionpolicy/default/policy-for-http-route/wasm/0 + wasmName: wasm-filter-4 - destination: name: httproute/default/httproute-2/rule/0 settings: @@ -294,16 +305,27 @@ xdsIR: parameter2: value3 failOpen: false httpWasmCode: + originalDownloadingURL: https://www.example.com/wasm-filter-1.wasm + servingURL: https://envoy-gateway:18002/5c90b9a82642ce00a7753923fabead306b9d9a54a7c0bd2463a1af3efcfb110b.wasm sha256: 746df05c8f3a0b07a46c0967cfbc5cbe5b9d48d0f79b6177eeedf8be6c8b34b5 - url: https://www.example.com/wasm-filter-1.wasm - name: envoyextensionpolicy/envoy-gateway/policy-for-gateway/0 + name: envoyextensionpolicy/envoy-gateway/policy-for-gateway/wasm/0 wasmName: wasm-filter-1 - config: parameter1: value1 parameter2: value2 failOpen: false httpWasmCode: - sha256: a1efca12ea51069abb123bf9c77889fcc2a31cc5483fc14d115e44fdf07c7980 - url: https://www.example.com/wasm-filter-2.wasm - name: envoyextensionpolicy/envoy-gateway/policy-for-gateway/1 + originalDownloadingURL: oci://www.example.com/wasm-filter-2:v1.0.0 + servingURL: https://envoy-gateway:18002/7abf116e5cd5a20389604a5ba0f3bd04fdf76f92181fe67506b42c2ee596d3fd.wasm + sha256: 314100af781b98a8ca175d5bf90a8bf76576e20a2f397a88223404edc6ebfd46 + name: envoyextensionpolicy/envoy-gateway/policy-for-gateway/wasm/1 + rootID: my-root-id wasmName: wasm-filter-2 + - config: null + failOpen: false + httpWasmCode: + originalDownloadingURL: oci://www.example.com:8080/wasm-filter-3:latest + servingURL: https://envoy-gateway:18002/42d30b4a4cc631415e6e48c02d244700da327201eb273f752cacf745715b31d9.wasm + sha256: 2a19e4f337e5223d7287e7fccd933fb01905deaff804292e5257f8c681b82bee + name: envoyextensionpolicy/envoy-gateway/policy-for-gateway/wasm/2 + wasmName: envoyextensionpolicy/envoy-gateway/policy-for-gateway/wasm/2 diff --git a/internal/gatewayapi/translator.go b/internal/gatewayapi/translator.go index 191a75d460f..1be3f59f229 100644 --- a/internal/gatewayapi/translator.go +++ b/internal/gatewayapi/translator.go @@ -17,6 +17,7 @@ import ( egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/ir" + "github.com/envoyproxy/gateway/internal/wasm" ) const ( @@ -106,6 +107,9 @@ type Translator struct { // Namespace is the namespace that Envoy Gateway runs in. Namespace string + + // WasmCache is the cache for Wasm modules. + WasmCache wasm.Cache } type TranslateResult struct { diff --git a/internal/gatewayapi/translator_test.go b/internal/gatewayapi/translator_test.go index 1f5c43b1762..062808af580 100644 --- a/internal/gatewayapi/translator_test.go +++ b/internal/gatewayapi/translator_test.go @@ -7,6 +7,9 @@ package gatewayapi import ( "bufio" + "context" + "crypto/sha256" + "encoding/hex" "flag" "fmt" "os" @@ -32,6 +35,7 @@ import ( egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/utils/field" "github.com/envoyproxy/gateway/internal/utils/file" + "github.com/envoyproxy/gateway/internal/wasm" ) var overrideTestData = flag.Bool("override-testdata", false, "if override the test output data.") @@ -85,6 +89,7 @@ func TestTranslate(t *testing.T) { BackendEnabled: backendEnabled, Namespace: "envoy-gateway-system", MergeGateways: IsMergeGatewaysEnabled(resources), + WasmCache: &mockWasmCache{}, } // Add common test fixtures @@ -812,3 +817,22 @@ func TestServicePortToContainerPort(t *testing.T) { assert.Equal(t, tc.containerPort, got) } } + +var _ wasm.Cache = &mockWasmCache{} + +type mockWasmCache struct{} + +func (m *mockWasmCache) Start(_ context.Context) {} + +func (m *mockWasmCache) Get(downloadURL string, _ wasm.GetOptions) (url string, checksum string, err error) { + // This is a mock implementation of the wasm.Cache.Get method. + sha := sha256.Sum256([]byte(downloadURL)) + hashedName := hex.EncodeToString(sha[:]) + salt := []byte("salt") + salt = append(salt, hashedName...) + sha = sha256.Sum256(salt) + checksum = hex.EncodeToString(sha[:]) + return fmt.Sprintf("https://envoy-gateway:18002/%s.wasm", hashedName), checksum, nil +} + +func (m *mockWasmCache) Cleanup() {} diff --git a/internal/gatewayapi/validate.go b/internal/gatewayapi/validate.go index f2172f2726c..5c442812d8b 100644 --- a/internal/gatewayapi/validate.go +++ b/internal/gatewayapi/validate.go @@ -895,6 +895,8 @@ func (t *Translator) validateHostname(hostname string) error { // 2. If the secret reference is a cross-namespace reference, // is it permitted by any ReferenceGrant // 3. Does the secret exist +// +// nolint:unparam func (t *Translator) validateSecretRef( allowCrossNamespace bool, from crossNamespaceFrom, diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/custom.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/custom.yaml index 8c49a94297c..b116a942356 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/custom.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/custom.yaml @@ -163,6 +163,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/default-env.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/default-env.yaml index ac875d88b8e..9b374a2f59c 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/default-env.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/default-env.yaml @@ -161,6 +161,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/default.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/default.yaml index b27d515a915..88a041e64c2 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/default.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/default.yaml @@ -161,6 +161,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/disable-prometheus.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/disable-prometheus.yaml index 842407c32c7..9fe8dfd69e8 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/disable-prometheus.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/disable-prometheus.yaml @@ -135,6 +135,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/extension-env.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/extension-env.yaml index 97a00431c75..d2049910591 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/extension-env.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/extension-env.yaml @@ -161,6 +161,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/override-labels-and-annotations.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/override-labels-and-annotations.yaml index 5a19dc72f0d..d294b0b680a 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/override-labels-and-annotations.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/override-labels-and-annotations.yaml @@ -172,6 +172,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/patch-daemonset.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/patch-daemonset.yaml index 4728969d70b..c49c85dd504 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/patch-daemonset.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/patch-daemonset.yaml @@ -161,6 +161,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/shutdown-manager.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/shutdown-manager.yaml index a5a5e85e728..ed8c1ab331b 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/shutdown-manager.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/shutdown-manager.yaml @@ -161,6 +161,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/volumes.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/volumes.yaml index 8553fda9705..9f1b54d5f14 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/volumes.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/volumes.yaml @@ -161,6 +161,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-annotations.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-annotations.yaml index bd8b749a1d3..f59049a6776 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-annotations.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-annotations.yaml @@ -166,6 +166,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-extra-args.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-extra-args.yaml index 8e7a529ce1c..93b5b6ed312 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-extra-args.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-extra-args.yaml @@ -161,6 +161,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-image-pull-secrets.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-image-pull-secrets.yaml index 1469fb21616..f1619138643 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-image-pull-secrets.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-image-pull-secrets.yaml @@ -161,6 +161,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-name.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-name.yaml index 3820ca17ead..9623f718755 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-name.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-name.yaml @@ -161,6 +161,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-node-selector.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-node-selector.yaml index d0618604cf5..c41878beee9 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-node-selector.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-node-selector.yaml @@ -161,6 +161,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-topology-spread-constraints.yaml b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-topology-spread-constraints.yaml index 0986faca79a..1152f25816a 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-topology-spread-constraints.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/daemonsets/with-topology-spread-constraints.yaml @@ -161,6 +161,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml index be6b2d5d225..7b6dbacf6c1 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml @@ -168,6 +168,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom_with_initcontainers.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom_with_initcontainers.yaml index d340fce372d..fc4f7d4db4a 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom_with_initcontainers.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom_with_initcontainers.yaml @@ -168,6 +168,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/default-env.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/default-env.yaml index ea0fa6907a0..ae5de51bab0 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/default-env.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/default-env.yaml @@ -166,6 +166,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml index 3fd0948f2ee..6d7c6736dff 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml @@ -165,6 +165,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/disable-prometheus.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/disable-prometheus.yaml index 14342f83286..07ba78e2f6c 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/disable-prometheus.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/disable-prometheus.yaml @@ -139,6 +139,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/extension-env.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/extension-env.yaml index 2c371d90811..4afb4a82326 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/extension-env.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/extension-env.yaml @@ -166,6 +166,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/override-labels-and-annotations.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/override-labels-and-annotations.yaml index 1e65cc299ec..f856c4231a6 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/override-labels-and-annotations.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/override-labels-and-annotations.yaml @@ -176,6 +176,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/patch-deployment.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/patch-deployment.yaml index 72757a325b7..636b505295b 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/patch-deployment.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/patch-deployment.yaml @@ -165,6 +165,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/shutdown-manager.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/shutdown-manager.yaml index 12c9ad5766f..ba51e4461c6 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/shutdown-manager.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/shutdown-manager.yaml @@ -165,6 +165,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/volumes.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/volumes.yaml index 2fecd59712f..e91cc8f9fbd 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/volumes.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/volumes.yaml @@ -166,6 +166,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-annotations.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-annotations.yaml index 65cea34a8d9..81186f66df5 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-annotations.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-annotations.yaml @@ -170,6 +170,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-empty-memory-limits.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-empty-memory-limits.yaml index f60fec2e8eb..7a02bcef713 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-empty-memory-limits.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-empty-memory-limits.yaml @@ -165,6 +165,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-extra-args.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-extra-args.yaml index 2085acacc0e..4db28471d1b 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-extra-args.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-extra-args.yaml @@ -165,6 +165,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-image-pull-secrets.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-image-pull-secrets.yaml index e1e2b22b957..b4e9d58e882 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-image-pull-secrets.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-image-pull-secrets.yaml @@ -165,6 +165,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-name.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-name.yaml index 91b38ecbd3d..2e2844be152 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-name.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-name.yaml @@ -165,6 +165,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-node-selector.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-node-selector.yaml index 08f9e6a0b85..9b29ce09f59 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-node-selector.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-node-selector.yaml @@ -165,6 +165,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-topology-spread-constraints.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-topology-spread-constraints.yaml index 1b734654109..6daf8140b3c 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-topology-spread-constraints.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/with-topology-spread-constraints.yaml @@ -165,6 +165,44 @@ spec: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/ir/xds.go b/internal/ir/xds.go index d3eb48d853d..b3d3ed3efd9 100644 --- a/internal/ir/xds.go +++ b/internal/ir/xds.go @@ -2206,7 +2206,7 @@ type ExtProc struct { // +k8s:deepcopy-gen=true type Wasm struct { // Name is a unique name for an Wasm configuration. - // The xds translator only generates one ExtProc filter for each unique name. + // The xds translator only generates one Wasm filter for each unique name. Name string `json:"name"` // RootID is a unique ID for a set of extensions in a VM which will share a @@ -2227,17 +2227,26 @@ type Wasm struct { // during the initialization or the execution of the Wasm extension. FailOpen bool `json:"failOpen"` - // HTTPWasmCode is the HTTP Wasm code source. - HTTPWasmCode *HTTPWasmCode `json:"httpWasmCode,omitempty"` + // Code is the HTTP Wasm code source. + // Envoy only supports HTTP Wasm code source. EG downloads the Wasm code from the + // original URL(either an HTTP URL or an OCI image) and serves it through the + // local HTTP server. + Code *HTTPWasmCode `json:"httpWasmCode,omitempty"` } // HTTPWasmCode holds the information associated with the HTTP Wasm code source. +// +k8s:deepcopy-gen=true type HTTPWasmCode struct { - // URL is the URL of the Wasm code. - URL string `json:"url"` + // ServingURL is the URL of the Wasm code served by the local EG HTTP server. + ServingURL string `json:"servingURL"` - // SHA256 checksum that will be used to verify the wasm code. + // SHA256 checksum that will be used by the Envoy to verify the Wasm code. + // It's different from the digest of the OCI image. SHA256 string `json:"sha256"` + + // OriginalURL is the original downloading URL of the Wasm code. + // Note: This field is just used for testing. It's not used to generate the Envoy configuration. + OriginalURL string `json:"originalDownloadingURL"` } // DestinationFilters contains HTTP filters that will be used with the DestinationSetting. diff --git a/internal/ir/zz_generated.deepcopy.go b/internal/ir/zz_generated.deepcopy.go index f4882032c03..4b87aa93b9c 100644 --- a/internal/ir/zz_generated.deepcopy.go +++ b/internal/ir/zz_generated.deepcopy.go @@ -1251,6 +1251,21 @@ func (in *HTTPTimeout) DeepCopy() *HTTPTimeout { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HTTPWasmCode) DeepCopyInto(out *HTTPWasmCode) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPWasmCode. +func (in *HTTPWasmCode) DeepCopy() *HTTPWasmCode { + if in == nil { + return nil + } + out := new(HTTPWasmCode) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HeaderSettings) DeepCopyInto(out *HeaderSettings) { *out = *in @@ -2822,8 +2837,8 @@ func (in *Wasm) DeepCopyInto(out *Wasm) { *out = new(apiextensionsv1.JSON) (*in).DeepCopyInto(*out) } - if in.HTTPWasmCode != nil { - in, out := &in.HTTPWasmCode, &out.HTTPWasmCode + if in.Code != nil { + in, out := &in.Code, &out.Code *out = new(HTTPWasmCode) **out = **in } diff --git a/internal/provider/kubernetes/controller.go b/internal/provider/kubernetes/controller.go index 343d4edc9a7..c1135c9c6f4 100644 --- a/internal/provider/kubernetes/controller.go +++ b/internal/provider/kubernetes/controller.go @@ -1831,6 +1831,7 @@ func (r *gatewayAPIReconciler) processExtensionServerPolicies( // processEnvoyExtensionPolicyObjectRefs adds the referenced resources in EnvoyExtensionPolicies // to the resourceTree // - BackendRefs for ExtProcs +// - SecretRefs for Wasms func (r *gatewayAPIReconciler) processEnvoyExtensionPolicyObjectRefs( ctx context.Context, resourceTree *gatewayapi.Resources, resourceMap *resourceMappings, ) { @@ -1881,5 +1882,23 @@ func (r *gatewayAPIReconciler) processEnvoyExtensionPolicyObjectRefs( } } } + + // Add the referenced SecretRefs in EnvoyExtensionPolicies to the resourceTree + for _, wasm := range policy.Spec.Wasm { + if wasm.Code.Image != nil && wasm.Code.Image.PullSecretRef != nil { + if err := r.processSecretRef( + ctx, + resourceMap, + resourceTree, + gatewayapi.KindSecurityPolicy, + policy.Namespace, + policy.Name, + *wasm.Code.Image.PullSecretRef); err != nil { + r.log.Error(err, + "failed to process Wasm Image PullSecretRef for EnvoyExtensionPolicy", + "policy", policy, "secretRef", wasm.Code.Image.PullSecretRef) + } + } + } } } diff --git a/internal/provider/kubernetes/indexers.go b/internal/provider/kubernetes/indexers.go index ba5bfd51db2..3b6883746a4 100644 --- a/internal/provider/kubernetes/indexers.go +++ b/internal/provider/kubernetes/indexers.go @@ -43,6 +43,7 @@ const ( backendEnvoyExtensionPolicyIndex = "backendEnvoyExtensionPolicyIndex" backendEnvoyProxyTelemetryIndex = "backendEnvoyProxyTelemetryIndex" secretEnvoyProxyIndex = "secretEnvoyProxyIndex" + secretEnvoyExtensionPolicyIndex = "secretEnvoyExtensionPolicyIndex" ) func addReferenceGrantIndexers(ctx context.Context, mgr manager.Manager) error { @@ -652,6 +653,12 @@ func addEnvoyExtensionPolicyIndexers(ctx context.Context, mgr manager.Manager) e return err } + if err = mgr.GetFieldIndexer().IndexField( + ctx, &egv1a1.EnvoyExtensionPolicy{}, secretEnvoyExtensionPolicyIndex, + secretEnvoyExtensionPolicyIndexFunc); err != nil { + return err + } + return nil } @@ -673,3 +680,22 @@ func backendEnvoyExtensionPolicyIndexFunc(rawObj client.Object) []string { return ret } + +func secretEnvoyExtensionPolicyIndexFunc(rawObj client.Object) []string { + envoyExtensionPolicy := rawObj.(*egv1a1.EnvoyExtensionPolicy) + + var ret []string + + for _, wasm := range envoyExtensionPolicy.Spec.Wasm { + if wasm.Code.Image != nil && wasm.Code.Image.PullSecretRef != nil { + secretRef := wasm.Code.Image.PullSecretRef + ret = append(ret, + types.NamespacedName{ + Namespace: gatewayapi.NamespaceDerefOr(secretRef.Namespace, envoyExtensionPolicy.Namespace), + Name: string(secretRef.Name), + }.String()) + } + } + + return ret +} diff --git a/internal/provider/kubernetes/predicates.go b/internal/provider/kubernetes/predicates.go index a2d456c72ac..0e0f984c69c 100644 --- a/internal/provider/kubernetes/predicates.go +++ b/internal/provider/kubernetes/predicates.go @@ -158,6 +158,10 @@ func (r *gatewayAPIReconciler) validateSecretForReconcile(obj client.Object) boo return true } + if r.isExtensionPolicyReferencingSecret(&nsName) { + return true + } + return false } @@ -186,6 +190,7 @@ func (r *gatewayAPIReconciler) isEnvoyProxyReferencingSecret(nsName *types.Names } } } + return false } @@ -623,3 +628,15 @@ func (r *gatewayAPIReconciler) isEnvoyProxyReferencingBackend(nn *types.Namespac return len(proxyList.Items) > 0 } + +func (r *gatewayAPIReconciler) isExtensionPolicyReferencingSecret(nsName *types.NamespacedName) bool { + eepList := &egv1a1.EnvoyExtensionPolicyList{} + if err := r.client.List(context.Background(), eepList, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(secretEnvoyExtensionPolicyIndex, nsName.String()), + }); err != nil { + r.log.Error(err, "unable to find associated ExtensionPolicies") + return false + } + + return len(eepList.Items) > 0 +} diff --git a/internal/provider/kubernetes/predicates_test.go b/internal/provider/kubernetes/predicates_test.go index 41f829fe821..6379263bdb0 100644 --- a/internal/provider/kubernetes/predicates_test.go +++ b/internal/provider/kubernetes/predicates_test.go @@ -316,6 +316,47 @@ func TestValidateSecretForReconcile(t *testing.T) { secret: test.GetSecret(types.NamespacedName{Name: "secret"}), expect: false, }, + { + name: "references EnvoyExtensionPolicy Wasm OCI Image", + configs: []client.Object{ + test.GetGatewayClass("test-gc", egv1a1.GatewayControllerName, nil), + test.GetGateway(types.NamespacedName{Name: "scheduled-status-test"}, "test-gc", 8080), + &egv1a1.EnvoyExtensionPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "wasm-oci", + }, + Spec: egv1a1.EnvoyExtensionPolicySpec{ + PolicyTargetReferences: egv1a1.PolicyTargetReferences{ + TargetRefs: []gwapiv1a2.LocalPolicyTargetReferenceWithSectionName{ + { + LocalPolicyTargetReference: gwapiv1a2.LocalPolicyTargetReference{ + Kind: "Gateway", + Name: "scheduled-status-test", + }, + }, + }, + }, + Wasm: []egv1a1.Wasm{ + { + Name: ptr.To("wasm-filter"), + RootID: ptr.To("my_root_id"), + Code: egv1a1.WasmCodeSource{ + Type: egv1a1.ImageWasmCodeSourceType, + Image: &egv1a1.ImageWasmCodeSource{ + URL: "https://example.com/testwasm:v1.0.0", + PullSecretRef: &gwapiv1b1.SecretObjectReference{ + Name: "secret", + }, + }, + }, + }, + }, + }, + }, + }, + secret: test.GetSecret(types.NamespacedName{Name: "secret"}), + expect: true, + }, } // Create the reconciler. @@ -334,6 +375,7 @@ func TestValidateSecretForReconcile(t *testing.T) { WithIndex(&gwapiv1.Gateway{}, secretGatewayIndex, secretGatewayIndexFunc). WithIndex(&egv1a1.SecurityPolicy{}, secretSecurityPolicyIndex, secretSecurityPolicyIndexFunc). WithIndex(&egv1a1.EnvoyProxy{}, secretEnvoyProxyIndex, secretEnvoyProxyIndexFunc). + WithIndex(&egv1a1.EnvoyExtensionPolicy{}, secretEnvoyExtensionPolicyIndex, secretEnvoyExtensionPolicyIndexFunc). Build() t.Run(tc.name, func(t *testing.T) { res := r.validateSecretForReconcile(tc.secret) diff --git a/internal/wasm/cache.go b/internal/wasm/cache.go new file mode 100644 index 00000000000..8bd94508bdf --- /dev/null +++ b/internal/wasm/cache.go @@ -0,0 +1,495 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// Copyright Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wasm + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "github.com/google/go-containerregistry/pkg/name" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/envoyproxy/gateway/internal/logging" +) + +const ( + // oci URL prefix + ociURLPrefix = "oci://" + // sha256 scheme prefix + sha256SchemePrefix = "sha256:" +) + +// Cache models a Wasm module cache. +type Cache interface { + Get(downloadURL string, opts GetOptions) (url string, checksum string, err error) + Start(ctx context.Context) +} + +// localFileCache for downloaded Wasm modules. It stores the Wasm module as local files. +type localFileCache struct { + // Map from Wasm module key to cache entry. + modules map[moduleKey]*cacheEntry + // Map from downloading URL to checksum + checksums map[string]*checksumEntry + // http fetcher fetches Wasm module with HTTP get. + httpFetcher *HTTPFetcher + + // mux is needed because stale Wasm module files will be purged periodically. + mux sync.Mutex + + // option sets for configuring the cache. + CacheOptions + + // logger + logger logging.Logger +} + +func (c *localFileCache) Start(ctx context.Context) { + go c.purge(ctx) +} + +var _ Cache = &localFileCache{} + +type checksumEntry struct { + checksum string + // Keeps the resource version per each resource for dealing with multiple resources which pointing the same image. + resourceVersionByResource map[string]string +} + +// moduleKey is a unique identifier for a Wasm module consisting of the name and checksum. +type moduleKey struct { + // Identifier for the wasm module. + // If the wasm module is an HTTP URL, it is the original download URL. + // e.g. http://example.com/test.wasm + // If the wasm module is an OCI image, it should be the image name without tag or digest. + // e.g. oci://docker.io/test + name string + // sha256 checksum of the wasm file or the image. + // Note that the checksum is different from the checksum of the wasm file if + // the module is extracted from an OCI image. + checksum string +} + +type cacheKey struct { + moduleKey + // URL to download the wasm module. + // e.g. http://example.com/test.wasm or oci://docker.io/test:v1.0.0 + downloadURL string + // Resource name of the wasm module. This should be a fully-qualified name. + // e.g. "envoyextensionpolicy/envoy-gateway/policy-for-gateway/wasm/0" + resourceName string + // Resource version of EnvoyExtensionPolicy resource. Even though PullPolicy is Always, + // if there is no change of resource state, a cached entry is used instead of pulling newly. + resourceVersion string +} + +// cacheEntry contains information about a Wasm module cache entry. +type cacheEntry struct { + // File path to the downloaded wasm modules. + modulePath string + // Last time that this local Wasm module is referenced. + last time.Time + // set of URLs referencing this entry + referencingURLs sets.Set[string] + // isPrivate is true if the module is from a private registry. + isPrivate bool + // checksum is the sha256 checksum of the module. + // It is different from the checksum of the image if the module is from an OCI image. + checksum string + // size is the size of the module. + size int +} + +// newLocalFileCache create a new Wasm module cache which downloads and stores Wasm module files locally. +func newLocalFileCache(options CacheOptions, logger logging.Logger) *localFileCache { + options = options.sanitize() + cache := &localFileCache{ + httpFetcher: NewHTTPFetcher(options.HTTPRequestTimeout, options.HTTPRequestMaxRetries, logger), + modules: make(map[moduleKey]*cacheEntry), + checksums: make(map[string]*checksumEntry), + CacheOptions: options, + logger: logger, + } + + return cache +} + +func moduleNameFromURL(fullURLStr string) string { + if strings.HasPrefix(fullURLStr, ociURLPrefix) { + if tag, err := name.ParseReference(fullURLStr[len(ociURLPrefix):]); err == nil { + // remove tag or sha + return ociURLPrefix + tag.Context().Name() + } + } + return fullURLStr +} + +func getModulePath(baseDir string, mkey moduleKey) (string, error) { + // Use sha256 checksum as the name of the module. + sha := sha256.Sum256([]byte(mkey.name)) + hashedName := hex.EncodeToString(sha[:]) + moduleDir := filepath.Join(baseDir, hashedName) + if err := os.Mkdir(moduleDir, 0o755); err != nil && !os.IsExist(err) { + return "", err + } + return filepath.Join(moduleDir, fmt.Sprintf("%s.wasm", mkey.checksum)), nil +} + +// Get returns path the local Wasm module file and its checksum. +func (c *localFileCache) Get(downloadURL string, opts GetOptions) (localFile string, checksum string, err error) { + // If the checksum is not provided, try to extract it from the OCI image URL. + originalChecksum := opts.Checksum + if len(opts.Checksum) == 0 && strings.HasPrefix(downloadURL, ociURLPrefix) { + if d, err := name.NewDigest(downloadURL[len(ociURLPrefix):]); err == nil { + // If there is no checksum and the digest is suffixed in URL, use the digest. + dstr := d.DigestStr() + if strings.HasPrefix(dstr, sha256SchemePrefix) { + originalChecksum = dstr[len(sha256SchemePrefix):] + } + } + } + + // Construct Wasm cache key with downloading URL and provided checksum of the module. + key := cacheKey{ + downloadURL: downloadURL, + moduleKey: moduleKey{ + name: moduleNameFromURL(downloadURL), + checksum: originalChecksum, + }, + resourceName: opts.ResourceName, + resourceVersion: opts.ResourceVersion, + } + + entry, err := c.getOrFetch(key, opts) + if err != nil { + return "", "", err + } + + return entry.modulePath, entry.checksum, err +} + +func (c *localFileCache) getOrFetch(key cacheKey, opts GetOptions) (*cacheEntry, error) { + var ( + u *url.URL + insecure bool + isPrivate bool + err error + ) + + if u, err = url.Parse(key.downloadURL); err != nil { + return nil, fmt.Errorf("fail to parse Wasm module fetch url: %s, error: %w", key.downloadURL, err) + } + insecure = c.allowInsecure(u.Host) + + requestTimout := DefaultPullTimeout + if opts.RequestTimeout != 0 { + requestTimout = opts.RequestTimeout + } + ctx, cancel := context.WithTimeout(context.Background(), requestTimout) + defer cancel() + + // First check if the cache entry is already downloaded and policy does not require pulling always. + ce := c.getEntry(key, opts.PullPolicy, u) + if ce != nil { + // We still need to check if the pull secret is correct if it is a private OCI image. + if u.Scheme == "oci" && ce.isPrivate { + if err = c.checkPermission(ctx, u, insecure, opts); err != nil { + return nil, err + } + } + return ce, nil + } + + // Fetch the image now as it is not available in cache. + var ( + b []byte // Byte array of Wasm binary. + dChecksum string // Hex-Encoded sha256 checksum of binary. + imageBinaryFetcher func() ([]byte, error) + ) + + switch u.Scheme { + case "http", "https": + // Download the Wasm module with http fetcher. + b, err = c.httpFetcher.Fetch(ctx, key.downloadURL, insecure) + if err != nil { + wasmRemoteFetchCount.With(resultTag.Value(downloadFailure)).Increment() + return nil, err + } + + // Get sha256 checksum and check if it is the same as the provided one. + sha := sha256.Sum256(b) + dChecksum = hex.EncodeToString(sha[:]) + case "oci": + if opts.PullSecret != nil && len(opts.PullSecret) > 0 { + isPrivate = true + } + if imageBinaryFetcher, dChecksum, err = c.prepareFetch(ctx, u, insecure, opts); err != nil { + wasmRemoteFetchCount.With(resultTag.Value(manifestFailure)).Increment() + return nil, fmt.Errorf("could not fetch Wasm OCI image: %w", err) + } + default: + return nil, fmt.Errorf("unsupported Wasm module downloading URL scheme: %v", u.Scheme) + } + + // If the checksum is provided, check if it matches the downloaded binary. + if key.checksum != "" { + if dChecksum != key.checksum { + wasmRemoteFetchCount.With(resultTag.Value(checksumMismatch)).Increment() + return nil, fmt.Errorf("module downloaded from %v has checksum %v, which does not match: %v", key.downloadURL, dChecksum, key.checksum) + } + } else { + // Update the checksum with the one from the downloaded binary. + key.checksum = dChecksum + } + + if imageBinaryFetcher != nil { + b, err = imageBinaryFetcher() + if err != nil { + wasmRemoteFetchCount.With(resultTag.Value(downloadFailure)).Increment() + return nil, fmt.Errorf("could not fetch Wasm binary: %w", err) + } + } + + if !isValidWasmBinary(b) { + wasmRemoteFetchCount.With(resultTag.Value(fetchFailure)).Increment() + return nil, fmt.Errorf("fetched Wasm binary from %s is invalid", key.downloadURL) + } + + wasmRemoteFetchCount.With(resultTag.Value(fetchSuccess)).Increment() + return c.addEntry(key, b, isPrivate) +} + +func (c *localFileCache) checkPermission(ctx context.Context, u *url.URL, insecure bool, opts GetOptions) error { + // Try to get the image metadata to check if the pull secret is correct. + if _, _, err := c.prepareFetch(ctx, u, insecure, opts); err != nil { + return fmt.Errorf("failed to login to private registry: %w", err) + } + return nil +} + +// prepareFetch won't fetch the binary, but it will prepare the binaryFetcher and actualDigest. +func (c *localFileCache) prepareFetch( + ctx context.Context, url *url.URL, insecure bool, opts GetOptions) ( + binaryFetcher func() ([]byte, error), actualDigest string, err error, +) { + imgFetcherOps := ImageFetcherOption{ + Insecure: insecure, + } + if opts.PullSecret != nil && len(opts.PullSecret) > 0 { + imgFetcherOps.PullSecret = opts.PullSecret + } + fetcher := NewImageFetcher(ctx, imgFetcherOps, c.logger) + if binaryFetcher, actualDigest, err = fetcher.PrepareFetch(url.Host + url.Path); err != nil { + return nil, "", err + } + return binaryFetcher, actualDigest, nil +} + +func (c *localFileCache) updateChecksum(key cacheKey) { + ce := c.checksums[key.downloadURL] + if ce == nil { + ce = new(checksumEntry) + ce.resourceVersionByResource = make(map[string]string) + c.checksums[key.downloadURL] = ce + } + ce.checksum = key.checksum + ce.resourceVersionByResource[key.resourceName] = key.resourceVersion +} + +// addEntry adds a wasmModule to cache with cacheKey, writes the module to the local file system, +// and returns the created entry. +func (c *localFileCache) addEntry(key cacheKey, wasmModule []byte, isPrivate bool) (*cacheEntry, error) { + c.mux.Lock() + defer c.mux.Unlock() + + // Check if the cache size exceeds the limit. + if c.size()+len(wasmModule) > c.MaxCacheSize { + return nil, fmt.Errorf("wasm cache size exceeded the limit: %d", c.MaxCacheSize) + } + + c.updateChecksum(key) + + // Check if the module has already been added. If so, avoid writing the file again. + if ce, ok := c.modules[key.moduleKey]; ok { + // Update last touched time. + ce.last = time.Now() + ce.referencingURLs.Insert(key.downloadURL) + return ce, nil + } + + modulePath, err := getModulePath(c.CacheDir, key.moduleKey) + if err != nil { + return nil, err + } + // Materialize the Wasm module into a local file. Use checksum as name of the module. + if err := os.WriteFile(modulePath, wasmModule, 0o600); err != nil { + return nil, err + } + + // Calculate the checksum of the wasm module. It is different from the checksum of the image. + wasmChecksum := strings.ToLower(fmt.Sprintf("%x", sha256.Sum256(wasmModule))) + ce := cacheEntry{ + modulePath: modulePath, + last: time.Now(), + referencingURLs: sets.New[string](), + isPrivate: isPrivate, + checksum: wasmChecksum, + size: len(wasmModule), + } + ce.referencingURLs.Insert(key.downloadURL) + c.modules[key.moduleKey] = &ce + wasmCacheEntries.Record(float64(len(c.modules))) + return &ce, nil +} + +// getEntry finds a cached module, and returns the found cache entry and its checksum. +// If the module is not found in the cache, it returns nil. +// If the module is found in the cache, but the module needs to be re-pulled, it returns nil. +func (c *localFileCache) getEntry(key cacheKey, pullPolicy PullPolicy, u *url.URL) *cacheEntry { + cacheHit := false + + c.mux.Lock() + defer func() { + c.mux.Unlock() + wasmCacheLookupCount.With(hitTag.Value(strconv.FormatBool(cacheHit))).Increment() + }() + + // If no checksum is provided, check if a wasm module with the same downloading URL has been pulled before. + if len(key.checksum) == 0 { + // If an image with the same downloading URL was pulled before, there should be a checksum of the most recently pulled image. + if ce, found := c.checksums[key.downloadURL]; found { + // If it is an OCI image and the tag is "latest", default pull policy is Always. + // Otherwise, default pull policy is IfNotPresent. + if pullPolicy == Unspecified { + if u.Scheme == "oci" && strings.HasSuffix(u.Path, ":latest") { + pullPolicy = Always + } else { + pullPolicy = IfNotPresent + } + } + + // Check if we need to re-pull the wasm module. + needPull := true + switch pullPolicy { + case IfNotPresent: + needPull = false + case Always: + // If the resource version is not changed, use the cached wasm module. + // Otherwise, pull the new one from its original URL. + if key.resourceVersion == ce.resourceVersionByResource[key.resourceName] { + needPull = false + } + } + + // If we need to re-pull this wasm module, return nil. + if needPull { + return nil + } + + // If we don't need to pull the module again, return the cached module. + key.checksum = ce.checksum + existingModule := c.modules[key.moduleKey] + // Update last touched time. + existingModule.last = time.Now() + cacheHit = true + // Update the checksum map as the same downloading URL can be referenced + // by multiple EnvoyExtensionPolicy resources. + c.updateChecksum(key) + return existingModule + } + + // If no previous checksum is found, return nil. + return nil + } + + // If the checksum is provided, check if the module with the same checksum has been pulled before. + if existingModule, ok := c.modules[key.moduleKey]; ok { + // Update last touched time. + existingModule.last = time.Now() + cacheHit = true + // Update the checksum map as the same downloading URL can be referenced + // by multiple EnvoyExtensionPolicy resources. + c.updateChecksum(key) + return existingModule + } + return nil +} + +func (c *localFileCache) size() int { + cacheSize := 0 + for _, entry := range c.modules { + cacheSize += entry.size + } + return cacheSize +} + +// Purge periodically clean up the stale Wasm modules local file and the cache map. +func (c *localFileCache) purge(ctx context.Context) { + ticker := time.NewTicker(c.PurgeInterval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + c.mux.Lock() + for k, m := range c.modules { + if !m.expired(c.ModuleExpiry) { + continue + } + // The module has not be touched for expiry duration, delete it from the map as well as the local dir. + if err := os.Remove(m.modulePath); err != nil { + c.logger.Error(err, "failed to purge Wasm module", "path", m.modulePath) + } else { + for downloadURL := range m.referencingURLs { + delete(c.checksums, downloadURL) + } + delete(c.modules, k) + c.logger.Info("successfully removed stale Wasm module", "path", m.modulePath) + } + } + wasmCacheEntries.Record(float64(len(c.modules))) + c.mux.Unlock() + case <-ctx.Done(): + return + } + } +} + +// Expired returns true if the module has not been touched for Wasm module Expiry. +func (ce *cacheEntry) expired(expiry time.Duration) bool { + now := time.Now() + return now.Sub(ce.last) > expiry +} + +var wasmMagicNumber = []byte{0x00, 0x61, 0x73, 0x6d} + +func isValidWasmBinary(in []byte) bool { + // Wasm file header is 8 bytes (magic number + version). + return len(in) >= 8 && bytes.Equal(in[:4], wasmMagicNumber) +} diff --git a/internal/wasm/cache_test.go b/internal/wasm/cache_test.go new file mode 100644 index 00000000000..ba40c0625d8 --- /dev/null +++ b/internal/wasm/cache_test.go @@ -0,0 +1,993 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// Copyright Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wasm + +import ( + "context" + "crypto/sha256" + "crypto/tls" + "encoding/hex" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/types" + "k8s.io/apimachinery/pkg/util/sets" + + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" + "github.com/envoyproxy/gateway/internal/logging" +) + +const wasmTestData = "this is wasm plugin" + +// Wasm header = magic number (4 bytes) + Wasm spec version (4 bytes). +var wasmHeader = append(wasmMagicNumber, []byte{0x1, 0x00, 0x00, 0x00}...) + +func TestWasmCache(t *testing.T) { + // Setup http server. + tsNumRequest := int32(0) + + wasmData := wasmHeader + wasmData = append(wasmData, wasmTestData...) + invalidHTTPData := []byte("invalid binary") + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&tsNumRequest, 1) + + switch r.URL.Path { + case "/different-url": + _, _ = w.Write(append(wasmData, []byte("different data")...)) + case "/invalid-wasm-header": + _, _ = w.Write(invalidHTTPData) + default: + _, _ = w.Write(wasmData) + } + })) + defer ts.Close() + wasmDataSha := sha256.Sum256(wasmData) + wasmDataCheckSum := hex.EncodeToString(wasmDataSha[:]) + invalidHTTPDataSha := sha256.Sum256(invalidHTTPData) + invalidHTTPDataCheckSum := hex.EncodeToString(invalidHTTPDataSha[:]) + + reg := registry.New() + // Set up a fake registry for OCI images. + tos := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&tsNumRequest, 1) + reg.ServeHTTP(w, r) + })) + defer tos.Close() + ou, err := url.Parse(tos.URL) + if err != nil { + t.Fatal(err) + } + + dockerImageDigest, invalidOCIImageDigest := setupOCIRegistry(t, ou.Host) + + ociWasmFile := fmt.Sprintf("%s.wasm", dockerImageDigest) + ociURLWithTag := fmt.Sprintf("oci://%s/test/valid/docker:v0.1.0", ou.Host) + ociURLWithLatestTag := fmt.Sprintf("oci://%s/test/valid/docker:latest", ou.Host) + ociURLWithDigest := fmt.Sprintf("oci://%s/test/valid/docker@sha256:%s", ou.Host, dockerImageDigest) + + // Calculate cachehit sum. + cacheHitSha := sha256.Sum256([]byte("cachehit")) + cacheHitSum := hex.EncodeToString(cacheHitSha[:]) + + cases := []struct { + name string + initialCachedModules map[moduleKey]cacheEntry + initialCachedChecksums map[string]*checksumEntry + fetchURL string + purgeInterval time.Duration + wasmModuleExpiry time.Duration + checkPurgeTimeout time.Duration + getOptions GetOptions + wantCachedModules map[moduleKey]*cacheEntry + wantCachedChecksums map[string]*checksumEntry + wantFileName string + wantErrorMsgPrefix string + wantVisitServer bool + }{ + { + name: "cache miss", + initialCachedModules: map[moduleKey]cacheEntry{}, + initialCachedChecksums: map[string]*checksumEntry{}, + fetchURL: ts.URL, + getOptions: GetOptions{ + Checksum: wasmDataCheckSum, + ResourceName: "namespace.resource", + ResourceVersion: "0", + RequestTimeout: time.Second * 10, + }, + wantCachedModules: map[moduleKey]*cacheEntry{ + {name: ts.URL, checksum: wasmDataCheckSum}: {modulePath: wasmDataCheckSum + ".wasm", checksum: wasmDataCheckSum, size: 27}, + }, + wantCachedChecksums: map[string]*checksumEntry{ + ts.URL: {checksum: wasmDataCheckSum, resourceVersionByResource: map[string]string{"namespace.resource": "0"}}, + }, + wantFileName: fmt.Sprintf("%s.wasm", wasmDataCheckSum), + wantVisitServer: true, + }, + { + name: "cache hit", + initialCachedModules: map[moduleKey]cacheEntry{ + {name: moduleNameFromURL(ts.URL), checksum: cacheHitSum}: {modulePath: "test.wasm"}, + }, + initialCachedChecksums: map[string]*checksumEntry{}, + fetchURL: ts.URL, + getOptions: GetOptions{ + Checksum: cacheHitSum, + ResourceName: "namespace.resource", + ResourceVersion: "0", + RequestTimeout: time.Second * 10, + }, + wantCachedModules: map[moduleKey]*cacheEntry{ + {name: ts.URL, checksum: cacheHitSum}: {modulePath: "test.wasm"}, + }, + wantCachedChecksums: map[string]*checksumEntry{ + ts.URL: {checksum: cacheHitSum, resourceVersionByResource: map[string]string{"namespace.resource": "0"}}, + }, + wantFileName: "test.wasm", + wantVisitServer: false, + }, + { + name: "invalid scheme", + initialCachedModules: map[moduleKey]cacheEntry{}, + initialCachedChecksums: map[string]*checksumEntry{}, + fetchURL: "foo://abc", + getOptions: GetOptions{ + Checksum: wasmDataCheckSum, + RequestTimeout: time.Second * 10, + }, + wantCachedModules: map[moduleKey]*cacheEntry{}, + wantCachedChecksums: map[string]*checksumEntry{}, + wantFileName: fmt.Sprintf("%s.wasm", wasmDataCheckSum), + wantErrorMsgPrefix: "unsupported Wasm module downloading URL scheme: foo", + wantVisitServer: false, + }, + { + name: "download failure", + initialCachedModules: map[moduleKey]cacheEntry{}, + initialCachedChecksums: map[string]*checksumEntry{}, + fetchURL: "https://-invalid-url", + getOptions: GetOptions{ + RequestTimeout: time.Second * 10, + }, + wantCachedModules: map[moduleKey]*cacheEntry{}, + wantCachedChecksums: map[string]*checksumEntry{}, + wantErrorMsgPrefix: "wasm module download failed after 5 attempts, last error: Get \"https://-invalid-url\"", + wantVisitServer: false, + }, + { + name: "wrong checksum", + initialCachedModules: map[moduleKey]cacheEntry{}, + initialCachedChecksums: map[string]*checksumEntry{}, + fetchURL: ts.URL, + getOptions: GetOptions{ + Checksum: "wrongchecksum\n", + RequestTimeout: time.Second * 10, + }, + wantCachedModules: map[moduleKey]*cacheEntry{}, + wantCachedChecksums: map[string]*checksumEntry{}, + wantErrorMsgPrefix: fmt.Sprintf("module downloaded from %v has checksum %s, which does not match", ts.URL, wasmDataCheckSum), + wantVisitServer: true, + }, + { + // this might be common error in user configuration, that url was updated, but not checksum. + // Test that downloading still proceeds and error returns. + name: "different url same checksum", + initialCachedModules: map[moduleKey]cacheEntry{ + {name: moduleNameFromURL(ts.URL), checksum: wasmDataCheckSum}: {modulePath: fmt.Sprintf("%s.wasm", wasmDataCheckSum)}, + }, + initialCachedChecksums: map[string]*checksumEntry{}, + fetchURL: ts.URL + "/different-url", + getOptions: GetOptions{ + Checksum: wasmDataCheckSum, + RequestTimeout: time.Second * 10, + }, + wantCachedModules: map[moduleKey]*cacheEntry{ + {name: ts.URL, checksum: wasmDataCheckSum}: {modulePath: wasmDataCheckSum + ".wasm"}, + }, + wantCachedChecksums: map[string]*checksumEntry{}, + wantErrorMsgPrefix: fmt.Sprintf("module downloaded from %v/different-url has checksum", ts.URL), + wantVisitServer: true, + }, + { + name: "invalid wasm header", + initialCachedModules: map[moduleKey]cacheEntry{ + {name: moduleNameFromURL(ts.URL), checksum: wasmDataCheckSum}: {modulePath: fmt.Sprintf("%s.wasm", wasmDataCheckSum)}, + }, + initialCachedChecksums: map[string]*checksumEntry{}, + fetchURL: ts.URL + "/invalid-wasm-header", + getOptions: GetOptions{ + Checksum: invalidHTTPDataCheckSum, + RequestTimeout: time.Second * 10, + }, + wantCachedModules: map[moduleKey]*cacheEntry{ + {name: ts.URL, checksum: wasmDataCheckSum}: {modulePath: wasmDataCheckSum + ".wasm"}, + }, + wantCachedChecksums: map[string]*checksumEntry{}, + wantErrorMsgPrefix: fmt.Sprintf("fetched Wasm binary from %s is invalid", ts.URL+"/invalid-wasm-header"), + wantVisitServer: true, + }, + { + name: "purge on expiry", + initialCachedModules: map[moduleKey]cacheEntry{}, + initialCachedChecksums: map[string]*checksumEntry{}, + fetchURL: ts.URL, + purgeInterval: 1 * time.Millisecond, + wasmModuleExpiry: 1 * time.Millisecond, + checkPurgeTimeout: 5 * time.Second, + getOptions: GetOptions{ + Checksum: wasmDataCheckSum, + RequestTimeout: time.Second * 10, + }, + wantCachedModules: map[moduleKey]*cacheEntry{}, + wantCachedChecksums: map[string]*checksumEntry{}, + wantFileName: fmt.Sprintf("%s.wasm", wasmDataCheckSum), + wantVisitServer: true, + }, + { + name: "fetch oci without digest", + initialCachedModules: map[moduleKey]cacheEntry{}, + initialCachedChecksums: map[string]*checksumEntry{}, + fetchURL: ociURLWithTag, + getOptions: GetOptions{ + ResourceName: "namespace.resource", + ResourceVersion: "0", + RequestTimeout: time.Second * 10, + }, + wantCachedModules: map[moduleKey]*cacheEntry{ + {name: moduleNameFromURL(ociURLWithTag), checksum: dockerImageDigest}: {modulePath: ociWasmFile, checksum: wasmDataCheckSum, size: 27}, + }, + wantCachedChecksums: map[string]*checksumEntry{ + ociURLWithTag: {checksum: dockerImageDigest, resourceVersionByResource: map[string]string{"namespace.resource": "0"}}, + }, + wantFileName: ociWasmFile, + wantVisitServer: true, + }, + { + name: "fetch oci with digest", + initialCachedModules: map[moduleKey]cacheEntry{}, + initialCachedChecksums: map[string]*checksumEntry{}, + fetchURL: ociURLWithTag, + getOptions: GetOptions{ + Checksum: dockerImageDigest, + ResourceName: "namespace.resource", + ResourceVersion: "0", + RequestTimeout: time.Second * 10, + }, + wantCachedModules: map[moduleKey]*cacheEntry{ + {name: moduleNameFromURL(ociURLWithTag), checksum: dockerImageDigest}: {modulePath: ociWasmFile, checksum: wasmDataCheckSum, size: 27}, + }, + wantCachedChecksums: map[string]*checksumEntry{ + ociURLWithTag: {checksum: dockerImageDigest, resourceVersionByResource: map[string]string{"namespace.resource": "0"}}, + }, + wantFileName: ociWasmFile, + wantVisitServer: true, + }, + { + name: "cache hit for tagged oci url with digest", + initialCachedModules: map[moduleKey]cacheEntry{ + {name: moduleNameFromURL(ociURLWithTag), checksum: dockerImageDigest}: {modulePath: ociWasmFile}, + }, + initialCachedChecksums: map[string]*checksumEntry{}, + fetchURL: ociURLWithTag, + getOptions: GetOptions{ + Checksum: dockerImageDigest, + ResourceName: "namespace.resource", + ResourceVersion: "0", + RequestTimeout: time.Second * 10, + }, + wantCachedModules: map[moduleKey]*cacheEntry{ + {name: moduleNameFromURL(ociURLWithTag), checksum: dockerImageDigest}: {modulePath: ociWasmFile}, + }, + wantCachedChecksums: map[string]*checksumEntry{ + ociURLWithTag: {checksum: dockerImageDigest, resourceVersionByResource: map[string]string{"namespace.resource": "0"}}, + }, + wantFileName: ociWasmFile, + wantVisitServer: false, + }, + { + name: "cache hit for tagged oci url without digest", + initialCachedModules: map[moduleKey]cacheEntry{ + {name: moduleNameFromURL(ociURLWithTag), checksum: dockerImageDigest}: {modulePath: ociWasmFile}, + }, + initialCachedChecksums: map[string]*checksumEntry{ + ociURLWithTag: { + checksum: dockerImageDigest, + resourceVersionByResource: map[string]string{ + "namespace.resource": "123456", + }, + }, + }, + fetchURL: ociURLWithTag, + getOptions: GetOptions{ + ResourceName: "namespace.resource", + ResourceVersion: "0", + RequestTimeout: time.Second * 10, + }, + wantCachedModules: map[moduleKey]*cacheEntry{ + {name: moduleNameFromURL(ociURLWithTag), checksum: dockerImageDigest}: {modulePath: ociWasmFile}, + }, + wantCachedChecksums: map[string]*checksumEntry{ + ociURLWithTag: {checksum: dockerImageDigest, resourceVersionByResource: map[string]string{"namespace.resource": "0"}}, + }, + wantFileName: ociWasmFile, + wantVisitServer: false, + }, + { + name: "cache miss for tagged oci url without digest", + initialCachedModules: map[moduleKey]cacheEntry{ + {name: moduleNameFromURL(ociURLWithTag), checksum: dockerImageDigest}: {modulePath: ociWasmFile}, + }, + initialCachedChecksums: map[string]*checksumEntry{}, + getOptions: GetOptions{ + ResourceName: "namespace.resource", + ResourceVersion: "0", + RequestTimeout: time.Second * 10, + }, + fetchURL: ociURLWithTag, + wantCachedModules: map[moduleKey]*cacheEntry{ + {name: moduleNameFromURL(ociURLWithTag), checksum: dockerImageDigest}: {modulePath: ociWasmFile}, + }, + wantCachedChecksums: map[string]*checksumEntry{ + ociURLWithTag: {checksum: dockerImageDigest, resourceVersionByResource: map[string]string{"namespace.resource": "0"}}, + }, + wantFileName: ociWasmFile, + wantVisitServer: true, + }, + { + name: "cache hit for oci url suffixed by digest", + initialCachedModules: map[moduleKey]cacheEntry{ + {name: moduleNameFromURL(ociURLWithTag), checksum: dockerImageDigest}: {modulePath: ociWasmFile}, + }, + initialCachedChecksums: map[string]*checksumEntry{}, + fetchURL: ociURLWithDigest, + getOptions: GetOptions{ + ResourceName: "namespace.resource", + ResourceVersion: "0", + RequestTimeout: time.Second * 10, + }, + wantCachedModules: map[moduleKey]*cacheEntry{ + {name: moduleNameFromURL(ociURLWithTag), checksum: dockerImageDigest}: {modulePath: ociWasmFile}, + }, + // We don't need checksum for OCI images with digest, but we still store it to make the code simpler. + wantCachedChecksums: map[string]*checksumEntry{ + ociURLWithDigest: {checksum: dockerImageDigest, resourceVersionByResource: map[string]string{"namespace.resource": "0"}}, + }, + wantFileName: ociWasmFile, + wantVisitServer: false, + }, + { + name: "pull due to pull-always policy when cache hit", + initialCachedModules: map[moduleKey]cacheEntry{ + {name: moduleNameFromURL(ociURLWithTag), checksum: dockerImageDigest}: {modulePath: ociWasmFile}, + }, + initialCachedChecksums: map[string]*checksumEntry{ + ociURLWithTag: { + checksum: dockerImageDigest, + resourceVersionByResource: map[string]string{ + "namespace.resource": "123456", + }, + }, + }, + fetchURL: ociURLWithTag, + getOptions: GetOptions{ + ResourceName: "namespace.resource", + ResourceVersion: "0", + RequestTimeout: time.Second * 10, + PullPolicy: Always, + }, + wantCachedModules: map[moduleKey]*cacheEntry{ + {name: moduleNameFromURL(ociURLWithTag), checksum: dockerImageDigest}: {modulePath: ociWasmFile}, + }, + wantCachedChecksums: map[string]*checksumEntry{ + ociURLWithTag: {checksum: dockerImageDigest, resourceVersionByResource: map[string]string{"namespace.resource": "0"}}, + }, + wantFileName: ociWasmFile, + wantVisitServer: true, + }, + { + name: "do not pull due to resourceVersion is the same", + initialCachedModules: map[moduleKey]cacheEntry{ + {name: moduleNameFromURL(ociURLWithTag), checksum: dockerImageDigest}: {modulePath: ociWasmFile}, + }, + initialCachedChecksums: map[string]*checksumEntry{ + ociURLWithTag: { + checksum: dockerImageDigest, + resourceVersionByResource: map[string]string{ + "namespace.resource": "123456", + }, + }, + }, + fetchURL: ociURLWithTag, + getOptions: GetOptions{ + ResourceName: "namespace.resource", + ResourceVersion: "123456", + RequestTimeout: time.Second * 10, + PullPolicy: Always, + }, + wantCachedModules: map[moduleKey]*cacheEntry{ + {name: moduleNameFromURL(ociURLWithTag), checksum: dockerImageDigest}: {modulePath: ociWasmFile}, + }, + wantCachedChecksums: map[string]*checksumEntry{ + ociURLWithTag: {checksum: dockerImageDigest, resourceVersionByResource: map[string]string{"namespace.resource": "123456"}}, + }, + wantFileName: ociWasmFile, + wantVisitServer: false, + }, + { + name: "pull due to if-not-present policy when cache hit", + initialCachedModules: map[moduleKey]cacheEntry{ + {name: moduleNameFromURL(ociURLWithTag), checksum: dockerImageDigest}: {modulePath: ociWasmFile}, + }, + initialCachedChecksums: map[string]*checksumEntry{ + ociURLWithTag: { + checksum: dockerImageDigest, + resourceVersionByResource: map[string]string{ + "namespace.resource": "123456", + }, + }, + }, + fetchURL: ociURLWithTag, + getOptions: GetOptions{ + ResourceName: "namespace.resource", + ResourceVersion: "0", + RequestTimeout: time.Second * 10, + PullPolicy: IfNotPresent, + }, + wantCachedModules: map[moduleKey]*cacheEntry{ + {name: moduleNameFromURL(ociURLWithTag), checksum: dockerImageDigest}: {modulePath: ociWasmFile}, + }, + wantCachedChecksums: map[string]*checksumEntry{ + ociURLWithTag: {checksum: dockerImageDigest, resourceVersionByResource: map[string]string{"namespace.resource": "0"}}, + }, + wantFileName: ociWasmFile, + wantVisitServer: false, + }, + { + name: "do not pull in spite of pull-always policy due to checksum", + initialCachedModules: map[moduleKey]cacheEntry{ + {name: moduleNameFromURL(ociURLWithTag), checksum: dockerImageDigest}: {modulePath: ociWasmFile}, + }, + fetchURL: ociURLWithTag, + getOptions: GetOptions{ + Checksum: dockerImageDigest, + ResourceName: "namespace.resource", + ResourceVersion: "0", + RequestTimeout: time.Second * 10, + PullPolicy: Always, + }, + wantCachedModules: map[moduleKey]*cacheEntry{ + {name: moduleNameFromURL(ociURLWithTag), checksum: dockerImageDigest}: {modulePath: ociWasmFile}, + }, + wantCachedChecksums: map[string]*checksumEntry{ + ociURLWithTag: {checksum: dockerImageDigest, resourceVersionByResource: map[string]string{"namespace.resource": "0"}}, + }, + wantFileName: ociWasmFile, + wantVisitServer: false, + }, + { + name: "pull due to latest tag", + initialCachedModules: map[moduleKey]cacheEntry{ + {name: moduleNameFromURL(ociURLWithLatestTag), checksum: dockerImageDigest}: {modulePath: ociWasmFile}, + }, + initialCachedChecksums: map[string]*checksumEntry{ + ociURLWithLatestTag: { + checksum: dockerImageDigest, + resourceVersionByResource: map[string]string{ + "namespace.resource": "123456", + }, + }, + }, + fetchURL: ociURLWithLatestTag, + getOptions: GetOptions{ + ResourceName: "namespace.resource", + ResourceVersion: "0", + RequestTimeout: time.Second * 10, + PullPolicy: Unspecified, // Default policy + }, + wantCachedModules: map[moduleKey]*cacheEntry{ + {name: moduleNameFromURL(ociURLWithLatestTag), checksum: dockerImageDigest}: {modulePath: ociWasmFile}, + }, + wantCachedChecksums: map[string]*checksumEntry{ + ociURLWithLatestTag: {checksum: dockerImageDigest, resourceVersionByResource: map[string]string{"namespace.resource": "0"}}, + }, + wantFileName: ociWasmFile, + wantVisitServer: true, + }, + { + name: "do not pull in spite of latest tag due to checksum", + initialCachedModules: map[moduleKey]cacheEntry{ + {name: moduleNameFromURL(ociURLWithLatestTag), checksum: dockerImageDigest}: {modulePath: ociWasmFile}, + }, + initialCachedChecksums: map[string]*checksumEntry{ + ociURLWithLatestTag: { + checksum: dockerImageDigest, + resourceVersionByResource: map[string]string{ + "namespace.resource": "123456", + }, + }, + }, + fetchURL: ociURLWithLatestTag, + getOptions: GetOptions{ + ResourceName: "namespace.resource", + ResourceVersion: "0", + RequestTimeout: time.Second * 10, + Checksum: dockerImageDigest, + PullPolicy: Unspecified, // Default policy + }, + wantCachedModules: map[moduleKey]*cacheEntry{ + {name: moduleNameFromURL(ociURLWithLatestTag), checksum: dockerImageDigest}: {modulePath: ociWasmFile}, + }, + wantCachedChecksums: map[string]*checksumEntry{ + ociURLWithLatestTag: {checksum: dockerImageDigest, resourceVersionByResource: map[string]string{"namespace.resource": "0"}}, + }, + wantFileName: ociWasmFile, + wantVisitServer: false, + }, + { + name: "do not pull in spite of latest tag due to IfNotPresent policy", + initialCachedModules: map[moduleKey]cacheEntry{ + {name: moduleNameFromURL(ociURLWithLatestTag), checksum: dockerImageDigest}: {modulePath: ociWasmFile}, + }, + initialCachedChecksums: map[string]*checksumEntry{ + ociURLWithLatestTag: { + checksum: dockerImageDigest, + resourceVersionByResource: map[string]string{ + "namespace.resource": "123456", + }, + }, + }, + fetchURL: ociURLWithLatestTag, + getOptions: GetOptions{ + ResourceName: "namespace.resource", + ResourceVersion: "0", + RequestTimeout: time.Second * 10, + PullPolicy: IfNotPresent, + }, + wantCachedModules: map[moduleKey]*cacheEntry{ + {name: moduleNameFromURL(ociURLWithLatestTag), checksum: dockerImageDigest}: {modulePath: ociWasmFile}, + }, + wantCachedChecksums: map[string]*checksumEntry{ + ociURLWithLatestTag: {checksum: dockerImageDigest, resourceVersionByResource: map[string]string{"namespace.resource": "0"}}, + }, + wantFileName: ociWasmFile, + wantVisitServer: false, + }, + { + name: "purge OCI image on expiry", + initialCachedModules: map[moduleKey]cacheEntry{ + {name: moduleNameFromURL(ociURLWithTag) + "-purged", checksum: dockerImageDigest}: { + modulePath: ociWasmFile, + referencingURLs: sets.New[string](ociURLWithTag), + }, + }, + initialCachedChecksums: map[string]*checksumEntry{ + ociURLWithTag: { + checksum: dockerImageDigest, + resourceVersionByResource: map[string]string{ + "namespace.resource": "123456", + }, + }, + "test-url": { + checksum: "test-checksum", + resourceVersionByResource: map[string]string{ + "namespace.resource2": "123456", + }, + }, + }, + fetchURL: ociURLWithDigest, + purgeInterval: 1 * time.Millisecond, + wasmModuleExpiry: 1 * time.Millisecond, + getOptions: GetOptions{ + ResourceName: "namespace.resource", + ResourceVersion: "0", + RequestTimeout: time.Second * 10, + }, + checkPurgeTimeout: 5 * time.Second, + wantCachedModules: map[moduleKey]*cacheEntry{}, + wantCachedChecksums: map[string]*checksumEntry{ + "test-url": {checksum: "test-checksum", resourceVersionByResource: map[string]string{"namespace.resource2": "123456"}}, + }, + wantFileName: ociWasmFile, + wantVisitServer: true, + }, + { + name: "fetch oci with wrong digest", + initialCachedModules: map[moduleKey]cacheEntry{}, + fetchURL: ociURLWithTag, + getOptions: GetOptions{ + ResourceName: "namespace.resource", + ResourceVersion: "0", + RequestTimeout: time.Second * 10, + Checksum: "wrongdigest", + }, + wantCachedModules: map[moduleKey]*cacheEntry{}, + wantCachedChecksums: map[string]*checksumEntry{}, + wantErrorMsgPrefix: fmt.Sprintf( + "module downloaded from %v has checksum %v, which does not match:", fmt.Sprintf("oci://%s/test/valid/docker:v0.1.0", ou.Host), dockerImageDigest, + ), + wantVisitServer: true, + }, + { + name: "fetch invalid oci", + initialCachedModules: map[moduleKey]cacheEntry{}, + fetchURL: fmt.Sprintf("oci://%s/test/invalid", ou.Host), + getOptions: GetOptions{ + ResourceName: "namespace.resource", + ResourceVersion: "0", + Checksum: invalidOCIImageDigest, + RequestTimeout: time.Second * 10, + }, + wantCachedModules: map[moduleKey]*cacheEntry{}, + wantCachedChecksums: map[string]*checksumEntry{}, + wantErrorMsgPrefix: `could not fetch Wasm binary: the given image is in invalid format as an OCI image: 2 errors occurred: + * could not parse as compat variant: invalid media type application/vnd.oci.image.layer.v1.tar (expect application/vnd.oci.image.layer.v1.tar+gzip) + * could not parse as oci variant: number of layers must be 2 but got 1`, + wantVisitServer: true, + }, + { + name: "cache size limit", + initialCachedModules: map[moduleKey]cacheEntry{ + {name: moduleNameFromURL(ociURLWithTag), checksum: cacheHitSum}: {modulePath: "test.wasm", size: DefaultMaxCacheSize - 1}, + }, + initialCachedChecksums: map[string]*checksumEntry{}, + fetchURL: ts.URL, + getOptions: GetOptions{ + ResourceName: "namespace.resource", + ResourceVersion: "0", + Checksum: wasmDataCheckSum, + RequestTimeout: time.Second * 10, + }, + wantCachedModules: map[moduleKey]*cacheEntry{ + {name: moduleNameFromURL(ociURLWithTag), checksum: cacheHitSum}: {modulePath: "test.wasm", size: DefaultMaxCacheSize - 1}, + }, + wantCachedChecksums: map[string]*checksumEntry{}, + wantErrorMsgPrefix: `wasm cache size exceeded the limit`, + wantVisitServer: true, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + tmpDir := t.TempDir() + options := defaultCacheOptions() + options.CacheDir = tmpDir + if c.purgeInterval != 0 { + options.PurgeInterval = c.purgeInterval + } + if c.wasmModuleExpiry != 0 { + options.ModuleExpiry = c.wasmModuleExpiry + } + cache := newLocalFileCache(options, logging.DefaultLogger(egv1a1.LogLevelInfo)) + cache.httpFetcher.initialBackoff = time.Microsecond + ctx, cancel := context.WithTimeout(context.Background(), time.Second*60) + cache.Start(ctx) + defer cancel() + + var cacheHitKey *moduleKey + initTime := time.Now() + cache.mux.Lock() + for k, m := range c.initialCachedModules { + filePath := generateModulePath(t, tmpDir, k.name, m.modulePath) + err := os.WriteFile(filePath, []byte("data/\n"), 0o600) + if err != nil { + t.Fatalf("failed to write initial wasm module file %v", err) + } + mkey := moduleKey{name: k.name, checksum: k.checksum} + + cache.modules[mkey] = &cacheEntry{modulePath: filePath, last: initTime, size: m.size} + if m.referencingURLs != nil { + cache.modules[mkey].referencingURLs = m.referencingURLs.Clone() + } else { + cache.modules[mkey].referencingURLs = sets.New[string]() + } + + if moduleNameFromURL(c.fetchURL) == k.name && c.getOptions.Checksum == k.checksum { + cacheHitKey = &mkey + } + } + + for k, m := range c.initialCachedChecksums { + cache.checksums[k] = m + } + + // put the tmp dir into the module path. + for k, m := range c.wantCachedModules { + c.wantCachedModules[k].modulePath = generateModulePath(t, tmpDir, k.name, m.modulePath) + } + cache.mux.Unlock() + + atomic.StoreInt32(&tsNumRequest, 0) + if c.getOptions.PullSecret == nil { + c.getOptions.PullSecret = []byte{} + } + gotFilePath, _, gotErr := cache.Get(c.fetchURL, c.getOptions) + serverVisited := atomic.LoadInt32(&tsNumRequest) > 0 + + if c.checkPurgeTimeout > 0 { + moduleDeleted := false + for start := time.Now(); time.Since(start) < c.checkPurgeTimeout; { + fileCount := 0 + err = filepath.Walk(tmpDir, + func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + fileCount++ + } + return nil + }) + // Check existence of module files. files should be deleted before timing out. + if err == nil && fileCount == 0 { + moduleDeleted = true + break + } + } + + if !moduleDeleted { + t.Fatalf("Wasm modules are not purged before purge timeout") + } + } + + cache.mux.Lock() + if cacheHitKey != nil { + if entry, ok := cache.modules[*cacheHitKey]; ok && entry.last == initTime { + t.Errorf("Wasm module cache entry's last access time not updated after get operation, key: %v", *cacheHitKey) + } + } + + if diff := cmp.Diff(c.wantCachedModules, cache.modules, + cmpopts.IgnoreFields(cacheEntry{}, "last", "referencingURLs"), + cmp.AllowUnexported(cacheEntry{}), + ); diff != "" { + t.Errorf("unexpected module cache: (-want, +got)\n%v", diff) + } + + if diff := cmp.Diff(c.wantCachedChecksums, cache.checksums, + cmp.AllowUnexported(checksumEntry{}), + ); diff != "" { + t.Errorf("unexpected checksums: (-want, +got)\n%v", diff) + } + + cache.mux.Unlock() + + wantFilePath := generateModulePath(t, tmpDir, moduleNameFromURL(c.fetchURL), c.wantFileName) + if c.wantErrorMsgPrefix != "" { + if gotErr == nil { + t.Errorf("Wasm module cache lookup got no error, want error prefix `%v`", c.wantErrorMsgPrefix) + } else if !strings.Contains(gotErr.Error(), c.wantErrorMsgPrefix) { + t.Errorf("Wasm module cache lookup got error `%v`, want error prefix `%v`", gotErr, c.wantErrorMsgPrefix) + } + } else if gotFilePath != wantFilePath { + t.Errorf("Wasm module local file path got %v, want %v", gotFilePath, wantFilePath) + if gotErr != nil { + t.Errorf("got unexpected error %v", gotErr) + } + } + if c.wantVisitServer != serverVisited { + t.Errorf("test wasm binary server encountered the unexpected visiting status got %v, want %v", serverVisited, c.wantVisitServer) + } + }) + } +} + +func setupOCIRegistry(t *testing.T, host string) (dockerImageDigest, invalidOCIImageDigest string) { + // Push *compat* variant docker image (others are well tested in imagefetcher's test and the behavior is consistent). + ref := fmt.Sprintf("%s/test/valid/docker:v0.1.0", host) + binary := wasmHeader + binary = append(binary, wasmTestData...) + transport := remote.DefaultTransport.(*http.Transport).Clone() + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} // nolint: gosec // test only code + fetchOpt := crane.WithTransport(transport) + + // Create docker layer. + l, err := newMockLayer(types.DockerLayer, + map[string][]byte{"plugin.wasm": binary}) + if err != nil { + t.Fatal(err) + } + img, err := mutate.Append(empty.Image, mutate.Addendum{Layer: l}) + if err != nil { + t.Fatal(err) + } + + // Set manifest type. + manifest, err := img.Manifest() + if err != nil { + t.Fatal(err) + } + manifest.MediaType = types.DockerManifestSchema2 + + // Push image to the registry. + err = crane.Push(img, ref, fetchOpt) + if err != nil { + t.Fatal(err) + } + + // Push image to the registry with latest tag as well + ref = fmt.Sprintf("%s/test/valid/docker:latest", host) + err = crane.Push(img, ref, fetchOpt) + if err != nil { + t.Fatal(err) + } + + // Calculate sum + d, _ := img.Digest() + dockerImageDigest = d.Hex + + // Finally push the invalid image. + ref = fmt.Sprintf("%s/test/invalid", host) + l, err = newMockLayer(types.OCIUncompressedLayer, map[string][]byte{"not-wasm.txt": []byte("a")}) + if err != nil { + t.Fatal(err) + } + img2, err := mutate.Append(empty.Image, mutate.Addendum{Layer: l}) + if err != nil { + t.Fatal(err) + } + + // Set manifest type so it will pass the docker parsing branch. + img2 = mutate.MediaType(img2, types.OCIManifestSchema1) + + d, _ = img2.Digest() + invalidOCIImageDigest = d.Hex + + // Push image to the registry. + err = crane.Push(img2, ref, fetchOpt) + if err != nil { + t.Fatal(err) + } + return +} + +func TestWasmCachePolicyChangesUsingHTTP(t *testing.T) { + tmpDir := t.TempDir() + options := defaultCacheOptions() + options.CacheDir = tmpDir + cache := newLocalFileCache(options, logging.DefaultLogger(egv1a1.LogLevelInfo)) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*60) + cache.Start(ctx) + defer cancel() + + gotNumRequest := 0 + binary1 := wasmHeader + binary1 = append(binary1, 1) + binary2 := wasmHeader + binary2 = append(binary2, 2) + + // Create a test server which returns 0 for the first two calls, and returns 1 for the following calls. + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if gotNumRequest <= 1 { + _, _ = w.Write(binary1) + } else { + _, _ = w.Write(binary2) + } + gotNumRequest++ + })) + defer ts.Close() + url1 := ts.URL + url2 := ts.URL + "/next" + wantFilePath1 := generateModulePath(t, tmpDir, url1, fmt.Sprintf("%x.wasm", sha256.Sum256(binary1))) + wantFilePath2 := generateModulePath(t, tmpDir, url2, fmt.Sprintf("%x.wasm", sha256.Sum256(binary2))) + var defaultPullPolicy PullPolicy + + testWasmGet := func(downloadURL string, policy PullPolicy, resourceVersion string, wantFilePath string, wantNumRequest int) { + t.Helper() + gotFilePath, _, err := cache.Get(downloadURL, GetOptions{ + ResourceName: "namespace.resource", + ResourceVersion: resourceVersion, + RequestTimeout: time.Second * 10, + PullSecret: []byte{}, + PullPolicy: policy, + }) + if err != nil { + t.Fatalf("failed to download Wasm module: %v", err) + } + if gotFilePath != wantFilePath { + t.Fatalf("wasm download path got %v want %v", gotFilePath, wantFilePath) + } + if gotNumRequest != wantNumRequest { + t.Fatalf("wasm download call got %v want %v", gotNumRequest, wantNumRequest) + } + } + + // 1st time: Initially load the binary1. + testWasmGet(url1, defaultPullPolicy, "1", wantFilePath1, 1) + // 2nd time: Should not pull the binary and use the cache because defaultPullPolicy is IfNotPresent + testWasmGet(url1, defaultPullPolicy, "2", wantFilePath1, 1) + // 3rd time: Should not pull the binary because the policy is IfNotPresent + testWasmGet(url1, IfNotPresent, "3", wantFilePath1, 1) + // 4th time: Should not pull the binary because the resource version is not changed + testWasmGet(url1, Always, "3", wantFilePath1, 1) + // 5th time: Should pull the binary because the resource version is changed. + testWasmGet(url1, Always, "4", wantFilePath1, 2) + // 6th time: Should pull the binary because URL is changed. + testWasmGet(url2, Always, "4", wantFilePath2, 3) +} + +func TestAllInsecureServer(t *testing.T) { + tmpDir := t.TempDir() + options := defaultCacheOptions() + options.CacheDir = tmpDir + options.InsecureRegistries = sets.New[string]("*") + cache := newLocalFileCache(options, logging.DefaultLogger(egv1a1.LogLevelInfo)) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*60) + cache.Start(ctx) + defer cancel() + + // Set up a fake registry for OCI images with TLS Server + // Without "insecure" option, this should cause an error. + tos := httptest.NewTLSServer(registry.New()) + defer tos.Close() + ou, err := url.Parse(tos.URL) + if err != nil { + t.Fatal(err) + } + + dockerImageDigest, _ := setupOCIRegistry(t, ou.Host) + ociURLWithTag := fmt.Sprintf("oci://%s/test/valid/docker:v0.1.0", ou.Host) + var defaultPullPolicy PullPolicy + + gotFilePath, _, err := cache.Get(ociURLWithTag, GetOptions{ + ResourceName: "namespace.resource", + ResourceVersion: "123456", + RequestTimeout: time.Second * 10, + PullSecret: []byte{}, + PullPolicy: defaultPullPolicy, + }) + if err != nil { + t.Fatalf("failed to download Wasm module: %v", err) + } + + wantFilePath := generateModulePath(t, tmpDir, moduleNameFromURL(ociURLWithTag), fmt.Sprintf("%s.wasm", dockerImageDigest)) + if gotFilePath != wantFilePath { + t.Errorf("Wasm module local file path got %v, want %v", gotFilePath, wantFilePath) + } +} + +func generateModulePath(t *testing.T, baseDir, resourceName, filename string) string { + t.Helper() + sha := sha256.Sum256([]byte(resourceName)) + moduleDir := filepath.Join(baseDir, hex.EncodeToString(sha[:])) + if _, err := os.Stat(moduleDir); errors.Is(err, os.ErrNotExist) { + err := os.Mkdir(moduleDir, 0o755) + if err != nil { + t.Fatalf("failed to create module dir %s: %v", moduleDir, err) + } + } + return filepath.Join(moduleDir, filename) +} diff --git a/internal/wasm/httpfetcher.go b/internal/wasm/httpfetcher.go new file mode 100644 index 00000000000..6850ef9974b --- /dev/null +++ b/internal/wasm/httpfetcher.go @@ -0,0 +1,224 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// Copyright Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wasm + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/tls" + "fmt" + "io" + "net/http" + "time" + + "github.com/cenkalti/backoff/v4" + + "github.com/envoyproxy/gateway/internal/logging" +) + +// Default values for ExponentialBackOff. +const ( + defaultInitialInterval = 500 * time.Millisecond + defaultMaxInterval = 60 * time.Second + maxWasmSize = 1024 * 1024 * 256 +) + +var ( + // Referred to https://en.wikipedia.org/wiki/Tar_(computing)#UStar_format + tarMagicNumber = []byte{0x75, 0x73, 0x74, 0x61, 0x72} + // Referred to https://en.wikipedia.org/wiki/Gzip#File_format + gzMagicNumber = []byte{0x1f, 0x8b} +) + +// HTTPFetcher fetches remote wasm module with HTTP get. +type HTTPFetcher struct { + client *http.Client + insecureClient *http.Client + initialBackoff time.Duration + requestMaxRetry int + logger logging.Logger +} + +// NewHTTPFetcher create a new HTTP remote wasm module fetcher. +// requestTimeout is a timeout for each HTTP/HTTPS request. +// requestMaxRetry is # of maximum retries of HTTP/HTTPS requests. +func NewHTTPFetcher(requestTimeout time.Duration, requestMaxRetry int, logger logging.Logger) *HTTPFetcher { + if requestTimeout == 0 { + requestTimeout = 5 * time.Second + } + transport := http.DefaultTransport.(*http.Transport).Clone() + // nolint: gosec + // This is only when a user explicitly sets a flag to enable insecure mode + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + return &HTTPFetcher{ + client: &http.Client{ + Timeout: requestTimeout, + }, + insecureClient: &http.Client{ + Timeout: requestTimeout, + Transport: transport, + }, + initialBackoff: time.Millisecond * 500, + requestMaxRetry: requestMaxRetry, + logger: logger, + } +} + +// Fetch downloads a wasm module with HTTP get. +func (f *HTTPFetcher) Fetch(ctx context.Context, url string, allowInsecure bool) ([]byte, error) { + c := f.client + if allowInsecure { + c = f.insecureClient + } + attempts := 0 + b := backoff.NewExponentialBackOff() + b.InitialInterval = defaultInitialInterval + b.MaxInterval = defaultMaxInterval + b.InitialInterval = f.initialBackoff + + var lastError error + for attempts < f.requestMaxRetry { + attempts++ + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + f.logger.Info("wasm module download request failed", "error", err) + return nil, err + } + resp, err := c.Do(req) + if err != nil { + lastError = err + f.logger.Info("wasm module download request failed", "error", err) + if ctx.Err() != nil { + // If there is context timeout, exit this loop. + return nil, fmt.Errorf("wasm module download failed after %v attempts, last error: %w", attempts, lastError) + } + time.Sleep(b.NextBackOff()) + continue + } + if resp.StatusCode == http.StatusOK { + // Limit wasm module to 256mb; in reality, it must be much smaller + body, err := io.ReadAll(io.LimitReader(resp.Body, maxWasmSize)) + if err != nil { + return nil, err + } + err = resp.Body.Close() + if err != nil { + f.logger.Info("wasm server connection is not closed", "error", err) + } + return unboxIfPossible(body), err + } + lastError = fmt.Errorf("wasm module download request failed: status code %v", resp.StatusCode) + if retryable(resp.StatusCode) { + // Limit wasm module to 256mb; in reality it must be much smaller + body, err := io.ReadAll(io.LimitReader(resp.Body, maxWasmSize)) + if err != nil { + return nil, err + } + f.logger.Info("wasm module download failed", "status code", resp.StatusCode, "body", string(body)) + err = resp.Body.Close() + if err != nil { + f.logger.Info("wasm server connection is not closed", "error", err) + } + time.Sleep(b.NextBackOff()) + continue + } + err = resp.Body.Close() + if err != nil { + f.logger.Info("wasm server connection is not closed", "error", err) + } + break + } + return nil, fmt.Errorf("wasm module download failed after %v attempts, last error: %w", attempts, lastError) +} + +func retryable(code int) bool { + return code >= 500 && + !(code == http.StatusNotImplemented || + code == http.StatusHTTPVersionNotSupported || + code == http.StatusNetworkAuthenticationRequired) +} + +func isPosixTar(b []byte) bool { + return len(b) > 262 && bytes.Equal(b[257:262], tarMagicNumber) +} + +// wasm plugin should be the only file in the tarball. +func getFirstFileFromTar(b []byte) []byte { + buf := bytes.NewBuffer(b) + + // Limit wasm module to 256mb; in reality it must be much smaller + tr := tar.NewReader(io.LimitReader(buf, maxWasmSize)) + + h, err := tr.Next() + if err != nil { + return nil + } + + ret := make([]byte, h.Size) + _, err = io.ReadFull(tr, ret) + if err != nil { + return nil + } + return ret +} + +func isGZ(b []byte) bool { + return len(b) > 2 && bytes.Equal(b[:2], gzMagicNumber) +} + +func getFileFromGZ(b []byte) []byte { + buf := bytes.NewBuffer(b) + + zr, err := gzip.NewReader(buf) + if err != nil { + return nil + } + + ret, err := io.ReadAll(zr) + if err != nil { + return nil + } + return ret +} + +// Just do the best effort. +// If an error is encountered, just return the original bytes. +// Errors will be handled upper layers. +func unboxIfPossible(origin []byte) []byte { + b := origin + for { + switch { + case isValidWasmBinary(b): + return b + case isGZ(b): + if b = getFileFromGZ(b); b == nil { + return origin + } + case isPosixTar(b): + if b = getFirstFileFromTar(b); b == nil { + return origin + } + default: + return origin + } + } +} diff --git a/internal/wasm/httpfetcher_test.go b/internal/wasm/httpfetcher_test.go new file mode 100644 index 00000000000..a6ba760b188 --- /dev/null +++ b/internal/wasm/httpfetcher_test.go @@ -0,0 +1,277 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// Copyright Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wasm + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "fmt" + "net/http" + "net/http/httptest" + "regexp" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" + "github.com/envoyproxy/gateway/internal/logging" +) + +func TestWasmHTTPFetch(t *testing.T) { + var ts *httptest.Server + + cases := []struct { + name string + handler func(http.ResponseWriter, *http.Request, int) + timeout time.Duration + wantNumRequest int + wantErrorRegex string + }{ + { + name: "download ok", + handler: func(w http.ResponseWriter, r *http.Request, num int) { + fmt.Fprintln(w, "wasm") + }, + timeout: 10 * time.Second, + wantNumRequest: 1, + }, + { + name: "download retry", + handler: func(w http.ResponseWriter, r *http.Request, num int) { + if num <= 2 { + w.WriteHeader(http.StatusInternalServerError) + } else { + fmt.Fprintln(w, "wasm") + } + }, + timeout: 10 * time.Second, + wantNumRequest: 4, + }, + { + name: "download max retry", + handler: func(w http.ResponseWriter, r *http.Request, num int) { + w.WriteHeader(http.StatusInternalServerError) + }, + timeout: 10 * time.Second, + wantNumRequest: 5, + wantErrorRegex: "wasm module download failed after 5 attempts, last error: wasm module download request failed: status code 500", + }, + { + name: "download is never tried by immediate context timeout", + handler: func(w http.ResponseWriter, r *http.Request, num int) { + w.WriteHeader(http.StatusInternalServerError) + }, + timeout: 0, // Immediately timeout in the context level. + wantNumRequest: 0, // Should not retried because it is already timed out. + wantErrorRegex: "wasm module download failed after 1 attempts, last error: Get \"[^\"]+\": context deadline exceeded", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + gotNumRequest := 0 + wantWasmModule := "wasm\n" + ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + c.handler(w, r, gotNumRequest) + gotNumRequest++ + })) + defer ts.Close() + fetcher := NewHTTPFetcher(DefaultHTTPRequestTimeout, DefaultHTTPRequestMaxRetries, logging.DefaultLogger(egv1a1.LogLevelInfo)) + fetcher.initialBackoff = time.Microsecond + ctx, cancel := context.WithTimeout(context.Background(), c.timeout) + defer cancel() + b, err := fetcher.Fetch(ctx, ts.URL, false) + if c.wantNumRequest != gotNumRequest { + t.Errorf("Wasm download request got %v, want %v", gotNumRequest, c.wantNumRequest) + } + if c.wantErrorRegex != "" { + if err == nil { + t.Errorf("Wasm download got no error, want error regex `%v`", c.wantErrorRegex) + } else if matched, regexErr := regexp.MatchString(c.wantErrorRegex, err.Error()); regexErr != nil || !matched { + t.Errorf("Wasm download got error `%v`, want error regex `%v`", err, c.wantErrorRegex) + } + } else if string(b) != wantWasmModule { + t.Errorf("downloaded wasm module got %v, want wasm", string(b)) + } + }) + } +} + +func TestWasmHTTPInsecureServer(t *testing.T) { + var ts *httptest.Server + + cases := []struct { + name string + handler func(http.ResponseWriter, *http.Request, int) + insecure bool + wantNumRequest int + wantErrorSuffix string + }{ + { + name: "download fail", + handler: func(w http.ResponseWriter, r *http.Request, num int) { + fmt.Fprintln(w, "wasm") + }, + insecure: false, + wantErrorSuffix: "x509: certificate signed by unknown authority", + wantNumRequest: 0, + }, + { + name: "download ok", + handler: func(w http.ResponseWriter, r *http.Request, num int) { + fmt.Fprintln(w, "wasm") + }, + insecure: true, + wantNumRequest: 1, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + gotNumRequest := 0 + wantWasmModule := "wasm\n" + ts = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + c.handler(w, r, gotNumRequest) + gotNumRequest++ + })) + defer ts.Close() + fetcher := NewHTTPFetcher(DefaultHTTPRequestTimeout, DefaultHTTPRequestMaxRetries, logging.DefaultLogger(egv1a1.LogLevelInfo)) + fetcher.initialBackoff = time.Microsecond + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + b, err := fetcher.Fetch(ctx, ts.URL, c.insecure) + if c.wantNumRequest != gotNumRequest { + t.Errorf("Wasm download request got %v, want %v", gotNumRequest, c.wantNumRequest) + } + if c.wantErrorSuffix != "" { + if err == nil { + t.Errorf("Wasm download got no error, want error suffix `%v`", c.wantErrorSuffix) + } else if !strings.HasSuffix(err.Error(), c.wantErrorSuffix) { + t.Errorf("Wasm download got error `%v`, want error suffix `%v`", err, c.wantErrorSuffix) + } + } else if string(b) != wantWasmModule { + t.Errorf("downloaded wasm module got %v, want wasm", string(b)) + } + }) + } +} + +func createTar(t *testing.T, b []byte) []byte { + t.Helper() + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + hdr := &tar.Header{ + Name: "plugin.wasm", + Mode: 0o600, + Size: int64(len(b)), + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + if _, err := tw.Write(b); err != nil { + t.Fatal(err) + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func createGZ(t *testing.T, b []byte) []byte { + t.Helper() + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + if _, err := zw.Write(b); err != nil { + t.Fatal(err) + } + + if err := zw.Close(); err != nil { + t.Fatal(err) + } + + return buf.Bytes() +} + +func TestWasmHTTPFetchCompressedOrTarFile(t *testing.T) { + wasmBinary := wasmMagicNumber + wasmBinary = append(wasmBinary, 0x00, 0x00, 0x00, 0x00) + tarball := createTar(t, wasmBinary) + gz := createGZ(t, wasmBinary) + gzTarball := createGZ(t, tarball) + cases := []struct { + name string + handler func(http.ResponseWriter, *http.Request, int) + }{ + { + name: "plain wasm binary", + handler: func(w http.ResponseWriter, r *http.Request, num int) { + _, _ = w.Write(wasmBinary) + }, + }, + { + name: "tarball of wasm binary", + handler: func(w http.ResponseWriter, r *http.Request, num int) { + _, _ = w.Write(tarball) + }, + }, + { + name: "gzipped wasm binary", + handler: func(w http.ResponseWriter, r *http.Request, num int) { + _, _ = w.Write(gz) + }, + }, + { + name: "gzipped tarball of wasm binary", + handler: func(w http.ResponseWriter, r *http.Request, num int) { + _, _ = w.Write(gzTarball) + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + gotNumRequest := 0 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + c.handler(w, r, gotNumRequest) + gotNumRequest++ + })) + defer ts.Close() + fetcher := NewHTTPFetcher(DefaultHTTPRequestTimeout, DefaultHTTPRequestMaxRetries, logging.DefaultLogger(egv1a1.LogLevelInfo)) + fetcher.initialBackoff = time.Microsecond + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + b, err := fetcher.Fetch(ctx, ts.URL, false) + if err != nil { + t.Errorf("Wasm download got an unexpected error: %v", err) + } + + if diff := cmp.Diff(wasmBinary, b); diff != "" { + if len(diff) > 500 { + diff = diff[:500] + } + t.Errorf("unexpected binary: (-want, +got)\n%v", diff) + } + }) + } +} diff --git a/internal/wasm/httpserver.go b/internal/wasm/httpserver.go new file mode 100644 index 00000000000..9b1d0b32c90 --- /dev/null +++ b/internal/wasm/httpserver.go @@ -0,0 +1,247 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package wasm + +import ( + "context" + "crypto/sha256" + "crypto/tls" + "encoding/base64" + "fmt" + "net/http" + "strings" + "sync" + "time" + + "github.com/envoyproxy/gateway/internal/logging" +) + +const ( + serverHost = "envoy-gateway" + serverPort = 18002 + defaultMaxFailedAttempts = 10 + defaultAttemptsResetInterval = 5 * time.Minute + defaultAttemptResetDelay = 1 * time.Hour +) + +var _ Cache = &HTTPServer{} + +type SeverOptions struct { + // Salt is used as a hash salt to generate an unguessable path for the Wasm module. + Salt []byte + // TLSConfig is the TLS configuration for the HTTP server. + TLSConfig *tls.Config + MaxFailedAttempts int + FailedAttemptsResetInterval time.Duration + FailedAttemptResetDelay time.Duration +} + +// setDefault sets the default values for the server options if they are not set. +func (o *SeverOptions) setDefault() { + if o.MaxFailedAttempts == 0 { + o.MaxFailedAttempts = defaultMaxFailedAttempts + } + if o.FailedAttemptsResetInterval == 0 { + o.FailedAttemptsResetInterval = defaultAttemptsResetInterval + } + if o.FailedAttemptResetDelay == 0 { + o.FailedAttemptResetDelay = defaultAttemptResetDelay + } +} + +// HTTPServer wraps a local file cache and serves the Wasm modules over HTTP. +type HTTPServer struct { + SeverOptions + sync.Mutex + // map from the mapping path to the wasm file path in the local cache. + // The mapping path is a generated unguessable path to prevent unauthorized users + // from accessing the Wasm module using EnvoyPatchPolicy. Unless the user is + // an admin who can dump the configuration of the Envoy proxy, the mapping path + // is not exposed to the user. + mappingPath2Cache map[string]wasmModuleEntry + // map from the original URL to the number of failed attempts to download the Wasm module. + // If the number of failed attempts exceeds the maximum number of attempts, we will not + // try to download the Wasm module again for attemptResetDelay. This is used + // to prevent the cache from being flooded by failed requests. + failedAttempts map[string]attemptEntry + // local file cache + cache Cache + // HTTP server to serve the Wasm modules to the Envoy Proxies. + server *http.Server + // logger + logger logging.Logger +} + +type attemptEntry struct { + fails int + last time.Time + delay time.Duration +} + +func (a *attemptEntry) expired() bool { + return time.Since(a.last) > a.delay +} + +type wasmModuleEntry struct { + name string + originalURL string + localFile string +} + +// NewHTTPServerWithFileCache creates a HTTP server with a local file cache for Wasm modules. +// The local file cache is used to store the Wasm modules downloaded from the original URL. +// The HTTP server serves the cached Wasm modules over HTTP to the Envoy Proxies. +func NewHTTPServerWithFileCache(serverOptions SeverOptions, cacheOptions CacheOptions, logger logging.Logger) *HTTPServer { + logger = logger.WithName("wasm-cache") + serverOptions.setDefault() + return &HTTPServer{ + SeverOptions: serverOptions, + mappingPath2Cache: make(map[string]wasmModuleEntry), + failedAttempts: make(map[string]attemptEntry), + cache: newLocalFileCache(cacheOptions, logger), + logger: logger, + } +} + +func (s *HTTPServer) Start(ctx context.Context) { + s.logger.Info(fmt.Sprintf("Listening on :%d", serverPort)) + + handler := http.NewServeMux() + handler.Handle("/", s) + + s.server = &http.Server{ + Addr: fmt.Sprintf(":%d", serverPort), + Handler: handler, + TLSConfig: s.TLSConfig, + ReadHeaderTimeout: 15 * time.Second, + } + + var err error + go func() { + if s.enableTLS() { + err = s.server.ListenAndServeTLS("", "") + } else { + err = s.server.ListenAndServe() + } + if err != nil { + s.logger.Error(err, "Failed to start Wasm HTTP server") + return + } + }() + s.cache.Start(ctx) + go s.resetFailedAttempts(ctx) +} + +func (s *HTTPServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + s.logger.Sugar().Debugw("Received wasm request", "path", r.URL.Path) + + path := strings.TrimPrefix(r.URL.Path, "/") + if entry, ok := s.mappingPath2Cache[path]; ok { + http.ServeFile(w, r, entry.localFile) + } else { + w.WriteHeader(http.StatusNotFound) + } +} + +// Get returns the HTTP URL of the Wasm module serving by the EG HTTP Wasm server +// and the checksum of the Wasm module. +// EG downloads the Wasm module from its original URL, caches it locally in the +// file system, and serves it through an HTTP server. +func (s *HTTPServer) Get(originalURL string, opts GetOptions) (servingURL string, checksum string, err error) { + var ( + mappingPath string + localFile string + ) + + s.Lock() + defer s.Unlock() + attempt, attempted := s.failedAttempts[originalURL] + + if attempted && attempt.fails > s.MaxFailedAttempts { + err = fmt.Errorf("failed to get Wasm module %s after %d attempts", originalURL, s.MaxFailedAttempts) + s.logger.Error(err, "") + return "", "", err + } + + // Get the local file path of the cached Wasm module. + // Even it's already cached, the file cache may still download the Wasm module + // again if it is expired or it needs to be updated. + if localFile, checksum, err = s.cache.Get(originalURL, opts); err != nil { + s.logger.Error(err, "Failed to get Wasm module", "URL", originalURL) + attempt, attempted = s.failedAttempts[originalURL] + if !attempted { + attempt = attemptEntry{fails: 0, last: time.Now(), delay: s.FailedAttemptResetDelay} + } + attempt.fails++ + attempt.last = time.Now() + s.failedAttempts[originalURL] = attempt + return "", "", err + } + delete(s.failedAttempts, originalURL) + + // Generate a new path with the hash of the original url and a salt to + // make the URL unpredictable. + // The unguessable path is used to prevent unauthorized users from accessing + // an unauthorized private Wasm module. + mappingPath = generateUnguessablePath(originalURL, s.Salt) + s.mappingPath2Cache[mappingPath] = wasmModuleEntry{ + name: opts.ResourceName, + originalURL: originalURL, + localFile: localFile, + } + + entry := s.mappingPath2Cache[mappingPath] + entry.localFile = localFile + s.mappingPath2Cache[mappingPath] = entry + + scheme := "http" + if s.enableTLS() { + scheme = "https" + } + servingURL = fmt.Sprintf("%s://%s:%d/%s", scheme, serverHost, serverPort, mappingPath) + return servingURL, checksum, nil +} + +// Generate an unguessable downloading path for a Wasm module. +func generateUnguessablePath(originalURL string, salt []byte) string { + saltedData := []byte(originalURL) + saltedData = append(saltedData, salt...) + hash := sha256.Sum256(saltedData) + return fmt.Sprintf("%s.wasm", base64.URLEncoding.EncodeToString(hash[:])) +} + +func (s *HTTPServer) close() { + if s != nil { + _ = s.server.Close() + } +} + +func (s *HTTPServer) enableTLS() bool { + return s.TLSConfig != nil +} + +// resetFailedAttempts resets the failed attempts. +// After reset, the cache will try to download the failed Wasm module again the +// next time it is requested. +func (s *HTTPServer) resetFailedAttempts(ctx context.Context) { + ticker := time.NewTicker(s.FailedAttemptsResetInterval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + s.Lock() + for k, m := range s.failedAttempts { + if m.expired() { + s.logger.Info("Reset failed attempts", "URL", k) + delete(s.failedAttempts, k) + } + } + s.Unlock() + case <-ctx.Done(): + return + } + } +} diff --git a/internal/wasm/httpserver_test.go b/internal/wasm/httpserver_test.go new file mode 100644 index 00000000000..0f054ea8416 --- /dev/null +++ b/internal/wasm/httpserver_test.go @@ -0,0 +1,368 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package wasm + +import ( + "context" + "fmt" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" + + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/registry" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/stretchr/testify/require" + + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" + "github.com/envoyproxy/gateway/internal/logging" +) + +const ( + validWasmModule = "valid.wasm" + nonExistingWasmModule = "non-existing.wasm" + resourceName = "envoyextensionpolicy/envoy-gateway/policy-for-gateway/wasm/0" +) + +func Test_httpServerWithOCIImage(t *testing.T) { + var ( + registryURL *url.URL + err error + ) + + // Set up a fake registry. + r := httptest.NewServer(registry.New()) + defer r.Close() + + if registryURL, err = url.Parse(r.URL); err != nil { + t.Fatal(err) + } + if err = setupFakeRegistry(registryURL.Host); err != nil { + t.Fatal(err) + } + + t.Run("get wasm module from EG HTTP server", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var ( + server *HTTPServer + client = newHTTPClient() + resp *http.Response + servingURL string + ) + + if server, err = startLocalHTTPServer( + ctx, + t.TempDir(), + defaultMaxFailedAttempts, + defaultAttemptResetDelay, + defaultAttemptsResetInterval); err != nil { + t.Fatal(err) + } + defer server.close() + + // Call server.Get() to initialize the local file cache. + servingURL, _, err = server.Get( + fmt.Sprintf("oci://%s/%s", registryURL.Host, validWasmModule), + GetOptions{ + ResourceName: resourceName, + RequestTimeout: time.Second * 1000, + }) + require.NoError(t, err) + + // Get wasm module from the EG HTTP server. + t.Logf("Get wasm module from the EG HTTP server: %s", servingURL) + resp, err = client.Get(servingURL) + _ = resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + + // Call server.Get() again to get the serving URL for the same wasm module. + // The serving URL should be the same as the previous one. + servingURL1, _, err := server.Get( + fmt.Sprintf("oci://%s/%s", registryURL.Host, validWasmModule), + GetOptions{ + ResourceName: resourceName, + RequestTimeout: time.Second * 1000, + }) + require.NoError(t, err) + require.Equal(t, servingURL, servingURL1) + + // Get wasm module from the EG HTTP server. + t.Logf("Get wasm module from the EG HTTP server: %s", servingURL1) + resp, err = client.Get(servingURL1) + require.NoError(t, err) + _ = resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + }) + + t.Run("get non-existing wasm module from EG HTTP server", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var server *HTTPServer + + if server, err = startLocalHTTPServer( + ctx, + t.TempDir(), + defaultMaxFailedAttempts, + defaultAttemptResetDelay, + defaultAttemptsResetInterval); err != nil { + t.Fatal(err) + } + defer server.close() + + // Initialize the local cache. + _, _, err = server.Get(fmt.Sprintf("oci://%s/%s", registryURL.Host, nonExistingWasmModule), + GetOptions{ + ResourceName: resourceName, + RequestTimeout: time.Second * 10, + }) + if err == nil || !strings.Contains(err.Error(), "Unknown name") { + t.Errorf("Get() error = %v, expect error contains 'Unknown name'", err) + } + }) +} + +func Test_httpServerWithHTTP(t *testing.T) { + var ( + fakeServerURL string + err error + ) + + // Set up a fake HTTP server. + httpData := append([]byte{}, wasmHeader...) + httpData = append(httpData, []byte("data")...) + r := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == fmt.Sprintf("/%s", validWasmModule) { + _, _ = w.Write(httpData) + } else { + w.WriteHeader(http.StatusNotFound) + } + })) + fakeServerURL = r.URL + defer r.Close() + + t.Run("get wasm module from EG HTTP server", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var ( + server *HTTPServer + client = newHTTPClient() + resp *http.Response + servingURL string + ) + + if server, err = startLocalHTTPServer( + ctx, + t.TempDir(), + defaultMaxFailedAttempts, + defaultAttemptResetDelay, + defaultAttemptsResetInterval); err != nil { + t.Fatal(err) + } + defer server.close() + + getOptions := GetOptions{ + ResourceName: resourceName, + RequestTimeout: time.Second * 10, + } + + // Call server.Get() to initialize the local file cache. + servingURL, _, err = server.Get(fmt.Sprintf("%s/%s", fakeServerURL, validWasmModule), getOptions) + require.NoError(t, err) + + // Get wasm module from the EG HTTP server. + t.Logf("Get wasm module from the EG HTTP server: %s", servingURL) + resp, err = client.Get(servingURL) + require.Equal(t, http.StatusOK, resp.StatusCode) + _ = resp.Body.Close() + + // Call server.Get() again to get the serving URL for the same wasm module. + // The serving URL should be the same as the previous one. + servingURL1, _, err := server.Get(fmt.Sprintf("%s/%s", fakeServerURL, validWasmModule), getOptions) + require.NoError(t, err) + require.Equal(t, servingURL, servingURL1) + + // Get wasm module from the EG HTTP server. + t.Logf("Get wasm module from the EG HTTP server: %s", servingURL1) + resp, err = client.Get(servingURL1) + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode) + _ = resp.Body.Close() + }) + + t.Run("get non-existing wasm module from EG HTTP server", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var server *HTTPServer + + if server, err = startLocalHTTPServer( + ctx, + t.TempDir(), + defaultMaxFailedAttempts, + defaultAttemptResetDelay, + defaultAttemptsResetInterval); err != nil { + t.Fatal(err) + } + defer server.close() + + // Initialize the local cache. + _, _, err = server.Get(fmt.Sprintf("%s/%s", fakeServerURL, nonExistingWasmModule), GetOptions{ + ResourceName: resourceName, + RequestTimeout: time.Second * 10, + }) + if err == nil || !strings.Contains(err.Error(), "404") { + t.Errorf("Get() error = %v, expect error contains 'Unknown name'", err) + } + }) +} + +func Test_httpServerFailedAttempt(t *testing.T) { + var ( + registryURL *url.URL + err error + ) + + // Set up a fake registry. + r := httptest.NewServer(registry.New()) + defer r.Close() + + if registryURL, err = url.Parse(r.URL); err != nil { + t.Fatal(err) + } + if err = setupFakeRegistry(registryURL.Host); err != nil { + t.Fatal(err) + } + + t.Run("failed attempts exceed the max", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var ( + server *HTTPServer + maxFailedAttempts = 5 + attemptResetDelay = time.Millisecond * 200 + attemptsResetInterval = time.Millisecond * 100 + ) + + if server, err = startLocalHTTPServer( + ctx, + t.TempDir(), + maxFailedAttempts, + attemptResetDelay, + attemptsResetInterval); err != nil { + t.Fatal(err) + } + defer server.close() + + // The 6th Get() should return an error immediately because the max failed attempts is 5. + for i := 0; i <= 6; i++ { + _, _, err = server.Get( + fmt.Sprintf("oci://%s/%s", registryURL.Host, nonExistingWasmModule), + GetOptions{ + ResourceName: resourceName, + RequestTimeout: time.Second * 1000, + }) + } + require.ErrorContains(t, err, "after 5 attempts") + + // The 7th Get() should return a normal error because the failed attempts have been reset. + err = nil + for i := 0; i < 3; i++ { + time.Sleep(300 * time.Millisecond) + _, _, err = server.Get( + fmt.Sprintf("oci://%s/%s", registryURL.Host, nonExistingWasmModule), + GetOptions{ + ResourceName: resourceName, + RequestTimeout: time.Second * 1000, + }) + if err != nil { + break + } + } + require.ErrorContains(t, err, "Unknown name") + }) +} + +func setupFakeRegistry(host string) error { + var ( + l v1.Layer + img v1.Image + err error + ) + + ref := fmt.Sprintf("%s/%s", host, validWasmModule) + binary := wasmHeader + binary = append(binary, []byte("this is wasm plugin")...) + + // Create OCI compressed layer. + if l, err = newMockLayer(types.OCILayer, map[string][]byte{"plugin.wasm": binary}); err != nil { + return err + } + + if img, err = mutate.Append(empty.Image, mutate.Addendum{Layer: l}); err != nil { + return err + } + + img = mutate.MediaType(img, types.OCIManifestSchema1) + + // Push image to the registry. + if err = crane.Push(img, ref); err != nil { + return err + } + return nil +} + +func startLocalHTTPServer(ctx context.Context, cacheDir string, maxFailedAttempts int, failedAttemptResetDelay, failedAttemptsResetInterval time.Duration) (*HTTPServer, error) { + logger := logging.DefaultLogger(egv1a1.LogLevelInfo) + s := NewHTTPServerWithFileCache( + SeverOptions{ + Salt: []byte("salt"), + MaxFailedAttempts: maxFailedAttempts, + FailedAttemptResetDelay: failedAttemptResetDelay, + FailedAttemptsResetInterval: failedAttemptsResetInterval, + }, + CacheOptions{ + CacheDir: cacheDir, + }, logger) + go s.Start(ctx) + + // Wait for the server to start + var ( + retries = 10 + response *http.Response + err error + ) + for i := 0; i < retries; i++ { + if response, err = http.Get(fmt.Sprintf("http://127.0.0.1:%d", serverPort)); err == nil { + _ = response.Body.Close() + break + } + time.Sleep(200 * time.Millisecond) + } + return s, err +} + +func newHTTPClient() *http.Client { + return &http.Client{ + Transport: &http.Transport{ + DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) { + d := net.Dialer{} + return d.DialContext(ctx, network, fmt.Sprintf("127.0.0.1:%d", serverPort)) + }, + }, + } +} diff --git a/internal/wasm/imagefetcher.go b/internal/wasm/imagefetcher.go new file mode 100644 index 00000000000..5c97ad5f4a4 --- /dev/null +++ b/internal/wasm/imagefetcher.go @@ -0,0 +1,377 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// Copyright Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wasm + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "net/http" + "path/filepath" + "strings" + + "github.com/docker/cli/cli/config/configfile" + dtypes "github.com/docker/cli/cli/config/types" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/hashicorp/go-multierror" + + "github.com/envoyproxy/gateway/internal/logging" +) + +// This file implements the fetcher of "Wasm Image Specification" compatible container images. +// The spec is here https://github.com/solo-io/wasm/blob/master/spec/README.md. +// Basically, this supports fetching and unpackaging three types of container images containing a Wasm binary. + +type ImageFetcherOption struct { + PullSecret []byte + Insecure bool +} + +func (o *ImageFetcherOption) useAnonymous() bool { + return o.PullSecret == nil || len(o.PullSecret) == 0 +} + +func (o *ImageFetcherOption) String() string { + if o.PullSecret == nil { + return fmt.Sprintf("{Insecure: %v}", o.Insecure) + } + return fmt.Sprintf("{Insecure: %v, PullSecret: }", o.Insecure) +} + +type ImageFetcher struct { + fetchOpts []remote.Option + logger logging.Logger +} + +func NewImageFetcher(ctx context.Context, opt ImageFetcherOption, logger logging.Logger) *ImageFetcher { + fetchOpts := make([]remote.Option, 0, 2) + if opt.useAnonymous() { + // Use anonymous auth if no pull secret is provided. + fetchOpts = append(fetchOpts, remote.WithAuth(authn.Anonymous)) + } else { + fetchOpts = append(fetchOpts, remote.WithAuthFromKeychain(&wasmKeyChain{data: opt.PullSecret})) + } + + if opt.Insecure { + t := remote.DefaultTransport.(*http.Transport).Clone() + // nolint: gosec + // This is only when a user explicitly sets a flag to enable insecure mode + t.TLSClientConfig = &tls.Config{ + InsecureSkipVerify: opt.Insecure, + } + fetchOpts = append(fetchOpts, remote.WithTransport(t)) + } + + return &ImageFetcher{ + fetchOpts: append(fetchOpts, remote.WithContext(ctx)), + logger: logger, + } +} + +// PrepareFetch is the entrypoint for fetching Wasm binary from Wasm Image Specification compatible images. +// Wasm binary is not fetched immediately, but returned by `binaryFetcher` function, which is returned by PrepareFetch. +// By this way, we can have another chance to check cache with `actualDigest` without downloading the OCI image. +func (o *ImageFetcher) PrepareFetch(url string) (binaryFetcher func() ([]byte, error), actualDigest string, err error) { + ref, err := name.ParseReference(url) + if err != nil { + err = fmt.Errorf("could not parse url in image reference: %w", err) + return + } + o.logger.Info("fetching image", "image", ref.Context().RepositoryStr(), + "registry", ref.Context().RegistryStr(), "tag", ref.Identifier()) + + // fallback to http based request, inspired by [helm](https://github.com/helm/helm/blob/12f1bc0acdeb675a8c50a78462ed3917fb7b2e37/pkg/registry/client.go#L594) + // only deal with https fallback instead of attributing all other type of errors to URL parsing error + desc, err := remote.Get(ref, o.fetchOpts...) + if err != nil && strings.Contains(err.Error(), "server gave HTTP response") { + o.logger.Info("fetching image with plain text", "url", url) + ref, err = name.ParseReference(url, name.Insecure) + if err == nil { + desc, err = remote.Get(ref, o.fetchOpts...) + } + } + + if err != nil { + err = fmt.Errorf("could not fetch manifest: %w", err) + return + } + + // Fetch image. + img, err := desc.Image() + if err != nil { + err = fmt.Errorf("could not fetch image: %w", err) + return + } + + // Check Manifest's digest if expManifestDigest is not empty. + d, _ := img.Digest() + actualDigest = d.Hex + binaryFetcher = func() ([]byte, error) { + manifest, err := img.Manifest() + if err != nil { + return nil, fmt.Errorf("could not retrieve manifest: %w", err) + } + + if manifest.MediaType == types.DockerManifestSchema2 { + // This case, assume we have docker images with "application/vnd.docker.distribution.manifest.v2+json" + // as the manifest media type. Note that the media type of manifest is Docker specific and + // all OCI images would have an empty string in .MediaType field. + ret, err := extractDockerImage(img) + if err != nil { + return nil, fmt.Errorf("could not extract Wasm file from the image as Docker container %w", err) + } + return ret, nil + } + + // We try to parse it as the "compat" variant image with a single "application/vnd.oci.image.layer.v1.tar+gzip" layer. + ret, errCompat := extractOCIStandardImage(img) + if errCompat == nil { + return ret, nil + } + + // Otherwise, we try to parse it as the *oci* variant image with custom artifact media types. + ret, errOCI := extractOCIArtifactImage(img) + if errOCI == nil { + return ret, nil + } + + // We failed to parse the image in any format, so wrap the errors and return. + return nil, fmt.Errorf("the given image is in invalid format as an OCI image: %w", + multierror.Append(err, + fmt.Errorf("could not parse as compat variant: %w", errCompat), + fmt.Errorf("could not parse as oci variant: %w", errOCI), + ), + ) + } + return +} + +// extractDockerImage extracts the Wasm binary from the +// *compat* variant Wasm image with the standard Docker media type: application/vnd.docker.image.rootfs.diff.tar.gzip. +// https://github.com/solo-io/wasm/blob/master/spec/spec-compat.md#specification +func extractDockerImage(img v1.Image) ([]byte, error) { + layers, err := img.Layers() + if err != nil { + return nil, fmt.Errorf("could not fetch layers: %w", err) + } + + // The image must have at least one layer. + if len(layers) == 0 { + return nil, errors.New("number of layers must be greater than zero") + } + + layer := layers[len(layers)-1] + mt, err := layer.MediaType() + if err != nil { + return nil, fmt.Errorf("could not get media type: %w", err) + } + + // Media type must be application/vnd.docker.image.rootfs.diff.tar.gzip. + if mt != types.DockerLayer { + return nil, fmt.Errorf("invalid media type %s (expect %s)", mt, types.DockerLayer) + } + + r, err := layer.Compressed() + if err != nil { + return nil, fmt.Errorf("could not get layer content: %w", err) + } + defer r.Close() + + ret, err := extractWasmPluginBinary(r) + if err != nil { + return nil, fmt.Errorf("could not extract wasm binary: %w", err) + } + return ret, nil +} + +// extractOCIStandardImage extracts the Wasm binary from the +// *compat* variant Wasm image with the standard OCI media type: application/vnd.oci.image.layer.v1.tar+gzip. +// https://github.com/solo-io/wasm/blob/master/spec/spec-compat.md#specification +func extractOCIStandardImage(img v1.Image) ([]byte, error) { + layers, err := img.Layers() + if err != nil { + return nil, fmt.Errorf("could not fetch layers: %w", err) + } + + // The image must have at least one layer. + if len(layers) == 0 { + return nil, fmt.Errorf("number of layers must be greater than zero") + } + + layer := layers[len(layers)-1] + mt, err := layer.MediaType() + if err != nil { + return nil, fmt.Errorf("could not get media type: %w", err) + } + + // Check if the layer is "application/vnd.oci.image.layer.v1.tar+gzip". + if types.OCILayer != mt { + return nil, fmt.Errorf("invalid media type %s (expect %s)", mt, types.OCILayer) + } + + r, err := layer.Compressed() + if err != nil { + return nil, fmt.Errorf("could not get layer content: %w", err) + } + defer r.Close() + + ret, err := extractWasmPluginBinary(r) + if err != nil { + return nil, fmt.Errorf("could not extract wasm binary: %w", err) + } + return ret, nil +} + +// Extracts the Wasm plugin binary named "plugin.wasm" in a given reader for tar.gz. +// This is only used for *compat* variant. +func extractWasmPluginBinary(r io.Reader) ([]byte, error) { + gr, err := gzip.NewReader(r) + if err != nil { + return nil, fmt.Errorf("failed to parse layer as tar.gz: %w", err) + } + + // The target file name for Wasm binary. + // https://github.com/solo-io/wasm/blob/master/spec/spec-compat.md#specification + const wasmPluginFileName = "plugin.wasm" + + // Search for the file walking through the archive. + + // Limit wasm binary to 256mb; in reality it must be much smaller + tr := tar.NewReader(io.LimitReader(gr, maxWasmSize)) + for { + h, err := tr.Next() + if err == io.EOF { + break + } else if err != nil { + return nil, err + } + + ret := make([]byte, h.Size) + if filepath.Base(h.Name) == wasmPluginFileName { + _, err := io.ReadFull(tr, ret) + if err != nil { + return nil, fmt.Errorf("failed to read %s: %w", wasmPluginFileName, err) + } + return ret, nil + } + } + return nil, fmt.Errorf("%s not found in the archive", wasmPluginFileName) +} + +// extractOCIArtifactImage extracts the Wasm binary from the +// *oci* variant Wasm image: https://github.com/solo-io/wasm/blob/master/spec/spec.md#format +func extractOCIArtifactImage(img v1.Image) ([]byte, error) { + layers, err := img.Layers() + if err != nil { + return nil, fmt.Errorf("could not fetch layers: %w", err) + } + + // The image must be two-layered. + if len(layers) != 2 { + return nil, fmt.Errorf("number of layers must be 2 but got %d", len(layers)) + } + + // The layer type of the Wasm binary itself in *oci* variant. + const wasmLayerMediaType = "application/vnd.module.wasm.content.layer.v1+wasm" + + // Find the target layer walking through the layers. + var layer v1.Layer + for _, l := range layers { + mt, err := l.MediaType() + if err != nil { + return nil, fmt.Errorf("could not retrieve the media type: %w", err) + } + if mt == wasmLayerMediaType { + layer = l + break + } + } + + if layer == nil { + return nil, fmt.Errorf("could not find the layer of type %s", wasmLayerMediaType) + } + + // Somehow go-containerregistry recognizes custom artifact layers as compressed ones, + // while the Solo's Wasm layer is actually uncompressed and therefore + // the content itself is a raw Wasm binary. So using "Uncompressed()" here result in errors + // since internally it tries to umcompress it as gzipped blob. + r, err := layer.Compressed() + if err != nil { + return nil, fmt.Errorf("could not get layer content: %w", err) + } + defer r.Close() + + // Just read it since the content is already a raw Wasm binary as mentioned above. + ret, err := io.ReadAll(r) + if err != nil { + return nil, fmt.Errorf("could not extract wasm binary: %w", err) + } + return ret, nil +} + +type wasmKeyChain struct { + data []byte +} + +// Resolve an image reference to a credential. +// The function code is borrowed from https://github.com/google/go-containerregistry/blob/v0.8.0/pkg/authn/keychain.go#L65, +// by making it take dockerconfigjson directly as bytes instead of reading from files. +func (k *wasmKeyChain) Resolve(target authn.Resource) (authn.Authenticator, error) { + if bytes.Equal(k.data, []byte("null")) { + // Filter out key chain with content "null" to prevent crash at underlying docker library. + // Remove this check when https://github.com/docker/cli/pull/3434 is merged. + return nil, fmt.Errorf("") + } + reader := bytes.NewReader(k.data) + cf := configfile.ConfigFile{} + if err := cf.LoadFromReader(reader); err != nil { + return nil, err + } + key := target.RegistryStr() + if key == name.DefaultRegistry { + key = authn.DefaultAuthKey + } + cfg, err := cf.GetAuthConfig(key) + if err != nil { + return nil, err + } + + empty := dtypes.AuthConfig{} + if cfg == empty { + return authn.Anonymous, nil + } + authConfig := authn.AuthConfig{ + Username: cfg.Username, + Password: cfg.Password, + Auth: cfg.Auth, + IdentityToken: cfg.IdentityToken, + RegistryToken: cfg.RegistryToken, + } + return authn.FromConfig(authConfig), nil +} diff --git a/internal/wasm/imagefetcher_test.go b/internal/wasm/imagefetcher_test.go new file mode 100644 index 00000000000..833680e3955 --- /dev/null +++ b/internal/wasm/imagefetcher_test.go @@ -0,0 +1,636 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// Copyright Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wasm + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "fmt" + "io" + "net/http/httptest" + "net/url" + "reflect" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +func TestImageFetcherOption_useAnonymous(t *testing.T) { + cases := []struct { + name string + opt ImageFetcherOption + exp bool + }{ + {name: "anonymous", exp: true}, + {name: "use secret config", opt: ImageFetcherOption{PullSecret: []byte("secret")}}, + {name: "missing secret", opt: ImageFetcherOption{}, exp: true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + actual := c.opt.useAnonymous() + if actual != c.exp { + t.Errorf("anonymous got %v want %v", actual, c.exp) + } + }) + } +} + +func TestImageFetcher_Fetch(t *testing.T) { + // Fetcher with anonymous auth. + fetcher := ImageFetcher{fetchOpts: []remote.Option{remote.WithAuth(authn.Anonymous)}} + + // Set up a fake registry. + s := httptest.NewServer(registry.New()) + defer s.Close() + u, err := url.Parse(s.URL) + if err != nil { + t.Fatal(err) + } + + t.Run("docker image", func(t *testing.T) { + ref := fmt.Sprintf("%s/test/valid/docker", u.Host) + exp := "this is wasm plugin" + + // Create docker layer. + l, err := newMockLayer(types.DockerLayer, + map[string][]byte{"plugin.wasm": []byte(exp)}) + if err != nil { + t.Fatal(err) + } + img, err := mutate.Append(empty.Image, mutate.Addendum{Layer: l}) + if err != nil { + t.Fatal(err) + } + + // Set manifest type. + manifest, err := img.Manifest() + if err != nil { + t.Fatal(err) + } + manifest.MediaType = types.DockerManifestSchema2 + + // Push image to the registry. + err = crane.Push(img, ref) + if err != nil { + t.Fatal(err) + } + + // Fetch docker image with digest + d, err := img.Digest() + if err != nil { + t.Fatal(err) + } + + // Fetch OCI image. + binaryFetcher, actualDiget, err := fetcher.PrepareFetch(ref) + if err != nil { + t.Fatal(err) + } + actual, err := binaryFetcher() + if err != nil { + t.Fatal(err) + } + if string(actual) != exp { + t.Errorf("ImageFetcher.binaryFetcher got %s, but want '%s'", string(actual), exp) + } + if actualDiget != d.Hex { + t.Errorf("ImageFetcher.binaryFetcher got digest %s, but want '%s'", actualDiget, d.Hex) + } + }) + + t.Run("OCI standard", func(t *testing.T) { + ref := fmt.Sprintf("%s/test/valid/oci_standard", u.Host) + exp := "this is wasm plugin" + + // Create OCI compressed layer. + l, err := newMockLayer(types.OCILayer, + map[string][]byte{"plugin.wasm": []byte(exp)}) + if err != nil { + t.Fatal(err) + } + img, err := mutate.Append(empty.Image, mutate.Addendum{Layer: l}) + if err != nil { + t.Fatal(err) + } + img = mutate.MediaType(img, types.OCIManifestSchema1) + + // Push image to the registry. + err = crane.Push(img, ref) + if err != nil { + t.Fatal(err) + } + + // Fetch OCI image with digest + d, err := img.Digest() + if err != nil { + t.Fatal(err) + } + + // Fetch OCI image. + binaryFetcher, actualDiget, err := fetcher.PrepareFetch(ref) + if err != nil { + t.Fatal(err) + } + actual, err := binaryFetcher() + if err != nil { + t.Fatal(err) + } + if string(actual) != exp { + t.Errorf("ImageFetcher.binaryFetcher got %s, but want '%s'", string(actual), exp) + } + if actualDiget != d.Hex { + t.Errorf("ImageFetcher.binaryFetcher got digest %s, but want '%s'", actualDiget, d.Hex) + } + }) + + t.Run("OCI artifact", func(t *testing.T) { + ref := fmt.Sprintf("%s/test/valid/oci_artifact", u.Host) + + // Create the image with custom media types. + wasmLayer, err := random.Layer(1000, "application/vnd.module.wasm.content.layer.v1+wasm") + if err != nil { + t.Fatal(err) + } + configLayer, err := random.Layer(1000, "application/vnd.module.wasm.config.v1+json") + if err != nil { + t.Fatal(err) + } + img, err := mutate.Append(empty.Image, mutate.Addendum{Layer: wasmLayer}, mutate.Addendum{Layer: configLayer}) + if err != nil { + t.Fatal(err) + } + img = mutate.MediaType(img, types.OCIManifestSchema1) + + // Push image to the registry. + err = crane.Push(img, ref) + if err != nil { + t.Fatal(err) + } + + // Retrieve the wanted image content. + wantReader, err := wasmLayer.Compressed() + if err != nil { + t.Fatal(err) + } + defer wantReader.Close() + + want, err := io.ReadAll(wantReader) + if err != nil { + t.Fatal(err) + } + + // Fetch OCI image with digest + d, err := img.Digest() + if err != nil { + t.Fatal(err) + } + + // Fetch OCI image. + binaryFetcher, actualDiget, err := fetcher.PrepareFetch(ref) + if err != nil { + t.Fatal(err) + } + actual, err := binaryFetcher() + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(actual, want) { + t.Errorf("ImageFetcher.binaryFetcher got %s, but want '%s'", string(actual), string(want)) + } + if actualDiget != d.Hex { + t.Errorf("ImageFetcher.binaryFetcher got digest %s, but want '%s'", actualDiget, d.Hex) + } + }) + + t.Run("invalid image", func(t *testing.T) { + ref := fmt.Sprintf("%s/test/invalid", u.Host) + + l, err := newMockLayer(types.OCIUncompressedLayer, map[string][]byte{"not-wasm.txt": []byte("a")}) + if err != nil { + t.Fatal(err) + } + img, err := mutate.Append(empty.Image, mutate.Addendum{Layer: l}) + if err != nil { + t.Fatal(err) + } + img = mutate.MediaType(img, types.OCIManifestSchema1) + + // Push image to the registry. + err = crane.Push(img, ref) + if err != nil { + t.Fatal(err) + } + + // Try to fetch. + binaryFetcher, _, err := fetcher.PrepareFetch(ref) + if err != nil { + t.Fatal(err) + } + actual, err := binaryFetcher() + if actual != nil { + t.Errorf("ImageFetcher.binaryFetcher got %s, but want nil", string(actual)) + } + + expErr := `the given image is in invalid format as an OCI image: 2 errors occurred: + * could not parse as compat variant: invalid media type application/vnd.oci.image.layer.v1.tar (expect application/vnd.oci.image.layer.v1.tar+gzip) + * could not parse as oci variant: number of layers must be 2 but got 1` + if actual := strings.TrimSpace(err.Error()); actual != expErr { + t.Errorf("ImageFetcher.binaryFetcher get unexpected error '%v', but want '%v'", actual, expErr) + } + }) +} + +func TestExtractDockerImage(t *testing.T) { + t.Run("no layers", func(t *testing.T) { + _, err := extractDockerImage(empty.Image) + if err == nil || err.Error() != "number of layers must be greater than zero" { + t.Fatal("extractDockerImage should fail due to empty image") + } + }) + + t.Run("valid layers", func(t *testing.T) { + previousLayer, err := newMockLayer(types.DockerLayer, nil) + if err != nil { + t.Fatal(err) + } + + exp := "this is wasm binary" + lastLayer, err := newMockLayer(types.DockerLayer, map[string][]byte{ + "plugin.wasm": []byte(exp), + }) + if err != nil { + t.Fatal(err) + } + + tCases := map[string]int{ + "one layer": 0, + "more than one layer": 1, + } + + for name, numberOfPreviousLayers := range tCases { + t.Run(name, func(t *testing.T) { + img := empty.Image + for i := 0; i < numberOfPreviousLayers; i++ { + img, err = mutate.Append(img, mutate.Addendum{Layer: previousLayer}) + if err != nil { + t.Fatal(err) + } + } + + img, err = mutate.Append(img, mutate.Addendum{Layer: lastLayer}) + if err != nil { + t.Fatal(err) + } + actual, err := extractDockerImage(img) + if err != nil { + t.Fatalf("extractDockerImage failed: %v", err) + } + + if string(actual) != exp { + t.Fatalf("got %s, but want %s", string(actual), exp) + } + }) + } + }) + + t.Run("invalid media type", func(t *testing.T) { + l, err := newMockLayer(types.DockerPluginConfig, nil) + if err != nil { + t.Fatal(err) + } + img, err := mutate.Append(empty.Image, mutate.Addendum{Layer: l}) + if err != nil { + t.Fatal(err) + } + _, err = extractDockerImage(img) + if err == nil || !strings.Contains(err.Error(), "invalid media type") { + t.Fatal("extractDockerImage should fail due to invalid media type") + } + }) +} + +func TestExtractOCIStandardImage(t *testing.T) { + t.Run("no layers", func(t *testing.T) { + _, err := extractOCIStandardImage(empty.Image) + if err == nil || err.Error() != "number of layers must be greater than zero" { + t.Fatal("extractDockerImage should fail due to empty image") + } + }) + + t.Run("valid layers", func(t *testing.T) { + previousLayer, err := newMockLayer(types.DockerLayer, nil) + if err != nil { + t.Fatal(err) + } + + exp := "this is wasm binary" + lastLayer, err := newMockLayer(types.OCILayer, map[string][]byte{ + "plugin.wasm": []byte(exp), + }) + if err != nil { + t.Fatal(err) + } + + tCases := map[string]int{ + "one layer": 0, + "more than one layer": 1, + } + + for name, numberOfPreviousLayers := range tCases { + t.Run(name, func(t *testing.T) { + img := empty.Image + for i := 0; i < numberOfPreviousLayers; i++ { + img, err = mutate.Append(img, mutate.Addendum{Layer: previousLayer}) + if err != nil { + t.Fatal(err) + } + } + + img, err = mutate.Append(img, mutate.Addendum{Layer: lastLayer}) + if err != nil { + t.Fatal(err) + } + actual, err := extractOCIStandardImage(img) + if err != nil { + t.Fatalf("extractOCIStandardImage failed: %v", err) + } + + if string(actual) != exp { + t.Fatalf("got %s, but want %s", string(actual), exp) + } + }) + } + }) + + t.Run("invalid media type", func(t *testing.T) { + l, err := newMockLayer(types.DockerLayer, nil) + if err != nil { + t.Fatal(err) + } + img, err := mutate.Append(empty.Image, mutate.Addendum{Layer: l}) + if err != nil { + t.Fatal(err) + } + _, err = extractOCIStandardImage(img) + if err == nil || !strings.Contains(err.Error(), "invalid media type") { + t.Fatal("extractOCIStandardImage should fail due to invalid media type") + } + }) +} + +func newMockLayer(mediaType types.MediaType, contents map[string][]byte) (v1.Layer, error) { + var b bytes.Buffer + hasher := sha256.New() + mw := io.MultiWriter(&b, hasher) + tw := tar.NewWriter(mw) + defer tw.Close() + + for filename, content := range contents { + if err := tw.WriteHeader(&tar.Header{ + Name: filename, + Size: int64(len(content)), + Typeflag: tar.TypeReg, + }); err != nil { + return nil, err + } + if _, err := io.CopyN(tw, bytes.NewReader(content), int64(len(content))); err != nil { + return nil, err + } + } + return partial.UncompressedToLayer( + &mockLayer{ + raw: b.Bytes(), + diffID: v1.Hash{ + Algorithm: "sha256", + Hex: hex.EncodeToString(hasher.Sum(make([]byte, 0, hasher.Size()))), + }, + mediaType: mediaType, + }, + ) +} + +type mockLayer struct { + raw []byte + diffID v1.Hash + mediaType types.MediaType +} + +func (r *mockLayer) DiffID() (v1.Hash, error) { return v1.Hash{}, nil } +func (r *mockLayer) Uncompressed() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewBuffer(r.raw)), nil +} +func (r *mockLayer) MediaType() (types.MediaType, error) { return r.mediaType, nil } + +func TestExtractOCIArtifactImage(t *testing.T) { + t.Run("valid", func(t *testing.T) { + // Create the image with custom media types. + wasmLayer, err := random.Layer(1000, "application/vnd.module.wasm.content.layer.v1+wasm") + if err != nil { + t.Fatal(err) + } + configLayer, err := random.Layer(1000, "application/vnd.module.wasm.config.v1+json") + if err != nil { + t.Fatal(err) + } + img, err := mutate.Append(empty.Image, mutate.Addendum{Layer: wasmLayer}, mutate.Addendum{Layer: configLayer}) + if err != nil { + t.Fatal(err) + } + + // Extract the binary. + actual, err := extractOCIArtifactImage(img) + if err != nil { + t.Fatalf("extractOCIArtifactImage failed: %v", err) + } + + // Retrieve the wanted image content. + wantReader, err := wasmLayer.Compressed() + if err != nil { + t.Fatal(err) + } + defer wantReader.Close() + want, err := io.ReadAll(wantReader) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(actual, want) { + t.Errorf("extractOCIArtifactImage got %s, but want '%s'", string(actual), string(want)) + } + }) + + t.Run("invalid number of layers", func(t *testing.T) { + l, err := random.Layer(1000, "application/vnd.module.wasm.content.layer.v1+wasm") + if err != nil { + t.Fatal(err) + } + img, err := mutate.Append(empty.Image, mutate.Addendum{Layer: l}) + if err != nil { + t.Fatal(err) + } + _, err = extractOCIArtifactImage(img) + if err == nil || !strings.Contains(err.Error(), "number of layers must be") { + t.Fatal("extractOCIArtifactImage should fail due to invalid number of layers") + } + }) + + t.Run("invalid media types", func(t *testing.T) { + // Create the image with invalid media types. + layer, err := random.Layer(1000, "aaa") + if err != nil { + t.Fatal(err) + } + img, err := mutate.Append(empty.Image, mutate.Addendum{Layer: layer}, mutate.Addendum{Layer: layer}) + if err != nil { + t.Fatal(err) + } + + _, err = extractOCIArtifactImage(img) + if err == nil || !strings.Contains(err.Error(), + "could not find the layer of type application/vnd.module.wasm.content.layer.v1+wasm") { + t.Fatal("extractOCIArtifactImage should fail due to invalid number of layers") + } + }) +} + +func TestExtractWasmPluginBinary(t *testing.T) { + t.Run("ok", func(t *testing.T) { + buf := bytes.NewBuffer(nil) + gz := gzip.NewWriter(buf) + tw := tar.NewWriter(gz) + + exp := "hello" + if err := tw.WriteHeader(&tar.Header{ + Name: "plugin.wasm", + Size: int64(len(exp)), + }); err != nil { + t.Fatal(err) + } + + if _, err := io.WriteString(tw, exp); err != nil { + t.Fatal(err) + } + + tw.Close() + gz.Close() + + actual, err := extractWasmPluginBinary(buf) + if err != nil { + t.Errorf("extractWasmPluginBinary failed: %v", err) + } + + if string(actual) != exp { + t.Errorf("extractWasmPluginBinary got %v, but want %v", string(actual), exp) + } + }) + + t.Run("ok with relative path prefix", func(t *testing.T) { + buf := bytes.NewBuffer(nil) + gz := gzip.NewWriter(buf) + tw := tar.NewWriter(gz) + + exp := "hello" + if err := tw.WriteHeader(&tar.Header{ + Name: "./plugin.wasm", + Size: int64(len(exp)), + }); err != nil { + t.Fatal(err) + } + + if _, err := io.WriteString(tw, exp); err != nil { + t.Fatal(err) + } + + tw.Close() + gz.Close() + + actual, err := extractWasmPluginBinary(buf) + if err != nil { + t.Errorf("extractWasmPluginBinary failed: %v", err) + } + + if string(actual) != exp { + t.Errorf("extractWasmPluginBinary got %v, but want %v", string(actual), exp) + } + }) + + t.Run("not found", func(t *testing.T) { + buf := bytes.NewBuffer(nil) + gz := gzip.NewWriter(buf) + tw := tar.NewWriter(gz) + if err := tw.WriteHeader(&tar.Header{ + Name: "non-wasm.txt", + Size: int64(1), + }); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte{1}); err != nil { + t.Fatal(err) + } + tw.Close() + gz.Close() + _, err := extractWasmPluginBinary(buf) + if err == nil || !strings.Contains(err.Error(), "not found") { + t.Errorf("extractWasmPluginBinary must fail with not found") + } + }) +} + +func TestWasmKeyChain(t *testing.T) { + dockerjson := fmt.Sprintf(`{"auths": {"test.io": {"auth": %q}}}`, encode("foo", "bar")) + keyChain := wasmKeyChain{data: []byte(dockerjson)} + testRegistry, _ := name.NewRegistry("test.io", name.WeakValidation) + _, _ = keyChain.Resolve(testRegistry) + auth, err := keyChain.Resolve(testRegistry) + if err != nil { + t.Fatalf("Resolve() = %v", err) + } + got, err := auth.Authorization() + if err != nil { + t.Fatal(err) + } + want := &authn.AuthConfig{ + Username: "foo", + Password: "bar", + } + if !reflect.DeepEqual(got, want) { + t.Errorf("got %+v, want %+v", got, want) + } +} + +func encode(user, pass string) string { + delimited := fmt.Sprintf("%s:%s", user, pass) + return base64.StdEncoding.EncodeToString([]byte(delimited)) +} diff --git a/internal/wasm/metrics.go b/internal/wasm/metrics.go new file mode 100644 index 00000000000..9de7e6208f0 --- /dev/null +++ b/internal/wasm/metrics.go @@ -0,0 +1,54 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// Copyright Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wasm + +import "github.com/envoyproxy/gateway/internal/metrics" + +// Const strings for label value. +const ( + // For remote fetch metric. + fetchSuccess = "success" + fetchFailure = "fetch_failure" + downloadFailure = "download_failure" + manifestFailure = "manifest_failure" + checksumMismatch = "checksum_mismatched" +) + +var ( + hitTag = metrics.NewLabel("hit") + resultTag = metrics.NewLabel("result") + + wasmCacheEntries = metrics.NewGauge( + "wasm_cache_entries", + "number of Wasm remote fetch cache entries.", + ) + + wasmCacheLookupCount = metrics.NewCounter( + "wasm_cache_lookup_count", + "number of Wasm remote fetch cache lookups.", + ) + + wasmRemoteFetchCount = metrics.NewCounter( + "wasm_remote_fetch_count", + "number of Wasm remote fetches and results, including success, download failure, and checksum mismatch.", + ) +) + +// TODO zhaohuabing export metrics to control plane dashboard. diff --git a/internal/wasm/options.go b/internal/wasm/options.go new file mode 100644 index 00000000000..e3838bac33c --- /dev/null +++ b/internal/wasm/options.go @@ -0,0 +1,108 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// Copyright Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wasm + +import ( + "time" + + "k8s.io/apimachinery/pkg/util/sets" +) + +const ( + DefaultPurgeInterval = 1 * time.Hour + DefaultModuleExpiry = 24 * time.Hour + DefaultHTTPRequestTimeout = 15 * time.Second + DefaultHTTPRequestMaxRetries = 5 + DefaultPullTimeout = 5 * time.Minute + DefaultMaxCacheSize = 1024 * 1024 * 1024 // 1GB +) + +// CacheOptions contains configurations to create a Cache instance. +type CacheOptions struct { + PurgeInterval time.Duration + ModuleExpiry time.Duration + // InsecureRegistries is a set of registries that are allowed to be accessed without TLS. + InsecureRegistries sets.Set[string] + HTTPRequestTimeout time.Duration + HTTPRequestMaxRetries int + MaxCacheSize int + CacheDir string +} + +// allowInsecure returns true if the host is allowed to be accessed without TLS. +func (o *CacheOptions) allowInsecure(host string) bool { + return o.InsecureRegistries.Has(host) || o.InsecureRegistries.Has("*") +} + +func (o *CacheOptions) sanitize() CacheOptions { + ret := defaultCacheOptions() + if o.InsecureRegistries != nil { + ret.InsecureRegistries = o.InsecureRegistries + } + if o.PurgeInterval != 0 { + ret.PurgeInterval = o.PurgeInterval + } + if o.ModuleExpiry != 0 { + ret.ModuleExpiry = o.ModuleExpiry + } + if o.HTTPRequestTimeout != 0 { + ret.HTTPRequestTimeout = o.HTTPRequestTimeout + } + if o.HTTPRequestMaxRetries != 0 { + ret.HTTPRequestMaxRetries = o.HTTPRequestMaxRetries + } + if o.MaxCacheSize != 0 { + ret.MaxCacheSize = o.MaxCacheSize + } + if o.CacheDir != "" { + ret.CacheDir = o.CacheDir + } + + return ret +} + +func defaultCacheOptions() CacheOptions { + return CacheOptions{ + PurgeInterval: DefaultPurgeInterval, + ModuleExpiry: DefaultModuleExpiry, + InsecureRegistries: sets.New[string](), + HTTPRequestTimeout: DefaultHTTPRequestTimeout, + HTTPRequestMaxRetries: DefaultHTTPRequestMaxRetries, + MaxCacheSize: DefaultMaxCacheSize, + } +} + +type PullPolicy int32 + +const ( + Unspecified PullPolicy = 0 + IfNotPresent PullPolicy = 1 + Always PullPolicy = 2 +) + +// GetOptions is a struct for providing options to Get method of Cache. +type GetOptions struct { + Checksum string + ResourceName string + ResourceVersion string + RequestTimeout time.Duration + PullSecret []byte + PullPolicy PullPolicy +} diff --git a/internal/xds/bootstrap/bootstrap.go b/internal/xds/bootstrap/bootstrap.go index eb5e4398dc5..e74620483f4 100644 --- a/internal/xds/bootstrap/bootstrap.go +++ b/internal/xds/bootstrap/bootstrap.go @@ -35,6 +35,10 @@ const ( // DefaultXdsServerPort is the default listening port of the xds-server. DefaultXdsServerPort = 18000 + wasmServerHost = envoyGatewayXdsServerHost + // DefaultWasmServerPort is the default listening port of the wasm HTTP server. + wasmServerPort = 18002 + envoyReadinessAddress = "0.0.0.0" EnvoyReadinessPort = 19001 EnvoyReadinessPath = "/ready" @@ -56,7 +60,9 @@ type bootstrapConfig struct { // bootstrapParameters defines the envoy Bootstrap configuration. type bootstrapParameters struct { // XdsServer defines the configuration of the XDS server. - XdsServer xdsServerParameters + XdsServer serverParameters + // WasmServer defines the configuration of the Wasm HTTP server. + WasmServer serverParameters // AdminServer defines the configuration of the Envoy admin interface. AdminServer adminServerParameters // ReadyServer defines the configuration for health check ready listener @@ -79,7 +85,7 @@ type bootstrapParameters struct { OverloadManager overloadManagerParameters } -type xdsServerParameters struct { +type serverParameters struct { // Address is the address of the XDS Server that Envoy is managed by. Address string // Port is the port of the XDS Server that Envoy is managed by. @@ -214,10 +220,14 @@ func GetRenderedBootstrapConfig(opts *RenderBootstrapConfigOptions) (string, err cfg := &bootstrapConfig{ parameters: bootstrapParameters{ - XdsServer: xdsServerParameters{ + XdsServer: serverParameters{ Address: envoyGatewayXdsServerHost, Port: DefaultXdsServerPort, }, + WasmServer: serverParameters{ + Address: wasmServerHost, + Port: wasmServerPort, + }, AdminServer: adminServerParameters{ Address: EnvoyAdminAddress, Port: EnvoyAdminPort, diff --git a/internal/xds/bootstrap/bootstrap.yaml.tpl b/internal/xds/bootstrap/bootstrap.yaml.tpl index af05d9752a5..b7d26c7d4a9 100644 --- a/internal/xds/bootstrap/bootstrap.yaml.tpl +++ b/internal/xds/bootstrap/bootstrap.yaml.tpl @@ -199,6 +199,44 @@ static_resources: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: {{ .WasmServer.Address }} + port_value: {{ .WasmServer.Port }} + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/xds/bootstrap/testdata/merge/default.out.yaml b/internal/xds/bootstrap/testdata/merge/default.out.yaml index 0fc11f219be..e0a187fd8bc 100644 --- a/internal/xds/bootstrap/testdata/merge/default.out.yaml +++ b/internal/xds/bootstrap/testdata/merge/default.out.yaml @@ -97,6 +97,44 @@ staticResources: connectionKeepalive: interval: 30s timeout: 5s + - connectTimeout: 10s + loadAssignment: + clusterName: wasm_cluster + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: envoy-gateway + portValue: 18002 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + name: wasm_cluster + transportSocket: + name: envoy.transport_sockets.tls + typedConfig: + '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + commonTlsContext: + tlsCertificateSdsSecretConfigs: + - name: xds_certificate + sdsConfig: + pathConfigSource: + path: /sds/xds-certificate.json + resourceApiVersion: V3 + tlsParams: + tlsMaximumProtocolVersion: TLSv1_3 + validationContextSdsSecretConfig: + name: xds_trusted_ca + sdsConfig: + pathConfigSource: + path: /sds/xds-trusted-ca.json + resourceApiVersion: V3 + type: STRICT_DNS + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicitHttpConfig: + http2ProtocolOptions: {} listeners: - address: socketAddress: diff --git a/internal/xds/bootstrap/testdata/merge/stats_sinks.out.yaml b/internal/xds/bootstrap/testdata/merge/stats_sinks.out.yaml index c805025fd13..40d2392a98d 100644 --- a/internal/xds/bootstrap/testdata/merge/stats_sinks.out.yaml +++ b/internal/xds/bootstrap/testdata/merge/stats_sinks.out.yaml @@ -91,6 +91,44 @@ staticResources: connectionKeepalive: interval: 30s timeout: 5s + - connectTimeout: 10s + loadAssignment: + clusterName: wasm_cluster + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: envoy-gateway + portValue: 18002 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + name: wasm_cluster + transportSocket: + name: envoy.transport_sockets.tls + typedConfig: + '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + commonTlsContext: + tlsCertificateSdsSecretConfigs: + - name: xds_certificate + sdsConfig: + pathConfigSource: + path: /sds/xds-certificate.json + resourceApiVersion: V3 + tlsParams: + tlsMaximumProtocolVersion: TLSv1_3 + validationContextSdsSecretConfig: + name: xds_trusted_ca + sdsConfig: + pathConfigSource: + path: /sds/xds-trusted-ca.json + resourceApiVersion: V3 + type: STRICT_DNS + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicitHttpConfig: + http2ProtocolOptions: {} - connectTimeout: 1s dnsLookupFamily: V4_ONLY dnsRefreshRate: 30s diff --git a/internal/xds/bootstrap/testdata/render/custom-stats-matcher.yaml b/internal/xds/bootstrap/testdata/render/custom-stats-matcher.yaml index 3a588cb9369..e23e57ff515 100644 --- a/internal/xds/bootstrap/testdata/render/custom-stats-matcher.yaml +++ b/internal/xds/bootstrap/testdata/render/custom-stats-matcher.yaml @@ -132,6 +132,44 @@ static_resources: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/xds/bootstrap/testdata/render/disable-prometheus.yaml b/internal/xds/bootstrap/testdata/render/disable-prometheus.yaml index 86b4ea3ee00..02902fec330 100644 --- a/internal/xds/bootstrap/testdata/render/disable-prometheus.yaml +++ b/internal/xds/bootstrap/testdata/render/disable-prometheus.yaml @@ -99,6 +99,44 @@ static_resources: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/xds/bootstrap/testdata/render/enable-prometheus-gzip-compression.yaml b/internal/xds/bootstrap/testdata/render/enable-prometheus-gzip-compression.yaml index ca82e1996b4..39219431305 100644 --- a/internal/xds/bootstrap/testdata/render/enable-prometheus-gzip-compression.yaml +++ b/internal/xds/bootstrap/testdata/render/enable-prometheus-gzip-compression.yaml @@ -128,6 +128,44 @@ static_resources: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/xds/bootstrap/testdata/render/enable-prometheus.yaml b/internal/xds/bootstrap/testdata/render/enable-prometheus.yaml index 347bccd5376..f2e0b49b859 100644 --- a/internal/xds/bootstrap/testdata/render/enable-prometheus.yaml +++ b/internal/xds/bootstrap/testdata/render/enable-prometheus.yaml @@ -121,6 +121,44 @@ static_resources: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/xds/bootstrap/testdata/render/otel-metrics-backendref.yaml b/internal/xds/bootstrap/testdata/render/otel-metrics-backendref.yaml index db865b4cb8b..6079f777dc8 100644 --- a/internal/xds/bootstrap/testdata/render/otel-metrics-backendref.yaml +++ b/internal/xds/bootstrap/testdata/render/otel-metrics-backendref.yaml @@ -124,6 +124,44 @@ static_resources: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/xds/bootstrap/testdata/render/otel-metrics.yaml b/internal/xds/bootstrap/testdata/render/otel-metrics.yaml index db865b4cb8b..6079f777dc8 100644 --- a/internal/xds/bootstrap/testdata/render/otel-metrics.yaml +++ b/internal/xds/bootstrap/testdata/render/otel-metrics.yaml @@ -124,6 +124,44 @@ static_resources: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/xds/bootstrap/testdata/render/with-max-heap-size-bytes.yaml b/internal/xds/bootstrap/testdata/render/with-max-heap-size-bytes.yaml index e79291a7bb0..9eebf9d010c 100644 --- a/internal/xds/bootstrap/testdata/render/with-max-heap-size-bytes.yaml +++ b/internal/xds/bootstrap/testdata/render/with-max-heap-size-bytes.yaml @@ -121,6 +121,44 @@ static_resources: path_config_source: path: "/sds/xds-trusted-ca.json" resource_api_version: V3 + - name: wasm_cluster + type: STRICT_DNS + connect_timeout: 10s + load_assignment: + cluster_name: wasm_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18002 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: "/sds/xds-certificate.json" + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: "/sds/xds-trusted-ca.json" + resource_api_version: V3 overload_manager: refresh_interval: 0.25s resource_monitors: diff --git a/internal/xds/translator/testdata/in/xds-ir/custom-filter-order.yaml b/internal/xds/translator/testdata/in/xds-ir/custom-filter-order.yaml index 8dcefc9c880..4650e1ea2a3 100644 --- a/internal/xds/translator/testdata/in/xds-ir/custom-filter-order.yaml +++ b/internal/xds/translator/testdata/in/xds-ir/custom-filter-order.yaml @@ -70,7 +70,7 @@ http: failOpen: false httpWasmCode: sha256: 746df05c8f3a0b07a46c0967cfbc5cbe5b9d48d0f79b6177eeedf8be6c8b34b5 - url: https://www.example.com/wasm-filter-1.wasm + servingURL: https://envoy-gateway:18002/42d30b4a4cc631415e6e48c02d244700da327201eb273f752cacf745715b31d9.wasm name: envoyextensionpolicy/envoy-gateway/policy-for-gateway/0 wasmName: wasm-filter-1 - config: @@ -79,6 +79,6 @@ http: failOpen: false httpWasmCode: sha256: a1efca12ea51069abb123bf9c77889fcc2a31cc5483fc14d115e44fdf07c7980 - url: https://www.example.com/wasm-filter-2.wasm + servingURL: https://envoy-gateway:18002/7abf116e5cd5a20389604a5ba0f3bd04fdf76f92181fe67506b42c2ee596d3fd.wasm name: envoyextensionpolicy/envoy-gateway/policy-for-gateway/1 wasmName: wasm-filter-2 diff --git a/internal/xds/translator/testdata/in/xds-ir/wasm.yaml b/internal/xds/translator/testdata/in/xds-ir/wasm.yaml index 06f767957c1..faa729eec98 100644 --- a/internal/xds/translator/testdata/in/xds-ir/wasm.yaml +++ b/internal/xds/translator/testdata/in/xds-ir/wasm.yaml @@ -9,10 +9,7 @@ http: mergeSlashes: true port: 10080 routes: - - backendWeights: - invalid: 0 - valid: 0 - destination: + - destination: name: httproute/default/httproute-1/rule/0 settings: - addressType: IP @@ -37,10 +34,11 @@ http: key3: value3 failOpen: true httpWasmCode: + servingURL: https://envoy-gateway:18002/fe571e7b1ef5dc626ceb2c2c86782a134a92989a2643485238951696ae4334c3.wasm + originalDownloadingURL: https://www.test.com/wasm-filter-4.wasm sha256: a1f0b78b8c1320690327800e3a5de10e7dbba7b6c752e702193a395a52c727b6 - url: https://www.test.com/wasm-filter-3.wasm - name: envoyextensionpolicy/default/policy-for-http-route/0 - wasmName: wasm-filter-3 + name: envoyextensionpolicy/default/policy-for-http-route/wasm/0 + wasmName: wasm-filter-4 - destination: name: httproute/default/httproute-2/rule/0 settings: @@ -65,17 +63,27 @@ http: parameter2: value3 failOpen: false httpWasmCode: + servingURL: https://envoy-gateway:18002/5c90b9a82642ce00a7753923fabead306b9d9a54a7c0bd2463a1af3efcfb110b.wasm + originalDownloadingURL: https://www.example.com/wasm-filter-1.wasm sha256: 746df05c8f3a0b07a46c0967cfbc5cbe5b9d48d0f79b6177eeedf8be6c8b34b5 - url: https://www.example.com/wasm-filter-1.wasm - name: envoyextensionpolicy/envoy-gateway/policy-for-gateway/0 + name: envoyextensionpolicy/envoy-gateway/policy-for-gateway/wasm/0 wasmName: wasm-filter-1 - rootID: my-root-id - config: parameter1: value1 parameter2: value2 failOpen: false httpWasmCode: - sha256: a1efca12ea51069abb123bf9c77889fcc2a31cc5483fc14d115e44fdf07c7980 - url: https://www.example.com/wasm-filter-2.wasm - name: envoyextensionpolicy/envoy-gateway/policy-for-gateway/1 + servingURL: https://envoy-gateway:18002/7abf116e5cd5a20389604a5ba0f3bd04fdf76f92181fe67506b42c2ee596d3fd.wasm + originalDownloadingURL: oci://www.example.com/wasm-filter-2:v1.0.0 + sha256: 314100af781b98a8ca175d5bf90a8bf76576e20a2f397a88223404edc6ebfd46 + name: envoyextensionpolicy/envoy-gateway/policy-for-gateway/wasm/1 wasmName: wasm-filter-2 + rootID: my-root-id + - config: null + failOpen: false + httpWasmCode: + servingURL: https://envoy-gateway:18002/42d30b4a4cc631415e6e48c02d244700da327201eb273f752cacf745715b31d9.wasm + originalDownloadingURL: oci://www.example.com:8080/wasm-filter-3:latest + sha256: 2a19e4f337e5223d7287e7fccd933fb01905deaff804292e5257f8c681b82bee + name: envoyextensionpolicy/envoy-gateway/policy-for-gateway/wasm/2 + wasmName: envoyextensionpolicy/envoy-gateway/policy-for-gateway/wasm/2 diff --git a/internal/xds/translator/testdata/out/xds-ir/custom-filter-order.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/custom-filter-order.clusters.yaml index ae5259499fd..0e10ab58f0c 100644 --- a/internal/xds/translator/testdata/out/xds-ir/custom-filter-order.clusters.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/custom-filter-order.clusters.yaml @@ -61,39 +61,3 @@ perConnectionBufferLimitBytes: 32768 respectDnsTtl: true type: STRICT_DNS -- circuitBreakers: - thresholds: - - maxRetries: 1024 - commonLbConfig: - localityWeightedLbConfig: {} - connectTimeout: 10s - dnsLookupFamily: V4_ONLY - dnsRefreshRate: 30s - lbPolicy: LEAST_REQUEST - loadAssignment: - clusterName: www_example_com_443 - endpoints: - - lbEndpoints: - - endpoint: - address: - socketAddress: - address: www.example.com - portValue: 443 - loadBalancingWeight: 1 - loadBalancingWeight: 1 - locality: - region: www_example_com_443/backend/0 - name: www_example_com_443 - outlierDetection: {} - perConnectionBufferLimitBytes: 32768 - respectDnsTtl: true - transportSocket: - name: envoy.transport_sockets.tls - typedConfig: - '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext - commonTlsContext: - validationContext: - trustedCa: - filename: /etc/ssl/certs/ca-certificates.crt - sni: www.example.com - type: STRICT_DNS diff --git a/internal/xds/translator/testdata/out/xds-ir/custom-filter-order.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/custom-filter-order.listeners.yaml index 5c87d208ac1..5f54802ba05 100644 --- a/internal/xds/translator/testdata/out/xds-ir/custom-filter-order.listeners.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/custom-filter-order.listeners.yaml @@ -36,9 +36,9 @@ code: remote: httpUri: - cluster: www_example_com_443 + cluster: wasm_cluster timeout: 10s - uri: https://www.example.com/wasm-filter-1.wasm + uri: https://envoy-gateway:18002/42d30b4a4cc631415e6e48c02d244700da327201eb273f752cacf745715b31d9.wasm sha256: 746df05c8f3a0b07a46c0967cfbc5cbe5b9d48d0f79b6177eeedf8be6c8b34b5 runtime: envoy.wasm.runtime.v8 vmId: envoyextensionpolicy/envoy-gateway/policy-for-gateway/0 @@ -55,9 +55,9 @@ code: remote: httpUri: - cluster: www_example_com_443 + cluster: wasm_cluster timeout: 10s - uri: https://www.example.com/wasm-filter-2.wasm + uri: https://envoy-gateway:18002/7abf116e5cd5a20389604a5ba0f3bd04fdf76f92181fe67506b42c2ee596d3fd.wasm sha256: a1efca12ea51069abb123bf9c77889fcc2a31cc5483fc14d115e44fdf07c7980 runtime: envoy.wasm.runtime.v8 vmId: envoyextensionpolicy/envoy-gateway/policy-for-gateway/1 diff --git a/internal/xds/translator/testdata/out/xds-ir/wasm.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/wasm.clusters.yaml index a70d771a1db..6a277bb94f6 100755 --- a/internal/xds/translator/testdata/out/xds-ir/wasm.clusters.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/wasm.clusters.yaml @@ -32,75 +32,3 @@ outlierDetection: {} perConnectionBufferLimitBytes: 32768 type: EDS -- circuitBreakers: - thresholds: - - maxRetries: 1024 - commonLbConfig: - localityWeightedLbConfig: {} - connectTimeout: 10s - dnsLookupFamily: V4_ONLY - dnsRefreshRate: 30s - lbPolicy: LEAST_REQUEST - loadAssignment: - clusterName: www_test_com_443 - endpoints: - - lbEndpoints: - - endpoint: - address: - socketAddress: - address: www.test.com - portValue: 443 - loadBalancingWeight: 1 - loadBalancingWeight: 1 - locality: - region: www_test_com_443/backend/0 - name: www_test_com_443 - outlierDetection: {} - perConnectionBufferLimitBytes: 32768 - respectDnsTtl: true - transportSocket: - name: envoy.transport_sockets.tls - typedConfig: - '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext - commonTlsContext: - validationContext: - trustedCa: - filename: /etc/ssl/certs/ca-certificates.crt - sni: www.test.com - type: STRICT_DNS -- circuitBreakers: - thresholds: - - maxRetries: 1024 - commonLbConfig: - localityWeightedLbConfig: {} - connectTimeout: 10s - dnsLookupFamily: V4_ONLY - dnsRefreshRate: 30s - lbPolicy: LEAST_REQUEST - loadAssignment: - clusterName: www_example_com_443 - endpoints: - - lbEndpoints: - - endpoint: - address: - socketAddress: - address: www.example.com - portValue: 443 - loadBalancingWeight: 1 - loadBalancingWeight: 1 - locality: - region: www_example_com_443/backend/0 - name: www_example_com_443 - outlierDetection: {} - perConnectionBufferLimitBytes: 32768 - respectDnsTtl: true - transportSocket: - name: envoy.transport_sockets.tls - typedConfig: - '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext - commonTlsContext: - validationContext: - trustedCa: - filename: /etc/ssl/certs/ca-certificates.crt - sni: www.example.com - type: STRICT_DNS diff --git a/internal/xds/translator/testdata/out/xds-ir/wasm.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/wasm.listeners.yaml index 4d59b5842d5..e5ecdbb8156 100755 --- a/internal/xds/translator/testdata/out/xds-ir/wasm.listeners.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/wasm.listeners.yaml @@ -15,7 +15,7 @@ maxConcurrentStreams: 100 httpFilters: - disabled: true - name: envoy.filters.http.wasm/envoyextensionpolicy/default/policy-for-http-route/0 + name: envoy.filters.http.wasm/envoyextensionpolicy/default/policy-for-http-route/wasm/0 typedConfig: '@type': type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm config: @@ -23,19 +23,19 @@ '@type': type.googleapis.com/google.protobuf.StringValue value: '{"parameter1":{"key1":"value1"},"parameter2":{"key2":{"key3":"value3"}}}' failOpen: true - name: wasm-filter-3 + name: wasm-filter-4 vmConfig: code: remote: httpUri: - cluster: www_test_com_443 + cluster: wasm_cluster timeout: 10s - uri: https://www.test.com/wasm-filter-3.wasm + uri: https://envoy-gateway:18002/fe571e7b1ef5dc626ceb2c2c86782a134a92989a2643485238951696ae4334c3.wasm sha256: a1f0b78b8c1320690327800e3a5de10e7dbba7b6c752e702193a395a52c727b6 runtime: envoy.wasm.runtime.v8 - vmId: envoyextensionpolicy/default/policy-for-http-route/0 + vmId: envoyextensionpolicy/default/policy-for-http-route/wasm/0 - disabled: true - name: envoy.filters.http.wasm/envoyextensionpolicy/envoy-gateway/policy-for-gateway/0 + name: envoy.filters.http.wasm/envoyextensionpolicy/envoy-gateway/policy-for-gateway/wasm/0 typedConfig: '@type': type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm config: @@ -43,19 +43,18 @@ '@type': type.googleapis.com/google.protobuf.StringValue value: '{"parameter1":{"key1":"value1","key2":"value2"},"parameter2":"value3"}' name: wasm-filter-1 - rootId: my-root-id vmConfig: code: remote: httpUri: - cluster: www_example_com_443 + cluster: wasm_cluster timeout: 10s - uri: https://www.example.com/wasm-filter-1.wasm + uri: https://envoy-gateway:18002/5c90b9a82642ce00a7753923fabead306b9d9a54a7c0bd2463a1af3efcfb110b.wasm sha256: 746df05c8f3a0b07a46c0967cfbc5cbe5b9d48d0f79b6177eeedf8be6c8b34b5 runtime: envoy.wasm.runtime.v8 - vmId: envoyextensionpolicy/envoy-gateway/policy-for-gateway/0 + vmId: envoyextensionpolicy/envoy-gateway/policy-for-gateway/wasm/0 - disabled: true - name: envoy.filters.http.wasm/envoyextensionpolicy/envoy-gateway/policy-for-gateway/1 + name: envoy.filters.http.wasm/envoyextensionpolicy/envoy-gateway/policy-for-gateway/wasm/1 typedConfig: '@type': type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm config: @@ -63,16 +62,36 @@ '@type': type.googleapis.com/google.protobuf.StringValue value: '{"parameter1":"value1","parameter2":"value2"}' name: wasm-filter-2 + rootId: my-root-id + vmConfig: + code: + remote: + httpUri: + cluster: wasm_cluster + timeout: 10s + uri: https://envoy-gateway:18002/7abf116e5cd5a20389604a5ba0f3bd04fdf76f92181fe67506b42c2ee596d3fd.wasm + sha256: 314100af781b98a8ca175d5bf90a8bf76576e20a2f397a88223404edc6ebfd46 + runtime: envoy.wasm.runtime.v8 + vmId: envoyextensionpolicy/envoy-gateway/policy-for-gateway/wasm/1 + - disabled: true + name: envoy.filters.http.wasm/envoyextensionpolicy/envoy-gateway/policy-for-gateway/wasm/2 + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm + config: + configuration: + '@type': type.googleapis.com/google.protobuf.StringValue + value: "" + name: envoyextensionpolicy/envoy-gateway/policy-for-gateway/wasm/2 vmConfig: code: remote: httpUri: - cluster: www_example_com_443 + cluster: wasm_cluster timeout: 10s - uri: https://www.example.com/wasm-filter-2.wasm - sha256: a1efca12ea51069abb123bf9c77889fcc2a31cc5483fc14d115e44fdf07c7980 + uri: https://envoy-gateway:18002/42d30b4a4cc631415e6e48c02d244700da327201eb273f752cacf745715b31d9.wasm + sha256: 2a19e4f337e5223d7287e7fccd933fb01905deaff804292e5257f8c681b82bee runtime: envoy.wasm.runtime.v8 - vmId: envoyextensionpolicy/envoy-gateway/policy-for-gateway/1 + vmId: envoyextensionpolicy/envoy-gateway/policy-for-gateway/wasm/2 - name: envoy.filters.http.router typedConfig: '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router diff --git a/internal/xds/translator/testdata/out/xds-ir/wasm.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/wasm.routes.yaml index 8fb6f03a0f0..1e07a621d5a 100755 --- a/internal/xds/translator/testdata/out/xds-ir/wasm.routes.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/wasm.routes.yaml @@ -13,7 +13,7 @@ upgradeConfigs: - upgradeType: websocket typedPerFilterConfig: - envoy.filters.http.wasm/envoyextensionpolicy/default/policy-for-http-route/0: + envoy.filters.http.wasm/envoyextensionpolicy/default/policy-for-http-route/wasm/0: '@type': type.googleapis.com/envoy.config.route.v3.FilterConfig config: {} - match: @@ -24,9 +24,12 @@ upgradeConfigs: - upgradeType: websocket typedPerFilterConfig: - envoy.filters.http.wasm/envoyextensionpolicy/envoy-gateway/policy-for-gateway/0: + envoy.filters.http.wasm/envoyextensionpolicy/envoy-gateway/policy-for-gateway/wasm/0: '@type': type.googleapis.com/envoy.config.route.v3.FilterConfig config: {} - envoy.filters.http.wasm/envoyextensionpolicy/envoy-gateway/policy-for-gateway/1: + envoy.filters.http.wasm/envoyextensionpolicy/envoy-gateway/policy-for-gateway/wasm/1: + '@type': type.googleapis.com/envoy.config.route.v3.FilterConfig + config: {} + envoy.filters.http.wasm/envoyextensionpolicy/envoy-gateway/policy-for-gateway/wasm/2: '@type': type.googleapis.com/envoy.config.route.v3.FilterConfig config: {} diff --git a/internal/xds/translator/translator_test.go b/internal/xds/translator/translator_test.go index 81563abf050..44d6d127bca 100644 --- a/internal/xds/translator/translator_test.go +++ b/internal/xds/translator/translator_test.go @@ -130,6 +130,7 @@ func TestTranslateXds(t *testing.T) { tCtx, err := tr.Translate(x) if !strings.HasSuffix(inputFileName, "partial-invalid") && len(cfg.errMsg) == 0 { + t.Logf(inputFileName) require.NoError(t, err) } else if len(cfg.errMsg) > 0 { require.Error(t, err) diff --git a/internal/xds/translator/wasm.go b/internal/xds/translator/wasm.go index 1e096318df8..01d7411c6a2 100644 --- a/internal/xds/translator/wasm.go +++ b/internal/xds/translator/wasm.go @@ -22,8 +22,9 @@ import ( ) const ( - wasmFilter = "envoy.filters.http.wasm" - vmRuntimeV8 = "envoy.wasm.runtime.v8" + wasmFilter = "envoy.filters.http.wasm" + vmRuntimeV8 = "envoy.wasm.runtime.v8" + wasmHTTPServerCluster = "wasm_cluster" ) func init() { @@ -103,18 +104,12 @@ func wasmFilterName(wasm ir.Wasm) string { func wasmConfig(wasm ir.Wasm) (*wasmfilterv3.Wasm, error) { var ( - uc *urlCluster pluginConfig = "" configAny *anypb.Any filterConfig *wasmfilterv3.Wasm err error ) - // We only support HTTP Wasm code source for now - if uc, err = url2Cluster(wasm.HTTPWasmCode.URL); err != nil { - return nil, err - } - if wasm.Config != nil { pluginConfig = string(wasm.Config.Raw) } @@ -134,15 +129,15 @@ func wasmConfig(wasm ir.Wasm) (*wasmfilterv3.Wasm, error) { Specifier: &corev3.AsyncDataSource_Remote{ Remote: &corev3.RemoteDataSource{ HttpUri: &corev3.HttpUri{ - Uri: wasm.HTTPWasmCode.URL, + Uri: wasm.Code.ServingURL, HttpUpstreamType: &corev3.HttpUri_Cluster{ - Cluster: uc.name, + Cluster: wasmHTTPServerCluster, }, Timeout: &durationpb.Duration{ Seconds: defaultExtServiceRequestTimeout, }, }, - Sha256: wasm.HTTPWasmCode.SHA256, + Sha256: wasm.Code.SHA256, }, }, }, @@ -170,27 +165,11 @@ func routeContainsWasm(irRoute *ir.HTTPRoute) bool { } // patchResources patches the cluster resources for the http wasm code source. -func (*wasm) patchResources(tCtx *types.ResourceVersionTable, - routes []*ir.HTTPRoute, -) error { - if tCtx == nil || tCtx.XdsResources == nil { - return errors.New("xds resource table is nil") - } - - var err, errs error - for _, route := range routes { - if !routeContainsWasm(route) { - continue - } - - for _, w := range route.Wasms { - if err = addClusterFromURL(w.HTTPWasmCode.URL, tCtx); err != nil { - errs = errors.Join(errs, err) - } - } - } - - return errs +func (*wasm) patchResources(_ *types.ResourceVersionTable, _ []*ir.HTTPRoute) error { + // EG always serves the Wasm module through the built-in HTTP server, which + // has been configured in the bootstrap configuration. So we don't need to + // create a cluster for the Wasm module. + return nil } // patchRoute patches the provided route with the wasm config if applicable. diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index 953a5d29783..37a76398121 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -234,7 +234,7 @@ _Appears in:_ | Field | Type | Required | Description | | --- | --- | --- | --- | -| `name` | _string_ | false | Name is a user-friendly name for the rule. It's just for display purposes. | +| `name` | _string_ | false | Name is a user-friendly name for the rule.
If not specified, Envoy Gateway will generate a unique name for the rule.n | | `action` | _[AuthorizationAction](#authorizationaction)_ | true | Action defines the action to be taken if the rule matches. | | `principal` | _[Principal](#principal)_ | true | Principal specifies the client identity of a request. | @@ -1869,14 +1869,15 @@ _Appears in:_ -HTTPWasmCodeSource defines the HTTP URL containing the wasm code. +HTTPWasmCodeSource defines the HTTP URL containing the Wasm code. _Appears in:_ - [WasmCodeSource](#wasmcodesource) | Field | Type | Required | Description | | --- | --- | --- | --- | -| `url` | _string_ | true | URL is the URL containing the wasm code. | +| `url` | _string_ | true | URL is the URL containing the Wasm code. | +| `sha256` | _string_ | false | SHA256 checksum that will be used to verify the Wasm code.

If not specified, Envoy Gateway will not verify the downloaded Wasm code.
kubebuilder:validation:Pattern=`^[a-f0-9]{64}$` | #### Header @@ -1978,19 +1979,35 @@ _Appears in:_ | `port` | _integer_ | true | Port defines the port of the backend endpoint. | +#### ImagePullPolicy + +_Underlying type:_ _string_ + +ImagePullPolicy defines the policy to use when pulling an OIC image. + +_Appears in:_ +- [WasmCodeSource](#wasmcodesource) + +| Value | Description | +| ----- | ----------- | +| `IfNotPresent` | ImagePullPolicyIfNotPresent will only pull the image if it does not already exist in the EG cache.
| +| `Always` | ImagePullPolicyAlways will pull the image when the EnvoyExtension resource version changes.
Note: EG does not update the Wasm module every time an Envoy proxy requests the Wasm module.
| + + #### ImageWasmCodeSource -ImageWasmCodeSource defines the OCI image containing the wasm code. +ImageWasmCodeSource defines the OCI image containing the Wasm code. _Appears in:_ - [WasmCodeSource](#wasmcodesource) | Field | Type | Required | Description | | --- | --- | --- | --- | -| `url` | _string_ | true | URL is the URL of the OCI image. | -| `pullSecret` | _[SecretObjectReference](https://gateway-api.sigs.k8s.io/references/spec/#gateway.networking.k8s.io/v1.SecretObjectReference)_ | true | PullSecretRef is a reference to the secret containing the credentials to pull the image. | +| `url` | _string_ | true | URL is the URL of the OCI image.
URL can be in the format of `registry/image:tag` or `registry/image@sha256:digest`. | +| `sha256` | _string_ | false | SHA256 checksum that will be used to verify the OCI image.

It must match the digest of the OCI image.

If not specified, Envoy Gateway will not verify the downloaded OCI image.
kubebuilder:validation:Pattern=`^[a-f0-9]{64}$` | +| `pullSecretRef` | _[SecretObjectReference](https://gateway-api.sigs.k8s.io/references/spec/#gateway.networking.k8s.io/v1.SecretObjectReference)_ | false | PullSecretRef is a reference to the secret containing the credentials to pull the image.
Only support Kubernetes Secret resource from the same namespace. | #### InfrastructureProviderType @@ -3634,7 +3651,7 @@ _Appears in:_ -Wasm defines a wasm extension. +Wasm defines a Wasm extension. Note: at the moment, Envoy Gateway does not support configuring Wasm runtime. @@ -3645,9 +3662,9 @@ _Appears in:_ | Field | Type | Required | Description | | --- | --- | --- | --- | -| `name` | _string_ | true | Name is a unique name for this Wasm extension. It is used to identify the
Wasm extension if multiple extensions are handled by the same vm_id and root_id.
It's also used for logging/debugging. | -| `rootID` | _string_ | true | RootID is a unique ID for a set of extensions in a VM which will share a
RootContext and Contexts if applicable (e.g., an Wasm HttpFilter and an Wasm AccessLog).
If left blank, all extensions with a blank root_id with the same vm_id will share Context(s).
RootID must match the root_id parameter used to register the Context in the Wasm code. | -| `code` | _[WasmCodeSource](#wasmcodesource)_ | true | Code is the wasm code for the extension. | +| `name` | _string_ | false | Name is a unique name for this Wasm extension. It is used to identify the
Wasm extension if multiple extensions are handled by the same vm_id and root_id.
It's also used for logging/debugging.
If not specified, EG will generate a unique name for the Wasm extension. | +| `rootID` | _string_ | true | RootID is a unique ID for a set of extensions in a VM which will share a
RootContext and Contexts if applicable (e.g., an Wasm HttpFilter and an Wasm AccessLog).
If left blank, all extensions with a blank root_id with the same vm_id will share Context(s).

Note: RootID must match the root_id parameter used to register the Context in the Wasm code. | +| `code` | _[WasmCodeSource](#wasmcodesource)_ | true | Code is the Wasm code for the extension. | | `config` | _[JSON](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#json-v1-apiextensions-k8s-io)_ | false | Config is the configuration for the Wasm extension.
This configuration will be passed as a JSON string to the Wasm extension. | | `failOpen` | _boolean_ | false | FailOpen is a switch used to control the behavior when a fatal error occurs
during the initialization or the execution of the Wasm extension.
If FailOpen is set to true, the system bypasses the Wasm extension and
allows the traffic to pass through. Otherwise, if it is set to false or
not set (defaulting to false), the system blocks the traffic and returns
an HTTP 5xx error. | @@ -3656,32 +3673,32 @@ _Appears in:_ -WasmCodeSource defines the source of the wasm code. +WasmCodeSource defines the source of the Wasm code. _Appears in:_ - [Wasm](#wasm) | Field | Type | Required | Description | | --- | --- | --- | --- | -| `type` | _[WasmCodeSourceType](#wasmcodesourcetype)_ | true | Type is the type of the source of the wasm code.
Valid WasmCodeSourceType values are "HTTP" or "Image". | -| `http` | _[HTTPWasmCodeSource](#httpwasmcodesource)_ | false | HTTP is the HTTP URL containing the wasm code.

Note that the HTTP server must be accessible from the Envoy proxy. | -| `image` | _[ImageWasmCodeSource](#imagewasmcodesource)_ | false | Image is the OCI image containing the wasm code.

Note that the image must be accessible from the Envoy Gateway. | -| `sha256` | _string_ | true | SHA256 checksum that will be used to verify the wasm code.

kubebuilder:validation:Pattern=`^[a-f0-9]{64}$` | +| `type` | _[WasmCodeSourceType](#wasmcodesourcetype)_ | true | Type is the type of the source of the Wasm code.
Valid WasmCodeSourceType values are "HTTP" or "Image". | +| `http` | _[HTTPWasmCodeSource](#httpwasmcodesource)_ | false | HTTP is the HTTP URL containing the Wasm code.

Note that the HTTP server must be accessible from the Envoy proxy. | +| `image` | _[ImageWasmCodeSource](#imagewasmcodesource)_ | false | Image is the OCI image containing the Wasm code.

Note that the image must be accessible from the Envoy Gateway. | +| `pullPolicy` | _[ImagePullPolicy](#imagepullpolicy)_ | false | PullPolicy is the policy to use when pulling the Wasm module by either the HTTP or Image source.
This field is only applicable when the SHA256 field is not set.

If not specified, the default policy is IfNotPresent except for OCI images whose tag is latest.

Note: EG does not update the Wasm module every time an Envoy proxy requests
the Wasm module even if the pull policy is set to Always.
It only updates the Wasm module when the EnvoyExtension resource version changes. | #### WasmCodeSourceType _Underlying type:_ _string_ -WasmCodeSourceType specifies the types of sources for the wasm code. +WasmCodeSourceType specifies the types of sources for the Wasm code. _Appears in:_ - [WasmCodeSource](#wasmcodesource) | Value | Description | | ----- | ----------- | -| `HTTP` | HTTPWasmCodeSourceType allows the user to specify the wasm code in an HTTP URL.
| -| `Image` | ImageWasmCodeSourceType allows the user to specify the wasm code in an OCI image.
| +| `HTTP` | HTTPWasmCodeSourceType allows the user to specify the Wasm code in an HTTP URL.
| +| `Image` | ImageWasmCodeSourceType allows the user to specify the Wasm code in an OCI image.
| #### WithUnderscoresAction diff --git a/site/content/en/latest/install/gateway-helm-api.md b/site/content/en/latest/install/gateway-helm-api.md index e1b68a6ae7a..9f2046a537f 100644 --- a/site/content/en/latest/install/gateway-helm-api.md +++ b/site/content/en/latest/install/gateway-helm-api.md @@ -48,9 +48,12 @@ The Helm chart for Envoy Gateway | deployment.ports[1].name | string | `"ratelimit"` | | | deployment.ports[1].port | int | `18001` | | | deployment.ports[1].targetPort | int | `18001` | | -| deployment.ports[2].name | string | `"metrics"` | | -| deployment.ports[2].port | int | `19001` | | -| deployment.ports[2].targetPort | int | `19001` | | +| deployment.ports[2].name | string | `"wasm"` | | +| deployment.ports[2].port | int | `18002` | | +| deployment.ports[2].targetPort | int | `18002` | | +| deployment.ports[3].name | string | `"metrics"` | | +| deployment.ports[3].port | int | `19001` | | +| deployment.ports[3].targetPort | int | `19001` | | | deployment.replicas | int | `1` | | | global.images.envoyGateway.image | string | `nil` | | | global.images.envoyGateway.pullPolicy | string | `nil` | | diff --git a/site/content/zh/latest/api/extension_types.md b/site/content/zh/latest/api/extension_types.md index 953a5d29783..37a76398121 100644 --- a/site/content/zh/latest/api/extension_types.md +++ b/site/content/zh/latest/api/extension_types.md @@ -234,7 +234,7 @@ _Appears in:_ | Field | Type | Required | Description | | --- | --- | --- | --- | -| `name` | _string_ | false | Name is a user-friendly name for the rule. It's just for display purposes. | +| `name` | _string_ | false | Name is a user-friendly name for the rule.
If not specified, Envoy Gateway will generate a unique name for the rule.n | | `action` | _[AuthorizationAction](#authorizationaction)_ | true | Action defines the action to be taken if the rule matches. | | `principal` | _[Principal](#principal)_ | true | Principal specifies the client identity of a request. | @@ -1869,14 +1869,15 @@ _Appears in:_ -HTTPWasmCodeSource defines the HTTP URL containing the wasm code. +HTTPWasmCodeSource defines the HTTP URL containing the Wasm code. _Appears in:_ - [WasmCodeSource](#wasmcodesource) | Field | Type | Required | Description | | --- | --- | --- | --- | -| `url` | _string_ | true | URL is the URL containing the wasm code. | +| `url` | _string_ | true | URL is the URL containing the Wasm code. | +| `sha256` | _string_ | false | SHA256 checksum that will be used to verify the Wasm code.

If not specified, Envoy Gateway will not verify the downloaded Wasm code.
kubebuilder:validation:Pattern=`^[a-f0-9]{64}$` | #### Header @@ -1978,19 +1979,35 @@ _Appears in:_ | `port` | _integer_ | true | Port defines the port of the backend endpoint. | +#### ImagePullPolicy + +_Underlying type:_ _string_ + +ImagePullPolicy defines the policy to use when pulling an OIC image. + +_Appears in:_ +- [WasmCodeSource](#wasmcodesource) + +| Value | Description | +| ----- | ----------- | +| `IfNotPresent` | ImagePullPolicyIfNotPresent will only pull the image if it does not already exist in the EG cache.
| +| `Always` | ImagePullPolicyAlways will pull the image when the EnvoyExtension resource version changes.
Note: EG does not update the Wasm module every time an Envoy proxy requests the Wasm module.
| + + #### ImageWasmCodeSource -ImageWasmCodeSource defines the OCI image containing the wasm code. +ImageWasmCodeSource defines the OCI image containing the Wasm code. _Appears in:_ - [WasmCodeSource](#wasmcodesource) | Field | Type | Required | Description | | --- | --- | --- | --- | -| `url` | _string_ | true | URL is the URL of the OCI image. | -| `pullSecret` | _[SecretObjectReference](https://gateway-api.sigs.k8s.io/references/spec/#gateway.networking.k8s.io/v1.SecretObjectReference)_ | true | PullSecretRef is a reference to the secret containing the credentials to pull the image. | +| `url` | _string_ | true | URL is the URL of the OCI image.
URL can be in the format of `registry/image:tag` or `registry/image@sha256:digest`. | +| `sha256` | _string_ | false | SHA256 checksum that will be used to verify the OCI image.

It must match the digest of the OCI image.

If not specified, Envoy Gateway will not verify the downloaded OCI image.
kubebuilder:validation:Pattern=`^[a-f0-9]{64}$` | +| `pullSecretRef` | _[SecretObjectReference](https://gateway-api.sigs.k8s.io/references/spec/#gateway.networking.k8s.io/v1.SecretObjectReference)_ | false | PullSecretRef is a reference to the secret containing the credentials to pull the image.
Only support Kubernetes Secret resource from the same namespace. | #### InfrastructureProviderType @@ -3634,7 +3651,7 @@ _Appears in:_ -Wasm defines a wasm extension. +Wasm defines a Wasm extension. Note: at the moment, Envoy Gateway does not support configuring Wasm runtime. @@ -3645,9 +3662,9 @@ _Appears in:_ | Field | Type | Required | Description | | --- | --- | --- | --- | -| `name` | _string_ | true | Name is a unique name for this Wasm extension. It is used to identify the
Wasm extension if multiple extensions are handled by the same vm_id and root_id.
It's also used for logging/debugging. | -| `rootID` | _string_ | true | RootID is a unique ID for a set of extensions in a VM which will share a
RootContext and Contexts if applicable (e.g., an Wasm HttpFilter and an Wasm AccessLog).
If left blank, all extensions with a blank root_id with the same vm_id will share Context(s).
RootID must match the root_id parameter used to register the Context in the Wasm code. | -| `code` | _[WasmCodeSource](#wasmcodesource)_ | true | Code is the wasm code for the extension. | +| `name` | _string_ | false | Name is a unique name for this Wasm extension. It is used to identify the
Wasm extension if multiple extensions are handled by the same vm_id and root_id.
It's also used for logging/debugging.
If not specified, EG will generate a unique name for the Wasm extension. | +| `rootID` | _string_ | true | RootID is a unique ID for a set of extensions in a VM which will share a
RootContext and Contexts if applicable (e.g., an Wasm HttpFilter and an Wasm AccessLog).
If left blank, all extensions with a blank root_id with the same vm_id will share Context(s).

Note: RootID must match the root_id parameter used to register the Context in the Wasm code. | +| `code` | _[WasmCodeSource](#wasmcodesource)_ | true | Code is the Wasm code for the extension. | | `config` | _[JSON](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#json-v1-apiextensions-k8s-io)_ | false | Config is the configuration for the Wasm extension.
This configuration will be passed as a JSON string to the Wasm extension. | | `failOpen` | _boolean_ | false | FailOpen is a switch used to control the behavior when a fatal error occurs
during the initialization or the execution of the Wasm extension.
If FailOpen is set to true, the system bypasses the Wasm extension and
allows the traffic to pass through. Otherwise, if it is set to false or
not set (defaulting to false), the system blocks the traffic and returns
an HTTP 5xx error. | @@ -3656,32 +3673,32 @@ _Appears in:_ -WasmCodeSource defines the source of the wasm code. +WasmCodeSource defines the source of the Wasm code. _Appears in:_ - [Wasm](#wasm) | Field | Type | Required | Description | | --- | --- | --- | --- | -| `type` | _[WasmCodeSourceType](#wasmcodesourcetype)_ | true | Type is the type of the source of the wasm code.
Valid WasmCodeSourceType values are "HTTP" or "Image". | -| `http` | _[HTTPWasmCodeSource](#httpwasmcodesource)_ | false | HTTP is the HTTP URL containing the wasm code.

Note that the HTTP server must be accessible from the Envoy proxy. | -| `image` | _[ImageWasmCodeSource](#imagewasmcodesource)_ | false | Image is the OCI image containing the wasm code.

Note that the image must be accessible from the Envoy Gateway. | -| `sha256` | _string_ | true | SHA256 checksum that will be used to verify the wasm code.

kubebuilder:validation:Pattern=`^[a-f0-9]{64}$` | +| `type` | _[WasmCodeSourceType](#wasmcodesourcetype)_ | true | Type is the type of the source of the Wasm code.
Valid WasmCodeSourceType values are "HTTP" or "Image". | +| `http` | _[HTTPWasmCodeSource](#httpwasmcodesource)_ | false | HTTP is the HTTP URL containing the Wasm code.

Note that the HTTP server must be accessible from the Envoy proxy. | +| `image` | _[ImageWasmCodeSource](#imagewasmcodesource)_ | false | Image is the OCI image containing the Wasm code.

Note that the image must be accessible from the Envoy Gateway. | +| `pullPolicy` | _[ImagePullPolicy](#imagepullpolicy)_ | false | PullPolicy is the policy to use when pulling the Wasm module by either the HTTP or Image source.
This field is only applicable when the SHA256 field is not set.

If not specified, the default policy is IfNotPresent except for OCI images whose tag is latest.

Note: EG does not update the Wasm module every time an Envoy proxy requests
the Wasm module even if the pull policy is set to Always.
It only updates the Wasm module when the EnvoyExtension resource version changes. | #### WasmCodeSourceType _Underlying type:_ _string_ -WasmCodeSourceType specifies the types of sources for the wasm code. +WasmCodeSourceType specifies the types of sources for the Wasm code. _Appears in:_ - [WasmCodeSource](#wasmcodesource) | Value | Description | | ----- | ----------- | -| `HTTP` | HTTPWasmCodeSourceType allows the user to specify the wasm code in an HTTP URL.
| -| `Image` | ImageWasmCodeSourceType allows the user to specify the wasm code in an OCI image.
| +| `HTTP` | HTTPWasmCodeSourceType allows the user to specify the Wasm code in an HTTP URL.
| +| `Image` | ImageWasmCodeSourceType allows the user to specify the Wasm code in an OCI image.
| #### WithUnderscoresAction diff --git a/site/content/zh/latest/install/gateway-helm-api.md b/site/content/zh/latest/install/gateway-helm-api.md index e1b68a6ae7a..9f2046a537f 100644 --- a/site/content/zh/latest/install/gateway-helm-api.md +++ b/site/content/zh/latest/install/gateway-helm-api.md @@ -48,9 +48,12 @@ The Helm chart for Envoy Gateway | deployment.ports[1].name | string | `"ratelimit"` | | | deployment.ports[1].port | int | `18001` | | | deployment.ports[1].targetPort | int | `18001` | | -| deployment.ports[2].name | string | `"metrics"` | | -| deployment.ports[2].port | int | `19001` | | -| deployment.ports[2].targetPort | int | `19001` | | +| deployment.ports[2].name | string | `"wasm"` | | +| deployment.ports[2].port | int | `18002` | | +| deployment.ports[2].targetPort | int | `18002` | | +| deployment.ports[3].name | string | `"metrics"` | | +| deployment.ports[3].port | int | `19001` | | +| deployment.ports[3].targetPort | int | `19001` | | | deployment.replicas | int | `1` | | | global.images.envoyGateway.image | string | `nil` | | | global.images.envoyGateway.pullPolicy | string | `nil` | | diff --git a/test/e2e/testdata/wasm.yaml b/test/e2e/testdata/wasm-http.yaml similarity index 93% rename from test/e2e/testdata/wasm.yaml rename to test/e2e/testdata/wasm-http.yaml index 76723cafb8c..e684da26765 100644 --- a/test/e2e/testdata/wasm.yaml +++ b/test/e2e/testdata/wasm-http.yaml @@ -52,4 +52,4 @@ spec: type: HTTP http: url: https://raw.githubusercontent.com/envoyproxy/envoy/main/examples/wasm-cc/lib/envoy_filter_http_wasm_example.wasm - sha256: 79c9f85128bb0177b6511afa85d587224efded376ac0ef76df56595f1e6315c0 + sha256: 79c9f85128bb0177b6511afa85d587224efded376ac0ef76df56595f1e6315c0 diff --git a/test/e2e/testdata/wasm-oci-registry-test-server.yaml b/test/e2e/testdata/wasm-oci-registry-test-server.yaml new file mode 100644 index 00000000000..3eed167d9a2 --- /dev/null +++ b/test/e2e/testdata/wasm-oci-registry-test-server.yaml @@ -0,0 +1,57 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: htpasswd + namespace: gateway-conformance-infra +data: + htpasswd: "testuser:$2y$05$NLYuo.x7JAL4EL7OOEHGjOUznJagjXCUczoWwc.dW1/5Qo6h5NiwO" # password is "testpassword" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: oci-registry + namespace: gateway-conformance-infra +spec: + replicas: 1 + selector: + matchLabels: + app: oci-registry + template: + metadata: + labels: + app: oci-registry + spec: + containers: + - name: registry + image: registry:2 + ports: + - containerPort: 5000 + env: + - name: REGISTRY_AUTH + value: htpasswd + - name: REGISTRY_AUTH_HTPASSWD_REALM + value: Registry Realm + - name: REGISTRY_AUTH_HTPASSWD_PATH + value: /auth/htpasswd + volumeMounts: + - name: htpasswd + mountPath: /auth + volumes: + - name: htpasswd + configMap: + name: htpasswd +--- +apiVersion: v1 +kind: Service +metadata: + name: oci-registry + namespace: gateway-conformance-infra +spec: + selector: + app: oci-registry + ports: + - protocol: TCP + port: 5000 + targetPort: 5000 + type: LoadBalancer # Expose the registry for testing diff --git a/test/e2e/testdata/wasm-oci.yaml b/test/e2e/testdata/wasm-oci.yaml new file mode 100644 index 00000000000..98f92cb97bf --- /dev/null +++ b/test/e2e/testdata/wasm-oci.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: http-with-oci-wasm-source + namespace: gateway-conformance-infra +spec: + parentRefs: + - name: same-namespace + hostnames: ["www.example.com"] + rules: + - matches: + - path: + type: PathPrefix + value: /wasm-oci + backendRefs: + - name: infra-backend-v1 + port: 8080 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: http-without-wasm + namespace: gateway-conformance-infra +spec: + parentRefs: + - name: same-namespace + hostnames: ["www.example.com"] + rules: + - matches: + - path: + type: PathPrefix + value: /no-wasm + backendRefs: + - name: infra-backend-v1 + port: 8080 +--- +# EnvoyExtensionPolicy for OCI Wasm source test is created in the test code because we can't get the OCI registry's LB +# address in advance. diff --git a/test/e2e/testdata/wasm/Dockerfile b/test/e2e/testdata/wasm/Dockerfile new file mode 100644 index 00000000000..422da60914a --- /dev/null +++ b/test/e2e/testdata/wasm/Dockerfile @@ -0,0 +1,3 @@ +FROM scratch +LABEL org.opencontainers.image.title test-wasm +COPY plugin.wasm ./ diff --git a/test/e2e/testdata/wasm/plugin.wasm b/test/e2e/testdata/wasm/plugin.wasm new file mode 100644 index 0000000000000000000000000000000000000000..df2554e971e8538ce58066a2bdacfddf4c1b1bab GIT binary patch literal 59641 zcmeIb3zS{gS?9UWIrmj{?yb6#O17j@;&U&OQ;wxXP8`{h9bCr`S+bp2N!-LVFy*pT zwyR1i)l0J6ajZ&W(t-pqO@I)>poBCrN!++QVL><0(J;`RSq;;MFioamx($;>vw*e6 z3|T{F$l&??zrD|W)XR24z-zGOQ|H`$_T$^%{$Bgr-`>YfE}e>;bI}K)k^7?ajw?EU zUvmC@adY>@=Sk)d=_S(nQPz#SbN5y3v6EKhvHN3>so-NOSoPUdhF_KQkLov1$zyhl zAA2l%^uGAf^Ug@E?!`G;e;}cWork9;l9b3C~@H9WQxcrDq z_2M%BVu{M{Kl8u?RDS;>%Tr6vAPFzk^4gcL@+B9iPMtnG)eG6#$&+WMJm{}3^30!{ zJibmQYbEq=^2CX?wOEFB7slo3Oak`}uC(+h=y<9F+mOrtc;!@tZh7MP;uN*HAC9`O=B6H2C3929 zt?I|49i3WEoPKz2X_+FYCIDj6eJBdVM2%Z3@({m4Yv;NZ_T=rJ#zAxI=Uxy#r?~g`0$qd<NoTs%9MwLMC|1zqaqEn}jyXyzeMlO%i?|4&d_lbObrnUROyXw2Y>mRpv@7#Is z?spw~$3NQr_IKX%FQTWS+iv^O=)=*q7o*nL&QC`>cm7=T#y7rk=l&<6e;$1@`kCkx z(Z{3DAWc3M{cP0QJ=Xf^=)aGCD*9M7)^6|L|C7;cJ{rB|C!*K-`N_tUm zM~6$}`Bz)~PqF;5EPi<5VC?EiN`~lQT+gDcbTF>f<5c-~v?@SkAW&4TST(hFtd?Rb z+?8f=Q7~4)C{>AkWTwq)fTb%*-t7L$T`M9}Lh)8LE7A4dtPxpTR5%tlRW!}2xdXm5 ztAL9Fl-=|zUd}(cXQo}E4SM8(2WgEc|5}UxS9h+GH}jFKLbX|PcyvY(ll!&?-Hfuz z!FY#y+ib^F6=n3J>3g-qurSh&H4^q>B)p&tS$QlL{K-O|+}?2c*ISuj&B~13V_+3j zF~HHHo~!~VZPGlf6_8H)3z46 z^%mpbNNCx97K?Nz)0MWqo3?R3UHe)K=~1<= zA#K&eMB-tGbu)MQPHKtH&hqz)!=uYyI3dFjV3EX6$ynuZLqNakY>SG4hFoHi(!xx; z3K7C^?Z~8=8UP`pcgdClZ9<~jqrO_(a9XDVr82bO6!D}Y0mHR4|8SfCsWv|!M1n&A z{mR}}0$4~g*9O}&EyjaZmH9mVO4B-{AbdC9&}^iykK*%;LzKDEmPTl>J?!K2-{$h>Y`e0@DSJEFNA2!N>Ac#kX-J>1E&x(FA57b2sxLB- z8B7^Q_X9_}1Um1Q29Z<;icpV-UD*=KXC>rx7B@#FA5b6~#NHY;O1{o4Iz+87n>*x& z$z7hwD*N5=A?Ilo?o|z0c^0H1azMbh8*9$&ahXfsSBjGJ4BN^jCzy=kns3b<1PlVe zqWYReOH#-s;fWDGqKvKBdgRkPKU*UqW?DuSbRkRbMv#J-u&j{YdNFj|b zm1L!%>&ctUGKMP}zLofBI~r}iID zfs9>NfirCSn+QB%Tas^{yQQMa=k@5DhKBO43o@z~t)j zVevoRw!SyN^R|C5_^z#oZzXzs|8MoE7W6;~;)H3DLK^SndQzT4!bC^gwNa)P5I(Xyk8-9ECwDeb zw0Sg(pQjX|&b}0x5~bp~Huk_#6uSy1Cw-GiJ~-2+8)_%KqF#e0q01Y-Vg|HPsjVBo zx@_ZDyElII>WyD9W!eagv5jB7cH>vC+xXSB8^1cV@v9p*e)Vh2e>Q^t-)-{li<`Xr zpEr5;JDa@w@+R+|-Q?X@HhK5Qo4oteP2T-u>7n?>@H4yH9NL?q@f7_v9w;KD)`gf3?ZG&u{YX-)!>k-){2m(kAbI zeUo?pZj*P<^u0?Eb{1f+Se9W#sk@K#ydQGO}hnKHQMM^GI3 zb4qDm=hDBxXMa9=jK*zxM4eJ17+V;o){<`6_cG^`{ITA2bV&Z5-UQ7Ut3KnSz3Es< z`GC@$6xS@1g*K+MOkB*w{Jrmcymqp117UK`f}b`}NT2zRhtf6~Xa1-}O-9d!*HOOQ z$+**+R}RfExsCXnrl=Yvip_V5o$Ne4)Ok8lR7K}X3;B-4c7)BEX9oUL?efjp2eR_x zkO94C7LbhdP`)nN;crcUtNEKypZTf6eb{FC6*7*~4Q=83dTrtNM{(lE#ofHZf6aR| z)~zyRrZ#2lMG%6!d2j`w1|NIRr}6`fHe@$z*80=YcgVHXV(iprR{oS+f!&fCs=+x% zdhq0ICRjM z?>Gb2-I3AO0HTGCH6S#J0;)~20dqlTvGU7BWckV69vr>BAvU5w6_8^FYJf5gVX$g{ zvcb`8pgGcD5+fh7d47HVtn@-Pi>fGwJY&uPQ5 zOT9HnFKYOPJSg#Kn=&qIw5!~@8RKy(9Te>jsuA);EgaLbYvZhmYn<^{iNnyJfkWMi zxx|^-*hCCA##;bIweZm~?##1~Srxr84(Cs$To$SSvw zk}Q8*7D>0|&gQk9B2iJKtnp$}tNATA=C_bk<#w$s^p%d%ya}6^gnV2ZP^$@8e#@PM zfR>cDM3=8z`C9VS@~>+;&nSPcH4u-(KkB{k{<#ll1K*QJOZr){+W zi~Nn27vaC9&(y{3IH#fO+If!QU^8fa2z$i z2f@9Iv@E(R#$|0!482AsYQ81CUOp8kXST}GAWo0wTPV}3o=G1Y0kFjOJOQ~c1t{r(!)XZ zj57xn1PB@31)X0v)7r-E`k7WUcS3Q(P-P8dqwwkB%o?+Ic$?}iLe=?JLwX^~8c12> z#8`Y+dq73)A1t``Up9dBX;-8v{mZN(8>L+p1w>xYRP(*JBfR(p!{LpI{BJFiVYO5R zZbcFeMTAoi$yjYg2)(@$YSK~zaxzL$I0!kClPqd%1=!)7Dh*1bAc4D&%8$+DgLkWs zRF?07bF;`NTY)WkB4sCY1gliI0y|pbB$I0o<=7pA*c}Fa)v(b*%4UPhB!Ay;m$Xzz zF9f?pufd)TE(&%U>6WLz^rE+*Izux<`D;xjN}^@eUU&`NtALJ|@mlAX9-~VPknc#c zj3Je;yhpOOmF7`v3&12Zf^|y+Pe*GD7j#J%;At{p7?x(NxRvk&3;3yZKTGrTcd%;Y zvUHYWl^iEH{ZV!0{5Rc9^Om+e{0&?Gx(k*E^EKg%fzMxO=+%&_S4!On(u)x&dX#jA zLF#{XkF-MF)1#Up7jsu*BXzZ(<4YdC8PNFWmZM)|}7lpW>6tVrYQin6VE zyYeS7Acpentt_>Si|{m1>NGT6F;dUJ-$?cJw&%cXqly7#~ z$a~r&(76DHTI)z&D7cB~VJUy8l>CoU`G8RY3+?)-))Z7p)@+`l+6oK8w;_9KSi1wZ z&O0M87>=59*7BOJJNc=s$gTWq5JXW0Xvh|tCp+~p&!FmKObif-XK8f?oe+bXBo!UO z<)*uGmZfn*_x_WqJv&roj#;XgD~<-hS=NB1>kNJO2eUQ*=UDoK+X z35GQ&$KwoH=_$O9fB~C>B-gURBctshbQx5EJmZJto!HA|!&#lRDm*!iKWWTZOX3M7 z3?Uz!Wn%0tQ&wa$#t}<^Q&AVT35!)(J^jB2;#B55%Q(>Ne$39^$w#|1+O9G+;jN=8 zeqq>VRi>^bEy~&tbIeC$&gDP)rGNil$|te7BS}I%wyv1t+gk^eY^-RESgU1KDQ`7P&5jm<=^U*<>2uN=vt8`mAtF)PmGvyFx zAbn_RK$t?fLVpr~67?8wVo@{(HIQAFn@ErvN=naY13}pw;Wzk`bdDNKr(DXIsPnBx zZy$2RRMuF?>de5RZw&C#CV(TnUIgIGoBIIQzz4vg>>bTIbX1!`LQjiWy@QQ;!3cP? zInRe}1~-&Hp-~1gpKA?^0Q?MY$YyZE#SD%~q0qI#3=Zp*@EOhHhO;5&aYJDqr@Qm> zxS?VmhsT9~@lpHM33QhMX%_F|;ij3w8Ge;KS&U>&w zms>A3Z&Nc&`c+oxX9!=J3O{#MV7>j0o^aqTHe!= zhlbJiZ3r;6A$6RBu+C5`Epu6zcv_YpiWFgjjxid*sjcC#6>8x2eubdpeaxq1if1Ai zBVsUp1ce9?epv!f&;P%F{V=or6}-XcRGAnF9u*p5vq>{vvt7WIf<NPLCi*)TA8S#y zsQ;MvaGEP;EWY)ds`SR9QsPZC#G%py-AV-;AnuR=6pTFMf?E&3v7)ZAzPfgQBXx}w zb;0o+xc0ei-Zu$0Y6fZg7T5fFLQ?md?*Ipezpq>=_wU!4*^5+t+T-E=;qJS%?3&-^ z!rw3MkJ7O0o+=b&y?@~UtF{Jj(Y-joq^;8#i%`o`#6nrIe$E0n9qRlU?lZ5VKzq~WYO0)F~YK|ir$h< zy-l=Af>|YH(4tMWVbR~Rj4F=NoQYplhX)is3V=RyoOi4v>9c{;>loC>Th%#rZ z4A%D@nDjzvGYlsgvQgw<;r!vK*<0`LNIM`axcqCvQ zv)m>K%-{)atyySm&4;!k@LSHt2+w{o9M}^w4HC$$qGbj0FgnzMNkp=SP(}yiOZclI zUWF!kA20|AEM=e)oYKpTpw~a(b5UT#S#24J={1~Qg`+hP{`bbbP7d+9gxE{u^)41# zQqLl~WW1N`WR(a=vC$}91WrBRJXVJYea#|JOLtQv0@S~BcpSp%%d7z~K)yU+BW{O5~T2S$<;fYNsG3rYMEUrsv(}>t-q($2=0q#4riEBu- z3IAK1KQvTkjt~Tki^sv9cr0%!714m?a1s@eDMF7Q!a!%!LWGH{G~Rl_CJ}g{ZEaB6 z?!079Wm2A}ekl*8O)`pCYzu?n!RN#e4~Wd8{6lU2x2_8aYf#olv2&u3soknvO8~td zjRZ&i=3a_YMyg(irN5V4{ z|KAx#EPBuP^qEF8clW3y-s8}`>Xekte`&9_9l8ATdo8hu(a%59_5`hWmpoM_;7Qh5 zEJKCfnt8%|9-kz^m=@75<&)V`CCpXH?=@vamvch$H88aqo|qY8i>UGsi36eHz-~#0 z-J+SCi11SWL_7Zjp@Yow_`3bBjyp~=M|4E%qF#`uLB{c0QjJM|E=xj-TH&@9u`Z3x zSK4746Zi&O=qq&s^n|9VaGj=Q?gVTqT%y2Y7AYE4%Q&qVB(|KGuJ{MWY&dK!Jq(MF z^bNg`__tERtyqk<#g~3Okt;o%iK=6^Bc)*@#Z6=G`wVF3)}UCDp^#ZI@8f`wI3PqK z5`QXY0a6S-AC!}Q?S;&tjq~^#M zt%%!EsFHO3d${0&x||lipi0R2G6h!_v0;35e!)3fnqtrrG;|qZ)ti;+j^fcO(uEY2 zU{JsDg`%koP`SMy5;w9SwFOQ zkUZ?~AwBHjvKQLJ6<)9&(TXxm@EHkILsBVh8tjZUufxzW`rq_kEwu)5;Xc<*K46fe z8@yQ)ZB)C~={LAbYhCLUA>*x66#Cj-S~R{pt7--ea)XT~YdKiZNG0t-^Rpl}YGDr9 zEQoD{qG(MV%#5XOc4$T3jD;8(SzB7ccM>dFypjUXc4$%SGK3-ExyORh> z-bdEi!Ah!jxA;!j+>#Q39GSUxH-WbBX21BYgCa)>3&@dMm`Uqq;!VeFqVI!Vv};n5q36m zv_8-u0dM;TOGcH*Y^c(Cszj8nrOMJ_T;$GECApUwje;soj2Trjx$>I^MU>_z_6Rds z3*$oK@>>|K8e`P_3{?bdV%b<7FU5d{-N}>+MGW%{u*M;guv8SN6-jJjQW)*%DMi^r z{yeb`=gAyOm+B;`V?}Iew6nxk#O^vkh_xaS#4_x+8`H%j+eCdPN0Y}USTnIqj~>h1 z!-STI@B4*vY0bu36Uu%HMMaF6@JCgKKhn+Xz;C@6DP90koDa(ifYa0`jIO}B#3MM< z`auaJci1u%Uh^dD9NMtlpfgOG3Z-h=RBO8CrigR%lu+6w?lJXHdJ zKUK=&UqzL|P>%x=Gd55qdS zCMuu~VG@=>jF{U|y_RnQw9B|L)Uv#cF53gZ>CZ&`%?X%)6P+_#I@SuLC0(loY$>@1 zNzfiu8@?Uf_(fcLt{JOx9SBdV>7c^GGTEnkv!fm7{#hVE7WnphO2DaBMN!ZiK1c%; z-NQVJM$`0rh{96>zN|i2B!dGiwg-VxDO%0x29OCWJG6Ju>gN`U^wvH|V*jCG|Mes12!A?R106^%RzkkG90xHo|g=q@z*| zah~3tDtL<*I9`}#`E8fn(!-Y@=4f(H<6D=X(9VEiqCdu0CLzdhtoigLwQ_q(e00ce~4 z9=3cpx4CqfH&o_F=xu2_rYC9doD&G>hUO%`g@{0|j6b#J!kZJ9{-?{5E$3rS>;m~u z1M2{|8qo0U1MktVwlDXgG0{ic(XX~H&!ZMm9>WrfYzPivF(di^u9WUk?H2@Qj9D?dJRQtC{Gr(LgC^g9Y+gAFIGw zZR;?D=qCcW&*sMgE&8;^$tX;5=Fbez0Cm=|T_yM>e(~cyor^$)0wwUlsnS_VAV-=4 zcaq%Qp^0wD?%bpnM9TT=wAGyu5BvkrcjxW>LXaQsL0fvapm0U>YF5i=_JelMj=KV^$@$Go&L5dJp}kz0XW}4ZI?q35ZVW%58*xneiy=j2xah; zLobf_NZJmhlOh^n+@e7O`J9+(r#fYz=+M*oCm79K3kWP1Fhuey4&=|knmEdcM{2Zc zi~#bfQ{=h)c<0%<2W%npGa3s*illNFN9Hv)v-l-H2vfiHTlpt?$r&7iedLbLaj4-} zZ?r7jmm7_x`KxrX=|m)4Q`8V{Q$?cJpK9&6*|jbwQw4rRU_L+m{;cvQ8r|`MN86W+ zM?d^$e~_%e>h|#1d%m4_l~4t11Zj5pr%)->Wzk6ZWPvzU57mZbuVvPD37HKrn&B|3 z&w9L7=omePVtB^_l(xbQ<~wKdfm6^|YdcYFf9wuxqfU557QIkR2k6=QUI@F3GXLOePe{+YGcKZVKOrwoZVgB%p zNP<;1Z5kc|s}_AlOnm64`M`$((KjrI{@YBwII z-AdLJmbM?xngVnfW@B?XtV12!*mV;)=GmLV!*j*M*KW$TQ8`Q5^~0m#C25sxJL_xP zvii}{*3MfalhNb1g!kjOWW=HpBrKUj$d$`3GifRlYv)H1vk|g1B{Xn2+cw+g1QA4! zq-r!t2JQPpwHp_K1Ca$3Ll4u1jgA*(pZ^ic%B?`i>o+i5 zhA<5b^gw)PC5_KR(V()8%L?!PE`~sCX>8)ac-m^xUyZAI8YRHDLfQ}m#?N_%I%3Bc zELOXNq<7yT9Vp*1yQ?tZc`hr6O>)1z6@4{eqO2+GsU3~l5I;2fkYWW456&B{g{=#W zrD!dgClfuc_tF!DAuTDk#AEAPTL~6QhSQ|+5*|LzRNZ`v_qjHQ9zz@z0Q@gbmXkS1;%f%D=Kn( zR$gB}ZppUK3W|}LY3I&-(YE;<_LViX+;n|D@54cYkv1!B^W6`=<0J8lNI$}X&)GV4zq7NwaLop_ko7yqG7Cj)x1aTv)38qG$mE-#emWZ2w@mK;S z>b3p~&aJoQIHTz~;;08DLR>>+SYfmWv&tRTH4^AUko;wjGj@@P@)aCBm~WlQw*~`Y zJ7l#T#LH(hXk$T+3WlQ8Q?qGBWV)LD;e+l6u$?tiR3fK29T`n8#?AtpZbIx|)AfcgFrL-f` zpgZABlUvz`G16*=e9X9b>vv80Ef8_~Sj=g&INN@;o!orwg-sM{H@%G~CJDS4!9*L3VR1u`>!4mY?u7m-h4f^1}zW5OYE zBUDGvE=8}(gbb!RI(Un~C|hSIKol_XKUsjN^Jgbfom{kp&=%pSc2*Z@3P%m~S}1DR z-Y8Fz;?X-$5Ojs$h$B#aD}7c@Tq17l5RA|+EXyq*FshKl_Xe)HUsPTn6P=H%r(*aAd4s^ja z7lHxMQb9ksMl7BPyn(_d7{BRw3*rqJbI~N7Z$6fOK5WakGB%&{2a|aMEBO%MR?OWy z{^S~nRriG}jldu-A0=$Nc-A25eqX+=0UJB30Sv(xs@d<$C!3ZtZ8sZx z^wo^|6u^n-Z3mo)wxlvu1P(}vD~xH&%t}CEKNMX+Ikzj^x*6kfDlIlbP~m$bB2jK8G#;Z-eUXb`f9Qz}Ri-qy#O-`&O5Pv|OyYCFPmf=@ik~j{gf#uvkq+E=4{k%rV!zn{*5bL? zY~ZxnASzC$)66z%a;93$-G>yiE^?Gpz0Ug$+n@8h-q~$Xy2aT}b={rc;ZO=Y99pcN zPyyys7Ek-#1~i^$)~t3LxIFAOu+|+-MX=OpoBWzNy5`|DeIgDzg_;maif`sdT*Pc% zI+I0y!V_!xky5(Jd#D^3feM#F9MZ|2JuF}V>~g3jg1dAE>yq);qRPrjgmHud((VoI z{Uk?{Cuf*1vv20;s1|CTM;%j0k2e9Qo%&G}mHWP=vq2ya%tQ84%rFzCbrN4KGJ6|H zg*S15a-a=Bq5*h^k`=^=b^ilZ;Z*WEI7v8z>^cMSdBRLW`gF@RLxIOaxOV zx=q2hVt6|?380`LW)iAF3+5c^KqW{iIJH|T!me}viO#mvcLH=-+q;Z_D zWcEk0t~msq5#nx#J`rBPUHrpRK@8ia=~rUT=7Rjm1TFJG`I$`dnIsXqXlx9NS;dGz zZ_??UUVm7mUdU6G*nGjZRsjZ9z9gbFySy7gZEIl)4U9Iig@2vT>xGAO4RJ1bnSEJo zGdh6$2Y|#N3Kb#Q?NndhDn63j%7~K+m<703zlldO#ADwg>%DRok-Q`KZQH6X;yY&*dtaJfphsP$`OIHh+wa0);V)4rUr zAll5`4Uw&f7g&sQG*%B5MZ<4W|8oknq_W>#i*@KjL=Ad6jJ?7ZwVrrNwg)y!eM(4yL z_`gKYdXk$0a%X7sFK^S@KkVrv4L}+Z_yo1(E`N5_O&Swf2o>TFiz5LUmj{YJnA38YVk;HLQ$)4 zLuAU+fd(Ve!9PzFH@f9~4##Es*65u%bvB%X*EORZU+CfTXm;XOo(jSX( zjd}T279!>F(z2458#eHO$5{)dWIQ%UP49d2l+bZ4etP#3H6YCZJpT;K-_6&zL9^^| z65ooRF~$QS-G}V-l<1Lb#rMnZotYzbUvr7-UBH`~cq0 z&gTa}K15vIiuA4V8n5NMc z_eZphA}Ba;=&hstLGALM831z4d@%(8P^`GnVB~4@uN5GNELv+8GDG@6G(BG{AVP&L z!589AgbzfWpJorjrr|x6{_q|^4dFdJ(VJD_J!OUW$kPO^;tkPS^igw-m_4AFpn;bh z-c#)f?_tK*3Fpy-*Sf166>v#6mZt){gEr$j#~e=cX&=kuImpNAz~n2nbP@^Vc^rMh z!1u>Ec}=2t5cm**=RXb6GbFJBf`*h}Z--2z}JYB`Ij(3P#vi&JFA7T(Zn%+pI(l;aC+Senh zl^|#V#d2lD#DjX2gJCsuCH=+7)8j=poPi_&XDif&X8wdS$KwJ9ESUSJs)!85I7cmv zyZoo65VbG>(AWCgZjxD!;ebC{MAoQHsc)OapaK;mY58|pJJ3+loNLh!DO&utoNque z9(Ej_6#_*|3dRnH{;9S6V&pCTYw5@_{Ktuhkdp3GI!!DDH^t$|+|q?M*nNlT>?qy8 zULp1~6bI}keYyWkr%o_R&wv|EIN*YV@o!?BVIU*|1ew{-yplk%%UDRd8fVFPZ7{ZC)H>~+2L zI}f-um1#s3dm|W%;Z&rw#gOQYYr`uF%_W&6`lC(q%qi&|fZBLQ>62pC80J(UYY9Fq zVa7D)Aq=HJ@loHGC|Z6WOerT`@Ate}4Vc?XD8R5VJM zD2%zzd*MYDVJvZScfi{ry5zf21tY+Pvd4^KZj|Tj^4@no{njl6 zS*(*jzCrqt4btyY`p5fFqeB&CVSCghA@l=)JZiMo#%8>L8VL!^ET+D65jF1ECt%Fj3@{ooBBPk36Phl8jIn~>=VHC9+YU{9p6r8V5C!rwy{a4?T zX%?c3uf8D^knNhH=sbK|3RFw-A9}8k&1gki)tL3o?piH;1UUY)zvu-`6X+2(wP`zWwvUT~ z;ADr=15RJsD|aj5Ql(a8T?)&;RB3hrz0gXnIunclTd6fjEZ*d$mMqCcx5j-P0k=Mm zEcA$NN1X^VYaUZd5oGjC?*4*L)j9tG5LG4c5M@-3z*8ID^X;`*Yxu3V!pk~ZOPKR~ z=ND@gsFao@%)kGNxWVzOn862ZQtFKn$<*F|a%|{31rQ3&@hJ{XL|OOI4O`M+Scm-j ziBD_Of#mv#uk$emBe^P}8o(=}wTNF&2(pBD61MCs3i?sVqf{nIk&OWtbyhuLa)U5+ zj!oEuM=8;R9XxcUzNRqT`7Xfc2o=|)$jn2WBYaJvS)9cgNKQa;YRc_E*Zr1)Tsnl! z6QMGEi?aw7-JOq68Sb9J$;>q)3Z?j#f_%q7iglq7^U2?$Fg32y$|DS;Yd zBXx`UvdBx3HdG-x;G+umnV4#bI;07>&+JoCui6TPC#z7=i|aJh+l*BwSePODHh@Y^ ztp28gEOC7?RGt@jsKjwP%`0gRj${u&S5oPVf$c%gM3glvx62@25W=|ZGe+2|B@q)o z3+8c2QwYpd-6ZWPdb1CWOHx)(%$VlU2SMi@bgjA}@c#|8x1v^}w zr$bI2Bs(I+ThfGLGZ|Lo3@F0wfYF1S^dkzH;)GlHpbj4Zj~IZncV1=u78pUGBEnTa zJ0sUHo<`ilc7lN87uaaUc_dH)Lj46{BUx#`)0xYCL1}Frge-JgRtrM|FI7bsnYTzk zY(+R8bDgu0YdQwdD7h`dY(?eEeZ<<$s!RCJEa3l#6fR$Z-S8YO(Il;}cfEpBG@WDVqeIR1cm zMMWoS`(P&=kZmA#7IG}Fx1epYZ&5Zsv769sJE42*2kD%J>@lTtl6ey;v!-xKRwp{Y1??!+=t~AEG-i&ozW2;<1FNR;ulvweSW=4;NyGGtpNe2e zg&=fj+dNl%cQ@)1vFwb(0*fHxUM1)O5;;pq6QLMc4Vtk4y^(mJWNa1mVV8VSiJTE< zpxB~juTIT;3`5bD?qlG-M;%UTDJan=>uAfN8`3T>pRgfjdVo=^ z0UCrl1dM&GP(nenRC}WMw2lFzeOn$Fbi4;9v&Q-IykH=7K{v=t-PC!%dwdGpqx9m& z?hC%Gt(05+?O6B4T|VXBZpwY2h^{E`8kP{w`F`$|M$nq#N-;&InAWHz4KPJabL&s@ z8Y^X5(o!WH|9`KRq=-xz16dZZ>Zi~&S2nyt1euXvHtVL(11W(-(s z$sqR3iDr^pMeJxHI8sRFLG&4DU*GRt` zr>UPawma3;Rh?+G?kh*{p_mKW_ih7qG7gn*w6`k#tFm9oeVUcrh!q;s(5ut1D*U?GaLAExOY=Q+cq5=NRQ4pUuDCoBQ5QZd9TaWH&b^N&b7T zxy4%1nlQ|pQANhnes^`d=lh!&_Pj**BI%lq4k!m>nMNI3(L7q_W5

?y#PhKmAEtSunBwRr_OqWmv+HTSeJ*$kgM zNOFeHre70vS=`0v6Reoibses#*}5j1t~=r3yQI9-ifJso6jeScKzyASdGL_?N^?Zn zSWIf9rFvEP8HjLoji02Hs$}LywB2X0inac31!X^zHrh!ti<*>RW0Oh2d&={Z-Ess@ z__Pmp(^iVKcX#TV54mPLDffkxt2-%KNcpZ#%CV4gXD4Mmq}tM7jYQ6!gYs_GR29m}kkm+eRggKO62WbpYG}GiWb1r@V&Zwej zi04-E5Tt25bRW4rDfThZ3zK3W7i~(4ifYK;68H$B)HofZQDdq7?z`49(mksgX>-br zh~(YFO2r^#xIE^eSxRfE5N|?Gt(El1um^BkY}1BmCX zCZq9*(W}^S?KEZhsDCsjI^?z*`Y(Z$INu6~Bnpc|h<#bLC&qQp@ZdS58rV*s@ay5$ zXFN)wf4zWX?(Ah4FPd%@ZFm;+Qu0sKfM>yHdFT}$Ij_cIQlSgaf`8V>g1*MDP)b#L z7W}g=+dNya&02E*{cf&jEEnEmG2(ydre7*b|8h6&=^|}CMqiJK*RtL>#@52O;eq=z zY~d;}qVShyw96)|Y{nC^S}Z8&=^0#?_?Bh9IkIyTra^D=Y!r_Ns|yk21DHLAlja>V zqIFDzc3fy%7)R%EUI;D%mr#gpb~uu(9c@ZMF1t2F!b+SuP?f+^Ko~U+zveizjDRrw zEKDx-Ek1&OaO!A+sW~w-Ni23;?#jU+%8!je)D)6h(TFA)U{TVE^zI zY~rpukWMqnP-K@N@!98BXO}L%ROnZ_qQ znKJSZznR;>A$LP2=n*;m#k?8lQJr7x0(5C@{W49)Vp!8?McYlAeAkFtBwXi@At_FKVZL0sjiw{!~JmnTD7^JELg23_qlII zI;*!yuokANu@)($LH>cWyhw)e0x=&e zkezfnON0%F9#SH8fyj93&9md3o4%}f|#9?3CilG1+}5s_;CLG3xy z5sMZ4j1GkBtCN0FhYBRMGj8&uY5L65A$K($wvB^|OtjTh;$QuC=&-0mzF@6x$YI3< zVHKL!l@^^|0a&KRCQ~0x+CV}O&zN8Q&tH45c{0_mmlAZUWKzOoYh5YPsh3{Xe5G)w zrj&*)^Dfn1EVig1=s01^!V3*sAPu~3j;bJxgnr18&)sy*dygmFkNxDht|apPc_B&U zYcxsJu~ycTL}`cLi|N67l4!J(s~JrA^(4_+srH$!HFL4BW`44p_7bNbYvti=2d-j5 z(l482Wv!BNF`R&)Dx3DthR`v6_(R`>`VY+4V zjk&%be_4S-gqD!8<*Kshm}@YJQ2?KOM7sW>>i0HiBJY^U^YWnPcy@mRy3U=dvl*Y0 zG2bkE;|fC1WUz(POHWUjafP8H#r=C7G&n-T-GltVu5cldQ0IX&BuHtlAW5#!6B5y`8#`t*)DT zB;L(bDY8G*$$sBD*%9jb^P0aD+0S*d&##kR0^7uCksX0u;AI6a?ColZ&iIktHf7Q0 zY<|E$vK%c5q-`wV9)qF`Ni;eTAkvE}pIB9;T#q`{jvJ8!lr{+@gCao!<@1DTnDShA z;hjo5GP*_+h#LU#^Ikkx-pv`?o{+Xyzj2f!rq@|}H@G1#jgD$C+0*aD;SJ`smpz3) z3C%-mHWj7aD!^_X_@Uh+BTED}u?zY*5{0_h1Iahv%rwS&H z#l+sJDMcLTTVC-yjK!AbPigl&^c?ni?%7IN8Zd?cOn6&o29Rtd=0O-9SSB7YK)OLH3kd*hpsW+ z5d}g0k;~K{xxN|Uu%0LgDpwR3M8OLShmJC*l*EkI1=ctnWKJbqjJo&g9l^+xl}w7! z`nS^ca9v<7nDS-ec?hiacj$S*V3fsM(lET@c>n`lL9~wN!TdC@cpifFzkx6s?N|-X zAS%`qMr#Gzi?vw~f*@GUZhnsaav=Qw_2otQzw>uP z_+Ng7@DXw^VJ7>xO!!;mw?h!||GitBW7paL_4gkm_WEG%zC2XGfBn1Rgg<$O6aHQ~ zp#~o`uSIeEb&8`k9`FAr9@*Ds4*|Id$hWh$m?xIm|D&^(%iaCE5uqYFcu;38m*>-V zt=IzP&vJ&mCqj&aq zG*rluQ#vG}%CHjOZySPo$U;c75ePfi_)!;uYCT@TZ!NwbPD@%*6iR6cM!Uy8?gvUM zo$nfGhgs&rmS;{Ou}_52Y{fP`Q|6d@ZYMPKY)03%bgE(FtH@b(1ah`!IHO(U3}tX2 zdDsj^O)NXv4s;&b*eJ36;i(`OTX!)m8aiDd6K&2S)pPR^C8EU;{DE8O^>k$FvDEZFc_vsWEP`_gR~O%eHrsJEPy}s}JlqNX7r2xPg`J&}@)0uoFR2 zYKP%6E+nHLuD)*T43aL`J%^L3TPPG^K`G{$*xaeuRL(rGXSW7%_%DZ0DyV+Iz45XgfcQ$vB2SXxSQ~ zdTAv~a$Z>bHhcwo{oeb~yyTxhV+m=d<|Wj$_a5$Jho(Mn7pPH_TB8~#=wK-OX-%-* zgnZx4Kp%|1TO(A5BWx{zKzi$==RO}nFIQ&zyJe1P>=*suglwA&|E>h-7ycayVmsG8 zmx_jB9fTbP5_X(IXua#G>)XW}SKg z;h#HE?n@N@FVI^_0fa(F`1gDxfg^P6=mSMW6vAKMNazgRD&fz09NP71pGt7OLuY!M zp?%51zqYyXw@)rm?@JK=!k{PZrjBMw|2*=C%8_=Dw?25164W^ZvR{&3C5R{rO0e(z zk%CVVAYSALB32#R+YX;ZCJej<1EYr1Ll|$GGYbYH6kN7Xr4aeOg$TU&7L3k@fZ~hO zM~d3UZzGaH~HFeIpABA;WlxJ+a z#udL<~v&`@&IsQwDj2@(tFyb zSO5$$7XbGt^8On5+KCktEeAZU>Zi3Ij=L$ocH&0~cEa=m69}fRB9Bqz;nPGR)=Yz+ z@8*O}=?(;^KT9Mi(%!F~=##HtluBhxWU=XS>jIy9urSY7+y&+`KvI^|)x&Ta`WxNi_*NqK>4;{qZ%KjeKT$JQE%}->%>pRgU8V^h$mI zX~`es`a~x#&M(&S0eD-iBftz?5qM^C!bR-X7^yy_E$ffmvfSCKkhV<%;PU-=F4O{X z!SswLg)r%xN}Urcd@LIy+8@hyqyqv{v^RLnsw1$uFO=g;C1G2(eFnMe(m^^7G*vf@^f;t$o|eh1P!HTS{wvwf_U9HQciQ=if86jlEo`?MJ_*)Yez~KQL;`|D@U+ z(0SK{^b&WalcxOE952)i44^@iTu`W7?YlQup zIwlMiSW>)<4pQNq7^T$6W#2K}Jkv*fCcG5!xRgk}-Q4eLoVsVHeerFBuy?acoj3_) zci^#Mj|z%aHblh=mi!(Sox`9#Dmn`%!2+%I#6NQmoh*&2oy4D2?Tg;}rTFJ+dyxE0 z95KQa|6HkQA1jxUY2_5Be!+*H$OyT;x?Ih5=$F#BPLftANx z=P)o1wK|(silexgm~{^Wy>QMq*H1c!k! zHN%M`6o5VE@x@_ad{KZ%f#Ucte)Sy&#_!(4zyPA}+^>?({VFYmARIfA#oQXYyv?wHL7 zsN32@*-#hhmC-E4t%b?RQDDO&fqGWasVk`t@~YYAJ$sJ=8xo%QL_Ft-`J=!9T8G`$ zvtjMx90nJ~QDD?iQhG)kwJdFp*f}LKRoi7w&%~F-h}blE3&we;4_H5yYzqg0-9D-_ ziN6uRImT;4z(LB309*(pN=m`jDvt{RH;sGWB8eCiQg?hA>sxY5!ptQX>5|VR zUtBQ{dr^(4#?X)-BZin1s?_9@Mz{|ct{CEQ3`xDk*9Z{Igqi79f_1p*2sX}?bZrNu zxZ6+}azz`-h;l00eh24dEjTr68dWg`)Cu>J!}e_k{OE=lKd<9FO3ZeBB1-ZUt-~4Qc` zACTGW1ASl`eV~If3w=NhCHMhP3LDe(?duKo!Ejd}#J8gl_`)G)aIMh?FB))IZ9~9= zK6nAZIYF%t@NZ5ZWNY=opw|Zw!3*hw`lkB8y####C-M&g%+6P@5A=K0hpmk;S&x)H z7=(3AA5cpAfMZpUwrUWq!2VpOKD>ZF;QL%Lw4|~@A3!g8W?on61C|fGJ_x!<096E_ zZ_og+19V{;ln1ouOdlYII}Woo`aql&^nqk;p$|w#f9sG<(+52C>I1|T_*tb7HgC*W z*f+GHJ{UAy1$|&^YeTjus1wonscnheqz{G!s!pt8VWFcBkgn1Pc=>wt0SD}0Wk?_3 zMO~#2qTuKAM|}0@10=t`j^#JU{z$B2eas1p{IT}^nQG>j!cQGen2gl2{HnmAjo|(u zkM73FRTGCYih}-N^Dvn)ML{7|L5+46h%`S>I%v{Q7tC=TPQ5Iq3kJh(?;>M2t7b!E zv>oP0dwj|Pf8&&i;srZRl=W7BJJx-1mruF3n}UBYf9`V80)xQKBLydw4n}5)(E}X1 z{`jtL#`HquJ*ONs?l%;3`d%Ft`s!sWEsy(plCvTiw<&(LB@YNwN_}jKN@nx;Oe;c?dKzNH zIJZ}WTc@|tSxt8I1$7chV#sX*c+^f2R*DXQu`;%3w=|=LCEMjHn_ZMcOhw5x-;}05 zxcya5-==%V((;Kzhwh&|G4a5ex#P>zr{^Y)Po6yK?7t%==VG1`E*{<6-nsn7iHSE{ zbKRQ!x8as{ZXcNr41lp(`k|yKSf*f=i}$X<1zOxTuFJVz#dQVOUaqUSuIA$0Bljk* z8@S%Wbqm++Tt~U?=6WaBcXCZ~O>sTQHOqC1Yk})L*ZaA?kL!b6Kg#vvTtCJ2GhCnI zdW!3FT))8e1+M>r>o>UmJ=d4GBCfwHcpssy)iD1MY3srLbENme{>%Jc4gYVGw*TrZ z%MRRo;G7$qJ9F~nRSb1KY{A}I6vG# zg+HmQsD#Spzgw}Fns=NHh8|n7Cs<%G__E*om`VDdn zQ5!i#5GDO_14H`j_{*#3$BLU`arq^*Sz74*S_iCb=SW+zxkG1Z#%sI zz`a)=xEG}EKXC5q1LrbuJ3aS+JNmZ6ZfbJ?QqHaC^_Vy(iB&`0VuY$>r0FH{SE@hYp2bQ)5fh|72=nIh(sG zd(GTyvZ-^&r>0IUWv3?3O`ke*DqA`;KYx00dFn)_eD3VjaVl`e#mYJ_7RIf$HT|^` ztmta$IhRoI&gD0Ev+(=NBxXhe&V4s`AxPNJc)WA@nrlv;eh{#y78hwiFYh9c>ePQv zg!~${kpB%Qr{*49eyAwG3(7o7KGjXz%a6=YO@Oo0DzCObM*7>XIWRGC{M@<8`=`%d zbIrktiKU08Pfc{P-ln{&=kw(K?3(=7QTF8I($cB|2fV6g_sKLR{! z^S^mwV*d2>-15|7UkUQxO8)D)^xrRW7ln#G#VaC5Nw~hgy#dLIh&?fRYHDtI^5p9J zo(S!qrQD-D|9Ysq7U~`$y{LOT&zEsYg6`nbe@B2xFzn>&&9{r+qg=bW`r!fZOVjJ3 zPx#KbXgh!QvNZm4c=ox=(%mbhoqA+q@$~8CiRlx53@?!O#yu0)xctMHr(cPG3Htdq zn%cv)m+MNd*KpzeGXeiv?pJf|kV9Q_d1?m zsNU2ut{9UeVuIYa#w)g`}5BYC~S}CIRuaXX`471uA8}pYU$5gxkTxQxo+n=!gZAE4z4@7?&5kY*W0-6<~qhDFc)F($GJ|=-FkX% zD%>)6WD3c(FMIg(;>i;vzGG@>{`B0^)RC#l6H|*zYg01!kbiY3bNB9l7r5Mi{LIqw z=~M3T-FLg=!pz+Ae(|Iej$S{1a&mg^hV1x5lZ#7J%WpZe{J{R}JK4OHu|^ioOf4wyG}po|LVsri&K;K-almSnrp*&cE=JN zJpq0+l-Cuh`NdbI|11{TG(2zTDuzd#FwS)qSHTJE4bMAxui<$omxkvaE)CB=;?nSZ z2iN?`GY?MBb(nE}@$|VzCLWwxp5SSIYH|6Iv37r!_CC`Xryp4J;;L+acFn%*%D~tY z^UI4pzocfa`diUe`LA4&R%BXz5yA4+ERDa6j=y+inq1m;%w4%{>eSNl#p!uyVdB)( c^5luh2Nf+*n*aa+ literal 0 HcmV?d00001 diff --git a/test/e2e/tests/utils.go b/test/e2e/tests/utils.go index b0a5d24269c..8e72ca0e0c5 100644 --- a/test/e2e/tests/utils.go +++ b/test/e2e/tests/utils.go @@ -9,6 +9,7 @@ import ( "context" "fmt" "io" + "strings" "testing" "time" @@ -25,6 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" "sigs.k8s.io/gateway-api/conformance/utils/config" + "sigs.k8s.io/gateway-api/conformance/utils/tlog" egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" ) @@ -64,6 +66,7 @@ func WaitForPods(t *testing.T, cl client.Client, namespace string, selectors map } } + t.Logf("pod %s/%s status: %v", p.Namespace, p.Name, p.Status) return false } @@ -200,6 +203,48 @@ func policyAcceptedByAncestor(ancestors []gwapiv1a2.PolicyAncestorStatus, contro return false } +// EnvoyExtensionPolicyMustFail waits for an EnvoyExtensionPolicy to fail with the specified reason. +func EnvoyExtensionPolicyMustFail( + t *testing.T, client client.Client, policyName types.NamespacedName, + controllerName string, ancestorRef gwapiv1a2.ParentReference, message string, +) { + t.Helper() + + policy := &egv1a1.EnvoyExtensionPolicy{} + waitErr := wait.PollUntilContextTimeout( + context.Background(), 1*time.Second, 60*time.Second, + true, func(ctx context.Context) (bool, error) { + err := client.Get(ctx, policyName, policy) + if err != nil { + return false, fmt.Errorf("error fetching EnvoyExtensionPolicy: %w", err) + } + + if policyFailAcceptedByAncestor(policy.Status.Ancestors, controllerName, ancestorRef, message) { + t.Logf("EnvoyExtensionPolicy has been failed: %v", policy) + return true, nil + } + + return false, nil + }) + + require.NoErrorf(t, waitErr, "error waiting for EnvoyExtensionPolicy to fail with message: %s policy %v", message, policy) +} + +func policyFailAcceptedByAncestor(ancestors []gwapiv1a2.PolicyAncestorStatus, controllerName string, ancestorRef gwapiv1a2.ParentReference, message string) bool { + for _, ancestor := range ancestors { + if string(ancestor.ControllerName) == controllerName && cmp.Equal(ancestor.AncestorRef, ancestorRef) { + for _, condition := range ancestor.Conditions { + if condition.Type == string(gwapiv1a2.PolicyConditionAccepted) && + condition.Status == metav1.ConditionFalse && + strings.Contains(condition.Message, message) { + return true + } + } + } + } + return false +} + // EnvoyExtensionPolicyMustBeAccepted waits for the specified EnvoyExtensionPolicy to be accepted. func EnvoyExtensionPolicyMustBeAccepted(t *testing.T, client client.Client, policyName types.NamespacedName, controllerName string, ancestorRef gwapiv1a2.ParentReference) { t.Helper() @@ -222,3 +267,25 @@ func EnvoyExtensionPolicyMustBeAccepted(t *testing.T, client client.Client, poli require.NoErrorf(t, waitErr, "error waiting for EnvoyExtensionPolicy to be accepted") } + +func WaitForLoadBalancerAddress(t *testing.T, client client.Client, timeout time.Duration, nn types.NamespacedName) (string, error) { + t.Helper() + + var ipAddr string + waitErr := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + s := &corev1.Service{} + err := client.Get(ctx, nn, s) + if err != nil { + tlog.Logf(t, "error fetching Service: %v", err) + return false, fmt.Errorf("error fetching Service: %w", err) + } + + if len(s.Status.LoadBalancer.Ingress) > 0 { + ipAddr = s.Status.LoadBalancer.Ingress[0].IP + return true, nil + } + return false, nil + }) + require.NoErrorf(t, waitErr, "error waiting for Service to have at least one load balancer IP address in status") + return ipAddr, nil +} diff --git a/test/e2e/tests/wasm.go b/test/e2e/tests/wasm_http.go similarity index 93% rename from test/e2e/tests/wasm.go rename to test/e2e/tests/wasm_http.go index 7211c13e943..6ce3078f3ab 100644 --- a/test/e2e/tests/wasm.go +++ b/test/e2e/tests/wasm_http.go @@ -22,14 +22,14 @@ import ( ) func init() { - ConformanceTests = append(ConformanceTests, WasmTest) + ConformanceTests = append(ConformanceTests, HTTPWasmTest) } -// WasmTest tests Wasm extension for an http route with HTTP Wasm configured. -var WasmTest = suite.ConformanceTest{ - ShortName: "Wasm", +// HTTPWasmTest tests Wasm extension for an http route with HTTP Wasm configured. +var HTTPWasmTest = suite.ConformanceTest{ + ShortName: "Wasm HTTP Code Source", Description: "Test Wasm extension that adds response headers", - Manifests: []string{"testdata/wasm.yaml"}, + Manifests: []string{"testdata/wasm-http.yaml"}, Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { t.Run("http route with http wasm source", func(t *testing.T) { ns := "gateway-conformance-infra" diff --git a/test/e2e/tests/wasm_oci.go b/test/e2e/tests/wasm_oci.go new file mode 100644 index 00000000000..1a41187092b --- /dev/null +++ b/test/e2e/tests/wasm_oci.go @@ -0,0 +1,427 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build e2e +// +build e2e + +package tests + +import ( + "bufio" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "testing" + "time" + + dockertypes "github.com/docker/docker/api/types" + "github.com/docker/docker/client" + "github.com/docker/docker/pkg/archive" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/daemon" + "github.com/google/go-containerregistry/pkg/v1/remote" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" + "sigs.k8s.io/gateway-api/conformance/utils/http" + "sigs.k8s.io/gateway-api/conformance/utils/kubernetes" + "sigs.k8s.io/gateway-api/conformance/utils/suite" + + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" + "github.com/envoyproxy/gateway/internal/gatewayapi" +) + +const ( + dockerUsername = "testuser" + dockerPassword = "testpassword" + dockerEmail = "your-email@example.com" + testNS = "gateway-conformance-infra" + testGW = "same-namespace" + testEEP = "oci-wasm-source-test" + pullSecret = "registry-secret" + httpRouteWithWasm = "http-with-oci-wasm-source" + httpRouteWithoutWasm = "http-without-wasm" +) + +func init() { + ConformanceTests = append(ConformanceTests, OCIWasmTest) +} + +// OCIWasmTest tests Wasm extension for an http route with OCI Wasm configured. +var OCIWasmTest = suite.ConformanceTest{ + ShortName: "Wasm OCI Image Code Source", + Description: "Test OCI Wasm extension", + Manifests: []string{"testdata/wasm-oci.yaml", "testdata/wasm-oci-registry-test-server.yaml"}, + Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { + // Get the LoadBalancer IP of the registry + registryNN := types.NamespacedName{Name: "oci-registry", Namespace: testNS} + registryIP, err := WaitForLoadBalancerAddress(t, suite.Client, 10*time.Second, registryNN) + if err != nil { + t.Fatalf("failed to get registry IP: %v", err) + } + registryAddr := fmt.Sprintf("%s:5000", registryIP) + + // Push the wasm image to the registry + digest := pushWasmImageForTest(t, suite, registryAddr) + + // Create the pull secret for the wasm image + secret := createPullSecretForWasmTest(t, suite, registryAddr, dockerPassword) + + // Create the EnvoyExtensionPolicy referencing the wasm image + eep := createEEPForWasmTest(t, suite, registryAddr, digest, true) + + // Wait for the EnvoyExtensionPolicy to be accepted + ancestorRef := gwapiv1a2.ParentReference{ + Group: gatewayapi.GroupPtr(gwapiv1.GroupName), + Kind: gatewayapi.KindPtr(gatewayapi.KindGateway), + Namespace: gatewayapi.NamespacePtr(testNS), + Name: gwapiv1.ObjectName(testGW), + } + + EnvoyExtensionPolicyMustBeAccepted( + t, suite.Client, + types.NamespacedName{Name: testEEP, Namespace: testNS}, + suite.ControllerName, + ancestorRef) + + // HTTPRoute configured with the correct wasm extension should modify the response + t.Run("http route with oci wasm source", func(t *testing.T) { + // Wait for the HTTPRoute to be accepted + routeNN := types.NamespacedName{Name: httpRouteWithWasm, Namespace: testNS} + gwNN := types.NamespacedName{Name: testGW, Namespace: testNS} + gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), routeNN) + + // Make a request to the gateway and expect the wasm filter to add a response header + expectedResponse := http.ExpectedResponse{ + Request: http.Request{ + Host: "www.example.com", + Path: "/wasm-oci", + }, + + // Set the expected request properties to empty strings. + // This is a workaround to avoid the test failure. + // These values can't be extracted from the json format response + // body because the test wasm code appends a "Hello, world" text + // to the response body, invalidating the json format. + ExpectedRequest: &http.ExpectedRequest{ + Request: http.Request{ + Host: "", + Method: "", + Path: "", + Headers: nil, + }, + }, + Namespace: "", + + Response: http.Response{ + StatusCode: 200, + Headers: map[string]string{ + "x-wasm-custom": "FOO", // response header added by wasm + }, + }, + } + + http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, expectedResponse) + }) + + // HTTPRoute without wasm should not modify the response + t.Run("http route without wasm", func(t *testing.T) { + EnvoyExtensionPolicyMustBeAccepted( + t, suite.Client, + types.NamespacedName{Name: testEEP, Namespace: testNS}, + suite.ControllerName, + ancestorRef) + + ns := testNS + routeNN := types.NamespacedName{Name: httpRouteWithoutWasm, Namespace: ns} + gwNN := types.NamespacedName{Name: testGW, Namespace: ns} + gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), routeNN) + + expectedResponse := http.ExpectedResponse{ + Request: http.Request{ + Host: "www.example.com", + Path: "/no-wasm", + }, + Response: http.Response{ + StatusCode: 200, + AbsentHeaders: []string{"x-wasm-custom"}, + }, + Namespace: ns, + } + + req := http.MakeRequest(t, &expectedResponse, gwAddr, "HTTP", "http") + cReq, cResp, err := suite.RoundTripper.CaptureRoundTrip(req) + if err != nil { + t.Errorf("failed to get expected response: %v", err) + } + + if err := http.CompareRequest(t, &req, cReq, cResp, expectedResponse); err != nil { + t.Errorf("failed to compare request and response: %v", err) + } + }) + + // Verify that the wasm module can't be loaded if the pull secret is missing + // even if the wasm image is already cached. + t.Run("without pull secret", func(t *testing.T) { + // Delete the EnvoyExtensionPolicy with pull secret + _ = suite.Client.Delete(context.Background(), eep) + + // Create the EnvoyExtensionPolicy without pull secret + createEEPForWasmTest(t, suite, registryAddr, digest, false) + + defer func() { + _ = suite.Client.Delete(context.Background(), eep) + }() + + // Wait for the EnvoyExtensionPolicy to be failed due to missing pull secret + ancestorRef := gwapiv1a2.ParentReference{ + Group: gatewayapi.GroupPtr(gwapiv1.GroupName), + Kind: gatewayapi.KindPtr(gatewayapi.KindGateway), + Namespace: gatewayapi.NamespacePtr(testNS), + Name: gwapiv1.ObjectName(testGW), + } + + EnvoyExtensionPolicyMustFail( + t, suite.Client, + types.NamespacedName{Name: testEEP, Namespace: testNS}, + suite.ControllerName, + ancestorRef, "failed to login to private registry") + }) + + // Verify that the wasm module can't be loaded if the password is incorrect + // even if the wasm image is already cached. + t.Run("with wrong password", func(t *testing.T) { + // Delete the EnvoyExtensionPolicy with pull secret + _ = suite.Client.Delete(context.Background(), eep) + + // Delete the pull secret + _ = suite.Client.Delete(context.Background(), secret) + + // Create the pull secret with a wrong password + secret = createPullSecretForWasmTest(t, suite, registryAddr, "wrongpassword") + + // Create the EnvoyExtensionPolicy without pull secret + eep = createEEPForWasmTest(t, suite, registryAddr, digest, true) + + defer func() { + _ = suite.Client.Delete(context.Background(), eep) + _ = suite.Client.Delete(context.Background(), secret) + }() + + // Wait for the EnvoyExtensionPolicy to be failed due to missing pull secret + ancestorRef := gwapiv1a2.ParentReference{ + Group: gatewayapi.GroupPtr(gwapiv1.GroupName), + Kind: gatewayapi.KindPtr(gatewayapi.KindGateway), + Namespace: gatewayapi.NamespacePtr(testNS), + Name: gwapiv1.ObjectName(testGW), + } + + EnvoyExtensionPolicyMustFail( + t, suite.Client, + types.NamespacedName{Name: testEEP, Namespace: testNS}, + suite.ControllerName, + ancestorRef, "failed to login to private registry") + }) + }, +} + +func pushWasmImageForTest(t *testing.T, suite *suite.ConformanceTestSuite, registryAddr string) string { + // Wait for the registry pod to be ready + podReady := corev1.PodCondition{Type: corev1.PodReady, Status: corev1.ConditionTrue} + WaitForPods( + t, suite.Client, testNS, + map[string]string{"app": "oci-registry"}, corev1.PodRunning, podReady) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*120) + defer cancel() + + var ( + cli *client.Client + tar io.Reader + res dockertypes.ImageBuildResponse + digest v1.Hash + err error + ) + + tag := fmt.Sprintf("%s/testwasm:v1.0.0", registryAddr) + + if cli, err = client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()); err != nil { + t.Fatalf("failed to create docker client: %v", err) + } + + if tar, err = archive.TarWithOptions("testdata/wasm", &archive.TarOptions{}); err != nil { + t.Fatalf("failed to create tar: %v", err) + } + + opts := dockertypes.ImageBuildOptions{ + Dockerfile: "Dockerfile", + Tags: []string{tag}, + Remove: true, + } + if res, err = cli.ImageBuild(ctx, tar, opts); err != nil { + t.Fatalf("failed to build image: %v", err) + } + defer func() { + _ = res.Body.Close() + }() + if err = printDockerCLIResponse(res.Body); err != nil { + t.Fatalf("failed to print docker cli response: %v", err) + } + + ref, err := name.ParseReference(tag, name.Insecure) + if err != nil { + t.Fatalf("failed to parse reference: %v", err) + } + + // Retrieve the image from the local Docker daemon + img, err := daemon.Image(ref) + if err != nil { + t.Fatalf("failed to retrieve image: %v", err) + } + + authOption := remote.WithAuth(&authn.Basic{ + Username: dockerUsername, + Password: dockerPassword, + }) + + const retries = 5 + for i := 0; i < retries; i++ { + // Push the image to the remote registry + // err = crane.Push(img, tag) + err = remote.Write(ref, img, authOption) + if err == nil { + break + } + t.Logf("failed to push image: %v", err) + } + if err != nil { + t.Fatalf("failed to push image: %v", err) + } + + if img, err = remote.Image(ref, authOption); err != nil { + t.Fatalf("failed to retrieve image: %v", err) + } + if digest, err = img.Digest(); err != nil { + t.Fatalf("failed to get image digest: %v", err) + } + + t.Logf("pushed image %s with digest: %s", tag, digest.Hex) + return digest.Hex +} + +type ErrorLine struct { + Error string `json:"error"` + ErrorDetail ErrorDetail `json:"errorDetail"` +} + +type ErrorDetail struct { + Message string `json:"message"` +} + +func printDockerCLIResponse(rd io.Reader) error { + var lastLine string + + scanner := bufio.NewScanner(rd) + for scanner.Scan() { + lastLine = scanner.Text() + fmt.Println(scanner.Text()) + } + + errLine := &ErrorLine{} + _ = json.Unmarshal([]byte(lastLine), errLine) + if errLine.Error != "" { + return errors.New(errLine.Error) + } + + if err := scanner.Err(); err != nil { + return err + } + + return nil +} + +func createPullSecretForWasmTest(t *testing.T, suite *suite.ConformanceTestSuite, registryAddr string, password string) *corev1.Secret { + // Create Docker config JSON + dockerConfigJSON := fmt.Sprintf(`{"auths":{"%s":{"username":"%s","password":"%s","email":"%s","auth":"%s"}}}`, + registryAddr, dockerUsername, password, dockerEmail, + base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", dockerUsername, password)))) + + // Create a Secret object + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: pullSecret, + Namespace: testNS, + }, + Type: corev1.SecretTypeDockerConfigJson, + Data: map[string][]byte{ + corev1.DockerConfigJsonKey: []byte(dockerConfigJSON), + }, + } + + // Create the secret in the specified namespace + _ = suite.Client.Delete(context.Background(), secret) + if err := suite.Client.Create(context.Background(), secret); err != nil { + t.Fatalf("failed to create secret: %v", err) + } + return secret +} + +func createEEPForWasmTest( + t *testing.T, suite *suite.ConformanceTestSuite, + registryAddr string, digest string, withPullSecret bool, +) *egv1a1.EnvoyExtensionPolicy { + eep := &egv1a1.EnvoyExtensionPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: testEEP, + Namespace: testNS, + }, + Spec: egv1a1.EnvoyExtensionPolicySpec{ + PolicyTargetReferences: egv1a1.PolicyTargetReferences{ + TargetRefs: []gwapiv1a2.LocalPolicyTargetReferenceWithSectionName{ + { + LocalPolicyTargetReference: gwapiv1a2.LocalPolicyTargetReference{ + Group: "gateway.networking.k8s.io", + Kind: "HTTPRoute", + Name: httpRouteWithWasm, + }, + }, + }, + }, + + Wasm: []egv1a1.Wasm{ + { + Name: ptr.To("wasm-filter"), + RootID: ptr.To("my_root_id"), + Code: egv1a1.WasmCodeSource{ + Type: egv1a1.ImageWasmCodeSourceType, + Image: &egv1a1.ImageWasmCodeSource{ + URL: fmt.Sprintf("%s/testwasm:v1.0.0", registryAddr), + SHA256: &digest, + }, + }, + }, + }, + }, + } + if withPullSecret { + eep.Spec.Wasm[0].Code.Image.PullSecretRef = &gwapiv1b1.SecretObjectReference{ + Name: gwapiv1.ObjectName(pullSecret), + } + } + // Create the EnvoyExtensionPolicy in the specified namespace + if err := suite.Client.Create(context.Background(), eep); err != nil { + t.Fatalf("failed to create envoy extension policy: %v", err) + } + return eep +} diff --git a/test/helm/gateway-helm/control-plane-with-pdb.out.yaml b/test/helm/gateway-helm/control-plane-with-pdb.out.yaml index d555ce7e6b6..63d999c0b5c 100644 --- a/test/helm/gateway-helm/control-plane-with-pdb.out.yaml +++ b/test/helm/gateway-helm/control-plane-with-pdb.out.yaml @@ -355,6 +355,9 @@ spec: - name: ratelimit port: 18001 targetPort: 18001 + - name: wasm + port: 18002 + targetPort: 18002 - name: metrics port: 19001 targetPort: 19001 @@ -415,6 +418,8 @@ spec: name: grpc - containerPort: 18001 name: ratelimit + - containerPort: 18002 + name: wasm - containerPort: 19001 name: metrics readinessProbe: diff --git a/test/helm/gateway-helm/default-config.out.yaml b/test/helm/gateway-helm/default-config.out.yaml index 2169dee5233..e01d1c025e4 100644 --- a/test/helm/gateway-helm/default-config.out.yaml +++ b/test/helm/gateway-helm/default-config.out.yaml @@ -341,6 +341,9 @@ spec: - name: ratelimit port: 18001 targetPort: 18001 + - name: wasm + port: 18002 + targetPort: 18002 - name: metrics port: 19001 targetPort: 19001 @@ -401,6 +404,8 @@ spec: name: grpc - containerPort: 18001 name: ratelimit + - containerPort: 18002 + name: wasm - containerPort: 19001 name: metrics readinessProbe: diff --git a/test/helm/gateway-helm/deployment-custom-topology.out.yaml b/test/helm/gateway-helm/deployment-custom-topology.out.yaml index 40f18f8f849..47b89266c24 100644 --- a/test/helm/gateway-helm/deployment-custom-topology.out.yaml +++ b/test/helm/gateway-helm/deployment-custom-topology.out.yaml @@ -341,6 +341,9 @@ spec: - name: ratelimit port: 18001 targetPort: 18001 + - name: wasm + port: 18002 + targetPort: 18002 - name: metrics port: 19001 targetPort: 19001 @@ -429,6 +432,8 @@ spec: name: grpc - containerPort: 18001 name: ratelimit + - containerPort: 18002 + name: wasm - containerPort: 19001 name: metrics readinessProbe: diff --git a/test/helm/gateway-helm/deployment-images-config.out.yaml b/test/helm/gateway-helm/deployment-images-config.out.yaml index 261544dba3b..3cba2b4a50b 100644 --- a/test/helm/gateway-helm/deployment-images-config.out.yaml +++ b/test/helm/gateway-helm/deployment-images-config.out.yaml @@ -341,6 +341,9 @@ spec: - name: ratelimit port: 18001 targetPort: 18001 + - name: wasm + port: 18002 + targetPort: 18002 - name: metrics port: 19001 targetPort: 19001 @@ -401,6 +404,8 @@ spec: name: grpc - containerPort: 18001 name: ratelimit + - containerPort: 18002 + name: wasm - containerPort: 19001 name: metrics readinessProbe: diff --git a/test/helm/gateway-helm/envoy-gateway-config.out.yaml b/test/helm/gateway-helm/envoy-gateway-config.out.yaml index bdb60c8f131..b80001eb80b 100644 --- a/test/helm/gateway-helm/envoy-gateway-config.out.yaml +++ b/test/helm/gateway-helm/envoy-gateway-config.out.yaml @@ -343,6 +343,9 @@ spec: - name: ratelimit port: 18001 targetPort: 18001 + - name: wasm + port: 18002 + targetPort: 18002 - name: metrics port: 19001 targetPort: 19001 @@ -403,6 +406,8 @@ spec: name: grpc - containerPort: 18001 name: ratelimit + - containerPort: 18002 + name: wasm - containerPort: 19001 name: metrics readinessProbe: diff --git a/test/helm/gateway-helm/global-images-config.out.yaml b/test/helm/gateway-helm/global-images-config.out.yaml index 4f1490d32e7..4537cf99b49 100644 --- a/test/helm/gateway-helm/global-images-config.out.yaml +++ b/test/helm/gateway-helm/global-images-config.out.yaml @@ -345,6 +345,9 @@ spec: - name: ratelimit port: 18001 targetPort: 18001 + - name: wasm + port: 18002 + targetPort: 18002 - name: metrics port: 19001 targetPort: 19001 @@ -405,6 +408,8 @@ spec: name: grpc - containerPort: 18001 name: ratelimit + - containerPort: 18002 + name: wasm - containerPort: 19001 name: metrics readinessProbe: diff --git a/tools/docker/envoy-gateway/Dockerfile b/tools/docker/envoy-gateway/Dockerfile index 074320308eb..79b85852c9c 100644 --- a/tools/docker/envoy-gateway/Dockerfile +++ b/tools/docker/envoy-gateway/Dockerfile @@ -1,8 +1,13 @@ +FROM busybox as source +# Create the data directory for eg +RUN mkdir -p /var/lib/eg + # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details FROM gcr.io/distroless/static:nonroot@sha256:e9ac71e2b8e279a8372741b7a0293afda17650d926900233ec3a7b2b7c22a246 ARG TARGETPLATFORM COPY $TARGETPLATFORM/envoy-gateway /usr/local/bin/ +COPY --from=source --chown=65532:65532 /var/lib /var/lib USER 65532:65532 From 7338994632bcb73c00bbebc49b39747d1eda9f02 Mon Sep 17 00:00:00 2001 From: sh2 Date: Sat, 29 Jun 2024 07:55:59 +0800 Subject: [PATCH 22/23] feat: add resources dashboard for envoy gateway (#3689) * add resources dashboard for envoy gateway Signed-off-by: shawnh2 * fix gen-check Signed-off-by: shawnh2 --------- Signed-off-by: shawnh2 --- .../dashboards/envoy-gateway-resource.json | 263 +++++++++++++++++ .../observability/grafana-integration.md | 6 + .../img/envoy-gateway-resources-dashboard.png | Bin 0 -> 130568 bytes .../helm/gateway-addons-helm/default.out.yaml | 264 ++++++++++++++++++ test/helm/gateway-addons-helm/e2e.out.yaml | 264 ++++++++++++++++++ 5 files changed, 797 insertions(+) create mode 100644 charts/gateway-addons-helm/dashboards/envoy-gateway-resource.json create mode 100644 site/static/img/envoy-gateway-resources-dashboard.png diff --git a/charts/gateway-addons-helm/dashboards/envoy-gateway-resource.json b/charts/gateway-addons-helm/dashboards/envoy-gateway-resource.json new file mode 100644 index 00000000000..0dada1c06ef --- /dev/null +++ b/charts/gateway-addons-helm/dashboards/envoy-gateway-resource.json @@ -0,0 +1,263 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Envoy Gateway Memory and CPU Usage", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 6, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(namespace) (rate(container_cpu_usage_seconds_total{container=\"envoy-gateway\"}[5m]))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Envoy Gateway CPU Usage (m)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(namespace) (container_memory_working_set_bytes{container=\"envoy-gateway\"}/1024/1024)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Envoy Gateway Memory Usage (MiB)", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "tags": [ + "Control Plane" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": {}, + "timezone": "", + "title": "Envoy Gateway Resources", + "uid": "edq1b2tldspa8d", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/site/content/en/latest/tasks/observability/grafana-integration.md b/site/content/en/latest/tasks/observability/grafana-integration.md index 600a7f5d550..986f8af1986 100644 --- a/site/content/en/latest/tasks/observability/grafana-integration.md +++ b/site/content/en/latest/tasks/observability/grafana-integration.md @@ -65,6 +65,12 @@ This dashboard example shows the overall stats exported by Envoy Gateway fleet. ![Envoy Gateway Global: Infrastructure Manager](/img/envoy-gateway-global-infra-manager.png) +### [Envoy Gateway Resources](https://raw.githubusercontent.com/envoyproxy/gateway/main/charts/gateway-addons-helm/dashboards/envoy-gateway-resource.json) + +This dashboard example shows the overall resources stats for each Envoy Gateway fleet. + +![Envoy Gateway Resources](/img/envoy-gateway-resources-dashboard.png) + ## Update Dashboards The example dashboards cannot be updated in-place by default, if you are trying to diff --git a/site/static/img/envoy-gateway-resources-dashboard.png b/site/static/img/envoy-gateway-resources-dashboard.png new file mode 100644 index 0000000000000000000000000000000000000000..8b7128658bbbf00d094400ac59cedc247639ada6 GIT binary patch literal 130568 zcmZU)WmH>T)GbVlyR{?~cPJ91#ickDic}~RfBc8UJ@@qfy-8SN^hi@k96_#${819;9<6*g;BQoLK_qaI8QeIbogwav3 zKc-VRpzX=^?3LfhO}ETv%wuk6g!F3f{0e-7+3-5vB6zWRveLZ2^B4VkZ}!K}Npqxm z;H~BIvXxPT75!jexz!ub_hcLqG%6(j=O>2lQ~Re8amxeI%P=*-{~7-OcZdOV#fWSQ zD*^qw4Svl%gT?>v14%g49Qy%V;{VTZw+P3m`v08ZzbzOAKYQ~;@+Rm1b!rmB_t;pG zYo6x+*?HIFr|sK@$0iR`Q!diQtACnk*c&yKyq}2b7w3uHf!QA4O~(1H7?XUR&fZLs z_P-{qLid*$4Q@Yu@0#$XzB=k5l<38~cY4DCJr^0`)xS*Yh*-*ea=3o`?0nEjL%;hS zW#Y4EZ=Ps7Hh{by1$@5pKJIe$Ql*tb=_aeRY)#Z9Q39WI zUL3K#tJ;*BxS(Wa4W_EvuG;}@TMYCKRJ{`>Hbsw{SFw7+W6oO5#%_u)g-cz3sc@Oc zTsSVY|N7@P|M+M85#sK8r%5phv&T%PEqV&%!O>~NSoP-0QXO!t$p1VVJF1(EM|Dio zUptD*-u(_B%vk@)f3z<#{iR`h`Bs1O-s>Mh~S+?2s_lJ&~+)B5$q)Ts5PH^viCMl~g-ush?(789IK_)7fN%8^E<9(^_>%XlC zA}UtL)Ne!85*tFbj5Qmh{GvBd*LP z+O_3~>US1w@GFOr)Z^Dt@CV_WFPl4)ExW$`cK$=l)tX8Ll!qA*$#Rj~{rzlcSh~NH zC+nXFCM?=Ic%nje-a9>;K4Or6fQ8ulo7h-;{1R1lc>CwGw9xa0?@ah&E}_{!u7Vve zv3M?q8NpSp%ID-c3I-F`wMxr1Z(l;5z7`J7AI#}svRB>68LlEU(_XG8s2T?{cGM}7Z*ZOJNo1CpZ9dsV`&4Ld zWToR^8&>x*1)I%e8Txc+?G?ln9@e!7vopMzG;<BPxKRxKB0?Xxs>VXj{_RXfyDus6!^1K=TACj64$i zDP7O6BQ~@jdt8r7U7A6dhQ8r8Z6FNE%h;0(v8iaUAC~pnH{tDNbR3GKP=>bOMJx1E zW+}g-&2J_T+tTzD*N@1Uz^XDxMc*G_&pnppXxo$1a8 zhE-od^*hrG5W}IE{RGC*L1z0AMZ`BfIf-V?oQwyCf6SMt;JYxU`-L-m)1a&n&C<1PIm8Y9@G$T4R4e8pHQebSkFZ zA^^@$u^R1@<{^3D`qjs$^b_S%)9)HEtD#-Zg5{?lj&~Kl4>5ea?iF(5zM~TPaPL~L*?YFvpwHW)=BQQ z%%+WR_}wb{Uu~5zm}_}H_rk0S{(Px%u2-UwMCxmav)iMT+%-1_Du6u{C&()R6=Dud@ zSEKVP-4A&tziR88(({hbFg0$z{FR?4b9)~U)mU)A)Ufq;7+zXn_qM_&A*Zg&P&276 z9@Nx10=JB;b6BuyxU}>S^OaPjUfj8Q!LAxZAPiL%e0xkvPV%N@b^P`wMx^9{`S(7Q zo^+2SI51l7Sjx4zO{=_#VbvzptmRz9)Tdw8x0B%nCPFEFCS?)YpwPaa9z>;!oiBgo z8@PAO;^W~pj+!P#yX(eKXm41R4zRM(GoQSEAEb9dLf6hk7xCoO9$lF0JAa%-ZZf5Q z?2dYI8VC+LI-K%BLl^b$wYMf z|G193L%_KR$2D+N@WayS4*%0wsi8SF{i>yut%*r<{K3U?MqY||oW6@M7&S)q`qi=sV zCqRiNoIp#?IwJ2-{yk@qj|1^DnJHRgV={^hkII9RJi^dCyp%+YHr_-qVn*pZ1Ey!YR)QfiynM?%!`z95LF_yXK zkYV^=J-OiWOX?B%o%=bm*FwMER8+}{6H~HoNz&7bW|}pJ6p(|aL2F?qEv34-RttfY zm9L6geKAk}BEr{PHj@#v5!w=Eucj1TQ&A2!)#g@`hK^rHooat(Kvl!F+>kn26 z%9OI@F&`FfyND@}yITZQ37DHjO&gXUUA*RCzrZ3kY?^ zgx*Mz-`*Uv)wC4c7Lki^AulIxjxo2h$NE=`vJw1Q5oeuCK?PToqu+Lhq~oHNI!{;J zLUcCxSOQJGNKctpyLtc>xRRdvQn~~eQ?+9SpZOm~3eA|#=?uHxtU9IbO?q-D)0pzb zk&gXYg|SEaQ%6HTILK8#dH+;YBwO15TRh9!(}kwA*uHSV$t6pjjB+2%2F*fn?PT+b z)-4684Hw*XQV*_uChR@md0MwEiFF{)P`1AB&<83AvEb>QJ1}a&;u2BsXXZ=B*iOFP1!75II9QK0E%I zt@|OTyNHEvn?4Qte5qdbv{jPiI#ymPVV{^&Di*3-tUu)|_~^*wibaSqW4~R8_!_w4 zyYNuztote@AMkZVFn4pTLUgo;JnMN!#LGf)F(dc)`>Y<1?A~&%^NfzFpfq-MmBht^a2RI)?3&Rr7b2sbz`QWJiFJ;~JZ67si^ zd~vJXb#0Zk?YH4;&ujUWvt0Dor%ZY8XzS~aXn3W0Fma3300wBhpjpjC@2O~4e8MIkHYV;-dy<@w#LjFD9G$u(tGU;8 z+S;leALN1ajq%%Brw-nGbzznp{cf_#3B$j@1zr(x3C6r|{g`Y{J>)K3Yu z=dDW^&^0wIU%V!286Anw=D!EU+6&L2kKHKbSdH3Xs51&sz=a$9`{HK9{FIqg?ozE? zODoPJt2EzY;#%wSv(h#tDoJrd_&vV;=d<|716+GUPo;TDg3W#zdRYh=2UQ`hmF$nC zk@)_677>9byQ!vZ!|wuvBbwQ7tqhdgy!S?zzBKHAo}zAyzBZ2^V3;ilM(2kFk0jt8 zM%#NbBcjEz{nrD~%8AWJn7df<`>h1-?Zo=lE9~&3=zuElfKV&xgS8fJDAwZH0&Y9j zXSJz!sC9zTJr{93)A}Y*%Jw^8tZ=`{*SO*dFxXoL4!*$UtmhYefy}1**hjI`%95zA zoW~Ychjb;l9z{c#gVUVc!S;MspFg;iBP8zQfARXFah-{FaA3kt-(>`+nfh0YkA5D? zqP=8ieE^t2sAdKt zbR=6c&#V+ML{RWTL&4e@%e2(+eom&M{Z%A&Na<32+<->DA8)CmTqhy>$u-L8(MYD&*TbX^UEpYsKT{!dCgqegHZugbv%ocb$aW;#z5` zW}gr^C0yqVrT;cmd#ZHSl%VJVQz}>rxRfw$+@$kmcMts_gx=k*dHY}GTf$dwXfsO4 z$8^#&?~gto)390T?j(YN1$+dR;kmvCwy6UPC)cP)a=?4K)xrhl2< zzQ4Vi3U!(J4X@S~gx3aIT&2c?$fZJEx)_42Z*P#HF5@=v8gtM3Iql34t6;QCgGdu6 z*{XJuMEy*bT|tXq*KOBJTP{H8`z}C;^)+(V$_8{kkA(aXzhMQGNUY842Wi(-rkPmn zx{MOS$p%-&h`Jy&y%x1ww5mzi>Kss9Ssnf~*BsKnB z=Gm+ehF*H8atbP|Y};F&^W)O1Od{S;NI_sSIH+HCXROE~A8rY`sPfF$4w6>|#xFCe za9h2!0AhG>!^ZI*4^hOus%DwE@A$nGLyh{0ra(+&V7?fkjrxeyRd?3EU@pVi7$LK8 zZHtvhj5CMc0!)sdK|#8MBJJ}G1@L){+}H}Rcf?9)7h#0N+^*iHCG2>D^J!b;$_sTP z>%dq=kf2rzIC_6kDM|LEI7qRTdtZrJejv#%7BS(0UF1CSRTEz%CD|1;GT+10dMUCG zbWI2S+jiQ`LLt`AWljk$Vb^0F*WsI;5pgLe+81)yexdZ>PKm1TPXuhV2ThpGh)3-# zlaocl=*U1TC1wo8bD?%`NnwNG)Yn69`KpiK96K?489(&3{T3%8>NhQRGM(|(?3o|t zx?m3M>*k@F?1gu@S{|5R`(kRrjxFR!r@&v_=gP5OHo!9IpaZnqMIGuWwQ%)pQ8n+0 zhnnKyn$wM(3PQL+acvNrR*OGosmT^tEeFvF(EEs@1=GCkOc7d(#}>3LbAlkqcFpOI zbn!CNiZGu`mL!ed1wcD=5Ql`LS1Ayes!sy^7lL-e_m4VBPu5;lTvB|)r~b7IBS-%> z4Q*4noS)0z_e8>X62-R@axf=yY&Dnrp}nqc*H%hwnwS5W+y)mmg%iyk{*`-rB}BuT zZwJQxwzs6@g^~k`S=}q?ZcYb&nJOf$Me?kY7}|ZiI!H#${i<*|J@Lo1RqXcWU`{3Z zzI?ihlH~dsR}azV9#Q4DX{wqsU)uK%JWo#wj#6(f86(3-vEYBNby1t~kF{txFaNIB zj@@~+FUI%cq}KA^7pL8h=uie0&xW@1jZ$+ZsWBvH5lrLAI%KUXbNA;jxBlRmiGzzT zQny!qTDL=NKGEtfNXzIChQt$0gF_r2t1`BVlG>b^%9~|w&mkT+rRs`8Vt@7J{o@pF zSBVXqE#`DUp=&jxw6#%5%zNHTcTuoqrS}rne%sW*c&x}ZFt}p)1t4UyzFzWTF=;S==ZkslTwjQ!_1?G zc+PM%#PGHxWRq#|t(jv*d;4&Oe)bs?iUbvPW!C6bha@P{J?eUzX#qjXUuG0o%!g+# z`Bznv@{L@3f%-c8K+Uz)uSW^PvuoYuYZN7S>_CccO-y`i1to{TAdg!txI*y|7ba=z z-d7c2`!)`%$q%YI29j8?rYCfW(n{7+s|cXE#2-qBUYHP?>#jMj7H7NWvfD1KSTR`HRckL#n?+;mW zdJw`hv92w4P5aR+WiJ_<5m@VbcO$RH6YoJ>^p%Y*JrO+M=~U}vQP}rr%V!*tBiRko zk1sWD^W+cpzRJf)Mp)qjjW0f#moIJZ}<__?#%a!5vS`@bB8BYZo~PyJ_GZIK>W1o(VON z9i4Z6-Ic$MWqudB{=x^@rhno28S+QrR54xZ-X}Bz#$-|AROtY!shW&W7P)8eXudsR_p7S{@pNP|cvw1G zUi4n{aA*~(GE^@|b_$^}EkErB(LmH;rn^4Cs(HtSgx3nWVdUr5SJ;C)a16Zpnb0ts z!Dl|~xoV9OB~22Ou#xMX-VH{)XP9^jFL;!WszO*CiZP-z6HS?C{%V`X!n2osE{^=o zd;f|AB?S)=sQ2phBt+k5ZfJp#C-dTOU@AeOxV&IobZCIBPJBO`KlcD^2VRfL4B6=H zUxHs&w6*V2^+?L5-zdweNg``nH;q`i_S2(vJPCT`i@aNdpeU_qv$@Z0smO6^s+( z9je)z4}zyXV?7vb(roAW3f&&^HUDiFp7Zeu`-h<4k!-2C*xYQ_yh8c$mWzq3G+*hp z1Kns?{&WlM*PS)&Tr(!mHgqa<=0Do4iAeh9{Y3FA$mn@CMJ}0*Sca6Xv+NGkvj{R2 zU=bX4$cZw=B9_u(13IVruIogbd#^l2jsmu(>oPN=*XjDXQ0%=jX@aa6yK}cp^(jkc zCL_16xoTJrtZ4bWrpK&PAKF;kh%d+l7Xpq>ZHzfSE%}{9*o|d9>$kOm{q1Z_sxr8| zj~R^M7IFfhLU1&&zeh$uPE4DZYi;}JQ%zQd z-rM%|`sn#AovCGQX`*PwD0ZriF)y>k7jYuXiNpND%!|@9e#vIu!@o}kezDDIqx5|> z4|<*4eDM=ZD5N1C5gf(roL0FjBt`SJO5Slae<+yD^OYU$+XI$$zYXo~gn#@fEp}Rl zYuz{cJ+ryjjgOg}V?A-IMvi=5K{Vcr{WG>;@$SvMb{vYCE6c#0qT{saA|KdB5E{_8 z;%eZWP#*ToHOqt|DT5-R`*;RBoXl zP)^e9`N7A+giTPZ?I)b9zpm_eIU*t8I0%kSmfQDN<|5+|qIsjXPyzM@zxG`^A{IVV%7tQA1amjM7T?IMp2LpOw%AKTqdRR}OQ0q;vk8jK=qv*sq zpTb=%B)n9keVH_^9m3{+q^=*Xi1Z+NLGk3fzKWKB9LR4w!;aQtANy%tk=;7#-oU`W zqrp6h0W$pXNmW~rIa-1zi1II;NK4wCo{;Gx=XEUE?T+fvpF4C`|M=&CNq%?p`(m^I zrucc6f6}PmX!YRU(RWt)tEb}J0(=@GfeDty5`8^}BY!d-?&a$X8kw{Q7YbtcGU&v^ZOlF93-niyj&P+t6MwFyA-YRi0Q&*;w2Ng(?Ae&JF zjUKuB;o>sueVqMn+&=eVdR_c+!gl0Ss~@H#$|Uwa4?s4saT?{Aa^>9>8P21k~#earQU zY5-uEBDk^bkHtGtc6im`zjkF!FN+iGJ=7=%Dzjw{X!|*!Aoagc0qj z^y=HhU;BL0vVy_u@kW2s3dqSjCZ9DAbEG_EI+TbuzHX?W+(m1_@NI2&W?tA#3@Z3# z=Naj^^WN-EneS`gnDLmMiM`KO2(?pf%~0RU70Xsuw?j-dZ23Y{?KbPT z`*p!vC6i&dv?xige=kctKC|U_e1g2^Nt2>P-eK=YE^EZo6Pj~7`9bH)r^&*#g7iBn z1PFZ(7&(eMh!bi7w6IPRbL~;=V(s#QU*_`#56O#MBvcK{VS-j{l$?1+?8w#$`OKe?6Tjd;9A0gzdHpb8~EAdi%$1uYI0B>#jyB@{S3Cqneav zF*jFkrXO%pJ~LqW$&#!81$F-Z&K&$0-q3m&>eAtDG1rGvtg*OWDX6~&+*XcHg{&hE zF`YT>hcTJ9izktrbk3H&Iny15ERG24QuBG5eSU*NRRwM-ZwgPoS zzt45W5DaS%(+acv^ONpK|4KTBYF;xY+*u>>YgkuS{}40M*wx{QM(kU+>1=7*#NG#6EWl=V! zG6!Y=+pNHJt`*DYDt{Cp8x!AnHX@pMQL?;kJcIA(fCbfFQd^wHkz)G?4y^;g7NL_K zZQ`lI#DgZ$ZzYPr=$a>Q9?MrGGC!RKmg1xWiy2PJrpG_)cdKl38N4>%6qL8v;`31A zzYJJ=#HVpeg~_kP(oC(emQvS{_2oCZ%J%6Pn?rro!%hPqWgR_V*wu%{H_UxDtX~<( z3HH~xg8cCA!{{!r-&nfI=KoM@;u#m9Tx&a10H-|QXto5JwO*0UwWAttREU|orKjUR zWnNCYz(dKGantRvCZNvusaVMd1>^KF8fbED`-}R4}R1U$r&b5z- zmRRHhTG0=~h-f2cPGdx+dqX}PSnWr^n1fdFzJ0*7-8#ZI?nS@;ke=U1FlOr#pdzfF zBZ*{L7HW8%A|3#Vw9sg&GIe(AD&LZ?_t(Id@tZ)&{s5Q8YiX(A{6t8bZp2Xc}e}2Zn!-=U8?4 zH~T{5=xaC1;2SkG!Q3l1sZT}e8L-8xhe#>9B+d8{G@4^uTw@t&?uQgH^{xXI^bTg- zMAxJ~aqnDRE@2Lb5pz2&M)}xTFu{oXKqJp{gCod@!o%%y9niLU-?-#yvcg6vNOCLG z6)N#3o`Sf`6U5Tecf5KkA#C}cMK`MOmaDj7R)$0?mA224fotrOe=+X-^p4$sR!R-w zYjz(!M#{Xl_wq0EKV)@yX&tc_gDNv=tx0+rb(}{}`|;7JTyzjLXrMR$1&U9vCY4XO z?iK8bN8(sWriLAlY>W>lS!8ohhwg&)YXikGL|C9-x^w79&d;N_w34Age+yl+R;1N; zpGg;iIN~aKy}l=z2S=ZI`)R*&i?M&}+}LrszOjRey;%Pxl68T#Qn&~Vn1`h0tylE1 zQ5!|*b#rijpb-zD1K&1|minbpN#MQ@rs`*J2WJNrM;!tPPw9ymiRjlM&)%eDM#PH* z+l=xgb~jE=YDDoQy4#O@7J-qwL{9(Ffr@+RTYcoz7|Y;GQX0;?s&aoIcP&;+vqZxn z3Z-R^-B|G-{$KT|>1#qF@IzVyIns!THztkZ`-k_FIG=UJ!WWHGqMt#?hanQbdX76jQk3dcm-`S7 zbk?~)U>6;~k;lAD;7lYvy$7Q}D!@v`J)bvdfB(<5;KSnkhgS{Ij?UW~$s98Kxj&D< zyS&fG?6bWX0gba-|k~77S%0XEe1;uB>V;9fMue7Th{I{4El#kC zT;&gGloV%)$2K^$tF673k!o{H`b@fBNDYtyw9&2{+`me1yFFi0xE~n`3OWEVd7V%t zz2j(JubR4b{;hvGO_W>L6@n`boguckT0rjkZcyd4t)N9Jq`7yaZQ`P#!Ka1H_ruLH^T6I;g7vgT0&%La8NKUxOQd=E#dVP-^cSCml z3TiXO&p}XfT0n{;FR9|^PBnm~bJuD6P!+ji!gc#Q_>BgDk_HSUr~en`7QNm{3jm^V zp%=t94BZd*yKI+W`M?$~*~*vq+S5@QNvEhLYQ`NtWFgc>Ki~T8=jiXQ2dHGF-auVj zo1mWHaP~hy%4ks=+upBc;Q?x197yP%Rubf%%J2oHRFef!0o^fhgc>$@r;}eae)ixK zBoj?eui|ucXG7^If8<=!JwpLAgl*xEP7(SH%#ILiw5#^qlf#Ni+T|C{n>i#niYzcBoBdhVkC zv}{S&=Hbi>JoBrbYEoAu@hV@T(!>x8bG3nt9*X5s-m%M;sTCVrvA@Z>I;82X8Oj@VfI zvr&Xa{T*Z1UnCsUtlY;adu6P|myP!N>oC7Y&o}H25!BmVBj;mTZpZ24JVV+(2B38T<-m2$WRk}a` z$nwO?ic~Q1R9+1MWtHn^S!8KG@+uWRSAIRy1kiojA}8yHs`EXgz;~s}=;!2&qInNB zWImF%wHdXm)kH2k*!L=}Lj-tX@hTH!j$G1K&{gMkp14*02>KBp7%_Ky5>LAbqC?4nM7>2JbUEZ23p0HCelmgrTGN8Es9C(3*pS8me zvfp%7yAOTi9G-&7mQq_U-%)WewbvrEdF^qi3`P@uC0qu%!V!o0 z2NLsA-7ov6S~*nUYt}9Sz2M3r3-e&Lziu!lEPqi1XdF4I!~?lyFVBUMy68|0#uK z#si(Fo&JmTUq(KVy{UfRjn>qV@1;%jn`Pq?1E-N6bjk*A%VVT&=;AL(K_E$U?hk}0 zv0dV^qN!&99D_)EH!=RNB1yBLMLQ9oJJRw%FIT>6C#e=0M}%@-?ECXKwmkBg^10}= zKdD6HA3V(=^JcFEq}j928p~D&VO9F=qRS*<2s~~T!lJAIlra|Zw28V8L?m)e8lGRT zH5H=l#X&~Mhkpw>;c}F6nCQg(sx)H?H^SzmbCrz*A<6~l00zzxKCFHt|~F>tQ)}{IKI$}r>#)jommITm@8Gt#yd{` zpSdYN%-1JmlqZB5<*syc=Aqa@X_~NZ-}zETSFOWn<}Hj|kk^vD%Im35NbAJjb2+D2Y^${&3oDnt7nlz|5S3=@oe%I?mA6#I5}*dfRZ8LU5dJqmm{8qpWD3%kLO zzHE%Ck{Pk;)t?@?(v!(HK~6`JKAG0iePZ3ocmwr1agnbtW0-jjX#U%&CfVyGG@}+! z;wU&KKt4*ST#U({`;DsoEfQ2IHJG5^OUtiTPhol%d%d%CTbGl6)du35ge>~ZUEo5e z5f;~7nKUh{VN26&w^+B!OQl;UxsY?}q&R;Y)AG$~l8-Qx14SD_MJ^2nOjY%&HYRZYMknDDnEg%>xHnT5_M`?H||M7wwq!D!zBboR0nKy+PM}P^W8Nrv-I9cV|)!DTK4%cxy)(g3nv^Qt=81-D)(Kb&$Av2ZgBGT8XzQP@b#Xma-YV z?GH((N?cLs@XNc(T+0RDVh2wWpH1apc+?JP9;zdFZpWUea81f^9Re$vY zox+x*7Mn_S@IxJNEF%zmj|c^@yR(J=v2gu*|J#N0hq*Wsydj~EY$i28ZgpX645f6r z$vbGY44UHu!D!?YF*Acn1TBxKN3+z%9shDv;`{tUbUi>VorXwT>$Ub27S&q3+37Ex zW0+6N=uU?nv(YXGm|y&1lNZYDQY8r&ah{06&&=Fidxdm|C?$Si-U9xX_e+71JKmL~ ze*LA>qARJHZ0MCE9-*|`7RwatiB&vwfb^z^E$7fOv`Y-pMO^hMNA``ERjlYNwfh+t z!|37)N78}D4Kl0cFD@?ZBR=j-j6TQ>a8jdeYbo$8MUh{cikRC92L!YZ#XE7$M#Ruu z&`_D>9v{&TMEU*wwM2ttgOS4wXuy6nT-5}lgedFPv~1Sjo$YE#ZvHbyIYYZaSM{)Ts40*M4J>ntgvVgzTQF#;Dt z$!~#PR~gO~cND+$C$TRyLNvZQ*7C)1BJHI*8ureccGg;m#5I^20Jga4YsLhTrhOEI zxc=izFdNRO)XM)#)l&O3mr%#W$2Ec&Z6oa&avOF?lt7}iHSnjY%q}i@IBK6+RSD8% zxJ}oAugcPu3J`V_V~2dyX+LKPuMr)Mlw6mQa_BQ0)rl+N*E}DC&XX7SAJh0baOFqF zD9r*5?=D}5iRgvp#BC}YsxH>#idUJ7?L+qie;ZUx@HsVDr_pOD-lRF+Kv&Mc z?9VV~094ED>Uo!F7f>9%qc|B>2MVnCIOI|t$%jSasxTi{SQRw01q8hj&RphawLCjy zsl0m#oq{?u+_qB7w|JwNw`=61k>7t-@(MW8DX1!->A|q-mm2xu@@VZ$|ECuBX^-ZV%qay|HVVFpAwtck%KRDwGdK|7!}}qAtnfvdv6gd zGX0uG_Vh!v_z4wiUK**XZj+zjN?hQ3L)PHreqeoSnJR8YG8eQ!&$_;sk~?tqxG z)@R&LZZ$rvd{)HCHumUc~LN7NR@8w=a%1Z_qIn!d2@zoDW)sEZ1&arNoJx0psYv ztLr`11$2sr1D)G++M|(g9HogyBb~$pL#C4v%@uKWU&8E0WumUUq1!}o;?zTa;t*NR zn^Zet%3>Yx*op3tu2eZlN2{Q@Fwr=e_g_KgTIHrxZ`QQMZ{Stxmn zk^}w)!Fqgf_u;g^@9Z~j(fV@3hC=u+W2}OID29?>FqKAyo8e}2)eoBn#X zzQp`AV)Oo5q;{(!hvWd;V2kJNDiKOOuNXnsIaOOk_GZ}a}=zhzc|^9-tWowjOBf%+q^y* zeYHLSCCIQSYQ4Qo#aAe}Dky1=Ia@AN!a+UTT+b4M)!g-7O6Gb87iBu%@uefaoIJ8# zTU#UE_Dcr~uk_(tpvU&VQDqP!3F~{uFZj7R$w9`m;omb&U4y6da+GA3*+Z+^PiB1p z55{6ej`-}iDxDbztv$d54R_{5zZNVFhg<9Snfbh+S0VQHE^{|b0A4LFlM3zcuT&a6 z&Z+_G+ETCik|wAP)ky-67`LUIYZE*=kK^gV4iL`gG2B)aECi%Pin*pdHO-okDl}=b zwh*d&kuf4c>9ocdl}%#Uv}L0vP21Cb-rj2DI&)2k0_9ATtXv%V!V(n|FGvP495Aat zm1~T_Rt^(4Bw(S``m;}it>&R;0kMqkH#_tx`d&?zk)^7ppQ%9dP+Lw@*$-yp zPZB9lV*O!Up{cBPoL>?SJnSLg=)wGa;nc7jKV;8+lW*jD^7tm=DT=y}VytV$rV87& zxzuWo;;};-Syv-%@n>Pm4>euBRn&mP#9t}NM`vBDDR{WyAsKm9MQu{+$vE5pBWX@p zr}dS+IvBN!NobL3I+mwaIijvsbg}x-*j>(yTv9}7jo9gvY0$K-?mlTCJ(eZ4q>p|? zdsnn~a7f&|_Uxra+?Uo2TeG*!#jnyfJ2BHKy0hIxones*Q7jP)16gq;Be6j1)??Qv z8umQ+M#q*EZR@d9@|3*@P}@;qQu;`ev#XJT<*giV_QR(dhm!qy&JDk)W~*tyG_nE6 zh5f<6L(FfM(3;Z48~DkN(#>FG2IJcxyF0(rDtzH&mkoQDz28n}Mwax_ytz4>3JrWs zF&KA<+0(ZkU2j9)+QZKll&;@_A+E=$hLB_GB&ZczHhS?go(;aSXn~OmY{qjzJSTvX z$3{JN?WFgSpl4vTITGr~SE~*Exk<nuBysP z$+A^_IW(&-E8&rU<#Qae_>iC4TR$)AMpaS8{H**ATZ#CH`iuGBEPnE_uWqPd5I_1} ztEjYHAZh)H|Avg<^CA7W%=g%<{Yua*=;1y-udc)l{q5r$J>F!7I|!~8$ommTCFrs! zfXk-?s9soC5awME;opHu&dGU67XulU8PaA>HXmvPb20R<3Z_g5u9U1c&9! zo(9E06YrXt?#wd)RuPksNV${k3Ab&gPq+na-$yIQQs1TPF(nU053XuP`I=1`w$g`A zZ$hwJnBTsklao-EZ@XI`_BL9sI|j_}H!O~99I&02;blAtKn?5f9U?6H6Bzj0>0iQI zMQM~p@pPKCr9SIR7NIY8Dq8!w%=eowNdY>n!Mi=J0Q$lK=~CKf3_{J_MGB%P?%#1R zj^A_peiMuSr=BgFDT8gyBt}7+3x9of5#w&l$>!(@${MD6%vl~&we+7mSD(k_W|13g z=V~aEveQKGV#pZnCt1aL@iAL^H&!*`eq!Av>|(8_Kc2iRIa^MMul}G_C)ka;d~lad z4Pc%&SARR%e6)#{da##Z_yH5F-%^y9Fz8N3-}8yRTQ;S2S=5}yB+IZ{=xa2q|Dx0f zbctDtHAeql^sr5Zd*j2PNPDH5G z8$8hvUX!Yy;d-%)Q@U`o@)x?mg8^xoh4P~O5YFFy7ca+n*G90bL5e>#Pxv_>|B&`b zTJmEz4@Cus6#UI!%W)9c{-TvRko^zLZ0#?#on9BiD9ZlYI%1imv>>SVmnoceb0~y_KyAy1FWIbV^`{U!?a)qJJ0jW z7~`io0ok2% z|2=)?km5TjJ;X0$S}+bmRLitv9yO)*)v+svr#~eHV;Kk9(?+3nZ+7N zDM^>*y-~UjNwo*bAPQwQ0hLx&xfFes4FNtXg7QI%iHCyaCuru=g}Y`h=ypIVb(Ha= zh>}HqKux-X{%kIU8=rN5Dbf&JoI~2sa)u}J^HsMoRBF=4zFq)%XEs%%86>N2FmU7? zyK596F_g5#=V%@(fOqriXYawtefo+84)aN-om|6h5r#w3tI|<8Q=u^#@X1LN9^8C-8>BWW@Kf#*f=*`9{$fuc7wj{E*A`aZs1LFl_ypxHZo zZF{~twKS6?=rcGD_Z^Ky5?kE7TC9^rPTM2Q&$1OR_bp&M-8%6LmFuUIw`F^m zFN$??&7T4kvWz2ZWS#Ny<*7FDp}&BhV#yMvpZmRj9LQa#h_RrQVIoO7szCMfLd}-&*0j;pPxl zDnkN_@{Th~gj=&+f`B0Ei(rvcHB{V2tT!5(aDa4|Kb1nV><34@QOo4fF}i$yUmkD} zE7og82cZFH{q_6zui;|NHpm2u(@L$D_*UY|m}IDUvb1KRx!|Ehx7Qg*tt)OT&W{k= z$a@M=Zk?YV`=_Y4uE?v9FEL(psFbREp4T7Ie%%$%1FnS5dNEXe~1)0MZ8)hB9oDhtT6)>O>kyg>AQsnowQ{z0_WaGS^WF_|ShVMrlp)!*d>o#p zfqRH}JON64S74RN|M!9|wF35&8#i*1_Qz$$17ae8QN>u4ClU(UQ| za`S5|n)azIW@Gr1ZSxq2j!oWm5c10riC-pK0ujfzALV{%$4eIHyGut=&XSt+086h{ z!t`0hx*|NqehR#8B)ZkCT$UQU!{k(=i=7IzC;rMo)-!b}(h^O)_z!byhZ^rZvq1&H ziJ}ZAUdn3gQsp`eHmw|4qBcCaNmeSM#?{U*%!A$M+eUqJeNn@wm(?~MmBpcl)i(Y$ zcl{cDEvK8NwNBH-_i2P1Z?_tbelQ>wD&^Y*Up8)N&BVjz1%n^lHjO5VyKB49hBA{ud3pCAAJ->=My*w zji_owHuas-GBno;4XW&BKt}lqbm;;p-FH*pQhd0mnR0W@_Y1Qr_E5M(eyLAo|D)_I!=l{UzQ2_&DV6S0ltH>dU??e(kVYD$1cojN z3F&SSknV;dMnXEIyJP6?0p`8j&wlQE-}`>Pyx%yE;acmuR-Ws>&hvM&FlGWlhW2na zXR27Uq@y;63R4@QIMkDYFi5ujr>dVGrz^|Cd{Y^ATd^)lrbC(;J*Ejtp_gUzUON8V zfNtsYqQP*3NoDq8K26|W^C%m$I&cYi)HJS>>3_am&)mzB$U%SbY*olM^U9w1aIZOiueX980Ul4lyKh z+ax8LCp_0Jd^=dRcc#Z25+QQUZ9w(eUbi`49@bnbu*RIgY@F7?Shn6GrMGfvT27yD@LxCfN6H-yHB zKRDSGmtRx0@rJyi^etux9zjUu*=p@|T40lWk#7w4lW6YOs}wKha*Qyqg__?qQ@;PG zU1&rATluEw72VYjcuX^YmxHf7=tRnU#G;$)o*-g=oc9>NYVcEeN@?TPtsZVZk^==M z8YLH6yOT>oj@;rsKVA4+{6eSNYTMOQNuhv54$C8#kBzImy$(5viHfe%xp{J}nkjfW z@y3!AG4ORB6vIs1(Xjrf6xD^-EL6Df`{36vvz{fS)UP|qzu5{U{bO9rM^$;niD?3QVpEO>9kBUxl2ix3J<9zzHTt>f3jnLG_^xL{4q3%-=I16lz8h?as zdsI=e8xbtlaZ*p^bL#o1UtqI%ZoV~0Qz1@ME}<9uj_JIKe__v4B{*w--0Di`_GzDt z&w7S(0{eRGJ6?=ohA~~D+D37hY|Y{2j)l%6y`uGqcV)5-8s$&GHEhUJc#YA>9_A~vSGPqJ zKtg$TO@(0cKYb@BxO#~ODXH?F-VfovL|v~exQX;xYH@h%5x$qb`|C0A#3se@y? zRsQm>EXvwr)YMTf>@_9Y0J@~bvwnowkl1!EyQKZbi-SwO+y=;XxVfQW_hT1ttxt<+ ziPxj3r1rRK?IviVA!Th9#&v|NcNs%}@^7$%HcXON`Z$L@tB^XZFPnB?k%DaiG(h|N z+X}uTysx{R#0FQ4?=qN-KNr~TVKeV8^RrnsRg_32-r;o-n6g3l%761H?9t8@)=A}w zf-oPZP4R<6{8)sLS;1{?7}K^_uDX>`XC_y@F;QTR-mR zbUWQNh$6v?76wrXt6_C8r9~-jX=Oxw@|4aL9oG zOdMCU-!Bvw`^o6SEt%UbA*d^X@l1cutzD$Y9NLrfp(2QBF$RVMOj)DD=%GptQbhk1 zEkD1n*L_m-VAaeX5G81EwH~YY<%|(33wi!R0upp6Ii6k0UOqI<3@#$kKuYYf2bjN< z&nJ0uUBgGqKF?Ru+D_z}4n04yyAgZ^0xsIEBrT* zkm#*Ji53ei`f`b z#_KN{`H^xaHZ_~}DYTZ%%kkPzn8qiEolPU|FtpBgzPWUwY^g?92>%NK0063v5phhZ zvo%K>hdMgkK}h;)Bn_V)#}H^-1zdn6oc00L$Fg+bn=sEE$`=y?MyS1 z8#DIL0tgU(C-?IM_05_~_s_F&*k%6?SNgLUCVFHB=ng|`zb3mMgw3vPr0~Cm|Iha< z5udQJKLsbUibI}rt>i9$`HdC&tNP!6u@e0Z1vNDdB>lYwe!tzZ$hWs6Obn%4RYL9Z zZK1zM``;c^dkkQ&fVh>K*>iHu_yI~1J$+nlVB3k^;nEIqM6Rn(ELQBtnfXgoqP z+MHP64(iAp{YEhS?d4VO#qlktDx0W0ZbL5oMZGXI_n~-fIUkiM7h<$p;>Ifmwz#;8 zr9nKZm`hA@BGP|*6!@BKPd}x1<=(^;+!h6Q-*oCT64F)29*KUBp>nX;&@GBfk;KO_ z`OilrJlXlLez9tlx>N0rHNoz_d0grzsFo&#vNbWJQKAQ76#autrsn!c;AKbARDqQj z0+)6)H*sSfa5EUiVJG_Ac>A-py%<`y9D@IgUN#zK05No{}P~_KdN9 z|F?<*2sSe58T7ERS)@W);#tkL(%r*L4c9aX*-8y#pAvf3vBpXNdaNC)TacKx8bZpwr!L|+VMEd{!AYkk~tis8bcVGEpY&B(L4Uo8M zKJx5tyyM$IJ|FkH$tL(XVP&nTT~o`dUc4ayzXpG@9IdG}HJ|!AD;LLCmF--t-jr2q zIrfwD-K@snn+p8%n}Sp#dbqacx%PSK8#A8%o+5qznYWUUfyrU>lB>x~bR^fbSScY^ zRme7x+lICeMZH9QJOcC8ZF)1^f6TEsgNxuTzlwJ}P0}mtuN74=zl4IQ^dQmIt-`VY zfZMS?!y3tzc`(A%azsC3s!TtUOV9A1ox#d_EtP0Xq<4YA2vsxrO-}oFYk;KydYRYb z?J6Yg|Mj4@zO@)QTV6`~KgW!k8o(n9l7$rZ$O3Egzsn2$+6VdyLi+-YNYn0vuOJL6 z-QGw1N56tQPDCv;*XrPk*=e6`zLn>xjl z>CH>gu=wJ#Iv|*hZYSxyPdz}X_aBqsNxT<5)C3?5_Nf}`2zH^K2r34HbUgU1n@k@KRMGVZ|wSw6Cz;__QQhbC!`{Sn!90e0|_>e z!3KEoIeh3aW}z1JgqfA`a;(QW>7b6CMmyXqgbiOSQ_`6fs)^-E~TlbcS?M z1-xyZG@2|)c11RcDD1mX z?zO5_5X~EXJM{=*8N}0+Af_^ zFDt}j%j*PHptpKC;#RwSw&^^(4q=4pl4?r%S&k*$l_c884DOAC<#VtpNV zljeJ0pOr7@xzf>0>~Pm+bb~_595bDS+@saYr#Lg8#YO=4vj;7Qmi-0VgWm2xvMxGk zs6L3eKRPN%42df)*2gcJ&^tc4d`)6@vfVkyKQkH~!vWW}b}XLU9ODy6h^n_>$5a&O}Ib@@#H~ zl(a7@Lpj)>k#*Zs_hZZ@H+|CV-VU z1L8C1YQlYL*GO6ka!*~vFdm>Co1ARGszQoeYu27NFHh}?*`}+Pz57XCOYsX%cZK|R^qISY^a`0z%4qVK1jPI@^LgZ3F=&ndirn}i} zVwX2^_gVPzJB{pquS+pwfr*bEO}A&CC52bf?2U)!rG5{CEhDu4NRjh13g^K*A?u!B zIi>n{%ojiWhuy%q@^OT0rx!w*b4Lu3j?v$}{mp)}DQ0%<_C95;-B27HbIa|HbnDkR z0i+-?kNbt-%M$xo4eH*}0(*G_*1azzV!_@^*7%}lX)Wg$=B-l-{TftzKqT@R_d&B& zD4VutVJHZbPWN&K2!}kUusss?x*D0{y$J|$i)J6o%QU6&yuuO7Y;HY0AOR7nm&L3( zeL|uP-zQil7m&&DTdVsK$Sp?1P{MdCH^4egDZ_^*&XFKzIlf#5G ziBS&%-}^{R1zEH~--pMy2sa`aVZwaLAd~(9c?uIdH*E*$*yrL%$ly|&B9X}3&Py*h z09#_y5zp83VYJYZUCjJ`MqU^9Mhv-xzrSvue=W+3f=L!AH*LnSpLxod{32Fv-JB}dW$MpyK4`p@ zD^{(ypLHDluML{%D+E|4Mw?|v(Asjeri?|Z0)i9Xkdd|W_4hPJ&R%U~YAfwaC+HPt zoeLk(3$5Jp-8PW{3x-r8u$7m(U9RcXSj@Vqz0_7K-ta?LVO#?(*ybBtlcYAzl-LJl zfo9VnBN)7EP@kCwdFScRSQJ24)+LaoOV;#_oMt+61Pb$PbB_)+%y{-zndwD16QcP7 z5h$GZRm0i0qC|I3Gy6x4Rm<;FCoE?tF5L~cU-h8^o6D}L)LTKz85z)Yh4Abz%X&nk zTxD(vII);-ac31vK8#M3;HUWa9yhqbX1@cSHLW{A7E!_yeR}{?3qQ$-pVjRZ3&xy% zcVvBJy|*!y&^hEdum%8?=F&)x_X1~RS1@)oBPJzfcdeEmuCH$}@u+EGxmNXXW#QNm zB|;i-NvYG?Q&t29QV$uIcEX?~ag!yriqzgbunil5*b!s5T3~q?7K%8DWmY zI}S1(=FfXJf@Ys@UheCycM(AsejtMG3fsKt6~sGN7NI*6Zryk?jPIxBA2=AQ8&g92 z&lww|4A~yd4Kg)9)B?IHAh&h)5&wh~en7bKrr6p;d5ySeaaCTA@!qI0ySk?$N)^UK z-m^+t$GN6c)&LHK`+#4#?>1|)4*8fwom*6m6`?WYohAT$e5t4e?1o$h}J7 zF4ifR(;hLM*a-)Xj~8f(@xkH&zpN8pw?1nU&EQS(D_X@-7~n;zJO3Os`UNu;FfjMz ziz>mh05a(GtZdG+p60#g_?rfj%}lQ>cu?+GkK;*0_CK)JPvTox0ULAks!BIj1KHQ5 zdlg{^S;|`i1KSg9wl~q*QG9*cF|$6zs#)z4KVoA z?2-j)bq8KUEN2``?96<)J&NahPK^Ea0t75)5R=Vw_N_tKD(u8lo@?8<84mt3KWKYs z6DaGK%~h!HjcUnVK8d5M1bbM-pC-u1(|M1cShJ-22?4W=mHz6b?6R8u@_9 zW!vsy^G)&322WT}?Q_!hL z|xzS}dkalP)KWX(D zx)Pf^*Yp15a`pe}nb+|}$&=o{AgRuSF1+U{u(|bGSY=4Dx8WLS#&(8h*8_E6JbY<3 z?%bzBt{7IST7K1%?l5BxDjNSdKY~UIj_@Vsa|ic+uQiw(FGd>0vLh8C^CYcVB8b9q zo5cm@lH1WdJHVz2A5D3hJ9D>n>a`S4ey%gO^^?7+fpb5_=Suo=KQie+0yCObW(YdP zV322ku%l>NZ+UhH(;i-9cMuOKNL4wd?$@CX3yv2x*0|E=1eQJC_Z2g%*MmJpW?o9V zmRx+u?mXRNq14J5>%CDI-M2=ODc`5xCH!UZsAyRvu5-wZFr{^m8tjE~njaWhjn!3$ z3w{if8L)2NbS_pCnEMg~1elCDN~Kosr@O#vp66$*(96K?CH3BFx%@n*wrO`Tv1>g~ z2Fz5&#lSPbM8M?1qo}Q5Q*QoPzi=Ncv&?O^I5Z^SVi`AuFXEMLIBD0=tMB2JO69mG z*J_7LD2&MizPmld$3I$|84GMN(WJK?T~EFMqVf>>?BFZL-IwvdmSUF@asO$p&g^E( zPwef0_{Z(WhG)+*w^A3UY)5-@^a+FsSa<4otZXn8>H=*rw9=XNJVVL{cARdTwZjE9 z`}K`W#OEGpG=5CL;jp<;tN+loTxwdyo_WtY4d<$%GEp*Uz>1GqNMvHe$T-|7q;~RBXVu;+Mkk`>S6zw6IB;c4W zE+v+JpS7{_XbBp1$ zv%QF|cDuop`g7wK>O-tY9)N3CD?f;bZ8pjgdbwq1tempij@d4wDP8OkPibI=Ysgq5 zgoF!q5)Taz6j|;ILF%<@DK^h>M)xLL4B|Emd2f5xN&$=A&GN*%5zbb%c#3-m*+Sif7lW^Y-A^LyL5 z)s*GB$3ogGv+PhQGWBAz5x(9VR-OM#!npu0!$1O+ujsL>gtBoe^5yIi;YS`&V_6Du ze+$A3UpZ;7Pf9z|r?)h|C>(U>x8tYp#sb!*&nf#44VRMn2XEcv6PNgRn3TAj>{7;; zdXxbxwy8#Z*v2@@U=Pge^d7M+bnIs!y}!%`_CP#%r9sQ)<61 zS1U$y+b#-bmj+vmW~WaSE_FUqwtTkJ;C>FB%u_VCT7bsZ5^Y3kq?ahpfhG%7LLc&F z|6K$RK2c7Vf}*mj4^@Q~O`OD%73e=xm!bxG8cwYH(EdA^#~_9E#YYyEds^UDq3j0rZ(@Q56eN1W6P78XGfgDXV!A_4WU3VZLrhe|9SsHn9%eC*c=02jCq4K}{=4$o zYcY3K-+&$<_xGx&w0UU>+;OnzYJPbfwp@a!9zW@P=!{nR;;Yg7&a!73YL5xFvq2tu zR+$S!i-@>g!KriP?QQpAa@Z>_u5JP&Su@TcXEWM}d_{cx46Jo)rA^vNo9iWe4R$J|m>53jI)M%UQRaeOZSt1Ta-zl2rz zZ=jPisrP(u7v1aUFdb(2OCB|sWa)olVI|;1kuX9HllhVi+8d)U|D|1PYlr7gobZb~ zqM3-C=P$WDc(kU6y=qb_y+mVwp3{gqz-F(bwXplIo0rZRKlA9NJpWp2b;#rs_9q{q z>-DyFfU8ke+Pt(zx%e9DIfDhJ;+ulG*MEJ%8&#M+m(^Hn=tP++EG!AjkOEO*w{!g0 zYuneYkK0*?r@|4|`s&uu8pp9*P~MwQ#+3~-qiTPZKwr4jaIKXD8tkWJ|KBgd=?Yc& zE5`~rTr%&kX3egO{@*+!1=0WNEH&PL^#%{ozgoI4k6s~B{9nH$2mY%(Q&qyeP}cwL zK(e5}{m_;tPyDYE!e0No-og)K8}sj><9&s8B%7#wQL#GhUmgHh>1U^(lF{2*3v53j zop#XHwfEie`u9|0SKQoeGmzQKB1Dnv`^}PdL5Fc(JL|lxwaV%9odP?&7P>o*kv0`R zYC<1`;0h#;uyV?wzy(*I=a)N-xjgTo&n)3hLSQ$6@^?+r`NYnOFa&K@7;=g2`~_`U~{N=6z*l8h|B)S zFtc(hd|1rrjL~Yk#x&%$Y32ienVA5&YU}wSh(mGQkioe_>YDh(5|_4UXyor@3#^z!ub2qrwL!h!G#REKzW(Tk6>F{)Nn1WV zE=<&erh|UXH5;Ov<_RtEf-j%mm|leYN9@g>_c?Ew)2|Qj3!P&wPQ9E8XT$mWriGzK z$~*PZ{iB(DMS{qk`IU@cU^iZ~Tkc&k3vi)L5`KEqCD!bYwbeo$zj0=)xy2~`61Cfz zo5x;A;;l>7s|Eva_~*;`qK%Zgmq(ZKskT#N;>S=4oOyRP<}(vzxb|j%bagb#5g)Plix04v z4z=&3bJyfnMIfE72vG|b9UAvpnaeM84F{5Q8MQRtSD6`M`4mqQu#RwnoC}7q+Y{Jc zOn=#Nw00q)v(2Li$Gcnf>Zri0;j&?aB;o1HKSwK1^jA*3o>c8kl9{&Q)(g)JUEcNl zOl1E-3(|fGuHbOf7pP<|n7iZ-|Gb+=<9&-C)ExCdr7(VD8&l})y@7i?n?d6wUctB* zXE@Hssp|eo+cWd~1B6etlsxEpyyWTKlif0x@S{6?q%96ef1~?*Mn3d4qT{W=@pbt{ zd+6bn5cb^Kb?wfq6W_cc?EVI3vzhiY_v|fm+mBZY;%BSjd`mWPJ+noO&7VKO-`)p36p+ zS2vy}&fH49uvz(8Y%@CEC~h<}N)sF~CjV7gJ>ba8p;v59Ri@|rSH=iDvEua9S7_N0 zkvA`I(^S;UTkZ{MZSV7KeD=^_1~E_(8p`8!LP>gIY)`L zKErfPXD3!RZQ1Ga<4AHydbm8k%JW9bX z;!Pdfrls)YHp%V;PCY^AZOcq_#DV8Mo7JzYnjfEkL9TgaD`;Gg3aj$^JBDxSenI3S z@?ppinI;@5lr*d42dR{iN5Y=UxbKX0FVf@F->}6e>U$3Z<9sOd3Gvqvw~$kogW)B` zqxj+JjH~1kosd&{qcZ5`Q@bE8GlQ2Lzb9TWxw2F$D3khjR`hl&iA^8c<%CIfPQu`RMjhL${%Rlpj!RQp16b@l>bacIx?q1}Uaig5|=lA&9nW$9?K z-9!2Ew)avuW^Pj1B4^!8HUG;S5m82Ny{!r)F#S6+JiO~SGRV`=)cSHpNPB|Z60D&C zHzS91SE6no*N>8k5NVUCo(YsqCU=;o#yaj5?^gwsGPxK9yzPhk&67Yfk#}*EkQvvz zr-4YM;m_vc)?y)&#D+O6mQMaXaj~8CyZ}X**+F_`aWk$aAIsB)(tX|PUL`w!7V<{nYJGJ!;ar)sqv6K8SQTdV-0yeN|y z(SvCRIIqX2xoUA*LDVVB52}C!QbbO4+;reJ?tVtLo_?Q11rTfMC{PTu2i_OqXW+-^ z)UmJ>&o6yd-}PJ{6uA^RkfV`1Fz`c-*{&&YzCNoiZ(*QP&2>_#)N?x@)IUG&EB3JN za7y(b7QAsQgKaAbxo%M)Zr95%x(<{d5>~c_lf<-#qO5eE8!p^Xh*+)*hJibRw-)RQ z4*TJ8R(4RYVjR2!uY=nS@A)H@SY3-V+>G-JA?m9428bU&dsj^p-A0uJ_&8=Ars`r$ z63tE5DLK-)e6*e)|KQDgoYTIM)56mH{Z4q%=aWI_if^g1`v!V@vKHQ~uu( zlZux*O{=!vRNwX&ApIg2C&h;dM`K1Fn;9tBgdkOZQkS@<#pTYMSvh_8xYgzq1@u-x zrem4JeCF$m+bcj#?Ef<}hGo=ClbFNk!mSAEyO9j{&Dxj-h6{kRtq!B^4^zOvJj4(~bUB)c z%9p4nX)^fDhv|lTCMhjH=X&aTaOn^=94mh_uHG2FKTBKBl9_`oI>6RzJi7lj@E#|3PyOh&jnKUP2hzf1=f^4@8TZm<9e24(EJ z@d)}h&z{0*r&#LEX7%z+SIMr^jCU~RlG?YASq!R-yM4~)?{nWTI^w-H=V@fQR07?A zk0{lszm*5GaJXJY!-bva*;R2J7jwm_2dY)GBFK)eV#}{;+Oz7WFM1w%cumRF-Bpr! zef@BNw_jrnPpxwwR1Xwrb99K1N73todtuHW1^| zuS_CR<(eP`q>wdP>8Q+C&=6$PcE|PMIiD-_&{CFa_A>BpjtiZKnikLiX1aZUSBFPNeehfTZ1``S(mmu}$~VDa3uf*g z7+x9bcM-AOzT9ftc6#h_hVRq*WAI(kcmf&>qq7???c3dG*9`lD$e(^wY0sY+Le&n= zjPex{fymeRBxX8BgAQrM3Og8Liq`FZM0C+*bkQ5GN(cFJW=iBiKG7FK6-?{_yjHkt4Ew`cJ7i+r<=XH9?uu(n2fmo_PWnAS)@6&o zt)e<^6S1cX&~hy&#oHHhst1UPsO)sZDexz={pHPeV#pJNi_HVCi>}m6S2FGiHb!5* zdDncFI9n*{+_NB=^-}zpyFN{cRm;GH#cyn3$E zsC~*}$oy1oINC6U?HYe4JZvVkRHyvD)y@JM6|BmuBr)LGq}Ka^==R#r(GD8*$$+La z`SNE|_m3QM%c)cv42hywt}s2nx=-_B9vc(*do7pvZr4Iaf*eE@l{T$w;xO~YCtcku z%gF@To=bai0@W+fIT5p_rP;%WsC2iZ{f8dvYA0K-UXyZRaC9x4^DV5p&>R+WK0Bw6e+gT~ADfoL?G3ziAZ^d>DLF(^n6S zfdL&mnj_<$1-C_^!kJe-7>2(GkbrhSU_`-;W7=~jn=(dv+%kR}t9hZBe`=zsd*Q5IeU zpM&KIJkifw9Onb{cB5S|gFO~OtXeaSA+Dr&F#LEJS6hPXTX#hIRc|p9scu*67CMAp zFx)CS2Or3_ajtQxk8 z9?L!=O4_jbd^~MYo3|8-F@YAEtz-Hj`Uue9Pi;!sh++K!snFzTGaH7AqbT8r+sN zeRtulJYR-QR1DKq>sEZSX=%#+XaY;oFDNI6s z5H`S!(aU0l$`I($P5Nyt12s-lj|7t$EC)aEk`ExcOnnLA_g+xvbGQDTWs-P)eY%1@ zGy0RGZ6^WzHyG%BCcI1|Nm2Xp#BKHr>3mK+$Wubgb-pR*-7jI45P8 zxXcCqBl{&h&b|ijp95+j^_7+)q{lsOzav)7i($wz|IVPgjE5D#VG~!B^$XWXquObJ^z z0nFRwc>47ch^?x9gq**#p$HBQ?!~}v#;{5wk#lporc>?`hC0Ez3q*HJW#HzC*lpC7 zg0g;ffYrI$`okp4_qQvk{%}xnO}yu z2Tc%SnDC*yixzm9xOtmt&Q zl9zhuOvU4IFiJb+-Rb!8+g~hJSe3Mf+$A z7Z=-`uTJd7FI5gDFsw@B%f4rkNSd>n-h4>{`M(%}7zR9H(9Lk& zQJ=e69uqAk3iMCxpX#$8`|=&|H#$5=4K)e*h%I<8=YWKWv$0RH%&eIc$r7_WkckP-xWFl1_DoZG8-CwtH30E5@zSn{5OwQ$3XZJnG>cQkL>p!={48UfnClCUV z`E}37;EfG#n=^XUcG9JE+#Hs{DDhA59=83cG3#_tk3!9lK&Ow+181BP@2YY6K0;P} zM0utM*CKq4DcFg77*6Cgz!ib7m?$pU^*sqBa$m9CUchq@bQQg)n(X&#S#uz%f)iT> zyn)p#qRuEPHJ{IjOt-oHu65x^`_&CU3A{H95BniWYYZ)cuCwwTCz0<#RS$9LF2wZR(@jCb+DN{g6d?F|-GD3R`cEO4BT(zV&{_w8v40&N?Yor0ZIRn9L z`_aQOBM=19)~`g*8&0`Ke7m` zQ>+)2;K`bcU2@rMN|ZauNdC$iK7I8^m@uh7-S~qo9LOK(v$*6rMnc?r?av))a8Dq( zPfwK3Pz7FS`@g}3JnSA3K;GgUx|-idGc0*bJh<%is)!QM_HDB^WDyz;U1|HkjXs){ z$S*b=As+B(&H>(0rqi=u`00bRuL%`;*0uy(=n4M%5M&@S`PD-C;AiFm`aBkfO~^qG zUD7+;>d$vt1L?ZyWN1AhJ{az^A0Y0GY}Qj=VL4;t1f|8Y%N_>M5n%$(gz5ezqdAdiqHL+@A^fM6g_fp9q zQbHd=SVE@%^pR-!LPc+JLA&KkE+B1nhEdJt}Qq7R&pZ=XFGF=_t%%!>3i9g}!CG{jtZq(1g(C{AJRvymzA5yvM-Y<4a`(8J}TbX6&2f@vHx_ z((#@cLDkH1<+pkC9E5vuWbQrSH_ykuY^Pq40h-kML#(#8 z6bi;nND70|GxbL@U+oX5gj`tiT6bpW#Ed^aCsHQz%)@)F`*u58P>;BGmROmyWg0fK zykabp%y>*qNhwp?#U)PEcg~@%jmGiKBo$0<-~RXkV;R<%X@eMG4Zn?V8)l5djC4V% zl={*^VduT4{sQyG=6EhwhKhKS1mi$(kvx0TqzvK+mv7M_lBJ_VTInnx?mMynXOG#_ z6+#k%u&oY=AvurqGn>VeD7)@3Aph3XL*Jv8ZYC!cu+gwPgWAG}BU;7|8^#kg?SZuF z&Z#Iv18EbES!_c2TD;xpqallLSu6IZrI!ucoMyKWF=kJ(5)#QK*~A|#sUT9Pus-?4 zKn@inDC!|bC8R^q#U1U5J%cE|ozIXI(J49%29K)e$V-yEeJ4Sa0t(xd#H0-rc>nB| z0LiHcgL4b+Js>E%EDC9!3|WY=!){w6BXBc|zvE36 z%5-;$k&PKGF;FL#P!P8BGzHX{1hSh6Ar@=;5clULO>l|=a9Cdfeh4R0{SoNy@5`o^ zYYdI5R!kC}6AJ4!=$+~5?wRNrgbx_(NO$zfmsRn+w}zxkoIg+%9qCohQ2Zj=?Wo87 zP%xnA(}^zR*T|aKY)<69!dc}Xe|-K!4lJ(LvOGl>srY`otx};w%67Qsy_)j)qZ20w zcW#2+_HV`opKZrWOuywYr;R>9ZmQ2K&<%_mWNo*r^<-`PWyqEI9v6vGsM8qg%k$vG zPlQ;e?zoi=J~mftq7=KYhF6mksFe!e4Oofxgn6vqYmpH8V=)A=#>x2UUaXV&oYr!< zQJ>hOOGo?U(D7fG76r!O21fmYv{`)97E22ZFZ>KR;~y`#A`euHD+{15XY9J$@Re&N zPoU_7d6eNZKt{fKEy#`5eKf$gSfTQHqDw&~JtrPiu}Cl*V3jiy zOcu|7JMda&RnZA+EV5C_>rT?3DbD4`q0aL$Cf7-xHbi!j)kutdUlORq9DF{}%kJ%1 zOzI=H3U!t{*Vj_?# zdo~kgWYv)_v*_3BtAIQl1~&;pupfB}C`(+@oI8XAYdulCN-7?kAtDf$W;YLm-v3Th z=DQ>u`~z#H)VsH6iIf8v+8^flB3OKxe5DyHH5tR#_FeGZO>vEaT^U9jFk&1ce7081 z7f5BjoWWFPn8QKhQqR3Ly}&Vp*(pBAd?Z`~WyLq@OcOOb@hjshWts01f4jIKWFAie@3bm#qP%udswmSdm(~=|e*RFwi80e1? zwF7oK(yvAX3)UrMD>PXS3x)G=F5k@S6`=cRkA-6Wd4P7bRY(*O{e>NiMkTQ*l@V_Z zeRZd90j=n92c&>uPE)o`w22GIPArRYNz_=Go{#_TgFFi9pj&~SkV0WPI0F*z1#QnU z>36Aky!4}&q#4@V;!3@(xPq%01zWV`jQIE6r=SxhAn6ghXeIk@`G@5Xqy#wVIF%W_ zGBh;o9C}-CWrRL?p9~8f%x_*wqsbcmp+O8J50AoE$WhJo(kRnEw2BX~dX&xn>n&tc zRZg-NH}A;=Z|6p=mw{_7&TQb0w~@>j)JxftsWRAznDJ+F|FHygH;0cuMHo{-7HCGk z&wqU8$TK&7OXYSBrQI}iLOV9JNyzE;KJi#inhzW-L4G4_Lr@%C{%}vj8r#U=;?yIW zkPY$_Qte@49d&G(r5^hLIigO1*V~Kug#gWS)H=en#XEU!_-cM=*^49neUaI+exdUX zzpKm^HS5AiWQ}|cAlmKO$3AH3cXNA$p3YG$38Wa)h27rvBTt?$CV}t=9Z%UVq4&9L zoK&2G$!|z%`29DCcEOuX36gdM?`Lr@xh(s1monR|SAKNfzE*yx5^FG>HE?&@B>sgM zWv>iBg02o&Vdrt;^*S%M&0Vf%C2E-RNY;e6ZUHMDda(uXVu7KMs#)+uKsM?8YzDPM zPUNaW`9q?#ziljgUN!l#jp<_=|Mj~!<*eVh(_*Y+VD91)!X}*!OFVkA4gwJ$QKZj* zSyCss$L_^`Rs{_Ta;z~A9yyuFPL}KXsD@Z}j4NPL5gjOtA&On5jXA%gPs=2sg|LQx zH*S6#!ePG1sYA#J0i?A;7AFOcFFo11X(rLnF*U(a=8V^yWP}GKp*}V2#>qEAX%o-(QU0Vvqz$G!MPo?QKwS;C z9B&{|VA^!(g3w7P;f3ilNb8!%g!YZnv!ikas^|`Ig$HNPnrTGc(m!*31V$Ny+5=<% zq3`z8sQL%m^@0LJZtAHG{d`oL&)-u}Glzo>p&q?h0hGDxY74IfqcsQhD%$nu&a*;a zzFBKzK3*4XMBo27U4SLZM8Wa;jP7#>nv|Q2Ldy?CTb!}J4JsavqW9?n4B8ubBu(%k ztuik(G-dSwq=`>?TRaG4x9Pti`x#`B#$#uhjDS;dF7cQ>didnQTk5edUIkHt-m8hP zOPXyJyb@WRfA;$J6oh&C9p<1xs8{u#>2_ zvUL_q{W1gRnd0uqWe>7XFeJ5mDD zTSD(unsgA5-V`JhrT5-@?t3nHF=YF&z`+!=9y<^ z7m8!Sr1Z|SeCo8HU{t&n`gqqagbX%FA>Y*FEZ#tLlI_34M6NWSX zu7Pwo-dd)IcIPvmpgym`;cqB;Xv;j~Hj$D);w7Qmk3GcY?87W^zux!+5RepJh$d1V z{`@+A7JEjL?~FG)KqBVj8(&$dJRNoK=lD? z3%g|@UKnGbPd(GGizqLUt|_3gVTs4c3No^eBfRPK`n3Wtrc_FnQ37C|WhMBVm0 zJ-{q`>SS3$vFPf&>+!HB;LW}(495pg-8@S02CVMoK{1{Rw^riZINZJWv*(F%NKoNV zOBgSYlqQbkjnukvv-3eG<85M!m5?ow-LDB~JNiWJM~I1@9d5eD(|A~jfHNZ-@9{bh z{es@CPD{tCu-%hblVYmmjii%C|5O=s0^fqr7a>&6pRyY-x7)2OBDG-n^OckS5kgN+ zo>bf-Z^3S&;Q-^)JZbne_#8X%6)}xb!+gYSVZSZS;{cq4k;iwORa4HyA57ed)bo{n zYwF$Ecp#S?&&r6;1~h-S)on%SU?QlU#SLnXWc6GMcip7b+m4z_wNLqvDw}VLPvbvN zrIh}z-+U1D(KbAUb#ZdxM4I@~i`Vb1h4`8JTVCAD3~Uu6%OFa+?89Jdc5Auya{HP@ z-Ps+lS4A>+B&OJ`@tfjkLmH}Gnse)(8u`x)uyej|r+0e&9rG3ZWBL|}%JJJr zStK%kg??E^jBH&S52+=7WG6H_E%4t8w*qY$rS?(d%az!^YJ3yR{2l9_k3TCi(tjVez6DmThEz6n$hd#xS)4EtM>0v{@C<7p$;={{<*D0*8`s zFA5PXEkdrEy&ElSIkx+*0aw0IHq&w94brDdRz^n(^hU+KN-`Xux1%$)Q>>Gs)9Xc; z{r&ZZyP10#Q;D|h^1;*hWrT_P{6+3gl^A;MH1mcn_oG4kt)cGxBYOxB)(J_QudIX3 z)vmVs@JIgCik|rLks}=O+sUUSM9Y^XvMnF{MzYx#1ld$rKXtUe9LD6KH&+|d?R(FW zW1yJ&P*H}MabVC{E7o)*OA%3QFzmwKsw6m=8Y0sQrrtMcsu?QMH%k4 z4dCgJDd8 z;i{|&xM5m?@n_FtVC{@52X0;!3$Z|t4p=;n7|H_$>S^xW_D~AeNqo{Z68LX8*jVvJqrAWJmhw^)^>x0G;J>G+46@rA8YWy_KCroZ ziE&T-NTPcFl9e`2G9>nX(4DTK*Wrn)8K{LyXRoBF<#+|dqr{S@Z`uHRNd{zTsPDxp z^;0D!e&wB%e(bPB*S9-pXSYjeyBc;^zd?d$(VM4nT6Ey6pe1`!;sLwgRDl8^baCW` zU{LsrsAl_j6Igq#5)~inxMbCrp4?#TLg`LjCot7hxF0|lxl*wdz;YAc`rSE4!=r`b z<*%tb@rq$ZE{nqO#mX1laFf?9AD}Hk2+3*3$6)~hf*2pps$b%l-#r!?Ie*%sgu=%w zEB=T$j1ShTp)D^|Hqog&HGY&plI*9eNJ8F^!hy5sRNtt_nU*8*Z`qL&ss|yAL1yA6 zkM$J&{t3ROr%A`|YK64Wmp?{eQ>)<}u-HKNpFa?nTb`C7ICCq|$yuG(5(Xu>QS866 zsR)cRc15^nTGR^UXG+E+rM>(jL}+;_`lWG?T%9Q>=BEi&Sf1^Ntv9>xIh#@pG}>xO zlcr>VtsUzI z_csdD?H>X}vr4Z950WPy;f~#Odeks#0XnBU{#m> z;C_!<`&wiPUc0+`ZxWj!`lg%dKxV>yEA}V1`6-c7AF7mjC_bI=SEd@ROjoY3a#KS9VCKCY`T<)9X11i&BvJ!`*I%G=$TmlB!sm@(#h?VJd;!I$Dz*Qqv3ekeC2t`Y z=sC>+F&32_IDTl@E>S#kFQOi+el0{&UzwO@XR+mJpYc#Ee827eKzE)DW5LV@!9yB_?KsOhoXAFdOf^VAberzxC{zB<)Fu&+2_?+y zU_CuJ=xV?{O(_e~^n*vUS4sPH=CwZC!S#CMZ1_(~cwY=Q7dT(3tg7)( zKK}f@BEJNqDq$WKMXuG@Nmz4}h*Eq`t)zL%79{FtLSp^npA6=aSHnPaBJ^z$lYg+8 zMxNNXtn{#6|uzuvkC*=ZImjQM zF?;gM7oqbbhQE{C+3))YJHrYLV^7}gxrhf=TnG4`+WDWf;DeGU~fK+k(@eLif{um%*BA`8gO6Sg> zZ~RXW)MvNO{Ko^m`1N;nf8qK+9{xqfz#s+>^?u4;G$7=^64wEY`%cVy zRAJRNLU=DQ6UYI#(%)Q64@)p1wu0tJEfn_=|Md9S|kkr`A@E_w3+>EM!X*?wPQ2_T1OX&5MzKvFqo7_ft&{tBGWMV9YfvTt^29 zuYj4)vFn5ouvA9VKXgm25`gcWmdLxMJ))D*Sf-C0xoKE#%%|&Sb}kgWzqn_befr9a z)nYbB-qZE5yxZaz43WLS52qaFqZ)gGk2Du)hG~mG*A8yq-#i>}9xX8x3B9U)#u6mQ z(c+sV43lyhcx*yYnp^}t*>Hin)HuqF&ybq#kdQ%eaJM@8Jf3INApzm6V=rUD$Db%2 zODela+pQeXv;=%Df!^a`+5}cXcA}xBy@+*!Kxsdx`>9R_RV`7>Fj42un9~ls&Sxf; z?BKMBYn4=iCmAeWpZ>9Ud+nD-jSV z9c$8b+Mz5F-VVI|gq>(n!EF)v8M(4J>=oMI2i9p9C=L_lM~~+f1!AxRxP;PE#)Ag& zfr5d(4!bwu4s@ZCMj83`Yq zI@yWRc7Y)W`o;l&=t9D!{SKYnfDb=bqNZ#`KhvH;aJIa55iRfp>^M_^G))W`ThdQ~ zZyT|u3C#<&W$cyFgTgU8B|sLVr%4NkAZPUpnxsp;NNJ%%q{^ z1UynCr6nIbSWh8Cij#UT(vb1h0(!o0qn}fTgeo3;eGfhES3l611UH5 zLAz`+lX%|B(C_{4b-5fHZ4x=`^GZ8d_WLKo6o!D%0@nBTo$V?M(C*m(6NKr->)lxp zP|8t@!YzDQ%u<$ioIC@`R>}7&wa|nN8-Wh>T~El&UNE$)(oUEEP4STqX8@XX=gI&0 z+>OHV^;XgD$CKq{2p<}UAE$E1htKn|yubU5=D-_*s*j7t9o+!}!;g0;JU3JSyY9xt zMl2x1ni0H9L6BzQSlA7e+to}xyyrzQl7)D*aJv?3Ler9>Ku+l5n|sj?4xig8-&~B@ zlrm8XH=wQmP{0pyK>KP^;1XDkXk|k$FhFnC_sW2)7!GuG40}Y2R7^I$-_JFHUO*hW zZU&5B+kY;|pSlVteIUK2wBLRt^Ij*%u%8omH&Y1))}`6_jFmq^t@a$Zu>Y>Rd2noA zq33BgrbBOO`DC5Djt0WQ-EEF<+off8&bB21IH$$zUC$i~!C9HaBel6L9X3@ov(KR$ zdNRoely%%quF@=y$pz3C!uH3f>bxe^#)n_gspx;&j zK;ttq2hF{^LiQsjImx&t0dA6JI?VYv6US__l#F{4U|}$`f#bx#>&I$yz1w{%L=5Zx z{zA%)B9;rutLdTZs?)#1LOf=zkMNnF26i5-pW6$d?;AYf;($=i_|RhffL_VKuHaIG z1YU^2{#W(ywY#PlCK9PsAK{tN1mL-;Gp`z-txsfRBBx5nwcXai@G_`BQfN0ik@5Qx zK?=r_A18SKIxc`=`wj;%t9w*J=PunACn__-b0wV|{-&A?!K5tXLGhCR%SLUGwoc&v z21s6V@Ok_&u69G(t5#cJd;{lu;Zku93SKUMtvy|=K3>yjIj>?Khg9VDR5v-j;RH|k zhphwJp~vS?^N<{Mk*(jYO?Aqr8E@6QdgkX5o(+HXygxQ!r8FOV`_hymz+lb!vZ#b0RJP=(niB6pkT)GuR{Z`#EXm+cq;bx2(hLcc~7#5@;Hd ze!G?auHF#L-FRnSYKG3cOlhJ|qa2x47Io9+T4}P0MyaVWddy#tC1%6gdx2haxg-6r z(vZukUQtAcuKDu`BBMjw^5<`d`c78k;WX(aY-!N z!>7-8>TJbHuR1zeBo*T0qXSbFKo|b@JCQDVnS-7LR#q7UdHQaO9nvmnNIb1G* z;aFXCdAk~wytukjEDxl+)HcOvVX#jGe2+(TxExLJ zbnj?td<+=r9~K!Nbh4$%M$EOTg}f!nBhzQn3lnD0s@>lL!hw>X;QIvB3t24UAsLC| z6|xpD=Nxo}dAy&^*q@RT*|pq7eWlci{!f+=pw<(Ca$QSGrRKnTn?jmUHIJ&UF{Gwx za=TqAul;j`r#aoDVcHZ38&Q{3SquFz0J~jRp>#baFSM*XQ>)%gK{^1LpQ#vTIOH-d zI?s4atFDnrlWXSk-R^}rPy=$?b#LXQX4tA@*VNU|-KwC-~_OQPV%&F_}&X8~caX zl8d(eg9)I$-#xDqz%8PMb_98Xb#-9p9%sltQ6$=R`V4|V2RKprZ{K)Gyb|9m zYd!8_a^2cp*g;0g$xeC6U2Wi~J(dM}IKBgzBD?)3NK}*5q0U_CI3Tl)Hk|q=@N%IY zTaePW=VD8~13>4%y!w<7%Mm93NZ`S(bREwNA&YIT`?tP~yjR2gPyQ^pD^e-Hu#{q8 zFmIyzvBFA`X9n2bsg&8nBF~;=h@pd!0MPc>(7#%9%z(4A(oL6)E>oMD1i@(!UD>>= zq^z`jP4+mZ@6&iPpLVr6g;E8}bj4NY0ek#+Tw~?zb?+~1y8958+EjjM3g;by0;VxA zqD^X?@cd*a-fA^qh*)ZyGMvA0Y1`BT)gV4X__;q)yQNrajpYO`64aXI_lg{M zZdbc^zdT0Zn8Ckh+alKnsi3D>fn$$Ojb&7NJEQ8=^Xq)1@_bI!t$^lvfSswwiU*Bi z{cCR@+@(mL8F3UhHLO}&*b!4-)Sz1HTo_ts_BFK1#@AD5DTP6GB;SXOMOrMz#f^*l zmo-f&6I;9xUXu!N9V)$DBmbUno2@wPXLF8KCTPl+;yN6P=h4P`wwtbB25FsAq-j0H zLaipEwyHMEC!8CP9`12EO}s@d2G>_w57mwfb6bjsq%6ZpOdTeP$dI)scQu1mq2Hag zzYy84EfTRkGwQT6$^`0OJuZNPs>yaLPmA(tmsE5`XyOawLg`pD?|<#hfiiS^W9~Nd z!n?+a)iVddxz^wK&1WioMzhB!dgE&PppYh9edtjlF??Bu@cU<1O(8mZZoVb@KV^|w zl?dX*{oA1iQ6dNzs5EjsBkxd--(`&>??+BU6S;%{R+qflTwX;Y+H(cUg2Q%v-V6a1 z!aBY-10M&uSl?qu1O9DQVN)RqD8cFyZY)jP>3Kf zR##D7TaO3;M(z7=MxELCwsS|rKuRacQo#6dzZ86$IQ)HOT~?P-lT{ttrKFT0k$z%h zCK&qTNdLQ`^_N6;`%Pmta>M`2xKqWVYRhRGxB4K@U+T_-I>j@aaWGs;uo<6Wd#YRT z;?SIo(;;iA}(SZ-#TY?T0umJ`!T=0klWxEAr2?$e}%r}ze!HKU+&klcMc-qH@ss3T9;x0xBHx^>_mJ>+B# z>+;>%v2W>_83pZa6M3L0V)%C~SmhbJ&5V_~s`9%R_MJ3EG%0q18NuNa!fpu&dSBTM zE`{2~s0680Tyz|Yb}xQLzv2`Kw9os`@VUe8%-x{s>uXr|4q*bwC#TEK`A1+Z1eMb-4Of5^(lQ5{SB@izB}pwL<35d~|HZdHK?aQ-)k z;ns84S$5RvQq&j>jEKD1koF0(IcfY6h_ATIR`+om1rU>(mOJG$mdJ8+;AhB`S8#!< zFM^lzn*afeY_%1w=O_HiANwjdd-pXoJruCe1wdl58^;j;)B88}Ofycuwk{2qrMJ`D zCrSj#=~lDDeHRa0cdxQ#P<@n{Bhbm~$@IBX<)xV6y=9%{At!Hr_X2|Dir>@q;?xkp z;1$|jI}r~BnJ3qn#ML(!?~YZNdQ2^u5iny1Jzk>0&dmU7&=v`=?J<0(+xwEv$(bss zWKd=B0PT$agf>EX`ZlU zXM5nA-I!(4$pYEA(u!x_0va+Fpi5vku&yNDjvFGQxhF0$bToh>gc!AfKEWy1X)bNg z>SUd7d84@3i z2%fK1$f*DxOL?R9cx7W0=^^WGG-ex~=dDW5I=Pt?q?20$lb`}V;Uzu+6>R$r@IStL zA~#?bgqe%OqI-efS6FsVM##C8c}q#42%Zbs_lE0Y`;Ds5X$nJ26VjM8mDiq3-J-y9 zd~XJNxl|093_i%cO1m5@5W`7@eTQ8lMafg&xD^pCzRXBoj7r(KpE|;S4*9qqslF>{ zSoI1(VgQVUR4*;;zyyFy(#r@#<-l6?@tlw(;$Fs%Q$sQS#M| zd*`s_vH2c4Q89VEb2WsfxC^}n>L}bPVu8p6g0A)kcxuM4!!J$3h4zX7uY?-4*j+yv ztwzhJ2kPoP%86#FKdlBjr#2sV93zmYg{jb<({%^!Tyio&vos2!%i!Ac4N6Ta@`DX? zs;Rp9X`oTRtnyHa*^*Ky+eVpr|E0t3Ja?JXq@kfVSOsuS$DRc?Ev0j!Q6|9I0{}9O z;wJ#i1bBD17eBmuMs{^P0{!?l-SBXf2D)9gZt-z>IVP-Gy_KGaYO{94a%kE`dUHi7 zntArcxJ&=GK5hcn38#+P)Ep}>b3c!ux;y)dD4z#BDJ|FkNgCeZe zsS?&e97_a=b<}mrQ6vVWwGCk#lGLJEuVORHJ%L~H%nzVP+Z{OYQOku z@$Emk*IoLIBR5#JTx53)nY1-c{A%H@ev%u4yRR3U+2UA2_cf+I8)87uzTvpf#4bjF zd%i&)=cy-u{5vpc=Ce`^ToF#e58YF8suNZ2*HCI2DC*?L_EKD;Xk_tEh*3Na)QJ~) zuFhdxY-!rJ%^NTB@4oheRcb~W{~Pn|wQsM_62e9Qh5a&{SOk*wMaekWzTJYQR4rwtw0R<#k0Fi5i1=Y=xgG z4N)Nf@rtnRHB8jyXFercTIma;@TWT(Tpi=qrc+g;>MW`{>wBgjD~v;;EeNaO-t@+k zL&ONpxw8wm5_nUl5qvS(Hg~7S;~`s@%-<6G4dn_)in*CpA>Luq@-C*%BQh>?Q(V&0 z60dHo$-`Nn3aqK}=9eKuEpE-Tb5V1liU~6W2V-%6y!_~1 zqg&;!_sua^2de6g-9Mq4fYN4U+B?k+wrC&|;#AbC{FG&q07!eqV&qFUs~odf%>)!SSqt)JO3T3Nwsa{9Hg>Uh$nm%%qrN1C${o#G}yM(EGfP{e=H z4>P3&t@jIx6PdMY6?Vx3GaDlesSDTR@u_^(szq8Y%H&syEUqN0&v(r<4-X;cg#7e0 zOF3o<*Zz|>7ga9(H~}o7Oj&vI)QeE){E!#dQc7oIe{a{J^Si3gC+1iQu>Pbvv}YgF z?P1y%4gH@)9eEojD&~D{umz?~LBp$2F`wE60e?}vm7OIFqQs>s1*Hk3+k`r00&UCha|H$gS1_; zJy#H{FcJnJ_Me=_?1o;Zsf^{Sd2;JU_It%E(wBe1C1Yzu_k3{nqFa1^CpN|X=44Z_ z8vOhG0gz}yo$*`&KnUcrDVodi(#E*2#*hoz=KV-bYPopW81!tCi)F$RXuI}sI9?o} zf&38#MTSx!jDyfiu@w2SJ-p84YiO;5-PXH8MVQpbIrKJ&#{W0Cp(#ffG=I&an@Yz5 zIa~IR|G~kQeooYdaiU%Bp_FJf{5J7oo`d+2J(Edjl-a%1n}PNa8#31B{iS(J~!MBr8*ZTGUPcR z0k1ysrQkm;Mb2Gicv8GDvU$@g6_;*VVNpzlT8Z^67kSw}h3?hm+ga>>b#g*p_R=3X z%Cww*9q6V&gk2&e2Cry^g{5HKQLe0M7AYVuT_5ApL-mXE)?djEW|cj^ORk&hdHGEy ziM>5MnFE?f#tJkqa9mp6cBL#H*G6?{lrxvDI{N`JYVz?997|1~HfH{9X$&sUK}mS| zl~lD+GaaARJQEp*U5KTt?TKJDVzyn2s+4KlY^T<9BCqaPwfZUjIFO4gs~Lml8j5%$ zQsB^jfvXs`o*sbVc-QS*%^Y4BjIdzd5B_f2J!ZIFpEE!Ct2m*6Qu?5kG8g$-MWe(L z8k_>0h=6kZF|%<9P@GdcPujuUZEae7=Z@jI>;Hv+5wBO}tArY{jKtqSq!AuKej#Hh z(<@O^+zJ)6MNJf`J$U;KBt|4S^^Ld9zg{7ra21IS>ETRA+w1EJTwMuRi(&ms;zP;I zzXTY4(d$hdy5OplQlO5Ybb7u5;6UWMTRQ?dch}U*O>{cGdVAIa4fN`}i6E;S#s%QW z%H--?kMvigw-IZjjs^BWWp~vH`8zB;4}BlGco1;n!Xl#GK({`*zuvGM>d{~}ZjKMJ z+*0~jIMyAYKsa;ykWaVjV#%3!DxsU(Nu_sUZ@5;24rmQJW|uAgdv@PIsr_scO`3t2 z@!g5cA#+Q1gFBJc=P)wYd!NYHE6*yeHerAxxL*e0+4MInO$&{Xu-SOTz6 zIJXs>W0-3RZ6~{NAY)^usspu9zpxw@ygz9HksBtAzNblTgjHxDSp$BJBG+N&Fj(&L zUXHC>oL(r&iX&4ILn7S-;Gn_hqkVD}w57Op7#DZ4e_ic{Rqx5DUQ}j|8JZg4LEO7$ zolicO_OR@?ZleyVHi#IN%Ym9q5WIXxiVP?!x5Vjhw2q?(N))8xK+SFEBL-h4o66oO zA*K+(vb82OdP0#HH;cxaDGxX8Q!?cAlhPIlgBam^?Ni8JWQT-D( zQk+6F$7q%QVmsLz?R3Gu_tfr`e+$>rQ=@!qszz1ULR?@z0#rGMfqqp?HKl9TgJqsr znw92XSm@U2L=!G9pN?`0cD}o#6g}=?Vw_kFt?IuvW9yPXKS^LMJ#z&hTz|L$wjdee zM;_9gZ6i9$Nd-nY(X?({uG<8u>|4O89EGo`TnERs)!o-eRp|{I0_0R;^s07W42K_4 z@ppxhpEX)kA_ch}n@;Q%tU}qAV3VCup2ciSnXz#kGp@J=>W%hr`>hB3O2)3|@#6qi zK-W9$7_jRY!^Y!?g)=#Gr`<6Uj5$PeA$JAQnlInQF<|o zs`GQEt#)Ui-hzC%_C%Pk;ED-;xprWo=!^2#Fli6t-kPMaJfDkMJ_)#{#K&`WR%!zQ z+AVnW0fusYGe0V^P}{PWoT0m)$`f!~oQc8> zphOHfV8hin%P2-2hJ-=#!*JQ+c~6^eE-|jF2=y_~+272kIqG`^6##}nAGYF@?C>h4 zDu0FfXa5Ip6Z0r$>?8z0WS`Oefja5mCH`be))pcls9AbWkzV-=rOc2RHbcbNZVxhc*_V={6j++UvbrZ<}G zu&4PhD|>h~9rU@X72#<5G0Vbqpr-Ojhod2eU#K5&@79!P_s=HtPJr$3PlpreL_t%G zU@lF6?OD<)H9m3XbyxW1JEsBXqscFp&=p$XKwaVy>?|aTlJ7Vwf3#_?d`zsB@=AFa zvvv?@wC0LP_CX(XbbcjoCu@@MnX^qcPM5y^v{yVld4sfNhkk$&Ag}ap*SJ=vUj5Lp zM@DjvPZ+1Vn`4Sm3~xNi(}}O3j#iV`yB|0-aI=%CIfqVx6pcy~s9wvkAe5P|5J%G* zXvwBzldD1@Bj@+HF3;KO!lX8idC;ZJ7pB=zf2Uq|r>di*Nw`_Dg?@s*Id6+EIW5@j z)pd_@MIpk>DcRzf+5Wc_Q?h24ZZIoIDHj-@53uy*fT=2sFDAyio|zj?F6Az-{g}me z=+g@{uFT~&(xTY-atq)0#X-zjQ?vl;4u}DHf^2+&b%%02*zj?j71L%CyAi&yjK=+m z6bxRB<3$lR&HI=>uT`fC?4KVH^V!tx>Gmt z6BV;aolkDaZdaxI)rc4{zPrT%nQbx>;_0QEWfx@2oqKVaD4)D&%XKh9!`gL~WL2JJE(&;DFbS|RY&)3HtP%-|HZlIlRiYry|y>z-t zwx&<-UWZ-21M2Sc!PVBq6ugJI0!0_jb8Q^@%AXI{1O~9|j;MH=WRp%?0Z<=-@I<#Z zYp1$wrkpBk=7P?eInLD1O zw1iPzOS^P>AJ&$;ZXy$FV&($>BKO6~fH?7FP1<}a zL9o5p93$Jj%2ysK^sZAl(R%I^<4#l-V?ZFl;qXVZd0Mi|0PJ6nfen09En?lfAnx8R zoEzOKcwsk0SNY&)Kj1(zr|i?hy-RB08`S)>u2&y}6F6hRn>TS`ZxS7nkw;)x_g7Q1 zwe)UZCoA4Ocds*eHM$>dj+`i@LSCHUp^hAOW{r`-)hMn9-ENDWu|EBt1t+sh)Adum z<+Gc5j|oZTesvKPvtixY5%aj3WhGuxSm)R6n?Vm3gXR2(Ng|$cGIf;$M{22uXV*sV|GBZ~Twasc3ox4q zKFyD`ldb0+y^;2S1<|8(lc-B&ho&U|_`#jX>Kt?RT;29eoX=yCcyrA5eX^qhMfJ*< ztNZMV?2L=+V}Di`^buVOk7D+*r_kI*)b;ou#chyDAUg~Pn!TnnzO%8~ZuYM~SpSXZ zBniz0{) zO^k6;k_>a-j%Os9KmMW6bZQWk{n6CPjlIQ=n$2RSI{7GNxSjXTPLhcfqC)11a3qa+Frk^ZsJIQhp$su3|JR)Y`k6M_7$2Ynf{;YJdM%H}N-T z!A%%z=(wrPj{~doXE|@<#z<^^O`A8kB$j!%qT-5p?s=Br4oCT|-couHE9e-u)6dn{ zZLGkHfo%SyuN9@wzD{Aw@(a5sJDUPQ;Q3Cn zo=?nO={s?fwr(h?wG-L+JSG9m1fM2fD78XXUxFwZopf_`G;f z*D(kF6Id|bXVTT?@G#UGi?{Gf*N06mxU<)15q#ax?~)G5c&5wCkprz)x54n{2-9guA@9y6=CQ>RHiJ?TArgmk?{{c@KZN}t%UL7Z#Rf5H82@??m4o7rThj(1F+ zZ^uK5DdRS16yhv7^OR#-U_h#r3z_#O23~RaxF}kTBvqP~i2Yu;oJt}{Sq>ALF`SiG zwpyFvaI_!=nW>BBGCBJk*_W4-#v6zI9X{mfA*6Y{??%-)^oA*m~ zucsZ1j1_s42!&X`e9-kLF&J=;f_Ys3!p+YP1jOw34>9XH_y``HPbT@pQa6~MR`qfX3MzDHo!J69 z-NF;APbTxgRP73)~miU%!>@G=ME zK&S2XA!v-gy+peX8N%xz7Yd5E1LCbBmyUP}rAFVqY(*t%pS{y0sbN1Z=NyUww$Mrf zS*kYUC-nd%RS<*Jgf{`O^|{YCD3YFOs|$na%JX>kK+5c!=E;Rk`-Jh%c;NI41o_&u z3?j)k%fPjt_(v*cZ& zxtdk{j9}Go5%L@L5(}yWI>IyvmuzPpfy_rfzj4*ppWpOVH1Um{G`{N;ob`@nR*#Ch zDh}t6o4P8C<`pK?*L}oGhJVm{`RK?U%r!Qq~Sc+{X(}i z;dd>FNn)|Xenl|l4&?DGptaPR9>q4FsNGOUV7Ow(J~j=4SmseDRLFfpp~Pk4~p}-Lz|^GepmB z-&*y5#QXd*e|j5P;NE12I=zMJ`R%xIS-h;j2kAgffrUXZV#)wkm+aQr%nY(8x__Uj zGqryOI5Yl^RFGjKT{cJb0~RkI!&MI#-9kKVk}l|V=Xpxgck$ftv>-E{BB*3gpRbf*}X5`9-7|;84hG5yk!R;vRu|)|!fTF2;Wp0%h=GX! z0dp6MPgHX6SfQPe0@3#>0JPzxG1qwXZclLshvL(Cq8!eH-owsbOy)t%cVQ!Ry?o!k z?p56j77hhIT1@kbc=Ohvp~D?waE_Vg+VP2JMoEvG?2=znYgY08s%9#o4eiU_-|)p{ zlbwTQlWKY*Kph!dECM9>g+->_7I|=+mEj$eqba1@oq%U&ek*n-+n$Qn>daJR82kZj z5&|FIG%>Za4Djt}t@Nu4M`+9QaFrRaJ}3(C(U;3-*1v{FD{W(aPEUV&4 z%JS24SH1$d5`$C;U8-*d?gWYxf?d#K{3ildO0Pr8b!b|RGZo)i-S_i_ z8()3?__WKkH$7t5{eE=PQ<=6czAyplNqR2pHZDsnHK>pbtR|#e%T}2g3{+8^o2> zdiXV!=gWQyCqMIiU$HiYuXb9u7|h3zpEb)bJLb-MeGakgWoo6`x`g1-q3#pH&o6M>(7a3sP{*?rU^jB%xAapEu`_@znEtbuL~UVt|=^|Tz0 zo#~-L-Y|Fz?5~-P+BL|%9;RV6z0Kn=3d1&=P-zy+j)osK=>Z)nKf!Ar*5yBL%Dx=b z(AYc1A72mDBBih8|6zK0-BT8$?MV5$x3d*#%>9>pVho=qlf?YGak7o1Xfs1M`Nl-j zEf3{_7?YLKu%~$I>)$If4l}Sbhk5I#cL+&4bWzumWRgUAV2E{8O1${!fmiCi3x3t| z&hu+A8ULtEDPIo56Y8LPgoIEk6ORrLfA8+rf;Upp)#^|c@+4FLr9rE=-{tdTdm^Kv z@WI{Z%8$9Y19#|e`D@mLnG1uiQZ^LMj__}9@+8t5qXfvAw?jCFiUUEXxIM0HIx;bFru;a{w*_%Z z?&uTDfBY5@3EGWf;$9!_uS>5*-L=^$h}C(~kHJlz`L!mj?&>n%?s&S|Zg>9MA$;yq zE=<|0gWv&?95D-6*9&2~?ifZc(5t}uZ5*13qSO#xQ3*X<3imYDHA3WNu3ene!_3D$ zU?UloN(_A=uPh7{ieKTDg=ej&VM7)rWQe6^VggEd%%+v(UJm^huvsQO#w@>W$#P1; zGq@1fZ_)n+Gg!-ZM|~?@vzNe8L_rO6Q4)DCX|$O>;hD1< zTq1P-M&9m8rln}#oZ(Bqq?9nsTTF9~TyfLy9vK}LV{vIs#fi}%&P^&u9t2!(x(NDY z5o}zTof6k7XrwpGFVwZ(m5!XtT~T&^hsxy~xs;CLSO}hXe`}SrwbZraN@%s=^0Jq} zb?*V{)?lmF=la{fPBBB6tpG6dyeUJD1s!*7+aii$_kfr zJI_8Rym_mE4zSqb0idykIzTUEz6XFrKty1xeUG8{?yWh@a6rG75v!fKqQ40X@pacD zP&C~yk;qCr|>!ed+HV{3nS6Vdr6B0?Q~@xQiM2b zQd)ivvR{!U5`^3X7ret$=a-k_V8!<6jYl~>E(pX2mjG+hzGCOrpco-jTCh0{_!;s0 zLtT6xc_qWD0k2yz&cSZ5%hiNuQK-k>V(QD3O+EbN`fMA`pO3rq>wVlKRD1UMhQrOz zKw-BFu~6~UO|9P39iqzeK1e_jRNh5Z27ySHPjC3`nnCVYSWdwkFuNBT?8O7}A6TW( z2&lAQ8fiSnB|oxI3U=q0w2OaG>aD^Txp3J+b=Q|s&Bd6|n5X+{iBV=9R&#f(^1XIVQ^_+?5T zYbKvZ45nR&2<=bU0?*}CE>Hgempr3T(<^b84mJokZ1&^W8^h0avGipe zi!C;Jm_X8-eLzZ?o8hHa#pC@Q?DF72=mP~eUF^*Lm&$~}toZkFYv#1P$^)>E7nzR% zNmdGb5(0}xRCz#r;x)xY3n4keg&Mt4e%W~kNgR*rSHJJ(^D=p=uPgvG@nNo2izJ>R ziQzo%p;q}{ra~zIbiA;3GRmfJ96_cYfNfUb4ooRFMDN>LQ^Yl?>kO6rp%gnx_L_7Q zn_U<7*Sx)6!Jqe%>!zPbx(B=WJg$~EqX9$!27y{Y+Jv?Qd>uGT%<`LKx_CZqXw&d z!y?1u5i4@FTZ`iJ%xT`WPQk#y^*{r2XpR1dD~iQs?bu$K0%$CS;|?E5_hbEQZIucS z`}!SpgV(MXye-s&J}ea)-a52pW%@rfocjT|2B$K@FYEJ7`0D^rP6HiauVN0RB?w1r z%2SeE)uAtnY}K${A3+-8$zw>G6}Yz6#`nmd9kcnYUB5<4#~rbch5aB%FiIrfFYeZazNx%8ruq#nc>SmLweZNE2{FZDM(I?d?YWPs#^)x0i%pW&j ze|ZB9M_o6a#it!S5XCa&FjQ(BV7G8|Q!UN*rr6KpVF8YIp>k|#7xEQt%H`{&7F?n% z8F3VOA`P)1APOf^LNq>*ZRW8l3nz6LQMbp~M#I&A|KnsbIh_QX}QnnTj z)W1ey4C+y(ixJ-<%?kIfB7dqK9)S7kyuq3>g`j)zwFEV$F)#K69{EmMV0VDVdSZ}9 z_N>+$=i0MRZ<8dy+#bA-57HqVe;aQ%eVKp_S$**q1GRXzL~QWgeGAjaSnK%>;#(}t z;oOSLLr$45GQiY~Z_Re+1v`*k>6dENCM$f+&px&hJ&~Jf5AbLd6dAndHd9tsm{QRs z`2X?s)=^QnUH3TQNQpEpIst`|Q2X`E1_EO05~kLBHmkG3ma=&wnSFR8^<` zLc8{^mIBv@pt&jC=E5kQ^{`J$J4ZaoSl(81VSj8n&RZRO3J^b#7>MC980YGsf5O$0 zr8Q^tFeCHw$Z7;=xOi>=fn)itmyqJyp?i`Jv|GJL4;5Ccx>wXNq5s^T z5*aNl=U#d9h#r6`3lg}Bd_i>mf2Rh3yD@N9>}Lv;VhJpEBWc_=X8@6y(wXhxaX)`(58<8~p}oU-hcyQ)pS6y@sEDnK z@}mWxL$34O2I#IYZcgO2roHF1)QV!Qr>jVOh78W-`DDIJl1SN;x5=?Cda5XsQpU5e z?-GGAx14Xg+Gl7jjt0BglisYI~ylI+d~*BK%ZA`f5?+t^~swepEGX``tLA}*tdB0pS9{CciW&$ zmj8LDP2|>m?6fmvKKNMt1_fh%7V>fp^5yf1Au;t&Z z(XfeUWjUm9VSj;k&USv>gIsWGhV{~;#T0&KXz%CCSkAl^a0AyHLhEnygY+9kGw=_q zL_9L86d=+{C5|q}@;uCUtiLyQHvPuuyiBm@RqI%G5S@_DdBKY8lRBQ_?qK>#6)Lv> z>W!4gUGJvoZrII-OYM3ZLJ|A!-fw0kb_LlISgk)uxQZcR^1h2RaDh3mEX*AJs^`{5 zo_;r@W0ZFu^4Imd(E2dRxq}lV`3KUAi*wVzs!@}bTW0T|L^}lEV+*2{hEElh`ZhqML%xS)|;qU zsR~Qf^*2RP-CQ1PlGi}IIHfxj@A-Z&SKzUG(9CZrO)rulDlZ@Gqfa|kvh$hg`He3l zV;!LHG$zf!(qLegsdCOy!V~KoU!yZI^3^}rV0D4xXtcu+*|ASXqx+g6b zMm1d|DuWZ)^If!0IwxubL|qa~Okqp3sWOL*%pRKH_#~bp%yJ2*_+W+II&7QsFx&*1H742O1>*J?6F81*vuh%;C?LifB`V_8^G7>BAH z8hNq$w~xi@4kjK|pR7Hid*7L8+iQPd5xNXWEY^R-=oH;j*%fzEXF1*OP2!&wwarWWEi*|L#`%SSA*KT8HrEzt!hp@fkoS*#KD78 zqNB2*RgGpooUP}vI`g82=o3c1dm<& zNf+_fq^Ha|N-4D5qTemQ{@{8$rKj#8am(|^!}GGoc9U==_4w~Pd%v#R1vCXc&ULU5 zC_4XXCGEKsq$jO+G%LI+HV%_o8ZX&0xJ?ipA>4_DQE)~pJ#wGb8f`Og%=kVaFto-W zY+{#YjF1%#hY(2m{g%-Gf1kh6wCrJ2R+;30RZy6$Z|zE!JT&m<(Q_M@V|9P*sm?ct zSeXD;&@wj|grnn#;de-Ot^^M##78Bor0pPZglrXrV3p*C|_Z6=dR# zQu-RqFB)j1YZyn~jaf1}bv!~wlb$hLGnKR_{u*qNm+GsHfz{u(^XzR{9SdeZCT)Oqkzxc#*2&n-Q|oI z=1SuU$X~4g4u0@jt`6Sh7FRz>P5e*!v7B`Tq1DGnQFpge=T6S6Q{?Dj+G^&k~< z!^|1UBh;k|)I(~zH_mg)jzbh6b|cgY5WflZd-0QjR4SuFr|r zgN%!Cf8!5-dGG5Lx>nP+6NAoK`#9#z<7Eyz+))Jg`zFMi+pg|L?#9zZhTnZrBHWD;>8c{((GHRoXA~%jvbx13U}Tf(jJ%+m(;B3a=qmZc@Zj zd_nf=sIU85Xe~JK71_wPXnjL)nCffPOPw?YQ*np!#=!@oWei!bK~fOTInP$>I9 z<9o7YO$0vWlB>5XPfD3Qeq{hnBmcbx`#uhyu5+CTTk~i{6SZIj3}fr<%$PD49WU}F z1sjjE5hAc2U*@C*50{{*spctYI1)Nu5=!4D18MhviNYCC5&SER1w-Hj#7BKx_~!Gv zi*wK0s8ZEEu{)3RyQ=NHF(2AW(k18mKWHjBcNemGMd9#bGsQ7!c4GK(9vS6;&!&Ep zpc;bD%LUWBV0W1E(FZZ9zT{%pcS+s9mWvk34_taV=RbuP2REI5Mr7!w_D&N<#WA%K z`(W@(u~E=HN=0@%KkVXFeUriaJ$d9+PgfKaJs|1jg@w%|+Cl62T}+|)USqwsyZMrz z$_-wbQ6*l@yDI0*$&~fZLt^@4(k8bW_>)L(bKEik&|fER;b3vEIr>5bVJf7B8EB-Z z!0h|R$4LrAXu7?@y`k9*6^55`Y6L=P+$W$V#Ap%}^@W@d-x|Edcu9 zUImdVfd2msx>zVhKmJ6|H6%gS%GH<6dyedF*F;K175bJ>0(v(;=OIzHiClc1&IS0- z+k%yGM4xEgq8b**cji*m2jM^I<&lauhI@<1wH7q@oEOasY6 zaEBkDPHv8R1rfs88eX3xtH|n1@2>Dr^0cQA+HeHt4ZYGw{&J#}s6y~$7Zsm9nNxsNbsP!=En``)wM9OIo!N6L#`tnc zDNU5vYDz(w2i6AiLy@`~Nu=b0oNvuMcCduB*u3?$o(h|hHOZSZD#bsgG%Ffwx@F=+ znz(2XOB0RQr=*x^QbO&6%3S<-mB5C;%bDQ=qDj}ivQ3D+;4t{}=#k3p@6yKiLH}eN ze!G!1Eh=2qyNmMlsg3rHl$f-N;wWEo4oQYMatl>Sk-w(`sJ`)vVq%i}s#w9c+fPJN z-ordj+FEEM$Hi&UCOLr+Ge z(5hq~A+2|uQ2o=w_*uFCSwE~ivuI<%e-QAPZqL%sP!*EW(>#!{zflrzQNA2pZzN}q z9}B0Oxq-WxBk+mlAE=aMeLnM2`;`XwG<6>6SargX zDd<n%F)Wrz$Bn6x zaHmUA!uYa81J%BsA3FElYHhP@MV0FeDS7K&H9j&lA&;JDQ z&rk8y$QK9q=4kko*=!>1CIVD7Zl@b$^`d=r@CA7P(hJqENGS@?1AL;%sja`1`mU!% z+N3=If685Ei6=y^9Dh-D-}}M1(pyvugoVooL|gvB5Hj3btoimU;fxoG`sEiK8+*5b z-`9yls`PmnB^@@J?Tyv&7agZzV9^U4IiPK=hzQw|bErQ*H(h33k(gG!Gy%#)7uy>b zc~x0+$obhw>vE~R%!qaF^qcaQ)5shd*QoNZHymT#uBh_=Vg264xd{SP z$k5|<>s#GjE`_vaBtctxmHQ)34+a|-j>Z>;|7b}KNZ_Ear%t5(>8;iNt^(1|)S%2~ z9%tokA7SB2TEARFZPH&6sT2jfAa1}CdqjH;_N}j?=NBb1`Wa&=ZWI$sHwqWBPWDL_|4qpn#=xy#hGoY)5FCr+lfxJcWa>`#C>m( z(bmBN`y72vw6hhk2V@^%fs}F_wB?5)Nu~V1y^|Fwg=m<-s$7N*bHppaikp!MiVu7Z z1{tAUih+u}%eE^ut}Eh}R0&<(a3j@2Dyxlm^HJyjtT-(iPm>S)2xPEJ<@okLX(;mN7bdPoqc< zMF>z`RsVGjM6&Om+l;|`gdWF8rmf6KN<>e?pYIZI7Z*7tUj~?6SiE&wEdV1JYCttP(~u|cJ)}o_ z)3O+}9RK!v+`2c#|tH*EjZhR>MZG6dGJ>`ic z;r@M5mWG*0S)hdI@*B)(p?)lL&z4l`it5VO)1T-ZyhuUUV^tGU@)nI+t9Y6y3b1GB zXUe(<@v@A}aRk18j!^xKVWw+@nMH&g!w-+2{ZB43aF@le zTz+>=gSR_`UV;U-k0reG1O>VGw0z1$ov_hv$-xg?5j&SzSU2)s0JEI)gYrISqjty@ zw|)ov-6C#GQ~}W0bnDArcIZetEhGv4>vj}5xy+D8ceFa{KmO!g$lJPWq&42710-xr zVs+x34?`l)dmchz{%YmThtikbXI(AdzDx9$^l0SzWJ({;*q?6a(hJ4=3#911N%hnuXyx5-JKR9S~U;B}} z+#NgpVJ2h`+|R$*8=xdn<%hB>x;>czL_qza@gT|*AD2jEzv(#;k@T*>Eb!q#A;j1P ze$5teF~@VO6!-G-yo200EtN;&U;RGELxUPfmYp=*#{{egS)jS z^;ItN?wAJiucl4!A^1F0gmdes=!5u)F26_BzNpWxh^SbIR1`}&^uMIuD80Qbv7M&t z^!_C}ZvWR~pc$~GyrIc?%SgrB@cO-lTT-3p`pqxGHl=T*4%Uw>CJcc@Idc-+FMS2K z25=D8LO?~F^Ga(zN`KWZB&Ov41xP4{;PDw~wf9#bZcxIBuNxjY6ME1;n}epjvhxI` z1=g^Qzx>&m#`|M*s<749lF!*)(i$WbXHPK8fO`f~Hy`EBQ1EqZFxKo!+vHmI`m9bz zPA6ntIcW6nJDtfPh>J#2I1y<*c6>9c)X1?hS3*#cfv0IYjIboa&!m=Fb``ZI#2qmjX zQ(V5&$6ilkYW%CBwhkOc^hd@{0=Qmrya9Lx28p)d644tv9qhL4i!~j@2`-Ost#rI@ zO2VSOu_58GHg7=FDtfO{>;X>`$B~Q3Hq6GmIu3HP`g`h{Yv%W@cX<}jBgXyg)vqbl zJFA~`r-N)pJ&IFO)7yS{!?w;TCLq1%{0p5WK)-INY(o4v?c zl^E62KKnpE@{2YCsU4T=*hP(0jYK8F_6on??J}_9++i2#jm`?FG;n!=SmND%C9pG- zVmT~OG4+^AS})W5H#Uy~EKo|gWsQ~5C}W=^-tm?zD=jxyI(1Ea5RyQz)>4Q%**bK1 zplfyup9dLyI~s=J8PBbZ1fAFp#Sz=7r zWO5vfm$}G(JdR#z@geXDa9b}NT@E2C-}5GLBkjxT&R7l>Q~K!MB;j=-8^0mJ5iOCl zu}l`BOe2$1%H&~a=_WGOat;Mjbv4aKu)vKR)-tTP(y@ZOo^l+ucov*2>_HZ!IzKgf z9m}vj(%T8^@2%=k!e`nVKE6mjpwKd_ut!Ie3Xb)>YhzcIprWJ@vCO=;Dp46po|M3H zI&iDqT!1_`TV+iq>@)N9QiQkR!6xn%dRF&?$Lwc14&Fs|^XaRJZOLHN8PHDj9|GKM4UKw6TxFfrwm_BhkDq^fj_O#W=ZDH7V%+@ zgX}}=OxqFtqr>c9Q1QY52C2D8IAKm-@1QZ>_}jUaR9i>3iQ+x8(!L3NrngVme`6@I z8B%&WG}OEt=B_%H*2-z6u81h>6{zO@eBs?A54}GUZO9Ir;}+4noWZwcN)^3NTxra~ zt{uxeRqc8}Y4>c+^7iYeav@uxN?kfzZP(r#e0-(CO?y1ndJ)IC08Jeki;BH_rzS4d ziZ63K4(|rLEL^J3Hq3oJG?f>;?oSZN_} zqbL*go62iSdxV10jl)o7>H0#MqB>UFy&daLKZ00nZ{3Y=XY1aR*}%e+UeQR9xgNiohdVP!K}!lMU`MbS;kLw!z=~V?Hz2Ge>P^gofRe9=HuyVHockmLsSL3=ugvB<{>`-r~p33cSOqo?`#ch*=nl(Z5w*^PcJ^N^4WCD$9dBhi z7#vFqEh^)ikys`?&M|i%goG}XAL%?%Y+6BP(*^x8M1&TRL%Ugk$fh$)AwJNkJGGs9Kb*GYsmiYrA^LH*69ZK%! zr%o%g*_y+N`9(gPL1Cs58a@uw%%_W))0GchFPp+Xc<;?k*QF|WyJNx8r`D%)jF*{l zE#9d)65IInQob*`Zwqg4)p*<-o@%^gqrbG&uzxXUaq~&4XSYv825dDw&kE0u07)!B zXA{O6mPLvaIB^H|gZ6LQp&}b=q!}F{5rS_*q90=Avlu*l|4aM9Bid4qJmM8ZOcjwo zt3hbS&zI^HYkjvz2&uiKDe2y?U^&yvq7v@{nb5lWx$^7Dp6oXCYX^1CgQg*kr=@*1 zKg~$cA7kwJG*l_*RDcw(+weBI=a_iAe@4IE&G6(EcThag_QeiV9K-fa>Zg2O)KIjY zadUA8RW<1X?j1Ok$m{R^V$~^|tDmh{*0ux6n=JP4-lg0?O-)G;d78R#m$5UE>5Ci) z5qWC0@h$nJm%HMp3u%|g*#&!>L{lb@1K1KCp4?t#hr>n#{r;3!;GzxX4P8%}n%tK9 zoyL+!R%x)n-Xxl)cg#agdjpR&u%8E~@NOWoH0vu0q9|2cMZ{Zjo&{b)Rr)?R7E!&? z0whFC@yyLXQNv%-9&P@6ml&eJoE>T(I8A9rR!#yx+rh#usF zA+fGEUgVk68HP*~sK&fHIGX(HWcrV^0|>uCpLf4a&M$DGuKS-bTX>6q_Kfh$xwlVZ zqYmO_e9}h(&C7z7Q;TvDiTSz-3^(@Dc_$8sqI*p8!GdBP>mywAxR7PvZ(xTt+7SOG zyRE36I)Zj%?Bv>sDjA>uR^rs)P)_sU&d$OdLw+LaU;VMy4(^@xoua)sx+wf18nAHN&fsXW;+n~HivxV zmR$c1A#S0_^jXYJMyao*DA!%p;!+&4LNg7bhmR)YytXcGFHk1Crne3+mjRZ~QXb;< ze8VzyIXHz5v;FS23iMG~IsemS0x%i*z#Y?uAi^A&1iv=mw^`)YN8vv_!tUtwp)dzb z6FD8dZAMU`pgF*`3KV}0nSdv&247g&ZgNDtjifpH)_Cw~l%#rn?g0=wFe2E}p=M>< z+^$gq4&iQCC7t|yLAW6&J^Nd~1yv{RMYPjFE_aR_27R` z&#C~4@j922^D)L(qL8pE7?dMJ+Wy{UQh54_mVw@;0MUQ*O1AL`UEbxct9281vt8D$ zy*sMVi$Hl*J7+Aj#7ZS`57)imK~qf*)v_q9%Mb0kA?OmX$zlhvSdaC*GIPj%m@H=r zz0M&iq(^f^>-9`A5@u7{_8FH(Y=pZu<;yB0zx7(EqVhRT2|BnK^pkHVAI=Kc9`eqJ zPwBWPvQuVHb?z^ud2zr5-~)*~>O90xT^D-QqarlYwI2)2iXQH+WntYQBl3T-l`HD; zDlWI~^EBe9x!k7jN6)dFPaA1ZNe8>$v&BJv=BOBTWX~akS>R_l?+mN3*(dO>WwmkvWmUF~*BL zl?cCYyfa_Q8h`y6++PH~C__iRU%bad(4#T9`ygkq{&->({y6r4pBm~|Ef0~LndhRQw>6u^*c!|r zKd)Tckx6MM3&9-e2egc@fFCo*{}@I1xYm3r`86q7u8Q9w!|T!M@c|>I5lEBG_SVubXL(4i<3CL&{JgYD= zUmru)Lpb;rJT<#|Ojn^OExOcL59zLtOJ}xDyC%(D!Y*qT(01D>K z$#t|xJ|>-5Gto2ge2+;|{8!;1gO}Wx60ImP7-E@Cu0Xl`*dW6Qcf$2xd{#8>_^1hM zAoG{Yy%A~lP7)>J7Cw3C9w7T1V@N4xW|ZF>s@xy0gA#dj{!aOcfpoZ$VWZF7UD zoXkh=?I%oCYDasH#(kfPaq&U8sNS8}USWvyMDzX)iXA+^8s36+yl5TeuZL?5Tj*&2bqul}n4SB$_{rhY+g(OKljSt;t#9c7@QMM^JgxrF8n zF#M2b`#7c|zLAM;=?dY)0^(Tarj*9LQH*nZrvnb6I_~XJ7j?l;KATZTc856-l zF3#9`SGh7GA;VuY-OwG=xLicw#Txind_li=YdzMaQSadXH3W~OnBuBs$gspzx_u3# za5;F8iIVv@<-r?CHnNzBC>XK0cG}|bd!s;Piha%S`5yE^Qn$hBe3Fyxr5hjWK0(^{ zyqK^+EF|N{VF1k2)MQG&)yG*H3cfKuFkMm`n6>F=!z^_%zqfw^oz^LEXE^$5q}1CM3oX*K{X6>dqW^u{K@Y?&KDBoqX z%+lxEp*YOfX^vZ68*4ic@jTumDpX!u*;AC?seDHajFor{p%w8z2OkktVBvBHjpsaj zDWfw67=Bf8S)pyLNag4Sy0WkFBHJpmH?We?J~R%w6+buT-p?Q~!i!3Cq&8mw6r8Fo zVa~YLCLQtQ`v;;^gUdd+F0fexfh4zbDyb51jk02KIRsZ0XB*8c71s_7dMnXPSU5ax zd877SEMzeANciEwqRMxKaZu^ToJJdahf7&pZ8Xey4k;RzJ=ptg5xHeEr-p*`JkKT( zi%`DCnp;%Uu|!8Mhv*S6_P$}uzO^A?MG0g%sF3;#jiZP6Lf1#MH}h|ok;93u)j64p>siAxH)4oiQcCmUEsLBVpOgx94ids7i+ubL~h4kvP+#t9FQ!=D%a zP#(?Gt1afly;SNtbzMy!@bV2Mf3DsSYrrUQz3-URJ>>yAY6VJpJDn*99tOs9j0f_v zy3KHwgZIBIvztDT_B(bUfa--IU-xP8)`xpB%UoXZ-tAX6Mn$?}A?9Pc(!~$JdR&Wh zMvm_yq?hk)mderhA7b~2WOm8J6uR69)6zqA$1k?$+yT2|113g_v_obMl$leLOgrSh z67&q9q8=H|r&#gb$Qx8Q%oT=)N=$Cz=#^~_fihWl z3mCTn=UVJ)37Hr&x(59(@(OrCEu)Ju61zk=lJ{_+apW|qTR(CCaT%OkZHXk=*==!S zp9@_w;s~>d^m5~})(XF^7Q(z98#)&vCJD#nMF+m`SfL1K0S{VLsAR&-%dvc8{UHL!T>Ls+?@mx!mE$V@(E4Vl~*X!9y6jkC&Rkd+e3C)ht3I-6O!$lkuzKSjMi~jF!dn4I?yHia(G02JWiy>7=H}mf8DuY2o)Lo9*MCvh(e4nEOFd*DJk=>%c;{wCY{D zjJ3qZ;;S@pp!M*^nWJF2={~(9(AUh`iN^EL$`ZV^CX3pbX0RDL?0lBGEhee7k4M)W ze&0kA%#iyjByk>UCt3^&Ev@0B!bV)f!ca3`7<$+GL>zN7fE}0u6}CmPQWbapmF{U_ z%z)c%XL=2CxH0VtO9de~oLgLV%nppuNSODK^<8V!r_B50JcPcKhSdA%;L;;tchb#R zqzpFVR!b-XDA61W5DmkTbOn29s>0wFtedt6is(w9jv%A{SnsbUe8$JPxD51KW8qY0 zE>raETmx--=sGa3`>!#So<~Tm?&{*f;fm59izm2xqCE?@cdG7i(Ci()LNCXbuFu8& z&C()KusZ9+AO7odGHfY@C2>yj)2QyBpanFVKb%Lkal^0SRJk_J9)H=wBfnejSEne{ zKqBk{MM0eHYAY>(k)u1X@pl%Sxq%5nH;(5K0cT?u!@oKX$c%V!;nFDJ_kfAb8q=^o zUgEhuP!z&hMLyVrzAJQNjs=V_DNwVEZ6bekfWf82M14Q`;h7znN@?Frdw7@hhytXE z_(~Z68s5iQin!zdUDdJYKS^Wo?gXJ)K{|mdNfp3eFiX(@quPuI_lFz(@ zT5nV4QqA70x*1pZ;ZfnY3x{ZC58VBlcnR!G+@5;zU+fKSB#jO>)J5HjT(a-iC=p7u zQu0s?89B6|>*+0h(Mw0jQkb|Y$TFP5e71V+1KC%9rCS~034EV>F8+9Q>Bc@ee49b> zF0S^Q!cpsnM|W|LvJC;yw(me5fKk;Kg~A@!j5FQj(zu%yLeN*E zX8$H=GC|w!ENvmg!J?fqA(>b|-UI)4Z1_gui?Ux^Tx=$+D)ZO3e2cMP$c(QV=W_B^ zM`~kk^Is1n5>{7BB86Gxqic2cE=~os)F@SVSqWx2Ocv2lfdx<2Hlfq&VGp;W>9bzg z*Bo6t^b!HV-|(==YFl2Ui0a24{7u?&Fr-y<3sChXR%_)eTT_rt31I+a%GvL#mC26A z(ztIfSa$xwF%im8H}FC(kim@|hbTk$AT0Vs2Bp$z+Ps37x|C+gSLzeU7e`o`OwCyM zevceWkjPmU6KeP!Ew$1DVGWq3S7Ta`iSy-uXe(FGL|+SYYqjE&&EtczGdiy#ua8pw z-=#OBh~fQ^KaNpgv6)HP!tN_xvS~cGC7+#7ToxNv?0Z1~ZxJ`r1gWJW(mAViju%N# zMx?QKktSjLGr{qw7<9JFYz+qZ-}%6t4J*;t3Vvv`B8kt4N3nW zJJ}sQ<+v*_{w8?6BC-+m6P&~g!Q}qMI#$v5Y>9j$0JTya{Jqe!;K{h8bV%|K;a1J$(X3ZsH!xGuSK9C0X?_pzB!w zr)DS-X<_#465 zur(RlG}9ljEdLDpf2PTG3JRv`yE4CKY9O?ZT_)%{+W$Ihwezm z{XA471NOS_jvf_4`xWs;7=IEZP{7Slj74vK-)ovIi-n`LugV5Fd`A8djNCWCinlBx zu%|Rl+(S=0JtP(>U9dCjs=y#hIe9#V7J$|*I9UjK0PaK~lMSw`(!&FR5@>%oN%x`vVw_(5b|K8VG$J`wj~Imgxxgj*#vslE}m<4{*RxBJGcJE1mHko9zD*r zc|8uc`4sa4-0BM~_1uTtqxuimbYfrd90tX6H43I(*BodrmUQ^Ni6j`9xXpBB8_Itf zNm_(n&*E&D$-Y@>zJAvB_s>XZ1YW+D)ic>hx-=hUTEQ)unEpEWT^vJE~JuyZ!y z;`#0RtXEJb5Xq5jt7A;sJe8tP2Rw(s6b-@X>qy+RUn|YPl)UEd^7*^-bme`(L+1kF z|69}OnIWFDqn!JWGk66`y4%;lD~JSR(nl>r8bT9qC)t-?|FN8}@)(oe3%Hn_pM!F( zTSERB_jDu_b?G)fG^3WCxB64a%=Bmf`#1AwVNUnBmD)NI6MWYBwJta2Gsj{PgImHX6AH#go-+`PC=6HtxX>CA$vH zP&R{!o?HxkHEUijH#CXi7-ceG$BoWs^aQiX+gWl{muA^zh?nn(<$>qNl%F##mPO0K zH9G`^a}B~j=@%(q|MN#jz?5aWRvt(LwupM?S!T{(D-;-?*J>Ac0Q+x>S^jLkxf-b! z1|u|QElKQcfM>9~A6A01clbL2F^T){D+WJz@Lg~amZ$PEMI@m$AO1boaY(5XTtL$~ z6(LFTwNa#-R4ZTQ{l?|rd@@zhsZdv=hF+)8ai)c}-nvx0oaw`+ z+h$~%BycnArqNZe@4LZR)_Al7#PA`Rgz2bn@429)G&2?;L1AB2{#Ao@u=-=|&mi)Dv;(4C zTxK6_d=c#GO=8Nj0yzThuj29#U7-J-+7*=Xf zBOO91_C>ZZOmKa!R8$WAAv*yRHEb} zm4*%lPt*Y04>))@yElDJD%+RrknG{P`|ML6;ZAR=T*Uy*zUrXAR5O3 z(vT}ij18I$^gI!Hh4V6NEcb>~T+U1u9R*wk{EMH>D|)sR&u~^CvUnP1*r=JC@9H;; z$URutJU0uynjVY;6Zg6mH~-lyb|QoK2kr;ikiUk}a4+hL3m&>y($&OgzOG%yLK_0s z`Dvr8jD0SrOm3g>M??(P(v@lgoGIH<({03_9g9cM<(yws+=ZpW`e6G@87Ia@F+!61 zZwTB)K`z2Vk>oUuLxTgpet$azW9A0uf=DM~s2d**?+0%6toIca#MJG_X`#BYt zKM6CvUxIw^lvmcbymn5WO2^9gy5>Xfdtk+@P{$4~*)t<}^p4xjz7lB0 zabLF$|83TQq(<%>BjuvjmMqYF=!bON4R3ulGHV5%PzC~?eZ0D!9D>wi)vZ=s{KyD% zYk`Xz&l|+#lwZ*^f^^tD9mi7g;=$U%FN76H3(4fS{iS}cY%gt9M2E4h5=BH}<7?Ht>w;s3`U-nQ~fF}h^V zjJs&tZGM~%Y*eVobFNFcZnAP~I^kh_RKrrs+-m^KL~(i)9t?IesBmctPO*p?XObzT zF@dK~T%VZD&E-w%z<4_fmhNy(QDF#l za#jg_1u9sb3J8)|v7Pm9xm;i?4QJ(f7v$og=XaAZXZiPI<2dmhb23eGxHE4ZerdSx*+V2t5&<}C!!&pNkyveyZNQ)e0OiUHlP zq*nWg8Cq1!z5NMmdF^w3e@JNftJN2^IRCGhR@H6uYj!S(pnnA{@(g=0_f$lGTn@5t zPe}vyz0|8X&<5f-)snV(wy#M;XQFVVx|dyOc=E@UHgDT3u0@JE1(YFQaiO!2PsALPp%D4t}wua}jtdKqeULzsuvYGl>Ar6*ucc*u$c zt*75?{#xKkmgEl<$d=v{mO@YVxdT(oHxYhWp#R>+e|dhvH>0ouuBZLPG1tgH)+fUAJ7*1wKToymk^e=)q)lbQ~b0m6}YoY z@%Ei<-ley*^kB9$`b^kyY#;Z0!im8wapCm9^v!VG`C>bSPAb3Ydj9i`tMUOyc0A#I z{-$U3r`D34H8%9QMr2A(;(hW&%2Inwo5uAywtrQ@2w0ZlZQ&@W*Bd=vkBo9z0Pl@C z8`@m5^wPc{Gn|}@N6z>}y>i=#%c!vvCrvPA@#5A>z!hcRG5Sz)nWC{vRLghgGXn}@ z+x50qSF6X3H|s%<;dz^y^Rx?c+2V`FkXtq4&Rl|9PJEojD?wnA4FUE40Wuh;p85IH9f94eakK!k&0J$-#^}6zML8HWUU~)4~bVS-pkqC?yvuhpS zlurgcI;F&qvcPnP^wx;qg>&$F*Cd_rl;rQjc}to@Xh!3rgb2|uiyx}62SvxG^-qtQ zkIE{ScZWLI$cgFNx6~anTgTFF()p)=Z-X1z@=$)6?ySR}&c5x~H^9B9oc1dS@09UK zN_oOdx?Sr|8VsHnFMW~fsT>%t*j@QNUlNoEp{vbCen4Z--9_(S%@a$kEbhXK_&HDA zjqvZSEiYH}NdE#8sDP}kYhGr_7Fl(vmKsq5f^2rfF8uG&J13$eoa4%r!Llm_4N|3< zPkg32D)8{}W2b$#1?fYKOz#iN!ABDvY(cuMrKw>Od<3d^_@e zJEA8NOn{t|W@gG?(RA{@S$)tv1MJa7IuzTWHwu!3r>4tbTw##$`uOXZH2u3XU+18v zEv(#E3cLc5KKI7immam$=JB+KZ0u2<3@kGp-{=t@nFp-wDnKl^8n#2{oN0meATNyI zAk(?-FDU;%V4IF^8=5|vv!a-#5WmE)U!dFVn)P^3|1ts*Ddztij0_V-5 z$k>Y$Rc~EMrO#P1mjdwzJ5xbdL zQR8~1?=^xA5K^hrrfmmFV8>3DFZ~sAgwl0y8CZ`S6I6R_9CCq8ZG9`WDHBWhaf!OA zik;TJfG)|PM}eJI5aD*o>6AkkA!WfuMg~PdZg5ly`3*P*`>CG1qB(xwwyX>WWO`5s z#C>270zNQCbo?A#?r*`)oHhLE@;WTPkw08#(~_unr|SJoV_X!B%__pZ9lt6f{n|i& zlhu?b$P_I_4+~wV_xt@bfO%E#-7mRU^$Nq*Nl8>IW6@_=C1sT%Kgyqf zMOrv#iFe7}^4#;YjjQ=y0iXF0FN$h*et zdg_M-Mu!db$9oo(4}+)b)CY(jswuuai>pG#)zv2FmSE<3Pw0I8R^2GuGWdRdj?;NM zQvLPkG5MlHdOE1%2%;jJ>xZ7QBUHdLw?5c%oEC5S2C8d&-y;29sM77B69PuJ_Fj;% zUTjXD+y-c@&RQFrpZSZao`z|_LDDDs2Hyjv)6=y^U2UzNjz0fL!c0%p;PADjP9mr6 zTMP3gBAf=kL*}GO2`@Pj0ol*|_asi2^PRj|LOWiKQL+eix7^Mtn!Y#;+_9@c4zC)r zJ~IX*zV7~N*+)W8Bc~a)*3@i@;IRT6*f4oTlcBle528&XMNf!ZjlaROa{3K^y zi<@x(L~ye_bd2zKcTP`EK4ZW!_oMGMLkJ??nL++b@69%D_my3WIK9~nu3@UxtKF*I zyp1atC(&;{k=P>%4$F&eQx};7^MKAhk4Ut-fwuS+0x?xGmF(-e?^}s3JV|L%;ZJzG zHeQTuh-7#?-}&)rh7o%Ca6@Kf_#IX$!LajJ1I$gXZ9j9=j%1ncGqTU6CHM1xTREk* z@jw1eEB8IKw0L3X`tA-<_PK<3unk|9CRn7Z`!P>DgWyX^TUPAX!yP#@vSWGvMZITa zOucMTJ!+djy!;E7r1V0LTE8hM^hMk3kSg29BGCUZ`;waq$Chj^q9))JP>eWw?@UzRk2nIq@^5gn%n|_}=j2CfKB!QRn z({(LQ5vTJ+U?(hiphO)0EpYGa7KgOhTn^s#i@@LWPkqr-rQ{^ppZR|6fi1Xh<*kFk zH`s3(gw+JhvC>_bKGxzZ*QPb>BI6{lX>whlEa&K)xAI?LhrOiEOyE*?eY{dy9)P4L zl9BZaIK`pBM|JT!1I6h7BkU`~qTsf+0SPHl5Ger_2@z?AE~UFfL>lSt?vO^5?hfga z25IRSI;9!9JH9|FVP)(Iz+^}z$SdX@Y-2hQ7min`?N;cn;Tmjbp|mttoa-UWjmKOWb2mf5FZ}$o>zov)2=~*vTj`e%N>dcuE}!SywDRvq3V*u53afnSl8}V|1tohky%?LwlL)MiP z`V_}+#Oj{hYxyA2bG*@CjZ4e%^iVHsFg2|En*VjmjRj^k|{c^)+TJ_rKx4qSECGoG_^$XE-!*noB7Nc>6(R%Sbu6;q*U_5>w_K-fcZ#+5{Rbump{Fx zqKFT5t&uoF7^e-n0}4R+gSq}QuH_6G10S}pQV)GQ6``>vl2e016&*STKWoYH`uE)VRe}bbqo-%)C>;7kWAi**Vc zmzoUkE>6H_Kl%Z8`y5`U*1BCXTutA%Il$;C6sh@pB1x&R(hr?tFhZh#V1O;#)b?r} zyML13wG6@5DV|=2;=|Tb;a@2?HyBiC^E@hRc%9_>p}*>5uE2cY0pJ?leLX3Aq)<&l zEp4}=d%BZ>T{Ni5;B{vx4lvLmF&tDS&3GwjBC-NN0m?9}QuGi-&K)K?)boGg@<(G}CfvpZj z689;Yo5STmg>|2@#*lwtzYpdkMRBJjFaZRN{s6$2h9}}MvM%Um0k}t;{^Ande<`vu zf^&aSH=RzOvMaVyRGRZL4Y%`A!O!{E`NOAun$IZHb-sRF{K6TE{QZ59w~qFsXZuh6 zRTMlIY=pXGQw;ru+hsXldAI61dORV2cQY;<@3`=|@S6US{{qMTcdl9UvJ!pi6(O$4 znrUPq2Unp;Fx&Rs;Zq$npwSx0yS7TEc{1me8Wq$TWJj96SOwQs zmr(C?fmy6(L8)L+u^{UQ05onyO-(7Bsn>*41I>^D7+-LNgPD<>u?=t;mYmCn;E76~n#R`N~ToE{}zbyiu+D z>SP{^dm{#bz4#1T^0P7+60_i734DevJ1&*Nga??T%D^JUs)pho9(?QD)$S2F-9Y*s zCn#jnh3oJRh768P{${!ag$ zhbI+llS|llFAc$5Z@6!kRk2NkZu=xoyjX!%00kI^Y~a_9?0(+H{J|bR=U)L7cjw0e z#7}tn(koYIGR)}?U+ZI}TiPaWw=aR~L_PD-yDx@rW?j}q#lD)x)zJ)0i&?@4`58du z5%2lb0|_ap>#tt7NUbIQ)(-?L{)XB_bOfh+-5zU0E*)9^B5sw_gUO%ad*@oo^0I9W zdw6{U2oK@2+OwX4OVDe(#h5G^iI;em^-?l#Q|{$$;hNLZ1R(!^o*F!vPrwfEclR&` z6NSwIyeh;>G8{~VAF7Q!;{Z6>AJ4ivN}a@6t&76VLac(U$$6VAYbpWuW-9Q|>7*ux zL!D44BC~?-;$-@_m^GL_*7n!oQ7=-h-KOkSO;(AX^c#}dnnMQ}aR^75f04#@#JkT6 z+!Ht5`D?JF$*Qa+I&EF;l23_Upi@h#UC$0oA8*RdaEbNkd^CiDLpkFZYgvtS7enPK z0a3!wzh_48m$|)OHBJd2v{wm9+*wMG@M@+h)7Rr?M?DD)lTEGwgX8Bw75$(To#i_^ z)%MpXlhh&;oy2MKr9je3mE1=Q&JXVqKJ``x;@m)~cwT?iz;a#5Zo7=}ggis2=Ir5Y zae|vIg097i`;cvbBx%Yp5F@}iusbAY0m=);&T2`lVF^8&|0P$%R$lSKMA=|O zZKy1~iUP=4foR0rnT%-on#DDHJUi+r>=_7XKxTelRgRtTnf=3@ilpn){a)dz`~_%3 zB5F_cE{FHKAbQ+rQ}zJ+F5@@P<1gxDz??y*tJkf$0Atmp}*E*4(W!>nu=oW!0qtS@XeY=Ht2)eFNI?X zieDBSxN1!aeh=@icIUavG+0vLi+6=FO+CVQ`D;}=Poq0u{SD{9#!Tbxy&qJL_gnZJ zncySfA6@>6OP!&Lt&oi^2`6?W|KY_wZV+DeWGf?-a#&`fldA*=8HIPt=hQ>h3>6R? zLV`>YJb7<-0KilcUM?S&u?BLYUX&>?rgbAX5aW!*NVGP*0$R54T9MlTKH#He9_U3l z)d>~K!v#3{4O-QO2xeFoUNEt`TLG<&Jj?4_7dWl}rbcn3zb4kSCry;XgY?+UKSRb! zUQa84WGYzi9FBvzCO{_TiBh!K6))MfffNHeOLxb23$j+JiC1>HIGIPb0Gt!CuEXhm zS>6Y4m+~`P;dAzyw|5yraSHkhEr)HyIIM$O!E&fO#r=YsAm5i!K}}h*>Gt|o{0)T( zyro6X#DyQ+0~{{9)mL0j2h6h#mnkDq{tmnMTKDh11PCL$Gf3)>+6m^N)bGr#=k$cC zY)?Y=O6T2nC7rLjT8~EG@k6bQ4dR zd~dq5Kc4<)B}QdrD~9@>0t?cF(8zDRTfV#v&}Z^cVSFibA?bT|zHSv>PzCt>T(o|5 z^(~49oCO5Xt13;2cd^>xjK*VR)V@LmNy zt~!G6s}ONZ1|Vju>GcRUu~7m;?p19OX{=M?6vbRlNi*MohVu0@)Yqk5D10}OpX(z} zws-|IB{w)c`Q_cPsZCLe10 zdlo2ij6kLRFd>%fZ7S|`#FQz}roLc0zBSlwXv(iHbDL+Ht6oZSk#*okKzbf&6ZXU# z0sU#Z?`MRkk@5o1y_mFqbaWu1C2rgQ=zu^V6TguAAAD_okXU-$xCoR zV;!K)q`b|n>Iga+C&|XD^==VMSdVPPYq5&L_4~SPcseu%zFI=m5so?!bO!ljVvrx_ zHs!!th=~enn>xe>{4_fK@g8#{g8yLEEVE%6ovEaIa!!}r_bFS_R8E?pnTRFglLf6W zX2AfXGy$t;$)}zU2d-Q#SOrtk``3S3E8bm2#Fg8bFon4-2s7kqW_Ops`VwlJtrbB4 zJJ=q_c#L{AsgHCF1U~&lqt5&0ZBq!&*ZUBO7v0O0om-tD$l#xT8^)486y+4aVgiK& z#c7kTY#cj+uL(V#m1a<{XZK_h&WTnpMNtI>k*ka^y99-0)P7Qyo2)JKJe6FySTL_e zSl6hx_UH6zfk1ykZJU1tzQ-=^F%>3!60}%!bVzqCN<#I*M+u@3LW3mW%*5!W4=! z-p>4;w3t=#CjD8^-uiQY!FO}K;W1TZVU@RYDSHipFr8?puR6IGC$w|BlQyV+dbPiJ zJKYV`J5^ukD))QZVu^Ctdu!Dq7^ZM?$8d!@(B?hy36;KNYD5~K%I+cT1a=*Vv4FhE zf(1qC)Z%;~b=@xZP?PQ13`X)hDFU`=J{?+))(fywAt@Bz^l&mJ^&{kNB>~=49jGf= z8W$%Q{m1TZ0T`a?zYkj_8;J`+A-LS}ed$r$r2F>$J%XPk1_jB&g`bFB$KqZB7h(8} zbj^6ixHk0=hZLq>%%g zSx)~A+ybQ-8Idl1S;-4>VEOtd+oL*_ql^@d3xRKyrsRngd@8O`I%Pgf5qNTS&s1h9 z-TsuIFuP*^Otyv()eF(3IvAApv-CM8@`ri%+K9~3ZSc*Bl;(A%v2BQAM5R-Ea}O-M0*XaJuQ%Mh3tZm3xKAYV+=!m+-w|Q{O`(4ow|f zBQacvMiI;>M8P$D27aX^ITL=Owi~e9XW8ClL!#la^v*~|w0qeh+>hj!6zvdek{Rx1 z@-+)d5jc1`;z8RHGr~e2?3BKE5&1b<2J^e}V312pf=}J+>YGsctIErKqy4_dNk$d0 zLxVzG{SAU>#B&+dBT5otvZsOP?8KjCR+Ete>rJ0P#ENU}@?_`2BfxEkZ$r0N7t?SnfYOhM_h)}IC(50WWHYY?ZivUwM1kEMF zukDE|8wP)}+$*h8&h%n{-W`CbLbKHLYpY3scLT2*KP?!zFC7yQIFzVu?p)>b<)DS4 z)fepUf4<|Nvyo2d#pr)#hVFh>JgL7zG~uj^64=0#`j^d{9*q_E)b$uGac`rNj=0rF zaT|D-j(lbiG|*}u&g1#QPs?XDVut>=lRHIqrkhX@OpsY5RAvcWTjdxTwSWr53WtfpH_2rFORe`|XfL*i)9#K*NHMiu(hM+Hgc?~DC0@}pPed2+QJaOWGY++pL(&%OLAu!%?a zuRTEZhO}3#Kw;5`35Ge%uh!V1-v?@6n8;{|EmLwa5|dC|aEG%hHZP{jSET^!x+9&r z*;xKXFoalD$sEezcwlCv_xOjcVyC+`z9CdbKPW+DJ45rzy$Zh2} zDhg8Xtwv4^EulhDN_iZ%)hP5stQWUL!qWP3xE%5Jv^v9L>F+FLu|XAcH&yZ@WOx%# zw`xaPU#|E%$L&aR*w1Ly67DLWUpTp~w3$;BLKAdY5MWh&`dHh| zU#m8ivDH7CW=2jP^JHXM08twJXk8dZwl_-wxDyg>N`n*^>%e)-rb?r*W-?1yJ1!d= zTR{fgOUak0kP2l>{uKa-exB_QRV+sfS6o`-Y;%-HOu9K?_L;}ECQwg@DZdbLHpZA$nVa*GUeGhH&0WZA>H96O=qLBXW@jdKo?jDIJ}}$|tuE_N zLko^te#Ok3qhyfLAOWL*sILaY7L9>>m{Gd>P9^1OEWP7GiN1a-dEfCAlXZ`KuL|-2 z?H}BOw>GVo&_n$;<#~D7*PvQ{bbr2Pfxq2@-V+{ZmOj@uN3ee*C$R^3&McC*SZjpJ zk3@hP54E7sB_m4oGCUib@^1%uXCaOro6@cP=U^Tx^bU!-ZZS%T2+s4^W45Y>$k0bK zcspxUOYXx4tH_ls$~vOqBvJ%Gaz(0FaT!F!UB_UyN|_JxI@+J56ySn5;^c)WK~%NR zT0YJm_C1=rWQsLff&66yD?-AuDBI`Gm@#-_EOeM_`PUA=_u*r9a2*q#lcnNQHElOk zCiTx&9R^tkTs=JTT3D=}{ur#_$V0QjV=?+m_;hM%9w-WiuV;XifVY_upAqvBMMjCc zjRIe^tEGk8Q^E`6JAcQA7_U@3Dl_Lt+RJXIfshTZDGAEZa<{Sqzn6?tI1#1jBPXC3 z7d;AwrK84+b)E|h>JKC)h$hT$F5esMA6GTJ_!hUCOdoO2rRa=g3x=z^!{l?>JcsDY zFX%r0+fx9${xub`k6y99>bL4n&%nN47TH{6u-MpUyN~K5WiI@*lCQ}h(l{+R>hkgh z6{XWwM%071Iy*C2o_v?RHdJ73NWDf6O1}?Z59s0Xu#R(U`$~s9yaL zN9;InIG2V|`p;_Q6)W8bJo$g95zvG320zF^_-cZpx73z51c?QQL=lP8RJiV()d~VL zsuQRWgbm{y3A}%IJ0U^4O_R0m`&>e7fbk$Kf&|YTdDp&u72}&rNinp6L&v4y zd)%ncKa-MGzA;cS#VtaIeQwOPvk5|!dz}TFyh%hFCAZdrg(Gp;hg?C;fCU7zh>^z@ zyx^Qv+c~dwC)u#vs{R2gLB6IB;vIGPB)d=t%(X|qlY+82rMm_y(kc zSpJaSzivYo`7!~fGp*r@8{5oNJfb84t3cfWrC$w9IePM^``_iwWd4;MUH_VGSwFx( zRX(IsY70x?^0y+-cyBxq1M(xQV3^Lr_X%K!J`pneT|{#!nEG4QZMLPqF8B89{A_|? z%pFX4@6Bj^xeLNOYz6(W6&J=+R3xonFN(H#_6Y_-=>E@nU7r$Tw?v3(>-tns-=UL_ zNU^S(z~IsT57krWZOPDzH3=M__w9_@B2zC+fW`k6{*g(WC-u%pPIM?JSL5}UVj=h; z^`$V_D93Vbvx`%(5v$C=Q-x+4n+oAUs)&hw5pap8r3l#J z{&mR5q30&{MX;^xB!CRLOLd@Tk9F^b6jCK-BOaUg7oJIm7V9gB3UBi!VU?u zhY|NTk8~oD7E|_>J3+fi4Is>C9`6k%1Gx3JK|brh#QejK8(xWJpZp{lYn%`h?zuP( ze%zCdGzY6Ov&q}f8*MOi-ql6brQ2jtg_mNw%5SkOJnMIkmAhXsY?(BH+Y0~r;1a-p z5vcjB{DJEwcohv(p#SYg-h)C4go_qi!8>G67a*PPU7v&F}uadeYYG1TRA;(ui^5v{tuwKzgfb zVKB|W!?*Egu~-~+75fWsb~&odVF*T^gpNzy23 z@4#g#JGdeK*5FVh%dUt)`R7CV4$_9UnG=N#p5Zo<^s}q{UEH8+*TPMCu01W7-2#(PTjy({SV9=KUitgj0f@Y)pOEhsCkYlJ|9> z$^$Coo8WXWM{$Z%&q;bjSrg0(Hd%;M#j70>DKrzBjQH4iE933S++)W?*n`g#x%&(8X=}s2a}>!v z%?^&Y3{Psl53ghY*Jee;(VMqPeY8*%WYl(hEJdI+6FpZJ8=qXn!q;gv*MyDsXQRo^ z)AG945hQ(>x9x|TJi3rSNz(X4zWikpI-_4JU_nSh zE#vyyqT!me@S6qvslmaj`J1Tdame7V1)*$@@y7YCn*;A_(9>;u)z*1*gSJ*Z#+Fuh+z`QZl=fi^)VQSOMkJjF>lnGO*%4E=?8|qeQ;in@Ok)Ph1c7X3l5&pp zzczL8QB1^Zk~wpnvocKcJMe4SF)S*D0-_d^dSpOes_4-WhfsV;25ZZ$tYjCAyjn;DXlLy zwLaJ~6&XxP@=;zDlWe~?eE08VBTcPNb*$-9o9Uf(aq z6YCXv1QkZv)MZg;z=_47msP8cc!wg>4-$4lTtPkCM+i`1=NGM(o#_LzjUvsmuIt-DUC;srz z_3z&KaH?q6N9?@4?sBxaz+TfjIgGVN$nO*+k0zdBv*w-|n&2>eyaY%AGiH9YsplN1 zO}s}fR`CzF-N+GOUmz3-T%lYRwXN3g8c|Ql!?H1L`;ud10j5I3M6GTF@%PC?cy;jT z(E+oZJo+^g@52_sln5m0meQri`ugIBrpDYA{^@vMBg-khAFf3tXAf1UM+jQ{y`V`)Kn>caVoRu1q6EGHu&mKlL8 zwB5M%*Nn?9hrqn+AxYnh4{aXPyI#Akh76%3{C=wyeQD7?Ce`;IGfh4gTE(%nu<9~O zY?^K&$rl({Uw*EkUt&4QI=40L9)hBC|5(E0$-K(*mY+C`x!9NJ_{uU%GB%e?w0~{< z;#cV9L5wAW+3O=wBT4?1rNDFdS`|bVGAIQbT#`s^xwCF_1DFGCuL0 zA7Q9h@3A!8Han>*mv?M(0eJf(le$s~A6Ygg7g9;{(gpi?d_oPaNRKb(pCekg(x6xW zTdZFbHd&txYCY9sTyoQMygNcMfz06kMQXW|gw8VnFNcU$=133RJJWgu;<}-xR&;l% zF8N?5xn_DneO`xwqTx)N*p$t*n@GYUfvQ4f5OW!((e5_d`au-GqS5&x;x;BV6eVSz z@PT+j=y!}0=RAw`R1#{LliEg<4aR*DK5s$2W1PawZ|agxO@wX!{GJP6B0dNN8pUbM zhl+oJz8{F^&NrIep$(EAGzms@hfSUre7&wK6_Qe;ebr<}R9Ii7FDK0-E`fvNCW6G; z)%z;bcLb|=x8u1;a#U=DpxGy4!{!T=ha}9&2W)jOT#!7Z?vRW{z7ka|ad8@DrY`ec@|*);Qq z<@woH$%^zMKk(+`7D4N)Ya}O)^X}{5B2Y$V1I;8wLu3ep~p>=CchpWBLp*u=9 z$4*Vf8J9Be9zvaK@uBBxqKQQE$y}NK+P?d{{Wzogbl+=2>K5UY2n1=ZDQ%HY;N0(? zSiS$A^W{`5{vn$qPbxpMUUFL)`@$pcRung3#z9K^dM_#-$2UjvI)$~*v3zx@Uqn%J z`@Y_q*c^DHhm}AdanxI0JY34ox14uoSV5E@0!5?LQo5(_v)P_O6g2C%aw{Va<%=2PojF&1vbYlmITfj zZnsZa>ueFPuMRyFCa<#G#?b*NxxUY;&Pd5=puq9dCxbEZVRsbf#^K&@z@D~}8 zibQjf>&17&itisHZ{M%N8_X5PYW2zf{W#CJT08Vga`Q~@EXB!K^o{~mj}4ZmEi=>~ zXEzHgTs859JE-;w2rjL>5C&jWP@qQQ0%BGq2J-KD1_XUcRk4rgi_+2;Ic?tn^E})U z1;GBwOX3&Y7R1{PONc<5(L`((OBty*v-SpuoOOyw(vhDqmY$&h)--YLk$FzRH;W}f z1GE%=kwaQqphafuOVEU|t@omj0AZ6X+hTyQRRVp(ZRNFfZ3{xtc#c|JA>iy|vek5sm< z(~9t%&BnfF=<|8#D#JJZa@6@g&5DqmWr6{^tgIg8{ezD+eyhFsu1&gYe1KkOHDF{^ z)@szcxE`F)5uthslPs;?D_je)t8`k6n>ys@ukiuH5AHjBEGv_I=A3g(?(}+F4v{rfaOMcZs zHLnFo)jw2YdlQ|+whzyIpELB3$~;@fx>#Qd9w{|{>EW?glTrV0a!u2up+1}lJD|#vQN7?4qkKw&yT_j=|5c(C3gN4j%{BS!jFLw8PFOyE>J~K1~UE(M_nt8pA>pqR5v8mR=nrqiKN@cY8ktX6u?# z@A>aVvtpWrp>aoBNaj2X?A#9X(wZLE7DqY!*A`QiYsqtLF-g1lr+74$13&Nf^>NNN z3CgExyM@H?GIK>3RGD&33IZTwD)g)ceci_aZNHV2`d+6mOEjM$@>CO^^Rf6tH>Fgb zAGW4l2}ywaOSPhdUpEz>akh)2c$~5Mb;}TO@pR$oM+%oOrCHrZrEZc98Yet9(Oipg zcHFG7q6%MTU<26)QOBXBdN-OKnhUQpM|Gbq;@1rASp5Uj%ghnFuN9^{QXQvw&-!HA zDhga_6Cnwq_!aDQuLh}_d+`s|vN`*FbVHh!N(KaJ`)6G=?p*PX^s}l_Fw=XnyQM%O z4nOsq<(ki1n{5R}_pU+@CvCOgXa}sm<%q*r=aHrFU8d~7Jske@vHq_!3ONq5c{#gD zt}c#QMNe3^fCN%BynCfA#Q#VQtx91!kU^RH^B3UCBgjHUeQ+u2`EIXBAX1E^a zacouIIa@Idxf=;Eg~pN~gIcMSw<2$Q-a$cHB5&`0D;1MqKG%bKkC|W#{Z0uHRW!mx z?Kb3M=khkS_5#wm(q2$$xSeo#hRK>+%J~3NZGDL<-0XSW)t%M0O!sZ7AyO3wV!8B) z^!@M4M&tbLCQUcn^ipgNU-GzLzjMuyqdoL{|i3 z2gQh3tnd!DzKdSeYPpr4Td04znpJfihGTyR|Cqdy3+!_=Att+<{~cDMP13a`Y9Yn< z?}*s=>*D(s(g^Jy0R{^F(*zMV`0EAt{12e*L}8>%>J`;vwup=(_Zl1%!7J&{6X+-Q z+~=;c>N}Wv@##4A`;40nnPQo^?Pt3Z6c<$rid|1;!Xx7&IydQ-955ffJ9GfT2O$Ph z_pkSwt{@iX@9V&f*?6w=TLcz0#-7!k>+s{g+xOKtxZRQx&^Ddk*Du(bZxC*Rf~Tc!#yQsuunxYshDQ?hBB$6 zc8&sIsGuO%B+Z2X=Bq(+FTJ{0r>v{gZv7#O0Y>b()%>+b=2Vn)J$aA%DUU=9!)=^# zNK?${*dZA`8kw_yU)+n1H;Yxf$vo0FJu!p+(%kDCl^$|$YzVGz{P(X;D@|@QlMCyN zHjX%L55wGSk2mI>{6nBpF#@_&Wm_OX#54v38*EgF*4#^$7ZMCNJjEw-nOXcEc zG9};^%5s8nSHbD8R1g-bCP!?6uK$|5%-VFvmjb z>cwy*NE*bB;ZYN~ac@LHN}_D@qzwLAmyoYZBUn^pK$Y_`igar@JXge62H9zg+ zggzC>ANy3oNHw4QCT+5VfhZtx0S-t9=Svg39I~IWrdF@t!%S-0V@cvU#*gYP{546_ z<OfGU>MIenYZJ%75Ma;0BB5wuQvarCMuMV!wbHVoFcAR-=_@F<^FTmG*IG?X!f# zu16%j`b89bep`0z2lHCHA8yEa@Cwrk2|FY407=#LP-n`u?CYwI@uly=yTuOX0C2ef zS&W0J_>@p9<<1p%ofD+8X+#XON_mf&*2hldUd#~4hSEJo_J`Fa&sa9+W_@GnDbQeX zpTEk$S$GTW2MMMpy8f7pW(D!|>vxlDUAZcG-pLMit!f0E=%P|QmPT1N%fB}==$#xS z7(<9D>IPUMqOYWDj4ae0a((JOxvM*R6!e&IOVrox6a|*}|D&opar7(Ztm*}y%1^Aj zQnqpf67Hf zOlXm#Xv*&pTneUs&0DEq;{nq)*F=XSHroP@1<_*VKgGY-d`&WMlr!AVypwo3>UK`d zq%MNiTbb7rRd#B=w1h^`N4~j`40+?CaMf7{>t|F)yK>hTXKnw}!f@ zt-Z&>&J=SU@h~0iu@=+NGCMsA$nUi}ju}#DwuXG0!GNlLC)a%tc7cU&$UMf?KSU)< z|1)(5&n%Lc!9%Dg=sj4aK3zu`13V>Kn+GV&zg<2@EPZJat9lD5+~P|Gz^ilLt3#AF z+%rP$)n|IMd-`n>9LFTPN9f23^>5Xyy<@>w{6}Vzf(c&@eZV;|t@`MpYMIT0@RL{H z==I;|vzFiU&&SyiQCTf;*2FL59+QBvl!=ZaKB9pyGnZSmom%d4L1ga8l@%7|NMe$W zc|`MpJ$#I(cKFKl;qi(k8@^VB(!9KIRNb-5f^R6Dv%}X8VZPO~u>U-u|oTVo*@s6AV zy~yR{ERS18l{~|0@P#V?`!mO=etfm?Z9vYGN;hxG_3G?J{eW!iSG%iI!|62;x`|9m zm*8f62hM5z8(w<=r4(YsqpnUwQq5{L-oQy{rH1 zQx42q-ieS%RfRK@o2ul-5u`Z-_RR$Z>s}FWBNTC8bZcdj9ER$pM*QhSzvbNz+Ed8G z)1`6KzgOsFN(o@%9+%;?4C~wsFSOB526sE*Mt%zwfFN0+=W!UM=U zK~f-zGj;ARiT?hk-mezsz1Hq7kyQL$?xY?m#UATZWuB+Ki0$J_zIU>K7wx_NX4>gn z$>)P0{kmc4v603wN5a%Swpf=-j*3~)AaHOUueZ+Nj2>Lw_doo~N2k*J%~H#yX0#=f zI;;pwy=A3g_c<*ft<^FJH2|-bj43D)tec2;(e_-c&HTf-I`Xlvb#f;y< z(@?Il$fB5MeTr-B*64LWfPcJgwdPcyPM-Wq@iXAxNSwlBCn}boj%LgNvg!tLpOx}y z1dxVg#~o94hYoB%nR`8H8^`!GUSe9(>3p$^DH~CjU!T>tF7Li}+godGszUy`j`Gv3 zbLGF*Xxzy;%n8ca_>{u673NRzDOhkNl&|#`GSWXz?m{Z&9H^*?St2@P^9HB$?&GKY zz%M_h9#iOypwietFo`?NEq@B}{>_a2QPQakge~Z{TH33aSTU;x8%C3JSr20(bnHJK zN3S?GEq;MI8(@kmj?M67@QsQ9W9{w4GU%dV2k!j#$IqiJr*pXTmWYez%#3ZmB>a|* zDfKhtiyEkLq`FXuRux?Db%vrc;rCwU?{{(0ZqpsA>PQ`di14~{>9O7*NHqlc1Um9RioTyxF~3bzmsZ7J!fM;r{3z8&%l`0jki-vM;vv?V2L}KBB+(6^ z?MQan>zU70OR~JDGz?Q0qgZO*;3ajI=DmpglF%nv(x00k4$tSxikNpes>11U^nhN7 zVf|t81WyWg7qw_7w(7gKOTwUR%pDSn{SWtw(&CQGub6Sk1M;*wMhg5VM-Hxt=a$+J zxraotS<&0WY$6r?-a>N^NA-hx#EygG(BY=~trQRB6#qFM9s$V%aci#c$}5`XyiOGf zQ^5h(AaO}D+j14s|GqqsDft?d5!&VUPCAY336Z{nK`r7uSW@+A(icSbw*MypI%3CfZNdT^wg^OnYr~Zkhf9f6gGtGoH zmDyj$Gi0)+;sTUuhZ@vxXKN$B91PbWvQ7=f{$I<|d!pQ_H)=3()C2nskL{(!>e7A%koMp>_}n1mA4ho=FCn4;RUOG))ohho=mxj5oXKnew~ zZ6xS28rqWZ2x~onz->g}tK0@h)$o1EpjUH^KP}2TfqGLO&$txN@Qg<=!BPbkqX zo3r9Tp8+(OLqRV2FDf~LUoO6R&+bg~TLlNlD*mH%7QoXM9ib_&ZB*9=cY1g$mU>0> z$FsN2`IZEOzAMN-7Ol&Bxl*2Ebu=OOgY#7$%lzgfE8h zm2gi`-jW!La7V{!r63$)cpk~sFTWaTA1MzA4jJ9I$qO(4uNB!!p_sfGho{0IK``T+ z6vUPtled4*SIoV$`QrNjI+}Kpku>XWOhX`Bfy-8Hd;EZOb&E=4tbP3WWz~~x$zK70 zs;)d$QamP&QZsP)#=hu=g%3r!6k2qDH>ddqvr4DK?{@wH6TZ~m)lirC$hj)!it@t~ zBL;vbQl$-Bg&pyT``c5D(sSLrTwx#kfw@ZM-;<_%>7S`%J4=D3FgURh-5)ks>8P zLM-z;0ddGdj_*9nCDRQ&r%c1;ZhrlbIw1~*`wVDqo+%<`bcA$AuQjA9@N!Zm>yO3k zi8v>rvTPX0e^m44Eb9tS`ae%l7#p=L74In%MQbq$L7kUC9DOt_!nDE&VX+^2mx$j8 zqn$cS(O86joMf-+lr#O@q04Ae&7rEFTicoyaNjf(t%VjB4t zEKo)DRD=Yl+lLCUzlMt!VKy1B(C-vpgwcMR)g&!B#Q?P-)pH%{fPYrPTD5IKu*KKl z=s)HCFsT&j+^33E#Km<&e$4k$g(pfiMYFa1LpN8Cp@hPhb9IEBnE#&10HiT6MCMoj zS!t2%6QHQ22+WH+I>Tcat^u5KyJXYBEg;k4rP98K^}%}xhqEL-(!=>0g+?o}O|vHk z_6J+hV4KCap%Bw4@$v8!!;vP1_3zWMY0+d-Eem&1 zF&2q#Sa(#@Anzr4#=lYhR)teMSdG3 z+_^gPmE*jZcwu?#92(L2u5Zo}ph)vIv$ldCK6drDR~$61=qEh)Jl15wNA;d~Utxw= z$T*Bt9uD`hZGEI`VojM>Sji|cEKN_IxIA})?_*V`ObHxuc*K`!^b=)1ceXT1y(mWT`8lt5hwGnEz>f$1H~*Tn={P+TdrOJS|j} zINwH6T?C4SE$r;9+B#FX`H)9tK&s+ zQ&aLz?d1dK!@>bah@@^0SFP*0Ea=avz3K6|CY4Tk+d(B|bdIZ`kF_Qy(UT2yVN!lj ziS_L&{v6V}oyEi5y2GdOkc>!!{>QF6gdV6YtNsC9wzJiDB?joVd<=TNe}HKg*F_lo z)bf{fwMLyE-?c~-03iF8-TVzIX~7GiT=ZeHks)Ki_Z!$RfVXy$b!jRIfle-YH4$3r z#K#k-Z>0PHKOdG>0h@$3k1aAQa+(W~8>9w);O^&aNMADS!+Qx@dxQlH!r>@UM_(zt zqwaqtQC5C1Z`eg{qK@0g0?iQrP}=4WlFnx=Vtu_!Y!STU3LO@f^irFhr}7;>PVV7{ z>0e@W)X+ko$=bAEnc2=;$ft&sW_aH`p6%P(a?brg3DSU=j2`E-Be0| znT?e*(3@>k$zpiZab9n}DV<_lzEB1t`IMDC#cgm9tNWVW4MH=#M-2IovNTI3d=ZK2 z^(70Ck#0X&es5@@H4qif-zWw!-VkhD)`~foTRPsey>hsyA({UOwc&VdZHBD$sMDgR z;Xx93h0I-;+hJB3-oR;gFL5a^C;TXme$2u=B-T`IN&aG^7iZ>?_d>bBuG`U)>}}Wh zCD2O3ywFk>PP*h!bIyrJb`97f2d7#?>upOab22_i6A;n?MfAwIF$+FjQa6jnR_7UL z<}y9{&I{N)Xh8=qMd+dsIm%#LA?eyr%vB4z9Khi0&(DxAS|;+t4L?D*6Z;>YnR(Fm z1$sRU>Fe_#>gUK#F(W=S@k6lK+C52!R-KujC` z2D~bx!y9g%;y)T#-7U@21Dvk&LOWi)`+*-OKUCd;-}vE7m)kfXGRZU)&w(`Na2TO{ zRhfgBVSUKyiM5#}^4hA|0X^X!KiY?N6@JuTadk`S6UHD#GB~ z)fLoZjaSqC`q<;3Ao_06;pt}httiPhrYw;7G>4QX&>Nzy5cDk*vG9I4-7|d@%W{lO zdUS=c_@U$yLOF~A!Bw;$; z-oHHcLSPDT&SLu^8?f@-z)18(p_416bTXi-JgFvfUf$n5w)|1&yV5q+FeDLp4&&ME zbC-JJnHC6X0(ubhP|~XV_%T1Pjc~c=1n?>jg~{2*ZUS2bIP4d{R;MsM)alglAfr9o zHk6Jz(CW1}E{h%lG4cB>r;3v-k-5&y)gnEm z8W$#u$C?!dvBL%{@VD98GoK$yJ>a3BeUaU@VAhLOdB?S-%`hlLD)VQqFXAZ+LSF&C zsa8PfF4SOEj!D1JG> zPAx{^cIx591Om{;05Nvd!DnFiG&rbDL17(t}dh`Fs#$OY-nfYR4iqOT`wN7sziX2mop zOnQNGsN}Wh^+f7t)0vDR{op|9tg`RS8;*H_UznNCZHjg82q>sN+C9Fb+kFEN_PAG* zhutA z?f5J;@4Bx_ldhulrl^P@y;l(h=^)aZlu!)4LkJO2P-)V8mnKbmO#qc%LhleX zKjTe^=+cNys6S-o=+aU6eWJ0RP0qXwwQb0;6-VZvh{JZ zH08NFQ>G>RFGVgp?i=jiKsI(&?gJT3Xhl*JxXw;qE9@{8XSTd?@HmnD8$EtZ{UL7EsF{bHH7dCqKCUMp|E$Y56> zDqokD#DaTW9-xbGzFhR=S)_+&&?%8c!aYI6skk$QMDg$jfq5wgIEPR?G^ZSIDG)LH`2$SIVL1~ z3z&oWH8L-dT}swb?e7j*mCQ_IkXgXv&ceds&xC%U&!x7*@|}a_rQj_NP*$rcu(~lK zQmAb&v|cVD?lFG)XT3Z4@bP~FQ~&@+M)fZM?!#%i(g@&YDQvc*c4bX970hJ0$*5WH zd}@)O>yd{15Jtv+tE<~ayS07wBdBW$l^oNmj${vvs^CzFT@2jRTP(g6MIU)h9r-v% zpzSO=&7?VMUwM~ZV6`dFPgR7E+AZGwiD>bg_Vz6Rq?>N{IR61U7z4I*J-pl?$HP8R zX4&NRkek7r5kKS~@$@E`WOR0QA2g@Fp49J3N7~f*4^VaRG4ahzoO^JwaC*~Dj$XP| zM*v5N&%V>8vF#WtUa3OGMJ(b_53SGgv0VQRf*Z`6YF2gmrql^TC#Zl%S)S`8g?Oad zg&j*BeEO+3+1rz5pnb6?1yE`JJVJaiq+`E4kEjKx#RrD`T9__Pz?T}&r7<3O8vuy> zOPBoXE5Lp+%+0FtA@&wOeo2bp+^q32vhXn7XBeq);Ty2jJ|+rNyz`{OOcj9Qnzq0H z<23*#a+-GA(}}j3_$<#ZY3HN`9F?GXR?w<`z|P6k ziEGCGIg$3r`WR~esW6z7GP=8r0OUE!ly7h4zEB7IAIaPO&KG%-;uuv=MC#>yTAN-U zSQ`zrS&NoKA`$h^a}-VaYz%1Xn4C+WaQHB77q~n|=?sbNZ{yfs+G;bn@;xz-(@~N^ zy5wnc%Q)=JHW7HA5;OkNn#_>Rms@;ps<6K5<7KTCpkpHiI16T@>d!R0PA86S*&Zx^ zC0Pj)Y@{QQtf{P>5C?YI z8KyB+MgLZ=HY{KfyG+pYL*o|*Hziu@lB6xcc69bLY>CQ5HtcsQQ?!_BJpGu2ENG~Z zG99h*J0y1=ArwyKVy@nh@te`TN%^6Un21_bQTaNw>n&9zsX9lgh!W%V&3J(1eCq!$ zRxu7I37Cu((MCW`3l(HDJ8*xPa8NasA07(SJoA=iBQAdEV->HR`Iv)sd&9;j*OZpG zeGxf;qoqP=!5hgX+d`5J^Tw0S))Rcrp+KE-VRr%5K1kzNy#&Yu<{qq;F$M-L<=U^u zcGK)$)I>rE%VYiHaQD-9^`xW1cOS&FL(guQw;o5hD=6%G^y(D>RrkuRM}Mhel~1oJ z_gARP&ATHoee_xDF-&K{5Ba%1$P`Q=rrU|efQs%H?t`LzAX1Ul7KlHPU|f@^38F8e zo2_mzo7R|Xn{Ou~5>oHYLYxy<#U~^Wh29a$iM2V|e4@NH{brGzqbzWi;l{9>cwq?tQ?{963|tuf(8QX&zfl%715Ud(lDz(^uu zDu&mA^{}ScLc`?0CY?q#BI0+WU@`kk8Imd}@2x40{G6Y6#dwpa@_wZy9ASU5t%Zt7 zYeHMwXnpyh-zQ56GP#D?Te21^Gv}FKhE4=0{k>163p<`2jD*VF1nPGHginYy<-|X_ z{s20B)45Wc_6{tapT0_2uGtkvR*J~7rPmr47ouZ78V{h$BK7S2*o|($0iYGEqBmUqZK%Q`xhT|3g{J3BM~HxkWe z>HPIj@iVCCbSCvg4LMQ)%J${DcsjkB>DM!L zr|gU2D<2qQ#!4$w`(vzJllDI7du?~abWTFl#Pz?wW=f)z1nhQcdi5VGC9L*tyZjOb zj67+DTJ&^Nyy@JWo0q!st?Ab4 z^06Z&diKSEzlCR~Jt@uMcwdvTX<(XN`b*971uraA+auh;;x|>Z->zZ6Udhkb!VMwp z{#*X|)pV3!7e#cr@x(9n_f}FuDGx6VfAEt@`!LDgR;~R-DA^id{9(96(;r*vyihMS z&{bW(w)pkdrmzidGLTC;oL9i>S4cg;QRBY+B!E*9Fux0AuGS~M`WVu40BWGmzrFbZP2g7S^qEJ9w4<&JQ z>hV#vX)W~$W8V&Si0RfwE{VETiiH_}L-67va-Z0M0#Wy4u#GD;V!M1~!^cJs=lLDy_IX6WKq zVSi+t$GN$k*3C@WlRKGGkVJXHxLMK3zzm0fiBP0L=)38GTVU>NpbB9FXjPyRp~k-s zqI~`9Uj8kf$I?P;)Z`NX#ljHKdC9!4*&J!UvU`YZruAw0wUrRPGIMKkG406a8n6&h z9GZ+zsFLyng>8*`^|Hv4+%Hox>@1MxinUVH@fAhI!_6Y)b>ek*&9 zjus|UUf^Vv)qM)tH1lbNZD!TQKLLD@dW!$WzUn`TZefjd<(a^~3Zz_ICQ~$BKGXl9 zHTsT-`TGtacR+*W&VMeV{a-adaB2T72<@LJ6+rzOb+5QY#BRv|&ZK|e<`guy2C&b& z0O?nf6Y`&n8NaO`M<%$90Mg?>2?fAzaT#6rPx{aV@P_^$I`(M_BL=+Ox&FHj|8Gpx zf6q_zSZf}E@$7ys9qe}E|eZ?R`UQVtLw^>5K;z)A$fnE`kBzmv;>JGLLVOaK#| zLh<-N$2I8q3xB*MX8;i5XkmY#khxh@ zYR`Yqc^~-4TmJuYCAiFX{3qNHDBXZq!pj!;uV|p`{dZgZzxmex|6~8(Su^0A+|_xf zcK6#uVQIyK5l$dD3V7MGd2yi2#io6ks|mCWMpU#VFHK^nH!>%35cl!LmjQ#~|J>K& zz>1&ug5wVGSN;v-@0Aql1OP%$$edX5#y=l77VGBgA)NnwZa4&5HTP&R{OhY5X5lIP z|N4rPn3*G;zv_Se?GnW=ubclm$lc-NX)hqJfeUo@^-;6S*>h6B@j~;->(Y|yl94;{ zUiJ3XDAu8?4>h92V^~~6E+AR7C`*N)Td-))4RO97-YzLQhzfuHeP;ASnpS&D5f^J) zl&5BjEU4Yrk91Rhw{j=7OFK5(&Y4-7Nqm=7|v7)D-7P< zfo2qB>M>6T(-W{(Uk$gY8VyZ@mIUw>Qj4c)tAsRnmy3mVo3OtcWijWeTkduW4BZgd z44T5aO2K_AUXC=;CHNJCMeCR+l{UATjqtzCs}w3RLX$Ep2{1BYZ6E8W&C`31rA{6J zzW5kj47pv9%qOvGg$5GvbTY8%9=9BHf2##UMTd9*XecaaT1b(q+XQrrWI%36KC2maV!c%Jdc5-D3&N%)CPK+H3j zQRnxQ1%9eH-zdnuXGu2Pp~rA~^4D}>f1?5{z&+sW>wrCR*mb#xVU)Jp2q?+l+Oq59a#LzYyegi*R?r4UXgilfXz=%xASsPmWB{wX&+c&EHg zt0a0Q*gv6MZY+L87x$ePd{_!Vhn5UXs(^`Q7II4D1%zb10(obm<)V^4>Dv7^XsJ%!S-fCeXvWE> zSqN8RO^*5Hc@`RhF=g2cEcJR$)@g>Tzq*;TpP0N-@|{0j$g|IGb?9oS&sNJ z|JwBIb#fDZRM&^f$3byXz5)A5=79@Y<&b07&1MI_fiw|RW7ky#78uNNET;Fo6~Ga! zW#^Q-4%rJk6A4pF1to80`J&6iR{2$&|84<#x&z?lVXoShoG5c*zJWcy2RXhCA7}Bu~cGCvhqngl2QOnw| z_1N2DQ3AD$ukvSCJR5_1idk5AQJui3NTEh8HWkDqE${3%5++9wr zgAg0DV$rxz%!~67S$6E5J%4qg3~nK@_N}vkt1lKdIU+BZW1{$_nl%+z! zo|@^#ZWektRlUPA1!J)rfOiTiEn8B`_IsYu^lBr6?1xn{NpZ%&DtQ){le>+hp6}_Y04b$&Muki!Ghzf@EN*`TAg)oNYFG1Tl^&{>wA- z&4$Ay-G-yYJAOW2=c?1h%=mv3+&{CinmV(FdO$8sT5>%B4P`h|(2WDjhDE;8%!7`q zM;yB}X)BX1YDbR*ZX!wa?A@6@wwCzl*BAzzFq_GcVBxuxJ8h*9mQ7G6jp$nki*78` zVuY#^^KAC+#dE39R;Cr2MY>YDs{yOa47fxU^96~+OjMA-v6GtFEe@OYP~Q>h$pEr) zPcdfPN~$jzKy-&iPwYIE8uE|UJ<8vH1Ba~2Bo6v-2XQ@|o0V`!{JAZ1Jm`^c_OvOc z`R1a{S%h!GW@49&ewBmNf;%ufAz2q?-0YONm-*u~JRoHvtixNb|EXE&r~}rG$+ngD zk@oAu$dPxRLp-L>8(3CSC(b{mI4u%Q-Ju#d!;=qH)kx^C?Z*6u0g3FXb*M*cg^JWwO?2F7BURn3+Q{w@unH?mYl@8e;>wkwGcPFPNQZD}Fz!;HNrOlCGn9-0wATb;Kt(7fXv86D{ zyNq_vZ9ZRL<#)9JZ+1O)x*H|Y;Iog>0=CBtUl5cxnQNFF8)^?FKMWezA9ew*-r5J7 zVdbY=Ic#7@2YB0&ENWeLvdJ!)e``~uSa1WA)ty)NG`+iZiojZcGPEMX6XJ_3D3NSnsT*W&ZLaIFm(*0d08toq7_Vfgu2EW z5x})~L{M(dbGC7OGr>EIeUIsT^6029epjxqmp(t*4@+1}Z(i8q+9)@4W3n30)aicj z0+tB~*o<3bK4qR)*qGnWl!JQj04DGtJ2MV^4m4PpoNz-6OLEXD^hkXesV`g8#@B(Z z*)Q5{86K(Y+)Z^GGWO(cWNo&;KwlzSF^nQ&Ks*(?b@pW_tCEd=cCbJP6tsGKWKd=> zkj9bqj!hEJG0h40B17%fTYAFqEb8?>sAU(#&C(UcART+!|E9)iz6JZ#)!(8hki$*x zz&>qdj*eA5zR<7!Fu(DF+?i|nbS>LMjn^^+V+4qV{%o_i#_aA{?zm!*K$Cmr{BgcU z4Mye)E;8krDT6mrjcflIeCAwDurKMbOm4RNBUh1qb+q%_^_?e`I$A^PGW2Qppk;fl zV}8h+`&zhVUN`4jwCe)+NT{q>WE1$USmlt|?!1_r&Ub3znbu_fMH2$kR<)msejTDj zhAd4G?@vkG&rR|Nbmd}F!*MvN+{nKJKX>KuzsrQFp;}tfWX$u~Dg27WsatAjk~RRN z`d50E{rYHuWm8BfC%Y>#&$ZQXj&tQoHfZ)x_C@NVRBg@MIjFMtRH zDZQV*BfF{qQKzy7kcrPK5k7s#IMiCN0{g03DrVmZ6L7GrbT;1|Wgjem>H3vOSqjUG zzqS5%M1|pG|58B%+ehmNir8#s^vq`k9qBiF6kIgND4g8#_4(bY{9!jwrkn*F>H?B2@7lI6#lY+qKl0oA{7n1c1cUnP>-X^W>M1XtfKdwD>)MT3+iC){ zDkM{x=h8h!3}0dUdztUX5#$wIr;N|5S_4C`P;_*vgQIX8+6R+!*(1GrmP}KpQ7rfU z1il%HGGc$}ZK&kW|jm?2zm+(#-3}G?-_(p|VS%w8;Gv#L23JOD$fB z z7SA)#n8;WkpzlvGvFa4bvmti>EUC$ z#eS6X$Q^VFZc#yR!Y!xyP5JqK~ZlYhpvE8V1@NqDuN^18?d%WsN1<+~!zqoSEtZtwgC2C?N- zpQtx#Jmt&X!f(1>RK}9E4NKrF1YCE@&9dcAxsrKYnL!jgZ6$T-b?UNN59dH38zY|C+=1o}TW%(OCglD$kjZDx%lVN@7xd+|9%} zW8|&Ba_rqrAa|I5^R7LVR!-fpk`NknvI1ko`crmXug3I$ge}umw(Z`{6!+@%v|~KE zZcS^#nSH!f!+W-K*HtMr_X1D2qgx~E2Z;Ax&zjEDcGDTD7}uQdx1Hm|GmvOA6!aA9 zq+={Sm+e}cf{83rkS9=J6jY)!mT_y-RkV#tH!^LD-{ouI`z_C7bi;Ob*y@<)Bz|7Q z=j`8c5B6BsQwm4QUo=<7Mhasl47GP#f?=^ZS(}6f1G0A{L&*tggUs1td#(wl8FpQ5 zkFf6i2}B>Y6G$GRf&{(Ze?nKblHoq!*s(Gj&L6?cP;I6F&5RJN@VXEh+U&EPeF+CR zGM9HjXI|gkX(o(TixG#Q+zRzlEE4ut88S3H?wh7Ezvs~Zhw%FKQuBI*X zJE&4B6N~PYxy;BqVN;U3wIaW0-rCKo(BPET&nYRu`RiV)?%gMlLlq!z7 z&A1y=%{N(0#F?ZMUNg$H8UY%j@vDq%MK@tAF-D3lL?F7wu|1N+gK!pZ#>00Hqt|KW zWxG#i8&IjT#~m^Hpa5_b?7N)(vg0gII&&Xenh70dpz@KN=M=Ye=&=Z zGs(Dsj{UQs_jm#=>LQ9g@OTD+tJ20S1~5wZAue(+nk5;e4MuK$Zao;Lbal(Jo8SNj zKneQ$c${35z2H;w98c7p>Ja&Dvi7k<)Fk?eAK zSm@PHFK*wbyms%N;;YyfH@;+qz9j9<$;sh-dFyJg?$g+>A)KT-Pp>KpVIc4m4m@i2 zMVkd&9x1&)UNLyJYP#t84z|O`#tLj76_R0kVUWky1m9>0%-%~(b ztm*g{-6SIX+l8dl^b*=Tu&$UJGfu0XeP&Y59dlTp$C-=xbAQqfc1*Y^d;h~^zmT;q zsFHe}0(rRUIm1er|7v)Bm@#`c^(TDtBq*qQqK;!ObU*OGqM`_Gy5GOr2>UKYtOmsx zUn4lN?=SH>yUX=TE1Fzpf+v5p1VFo`hk_S3L>{*gdq+w+OO~~@ZC7k# zJB_D9B4&R{ip%%jA3A^7W=FK$F66`yS)F1J%yXiwFXMCXU_1}Eyui0jvO2Yap*xo1 zU7-Zu0}APEar$3z?;R{ggo2(4Y-8B%8|7TUUpP6lMZ;p#OyPP4iEy>tD#=W`-ttVndtmq0M*Rlsr z0XCZS7f0B+^3`_NAOh;`c~pHVY$99JIqF!gtuR)1r_)k+x=MYDRida?4ibkABMK6q zQ#m1)LL4pmLQ?mC#(8e>X+fPiV}}FBFtZvVv3rq6Cmw`GdqXi)veeNGugl|1M zxN`6{IR(A68<+2T-(2-V9yZ7(U{aZonjKljslHCOA<$@d-l+Ai_Tby%WL1SJDqU`F zwxCYz6wxba^IpN7Kuyw`yklVGShv%~(q?L-WmiIDwexmYy0&yyJMp>f21!@sk#Vk%T?7&;CuWCDOL-^Al zu2pj~UWsI;P^2!l5Nd6buy$=;?_Xe{H!hP9uIEwo>IskL?Tol;<$lNfA?J1JFyEG` z&SVQi3UV7eFSpdc!s(1Z6JS~gz%J1YS)D;0d?qMA;M2Gs+EHif+UBY0rj|ht!o642 z%t2IPf0kA>m(zR38mu6Svn69>_d?IH{>Cwt^r3nL_JFnCTdyOIR{ng=dlZQOQQF)F z+s`Xtd^|P8G2N07W$ufroCOzU;>uJ9m^dF}EfS3Tg4=(f;Ep3Fx0Qc%SbM4nT)!>? z>z}l1tfe9!N(UJ-vigNs3$*^#u4C!)-2SyY&xUGKsUtLD%tu++klwDaZoEECMC*$I zRJ_^ig@{q_&P0i+i{zas>T7EorLNfX3RjNQ1%;Dd4&L_RYupNfJH88wE0JPBiw0Vm zJ_6DAwG!K)W2UQMj*GS52E8Gp&~=zAf1-;6o7A{4BDi(%@lc zP%;35X!_#3Swr}{A_9cR2lBOXeSf&%e_mMaO4vktW<1+r4r@3{9b@^v8rvQzT~AW9 z&=2P^y{&6ZIoo&7bvJkE_&g~{*LuEZ+(V&HpmPXSQlL=tFb^d6Fa9+j76V3BR^e zMk#c0D$Pia*0cI&jLxb+jE$JCRrr6G5R`P3sxjPxsZ^bu)%j}hK0zNs>KcohwCOXZ zlRvOdVp&O)zB|oWke^scnIuG6Y1_q_&lWF)ZgmL5HH5LXr9nsVdiZ&mK;o;43aWdD zcjwH(6=^jItcp^g8#-1?y*lGs_#W0jvVZ6H6`&xd&>bP3R9TVG3_H&L0}=Mfn|(AO zi;Hv)ygs3qVMUB4L4WwAf93J~V8L^q$~KZp$zgNfU*XzYI~$nwlA>%wuz^%un#NAl z60*la zNW&#)kygKJ6ZU-Tc11_}(A%`(%%Z*bkUP^ut8Tj)O6Z}bSVgtE$9lZceGTGzO9s>1 zm@A7)~-k@Tc0y0Jkd_D7GU16|2U?Zyy`os5?Y#n334H7*k@gpe=Op^Q0vD$T(9uPf*25{RZ-T4i#={!mxvJ)E7Xi=psaMwpUE*oEg) zx8{rObovq@)xw7NQQ!)AO$$hke@ue^VoP%Yjtc?@kVS}%UqorzBq?e!{B01@6sMqP z!P<7dAT<9*@Z!Lf-QT>c)~(Rq<~PQ1as3sWi1A8prSI-d9)JClu7pu}l0Os{9Aw&z zZR*W+)M^YH@DQkuWW}_D@wp!OHhk}QevF|4Y#xl0qbc^O{~7G$uy4jBxiugs{T{p? z&-$D@8~d)w~M`}!mnTA#|Gq6V^q z{PV!ADvl`{BAX6=c^aQyvbwI}A@r8Aurc$;aT~OoQvMa)eF8`dPC+kj;MIk zNnM;+7zgxVedk?h3?D(&K)xs$Vx+R3x_tj+t-;AjOCH>DJo2nEEX@*1kQ~vH+D1>r z_(YZD?`k%PMklKHXBOtPh9X4QJn)%Mj+#nc$|2t{43N3)Rg3T8RAl`Po(z{R6$jL2^lGCW+{3yz`r0G#r`HWfj zq0PIYs`D0a{kExuth?q-9Z`V!r2YE7+F@Yn8Z4`z#=MNkSHpvxI4!$3nsikpghY(LR3ta9esSwKcFJ*@13{nCb zbck`q9uAhaAYtc}3sBlXi1by5FQ&ZuZ@SKw-iS*qWbdqHRhWX|)NflSm`7r;z6Z@2 zyZj?o;+a42Cpk*dJWbr+*Il_)Un26x0_0iB2ixjfyoaxnv4o&a=1e3axyXxd!A}s= zHw-Pj)lXuREUXIBXp=3i^&@IOa8ZUPk$7~GKrYT7SKB%SH)$*!# zn^4z-%Brf~U{&N3f>Ze$nJ=`VY%R*YikpJ|oScMMy&-;?xW!SgR6ES+>8yp%-V~#{ zyolXE2M8}cHZw65t#9*dzD8M{ndPv%iMEZoNqG7#Az)(8^OjbR!e-O2gi(?JsjnPc zUq@-@TrNh7KDII^#K<8p-pYz9q})e8NV3Rvx-fNtw{;yfKp+~!FXRT;HiM$-2(*>L z`vSZ*RVw_*1sBW?HN0He5DgIypT7W})XI7%sN8Xe!8fsBIVZh$brt0|CpUea8yyLQ z*_{$ia1=f`YrwE*o1M` z{k~X=OH)zH?y3L|B{0e6f6#jfe-ZYjd1bQ0inl#vl~r&Fx54mAhK&)s^P(hw-ks5H zXK_|LrN6SV3c+=I?k3%N{T;J|sd~j&mxyo_q!7}-A>mJ&5B8v2B)&T!b$23EDT^pZ z>BXthvF*?FgTZx0rbL3>t+_m9hYG~xmpcO^{Y|PTwdJqA!g-7{!ZHHr^}~o(HVoVF zF%+y0uxR}`3i8$n{_{Ydk}N4%-_F^)oIj0p)R+>3uhp)-QpY(XUhr@V_?C@^JnOih z!$i8{Jkq*4EA!axnNXcfv6L`WAb;tbh2wBgy#@;rtIzFR3igADu%p<_r=Ny*HXikK z<(~Pv2C0*4(8xetZW_>}+9lk^85df*&5e*`wRFpq8IqzQ;)~Vv;t}29ko+zvU%d?C zA{2Bb+O9>D!c*hnlv_f3^`R`3faUM5c`**}$?Ui)@<=p6-!o?P8A4|ZBt+#5x2-JB z)QJ$tZ|3od|N113b@}H=;7?t{+rDZW)`X0=XoKN))}?{vkN47fOhUY8LzQYW{riMC zKR!D#uar2KwQnO*FzetIPW8-0|8TdSO&oPI7ibyy-SO9=Jb@*w1xf7B6XCiA(u(g* zBya3(MKjt{5}uRWf3LIe4ZACl0CF><>X7?WIR(T9ns zM$DCZdcizK?MX4uPCwbrE$u0h$o@WCvw)_6B(B;4b<=f{7N(SIBHmd}3NB~a8hZq% zNEV|?15?ih>B6Tc2V*7#Kg^X}HxHNDs%MIci8dNvIHGgnqY8&^>w~*>hCu3M-z#RG zpqZ$+KG$Ewo3CbYhMp{mC?2cx!z;QzZ9TnY-)g$=!7SMhEo}qo*c>b+q3cv^@Ws+w;S@^%N}}+4veK&Q7e> zqbL0GbhgQ(3Y~W>kN}b}%U6n>tCRRvWhM7g;=z%#JTG<<;xnn%F%{k;$1^R- zkV*dP0i)V|t+4xSiOyDAqm~a;r93*L$??NCA}v7D!-^7B=Rp`5g~xNTS`XLW$hlkY zRXh-)>XnVJVp&kx`?6)Y!s#i{_}5c!C>#jrIs3SCs!hlXN!fT)UPUunZjOcEhbQN@ zUuiuiXAt&}*#@QOKbze#9#(y@Qr9LGTn6*9-@beD{!ARn77mKahfFl>6FlphI1|M} zokv;M`*NRbyP?-4EW0J;Cf4ZSJj_I$H|sos!FvXPn9z z+cQYA=?Xso5nun{RLUUie&ug-%H`;KJyt&rHk(cF~2W2+3I>Njmov$$*kmnQri-?>#|iQUEMMm^dDh*m!95+U;}xE4PpE!`BMP zwMBfgQWT%iA}dRQTwi6UvwsM)n*(!Rpd`-jaM<>Kog;Yu*7$K~E zU|17Ma(|6GI#e%ubVOUWbaompG3(`WIC$#A?h_t4<$5bcPeD2k}vimu?goFC-(O(#ViK)Lsy){ zwYIu!U~ad-44W;>4RZl4u`%!2rFsgjrd6jQE=h!-8IUlengs>!W+PSOT5 zOg*A)Nr$g83hu=sf5yIYa&a!RSe;TW$}ryt0CFgF=;C?BnCVR6H}lss`woNmQjgvmrl*RI~XIz(%2wXZV_r`R6XY$W956*d~)6+AoBSG?-8x#8Q9TWl)^3|uO=N3K`P1*j%)b}%uYVHqF zqt`GXN*0-eY&J+s$*m6-eA5D-YQnw?uYQxBfPcE;T3BKgHwrxTo zd-vrVfTg$#zqb2SivG-^i%}&p-=9i4x~z}S$Yc7!Gabg1yrby5H~n(>rlY~Qk6c~} z1#;>J#pSWc!%+PkB?IZ;_D7gUzEM}5o=yLeV^JglakrCyx6zigQCr(FT3O|5T5VP& zXRI(UY$=y`#mZqFN6CKN<^YyjN9_6W%+9L#zZe8D-FO$t5~1q3=FTqp>Ql~?&ut2O zQaW+g(dxrF&$_F>+pkx*zdC-tyO4Y(_LGWK&9sKe&2)*N)r5ADpzB#yciReQi`go3 zGFZ>4>`FiylGltnGrd7*&LkMy`ClTc%dFy z?=f=?7ULWO&7RTxBSIne!N$dvwE@kzILCW&`(iLa$tg}nQB)a|Qa&my#;6zS={|#x zzQW)&(w1-+-7&&Y{g}97R@W0BufrfY3qdJ89(%Vx&rjxrcg44!xU$P#EyuzB(8AWUVVV6GWwr z6iIp%I*|=ErNo~&U6&n7+7Ilh9>>jot|?-&h!nT!X)QVs-1=75vOTvj$xBMb*rIz- z8mTeW68>+nt58p)b1l2H(-E&wJrf28Tz0D<{p%q>d?ULEQiTqW&G z-t%nKwaa7yo5dOqE!r^($MJlxwwBD9O+y2(_s$?H&($v@eTq9^u+1;;YK}uI5wq@1 z5guZTS=qc6BZ)07hpXF3i38K$&v`5Beg>ceFpEJrh`{T2$}Pu>C-6xh8clDp#XD;l575_Sr{>14=_NNm(+5VcbFAK$r;GN@ z%!bEuUBB1kJb08EU<~738)d=wJ(Mx7&lP@ER?a5u5q@O+&8N!<#m22G+&!HkG9NB) zWAT;+>HBMg1imviR3{$P%hG(MiQm(^nF=2K>xA2LtghUwYw12)a=vJ4{uysYbpVKZ zakg@O(0@b5qc(&eXb-M3bHysf1-`aWQFYVcsRlip91Ok zYFnv4eM_xFgM9T69b2(+{OMuVrSZf@j-1z9;EBs-8!B=Cde>%@gmkL`5MyR6yN2Q8?xzo;F*0w+Qs49?Cb<~(6${) zYIave0pCq=E2SpRgmvQK)t7S~RJ7fiw9a`NAs{C^^ht$)+-q<#LO3g0!OFqVnQM6M zAk3;^$`%BffT$sNWwD|+UKVlGA7;pP+1>UMqAyduwzBE`-F#kjBbXz zl%WO-C_ZaMk$4(e=i1ZY@M`O9fn?WcZt>1#6hZ3lE6Rl47p$e@D&tv9wnq#d>~yH= zc-WWZvPq6fz9m7Wl@|_X*o}nnJKfR>@;aE)Q~=G%5HYT<0@p9|Jw97IaL78*BnFXje#MsG3Lq0Q~%yZc#%9y<=j_BuaMj) zJ>3jJXWcGX3k8?#hMfCzqbPjZy!$|vH|qG!SiN}h!YRRIke~e)V3S^G-2fv-wzso< zH$EPHwMh{vk6I(k;%4e19KSqHh}53?q8KOKim|ERn(pE>46E9@@M1(zDr#k?LK3$m z?|2Ed-51=y6HBs84xJw3_V zMyZE36lo=jwr+fHaTprUV5=Pa>TnoryR6c7;#7zH^3+Px^xLhhvh+X(+nyK6W@DK% zbFF>+YY*33;4SvjcmLvBHgL|)ueoJ@MRBxbN7DMFb0yVGj8Q1c%cx5JXiy=v4Zh7!Z-J9ARb#U_P zxUj-1iTOcMvS#H}Od>|T$|aSG-V9X4DY;s}=4`9$RyjGDuv9~0UU=IBRGg#k`bKYW zAfMiABz*QOpg18dhDkQ+XOd*Rv~kF0zm567lk%(jl+A97JSwGnELb1bNK*62J069- zVRoF&Q#ZU$G(jJznRt1%Tt|Ggk%t~@o)BY)GTk-Va4QGjA6~ zkMlkKMb}0%qBjZ<+j4RnbF*(YEu=69$@&HJD|}fL&9#}(a`X8X67xr9V@R-K^5RG9 zr*2OrNIoh`5}VUx-AR?DQ~J}c_@F38;dfaHf(pb17&;wsZaeHCBBL7F&^Mn!5jS!3_LH@}of+!pcX4Y>R8$^T zUYSm$?v%~7?fF|Pmj2Eb%(22q-5go5xf)63Dhy}aT@f*E zx#MN*>DTLGda+KzTC=>Rfwp&uFW@DAhfeWKr2#E^%u_2SkE!|4u9%d|?=N1p323#5 ze|BQJnq2k9-ix;_$VD;Pr*&GV=#HAfb)6_O^T5dv{vg_DQh|&64OqnKZ*C$Ux_!%tpyB!=TNdrmD=`$I;VyV+}`YpM;YO9>#@xwMwG+J1&FYr%nAK;d&rJgytrVyi~WIEfHfQJ;L`j3FQv zx)t12*Vnvu?*zLunLWJwraH!9b(x9>l$&?Ss_(;B-}?j>Pv6t*i=IJ^#qhB<-08BRHfJbCx#7I%ekOiR~Y4f=rx%R7qAJe zdV)%}@cqQ*>{n=FR5jk?94^c8lpcao-P%{2F7f!S3ChhYVmQI)q}CPEOny=*LoUo_Rhfb)UwoQ2M( zS9F~__@oPoGkaB`LD!o#NH~)iy!?y3K7O!@sSS#!atNHexv-inW(z!;TQ^aTdM%&A zbNau$T$c~w_tKHMS(hvBpqM;@Q?5I$t$J&Jn_Czj0xIv3^KCUUUvpTAFK&m zB&+6Yg~Vua-=%x&yFB`2CSZLXxYQN{m~s?}kxM{oP6V>&5jY!G)YDqD54ZOh`{XD*n zAu$h&S2!*jD5n-rqiA+gAU2wBfq%<~id$5E_)s4(p_p6_Y*6|s?{1S3X?!*BDhLL> zVfFd-N$efdEzXFxAh_<9nP5qe!|qXs5It)+;21G4t zA|jA6 zUr%ZaR+6}|uX3-4ZC!HVn>933(jz)8h)vxp%~>jx1$&7nKv>SR1;fkLENO|_vD%4S z=5W~3DfVj%g>roI?u)NXcSW{XV&edj6wf;najF5Cu;XFel~cKi{lgL$+T?0Y8LlpU@n&ObWWRy%c!?b>4qtMHoSL%M@dP{4FD|4J39n zU!@wm*7at$pH94!aqE52yfHu?A!%u6Ehc*)bb}Jp`{_Y*ROa?J11rpZ=RbRQEX>my|67uPae8zr78aE|W zc`~i)TTu3Ci_78a^IfrH{wkL|Ex#FsDT6&Yl3gQ5OH#exMr)oi*WvA9N-GR6JSP;j ze%A)M_m63|E^titbvvey{C(WIUd<-!0xU|{wj#z&N?wQ*>Wf^X`IjBt(Ob z#G*BK5WQ|AE-y*L^|L!wo`r4tVjO3sJmQq8!~9~%8Nj8CgM(t0yb{}K5mSarUdzQN$MF8_P=!lGG&%G;W)9u`8PB{V zLrK4c&>nvz1KS_45*e^C%0maucyKBZb8o2XT%uk=iGH~QaO>!Wst@N9f4;UMzAe3j+4YtgarouY@#ve)J5+8* z8~eN}Dtz3tQU1esG4Je|t+Tjv0-FPi61Z7geEYX5VYl&!#uf-kbWoB)3an z0maOYHm=EZg|T5H*^oS~EA=|ah4ksUSOYP=Vxm1=ELXm2-1rU`ri{s5&Mxq?=iDwA z#;L(;rFidj+GQt$Ansl7rt+qI9EsBPDoWeVr>zFH7kt0)!mzCqgxV2qdap>7rUtOQ z%U$93=td^oMM=2I^pqVTYrZ7v$hid$^yUv}>k)>UIv;)h)}!9$b5jqIVNue7h`PV& zOhzZ7KszL4_RRAIZQPw@3LPe=S-W8+XUe=AKsnA9SPL@#(Mf>IM?aV39gPGJS;43rbnVHlY%euCb#vogAIsGi88CDzW7iPI#a2;76J9%fUC66( z!G5{DH(&zlE8_5|M=9wT_Bt`{RMTEA8X;*DqzN(!mkO$@EN%n!6N$Zirw~V;18ThK>Sn4RoLD zr8Hu0N9>6hrCiw}qz#T(dq3*Q{;ntkVeG;Ty$*|pcEc$}3y#A1R!XAYF*~tWk#O?h znbbMWE3X@B$AlO62~Wt7Zsx>X3R7+}5rmKIepE32CDGmJU5R$aBa;%rZB|A5oi=Jq z2)SEYQKrB$&5nNLF(|kr?&fmYPPa~R*&{jmF_Jd!WF^t%g0GR(W@=TbE4rm{JK8b7 zJ=Ecvcr8SK6vE{>J$?N|CE=is`wPzaGI0M|=Yg`n`y3~RtAO`sKw6cS4ecMc96c$t zwPK@(hRB;>n>s1giljST5aJi(6jZ4wK<@4b$iZuqqSwW)KS^L+xY2gvi1D>IU6WK{ z%sn&MgQb5!Za|ky&`uK}@K>k#*M;48*H-mgsm&=Y3uV9p>OJ9$fwU~>5x<$v!_aN_ zSZ>Zz$up^%YgK+kKBa%&|Jp-vw?+eTsP?{>H99@)=^f(R&U%M)$@PS}XX`b4^}2JH zVZ}=aB9PTdgVz0`{u8ecHXcU}J(UaE{P!09(+!bKLduuMm9op&I>3~CyTAHYP4`;O z8ad>b)84Pc40|MuE=e_)u&kHl7$=`qW99Rmwx7a$YrAQLIeV7}isQ_nv$rdxT13rU?o~y3y>!7` z$`ahcM1pCmwIaSEbZMogC`dwvg>{0bU4D~r<6>&?UsaYxh|(H%?>_TPOW^ZaklRow zH#T1PR%BR>Q?b_6sQopJJ;w!TW}uE2IR`Pg9wMv@a+)gGs0`jTnj<}Ph_U%N!+D7OVV(c3+ zAtZ*`U@Nk;1Yc>iFrU?y^PQ3W{>FAP(Z1hN^_eB`+NZnRamm_bZ|*)TW?TbqSLcHz ztMKQmZqZVV?knQ~3Gk7ajm-%aqV2tIYot)+SqIsXDv~OIxI;Lie#P6VrzKCN={05F zSzQ0KoQzN>^0m>ZdSpy2BcaUkZrf_cI@U()WAya!raUi6C6f+9=J{U0$ghQk#Hk1{6Z}DZt12c>F+CNRzsv9&)7Z)v{0tkm9K<;TL#|x+xox9qQL(PyY|kLhk3W8B{XL5B()9hT^nLb|nkz0A zihk@gnLunt*S(Ha?yTGJtwxxX8J&5;NLO2;dF@+KFKF%z1$K+byoA`+mO0Q(MNJhQ zH|N!&!nag(So`=zuxTKlB;;qvLsh5R%pEUv`}Giy-l^F{B$$(?zxmw?0hB~YmI3;y z8q}foNI~*B0z&RS*m#~`0cwE&YD*_bdz24))W@5i zqBCHK({A_LFS~5=d0*wA(M)AMwAhb3pskzl2J{B|b}k9_{AWwa(IphEk4HX?Oqk|L zEj4jKT-iH;Vzy;k*v`Lc_ND_KzPzg7htQ=&wRGid+EyMl=}5l7i>QS7^nmdsJ3{2Z z_TA-$%~XwTshzhlEgrCY!wag zxeUnNeA!*-X%~pW%xj;m(z)hGW?dr_IIM}DN%lhrkc^qI?eHpge=)q%up2#*2s6%$ zbs-=;7$Tb5)mq;#-vu7PQZ9u)k&A~+bT|`}^&ovBx_9D-;{+d@-tTK+xAZx1G{`O} zZh{-wF3uys&TrX0cexkgYY6@%(ceTU>9ln_la{=u!svK&)tx@|kyQ~!3Dg~yhYY@A z#@y@+UpUoMzwWZDBfQgjJ+KuBWUcMcWxr*p?DOq~gkEjJ-DBGupR#_$^q9JO|J zc{eqM$|f^<;yNLp3DF}2a;4+tkvly2^3EEnCUVH=J5Wz3rK&qPVsIrdeo5nUN6#*^ zN6FDDj#0#e%}EX#wWL@5aG>AL5uo5AUtH!eV$_8A>XNZs$`cOPUS_SC2QPM$)CI5v z_E;T@AmoM(@cPvLH_WYNz2*`|MFeDRqpe^r0T@S+?79jmr7w zGd_R8!1ZqolP#JEkpjTD{7 z@e=OWW|l~VXOc`Nr5sFUq}dXkRXO$F?n^#_9@=)|Lt(&8M1i41$>VoLnQi3hsls*N zxfm7aC)}@O;hTxC$_mU-8l*7G_L{u9S%y>WS)=23yE6Ayl%0ML%+FM%KhGB+WelRIikWKY>bB~t!sG*hJF(M=bJOJH7-%NDIHo4ePV|;9-UY?X|@q;eDwOA zXs0d-pTQu-7(qh3RxP8o&LJ^X>OqCEBmur~AG;~KvUwC0W?EOdkgL-ovvuvJ6-iz8 z_)zyAnnJl%MtT8QRy9w{p~YkI6UTM^p)jqn_fp=zq)CP5yh@wL4XH@mV>9VrqH6nK zno-D2JT{09hf$IMCeqCE1g)D-nl|a4xw&rFVHMm|!s^ebiENS%B(xLm%e}RXH;)QH zeIK%6F!uB5G1QeqZdnU1L2;dNhdN{c8`+|PZ@d=1Tj~`$K2?vgWYY?$3@I9=Z<=(mdaM1dWiAUd_D`w4X^z~khT{t(L&mIcokFWC~S2-SGY5 zcDJl%;f^f6RQ}Xe+LSuQaq*|a&vqwxBK;_~lp2stY|dmcGh9w{_EFmA&MwYW$a+XS z?vT&4!uJM@`ras86!3eGL!olhE4KmjAso|$G_wVDL#t(%=F@Uk%)|<=4xhI9(qZC^ zX9JW)V8d3OXr~e3wl55tw3~Hm3hahoE^|@2+%i`ErkI`T8c?`<;(GQY|dfZ<~93jYd7LqB_#S5at2}i29V*u*xZIFT}ms@T2qWb;?1lNG)6>y zz3=kmooL`k2vnY-b|i}!8|+si2Q5^2u=AeG9kpRdQ+Re^fJ5|~!bizUF2E)*58B<; z1q&?A7RhM3W7Dd_TPZsV)}1)qBnbG)7sASTpbTf)g`$L#JO7%NdpEb)!B&rpAc3Tr zSdbG=XMH8+YW}x9?1Qq8gtaPnbu7lDc&?M;GE zy@KQ9k)xcU^op%L88p{xo-CXGEgegF#;B>)Lx(+AkY_{hzJsok{pL36z7QNM#Ji6! zIkm*g{$yW%3%1 z9{q}~na&1jjP$W?7WJfFcJS**rnk76vXzL-c`DUK&7lXI-ceAD4qK+j3vkdW$Njw2 zj&ir>9_=p}?=Qay*DznbP|JYK%bWym-p>hWCVRb&W+0Q|mvhcmU=Cn$D>v(OVfzIU zQ{TOT!KO7MUgT$YsjDjakUCyxNQtHU@EZY!+p=?uzFXC4X{1i8ZE%@9e_Gy{+!K41 z&SvLtT18&m><~@^=noD*(vj9(IUF)bBT&yD$-gCdcBZC z8|*!js?C5m8Cwa7Vd~e!>%qa>>+B& zk+Z%cRVrGhSvR)Qj-EFgYDjJ3^XjeS3sssZs%gO$R#!unL+*>Jk4UQOI9Y4!4W*1# zy{t}3zy99P$mupF{L}ekt7aEhT5`-XyU5L4D5cWA|PKRAFsfT&P z%c3p|y8yB>sDmeqn4{~`68Tm4(D{;Xt;)G;-o@zdch3$QK0hzt)ZL@|G)_S*TpM8J z7T#Yy)m}8*@w;;5ncM%Iq~K?`M~iG^P4uJO>?E? z;d=(H)4*%jt~y~|vv$eDuSe%Y#HO>Qz@&^m0Iq5FQAsNz%c2(Mya-hp!C&%vMDD=8nSdu+iDiyCQw1GB}(uQ3fm75K^qiURiI5+z`ZX?pH zDnHF~_Hdz#?}C*g_$~NklutD=fYSlqcpk1lBhxhDolxd!^y+*}`FG-)9{?}Hk6uuFEdqIen}mu9;4i2dO9`3}K{ zxU?K=D!9 z_`2iGK&q9am{CxB#lyP%&#)^BIXI_)XuM0Ds1vyY`0aiC2TD}L{ke3lQ%e=mCLTPp zYJ1130nDLvOVMb1s*!R$1C?s)a!NI?!^V3Q>oomvlUo`^%sYS)2Uv%gJ2Xo(hS+08 z%K_BHC_T3qY?xGc4(dUltGqs(I2Y|l%UL5Bu`F%&ynbT#{rMhv6-f-kqAsXdI;F)? zFmu*u5v)s#$Ns9dB*YEcQY>1-ak$b!MXO{prcDa;u*Li4elZO&fpAA_)Dm+CKeKyi zN0xb@t*5_oez(a)5;HeAv-AG+v}^dtc#J zx!K_da{7;LF+h(dc*dX0(PbbEW2 z6y#o~Fcl47!nOJ@b_w=?_SAmJ1$0cv$f2QKRaaV{Y@?brg|Ft)mx&MHGtb8hGr>5o zJ87PTXk~Y}f=erK(N9y%FWGWP$Sp48ROke_^ST{AK4Qq1>JksdO+9{t+3Dl%U(70; zVo{&E599#@5zwX{hYr%FT^TM8f~Sqaufd4L?4`JzL0z47PjG=spTgK!5zcJ6w#^L5 zk~TZ6$w}_dqx4RxY5$Fl#Z9@UIJfZ;~&rG_a&{m#m9pQ7{dMU)F_UScLag46zrDCnmPfYCW^;|8N%xtfkPcx_cyY~^k z&(gkes9UnH zt;IuLUT?OPKHz0E+fBX)wc!?7IzI8r*Iho;u*VHQG8L5PnBw ziW68ouLwR3=6LuTmVrEmyp9#MD z3%8f^V1>$g(T=IO?X8O`a|APiIDNMm6eiphh(*Y%%0`kA3*!d9jMbs0V9|xjk6D!^ zt2aj@qn$lsEeY>umNN@=TJ-eo^(|u)b0oKIi2*iz$3@({@E=r zZlGoE#=1Svt7;gx!{3$)($y{yZR&L!)BX!wCv6pF;*Q7UWdr~@m|hVSEON5_GmG)D zRPAwu_f97np2zj~R=}`2m2@DG5ir{`d#2(`t#(03dm@-E18!e9AIst52p*pg;)6A2 z*E0DvOPRf{aJA*;xr$xqU={H2g{3~f{j{Z_2!J0gk~_L0%?Ub3ytm>Wtcda&*M_fr zgLTU2jk8nFs8zNGxiPPatRL`P{s+0n~LDFqlftM;DI~g6_yD;Kk1IgU!?4G4k z)5mG&+nHtrX)KMVYDgBIvUBzrJeT1?9kIJSi1(LHG8l*FoNMz~$hxAQbT0MH&VAsn zl};1dV!mV7r&q0NDzBMeXgQfV_~Y^FZZtLAR$js4E}B{`pqVo6(=1Z=bcHYBht;hvajk5+4kuNPd|ZDF>HK(if6~G@`15pNjf(mM5DMPpD@n^K zZZfOv7Ii3`(aG_-lpRtFUn3Q(+`80E$huQ~X=2pG!MCqtpe|>^NJj6JzdERDL07&c zUJ&pxqSle-Wj#2_F^PTwQ?iU||EPd53ZEnz!3$ch#uqRig8y4A=*qJioRwRWaUbpIfUdcSw!o>F9nYq|bPepF8w? z#4)dCg3;?;y+CdQE^{N{`_u;8UB=m-ZVqYjIezdO*nQo!M~U!*YOND}u%^kT>2q0E z5)p|3zPQ?fkFv8)q6qHzq+Lm~(^}R;%oQ~U3Ia(z3z?)cQt<4g6F%IaHLlg6x+ccp zP=%Mw3gCt6(P{AuA_t%DzTi)a%{Kl~61jh0?L~I`=w{o0|07azR$P$HRZvId;7RwR zfBCkWCH+%=y8DwG;&-JTY1@QtZ(Z&=@}u6<`P2jYziNGQ{-mEY_lrcW`tM4mrB{B} zM<4#5`bU2ksd4|SgVrs!>ZZRN5DxyX^~qUWKdEM9<`*l-|I3OoLXxX=A;Mv839n=< zWv&*q|HtS-{KM0W1p8K}M83I0C~4PZ*b*8`u9)%Klm zgZL!cEErOr{ooz1Cot$Le`=eXZ#(j1Tgc1$c5hGH+*#myz`x?;sl{fu50&(1w}10V zq)Au@nXwa(BxZUxe|~GM+#gbXw|e+``vALmsmARt@ZrUiRRbtLET0?H%jvw%w8nTW zr}nz61l5}4tdOy<9oF`@<-6aupP85~*8o)AfX>cLL9%MTr_vde+TrCTcwLLQ&-OYQ z-LyJoU3ng?7KA+6c-Fus8sFfD^XyK3@i6B|1O!|s2Wa;VNW2@!VT`T%a42H@REs*vL^+B5P<@H%`kgT4oWSY#q|+bF`=8A) zI?bf3g-;Kk3hM%z?{e1b=no^g+`xnJ*ipoSMFxqN1|PtsQPh(O4=BW!c+&hAK~Oki z4jgNk&!`+0Rmc0=u7#{IdZ&8Fn~4u=#neXjj#s#a2kvBB>9Xaq9s^y{v_jU13@nXZ zs$e$lQNtvoUCNu==pWE5b{3~7vzK42GVIxi$i9`BM2-BRm3cy#GW$cAa!5gjvN5aM zCg{Zk+ECJl^>^9rZgC$_%BQQX1)1xgdW6KkF^8cqfHOE9Z|5W*7VLCVimO5aQBfw4zf0Mh;3fTWPT-2{ycHQQ%7^ zg2BJny?<%TISAR0vA!;!+&jV`?`3sX6L3KKXEiR2P5`4;)k$W8b1mM$8gY*Eoj$)S zYZ$qTU={`y*G%C&A$B1v1s)8XnWqgHdHp*D*M&hh!g4maAu;-J7J{ih$I|&~P^lDt zkeP`)#UitNKBU#@!0loWTZLqVI#Qnm)HsJDp-wh5af0Ku3 zHS7jbQyN$Y>>u@(+|G13&;5YEMWo-*|4@mCL80{xMi4;&Lz6Fwg6*9of#3m-ZL1km zvd~Y`%nq$~Y-I_Nh$+_#wqW;X@qD07)>*S9@|?>fEE;z@dEAVu9>^G-Vcmi9qS3Z` zAMKdbULI7&Q@vU~Qpp#%49@{^{PS@Ct}i9F&T7*atnRum=ru~rlrYm4xKaJfR5^0B z&CDD3XIkyZ8`_NRSY@0bl*yOjKNU^_@#-Sr(<|gxHr2~C8f5MK$R=aNr~{tFrdEwk z!gGZIb$uV0#BrETTJ(kd{H*dF)AgN0Z|oR&;znv6tY4aGhj&$CbqJ7MLs*x7#l8U*G6UDvNiP)P6-1n;TAhV(Lk6^VN#~|8 zxOz5~>dTd1{s*2)(Q??`=vT-D&!j5EXjRSvzLwpj1h*C2j=T}?I2XZF9|jaIgUjp& zq3y_sDo{wHDJIPmTg`3gubUGCzGdW84i(mS5-AJC^ydnCeR2f048|B1pltJynhCBV z)`YN*!=>?s>-I5nUIRwx!J)g*OWfl>7Uf#?)xw=|cIOdU}b6U36G{at&zh)*|$ z>W#veh~1D~A>&*UJ9Fg6x-MHw&Mc%nkEXyim0REmWG$$tHF1=%ZXt^j3t*$o}XQo zQTy#Z=?gq)9s2^CwjNmbSNe!O9r1VR?hgNP8wrLkbP!1b1h0CM@_10sJHD-S77*7* z@?A=5_h0$G#mZNYv!Lf26TFl{=VI@B`Pdx} z^%%}O=K2jy3qU9slv--%!u5~N#PMp{t-Ef_q@A^*+b*?3UEp(}pb{QVP>L z&05M0GfoZcw>O4ltqn|&mb&?G+&6W*%fE6R;S0`ldED&{jB&h>}-nLF7>3x&nQ8QYf5bp zd)TooPDv#l)1E1XluUESLcybOrW=Judn3>64lK^()1x&?Hnf(*d7JOAnW>Tm_7hK! zTGt|kA@j|L|C)ONXXzD%9-;6X0JENK$Fy3iP8f-~0u8yVE;)SH?7+(MGb@NW3(k%< z$|``7`1Q~a=D3CF{=q6JnR$=t<;MTHS520vPoq}i+LOk-n|2`>T`TKeq#3>BEH!So zS5v>`xqd!xZX3yhy`O-rV9jQ@7cMky$hE1GdbaYqzl%n$m=-4ys2^Wc*7Gu8g}VgRP7Jd6L$4il|T`apE% z+M=`8ma|Uuo5F(DMvdDZ-N?VmS@-bUhw2O$EwNi)q+oQ|S{225E2R!6J~~#n%R%8t z%o`0?@~DhABv&L|B>A)$b5IaBY_(C5t+_g2zIIWHABN#|o-G)Mv6Has$1Our=Y;MR7L}Q?k&JWr3~x@2({n^j%14WGA~)wLCB)>8*S09t;*hbnvHqRR!Pxn?>dzk{3&1-x=fzSK5mI z*fYJvOmy3wPov+k8^mjSbIq z>&A}S+*sXgqE7V25SVZ0#LMS~>*m$05|FvbVHsb4Z~!MI>!i$;S4|W<9J`mksO0g1 zU#@otdy5K|nOJQ(?qU@xOEi(mfHVaRf&`+2uzB)pC;+~Ir$+9hmErXV6U~HP)#veO zh*d5QCoC7}+pXfK*B}ph`8YmfCG?nJ)p4o57$Qh&%L;#b8XVk^uV-}y#Bm@&!FQPs z<lVLNW9llnElun9Jca7KeLDtufu*1u^oL>5HUQEgrYo z&&{(E#0#!fxfij$nYTl4)q%b0hU>9?G+|@{PK3b0$ffHk7u^g8zOL4`iGl+wjDm6BzI1= zy|`{H5-vy#uh5rC-I=wMTy9MZ{Og^)BOF~Pm`gf$8zN6l(tAz~%1G|gUB0*_LP}J% zath8?8tN9=?Fw2$VLfW0{iX zSyN3;`?d}ZUTC0~V>3;Xu*KEebYWjibAjcCvTE*%P3yvP7U=Tx&kl56*atuu?m840y16pZ?m+_T1wxxOP{0Z@+LCj)=F87JF7<1P_B3jw!NxZ~ zn322+tuXN#>Q~(BqO10)T+ugT6ctka>0zc?`5S`*vU9|)*ItiiCh*XybeI?W#gg(V z!(aw)R~DqcJ^i7Pt;H*84bSNWH)0d*dhF_#(<}ab81VdD+)5jMKOpDnp{{K4&ovwZ zV1S+o=TMK61lYK24mPSv+rXd5#>1|rH_zD-+nnk~QURFuv!coyg-5Q@seHZZory}t zwYi9$yXUe;1(8mr3eF4L?`Wo)lr|T~m1 zPc0kgc~g#LDZ@?Q$P^rAK0x+3_N|NR=pU7K61qs}`fC&7V2_y~^vU?<>Gr=Ig`B5? zXHoQ`ay>yx!dWe+lPa5~V^mjOkN*?VyxCU5?n9;*15TM!{y6fN!QcrFf zcNLRJQpv(DsHXQ4kIJvCbOF%a$Hap-pPlZUVxdEos;5EWxr72 zhnBBn4Cl`4y~W4UJj%vHO&8WdF_QK7EPGOBy5kUslgkX}aIZdphbssjkhJwSo#`QO zoACi&u85H1F8hfr_q1aHWn1=a-j+DC^8%}>tU1e~s>=$N)@;!so>40w4=$b36;9fD z;MKSCI{3KFY%ruGC@+L1hMER}Yfa}X1 zT?#pth9E8D$YS;`D;Zx~8zT1G(n`|jBI3<-^V?=Mrb0L`QQdiA%2?!~-A9tY;8up4 z0UvNo?D-`1+G(|{62@%H1eu&BhFFK1XQ=;(jZFM@riDW$$QXu1D05or8f%ZC&$0IK)i?dGcuI__MJ)i zE{HrcQM$Tx7EiLq*h*bQRK0yq6jOfH_;H>cF5_v<(7Qi;y4c;VntIVW;cHYu& zf$O+^dN*8H9Qr72IbU4x6yfD zF!IF*qB8)9DS`{XQR(rUF1KP{m-dbkbr>Lguhc3>fx-ypEQJ*KB!b5zpRW#<@ZG>L zzTJ8|~hn71H&F_ucwZskKGyDeqTiP4db=AI4o^i*0I?b^A`1X1$V zg#oSqi%CxCtRmoWNkxt>*v&`ux1(#k)otN>87>u6(Dejp(OfUJw{C5)&dU3>Du1P! ze@*=ppQ90C$3Xic6%t5{Hzj%&SFGKO2FWbG5vfnXJ-6nB^z#!qb)<1Y%DZC5%xZr6`U{laWC=bU zWjtWDmrS^4BWu-ph%TEm4#w@+S=0Zo7g@RA1lenb;vX_jXN#^`Da1 z+NIol0U`$t7+oEFc(1J-ttCIQL#3pKg;0Iy?&Ko1O7?U@@y2%Xjc9KJw4UAhY6SUv z(}l-|kDKc>$iC3y3suu<_iBsc%9un7k4->%7)3vR`89@g3UsK(!Ly?!6q5s>RthGB zQCRDV{;jj$j)hYAVqnx%t*Ht-UN^?t+OWg_GPqd( z2_C}6@GVlHvs$36p~*D&DnBsS<&`Fg7J)4b@Sq$8+wj$iA=v;*-H_O!Fo@>9G1}kn z#Gi_vvme&Hs(;TZgQ;;s;YL-)y*RZr?AN%m?{{246t5}#>?(H->Mxge&nP<3mbXeo z6B#SBzCDJw$X}9d`6%Acd$5h*9=v5)@QF|VGOXYN%$g8Vre#82X?|+!dfu)JGdQBZ z?OZJ^r?btqJnfUpErT8>Hj~KeBv?oltT%QyLsN-pp#XIKeqJ*`pl^EuwtP4>``4=ghw#o+6#cI7=!4jDb6 zaT<7S2TF~bptDEPqMsa|?j<-J4Pui|RZ+V*(_8%3@MNys9czcoob8K~Oe^^mMm{yF^(V4&0ZWF^wQnn|wof-XLgFF7y}!Euwe7t&^o-itp` zJyj#X`)QiJVXg`brZ6#(n_*YA_T$trR{6z!tF7}6?YWmqCB1UNpSR_f3-Z`2&So4I zkKHUYJ&9Pc4VoE)pYZizoPCr32_e-SJ|ai)wbBn|sttyy#A>h{dPzxFE69q;z7 zNIh_M#2?^bM2P-Ic28B)g9MYHD<01XUiQ$~@ff&rO$XFTo(VZfY>T}Q;a7~mi&n(A z$bxGvJWI4lk4>AZZrG`(y1ni+E*8j_d@dugHSclObE=*!2C@Xs>hvnhJ3T_Q=Oqo3sdkk8b43B_7lGFI zmQz_6=lI?k9dlFyv_a&x$fH_u8^?-KSN+}1xun%E5wqA+Tr}afKOmsNP zY|y8J{;=Di;hOpAKrnLzSt&J4}``2GrUAZvU*!?dbGk)lD)PiQ*7Qs@Sq6MeB{8JAMu)$uU=O z%^wnYo;`ZMOxCs}KUnEHTP(+i|NDM)aBa0{6p5bHCY>lmV(SG*eSsr2^h?t1yzOO8 zcNpo-1ggLt%S&?Vgw~c%+5RMYdS{w_ij!4P>_Ihz;K_BJ`$MR^FWlaJ%hn;o<3(;F zES9#ApFC`qWYHv#nzoPD`q^^+<$d!GM~A5XV(H&QI4Lr&-mK&lp54A(aF15)vXKDS ze%$c0!B5H}CBn`k=vq#O^7wB-{N&&%9{5{qKVR7onmqpf*8Pjl(fT$0B-1^=p3Z;j zR50W_KavRl`u!)-lbTwflmGW`_l5rG`SeD?oytb>mnr-jP-*>drT^;v)8^kV^}jg! zPkO)60l)fA-tZsHN%*Fu{X!W0?jbZjSE8me}wu6fwh#5-T#9;_$9NadP#o@ z?fh>$`yZO?+3k6M&^#$}>VHqDU!#&pmjy`t?%@ZG(;NST$^IYdFMc8;e(`?$9;*K* z2I5!m|38HGAI$AQcl-uH{O+OQHzwovxVN1a9ABUPL15iPV3U8UI?MP^q5Vd1{4bgz zJx=|9(3E)kk8=OeMsDkj#A{!Th!UjoJCloZu73p~`=#dR~wN`R5k&YgYc}Y9z>zG;-z-iEnu$P_FZ@ zJ?uAu(D?B$Kl=T|r;z2|x$eApe=jLiA)_YYypS_r>3ZcX3u5 zHf%V5-gl$m$A%3eqrDTmZGRLuG@ble5PG}g$6x-Z)^p{5t8DDu@}qFz0Q%oQY`!S@ zqwu&M*gJul7W(na{1wJUN#mdY=obHfQoY;j#Fw+J(&(L_%HwL+{=R{Ow=7xwOOM~F zm#F-69Q8&oe)j(ah?dR&*~{t5|EZ9?WdHAY{v6=`UZ*`jN3iubgG%W1U$kH6{$#d3 z>lc$#U;ZT~)D`)!l>UdyzilBQKUwdv{zC?Gwg}6gf=#|-EcsJd_c{Vp{$=X%&!O-B zC32ydY1A*?HYNUI@^4$SUrpQkehU1n`A>O1{>|#|Vg6Fr*O$r|y`Q{J{^s@fe-X;f z$A1pl1eqK8ON^7XQa?ubr?v$zI3vIG`~Qs*7M};Y^x!d&`Vtc10w0#c0uk<2xHDl% zroeUjU04to!~6=5uLAe!k42zvUv}-1D>N7-isN{|uEUbzly~L8BC!D$i3zUL;W4lY zmd1Zvh6N(bs|VoDgvH2?MW^4vg4hz~S9pBwSbVy$3mQGa*)Nr#!PsM|&jxlKB0}Mz zEf{MCi^K!4NZhdKG&~02spk(Y5DQ>leE@SNJeLA<%6DiGAD9F4D_j&8$z$c# z5e20boFyt=akR{ E0C4tfy8r+H literal 0 HcmV?d00001 diff --git a/test/helm/gateway-addons-helm/default.out.yaml b/test/helm/gateway-addons-helm/default.out.yaml index 45cd9480326..805e6520b95 100644 --- a/test/helm/gateway-addons-helm/default.out.yaml +++ b/test/helm/gateway-addons-helm/default.out.yaml @@ -5965,6 +5965,270 @@ data: "version": 1, "weekStart": "" } + envoy-gateway-resource.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Envoy Gateway Memory and CPU Usage", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 6, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(namespace) (rate(container_cpu_usage_seconds_total{container=\"envoy-gateway\"}[5m]))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Envoy Gateway CPU Usage (m)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(namespace) (container_memory_working_set_bytes{container=\"envoy-gateway\"}/1024/1024)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Envoy Gateway Memory Usage (MiB)", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "tags": [ + "Control Plane" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": {}, + "timezone": "", + "title": "Envoy Gateway Resources", + "uid": "edq1b2tldspa8d", + "version": 2, + "weekStart": "" + } envoy-global.json: |- { "annotations": { diff --git a/test/helm/gateway-addons-helm/e2e.out.yaml b/test/helm/gateway-addons-helm/e2e.out.yaml index 1440dd9fba3..e53828af9b3 100644 --- a/test/helm/gateway-addons-helm/e2e.out.yaml +++ b/test/helm/gateway-addons-helm/e2e.out.yaml @@ -5996,6 +5996,270 @@ data: "version": 1, "weekStart": "" } + envoy-gateway-resource.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Envoy Gateway Memory and CPU Usage", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 6, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(namespace) (rate(container_cpu_usage_seconds_total{container=\"envoy-gateway\"}[5m]))", + "fullMetaSearch": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Envoy Gateway CPU Usage (m)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(namespace) (container_memory_working_set_bytes{container=\"envoy-gateway\"}/1024/1024)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Envoy Gateway Memory Usage (MiB)", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "tags": [ + "Control Plane" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": {}, + "timezone": "", + "title": "Envoy Gateway Resources", + "uid": "edq1b2tldspa8d", + "version": 2, + "weekStart": "" + } envoy-global.json: |- { "annotations": { From 51c6eb4b392859afe744c3157aca0975422876d9 Mon Sep 17 00:00:00 2001 From: zirain Date: Sat, 29 Jun 2024 08:40:48 +0800 Subject: [PATCH 23/23] feat: AccessLog support CEL Filter (#3688) * translator Signed-off-by: zirain * gatewayapi Signed-off-by: zirain * filename Signed-off-by: zirain * share CEL Env Signed-off-by: zirain --------- Signed-off-by: zirain --- go.mod | 3 + go.sum | 6 + internal/gatewayapi/listener.go | 25 ++ ...oyproxy-accesslog-cel-with-invalid.in.yaml | 90 +++++++ ...yproxy-accesslog-cel-with-invalid.out.yaml | 153 ++++++++++++ .../testdata/envoyproxy-accesslog-cel.in.yaml | 89 +++++++ .../envoyproxy-accesslog-cel.out.yaml | 167 +++++++++++++ internal/ir/xds.go | 1 + internal/ir/zz_generated.deepcopy.go | 5 + internal/xds/translator/accesslog.go | 52 ++++- .../testdata/in/xds-ir/accesslog-cel.yaml | 52 +++++ .../in/xds-ir/accesslog-multi-cel.yaml | 53 +++++ .../out/xds-ir/accesslog-cel.clusters.yaml | 49 ++++ .../out/xds-ir/accesslog-cel.endpoints.yaml | 12 + .../out/xds-ir/accesslog-cel.listeners.yaml | 184 +++++++++++++++ .../out/xds-ir/accesslog-cel.routes.yaml | 14 ++ .../xds-ir/accesslog-multi-cel.clusters.yaml | 49 ++++ .../xds-ir/accesslog-multi-cel.endpoints.yaml | 12 + .../xds-ir/accesslog-multi-cel.listeners.yaml | 220 ++++++++++++++++++ .../xds-ir/accesslog-multi-cel.routes.yaml | 14 ++ 20 files changed, 1246 insertions(+), 4 deletions(-) create mode 100644 internal/gatewayapi/testdata/envoyproxy-accesslog-cel-with-invalid.in.yaml create mode 100644 internal/gatewayapi/testdata/envoyproxy-accesslog-cel-with-invalid.out.yaml create mode 100644 internal/gatewayapi/testdata/envoyproxy-accesslog-cel.in.yaml create mode 100644 internal/gatewayapi/testdata/envoyproxy-accesslog-cel.out.yaml create mode 100644 internal/xds/translator/testdata/in/xds-ir/accesslog-cel.yaml create mode 100644 internal/xds/translator/testdata/in/xds-ir/accesslog-multi-cel.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/accesslog-cel.clusters.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/accesslog-cel.endpoints.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/accesslog-cel.listeners.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/accesslog-cel.routes.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/accesslog-multi-cel.clusters.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/accesslog-multi-cel.endpoints.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/accesslog-multi-cel.listeners.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/accesslog-multi-cel.routes.yaml diff --git a/go.mod b/go.mod index 9d4f56f40d1..b65d907503d 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/go-logr/zapr v1.3.0 github.com/gogo/protobuf v1.3.2 github.com/golang/protobuf v1.5.4 + github.com/google/cel-go v0.20.1 github.com/google/go-cmp v0.6.0 github.com/google/go-containerregistry v0.19.1 github.com/grafana/tempo v1.5.0 @@ -78,6 +79,7 @@ require ( github.com/Masterminds/squirrel v1.5.4 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/Microsoft/hcsshim v0.12.3 // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/containerd/containerd v1.7.17 // indirect @@ -120,6 +122,7 @@ require ( github.com/rubenv/sql-migrate v1.6.1 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/spf13/cast v1.6.0 // indirect + github.com/stoewer/go-strcase v1.2.0 // indirect github.com/vbatts/tar-split v0.11.5 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect diff --git a/go.sum b/go.sum index 83196a0ef50..8bc7d41c370 100644 --- a/go.sum +++ b/go.sum @@ -67,6 +67,8 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alessio/shellescape v1.2.2/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= @@ -322,6 +324,8 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/cel-go v0.20.1 h1:nDx9r8S3L4pE61eDdt8igGj8rf5kjYR3ILxWIpWNi84= +github.com/google/cel-go v0.20.1/go.mod h1:kWcIzTsPX0zmQ+H3TirHstLLf9ep5QTsZBN9u4dOYLg= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -624,6 +628,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= diff --git a/internal/gatewayapi/listener.go b/internal/gatewayapi/listener.go index 6978d806ad6..162a9eca976 100644 --- a/internal/gatewayapi/listener.go +++ b/internal/gatewayapi/listener.go @@ -9,8 +9,10 @@ import ( "errors" "fmt" + "github.com/google/cel-go/cel" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/utils/ptr" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" @@ -301,6 +303,22 @@ func (t *Translator) processAccessLog(envoyproxy *egv1a1.EnvoyProxy, resources * irAccessLog.OpenTelemetry = append(irAccessLog.OpenTelemetry, al) } } + + var ( + validExprs []string + errs []error + ) + for _, expr := range accessLog.Matches { + if !validCELExpression(expr) { + errs = append(errs, fmt.Errorf("invalid CEL expression: %s", expr)) + continue + } + validExprs = append(validExprs, expr) + } + if len(errs) > 0 { + return nil, utilerrors.NewAggregate(errs) + } + irAccessLog.CELMatches = validExprs } return irAccessLog, nil @@ -411,3 +429,10 @@ func destinationSettingFromHostAndPort(host string, port uint32) []*ir.Destinati }, } } + +var celEnv, _ = cel.NewEnv() + +func validCELExpression(expr string) bool { + _, issue := celEnv.Parse(expr) + return issue.Err() == nil +} diff --git a/internal/gatewayapi/testdata/envoyproxy-accesslog-cel-with-invalid.in.yaml b/internal/gatewayapi/testdata/envoyproxy-accesslog-cel-with-invalid.in.yaml new file mode 100644 index 00000000000..dd96ed4b01e --- /dev/null +++ b/internal/gatewayapi/testdata/envoyproxy-accesslog-cel-with-invalid.in.yaml @@ -0,0 +1,90 @@ +envoyProxyForGatewayClass: + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + namespace: envoy-gateway-system + name: test + spec: + telemetry: + accessLog: + settings: + - matches: + - "response.code >= 400" + - ")++++" # invalid CEL expression will be ignored + format: + type: Text + text: | + [%START_TIME%] "%REQ(:METHOD)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"\n + sinks: + - type: File + file: + path: /dev/stdout + - type: OpenTelemetry + openTelemetry: + host: otel-collector.monitoring.svc.cluster.local + port: 4317 + resources: + k8s.cluster.name: "cluster-1" + provider: + type: Kubernetes + kubernetes: + envoyService: + type: LoadBalancer + envoyDeployment: + replicas: 2 + container: + env: + - name: env_a + value: env_a_value + - name: env_b + value: env_b_name + image: "envoyproxy/envoy:distroless-dev" + resources: + requests: + cpu: 100m + memory: 512Mi + securityContext: + runAsUser: 2000 + allowPrivilegeEscalation: false + pod: + annotations: + key1: val1 + key2: val2 + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cloud.google.com/gke-nodepool + operator: In + values: + - router-node + tolerations: + - effect: NoSchedule + key: node-type + operator: Exists + value: "router" + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + fsGroupChangePolicy: "OnRootMismatch" + volumes: + - name: certs + secret: + secretName: envoy-cert +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: Same diff --git a/internal/gatewayapi/testdata/envoyproxy-accesslog-cel-with-invalid.out.yaml b/internal/gatewayapi/testdata/envoyproxy-accesslog-cel-with-invalid.out.yaml new file mode 100644 index 00000000000..e23a524604d --- /dev/null +++ b/internal/gatewayapi/testdata/envoyproxy-accesslog-cel-with-invalid.out.yaml @@ -0,0 +1,153 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + creationTimestamp: null + name: gateway-1 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: Same + name: http + port: 80 + protocol: HTTP + status: + conditions: + - lastTransitionTime: null + message: 'Invalid access log backendRefs: invalid CEL expression: )++++' + reason: Invalid + status: "False" + type: ListenersNotValid + listeners: + - attachedRoutes: 0 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +infraIR: + envoy-gateway/gateway-1: + proxy: + config: + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + creationTimestamp: null + name: test + namespace: envoy-gateway-system + spec: + logging: {} + provider: + kubernetes: + envoyDeployment: + container: + env: + - name: env_a + value: env_a_value + - name: env_b + value: env_b_name + image: envoyproxy/envoy:distroless-dev + resources: + requests: + cpu: 100m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + runAsUser: 2000 + pod: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cloud.google.com/gke-nodepool + operator: In + values: + - router-node + annotations: + key1: val1 + key2: val2 + securityContext: + fsGroup: 2000 + fsGroupChangePolicy: OnRootMismatch + runAsGroup: 3000 + runAsUser: 1000 + tolerations: + - effect: NoSchedule + key: node-type + operator: Exists + value: router + volumes: + - name: certs + secret: + secretName: envoy-cert + replicas: 2 + envoyService: + type: LoadBalancer + type: Kubernetes + telemetry: + accessLog: + settings: + - format: + text: | + [%START_TIME%] "%REQ(:METHOD)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"\n + type: Text + matches: + - response.code >= 400 + - )++++ + sinks: + - file: + path: /dev/stdout + type: File + - openTelemetry: + host: otel-collector.monitoring.svc.cluster.local + port: 4317 + resources: + k8s.cluster.name: cluster-1 + type: OpenTelemetry + status: {} + listeners: + - address: null + name: envoy-gateway/gateway-1/http + ports: + - containerPort: 10080 + name: http-80 + protocol: HTTP + servicePort: 80 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway/gateway-1 +xdsIR: + envoy-gateway/gateway-1: + http: + - address: 0.0.0.0 + hostnames: + - '*' + isHTTP2: false + name: envoy-gateway/gateway-1/http + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10080 diff --git a/internal/gatewayapi/testdata/envoyproxy-accesslog-cel.in.yaml b/internal/gatewayapi/testdata/envoyproxy-accesslog-cel.in.yaml new file mode 100644 index 00000000000..3d9f52018dc --- /dev/null +++ b/internal/gatewayapi/testdata/envoyproxy-accesslog-cel.in.yaml @@ -0,0 +1,89 @@ +envoyProxyForGatewayClass: + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + namespace: envoy-gateway-system + name: test + spec: + telemetry: + accessLog: + settings: + - matches: + - "response.code >= 400" + format: + type: Text + text: | + [%START_TIME%] "%REQ(:METHOD)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"\n + sinks: + - type: File + file: + path: /dev/stdout + - type: OpenTelemetry + openTelemetry: + host: otel-collector.monitoring.svc.cluster.local + port: 4317 + resources: + k8s.cluster.name: "cluster-1" + provider: + type: Kubernetes + kubernetes: + envoyService: + type: LoadBalancer + envoyDeployment: + replicas: 2 + container: + env: + - name: env_a + value: env_a_value + - name: env_b + value: env_b_name + image: "envoyproxy/envoy:distroless-dev" + resources: + requests: + cpu: 100m + memory: 512Mi + securityContext: + runAsUser: 2000 + allowPrivilegeEscalation: false + pod: + annotations: + key1: val1 + key2: val2 + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cloud.google.com/gke-nodepool + operator: In + values: + - router-node + tolerations: + - effect: NoSchedule + key: node-type + operator: Exists + value: "router" + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + fsGroupChangePolicy: "OnRootMismatch" + volumes: + - name: certs + secret: + secretName: envoy-cert +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: Same diff --git a/internal/gatewayapi/testdata/envoyproxy-accesslog-cel.out.yaml b/internal/gatewayapi/testdata/envoyproxy-accesslog-cel.out.yaml new file mode 100644 index 00000000000..3d0c0ae526d --- /dev/null +++ b/internal/gatewayapi/testdata/envoyproxy-accesslog-cel.out.yaml @@ -0,0 +1,167 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + creationTimestamp: null + name: gateway-1 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: Same + name: http + port: 80 + protocol: HTTP + status: + listeners: + - attachedRoutes: 0 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +infraIR: + envoy-gateway/gateway-1: + proxy: + config: + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + creationTimestamp: null + name: test + namespace: envoy-gateway-system + spec: + logging: {} + provider: + kubernetes: + envoyDeployment: + container: + env: + - name: env_a + value: env_a_value + - name: env_b + value: env_b_name + image: envoyproxy/envoy:distroless-dev + resources: + requests: + cpu: 100m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + runAsUser: 2000 + pod: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cloud.google.com/gke-nodepool + operator: In + values: + - router-node + annotations: + key1: val1 + key2: val2 + securityContext: + fsGroup: 2000 + fsGroupChangePolicy: OnRootMismatch + runAsGroup: 3000 + runAsUser: 1000 + tolerations: + - effect: NoSchedule + key: node-type + operator: Exists + value: router + volumes: + - name: certs + secret: + secretName: envoy-cert + replicas: 2 + envoyService: + type: LoadBalancer + type: Kubernetes + telemetry: + accessLog: + settings: + - format: + text: | + [%START_TIME%] "%REQ(:METHOD)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"\n + type: Text + matches: + - response.code >= 400 + sinks: + - file: + path: /dev/stdout + type: File + - openTelemetry: + host: otel-collector.monitoring.svc.cluster.local + port: 4317 + resources: + k8s.cluster.name: cluster-1 + type: OpenTelemetry + status: {} + listeners: + - address: null + name: envoy-gateway/gateway-1/http + ports: + - containerPort: 10080 + name: http-80 + protocol: HTTP + servicePort: 80 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway/gateway-1 +xdsIR: + envoy-gateway/gateway-1: + accessLog: + celMatches: + - response.code >= 400 + openTelemetry: + - authority: otel-collector.monitoring.svc.cluster.local + destination: + name: accesslog-0 + settings: + - endpoints: + - host: otel-collector.monitoring.svc.cluster.local + port: 4317 + protocol: GRPC + weight: 1 + resources: + k8s.cluster.name: cluster-1 + text: | + [%START_TIME%] "%REQ(:METHOD)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"\n + text: + - format: | + [%START_TIME%] "%REQ(:METHOD)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"\n + path: /dev/stdout + http: + - address: 0.0.0.0 + hostnames: + - '*' + isHTTP2: false + name: envoy-gateway/gateway-1/http + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10080 diff --git a/internal/ir/xds.go b/internal/ir/xds.go index b3d3ed3efd9..92e6f748e8c 100644 --- a/internal/ir/xds.go +++ b/internal/ir/xds.go @@ -1599,6 +1599,7 @@ type RateLimitValue struct { // AccessLog holds the access logging configuration. // +k8s:deepcopy-gen=true type AccessLog struct { + CELMatches []string `json:"celMatches,omitempty" yaml:"celMatches,omitempty"` Text []*TextAccessLog `json:"text,omitempty" yaml:"text,omitempty"` JSON []*JSONAccessLog `json:"json,omitempty" yaml:"json,omitempty"` OpenTelemetry []*OpenTelemetryAccessLog `json:"openTelemetry,omitempty" yaml:"openTelemetry,omitempty"` diff --git a/internal/ir/zz_generated.deepcopy.go b/internal/ir/zz_generated.deepcopy.go index 4b87aa93b9c..e98364722d9 100644 --- a/internal/ir/zz_generated.deepcopy.go +++ b/internal/ir/zz_generated.deepcopy.go @@ -19,6 +19,11 @@ import ( // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AccessLog) DeepCopyInto(out *AccessLog) { *out = *in + if in.CELMatches != nil { + in, out := &in.CELMatches, &out.CELMatches + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.Text != nil { in, out := &in.Text, &out.Text *out = make([]*TextAccessLog, len(*in)) diff --git a/internal/xds/translator/accesslog.go b/internal/xds/translator/accesslog.go index ce58b507c96..05e05bee30d 100644 --- a/internal/xds/translator/accesslog.go +++ b/internal/xds/translator/accesslog.go @@ -13,6 +13,7 @@ import ( accesslog "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v3" cfgcore "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" fileaccesslog "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/file/v3" + cel "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/filters/cel/v3" grpcaccesslog "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/grpc/v3" otelaccesslog "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/open_telemetry/v3" celformatter "github.com/envoyproxy/go-control-plane/envoy/extensions/formatter/cel/v3" @@ -54,6 +55,8 @@ const ( reqWithoutQueryCommandOperator = "%REQ_WITHOUT_QUERY" metadataCommandOperator = "%METADATA" celCommandOperator = "%CEL" + + celFilter = "envoy.access_loggers.extension_filters.cel" ) // for the case when a route does not exist to upstream, hcm logs will not be present @@ -217,16 +220,57 @@ func buildXdsAccessLog(al *ir.AccessLog, forListener bool) []*accesslog.AccessLo }) } - // add filter for listener access logs + // add filter for access logs + filters := make([]*accesslog.AccessLogFilter, 0) + for _, expr := range al.CELMatches { + filters = append(filters, celAccessLogFilter(expr)) + } if forListener { - for _, al := range accessLogs { - al.Filter = listenerAccessLogFilter - } + filters = append(filters, listenerAccessLogFilter) + } + + f := buildAccessLogFilter(filters...) + + for _, log := range accessLogs { + log.Filter = f } return accessLogs } +func celAccessLogFilter(expr string) *accesslog.AccessLogFilter { + fl := &cel.ExpressionFilter{ + Expression: expr, + } + + return &accesslog.AccessLogFilter{ + FilterSpecifier: &accesslog.AccessLogFilter_ExtensionFilter{ + ExtensionFilter: &accesslog.ExtensionFilter{ + Name: celFilter, + ConfigType: &accesslog.ExtensionFilter_TypedConfig{TypedConfig: protocov.ToAny(fl)}, + }, + }, + } +} + +func buildAccessLogFilter(f ...*accesslog.AccessLogFilter) *accesslog.AccessLogFilter { + if len(f) == 0 { + return nil + } + + if len(f) == 1 { + return f[0] + } + + return &accesslog.AccessLogFilter{ + FilterSpecifier: &accesslog.AccessLogFilter_AndFilter{ + AndFilter: &accesslog.AndFilter{ + Filters: f, + }, + }, + } +} + func accessLogTextFormatters(text string) []*cfgcore.TypedExtensionConfig { formatters := make([]*cfgcore.TypedExtensionConfig, 0, 3) diff --git a/internal/xds/translator/testdata/in/xds-ir/accesslog-cel.yaml b/internal/xds/translator/testdata/in/xds-ir/accesslog-cel.yaml new file mode 100644 index 00000000000..b0623fd0842 --- /dev/null +++ b/internal/xds/translator/testdata/in/xds-ir/accesslog-cel.yaml @@ -0,0 +1,52 @@ +name: "accesslog" +accesslog: + celMatches: + - response.code >= 400 + text: + - path: "/dev/stdout" + format: | + [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" + json: + - path: "/dev/stdout" + json: + start_time: "%START_TIME%" + method: "%REQ(:METHOD)%" + path: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%" + protocol: "%PROTOCOL%" + response_code: "%RESPONSE_CODE%" + openTelemetry: + - text: | + [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" + attributes: + "response_code": "%RESPONSE_CODE%" + resources: + "cluster_name": "cluster1" + authority: "otel-collector.default.svc.cluster.local" + destination: + name: "accesslog-0" + settings: + - endpoints: + - host: "otel-collector.default.svc.cluster.local" + port: 4317 + protocol: "GRPC" +http: +- name: "first-listener" + address: "0.0.0.0" + port: 10080 + hostnames: + - "*" + path: + mergeSlashes: true + escapedSlashesAction: UnescapeAndRedirect + routes: + - name: "direct-route" + hostname: "*" + destination: + name: "direct-route-dest" + settings: + - endpoints: + - host: "1.2.3.4" + port: 50000 + directResponse: + body: "Unknown custom filter type: UnsupportedType" + statusCode: 500 diff --git a/internal/xds/translator/testdata/in/xds-ir/accesslog-multi-cel.yaml b/internal/xds/translator/testdata/in/xds-ir/accesslog-multi-cel.yaml new file mode 100644 index 00000000000..704c19863d6 --- /dev/null +++ b/internal/xds/translator/testdata/in/xds-ir/accesslog-multi-cel.yaml @@ -0,0 +1,53 @@ +name: "accesslog" +accesslog: + celMatches: + - response.code >= 400 + - request.url_path.contains('v1beta3') + text: + - path: "/dev/stdout" + format: | + [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" + json: + - path: "/dev/stdout" + json: + start_time: "%START_TIME%" + method: "%REQ(:METHOD)%" + path: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%" + protocol: "%PROTOCOL%" + response_code: "%RESPONSE_CODE%" + openTelemetry: + - text: | + [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" + attributes: + "response_code": "%RESPONSE_CODE%" + resources: + "cluster_name": "cluster1" + authority: "otel-collector.default.svc.cluster.local" + destination: + name: "accesslog-0" + settings: + - endpoints: + - host: "otel-collector.default.svc.cluster.local" + port: 4317 + protocol: "GRPC" +http: +- name: "first-listener" + address: "0.0.0.0" + port: 10080 + hostnames: + - "*" + path: + mergeSlashes: true + escapedSlashesAction: UnescapeAndRedirect + routes: + - name: "direct-route" + hostname: "*" + destination: + name: "direct-route-dest" + settings: + - endpoints: + - host: "1.2.3.4" + port: 50000 + directResponse: + body: "Unknown custom filter type: UnsupportedType" + statusCode: 500 diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog-cel.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog-cel.clusters.yaml new file mode 100644 index 00000000000..b8874bf24f9 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/accesslog-cel.clusters.yaml @@ -0,0 +1,49 @@ +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_ONLY + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: direct-route-dest + lbPolicy: LEAST_REQUEST + name: direct-route-dest + outlierDetection: {} + perConnectionBufferLimitBytes: 32768 + type: EDS +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_ONLY + dnsRefreshRate: 30s + lbPolicy: LEAST_REQUEST + loadAssignment: + clusterName: accesslog-0 + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: otel-collector.default.svc.cluster.local + portValue: 4317 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: accesslog-0/backend/0 + name: accesslog-0 + outlierDetection: {} + perConnectionBufferLimitBytes: 32768 + respectDnsTtl: true + type: STRICT_DNS + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicitHttpConfig: + http2ProtocolOptions: {} diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog-cel.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog-cel.endpoints.yaml new file mode 100644 index 00000000000..20c80b3aaaa --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/accesslog-cel.endpoints.yaml @@ -0,0 +1,12 @@ +- clusterName: direct-route-dest + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.2.3.4 + portValue: 50000 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: direct-route-dest/backend/0 diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog-cel.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog-cel.listeners.yaml new file mode 100644 index 00000000000..2ccfca8ce50 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/accesslog-cel.listeners.yaml @@ -0,0 +1,184 @@ +- accessLog: + - filter: + andFilter: + filters: + - extensionFilter: + name: envoy.access_loggers.extension_filters.cel + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.filters.cel.v3.ExpressionFilter + expression: response.code >= 400 + - responseFlagFilter: + flags: + - NR + name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + logFormat: + textFormatSource: + inlineString: | + [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" + path: /dev/stdout + - filter: + andFilter: + filters: + - extensionFilter: + name: envoy.access_loggers.extension_filters.cel + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.filters.cel.v3.ExpressionFilter + expression: response.code >= 400 + - responseFlagFilter: + flags: + - NR + name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + logFormat: + jsonFormat: + method: '%REQ(:METHOD)%' + path: '%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%' + protocol: '%PROTOCOL%' + response_code: '%RESPONSE_CODE%' + start_time: '%START_TIME%' + path: /dev/stdout + - filter: + andFilter: + filters: + - extensionFilter: + name: envoy.access_loggers.extension_filters.cel + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.filters.cel.v3.ExpressionFilter + expression: response.code >= 400 + - responseFlagFilter: + flags: + - NR + name: envoy.access_loggers.open_telemetry + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.open_telemetry.v3.OpenTelemetryAccessLogConfig + attributes: + values: + - key: k8s.namespace.name + value: + stringValue: '%ENVIRONMENT(ENVOY_GATEWAY_NAMESPACE)%' + - key: k8s.pod.name + value: + stringValue: '%ENVIRONMENT(ENVOY_POD_NAME)%' + - key: response_code + value: + stringValue: '%RESPONSE_CODE%' + body: + stringValue: | + [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" + commonConfig: + grpcService: + envoyGrpc: + authority: otel-collector.default.svc.cluster.local + clusterName: accesslog-0 + logName: otel_envoy_accesslog + transportApiVersion: V3 + resourceAttributes: + values: + - key: cluster_name + value: + stringValue: cluster1 + address: + socketAddress: + address: 0.0.0.0 + portValue: 10080 + defaultFilterChain: + filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + accessLog: + - filter: + extensionFilter: + name: envoy.access_loggers.extension_filters.cel + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.filters.cel.v3.ExpressionFilter + expression: response.code >= 400 + name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + logFormat: + textFormatSource: + inlineString: | + [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" + path: /dev/stdout + - filter: + extensionFilter: + name: envoy.access_loggers.extension_filters.cel + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.filters.cel.v3.ExpressionFilter + expression: response.code >= 400 + name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + logFormat: + jsonFormat: + method: '%REQ(:METHOD)%' + path: '%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%' + protocol: '%PROTOCOL%' + response_code: '%RESPONSE_CODE%' + start_time: '%START_TIME%' + path: /dev/stdout + - filter: + extensionFilter: + name: envoy.access_loggers.extension_filters.cel + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.filters.cel.v3.ExpressionFilter + expression: response.code >= 400 + name: envoy.access_loggers.open_telemetry + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.open_telemetry.v3.OpenTelemetryAccessLogConfig + attributes: + values: + - key: k8s.namespace.name + value: + stringValue: '%ENVIRONMENT(ENVOY_GATEWAY_NAMESPACE)%' + - key: k8s.pod.name + value: + stringValue: '%ENVIRONMENT(ENVOY_POD_NAME)%' + - key: response_code + value: + stringValue: '%RESPONSE_CODE%' + body: + stringValue: | + [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" + commonConfig: + grpcService: + envoyGrpc: + authority: otel-collector.default.svc.cluster.local + clusterName: accesslog-0 + logName: otel_envoy_accesslog + transportApiVersion: V3 + resourceAttributes: + values: + - key: cluster_name + value: + stringValue: cluster1 + commonHttpProtocolOptions: + headersWithUnderscoresAction: REJECT_REQUEST + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 + maxConcurrentStreams: 100 + httpFilters: + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + suppressEnvoyHeaders: true + mergeSlashes: true + normalizePath: true + pathWithEscapedSlashesAction: UNESCAPE_AND_REDIRECT + rds: + configSource: + ads: {} + resourceApiVersion: V3 + routeConfigName: first-listener + serverHeaderTransformation: PASS_THROUGH + statPrefix: http + useRemoteAddress: true + name: first-listener + drainType: MODIFY_ONLY + name: first-listener + perConnectionBufferLimitBytes: 32768 diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog-cel.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog-cel.routes.yaml new file mode 100644 index 00000000000..d4a7fa5ae20 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/accesslog-cel.routes.yaml @@ -0,0 +1,14 @@ +- ignorePortInHostMatching: true + name: first-listener + virtualHosts: + - domains: + - '*' + name: first-listener/* + routes: + - directResponse: + body: + inlineString: 'Unknown custom filter type: UnsupportedType' + status: 500 + match: + prefix: / + name: direct-route diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog-multi-cel.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog-multi-cel.clusters.yaml new file mode 100644 index 00000000000..b8874bf24f9 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/accesslog-multi-cel.clusters.yaml @@ -0,0 +1,49 @@ +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_ONLY + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: direct-route-dest + lbPolicy: LEAST_REQUEST + name: direct-route-dest + outlierDetection: {} + perConnectionBufferLimitBytes: 32768 + type: EDS +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_ONLY + dnsRefreshRate: 30s + lbPolicy: LEAST_REQUEST + loadAssignment: + clusterName: accesslog-0 + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: otel-collector.default.svc.cluster.local + portValue: 4317 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: accesslog-0/backend/0 + name: accesslog-0 + outlierDetection: {} + perConnectionBufferLimitBytes: 32768 + respectDnsTtl: true + type: STRICT_DNS + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicitHttpConfig: + http2ProtocolOptions: {} diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog-multi-cel.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog-multi-cel.endpoints.yaml new file mode 100644 index 00000000000..20c80b3aaaa --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/accesslog-multi-cel.endpoints.yaml @@ -0,0 +1,12 @@ +- clusterName: direct-route-dest + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.2.3.4 + portValue: 50000 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: direct-route-dest/backend/0 diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog-multi-cel.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog-multi-cel.listeners.yaml new file mode 100644 index 00000000000..0bca441a443 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/accesslog-multi-cel.listeners.yaml @@ -0,0 +1,220 @@ +- accessLog: + - filter: + andFilter: + filters: + - extensionFilter: + name: envoy.access_loggers.extension_filters.cel + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.filters.cel.v3.ExpressionFilter + expression: response.code >= 400 + - extensionFilter: + name: envoy.access_loggers.extension_filters.cel + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.filters.cel.v3.ExpressionFilter + expression: request.url_path.contains('v1beta3') + - responseFlagFilter: + flags: + - NR + name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + logFormat: + textFormatSource: + inlineString: | + [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" + path: /dev/stdout + - filter: + andFilter: + filters: + - extensionFilter: + name: envoy.access_loggers.extension_filters.cel + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.filters.cel.v3.ExpressionFilter + expression: response.code >= 400 + - extensionFilter: + name: envoy.access_loggers.extension_filters.cel + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.filters.cel.v3.ExpressionFilter + expression: request.url_path.contains('v1beta3') + - responseFlagFilter: + flags: + - NR + name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + logFormat: + jsonFormat: + method: '%REQ(:METHOD)%' + path: '%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%' + protocol: '%PROTOCOL%' + response_code: '%RESPONSE_CODE%' + start_time: '%START_TIME%' + path: /dev/stdout + - filter: + andFilter: + filters: + - extensionFilter: + name: envoy.access_loggers.extension_filters.cel + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.filters.cel.v3.ExpressionFilter + expression: response.code >= 400 + - extensionFilter: + name: envoy.access_loggers.extension_filters.cel + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.filters.cel.v3.ExpressionFilter + expression: request.url_path.contains('v1beta3') + - responseFlagFilter: + flags: + - NR + name: envoy.access_loggers.open_telemetry + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.open_telemetry.v3.OpenTelemetryAccessLogConfig + attributes: + values: + - key: k8s.namespace.name + value: + stringValue: '%ENVIRONMENT(ENVOY_GATEWAY_NAMESPACE)%' + - key: k8s.pod.name + value: + stringValue: '%ENVIRONMENT(ENVOY_POD_NAME)%' + - key: response_code + value: + stringValue: '%RESPONSE_CODE%' + body: + stringValue: | + [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" + commonConfig: + grpcService: + envoyGrpc: + authority: otel-collector.default.svc.cluster.local + clusterName: accesslog-0 + logName: otel_envoy_accesslog + transportApiVersion: V3 + resourceAttributes: + values: + - key: cluster_name + value: + stringValue: cluster1 + address: + socketAddress: + address: 0.0.0.0 + portValue: 10080 + defaultFilterChain: + filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + accessLog: + - filter: + andFilter: + filters: + - extensionFilter: + name: envoy.access_loggers.extension_filters.cel + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.filters.cel.v3.ExpressionFilter + expression: response.code >= 400 + - extensionFilter: + name: envoy.access_loggers.extension_filters.cel + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.filters.cel.v3.ExpressionFilter + expression: request.url_path.contains('v1beta3') + name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + logFormat: + textFormatSource: + inlineString: | + [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" + path: /dev/stdout + - filter: + andFilter: + filters: + - extensionFilter: + name: envoy.access_loggers.extension_filters.cel + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.filters.cel.v3.ExpressionFilter + expression: response.code >= 400 + - extensionFilter: + name: envoy.access_loggers.extension_filters.cel + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.filters.cel.v3.ExpressionFilter + expression: request.url_path.contains('v1beta3') + name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + logFormat: + jsonFormat: + method: '%REQ(:METHOD)%' + path: '%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%' + protocol: '%PROTOCOL%' + response_code: '%RESPONSE_CODE%' + start_time: '%START_TIME%' + path: /dev/stdout + - filter: + andFilter: + filters: + - extensionFilter: + name: envoy.access_loggers.extension_filters.cel + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.filters.cel.v3.ExpressionFilter + expression: response.code >= 400 + - extensionFilter: + name: envoy.access_loggers.extension_filters.cel + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.filters.cel.v3.ExpressionFilter + expression: request.url_path.contains('v1beta3') + name: envoy.access_loggers.open_telemetry + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.open_telemetry.v3.OpenTelemetryAccessLogConfig + attributes: + values: + - key: k8s.namespace.name + value: + stringValue: '%ENVIRONMENT(ENVOY_GATEWAY_NAMESPACE)%' + - key: k8s.pod.name + value: + stringValue: '%ENVIRONMENT(ENVOY_POD_NAME)%' + - key: response_code + value: + stringValue: '%RESPONSE_CODE%' + body: + stringValue: | + [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" + commonConfig: + grpcService: + envoyGrpc: + authority: otel-collector.default.svc.cluster.local + clusterName: accesslog-0 + logName: otel_envoy_accesslog + transportApiVersion: V3 + resourceAttributes: + values: + - key: cluster_name + value: + stringValue: cluster1 + commonHttpProtocolOptions: + headersWithUnderscoresAction: REJECT_REQUEST + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 + maxConcurrentStreams: 100 + httpFilters: + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + suppressEnvoyHeaders: true + mergeSlashes: true + normalizePath: true + pathWithEscapedSlashesAction: UNESCAPE_AND_REDIRECT + rds: + configSource: + ads: {} + resourceApiVersion: V3 + routeConfigName: first-listener + serverHeaderTransformation: PASS_THROUGH + statPrefix: http + useRemoteAddress: true + name: first-listener + drainType: MODIFY_ONLY + name: first-listener + perConnectionBufferLimitBytes: 32768 diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog-multi-cel.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog-multi-cel.routes.yaml new file mode 100644 index 00000000000..d4a7fa5ae20 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/accesslog-multi-cel.routes.yaml @@ -0,0 +1,14 @@ +- ignorePortInHostMatching: true + name: first-listener + virtualHosts: + - domains: + - '*' + name: first-listener/* + routes: + - directResponse: + body: + inlineString: 'Unknown custom filter type: UnsupportedType' + status: 500 + match: + prefix: / + name: direct-route