Skip to content

Commit

Permalink
feat: expose shutdownmanager image to EnvoyGageway API (#3269)
Browse files Browse the repository at this point in the history 
* expose shutdownmanager image to EnvoyGageway API

Signed-off-by: huabing zhao <zhaohuabing@gmail.com>

* fix test

Signed-off-by: huabing zhao <zhaohuabing@gmail.com>

* Update internal/infrastructure/kubernetes/proxy/testdata/deployments/shutdown-manager.yaml

Co-authored-by: Arko Dasgupta <arkodg@users.noreply.github.com>
Signed-off-by: Huabing Zhao <zhaohuabing@gmail.com>

* add comments

Signed-off-by: huabing zhao <zhaohuabing@gmail.com>

* fix gen

Signed-off-by: huabing zhao <zhaohuabing@gmail.com>

* set default shutdownmanger image in helm value template

Signed-off-by: huabing zhao <zhaohuabing@gmail.com>

* set default shutdownmanger image in helm value template

Signed-off-by: huabing zhao <zhaohuabing@gmail.com>

* fix README

Signed-off-by: huabing zhao <zhaohuabing@gmail.com>

* use the same Envoy Gateway version for ShutManager

Signed-off-by: huabing zhao <zhaohuabing@gmail.com>

* fix check

Signed-off-by: huabing zhao <zhaohuabing@gmail.com>

* fix typo

Signed-off-by: huabing zhao <zhaohuabing@gmail.com>

* fix gen

Signed-off-by: huabing zhao <zhaohuabing@gmail.com>

* fix gen

Signed-off-by: huabing zhao <zhaohuabing@gmail.com>

* fix typo

Signed-off-by: huabing zhao <zhaohuabing@gmail.com>

---------

Signed-off-by: huabing zhao <zhaohuabing@gmail.com>
Signed-off-by: Huabing Zhao <zhaohuabing@gmail.com>
Co-authored-by: Arko Dasgupta <arkodg@users.noreply.github.com>
  • Loading branch information
@zhaohuabing @arkodg
zhaohuabing and arkodg authored May 7, 2024
1 parent 019a5e1 commit 2ad24a8
Show file tree
Hide file tree
Showing 19 changed files with 126 additions and 60 deletions.
3 changes: 3 additions & 0 deletions api/v1alpha1/envoygateway_helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,9 @@ func (r *EnvoyGatewayProvider) GetEnvoyGatewayKubeProvider() *EnvoyGatewayKubern

r.Kubernetes.RateLimitDeployment.defaultKubernetesDeploymentSpec(DefaultRateLimitImage)

if r.Kubernetes.ShutdownManager == nil {
r.Kubernetes.ShutdownManager = &ShutdownManager{Image: ptr.To(DefaultShutdownManagerImage)}
}
return r.Kubernetes
}

Expand Down
10 changes: 10 additions & 0 deletions api/v1alpha1/envoygateway_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,10 @@ type EnvoyGatewayKubernetesProvider struct {
// If it's not set up, leader election will be active by default, using Kubernetes' standard settings.
// +optional
LeaderElection *LeaderElection `json:"leaderElection,omitempty"`

// ShutdownManager defines the configuration for the shutdown manager.
// +optional
ShutdownManager *ShutdownManager `json:"shutdownManager,omitempty"`
}

const (
Expand Down Expand Up @@ -535,6 +539,12 @@ type EnvoyGatewayAdminAddress struct {
Host string `json:"host,omitempty"`
}

// ShutdownManager defines the configuration for the shutdown manager.
type ShutdownManager struct {
// Image specifies the ShutdownManager container image to be used, instead of the default image.
Image *string `json:"image,omitempty"`
}

func init() {
SchemeBuilder.Register(&EnvoyGateway{})
}
25 changes: 25 additions & 0 deletions api/v1alpha1/zz_generated.deepcopy.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 4 additions & 1 deletion charts/gateway-helm/values.tmpl.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ deployment:
replicas: 1
pod:
affinity: {}
annotations:
annotations:
prometheus.io/scrape: 'true'
prometheus.io/port: '19001'
labels: {}
Expand All @@ -36,6 +36,9 @@ config:
controllerName: gateway.envoyproxy.io/gatewayclass-controller
provider:
type: Kubernetes
kubernetes:
shutdownManager:
image: ${ImageRepository}:${ImageTag}
logging:
level:
default: info
Expand Down
28 changes: 12 additions & 16 deletions internal/cmd/version/version.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,29 +18,26 @@ import (
)

type Info struct {
EnvoyGatewayVersion string `json:"envoyGatewayVersion"`
GatewayAPIVersion string `json:"gatewayAPIVersion"`
EnvoyProxyVersion string `json:"envoyProxyVersion"`
ShutdownManagerVersion string `json:"shutdownManagerVersion"`
GitCommitID string `json:"gitCommitID"`
EnvoyGatewayVersion string `json:"envoyGatewayVersion"`
GatewayAPIVersion string `json:"gatewayAPIVersion"`
EnvoyProxyVersion string `json:"envoyProxyVersion"`
GitCommitID string `json:"gitCommitID"`
}

func Get() Info {
return Info{
EnvoyGatewayVersion: envoyGatewayVersion,
GatewayAPIVersion: gatewayAPIVersion,
EnvoyProxyVersion: envoyProxyVersion,
ShutdownManagerVersion: shutdownManagerVersion,
GitCommitID: gitCommitID,
EnvoyGatewayVersion: envoyGatewayVersion,
GatewayAPIVersion: gatewayAPIVersion,
EnvoyProxyVersion: envoyProxyVersion,
GitCommitID: gitCommitID,
}
}

var (
envoyGatewayVersion string
gatewayAPIVersion string
envoyProxyVersion = strings.Split(*v1alpha1.DefaultKubernetesContainerImage(v1alpha1.DefaultEnvoyProxyImage), ":")[1]
shutdownManagerVersion string
gitCommitID string
envoyGatewayVersion string
gatewayAPIVersion string
envoyProxyVersion = strings.Split(v1alpha1.DefaultEnvoyProxyImage, ":")[1]
gitCommitID string
)

func init() {
Expand Down Expand Up @@ -70,7 +67,6 @@ func Print(w io.Writer, format string) error {
_, _ = fmt.Fprintf(w, "ENVOY_GATEWAY_VERSION: %s\n", v.EnvoyGatewayVersion)
_, _ = fmt.Fprintf(w, "ENVOY_PROXY_VERSION: %s\n", v.EnvoyProxyVersion)
_, _ = fmt.Fprintf(w, "GATEWAYAPI_VERSION: %s\n", v.GatewayAPIVersion)
_, _ = fmt.Fprintf(w, "SHUTDOWN_MANAGER_VERSION: %s\n", v.ShutdownManagerVersion)
_, _ = fmt.Fprintf(w, "GIT_COMMIT_ID: %s\n", v.GitCommitID)
}

Expand Down
11 changes: 5 additions & 6 deletions internal/infrastructure/kubernetes/proxy/resource.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,13 @@ package proxy

import (
"fmt"
"strings"

corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/utils/ptr"

egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
"github.com/envoyproxy/gateway/internal/cmd/envoy"
"github.com/envoyproxy/gateway/internal/cmd/version"
"github.com/envoyproxy/gateway/internal/envoygateway/config"
"github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/resource"
"github.com/envoyproxy/gateway/internal/ir"
Expand Down Expand Up @@ -103,6 +101,7 @@ func enablePrometheus(infra *ir.ProxyInfra) bool {
func expectedProxyContainers(infra *ir.ProxyInfra,
containerSpec *egv1a1.KubernetesContainerSpec,
shutdownConfig *egv1a1.ShutdownConfig,
shutdownManager *egv1a1.ShutdownManager,
) ([]corev1.Container, error) {
// Define slice to hold container ports
var ports []corev1.ContainerPort
Expand Down Expand Up @@ -230,7 +229,7 @@ func expectedProxyContainers(infra *ir.ProxyInfra,
},
{
Name: "shutdown-manager",
Image: expectedShutdownManagerImage(),
Image: expectedShutdownManagerImage(shutdownManager),
ImagePullPolicy: corev1.PullIfNotPresent,
Command: []string{"envoy-gateway"},
Args: expectedShutdownManagerArgs(shutdownConfig),
Expand Down Expand Up @@ -277,9 +276,9 @@ func expectedProxyContainers(infra *ir.ProxyInfra,
return containers, nil
}

func expectedShutdownManagerImage() string {
if v := version.Get().ShutdownManagerVersion; v != "" {
return fmt.Sprintf("%s:%s", strings.Split(egv1a1.DefaultShutdownManagerImage, ":")[0], v)
func expectedShutdownManagerImage(shutdownManager *egv1a1.ShutdownManager) string {
if shutdownManager != nil && shutdownManager.Image != nil {
return *shutdownManager.Image
}
return egv1a1.DefaultShutdownManagerImage
}
Expand Down
13 changes: 8 additions & 5 deletions internal/infrastructure/kubernetes/proxy/resource_provider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,15 @@ type ResourceRender struct {

// Namespace is the Namespace used for managed infra.
Namespace string

ShutdownManager *egv1a1.ShutdownManager
}

func NewResourceRender(ns string, infra *ir.ProxyInfra) *ResourceRender {
func NewResourceRender(ns string, infra *ir.ProxyInfra, gateway *egv1a1.EnvoyGateway) *ResourceRender {
return &ResourceRender{
Namespace: ns,
infra: infra,
Namespace: ns,
infra: infra,
ShutdownManager: gateway.GetEnvoyGatewayProvider().GetEnvoyGatewayKubeProvider().ShutdownManager,
}
}

Expand Down Expand Up @@ -199,7 +202,7 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) {
}

// Get expected bootstrap configurations rendered ProxyContainers
containers, err := expectedProxyContainers(r.infra, deploymentConfig.Container, proxyConfig.Spec.Shutdown)
containers, err := expectedProxyContainers(r.infra, deploymentConfig.Container, proxyConfig.Spec.Shutdown, r.ShutdownManager)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -288,7 +291,7 @@ func (r *ResourceRender) DaemonSet() (*appsv1.DaemonSet, error) {
}

// Get expected bootstrap configurations rendered ProxyContainers
containers, err := expectedProxyContainers(r.infra, daemonSetConfig.Container, proxyConfig.Spec.Shutdown)
containers, err := expectedProxyContainers(r.infra, daemonSetConfig.Container, proxyConfig.Spec.Shutdown, r.ShutdownManager)
if err != nil {
return nil, err
}
Expand Down
43 changes: 26 additions & 17 deletions internal/infrastructure/kubernetes/proxy/resource_provider_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,15 +88,16 @@ func TestDeployment(t *testing.T) {
require.NoError(t, err)

cases := []struct {
caseName string
infra *ir.Infra
deploy *egv1a1.KubernetesDeploymentSpec
shutdown *egv1a1.ShutdownConfig
proxyLogging map[egv1a1.ProxyLogComponent]egv1a1.LogLevel
bootstrap string
telemetry *egv1a1.ProxyTelemetry
concurrency *int32
extraArgs []string
caseName string
infra *ir.Infra
deploy *egv1a1.KubernetesDeploymentSpec
shutdown *egv1a1.ShutdownConfig
shutdownManager *egv1a1.ShutdownManager
proxyLogging map[egv1a1.ProxyLogComponent]egv1a1.LogLevel
bootstrap string
telemetry *egv1a1.ProxyTelemetry
concurrency *int32
extraArgs []string
}{
{
caseName: "default",
Expand Down Expand Up @@ -171,8 +172,7 @@ func TestDeployment(t *testing.T) {
"env":[
{"name":"env_a","value":"env_a_value"},
{"name":"env_b","value":"env_b_value"}
],
"image":"envoyproxy/gateway-dev:v1.2.3"
]
}]
}
}
Expand All @@ -189,6 +189,9 @@ func TestDeployment(t *testing.T) {
Duration: 15 * time.Second,
},
},
shutdownManager: &egv1a1.ShutdownManager{
Image: ptr.To("privaterepo/envoyproxy/gateway-dev:v1.2.3"),
},
},
{
caseName: "bootstrap",
Expand Down Expand Up @@ -542,11 +545,17 @@ func TestDeployment(t *testing.T) {
tc.infra.Proxy.Config.Spec.Shutdown = tc.shutdown
}

if tc.shutdownManager != nil {
cfg.EnvoyGateway.Provider.Kubernetes.ShutdownManager = tc.shutdownManager
} else {
cfg.EnvoyGateway.Provider.Kubernetes.ShutdownManager = nil
}

if len(tc.extraArgs) > 0 {
tc.infra.Proxy.Config.Spec.ExtraArgs = tc.extraArgs
}

r := NewResourceRender(cfg.Namespace, tc.infra.GetProxyInfra())
r := NewResourceRender(cfg.Namespace, tc.infra.GetProxyInfra(), cfg.EnvoyGateway)
dp, err := r.Deployment()
require.NoError(t, err)

Expand Down Expand Up @@ -967,7 +976,7 @@ func TestDaemonSet(t *testing.T) {
tc.infra.Proxy.Config.Spec.ExtraArgs = tc.extraArgs
}

r := NewResourceRender(cfg.Namespace, tc.infra.GetProxyInfra())
r := NewResourceRender(cfg.Namespace, tc.infra.GetProxyInfra(), cfg.EnvoyGateway)
ds, err := r.DaemonSet()
require.NoError(t, err)

Expand Down Expand Up @@ -1082,7 +1091,7 @@ func TestService(t *testing.T) {
provider.EnvoyService = tc.service
}

r := NewResourceRender(cfg.Namespace, tc.infra.GetProxyInfra())
r := NewResourceRender(cfg.Namespace, tc.infra.GetProxyInfra(), cfg.EnvoyGateway)
svc, err := r.Service()
require.NoError(t, err)

Expand Down Expand Up @@ -1125,7 +1134,7 @@ func TestConfigMap(t *testing.T) {

for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
r := NewResourceRender(cfg.Namespace, tc.infra.GetProxyInfra())
r := NewResourceRender(cfg.Namespace, tc.infra.GetProxyInfra(), cfg.EnvoyGateway)
cm, err := r.ConfigMap()
require.NoError(t, err)

Expand Down Expand Up @@ -1168,7 +1177,7 @@ func TestServiceAccount(t *testing.T) {

for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
r := NewResourceRender(cfg.Namespace, tc.infra.GetProxyInfra())
r := NewResourceRender(cfg.Namespace, tc.infra.GetProxyInfra(), cfg.EnvoyGateway)
sa, err := r.ServiceAccount()
require.NoError(t, err)

Expand Down Expand Up @@ -1249,7 +1258,7 @@ func TestHorizontalPodAutoscaler(t *testing.T) {

provider.GetEnvoyProxyKubeProvider()

r := NewResourceRender(cfg.Namespace, tc.infra.GetProxyInfra())
r := NewResourceRender(cfg.Namespace, tc.infra.GetProxyInfra(), cfg.EnvoyGateway)
hpa, err := r.HorizontalPodAutoscaler()
require.NoError(t, err)

Expand Down
2 changes: 1 addition & 1 deletion internal/infrastructure/kubernetes/proxy/testdata/deployments/shutdown-manager.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,7 +249,7 @@ spec:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
image: envoyproxy/gateway-dev:v1.2.3
image: privaterepo/envoyproxy/gateway-dev:v1.2.3
imagePullPolicy: IfNotPresent
lifecycle:
preStop:
Expand Down
4 changes: 2 additions & 2 deletions internal/infrastructure/kubernetes/proxy_configmap_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ func TestCreateOrUpdateProxyConfigMap(t *testing.T) {
Build()
}
kube := NewInfra(cli, cfg)
r := proxy.NewResourceRender(kube.Namespace, infra.GetProxyInfra())
r := proxy.NewResourceRender(kube.Namespace, infra.GetProxyInfra(), kube.EnvoyGateway)
err := kube.createOrUpdateConfigMap(context.Background(), r)
require.NoError(t, err)
actual := &corev1.ConfigMap{
Expand Down Expand Up @@ -170,7 +170,7 @@ func TestDeleteConfigProxyMap(t *testing.T) {
infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNamespaceLabel] = "default"
infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNameLabel] = infra.Proxy.Name

r := proxy.NewResourceRender(kube.Namespace, infra.GetProxyInfra())
r := proxy.NewResourceRender(kube.Namespace, infra.GetProxyInfra(), kube.EnvoyGateway)
cm := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Namespace: kube.Namespace,
Expand Down
6 changes: 3 additions & 3 deletions internal/infrastructure/kubernetes/proxy_deployment_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ func TestCreateOrUpdateProxyDeployment(t *testing.T) {
infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNamespaceLabel] = "default"
infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNameLabel] = infra.Proxy.Name

r := proxy.NewResourceRender(cfg.Namespace, infra.GetProxyInfra())
r := proxy.NewResourceRender(cfg.Namespace, infra.GetProxyInfra(), cfg.EnvoyGateway)
deploy, err := r.Deployment()
require.NoError(t, err)

Expand Down Expand Up @@ -119,7 +119,7 @@ func TestCreateOrUpdateProxyDeployment(t *testing.T) {
}

kube := NewInfra(cli, cfg)
r := proxy.NewResourceRender(kube.Namespace, tc.in.GetProxyInfra())
r := proxy.NewResourceRender(kube.Namespace, tc.in.GetProxyInfra(), cfg.EnvoyGateway)
err := kube.createOrUpdateDeployment(context.Background(), r)
require.NoError(t, err)

Expand Down Expand Up @@ -162,7 +162,7 @@ func TestDeleteProxyDeployment(t *testing.T) {
infra := ir.NewInfra()
infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNamespaceLabel] = "default"
infra.Proxy.GetProxyMetadata().Labels[gatewayapi.OwningGatewayNameLabel] = infra.Proxy.Name
r := proxy.NewResourceRender(kube.Namespace, infra.GetProxyInfra())
r := proxy.NewResourceRender(kube.Namespace, infra.GetProxyInfra(), kube.EnvoyGateway)

err := kube.createOrUpdateDeployment(context.Background(), r)
require.NoError(t, err)
Expand Down
Loading

0 comments on commit 2ad24a8

Please sign in to comment.