feat: add file resource provider for EG standalone mode (envoyproxy#3159

) * add validations for envoy-gateway file resource type Signed-off-by: shawnh2 <shawnhxh@outlook.com> * improve eg validation and add resource provider interface for various provider Signed-off-by: shawnh2 <shawnhxh@outlook.com> * extract common gatewayapi layer translate logic in egctl translate Signed-off-by: shawnh2 <shawnhxh@outlook.com> * add notifier support Signed-off-by: shawnh2 <shawnhxh@outlook.com> * fix lint and move read yaml bytes function back to translate Signed-off-by: shawnh2 <shawnhxh@outlook.com> * add resources store support Signed-off-by: shawnh2 <shawnhxh@outlook.com> * fix lint Signed-off-by: shawnh2 <shawnhxh@outlook.com> * fix ci Signed-off-by: shawnh2 <shawnhxh@outlook.com> * update infra provider api and address comments Signed-off-by: shawnh2 <shawnhxh@outlook.com> * update custom provider comments and validate method test Signed-off-by: shawnh2 <shawnhxh@outlook.com> * restore extension manager and add health probe server for file provider Signed-off-by: shawnh2 <shawnhxh@outlook.com> * update envoy gateway helper functions Signed-off-by: shawnh2 <shawnhxh@outlook.com> * add some unit tests Signed-off-by: shawnh2 <shawnhxh@outlook.com> * properly handle the remove event for the file provider Signed-off-by: shawnh2 <shawnhxh@outlook.com> * fix lint Signed-off-by: shawnh2 <shawnhxh@outlook.com> * no default to k8s for infra provider Signed-off-by: shawnh2 <shawnhxh@outlook.com> * fix runner Signed-off-by: shawnh2 <shawnhxh@outlook.com> --------- Signed-off-by: shawnh2 <shawnhxh@outlook.com> Co-authored-by: Xunzhuo <bitliu@tencent.com>
guydc · Sep 11, 2024 · f5693f6 · f5693f6
1 parent c109131
commit f5693f6
Show file tree

Hide file tree

Showing 26 changed files with 1,330 additions and 125 deletions.
diff --git a/api/v1alpha1/envoygateway_helpers.go b/api/v1alpha1/envoygateway_helpers.go
@@ -237,6 +237,7 @@ func (r *EnvoyGatewayProvider) GetEnvoyGatewayKubeProvider() *EnvoyGatewayKubern
 	if r.Kubernetes.ShutdownManager == nil {
 		r.Kubernetes.ShutdownManager = &ShutdownManager{Image: ptr.To(DefaultShutdownManagerImage)}
 	}
+
 	return r.Kubernetes
 }
 

diff --git a/api/v1alpha1/envoygateway_types.go b/api/v1alpha1/envoygateway_types.go
@@ -174,7 +174,7 @@ type ExtensionAPISettings struct {
 // EnvoyGatewayProvider defines the desired configuration of a provider.
 // +union
 type EnvoyGatewayProvider struct {
-	// Type is the type of provider to use. Supported types are "Kubernetes".
+	// Type is the type of provider to use. Supported types are "Kubernetes", "Custom".
 	//
 	// +unionDiscriminator
 	Type ProviderType `json:"type"`
@@ -186,7 +186,7 @@ type EnvoyGatewayProvider struct {
 	Kubernetes *EnvoyGatewayKubernetesProvider `json:"kubernetes,omitempty"`
 
 	// Custom defines the configuration for the Custom provider. This provider
-	// allows you to define a specific resource provider and a infrastructure
+	// allows you to define a specific resource provider and an infrastructure
 	// provider.
 	//
 	// +optional
@@ -271,7 +271,11 @@ type EnvoyGatewayCustomProvider struct {
 	// This provider is used to specify the provider to be used
 	// to provide an environment to deploy the out resources like
 	// the Envoy Proxy data plane.
-	Infrastructure EnvoyGatewayInfrastructureProvider `json:"infrastructure"`
+	//
+	// Infrastructure is optional, if provider is not specified,
+	// No infrastructure provider is available.
+	// +optional
+	Infrastructure *EnvoyGatewayInfrastructureProvider `json:"infrastructure,omitempty"`
 }
 
 // ResourceProviderType defines the types of custom resource providers supported by Envoy Gateway.
@@ -300,7 +304,7 @@ type EnvoyGatewayResourceProvider struct {
 // EnvoyGatewayFileResourceProvider defines configuration for the File Resource provider.
 type EnvoyGatewayFileResourceProvider struct {
 	// Paths are the paths to a directory or file containing the resource configuration.
-	// Recursive sub directories are not currently supported.
+	// Recursive subdirectories are not currently supported.
 	Paths []string `json:"paths"`
 }
 

diff --git a/api/v1alpha1/shared_types.go b/api/v1alpha1/shared_types.go
@@ -47,16 +47,15 @@ type GroupVersionKind struct {
 
 // ProviderType defines the types of providers supported by Envoy Gateway.
 //
-// +kubebuilder:validation:Enum=Kubernetes
+// +kubebuilder:validation:Enum=Kubernetes;Custom
 type ProviderType string
 
 const (
 	// ProviderTypeKubernetes defines the "Kubernetes" provider.
 	ProviderTypeKubernetes ProviderType = "Kubernetes"
 
-	// ProviderTypeFile defines the "File" provider. This type is not implemented
-	// until https://github.com/envoyproxy/gateway/issues/1001 is fixed.
-	ProviderTypeFile ProviderType = "File"
+	// ProviderTypeCustom defines the "Custom" provider.
+	ProviderTypeCustom ProviderType = "Custom"
 )
 
 // KubernetesDeploymentSpec defines the desired state of the Kubernetes deployment resource.

diff --git a/api/v1alpha1/validation/envoygateway_validate.go b/api/v1alpha1/validation/envoygateway_validate.go
@@ -35,6 +35,10 @@ func ValidateEnvoyGateway(eg *egv1a1.EnvoyGateway) error {
 		if err := validateEnvoyGatewayKubernetesProvider(eg.Provider.Kubernetes); err != nil {
 			return err
 		}
+	case egv1a1.ProviderTypeCustom:
+		if err := validateEnvoyGatewayCustomProvider(eg.Provider.Custom); err != nil {
+			return err
+		}
 	default:
 		return fmt.Errorf("unsupported provider type")
 	}
@@ -79,6 +83,54 @@ func validateEnvoyGatewayKubernetesProvider(provider *egv1a1.EnvoyGatewayKuberne
 	return nil
 }
 
+func validateEnvoyGatewayCustomProvider(provider *egv1a1.EnvoyGatewayCustomProvider) error {
+	if provider == nil {
+		return fmt.Errorf("empty custom provider settings")
+	}
+
+	if err := validateEnvoyGatewayCustomResourceProvider(provider.Resource); err != nil {
+		return err
+	}
+
+	if err := validateEnvoyGatewayCustomInfrastructureProvider(provider.Infrastructure); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func validateEnvoyGatewayCustomResourceProvider(resource egv1a1.EnvoyGatewayResourceProvider) error {
+	switch resource.Type {
+	case egv1a1.ResourceProviderTypeFile:
+		if resource.File == nil {
+			return fmt.Errorf("field 'file' should be specified when resource type is 'File'")
+		}
+
+		if len(resource.File.Paths) == 0 {
+			return fmt.Errorf("no paths were assigned for file resource provider to watch")
+		}
+	default:
+		return fmt.Errorf("unsupported resource provider: %s", resource.Type)
+	}
+	return nil
+}
+
+func validateEnvoyGatewayCustomInfrastructureProvider(infra *egv1a1.EnvoyGatewayInfrastructureProvider) error {
+	if infra == nil {
+		return nil
+	}
+
+	switch infra.Type {
+	case egv1a1.InfrastructureProviderTypeHost:
+		if infra.Host == nil {
+			return fmt.Errorf("field 'host' should be specified when infrastructure type is 'Host'")
+		}
+	default:
+		return fmt.Errorf("unsupported infrastructure provdier: %s", infra.Type)
+	}
+	return nil
+}
+
 func validateEnvoyGatewayLogging(logging *egv1a1.EnvoyGatewayLogging) error {
 	if logging == nil || len(logging.Level) == 0 {
 		return nil

diff --git a/api/v1alpha1/validation/envoygateway_validate_test.go b/api/v1alpha1/validation/envoygateway_validate_test.go
@@ -68,11 +68,173 @@ func TestValidateEnvoyGateway(t *testing.T) {
 			expect: false,
 		},
 		{
-			name: "unsupported provider",
+			name: "nil custom provider",
 			eg: &egv1a1.EnvoyGateway{
 				EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{
-					Gateway:  egv1a1.DefaultGateway(),
-					Provider: &egv1a1.EnvoyGatewayProvider{Type: egv1a1.ProviderTypeFile},
+					Gateway: egv1a1.DefaultGateway(),
+					Provider: &egv1a1.EnvoyGatewayProvider{
+						Type:   egv1a1.ProviderTypeCustom,
+						Custom: nil,
+					},
+				},
+			},
+			expect: false,
+		},
+		{
+			name: "empty custom provider",
+			eg: &egv1a1.EnvoyGateway{
+				EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{
+					Gateway: egv1a1.DefaultGateway(),
+					Provider: &egv1a1.EnvoyGatewayProvider{
+						Type:   egv1a1.ProviderTypeCustom,
+						Custom: &egv1a1.EnvoyGatewayCustomProvider{},
+					},
+				},
+			},
+			expect: false,
+		},
+		{
+			name: "custom provider with file resource provider and host infra provider",
+			eg: &egv1a1.EnvoyGateway{
+				EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{
+					Gateway: egv1a1.DefaultGateway(),
+					Provider: &egv1a1.EnvoyGatewayProvider{
+						Type: egv1a1.ProviderTypeCustom,
+						Custom: &egv1a1.EnvoyGatewayCustomProvider{
+							Resource: egv1a1.EnvoyGatewayResourceProvider{
+								Type: egv1a1.ResourceProviderTypeFile,
+								File: &egv1a1.EnvoyGatewayFileResourceProvider{
+									Paths: []string{"foo", "bar"},
+								},
+							},
+							Infrastructure: &egv1a1.EnvoyGatewayInfrastructureProvider{
+								Type: egv1a1.InfrastructureProviderTypeHost,
+								Host: &egv1a1.EnvoyGatewayHostInfrastructureProvider{},
+							},
+						},
+					},
+				},
+			},
+			expect: true,
+		},
+		{
+			name: "custom provider with file provider and k8s infra provider",
+			eg: &egv1a1.EnvoyGateway{
+				EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{
+					Gateway: egv1a1.DefaultGateway(),
+					Provider: &egv1a1.EnvoyGatewayProvider{
+						Type: egv1a1.ProviderTypeCustom,
+						Custom: &egv1a1.EnvoyGatewayCustomProvider{
+							Resource: egv1a1.EnvoyGatewayResourceProvider{
+								Type: egv1a1.ResourceProviderTypeFile,
+								File: &egv1a1.EnvoyGatewayFileResourceProvider{
+									Paths: []string{"foo", "bar"},
+								},
+							},
+						},
+					},
+				},
+			},
+			expect: true,
+		},
+		{
+			name: "custom provider with unsupported resource provider",
+			eg: &egv1a1.EnvoyGateway{
+				EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{
+					Gateway: egv1a1.DefaultGateway(),
+					Provider: &egv1a1.EnvoyGatewayProvider{
+						Type: egv1a1.ProviderTypeCustom,
+						Custom: &egv1a1.EnvoyGatewayCustomProvider{
+							Resource: egv1a1.EnvoyGatewayResourceProvider{
+								Type: "foobar",
+							},
+						},
+					},
+				},
+			},
+			expect: false,
+		},
+		{
+			name: "custom provider with file provider but no file struct",
+			eg: &egv1a1.EnvoyGateway{
+				EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{
+					Gateway: egv1a1.DefaultGateway(),
+					Provider: &egv1a1.EnvoyGatewayProvider{
+						Type: egv1a1.ProviderTypeCustom,
+						Custom: &egv1a1.EnvoyGatewayCustomProvider{
+							Resource: egv1a1.EnvoyGatewayResourceProvider{
+								Type: egv1a1.ResourceProviderTypeFile,
+							},
+						},
+					},
+				},
+			},
+			expect: false,
+		},
+		{
+			name: "custom provider with file provider and host infra provider but no host struct",
+			eg: &egv1a1.EnvoyGateway{
+				EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{
+					Gateway: egv1a1.DefaultGateway(),
+					Provider: &egv1a1.EnvoyGatewayProvider{
+						Type: egv1a1.ProviderTypeCustom,
+						Custom: &egv1a1.EnvoyGatewayCustomProvider{
+							Resource: egv1a1.EnvoyGatewayResourceProvider{
+								Type: egv1a1.ResourceProviderTypeFile,
+								File: &egv1a1.EnvoyGatewayFileResourceProvider{
+									Paths: []string{"a", "b"},
+								},
+							},
+							Infrastructure: &egv1a1.EnvoyGatewayInfrastructureProvider{
+								Type: egv1a1.InfrastructureProviderTypeHost,
+							},
+						},
+					},
+				},
+			},
+			expect: false,
+		},
+		{
+			name: "custom provider with file provider and unsupported infra provider",
+			eg: &egv1a1.EnvoyGateway{
+				EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{
+					Gateway: egv1a1.DefaultGateway(),
+					Provider: &egv1a1.EnvoyGatewayProvider{
+						Type: egv1a1.ProviderTypeCustom,
+						Custom: &egv1a1.EnvoyGatewayCustomProvider{
+							Resource: egv1a1.EnvoyGatewayResourceProvider{
+								Type: egv1a1.ResourceProviderTypeFile,
+								File: &egv1a1.EnvoyGatewayFileResourceProvider{
+									Paths: []string{"a", "b"},
+								},
+							},
+							Infrastructure: &egv1a1.EnvoyGatewayInfrastructureProvider{
+								Type: "foobar",
+							},
+						},
+					},
+				},
+			},
+			expect: false,
+		},
+		{
+			name: "custom provider with file provider and host infra provider but no paths assign in resource",
+			eg: &egv1a1.EnvoyGateway{
+				EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{
+					Gateway: egv1a1.DefaultGateway(),
+					Provider: &egv1a1.EnvoyGatewayProvider{
+						Type: egv1a1.ProviderTypeCustom,
+						Custom: &egv1a1.EnvoyGatewayCustomProvider{
+							Resource: egv1a1.EnvoyGatewayResourceProvider{
+								Type: egv1a1.ResourceProviderTypeFile,
+								File: &egv1a1.EnvoyGatewayFileResourceProvider{},
+							},
+							Infrastructure: &egv1a1.EnvoyGatewayInfrastructureProvider{
+								Type: egv1a1.InfrastructureProviderTypeHost,
+								Host: &egv1a1.EnvoyGatewayHostInfrastructureProvider{},
+							},
+						},
+					},
 				},
 			},
 			expect: false,

diff --git a/api/v1alpha1/validation/envoyproxy_validate_test.go b/api/v1alpha1/validation/envoyproxy_validate_test.go
@@ -52,7 +52,7 @@ func TestValidateEnvoyProxy(t *testing.T) {
 				},
 				Spec: egv1a1.EnvoyProxySpec{
 					Provider: &egv1a1.EnvoyProxyProvider{
-						Type: egv1a1.ProviderTypeFile,
+						Type: egv1a1.ProviderTypeCustom,
 					},
 				},
 			},

diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml
@@ -10227,6 +10227,7 @@ spec:
                       optional auxiliary control planes. Supported types are "Kubernetes".
                     enum:
                     - Kubernetes
+                    - Custom
                     type: string
                 required:
                 - type

diff --git a/go.mod b/go.mod
@@ -214,7 +214,7 @@ require (
 	github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect
 	github.com/evanphx/json-patch v5.9.0+incompatible
 	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
-	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/fsnotify/fsnotify v1.7.0
 	github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
 	github.com/go-errors/errors v1.5.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
-Original file line number
+Diff line change
@@ Expand Up @@
     	if r.Kubernetes.ShutdownManager == nil {
     		r.Kubernetes.ShutdownManager = &ShutdownManager{Image: ptr.To(DefaultShutdownManagerImage)}
     	}
     	return r.Kubernetes
     }
@@ Expand Down @@