Add a RoutingMode option to cniConfig (#6766)

* Add a RoutingMode option to cniConfig This adds two routing modes for CIlium to use in the CNI configuration. Tunneled keeps the default geneve tunnel mode. Direct adds a no tunnel mode with native as routing mode Signed-off-by: Maartje Eyskens <maartje.eyskens@isovalent.com> * Documet the routingmode flag for Cilium Signed-off-by: Maartje Eyskens <maartje.eyskens@isovalent.com> * Update API comments for native routing --------- Signed-off-by: Maartje Eyskens <maartje.eyskens@isovalent.com> Co-authored-by: Chris Doherty <chris.doherty4@gmail.com>
aws · Dec 12, 2023 · 91693d0 · 91693d0
1 parent 3e3b2ab
commit 91693d0
Show file tree

Hide file tree

Showing 6 changed files with 191 additions and 2 deletions.
diff --git a/config/crd/bases/anywhere.eks.amazonaws.com_clusters.yaml b/config/crd/bases/anywhere.eks.amazonaws.com_clusters.yaml
@@ -72,11 +72,34 @@ spec:
                               network interfaces are used for masquerading. Accepted
                               values are a valid interface name or interface prefix.
                             type: string
+                          ipv4NativeRoutingCIDR:
+                            description: IPv4NativeRoutingCIDR specifies the CIDR
+                              to use when RoutingMode is set to direct. When specified,
+                              Cilium assumes networking for this CIDR is preconfigured
+                              and hands traffic destined for that range to the Linux
+                              network stack without applying any SNAT. If this is
+                              not set autoDirectNodeRoutes will be set to true
+                            type: string
+                          ipv6NativeRoutingCIDR:
+                            description: IPv6NativeRoutingCIDR specifies the IPv6
+                              CIDR to use when RoutingMode is set to direct. When
+                              specified, Cilium assumes networking for this CIDR is
+                              preconfigured and hands traffic destined for that range
+                              to the Linux network stack without applying any SNAT.
+                              If this is not set autoDirectNodeRoutes will be set
+                              to true
+                            type: string
                           policyEnforcementMode:
                             description: PolicyEnforcementMode determines communication
                               allowed between pods. Accepted values are default, always,
                               never.
                             type: string
+                          routingMode:
+                            description: RoutingMode indicates the routing tunnel
+                              mode to use for Cilium. Accepted values are overlay
+                              (geneve tunnel with overlay) or direct (tunneling disabled
+                              with direct routing) Defaults to overlay.
+                            type: string
                           skipUpgrade:
                             description: SkipUpgrade indicicates that Cilium maintenance
                               should be skipped during upgrades. This can be used

diff --git a/docs/content/en/docs/getting-started/optional/cni.md b/docs/content/en/docs/getting-started/optional/cni.md
@@ -169,6 +169,29 @@ spec:
         egressMasqueradeInterfaces: "eth0"
 ```
 
+### RoutingMode option for Cilium plugin
+
+By default all traffic is sent by Cilium over Geneve tunneling on the network. The `routingMode` option allows users to switch to [native routing](https://docs.cilium.io/en/v1.12/concepts/networking/routing/#native-routing) instead. 
+
+This field can be set as follows:
+```yaml
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: Cluster
+metadata:
+  name: my-cluster-name
+spec:
+  clusterNetwork:
+    pods:
+      cidrBlocks:
+      - 192.168.0.0/16
+    services:
+      cidrBlocks:
+      - 10.96.0.0/12
+    cniConfig:
+      cilium: 
+        routingMode: "direct"
+```
+
 ### Use a custom CNI
 
 EKS Anywhere can be configured to skip EKS Anywhere's default Cilium CNI upgrades via the `skipUpgrade` field. 

diff --git a/internal/pkg/api/cluster.go b/internal/pkg/api/cluster.go
@@ -83,6 +83,16 @@ func WithCiliumSkipUpgrade() ClusterFiller {
 	}
 }
 
+// WithCiliumRoutingMode sets the tunnel mode with the provided mode option to use.
+func WithCiliumRoutingMode(mode anywherev1.CiliumRoutingMode) ClusterFiller {
+	return func(c *anywherev1.Cluster) {
+		if c.Spec.ClusterNetwork.CNIConfig == nil {
+			c.Spec.ClusterNetwork.CNIConfig = &anywherev1.CNIConfig{Cilium: &anywherev1.CiliumConfig{}}
+		}
+		c.Spec.ClusterNetwork.CNIConfig.Cilium.RoutingMode = mode
+	}
+}
+
 func WithClusterNamespace(ns string) ClusterFiller {
 	return func(c *anywherev1.Cluster) {
 		c.Namespace = ns

diff --git a/pkg/api/v1alpha1/cluster_types.go b/pkg/api/v1alpha1/cluster_types.go
@@ -604,6 +604,10 @@ func (n *CiliumConfig) Equal(o *CiliumConfig) bool {
 		return false
 	}
 
+	if n.RoutingMode != o.RoutingMode {
+		return false
+	}
+
 	oSkipUpgradeIsFalse := o.SkipUpgrade == nil || !*o.SkipUpgrade
 	nSkipUpgradeIsFalse := n.SkipUpgrade == nil || !*n.SkipUpgrade
 
@@ -804,6 +808,8 @@ type CNI string
 
 type CiliumPolicyEnforcementMode string
 
+type CiliumRoutingMode string
+
 type CNIConfig struct {
 	Cilium   *CiliumConfig   `json:"cilium,omitempty"`
 	Kindnetd *KindnetdConfig `json:"kindnetd,omitempty"`
@@ -814,6 +820,7 @@ func (n *CNIConfig) IsManaged() bool {
 	return n != nil && (n.Kindnetd != nil || n.Cilium != nil && n.Cilium.IsManaged())
 }
 
+// CiliumConfig contains configuration specific to the Cilium CNI.
 type CiliumConfig struct {
 	// PolicyEnforcementMode determines communication allowed between pods. Accepted values are default, always, never.
 	PolicyEnforcementMode CiliumPolicyEnforcementMode `json:"policyEnforcementMode,omitempty"`
@@ -826,6 +833,28 @@ type CiliumConfig struct {
 	// be used when operators wish to self manage the Cilium installation.
 	// +optional
 	SkipUpgrade *bool `json:"skipUpgrade,omitempty"`
+
+	// RoutingMode indicates the routing tunnel mode to use for Cilium. Accepted values are overlay (geneve tunnel with overlay)
+	// or direct (tunneling disabled with direct routing)
+	// Defaults to overlay.
+	// +optional
+	RoutingMode CiliumRoutingMode `json:"routingMode,omitempty"`
+
+	// IPv4NativeRoutingCIDR specifies the CIDR to use when RoutingMode is set to direct.
+	// When specified, Cilium assumes networking for this CIDR is preconfigured and
+	// hands traffic destined for that range to the Linux network stack without
+	// applying any SNAT.
+	// If this is not set autoDirectNodeRoutes will be set to true
+	// +optional
+	IPv4NativeRoutingCIDR string `json:"ipv4NativeRoutingCIDR,omitempty"`
+
+	// IPv6NativeRoutingCIDR specifies the IPv6 CIDR to use when RoutingMode is set to direct.
+	// When specified, Cilium assumes networking for this CIDR is preconfigured and
+	// hands traffic destined for that range to the Linux network stack without
+	// applying any SNAT.
+	// If this is not set autoDirectNodeRoutes will be set to true
+	// +optional
+	IPv6NativeRoutingCIDR string `json:"ipv6NativeRoutingCIDR,omitempty"`
 }
 
 // IsManaged returns true if SkipUpgrade is nil or false indicating EKS-A is responsible for
@@ -834,19 +863,26 @@ func (n *CiliumConfig) IsManaged() bool {
 	return n.SkipUpgrade == nil || !*n.SkipUpgrade
 }
 
+// KindnetdConfig contains configuration specific to the Kindnetd CNI.
 type KindnetdConfig struct{}
 
 const (
-	Cilium           CNI = "cilium"
+	// Cilium is the EKS-A Cilium.
+	Cilium CNI = "cilium"
+
+	// CiliumEnterprise is Isovalents Cilium.
 	CiliumEnterprise CNI = "cilium-enterprise"
-	Kindnetd         CNI = "kindnetd"
+
+	// Kindnetd is the CNI shipped with KinD.
+	Kindnetd CNI = "kindnetd"
 )
 
 var validCNIs = map[CNI]struct{}{
 	Cilium:   {},
 	Kindnetd: {},
 }
 
+// Policy enforcement modes for Cilium.
 const (
 	CiliumPolicyModeDefault CiliumPolicyEnforcementMode = "default"
 	CiliumPolicyModeAlways  CiliumPolicyEnforcementMode = "always"
@@ -859,6 +895,12 @@ var validCiliumPolicyEnforcementModes = map[CiliumPolicyEnforcementMode]bool{
 	CiliumPolicyModeNever:   true,
 }
 
+// Routing modes for Cilium.
+const (
+	CiliumRoutingModeOverlay CiliumRoutingMode = "overlay"
+	CiliumRoutingModeDirect  CiliumRoutingMode = "direct"
+)
+
 // FailureReasonType is a type for defining failure reasons.
 type FailureReasonType string
 

diff --git a/pkg/networking/cilium/templater.go b/pkg/networking/cilium/templater.go
@@ -237,6 +237,18 @@ func templateValues(spec *cluster.Spec, versionsBundle *cluster.VersionsBundle)
 		val["egressMasqueradeInterfaces"] = spec.Cluster.Spec.ClusterNetwork.CNIConfig.Cilium.EgressMasqueradeInterfaces
 	}
 
+	if spec.Cluster.Spec.ClusterNetwork.CNIConfig.Cilium.RoutingMode == anywherev1.CiliumRoutingModeDirect {
+		val["tunnel"] = "disabled"
+
+		if spec.Cluster.Spec.ClusterNetwork.CNIConfig.Cilium.IPv4NativeRoutingCIDR == "" &&
+			spec.Cluster.Spec.ClusterNetwork.CNIConfig.Cilium.IPv6NativeRoutingCIDR == "" {
+			val["autoDirectNodeRoutes"] = "true"
+		} else {
+			val["ipv4NativeRoutingCIDR"] = spec.Cluster.Spec.ClusterNetwork.CNIConfig.Cilium.IPv4NativeRoutingCIDR
+			val["ipv6NativeRoutingCIDR"] = spec.Cluster.Spec.ClusterNetwork.CNIConfig.Cilium.IPv6NativeRoutingCIDR
+		}
+	}
+
 	return val
 }
 

diff --git a/pkg/networking/cilium/templater_test.go b/pkg/networking/cilium/templater_test.go
@@ -278,6 +278,85 @@ func TestTemplaterGenerateManifestEgressMasqueradeInterfacesSuccess(t *testing.T
 	tt.Expect(tt.t.GenerateManifest(tt.ctx, tt.spec)).To(Equal(tt.manifest), "templater.GenerateManifest() should return right manifest")
 }
 
+func TestTemplaterGenerateManifestDirectRouteModeSuccess(t *testing.T) {
+	wantValues := map[string]interface{}{
+		"cni": map[string]interface{}{
+			"chainingMode": "portmap",
+		},
+		"ipam": map[string]interface{}{
+			"mode": "kubernetes",
+		},
+		"identityAllocationMode": "crd",
+		"prometheus": map[string]interface{}{
+			"enabled": true,
+		},
+		"rollOutCiliumPods":    true,
+		"tunnel":               "disabled",
+		"autoDirectNodeRoutes": "true",
+		"image": map[string]interface{}{
+			"repository": "public.ecr.aws/isovalent/cilium",
+			"tag":        "v1.9.11-eksa.1",
+		},
+		"operator": map[string]interface{}{
+			"image": map[string]interface{}{
+				"repository": "public.ecr.aws/isovalent/operator",
+				"tag":        "v1.9.11-eksa.1",
+			},
+			"prometheus": map[string]interface{}{
+				"enabled": true,
+			},
+		},
+	}
+
+	tt := newtemplaterTest(t)
+	tt.spec.Cluster.Spec.ManagementCluster.Name = "managed"
+	tt.spec.Cluster.Spec.ClusterNetwork.CNIConfig.Cilium.RoutingMode = v1alpha1.CiliumRoutingModeDirect
+	tt.expectHelmTemplateWith(eqMap(wantValues), "1.22").Return(tt.manifest, nil)
+
+	tt.Expect(tt.t.GenerateManifest(tt.ctx, tt.spec)).To(Equal(tt.manifest), "templater.GenerateManifest() should return right manifest")
+}
+
+func TestTemplaterGenerateManifestDirectModeManualIPCIDRSuccess(t *testing.T) {
+	wantValues := map[string]interface{}{
+		"cni": map[string]interface{}{
+			"chainingMode": "portmap",
+		},
+		"ipam": map[string]interface{}{
+			"mode": "kubernetes",
+		},
+		"identityAllocationMode": "crd",
+		"prometheus": map[string]interface{}{
+			"enabled": true,
+		},
+		"rollOutCiliumPods":     true,
+		"tunnel":                "disabled",
+		"ipv4NativeRoutingCIDR": "192.168.0.0/24",
+		"ipv6NativeRoutingCIDR": "2001:db8::/32",
+		"image": map[string]interface{}{
+			"repository": "public.ecr.aws/isovalent/cilium",
+			"tag":        "v1.9.11-eksa.1",
+		},
+		"operator": map[string]interface{}{
+			"image": map[string]interface{}{
+				"repository": "public.ecr.aws/isovalent/operator",
+				"tag":        "v1.9.11-eksa.1",
+			},
+			"prometheus": map[string]interface{}{
+				"enabled": true,
+			},
+		},
+	}
+
+	tt := newtemplaterTest(t)
+	tt.spec.Cluster.Spec.ManagementCluster.Name = "managed"
+	tt.spec.Cluster.Spec.ClusterNetwork.CNIConfig.Cilium.RoutingMode = v1alpha1.CiliumRoutingModeDirect
+	tt.spec.Cluster.Spec.ClusterNetwork.CNIConfig.Cilium.IPv4NativeRoutingCIDR = "192.168.0.0/24"
+	tt.spec.Cluster.Spec.ClusterNetwork.CNIConfig.Cilium.IPv6NativeRoutingCIDR = "2001:db8::/32"
+	tt.expectHelmTemplateWith(eqMap(wantValues), "1.22").Return(tt.manifest, nil)
+
+	tt.Expect(tt.t.GenerateManifest(tt.ctx, tt.spec)).To(Equal(tt.manifest), "templater.GenerateManifest() should return right manifest")
+}
+
 func TestTemplaterGenerateManifestError(t *testing.T) {
 	expectedAttempts := 2
 	tt := newtemplaterTest(t)