From f1962fa9c5dcac2f23c88bc48937b4f1ebdbeeba Mon Sep 17 00:00:00 2001 From: Tarun Chinmai Sekar Date: Fri, 27 Sep 2024 16:16:48 -0700 Subject: [PATCH 1/9] Initial groundwork for VLAN controller --- api/v1alpha1/conversion.go | 5 ++ api/v1alpha1/zz_generated.conversion.go | 16 ++-- api/v1alpha2/linodecluster_types.go | 5 ++ api/v1alpha2/linodecluster_webhook.go | 7 ++ ...cture.cluster.x-k8s.io_linodeclusters.yaml | 7 ++ ...uster.x-k8s.io_linodeclustertemplates.yaml | 7 ++ .../linodemachine_controller_helpers.go | 86 ++++++++++++++++++- util/helpers.go | 13 +++ 8 files changed, 135 insertions(+), 11 deletions(-) diff --git a/api/v1alpha1/conversion.go b/api/v1alpha1/conversion.go index 319415ca..28af4a0a 100644 --- a/api/v1alpha1/conversion.go +++ b/api/v1alpha1/conversion.go @@ -86,3 +86,8 @@ func Convert_v1alpha2_LinodeObjectStorageBucket_To_v1alpha1_LinodeObjectStorageB } return autoConvert_v1alpha2_LinodeObjectStorageBucket_To_v1alpha1_LinodeObjectStorageBucket(in, out, scope) } + +func Convert_v1alpha2_LinodeClusterSpec_To_v1alpha1_LinodeClusterSpec(in *infrastructurev1alpha2.LinodeClusterSpec, out *LinodeClusterSpec, scope conversion.Scope) error { + // VLAN is not supported in v1alpha1 + return nil +} diff --git a/api/v1alpha1/zz_generated.conversion.go b/api/v1alpha1/zz_generated.conversion.go index 975be8c8..8f77aefe 100644 --- a/api/v1alpha1/zz_generated.conversion.go +++ b/api/v1alpha1/zz_generated.conversion.go @@ -95,11 +95,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1alpha2.LinodeClusterSpec)(nil), (*LinodeClusterSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha2_LinodeClusterSpec_To_v1alpha1_LinodeClusterSpec(a.(*v1alpha2.LinodeClusterSpec), b.(*LinodeClusterSpec), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*LinodeClusterStatus)(nil), (*v1alpha2.LinodeClusterStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha1_LinodeClusterStatus_To_v1alpha2_LinodeClusterStatus(a.(*LinodeClusterStatus), b.(*v1alpha2.LinodeClusterStatus), scope) }); err != nil { @@ -320,6 +315,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1alpha2.LinodeClusterSpec)(nil), (*LinodeClusterSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha2_LinodeClusterSpec_To_v1alpha1_LinodeClusterSpec(a.(*v1alpha2.LinodeClusterSpec), b.(*LinodeClusterSpec), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1alpha2.LinodeMachineSpec)(nil), (*LinodeMachineSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_LinodeMachineSpec_To_v1alpha1_LinodeMachineSpec(a.(*v1alpha2.LinodeMachineSpec), b.(*LinodeMachineSpec), scope) }); err != nil { @@ -517,16 +517,12 @@ func autoConvert_v1alpha2_LinodeClusterSpec_To_v1alpha1_LinodeClusterSpec(in *v1 if err := Convert_v1alpha2_NetworkSpec_To_v1alpha1_NetworkSpec(&in.Network, &out.Network, s); err != nil { return err } + // WARNING: in.UseVlan requires manual conversion: does not exist in peer-type out.VPCRef = (*v1.ObjectReference)(unsafe.Pointer(in.VPCRef)) out.CredentialsRef = (*v1.SecretReference)(unsafe.Pointer(in.CredentialsRef)) return nil } -// Convert_v1alpha2_LinodeClusterSpec_To_v1alpha1_LinodeClusterSpec is an autogenerated conversion function. -func Convert_v1alpha2_LinodeClusterSpec_To_v1alpha1_LinodeClusterSpec(in *v1alpha2.LinodeClusterSpec, out *LinodeClusterSpec, s conversion.Scope) error { - return autoConvert_v1alpha2_LinodeClusterSpec_To_v1alpha1_LinodeClusterSpec(in, out, s) -} - func autoConvert_v1alpha1_LinodeClusterStatus_To_v1alpha2_LinodeClusterStatus(in *LinodeClusterStatus, out *v1alpha2.LinodeClusterStatus, s conversion.Scope) error { out.Ready = in.Ready out.FailureReason = (*errors.ClusterStatusError)(unsafe.Pointer(in.FailureReason)) diff --git a/api/v1alpha2/linodecluster_types.go b/api/v1alpha2/linodecluster_types.go index 87ed9f84..81dede70 100644 --- a/api/v1alpha2/linodecluster_types.go +++ b/api/v1alpha2/linodecluster_types.go @@ -43,6 +43,11 @@ type LinodeClusterSpec struct { // +optional Network NetworkSpec `json:"network"` + // UseVlan provisions a cluster that uses VLANs instead of VPCs. IPAM is managed internally. + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" + // +optional + UseVlan bool `json:"useVlan"` + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" // +optional VPCRef *corev1.ObjectReference `json:"vpcRef,omitempty"` diff --git a/api/v1alpha2/linodecluster_webhook.go b/api/v1alpha2/linodecluster_webhook.go index 355782db..7a05ddb8 100644 --- a/api/v1alpha2/linodecluster_webhook.go +++ b/api/v1alpha2/linodecluster_webhook.go @@ -106,6 +106,13 @@ func (r *LinodeCluster) validateLinodeClusterSpec(ctx context.Context, client Li } } + if r.Spec.UseVlan && r.Spec.VPCRef != nil { + errs = append(errs, &field.Error{ + Field: "Cannot use VLANs and VPCs together. Unset `useVlan` or remove `vpcRef`", + Type: field.ErrorTypeInvalid, + }) + } + if len(errs) == 0 { return nil } diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_linodeclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_linodeclusters.yaml index 569c40ce..e5159881 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_linodeclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_linodeclusters.yaml @@ -407,6 +407,13 @@ spec: region: description: The Linode Region the LinodeCluster lives in. type: string + useVlan: + description: UseVlan provisions a cluster that uses VLANs instead + of VPCs. IPAM is managed internally. + type: boolean + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf vpcRef: description: |- ObjectReference contains enough information to let you inspect or modify the referred object. diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_linodeclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_linodeclustertemplates.yaml index 4aef1cbd..1044e461 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_linodeclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_linodeclustertemplates.yaml @@ -335,6 +335,13 @@ spec: region: description: The Linode Region the LinodeCluster lives in. type: string + useVlan: + description: UseVlan provisions a cluster that uses VLANs + instead of VPCs. IPAM is managed internally. + type: boolean + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf vpcRef: description: |- ObjectReference contains enough information to let you inspect or modify the referred object. diff --git a/controller/linodemachine_controller_helpers.go b/controller/linodemachine_controller_helpers.go index 6f99d02d..52176e75 100644 --- a/controller/linodemachine_controller_helpers.go +++ b/controller/linodemachine_controller_helpers.go @@ -24,15 +24,18 @@ import ( "errors" "fmt" "net/http" + "net/netip" "slices" "sort" "github.com/go-logr/logr" "github.com/google/uuid" "github.com/linode/linodego" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/retry" "k8s.io/utils/ptr" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" cerrs "sigs.k8s.io/cluster-api/errors" @@ -120,7 +123,7 @@ func newCreateConfig(ctx context.Context, machineScope *scope.MachineScope, logg createConfig.RootPass = uuid.NewString() } - // if vpc, attach additional interface as eth0 to linode + // if vpc is enabled, attach additional interface as eth0 to linode if machineScope.LinodeCluster.Spec.VPCRef != nil { iface, err := getVPCInterfaceConfig(ctx, machineScope, createConfig.Interfaces, logger) if err != nil { @@ -134,6 +137,20 @@ func newCreateConfig(ctx context.Context, machineScope *scope.MachineScope, logg } } + // if vlan is enabled, attach additional interface as eth0 to linode + if machineScope.LinodeCluster.Spec.UseVlan { + iface, err := getVlanInterfaceConfig(ctx, machineScope, createConfig.Interfaces, logger) + if err != nil { + logger.Error(err, "Failed to get VLAN interface config") + + return nil, err + } + if iface != nil { + // add VLAN interface as first interface + createConfig.Interfaces = slices.Insert(createConfig.Interfaces, 0, *iface) + } + } + if machineScope.LinodeMachine.Spec.PlacementGroupRef != nil { pgID, err := getPlacementGroupID(ctx, machineScope, logger) if err != nil { @@ -345,6 +362,73 @@ func getFirewallID(ctx context.Context, machineScope *scope.MachineScope, logger return *linodeFirewall.Spec.FirewallID, nil } +func getNextIP(ips map[string]string, prefixStr string) string { + prefix := netip.MustParsePrefix(prefixStr) + currentIp := prefix.Addr().Next() + + ipString := currentIp.String() + for { + if _, exists := ips[ipString]; !exists { + break + } + currentIp = currentIp.Next() + ipString = currentIp.String() + } + return ipString +} + +func reserveNextIP(ctx context.Context, machineScope *scope.MachineScope, logger logr.Logger) (string, error) { + namespace := machineScope.Cluster.Namespace + clusterName := machineScope.Cluster.Name + + var ipsMap corev1.ConfigMap + err := machineScope.Client.Get(ctx, client.ObjectKey{Namespace: namespace, Name: fmt.Sprintf("%s-ips", clusterName)}, &ipsMap) + if err != nil && !apierrors.IsNotFound(err) { + return "", fmt.Errorf("retreiving ips configmap %s/%s: %w", namespace, fmt.Sprintf("%s-ips", clusterName), err) + } + + if ip, ok := ipsMap.Data[machineScope.LinodeMachine.Name]; ok { + return ip, nil + } + + var nextIP string + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + if err = machineScope.Client.Get(ctx, client.ObjectKeyFromObject(&ipsMap), &ipsMap); err != nil { + return fmt.Errorf("retreiving ips configmap %s/%s: %w", namespace, fmt.Sprintf("%s-ips", clusterName), err) + } + nextIP = getNextIP(util.InvertMap(ipsMap.Data), "10.0.0.0/11") + ipsMap.Data[machineScope.LinodeMachine.Name] = nextIP + if err := machineScope.Client.Update(ctx, &ipsMap); err != nil { + return fmt.Errorf("updating ips configMap: %w", err) + } + return nil + }) + if err != nil { + return "", err + } + + logger.Info("onbained IP for machine", "name", machineScope.LinodeMachine.Name, "ip", nextIP) + // if an IP is available + return nextIP, nil +} + +func getVlanInterfaceConfig(ctx context.Context, machineScope *scope.MachineScope, interfaces []linodego.InstanceConfigInterfaceCreateOptions, logger logr.Logger) (*linodego.InstanceConfigInterfaceCreateOptions, error) { + logger = logger.WithValues("vlanName", machineScope.Cluster.Name) + + // Try to obtain a IP for the machine using its name + + ip, err := reserveNextIP(ctx, machineScope, logger) + if err != nil { + return nil, err + } + + return &linodego.InstanceConfigInterfaceCreateOptions{ + Purpose: linodego.InterfacePurposeVLAN, + Label: machineScope.Cluster.Name, + IPAMAddress: ip, + }, nil +} + func getVPCInterfaceConfig(ctx context.Context, machineScope *scope.MachineScope, interfaces []linodego.InstanceConfigInterfaceCreateOptions, logger logr.Logger) (*linodego.InstanceConfigInterfaceCreateOptions, error) { name := machineScope.LinodeCluster.Spec.VPCRef.Name namespace := machineScope.LinodeCluster.Spec.VPCRef.Namespace diff --git a/util/helpers.go b/util/helpers.go index cc81cb0c..f9ab388d 100644 --- a/util/helpers.go +++ b/util/helpers.go @@ -61,3 +61,16 @@ func GetInstanceID(providerID *string) (int, error) { } return instanceID, nil } + +// InvertMap returns a given map with keys and values are swapped +func InvertMap[M ~map[K]V, K comparable, V comparable](m M) map[V]K { + if m == nil { + return nil + } + + r := make(map[V]K, len(m)) + for k, v := range m { + r[v] = k + } + return r +} From 1a3ebb76c2ae339cd590e15fd829bea4d9c9923d Mon Sep 17 00:00:00 2001 From: Tarun Chinmai Sekar Date: Fri, 27 Sep 2024 16:18:18 -0700 Subject: [PATCH 2/9] call auto generated one --- api/v1alpha1/conversion.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/v1alpha1/conversion.go b/api/v1alpha1/conversion.go index 28af4a0a..0e9bd1ae 100644 --- a/api/v1alpha1/conversion.go +++ b/api/v1alpha1/conversion.go @@ -89,5 +89,5 @@ func Convert_v1alpha2_LinodeObjectStorageBucket_To_v1alpha1_LinodeObjectStorageB func Convert_v1alpha2_LinodeClusterSpec_To_v1alpha1_LinodeClusterSpec(in *infrastructurev1alpha2.LinodeClusterSpec, out *LinodeClusterSpec, scope conversion.Scope) error { // VLAN is not supported in v1alpha1 - return nil + return autoConvert_v1alpha2_LinodeClusterSpec_To_v1alpha1_LinodeClusterSpec(in, out, scope) } From 3d236998370018b4265b4bdf624653e87b839486 Mon Sep 17 00:00:00 2001 From: Tarun Chinmai Sekar Date: Fri, 27 Sep 2024 19:23:15 -0700 Subject: [PATCH 3/9] preflight stuff --- .../linodemachine_controller_helpers.go | 61 +++++++++++-------- util/helpers.go | 13 ---- 2 files changed, 36 insertions(+), 38 deletions(-) diff --git a/controller/linodemachine_controller_helpers.go b/controller/linodemachine_controller_helpers.go index 52176e75..14e79f7b 100644 --- a/controller/linodemachine_controller_helpers.go +++ b/controller/linodemachine_controller_helpers.go @@ -31,6 +31,7 @@ import ( "github.com/go-logr/logr" "github.com/google/uuid" "github.com/linode/linodego" + "golang.org/x/exp/maps" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" @@ -83,24 +84,7 @@ func retryIfTransient(err error) (ctrl.Result, error) { return ctrl.Result{}, err } -func newCreateConfig(ctx context.Context, machineScope *scope.MachineScope, logger logr.Logger) (*linodego.InstanceCreateOptions, error) { - var err error - - createConfig := linodeMachineSpecToInstanceCreateConfig(machineScope.LinodeMachine.Spec) - if createConfig == nil { - err = errors.New("failed to convert machine spec to create instance config") - - logger.Error(err, "Panic! Struct of LinodeMachineSpec is different than InstanceCreateOptions") - - return nil, err - } - - createConfig.Booted = util.Pointer(false) - - if err := setUserData(ctx, machineScope, createConfig, logger); err != nil { - return nil, err - } - +func fillCreateConfig(createConfig *linodego.InstanceCreateOptions, machineScope *scope.MachineScope) { if machineScope.LinodeMachine.Spec.PrivateIP != nil { createConfig.PrivateIP = *machineScope.LinodeMachine.Spec.PrivateIP } else { @@ -122,6 +106,26 @@ func newCreateConfig(ctx context.Context, machineScope *scope.MachineScope, logg if createConfig.RootPass == "" { createConfig.RootPass = uuid.NewString() } +} + +func newCreateConfig(ctx context.Context, machineScope *scope.MachineScope, logger logr.Logger) (*linodego.InstanceCreateOptions, error) { + var err error + + createConfig := linodeMachineSpecToInstanceCreateConfig(machineScope.LinodeMachine.Spec) + if createConfig == nil { + err = errors.New("failed to convert machine spec to create instance config") + + logger.Error(err, "Panic! Struct of LinodeMachineSpec is different than InstanceCreateOptions") + + return nil, err + } + + createConfig.Booted = util.Pointer(false) + if err := setUserData(ctx, machineScope, createConfig, logger); err != nil { + return nil, err + } + + fillCreateConfig(createConfig, machineScope) // if vpc is enabled, attach additional interface as eth0 to linode if machineScope.LinodeCluster.Spec.VPCRef != nil { @@ -139,10 +143,9 @@ func newCreateConfig(ctx context.Context, machineScope *scope.MachineScope, logg // if vlan is enabled, attach additional interface as eth0 to linode if machineScope.LinodeCluster.Spec.UseVlan { - iface, err := getVlanInterfaceConfig(ctx, machineScope, createConfig.Interfaces, logger) + iface, err := getVlanInterfaceConfig(ctx, machineScope, logger) if err != nil { logger.Error(err, "Failed to get VLAN interface config") - return nil, err } if iface != nil { @@ -208,7 +211,7 @@ func buildInstanceAddrs(ctx context.Context, machineScope *scope.MachineScope, i Type: clusterv1.MachineExternalIP, }) - // Iterate over interfaces in config and find VPC specific ips + // Iterate over interfaces in config and find VPC or VLAN specific ips for _, iface := range configs[0].Interfaces { if iface.VPCID != nil && iface.IPv4.VPC != "" { ips = append(ips, clusterv1.MachineAddress{ @@ -216,6 +219,13 @@ func buildInstanceAddrs(ctx context.Context, machineScope *scope.MachineScope, i Type: clusterv1.MachineInternalIP, }) } + + if iface.Purpose == linodego.InterfacePurposeVLAN { + ips = append(ips, clusterv1.MachineAddress{ + Address: iface.IPAMAddress, + Type: clusterv1.MachineInternalIP, + }) + } } // if a node has private ip, store it as well @@ -362,13 +372,13 @@ func getFirewallID(ctx context.Context, machineScope *scope.MachineScope, logger return *linodeFirewall.Spec.FirewallID, nil } -func getNextIP(ips map[string]string, prefixStr string) string { +func getNextIP(ips []string, prefixStr string) string { prefix := netip.MustParsePrefix(prefixStr) currentIp := prefix.Addr().Next() ipString := currentIp.String() for { - if _, exists := ips[ipString]; !exists { + if slices.Contains(ips, ipString) { break } currentIp = currentIp.Next() @@ -396,7 +406,8 @@ func reserveNextIP(ctx context.Context, machineScope *scope.MachineScope, logger if err = machineScope.Client.Get(ctx, client.ObjectKeyFromObject(&ipsMap), &ipsMap); err != nil { return fmt.Errorf("retreiving ips configmap %s/%s: %w", namespace, fmt.Sprintf("%s-ips", clusterName), err) } - nextIP = getNextIP(util.InvertMap(ipsMap.Data), "10.0.0.0/11") + + nextIP = getNextIP(maps.Values(ipsMap.Data), "10.0.0.0/11") ipsMap.Data[machineScope.LinodeMachine.Name] = nextIP if err := machineScope.Client.Update(ctx, &ipsMap); err != nil { return fmt.Errorf("updating ips configMap: %w", err) @@ -412,7 +423,7 @@ func reserveNextIP(ctx context.Context, machineScope *scope.MachineScope, logger return nextIP, nil } -func getVlanInterfaceConfig(ctx context.Context, machineScope *scope.MachineScope, interfaces []linodego.InstanceConfigInterfaceCreateOptions, logger logr.Logger) (*linodego.InstanceConfigInterfaceCreateOptions, error) { +func getVlanInterfaceConfig(ctx context.Context, machineScope *scope.MachineScope, logger logr.Logger) (*linodego.InstanceConfigInterfaceCreateOptions, error) { logger = logger.WithValues("vlanName", machineScope.Cluster.Name) // Try to obtain a IP for the machine using its name diff --git a/util/helpers.go b/util/helpers.go index f9ab388d..cc81cb0c 100644 --- a/util/helpers.go +++ b/util/helpers.go @@ -61,16 +61,3 @@ func GetInstanceID(providerID *string) (int, error) { } return instanceID, nil } - -// InvertMap returns a given map with keys and values are swapped -func InvertMap[M ~map[K]V, K comparable, V comparable](m M) map[V]K { - if m == nil { - return nil - } - - r := make(map[V]K, len(m)) - for k, v := range m { - r[v] = k - } - return r -} From ca234c05369501a5eeb1a7767e1efec25544e67e Mon Sep 17 00:00:00 2001 From: Tarun Chinmai Sekar Date: Mon, 30 Sep 2024 12:54:33 -0700 Subject: [PATCH 4/9] move to networkspec --- api/v1alpha1/zz_generated.conversion.go | 2 +- api/v1alpha2/linodecluster_types.go | 10 ++--- api/v1alpha2/linodecluster_webhook.go | 4 +- ...cture.cluster.x-k8s.io_linodeclusters.yaml | 14 +++--- ...uster.x-k8s.io_linodeclustertemplates.yaml | 14 +++--- .../linodemachine_controller_helpers.go | 44 ++++++++++++------- 6 files changed, 51 insertions(+), 37 deletions(-) diff --git a/api/v1alpha1/zz_generated.conversion.go b/api/v1alpha1/zz_generated.conversion.go index 8f77aefe..8aa2f833 100644 --- a/api/v1alpha1/zz_generated.conversion.go +++ b/api/v1alpha1/zz_generated.conversion.go @@ -517,7 +517,6 @@ func autoConvert_v1alpha2_LinodeClusterSpec_To_v1alpha1_LinodeClusterSpec(in *v1 if err := Convert_v1alpha2_NetworkSpec_To_v1alpha1_NetworkSpec(&in.Network, &out.Network, s); err != nil { return err } - // WARNING: in.UseVlan requires manual conversion: does not exist in peer-type out.VPCRef = (*v1.ObjectReference)(unsafe.Pointer(in.VPCRef)) out.CredentialsRef = (*v1.SecretReference)(unsafe.Pointer(in.CredentialsRef)) return nil @@ -1171,6 +1170,7 @@ func autoConvert_v1alpha2_NetworkSpec_To_v1alpha1_NetworkSpec(in *v1alpha2.Netwo out.NodeBalancerID = (*int)(unsafe.Pointer(in.NodeBalancerID)) // WARNING: in.ApiserverNodeBalancerConfigID requires manual conversion: does not exist in peer-type // WARNING: in.AdditionalPorts requires manual conversion: does not exist in peer-type + // WARNING: in.UseVlan requires manual conversion: does not exist in peer-type return nil } diff --git a/api/v1alpha2/linodecluster_types.go b/api/v1alpha2/linodecluster_types.go index 81dede70..a16dea68 100644 --- a/api/v1alpha2/linodecluster_types.go +++ b/api/v1alpha2/linodecluster_types.go @@ -43,11 +43,6 @@ type LinodeClusterSpec struct { // +optional Network NetworkSpec `json:"network"` - // UseVlan provisions a cluster that uses VLANs instead of VPCs. IPAM is managed internally. - // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" - // +optional - UseVlan bool `json:"useVlan"` - // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" // +optional VPCRef *corev1.ObjectReference `json:"vpcRef,omitempty"` @@ -153,6 +148,11 @@ type NetworkSpec struct { // additionalPorts contains list of ports to be configured with NodeBalancer. // +optional AdditionalPorts []LinodeNBPortConfig `json:"additionalPorts,omitempty"` + + // UseVlan provisions a cluster that uses VLANs instead of VPCs. IPAM is managed internally. + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" + // +optional + UseVlan bool `json:"useVlan"` } type LinodeNBPortConfig struct { diff --git a/api/v1alpha2/linodecluster_webhook.go b/api/v1alpha2/linodecluster_webhook.go index 7a05ddb8..5138a21e 100644 --- a/api/v1alpha2/linodecluster_webhook.go +++ b/api/v1alpha2/linodecluster_webhook.go @@ -106,9 +106,9 @@ func (r *LinodeCluster) validateLinodeClusterSpec(ctx context.Context, client Li } } - if r.Spec.UseVlan && r.Spec.VPCRef != nil { + if r.Spec.Network.UseVlan && r.Spec.VPCRef != nil { errs = append(errs, &field.Error{ - Field: "Cannot use VLANs and VPCs together. Unset `useVlan` or remove `vpcRef`", + Field: "Cannot use VLANs and VPCs together. Unset `network.useVlan` or remove `vpcRef`", Type: field.ErrorTypeInvalid, }) } diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_linodeclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_linodeclusters.yaml index e5159881..160b746d 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_linodeclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_linodeclusters.yaml @@ -403,17 +403,17 @@ spec: nodeBalancerID: description: NodeBalancerID is the id of NodeBalancer. type: integer + useVlan: + description: UseVlan provisions a cluster that uses VLANs instead + of VPCs. IPAM is managed internally. + type: boolean + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf type: object region: description: The Linode Region the LinodeCluster lives in. type: string - useVlan: - description: UseVlan provisions a cluster that uses VLANs instead - of VPCs. IPAM is managed internally. - type: boolean - x-kubernetes-validations: - - message: Value is immutable - rule: self == oldSelf vpcRef: description: |- ObjectReference contains enough information to let you inspect or modify the referred object. diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_linodeclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_linodeclustertemplates.yaml index 1044e461..179eb147 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_linodeclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_linodeclustertemplates.yaml @@ -331,17 +331,17 @@ spec: nodeBalancerID: description: NodeBalancerID is the id of NodeBalancer. type: integer + useVlan: + description: UseVlan provisions a cluster that uses VLANs + instead of VPCs. IPAM is managed internally. + type: boolean + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf type: object region: description: The Linode Region the LinodeCluster lives in. type: string - useVlan: - description: UseVlan provisions a cluster that uses VLANs - instead of VPCs. IPAM is managed internally. - type: boolean - x-kubernetes-validations: - - message: Value is immutable - rule: self == oldSelf vpcRef: description: |- ObjectReference contains enough information to let you inspect or modify the referred object. diff --git a/controller/linodemachine_controller_helpers.go b/controller/linodemachine_controller_helpers.go index 14e79f7b..af398ae4 100644 --- a/controller/linodemachine_controller_helpers.go +++ b/controller/linodemachine_controller_helpers.go @@ -55,7 +55,11 @@ import ( // Size limit in bytes on the decoded metadata.user_data for cloud-init // The decoded user_data must not exceed 16384 bytes per the Linode API -const maxBootstrapDataBytes = 16384 +const ( + maxBootstrapDataBytes = 16384 + vlanIPRange = "10.0.0.0/8" + vlanIPFormat = "%s/11" +) var ( errNoPublicIPv4Addrs = errors.New("no public ipv4 addresses set") @@ -142,7 +146,7 @@ func newCreateConfig(ctx context.Context, machineScope *scope.MachineScope, logg } // if vlan is enabled, attach additional interface as eth0 to linode - if machineScope.LinodeCluster.Spec.UseVlan { + if machineScope.LinodeCluster.Spec.Network.UseVlan { iface, err := getVlanInterfaceConfig(ctx, machineScope, logger) if err != nil { logger.Error(err, "Failed to get VLAN interface config") @@ -378,7 +382,7 @@ func getNextIP(ips []string, prefixStr string) string { ipString := currentIp.String() for { - if slices.Contains(ips, ipString) { + if !slices.Contains(ips, ipString) { break } currentIp = currentIp.Next() @@ -390,37 +394,47 @@ func getNextIP(ips []string, prefixStr string) string { func reserveNextIP(ctx context.Context, machineScope *scope.MachineScope, logger logr.Logger) (string, error) { namespace := machineScope.Cluster.Namespace clusterName := machineScope.Cluster.Name - + var nextIP string var ipsMap corev1.ConfigMap err := machineScope.Client.Get(ctx, client.ObjectKey{Namespace: namespace, Name: fmt.Sprintf("%s-ips", clusterName)}, &ipsMap) - if err != nil && !apierrors.IsNotFound(err) { + if err != nil { + if apierrors.IsNotFound(err) { + nextIP = getNextIP([]string{}, vlanIPRange) + ipsMap.Name = fmt.Sprintf("%s-ips", clusterName) + ipsMap.Namespace = namespace + ipsMap.Data = make(map[string]string) + ipsMap.Data[machineScope.LinodeMachine.Name] = nextIP + if err := machineScope.Client.Create(ctx, &ipsMap); err != nil { + return "", fmt.Errorf("creating ips configMap: %w", err) + } + logger.Info("Machine got ip", machineScope.LinodeMachine.Name, nextIP) + return fmt.Sprintf(vlanIPFormat, nextIP), nil + } return "", fmt.Errorf("retreiving ips configmap %s/%s: %w", namespace, fmt.Sprintf("%s-ips", clusterName), err) } if ip, ok := ipsMap.Data[machineScope.LinodeMachine.Name]; ok { - return ip, nil + return fmt.Sprintf(vlanIPFormat, ip), nil } - var nextIP string err = retry.RetryOnConflict(retry.DefaultRetry, func() error { - if err = machineScope.Client.Get(ctx, client.ObjectKeyFromObject(&ipsMap), &ipsMap); err != nil { - return fmt.Errorf("retreiving ips configmap %s/%s: %w", namespace, fmt.Sprintf("%s-ips", clusterName), err) + if err := machineScope.Client.Get(ctx, client.ObjectKeyFromObject(&ipsMap), &ipsMap); err != nil { + return err } - nextIP = getNextIP(maps.Values(ipsMap.Data), "10.0.0.0/11") + nextIP = getNextIP(maps.Values(ipsMap.Data), vlanIPRange) ipsMap.Data[machineScope.LinodeMachine.Name] = nextIP if err := machineScope.Client.Update(ctx, &ipsMap); err != nil { - return fmt.Errorf("updating ips configMap: %w", err) + return err } return nil }) if err != nil { - return "", err + return "", fmt.Errorf("updating ips in configmap %s/%s: %w", namespace, fmt.Sprintf("%s-ips", clusterName), err) } - logger.Info("onbained IP for machine", "name", machineScope.LinodeMachine.Name, "ip", nextIP) - // if an IP is available - return nextIP, nil + logger.Info("onbtained IP for machine", "name", machineScope.LinodeMachine.Name, "ip", nextIP) + return fmt.Sprintf(vlanIPFormat, nextIP), nil } func getVlanInterfaceConfig(ctx context.Context, machineScope *scope.MachineScope, logger logr.Logger) (*linodego.InstanceConfigInterfaceCreateOptions, error) { From d70ff4a2f6c3e3fea63f62c7bed88c374c9b9939 Mon Sep 17 00:00:00 2001 From: Tarun Chinmai Sekar Date: Mon, 30 Sep 2024 13:51:49 -0700 Subject: [PATCH 5/9] add kubebuilder annotations --- config/rbac/role.yaml | 9 +++++++++ controller/linodemachine_controller.go | 1 + 2 files changed, 10 insertions(+) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 6866c4f3..eb6a2d01 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -4,6 +4,15 @@ kind: ClusterRole metadata: name: manager-role rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - get + - patch + - update - apiGroups: - "" resources: diff --git a/controller/linodemachine_controller.go b/controller/linodemachine_controller.go index 66fdfd96..624b4b0b 100644 --- a/controller/linodemachine_controller.go +++ b/controller/linodemachine_controller.go @@ -101,6 +101,7 @@ type LinodeMachineReconciler struct { // +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=machines,verbs=get;watch;list // +kubebuilder:rbac:groups="",resources=events,verbs=get;list;watch;create;update;patch // +kubebuilder:rbac:groups="",resources=secrets;,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=configmaps;,verbs=get;create;update;patch // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. From 8e6245f292412114b95aca4166092fadfd60151e Mon Sep 17 00:00:00 2001 From: Tarun Chinmai Sekar Date: Mon, 30 Sep 2024 14:42:50 -0700 Subject: [PATCH 6/9] Add list permissions --- config/rbac/role.yaml | 1 + controller/linodemachine_controller.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index eb6a2d01..55e84788 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -11,6 +11,7 @@ rules: verbs: - create - get + - list - patch - update - apiGroups: diff --git a/controller/linodemachine_controller.go b/controller/linodemachine_controller.go index 624b4b0b..24716ac6 100644 --- a/controller/linodemachine_controller.go +++ b/controller/linodemachine_controller.go @@ -101,7 +101,7 @@ type LinodeMachineReconciler struct { // +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=machines,verbs=get;watch;list // +kubebuilder:rbac:groups="",resources=events,verbs=get;list;watch;create;update;patch // +kubebuilder:rbac:groups="",resources=secrets;,verbs=get;list;watch -// +kubebuilder:rbac:groups="",resources=configmaps;,verbs=get;create;update;patch +// +kubebuilder:rbac:groups="",resources=configmaps;,verbs=get;list;create;update;patch // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. From 4e0bb7410312ad8b9f06d63b132362fa7caac870 Mon Sep 17 00:00:00 2001 From: Tarun Chinmai Sekar Date: Mon, 30 Sep 2024 15:38:44 -0700 Subject: [PATCH 7/9] fixups --- config/rbac/role.yaml | 1 + controller/linodemachine_controller.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 55e84788..76f6af91 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -14,6 +14,7 @@ rules: - list - patch - update + - watch - apiGroups: - "" resources: diff --git a/controller/linodemachine_controller.go b/controller/linodemachine_controller.go index 24716ac6..a1f4918e 100644 --- a/controller/linodemachine_controller.go +++ b/controller/linodemachine_controller.go @@ -101,7 +101,7 @@ type LinodeMachineReconciler struct { // +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=machines,verbs=get;watch;list // +kubebuilder:rbac:groups="",resources=events,verbs=get;list;watch;create;update;patch // +kubebuilder:rbac:groups="",resources=secrets;,verbs=get;list;watch -// +kubebuilder:rbac:groups="",resources=configmaps;,verbs=get;list;create;update;patch +// +kubebuilder:rbac:groups="",resources=configmaps;,verbs=get;watch;list;create;update;patch // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. From 11e3c266b3953e8bd9fdb616fe4a241f4921c357 Mon Sep 17 00:00:00 2001 From: Tarun Chinmai Sekar Date: Mon, 30 Sep 2024 16:58:35 -0700 Subject: [PATCH 8/9] fixups --- controller/linodecluster_controller.go | 14 ++++++++++ .../linodemachine_controller_helpers.go | 26 ++++++++++++++----- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/controller/linodecluster_controller.go b/controller/linodecluster_controller.go index 74a106d9..77ad23c3 100644 --- a/controller/linodecluster_controller.go +++ b/controller/linodecluster_controller.go @@ -262,6 +262,20 @@ func (r *LinodeClusterReconciler) reconcileDelete(ctx context.Context, logger lo return errors.New("waiting for associated LinodeMachine objects to be deleted") } + if clusterScope.LinodeCluster.Spec.Network.UseVlan { + var ipsMap corev1.ConfigMap + err := clusterScope.Client.Get(ctx, client.ObjectKey{Namespace: clusterScope.Cluster.Namespace, Name: fmt.Sprintf("%s-ips", clusterScope.Cluster.Name)}, &ipsMap) + if err != nil && !apierrors.IsNotFound(err) { + logger.Error(err, "failed to get ips configmap") + return err + } + err = clusterScope.Client.Delete(ctx, &ipsMap) + if err != nil && !apierrors.IsNotFound(err) { + logger.Error(err, "failed to delete ips configmap") + return err + } + } + if err := clusterScope.RemoveCredentialsRefFinalizer(ctx); err != nil { logger.Error(err, "failed to remove credentials finalizer") setFailureReason(clusterScope, cerrs.DeleteClusterError, err, r) diff --git a/controller/linodemachine_controller_helpers.go b/controller/linodemachine_controller_helpers.go index af398ae4..d9d1276d 100644 --- a/controller/linodemachine_controller_helpers.go +++ b/controller/linodemachine_controller_helpers.go @@ -225,8 +225,9 @@ func buildInstanceAddrs(ctx context.Context, machineScope *scope.MachineScope, i } if iface.Purpose == linodego.InterfacePurposeVLAN { + // vlan addresses have a /11 appended to them - we should strip it out. ips = append(ips, clusterv1.MachineAddress{ - Address: iface.IPAMAddress, + Address: netip.MustParsePrefix(iface.IPAMAddress).Addr().String(), Type: clusterv1.MachineInternalIP, }) } @@ -391,6 +392,21 @@ func getNextIP(ips []string, prefixStr string) string { return ipString } +func createIPsConfigMap(ctx context.Context, machineScope *scope.MachineScope, ip string) error { + return machineScope.Client.Create(ctx, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-ips", machineScope.Cluster.Name), + Namespace: machineScope.Cluster.Namespace, + Labels: map[string]string{ + "clusterctl.cluster.x-k8s.io/move": "true", + }, + }, + Data: map[string]string{ + machineScope.LinodeMachine.Name: ip, + }, + }) +} + func reserveNextIP(ctx context.Context, machineScope *scope.MachineScope, logger logr.Logger) (string, error) { namespace := machineScope.Cluster.Namespace clusterName := machineScope.Cluster.Name @@ -400,12 +416,8 @@ func reserveNextIP(ctx context.Context, machineScope *scope.MachineScope, logger if err != nil { if apierrors.IsNotFound(err) { nextIP = getNextIP([]string{}, vlanIPRange) - ipsMap.Name = fmt.Sprintf("%s-ips", clusterName) - ipsMap.Namespace = namespace - ipsMap.Data = make(map[string]string) - ipsMap.Data[machineScope.LinodeMachine.Name] = nextIP - if err := machineScope.Client.Create(ctx, &ipsMap); err != nil { - return "", fmt.Errorf("creating ips configMap: %w", err) + if err := createIPsConfigMap(ctx, machineScope, nextIP); err != nil { + return "", fmt.Errorf("creating Ips configmap: %w", err) } logger.Info("Machine got ip", machineScope.LinodeMachine.Name, nextIP) return fmt.Sprintf(vlanIPFormat, nextIP), nil From 14bb6c124e254be79668318d2fadf3fc96191a49 Mon Sep 17 00:00:00 2001 From: Tarun Chinmai Sekar Date: Tue, 1 Oct 2024 11:57:11 -0700 Subject: [PATCH 9/9] Add delete perms --- config/rbac/role.yaml | 1 + controller/linodemachine_controller.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 76f6af91..a071e34c 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -10,6 +10,7 @@ rules: - configmaps verbs: - create + - delete - get - list - patch diff --git a/controller/linodemachine_controller.go b/controller/linodemachine_controller.go index a1f4918e..eb0e62d0 100644 --- a/controller/linodemachine_controller.go +++ b/controller/linodemachine_controller.go @@ -101,7 +101,7 @@ type LinodeMachineReconciler struct { // +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=machines,verbs=get;watch;list // +kubebuilder:rbac:groups="",resources=events,verbs=get;list;watch;create;update;patch // +kubebuilder:rbac:groups="",resources=secrets;,verbs=get;list;watch -// +kubebuilder:rbac:groups="",resources=configmaps;,verbs=get;watch;list;create;update;patch +// +kubebuilder:rbac:groups="",resources=configmaps;,verbs=get;watch;list;create;update;patch;delete // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state.