Implement internal load balancer (#698)

Add `Internal` as a new `type` for `networking.LoadBalancer`s.
Update & extend all relevant code (`machinepoollet`, `machinebroker`,
`ori-machine`).

Update test cases.

api/networking/v1alpha1/loadbalancer_type.go

@@ -28,6 +28,8 @@ type LoadBalancerType string
 const (
 	// LoadBalancerTypePublic is a LoadBalancer that allocates and routes a stable public IP.
 	LoadBalancerTypePublic LoadBalancerType = "Public"
+	// LoadBalancerTypeInternal is a LoadBalancer that allocates and routes network-internal, stable IPs.
+	LoadBalancerTypeInternal LoadBalancerType = "Internal"
 )
 
 // LoadBalancerSpec defines the desired state of LoadBalancer
@@ -36,6 +38,8 @@ type LoadBalancerSpec struct {
 	Type LoadBalancerType `json:"type"`
 	// IPFamilies are the ip families the load balancer should have.
 	IPFamilies []corev1.IPFamily `json:"ipFamilies"`
+	// IPs are the ips to use. Can only be used when Type is LoadBalancerTypeInternal.
+	IPs []IPSource `json:"ips,omitempty"`
 	// NetworkRef is the Network this LoadBalancer should belong to.
 	NetworkRef corev1.LocalObjectReference `json:"networkRef"`
 	// NetworkInterfaceSelector defines the NetworkInterfaces

broker/machinebroker/aliasprefixes/aliasprefixes.go

@@ -121,7 +121,7 @@ func (m *AliasPrefixes) createAliasPrefix(
 		},
 	}
 	apiutils.SetManagerLabel(aliasPrefix, machinebrokerv1alpha1.MachineBrokerManager)
-	apiutils.SetNetworkHandle(aliasPrefix, key.networkHandle)
+	apiutils.SetNetworkHandleLabel(aliasPrefix, key.networkHandle)
 	apiutils.SetPrefixLabel(aliasPrefix, key.prefix)
 
 	if err := m.cluster.Client().Create(ctx, aliasPrefix); err != nil {
@@ -140,7 +140,7 @@ func (m *AliasPrefixes) createAliasPrefix(
 		},
 	}
 	apiutils.SetManagerLabel(aliasPrefixRouting, machinebrokerv1alpha1.MachineBrokerManager)
-	apiutils.SetNetworkHandle(aliasPrefixRouting, key.networkHandle)
+	apiutils.SetNetworkHandleLabel(aliasPrefixRouting, key.networkHandle)
 	apiutils.SetPrefixLabel(aliasPrefixRouting, key.prefix)
 	if err := ctrl.SetControllerReference(aliasPrefix, aliasPrefixRouting, m.cluster.Scheme()); err != nil {
 		return nil, nil, fmt.Errorf("error setting alias prefix routing to be controlled by alias prefix: %w", err)

broker/machinebroker/api/v1alpha1/common_types.go

@@ -19,6 +19,7 @@ import (
 	"sort"
 
 	commonv1alpha1 "github.com/onmetal/onmetal-api/api/common/v1alpha1"
+	networkingv1alpha1 "github.com/onmetal/onmetal-api/api/networking/v1alpha1"
 	corev1 "k8s.io/api/core/v1"
 )
 
@@ -41,6 +42,8 @@ const (
 
 	PrefixLabel = "machinebrokerlet.api.onmetal.de/prefix"
 
+	LoadBalancerTypeLabel = "machinebrokerlet.api.onmetal.de/load-balancer-type"
+
 	IPLabel = "machinebrokerlet.api.onmetal.de/ip"
 )
 
@@ -74,16 +77,18 @@ func LoadBalancerPortsKey(ports []LoadBalancerPort) string {
 }
 
 type LoadBalancerTarget struct {
-	IP    commonv1alpha1.IP
-	Ports []LoadBalancerPort
+	LoadBalancerType networkingv1alpha1.LoadBalancerType
+	IP               commonv1alpha1.IP
+	Ports            []LoadBalancerPort
 }
 
 func (t LoadBalancerTarget) Key() string {
 	portKeys := LoadBalancerPortsKey(t.Ports)
-	return fmt.Sprintf("%s%s", t.IP, portKeys)
+	return fmt.Sprintf("%s-%s%s", t.LoadBalancerType, t.IP, portKeys)
 }
 
 type LoadBalancer struct {
+	Type          networkingv1alpha1.LoadBalancerType
 	NetworkHandle string
 	IP            commonv1alpha1.IP
 	Ports         []LoadBalancerPort

broker/machinebroker/apiutils/apiutils.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/onmetal/controller-utils/metautils"
 	commonv1alpha1 "github.com/onmetal/onmetal-api/api/common/v1alpha1"
+	networkingv1alpha1 "github.com/onmetal/onmetal-api/api/networking/v1alpha1"
 	machinebrokerv1alpha1 "github.com/onmetal/onmetal-api/broker/machinebroker/api/v1alpha1"
 	orimeta "github.com/onmetal/onmetal-api/ori/apis/meta/v1alpha1"
 	"golang.org/x/exp/slices"
@@ -159,7 +160,7 @@ var (
 	reverseIPAndPrefixReplacer = strings.NewReplacer("-", "/", "_", ":")
 )
 
-func SetNetworkHandle(o metav1.Object, handle string) {
+func SetNetworkHandleLabel(o metav1.Object, handle string) {
 	metautils.SetLabel(o, machinebrokerv1alpha1.NetworkHandleLabel, handle)
 }
 
@@ -181,6 +182,10 @@ func GetPrefixLabel(o metav1.Object) (commonv1alpha1.IPPrefix, error) {
 	return UnescapePrefix(escapedPrefix)
 }
 
+func SetLoadBalancerTypeLabel(o metav1.Object, typ networkingv1alpha1.LoadBalancerType) {
+	metautils.SetLabel(o, machinebrokerv1alpha1.LoadBalancerTypeLabel, string(typ))
+}
+
 func EscapeIP(ip commonv1alpha1.IP) string {
 	return ipAndPrefixReplacer.Replace(ip.String())
 }

broker/machinebroker/loadbalancers/loadbalancers.go

@@ -84,10 +84,11 @@ func (m *LoadBalancers) getLoadBalancerByKey(ctx context.Context, key loadBalanc
 	if err := m.cluster.Client().List(ctx, loadBalancerList,
 		client.InNamespace(m.cluster.Namespace()),
 		client.MatchingLabels{
-			machinebrokerv1alpha1.ManagerLabel:       machinebrokerv1alpha1.MachineBrokerManager,
-			machinebrokerv1alpha1.CreatedLabel:       "true",
-			machinebrokerv1alpha1.NetworkHandleLabel: key.networkHandle,
-			machinebrokerv1alpha1.IPLabel:            apiutils.EscapeIP(key.target.IP),
+			machinebrokerv1alpha1.ManagerLabel:          machinebrokerv1alpha1.MachineBrokerManager,
+			machinebrokerv1alpha1.CreatedLabel:          "true",
+			machinebrokerv1alpha1.NetworkHandleLabel:    key.networkHandle,
+			machinebrokerv1alpha1.LoadBalancerTypeLabel: string(key.target.LoadBalancerType),
+			machinebrokerv1alpha1.IPLabel:               apiutils.EscapeIP(key.target.IP),
 		},
 	); err != nil {
 		return nil, nil, false, fmt.Errorf("error listing load balanceres by key: %w", err)
@@ -134,7 +135,8 @@ func (m *LoadBalancers) createLoadBalancer(
 	}
 	annotations.SetExternallyMangedBy(loadBalancer, machinebrokerv1alpha1.MachineBrokerManager)
 	apiutils.SetManagerLabel(loadBalancer, machinebrokerv1alpha1.MachineBrokerManager)
-	apiutils.SetNetworkHandle(loadBalancer, key.networkHandle)
+	apiutils.SetNetworkHandleLabel(loadBalancer, key.networkHandle)
+	apiutils.SetLoadBalancerTypeLabel(loadBalancer, key.target.LoadBalancerType)
 	apiutils.SetIPLabel(loadBalancer, key.target.IP)
 
 	if err := m.cluster.Client().Create(ctx, loadBalancer); err != nil {
@@ -159,7 +161,7 @@ func (m *LoadBalancers) createLoadBalancer(
 		},
 	}
 	apiutils.SetManagerLabel(loadBalancerRouting, machinebrokerv1alpha1.MachineBrokerManager)
-	apiutils.SetNetworkHandle(loadBalancerRouting, key.networkHandle)
+	apiutils.SetNetworkHandleLabel(loadBalancerRouting, key.networkHandle)
 	apiutils.SetIPLabel(loadBalancerRouting, key.target.IP)
 	if err := ctrl.SetControllerReference(loadBalancer, loadBalancerRouting, m.cluster.Scheme()); err != nil {
 		return nil, nil, fmt.Errorf("error setting load balancer routing to be controlled by load balancer: %w", err)
@@ -377,6 +379,7 @@ func (m *LoadBalancers) joinLoadBalancersAndRoutings(
 		)
 
 		res = append(res, machinebrokerv1alpha1.LoadBalancer{
+			Type:          loadBalancer.Spec.Type,
 			NetworkHandle: networkHandle,
 			IP:            ip,
 			Ports:         apiutils.ConvertNetworkingLoadBalancerPortsToLoadBalancerPorts(loadBalancer.Spec.Ports),

broker/machinebroker/natgateways/natgateways.go

@@ -126,7 +126,7 @@ func (m *NATGateways) createNATGateway(
 	}
 	annotations.SetExternallyMangedBy(natGateway, machinebrokerv1alpha1.MachineBrokerManager)
 	apiutils.SetManagerLabel(natGateway, machinebrokerv1alpha1.MachineBrokerManager)
-	apiutils.SetNetworkHandle(natGateway, network.Spec.Handle)
+	apiutils.SetNetworkHandleLabel(natGateway, network.Spec.Handle)
 	apiutils.SetIPLabel(natGateway, key.ip)
 
 	if err := m.cluster.Client().Create(ctx, natGateway); err != nil {
@@ -153,7 +153,7 @@ func (m *NATGateways) createNATGateway(
 		},
 	}
 	apiutils.SetManagerLabel(natGatewayRouting, machinebrokerv1alpha1.MachineBrokerManager)
-	apiutils.SetNetworkHandle(natGatewayRouting, network.Spec.Handle)
+	apiutils.SetNetworkHandleLabel(natGatewayRouting, network.Spec.Handle)
 	apiutils.SetIPLabel(natGateway, key.ip)
 	if err := ctrl.SetControllerReference(natGateway, natGatewayRouting, m.cluster.Scheme()); err != nil {
 		return nil, nil, fmt.Errorf("error setting nat gateway routing to be controlled by nat gateway: %w", err)

broker/machinebroker/networks/networks.go

@@ -97,7 +97,7 @@ func (m *Networks) createNetwork(ctx context.Context, handle string) (res *netwo
 	}
 	annotations.SetExternallyMangedBy(network, machinebrokerv1alpha1.MachineBrokerManager)
 	apiutils.SetManagerLabel(network, machinebrokerv1alpha1.MachineBrokerManager)
-	apiutils.SetNetworkHandle(network, handle)
+	apiutils.SetNetworkHandleLabel(network, handle)
 
 	if err := m.cluster.Client().Create(ctx, network); err != nil {
 		return nil, fmt.Errorf("error creating network: %w", err)

broker/machinebroker/server/common.go

@@ -113,6 +113,17 @@ func (s *Server) convertOnmetalPrefixes(prefixes []commonv1alpha1.IPPrefix) []st
 	return res
 }
 
+func (s *Server) convertOnmetalLoadBalancerType(typ networkingv1alpha1.LoadBalancerType) (ori.LoadBalancerType, error) {
+	switch typ {
+	case networkingv1alpha1.LoadBalancerTypePublic:
+		return ori.LoadBalancerType_PUBLIC, nil
+	case networkingv1alpha1.LoadBalancerTypeInternal:
+		return ori.LoadBalancerType_INTERNAL, nil
+	default:
+		return 0, fmt.Errorf("unrecognized load balancer type %q", typ)
+	}
+}
+
 func (s *Server) convertOnmetalProtocol(protocol corev1.Protocol) (ori.Protocol, error) {
 	switch protocol {
 	case corev1.ProtocolTCP:
@@ -142,6 +153,11 @@ func (s *Server) convertOnmetalLoadBalancerTargetPort(port machinebrokerv1alpha1
 func (s *Server) convertOnmetalLoadBalancerTargets(loadBalancerTargets []machinebrokerv1alpha1.LoadBalancerTarget) ([]*ori.LoadBalancerTargetSpec, error) {
 	res := make([]*ori.LoadBalancerTargetSpec, len(loadBalancerTargets))
 	for i, loadBalancerTarget := range loadBalancerTargets {
+		typ, err := s.convertOnmetalLoadBalancerType(loadBalancerTarget.LoadBalancerType)
+		if err != nil {
+			return nil, err
+		}
+
 		ports := make([]*ori.LoadBalancerPort, len(loadBalancerTarget.Ports))
 		for j, port := range loadBalancerTarget.Ports {
 			p, err := s.convertOnmetalLoadBalancerTargetPort(port)
@@ -153,8 +169,9 @@ func (s *Server) convertOnmetalLoadBalancerTargets(loadBalancerTargets []machine
 		}
 
 		res[i] = &ori.LoadBalancerTargetSpec{
-			Ip:    loadBalancerTarget.IP.String(),
-			Ports: ports,
+			LoadBalancerType: typ,
+			Ip:               loadBalancerTarget.IP.String(),
+			Ports:            ports,
 		}
 	}
 	return res, nil
@@ -225,6 +242,17 @@ func (s *Server) convertORIProtocol(protocol ori.Protocol) (corev1.Protocol, err
 	}
 }
 
+func (s *Server) convertORILoadBalancerType(typ ori.LoadBalancerType) (networkingv1alpha1.LoadBalancerType, error) {
+	switch typ {
+	case ori.LoadBalancerType_PUBLIC:
+		return networkingv1alpha1.LoadBalancerTypePublic, nil
+	case ori.LoadBalancerType_INTERNAL:
+		return networkingv1alpha1.LoadBalancerTypeInternal, nil
+	default:
+		return "", fmt.Errorf("unknown load balancer type %d", typ)
+	}
+}
+
 func (s *Server) parseIPPrefixes(prefixStrings []string) ([]commonv1alpha1.IPPrefix, error) {
 	var ipPrefixes []commonv1alpha1.IPPrefix
 	for _, prefixString := range prefixStrings {

broker/machinebroker/server/networkinterface_create.go

@@ -56,6 +56,11 @@ func (s *Server) prepareOnmetalVirtualIP(virtualIPSpec *ori.VirtualIPSpec) (*net
 }
 
 func (s *Server) prepareOnmetalLoadBalancerTarget(lbTgt *ori.LoadBalancerTargetSpec) (*machinebrokerv1alpha1.LoadBalancerTarget, error) {
+	typ, err := s.convertORILoadBalancerType(lbTgt.LoadBalancerType)
+	if err != nil {
+		return nil, err
+	}
+
 	ip, err := commonv1alpha1.ParseIP(lbTgt.Ip)
 	if err != nil {
 		return nil, err
@@ -76,8 +81,9 @@ func (s *Server) prepareOnmetalLoadBalancerTarget(lbTgt *ori.LoadBalancerTargetS
 	}
 
 	return &machinebrokerv1alpha1.LoadBalancerTarget{
-		IP:    ip,
-		Ports: ports,
+		LoadBalancerType: typ,
+		IP:               ip,
+		Ports:            ports,
 	}, nil
 }
 

broker/machinebroker/server/networkinterface_create_loadbalancer_test.go

@@ -16,6 +16,7 @@ package server_test
 
 import (
 	commonv1alpha1 "github.com/onmetal/onmetal-api/api/common/v1alpha1"
+	networkingv1alpha1 "github.com/onmetal/onmetal-api/api/networking/v1alpha1"
 	machinebrokerv1alpha1 "github.com/onmetal/onmetal-api/broker/machinebroker/api/v1alpha1"
 	ori "github.com/onmetal/onmetal-api/ori/apis/machine/v1alpha1"
 	orimeta "github.com/onmetal/onmetal-api/ori/apis/meta/v1alpha1"
@@ -67,6 +68,7 @@ var _ = Describe("NetworkInterfaceCreateLoadBalancerTarget", func() {
 		Expect(err).NotTo(HaveOccurred())
 
 		Expect(loadBalancers).To(ConsistOf(machinebrokerv1alpha1.LoadBalancer{
+			Type:          networkingv1alpha1.LoadBalancerTypePublic,
 			NetworkHandle: "foo",
 			IP:            commonv1alpha1.MustParseIP("10.0.0.1"),
 			Ports: []machinebrokerv1alpha1.LoadBalancerPort{

broker/machinebroker/server/networkinterface_create_test.go

@@ -121,6 +121,7 @@ var _ = Describe("CreateNetworkInterface", func() {
 
 		By("inspecting the load balancer list")
 		Expect(loadBalancers).To(ConsistOf(machinebrokerv1alpha1.LoadBalancer{
+			Type:          networkingv1alpha1.LoadBalancerTypePublic,
 			NetworkHandle: networkHandle,
 			IP:            commonv1alpha1.MustParseIP(loadBalancerIP),
 			Ports: []machinebrokerv1alpha1.LoadBalancerPort{

broker/machinebroker/server/networkinterface_list.go

@@ -179,8 +179,9 @@ func (s *Server) aggregateOnmetalNetworkInterface(
 		}
 
 		lbTgts = append(lbTgts, machinebrokerv1alpha1.LoadBalancerTarget{
-			IP:    loadBalancer.IP,
-			Ports: loadBalancer.Ports,
+			LoadBalancerType: loadBalancer.Type,
+			IP:               loadBalancer.IP,
+			Ports:            loadBalancer.Ports,
 		})
 	}
 

client-go/openapi/api_violations.report

@@ -15,6 +15,7 @@ API rule violation: list_type_missing,github.com/onmetal/onmetal-api/api/ipam/v1
 API rule violation: list_type_missing,github.com/onmetal/onmetal-api/api/networking/v1alpha1,AliasPrefixRouting,Destinations
 API rule violation: list_type_missing,github.com/onmetal/onmetal-api/api/networking/v1alpha1,LoadBalancerRouting,Destinations
 API rule violation: list_type_missing,github.com/onmetal/onmetal-api/api/networking/v1alpha1,LoadBalancerSpec,IPFamilies
+API rule violation: list_type_missing,github.com/onmetal/onmetal-api/api/networking/v1alpha1,LoadBalancerSpec,IPs
 API rule violation: list_type_missing,github.com/onmetal/onmetal-api/api/networking/v1alpha1,LoadBalancerSpec,Ports
 API rule violation: list_type_missing,github.com/onmetal/onmetal-api/api/networking/v1alpha1,LoadBalancerStatus,IPs
 API rule violation: list_type_missing,github.com/onmetal/onmetal-api/api/networking/v1alpha1,NATGatewayDestination,IPs
@@ -161,6 +162,7 @@ API rule violation: list_type_missing,k8s.io/apimachinery/pkg/runtime,RawExtensi
 API rule violation: list_type_missing,k8s.io/apimachinery/pkg/runtime,Unknown,Raw
 API rule violation: names_match,github.com/onmetal/onmetal-api/api/compute/v1alpha1,MachineSpec,ImagePullSecretRef
 API rule violation: names_match,github.com/onmetal/onmetal-api/api/compute/v1alpha1,NetworkInterfaceStatus,IPs
+API rule violation: names_match,github.com/onmetal/onmetal-api/api/networking/v1alpha1,LoadBalancerSpec,IPs
 API rule violation: names_match,github.com/onmetal/onmetal-api/api/networking/v1alpha1,LoadBalancerStatus,IPs
 API rule violation: names_match,github.com/onmetal/onmetal-api/api/networking/v1alpha1,NATGatewayDestination,IPs
 API rule violation: names_match,github.com/onmetal/onmetal-api/api/networking/v1alpha1,NATGatewaySpec,IPs