From eb0957464d119947154604f0f8756d0994d7e11f Mon Sep 17 00:00:00 2001 From: Axel Christ Date: Tue, 11 Apr 2023 14:13:39 +0200 Subject: [PATCH] Implement internal load balancer (#698) Add `Internal` as a new `type` for `networking.LoadBalancer`s. Update & extend all relevant code (`machinepoollet`, `machinebroker`, `ori-machine`). Update test cases. --- api/networking/v1alpha1/loadbalancer_type.go | 4 + .../v1alpha1/zz_generated.deepcopy.go | 7 + .../aliasprefixes/aliasprefixes.go | 4 +- .../api/v1alpha1/common_types.go | 11 +- broker/machinebroker/apiutils/apiutils.go | 7 +- .../loadbalancers/loadbalancers.go | 15 +- .../machinebroker/natgateways/natgateways.go | 4 +- broker/machinebroker/networks/networks.go | 2 +- broker/machinebroker/server/common.go | 32 +- .../server/networkinterface_create.go | 10 +- ...tworkinterface_create_loadbalancer_test.go | 2 + .../server/networkinterface_create_test.go | 1 + .../server/networkinterface_list.go | 5 +- .../applyconfigurations/internal/internal.go | 6 + .../networking/v1alpha1/loadbalancerspec.go | 14 + client-go/openapi/api_violations.report | 2 + client-go/openapi/zz_generated.openapi.go | 16 +- docs/api-reference/networking.md | 33 +- internal/apis/networking/loadbalancer_type.go | 4 + internal/apis/networking/v1alpha1/defaults.go | 26 +- .../v1alpha1/zz_generated.conversion.go | 2 + .../v1alpha1/zz_generated.defaults.go | 21 + internal/apis/networking/validation/common.go | 55 +++ .../networking/validation/loadbalancer.go | 14 +- .../validation/loadbalancer_test.go | 8 + .../networking/validation/networkinterface.go | 30 -- .../apis/networking/zz_generated.deepcopy.go | 7 + .../networking/loadbalancer/strategy.go | 22 + ori/apis/bucket/v1alpha1/api.pb.go | 11 +- ori/apis/machine/v1alpha1/api.pb.go | 445 ++++++++++-------- ori/apis/machine/v1alpha1/api.proto | 10 +- ori/apis/machine/v1alpha1/utils.go | 2 +- ori/apis/meta/v1alpha1/api.pb.go | 7 +- ori/apis/volume/v1alpha1/api.pb.go | 11 +- .../machine_controller_networkinterface.go | 21 +- 35 files changed, 592 insertions(+), 279 deletions(-) create mode 100644 internal/apis/networking/validation/common.go diff --git a/api/networking/v1alpha1/loadbalancer_type.go b/api/networking/v1alpha1/loadbalancer_type.go index 1bfa3cd8b..2cc76f7d7 100644 --- a/api/networking/v1alpha1/loadbalancer_type.go +++ b/api/networking/v1alpha1/loadbalancer_type.go @@ -28,6 +28,8 @@ type LoadBalancerType string const ( // LoadBalancerTypePublic is a LoadBalancer that allocates and routes a stable public IP. LoadBalancerTypePublic LoadBalancerType = "Public" + // LoadBalancerTypeInternal is a LoadBalancer that allocates and routes network-internal, stable IPs. + LoadBalancerTypeInternal LoadBalancerType = "Internal" ) // LoadBalancerSpec defines the desired state of LoadBalancer @@ -36,6 +38,8 @@ type LoadBalancerSpec struct { Type LoadBalancerType `json:"type"` // IPFamilies are the ip families the load balancer should have. IPFamilies []corev1.IPFamily `json:"ipFamilies"` + // IPs are the ips to use. Can only be used when Type is LoadBalancerTypeInternal. + IPs []IPSource `json:"ips,omitempty"` // NetworkRef is the Network this LoadBalancer should belong to. NetworkRef corev1.LocalObjectReference `json:"networkRef"` // NetworkInterfaceSelector defines the NetworkInterfaces diff --git a/api/networking/v1alpha1/zz_generated.deepcopy.go b/api/networking/v1alpha1/zz_generated.deepcopy.go index d9941f95b..40e574329 100644 --- a/api/networking/v1alpha1/zz_generated.deepcopy.go +++ b/api/networking/v1alpha1/zz_generated.deepcopy.go @@ -424,6 +424,13 @@ func (in *LoadBalancerSpec) DeepCopyInto(out *LoadBalancerSpec) { *out = make([]corev1.IPFamily, len(*in)) copy(*out, *in) } + if in.IPs != nil { + in, out := &in.IPs, &out.IPs + *out = make([]IPSource, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } out.NetworkRef = in.NetworkRef if in.NetworkInterfaceSelector != nil { in, out := &in.NetworkInterfaceSelector, &out.NetworkInterfaceSelector diff --git a/broker/machinebroker/aliasprefixes/aliasprefixes.go b/broker/machinebroker/aliasprefixes/aliasprefixes.go index a30722f87..419852ef3 100644 --- a/broker/machinebroker/aliasprefixes/aliasprefixes.go +++ b/broker/machinebroker/aliasprefixes/aliasprefixes.go @@ -121,7 +121,7 @@ func (m *AliasPrefixes) createAliasPrefix( }, } apiutils.SetManagerLabel(aliasPrefix, machinebrokerv1alpha1.MachineBrokerManager) - apiutils.SetNetworkHandle(aliasPrefix, key.networkHandle) + apiutils.SetNetworkHandleLabel(aliasPrefix, key.networkHandle) apiutils.SetPrefixLabel(aliasPrefix, key.prefix) if err := m.cluster.Client().Create(ctx, aliasPrefix); err != nil { @@ -140,7 +140,7 @@ func (m *AliasPrefixes) createAliasPrefix( }, } apiutils.SetManagerLabel(aliasPrefixRouting, machinebrokerv1alpha1.MachineBrokerManager) - apiutils.SetNetworkHandle(aliasPrefixRouting, key.networkHandle) + apiutils.SetNetworkHandleLabel(aliasPrefixRouting, key.networkHandle) apiutils.SetPrefixLabel(aliasPrefixRouting, key.prefix) if err := ctrl.SetControllerReference(aliasPrefix, aliasPrefixRouting, m.cluster.Scheme()); err != nil { return nil, nil, fmt.Errorf("error setting alias prefix routing to be controlled by alias prefix: %w", err) diff --git a/broker/machinebroker/api/v1alpha1/common_types.go b/broker/machinebroker/api/v1alpha1/common_types.go index a326e4127..45b91cbf2 100644 --- a/broker/machinebroker/api/v1alpha1/common_types.go +++ b/broker/machinebroker/api/v1alpha1/common_types.go @@ -19,6 +19,7 @@ import ( "sort" commonv1alpha1 "github.com/onmetal/onmetal-api/api/common/v1alpha1" + networkingv1alpha1 "github.com/onmetal/onmetal-api/api/networking/v1alpha1" corev1 "k8s.io/api/core/v1" ) @@ -41,6 +42,8 @@ const ( PrefixLabel = "machinebrokerlet.api.onmetal.de/prefix" + LoadBalancerTypeLabel = "machinebrokerlet.api.onmetal.de/load-balancer-type" + IPLabel = "machinebrokerlet.api.onmetal.de/ip" ) @@ -74,16 +77,18 @@ func LoadBalancerPortsKey(ports []LoadBalancerPort) string { } type LoadBalancerTarget struct { - IP commonv1alpha1.IP - Ports []LoadBalancerPort + LoadBalancerType networkingv1alpha1.LoadBalancerType + IP commonv1alpha1.IP + Ports []LoadBalancerPort } func (t LoadBalancerTarget) Key() string { portKeys := LoadBalancerPortsKey(t.Ports) - return fmt.Sprintf("%s%s", t.IP, portKeys) + return fmt.Sprintf("%s-%s%s", t.LoadBalancerType, t.IP, portKeys) } type LoadBalancer struct { + Type networkingv1alpha1.LoadBalancerType NetworkHandle string IP commonv1alpha1.IP Ports []LoadBalancerPort diff --git a/broker/machinebroker/apiutils/apiutils.go b/broker/machinebroker/apiutils/apiutils.go index ad2f923e8..84f73b8ef 100644 --- a/broker/machinebroker/apiutils/apiutils.go +++ b/broker/machinebroker/apiutils/apiutils.go @@ -22,6 +22,7 @@ import ( "github.com/onmetal/controller-utils/metautils" commonv1alpha1 "github.com/onmetal/onmetal-api/api/common/v1alpha1" + networkingv1alpha1 "github.com/onmetal/onmetal-api/api/networking/v1alpha1" machinebrokerv1alpha1 "github.com/onmetal/onmetal-api/broker/machinebroker/api/v1alpha1" orimeta "github.com/onmetal/onmetal-api/ori/apis/meta/v1alpha1" "golang.org/x/exp/slices" @@ -159,7 +160,7 @@ var ( reverseIPAndPrefixReplacer = strings.NewReplacer("-", "/", "_", ":") ) -func SetNetworkHandle(o metav1.Object, handle string) { +func SetNetworkHandleLabel(o metav1.Object, handle string) { metautils.SetLabel(o, machinebrokerv1alpha1.NetworkHandleLabel, handle) } @@ -181,6 +182,10 @@ func GetPrefixLabel(o metav1.Object) (commonv1alpha1.IPPrefix, error) { return UnescapePrefix(escapedPrefix) } +func SetLoadBalancerTypeLabel(o metav1.Object, typ networkingv1alpha1.LoadBalancerType) { + metautils.SetLabel(o, machinebrokerv1alpha1.LoadBalancerTypeLabel, string(typ)) +} + func EscapeIP(ip commonv1alpha1.IP) string { return ipAndPrefixReplacer.Replace(ip.String()) } diff --git a/broker/machinebroker/loadbalancers/loadbalancers.go b/broker/machinebroker/loadbalancers/loadbalancers.go index 8a10449d5..b648b5fd7 100644 --- a/broker/machinebroker/loadbalancers/loadbalancers.go +++ b/broker/machinebroker/loadbalancers/loadbalancers.go @@ -84,10 +84,11 @@ func (m *LoadBalancers) getLoadBalancerByKey(ctx context.Context, key loadBalanc if err := m.cluster.Client().List(ctx, loadBalancerList, client.InNamespace(m.cluster.Namespace()), client.MatchingLabels{ - machinebrokerv1alpha1.ManagerLabel: machinebrokerv1alpha1.MachineBrokerManager, - machinebrokerv1alpha1.CreatedLabel: "true", - machinebrokerv1alpha1.NetworkHandleLabel: key.networkHandle, - machinebrokerv1alpha1.IPLabel: apiutils.EscapeIP(key.target.IP), + machinebrokerv1alpha1.ManagerLabel: machinebrokerv1alpha1.MachineBrokerManager, + machinebrokerv1alpha1.CreatedLabel: "true", + machinebrokerv1alpha1.NetworkHandleLabel: key.networkHandle, + machinebrokerv1alpha1.LoadBalancerTypeLabel: string(key.target.LoadBalancerType), + machinebrokerv1alpha1.IPLabel: apiutils.EscapeIP(key.target.IP), }, ); err != nil { return nil, nil, false, fmt.Errorf("error listing load balanceres by key: %w", err) @@ -134,7 +135,8 @@ func (m *LoadBalancers) createLoadBalancer( } annotations.SetExternallyMangedBy(loadBalancer, machinebrokerv1alpha1.MachineBrokerManager) apiutils.SetManagerLabel(loadBalancer, machinebrokerv1alpha1.MachineBrokerManager) - apiutils.SetNetworkHandle(loadBalancer, key.networkHandle) + apiutils.SetNetworkHandleLabel(loadBalancer, key.networkHandle) + apiutils.SetLoadBalancerTypeLabel(loadBalancer, key.target.LoadBalancerType) apiutils.SetIPLabel(loadBalancer, key.target.IP) if err := m.cluster.Client().Create(ctx, loadBalancer); err != nil { @@ -159,7 +161,7 @@ func (m *LoadBalancers) createLoadBalancer( }, } apiutils.SetManagerLabel(loadBalancerRouting, machinebrokerv1alpha1.MachineBrokerManager) - apiutils.SetNetworkHandle(loadBalancerRouting, key.networkHandle) + apiutils.SetNetworkHandleLabel(loadBalancerRouting, key.networkHandle) apiutils.SetIPLabel(loadBalancerRouting, key.target.IP) if err := ctrl.SetControllerReference(loadBalancer, loadBalancerRouting, m.cluster.Scheme()); err != nil { return nil, nil, fmt.Errorf("error setting load balancer routing to be controlled by load balancer: %w", err) @@ -377,6 +379,7 @@ func (m *LoadBalancers) joinLoadBalancersAndRoutings( ) res = append(res, machinebrokerv1alpha1.LoadBalancer{ + Type: loadBalancer.Spec.Type, NetworkHandle: networkHandle, IP: ip, Ports: apiutils.ConvertNetworkingLoadBalancerPortsToLoadBalancerPorts(loadBalancer.Spec.Ports), diff --git a/broker/machinebroker/natgateways/natgateways.go b/broker/machinebroker/natgateways/natgateways.go index f8371fb1d..6c65dba2a 100644 --- a/broker/machinebroker/natgateways/natgateways.go +++ b/broker/machinebroker/natgateways/natgateways.go @@ -126,7 +126,7 @@ func (m *NATGateways) createNATGateway( } annotations.SetExternallyMangedBy(natGateway, machinebrokerv1alpha1.MachineBrokerManager) apiutils.SetManagerLabel(natGateway, machinebrokerv1alpha1.MachineBrokerManager) - apiutils.SetNetworkHandle(natGateway, network.Spec.Handle) + apiutils.SetNetworkHandleLabel(natGateway, network.Spec.Handle) apiutils.SetIPLabel(natGateway, key.ip) if err := m.cluster.Client().Create(ctx, natGateway); err != nil { @@ -153,7 +153,7 @@ func (m *NATGateways) createNATGateway( }, } apiutils.SetManagerLabel(natGatewayRouting, machinebrokerv1alpha1.MachineBrokerManager) - apiutils.SetNetworkHandle(natGatewayRouting, network.Spec.Handle) + apiutils.SetNetworkHandleLabel(natGatewayRouting, network.Spec.Handle) apiutils.SetIPLabel(natGateway, key.ip) if err := ctrl.SetControllerReference(natGateway, natGatewayRouting, m.cluster.Scheme()); err != nil { return nil, nil, fmt.Errorf("error setting nat gateway routing to be controlled by nat gateway: %w", err) diff --git a/broker/machinebroker/networks/networks.go b/broker/machinebroker/networks/networks.go index 8467a3b27..094494802 100644 --- a/broker/machinebroker/networks/networks.go +++ b/broker/machinebroker/networks/networks.go @@ -97,7 +97,7 @@ func (m *Networks) createNetwork(ctx context.Context, handle string) (res *netwo } annotations.SetExternallyMangedBy(network, machinebrokerv1alpha1.MachineBrokerManager) apiutils.SetManagerLabel(network, machinebrokerv1alpha1.MachineBrokerManager) - apiutils.SetNetworkHandle(network, handle) + apiutils.SetNetworkHandleLabel(network, handle) if err := m.cluster.Client().Create(ctx, network); err != nil { return nil, fmt.Errorf("error creating network: %w", err) diff --git a/broker/machinebroker/server/common.go b/broker/machinebroker/server/common.go index 0b1972213..81105ec33 100644 --- a/broker/machinebroker/server/common.go +++ b/broker/machinebroker/server/common.go @@ -113,6 +113,17 @@ func (s *Server) convertOnmetalPrefixes(prefixes []commonv1alpha1.IPPrefix) []st return res } +func (s *Server) convertOnmetalLoadBalancerType(typ networkingv1alpha1.LoadBalancerType) (ori.LoadBalancerType, error) { + switch typ { + case networkingv1alpha1.LoadBalancerTypePublic: + return ori.LoadBalancerType_PUBLIC, nil + case networkingv1alpha1.LoadBalancerTypeInternal: + return ori.LoadBalancerType_INTERNAL, nil + default: + return 0, fmt.Errorf("unrecognized load balancer type %q", typ) + } +} + func (s *Server) convertOnmetalProtocol(protocol corev1.Protocol) (ori.Protocol, error) { switch protocol { case corev1.ProtocolTCP: @@ -142,6 +153,11 @@ func (s *Server) convertOnmetalLoadBalancerTargetPort(port machinebrokerv1alpha1 func (s *Server) convertOnmetalLoadBalancerTargets(loadBalancerTargets []machinebrokerv1alpha1.LoadBalancerTarget) ([]*ori.LoadBalancerTargetSpec, error) { res := make([]*ori.LoadBalancerTargetSpec, len(loadBalancerTargets)) for i, loadBalancerTarget := range loadBalancerTargets { + typ, err := s.convertOnmetalLoadBalancerType(loadBalancerTarget.LoadBalancerType) + if err != nil { + return nil, err + } + ports := make([]*ori.LoadBalancerPort, len(loadBalancerTarget.Ports)) for j, port := range loadBalancerTarget.Ports { p, err := s.convertOnmetalLoadBalancerTargetPort(port) @@ -153,8 +169,9 @@ func (s *Server) convertOnmetalLoadBalancerTargets(loadBalancerTargets []machine } res[i] = &ori.LoadBalancerTargetSpec{ - Ip: loadBalancerTarget.IP.String(), - Ports: ports, + LoadBalancerType: typ, + Ip: loadBalancerTarget.IP.String(), + Ports: ports, } } return res, nil @@ -225,6 +242,17 @@ func (s *Server) convertORIProtocol(protocol ori.Protocol) (corev1.Protocol, err } } +func (s *Server) convertORILoadBalancerType(typ ori.LoadBalancerType) (networkingv1alpha1.LoadBalancerType, error) { + switch typ { + case ori.LoadBalancerType_PUBLIC: + return networkingv1alpha1.LoadBalancerTypePublic, nil + case ori.LoadBalancerType_INTERNAL: + return networkingv1alpha1.LoadBalancerTypeInternal, nil + default: + return "", fmt.Errorf("unknown load balancer type %d", typ) + } +} + func (s *Server) parseIPPrefixes(prefixStrings []string) ([]commonv1alpha1.IPPrefix, error) { var ipPrefixes []commonv1alpha1.IPPrefix for _, prefixString := range prefixStrings { diff --git a/broker/machinebroker/server/networkinterface_create.go b/broker/machinebroker/server/networkinterface_create.go index 0fb1783e3..56bfa7155 100644 --- a/broker/machinebroker/server/networkinterface_create.go +++ b/broker/machinebroker/server/networkinterface_create.go @@ -56,6 +56,11 @@ func (s *Server) prepareOnmetalVirtualIP(virtualIPSpec *ori.VirtualIPSpec) (*net } func (s *Server) prepareOnmetalLoadBalancerTarget(lbTgt *ori.LoadBalancerTargetSpec) (*machinebrokerv1alpha1.LoadBalancerTarget, error) { + typ, err := s.convertORILoadBalancerType(lbTgt.LoadBalancerType) + if err != nil { + return nil, err + } + ip, err := commonv1alpha1.ParseIP(lbTgt.Ip) if err != nil { return nil, err @@ -76,8 +81,9 @@ func (s *Server) prepareOnmetalLoadBalancerTarget(lbTgt *ori.LoadBalancerTargetS } return &machinebrokerv1alpha1.LoadBalancerTarget{ - IP: ip, - Ports: ports, + LoadBalancerType: typ, + IP: ip, + Ports: ports, }, nil } diff --git a/broker/machinebroker/server/networkinterface_create_loadbalancer_test.go b/broker/machinebroker/server/networkinterface_create_loadbalancer_test.go index d99d4376a..0ac55bbe6 100644 --- a/broker/machinebroker/server/networkinterface_create_loadbalancer_test.go +++ b/broker/machinebroker/server/networkinterface_create_loadbalancer_test.go @@ -16,6 +16,7 @@ package server_test import ( commonv1alpha1 "github.com/onmetal/onmetal-api/api/common/v1alpha1" + networkingv1alpha1 "github.com/onmetal/onmetal-api/api/networking/v1alpha1" machinebrokerv1alpha1 "github.com/onmetal/onmetal-api/broker/machinebroker/api/v1alpha1" ori "github.com/onmetal/onmetal-api/ori/apis/machine/v1alpha1" orimeta "github.com/onmetal/onmetal-api/ori/apis/meta/v1alpha1" @@ -67,6 +68,7 @@ var _ = Describe("NetworkInterfaceCreateLoadBalancerTarget", func() { Expect(err).NotTo(HaveOccurred()) Expect(loadBalancers).To(ConsistOf(machinebrokerv1alpha1.LoadBalancer{ + Type: networkingv1alpha1.LoadBalancerTypePublic, NetworkHandle: "foo", IP: commonv1alpha1.MustParseIP("10.0.0.1"), Ports: []machinebrokerv1alpha1.LoadBalancerPort{ diff --git a/broker/machinebroker/server/networkinterface_create_test.go b/broker/machinebroker/server/networkinterface_create_test.go index e3aa9f6a0..4faf01707 100644 --- a/broker/machinebroker/server/networkinterface_create_test.go +++ b/broker/machinebroker/server/networkinterface_create_test.go @@ -121,6 +121,7 @@ var _ = Describe("CreateNetworkInterface", func() { By("inspecting the load balancer list") Expect(loadBalancers).To(ConsistOf(machinebrokerv1alpha1.LoadBalancer{ + Type: networkingv1alpha1.LoadBalancerTypePublic, NetworkHandle: networkHandle, IP: commonv1alpha1.MustParseIP(loadBalancerIP), Ports: []machinebrokerv1alpha1.LoadBalancerPort{ diff --git a/broker/machinebroker/server/networkinterface_list.go b/broker/machinebroker/server/networkinterface_list.go index b7dc0d557..e44df9ba3 100644 --- a/broker/machinebroker/server/networkinterface_list.go +++ b/broker/machinebroker/server/networkinterface_list.go @@ -179,8 +179,9 @@ func (s *Server) aggregateOnmetalNetworkInterface( } lbTgts = append(lbTgts, machinebrokerv1alpha1.LoadBalancerTarget{ - IP: loadBalancer.IP, - Ports: loadBalancer.Ports, + LoadBalancerType: loadBalancer.Type, + IP: loadBalancer.IP, + Ports: loadBalancer.Ports, }) } diff --git a/client-go/applyconfigurations/internal/internal.go b/client-go/applyconfigurations/internal/internal.go index 23d440fd1..6461e053e 100644 --- a/client-go/applyconfigurations/internal/internal.go +++ b/client-go/applyconfigurations/internal/internal.go @@ -799,6 +799,12 @@ var schemaYAML = typed.YAMLObject(`types: elementType: scalar: string elementRelationship: atomic + - name: ips + type: + list: + elementType: + namedType: com.github.onmetal.onmetal-api.api.networking.v1alpha1.IPSource + elementRelationship: atomic - name: networkInterfaceSelector type: namedType: io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelector diff --git a/client-go/applyconfigurations/networking/v1alpha1/loadbalancerspec.go b/client-go/applyconfigurations/networking/v1alpha1/loadbalancerspec.go index bed327eb9..5b2618ffb 100644 --- a/client-go/applyconfigurations/networking/v1alpha1/loadbalancerspec.go +++ b/client-go/applyconfigurations/networking/v1alpha1/loadbalancerspec.go @@ -28,6 +28,7 @@ import ( type LoadBalancerSpecApplyConfiguration struct { Type *v1alpha1.LoadBalancerType `json:"type,omitempty"` IPFamilies []v1.IPFamily `json:"ipFamilies,omitempty"` + IPs []IPSourceApplyConfiguration `json:"ips,omitempty"` NetworkRef *v1.LocalObjectReference `json:"networkRef,omitempty"` NetworkInterfaceSelector *metav1.LabelSelectorApplyConfiguration `json:"networkInterfaceSelector,omitempty"` Ports []LoadBalancerPortApplyConfiguration `json:"ports,omitempty"` @@ -57,6 +58,19 @@ func (b *LoadBalancerSpecApplyConfiguration) WithIPFamilies(values ...v1.IPFamil return b } +// WithIPs adds the given value to the IPs field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the IPs field. +func (b *LoadBalancerSpecApplyConfiguration) WithIPs(values ...*IPSourceApplyConfiguration) *LoadBalancerSpecApplyConfiguration { + for i := range values { + if values[i] == nil { + panic("nil value passed to WithIPs") + } + b.IPs = append(b.IPs, *values[i]) + } + return b +} + // WithNetworkRef sets the NetworkRef field in the declarative configuration to the given value // and returns the receiver, so that objects can be built by chaining "With" function invocations. // If called multiple times, the NetworkRef field is set to the value of the last call. diff --git a/client-go/openapi/api_violations.report b/client-go/openapi/api_violations.report index bcd00f87b..77a1467b2 100644 --- a/client-go/openapi/api_violations.report +++ b/client-go/openapi/api_violations.report @@ -15,6 +15,7 @@ API rule violation: list_type_missing,github.com/onmetal/onmetal-api/api/ipam/v1 API rule violation: list_type_missing,github.com/onmetal/onmetal-api/api/networking/v1alpha1,AliasPrefixRouting,Destinations API rule violation: list_type_missing,github.com/onmetal/onmetal-api/api/networking/v1alpha1,LoadBalancerRouting,Destinations API rule violation: list_type_missing,github.com/onmetal/onmetal-api/api/networking/v1alpha1,LoadBalancerSpec,IPFamilies +API rule violation: list_type_missing,github.com/onmetal/onmetal-api/api/networking/v1alpha1,LoadBalancerSpec,IPs API rule violation: list_type_missing,github.com/onmetal/onmetal-api/api/networking/v1alpha1,LoadBalancerSpec,Ports API rule violation: list_type_missing,github.com/onmetal/onmetal-api/api/networking/v1alpha1,LoadBalancerStatus,IPs API rule violation: list_type_missing,github.com/onmetal/onmetal-api/api/networking/v1alpha1,NATGatewayDestination,IPs @@ -161,6 +162,7 @@ API rule violation: list_type_missing,k8s.io/apimachinery/pkg/runtime,RawExtensi API rule violation: list_type_missing,k8s.io/apimachinery/pkg/runtime,Unknown,Raw API rule violation: names_match,github.com/onmetal/onmetal-api/api/compute/v1alpha1,MachineSpec,ImagePullSecretRef API rule violation: names_match,github.com/onmetal/onmetal-api/api/compute/v1alpha1,NetworkInterfaceStatus,IPs +API rule violation: names_match,github.com/onmetal/onmetal-api/api/networking/v1alpha1,LoadBalancerSpec,IPs API rule violation: names_match,github.com/onmetal/onmetal-api/api/networking/v1alpha1,LoadBalancerStatus,IPs API rule violation: names_match,github.com/onmetal/onmetal-api/api/networking/v1alpha1,NATGatewayDestination,IPs API rule violation: names_match,github.com/onmetal/onmetal-api/api/networking/v1alpha1,NATGatewaySpec,IPs diff --git a/client-go/openapi/zz_generated.openapi.go b/client-go/openapi/zz_generated.openapi.go index 57fd88bee..e5ee1fefa 100644 --- a/client-go/openapi/zz_generated.openapi.go +++ b/client-go/openapi/zz_generated.openapi.go @@ -3084,6 +3084,20 @@ func schema_onmetal_api_api_networking_v1alpha1_LoadBalancerSpec(ref common.Refe }, }, }, + "ips": { + SchemaProps: spec.SchemaProps{ + Description: "IPs are the ips to use. Can only be used when Type is LoadBalancerTypeInternal.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/onmetal/onmetal-api/api/networking/v1alpha1.IPSource"), + }, + }, + }, + }, + }, "networkRef": { SchemaProps: spec.SchemaProps{ Description: "NetworkRef is the Network this LoadBalancer should belong to.", @@ -3116,7 +3130,7 @@ func schema_onmetal_api_api_networking_v1alpha1_LoadBalancerSpec(ref common.Refe }, }, Dependencies: []string{ - "github.com/onmetal/onmetal-api/api/networking/v1alpha1.LoadBalancerPort", "k8s.io/api/core/v1.LocalObjectReference", "k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector"}, + "github.com/onmetal/onmetal-api/api/networking/v1alpha1.IPSource", "github.com/onmetal/onmetal-api/api/networking/v1alpha1.LoadBalancerPort", "k8s.io/api/core/v1.LocalObjectReference", "k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector"}, } } diff --git a/docs/api-reference/networking.md b/docs/api-reference/networking.md index 4173aa595..86781b63e 100644 --- a/docs/api-reference/networking.md +++ b/docs/api-reference/networking.md @@ -300,6 +300,19 @@ LoadBalancerType +ips
+ + +[]IPSource + + + + +

IPs are the ips to use. Can only be used when Type is LoadBalancerTypeInternal.

+ + + + networkRef
@@ -1161,7 +1174,7 @@ VirtualIPTemplateSpec

IPSource

-(Appears on:NetworkInterfaceSpec) +(Appears on:LoadBalancerSpec, NetworkInterfaceSpec)

IPSource is the definition of how to obtain an IP.

@@ -1300,6 +1313,19 @@ LoadBalancerType +ips
+ + +[]IPSource + + + + +

IPs are the ips to use. Can only be used when Type is LoadBalancerTypeInternal.

+ + + + networkRef
@@ -1386,7 +1412,10 @@ for which this LoadBalancer should be applied

Description -

"Public"

+

"Internal"

+

LoadBalancerTypeInternal is a LoadBalancer that allocates and routes network-internal, stable IPs.

+ +

"Public"

LoadBalancerTypePublic is a LoadBalancer that allocates and routes a stable public IP.

diff --git a/internal/apis/networking/loadbalancer_type.go b/internal/apis/networking/loadbalancer_type.go index 1635e1e2c..cdca912f5 100644 --- a/internal/apis/networking/loadbalancer_type.go +++ b/internal/apis/networking/loadbalancer_type.go @@ -28,6 +28,8 @@ type LoadBalancerType string const ( // LoadBalancerTypePublic is a LoadBalancer that allocates and routes a stable public IP. LoadBalancerTypePublic LoadBalancerType = "Public" + // LoadBalancerTypeInternal is a LoadBalancer that allocates and routes network-internal, stable IPs. + LoadBalancerTypeInternal LoadBalancerType = "Internal" ) // LoadBalancerSpec defines the desired state of LoadBalancer @@ -36,6 +38,8 @@ type LoadBalancerSpec struct { Type LoadBalancerType // IPFamilies are the ip families the load balancer should have. IPFamilies []corev1.IPFamily + // IPs are the ips to use. Can only be used when Type is LoadBalancerTypeInternal. + IPs []IPSource // NetworkRef is the Network this LoadBalancer should belong to. NetworkRef corev1.LocalObjectReference // NetworkInterfaceSelector defines the NetworkInterfaces diff --git a/internal/apis/networking/v1alpha1/defaults.go b/internal/apis/networking/v1alpha1/defaults.go index ad6d138a1..c80e1eb95 100644 --- a/internal/apis/networking/v1alpha1/defaults.go +++ b/internal/apis/networking/v1alpha1/defaults.go @@ -35,32 +35,40 @@ func addDefaultingFuncs(scheme *runtime.Scheme) error { } func SetDefaults_NetworkInterfaceSpec(spec *v1alpha1.NetworkInterfaceSpec) { - if len(spec.IPFamilies) > 0 { - if len(spec.IPFamilies) == len(spec.IPs) { - for i, ip := range spec.IPs { + setDefaults_IPFamiliesIPSources(&spec.IPFamilies, &spec.IPs) +} + +func SetDefaults_LoadBalancerSpec(spec *v1alpha1.LoadBalancerSpec) { + setDefaults_IPFamiliesIPSources(&spec.IPFamilies, &spec.IPs) +} + +func setDefaults_IPFamiliesIPSources(ipFamilies *[]corev1.IPFamily, ipSources *[]v1alpha1.IPSource) { + if len(*ipFamilies) > 0 { + if len(*ipFamilies) == len(*ipSources) { + for i, ip := range *ipSources { if ip.Ephemeral != nil { if ip.Ephemeral.PrefixTemplate != nil { ephemeralPrefixSpec := &ip.Ephemeral.PrefixTemplate.Spec if ephemeralPrefixSpec.IPFamily == "" { - ephemeralPrefixSpec.IPFamily = spec.IPFamilies[i] + ephemeralPrefixSpec.IPFamily = (*ipFamilies)[i] } } } } } - } else if len(spec.IPs) > 0 { - for _, ip := range spec.IPs { + } else if len(*ipSources) > 0 { + for _, ip := range *ipSources { switch { case ip.Value != nil: - spec.IPFamilies = append(spec.IPFamilies, ip.Value.Family()) + *ipFamilies = append(*ipFamilies, ip.Value.Family()) case ip.Ephemeral != nil && ip.Ephemeral.PrefixTemplate != nil: - spec.IPFamilies = append(spec.IPFamilies, ip.Ephemeral.PrefixTemplate.Spec.IPFamily) + *ipFamilies = append(*ipFamilies, ip.Ephemeral.PrefixTemplate.Spec.IPFamily) } } } - for _, ip := range spec.IPs { + for _, ip := range *ipSources { if ip.Ephemeral != nil && ip.Ephemeral.PrefixTemplate != nil { templateSpec := &ip.Ephemeral.PrefixTemplate.Spec if templateSpec.Prefix == nil && templateSpec.PrefixLength == 0 { diff --git a/internal/apis/networking/v1alpha1/zz_generated.conversion.go b/internal/apis/networking/v1alpha1/zz_generated.conversion.go index 6b17f66d5..056b82b0a 100644 --- a/internal/apis/networking/v1alpha1/zz_generated.conversion.go +++ b/internal/apis/networking/v1alpha1/zz_generated.conversion.go @@ -822,6 +822,7 @@ func Convert_networking_LoadBalancerRoutingList_To_v1alpha1_LoadBalancerRoutingL func autoConvert_v1alpha1_LoadBalancerSpec_To_networking_LoadBalancerSpec(in *v1alpha1.LoadBalancerSpec, out *networking.LoadBalancerSpec, s conversion.Scope) error { out.Type = networking.LoadBalancerType(in.Type) out.IPFamilies = *(*[]corev1.IPFamily)(unsafe.Pointer(&in.IPFamilies)) + out.IPs = *(*[]networking.IPSource)(unsafe.Pointer(&in.IPs)) out.NetworkRef = in.NetworkRef out.NetworkInterfaceSelector = (*v1.LabelSelector)(unsafe.Pointer(in.NetworkInterfaceSelector)) out.Ports = *(*[]networking.LoadBalancerPort)(unsafe.Pointer(&in.Ports)) @@ -836,6 +837,7 @@ func Convert_v1alpha1_LoadBalancerSpec_To_networking_LoadBalancerSpec(in *v1alph func autoConvert_networking_LoadBalancerSpec_To_v1alpha1_LoadBalancerSpec(in *networking.LoadBalancerSpec, out *v1alpha1.LoadBalancerSpec, s conversion.Scope) error { out.Type = v1alpha1.LoadBalancerType(in.Type) out.IPFamilies = *(*[]corev1.IPFamily)(unsafe.Pointer(&in.IPFamilies)) + out.IPs = *(*[]v1alpha1.IPSource)(unsafe.Pointer(&in.IPs)) out.NetworkRef = in.NetworkRef out.NetworkInterfaceSelector = (*v1.LabelSelector)(unsafe.Pointer(in.NetworkInterfaceSelector)) out.Ports = *(*[]v1alpha1.LoadBalancerPort)(unsafe.Pointer(&in.Ports)) diff --git a/internal/apis/networking/v1alpha1/zz_generated.defaults.go b/internal/apis/networking/v1alpha1/zz_generated.defaults.go index 28a77160b..23950b514 100644 --- a/internal/apis/networking/v1alpha1/zz_generated.defaults.go +++ b/internal/apis/networking/v1alpha1/zz_generated.defaults.go @@ -32,6 +32,8 @@ import ( func RegisterDefaults(scheme *runtime.Scheme) error { scheme.AddTypeDefaultingFunc(&v1alpha1.AliasPrefix{}, func(obj interface{}) { SetObjectDefaults_AliasPrefix(obj.(*v1alpha1.AliasPrefix)) }) scheme.AddTypeDefaultingFunc(&v1alpha1.AliasPrefixList{}, func(obj interface{}) { SetObjectDefaults_AliasPrefixList(obj.(*v1alpha1.AliasPrefixList)) }) + scheme.AddTypeDefaultingFunc(&v1alpha1.LoadBalancer{}, func(obj interface{}) { SetObjectDefaults_LoadBalancer(obj.(*v1alpha1.LoadBalancer)) }) + scheme.AddTypeDefaultingFunc(&v1alpha1.LoadBalancerList{}, func(obj interface{}) { SetObjectDefaults_LoadBalancerList(obj.(*v1alpha1.LoadBalancerList)) }) scheme.AddTypeDefaultingFunc(&v1alpha1.NATGateway{}, func(obj interface{}) { SetObjectDefaults_NATGateway(obj.(*v1alpha1.NATGateway)) }) scheme.AddTypeDefaultingFunc(&v1alpha1.NATGatewayList{}, func(obj interface{}) { SetObjectDefaults_NATGatewayList(obj.(*v1alpha1.NATGatewayList)) }) scheme.AddTypeDefaultingFunc(&v1alpha1.NetworkInterface{}, func(obj interface{}) { SetObjectDefaults_NetworkInterface(obj.(*v1alpha1.NetworkInterface)) }) @@ -54,6 +56,25 @@ func SetObjectDefaults_AliasPrefixList(in *v1alpha1.AliasPrefixList) { } } +func SetObjectDefaults_LoadBalancer(in *v1alpha1.LoadBalancer) { + SetDefaults_LoadBalancerSpec(&in.Spec) + for i := range in.Spec.IPs { + a := &in.Spec.IPs[i] + if a.Ephemeral != nil { + if a.Ephemeral.PrefixTemplate != nil { + ipamv1alpha1.SetDefaults_PrefixSpec(&a.Ephemeral.PrefixTemplate.Spec) + } + } + } +} + +func SetObjectDefaults_LoadBalancerList(in *v1alpha1.LoadBalancerList) { + for i := range in.Items { + a := &in.Items[i] + SetObjectDefaults_LoadBalancer(a) + } +} + func SetObjectDefaults_NATGateway(in *v1alpha1.NATGateway) { SetDefaults_NATGatewaySpec(&in.Spec) } diff --git a/internal/apis/networking/validation/common.go b/internal/apis/networking/validation/common.go new file mode 100644 index 000000000..ccfd34053 --- /dev/null +++ b/internal/apis/networking/validation/common.go @@ -0,0 +1,55 @@ +// Copyright 2023 OnMetal authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package validation + +import ( + "fmt" + + commonvalidation "github.com/onmetal/onmetal-api/internal/apis/common/validation" + "github.com/onmetal/onmetal-api/internal/apis/networking" + corev1 "k8s.io/api/core/v1" + apivalidation "k8s.io/apimachinery/pkg/api/validation" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +func validateIPSource(ipSource networking.IPSource, idx int, ipFamily corev1.IPFamily, objectMeta *metav1.ObjectMeta, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + + var numSources int + if ip := ipSource.Value; ip.IsValid() { + numSources++ + allErrs = append(allErrs, commonvalidation.ValidateIP(ipFamily, *ip, fldPath.Child("value"))...) + } + if ephemeral := ipSource.Ephemeral; ephemeral != nil { + if numSources > 0 { + allErrs = append(allErrs, field.Invalid(fldPath.Child("ephemeral"), ephemeral, "cannot specify multiple ip sources")) + } else { + numSources++ + allErrs = append(allErrs, validateEphemeralPrefixSource(ipFamily, ephemeral, fldPath.Child("ephemeral"))...) + if objectMeta != nil && objectMeta.Name != "" { + prefixName := fmt.Sprintf("%s-%d", objectMeta.Name, idx) + for _, msg := range apivalidation.NameIsDNSLabel(prefixName, false) { + allErrs = append(allErrs, field.Invalid(fldPath, prefixName, fmt.Sprintf("resulting prefix name %q is invalid: %s", prefixName, msg))) + } + } + } + } + if numSources == 0 { + allErrs = append(allErrs, field.Invalid(fldPath, ipSource, "must specify an ip source")) + } + + return allErrs +} diff --git a/internal/apis/networking/validation/loadbalancer.go b/internal/apis/networking/validation/loadbalancer.go index 9d34b34aa..c862c164a 100644 --- a/internal/apis/networking/validation/loadbalancer.go +++ b/internal/apis/networking/validation/loadbalancer.go @@ -23,6 +23,7 @@ import ( "github.com/onmetal/onmetal-api/internal/apis/networking" corev1 "k8s.io/api/core/v1" apivalidation "k8s.io/apimachinery/pkg/api/validation" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1validation "k8s.io/apimachinery/pkg/apis/meta/v1/validation" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation" @@ -34,18 +35,20 @@ func ValidateLoadBalancer(loadBalancer *networking.LoadBalancer) field.ErrorList var allErrs field.ErrorList allErrs = append(allErrs, apivalidation.ValidateObjectMetaAccessor(loadBalancer, true, apivalidation.NameIsDNSLabel, field.NewPath("metadata"))...) - allErrs = append(allErrs, validateLoadBalancerSpec(&loadBalancer.Spec, field.NewPath("spec"))...) + allErrs = append(allErrs, validateLoadBalancerSpec(&loadBalancer.Spec, &loadBalancer.ObjectMeta, field.NewPath("spec"))...) return allErrs } -func validateLoadBalancerSpec(spec *networking.LoadBalancerSpec, fldPath *field.Path) field.ErrorList { +func validateLoadBalancerSpec(spec *networking.LoadBalancerSpec, lbMeta *metav1.ObjectMeta, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList allErrs = append(allErrs, validateLoadBalancerType(spec.Type, fldPath.Child("type"))...) allErrs = append(allErrs, onmetalapivalidation.ValidateIPFamilies(spec.IPFamilies, fldPath.Child("ipFamilies"))...) + allErrs = append(allErrs, validateNetworkInterfaceIPSources(spec.IPs, spec.IPFamilies, lbMeta, fldPath.Child("ips"))...) + for _, msg := range apivalidation.NameIsDNSLabel(spec.NetworkRef.Name, false) { allErrs = append(allErrs, field.Invalid(fldPath.Child("networkRef").Child("name"), spec.NetworkRef.Name, msg)) } @@ -94,6 +97,7 @@ func portRangesOverlap(x, y [2]int32) bool { var supportedLoadBalancerTypes = sets.New( networking.LoadBalancerTypePublic, + networking.LoadBalancerTypeInternal, ) func validateLoadBalancerType(loadBalancerType networking.LoadBalancerType, fldPath *field.Path) field.ErrorList { @@ -128,14 +132,14 @@ func ValidateLoadBalancerUpdate(newLoadBalancer, oldLoadBalancer *networking.Loa var allErrs field.ErrorList allErrs = append(allErrs, apivalidation.ValidateObjectMetaAccessorUpdate(newLoadBalancer, oldLoadBalancer, field.NewPath("metadata"))...) - allErrs = append(allErrs, validateLoadBalancerSpecPrefixUpdate(&newLoadBalancer.Spec, &oldLoadBalancer.Spec, field.NewPath("spec"))...) + allErrs = append(allErrs, validateLoadBalancerSpecUpdate(&newLoadBalancer.Spec, &oldLoadBalancer.Spec, field.NewPath("spec"))...) allErrs = append(allErrs, ValidateLoadBalancer(newLoadBalancer)...) return allErrs } -// validateLoadBalancerSpecPrefixUpdate validates the spec of a loadBalancer object before an update. -func validateLoadBalancerSpecPrefixUpdate(newSpec, oldSpec *networking.LoadBalancerSpec, fldPath *field.Path) field.ErrorList { +// validateLoadBalancerSpecUpdate validates the spec of a loadBalancer object before an update. +func validateLoadBalancerSpecUpdate(newSpec, oldSpec *networking.LoadBalancerSpec, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList allErrs = append(allErrs, onmetalapivalidation.ValidateImmutableField(newSpec.NetworkRef, oldSpec.NetworkRef, fldPath.Child("networkRef"))...) diff --git a/internal/apis/networking/validation/loadbalancer_test.go b/internal/apis/networking/validation/loadbalancer_test.go index 52e7ca990..29a6f7862 100644 --- a/internal/apis/networking/validation/loadbalancer_test.go +++ b/internal/apis/networking/validation/loadbalancer_test.go @@ -61,6 +61,14 @@ var _ = Describe("LoadBalancer", func() { &networking.LoadBalancer{}, ContainElement(RequiredField("spec.type")), ), + Entry("load balancer type public", + &networking.LoadBalancer{Spec: networking.LoadBalancerSpec{Type: networking.LoadBalancerTypePublic}}, + Not(ContainElement(RequiredField("spec.type"))), + ), + Entry("load balancer type internal", + &networking.LoadBalancer{Spec: networking.LoadBalancerSpec{Type: networking.LoadBalancerTypeInternal}}, + Not(ContainElement(RequiredField("spec.type"))), + ), Entry("duplicate ip family", &networking.LoadBalancer{ Spec: networking.LoadBalancerSpec{ diff --git a/internal/apis/networking/validation/networkinterface.go b/internal/apis/networking/validation/networkinterface.go index eb6622b55..cf6d5a4c3 100644 --- a/internal/apis/networking/validation/networkinterface.go +++ b/internal/apis/networking/validation/networkinterface.go @@ -20,7 +20,6 @@ import ( "fmt" onmetalapivalidation "github.com/onmetal/onmetal-api/internal/api/validation" - commonvalidation "github.com/onmetal/onmetal-api/internal/apis/common/validation" "github.com/onmetal/onmetal-api/internal/apis/ipam" ipamvalidation "github.com/onmetal/onmetal-api/internal/apis/ipam/validation" "github.com/onmetal/onmetal-api/internal/apis/networking" @@ -98,35 +97,6 @@ func validateNetworkInterfaceIPSources(ipSources []networking.IPSource, ipFamili return allErrs } -func validateIPSource(ipSource networking.IPSource, idx int, ipFamily corev1.IPFamily, nicMeta *metav1.ObjectMeta, fldPath *field.Path) field.ErrorList { - var allErrs field.ErrorList - - var numSources int - if ip := ipSource.Value; ip.IsValid() { - numSources++ - allErrs = append(allErrs, commonvalidation.ValidateIP(ipFamily, *ip, fldPath.Child("value"))...) - } - if ephemeral := ipSource.Ephemeral; ephemeral != nil { - if numSources > 0 { - allErrs = append(allErrs, field.Invalid(fldPath.Child("ephemeral"), ephemeral, "cannot specify multiple ip sources")) - } else { - numSources++ - allErrs = append(allErrs, validateEphemeralPrefixSource(ipFamily, ephemeral, fldPath.Child("ephemeral"))...) - if nicMeta != nil && nicMeta.Name != "" { - prefixName := fmt.Sprintf("%s-%d", nicMeta.Name, idx) - for _, msg := range apivalidation.NameIsDNSLabel(prefixName, false) { - allErrs = append(allErrs, field.Invalid(fldPath, prefixName, fmt.Sprintf("resulting prefix name %q is invalid: %s", prefixName, msg))) - } - } - } - } - if numSources == 0 { - allErrs = append(allErrs, field.Invalid(fldPath, ipSource, "must specify an ip source")) - } - - return allErrs -} - func validateVirtualIPSource(vipSource *networking.VirtualIPSource, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList diff --git a/internal/apis/networking/zz_generated.deepcopy.go b/internal/apis/networking/zz_generated.deepcopy.go index b8399461a..f48404dce 100644 --- a/internal/apis/networking/zz_generated.deepcopy.go +++ b/internal/apis/networking/zz_generated.deepcopy.go @@ -424,6 +424,13 @@ func (in *LoadBalancerSpec) DeepCopyInto(out *LoadBalancerSpec) { *out = make([]corev1.IPFamily, len(*in)) copy(*out, *in) } + if in.IPs != nil { + in, out := &in.IPs, &out.IPs + *out = make([]IPSource, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } out.NetworkRef = in.NetworkRef if in.NetworkInterfaceSelector != nil { in, out := &in.NetworkInterfaceSelector, &out.NetworkInterfaceSelector diff --git a/internal/registry/networking/loadbalancer/strategy.go b/internal/registry/networking/loadbalancer/strategy.go index 348f76907..3ceb23a62 100644 --- a/internal/registry/networking/loadbalancer/strategy.go +++ b/internal/registry/networking/loadbalancer/strategy.go @@ -65,9 +65,16 @@ func (loadBalancerStrategy) NamespaceScoped() bool { } func (loadBalancerStrategy) PrepareForCreate(ctx context.Context, obj runtime.Object) { + loadBalancer := obj.(*networking.LoadBalancer) + loadBalancer.Status = networking.LoadBalancerStatus{} + dropTypeDependentFields(loadBalancer) } func (loadBalancerStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) { + newLoadBalancer := obj.(*networking.LoadBalancer) + oldLoadBalancer := old.(*networking.LoadBalancer) + newLoadBalancer.Status = oldLoadBalancer.Status + dropTypeDependentFields(newLoadBalancer) } func (loadBalancerStrategy) Validate(ctx context.Context, obj runtime.Object) field.ErrorList { @@ -126,3 +133,18 @@ func (loadBalancerStatusStrategy) ValidateUpdate(ctx context.Context, obj, old r func (loadBalancerStatusStrategy) WarningsOnUpdate(cxt context.Context, obj, old runtime.Object) []string { return nil } + +func needsIPs(loadBalancer *networking.LoadBalancer) bool { + switch loadBalancer.Spec.Type { + case networking.LoadBalancerTypeInternal: + return true + default: + return false + } +} + +func dropTypeDependentFields(loadBalancer *networking.LoadBalancer) { + if !needsIPs(loadBalancer) { + loadBalancer.Spec.IPs = nil + } +} diff --git a/ori/apis/bucket/v1alpha1/api.pb.go b/ori/apis/bucket/v1alpha1/api.pb.go index 609e9c1db..710c463fa 100644 --- a/ori/apis/bucket/v1alpha1/api.pb.go +++ b/ori/apis/bucket/v1alpha1/api.pb.go @@ -6,12 +6,6 @@ package v1alpha1 import ( context "context" fmt "fmt" - io "io" - math "math" - math_bits "math/bits" - reflect "reflect" - strings "strings" - _ "github.com/gogo/protobuf/gogoproto" proto "github.com/gogo/protobuf/proto" github_com_gogo_protobuf_sortkeys "github.com/gogo/protobuf/sortkeys" @@ -19,6 +13,11 @@ import ( grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" + io "io" + math "math" + math_bits "math/bits" + reflect "reflect" + strings "strings" ) // Reference imports to suppress errors if they are not otherwise used. diff --git a/ori/apis/machine/v1alpha1/api.pb.go b/ori/apis/machine/v1alpha1/api.pb.go index 46290d5b9..a4c626336 100644 --- a/ori/apis/machine/v1alpha1/api.pb.go +++ b/ori/apis/machine/v1alpha1/api.pb.go @@ -6,12 +6,6 @@ package v1alpha1 import ( context "context" fmt "fmt" - io "io" - math "math" - math_bits "math/bits" - reflect "reflect" - strings "strings" - _ "github.com/gogo/protobuf/gogoproto" proto "github.com/gogo/protobuf/proto" github_com_gogo_protobuf_sortkeys "github.com/gogo/protobuf/sortkeys" @@ -19,6 +13,11 @@ import ( grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" + io "io" + math "math" + math_bits "math/bits" + reflect "reflect" + strings "strings" ) // Reference imports to suppress errors if they are not otherwise used. @@ -60,6 +59,31 @@ func (Protocol) EnumDescriptor() ([]byte, []int) { return fileDescriptor_00212fb1f9d3bf1c, []int{0} } +type LoadBalancerType int32 + +const ( + LoadBalancerType_PUBLIC LoadBalancerType = 0 + LoadBalancerType_INTERNAL LoadBalancerType = 1 +) + +var LoadBalancerType_name = map[int32]string{ + 0: "PUBLIC", + 1: "INTERNAL", +} + +var LoadBalancerType_value = map[string]int32{ + "PUBLIC": 0, + "INTERNAL": 1, +} + +func (x LoadBalancerType) String() string { + return proto.EnumName(LoadBalancerType_name, int32(x)) +} + +func (LoadBalancerType) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_00212fb1f9d3bf1c, []int{1} +} + type Power int32 const ( @@ -82,7 +106,7 @@ func (x Power) String() string { } func (Power) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_00212fb1f9d3bf1c, []int{1} + return fileDescriptor_00212fb1f9d3bf1c, []int{2} } type VolumeAttachmentState int32 @@ -110,7 +134,7 @@ func (x VolumeAttachmentState) String() string { } func (VolumeAttachmentState) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_00212fb1f9d3bf1c, []int{2} + return fileDescriptor_00212fb1f9d3bf1c, []int{3} } type NetworkInterfaceAttachmentState int32 @@ -138,7 +162,7 @@ func (x NetworkInterfaceAttachmentState) String() string { } func (NetworkInterfaceAttachmentState) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_00212fb1f9d3bf1c, []int{3} + return fileDescriptor_00212fb1f9d3bf1c, []int{4} } type MachineState int32 @@ -169,7 +193,7 @@ func (x MachineState) String() string { } func (MachineState) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_00212fb1f9d3bf1c, []int{4} + return fileDescriptor_00212fb1f9d3bf1c, []int{5} } type VolumeFilter struct { @@ -613,8 +637,9 @@ func (m *LoadBalancerPort) GetEndPort() int32 { } type LoadBalancerTargetSpec struct { - Ip string `protobuf:"bytes,1,opt,name=ip,proto3" json:"ip,omitempty"` - Ports []*LoadBalancerPort `protobuf:"bytes,2,rep,name=ports,proto3" json:"ports,omitempty"` + LoadBalancerType LoadBalancerType `protobuf:"varint,1,opt,name=load_balancer_type,json=loadBalancerType,proto3,enum=machine.v1alpha1.LoadBalancerType" json:"load_balancer_type,omitempty"` + Ip string `protobuf:"bytes,2,opt,name=ip,proto3" json:"ip,omitempty"` + Ports []*LoadBalancerPort `protobuf:"bytes,3,rep,name=ports,proto3" json:"ports,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_sizecache int32 `json:"-"` } @@ -651,6 +676,13 @@ func (m *LoadBalancerTargetSpec) XXX_DiscardUnknown() { var xxx_messageInfo_LoadBalancerTargetSpec proto.InternalMessageInfo +func (m *LoadBalancerTargetSpec) GetLoadBalancerType() LoadBalancerType { + if m != nil { + return m.LoadBalancerType + } + return LoadBalancerType_PUBLIC +} + func (m *LoadBalancerTargetSpec) GetIp() string { if m != nil { return m.Ip @@ -4085,6 +4117,7 @@ func (m *ExecResponse) GetUrl() string { func init() { proto.RegisterEnum("machine.v1alpha1.Protocol", Protocol_name, Protocol_value) + proto.RegisterEnum("machine.v1alpha1.LoadBalancerType", LoadBalancerType_name, LoadBalancerType_value) proto.RegisterEnum("machine.v1alpha1.Power", Power_name, Power_value) proto.RegisterEnum("machine.v1alpha1.VolumeAttachmentState", VolumeAttachmentState_name, VolumeAttachmentState_value) proto.RegisterEnum("machine.v1alpha1.NetworkInterfaceAttachmentState", NetworkInterfaceAttachmentState_name, NetworkInterfaceAttachmentState_value) @@ -4180,181 +4213,185 @@ func init() { func init() { proto.RegisterFile("api.proto", fileDescriptor_00212fb1f9d3bf1c) } var fileDescriptor_00212fb1f9d3bf1c = []byte{ - // 2783 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x5b, 0xcf, 0x73, 0xdb, 0xc6, - 0xf5, 0x17, 0xa8, 0x9f, 0x7c, 0x94, 0x64, 0x66, 0xf5, 0x23, 0x34, 0x6c, 0x51, 0x32, 0x62, 0xcb, - 0x8a, 0x62, 0x8b, 0x16, 0xf3, 0x4d, 0x62, 0xfb, 0xdb, 0xa4, 0xd1, 0x0f, 0x5a, 0xe6, 0x44, 0xa2, - 0x14, 0x88, 0x76, 0x1a, 0xd7, 0x1d, 0x16, 0x24, 0x57, 0x32, 0x6a, 0x12, 0x64, 0x00, 0x50, 0xb1, - 0x93, 0xc9, 0x4c, 0x33, 0xd3, 0x4b, 0x0f, 0x9d, 0xc9, 0xa9, 0x87, 0xe6, 0xd4, 0xe9, 0x3f, 0xd0, - 0x1e, 0x9b, 0xe9, 0x4c, 0xaf, 0x99, 0x1e, 0x3a, 0xbd, 0xb5, 0x87, 0x1e, 0x1a, 0x77, 0xa6, 0xc7, - 0xfe, 0x0d, 0x9d, 0xfd, 0x05, 0x81, 0xc4, 0x82, 0x00, 0xa5, 0x36, 0xd3, 0x9e, 0x8c, 0x7d, 0x78, - 0xfb, 0xde, 0xe7, 0xbd, 0x7d, 0xfb, 0x76, 0xf1, 0x31, 0x05, 0x49, 0xa3, 0x6d, 0xae, 0xb5, 0xed, - 0x96, 0xdb, 0x42, 0xe9, 0xa6, 0x51, 0x7b, 0x62, 0x5a, 0x78, 0xed, 0x64, 0xdd, 0x68, 0xb4, 0x9f, - 0x18, 0xeb, 0xea, 0xcd, 0x63, 0xd3, 0x7d, 0xd2, 0xa9, 0xae, 0xd5, 0x5a, 0xcd, 0xdc, 0x71, 0xeb, - 0xb8, 0x95, 0xa3, 0x8a, 0xd5, 0xce, 0x11, 0x1d, 0xd1, 0x01, 0x7d, 0x62, 0x06, 0xd4, 0xef, 0xfa, - 0xd4, 0x5b, 0x56, 0x13, 0xbb, 0x46, 0x43, 0xfc, 0x7b, 0xd3, 0x68, 0x9b, 0xb9, 0x96, 0x6d, 0xe6, - 0x8c, 0xb6, 0xe9, 0xe4, 0x88, 0x28, 0x27, 0xbc, 0xe4, 0x3c, 0x04, 0xda, 0x57, 0x0a, 0x4c, 0x3e, - 0x6c, 0x35, 0x3a, 0x4d, 0x7c, 0xcf, 0x6c, 0xb8, 0xd8, 0x46, 0xd3, 0x90, 0x30, 0xeb, 0x19, 0x65, - 0x49, 0x59, 0x49, 0xea, 0x09, 0xb3, 0x8e, 0xbe, 0x07, 0xd3, 0x0d, 0xa3, 0x8a, 0x1b, 0x15, 0x07, - 0x37, 0x70, 0xcd, 0x6d, 0xd9, 0x99, 0xc4, 0xd2, 0xf0, 0x4a, 0x2a, 0xbf, 0xbe, 0xd6, 0x8b, 0x7d, - 0xcd, 0x6f, 0x67, 0x6d, 0x97, 0x4c, 0x3a, 0xe4, 0x73, 0x0a, 0x96, 0x6b, 0x3f, 0xd7, 0xa7, 0x1a, - 0x7e, 0x99, 0xfa, 0x2e, 0xa0, 0xa0, 0x12, 0x4a, 0xc3, 0xf0, 0x53, 0xfc, 0x9c, 0x03, 0x20, 0x8f, - 0x68, 0x16, 0x46, 0x4f, 0x8c, 0x46, 0x07, 0x67, 0x12, 0x54, 0xc6, 0x06, 0x77, 0x13, 0xb7, 0x15, - 0xed, 0xcf, 0x09, 0x00, 0xe6, 0xf4, 0xb0, 0x8d, 0x6b, 0x68, 0x1e, 0xc6, 0xea, 0xb6, 0x79, 0x82, - 0x6d, 0x3e, 0x9b, 0x8f, 0x88, 0xfc, 0x89, 0x61, 0xd5, 0x1b, 0xc2, 0x02, 0x1f, 0xa1, 0x5d, 0x00, - 0xc3, 0x75, 0x6d, 0xb3, 0xda, 0x71, 0xb1, 0x93, 0x19, 0xa6, 0x61, 0xdd, 0x08, 0x0b, 0x8b, 0x78, - 0x58, 0xdb, 0xf0, 0xd4, 0x59, 0x44, 0xbe, 0xf9, 0x68, 0x0f, 0x52, 0x0e, 0xae, 0xd9, 0xd8, 0xad, - 0xd4, 0x0d, 0xd7, 0xc8, 0x8c, 0xc4, 0x30, 0x77, 0x48, 0xf5, 0xb7, 0x0d, 0xd7, 0xe0, 0xe6, 0x1c, - 0x4f, 0xa0, 0xbe, 0x0d, 0x17, 0x7a, 0xbc, 0x0d, 0x92, 0x1a, 0x32, 0xbd, 0xc7, 0x7a, 0xd4, 0xf4, - 0x49, 0x7f, 0x66, 0x3b, 0x30, 0xc6, 0x70, 0xa2, 0x3b, 0x30, 0x41, 0x6a, 0x87, 0xc6, 0x44, 0xa6, - 0xa6, 0xf2, 0x0b, 0x6b, 0x44, 0x70, 0x1a, 0xd0, 0x7e, 0xf5, 0x47, 0xb8, 0xe6, 0xee, 0x71, 0x25, - 0xdd, 0x53, 0x47, 0xb7, 0x60, 0xc4, 0x69, 0xe3, 0x1a, 0xb5, 0x9e, 0xca, 0x5f, 0xee, 0x97, 0x0a, - 0x9d, 0x6a, 0x6a, 0x7f, 0x54, 0x60, 0xbe, 0x84, 0xdd, 0x8f, 0x5b, 0xf6, 0xd3, 0xa2, 0xe5, 0x62, - 0xfb, 0xc8, 0xa8, 0x85, 0xd5, 0x65, 0x35, 0xa4, 0x2e, 0xff, 0x3f, 0xe8, 0x46, 0x6e, 0xf1, 0x5b, - 0xa9, 0xd0, 0x6b, 0x90, 0xe2, 0xde, 0x45, 0x85, 0xf2, 0x4a, 0x54, 0xfc, 0x95, 0xa8, 0x2d, 0xc2, - 0xd4, 0x43, 0xd3, 0x76, 0x3b, 0x46, 0xa3, 0x78, 0x40, 0x15, 0x49, 0xb4, 0x6d, 0x2f, 0xda, 0xb6, - 0xb6, 0x03, 0xe3, 0xa5, 0x8d, 0xb2, 0xec, 0x15, 0x42, 0x30, 0xd2, 0x6e, 0xd9, 0x2e, 0xf5, 0x3d, - 0xaa, 0xd3, 0x67, 0x94, 0x81, 0x71, 0x6c, 0xd5, 0x0f, 0x88, 0x78, 0x98, 0x8a, 0xc5, 0x50, 0x7b, - 0x06, 0xe9, 0xdd, 0x96, 0x51, 0xdf, 0x34, 0x1a, 0x86, 0x55, 0xc3, 0x36, 0x91, 0xa1, 0x37, 0x61, - 0x82, 0x36, 0x83, 0x5a, 0xab, 0x41, 0xed, 0x4e, 0xe7, 0xd5, 0x60, 0x12, 0x0f, 0xb8, 0x86, 0xee, - 0xe9, 0x0e, 0xe8, 0xb9, 0x0a, 0xf3, 0x7e, 0xcf, 0x65, 0xc3, 0x3e, 0xc6, 0xae, 0x34, 0xa2, 0xdb, - 0x30, 0x4a, 0x6c, 0x39, 0x7c, 0x45, 0xb5, 0x20, 0x98, 0xde, 0x10, 0x74, 0x36, 0x41, 0xfb, 0x43, - 0x02, 0x66, 0x7b, 0x57, 0x9b, 0xba, 0x78, 0x0b, 0xc6, 0x2d, 0x26, 0x3f, 0x2d, 0xe2, 0x90, 0x32, - 0xa1, 0xe5, 0x28, 0xb4, 0xc9, 0x62, 0x9b, 0x6d, 0x86, 0x24, 0xa9, 0x93, 0x47, 0xf4, 0x0e, 0xc0, - 0x09, 0x5b, 0xab, 0x8a, 0xd9, 0xa6, 0x41, 0xa6, 0xf2, 0x8b, 0x92, 0xda, 0xf6, 0xaf, 0xa7, 0x9e, - 0xe4, 0x53, 0x8a, 0x6d, 0xa4, 0x92, 0x6c, 0xe3, 0x23, 0xf3, 0x19, 0x76, 0x68, 0x93, 0x48, 0xea, - 0xde, 0x18, 0x3d, 0x86, 0xb9, 0x46, 0xcb, 0xa8, 0x57, 0xaa, 0x3c, 0xb6, 0x8a, 0x4b, 0xb3, 0xe4, - 0x64, 0x46, 0x69, 0x26, 0x56, 0xfa, 0x67, 0xe2, 0x34, 0xa5, 0xfa, 0x4c, 0x23, 0x20, 0x77, 0xd0, - 0x4d, 0x18, 0xb1, 0x0c, 0xd7, 0xc9, 0x8c, 0x51, 0x63, 0x17, 0x25, 0x19, 0x60, 0x25, 0xa6, 0x53, - 0x35, 0xed, 0xa7, 0x0a, 0xa4, 0x7b, 0x93, 0x79, 0x9e, 0x76, 0x70, 0xb7, 0xab, 0x1d, 0x2c, 0x47, - 0xef, 0x53, 0x5f, 0x63, 0xd0, 0x60, 0xb2, 0x78, 0x6c, 0x99, 0xae, 0xd9, 0xb2, 0xe8, 0x7a, 0x22, - 0x18, 0xf1, 0x20, 0x4c, 0xea, 0xf4, 0x59, 0xfb, 0x9d, 0x02, 0x53, 0x7b, 0xcc, 0x66, 0x48, 0xcf, - 0xf8, 0x30, 0xa4, 0x67, 0xe4, 0x83, 0x58, 0xba, 0x0c, 0x7d, 0x2b, 0xad, 0xe2, 0x31, 0x64, 0xb8, - 0xd3, 0xad, 0x86, 0xe1, 0x38, 0x5b, 0x46, 0xdb, 0xa8, 0x9a, 0x0d, 0xd3, 0x35, 0xb1, 0x83, 0x16, - 0x00, 0x6a, 0xed, 0x4e, 0xa5, 0x69, 0x36, 0x1a, 0xa6, 0x43, 0xcd, 0x0d, 0xeb, 0xc9, 0x5a, 0xbb, - 0xb3, 0x47, 0x05, 0xe8, 0x0a, 0x4c, 0x36, 0x71, 0xb3, 0x65, 0x3f, 0xaf, 0x54, 0x9f, 0x93, 0xa3, - 0x8c, 0xd8, 0x1e, 0xd1, 0x53, 0x4c, 0xb6, 0x49, 0x44, 0xda, 0xaf, 0x15, 0x18, 0xe7, 0xe6, 0xcf, - 0xb3, 0x86, 0xeb, 0x5d, 0x6b, 0xb8, 0x10, 0x9a, 0xb7, 0xd3, 0xa5, 0x43, 0x6f, 0xc1, 0x98, 0xe3, - 0x1a, 0x6e, 0xc7, 0x09, 0xdf, 0x2b, 0x62, 0x12, 0x55, 0xd3, 0xb9, 0xba, 0x76, 0x05, 0x92, 0xc5, - 0xa6, 0x71, 0xcc, 0x36, 0xf0, 0x2c, 0x8c, 0x9a, 0x64, 0xc0, 0x73, 0xc9, 0x06, 0xda, 0x1a, 0x4c, - 0x15, 0x9a, 0x6d, 0xf7, 0xf9, 0xb6, 0xe9, 0xb0, 0x06, 0xbb, 0x00, 0xe0, 0x98, 0x9f, 0x60, 0x9e, - 0x07, 0x85, 0xe6, 0x21, 0x49, 0x24, 0x2c, 0x0b, 0x5f, 0x2a, 0x90, 0x66, 0x87, 0xce, 0x86, 0xeb, - 0x1a, 0xb5, 0x27, 0x4d, 0x6c, 0xb9, 0xa4, 0x96, 0x2c, 0xa3, 0x29, 0x2c, 0xd3, 0x67, 0x7a, 0x95, - 0xc0, 0x27, 0x66, 0xcd, 0xbb, 0x32, 0xb0, 0x11, 0xba, 0x04, 0xc9, 0x13, 0x3a, 0xbf, 0x62, 0xd6, - 0x69, 0x3c, 0x49, 0x7d, 0x82, 0x09, 0x8a, 0x75, 0xd2, 0x19, 0x30, 0x41, 0x53, 0xa9, 0x9b, 0xce, - 0xd3, 0xcc, 0x48, 0x58, 0xb4, 0x5d, 0x88, 0xf5, 0x24, 0x16, 0x43, 0xad, 0x0a, 0x6a, 0xef, 0x16, - 0x88, 0x80, 0x79, 0x0b, 0x66, 0x79, 0xa3, 0xaa, 0x98, 0x62, 0x0a, 0x41, 0xc6, 0x40, 0x23, 0xab, - 0xc7, 0x5a, 0xb1, 0xae, 0xfd, 0x35, 0x01, 0x29, 0xdf, 0x1a, 0xa1, 0x9b, 0xa4, 0xd7, 0x7e, 0xcc, - 0xaf, 0x4c, 0xd3, 0xf9, 0x97, 0x25, 0x8d, 0x9f, 0xbc, 0xd6, 0x99, 0x16, 0x5a, 0x17, 0xcb, 0xc0, - 0x0a, 0xe0, 0x52, 0x50, 0xdd, 0x5b, 0x32, 0xbe, 0x46, 0x64, 0xe5, 0x6a, 0xa4, 0xa0, 0x79, 0xba, - 0xd8, 0x00, 0xdd, 0x85, 0x09, 0x93, 0x6f, 0x68, 0x9e, 0xa9, 0xac, 0xc4, 0x96, 0x6f, 0xcb, 0xeb, - 0x9e, 0x3e, 0xfa, 0x0e, 0x8c, 0xb3, 0x9c, 0x8b, 0xbe, 0xa8, 0x85, 0x5d, 0x2d, 0x4e, 0xd3, 0xa7, - 0x8b, 0x29, 0xe8, 0xfb, 0x80, 0x02, 0x39, 0x13, 0x3d, 0xf1, 0x46, 0x74, 0x53, 0xf2, 0x99, 0x7c, - 0xa9, 0x37, 0xbf, 0x8e, 0xf6, 0x55, 0xc2, 0xeb, 0x41, 0xac, 0x9a, 0x51, 0x0e, 0x66, 0x5a, 0x55, - 0x07, 0xdb, 0x27, 0xb8, 0x5e, 0x39, 0xc6, 0x16, 0xb6, 0x0d, 0x1a, 0x33, 0xdb, 0xc3, 0x48, 0xbc, - 0xda, 0xf1, 0xde, 0xa0, 0xff, 0x83, 0x51, 0xb2, 0x01, 0x58, 0x8a, 0xa7, 0x65, 0x69, 0xf1, 0x39, - 0xc0, 0x3a, 0x53, 0x26, 0x85, 0x49, 0xd3, 0x5d, 0xb1, 0xf1, 0x91, 0x28, 0x4c, 0x2a, 0xd0, 0xf1, - 0x11, 0xda, 0x3c, 0x4d, 0xd8, 0x48, 0xd8, 0x41, 0xd2, 0x9b, 0x30, 0xbe, 0x19, 0xbd, 0xb4, 0x19, - 0xd2, 0xb4, 0x8d, 0x86, 0xf5, 0xcf, 0xf0, 0xb4, 0x71, 0xc3, 0x92, 0xe4, 0x7d, 0xa1, 0xc0, 0xbc, - 0x1c, 0x86, 0xb4, 0xf8, 0x5f, 0x81, 0x29, 0xbe, 0x17, 0xbb, 0x6e, 0xf7, 0x93, 0x4c, 0x78, 0x9f, - 0xdd, 0xf1, 0xdf, 0x16, 0xd9, 0x1c, 0xa6, 0xd9, 0xbc, 0x1e, 0x2f, 0x70, 0x91, 0x56, 0xed, 0xb7, - 0x0a, 0x2c, 0x45, 0x85, 0x22, 0x05, 0x77, 0x1b, 0x32, 0xc1, 0x9d, 0xd9, 0x85, 0x73, 0xbe, 0x37, - 0x01, 0x1c, 0xf1, 0x4e, 0x37, 0xe2, 0xf5, 0x41, 0x73, 0xeb, 0x61, 0xb7, 0x61, 0xd2, 0x7f, 0xa0, - 0x48, 0x61, 0x96, 0x60, 0xb2, 0xe6, 0x3b, 0x68, 0xf8, 0xb6, 0x5e, 0x0d, 0xad, 0xb9, 0xc0, 0xd1, - 0xa4, 0x77, 0xcd, 0xd7, 0x56, 0x61, 0xfa, 0x21, 0xb6, 0x1d, 0xb3, 0x65, 0xe9, 0xf8, 0xa3, 0x0e, - 0x76, 0xe8, 0x85, 0xf0, 0x84, 0x49, 0xb8, 0x63, 0x31, 0xd4, 0x7e, 0x00, 0x17, 0x3c, 0x5d, 0xa7, - 0xdd, 0xb2, 0x1c, 0x4c, 0x0e, 0x32, 0xbb, 0x63, 0xb9, 0x66, 0x13, 0x57, 0x7c, 0x50, 0x53, 0x5c, - 0x56, 0x22, 0x88, 0xaf, 0xc3, 0x05, 0xa1, 0x22, 0xec, 0xb2, 0x7c, 0x4e, 0x73, 0x31, 0xb7, 0xa9, - 0x95, 0x60, 0x66, 0xd7, 0x74, 0x5c, 0x0e, 0xdc, 0x11, 0x78, 0xde, 0x82, 0xb1, 0x23, 0x7a, 0xa8, - 0xf3, 0xa3, 0x6f, 0x31, 0xe2, 0xec, 0xd7, 0xb9, 0xba, 0xb6, 0x07, 0xb3, 0xdd, 0xf6, 0x38, 0xe6, - 0x37, 0x60, 0x82, 0x5b, 0x20, 0x07, 0x4e, 0xc8, 0xcd, 0x8a, 0xcf, 0xd2, 0x3d, 0x55, 0xed, 0x3d, - 0x98, 0xdd, 0xb2, 0xb1, 0xe1, 0x62, 0xf1, 0x8a, 0xe3, 0x7b, 0x1d, 0xc6, 0xb9, 0x0e, 0x07, 0xd8, - 0xc7, 0x9a, 0xd0, 0xd4, 0x76, 0x61, 0xae, 0xc7, 0x18, 0x07, 0x77, 0x26, 0x6b, 0x6f, 0xc0, 0xec, - 0x36, 0x6e, 0xe0, 0x00, 0xb4, 0x05, 0x00, 0xae, 0x52, 0xf1, 0xae, 0x55, 0x49, 0x2e, 0x29, 0xd6, - 0xb5, 0x97, 0x61, 0xae, 0x67, 0x1a, 0x03, 0xa1, 0xfd, 0x43, 0x81, 0xc5, 0x07, 0xed, 0xfa, 0x29, - 0xbc, 0x0d, 0xcb, 0x6a, 0xb9, 0xb4, 0xdb, 0x39, 0xf1, 0x6c, 0xa3, 0x3a, 0xa4, 0x8c, 0xd3, 0x49, - 0xfc, 0xda, 0xb6, 0x19, 0x8c, 0x25, 0xc2, 0xcd, 0x9a, 0x4f, 0xc4, 0xae, 0x71, 0x7e, 0xb3, 0xea, - 0x3b, 0x90, 0xee, 0x55, 0x18, 0xe8, 0x0a, 0xa7, 0xc1, 0x52, 0x38, 0x00, 0x9e, 0x0c, 0x13, 0x2e, - 0x76, 0xe9, 0xb0, 0xe3, 0x35, 0x5e, 0x16, 0xbc, 0xc3, 0x3a, 0x11, 0xe7, 0xb0, 0xd6, 0x2e, 0x83, - 0x2a, 0x73, 0xc5, 0x81, 0x7c, 0x02, 0x0b, 0xac, 0x66, 0x02, 0x47, 0x65, 0x3c, 0x30, 0x77, 0x61, - 0x8c, 0x75, 0x5a, 0xde, 0x34, 0xe2, 0x1c, 0xc2, 0x7c, 0x86, 0xb6, 0x04, 0xd9, 0x30, 0xdf, 0x1c, - 0x9d, 0x0e, 0x0b, 0xac, 0x98, 0xce, 0x88, 0x4e, 0x34, 0xbb, 0xc4, 0x69, 0xb3, 0x23, 0x5e, 0xc3, - 0x6c, 0x72, 0xaf, 0xbf, 0x52, 0xe0, 0x3a, 0x03, 0xd6, 0xe7, 0xd8, 0x8f, 0x07, 0xe0, 0x43, 0x78, - 0x29, 0x70, 0x00, 0xf0, 0x4c, 0x0d, 0x76, 0xcb, 0x48, 0xf7, 0x9e, 0x13, 0xda, 0x2a, 0xac, 0x44, - 0x83, 0xe4, 0x11, 0x3d, 0x86, 0xeb, 0x2c, 0xe6, 0x73, 0x07, 0x24, 0xcb, 0xe8, 0x2a, 0xac, 0x44, - 0x5b, 0xe7, 0x48, 0x76, 0x01, 0x91, 0xfe, 0xc9, 0x72, 0xef, 0xed, 0xfb, 0x37, 0x7b, 0xda, 0x71, - 0xb6, 0x3f, 0xad, 0xe8, 0x75, 0xe3, 0x22, 0xeb, 0xee, 0x9e, 0x35, 0xde, 0xef, 0xf2, 0xa7, 0x37, - 0x1d, 0xd6, 0x8b, 0x33, 0x61, 0xf6, 0xbc, 0x9b, 0x8d, 0xb6, 0x03, 0x33, 0xfe, 0x62, 0x14, 0xc8, - 0x6e, 0x79, 0xf5, 0xcd, 0x90, 0x85, 0x5b, 0x12, 0x55, 0x7d, 0x5f, 0xb4, 0x74, 0x61, 0x88, 0x83, - 0x1a, 0xdc, 0x52, 0x1e, 0x66, 0xfc, 0x95, 0x2a, 0x20, 0x75, 0x7d, 0x7d, 0x28, 0xdd, 0x5f, 0x1f, - 0xda, 0xbc, 0xe8, 0xda, 0xdd, 0xde, 0xb5, 0x1f, 0xc2, 0x65, 0x92, 0xa9, 0xde, 0x15, 0xf2, 0x56, - 0xe0, 0xdd, 0x9e, 0x15, 0x58, 0x89, 0x4b, 0xa0, 0x79, 0x6b, 0x61, 0xc3, 0x42, 0x88, 0x07, 0x9e, - 0x80, 0xf7, 0xa5, 0x77, 0x47, 0x25, 0xec, 0xee, 0xde, 0x6b, 0x48, 0x76, 0x57, 0x6c, 0x8b, 0xee, - 0x15, 0x50, 0xe6, 0x61, 0xed, 0xcb, 0xf6, 0x9f, 0x12, 0xd6, 0xa9, 0x02, 0x56, 0x82, 0xbb, 0xee, - 0x23, 0xd1, 0xb3, 0x82, 0x1e, 0x79, 0x98, 0xff, 0x76, 0x97, 0xef, 0x8b, 0x26, 0x18, 0x16, 0x64, - 0xd8, 0xf7, 0x9f, 0x12, 0xfa, 0xfd, 0xe7, 0xf5, 0xc0, 0xb0, 0x28, 0xb4, 0x23, 0x71, 0x88, 0xf5, - 0x6a, 0x14, 0x0f, 0x9c, 0x33, 0xfb, 0x0d, 0xf2, 0x68, 0xda, 0x2b, 0x70, 0xa5, 0x8f, 0x1f, 0x0e, - 0xe6, 0x17, 0x0a, 0x2c, 0xcb, 0xb3, 0xee, 0xf1, 0x6b, 0x67, 0xc7, 0xd4, 0xcd, 0xe4, 0x25, 0x06, - 0x65, 0xf2, 0xb4, 0x57, 0xc3, 0x0e, 0x0b, 0x1f, 0x36, 0x5f, 0x1c, 0xf2, 0x68, 0xff, 0x3b, 0xe2, - 0x88, 0xc4, 0xc6, 0xe3, 0x68, 0xc1, 0x2b, 0xf2, 0x90, 0x0f, 0x28, 0x85, 0x79, 0xf6, 0x18, 0xe6, - 0x61, 0x8c, 0xb1, 0xa0, 0x82, 0x70, 0x61, 0x23, 0x6d, 0x19, 0xae, 0xf6, 0x77, 0x78, 0x0a, 0x4c, - 0x5e, 0xd7, 0xff, 0x41, 0x60, 0xfd, 0x1d, 0x72, 0x60, 0xbf, 0x57, 0x20, 0x27, 0x8f, 0x20, 0x48, - 0xdd, 0x9e, 0x1d, 0xe5, 0x23, 0x98, 0x95, 0x11, 0xc7, 0xbc, 0x18, 0xe2, 0xf3, 0xc6, 0x28, 0xc8, - 0x1b, 0x6b, 0x79, 0xb8, 0x15, 0x3f, 0x00, 0x5f, 0xd4, 0xf2, 0xf4, 0xfc, 0x0f, 0x45, 0x1d, 0x3f, - 0x00, 0x1e, 0xf5, 0xe7, 0x0a, 0x2c, 0xc9, 0x53, 0x55, 0xda, 0x28, 0x9f, 0x3d, 0xcc, 0xd7, 0x60, - 0xd8, 0x32, 0x44, 0x54, 0x7d, 0x68, 0x7b, 0xa2, 0x45, 0xda, 0x6a, 0x1f, 0x08, 0x1c, 0xe8, 0x53, - 0x58, 0x92, 0x07, 0x77, 0x2e, 0x9c, 0x73, 0x30, 0x66, 0x19, 0xae, 0xe8, 0x41, 0x49, 0x7d, 0xd4, - 0x32, 0xdc, 0x62, 0x9b, 0x20, 0xea, 0xe3, 0x8c, 0x23, 0x7a, 0x04, 0xcb, 0x72, 0xa5, 0xf3, 0xf7, - 0x47, 0xd2, 0xdf, 0x22, 0x6d, 0x73, 0x18, 0x97, 0xe0, 0xa2, 0xef, 0x23, 0x9f, 0xb2, 0x1d, 0xde, - 0x4d, 0x49, 0xc3, 0xa0, 0xca, 0x5e, 0xf2, 0xd3, 0x7f, 0x07, 0x2e, 0x88, 0xeb, 0x73, 0x8d, 0xbd, - 0xe2, 0x37, 0x9c, 0x6c, 0x7f, 0x36, 0x45, 0x9f, 0x6e, 0x76, 0x19, 0xd4, 0x6e, 0x40, 0xaa, 0xf0, - 0x0c, 0xd7, 0x62, 0x7e, 0x75, 0x2f, 0xc1, 0x24, 0xd3, 0xe6, 0x30, 0xd2, 0x30, 0xdc, 0xb1, 0x1b, - 0xe2, 0x7b, 0xb5, 0x63, 0x37, 0x56, 0x97, 0x61, 0x42, 0xfc, 0xe7, 0x1d, 0x1a, 0x87, 0xe1, 0xf2, - 0xd6, 0x41, 0x7a, 0x88, 0x3c, 0x3c, 0xd8, 0x3e, 0x48, 0x2b, 0x68, 0x02, 0x46, 0x0e, 0xb7, 0xca, - 0x07, 0xe9, 0xc4, 0xea, 0x55, 0x18, 0xa5, 0x5f, 0x88, 0x68, 0x12, 0x26, 0x0e, 0xf6, 0x3f, 0x28, - 0xe8, 0x95, 0xfd, 0x52, 0x7a, 0x08, 0x4d, 0x41, 0x92, 0x8f, 0xee, 0xdd, 0x4b, 0x2b, 0xab, 0x27, - 0x30, 0x27, 0x65, 0xcc, 0xd0, 0x02, 0x5c, 0x7c, 0xb8, 0xbf, 0xfb, 0x60, 0xaf, 0x50, 0xd9, 0x28, - 0x97, 0x37, 0xb6, 0xee, 0xef, 0x15, 0x4a, 0xe5, 0xca, 0x41, 0xa1, 0xb4, 0x5d, 0x2c, 0xed, 0xa4, - 0x87, 0x50, 0x16, 0xd4, 0xe0, 0x6b, 0xf6, 0x58, 0xd8, 0x4e, 0x2b, 0xf2, 0xf7, 0xdb, 0x05, 0xfe, - 0x3e, 0xb1, 0xfa, 0x4b, 0x05, 0x16, 0x23, 0x88, 0x2f, 0xb4, 0x02, 0x57, 0x4b, 0x85, 0xf2, 0x07, - 0xfb, 0xfa, 0x7b, 0x95, 0x62, 0xa9, 0x5c, 0xd0, 0xef, 0x6d, 0x6c, 0x85, 0xa0, 0x79, 0x15, 0xae, - 0xf5, 0xd5, 0xf4, 0x01, 0x8b, 0x52, 0xf5, 0x61, 0x3c, 0xf6, 0x18, 0x37, 0x86, 0x67, 0x06, 0x2e, - 0xec, 0x6d, 0x6c, 0xdd, 0x2f, 0x96, 0x0a, 0x3e, 0xd7, 0x3e, 0xa1, 0xfe, 0xa0, 0x54, 0x22, 0x42, - 0x05, 0xcd, 0xc1, 0x4b, 0x42, 0x78, 0xf8, 0xe0, 0x90, 0x28, 0x13, 0x83, 0x68, 0x1e, 0x90, 0x10, - 0x97, 0x0b, 0xfa, 0x5e, 0xb1, 0xb4, 0x51, 0x2e, 0x6c, 0xa7, 0x87, 0xf3, 0xff, 0xbc, 0x04, 0xd3, - 0x82, 0x65, 0x61, 0xac, 0x17, 0x3a, 0x80, 0x71, 0xce, 0x7c, 0xa1, 0x25, 0xc9, 0xd9, 0xdf, 0x45, - 0xca, 0xa9, 0x57, 0xfa, 0x68, 0xf0, 0x9d, 0x30, 0x84, 0x2a, 0x30, 0xe9, 0x27, 0xbc, 0xd0, 0x35, - 0x49, 0x3f, 0x0d, 0x12, 0x6c, 0xea, 0x72, 0x94, 0x9a, 0xe7, 0xa0, 0x0a, 0x53, 0x5d, 0xac, 0x15, - 0x92, 0x4c, 0x95, 0x71, 0x64, 0xea, 0xf5, 0x48, 0x3d, 0xbf, 0x8f, 0x2e, 0x52, 0x4a, 0xe6, 0x43, - 0x46, 0x76, 0xc9, 0x7c, 0xc8, 0xd9, 0xad, 0x21, 0xf4, 0xb9, 0x02, 0x99, 0x30, 0xde, 0x07, 0xad, - 0x0f, 0x4c, 0x52, 0xa9, 0xf9, 0x41, 0xa6, 0xf0, 0x6d, 0xdf, 0x02, 0x14, 0xe4, 0x7a, 0xd0, 0x6b, - 0x11, 0x96, 0xfc, 0xe4, 0x93, 0x7a, 0x23, 0x9e, 0x32, 0x77, 0xf8, 0x19, 0xcc, 0xcb, 0x29, 0x1c, - 0x94, 0x0b, 0x5b, 0x9d, 0x10, 0x2a, 0x47, 0xbd, 0x15, 0x7f, 0x82, 0x97, 0xf3, 0xcf, 0x60, 0x5e, - 0xce, 0xe5, 0xc8, 0xdc, 0xf7, 0x65, 0x92, 0x64, 0xee, 0x23, 0x68, 0xa2, 0x21, 0xf4, 0x65, 0xe8, - 0x49, 0xef, 0x43, 0x72, 0x27, 0x2c, 0xae, 0x48, 0x2e, 0x46, 0xbd, 0x7b, 0x96, 0xa9, 0x5d, 0xe8, - 0xa2, 0x78, 0x19, 0x19, 0xba, 0x98, 0x4c, 0x91, 0x0c, 0x5d, 0x6c, 0x1a, 0x68, 0x08, 0x3d, 0x86, - 0x94, 0x8f, 0xba, 0x41, 0x57, 0xe5, 0xfd, 0xa2, 0x9b, 0x27, 0x52, 0xaf, 0x45, 0x68, 0xf9, 0xbb, - 0x96, 0xbf, 0x78, 0x64, 0x5d, 0x4b, 0xc2, 0xf6, 0xa8, 0xcb, 0x51, 0x6a, 0x7e, 0x07, 0xfe, 0xf2, - 0x90, 0x39, 0x90, 0x70, 0x37, 0xea, 0x72, 0x94, 0x9a, 0xe7, 0xe0, 0x19, 0xcc, 0x49, 0xe9, 0x14, - 0xb4, 0x26, 0xcf, 0x41, 0x18, 0xb3, 0xa3, 0xe6, 0x62, 0xeb, 0xf3, 0x3d, 0xfd, 0xa9, 0xd8, 0xd3, - 0x81, 0x9f, 0x7d, 0xe4, 0xe2, 0xd6, 0x63, 0xe4, 0x9e, 0x0e, 0x65, 0x4f, 0x3e, 0x15, 0x3b, 0x3a, - 0x8e, 0xf3, 0xbe, 0xb4, 0x48, 0xf8, 0x8e, 0x0e, 0x75, 0xfe, 0x13, 0x45, 0xd0, 0xf2, 0x12, 0x36, - 0x02, 0x85, 0x36, 0xe4, 0x70, 0x8a, 0x44, 0x7d, 0x7d, 0xa0, 0x39, 0x1c, 0xc6, 0xcf, 0x15, 0x58, - 0x8c, 0xa0, 0x14, 0xd0, 0xed, 0xb8, 0x99, 0xed, 0xbd, 0x39, 0xab, 0x77, 0xce, 0x30, 0xd3, 0x07, - 0x2c, 0x82, 0x23, 0x90, 0x01, 0x8b, 0x47, 0x79, 0xc8, 0x80, 0xc5, 0x24, 0x24, 0x28, 0xb0, 0x88, - 0xcb, 0xbd, 0x0c, 0x58, 0xbc, 0x6f, 0x0d, 0xf5, 0xce, 0x19, 0x66, 0x72, 0x60, 0x3f, 0x53, 0xe0, - 0x72, 0x3f, 0xe6, 0x02, 0xbd, 0x11, 0x77, 0x35, 0xba, 0x18, 0x0c, 0xf5, 0xcd, 0x41, 0xa7, 0xf9, - 0xf0, 0xf4, 0x23, 0x2c, 0x64, 0x78, 0x62, 0x30, 0x2a, 0x32, 0x3c, 0x71, 0x78, 0x11, 0xf4, 0x1b, - 0x25, 0xec, 0x7f, 0x31, 0x82, 0x1f, 0xd8, 0x68, 0x23, 0x6e, 0xd0, 0xa1, 0xec, 0x82, 0xba, 0x79, - 0x1e, 0x13, 0x3e, 0xcc, 0x71, 0x49, 0x01, 0x19, 0xe6, 0x01, 0x19, 0x11, 0x19, 0xe6, 0x41, 0x39, - 0x09, 0xda, 0xd9, 0x42, 0x09, 0x01, 0x59, 0x67, 0x8b, 0x22, 0x30, 0x64, 0x9d, 0x2d, 0x92, 0x71, - 0xa0, 0x30, 0x42, 0x59, 0x00, 0x19, 0x8c, 0x28, 0x7e, 0x42, 0x06, 0x23, 0x92, 0x66, 0x20, 0xd7, - 0xe4, 0xe0, 0x27, 0xbc, 0xec, 0x9a, 0x1c, 0xca, 0x02, 0xc8, 0xae, 0xc9, 0x7d, 0x58, 0x81, 0x02, - 0x8c, 0x90, 0xcf, 0x73, 0x24, 0xf9, 0xa9, 0x9c, 0xef, 0x23, 0x5f, 0xcd, 0x86, 0xbd, 0x66, 0x66, - 0x36, 0x1f, 0x7e, 0xfd, 0x4d, 0x56, 0xf9, 0xcb, 0x37, 0xd9, 0xa1, 0x1f, 0xbf, 0xc8, 0x2a, 0x5f, - 0xbf, 0xc8, 0x2a, 0x7f, 0x7a, 0x91, 0x55, 0xfe, 0xf6, 0x22, 0xab, 0x7c, 0xf1, 0xf7, 0xec, 0xd0, - 0xa3, 0xdb, 0xb1, 0xff, 0x0a, 0x80, 0x39, 0xf1, 0xfe, 0x10, 0xa0, 0x3a, 0x46, 0x7f, 0xcc, 0xfb, - 0xfa, 0xbf, 0x02, 0x00, 0x00, 0xff, 0xff, 0x0e, 0x1f, 0xc1, 0x2e, 0x94, 0x30, 0x00, 0x00, + // 2835 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x5b, 0xcd, 0x6f, 0x1b, 0xc7, + 0x15, 0xd7, 0x52, 0x9f, 0x7c, 0x94, 0x64, 0x66, 0x2c, 0x29, 0xf4, 0xda, 0xa2, 0xe5, 0x8d, 0x2d, + 0x2b, 0x8a, 0x2d, 0x5a, 0x4c, 0x93, 0xd8, 0x6e, 0x93, 0x86, 0x92, 0x68, 0x99, 0x88, 0x44, 0x31, + 0x2b, 0xca, 0x69, 0x52, 0x17, 0xec, 0x92, 0x1c, 0xc9, 0x5b, 0xf3, 0x2b, 0xbb, 0x4b, 0xc5, 0x4a, + 0x10, 0xa0, 0x01, 0x7a, 0xe9, 0xa1, 0x40, 0x4e, 0x3d, 0x34, 0xa7, 0xa2, 0xff, 0x40, 0x0b, 0xf4, + 0xd2, 0xa0, 0x40, 0xaf, 0x41, 0x0f, 0x45, 0x6f, 0xed, 0xa1, 0x87, 0xc6, 0x05, 0x7a, 0xec, 0xdf, + 0x50, 0xcc, 0xd7, 0x6a, 0xc9, 0x9d, 0xe5, 0x2e, 0xa5, 0x36, 0x68, 0x4f, 0xde, 0x7d, 0xfb, 0xe6, + 0xbd, 0xdf, 0x7b, 0xf3, 0xe6, 0xcd, 0xcc, 0x8f, 0x32, 0xc4, 0x8d, 0x8e, 0xb9, 0xd6, 0xb1, 0xda, + 0x4e, 0x1b, 0x25, 0x9b, 0x46, 0xed, 0x89, 0xd9, 0xc2, 0x6b, 0xc7, 0xeb, 0x46, 0xa3, 0xf3, 0xc4, + 0x58, 0x57, 0x6f, 0x1f, 0x99, 0xce, 0x93, 0x6e, 0x75, 0xad, 0xd6, 0x6e, 0x66, 0x8e, 0xda, 0x47, + 0xed, 0x0c, 0x55, 0xac, 0x76, 0x0f, 0xe9, 0x1b, 0x7d, 0xa1, 0x4f, 0xcc, 0x80, 0xfa, 0x5d, 0x8f, + 0x7a, 0xbb, 0xd5, 0xc4, 0x8e, 0xd1, 0x10, 0xff, 0xde, 0x36, 0x3a, 0x66, 0xa6, 0x6d, 0x99, 0x19, + 0xa3, 0x63, 0xda, 0x19, 0x22, 0xca, 0x08, 0x2f, 0x19, 0x17, 0x81, 0xf6, 0xa5, 0x02, 0xd3, 0x8f, + 0xda, 0x8d, 0x6e, 0x13, 0x3f, 0x30, 0x1b, 0x0e, 0xb6, 0xd0, 0x2c, 0xc4, 0xcc, 0x7a, 0x4a, 0x59, + 0x52, 0x56, 0xe2, 0x7a, 0xcc, 0xac, 0xa3, 0xef, 0xc1, 0x6c, 0xc3, 0xa8, 0xe2, 0x46, 0xc5, 0xc6, + 0x0d, 0x5c, 0x73, 0xda, 0x56, 0x2a, 0xb6, 0x34, 0xba, 0x92, 0xc8, 0xae, 0xaf, 0xf5, 0x63, 0x5f, + 0xf3, 0xda, 0x59, 0xdb, 0x21, 0x83, 0xf6, 0xf9, 0x98, 0x7c, 0xcb, 0xb1, 0x4e, 0xf4, 0x99, 0x86, + 0x57, 0xa6, 0xbe, 0x0d, 0xc8, 0xaf, 0x84, 0x92, 0x30, 0xfa, 0x14, 0x9f, 0x70, 0x00, 0xe4, 0x11, + 0xcd, 0xc1, 0xf8, 0xb1, 0xd1, 0xe8, 0xe2, 0x54, 0x8c, 0xca, 0xd8, 0xcb, 0xfd, 0xd8, 0x5d, 0x45, + 0xfb, 0x4b, 0x0c, 0x80, 0x39, 0xdd, 0xef, 0xe0, 0x1a, 0x5a, 0x80, 0x89, 0xba, 0x65, 0x1e, 0x63, + 0x8b, 0x8f, 0xe6, 0x6f, 0x44, 0xfe, 0xc4, 0x68, 0xd5, 0x1b, 0xc2, 0x02, 0x7f, 0x43, 0x3b, 0x00, + 0x86, 0xe3, 0x58, 0x66, 0xb5, 0xeb, 0x60, 0x3b, 0x35, 0x4a, 0xc3, 0xba, 0x15, 0x14, 0x16, 0xf1, + 0xb0, 0x96, 0x73, 0xd5, 0x59, 0x44, 0x9e, 0xf1, 0x68, 0x17, 0x12, 0x36, 0xae, 0x59, 0xd8, 0xa9, + 0xd4, 0x0d, 0xc7, 0x48, 0x8d, 0x45, 0x30, 0xb7, 0x4f, 0xf5, 0xb7, 0x0c, 0xc7, 0xe0, 0xe6, 0x6c, + 0x57, 0xa0, 0xbe, 0x09, 0x17, 0xfa, 0xbc, 0x0d, 0x93, 0x1a, 0x32, 0xbc, 0xcf, 0x7a, 0xd8, 0xf0, + 0x69, 0x6f, 0x66, 0xbb, 0x30, 0xc1, 0x70, 0xa2, 0x7b, 0x30, 0x45, 0x6a, 0x87, 0xc6, 0x44, 0x86, + 0x26, 0xb2, 0x8b, 0x6b, 0x44, 0x70, 0x1a, 0xd0, 0x5e, 0xf5, 0x47, 0xb8, 0xe6, 0xec, 0x72, 0x25, + 0xdd, 0x55, 0x47, 0x77, 0x60, 0xcc, 0xee, 0xe0, 0x1a, 0xb5, 0x9e, 0xc8, 0x5e, 0x19, 0x94, 0x0a, + 0x9d, 0x6a, 0x6a, 0x7f, 0x52, 0x60, 0xa1, 0x88, 0x9d, 0x8f, 0xda, 0xd6, 0xd3, 0x42, 0xcb, 0xc1, + 0xd6, 0xa1, 0x51, 0x0b, 0xaa, 0xcb, 0x6a, 0x40, 0x5d, 0x7e, 0xdb, 0xef, 0x46, 0x6e, 0xf1, 0x1b, + 0xa9, 0xd0, 0x1b, 0x90, 0xe0, 0xde, 0x45, 0x85, 0xf2, 0x4a, 0x54, 0xbc, 0x95, 0xa8, 0x5d, 0x85, + 0x99, 0x47, 0xa6, 0xe5, 0x74, 0x8d, 0x46, 0xa1, 0x44, 0x15, 0x49, 0xb4, 0x1d, 0x37, 0xda, 0x8e, + 0xb6, 0x0d, 0x93, 0xc5, 0x5c, 0x59, 0xf6, 0x09, 0x21, 0x18, 0xeb, 0xb4, 0x2d, 0x87, 0xfa, 0x1e, + 0xd7, 0xe9, 0x33, 0x4a, 0xc1, 0x24, 0x6e, 0xd5, 0x4b, 0x44, 0x3c, 0x4a, 0xc5, 0xe2, 0x55, 0x7b, + 0x06, 0xc9, 0x9d, 0xb6, 0x51, 0xdf, 0x30, 0x1a, 0x46, 0xab, 0x86, 0x2d, 0x22, 0x43, 0xaf, 0xc3, + 0x14, 0x6d, 0x06, 0xb5, 0x76, 0x83, 0xda, 0x9d, 0xcd, 0xaa, 0xfe, 0x24, 0x96, 0xb8, 0x86, 0xee, + 0xea, 0x0e, 0xe9, 0xf9, 0xb7, 0x0a, 0x2c, 0x78, 0x5d, 0x97, 0x0d, 0xeb, 0x08, 0x3b, 0x34, 0xa4, + 0x12, 0xa0, 0x46, 0xdb, 0xa8, 0x57, 0xaa, 0xfc, 0x53, 0xc5, 0x39, 0xe9, 0x60, 0x0e, 0x45, 0xf3, + 0x43, 0xe9, 0xb1, 0x72, 0xd2, 0xc1, 0x7a, 0xb2, 0xd1, 0x27, 0xe1, 0x49, 0x8a, 0xb9, 0x49, 0xba, + 0x0b, 0xe3, 0x04, 0x9e, 0x58, 0xe5, 0x21, 0x46, 0x09, 0x5e, 0x9d, 0x0d, 0xd0, 0xfe, 0x18, 0x83, + 0xb9, 0xfe, 0x02, 0xa2, 0xa0, 0xdf, 0x80, 0xc9, 0x16, 0x93, 0x9f, 0xae, 0x8b, 0x80, 0xca, 0xa3, + 0x15, 0x2e, 0xb4, 0x49, 0xfd, 0x98, 0x1d, 0x9b, 0x96, 0x6b, 0x5c, 0x27, 0x8f, 0xe8, 0x2d, 0x80, + 0x63, 0x36, 0xfd, 0x15, 0xb3, 0x43, 0xf3, 0x96, 0xc8, 0x5e, 0x95, 0x2c, 0x17, 0x6f, 0x89, 0xe8, + 0x71, 0x3e, 0xa4, 0xd0, 0x41, 0x2a, 0x99, 0x40, 0x7c, 0x68, 0x3e, 0xc3, 0x36, 0xed, 0x3b, 0x71, + 0xdd, 0x7d, 0x47, 0x8f, 0x61, 0xbe, 0x2f, 0xb7, 0x34, 0xef, 0x76, 0x6a, 0x9c, 0x66, 0x62, 0x25, + 0x24, 0xbd, 0xee, 0x24, 0xe9, 0x17, 0x1b, 0x3e, 0xb9, 0x8d, 0x6e, 0xc3, 0x58, 0xcb, 0x70, 0xec, + 0xd4, 0x04, 0x35, 0x76, 0x49, 0x92, 0x01, 0x56, 0xb5, 0x3a, 0x55, 0xd3, 0x7e, 0xaa, 0x40, 0xb2, + 0x3f, 0x99, 0xe7, 0xe9, 0x30, 0xf7, 0x7b, 0x3a, 0xcc, 0x72, 0xf8, 0xd2, 0xf7, 0xf4, 0x1a, 0x0d, + 0xa6, 0x0b, 0x47, 0x2d, 0xd3, 0x31, 0xdb, 0x2d, 0x3a, 0x9f, 0x08, 0xc6, 0x5c, 0x08, 0xd3, 0x3a, + 0x7d, 0xd6, 0x7e, 0xaf, 0xc0, 0xcc, 0x2e, 0xb3, 0x19, 0xd0, 0x86, 0xde, 0x0f, 0x68, 0x43, 0x59, + 0x3f, 0x96, 0x1e, 0x43, 0xdf, 0x48, 0xf7, 0x79, 0x0c, 0x29, 0xee, 0x74, 0xb3, 0x61, 0xd8, 0xf6, + 0xa6, 0xd1, 0x31, 0xaa, 0x66, 0xc3, 0x74, 0x4c, 0x6c, 0xa3, 0x45, 0x80, 0x5a, 0xa7, 0x5b, 0x69, + 0x9a, 0x8d, 0x86, 0x69, 0x53, 0x73, 0xa3, 0x7a, 0xbc, 0xd6, 0xe9, 0xee, 0x52, 0x01, 0xba, 0x06, + 0xd3, 0x4d, 0xdc, 0x6c, 0x5b, 0x27, 0x95, 0xea, 0x09, 0xd9, 0x1d, 0x89, 0xed, 0x31, 0x3d, 0xc1, + 0x64, 0x1b, 0x44, 0xa4, 0xfd, 0x5a, 0x81, 0x49, 0x6e, 0xfe, 0x3c, 0x73, 0xb8, 0xde, 0x33, 0x87, + 0x8b, 0x81, 0x79, 0x3b, 0x9d, 0x3a, 0xf4, 0x06, 0x4c, 0xd8, 0x8e, 0xe1, 0x74, 0xed, 0xe0, 0xb5, + 0x22, 0x06, 0x51, 0x35, 0x9d, 0xab, 0x6b, 0xd7, 0x20, 0x5e, 0x68, 0x1a, 0x47, 0x6c, 0x01, 0xcf, + 0xc1, 0xb8, 0x49, 0x5e, 0x78, 0x2e, 0xd9, 0x8b, 0xb6, 0x06, 0x33, 0xf9, 0x66, 0xc7, 0x39, 0xd9, + 0x32, 0x6d, 0xd6, 0xb3, 0x17, 0x01, 0x6c, 0xf3, 0x63, 0xcc, 0xf3, 0xa0, 0xd0, 0x3c, 0xc4, 0x89, + 0x84, 0x65, 0xe1, 0x0b, 0x05, 0x92, 0x6c, 0x1f, 0xcb, 0x39, 0x8e, 0x51, 0x7b, 0xd2, 0xc4, 0x2d, + 0x87, 0xd4, 0x52, 0xcb, 0x68, 0x0a, 0xcb, 0xf4, 0x99, 0x9e, 0x4e, 0xf0, 0xb1, 0x59, 0x73, 0x4f, + 0x21, 0xec, 0x0d, 0x5d, 0x86, 0xf8, 0x31, 0x1d, 0x5f, 0x31, 0xeb, 0x34, 0x9e, 0xb8, 0x3e, 0xc5, + 0x04, 0x85, 0x3a, 0xe9, 0x0c, 0x98, 0xa0, 0xa9, 0xd4, 0x4d, 0xfb, 0x69, 0x6a, 0x2c, 0x28, 0xda, + 0x1e, 0xc4, 0x7a, 0x1c, 0x8b, 0x57, 0xad, 0x0a, 0x6a, 0xff, 0x12, 0x08, 0x81, 0x79, 0x07, 0xe6, + 0x78, 0xa3, 0xaa, 0x98, 0x62, 0x08, 0x41, 0xc6, 0x40, 0xa3, 0x56, 0x9f, 0xb5, 0x42, 0x5d, 0xfb, + 0x5b, 0x0c, 0x12, 0x9e, 0x39, 0x42, 0xb7, 0x49, 0xaf, 0xfd, 0x88, 0x9f, 0xc2, 0x66, 0xb3, 0x2f, + 0x4a, 0xf6, 0x12, 0xf2, 0x59, 0x67, 0x5a, 0x68, 0x5d, 0x4c, 0x03, 0x2b, 0x80, 0xcb, 0x7e, 0x75, + 0x77, 0xca, 0xf8, 0x1c, 0x91, 0x99, 0xab, 0x91, 0x82, 0xe6, 0xe9, 0x62, 0x2f, 0xe8, 0x3e, 0x4c, + 0x99, 0x7c, 0x41, 0xf3, 0x4c, 0xa5, 0x25, 0xb6, 0x3c, 0x4b, 0x5e, 0x77, 0xf5, 0xd1, 0x77, 0x60, + 0x92, 0xe5, 0x5c, 0xf4, 0x45, 0x2d, 0xe8, 0xb4, 0x72, 0x9a, 0x3e, 0x5d, 0x0c, 0x41, 0xdf, 0x07, + 0xe4, 0xcb, 0x99, 0xe8, 0x89, 0xb7, 0xc2, 0x9b, 0x92, 0xc7, 0xe4, 0x0b, 0xfd, 0xf9, 0xb5, 0xb5, + 0x2f, 0x63, 0x6e, 0x0f, 0x62, 0xd5, 0x8c, 0x32, 0x70, 0xb1, 0x5d, 0xb5, 0xb1, 0x75, 0x8c, 0xeb, + 0x95, 0x23, 0xdc, 0xc2, 0x96, 0x41, 0x63, 0x66, 0x6b, 0x18, 0x89, 0x4f, 0xdb, 0xee, 0x17, 0xf4, + 0x2d, 0x18, 0x27, 0x0b, 0x80, 0xa5, 0x78, 0x56, 0x96, 0x16, 0x8f, 0x03, 0xac, 0x33, 0x65, 0x52, + 0x98, 0x34, 0xdd, 0x15, 0x0b, 0x1f, 0x8a, 0xc2, 0xa4, 0x02, 0x1d, 0x1f, 0xa2, 0x8d, 0xd3, 0x84, + 0x8d, 0x05, 0x6d, 0x24, 0xfd, 0x09, 0xe3, 0x8b, 0xd1, 0x4d, 0x9b, 0x21, 0x4d, 0xdb, 0x78, 0x50, + 0xff, 0x0c, 0x4e, 0x1b, 0x37, 0x2c, 0x49, 0xde, 0xe7, 0x0a, 0x2c, 0xc8, 0x61, 0x48, 0x8b, 0xff, + 0x25, 0x98, 0xe1, 0x6b, 0xb1, 0xe7, 0xc2, 0x30, 0xcd, 0x84, 0x0f, 0xd9, 0xb5, 0xe1, 0x4d, 0x91, + 0xcd, 0x51, 0x9a, 0xcd, 0x9b, 0xd1, 0x02, 0x17, 0x69, 0xd5, 0x7e, 0xa7, 0xc0, 0x52, 0x58, 0x28, + 0x52, 0x70, 0x77, 0x21, 0xe5, 0x5f, 0x99, 0x3d, 0x38, 0x17, 0xfa, 0x13, 0xc0, 0x11, 0x6f, 0xf7, + 0x22, 0x5e, 0x1f, 0x36, 0xb7, 0x2e, 0x76, 0x0b, 0xa6, 0xbd, 0x1b, 0x8a, 0x14, 0x66, 0x11, 0xa6, + 0x6b, 0x9e, 0x8d, 0x86, 0x2f, 0xeb, 0xd5, 0xc0, 0x9a, 0xf3, 0x6d, 0x4d, 0x7a, 0xcf, 0x78, 0x6d, + 0x15, 0x66, 0x1f, 0x61, 0xcb, 0x36, 0xdb, 0x2d, 0x1d, 0x7f, 0xd8, 0xc5, 0x36, 0x3d, 0x63, 0x1e, + 0x33, 0x09, 0x77, 0x2c, 0x5e, 0xb5, 0x1f, 0xc0, 0x05, 0x57, 0xd7, 0xee, 0xb4, 0x5b, 0x36, 0x26, + 0x1b, 0x99, 0xd5, 0x6d, 0x39, 0x66, 0x13, 0x57, 0x3c, 0x50, 0x13, 0x5c, 0x56, 0x24, 0x88, 0x6f, + 0xc2, 0x05, 0xa1, 0x22, 0xec, 0xb2, 0x7c, 0xce, 0x72, 0x31, 0xb7, 0xa9, 0x15, 0xe1, 0xe2, 0x8e, + 0x69, 0x3b, 0x1c, 0xb8, 0x2d, 0xf0, 0xbc, 0x01, 0x13, 0x87, 0x74, 0x53, 0xe7, 0x5b, 0xdf, 0xd5, + 0x90, 0xbd, 0x5f, 0xe7, 0xea, 0xda, 0x2e, 0xcc, 0xf5, 0xda, 0xe3, 0x98, 0x5f, 0x83, 0x29, 0x6e, + 0x81, 0x6c, 0x38, 0x01, 0x27, 0x2b, 0x3e, 0x4a, 0x77, 0x55, 0xb5, 0x77, 0x60, 0x6e, 0xd3, 0xc2, + 0x86, 0x83, 0xc5, 0x27, 0x8e, 0xef, 0x55, 0x98, 0xe4, 0x3a, 0x1c, 0xe0, 0x00, 0x6b, 0x42, 0x53, + 0xdb, 0x81, 0xf9, 0x3e, 0x63, 0x1c, 0xdc, 0x99, 0xac, 0xbd, 0x06, 0x73, 0x5b, 0xb8, 0x81, 0x7d, + 0xd0, 0x16, 0x01, 0xb8, 0x4a, 0xc5, 0x3d, 0x56, 0xc5, 0xb9, 0xa4, 0x50, 0xd7, 0x5e, 0x84, 0xf9, + 0xbe, 0x61, 0x0c, 0x84, 0xf6, 0x4f, 0x05, 0xae, 0x1e, 0x74, 0xea, 0xa7, 0xf0, 0x72, 0xad, 0x56, + 0xdb, 0xa1, 0xdd, 0xce, 0x8e, 0x66, 0x1b, 0xd5, 0x21, 0x61, 0x9c, 0x0e, 0xe2, 0xc7, 0xb6, 0x0d, + 0x7f, 0x2c, 0x21, 0x6e, 0xd6, 0x3c, 0x22, 0x76, 0x8c, 0xf3, 0x9a, 0x55, 0xdf, 0x82, 0x64, 0xbf, + 0xc2, 0x50, 0x47, 0x38, 0x0d, 0x96, 0x82, 0x01, 0xf0, 0x64, 0x98, 0x70, 0xa9, 0x47, 0x87, 0x6d, + 0xaf, 0xd1, 0xb2, 0xe0, 0x6e, 0xd6, 0xb1, 0x28, 0x9b, 0xb5, 0x76, 0x05, 0x54, 0x99, 0x2b, 0x0e, + 0xe4, 0x63, 0x58, 0x64, 0x35, 0xe3, 0xdb, 0x2a, 0xa3, 0x81, 0xb9, 0x0f, 0x13, 0xac, 0xd3, 0xf2, + 0xa6, 0x11, 0x65, 0x13, 0xe6, 0x23, 0xb4, 0x25, 0x48, 0x07, 0xf9, 0xe6, 0xe8, 0x74, 0x58, 0x64, + 0xc5, 0x74, 0x46, 0x74, 0xa2, 0xd9, 0xc5, 0x4e, 0x9b, 0x1d, 0xf1, 0x1a, 0x64, 0x93, 0x7b, 0xfd, + 0x95, 0x02, 0x37, 0x19, 0xb0, 0x01, 0xdb, 0x7e, 0x34, 0x00, 0xef, 0xc3, 0x0b, 0xbe, 0x0d, 0x80, + 0x67, 0x6a, 0xb8, 0x53, 0x46, 0xb2, 0x7f, 0x9f, 0xd0, 0x56, 0x61, 0x25, 0x1c, 0x24, 0x8f, 0xe8, + 0x31, 0xdc, 0x64, 0x31, 0x9f, 0x3b, 0x20, 0x59, 0x46, 0x57, 0x61, 0x25, 0xdc, 0x3a, 0x47, 0xb2, + 0x03, 0x88, 0xf4, 0x4f, 0x96, 0x7b, 0x77, 0xdd, 0xbf, 0xde, 0xd7, 0x8e, 0xd3, 0x83, 0x99, 0x4a, + 0xb7, 0x1b, 0x17, 0x58, 0x77, 0x77, 0xad, 0xf1, 0x7e, 0x97, 0x3d, 0x3d, 0xe9, 0xb0, 0x5e, 0x9c, + 0x0a, 0xb2, 0xe7, 0x9e, 0x6c, 0xb4, 0x6d, 0xb8, 0xe8, 0x2d, 0x46, 0x81, 0xec, 0x8e, 0x5b, 0xdf, + 0x0c, 0x59, 0xb0, 0x25, 0x51, 0xd5, 0x0f, 0x45, 0x4b, 0x17, 0x86, 0x38, 0xa8, 0xe1, 0x2d, 0x65, + 0xe1, 0xa2, 0xb7, 0x52, 0x05, 0xa4, 0x9e, 0xdb, 0x87, 0xd2, 0x7b, 0xfb, 0xd0, 0x16, 0x44, 0xd7, + 0xee, 0xf5, 0xae, 0xfd, 0x10, 0xae, 0x90, 0x4c, 0xf5, 0xcf, 0x90, 0x3b, 0x03, 0x6f, 0xf7, 0xcd, + 0xc0, 0x4a, 0x54, 0x4e, 0xce, 0x9d, 0x0b, 0x0b, 0x16, 0x03, 0x3c, 0xf0, 0x04, 0xbc, 0x2b, 0x3d, + 0x3b, 0x2a, 0x41, 0x67, 0xf7, 0x7e, 0x43, 0xb2, 0xb3, 0x62, 0x47, 0x74, 0x2f, 0x9f, 0x32, 0x0f, + 0x6b, 0x4f, 0xb6, 0xfe, 0x94, 0xa0, 0x4e, 0xe5, 0xb3, 0xe2, 0x5f, 0x75, 0x1f, 0x8a, 0x9e, 0xe5, + 0xf7, 0xc8, 0xc3, 0xfc, 0x8f, 0xbb, 0x7c, 0x57, 0x34, 0xc1, 0xa0, 0x20, 0x83, 0xee, 0x7f, 0x4a, + 0xe0, 0xfd, 0xcf, 0xed, 0x81, 0x41, 0x51, 0x68, 0x87, 0x62, 0x13, 0xeb, 0xd7, 0x28, 0x94, 0xec, + 0x33, 0xfb, 0xf5, 0xf3, 0x68, 0xda, 0x4b, 0x70, 0x6d, 0x80, 0x1f, 0x0e, 0xe6, 0x17, 0x0a, 0x2c, + 0xcb, 0xb3, 0xee, 0xf2, 0x6b, 0x67, 0xc7, 0xd4, 0xcb, 0xe4, 0xc5, 0x86, 0x65, 0xf2, 0xb4, 0x97, + 0x83, 0x36, 0x0b, 0x0f, 0x36, 0x4f, 0x1c, 0xf2, 0x68, 0xff, 0x37, 0xe2, 0x08, 0xc5, 0xc6, 0xe3, + 0x68, 0xc3, 0x4b, 0xf2, 0x90, 0x4b, 0x94, 0xc2, 0x3c, 0x7b, 0x0c, 0x0b, 0x30, 0xc1, 0x58, 0x50, + 0x41, 0xb8, 0xb0, 0x37, 0x6d, 0x19, 0xae, 0x0f, 0x76, 0x78, 0x0a, 0x4c, 0x5e, 0xd7, 0xff, 0x45, + 0x60, 0x83, 0x1d, 0x72, 0x60, 0x7f, 0x50, 0x20, 0x23, 0x8f, 0xc0, 0x4f, 0xdd, 0x9e, 0x1d, 0xe5, + 0x07, 0x30, 0x27, 0x23, 0x8e, 0x79, 0x31, 0x44, 0xe7, 0x8d, 0x91, 0x9f, 0x37, 0xd6, 0xb2, 0x70, + 0x27, 0x7a, 0x00, 0x9e, 0xa8, 0xe5, 0xe9, 0xf9, 0x3f, 0x8a, 0x3a, 0x7a, 0x00, 0x3c, 0xea, 0xcf, + 0x14, 0x58, 0x92, 0xa7, 0xaa, 0x98, 0x2b, 0x9f, 0x3d, 0xcc, 0x57, 0x60, 0xb4, 0x65, 0x88, 0xa8, + 0x06, 0xd0, 0xf6, 0x44, 0x8b, 0xb4, 0xd5, 0x01, 0x10, 0x38, 0xd0, 0xa7, 0xb0, 0x24, 0x0f, 0xee, + 0x5c, 0x38, 0xe7, 0x61, 0xa2, 0x65, 0x38, 0x15, 0xf7, 0xb7, 0x9c, 0xf1, 0x96, 0xe1, 0x14, 0x3a, + 0x04, 0xd1, 0x00, 0x67, 0x1c, 0xd1, 0x07, 0xb0, 0x2c, 0x57, 0x3a, 0x7f, 0x7f, 0x24, 0xfd, 0x2d, + 0xd4, 0x36, 0x87, 0x71, 0x19, 0x2e, 0x79, 0x2e, 0xf9, 0x94, 0xed, 0x70, 0x4f, 0x4a, 0x1a, 0x06, + 0x55, 0xf6, 0x91, 0xef, 0xfe, 0xdb, 0x70, 0x41, 0x1c, 0x9f, 0x6b, 0xec, 0x13, 0x3f, 0xe1, 0xa4, + 0x07, 0xb3, 0x29, 0xfa, 0x6c, 0xb3, 0xc7, 0xa0, 0x76, 0x0b, 0x12, 0xf9, 0x67, 0xb8, 0x16, 0xf1, + 0xd6, 0xbd, 0x04, 0xd3, 0x4c, 0x9b, 0xc3, 0x48, 0xc2, 0x68, 0xd7, 0x6a, 0x88, 0xfb, 0x6a, 0xd7, + 0x6a, 0xac, 0x2e, 0xc3, 0x94, 0xf8, 0x3d, 0x10, 0x4d, 0xc2, 0x68, 0x79, 0xb3, 0x94, 0x1c, 0x21, + 0x0f, 0x07, 0x5b, 0xa5, 0xa4, 0x82, 0xa6, 0x60, 0x6c, 0x7f, 0xb3, 0x5c, 0x4a, 0xc6, 0x56, 0x6f, + 0xf5, 0xfe, 0xda, 0x48, 0x7f, 0x9a, 0x03, 0x98, 0x28, 0x1d, 0x6c, 0xec, 0x14, 0x36, 0x93, 0x23, + 0x68, 0x1a, 0xa6, 0x0a, 0xc5, 0x72, 0x5e, 0x2f, 0xe6, 0x76, 0x92, 0xca, 0xea, 0x75, 0x18, 0xa7, + 0xf7, 0x49, 0x22, 0x2e, 0xed, 0xbd, 0x97, 0xd7, 0x2b, 0x7b, 0xc5, 0xe4, 0x08, 0x9a, 0x81, 0x38, + 0x7f, 0x7b, 0xf0, 0x20, 0xa9, 0xac, 0x1e, 0xc3, 0xbc, 0x94, 0x5f, 0x43, 0x8b, 0x70, 0xe9, 0xd1, + 0xde, 0xce, 0xc1, 0x6e, 0xbe, 0x92, 0x2b, 0x97, 0x73, 0x9b, 0x0f, 0x77, 0xf3, 0xc5, 0x72, 0xa5, + 0x94, 0x2f, 0x6e, 0x15, 0x8a, 0xdb, 0xc9, 0x11, 0x94, 0x06, 0xd5, 0xff, 0x99, 0x3d, 0xe6, 0xb7, + 0x92, 0x8a, 0xfc, 0xfb, 0x56, 0x9e, 0x7f, 0x8f, 0xad, 0xfe, 0x52, 0x81, 0xab, 0x21, 0x34, 0x19, + 0x5a, 0x81, 0xeb, 0xc5, 0x7c, 0xf9, 0xbd, 0x3d, 0xfd, 0x9d, 0x0a, 0x8d, 0xeb, 0x41, 0x6e, 0x33, + 0x00, 0xcd, 0xcb, 0x70, 0x63, 0xa0, 0xa6, 0x07, 0x58, 0x98, 0xaa, 0x07, 0xe3, 0x91, 0xcb, 0xcf, + 0x31, 0x3c, 0x17, 0xe1, 0xc2, 0x6e, 0x6e, 0xf3, 0x61, 0xa1, 0x98, 0xf7, 0xb8, 0xf6, 0x08, 0xf5, + 0x83, 0x62, 0x91, 0x08, 0x15, 0x34, 0x0f, 0x2f, 0x08, 0xe1, 0xfe, 0xc1, 0x3e, 0x51, 0x26, 0x06, + 0xd1, 0x02, 0x20, 0x21, 0x2e, 0xe7, 0xf5, 0xdd, 0x42, 0x31, 0x57, 0xce, 0x6f, 0x25, 0x47, 0xb3, + 0xff, 0xba, 0x0c, 0xb3, 0x82, 0x93, 0x61, 0x1c, 0x19, 0x2a, 0xc1, 0x24, 0xe7, 0xc9, 0xd0, 0x92, + 0xe4, 0xa4, 0xd0, 0x43, 0xe1, 0xa9, 0xd7, 0x06, 0x68, 0xf0, 0x75, 0x33, 0x82, 0x2a, 0x30, 0xed, + 0xa5, 0xc7, 0xd0, 0x0d, 0x49, 0xf7, 0xf5, 0xd3, 0x71, 0xea, 0x72, 0x98, 0x9a, 0xeb, 0xa0, 0x0a, + 0x33, 0x3d, 0x1c, 0x17, 0x92, 0x0c, 0x95, 0x31, 0x6a, 0xea, 0xcd, 0x50, 0x3d, 0xaf, 0x8f, 0x1e, + 0x0a, 0x4b, 0xe6, 0x43, 0x46, 0x8d, 0xc9, 0x7c, 0xc8, 0xb9, 0xb0, 0x11, 0xf4, 0x99, 0x02, 0xa9, + 0x20, 0x96, 0x08, 0xad, 0x0f, 0x4d, 0x69, 0xa9, 0xd9, 0x61, 0x86, 0xf0, 0x26, 0xd1, 0x06, 0xe4, + 0x67, 0x86, 0xd0, 0x2b, 0x21, 0x96, 0xbc, 0x54, 0x95, 0x7a, 0x2b, 0x9a, 0x32, 0x77, 0xf8, 0x29, + 0x2c, 0xc8, 0x09, 0x1f, 0x94, 0x09, 0x9a, 0x9d, 0x00, 0xe2, 0x47, 0xbd, 0x13, 0x7d, 0x80, 0x9b, + 0xf3, 0x4f, 0x61, 0x41, 0xce, 0xfc, 0xc8, 0xdc, 0x0f, 0xe4, 0x9d, 0x64, 0xee, 0x43, 0x48, 0xa5, + 0x11, 0xf4, 0x45, 0xe0, 0xb9, 0xc0, 0x83, 0xe4, 0x5e, 0x50, 0x5c, 0xa1, 0xcc, 0x8d, 0x7a, 0xff, + 0x2c, 0x43, 0x7b, 0xd0, 0x85, 0xb1, 0x38, 0x32, 0x74, 0x11, 0x79, 0x25, 0x19, 0xba, 0xc8, 0xa4, + 0xd1, 0x08, 0x7a, 0x0c, 0x09, 0x0f, 0xd1, 0x83, 0xae, 0xcb, 0xfb, 0x45, 0x2f, 0xab, 0xa4, 0xde, + 0x08, 0xd1, 0xf2, 0x76, 0x2d, 0x6f, 0xf1, 0xc8, 0xba, 0x96, 0x84, 0x1b, 0x52, 0x97, 0xc3, 0xd4, + 0xbc, 0x0e, 0xbc, 0xe5, 0x21, 0x73, 0x20, 0x61, 0x7a, 0xd4, 0xe5, 0x30, 0x35, 0xd7, 0xc1, 0x33, + 0x98, 0x97, 0x92, 0x2f, 0x68, 0x4d, 0x9e, 0x83, 0x20, 0x1e, 0x48, 0xcd, 0x44, 0xd6, 0xe7, 0x6b, + 0xfa, 0x13, 0xb1, 0xa6, 0x7d, 0x7f, 0x24, 0x92, 0x89, 0x5a, 0x8f, 0xa1, 0x6b, 0x3a, 0x90, 0x6b, + 0xf9, 0x44, 0xac, 0xe8, 0x28, 0xce, 0x07, 0x92, 0x28, 0xc1, 0x2b, 0x3a, 0xd0, 0xf9, 0x4f, 0x14, + 0x41, 0xe2, 0x4b, 0xb8, 0x0b, 0x14, 0xd8, 0x90, 0x83, 0x09, 0x15, 0xf5, 0xd5, 0xa1, 0xc6, 0x70, + 0x18, 0x3f, 0x57, 0xe0, 0x6a, 0x08, 0x01, 0x81, 0xee, 0x46, 0xcd, 0x6c, 0xff, 0x39, 0x5b, 0xbd, + 0x77, 0x86, 0x91, 0x1e, 0x60, 0x21, 0x8c, 0x82, 0x0c, 0x58, 0x34, 0x82, 0x44, 0x06, 0x2c, 0x22, + 0x7d, 0x41, 0x81, 0x85, 0x5c, 0x05, 0x64, 0xc0, 0xa2, 0xdd, 0x4c, 0xd4, 0x7b, 0x67, 0x18, 0xc9, + 0x81, 0xfd, 0x4c, 0x81, 0x2b, 0x83, 0x78, 0x0e, 0xf4, 0x5a, 0xd4, 0xd9, 0xe8, 0xe1, 0x3b, 0xd4, + 0xd7, 0x87, 0x1d, 0xe6, 0xc1, 0x33, 0x88, 0xde, 0x90, 0xe1, 0x89, 0xc0, 0xbf, 0xc8, 0xf0, 0x44, + 0x61, 0x51, 0xd0, 0x6f, 0x94, 0xa0, 0xdf, 0x3c, 0xfc, 0xd7, 0x71, 0x94, 0x8b, 0x1a, 0x74, 0x20, + 0x17, 0xa1, 0x6e, 0x9c, 0xc7, 0x84, 0x07, 0x73, 0x54, 0x0a, 0x41, 0x86, 0x79, 0x48, 0xfe, 0x44, + 0x86, 0x79, 0x58, 0x06, 0x83, 0x76, 0xb6, 0x40, 0xfa, 0x40, 0xd6, 0xd9, 0xc2, 0xe8, 0x0e, 0x59, + 0x67, 0x0b, 0xe5, 0x27, 0x28, 0x8c, 0x40, 0xce, 0x40, 0x06, 0x23, 0x8c, 0xcd, 0x90, 0xc1, 0x08, + 0x25, 0x25, 0xc8, 0x31, 0xd9, 0x7f, 0xe1, 0x97, 0x1d, 0x93, 0x03, 0x39, 0x03, 0xd9, 0x31, 0x79, + 0x00, 0x87, 0x90, 0x87, 0x31, 0x72, 0x99, 0x47, 0x92, 0x3f, 0xac, 0xf3, 0x50, 0x02, 0x6a, 0x3a, + 0xe8, 0x33, 0x33, 0xb3, 0xf1, 0xe8, 0xab, 0xaf, 0xd3, 0xca, 0x5f, 0xbf, 0x4e, 0x8f, 0xfc, 0xf8, + 0x79, 0x5a, 0xf9, 0xea, 0x79, 0x5a, 0xf9, 0xf3, 0xf3, 0xb4, 0xf2, 0xf7, 0xe7, 0x69, 0xe5, 0xf3, + 0x7f, 0xa4, 0x47, 0x3e, 0xb8, 0x1b, 0xf9, 0xbf, 0x21, 0x30, 0x27, 0xee, 0xff, 0x44, 0xa8, 0x4e, + 0xd0, 0xbf, 0x26, 0x7e, 0xf5, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x19, 0x19, 0x44, 0xa8, 0x15, + 0x31, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -5800,7 +5837,7 @@ func (m *LoadBalancerTargetSpec) MarshalToSizedBuffer(dAtA []byte) (int, error) i = encodeVarintApi(dAtA, i, uint64(size)) } i-- - dAtA[i] = 0x12 + dAtA[i] = 0x1a } } if len(m.Ip) > 0 { @@ -5808,7 +5845,12 @@ func (m *LoadBalancerTargetSpec) MarshalToSizedBuffer(dAtA []byte) (int, error) copy(dAtA[i:], m.Ip) i = encodeVarintApi(dAtA, i, uint64(len(m.Ip))) i-- - dAtA[i] = 0xa + dAtA[i] = 0x12 + } + if m.LoadBalancerType != 0 { + i = encodeVarintApi(dAtA, i, uint64(m.LoadBalancerType)) + i-- + dAtA[i] = 0x8 } return len(dAtA) - i, nil } @@ -8504,6 +8546,9 @@ func (m *LoadBalancerTargetSpec) Size() (n int) { } var l int _ = l + if m.LoadBalancerType != 0 { + n += 1 + sovApi(uint64(m.LoadBalancerType)) + } l = len(m.Ip) if l > 0 { n += 1 + l + sovApi(uint64(l)) @@ -9707,6 +9752,7 @@ func (this *LoadBalancerTargetSpec) String() string { } repeatedStringForPorts += "}" s := strings.Join([]string{`&LoadBalancerTargetSpec{`, + `LoadBalancerType:` + fmt.Sprintf("%v", this.LoadBalancerType) + `,`, `Ip:` + fmt.Sprintf("%v", this.Ip) + `,`, `Ports:` + repeatedStringForPorts + `,`, `}`, @@ -11855,6 +11901,25 @@ func (m *LoadBalancerTargetSpec) Unmarshal(dAtA []byte) error { } switch fieldNum { case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LoadBalancerType", wireType) + } + m.LoadBalancerType = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.LoadBalancerType |= LoadBalancerType(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Ip", wireType) } @@ -11886,7 +11951,7 @@ func (m *LoadBalancerTargetSpec) Unmarshal(dAtA []byte) error { } m.Ip = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 2: + case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Ports", wireType) } diff --git a/ori/apis/machine/v1alpha1/api.proto b/ori/apis/machine/v1alpha1/api.proto index 56df36d69..5c8675c31 100644 --- a/ori/apis/machine/v1alpha1/api.proto +++ b/ori/apis/machine/v1alpha1/api.proto @@ -97,6 +97,11 @@ enum Protocol { SCTP = 2; } +enum LoadBalancerType { + PUBLIC = 0; + INTERNAL = 1; +} + message LoadBalancerPort { Protocol protocol = 1; int32 port = 2; @@ -104,8 +109,9 @@ message LoadBalancerPort { } message LoadBalancerTargetSpec { - string ip = 1; - repeated LoadBalancerPort ports = 2; + LoadBalancerType load_balancer_type = 1; + string ip = 2; + repeated LoadBalancerPort ports = 3; } message NetworkInterfaceSpec { diff --git a/ori/apis/machine/v1alpha1/utils.go b/ori/apis/machine/v1alpha1/utils.go index bba3b270b..3af897137 100644 --- a/ori/apis/machine/v1alpha1/utils.go +++ b/ori/apis/machine/v1alpha1/utils.go @@ -25,5 +25,5 @@ func (tgt *LoadBalancerTargetSpec) Key() string { portStrings[i] = fmt.Sprintf("%s:%d-%d", port.Protocol.String(), port.Port, port.EndPort) } sort.Strings(portStrings) - return fmt.Sprintf("%s%v", tgt.Ip, portStrings) + return fmt.Sprintf("%s-%s%v", tgt.LoadBalancerType.String(), tgt.Ip, portStrings) } diff --git a/ori/apis/meta/v1alpha1/api.pb.go b/ori/apis/meta/v1alpha1/api.pb.go index 4ed35ff55..5790a7fb2 100644 --- a/ori/apis/meta/v1alpha1/api.pb.go +++ b/ori/apis/meta/v1alpha1/api.pb.go @@ -5,15 +5,14 @@ package v1alpha1 import ( fmt "fmt" + _ "github.com/gogo/protobuf/gogoproto" + proto "github.com/gogo/protobuf/proto" + github_com_gogo_protobuf_sortkeys "github.com/gogo/protobuf/sortkeys" io "io" math "math" math_bits "math/bits" reflect "reflect" strings "strings" - - _ "github.com/gogo/protobuf/gogoproto" - proto "github.com/gogo/protobuf/proto" - github_com_gogo_protobuf_sortkeys "github.com/gogo/protobuf/sortkeys" ) // Reference imports to suppress errors if they are not otherwise used. diff --git a/ori/apis/volume/v1alpha1/api.pb.go b/ori/apis/volume/v1alpha1/api.pb.go index b1dd00d55..7d02df3e7 100644 --- a/ori/apis/volume/v1alpha1/api.pb.go +++ b/ori/apis/volume/v1alpha1/api.pb.go @@ -6,12 +6,6 @@ package v1alpha1 import ( context "context" fmt "fmt" - io "io" - math "math" - math_bits "math/bits" - reflect "reflect" - strings "strings" - _ "github.com/gogo/protobuf/gogoproto" proto "github.com/gogo/protobuf/proto" github_com_gogo_protobuf_sortkeys "github.com/gogo/protobuf/sortkeys" @@ -19,6 +13,11 @@ import ( grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" + io "io" + math "math" + math_bits "math/bits" + reflect "reflect" + strings "strings" ) // Reference imports to suppress errors if they are not otherwise used. diff --git a/poollet/machinepoollet/controllers/machine_controller_networkinterface.go b/poollet/machinepoollet/controllers/machine_controller_networkinterface.go index f6d02a9b3..a62fca360 100644 --- a/poollet/machinepoollet/controllers/machine_controller_networkinterface.go +++ b/poollet/machinepoollet/controllers/machine_controller_networkinterface.go @@ -62,6 +62,17 @@ func (r *MachineReconciler) convertProtocol(protocol corev1.Protocol) (ori.Proto } } +func (r *MachineReconciler) convertLoadBalancerType(loadBalancerType networkingv1alpha1.LoadBalancerType) (ori.LoadBalancerType, error) { + switch loadBalancerType { + case networkingv1alpha1.LoadBalancerTypePublic: + return ori.LoadBalancerType_PUBLIC, nil + case networkingv1alpha1.LoadBalancerTypeInternal: + return ori.LoadBalancerType_INTERNAL, nil + default: + return 0, fmt.Errorf("unknown load balancer type %q", loadBalancerType) + } +} + func (r *MachineReconciler) convertLoadBalancerPorts(ports []networkingv1alpha1.LoadBalancerPort) ([]*ori.LoadBalancerPort, error) { res := make([]*ori.LoadBalancerPort, len(ports)) for i, port := range ports { @@ -687,14 +698,20 @@ func (r *MachineReconciler) loadBalancerTargetsForNetworkInterface( } for _, ip := range loadBalancer.Status.IPs { + loadBalancerType, err := r.convertLoadBalancerType(loadBalancer.Spec.Type) + if err != nil { + return nil, err + } + ports, err := r.convertLoadBalancerPorts(loadBalancer.Spec.Ports) if err != nil { return nil, err } loadBalancerTarget := &ori.LoadBalancerTargetSpec{ - Ip: ip.String(), - Ports: ports, + LoadBalancerType: loadBalancerType, + Ip: ip.String(), + Ports: ports, } loadBalancerTargetsByKey[loadBalancerTarget.Key()] = loadBalancerTarget