From a2e42d578dfb64c3d980b60321a96b330750e94e Mon Sep 17 00:00:00 2001 From: Leo Pang <34628052+allthatjazzleo@users.noreply.github.com> Date: Wed, 9 Oct 2024 05:40:42 +0800 Subject: [PATCH] feat: add clusterIP override for headless svc and add ClusterDomain(cluster.local) override (#435) Co-authored-by: Andrew Gouin <andrew@gouin.io> --- api/v1/cosmosfullnode_types.go | 9 +++++++++ api/v1/zz_generated.deepcopy.go | 10 ++++++++++ .../bases/cosmos.strange.love_cosmosfullnodes.yaml | 13 +++++++++++++ internal/fullnode/peer_collector.go | 7 ++++++- internal/fullnode/service_builder.go | 4 ++++ 5 files changed, 42 insertions(+), 1 deletion(-) diff --git a/api/v1/cosmosfullnode_types.go b/api/v1/cosmosfullnode_types.go index 24d13e2c..54a2c34d 100644 --- a/api/v1/cosmosfullnode_types.go +++ b/api/v1/cosmosfullnode_types.go @@ -730,6 +730,10 @@ type ServiceSpec struct { // Overrides for the single RPC service. // +optional RPCTemplate ServiceOverridesSpec `json:"rpcTemplate"` + + // Overrides for default cluster domain name. + // +optional + ClusterDomain *string `json:"clusterDomain"` } // ServiceOverridesSpec allows some overrides for the created, single RPC service. @@ -743,6 +747,11 @@ type ServiceOverridesSpec struct { // +optional Type *corev1.ServiceType `json:"type"` + // Setting this to "None" makes a "headless service" (no virtual IP), which is useful when direct endpoint connections are preferred and proxying is not required. + // If not set, defaults to "". + // +optional + ClusterIP *string `json:"clusterIP"` + // Sets endpoint and routing behavior. // See: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#caveats-and-limitations-when-preserving-source-ips // If not set, defaults to "Cluster". diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index d4f5a544..a2ea16cb 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -745,6 +745,11 @@ func (in *ServiceOverridesSpec) DeepCopyInto(out *ServiceOverridesSpec) { *out = new(corev1.ServiceType) **out = **in } + if in.ClusterIP != nil { + in, out := &in.ClusterIP, &out.ClusterIP + *out = new(string) + **out = **in + } if in.ExternalTrafficPolicy != nil { in, out := &in.ExternalTrafficPolicy, &out.ExternalTrafficPolicy *out = new(corev1.ServiceExternalTrafficPolicyType) @@ -772,6 +777,11 @@ func (in *ServiceSpec) DeepCopyInto(out *ServiceSpec) { } in.P2PTemplate.DeepCopyInto(&out.P2PTemplate) in.RPCTemplate.DeepCopyInto(&out.RPCTemplate) + if in.ClusterDomain != nil { + in, out := &in.ClusterDomain, &out.ClusterDomain + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceSpec. diff --git a/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml b/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml index 92fde62f..33bc8869 100644 --- a/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml +++ b/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml @@ -5674,6 +5674,9 @@ spec: This allows a k8s admin to use the service in an Ingress, for example. Additionally, multiple p2p services are created for CometBFT peer exchange. properties: + clusterDomain: + description: Overrides for default cluster domain name. + type: string maxP2PExternalAddresses: description: |- Max number of external p2p services to create for CometBFT peer exchange. @@ -5688,6 +5691,11 @@ spec: description: Overrides for all P2P services that need external addresses. properties: + clusterIP: + description: |- + Setting this to "None" makes a "headless service" (no virtual IP), which is useful when direct endpoint connections are preferred and proxying is not required. + If not set, defaults to "". + type: string externalTrafficPolicy: description: |- Sets endpoint and routing behavior. @@ -5729,6 +5737,11 @@ spec: rpcTemplate: description: Overrides for the single RPC service. properties: + clusterIP: + description: |- + Setting this to "None" makes a "headless service" (no virtual IP), which is useful when direct endpoint connections are preferred and proxying is not required. + If not set, defaults to "". + type: string externalTrafficPolicy: description: |- Sets endpoint and routing behavior. diff --git a/internal/fullnode/peer_collector.go b/internal/fullnode/peer_collector.go index 75426d50..37c27c3a 100644 --- a/internal/fullnode/peer_collector.go +++ b/internal/fullnode/peer_collector.go @@ -105,6 +105,11 @@ func NewPeerCollector(client Getter) *PeerCollector { // Collect peer information given the crd. func (c PeerCollector) Collect(ctx context.Context, crd *cosmosv1.CosmosFullNode) (Peers, kube.ReconcileError) { peers := make(Peers) + + clusterDomain := "cluster.local" + if crd.Spec.Service.ClusterDomain != nil { + clusterDomain = *crd.Spec.Service.ClusterDomain + } for i := int32(0); i < crd.Spec.Replicas; i++ { secretName := nodeKeySecretName(crd, i) var secret corev1.Secret @@ -121,7 +126,7 @@ func (c PeerCollector) Collect(ctx context.Context, crd *cosmosv1.CosmosFullNode svcName := p2pServiceName(crd, i) peers[c.objectKey(crd, i)] = Peer{ NodeID: nodeKey.ID(), - PrivateAddress: fmt.Sprintf("%s.%s.svc.cluster.local:%d", svcName, secret.Namespace, p2pPort), + PrivateAddress: fmt.Sprintf("%s.%s.svc.%s:%d", svcName, secret.Namespace, clusterDomain, p2pPort), } if err := c.addExternalAddress(ctx, peers, crd, i); err != nil { return nil, kube.TransientError(err) diff --git a/internal/fullnode/service_builder.go b/internal/fullnode/service_builder.go index 6f880a0a..2831a12a 100644 --- a/internal/fullnode/service_builder.go +++ b/internal/fullnode/service_builder.go @@ -63,6 +63,7 @@ func BuildServices(crd *cosmosv1.CosmosFullNode) []diff.Resource[*corev1.Service svc.Spec.ExternalTrafficPolicy = *valOrDefault(crd.Spec.Service.P2PTemplate.ExternalTrafficPolicy, ptr(corev1.ServiceExternalTrafficPolicyTypeLocal)) } else { svc.Spec.Type = corev1.ServiceTypeClusterIP + svc.Spec.ClusterIP = *valOrDefault(crd.Spec.Service.P2PTemplate.ClusterIP, ptr("")) } p2ps[i] = diff.Adapt(&svc, i) @@ -131,6 +132,9 @@ func rpcService(crd *cosmosv1.CosmosFullNode) *corev1.Service { if v := rpcSpec.Type; v != nil { svc.Spec.Type = *v } + if v := rpcSpec.ClusterIP; v != nil { + svc.Spec.ClusterIP = *v + } return &svc }