From a2e42d578dfb64c3d980b60321a96b330750e94e Mon Sep 17 00:00:00 2001
From: Leo Pang <34628052+allthatjazzleo@users.noreply.github.com>
Date: Wed, 9 Oct 2024 05:40:42 +0800
Subject: [PATCH] feat: add clusterIP override for headless svc and add
 ClusterDomain(cluster.local) override (#435)

Co-authored-by: Andrew Gouin <andrew@gouin.io>
---
 api/v1/cosmosfullnode_types.go                      |  9 +++++++++
 api/v1/zz_generated.deepcopy.go                     | 10 ++++++++++
 .../bases/cosmos.strange.love_cosmosfullnodes.yaml  | 13 +++++++++++++
 internal/fullnode/peer_collector.go                 |  7 ++++++-
 internal/fullnode/service_builder.go                |  4 ++++
 5 files changed, 42 insertions(+), 1 deletion(-)

diff --git a/api/v1/cosmosfullnode_types.go b/api/v1/cosmosfullnode_types.go
index 24d13e2c..54a2c34d 100644
--- a/api/v1/cosmosfullnode_types.go
+++ b/api/v1/cosmosfullnode_types.go
@@ -730,6 +730,10 @@ type ServiceSpec struct {
 	// Overrides for the single RPC service.
 	// +optional
 	RPCTemplate ServiceOverridesSpec `json:"rpcTemplate"`
+
+	// Overrides for default cluster domain name.
+	// +optional
+	ClusterDomain *string `json:"clusterDomain"`
 }
 
 // ServiceOverridesSpec allows some overrides for the created, single RPC service.
@@ -743,6 +747,11 @@ type ServiceOverridesSpec struct {
 	// +optional
 	Type *corev1.ServiceType `json:"type"`
 
+	// Setting this to "None" makes a "headless service" (no virtual IP), which is useful when direct endpoint connections are preferred and proxying is not required.
+	// If not set, defaults to "".
+	// +optional
+	ClusterIP *string `json:"clusterIP"`
+
 	// Sets endpoint and routing behavior.
 	// See: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#caveats-and-limitations-when-preserving-source-ips
 	// If not set, defaults to "Cluster".
diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go
index d4f5a544..a2ea16cb 100644
--- a/api/v1/zz_generated.deepcopy.go
+++ b/api/v1/zz_generated.deepcopy.go
@@ -745,6 +745,11 @@ func (in *ServiceOverridesSpec) DeepCopyInto(out *ServiceOverridesSpec) {
 		*out = new(corev1.ServiceType)
 		**out = **in
 	}
+	if in.ClusterIP != nil {
+		in, out := &in.ClusterIP, &out.ClusterIP
+		*out = new(string)
+		**out = **in
+	}
 	if in.ExternalTrafficPolicy != nil {
 		in, out := &in.ExternalTrafficPolicy, &out.ExternalTrafficPolicy
 		*out = new(corev1.ServiceExternalTrafficPolicyType)
@@ -772,6 +777,11 @@ func (in *ServiceSpec) DeepCopyInto(out *ServiceSpec) {
 	}
 	in.P2PTemplate.DeepCopyInto(&out.P2PTemplate)
 	in.RPCTemplate.DeepCopyInto(&out.RPCTemplate)
+	if in.ClusterDomain != nil {
+		in, out := &in.ClusterDomain, &out.ClusterDomain
+		*out = new(string)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceSpec.
diff --git a/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml b/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml
index 92fde62f..33bc8869 100644
--- a/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml
+++ b/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml
@@ -5674,6 +5674,9 @@ spec:
                   This allows a k8s admin to use the service in an Ingress, for example.
                   Additionally, multiple p2p services are created for CometBFT peer exchange.
                 properties:
+                  clusterDomain:
+                    description: Overrides for default cluster domain name.
+                    type: string
                   maxP2PExternalAddresses:
                     description: |-
                       Max number of external p2p services to create for CometBFT peer exchange.
@@ -5688,6 +5691,11 @@ spec:
                     description: Overrides for all P2P services that need external
                       addresses.
                     properties:
+                      clusterIP:
+                        description: |-
+                          Setting this to "None" makes a "headless service" (no virtual IP), which is useful when direct endpoint connections are preferred and proxying is not required.
+                          If not set, defaults to "".
+                        type: string
                       externalTrafficPolicy:
                         description: |-
                           Sets endpoint and routing behavior.
@@ -5729,6 +5737,11 @@ spec:
                   rpcTemplate:
                     description: Overrides for the single RPC service.
                     properties:
+                      clusterIP:
+                        description: |-
+                          Setting this to "None" makes a "headless service" (no virtual IP), which is useful when direct endpoint connections are preferred and proxying is not required.
+                          If not set, defaults to "".
+                        type: string
                       externalTrafficPolicy:
                         description: |-
                           Sets endpoint and routing behavior.
diff --git a/internal/fullnode/peer_collector.go b/internal/fullnode/peer_collector.go
index 75426d50..37c27c3a 100644
--- a/internal/fullnode/peer_collector.go
+++ b/internal/fullnode/peer_collector.go
@@ -105,6 +105,11 @@ func NewPeerCollector(client Getter) *PeerCollector {
 // Collect peer information given the crd.
 func (c PeerCollector) Collect(ctx context.Context, crd *cosmosv1.CosmosFullNode) (Peers, kube.ReconcileError) {
 	peers := make(Peers)
+
+	clusterDomain := "cluster.local"
+	if crd.Spec.Service.ClusterDomain != nil {
+		clusterDomain = *crd.Spec.Service.ClusterDomain
+	}
 	for i := int32(0); i < crd.Spec.Replicas; i++ {
 		secretName := nodeKeySecretName(crd, i)
 		var secret corev1.Secret
@@ -121,7 +126,7 @@ func (c PeerCollector) Collect(ctx context.Context, crd *cosmosv1.CosmosFullNode
 		svcName := p2pServiceName(crd, i)
 		peers[c.objectKey(crd, i)] = Peer{
 			NodeID:         nodeKey.ID(),
-			PrivateAddress: fmt.Sprintf("%s.%s.svc.cluster.local:%d", svcName, secret.Namespace, p2pPort),
+			PrivateAddress: fmt.Sprintf("%s.%s.svc.%s:%d", svcName, secret.Namespace, clusterDomain, p2pPort),
 		}
 		if err := c.addExternalAddress(ctx, peers, crd, i); err != nil {
 			return nil, kube.TransientError(err)
diff --git a/internal/fullnode/service_builder.go b/internal/fullnode/service_builder.go
index 6f880a0a..2831a12a 100644
--- a/internal/fullnode/service_builder.go
+++ b/internal/fullnode/service_builder.go
@@ -63,6 +63,7 @@ func BuildServices(crd *cosmosv1.CosmosFullNode) []diff.Resource[*corev1.Service
 			svc.Spec.ExternalTrafficPolicy = *valOrDefault(crd.Spec.Service.P2PTemplate.ExternalTrafficPolicy, ptr(corev1.ServiceExternalTrafficPolicyTypeLocal))
 		} else {
 			svc.Spec.Type = corev1.ServiceTypeClusterIP
+			svc.Spec.ClusterIP = *valOrDefault(crd.Spec.Service.P2PTemplate.ClusterIP, ptr(""))
 		}
 
 		p2ps[i] = diff.Adapt(&svc, i)
@@ -131,6 +132,9 @@ func rpcService(crd *cosmosv1.CosmosFullNode) *corev1.Service {
 	if v := rpcSpec.Type; v != nil {
 		svc.Spec.Type = *v
 	}
+	if v := rpcSpec.ClusterIP; v != nil {
+		svc.Spec.ClusterIP = *v
+	}
 
 	return &svc
 }