feat: add clusterIP override for headless svc and add ClusterDomain(c…

…luster.local) override (#435) Co-authored-by: Andrew Gouin <andrew@gouin.io>
strangelove-ventures · Oct 8, 2024 · a2e42d5 · a2e42d5
1 parent 78954be
commit a2e42d5
Show file tree

Hide file tree

Showing 5 changed files with 42 additions and 1 deletion.
diff --git a/api/v1/cosmosfullnode_types.go b/api/v1/cosmosfullnode_types.go
@@ -730,6 +730,10 @@ type ServiceSpec struct {
 	// Overrides for the single RPC service.
 	// +optional
 	RPCTemplate ServiceOverridesSpec `json:"rpcTemplate"`
+
+	// Overrides for default cluster domain name.
+	// +optional
+	ClusterDomain *string `json:"clusterDomain"`
 }
 
 // ServiceOverridesSpec allows some overrides for the created, single RPC service.
@@ -743,6 +747,11 @@ type ServiceOverridesSpec struct {
 	// +optional
 	Type *corev1.ServiceType `json:"type"`
 
+	// Setting this to "None" makes a "headless service" (no virtual IP), which is useful when direct endpoint connections are preferred and proxying is not required.
+	// If not set, defaults to "".
+	// +optional
+	ClusterIP *string `json:"clusterIP"`
+
 	// Sets endpoint and routing behavior.
 	// See: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#caveats-and-limitations-when-preserving-source-ips
 	// If not set, defaults to "Cluster".

diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml b/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml
@@ -5674,6 +5674,9 @@ spec:
                   This allows a k8s admin to use the service in an Ingress, for example.
                   Additionally, multiple p2p services are created for CometBFT peer exchange.
                 properties:
+                  clusterDomain:
+                    description: Overrides for default cluster domain name.
+                    type: string
                   maxP2PExternalAddresses:
                     description: |-
                       Max number of external p2p services to create for CometBFT peer exchange.
@@ -5688,6 +5691,11 @@ spec:
                     description: Overrides for all P2P services that need external
                       addresses.
                     properties:
+                      clusterIP:
+                        description: |-
+                          Setting this to "None" makes a "headless service" (no virtual IP), which is useful when direct endpoint connections are preferred and proxying is not required.
+                          If not set, defaults to "".
+                        type: string
                       externalTrafficPolicy:
                         description: |-
                           Sets endpoint and routing behavior.
@@ -5729,6 +5737,11 @@ spec:
                   rpcTemplate:
                     description: Overrides for the single RPC service.
                     properties:
+                      clusterIP:
+                        description: |-
+                          Setting this to "None" makes a "headless service" (no virtual IP), which is useful when direct endpoint connections are preferred and proxying is not required.
+                          If not set, defaults to "".
+                        type: string
                       externalTrafficPolicy:
                         description: |-
                           Sets endpoint and routing behavior.

diff --git a/internal/fullnode/peer_collector.go b/internal/fullnode/peer_collector.go
@@ -105,6 +105,11 @@ func NewPeerCollector(client Getter) *PeerCollector {
 // Collect peer information given the crd.
 func (c PeerCollector) Collect(ctx context.Context, crd *cosmosv1.CosmosFullNode) (Peers, kube.ReconcileError) {
 	peers := make(Peers)
+
+	clusterDomain := "cluster.local"
+	if crd.Spec.Service.ClusterDomain != nil {
+		clusterDomain = *crd.Spec.Service.ClusterDomain
+	}
 	for i := int32(0); i < crd.Spec.Replicas; i++ {
 		secretName := nodeKeySecretName(crd, i)
 		var secret corev1.Secret
@@ -121,7 +126,7 @@ func (c PeerCollector) Collect(ctx context.Context, crd *cosmosv1.CosmosFullNode
 		svcName := p2pServiceName(crd, i)
 		peers[c.objectKey(crd, i)] = Peer{
 			NodeID:         nodeKey.ID(),
-			PrivateAddress: fmt.Sprintf("%s.%s.svc.cluster.local:%d", svcName, secret.Namespace, p2pPort),
+			PrivateAddress: fmt.Sprintf("%s.%s.svc.%s:%d", svcName, secret.Namespace, clusterDomain, p2pPort),
 		}
 		if err := c.addExternalAddress(ctx, peers, crd, i); err != nil {
 			return nil, kube.TransientError(err)

diff --git a/internal/fullnode/service_builder.go b/internal/fullnode/service_builder.go
@@ -63,6 +63,7 @@ func BuildServices(crd *cosmosv1.CosmosFullNode) []diff.Resource[*corev1.Service
 			svc.Spec.ExternalTrafficPolicy = *valOrDefault(crd.Spec.Service.P2PTemplate.ExternalTrafficPolicy, ptr(corev1.ServiceExternalTrafficPolicyTypeLocal))
 		} else {
 			svc.Spec.Type = corev1.ServiceTypeClusterIP
+			svc.Spec.ClusterIP = *valOrDefault(crd.Spec.Service.P2PTemplate.ClusterIP, ptr(""))
 		}
 
 		p2ps[i] = diff.Adapt(&svc, i)
@@ -131,6 +132,9 @@ func rpcService(crd *cosmosv1.CosmosFullNode) *corev1.Service {
 	if v := rpcSpec.Type; v != nil {
 		svc.Spec.Type = *v
 	}
+	if v := rpcSpec.ClusterIP; v != nil {
+		svc.Spec.ClusterIP = *v
+	}
 
 	return &svc
 }