Skip to content

Commit

Permalink
add service account and cluster role/binding reconciliation
Browse files Browse the repository at this point in the history
  • Loading branch information
agouin committed Oct 18, 2023
1 parent 7353e31 commit 992f5ad
Show file tree
Hide file tree
Showing 6 changed files with 319 additions and 18 deletions.
60 changes: 42 additions & 18 deletions controllers/cosmosfullnode_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,15 +43,18 @@ const controllerOwnerField = ".metadata.controller"
type CosmosFullNodeReconciler struct {
client.Client

cacheController *cosmos.CacheController
configMapControl fullnode.ConfigMapControl
nodeKeyControl fullnode.NodeKeyControl
peerCollector *fullnode.PeerCollector
podControl fullnode.PodControl
pvcControl fullnode.PVCControl
recorder record.EventRecorder
serviceControl fullnode.ServiceControl
statusClient *fullnode.StatusClient
cacheController *cosmos.CacheController
configMapControl fullnode.ConfigMapControl
nodeKeyControl fullnode.NodeKeyControl
peerCollector *fullnode.PeerCollector
podControl fullnode.PodControl
pvcControl fullnode.PVCControl
recorder record.EventRecorder
serviceControl fullnode.ServiceControl
statusClient *fullnode.StatusClient
serviceAccountControl fullnode.ServiceAccountControl
clusterRoleControl fullnode.ClusterRoleControl
clusterRoleBindingControl fullnode.ClusterRoleBindingControl
}

// NewFullNode returns a valid CosmosFullNode controller.
Expand All @@ -64,15 +67,18 @@ func NewFullNode(
return &CosmosFullNodeReconciler{
Client: client,

cacheController: cacheController,
configMapControl: fullnode.NewConfigMapControl(client),
nodeKeyControl: fullnode.NewNodeKeyControl(client),
peerCollector: fullnode.NewPeerCollector(client),
podControl: fullnode.NewPodControl(client, cacheController),
pvcControl: fullnode.NewPVCControl(client),
recorder: recorder,
serviceControl: fullnode.NewServiceControl(client),
statusClient: statusClient,
cacheController: cacheController,
configMapControl: fullnode.NewConfigMapControl(client),
nodeKeyControl: fullnode.NewNodeKeyControl(client),
peerCollector: fullnode.NewPeerCollector(client),
podControl: fullnode.NewPodControl(client, cacheController),
pvcControl: fullnode.NewPVCControl(client),
recorder: recorder,
serviceControl: fullnode.NewServiceControl(client),
statusClient: statusClient,
serviceAccountControl: fullnode.NewServiceAccountControl(client),
clusterRoleControl: fullnode.NewClusterRoleControl(client),
clusterRoleBindingControl: fullnode.NewClusterRoleBindingControl(client),
}
}

Expand Down Expand Up @@ -143,6 +149,24 @@ func (r *CosmosFullNodeReconciler) Reconcile(ctx context.Context, req ctrl.Reque
errs.Append(err)
}

// Reconcile service accounts.
err = r.serviceAccountControl.Reconcile(ctx, reporter, crd)
if err != nil {
errs.Append(err)
}

// Reconcile cluster roles.
err = r.clusterRoleControl.Reconcile(ctx, reporter, crd)
if err != nil {
errs.Append(err)
}

// Reconcile cluster role bindings.
err = r.clusterRoleBindingControl.Reconcile(ctx, reporter, crd)
if err != nil {
errs.Append(err)
}

// Reconcile pods.
podRequeue, err := r.podControl.Reconcile(ctx, reporter, crd, configCksums)
if err != nil {
Expand Down
60 changes: 60 additions & 0 deletions internal/fullnode/cluster_role_binding_control.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package fullnode

import (
"context"
"fmt"

cosmosv1 "github.com/strangelove-ventures/cosmos-operator/api/v1"
"github.com/strangelove-ventures/cosmos-operator/internal/diff"
"github.com/strangelove-ventures/cosmos-operator/internal/kube"
rbacv1 "k8s.io/api/rbac/v1"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
)

// ClusterRoleBindingControl creates or updates ClusterRoleBindings.
type ClusterRoleBindingControl struct {
client Client
}

func NewClusterRoleBindingControl(client Client) ClusterRoleBindingControl {
return ClusterRoleBindingControl{
client: client,
}
}

// Reconcile creates or updates cluster role bindings.
func (sc ClusterRoleBindingControl) Reconcile(ctx context.Context, log kube.Logger, crd *cosmosv1.CosmosFullNode) kube.ReconcileError {
var crs rbacv1.ClusterRoleBindingList
if err := sc.client.List(ctx, &crs,
client.InNamespace(crd.Namespace),
client.MatchingFields{kube.ControllerOwnerField: crd.Name},
); err != nil {
return kube.TransientError(fmt.Errorf("list existing cluster role bindings: %w", err))
}

current := ptrSlice(crs.Items)
want := BuildClusterRoleBindings(crd)
diffed := diff.New(current, want)

for _, cr := range diffed.Creates() {
log.Info("Creating cluster role binding", "clusterRoleBindingName", cr.Name)
if err := ctrl.SetControllerReference(crd, cr, sc.client.Scheme()); err != nil {
return kube.TransientError(fmt.Errorf("set controller reference on cluster role binding %q: %w", cr.Name, err))
}
// CreateOrUpdate (vs. only create) fixes a bug with current deployments where updating would remove the owner reference.
// This ensures we update the service with the owner reference.
if err := kube.CreateOrUpdate(ctx, sc.client, cr); err != nil {
return kube.TransientError(fmt.Errorf("create cluster role binding %q: %w", cr.Name, err))
}
}

for _, cr := range diffed.Updates() {
log.Info("Updating cluster role binding", "clusterRoleBindingName", cr.Name)
if err := sc.client.Update(ctx, cr); err != nil {
return kube.TransientError(fmt.Errorf("update cluster role binding %q: %w", cr.Name, err))
}
}

return nil
}
60 changes: 60 additions & 0 deletions internal/fullnode/cluster_role_control.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package fullnode

import (
"context"
"fmt"

cosmosv1 "github.com/strangelove-ventures/cosmos-operator/api/v1"
"github.com/strangelove-ventures/cosmos-operator/internal/diff"
"github.com/strangelove-ventures/cosmos-operator/internal/kube"
rbacv1 "k8s.io/api/rbac/v1"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
)

// ClusterRoleControl creates or updates ClusterRoles.
type ClusterRoleControl struct {
client Client
}

func NewClusterRoleControl(client Client) ClusterRoleControl {
return ClusterRoleControl{
client: client,
}
}

// Reconcile creates or updates cluster roles.
func (sc ClusterRoleControl) Reconcile(ctx context.Context, log kube.Logger, crd *cosmosv1.CosmosFullNode) kube.ReconcileError {
var crs rbacv1.ClusterRoleList
if err := sc.client.List(ctx, &crs,
client.InNamespace(crd.Namespace),
client.MatchingFields{kube.ControllerOwnerField: crd.Name},
); err != nil {
return kube.TransientError(fmt.Errorf("list existing cluster roles: %w", err))
}

current := ptrSlice(crs.Items)
want := BuildClusterRoles(crd)
diffed := diff.New(current, want)

for _, cr := range diffed.Creates() {
log.Info("Creating cluster role", "clusterRoleName", cr.Name)
if err := ctrl.SetControllerReference(crd, cr, sc.client.Scheme()); err != nil {
return kube.TransientError(fmt.Errorf("set controller reference on cluster role %q: %w", cr.Name, err))
}
// CreateOrUpdate (vs. only create) fixes a bug with current deployments where updating would remove the owner reference.
// This ensures we update the service with the owner reference.
if err := kube.CreateOrUpdate(ctx, sc.client, cr); err != nil {
return kube.TransientError(fmt.Errorf("create cluster role %q: %w", cr.Name, err))
}
}

for _, cr := range diffed.Updates() {
log.Info("Updating cluster role", "clusterRoleName", cr.Name)
if err := sc.client.Update(ctx, cr); err != nil {
return kube.TransientError(fmt.Errorf("update cluster role %q: %w", cr.Name, err))
}
}

return nil
}
2 changes: 2 additions & 0 deletions internal/fullnode/pod_builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ func NewPodBuilder(crd *cosmosv1.CosmosFullNode) PodBuilder {
Annotations: make(map[string]string),
},
Spec: corev1.PodSpec{
ServiceAccountName: serviceAccountName(crd),
SecurityContext: &corev1.PodSecurityContext{
RunAsUser: ptr(int64(1025)),
RunAsGroup: ptr(int64(1025)),
Expand Down Expand Up @@ -408,6 +409,7 @@ config-merge -f toml "$TMP_DIR/app.toml" "$OVERLAY_DIR/app-overlay.toml" > "$CON
Env: env,
ImagePullPolicy: tpl.ImagePullPolicy,
WorkingDir: workDir,
SecurityContext: &corev1.SecurityContext{},
})

return required
Expand Down
95 changes: 95 additions & 0 deletions internal/fullnode/rbac_builder.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
package fullnode

import (
cosmosv1 "github.com/strangelove-ventures/cosmos-operator/api/v1"
"github.com/strangelove-ventures/cosmos-operator/internal/diff"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func serviceAccountName(crd *cosmosv1.CosmosFullNode) string {
return crd.Name + "-vc"
}

func clusterRoleName(crd *cosmosv1.CosmosFullNode) string {
return crd.Name + "-cr"
}

// BuildServiceAccounts returns a list of service accounts given the crd.
//
// Creates a single service account for the version check.
func BuildServiceAccounts(crd *cosmosv1.CosmosFullNode) []diff.Resource[*corev1.ServiceAccount] {
diffSa := make([]diff.Resource[*corev1.ServiceAccount], 1)
svc := corev1.ServiceAccount{
TypeMeta: v1.TypeMeta{
Kind: "ServiceAccount",
},
ObjectMeta: v1.ObjectMeta{
Name: serviceAccountName(crd),
Namespace: crd.Namespace,
},
}

diffSa[0] = diff.Adapt(&svc, 0)

return diffSa
}

// BuildClusterRoles returns a list of cluster roles given the crd.
//
// Creates a single cluster role for the version check.
func BuildClusterRoles(crd *cosmosv1.CosmosFullNode) []diff.Resource[*rbacv1.ClusterRole] {
diffCr := make([]diff.Resource[*rbacv1.ClusterRole], 1)
cr := rbacv1.ClusterRole{
ObjectMeta: v1.ObjectMeta{
Name: clusterRoleName(crd),
Namespace: crd.Namespace,
},
Rules: []rbacv1.PolicyRule{
{
APIGroups: []string{""}, // core API group
Resources: []string{"namespaces", "pods"},
Verbs: []string{"get", "list"},
},
{
APIGroups: []string{"cosmos.strange.love"},
Resources: []string{"cosmosfullnodes"},
Verbs: []string{"get", "list", "patch"},
},
},
}

diffCr[0] = diff.Adapt(&cr, 0)

return diffCr
}

// BuildClusterRoles returns a list of cluster role bindings given the crd.
//
// Creates a single cluster role binding for the version check.
func BuildClusterRoleBindings(crd *cosmosv1.CosmosFullNode) []diff.Resource[*rbacv1.ClusterRoleBinding] {
diffCrb := make([]diff.Resource[*rbacv1.ClusterRoleBinding], 1)
crb := rbacv1.ClusterRoleBinding{
ObjectMeta: v1.ObjectMeta{
Name: crd.Name + "-crb",
Namespace: crd.Namespace,
},
Subjects: []rbacv1.Subject{
{
Kind: "ServiceAccount",
Name: serviceAccountName(crd),
Namespace: crd.Namespace,
},
},
RoleRef: rbacv1.RoleRef{
Kind: "ClusterRole",
Name: clusterRoleName(crd),
APIGroup: "rbac.authorization.k8s.io",
},
}

diffCrb[0] = diff.Adapt(&crb, 0)

return diffCrb
}
60 changes: 60 additions & 0 deletions internal/fullnode/service_account_control.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package fullnode

import (
"context"
"fmt"

cosmosv1 "github.com/strangelove-ventures/cosmos-operator/api/v1"
"github.com/strangelove-ventures/cosmos-operator/internal/diff"
"github.com/strangelove-ventures/cosmos-operator/internal/kube"
corev1 "k8s.io/api/core/v1"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
)

// ServiceControl creates or updates Services.
type ServiceAccountControl struct {
client Client
}

func NewServiceAccountControl(client Client) ServiceAccountControl {
return ServiceAccountControl{
client: client,
}
}

// Reconcile creates or updates service accounts.
func (sc ServiceAccountControl) Reconcile(ctx context.Context, log kube.Logger, crd *cosmosv1.CosmosFullNode) kube.ReconcileError {
var svcs corev1.ServiceAccountList
if err := sc.client.List(ctx, &svcs,
client.InNamespace(crd.Namespace),
client.MatchingFields{kube.ControllerOwnerField: crd.Name},
); err != nil {
return kube.TransientError(fmt.Errorf("list existing service accounts: %w", err))
}

current := ptrSlice(svcs.Items)
want := BuildServiceAccounts(crd)
diffed := diff.New(current, want)

for _, svc := range diffed.Creates() {
log.Info("Creating service account", "svcAccountName", svc.Name)
if err := ctrl.SetControllerReference(crd, svc, sc.client.Scheme()); err != nil {
return kube.TransientError(fmt.Errorf("set controller reference on service account %q: %w", svc.Name, err))
}
// CreateOrUpdate (vs. only create) fixes a bug with current deployments where updating would remove the owner reference.
// This ensures we update the service with the owner reference.
if err := kube.CreateOrUpdate(ctx, sc.client, svc); err != nil {
return kube.TransientError(fmt.Errorf("create service account %q: %w", svc.Name, err))
}
}

for _, svc := range diffed.Updates() {
log.Info("Updating service account", "svcAccountName", svc.Name)
if err := sc.client.Update(ctx, svc); err != nil {
return kube.TransientError(fmt.Errorf("update service account %q: %w", svc.Name, err))
}
}

return nil
}

0 comments on commit 992f5ad

Please sign in to comment.