Merge pull request #164 from buildkite/pdp-776-does-the-rbac-for-the-…

…controller-need-to

charts/agent-stack-k8s/templates/deployment.yaml.tpl

@@ -15,7 +15,7 @@ spec:
  checksum/config: {{ include (print $.Template.BasePath "/config.yaml.tpl") . | sha256sum }}
  checksum/secrets: {{ include (print $.Template.BasePath "/secrets.yaml.tpl") . | sha256sum }}
  spec:
- serviceAccountName: {{ .Release.Name }}
+ serviceAccountName: {{ .Release.Name }}-controller
  nodeSelector:
 {{ toYaml $.Values.nodeSelector | indent 8 }}
  containers:

charts/agent-stack-k8s/templates/rbac.yaml.tpl

@@ -1,7 +1,8 @@
 apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
+kind: Role
 metadata:
- name: {{ .Release.Name }}
+ name: {{ .Release.Name }}-controller
+ namespace: {{ .Release.Namespace }}
 rules:
  - apiGroups:
  - batch
@@ -23,20 +24,20 @@ rules:
  - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
+kind: RoleBinding
 metadata:
  name: {{ .Release.Name }}
 roleRef:
  apiGroup: rbac.authorization.k8s.io
- kind: ClusterRole
- name: {{ .Release.Name }}
+ kind: Role
+ name: {{ .Release.Name }}-controller
 subjects:
  - kind: ServiceAccount
- name: {{ .Release.Name }}
+ name: {{ .Release.Name }}-controller
  namespace: {{ .Release.Namespace }}
 ---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
- name: {{ .Release.Name }}
+ name: {{ .Release.Name }}-controller
  namespace: {{ .Release.Namespace }}

internal/controller/controller.go

@@ -2,6 +2,7 @@ package controller
 
 import (
  "context"
+ "fmt"
  "net/http"
  _ "net/http/pprof"
  "time"
@@ -10,6 +11,10 @@ import (
  "github.com/buildkite/agent-stack-k8s/v2/internal/controller/monitor"
  "github.com/buildkite/agent-stack-k8s/v2/internal/controller/scheduler"
  "go.uber.org/zap"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/labels"
+ "k8s.io/apimachinery/pkg/selection"
+ "k8s.io/client-go/informers"
  "k8s.io/client-go/kubernetes"
 )
 
@@ -49,7 +54,7 @@ func Run(
  })
  limiter := scheduler.NewLimiter(logger.Named("limiter"), sched, cfg.MaxInFlight)
 
- informerFactory, err := scheduler.NewInformerFactory(k8sClient, cfg.Tags)
+ informerFactory, err := NewInformerFactory(k8sClient, cfg.Namespace, cfg.Tags)
  if err != nil {
  logger.Fatal("failed to create informer", zap.Error(err))
  }
@@ -79,3 +84,28 @@ func Run(
  logger.Info("monitor failed", zap.Error(err))
  }
 }
+
+// returns an informer factory configured to watch resources (pods, jobs) created by the scheduler
+func NewInformerFactory(
+ k8s kubernetes.Interface,
+ namespace string,
+ tags []string,
+) (informers.SharedInformerFactory, error) {
+ hasTag, err := labels.NewRequirement(config.TagLabel, selection.In, config.TagsToLabels(tags))
+ if err != nil {
+ return nil, fmt.Errorf("failed to build tag label selector for job manager: %w", err)
+ }
+ hasUUID, err := labels.NewRequirement(config.UUIDLabel, selection.Exists, nil)
+ if err != nil {
+ return nil, fmt.Errorf("failed to build uuid label selector for job manager: %w", err)
+ }
+ factory := informers.NewSharedInformerFactoryWithOptions(
+ k8s,
+ 0,
+ informers.WithNamespace(namespace),
+ informers.WithTweakListOptions(func(opt *metav1.ListOptions) {
+ opt.LabelSelector = labels.NewSelector().Add(*hasTag, *hasUUID).String()
+ }),
+ )
+ return factory, nil
+}

internal/controller/scheduler/scheduler.go

@@ -18,9 +18,6 @@ import (
  corev1 "k8s.io/api/core/v1"
  "k8s.io/apimachinery/pkg/api/errors"
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/apimachinery/pkg/labels"
- "k8s.io/apimachinery/pkg/selection"
- "k8s.io/client-go/informers"
  "k8s.io/client-go/kubernetes"
  "k8s.io/utils/pointer"
 )
@@ -45,29 +42,6 @@ func New(logger *zap.Logger, client kubernetes.Interface, cfg Config) *worker {
  }
 }
 
-// returns an informer factory configured to watch resources (pods, jobs) created by the scheduler
-func NewInformerFactory(
- k8s kubernetes.Interface,
- tags []string,
-) (informers.SharedInformerFactory, error) {
- hasTag, err := labels.NewRequirement(config.TagLabel, selection.In, config.TagsToLabels(tags))
- if err != nil {
- return nil, fmt.Errorf("failed to build tag label selector for job manager: %w", err)
- }
- hasUUID, err := labels.NewRequirement(config.UUIDLabel, selection.Exists, nil)
- if err != nil {
- return nil, fmt.Errorf("failed to build uuid label selector for job manager: %w", err)
- }
- factory := informers.NewSharedInformerFactoryWithOptions(
- k8s,
- 0,
- informers.WithTweakListOptions(func(opt *metav1.ListOptions) {
- opt.LabelSelector = labels.NewSelector().Add(*hasTag, *hasUUID).String()
- }),
- )
- return factory, nil
-}
-
 type KubernetesPlugin struct {
  PodSpec *corev1.PodSpec
  GitEnvFrom []corev1.EnvFromSource