Support hybrid nodes with different attestors or config

charts/spire/README.md

@@ -75,7 +75,11 @@ kubectl delete crds clusterfederatedtrustdomains.spire.spiffe.io clusterspiffeid
 
 We only support upgrading one major version at a time. Version skipping isn't supported.
 
-### 0.18.x
+### 0.19.X
+
+- The spire-agent daemonset gained a new label. For those disabling the upgrade hooks, you need to delete the spire-agent daemonset before issuing the helm upgrade.
+
+### 0.18.X
 
 - SPIRE no longer emits x509UniqueIdentifiers in x509-SVIDS by default. The old behavior can be reenabled with spire-server.credentialComposer.uniqueID.enabled=true. See https://github.com/spiffe/spire/pull/4862 for details.
 - SPIRE agents will now automatically reattest when they can. The old behavior can be reenabled with spire-agent.disableReattestToRenew=true. See https://github.com/spiffe/spire/pull/4791 for details.

charts/spire/charts/spire-agent/README.md

@@ -126,6 +126,12 @@ A Helm chart to install the SPIRE agent.
 | `experimental.enabled`                            | Allow configuration of experimental features                                                                                                                                                      | `false`                                                                          |
 | `experimental.syncInterval`                       | Sync interval with SPIRE server with exponential backoff                                                                                                                                          | `5s`                                                                             |
 | `experimental.featureFlags`                       | List of developer feature flags                                                                                                                                                                   | `[]`                                                                             |
+| `agents`                                          | Configure multiple agent DaemonSets. Useful when you have different node types and nodeAttestors                                                                                                  | `{}`                                                                             |
+| `installAndUpgradeHook.enabled`                   | Enable Helm hook to autofix common install/upgrade issues (should be disabled when using `helm template`)                                                                                         | `true`                                                                           |
+| `tools.kubectl.image.registry`                    | The OCI registry to pull the image from                                                                                                                                                           | `docker.io`                                                                      |
+| `tools.kubectl.image.repository`                  | The repository within the registry                                                                                                                                                                | `rancher/kubectl`                                                                |
+| `tools.kubectl.image.pullPolicy`                  | The image pull policy                                                                                                                                                                             | `IfNotPresent`                                                                   |
+| `tools.kubectl.image.tag`                         | Overrides the image tag whose default is the chart appVersion                                                                                                                                     | `""`                                                                             |
 | `sockets.hostBasePath`                            | Path on which the agent socket is made available when admin.mountOnHost is true                                                                                                                   | `/run/spire/agent/sockets`                                                       |
 | `sockets.admin.enabled`                           | Enable the admin socket. Useful for admin tasks or the Delegated Identity API.                                                                                                                    | `false`                                                                          |
 | `sockets.admin.mountOnHost`                       | Enable the admin socket to be visible on the host.                                                                                                                                                | `false`                                                                          |

charts/spire/charts/spire-agent/templates/configmap.yaml

@@ -1,3 +1,4 @@
+{{- define "spire-agent.check-config-values" -}}
 {{- include "spire-lib.check-strict-mode" (list . "clusterName must be set" (eq (include "spire-lib.cluster-name" .) "example-cluster"))}}
 {{- include "spire-lib.check-strict-mode" (list . "trustDomain must be set" (eq (include "spire-lib.trust-domain" .) "example.org"))}}
 {{- range $type, $tvals := .Values.customPlugins }}
@@ -21,6 +22,7 @@
 {{- if hasPrefix (.Values.socketPath | dir | clean) (.Values.sockets.hostBasePath | clean) }}
 {{- fail "The sockets.hostBasePath can not be located under the socketPath direcotry" }}
 {{- end }}
+{{- end }}
 {{- define "spire-agent.yaml-config" -}}
 agent:
   {{- if .Values.disableReattestToRenew }}
@@ -132,10 +134,22 @@ telemetry:
         port: {{ .Values.telemetry.prometheus.port }}
 {{- end }}
 {{- end }}
+{{- $root := . }}
+{{- range $name := (concat (list "default") (keys .Values.agents)) | uniq }}
+{{- with (dict "Release" $root.Release "Chart" $root.Chart "Values" (deepCopy $root.Values)) }}
+{{- $nameSuffix := "" }}
+{{- if ne $name "default" }}
+{{-   $nameSuffix = printf "-%s" $name }}
+{{- end }}
+{{- if hasKey $root.Values.agents $name }}
+{{-   $_ := set . "Values" (mergeOverwrite .Values (index $root.Values.agents $name)) }}
+{{- end }}
+{{- include "spire-agent.check-config-values" . }}
+---
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: {{ include "spire-agent.fullname" . }}
+  name: {{ include "spire-agent.fullname" . }}{{ $nameSuffix }}
   namespace: {{ include "spire-agent.namespace" . }}
   {{- with .Values.configMap.annotations }}
   annotations:
@@ -144,3 +158,5 @@ metadata:
 data:
   agent.conf: |
     {{- include "spire-lib.reformat-and-yaml2json" (dict "config" (include "spire-agent.yaml-config" .) "root" .) | nindent 4 }}
+{{- end }}
+{{- end }}

charts/spire/charts/spire-agent/templates/daemonset.yaml

@@ -1,4 +1,14 @@
 {{- $configSum := (include (print $.Template.BasePath "/configmap.yaml") . | sha256sum) }}
+{{- $root := . }}
+{{- range $name := (concat (list "default") (keys .Values.agents)) | uniq }}
+{{- with (dict "Release" $root.Release "Chart" $root.Chart "Values" (deepCopy $root.Values)) }}
+{{- $nameSuffix := "" }}
+{{- if ne $name "default" }}
+{{-   $nameSuffix = printf "-%s" $name }}
+{{- end }}
+{{- if hasKey $root.Values.agents $name }}
+{{-   $_ := set . "Values" (mergeOverwrite .Values (index $root.Values.agents $name)) }}
+{{- end }}
 {{- $podSecurityContext := fromYaml (include "spire-lib.podsecuritycontext" .) }}
 {{- $mainSecurityContext := deepCopy .Values.securityContext }}
 {{- if .Values.nodeAttestor.tpmDirect.enabled }}
@@ -8,17 +18,20 @@
 {{- $cbh := eq (include "spire-agent.connect-by-hostname" .) "true" }}
 {{- $socketAlternateNames := index (include "spire-agent.socket-alternate-names" . | fromYaml) "names" }}
 {{- $socketPath := include "spire-agent.socket-path" . }}
+---
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  name: {{ include "spire-agent.fullname" . }}
+  name: {{ include "spire-agent.fullname" . }}{{ $nameSuffix }}
   namespace: {{ include "spire-agent.namespace" . }}
   labels:
     {{- include "spire-agent.labels" . | nindent 4 }}
+    app.kubernetes.io/component: {{ $name }}
 spec:
   selector:
     matchLabels:
       {{- include "spire-agent.selectorLabels" . | nindent 6 }}
+      app.kubernetes.io/component: {{ $name }}
   {{- with .Values.updateStrategy }}
   updateStrategy:
     {{- if not (has .type (list "RollingUpdate" "OnDelete")) }}
@@ -40,6 +53,7 @@ spec:
         {{- end }}
       labels:
         {{- include "spire-agent.selectorLabels" . | nindent 8 }}
+        app.kubernetes.io/component: {{ $name }}
         {{- with .Values.podLabels }}
           {{- toYaml . | nindent 8 }}
         {{- end }}
@@ -257,7 +271,11 @@ spec:
         {{- if eq (len .Values.trustBundleURL) 0 }}
         - name: spire-bundle
           configMap:
-            name: {{ include "spire-lib.bundle-configmap" . }}
+            name: {{ include "spire-lib.bundle-configmap" . }}{{ $nameSuffix }}
+        {{- end }}
+        {{- if .Values.nodeAttestor.tpmDirect.enabled }}
+        - name: tpm-direct
+          emptyDir: {}
         {{- end }}
         {{- if .Values.nodeAttestor.tpmDirect.enabled }}
         - name: tpm-direct
@@ -287,3 +305,5 @@ spec:
         {{- if gt (len .Values.extraVolumes) 0 }}
         {{- toYaml .Values.extraVolumes | nindent 8 }}
         {{- end }}
+{{- end }}
+{{- end }}

charts/spire/charts/spire-agent/templates/pre-upgrade-hook.yaml

@@ -0,0 +1,77 @@
+{{- if eq ((dig "installAndUpgradeHooks" "enabled" .Values.installAndUpgradeHook.enabled .Values.global) | toString) "true" }}
+{{- $ds := lookup "apps/v1" "DaemonSet" (include "spire-agent.namespace" .) (include "spire-agent.fullname" .) }}
+{{- if and $ds (not (hasKey $ds.metadata.labels "app.kubernetes.io/component")) }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "spire-agent.serviceAccountName" . }}-pre-upgrade
+  namespace: {{ include "spire-agent.namespace" . }}
+  labels:
+    {{- include "spire-agent.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": pre-upgrade
+    "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "spire-agent.fullname" . }}-pre-upgrade
+  namespace: {{ include "spire-agent.namespace" . }}
+  annotations:
+    "helm.sh/hook": pre-upgrade
+    "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed
+rules:
+  - apiGroups: ["apps"]
+    resources: ["daemonsets"]
+    resourceNames: [{{ include "spire-agent.fullname" . | quote }}]
+    verbs: ["get", "delete"]
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ include "spire-agent.fullname" . }}-pre-upgrade
+  namespace: {{ include "spire-agent.namespace" . }}
+  annotations:
+    "helm.sh/hook": pre-upgrade
+    "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "spire-agent.serviceAccountName" . }}-pre-upgrade
+    namespace: {{ include "spire-agent.namespace" . }}
+roleRef:
+  kind: Role
+  name: {{ include "spire-agent.fullname" . }}-pre-upgrade
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "spire-agent.fullname" . }}-pre-upgrade
+  namespace: {{ include "spire-agent.namespace" . }}
+  labels:
+    {{- include "spire-agent.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": pre-upgrade
+    "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed
+spec:
+  template:
+    metadata:
+      name: {{ include "spire-agent.fullname" . }}-pre-upgrade
+    spec:
+      restartPolicy: Never
+      serviceAccountName: {{ include "spire-agent.serviceAccountName" . }}-pre-upgrade
+      securityContext:
+        {{- include "spire-lib.podsecuritycontext" . | nindent 8 }}
+      containers:
+      - name: pre-upgrade
+        securityContext:
+          {{- include "spire-lib.securitycontext" . | nindent 10 }}
+        image: {{ template "spire-lib.kubectl-image" (dict "appVersion" $.Chart.AppVersion "image" .Values.tools.kubectl.image "global" .Values.global "KubeVersion" .Capabilities.KubeVersion.Version) }}
+        args:
+          - delete
+          - daemonset
+          - {{ include "spire-agent.fullname" . }}
+          - -n
+          - {{ include "spire-agent.namespace" . }}
+{{- end }}
+{{- end }}

charts/spire/charts/spire-agent/values.yaml

@@ -328,6 +328,37 @@ experimental:
   ## @param experimental.featureFlags [array] List of developer feature flags
   featureFlags: []
 
+## @param agents Configure multiple agent DaemonSets. Useful when you have different node types and nodeAttestors
+agents: {}
+#  default:
+#   nodeSelector:
+#     tpm: without
+#  tpm:
+#    nodeSelector:
+#      tpm: with
+#    nodeAttestor:
+#      k8sPsat:
+#        enabled: false
+#      tpmDirect:
+#        enabled: true
+
+installAndUpgradeHook:
+  ## @param installAndUpgradeHook.enabled Enable Helm hook to autofix common install/upgrade issues (should be disabled when using `helm template`)
+  enabled: true
+
+tools:
+  kubectl:
+    ## @param tools.kubectl.image.registry The OCI registry to pull the image from
+    ## @param tools.kubectl.image.repository The repository within the registry
+    ## @param tools.kubectl.image.pullPolicy The image pull policy
+    ## @param tools.kubectl.image.tag Overrides the image tag whose default is the chart appVersion
+    ##
+    image:
+      registry: docker.io
+      repository: rancher/kubectl
+      pullPolicy: IfNotPresent
+      tag: ""
+
 sockets:
   ## @param sockets.hostBasePath Path on which the agent socket is made available when admin.mountOnHost is true
   hostBasePath: /run/spire/agent/sockets

examples/tpm-direct/different-nodes.yaml

@@ -0,0 +1,22 @@
+spire-agent:
+  agents:
+    default:
+      nodeSelector:
+        tpm: without
+    tpm:
+      nodeSelector:
+        tpm: with
+      nodeAttestor:
+        k8sPsat:
+          enabled: false
+        tpmDirect:
+          enabled: true
+
+spire-server:
+  controllerManager:
+    # K8s labels have a 63 character limit. TPM hashes are 64 chars. So you need to label the node with two labels with half of the tpm's hash each. The 'node-restriction.kubernetes.io/' prefix is so that the
+    # nodes can't update the hash themselves, an important security constraint.
+    parentIDTemplate: 'spiffe://{{ .TrustDomain }}/spire/agent/{{if index .NodeMeta.Labels "node-restriction.kubernetes.io/tpm-pubhash"}}tpm/{{ index .NodeMeta.Labels "node-restriction.kubernetes.io/tpm-pubhash" }}{{ index .NodeMeta.Labels "node-restriction.kubernetes.io/tpm-pubhash2" }}{{ else }}spire/agent/k8s_psat/{{ .ClusterName }}/{{ .NodeMeta.UID }}{{ end }}'
+  nodeAttestor:
+    tpmDirect:
+      enabled: true

examples/tpm-direct/same-nodes.yaml

@@ -0,0 +1,15 @@
+spire-agent:
+  nodeAttestor:
+    k8sPsat:
+      enabled: false
+    tpmDirect:
+      enabled: true
+
+spire-server:
+  controllerManager:
+    # K8s labels have a 63 character limit. TPM hashes are 64 chars. So you need to label the node with two labels with half of the tpm's hash each. The 'node-restriction.kubernetes.io/' prefix is so that the
+    # nodes can't update the hash themselves, an important security constraint.
+    parentIDTemplate: 'spiffe://{{ .TrustDomain }}/spire/agent/tpm/{{ index .NodeMeta.Labels "node-restriction.kubernetes.io/tpm-pubhash" }}{{ index .NodeMeta.Labels "node-restriction.kubernetes.io/tpm-pubhash2" }}'
+  nodeAttestor:
+    tpmDirect:
+      enabled: true