Fix registry mirror issues in packages

Makefile

@@ -60,7 +60,7 @@ OUTPUT_DIR := _output
 OUTPUT_BIN_DIR := ${OUTPUT_DIR}/bin
 
 KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize
-KUSTOMIZE_VERSION := 4.2.0
+KUSTOMIZE_VERSION := 4.5.6
 
 KUSTOMIZE_OUTPUT_BIN_DIR="${OUTPUT_DIR}/kustomize-bin/"
 

config/default/kustomization.yaml

@@ -17,7 +17,6 @@ resources:
 
 bases:
 - ../crd
-- ../rbac
 - ../manager
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml

config/manifest/eksa-components.yaml

@@ -6633,6 +6633,28 @@ metadata:
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
+metadata:
+  creationTimestamp: null
+  name: eksa-packages-role
+  namespace: eksa-packages
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - packages.eks.amazonaws.com
+  resources:
+  - packagebundlecontrollers
+  verbs:
+  - delete
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
 metadata:
   name: eksa-leader-election-role
   namespace: eksa-system
@@ -6683,12 +6705,6 @@ rules:
   verbs:
   - patch
   - update
-- apiGroups:
-  - packages.eks.amazonaws.com
-  resources:
-  - packagebundlecontrollers
-  verbs:
-  - delete
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -6711,6 +6727,7 @@ rules:
   verbs:
   - create
   - delete
+  - get
 - apiGroups:
   - ""
   resources:
@@ -7062,6 +7079,20 @@ rules:
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
+metadata:
+  name: eksa-packages-rolebinding
+  namespace: eksa-packages
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: eksa-packages-role
+subjects:
+- kind: ServiceAccount
+  name: eksa-controller-manager
+  namespace: eksa-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
 metadata:
   name: eksa-leader-election-rolebinding
   namespace: eksa-system

config/prod/kustomization.yaml

@@ -2,6 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
 - ../default
+- ../rbac_default
 
 images:
 - name: controller

config/rbac/role.yaml

@@ -21,6 +21,7 @@ rules:
   verbs:
   - create
   - delete
+  - get
 - apiGroups:
   - ""
   resources:
@@ -376,13 +377,14 @@ kind: Role
 metadata:
   creationTimestamp: null
   name: manager-role
-  namespace: eksa-system
+  namespace: eksa-packages
 rules:
 - apiGroups:
   - ""
   resources:
   - secrets
   verbs:
+  - get
   - patch
   - update
 - apiGroups:
@@ -391,3 +393,19 @@ rules:
   - packagebundlecontrollers
   verbs:
   - delete
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  creationTimestamp: null
+  name: manager-role
+  namespace: eksa-system
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - patch
+  - update

config/rbac/role_binding.yaml

@@ -24,4 +24,19 @@ roleRef:
 subjects:
 - kind: ServiceAccount
   name: controller-manager
-  namespace: system
+  namespace: system
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: manager-rolebinding
+  namespace: eksa-packages
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: manager-role
+subjects:
+- kind: ServiceAccount
+  name: controller-manager
+  namespace: system

config/rbac_default/apply-namespace.yaml

@@ -0,0 +1,11 @@
+namePrefix:
+- path: metadata/namespace
+  kind: ServiceAccount
+- path: subjects/name
+  kind: RoleBinding
+- path: subjects/namespace
+  kind: RoleBinding
+- path: subjects/name
+  kind: ClusterRoleBinding
+- path: subjects/namespace
+  kind: ClusterRoleBinding

config/rbac_default/kustomization.yaml

@@ -0,0 +1,38 @@
+namePrefix: eksa-
+
+bases:
+- ../rbac
+
+configurations:
+- ./apply-namespace.yaml
+
+transformers:
+  - |-
+    apiVersion: builtin
+    kind: NamespaceTransformer
+    metadata:
+      name: notImportantHere2
+      namespace: eksa-system
+    unsetOnly: true
+
+patchesJson6902:
+- patch: |-
+    - op: replace
+      path: /metadata/name
+      value: eksa-packages-rolebinding
+  target:
+    group: rbac.authorization.k8s.io
+    kind: RoleBinding
+    name: eksa-manager-rolebinding
+    namespace: eksa-packages
+    version: v1
+- patch: |-
+    - op: replace
+      path: /metadata/name
+      value: eksa-packages-role
+  target:
+    group: rbac.authorization.k8s.io
+    kind: Role
+    name: eksa-manager-role
+    namespace: eksa-packages
+    version: v1

controllers/cluster_controller.go

@@ -58,6 +58,7 @@ type PackagesClient interface {
 	EnableFullLifecycle(ctx context.Context, log logr.Logger, clusterName, kubeConfig string, chart *v1alpha1.Image, registry *registrymirror.RegistryMirror, options ...curatedpackages.PackageControllerClientOpt) error
 	ReconcileDelete(context.Context, logr.Logger, curatedpackages.KubeDeleter, *anywherev1.Cluster) error
 	Reconcile(context.Context, logr.Logger, client.Client, *anywherev1.Cluster) error
+	UpdateSecrets(ctx context.Context, client client.Client, cluster *anywherev1.Cluster) error
 }
 
 type ProviderClusterReconcilerRegistry interface {
@@ -174,7 +175,7 @@ func (r *ClusterReconciler) SetupWithManager(mgr ctrl.Manager, log logr.Logger)
 // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch;update
 // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;delete
 // +kubebuilder:rbac:groups="",namespace=eksa-system,resources=secrets,verbs=patch;update
-// +kubebuilder:rbac:groups="",resources=namespaces,verbs=create;delete
+// +kubebuilder:rbac:groups="",resources=namespaces,verbs=create;delete;get
 // +kubebuilder:rbac:groups="",resources=nodes,verbs=list
 // +kubebuilder:rbac:groups=addons.cluster.x-k8s.io,resources=clusterresourcesets,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=anywhere.eks.amazonaws.com,resources=clusters;gitopsconfigs;snowmachineconfigs;snowdatacenterconfigs;snowippools;vspheredatacenterconfigs;vspheremachineconfigs;dockerdatacenterconfigs;tinkerbellmachineconfigs;tinkerbelldatacenterconfigs;cloudstackdatacenterconfigs;cloudstackmachineconfigs;nutanixdatacenterconfigs;nutanixmachineconfigs;awsiamconfigs;oidcconfigs;awsiamconfigs;fluxconfigs,verbs=get;list;watch;update;patch
@@ -194,7 +195,8 @@ func (r *ClusterReconciler) SetupWithManager(mgr ctrl.Manager, log logr.Logger)
 // +kubebuilder:rbac:groups=bmc.tinkerbell.org,resources=machines,verbs=list;watch
 // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=awssnowclusters;awssnowmachinetemplates;awssnowippools;vsphereclusters;vspheremachinetemplates;dockerclusters;dockermachinetemplates;tinkerbellclusters;tinkerbellmachinetemplates;cloudstackclusters;cloudstackmachinetemplates;nutanixclusters;nutanixmachinetemplates,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=packages.eks.amazonaws.com,resources=packages,verbs=create;delete;get;list;patch;update;watch
-// +kubebuilder:rbac:groups=packages.eks.amazonaws.com,namespace=eksa-system,resources=packagebundlecontrollers,verbs=delete
+// +kubebuilder:rbac:groups=packages.eks.amazonaws.com,namespace=eksa-packages,resources=packagebundlecontrollers,verbs=delete
+// +kubebuilder:rbac:groups="",namespace=eksa-packages,resources=secrets,verbs=get;patch;update
 // +kubebuilder:rbac:groups=anywhere.eks.amazonaws.com,resources=eksareleases,verbs=get;list;watch
 // The eksareleases permissions are being moved to the ClusterRole due to client trying to list this resource from cache.
 // When trying to list resources not already in cache, it starts an informer for that type using the scope of the cache.
@@ -386,6 +388,10 @@ func (r *ClusterReconciler) preClusterProviderReconcile(ctx context.Context, log
 		}
 	}
 
+	if err := r.packagesClient.UpdateSecrets(ctx, r.client, cluster); err != nil {
+		return controller.Result{}, err
+	}
+
 	return controller.Result{}, nil
 }
 

pkg/api/v1alpha1/nodeupgrade_types.go

@@ -38,6 +38,7 @@ type NodeUpgradeSpec struct {
 	Machine corev1.ObjectReference `json:"machine"`
 
 	// TODO(in-place): Determine if there's a way to get these dynamically instead of expecting it from the CRD.
+
 	KubernetesVersion string  `json:"kubernetesVersion"`
 	EtcdVersion       *string `json:"etcdVersion,omitempty"`
 	CoreDNSVersion    *string `json:"coreDNSVersion,omitempty"`

pkg/clustermanager/cluster_manager.go

@@ -1137,6 +1137,10 @@ func (c *ClusterManager) CreateEKSANamespace(ctx context.Context, cluster *types
 	return c.clusterClient.CreateNamespaceIfNotPresent(ctx, cluster.KubeconfigFile, constants.EksaSystemNamespace)
 }
 
+func (c *ClusterManager) CreatePackagesNamespace(ctx context.Context, cluster *types.Cluster) error {
+	return c.clusterClient.CreateNamespaceIfNotPresent(ctx, cluster.KubeconfigFile, constants.EksaPackagesName)
+}
+
 // CreateEKSAResources applies the eks-a cluster specs (cluster, datacenterconfig, machine configs, etc.), as well as the
 // release bundle to the cluster. Before applying the spec, we pause eksa controller cluster and datacenter webhook validation
 // so that the cluster spec can be created or updated in the cluster without webhook validation error.

pkg/curatedpackages/packagecontrollerclient.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	_ "embed"
 	"encoding/base64"
+	"encoding/json"
 	"fmt"
 	"strconv"
 	"strings"
@@ -38,6 +39,17 @@ const (
 	valueFileName     = "values.yaml"
 )
 
+type dockerConfig struct {
+	Auths map[string]*dockerAuth `json:"auths"`
+}
+
+type dockerAuth struct {
+	Username string `json:"username"`
+	Password string `json:"password"`
+	Email    string `json:"email"`
+	Auth     string `json:"auth"`
+}
+
 type PackageControllerClientOpt func(client *PackageControllerClient)
 
 type PackageControllerClient struct {
@@ -75,6 +87,7 @@ type PackageControllerClient struct {
 
 	// registryAccessTester test if the aws credential has access to registry
 	registryAccessTester RegistryAccessTester
+	flc                  bool
 }
 
 // ClientBuilder returns a k8s client for the specified cluster.
@@ -236,6 +249,56 @@ func (pc *PackageControllerClient) Enable(ctx context.Context) error {
 	return nil
 }
 
+func (pc *PackageControllerClient) UpdateSecrets(ctx context.Context, client client.Client, cluster *anywherev1.Cluster) error {
+	secretKey := types.NamespacedName{
+		Namespace: constants.EksaPackagesName,
+		Name:      "registry-mirror-cred",
+	}
+	secret := &corev1.Secret{}
+	credErr := client.Get(ctx, secretKey, secret)
+	err := fillRegistrySecret(cluster.Name, cluster.Spec.RegistryMirrorConfiguration, secret)
+	if err != nil {
+		return err
+	}
+
+	if apierrors.IsNotFound(credErr) {
+		return client.Create(ctx, secret)
+	} else if credErr == nil {
+		return client.Update(ctx, secret)
+	}
+	return credErr
+}
+
+func fillRegistrySecret(clusterName string, registry *anywherev1.RegistryMirrorConfiguration, secret *corev1.Secret) error {
+	caDataName := clusterName + "_ca.crt"
+	insecureDataName := clusterName + "_insecure"
+	secret.Data[caDataName] = []byte(registry.CACertContent)
+	secret.Data[insecureDataName] = []byte(strconv.FormatBool(registry.InsecureSkipVerify))
+
+	dconfig := &dockerConfig{Auths: make(map[string]*dockerAuth)}
+	err := json.Unmarshal(secret.Data["config.json"], dconfig)
+	if err != nil {
+		return err
+	}
+	username, password, err := config.ReadCredentials()
+	if err != nil {
+		return err
+	}
+	dconfig.Auths[registry.Endpoint] = &dockerAuth{
+		Username: username,
+		Password: password,
+		Email:    "test@test.com",
+		Auth:     base64.StdEncoding.EncodeToString([]byte(username + ":" + password)),
+	}
+
+	configJson, err := json.Marshal(dconfig)
+	if err != nil {
+		return err
+	}
+	secret.Data["config.json"] = configJson
+	return nil
+}
+
 // GetCuratedPackagesRegistries gets value for configurable registries from PBC.
 func (pc *PackageControllerClient) GetCuratedPackagesRegistries(ctx context.Context) (sourceRegistry, defaultRegistry, defaultImageRegistry string) {
 	sourceRegistry = publicProdECR

pkg/workflows/create.go

@@ -252,6 +252,12 @@ func (s *CreateWorkloadClusterTask) Run(ctx context.Context, commandContext *tas
 			return &CollectDiagnosticsTask{}
 		}
 
+		err = commandContext.ClusterManager.CreatePackagesNamespace(ctx, workloadCluster)
+		if err != nil {
+			commandContext.SetError(err)
+			return &CollectDiagnosticsTask{}
+		}
+
 		logger.Info("Installing cluster-api providers on workload cluster")
 		err = commandContext.ClusterManager.InstallCAPI(ctx, commandContext.ClusterSpec, commandContext.WorkloadCluster, commandContext.Provider)
 		if err != nil {