Merge pull request #273 from vshn/add/sli-exporter-control-plane-support

Add support for remote control plane to SLI exporter

apis/vshn/v1/dbaas_vshn_mariadb.go

@@ -171,6 +171,10 @@ type XVSHNMariaDB struct {
 	Status XVSHNMariaDBStatus `json:"status,omitempty"`
 }
 
+func (v *XVSHNMariaDB) GetInstanceNamespace() string {
+	return fmt.Sprintf("vshn-mariadb-%s", v.GetName())
+}
+
 // XVSHNMariaDBSpec defines the desired state of a VSHNMariaDB.
 type XVSHNMariaDBSpec struct {
 	// Parameters are the configurable fields of a VSHNMariaDB.

apis/vshn/v1/dbaas_vshn_redis.go

@@ -2,6 +2,7 @@ package v1
 
 import (
 	"fmt"
+
 	xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -166,6 +167,10 @@ type XVSHNRedis struct {
 	Status XVSHNRedisStatus `json:"status,omitempty"`
 }
 
+func (v *XVSHNRedis) GetInstanceNamespace() string {
+	return fmt.Sprintf("vshn-redis-%s", v.GetName())
+}
+
 // XVSHNRedisSpec defines the desired state of a VSHNRedis.
 type XVSHNRedisSpec struct {
 	// Parameters are the configurable fields of a VSHNRedis.

apis/vshn/v1/vshn_minio.go

@@ -2,6 +2,7 @@ package v1
 
 import (
 	"fmt"
+
 	xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -130,6 +131,10 @@ type XVSHNMinio struct {
 	Status XVSHNMinioStatus `json:"status,omitempty"`
 }
 
+func (v *XVSHNMinio) GetInstanceNamespace() string {
+	return fmt.Sprintf("vshn-minio-%s", v.GetName())
+}
+
 // XVSHNMinioSpec defines the desired state of a VSHNMinio.
 type XVSHNMinioSpec struct {
 	// Parameters are the configurable fields of a VSHNMinio.

cmd/sliexporter.go

@@ -2,12 +2,14 @@ package cmd
 
 import (
 	"os"
-	"strconv"
 	"time"
 
+	managedupgradev1beta1 "github.com/appuio/openshift-upgrade-controller/api/v1beta1"
 	"github.com/go-logr/logr"
 	"github.com/spf13/cobra"
+	vshnv1 "github.com/vshn/appcat/v4/apis/vshn/v1"
 	"github.com/vshn/appcat/v4/pkg"
+	"github.com/vshn/appcat/v4/pkg/common/utils"
 	maintenancecontroller "github.com/vshn/appcat/v4/pkg/sliexporter/maintenance_controller"
 	"github.com/vshn/appcat/v4/pkg/sliexporter/probes"
 	vshnkeycloakcontroller "github.com/vshn/appcat/v4/pkg/sliexporter/vshnkeycloak_controller"
@@ -16,16 +18,20 @@ import (
 	vshnpostgresqlcontroller "github.com/vshn/appcat/v4/pkg/sliexporter/vshnpostgresql_controller"
 	vshnrediscontroller "github.com/vshn/appcat/v4/pkg/sliexporter/vshnredis_controller"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/cluster"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/server"
 )
 
 type sliProber struct {
-	scheme                                                                                                                      *runtime.Scheme
-	metricsAddr, probeAddr                                                                                                      string
-	leaderElect, enableVSHNPostgreSQL, enableVSHNRedis, enableVSHNMinio, enableMaintenanceStatus, enableKeycloak, enableMariaDB bool
+	scheme                                    *runtime.Scheme
+	metricsAddr, probeAddr, serviceKubeConfig string
+	leaderElect                               bool
 }
 
 var s = sliProber{
@@ -39,21 +45,17 @@ var SLIProberCMD = &cobra.Command{
 	RunE:  s.executeSLIProber,
 }
 
+const (
+	startupGraceMin = 10
+)
+
 func init() {
 	SLIProberCMD.Flags().StringVar(&s.metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	SLIProberCMD.Flags().StringVar(&s.probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	SLIProberCMD.Flags().BoolVar(&s.leaderElect, "leader-elect", false, "Enable leader election for controller manager. "+
 		"Enabling this will ensure there is only one active controller manager.")
-	SLIProberCMD.Flags().BoolVar(&s.enableVSHNPostgreSQL, "vshn-postgresql", getEnvBool("APPCAT_SLI_VSHNPOSTGRESQL"),
-		"Enable probing of VSHNPostgreSQL instances")
-	SLIProberCMD.Flags().BoolVar(&s.enableVSHNRedis, "vshn-redis", getEnvBool("APPCAT_SLI_VSHNREDIS"),
-		"Enable probing of VSHNRedis instances")
-	SLIProberCMD.Flags().BoolVar(&s.enableVSHNMinio, "vshn-minio", getEnvBool("APPCAT_SLI_VSHNMINIO"),
-		"Enable probing of VSHNMinio instances")
-	SLIProberCMD.Flags().BoolVar(&s.enableMaintenanceStatus, "vshn-track-oc-maintenance-status", getEnvBool("APPCAT_SLI_TRACK_OC_MAINTENANCE_STATUS"),
-		"Enable oc maintenance status observer. Will set the labels 'maintenance' accordingly.")
-	SLIProberCMD.Flags().BoolVar(&s.enableKeycloak, "vshn-keycloak", getEnvBool("APPCAT_SLI_VSHNKEYCLOAK"), "Enable probing of VSHNKeycloak instances")
-	SLIProberCMD.Flags().BoolVar(&s.enableMariaDB, "vshn-mariadb", getEnvBool("APPCAT_SLI_VSHNMARIADB"), "Enable probing of VSHNMariaDB instances")
+	SLIProberCMD.Flags().StringVar(&s.serviceKubeConfig, "service-kubeconfig", os.Getenv("SERVICE_KUBECONFIG"),
+		"Kubeconfig for the service cluster itself, usually only for debugging purpose. The sliprober should run on the service clusters and thus use the in-cluster-config.")
 }
 
 func (s *sliProber) executeSLIProber(cmd *cobra.Command, _ []string) error {
@@ -74,8 +76,20 @@ func (s *sliProber) executeSLIProber(cmd *cobra.Command, _ []string) error {
 		return err
 	}
 
+	config, err := getServiceClusterConfig()
+	if err != nil {
+		return err
+	}
+
+	scClient, err := client.New(config, client.Options{
+		Scheme: pkg.SetupScheme(),
+	})
+	if err != nil {
+		return err
+	}
+
 	maintenanceRecociler := &maintenancecontroller.MaintenanceReconciler{
-		Client: mgr.GetClient(),
+		Client: scClient,
 	}
 
 	probeManager := probes.NewManager(log, maintenanceRecociler)
@@ -86,74 +100,88 @@ func (s *sliProber) executeSLIProber(cmd *cobra.Command, _ []string) error {
 		return err
 	}
 
-	if s.enableVSHNPostgreSQL {
+	if utils.IsKindAvailable(vshnv1.GroupVersion, "XVSHNPostgreSQL", ctrl.GetConfigOrDie()) {
+		log.Info("Enabling VSHNPostgreSQL controller")
 		if err = (&vshnpostgresqlcontroller.VSHNPostgreSQLReconciler{
 			Client:             mgr.GetClient(),
 			Scheme:             mgr.GetScheme(),
 			ProbeManager:       &probeManager,
-			StartupGracePeriod: 15 * time.Minute,
+			StartupGracePeriod: startupGraceMin * time.Minute,
 			PostgreDialer:      probes.NewPostgreSQL,
+			ScClient:           scClient,
 		}).SetupWithManager(mgr); err != nil {
 			log.Error(err, "unable to create controller", "controller", "VSHNPostgreSQL")
 			return err
 		}
 	}
-	if s.enableVSHNRedis {
+	if utils.IsKindAvailable(vshnv1.GroupVersion, "XVSHNRedis", ctrl.GetConfigOrDie()) {
 		log.Info("Enabling VSHNRedis controller")
 		if err = (&vshnrediscontroller.VSHNRedisReconciler{
 			Client:             mgr.GetClient(),
 			Scheme:             mgr.GetScheme(),
 			ProbeManager:       &probeManager,
-			StartupGracePeriod: 10 * time.Minute,
+			StartupGracePeriod: startupGraceMin * time.Minute,
 			RedisDialer:        probes.NewRedis,
+			ScClient:           scClient,
 		}).SetupWithManager(mgr); err != nil {
 			log.Error(err, "unable to create controller", "controller", "VSHNRedis")
 			return err
 		}
 	}
 
-	if s.enableVSHNMinio {
+	if utils.IsKindAvailable(vshnv1.GroupVersion, "XVSHNMinio", ctrl.GetConfigOrDie()) {
 		log.Info("Enabling VSHNRedis controller")
 		if err = (&vshnminiocontroller.VSHNMinioReconciler{
 			Client:             mgr.GetClient(),
 			Scheme:             mgr.GetScheme(),
 			ProbeManager:       &probeManager,
-			StartupGracePeriod: 10 * time.Minute,
+			StartupGracePeriod: startupGraceMin * time.Minute,
 			MinioDialer:        probes.NewMinio,
+			ScClient:           scClient,
 		}).SetupWithManager(mgr); err != nil {
 			log.Error(err, "unable to create controller", "controller", "VSHNRedis")
 			return err
 		}
 	}
 
-	if s.enableKeycloak {
+	if utils.IsKindAvailable(vshnv1.GroupVersion, "XVSHNKeycloak", ctrl.GetConfigOrDie()) {
 		log.Info("Enablign VSHNKeycloak controller")
 		if err = (&vshnkeycloakcontroller.VSHNKeycloakReconciler{
 			Client:             mgr.GetClient(),
 			Scheme:             mgr.GetScheme(),
 			ProbeManager:       &probeManager,
-			StartupGracePeriod: 10 * time.Minute,
+			StartupGracePeriod: startupGraceMin * time.Minute,
+			ScClient:           scClient,
 		}).SetupWithManager(mgr); err != nil {
 			log.Error(err, "unable to create controller", "controller", "VSHNKeycloak")
 			return err
 		}
 	}
 
-	if s.enableMaintenanceStatus {
+	if utils.IsKindAvailable(managedupgradev1beta1.GroupVersion, "UpgradeJob", config) {
 		log.Info("Enable OC maintenance observer")
-		if err = maintenanceRecociler.SetupWithManager(mgr); err != nil {
+		serviceCluster, err := getServiceCluster(config)
+		if err != nil {
+			return err
+		}
+		if err = maintenanceRecociler.SetupWithManager(mgr, serviceCluster); err != nil {
 			log.Error(err, "unable to create controller", "controller", "Maintenance Observer")
 			return err
 		}
+		err = mgr.Add(*serviceCluster)
+		if err != nil {
+			return err
+		}
 	}
-	if s.enableMariaDB {
+	if utils.IsKindAvailable(vshnv1.GroupVersion, "XVSHNMariaDB", ctrl.GetConfigOrDie()) {
 		log.Info("Enabling VSHNMariaDB controller")
 		if err = (&vshnmariadbcontroller.VSHNMariaDBReconciler{
 			Client:             mgr.GetClient(),
 			Scheme:             mgr.GetScheme(),
 			ProbeManager:       &probeManager,
-			StartupGracePeriod: 1 * time.Minute,
+			StartupGracePeriod: startupGraceMin * time.Minute,
 			MariaDBDialer:      probes.NewMariaDB,
+			ScClient:           scClient,
 		}).SetupWithManager(mgr); err != nil {
 			log.Error(err, "unable to create controller", "controller", "VSHNMariadb")
 			return err
@@ -179,7 +207,39 @@ func (s *sliProber) executeSLIProber(cmd *cobra.Command, _ []string) error {
 	return nil
 }
 
-func getEnvBool(key string) bool {
-	b, err := strconv.ParseBool(os.Getenv(key))
-	return err == nil && b
+func getServiceCluster(config *rest.Config) (*cluster.Cluster, error) {
+	serviceCluster, err := cluster.New(config, func(o *cluster.Options) {
+		o.Scheme = pkg.SetupScheme()
+	})
+	return &serviceCluster, err
+}
+
+// getServiceClusterConfig will create an incluster config by default.
+// If serviceKubeConfig is set, it will use that instead.
+func getServiceClusterConfig() (*rest.Config, error) {
+
+	if s.serviceKubeConfig != "" {
+		kubeconfig, err := os.ReadFile(s.serviceKubeConfig)
+		if err != nil {
+			return nil, err
+		}
+
+		clientConfig, err := clientcmd.NewClientConfigFromBytes(kubeconfig)
+		if err != nil {
+			return nil, err
+		}
+
+		config, err := clientConfig.ClientConfig()
+		if err != nil {
+			return nil, err
+		}
+		return config, nil
+	}
+
+	config, err := rest.InClusterConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	return config, nil
 }

pkg/apiserver/appcat/appcat.go

@@ -3,8 +3,8 @@ package appcat
 import (
 	crossplane "github.com/crossplane/crossplane/apis/apiextensions/v1"
 	appcatv1 "github.com/vshn/appcat/v4/apis/apiserver/v1"
-	"github.com/vshn/appcat/v4/pkg/apiserver"
 	"github.com/vshn/appcat/v4/pkg/apiserver/noop"
+	"github.com/vshn/appcat/v4/pkg/common/utils"
 	"k8s.io/apimachinery/pkg/runtime"
 	genericregistry "k8s.io/apiserver/pkg/registry/generic"
 	"k8s.io/apiserver/pkg/registry/rest"
@@ -37,7 +37,7 @@ func New() restbuilder.ResourceHandlerProvider {
 
 		noopImplementation := noop.New(s, &appcatv1.AppCat{}, &appcatv1.AppCatList{})
 
-		if !apiserver.IsTypeAvailable(crossplane.SchemeGroupVersion.String(), "Composition") {
+		if !utils.IsKindAvailable(crossplane.SchemeGroupVersion, "Composition", loopback.GetLoopbackMasterClientConfig()) {
 			return noopImplementation, nil
 		}
 

pkg/apiserver/common.go

@@ -6,8 +6,6 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/client-go/discovery"
-	"sigs.k8s.io/apiserver-runtime/pkg/util/loopback"
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -91,21 +89,3 @@ func GetBackupColumnDefinition() []metav1.TableColumnDefinition {
 		{Name: "Age", Type: "date", Description: desc["creationTimestamp"]},
 	}
 }
-
-func IsTypeAvailable(gv string, kind string) bool {
-	d, err := discovery.NewDiscoveryClientForConfig(loopback.GetLoopbackMasterClientConfig())
-	if err != nil {
-		return false
-	}
-	resources, err := d.ServerResourcesForGroupVersion(gv)
-	if err != nil {
-		return false
-	}
-
-	for _, res := range resources.APIResources {
-		if res.Kind == kind {
-			return true
-		}
-	}
-	return false
-}

pkg/apiserver/vshn/mariadb/backup.go

@@ -4,9 +4,9 @@ import (
 	k8upv1 "github.com/k8up-io/k8up/v2/api/v1"
 	appcatv1 "github.com/vshn/appcat/v4/apis/apiserver/v1"
 	vshnv1 "github.com/vshn/appcat/v4/apis/vshn/v1"
-	"github.com/vshn/appcat/v4/pkg/apiserver"
 	"github.com/vshn/appcat/v4/pkg/apiserver/noop"
 	"github.com/vshn/appcat/v4/pkg/apiserver/vshn/k8up"
+	"github.com/vshn/appcat/v4/pkg/common/utils"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	genericregistry "k8s.io/apiserver/pkg/registry/generic"
@@ -37,7 +37,7 @@ func New() restbuilder.ResourceHandlerProvider {
 
 		noopImplementation := noop.New(s, &appcatv1.VSHNMariaDBBackup{}, &appcatv1.VSHNMariaDBBackupList{})
 
-		if !apiserver.IsTypeAvailable(vshnv1.GroupVersion.String(), "XVSHNMariaDB") {
+		if !utils.IsKindAvailable(vshnv1.GroupVersion, "XVSHNMariaDB", loopback.GetLoopbackMasterClientConfig()) {
 			return noopImplementation, nil
 		}
 

pkg/apiserver/vshn/nextcloud/backup.go

@@ -4,10 +4,10 @@ import (
 	k8upv1 "github.com/k8up-io/k8up/v2/api/v1"
 	appcatv1 "github.com/vshn/appcat/v4/apis/apiserver/v1"
 	vshnv1 "github.com/vshn/appcat/v4/apis/vshn/v1"
-	"github.com/vshn/appcat/v4/pkg/apiserver"
 	"github.com/vshn/appcat/v4/pkg/apiserver/noop"
 	"github.com/vshn/appcat/v4/pkg/apiserver/vshn/k8up"
 	"github.com/vshn/appcat/v4/pkg/apiserver/vshn/postgres"
+	"github.com/vshn/appcat/v4/pkg/common/utils"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	genericregistry "k8s.io/apiserver/pkg/registry/generic"
@@ -41,7 +41,7 @@ func New() restbuilder.ResourceHandlerProvider {
 
 		noopImplementation := noop.New(s, &appcatv1.VSHNNextcloudBackup{}, &appcatv1.VSHNNextcloudBackupList{})
 
-		if !apiserver.IsTypeAvailable(vshnv1.GroupVersion.String(), "XVSHNNextcloud") {
+		if !utils.IsKindAvailable(vshnv1.GroupVersion, "XVSHNNextcloud", loopback.GetLoopbackMasterClientConfig()) {
 			return noopImplementation, nil
 		}
 

pkg/apiserver/vshn/postgres/backup.go

@@ -3,8 +3,8 @@ package postgres
 import (
 	appcatv1 "github.com/vshn/appcat/v4/apis/apiserver/v1"
 	vshnv1 "github.com/vshn/appcat/v4/apis/vshn/v1"
-	"github.com/vshn/appcat/v4/pkg/apiserver"
 	"github.com/vshn/appcat/v4/pkg/apiserver/noop"
+	"github.com/vshn/appcat/v4/pkg/common/utils"
 	"k8s.io/apimachinery/pkg/runtime"
 	genericregistry "k8s.io/apiserver/pkg/registry/generic"
 	"k8s.io/apiserver/pkg/registry/rest"
@@ -29,7 +29,7 @@ func New() restbuilder.ResourceHandlerProvider {
 
 		noopImplementation := noop.New(s, &appcatv1.VSHNPostgresBackup{}, &appcatv1.VSHNPostgresBackupList{})
 
-		if !apiserver.IsTypeAvailable(vshnv1.GroupVersion.String(), "VSHNPostgreSQL") {
+		if !utils.IsKindAvailable(vshnv1.GroupVersion, "VSHNPostgreSQL", loopback.GetLoopbackMasterClientConfig()) {
 			return noopImplementation, nil
 		}