diff --git a/cmd/ipam-controller/app/app.go b/cmd/ipam-controller/app/app.go index b21bcc4..aa2d87d 100644 --- a/cmd/ipam-controller/app/app.go +++ b/cmd/ipam-controller/app/app.go @@ -31,7 +31,6 @@ import ( "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/client" ctrlconf "sigs.k8s.io/controller-runtime/pkg/config" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/healthz" @@ -49,7 +48,6 @@ import ( cidrpoolctrl "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/controllers/cidrpool" ippoolctrl "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/controllers/ippool" nodectrl "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/controllers/node" - "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/migrator" "github.com/Mellanox/nvidia-k8s-ipam/pkg/version" ) @@ -135,34 +133,12 @@ func RunController(ctx context.Context, config *rest.Config, opts *options.Optio return err } - k8sClient, err := client.New(config, - client.Options{Scheme: mgr.GetScheme(), Mapper: mgr.GetRESTMapper()}) - if err != nil { - logger.Error(err, "failed to create k8sClient client") - os.Exit(1) - } - - migrationChan := make(chan struct{}) - m := migrator.Migrator{ - IPPoolsNamespace: opts.IPPoolsNamespace, - K8sClient: k8sClient, - MigrationCh: migrationChan, - LeaderElection: opts.EnableLeaderElection, - Logger: logger.WithName("Migrator"), - } - err = mgr.Add(&m) - if err != nil { - logger.Error(err, "failed to add Migrator to the Manager") - os.Exit(1) - } - ipPoolNodeEventCH := make(chan event.GenericEvent, 1) cidrPoolNodeEventCH := make(chan event.GenericEvent, 1) if err = (&nodectrl.NodeReconciler{ IPPoolNodeEventCH: ipPoolNodeEventCH, CIDRPoolNodeEventCH: cidrPoolNodeEventCH, - MigrationCh: migrationChan, PoolsNamespace: opts.IPPoolsNamespace, Client: mgr.GetClient(), Scheme: mgr.GetScheme(), @@ -186,7 +162,6 @@ func RunController(ctx context.Context, config *rest.Config, opts *options.Optio PoolsNamespace: opts.IPPoolsNamespace, Client: mgr.GetClient(), Scheme: mgr.GetScheme(), - MigrationCh: migrationChan, }).SetupWithManager(mgr); err != nil { logger.Error(err, "unable to create controller", "controller", "IPPool") return err diff --git a/pkg/ipam-controller/config/config.go b/pkg/ipam-controller/config/config.go deleted file mode 100644 index 3ed6356..0000000 --- a/pkg/ipam-controller/config/config.go +++ /dev/null @@ -1,78 +0,0 @@ -/* - Copyright 2023, NVIDIA CORPORATION & AFFILIATES - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package config - -import ( - "fmt" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - metaValidation "k8s.io/apimachinery/pkg/apis/meta/v1/validation" - validationField "k8s.io/apimachinery/pkg/util/validation/field" - - "github.com/Mellanox/nvidia-k8s-ipam/api/v1alpha1" -) - -const ( - // ConfigMapKey is the name of the key in ConfigMap which store - // configuration - ConfigMapKey = "config" -) - -// PoolConfig contains configuration for IP pool -type PoolConfig struct { - // subnet of the pool - Subnet string `json:"subnet"` - // amount of IPs to allocate for each node, - // must be less than amount of available IPs in the subnet - PerNodeBlockSize int `json:"perNodeBlockSize"` - // gateway for the pool, defaults to the first IP of the Subnet if not set - Gateway string `json:"gateway"` -} - -// Config contains configuration for IPAM controller -type Config struct { - // configuration for IP pools - Pools map[string]PoolConfig `json:"pools"` - // selector for nodes, if empty match all nodes - NodeSelector map[string]string `json:"nodeSelector"` -} - -// Validate validates IPAM controller config -func (c *Config) Validate() error { - if len(c.Pools) == 0 { - return fmt.Errorf("no IP pools in the config") - } - if errList := metaValidation.ValidateLabels(c.NodeSelector, - validationField.NewPath("config", "nodeSelector")); len(errList) > 0 { - return errList.ToAggregate() - } - for poolName, pool := range c.Pools { - return ValidatePool(poolName, pool.Subnet, pool.Gateway, pool.PerNodeBlockSize) - } - return nil -} - -// ValidatePool validates the IPPool parameters -func ValidatePool(name string, subnet string, gateway string, blockSize int) error { - ipPool := v1alpha1.IPPool{ - ObjectMeta: metav1.ObjectMeta{Name: name}, - Spec: v1alpha1.IPPoolSpec{ - Subnet: subnet, - PerNodeBlockSize: blockSize, - Gateway: gateway, - NodeSelector: nil, - }, - } - return ipPool.Validate().ToAggregate() -} diff --git a/pkg/ipam-controller/config/config_suite_test.go b/pkg/ipam-controller/config/config_suite_test.go deleted file mode 100644 index 39b0232..0000000 --- a/pkg/ipam-controller/config/config_suite_test.go +++ /dev/null @@ -1,26 +0,0 @@ -/* - Copyright 2023, NVIDIA CORPORATION & AFFILIATES - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package config_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestConfig(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Config Suite") -} diff --git a/pkg/ipam-controller/config/config_test.go b/pkg/ipam-controller/config/config_test.go deleted file mode 100644 index 990332f..0000000 --- a/pkg/ipam-controller/config/config_test.go +++ /dev/null @@ -1,65 +0,0 @@ -/* - Copyright 2023, NVIDIA CORPORATION & AFFILIATES - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package config_test - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/config" -) - -func getValidPool() config.PoolConfig { - return config.PoolConfig{Gateway: "192.168.1.1", Subnet: "192.168.1.0/24", PerNodeBlockSize: 10} -} - -var _ = Describe("Config", func() { - It("Empty config", func() { - cfg := &config.Config{} - Expect(cfg.Validate()).To(HaveOccurred()) - }) - It("Valid config", func() { - cfg := &config.Config{Pools: map[string]config.PoolConfig{"pool1": getValidPool()}} - Expect(cfg.Validate()).NotTo(HaveOccurred()) - }) - It("Invalid nodeSelector", func() { - cfg := &config.Config{Pools: map[string]config.PoolConfig{"pool1": getValidPool()}} - cfg.NodeSelector = map[string]string{",-_invalid": ""} - Expect(cfg.Validate()).To(HaveOccurred()) - }) - It("Invalid pool: invalid subnet", func() { - poolConfig := getValidPool() - poolConfig.Subnet = "aaaa" - cfg := &config.Config{Pools: map[string]config.PoolConfig{"pool1": poolConfig}} - Expect(cfg.Validate()).To(HaveOccurred()) - }) - It("Invalid pool: gw outside of the subnet", func() { - poolConfig := getValidPool() - poolConfig.Gateway = "10.10.10.1" - cfg := &config.Config{Pools: map[string]config.PoolConfig{"pool1": poolConfig}} - Expect(cfg.Validate()).To(HaveOccurred()) - }) - It("Invalid pool: subnet is too small", func() { - poolConfig := getValidPool() - poolConfig.PerNodeBlockSize = 255 - cfg := &config.Config{Pools: map[string]config.PoolConfig{"pool1": poolConfig}} - Expect(cfg.Validate()).To(HaveOccurred()) - }) - It("Invalid pool: perNodeBlockSize too small", func() { - poolConfig := getValidPool() - poolConfig.PerNodeBlockSize = 0 - cfg := &config.Config{Pools: map[string]config.PoolConfig{"pool1": poolConfig}} - Expect(cfg.Validate()).To(HaveOccurred()) - }) -}) diff --git a/pkg/ipam-controller/controllers/ippool/ippool.go b/pkg/ipam-controller/controllers/ippool/ippool.go index 4440011..a73451e 100644 --- a/pkg/ipam-controller/controllers/ippool/ippool.go +++ b/pkg/ipam-controller/controllers/ippool/ippool.go @@ -45,7 +45,6 @@ import ( type IPPoolReconciler struct { PoolsNamespace string NodeEventCh chan event.GenericEvent - MigrationCh chan struct{} client.Client Scheme *runtime.Scheme recorder record.EventRecorder @@ -53,11 +52,6 @@ type IPPoolReconciler struct { // Reconcile contains logic to sync IPPool objects func (r *IPPoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - select { - case <-r.MigrationCh: - case <-ctx.Done(): - return ctrl.Result{}, fmt.Errorf("canceled") - } reqLog := log.FromContext(ctx) if req.Namespace != r.PoolsNamespace { // this should never happen because of the watcher configuration of the manager from controller-runtime pkg diff --git a/pkg/ipam-controller/controllers/node/node.go b/pkg/ipam-controller/controllers/node/node.go index 7cb4fa3..6955c85 100644 --- a/pkg/ipam-controller/controllers/node/node.go +++ b/pkg/ipam-controller/controllers/node/node.go @@ -15,7 +15,6 @@ package controllers import ( "context" - "fmt" apiErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -35,18 +34,12 @@ type NodeReconciler struct { PoolsNamespace string IPPoolNodeEventCH chan event.GenericEvent CIDRPoolNodeEventCH chan event.GenericEvent - MigrationCh chan struct{} client.Client Scheme *runtime.Scheme } // Reconcile contains logic to sync Node objects func (r *NodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - select { - case <-r.MigrationCh: - case <-ctx.Done(): - return ctrl.Result{}, fmt.Errorf("canceled") - } reqLog := log.FromContext(ctx) reqLog.Info("Notification on Node", "name", req.Name) node := &corev1.Node{} diff --git a/pkg/ipam-controller/migrator/migrator.go b/pkg/ipam-controller/migrator/migrator.go deleted file mode 100644 index 42688fd..0000000 --- a/pkg/ipam-controller/migrator/migrator.go +++ /dev/null @@ -1,278 +0,0 @@ -/* - Copyright 2023, NVIDIA CORPORATION & AFFILIATES - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package migrator - -import ( - "context" - "encoding/json" - "fmt" - "net" - "os" - "reflect" - "strings" - - "github.com/go-logr/logr" - corev1 "k8s.io/api/core/v1" - apiErrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/sets" - "sigs.k8s.io/controller-runtime/pkg/client" - - ipamv1alpha1 "github.com/Mellanox/nvidia-k8s-ipam/api/v1alpha1" - "github.com/Mellanox/nvidia-k8s-ipam/pkg/common" - "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/config" - "github.com/Mellanox/nvidia-k8s-ipam/pkg/pool" -) - -const ( - // EnvDisableMigration contains the name of the environment variable which can be used - // to disable migration - EnvDisableMigration = "MIGRATOR_DISABLE_MIGRATION" - // EnvConfigMapName contains the name of the environment variable which can be used - // to specify the ConfigMap name containing the configuration to migrate from - EnvConfigMapName = "CONFIGMAP_NAME" - // EnvConfigMapNamespace contains the name of the environment variable which can be used - // to specify the namespace of the ConfigMap containing the configuration to migrate from - EnvConfigMapNamespace = "CONFIGMAP_NAMESPACE" - // DefaultConfigMapName is the default ConfigMap name used to read the configuration to - // migrate from - DefaultConfigMapName = "nvidia-k8s-ipam-config" -) - -// Migrator migrate from CM config to IPPool CR -type Migrator struct { - K8sClient client.Client - IPPoolsNamespace string - MigrationCh chan struct{} - LeaderElection bool - Logger logr.Logger -} - -// Implements manager.Runnable -func (m *Migrator) Start(ctx context.Context) error { - err := Migrate(ctx, m.Logger, m.K8sClient, m.IPPoolsNamespace) - if err != nil { - m.Logger.Error(err, fmt.Sprintf("failed to migrate NV-IPAM config from ConfigMap, "+ - "set %s env variable to disable migration", EnvDisableMigration)) - return err - } - close(m.MigrationCh) - return nil -} - -// Implements manager.NeedLeaderElection -func (m *Migrator) NeedLeaderElection() bool { - return m.LeaderElection -} - -// Migrate reads the ConfigMap with the IPAM configuration, reads the allocations -// from the Nodes annotation, create IPPool CRs and delete the ConfigMap and annotations -func Migrate(ctx context.Context, logger logr.Logger, c client.Client, poolNamespace string) error { - if os.Getenv(EnvDisableMigration) != "" { - logger.Info(fmt.Sprintf("%s set, skip controller migration", EnvDisableMigration)) - return nil - } - - cmName := DefaultConfigMapName - if os.Getenv(EnvConfigMapName) != "" { - cmName = os.Getenv(EnvConfigMapName) - } - cmNamespace := poolNamespace - if os.Getenv(EnvConfigMapNamespace) != "" { - cmNamespace = os.Getenv(EnvConfigMapNamespace) - } - - cfg := &corev1.ConfigMap{} - key := types.NamespacedName{ - Name: cmName, - Namespace: cmNamespace, - } - err := c.Get(ctx, key, cfg) - if err != nil { - if apiErrors.IsNotFound(err) { - logger.Info("ConfigMap not found, skipping migration") - return nil - } - logger.Error(err, "failed to read ConfigMap object") - return err - } - - confData, exist := cfg.Data[config.ConfigMapKey] - if !exist { - err = fmt.Errorf("invalid configuration: ConfigMap %s doesn't contain %s key", - key, config.ConfigMapKey) - logger.Error(err, "Invalid config, no data") - return err - } - controllerConfig := &config.Config{} - if err := json.Unmarshal([]byte(confData), controllerConfig); err != nil { - logger.Error(err, fmt.Sprintf("invalid configuration: ConfigMap %s contains invalid JSON", - config.ConfigMapKey)) - return err - } - if err := controllerConfig.Validate(); err != nil { - logger.Error(err, fmt.Sprintf("invalid configuration: ConfigMap %s contains invalid config", - config.ConfigMapKey)) - return err - } - - pools := buildIPPools(controllerConfig, poolNamespace) - - for name, p := range pools { - err = c.Create(ctx, p) - logger.Info(fmt.Sprintf("Creating IPPool: %v", p)) - if apiErrors.IsAlreadyExists(err) { - existingPool := &ipamv1alpha1.IPPool{} - err = c.Get(ctx, client.ObjectKeyFromObject(p), existingPool) - if err != nil { - logger.Info("fail to get existing pool", "pool name", name) - return err - } - if !reflect.DeepEqual(existingPool.Spec, p.Spec) { - logger.Info("existing pool has different spec than config map setting", "pool name", name) - return fmt.Errorf("existing pool has different spec than config map setting") - } - } else if err != nil { - logger.Info("fail to create pool", "pool name", name) - return err - } - } - - err = updateAllocations(ctx, c, logger, pools, poolNamespace) - if err != nil { - return err - } - - err = c.Delete(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: cmNamespace, Name: cmName}}) - if err != nil { - logger.Info("fail to delete nv-ipam config map") - return err - } - - return nil -} - -func updateAllocations(ctx context.Context, c client.Client, - logger logr.Logger, pools map[string]*ipamv1alpha1.IPPool, poolNamespace string) error { - nodeList := &corev1.NodeList{} - if err := c.List(ctx, nodeList); err != nil { - logger.Error(err, "failed to list nodes") - return err - } - nodesToClearAnnotation := sets.New[string]() - for poolName := range pools { - p := &ipamv1alpha1.IPPool{} - key := types.NamespacedName{ - Name: poolName, - Namespace: poolNamespace, - } - err := c.Get(ctx, key, p) - if err != nil { - logger.Info("fail getting IPPool", "reason", err.Error()) - return err - } - if len(p.Status.Allocations) > 0 { - logger.Info("skipping migration for IPPool, already has allocation", "ippool", poolName) - continue - } - allocs := make([]ipamv1alpha1.Allocation, 0) - for i := range nodeList.Items { - node := nodeList.Items[i] - nodeLog := logger.WithValues("node", node.Name) - poolCfg, err := pool.NewConfigReader(&node) - if err != nil { - nodeLog.Info("skip loading data from the node", "reason", err.Error()) - continue - } - nodesToClearAnnotation.Insert(node.Name) - nodeIPPoolConfig := poolCfg.GetPoolByKey(common.GetPoolKey(poolName, common.PoolTypeIPPool)) - if nodeIPPoolConfig == nil { - nodeLog.Info("skip loading data for pool from the node, pool not configured", "node", node.Name, "pool", poolName) - continue - } - alloc := ipamv1alpha1.Allocation{ - NodeName: node.Name, - StartIP: nodeIPPoolConfig.StartIP, - EndIP: nodeIPPoolConfig.EndIP, - } - allocs = append(allocs, alloc) - } - if len(allocs) != 0 { - p.Status.Allocations = allocs - logger.Info(fmt.Sprintf("Updating IPPool status: %v", p)) - err = c.Status().Update(ctx, p) - if err != nil { - logger.Info("fail to update pool allocation from node", "reason", err.Error()) - return err - } - } - } - - for _, nodeName := range sets.List(nodesToClearAnnotation) { - logger.Info("clear IPBlocksAnnotation from node", "name", nodeName) - fmtKey := strings.ReplaceAll(pool.IPBlocksAnnotation, "/", "~1") - patch := []byte(fmt.Sprintf("[{\"op\": \"remove\", \"path\": \"/metadata/annotations/%s\"}]", fmtKey)) - err := c.Patch(ctx, &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: nodeName, - }, - }, client.RawPatch(types.JSONPatchType, patch)) - if err != nil { - logger.Info("fail to remove IPBlocksAnnotation from node", "name", nodeName, "reason", err.Error()) - return err - } - } - return nil -} - -func buildIPPools(controllerConfig *config.Config, poolNamespace string) map[string]*ipamv1alpha1.IPPool { - var nodeSelector *corev1.NodeSelector - if len(controllerConfig.NodeSelector) > 0 { - nodeSelector = &corev1.NodeSelector{} - selectorsItems := make([]corev1.NodeSelectorTerm, 0, len(controllerConfig.NodeSelector)) - for k, v := range controllerConfig.NodeSelector { - selector := corev1.NodeSelectorTerm{ - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: k, - Operator: corev1.NodeSelectorOpIn, - Values: []string{v}, - }, - }, - MatchFields: nil, - } - selectorsItems = append(selectorsItems, selector) - } - nodeSelector.NodeSelectorTerms = selectorsItems - } - pools := make(map[string]*ipamv1alpha1.IPPool) - for pName, p := range controllerConfig.Pools { - // already validated by Validate function - _, subnet, _ := net.ParseCIDR(p.Subnet) - pools[pName] = &ipamv1alpha1.IPPool{ - ObjectMeta: metav1.ObjectMeta{ - Name: pName, - Namespace: poolNamespace, - }, - Spec: ipamv1alpha1.IPPoolSpec{ - Subnet: subnet.String(), - Gateway: p.Gateway, - PerNodeBlockSize: p.PerNodeBlockSize, - NodeSelector: nodeSelector, - }, - } - } - return pools -} diff --git a/pkg/ipam-controller/migrator/migrator_suite_test.go b/pkg/ipam-controller/migrator/migrator_suite_test.go deleted file mode 100644 index 883e135..0000000 --- a/pkg/ipam-controller/migrator/migrator_suite_test.go +++ /dev/null @@ -1,82 +0,0 @@ -/* - Copyright 2023, NVIDIA CORPORATION & AFFILIATES - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package migrator_test - -import ( - "context" - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/envtest" - - ipamv1alpha1 "github.com/Mellanox/nvidia-k8s-ipam/api/v1alpha1" -) - -const ( - TestNamespace = "test-ns" - TestConfigMapName = "nvidia-k8s-ipam-config" -) - -var ( - cfg *rest.Config - k8sClient client.Client - testEnv *envtest.Environment - cFunc context.CancelFunc - ctx context.Context -) - -func TestMigrator(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "IPAM Controller Migrator Suite") -} - -var _ = BeforeSuite(func() { - By("bootstrapping test environment") - testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{"../../../deploy/crds"}, - CRDInstallOptions: envtest.CRDInstallOptions{ - ErrorIfPathMissing: true, - }, - } - - ctx, cFunc = context.WithCancel(context.Background()) - - var err error - err = ipamv1alpha1.AddToScheme(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) - - // cfg is defined in this file globally. - cfg, err = testEnv.Start() - Expect(err).NotTo(HaveOccurred()) - Expect(cfg).NotTo(BeNil()) - - k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) - Expect(err).NotTo(HaveOccurred()) - Expect(k8sClient).NotTo(BeNil()) - - Expect(k8sClient.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: TestNamespace}})).To(BeNil()) -}) - -var _ = AfterSuite(func() { - cFunc() - By("tearing down the test environment") - err := testEnv.Stop() - Expect(err).NotTo(HaveOccurred()) -}) diff --git a/pkg/ipam-controller/migrator/migrator_test.go b/pkg/ipam-controller/migrator/migrator_test.go deleted file mode 100644 index 52a5661..0000000 --- a/pkg/ipam-controller/migrator/migrator_test.go +++ /dev/null @@ -1,235 +0,0 @@ -/* - Copyright 2023, NVIDIA CORPORATION & AFFILIATES - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package migrator_test - -import ( - "fmt" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" - apiErrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/klog/v2" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/config" - "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-controller/migrator" - "github.com/Mellanox/nvidia-k8s-ipam/pkg/pool" - - ipamv1alpha1 "github.com/Mellanox/nvidia-k8s-ipam/api/v1alpha1" -) - -const ( - pool1Name = "pool1" - pool2Name = "pool2" -) - -func updateConfigMap(data string) { - d := map[string]string{config.ConfigMapKey: data} - err := k8sClient.Create(ctx, &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: TestConfigMapName, Namespace: TestNamespace}, - Data: d, - }) - if err == nil { - return - } - if apiErrors.IsAlreadyExists(err) { - configMap := &corev1.ConfigMap{} - Expect(k8sClient.Get( - ctx, types.NamespacedName{Name: TestConfigMapName, Namespace: TestNamespace}, configMap)).NotTo(HaveOccurred()) - configMap.Data = d - Expect(k8sClient.Update( - ctx, configMap)).NotTo(HaveOccurred()) - } else { - Expect(err).NotTo(HaveOccurred()) - } -} - -var validConfig = fmt.Sprintf(` - { - "pools": { - "%s": { "subnet": "192.168.0.0/16", "perNodeBlockSize": 10 , "gateway": "192.168.0.1"}, - "%s": { "subnet": "172.16.0.0/16", "perNodeBlockSize": 50 , "gateway": "172.16.0.1"} - }, - "nodeSelector": {"foo": "bar"} - } -`, pool1Name, pool2Name) - -func createNode(name string) *corev1.Node { - node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: name}} - Expect(k8sClient.Create(ctx, node)).NotTo(HaveOccurred()) - return node -} - -func getNode(name string) *corev1.Node { - node := &corev1.Node{} - Expect(k8sClient.Get(ctx, types.NamespacedName{Name: name}, node)).NotTo(HaveOccurred()) - return node -} - -func updateNode(node *corev1.Node) *corev1.Node { - Expect(k8sClient.Update(ctx, node)).NotTo(HaveOccurred()) - return node -} - -func getRangeFromNode(nodeName string) map[string]*pool.Pool { - node := getNode(nodeName) - poolCfg, err := pool.NewConfigReader(node) - if err != nil { - return nil - } - return poolCfg.GetPools() -} - -var _ = Describe("Controller Migrator", func() { - - AfterEach(func() { - k8sClient.DeleteAllOf(ctx, &corev1.Node{}) - k8sClient.DeleteAllOf(ctx, &ipamv1alpha1.IPPool{}, client.InNamespace(TestNamespace)) - }) - - It("Basic tests", func() { - testNode1 := "node1" - testNode2 := "node2" - - By("Create valid cfg1") - updateConfigMap(validConfig) - - By("Set annotation with valid ranges for node1") - node1 := createNode(testNode1) - node1InitialRanges := map[string]*pool.Pool{pool1Name: { - Name: pool1Name, - Subnet: "192.168.0.0/16", - StartIP: "192.168.0.11", - EndIP: "192.168.0.20", - Gateway: "192.168.0.1", - }, pool2Name: { - Name: pool2Name, - Subnet: "172.16.0.0/16", - StartIP: "172.16.0.1", - EndIP: "172.16.0.50", - Gateway: "172.16.0.1", - }} - Expect(pool.SetIPBlockAnnotation(node1, node1InitialRanges)).NotTo(HaveOccurred()) - Expect(updateNode(node1)) - - By("Set annotation with valid ranges for node2") - node2 := createNode(testNode2) - node2InitialRanges := map[string]*pool.Pool{pool1Name: { - Name: pool1Name, - Subnet: "192.168.0.0/16", - StartIP: "192.168.0.21", - EndIP: "192.168.0.30", - Gateway: "192.168.0.1", - }, pool2Name: { - Name: pool2Name, - Subnet: "172.16.0.0/16", - StartIP: "172.16.0.51", - EndIP: "172.16.0.100", - Gateway: "172.16.0.1", - }} - Expect(pool.SetIPBlockAnnotation(node2, node2InitialRanges)).NotTo(HaveOccurred()) - Expect(updateNode(node2)) - - By("Run migrator") - Expect(migrator.Migrate(ctx, klog.NewKlogr(), k8sClient, TestNamespace)).NotTo(HaveOccurred()) - - By("Verify Pool1 Spec") - pool1 := &ipamv1alpha1.IPPool{} - Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: TestNamespace, Name: pool1Name}, pool1)).NotTo(HaveOccurred()) - Expect(pool1.Spec.Gateway == "192.168.0.1" && - pool1.Spec.Subnet == "192.168.0.0/16" && pool1.Spec.PerNodeBlockSize == 10).To(BeTrue()) - Expect(pool1.Spec.NodeSelector).NotTo(BeNil()) - - By("Verify Pool1 Allocations") - expectedAllocations := []ipamv1alpha1.Allocation{{NodeName: testNode1, StartIP: "192.168.0.11", EndIP: "192.168.0.20"}, - {NodeName: testNode2, StartIP: "192.168.0.21", EndIP: "192.168.0.30"}} - Expect(expectedAllocations).To(BeEquivalentTo(pool1.Status.Allocations)) - - By("Verify Pool2 Spec") - pool2 := &ipamv1alpha1.IPPool{} - Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: TestNamespace, Name: pool2Name}, pool2)).NotTo(HaveOccurred()) - Expect(pool2.Spec.Gateway == "172.16.0.1" && - pool2.Spec.Subnet == "172.16.0.0/16" && pool2.Spec.PerNodeBlockSize == 50).To(BeTrue()) - Expect(pool2.Spec.NodeSelector).NotTo(BeNil()) - - By("Verify Pool2 Allocations") - expectedAllocations = []ipamv1alpha1.Allocation{{NodeName: testNode1, StartIP: "192.168.0.11", EndIP: "192.168.0.20"}, - {NodeName: testNode2, StartIP: "192.168.0.21", EndIP: "192.168.0.30"}} - Expect(expectedAllocations).To(BeEquivalentTo(pool1.Status.Allocations)) - - By("Verify Nodes annotations are removed") - Expect(getRangeFromNode(testNode1)).To(BeEmpty()) - Expect(getRangeFromNode(testNode2)).To(BeEmpty()) - - By("Verify Config Map is deleted") - configMap := &corev1.ConfigMap{} - Expect(k8sClient.Get( - ctx, types.NamespacedName{Name: TestConfigMapName, Namespace: TestNamespace}, configMap)).To(HaveOccurred()) - }) - - It("No ConfigMap", func() { - By("Run migrator") - Expect(migrator.Migrate(ctx, klog.NewKlogr(), k8sClient, TestNamespace)).NotTo(HaveOccurred()) - }) - - Context("Negative flows", func() { - It("Invalid ConfigMap", func() { - By("Create invalid cfg - no data") - cm := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: TestConfigMapName, - Namespace: TestNamespace, - }, - } - Expect(k8sClient.Create(ctx, cm)).NotTo(HaveOccurred()) - By("Run migrator - should fail") - Expect(migrator.Migrate(ctx, klog.NewKlogr(), k8sClient, TestNamespace)).To(HaveOccurred()) - - By("Create invalid cfg - not a json data") - updateConfigMap("{{") - By("Run migrator - should fail") - Expect(migrator.Migrate(ctx, klog.NewKlogr(), k8sClient, TestNamespace)).To(HaveOccurred()) - - By("Create invalid cfg - Gateway not in subnet") - var inValidConfig = ` - { - "pools": { - "pool-1": { "subnet": "192.168.0.0/16", "perNodeBlockSize": 10 , "gateway": "172.20.0.1"} - } - }` - updateConfigMap(inValidConfig) - Expect(migrator.Migrate(ctx, klog.NewKlogr(), k8sClient, TestNamespace)).To(HaveOccurred()) - - By("Create valid cfg - IPPool exists with different spec") - updateConfigMap(validConfig) - pool1 := &ipamv1alpha1.IPPool{ - ObjectMeta: metav1.ObjectMeta{ - Name: pool1Name, - Namespace: TestNamespace, - }, - Spec: ipamv1alpha1.IPPoolSpec{ - Subnet: "192.168.0.0/16", - PerNodeBlockSize: 50, - Gateway: "192.168.0.1", - }, - } - Expect(k8sClient.Create(ctx, pool1)).NotTo(HaveOccurred()) - Expect(migrator.Migrate(ctx, klog.NewKlogr(), k8sClient, TestNamespace)).To(HaveOccurred()) - }) - }) -}) diff --git a/pkg/pool/annotations.go b/pkg/pool/annotations.go deleted file mode 100644 index ccfd948..0000000 --- a/pkg/pool/annotations.go +++ /dev/null @@ -1,49 +0,0 @@ -/* - Copyright 2023, NVIDIA CORPORATION & AFFILIATES - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pool - -import ( - "encoding/json" - "fmt" - - v1 "k8s.io/api/core/v1" -) - -// SetIPBlockAnnotation serialize IP pools settings for the node and add this info as annotation -func SetIPBlockAnnotation(node *v1.Node, pools map[string]*Pool) error { - annotations := node.GetAnnotations() - if annotations == nil { - annotations = map[string]string{} - } - data, err := json.Marshal(pools) - if err != nil { - return fmt.Errorf("failed to serialize pools config: %v", err) - } - annotations[IPBlocksAnnotation] = string(data) - node.SetAnnotations(annotations) - return nil -} - -// IPBlockAnnotationExists returns true if ip-block annotation exist -func IPBlockAnnotationExists(node *v1.Node) bool { - _, exist := node.GetAnnotations()[IPBlocksAnnotation] - return exist -} - -// RemoveIPBlockAnnotation removes annotation with ip-block from the node object -func RemoveIPBlockAnnotation(node *v1.Node) { - annotations := node.GetAnnotations() - delete(annotations, IPBlocksAnnotation) - node.SetAnnotations(annotations) -} diff --git a/pkg/pool/annotations_test.go b/pkg/pool/annotations_test.go deleted file mode 100644 index 7438528..0000000 --- a/pkg/pool/annotations_test.go +++ /dev/null @@ -1,111 +0,0 @@ -/* - Copyright 2023, NVIDIA CORPORATION & AFFILIATES - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pool_test - -import ( - "encoding/json" - - v1 "k8s.io/api/core/v1" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/Mellanox/nvidia-k8s-ipam/pkg/pool" -) - -var _ = Describe("annotations tests", func() { - Context("SetIPBlockAnnotation", func() { - testPools := make(map[string]*pool.Pool) - testPools["my-pool-1"] = &pool.Pool{ - Name: "my-pool-1", - Subnet: "192.168.0.0/16", - StartIP: "192.168.0.2", - EndIP: "192.168.0.254", - Gateway: "192.168.0.1", - } - testPools["my-pool-2"] = &pool.Pool{ - Name: "my-pool-2", - Subnet: "10.100.0.0/16", - StartIP: "10.100.0.2", - EndIP: "10.100.0.254", - Gateway: "10.100.0.1", - } - - It("sets annotation successfully", func() { - n := v1.Node{} - pool.SetIPBlockAnnotation(&n, testPools) - Expect(n.GetAnnotations()).ToNot(BeNil()) - - data, err := json.Marshal(testPools) - Expect(err).ToNot(HaveOccurred()) - Expect(n.GetAnnotations()[pool.IPBlocksAnnotation]).To(Equal(string(data))) - }) - - It("overwrites annotation successfully", func() { - n := v1.Node{} - annot := map[string]string{ - pool.IPBlocksAnnotation: `{"my-pool": {"some": "content"}}`, - } - n.SetAnnotations(annot) - - pool.SetIPBlockAnnotation(&n, testPools) - Expect(n.GetAnnotations()).ToNot(BeNil()) - - data, err := json.Marshal(testPools) - Expect(err).ToNot(HaveOccurred()) - Expect(n.GetAnnotations()[pool.IPBlocksAnnotation]).To(Equal(string(data))) - }) - }) - - Context("IPBlockAnnotationExists", func() { - It("returns true if annotation exists", func() { - n := v1.Node{} - ipBlockAnnot := make(map[string]string) - ipBlockAnnot[pool.IPBlocksAnnotation] = "foobar" - n.SetAnnotations(ipBlockAnnot) - - Expect(pool.IPBlockAnnotationExists(&n)).To(BeTrue()) - }) - - It("returns false if annotation does not exists", func() { - Expect(pool.IPBlockAnnotationExists(&v1.Node{})).To(BeFalse()) - }) - }) - - Context("RemoveIPBlockAnnotation", func() { - It("Succeeds if annotation does not exist", func() { - n := v1.Node{} - annot := map[string]string{ - "foo": "bar", - } - n.SetAnnotations(annot) - - pool.RemoveIPBlockAnnotation(&n) - Expect(n.GetAnnotations()).To(HaveKey("foo")) - }) - - It("removes annotation if exists", func() { - n := v1.Node{} - annot := map[string]string{ - "foo": "bar", - pool.IPBlocksAnnotation: "baz", - } - n.SetAnnotations(annot) - - pool.RemoveIPBlockAnnotation(&n) - Expect(n.GetAnnotations()).To(HaveKey("foo")) - Expect(n.GetAnnotations()).ToNot(HaveKey(pool.IPBlocksAnnotation)) - }) - }) -}) diff --git a/pkg/pool/manager.go b/pkg/pool/manager.go index 426e601..3e955bd 100644 --- a/pkg/pool/manager.go +++ b/pkg/pool/manager.go @@ -13,12 +13,53 @@ package pool -import "sync" +import ( + "encoding/json" + "sync" +) + +// Pool represents generic pool configuration +type Pool struct { + Name string `json:"-"` + Subnet string `json:"subnet"` + StartIP string `json:"startIP"` + EndIP string `json:"endIP"` + Gateway string `json:"gateway"` + Exclusions []ExclusionRange `json:"exclusions"` + Routes []Route `json:"routes"` + DefaultGateway bool `json:"defaultGateway"` +} + +// ExclusionRange contains range of IP to exclude from the allocation +type ExclusionRange struct { + StartIP string `json:"startIP"` + EndIP string `json:"endIP"` +} + +// Route contains a destination CIDR to be added as static route via gateway +type Route struct { + Dst string `json:"dst"` +} + +// String return string representation of the IPPool config +func (p *Pool) String() string { + //nolint:errchkjson + data, _ := json.Marshal(p) + return string(data) +} + +// ConfigReader is an interface to which provides access to the pool configuration +type ConfigReader interface { + // GetPoolByKey returns IPPool for the provided pool name or nil if pool doesn't exist + GetPoolByKey(key string) *Pool + // GetPools returns map with information about all pools + GetPools() map[string]*Pool +} // Manager provide access to pools configuration type Manager interface { ConfigReader - // Update Pool's config from IPPool CR + // Update Pool's config from provided Pool object UpdatePool(key string, pool *Pool) // Remove Pool's config by key RemovePool(key string) diff --git a/pkg/pool/reader.go b/pkg/pool/reader.go deleted file mode 100644 index 5d3a2eb..0000000 --- a/pkg/pool/reader.go +++ /dev/null @@ -1,102 +0,0 @@ -/* - Copyright 2023, NVIDIA CORPORATION & AFFILIATES - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pool - -import ( - "encoding/json" - "fmt" - - v1 "k8s.io/api/core/v1" -) - -const ( - IPBlocksAnnotation = "ipam.nvidia.com/ip-blocks" -) - -// Pool represents generic pool configuration -type Pool struct { - Name string `json:"-"` - Subnet string `json:"subnet"` - StartIP string `json:"startIP"` - EndIP string `json:"endIP"` - Gateway string `json:"gateway"` - Exclusions []ExclusionRange `json:"exclusions"` - Routes []Route `json:"routes"` - DefaultGateway bool `json:"defaultGateway"` -} - -// ExclusionRange contains range of IP to exclude from the allocation -type ExclusionRange struct { - StartIP string `json:"startIP"` - EndIP string `json:"endIP"` -} - -// Route contains a destination CIDR to be added as static route via gateway -type Route struct { - Dst string `json:"dst"` -} - -// String return string representation of the IPPool config -func (p *Pool) String() string { - //nolint:errchkjson - data, _ := json.Marshal(p) - return string(data) -} - -// ConfigReader is an interface to which provides access to the pool configuration -type ConfigReader interface { - // GetPoolByKey returns IPPool for the provided pool name or nil if pool doesn't exist - GetPoolByKey(key string) *Pool - // GetPools returns map with information about all pools - GetPools() map[string]*Pool -} - -type configReader struct { - poolByKey map[string]*Pool -} - -func NewConfigReader(node *v1.Node) (ConfigReader, error) { - if node == nil { - return nil, fmt.Errorf("nil node provided") - } - - blocks, ok := node.Annotations[IPBlocksAnnotation] - if !ok { - return nil, fmt.Errorf("%s node annotation not found", IPBlocksAnnotation) - } - - poolByKey := make(map[string]*Pool) - err := json.Unmarshal([]byte(blocks), &poolByKey) - if err != nil { - return nil, fmt.Errorf("failed to parse %s annotation content. %w", IPBlocksAnnotation, err) - } - - for poolName, pool := range poolByKey { - pool.Name = poolName - } - - return &configReader{ - poolByKey: poolByKey, - }, nil -} - -// GetPoolByKey implements ConfigReader interface -func (r *configReader) GetPoolByKey(key string) *Pool { - return r.poolByKey[key] -} - -// GetPools implements ConfigReader interface -func (r *configReader) GetPools() map[string]*Pool { - return r.poolByKey -} diff --git a/pkg/pool/reader_test.go b/pkg/pool/reader_test.go deleted file mode 100644 index 0759827..0000000 --- a/pkg/pool/reader_test.go +++ /dev/null @@ -1,92 +0,0 @@ -/* - Copyright 2023, NVIDIA CORPORATION & AFFILIATES - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package pool_test - -import ( - v1 "k8s.io/api/core/v1" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/Mellanox/nvidia-k8s-ipam/pkg/pool" -) - -var _ = Describe("pool tests", func() { - Context("NewConfigReader()", func() { - It("Creates a Manager successfully if node has ip-pool annotation", func() { - n := v1.Node{} - emptyAnnot := map[string]string{ - pool.IPBlocksAnnotation: "{}", - } - n.SetAnnotations(emptyAnnot) - r, err := pool.NewConfigReader(&n) - Expect(err).ToNot(HaveOccurred()) - Expect(r.GetPools()).To(HaveLen(0)) - - annot := map[string]string{ - pool.IPBlocksAnnotation: `{"my-pool": - {"subnet": "192.168.0.0/16", "startIP": "192.168.0.2", - "endIP": "192.168.0.254", "gateway": "192.168.0.1"}}`, - } - n.SetAnnotations(annot) - r, err = pool.NewConfigReader(&n) - Expect(err).ToNot(HaveOccurred()) - Expect(r.GetPools()).To(HaveLen(1)) - }) - - It("Fails to create Manager if node is missing ip-pool annotation", func() { - n := v1.Node{} - _, err := pool.NewConfigReader(&n) - Expect(err).To(HaveOccurred()) - }) - - It("Fails to create Manager if node has empty/invalid ip-pool annotation", func() { - n := v1.Node{} - emptyAnnot := map[string]string{ - pool.IPBlocksAnnotation: "", - } - n.SetAnnotations(emptyAnnot) - _, err := pool.NewConfigReader(&n) - Expect(err).To(HaveOccurred()) - }) - }) - - Context("GetPoolByKey()", func() { - var r pool.ConfigReader - - BeforeEach(func() { - var err error - n := v1.Node{} - annot := map[string]string{ - pool.IPBlocksAnnotation: `{"my-pool": - {"subnet": "192.168.0.0/16", "startIP": "192.168.0.2", - "endIP": "192.168.0.254", "gateway": "192.168.0.1"}}`, - } - n.SetAnnotations(annot) - r, err = pool.NewConfigReader(&n) - Expect(err).ToNot(HaveOccurred()) - }) - - It("returns nil if pool does not exist", func() { - p := r.GetPoolByKey("non-existent-key") - Expect(p).To(BeNil()) - }) - - It("returns pool if exists", func() { - p := r.GetPoolByKey("my-pool") - Expect(p).ToNot(BeNil()) - Expect(p.Subnet).To(Equal("192.168.0.0/16")) - }) - }) -})