v2.0.9 fixes and cli improvments for OBC

README.md

@@ -12,12 +12,12 @@ For Mac
 ```
 brew install noobaa/noobaa/noobaa
 # or
-wget https://github.com/noobaa/noobaa-operator/releases/download/v2.0.8/noobaa-mac-v2.0.8; mv noobaa-mac-* noobaa; chmod +x noobaa
+wget https://github.com/noobaa/noobaa-operator/releases/download/v2.0.9/noobaa-mac-v2.0.9; mv noobaa-mac-* noobaa; chmod +x noobaa
 ```
 
 For Linux
 ```
-wget https://github.com/noobaa/noobaa-operator/releases/download/v2.0.8/noobaa-linux-v2.0.8; mv noobaa-linux-* noobaa; chmod +x noobaa
+wget https://github.com/noobaa/noobaa-operator/releases/download/v2.0.9/noobaa-linux-v2.0.9; mv noobaa-linux-* noobaa; chmod +x noobaa
 ```
 
 - Run: `./noobaa --help` for CLI usage

deploy/internal/configmap-empty.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    app: noobaa
+data: {}

deploy/internal/text/system_status_readme_ready.tmpl

@@ -15,7 +15,7 @@
 
 		Open the management console service - take External IP/DNS or Node Port or use port forwarding:
 
-			kubectl port-forward -n {{.ServiceMgmt.Namespace}} service/{{.ServiceMgmt.Name}} 11443:8443 &
+			kubectl port-forward -n {{.ServiceMgmt.Namespace}} service/{{.ServiceMgmt.Name}} 11443:443 &
 			open https://localhost:11443
 
 	2. Test S3 client:

doc/backing-store-crd.md

@@ -5,7 +5,7 @@ BackingStore CRD represents a storage target to be used as underlying storage fo
 These storage targets are used to store deduped+compressed+encrypted chunks of data (encryption keys are stored separately).
 Backing-stores are referred to by name when defining [BucketClass](bucket-class-crd.md).
 
-Multiple types of backing-stores are currently supported: aws-s3, s3-compatible, google-cloud-storage, azure-blob.
+Multiple types of backing-stores are currently supported: aws-s3, s3-compatible, google-cloud-storage, azure-blob, pv-pool.
 Backing-store type pv-pool is not yet supported by the operator. Instead, the web management console must be used to administer pv-pool backing-stores.
 Adding support for a new type of backing-store is rather easy as it requires just GET/PUT key-value store, see [Backing-stores supported by NooBaa](https://github.com/noobaa/noobaa-core/tree/master/src/agent/block_store_services).
 
@@ -121,75 +121,29 @@ spec:
 ```
 
 #### PV-POOL type
-**Not yet implemented**
 
-Create NooBaa resources StatefulSet with PVC mounted in each pod. Each resource will connect to the NooBaa brain and provide the PV filesystem storage to be used for storing encrypted chunks of data.
-This action is supported from the NooBaa dashboard (Deploy Kubernetes Pool).
-It is possible to configure the number of pods to be used and their PV size.
-Here is an example of a StatefulSet with 3 pods and PV size of 30GB:
+Create NooBaa resources StatefulSet with PVC mounted in each pod. Each resource will connect to the NooBaa core and provide the PV filesystem storage to be used for storing encrypted chunks of data. It is possible to configure the number of pods to be used and their PV size.
+```shell
+noobaa -n noobaa backingstore create pv-pool bs --num-volumes 3 --pv-size-gb 32 --storage-class STORAGE-CLASS-NAME
+```
 ```yaml
----
-apiVersion: apps/v1
-kind: StatefulSet
+apiVersion: noobaa.io/v1alpha1
+kind: BackingStore
 metadata:
-  name: noobaa-agent
+  finalizers:
+  - noobaa.io/finalizer
   labels:
     app: noobaa
-    noobaa-module: noobaa-pool-impl
+  name: bs
+  namespace: noobaa
 spec:
-  selector:
-    matchLabels:
-      noobaa-module: noobaa-agent
-  serviceName: noobaa-agent
-  replicas: 3
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        app: noobaa
-        noobaa-module: noobaa-agent
-        noobaa-s3: "true"
-    spec:
-      containers:
-        - name: noobaa-agent
-          resources:
-            requests:
-              cpu: "100m"
-              memory: "500Mi"
-            limits:
-              cpu: "2"
-              memory: "2Gi"
-          env:
-            - name: CONTAINER_PLATFORM
-              value: KUBERNETES
-            - name: AGENT_CONFIG
-              value: "AGENT_CONFIG_VALUE"
-            - name: ENDPOINT_PORT
-              value: "6001"
-            - name: ENDPOINT_SSL_PORT
-              value: "6443"
-          command: ["/noobaa_init_files/noobaa_init.sh", "agent"]
-          ports:
-            - containerPort: 60101
-          volumeMounts:
-            - name: noobaastorage
-              mountPath: /noobaa_storage
-            - name: tmp-logs-vol
-              mountPath: /usr/local/noobaa/logs
-      volumes:
-        - name: tmp-logs-vol
-          emptyDir: {}
-  volumeClaimTemplates:
-    - metadata:
-        name: noobaastorage
-        labels:
-          app: noobaa
-      spec:
-        accessModes: ["ReadWriteOnce"]
-        resources:
-          requests:
-            storage: 30Gi
+  pvPool:
+    numVolumes: 3
+    resources:
+      requests:
+        storage: 32Gi
+    storageClass: STORAGE-CLASS-NAME
+  type: pv-pool
 ```
 
 

go.mod

@@ -1,25 +1,22 @@
 module github.com/noobaa/noobaa-operator/v2
 
-go 1.12
+go 1.13
 
 require (
 	github.com/aws/aws-sdk-go v1.23.8
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/coreos/prometheus-operator v0.29.0
 	github.com/docker/distribution v2.7.1+incompatible
 	github.com/go-openapi/spec v0.19.2
-	github.com/go-openapi/validate v0.18.0 // indirect
 	github.com/gobuffalo/flect v0.1.6 // indirect
 	github.com/google/uuid v1.1.1
-	github.com/googleapis/gnostic v0.2.0 // indirect
 	github.com/hashicorp/go-version v1.2.0
 	github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20190924175516-f3ba69cc601e
 	github.com/openshift/api v3.9.1-0.20190424152011-77b8897ec79a+incompatible
 	github.com/openshift/cloud-credential-operator v0.0.0-20190614194054-1ccced634f6c
 	github.com/openshift/custom-resource-status v0.0.0-20190801200128-4c95b3a336cd
 	github.com/operator-framework/operator-lifecycle-manager v0.0.0-20190605231540-b8a4faf68e36 // 0.11.0
 	github.com/operator-framework/operator-sdk v0.10.0
-	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/rook/rook v1.1.2
 	github.com/sirupsen/logrus v1.4.2
 	github.com/spf13/cobra v0.0.5

pkg/backingstore/backingstore.go

@@ -13,8 +13,8 @@ import (
 
 	"github.com/spf13/cobra"
 	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes/scheme"
@@ -381,7 +381,7 @@ func RunCreatePVPool(cmd *cobra.Command, args []string) {
 		if numVolumes > 20 {
 			log.Fatalf(`❌ Number of volumes seems to be too large %d %s`, numVolumes, cmd.UsageString())
 		}
-	
+
 		if pvSizeGB == 0 {
 			fmt.Printf("Enter PV size (GB): ")
 			_, err := fmt.Scan(&pvSizeGB)
@@ -395,7 +395,7 @@ func RunCreatePVPool(cmd *cobra.Command, args []string) {
 		}
 		backStore.Spec.PVPool = &nbv1.PVPoolSpec{
 			StorageClass: storageClass,
-			NumVolumes: int(numVolumes),
+			NumVolumes:   int(numVolumes),
 			VolumeResources: &corev1.ResourceRequirements{
 				Requests: corev1.ResourceList{
 					corev1.ResourceStorage: *resource.NewScaledQuantity(int64(pvSizeGB), resource.Giga),

pkg/backingstore/reconciler.go

@@ -362,24 +362,29 @@ func (r *Reconciler) ReadSystemInfo() error {
 	pool := r.PoolInfo
 	if r.BackingStore.Spec.Type == nbv1.StoreTypePVPool {
 		if pool != nil && pool.ResourceType != "HOSTS" {
-            return util.NewPersistentError("InvalidBackingStore",
-				fmt.Sprintf("BackingStore %q w/existing pool %+v has unexpected resource type %+v", r.BackingStore.Name, pool, pool.ResourceType))
+			return util.NewPersistentError("InvalidBackingStore", fmt.Sprintf(
+				"BackingStore %q w/existing pool %+v has unexpected resource type %+v",
+				r.BackingStore.Name, pool, pool.ResourceType,
+			))
 		}
 		pvPool := r.BackingStore.Spec.PVPool
 		qty := pvPool.VolumeResources.Requests[corev1.ResourceName(corev1.ResourceStorage)]
 		gbsize, _ := qty.AsInt64()
 		r.CreateHostsPoolParams = &nb.CreateHostsPoolParams{
-			Name:		r.BackingStore.Name,
-			IsManaged:	true,
-			HostCount:	int(pvPool.NumVolumes),
+			Name:       r.BackingStore.Name,
+			IsManaged:  true,
+			HostCount:  int(pvPool.NumVolumes),
 			HostConfig: nb.PoolHostsInfo{VolumeSize: gbsize},
 		}
 		return nil
 	}
-    if pool != nil && pool.ResourceType != "CLOUD" {
-        return util.NewPersistentError("InvalidBackingStore",
-            fmt.Sprintf("BackingStore %q w/existing pool %+v has unexpected resource type %+v", r.BackingStore.Name, pool, pool.ResourceType))
-    }
+
+	if pool != nil && pool.ResourceType != "CLOUD" {
+		return util.NewPersistentError("InvalidBackingStore", fmt.Sprintf(
+			"BackingStore %q w/existing pool %+v has unexpected resource type %+v",
+			r.BackingStore.Name, pool, pool.ResourceType,
+		))
+	}
 
 	conn, err := r.MakeExternalConnectionParams()
 	if err != nil {
@@ -517,8 +522,10 @@ func (r *Reconciler) MakeExternalConnectionParams() (*nb.AddExternalConnectionPa
 		}{}
 		err := json.Unmarshal([]byte(privateKeyJSON), privateKey)
 		if err != nil {
-			return nil, util.NewPersistentError("InvalidGoogleSecret",
-				fmt.Sprintf("Invalid secret for google type %q expected JSON in data.GoogleServiceAccountPrivateKeyJson", r.Secret.Name))
+			return nil, util.NewPersistentError("InvalidGoogleSecret", fmt.Sprintf(
+				"Invalid secret for google type %q expected JSON in data.GoogleServiceAccountPrivateKeyJson",
+				r.Secret.Name,
+			))
 		}
 		conn.Identity = privateKey.ID
 		conn.Secret = privateKeyJSON
@@ -567,13 +574,14 @@ func (r *Reconciler) fixAlternateKeysNames() {
 // ReconcileExternalConnection handles the external connection using noobaa api
 func (r *Reconciler) ReconcileExternalConnection() error {
 
+	// TODO we only support creation here, but not updates
 	if r.ExternalConnectionInfo != nil {
 		return nil
 	}
-
 	if r.AddExternalConnectionParams == nil {
 		return nil
 	}
+
 	res, err := r.NBClient.CheckExternalConnectionAPI(*r.AddExternalConnectionParams)
 	if err != nil {
 		if rpcErr, isRPCErr := err.(*nb.RPCError); isRPCErr {
@@ -623,11 +631,12 @@ func (r *Reconciler) ReconcileExternalConnection() error {
 // ReconcilePool handles the pool using noobaa api
 func (r *Reconciler) ReconcilePool() error {
 
+	// TODO we only support creation here, but not updates
 	if r.PoolInfo != nil {
 		return nil
 	}
 
-    poolName := ""
+	poolName := ""
 
 	if r.CreateHostsPoolParams != nil {
 		err := r.NBClient.CreateHostsPoolAPI(*r.CreateHostsPoolParams)
@@ -642,10 +651,10 @@ func (r *Reconciler) ReconcilePool() error {
 		if err != nil {
 			return err
 		}
-        poolName = r.CreateCloudPoolParams.Name
+		poolName = r.CreateCloudPoolParams.Name
 	}
 
-    if poolName != "" {
+	if poolName != "" {
 		err := r.NBClient.UpdateAllBucketsDefaultPool(nb.UpdateDefaultPoolParams{
 			PoolName: poolName,
 		})

pkg/bucket/bucket.go

@@ -20,6 +20,7 @@ func Cmd() *cobra.Command {
 	cmd.AddCommand(
 		CmdCreate(),
 		CmdDelete(),
+		CmdStatus(),
 		CmdList(),
 	)
 	return cmd
@@ -45,6 +46,16 @@ func CmdDelete() *cobra.Command {
 	return cmd
 }
 
+// CmdStatus returns a CLI command
+func CmdStatus() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "status <bucket-name>",
+		Short: "Show the status of a NooBaa bucket",
+		Run:   RunStatus,
+	}
+	return cmd
+}
+
 // CmdList returns a CLI command
 func CmdList() *cobra.Command {
 	cmd := &cobra.Command{
@@ -83,6 +94,46 @@ func RunDelete(cmd *cobra.Command, args []string) {
 	}
 }
 
+// RunStatus runs a CLI command
+func RunStatus(cmd *cobra.Command, args []string) {
+	log := util.Logger()
+	if len(args) != 1 || args[0] == "" {
+		log.Fatalf(`Missing expected arguments: <bucket-name> %s`, cmd.UsageString())
+	}
+	bucketName := args[0]
+	nbClient := system.GetNBClient()
+	b, err := nbClient.ReadBucketAPI(nb.ReadBucketParams{Name: bucketName})
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Printf("\n")
+	fmt.Printf("Bucket status:\n")
+	fmt.Printf("  %-22s : %s\n", "Bucket", b.Name)
+	if b.BucketClaim != nil {
+		fmt.Printf("  %-22s : %s\n", "OBC Namespace", b.BucketClaim.Namespace)
+		fmt.Printf("  %-22s : %s\n", "OBC BucketClass", b.BucketClaim.BucketClass)
+	}
+	fmt.Printf("  %-22s : %s\n", "Type", b.BucketType)
+	fmt.Printf("  %-22s : %s\n", "Mode", b.Mode)
+	if b.PolicyModes != nil {
+		fmt.Printf("  %-22s : %s\n", "ResiliencyStatus", b.PolicyModes.ResiliencyStatus)
+		fmt.Printf("  %-22s : %s\n", "QuotaStatus", b.PolicyModes.QuotaStatus)
+	}
+	if b.Undeletable != "" {
+		fmt.Printf("  %-22s : %s\n", "Undeletable", b.Undeletable)
+	}
+	if b.NumObjects != nil {
+		fmt.Printf("  %-22s : %d\n", "Num Objects", b.NumObjects.Value)
+	}
+	if b.DataCapacity != nil {
+		fmt.Printf("  %-22s : %s\n", "Data Size", nb.BigIntToHumanBytes(b.DataCapacity.Size))
+		fmt.Printf("  %-22s : %s\n", "Data Size Reduced", nb.BigIntToHumanBytes(b.DataCapacity.SizeReduced))
+		fmt.Printf("  %-22s : %s\n", "Data Space Avail", nb.BigIntToHumanBytes(b.DataCapacity.AvailableToUpload))
+	}
+	fmt.Printf("\n")
+}
+
 // RunList runs a CLI command
 func RunList(cmd *cobra.Command, args []string) {
 	nbClient := system.GetNBClient()
@@ -99,5 +150,7 @@ func RunList(cmd *cobra.Command, args []string) {
 		b := &list.Buckets[i]
 		table.AddRow(b.Name)
 	}
+	fmt.Printf("\n")
 	fmt.Print(table.String())
+	fmt.Printf("\n")
 }