Merge pull request #16 from vshn/update-component

Make component multi-instance capable

component/Makefile.vars.mk

@@ -5,7 +5,7 @@ COMPONENT_SUBDIR ?= $(shell basename ${PWD})
 compiled_path   ?= compiled/$(COMPONENT_NAME)/$(COMPONENT_NAME)
 root_volume     ?= -v "$${PWD}/../:/$(COMPONENT_NAME)"
 compiled_volume ?= -v "$${PWD}/$(compiled_path):/$(COMPONENT_NAME)"
-commodore_args  ?= --search-paths . -n $(COMPONENT_NAME)
+commodore_args  ?= --search-paths . -n $(COMPONENT_NAME) --alias $(instance)
 
 ifneq "$(shell which docker 2>/dev/null)" ""
 	DOCKER_CMD    ?= $(shell which docker)
@@ -43,5 +43,5 @@ KUBENT_ARGS     ?= -c=false --helm2=false --helm3=false -e
 KUBENT_IMAGE    ?= docker.io/projectsyn/kubent:latest
 KUBENT_DOCKER   ?= $(DOCKER_CMD) $(DOCKER_ARGS) $(root_volume) --entrypoint=/app/kubent $(KUBENT_IMAGE)
 
-instance ?= defaults
-test_instances = tests/defaults.yml
+instance ?= exoscale-metrics-collector
+test_instances = tests/exoscale-metrics-collector.yml tests/collector-cloudscale-lpg-2.yml

component/class/defaults.yml

@@ -1,10 +1,14 @@
 parameters:
   exoscale_metrics_collector:
+    =_metadata:
+      multi_instance: true
     secrets:
-      exoscale:
+      credentials:
         stringData:
-          api_key: "?{vaultkv:${cluster:tenant}/${cluster:name}/exoscale-metrics-collector/key}"
-          api_secret: "?{vaultkv:${cluster:tenant}/${cluster:name}/exoscale-metrics-collector/secret}"
+          EXOSCALE_API_KEY: "?{vaultkv:${cluster:tenant}/${cluster:name}/exoscale-metrics-collector/${_instance}/exoscale-key}"
+          EXOSCALE_API_SECRET: "?{vaultkv:${cluster:tenant}/${cluster:name}/exoscale-metrics-collector/${_instance}/exoscale-secret}"
+          K8S_SERVER_URL: "?{vaultkv:${cluster:tenant}/${cluster:name}/exoscale-metrics-collector/${_instance}/cluster-server}"
+          K8S_TOKEN: "?{vaultkv:${cluster:tenant}/${cluster:name}/exoscale-metrics-collector/${_instance}/cluster-token}"
     images:
       collector:
         registry: 'ghcr.io'

component/class/exoscale-metrics-collector.yml

@@ -5,7 +5,9 @@ parameters:
           - ${_base_directory}/component/app.jsonnet
         input_type: jsonnet
         output_path: apps/
+        output_type: yaml
       - input_paths:
           - ${_base_directory}/component/main.jsonnet
         input_type: jsonnet
-        output_path: exoscale-metrics-collector/
+        output_type: yaml
+        output_path: ${_instance}

component/component/main.jsonnet

@@ -5,18 +5,22 @@ local paramsACR = inv.parameters.appuio_cloud_reporting;
 local kube = import 'lib/kube.libjsonnet';
 local com = import 'lib/commodore.libjsonnet';
 local collectorImage = '%(registry)s/%(repository)s:%(tag)s' % params.images.collector;
+local alias = inv.parameters._instance;
+local alias_suffix = '-' + alias;
+local credentials_secret_name = 'credentials' + alias_suffix;
+local component_name = 'exoscale-metrics-collector';
 
 
 local labels = {
-  'app.kubernetes.io/name': 'exoscale-metrics-collector',
+  'app.kubernetes.io/name': component_name,
   'app.kubernetes.io/managed-by': 'commodore',
   'app.kubernetes.io/part-of': 'appuio-cloud-reporting',
-  'app.kubernetes.io/component': 'exoscale-metrics-collector',
+  'app.kubernetes.io/component': component_name,
 };
 
 local secrets = [
   if params.secrets[s] != null then
-    kube.Secret(s) {
+    kube.Secret(s + alias_suffix) {
       metadata+: {
         namespace: paramsACR.namespace,
       },
@@ -26,18 +30,20 @@ local secrets = [
 
 {
   assert params.secrets != null : 'secrets must be set.',
-  assert params.secrets.exoscale != null : 'secrets.exoscale must be set.',
-  assert params.secrets.exoscale.stringData != null : 'secrets.exoscale.stringData must be set.',
-  assert params.secrets.exoscale.stringData.api_key != null : 'secrets.exoscale.stringData.api_key must be set.',
-  assert params.secrets.exoscale.stringData.api_secret != null : 'secrets.exoscale.stringData.api_secret must be set.',
+  assert params.secrets.credentials != null : 'secrets.credentials must be set.',
+  assert params.secrets.credentials.stringData != null : 'secrets.credentials.stringData must be set.',
+  assert params.secrets.credentials.stringData.EXOSCALE_API_KEY != null : 'secrets.credentials.stringData.EXOSCALE_API_KEY must be set.',
+  assert params.secrets.credentials.stringData.EXOSCALE_API_SECRET != null : 'secrets.credentials.stringData.EXOSCALE_API_SECRET must be set.',
+  assert params.secrets.credentials.stringData.K8S_SERVER_URL != null : 'secrets.credentials.stringData.K8S_SERVER_URL must be set.',
+  assert params.secrets.credentials.stringData.K8S_TOKEN != null : 'secrets.credentials.stringData.K8S_TOKEN must be set.',
 
   secrets: std.filter(function(it) it != null, secrets),
 
   cronjob: {
     kind: 'CronJob',
     apiVersion: 'batch/v1',
     metadata: {
-      name: 'exoscale-metrics-collector',
+      name: alias,
       namespace: paramsACR.namespace,
       labels+: labels,
     },
@@ -57,6 +63,13 @@ local secrets = [
                     'exoscale-metrics-collector',
                   ],
                   command: [ 'sh', '-c' ],
+                  envFrom: [
+                    {
+                      secretRef: {
+                        name: credentials_secret_name
+                      }
+                    }
+                  ],
                   env: [
                     {
                       name: 'password',
@@ -80,24 +93,6 @@ local secrets = [
                       name: 'ACR_DB_URL',
                       value: 'postgres://$(username):$(password)@%(host)s:%(port)s/%(name)s?%(parameters)s' % paramsACR.database,
                     },
-                    {
-                      name: 'EXOSCALE_API_KEY',
-                      valueFrom: {
-                        secretKeyRef: {
-                          key: 'api_key',
-                          name: 'exoscale',
-                        },
-                      },
-                    },
-                    {
-                      name: 'EXOSCALE_API_SECRET',
-                      valueFrom: {
-                        secretKeyRef: {
-                          key: 'api_secret',
-                          name: 'exoscale',
-                        },
-                      },
-                    },
                   ],
                   resources: {},
                 },

component/tests/collector-cloudscale-lpg-2.yml

@@ -0,0 +1,12 @@
+applications:
+  - exoscale-metrics-collector as collector-cloudscale-lpg-2
+
+parameters:
+  appuio_cloud_reporting:
+    namespace: 'appuio-cloud-reporting'
+    database:
+      name: 'reporting'
+      host: 'reporting-db.appuio-reporting.svc'
+      parameters: 'sslmode=disable'
+      password: 'passw0rd'
+      port: 5432

component/tests/golden/collector-cloudscale-lpg-2/collector-cloudscale-lpg-2/collector-cloudscale-lpg-2/cronjob.yaml

@@ -0,0 +1,45 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  labels:
+    app.kubernetes.io/component: exoscale-metrics-collector
+    app.kubernetes.io/managed-by: commodore
+    app.kubernetes.io/name: exoscale-metrics-collector
+    app.kubernetes.io/part-of: appuio-cloud-reporting
+  name: collector-cloudscale-lpg-2
+  namespace: appuio-cloud-reporting
+spec:
+  concurrencyPolicy: Forbid
+  failedJobsHistoryLimit: 5
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+            - args:
+                - exoscale-metrics-collector
+              command:
+                - sh
+                - -c
+              env:
+                - name: password
+                  valueFrom:
+                    secretKeyRef:
+                      key: password
+                      name: reporting-db
+                - name: username
+                  valueFrom:
+                    secretKeyRef:
+                      key: username
+                      name: reporting-db
+                - name: ACR_DB_URL
+                  value: postgres://$(username):$(password)@reporting-db.appuio-reporting.svc:5432/reporting?sslmode=disable
+              envFrom:
+                - secretRef:
+                    name: credentials-collector-cloudscale-lpg-2
+              image: ghcr.io/vshn/exoscale-metrics-collector:v0.0.2
+              name: exoscale-metrics-collector-backfill
+              resources: {}
+          restartPolicy: OnFailure
+  schedule: 10 10,16,20 * * *
+  successfulJobsHistoryLimit: 3

component/tests/golden/collector-cloudscale-lpg-2/collector-cloudscale-lpg-2/collector-cloudscale-lpg-2/secrets.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+data: {}
+kind: Secret
+metadata:
+  annotations: {}
+  labels:
+    name: credentials-collector-cloudscale-lpg-2
+  name: credentials-collector-cloudscale-lpg-2
+  namespace: appuio-cloud-reporting
+stringData:
+  EXOSCALE_API_KEY: t-silent-test-1234/c-green-test-1234/exoscale-metrics-collector/collector-cloudscale-lpg-2/exoscale-key
+  EXOSCALE_API_SECRET: t-silent-test-1234/c-green-test-1234/exoscale-metrics-collector/collector-cloudscale-lpg-2/exoscale-secret
+  K8S_SERVER_URL: t-silent-test-1234/c-green-test-1234/exoscale-metrics-collector/collector-cloudscale-lpg-2/cluster-server
+  K8S_TOKEN: t-silent-test-1234/c-green-test-1234/exoscale-metrics-collector/collector-cloudscale-lpg-2/cluster-token
+type: Opaque

component/tests/golden/defaults/exoscale-metrics-collector/exoscale-metrics-collector/cronjob.yaml → component/tests/golden/exoscale-metrics-collector/exoscale-metrics-collector/exoscale-metrics-collector/cronjob.yaml

@@ -34,16 +34,9 @@ spec:
                       name: reporting-db
                 - name: ACR_DB_URL
                   value: postgres://$(username):$(password)@reporting-db.appuio-reporting.svc:5432/reporting?sslmode=disable
-                - name: EXOSCALE_API_KEY
-                  valueFrom:
-                    secretKeyRef:
-                      key: api_key
-                      name: exoscale
-                - name: EXOSCALE_API_SECRET
-                  valueFrom:
-                    secretKeyRef:
-                      key: api_secret
-                      name: exoscale
+              envFrom:
+                - secretRef:
+                    name: credentials-exoscale-metrics-collector
               image: ghcr.io/vshn/exoscale-metrics-collector:v0.0.2
               name: exoscale-metrics-collector-backfill
               resources: {}

component/tests/golden/exoscale-metrics-collector/exoscale-metrics-collector/exoscale-metrics-collector/secrets.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+data: {}
+kind: Secret
+metadata:
+  annotations: {}
+  labels:
+    name: credentials-exoscale-metrics-collector
+  name: credentials-exoscale-metrics-collector
+  namespace: appuio-cloud-reporting
+stringData:
+  EXOSCALE_API_KEY: t-silent-test-1234/c-green-test-1234/exoscale-metrics-collector/exoscale-metrics-collector/exoscale-key
+  EXOSCALE_API_SECRET: t-silent-test-1234/c-green-test-1234/exoscale-metrics-collector/exoscale-metrics-collector/exoscale-secret
+  K8S_SERVER_URL: t-silent-test-1234/c-green-test-1234/exoscale-metrics-collector/exoscale-metrics-collector/cluster-server
+  K8S_TOKEN: t-silent-test-1234/c-green-test-1234/exoscale-metrics-collector/exoscale-metrics-collector/cluster-token
+type: Opaque

docs/modules/ROOT/pages/how-tos/installation.adoc

@@ -4,6 +4,13 @@
 
 This component requires https://github.com/appuio/component-appuio-cloud-reporting[component-appuio-cloud-reporting] and is installed into the same namespace.
 This is required for this component to be able to access the billing database and its connection secrets.
+It also requires an Exoscale IAMKey and a Kubernetes/OpenShift Service Account token in the target cluster to get `buckets.exoscale.crossplane.io` resources.
+
+== Sources
+
+The data is matched from k8s cluster and Exoscale organization.
+The Kubernetes Service Account token is required to have `get` permissions on `Namespaces` and `buckets.exoscale.crossplane.io` resources.
+The Access Key (IAM Key) from an Exoscale organization is required to have read access across all SOS buckets.
 
 == Example
 
@@ -14,12 +21,7 @@ applications:
 
 parameters:
   exoscale_metrics_collector:
-    namespace: 'appuio-cloud-reporting'
-    secrets:
-      exoscale:
-        stringData:
-          api_key: "?{vaultkv:${cluster:tenant}/${cluster:name}/exoscale-metrics-collector/api_key}"
-          api_secret: "?{vaultkv:${cluster:tenant}/${cluster:name}/exoscale-metrics-collector/api_secret}"
+    schedule: '10 10,16,20 * * *'
 ----
 
 See the xref:references/parameters.adoc[parameters] reference for a full list of parameters.

docs/modules/ROOT/pages/how-tos/multi-instance.adoc

@@ -0,0 +1,44 @@
+= Deploy multiple instances
+
+This guide provides an example how to deploy multiple instances of this component.
+
+== Requirements
+
+
+. Prepare catalog by configuring 2 instances
++
+[source,yaml]
+----
+applications:
+  - exoscale-metrics-collector as collector-exoscale-ch-gva-2 <1>
+  - exoscale-metrics-collector as collector-cloudscale-rma-0 <2>
+parameters:
+  appuio_cloud_reporting:
+    namespace: 'appuio-cloud-reporting'
+    database:
+      name: 'reporting'
+      host: 'reporting-db.appuio-reporting.svc'
+      parameters: 'sslmode=disable'
+      password: 'passw0rd'
+      port: 5432
+----
+<1> Instance one with alias name collector-exoscale-ch-gva-2
+<2> Instance two with alias name collector-cloudscale-rma-0
++
+
+. Add relevant entries to Vault
++
+[source,bash]
+----
+parent="clusters/kv/${TENANT_ID}/${CLUSTER_ID}"
+instance_1="collector-exoscale-ch-gva-2"
+instance_2="collector-cloudscale-rma-0"
+
+vault kv put "${parent}/exoscale-metrics-collector/${instance_1}" exoscale-key=<key-1> exoscale-secret=<secret-1> cluster-server=<server-url-1> cluster-token=<token-1>
+vault kv put "${parent}/exoscale-metrics-collector/${instance_2}" exoscale-key=<key-2> exoscale-secret=<secret-2> cluster-server=<server-url-2> cluster-token=<token-2>
+----
++
+
+. Compile and push the cluster catalog
+. Wait until changes are applied
+. Verify that the instances are up and configured correctly

docs/modules/ROOT/pages/references/parameters.adoc

@@ -27,7 +27,7 @@ default:: `10 10,16,20 * * *`
 
 The cron schedule at which the metrics collection job is spawned.
 
-== `secrets.exoscale.stringData.api_key`
+== `secrets.credentials.stringData.EXOSCALE_API_KEY`
 
 [horizontal]
 type:: string
@@ -38,7 +38,7 @@ The Exoscale API key.
 You need to get the token from the https://portal.exoscale.com[Exoscale Console].
 You need to select the correct account (token is limited to one account), choose "IAM" in the menu and generate a new key pair.
 
-== `secrets.exoscale.stringData.api_secret`
+== `secrets.credentials.stringData.EXOSCALE_API_SECRET`
 
 [horizontal]
 type:: string
@@ -47,4 +47,22 @@ default:: Required.
 The Exoscale API secret.
 
 Second part of the Exoscale API credentials.
-See api_key for instructions.
+See EXOSCALE_API_KEY for instructions.
+
+== `secrets.credentials.stringData.K8S_SERVER_URL`
+
+[horizontal]
+type:: string
+default:: Required.
+
+The Kubernetes server URL.
+
+== `secrets.credentials.stringData.K8S_TOKEN`
+
+[horizontal]
+type:: string
+default:: Required.
+
+The token to connect to a Kubernetes cluster.
+
+The Service Account connected to this token should have `get` and `list` permissions to `buckets.exoscale.crossplane.io` managed resource.