Merge pull request #14 from vshn/component

Add Commodore component
vshn · Sep 28, 2022 · d02b844 · d02b844
2 parents 5f02563 + 422516e
commit d02b844
Show file tree

Hide file tree

Showing 16 changed files with 415 additions and 4 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -8,6 +8,9 @@ on:
     branches:
       - master
 
+env:
+  COMPONENT_NAME: exoscale-metrics-collector
+
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -34,8 +37,16 @@ jobs:
       - name: Upload code coverage report to Code Climate
         uses: paambaati/codeclimate-action@v3.0.0
         env:
-          # TODO: Update Reporter ID after importing in codeclimate.com
-          CC_TEST_REPORTER_ID: 8297d060a2a57278046db1ec5bb0e831a5e1c4e3624ad3991492157b60fc8433
+          CC_TEST_REPORTER_ID: 562205f0a6edaf974f08b29e0eecd479e6b3efd6fa40eee105e03c56e5efa4c5
         with:
           coverageLocations: cover.out:gocov
           prefix: github.com/${{ github.repository }}
+
+  golden:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Golden diff
+        working-directory: component/
+        run: make test golden-diff-all
diff --git a/component/Makefile b/component/Makefile
@@ -0,0 +1,85 @@
+MAKEFLAGS += --warn-undefined-variables
+SHELL := bash
+.SHELLFLAGS := -eu -o pipefail -c
+.DEFAULT_GOAL := all
+.DELETE_ON_ERROR:
+.SUFFIXES:
+
+include Makefile.vars.mk
+
+.PHONY: help
+help: ## Show this help
+	@grep -E -h '\s##\s' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = "(: ).*?## "}; {gsub(/\\:/,":", $$1)}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
+
+.PHONY: all
+all: lint
+
+.PHONY: lint
+lint: lint_jsonnet lint_yaml lint_adoc lint_kubent ## All-in-one linting
+
+.PHONY: lint_jsonnet
+lint_jsonnet: $(JSONNET_FILES) ## Lint jsonnet files
+	$(JSONNET_DOCKER) $(JSONNETFMT_ARGS) --test -- $?
+
+.PHONY: lint_yaml
+lint_yaml: ## Lint yaml files
+	$(YAMLLINT_DOCKER) -f parsable -c $(YAMLLINT_CONFIG) $(YAMLLINT_ARGS) -- .
+
+.PHONY: lint_adoc
+lint_adoc: ## Lint documentation
+	$(VALE_CMD) $(VALE_ARGS)
+.PHONY: lint_kubent
+lint_kubent: ## Check for deprecated Kubernetes API versions
+	$(KUBENT_DOCKER) $(KUBENT_ARGS) -f $(KUBENT_FILES)
+
+.PHONY: format
+format: format_jsonnet ## All-in-one formatting
+
+.PHONY: format_jsonnet
+format_jsonnet: $(JSONNET_FILES) ## Format jsonnet files
+	$(JSONNET_DOCKER) $(JSONNETFMT_ARGS) -- $?
+
+.PHONY: docs-serve
+docs-serve: ## Preview the documentation
+	$(ANTORA_PREVIEW_CMD)
+
+.PHONY: compile
+.compile:
+	mkdir -p dependencies
+	$(COMPILE_CMD)
+
+.PHONY: test
+test: commodore_args += -f tests/$(instance).yml
+test: .compile ## Compile the component
+
+.PHONY: gen-golden
+gen-golden: commodore_args += -f tests/$(instance).yml
+gen-golden: clean .compile ## Update the reference version for target `golden-diff`.
+	@rm -rf tests/golden/$(instance)
+	@mkdir -p tests/golden/$(instance)
+	@cp -R compiled/. tests/golden/$(instance)/.
+
+.PHONY: golden-diff
+golden-diff: commodore_args += -f tests/$(instance).yml
+golden-diff: clean .compile ## Diff compile output against the reference version. Review output and run `make gen-golden golden-diff` if this target fails.
+	@git diff --exit-code --minimal --no-index -- tests/golden/$(instance) compiled/
+
+.PHONY: golden-diff-all
+golden-diff-all: recursive_target=golden-diff
+golden-diff-all: $(test_instances) ## Run golden-diff for all instances. Note: this doesn't work when running make with multiple parallel jobs (-j != 1).
+
+.PHONY: gen-golden-all
+gen-golden-all: recursive_target=gen-golden
+gen-golden-all: $(test_instances) ## Run gen-golden for all instances. Note: this doesn't work when running make with multiple parallel jobs (-j != 1).
+
+.PHONY: lint_kubent_all
+lint_kubent_all: recursive_target=lint_kubent
+lint_kubent_all: $(test_instances) ## Lint deprecated Kubernetes API versions for all golden test instances. Will exit on first error. Note: this doesn't work when running make with multiple parallel jobs (-j != 1).
+
+.PHONY: $(test_instances)
+$(test_instances):
+	$(MAKE) $(recursive_target) -e instance=$(basename $(@F))
+
+.PHONY: clean
+clean: ## Clean the project
+	rm -rf .cache compiled dependencies vendor helmcharts jsonnetfile*.json || true
diff --git a/component/Makefile.vars.mk b/component/Makefile.vars.mk
@@ -0,0 +1,47 @@
+# Commodore takes the root dir name as the component name
+COMPONENT_NAME ?= exoscale-metrics-collector
+COMPONENT_SUBDIR ?= $(shell basename ${PWD})
+
+compiled_path   ?= compiled/$(COMPONENT_NAME)/$(COMPONENT_NAME)
+root_volume     ?= -v "$${PWD}/../:/$(COMPONENT_NAME)"
+compiled_volume ?= -v "$${PWD}/$(compiled_path):/$(COMPONENT_NAME)"
+commodore_args  ?= --search-paths . -n $(COMPONENT_NAME)
+
+ifneq "$(shell which docker 2>/dev/null)" ""
+	DOCKER_CMD    ?= $(shell which docker)
+	DOCKER_USERNS ?= ""
+else
+	DOCKER_CMD    ?= podman
+	DOCKER_USERNS ?= keep-id
+endif
+DOCKER_ARGS ?= run --rm -u "$$(id -u):$$(id -g)" --userns=$(DOCKER_USERNS) -w /$(COMPONENT_NAME)/$(COMPONENT_SUBDIR) -e HOME="/$(COMPONENT_NAME)"
+
+JSONNET_FILES   ?= $(shell find . -type f -not -path './vendor/*' \( -name '*.*jsonnet' -or -name '*.libsonnet' \))
+JSONNETFMT_ARGS ?= --in-place --pad-arrays
+JSONNET_IMAGE   ?= docker.io/bitnami/jsonnet:latest
+JSONNET_DOCKER  ?= $(DOCKER_CMD) $(DOCKER_ARGS) $(root_volume) --entrypoint=jsonnetfmt $(JSONNET_IMAGE)
+
+YAMLLINT_ARGS   ?= --no-warnings
+YAMLLINT_CONFIG ?= .yamllint.yml
+YAMLLINT_IMAGE  ?= docker.io/cytopia/yamllint:latest
+YAMLLINT_DOCKER ?= $(DOCKER_CMD) $(DOCKER_ARGS) $(root_volume) $(YAMLLINT_IMAGE)
+
+VALE_CMD  ?= $(DOCKER_CMD) $(DOCKER_ARGS) $(root_volume) --volume "$${PWD}"/docs/modules:/pages docker.io/vshn/vale:2.1.1
+VALE_ARGS ?= --minAlertLevel=error --config=/pages/ROOT/pages/.vale.ini /pages
+
+ANTORA_PREVIEW_CMD ?= $(DOCKER_CMD) run --rm --publish 35729:35729 --publish 2020:2020 --volume "${PWD}/.git":/preview/antora/.git --volume "${PWD}/docs":/preview/antora/docs docker.io/vshn/antora-preview:3.0.1.1 --style=syn --antora=docs
+
+
+COMMODORE_CMD  ?= $(DOCKER_CMD) $(DOCKER_ARGS) $(root_volume) docker.io/projectsyn/commodore:latest
+COMPILE_CMD    ?= $(COMMODORE_CMD) component compile . $(commodore_args)
+JB_CMD         ?= $(DOCKER_CMD) $(DOCKER_ARGS) --entrypoint /usr/local/bin/jb docker.io/projectsyn/commodore:latest install
+GOLDEN_FILES    ?= $(shell find tests/golden/$(instance) -type f)
+
+KUBENT_FILES    ?= $(shell echo "$(GOLDEN_FILES)" | sed 's/ /,/g')
+KUBENT_ARGS     ?= -c=false --helm2=false --helm3=false -e
+# Use our own kubent image until the upstream image is available
+KUBENT_IMAGE    ?= docker.io/projectsyn/kubent:latest
+KUBENT_DOCKER   ?= $(DOCKER_CMD) $(DOCKER_ARGS) $(root_volume) --entrypoint=/app/kubent $(KUBENT_IMAGE)
+
+instance ?= defaults
+test_instances = tests/defaults.yml
diff --git a/component/class/defaults.yml b/component/class/defaults.yml
@@ -0,0 +1,15 @@
+parameters:
+  exoscale_metrics_collector:
+    secrets:
+      exoscale:
+        stringData:
+          api_key: "?{vaultkv:${cluster:tenant}/${cluster:name}/exoscale-metrics-collector/key}"
+          api_secret: "?{vaultkv:${cluster:tenant}/${cluster:name}/exoscale-metrics-collector/secret}"
+    images:
+      collector:
+        registry: 'ghcr.io'
+        repository: 'vshn/exoscale-metrics-collector'
+        tag: 'v0.0.2'
+    # Times in UTC! Don't run job around midnight as exoscale API may return incomplete data
+    # default: Every day at minute 10 past hour 10, 16 and 20.
+    schedule: '10 10,16,20 * * *'
diff --git a/component/class/exoscale-metrics-collector.yml b/component/class/exoscale-metrics-collector.yml
@@ -0,0 +1,11 @@
+parameters:
+  kapitan:
+    compile:
+      - input_paths:
+          - ${_base_directory}/component/app.jsonnet
+        input_type: jsonnet
+        output_path: apps/
+      - input_paths:
+          - ${_base_directory}/component/main.jsonnet
+        input_type: jsonnet
+        output_path: exoscale-metrics-collector/
diff --git a/component/component/app.jsonnet b/component/component/app.jsonnet
@@ -0,0 +1,11 @@
+local kap = import 'lib/kapitan.libjsonnet';
+local inv = kap.inventory();
+local params = inv.parameters.exoscale_metrics_collector;
+local paramsACR = inv.parameters.appuio_cloud_reporting;
+local argocd = import 'lib/argocd.libjsonnet';
+
+local app = argocd.App('exoscale-metrics-collector', paramsACR.namespace);
+
+{
+  'exoscale-metrics-collector': app,
+}
diff --git a/component/component/main.jsonnet b/component/component/main.jsonnet
@@ -0,0 +1,113 @@
+local kap = import 'lib/kapitan.libjsonnet';
+local inv = kap.inventory();
+local params = inv.parameters.exoscale_metrics_collector;
+local paramsACR = inv.parameters.appuio_cloud_reporting;
+local kube = import 'lib/kube.libjsonnet';
+local com = import 'lib/commodore.libjsonnet';
+local collectorImage = '%(registry)s/%(repository)s:%(tag)s' % params.images.collector;
+
+
+local labels = {
+  'app.kubernetes.io/name': 'exoscale-metrics-collector',
+  'app.kubernetes.io/managed-by': 'commodore',
+  'app.kubernetes.io/part-of': 'appuio-cloud-reporting',
+  'app.kubernetes.io/component': 'exoscale-metrics-collector',
+};
+
+local secrets = [
+  if params.secrets[s] != null then
+    kube.Secret(s) {
+      metadata+: {
+        namespace: paramsACR.namespace,
+      },
+    } + com.makeMergeable(params.secrets[s])
+  for s in std.objectFields(params.secrets)
+];
+
+{
+  assert params.secrets != null : 'secrets must be set.',
+  assert params.secrets.exoscale != null : 'secrets.exoscale must be set.',
+  assert params.secrets.exoscale.stringData != null : 'secrets.exoscale.stringData must be set.',
+  assert params.secrets.exoscale.stringData.api_key != null : 'secrets.exoscale.stringData.api_key must be set.',
+  assert params.secrets.exoscale.stringData.api_secret != null : 'secrets.exoscale.stringData.api_secret must be set.',
+
+  secrets: std.filter(function(it) it != null, secrets),
+
+  cronjob: {
+    kind: 'CronJob',
+    apiVersion: 'batch/v1',
+    metadata: {
+      name: 'exoscale-metrics-collector',
+      namespace: paramsACR.namespace,
+      labels+: labels,
+    },
+    spec: {
+      concurrencyPolicy: 'Forbid',
+      failedJobsHistoryLimit: 5,
+      jobTemplate: {
+        spec: {
+          template: {
+            spec: {
+              restartPolicy: 'OnFailure',
+              containers: [
+                {
+                  name: 'exoscale-metrics-collector-backfill',
+                  image: collectorImage,
+                  args: [
+                    'exoscale-metrics-collector',
+                  ],
+                  command: [ 'sh', '-c' ],
+                  env: [
+                    {
+                      name: 'password',
+                      valueFrom: {
+                        secretKeyRef: {
+                          key: 'password',
+                          name: 'reporting-db',
+                        },
+                      },
+                    },
+                    {
+                      name: 'username',
+                      valueFrom: {
+                        secretKeyRef: {
+                          key: 'username',
+                          name: 'reporting-db',
+                        },
+                      },
+                    },
+                    {
+                      name: 'ACR_DB_URL',
+                      value: 'postgres://$(username):$(password)@%(host)s:%(port)s/%(name)s?%(parameters)s' % paramsACR.database,
+                    },
+                    {
+                      name: 'EXOSCALE_API_KEY',
+                      valueFrom: {
+                        secretKeyRef: {
+                          key: 'api_key',
+                          name: 'exoscale',
+                        },
+                      },
+                    },
+                    {
+                      name: 'EXOSCALE_API_SECRET',
+                      valueFrom: {
+                        secretKeyRef: {
+                          key: 'api_secret',
+                          name: 'exoscale',
+                        },
+                      },
+                    },
+                  ],
+                  resources: {},
+                },
+              ],
+            },
+          },
+        },
+      },
+      schedule: params.schedule,
+      successfulJobsHistoryLimit: 3,
+    },
+  },
+}
diff --git a/component/jsonnetfile.json b/component/jsonnetfile.json
@@ -0,0 +1,14 @@
+{
+    "version": 1,
+    "dependencies": [
+        {
+            "source": {
+                "git": {
+                    "remote": "https://github.com/bitnami-labs/kube-libsonnet"
+                }
+            },
+            "version": "v1.19.0"
+        }
+    ],
+    "legacyImports": true
+}
diff --git a/component/jsonnetfile.lock.json b/component/jsonnetfile.lock.json
@@ -0,0 +1,16 @@
+{
+  "version": 1,
+  "dependencies": [
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/bitnami-labs/kube-libsonnet.git",
+          "subdir": ""
+        }
+      },
+      "version": "20bf70d8d24b3a50baaae8f951dcbc62a66b464f",
+      "sum": "e0p4pbwKnpRfUm8CXqe+WC1Hcwih1lqIm1ODPsCV96Q="
+    }
+  ],
+  "legacyImports": false
+}
diff --git a/component/tests/defaults.yml b/component/tests/defaults.yml
@@ -0,0 +1,9 @@
+parameters:
+  appuio_cloud_reporting:
+    namespace: 'appuio-cloud-reporting'
+    database:
+      name: 'reporting'
+      host: 'reporting-db.appuio-reporting.svc'
+      parameters: 'sslmode=disable'
+      password: 'passw0rd'
+      port: 5432
diff --git a/docs/modules/ROOT/partials/.gitkeep → ...ctor/apps/exoscale-metrics-collector.yaml b/docs/modules/ROOT/partials/.gitkeep → ...ctor/apps/exoscale-metrics-collector.yaml