From 5e46b8c6c735e5550505ff76352e035674d4dd21 Mon Sep 17 00:00:00 2001 From: arttii Date: Fri, 7 Aug 2020 17:23:16 +0200 Subject: [PATCH 1/5] Add gitopsable structure, only for kapp-controller --- .vscode/launch.json | 16 +++- cmd/apply.go | 60 +++++++++----- cmd/applyGitOps.go | 43 ++++++++++ cmd/applyNamespace.go | 4 +- realm/namespaces/_defaults.yaml | 17 +++- realm/namespaces/default/_namespace.yaml | 2 + .../default/monitoring/_app_group.yaml | 2 +- .../default/monitoring/dummy.yaml.gitops | 46 +++++++++++ .../default/monitoring/test.yaml.gitops | 46 +++++++++++ realm/templates/templates.yaml | 55 +++++++++++++ .../env/dev/default/monitoring/k8s/dummy.yaml | 80 +++++++++++++++++++ realm/values/env/dev/default/values.yaml | 1 + realm/values/env/dev/values.yaml | 15 ++++ 13 files changed, 360 insertions(+), 27 deletions(-) create mode 100644 cmd/applyGitOps.go create mode 100644 realm/namespaces/default/_namespace.yaml create mode 100644 realm/namespaces/default/monitoring/dummy.yaml.gitops create mode 100644 realm/namespaces/default/monitoring/test.yaml.gitops create mode 100644 realm/templates/templates.yaml create mode 100644 realm/values/env/dev/default/monitoring/k8s/dummy.yaml create mode 100644 realm/values/env/dev/default/values.yaml create mode 100644 realm/values/env/dev/values.yaml diff --git a/.vscode/launch.json b/.vscode/launch.json index 2f9d910..618f01f 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -47,7 +47,21 @@ "host": "127.0.0.1", "program": "${workspaceRoot}/main.go", // "envFile": "${workspaceRoot}/.dev.env", - "args": ["apply", "-e","dev", "-n", "default","group","--app","dummy","monitoring"], + "args": ["apply", "-e","dev", "-n", "default","monitoring"], + "showLog": true + }, + { + "name": "Launch Gitops", + "type": "go", + "request": "launch", + "mode": "debug", + "remotePath": "", + "cwd": "${workspaceRoot}", + "port": 2345, + "host": "127.0.0.1", + "program": "${workspaceRoot}/main.go", + // "envFile": "${workspaceRoot}/.dev.env", + "args": ["apply","-e","dev", "-n", "default","gitops-defs","--app","dummy","monitoring"], "showLog": true }, { diff --git a/cmd/apply.go b/cmd/apply.go index e96bfeb..d888c20 100644 --- a/cmd/apply.go +++ b/cmd/apply.go @@ -19,11 +19,14 @@ import ( "fmt" "os" "path/filepath" + "regexp" "github.com/DataReply/korgi/pkg/utils" "github.com/spf13/cobra" ) +const DefaultMatcher = "(.yaml|.yml)$" + func templateApp(app string, inputFilePath string, appGroupDir string, lint bool) error { targeAppDir := utils.ConcatDirs(appGroupDir, app) @@ -53,7 +56,7 @@ func getFinalOutputDir(outputDir string, isolated bool) string { return outputDir } -func applyAppGroup(group string, namespace string, outputDir string, appFilter string, lint bool, dryRun bool) error { +func applyAppGroup(group string, namespace string, outputDir string, appFilter string, lint bool, dryRun bool, match string) error { log.V(0).Info("applying", "group", group, "namespace", namespace, "app", appFilter, "lint", lint, "dry", dryRun) namespaceDir := utils.GetNamespaceDir(namespace) @@ -75,6 +78,8 @@ func applyAppGroup(group string, namespace string, outputDir string, appFilter s return fmt.Errorf("listing group directory: %w", err) } + matchers, _ := regexp.Compile(match) + for _, matchedAppFile := range matches { appFile := filepath.Base(matchedAppFile) if appFile != "_app_group.yaml" { @@ -86,9 +91,11 @@ func applyAppGroup(group string, namespace string, outputDir string, appFilter s } } - err = templateApp(app, matchedAppFile, targetAppGroupDir, lint) - if err != nil { - return fmt.Errorf("templating app: %w", err) + if matchers.MatchString(appFile) { + err = templateApp(app, matchedAppFile, targetAppGroupDir, lint) + if err != nil { + return fmt.Errorf("templating app: %w", err) + } } } @@ -112,40 +119,49 @@ func applyAppGroup(group string, namespace string, outputDir string, appFilter s return nil } -// applyCmd represents the apply command -var applyCmd = &cobra.Command{ - Use: "apply", - Short: "Apply resources to k8s", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { +func runApplyWithMatch(cmd *cobra.Command, args []string, match string) error { + + namespace, _ := cmd.Flags().GetString("namespace") - namespace, _ := cmd.Flags().GetString("namespace") + lint, _ := cmd.Flags().GetBool("lint") - lint, _ := cmd.Flags().GetBool("lint") + dryRun, _ := cmd.Flags().GetBool("dry-run") - dryRun, _ := cmd.Flags().GetBool("dry-run") + appFilter, _ := cmd.Flags().GetString("app") - appFilter, _ := cmd.Flags().GetString("app") + outputDir, _ := cmd.Flags().GetString("output-dir") - outputDir, _ := cmd.Flags().GetString("output-dir") + isolated, _ := cmd.Flags().GetBool("isolate") + + err := applyAppGroup(args[0], namespace, getFinalOutputDir(outputDir, isolated), appFilter, lint, dryRun, match) + if err != nil { + return err + } + return nil +} - isolated, _ := cmd.Flags().GetBool("isolated") +// applyCmd represents the apply command +var applyCmd = &cobra.Command{ + Use: "apply", + Short: "Apply resources to k8s", + Args: cobra.ExactArgs(1), + TraverseChildren: true, + RunE: func(cmd *cobra.Command, args []string) error { - err := applyAppGroup(args[0], namespace, getFinalOutputDir(outputDir, isolated), appFilter, lint, dryRun) + err := runApplyWithMatch(cmd, args, DefaultMatcher) if err != nil { return err } - return nil }, } func init() { - rootCmd.AddCommand(applyCmd) - applyCmd.Flags().BoolP("lint", "l", false, "Lint temlate") - applyCmd.Flags().BoolP("dry-run", "d", false, "Dry Run") - applyCmd.Flags().StringP("namespace", "n", "", "Target namespace") + rootCmd.AddCommand(applyCmd) + applyCmd.PersistentFlags().BoolP("lint", "l", false, "Lint temlate") + applyCmd.PersistentFlags().BoolP("dry-run", "d", false, "Dry Run") + applyCmd.PersistentFlags().StringP("namespace", "n", "", "Target namespace") applyCmd.MarkFlagRequired("namespace") } diff --git a/cmd/applyGitOps.go b/cmd/applyGitOps.go new file mode 100644 index 0000000..bcf6026 --- /dev/null +++ b/cmd/applyGitOps.go @@ -0,0 +1,43 @@ +/* +Copyright © 2020 Artyom Topchyan a.topchyan@reply.de + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package cmd + +import ( + "github.com/spf13/cobra" +) + +const GitOpsMatcher = "(.yaml.gitops|.yml.gitops)$" + +// applyAppDefs represents the apply command +var applyAppDefs = &cobra.Command{ + Use: "gitops-defs", + Short: "Apply custom definition for generated app resources to k8s", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + err := runApplyWithMatch(cmd, args, GitOpsMatcher) + if err != nil { + return err + } + return nil + + }, +} + +func init() { + + applyCmd.AddCommand(applyAppDefs) + +} diff --git a/cmd/applyNamespace.go b/cmd/applyNamespace.go index 2ce1219..50006bb 100644 --- a/cmd/applyNamespace.go +++ b/cmd/applyNamespace.go @@ -38,7 +38,7 @@ var namespaceCmd = &cobra.Command{ outputDir, _ := cmd.Flags().GetString("output-dir") - isolated, _ := cmd.Flags().GetBool("isolated") + isolated, _ := cmd.Flags().GetBool("isolate") namespace := args[0] @@ -51,7 +51,7 @@ var namespaceCmd = &cobra.Command{ if info.IsDir() && path != namespaceDir { group := filepath.Base(path) - err := applyAppGroup(group, namespace, getFinalOutputDir(outputDir, isolated), appFilter, lint, dryRun) + err := applyAppGroup(group, namespace, getFinalOutputDir(outputDir, isolated), appFilter, lint, dryRun, DefaultMatcher) if err != nil { return err } diff --git a/realm/namespaces/_defaults.yaml b/realm/namespaces/_defaults.yaml index b1668a0..341a6f5 100644 --- a/realm/namespaces/_defaults.yaml +++ b/realm/namespaces/_defaults.yaml @@ -2,7 +2,22 @@ repositories: # To use official "stable" charts a.k.a https://github.com/helm/charts/tree/master/stable - name: stable url: https://kubernetes-charts.storage.googleapis.com - + +helmDefaults: + #tillerNamespace: tiller-namespace #dedicated default key for tiller-namespace + tillerless: true #dedicated default key for tillerless + cleanupOnFail: false #dedicated default key for helm flag --cleanup-on-fail + verify: true + wait: true + timeout: 600 + recreatePods: true + + +environments: + default: + values: + - test: test + releases: - name: dummy chart: stable/raw \ No newline at end of file diff --git a/realm/namespaces/default/_namespace.yaml b/realm/namespaces/default/_namespace.yaml new file mode 100644 index 0000000..6d6a243 --- /dev/null +++ b/realm/namespaces/default/_namespace.yaml @@ -0,0 +1,2 @@ +values: +- namespace: default \ No newline at end of file diff --git a/realm/namespaces/default/monitoring/_app_group.yaml b/realm/namespaces/default/monitoring/_app_group.yaml index 02eaff4..bebcde3 100644 --- a/realm/namespaces/default/monitoring/_app_group.yaml +++ b/realm/namespaces/default/monitoring/_app_group.yaml @@ -1,4 +1,4 @@ values: - - appGroup: testing + - appGroup: monitoring diff --git a/realm/namespaces/default/monitoring/dummy.yaml.gitops b/realm/namespaces/default/monitoring/dummy.yaml.gitops new file mode 100644 index 0000000..1b8b9de --- /dev/null +++ b/realm/namespaces/default/monitoring/dummy.yaml.gitops @@ -0,0 +1,46 @@ + + +bases: + - ../../_defaults.yaml + - ../_namespace.yaml + - ../../../values/env/{{.Environment.Name}}/values.yaml + - _app_group.yaml + +--- +templates: +{{ tpl ( readFile "./../../../templates/templates.yaml") . }} + +releases: + - name: {{.Values.app}} + <<: *base-local + chart: incubator/raw + version: 0.2.3 + values: + - templates: + - | + apiVersion: kappctrl.k14s.io/v1alpha1 + kind: App + metadata: + name: {{.Values.appGroup}}-{{.Values.app}} + namespace: {{.Values.namespace}} + spec: + serviceAccountName: {{.Values.namespaceDeployServiceAccount}} + fetch: + - image: + url: arttii/dummydeploy + # secretRef: + # name: {{.Values.dockerSecret}} + subPath: {{.Values.appGroup}}/dummy + template: + - ytt: + ignoreUnknownComments: true + strict: false + deploy: + - kapp: + rawOptions: ["--dangerous-override-ownership-of-existing-resources=true","--apply-concurrency=10"] + delete: + rawOptions: ["--apply-ignored=true"] + + + + diff --git a/realm/namespaces/default/monitoring/test.yaml.gitops b/realm/namespaces/default/monitoring/test.yaml.gitops new file mode 100644 index 0000000..4410511 --- /dev/null +++ b/realm/namespaces/default/monitoring/test.yaml.gitops @@ -0,0 +1,46 @@ + + +bases: + - ../../_defaults.yaml + - ../_namespace.yaml + - ../../../values/env/{{.Environment.Name}}/values.yaml + - _app_group.yaml + +--- +templates: +{{ tpl ( readFile "./../../../templates/templates.yaml") . }} + +releases: + - name: {{.Values.app}} + <<: *base-local + chart: incubator/raw + version: 0.2.3 + values: + - templates: + - | + apiVersion: kappctrl.k14s.io/v1alpha1 + kind: App + metadata: + name: {{.Values.appGroup}}-{{.Values.app}} + namespace: {{.Values.namespace}} + spec: + serviceAccountName: {{.Values.namespaceDeployServiceAccount}} + fetch: + - image: + url: arttii/dummydeploy + # secretRef: + # name: {{.Values.dockerSecret}} + subPath: {{.Values.appGroup}}/test + template: + - ytt: + ignoreUnknownComments: true + strict: false + deploy: + - kapp: + rawOptions: ["--dangerous-override-ownership-of-existing-resources=true","--apply-concurrency=10"] + delete: + rawOptions: ["--apply-ignored=true"] + + + + diff --git a/realm/templates/templates.yaml b/realm/templates/templates.yaml new file mode 100644 index 0000000..8b42739 --- /dev/null +++ b/realm/templates/templates.yaml @@ -0,0 +1,55 @@ + base: &base-local + missingFileHandler: Info + namespace: {{ .Values.namespace }} + hooks: [] + secrets: + # - ../../../values/defaults/secrets.yaml + - ../../../values/env/{{`{{ .Environment.Name }}`}}/secrets.yaml + + # - ../../../values/defaults/{{`{{ .Release.Namespace }}`}}/secrets.yaml + - ../../../values/env/{{`{{ .Environment.Name }}`}}/{{`{{ .Release.Namespace }}`}}/secrets.yaml + + # - ../../../values/defaults/{{`{{ .Release.Namespace }}`}}/{{ .Values.appGroup }}/secrets.yaml + - ../../../values/env/{{`{{ .Environment.Name }}`}}/{{`{{ .Release.Namespace }}`}}/{{ .Values.appGroup }}/secrets.yaml + + # - ../../../values/defaults/{{`{{ .Release.Namespace }}`}}/{{ .Values.appGroup }}/{{ .Values.app }}/secrets.yaml + - ../../../values/env/{{`{{ .Environment.Name }}`}}/{{`{{ .Release.Namespace }}`}}/{{ .Values.appGroup }}/{{ .Values.app }}/secrets.yaml + + # - ../../../values/defaults/{{`{{ .Release.Namespace }}`}}/{{ .Values.appGroup }}/{{ .Values.app }}/{{`{{ .Release.Name }}`}}.secrets.yaml + - ../../../values/env/{{`{{ .Environment.Name }}`}}/{{`{{ .Release.Namespace }}`}}/{{ .Values.appGroup }}/{{ .Values.app }}/{{`{{ .Release.Name }}`}}.secrets.yaml + valuesTemplate: + - namespace: {{ .Values.namespace }} + + # top level generics + # - ../../../values/defaults/values.yaml + # - ../../../values/defaults/values.gotmpl + - ../../../values/env/{{`{{ .Environment.Name }}`}}/values.yaml + - ../../../values/env/{{`{{ .Environment.Name }}`}}/values.gotmpl + + # defaults and overrides on a namespace level + # - ../../../values/defaults/{{`{{ .Release.Namespace }}`}}/values.yaml + # - ../../../values/defaults/{{`{{ .Release.Namespace }}`}}/values.gotmpl + - ../../../values/env/{{`{{ .Environment.Name }}`}}/{{`{{ .Release.Namespace }}`}}/values.yaml + - ../../../values/env/{{`{{ .Environment.Name }}`}}/{{`{{ .Release.Namespace }}`}}/values.gotmpl + + # defaults and overrides on a namespace+app-group level + # - ../../../values/defaults/{{`{{ .Release.Namespace }}`}}/{{ .Values.appGroup }}/values.yaml + # - ../../../values/defaults/{{`{{ .Release.Namespace }}`}}/{{ .Values.appGroup }}/values.gotmpl + - ../../../values/env/{{`{{ .Environment.Name }}`}}/{{`{{ .Release.Namespace }}`}}/{{ .Values.appGroup }}/values.yaml + - ../../../values/env/{{`{{ .Environment.Name }}`}}/{{`{{ .Release.Namespace }}`}}/{{ .Values.appGroup }}/values.gotmpl + + # defaults and overrides on a namespace+app-group+app level + # - ../../../values/defaults/{{`{{ .Release.Namespace }}`}}/{{ .Values.appGroup }}/{{ .Values.app }}/values.yaml + # - ../../../values/defaults/{{`{{ .Release.Namespace }}`}}/{{ .Values.appGroup }}/{{ .Values.app }}/values.gotmpl + - ../../../values/env/{{`{{ .Environment.Name }}`}}/{{`{{ .Release.Namespace }}`}}/{{ .Values.appGroup }}/{{ .Values.app }}/values.yaml + - ../../../values/env/{{`{{ .Environment.Name }}`}}/{{`{{ .Release.Namespace }}`}}/{{ .Values.appGroup }}/{{ .Values.app }}/values.gotmpl + + # defaults and overrides on a namespace+app-group+app+release level + # - ../../../values/defaults/{{`{{ .Release.Namespace }}`}}/{{ .Values.appGroup }}/{{ .Values.app }}/{{`{{ .Release.Name }}`}}.yaml + # - ../../../values/defaults/{{`{{ .Release.Namespace }}`}}/{{ .Values.appGroup }}/{{ .Values.app }}/{{`{{ .Release.Name }}`}}.gotmpl + - ../../../values/env/{{`{{ .Environment.Name }}`}}/{{`{{ .Release.Namespace }}`}}/{{ .Values.appGroup }}/{{ .Values.app }}/{{`{{ .Release.Name }}`}}.yaml + - ../../../values/env/{{`{{ .Environment.Name }}`}}/{{`{{ .Release.Namespace }}`}}/{{ .Values.appGroup }}/{{ .Values.app }}/{{`{{ .Release.Name }}`}}.gotmpl + + + + diff --git a/realm/values/env/dev/default/monitoring/k8s/dummy.yaml b/realm/values/env/dev/default/monitoring/k8s/dummy.yaml new file mode 100644 index 0000000..dce3231 --- /dev/null +++ b/realm/values/env/dev/default/monitoring/k8s/dummy.yaml @@ -0,0 +1,80 @@ +appLabel: "k8s-tests" + +serviceAccountName: k8s-tests + +secretPostgres: testuser.usecase-postgres-db.credentials.postgresql.acid.zalan.do + +configMap: + name: k8s-tests-validation-configs + data: + - name: NODES_COUNT + value: 6 + - name: MASTERS_COUNT + value: 3 + - name: DBHOST + value: usecase-postgres-db.usecase + - name: DBNAME + value: postgres + - name: DBPORT + value: 5432 + - name: DBSECRET + value: testuser.usecase-postgres-db.credentials.postgresql.acid.zalan.do + - name: LOKI_URL + value: http://loki.dap-base:3100 + - name: PROMETHEUS_URL + value: http://prometheus-operated.dap-base:9090 + - name: GRAFANA_URL + value: http://grafana.dap-base:3000 + - name: REGISTRY_URL + value: docker-registry.dap-base:5000 +# - name: REGISTRY_IMAGE +# value: busybox:local +# - name: REGISTRY_USER +# value: ... +# - name: REGISTRY_PASS +# value: ... + {{ if eq (.Values.auth|default false) true }} + - name: AUTH + value: true + {{ end }} + {{ if eq (.Values.blockStorageClass | default "isilon") "isilon" }} + - name: STORAGE_CLASS + value: isilon + {{ end }} + +validation: + name: k8s-tests-validation + schedule: "0 2 * * *" + container: + name: validation + image: datareply/k8s-tests:0.2 + imagePullPolicy: "IfNotPresent" + command: "python3" + args: "validation.py" + restartPolicy: OnFailure + backoffLimit: 4 + +checks: + name: k8s-tests-checks + schedule: "0 2 * * *" + volumes: + - name: docker-socket-volume + hostPath: + path: /var/run/docker.sock + type: File + - name: docker-bin-volume + hostPath: + path: /usr/bin/docker + container: + name: checks + image: datareply/k8s-tests:0.2 + imagePullPolicy: "IfNotPresent" + command: "python3" + args: "checks.py" + volumeMounts: + - mountPath: /var/run/docker.sock + name: docker-socket-volume + - mountPath: /usr/bin/docker + name: docker-bin-volume + restartPolicy: OnFailure + backoffLimit: 4 diff --git a/realm/values/env/dev/default/values.yaml b/realm/values/env/dev/default/values.yaml new file mode 100644 index 0000000..93c48c3 --- /dev/null +++ b/realm/values/env/dev/default/values.yaml @@ -0,0 +1 @@ +PLATFORM_ENV: dev \ No newline at end of file diff --git a/realm/values/env/dev/values.yaml b/realm/values/env/dev/values.yaml new file mode 100644 index 0000000..93e24f6 --- /dev/null +++ b/realm/values/env/dev/values.yaml @@ -0,0 +1,15 @@ +--- +values: + - imagePullPolicy: Always + - imageVersion: "latest" + - dockerSecret: "auth" + - namespaceDeployServiceAccount: 'default-ns-sa' + + +environments: + default: + values: + - tmp: "tmp" + dev: + values: + - kubeContext: "kubernetes-admin@cluster.local" From d70a5dcd8b82661e6865e533a28524cb6ec12c32 Mon Sep 17 00:00:00 2001 From: Anil Karatas Date: Thu, 6 Aug 2020 01:45:03 +0200 Subject: [PATCH 2/5] Increase Scanner buffer from 64*1024 to 1024*1024 --- .vscode/settings.json | 3 +++ pkg/utils/exec.go | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..304c3d7 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "go.inferGopath": false +} \ No newline at end of file diff --git a/pkg/utils/exec.go b/pkg/utils/exec.go index 95116e0..f473b5e 100644 --- a/pkg/utils/exec.go +++ b/pkg/utils/exec.go @@ -31,6 +31,9 @@ type ExecLine struct { func consume(wg *sync.WaitGroup, r io.Reader, logHook func(string)) { defer wg.Done() s := bufio.NewScanner(r) + buffer := make([]byte, 0, 1024*1024) + s.Buffer(buffer, 1024*1024) + for s.Scan() { line := s.Text() if line != "" { From 8613eef2cec302981be81b9ce175faeed282c3ec Mon Sep 17 00:00:00 2001 From: Anil Karatas Date: Mon, 10 Aug 2020 14:22:38 +0200 Subject: [PATCH 3/5] Add .vscode to gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 961583e..84e839f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .checkpoint .checkpoint.idx bin/ -*.csv.gz \ No newline at end of file +*.csv.gz +.vscode/ \ No newline at end of file From eef0a9fb5c24846d40158dac5508d23bb15e6495 Mon Sep 17 00:00:00 2001 From: Anil Karatas Date: Mon, 10 Aug 2020 14:22:51 +0200 Subject: [PATCH 4/5] Add OS and ARCH vars --- Makefile | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index f67f93a..642434b 100755 --- a/Makefile +++ b/Makefile @@ -1,22 +1,23 @@ -PROJECT_NAME :=korgi +PROJECT_NAME:=korgi GOFILES:=$(shell find . -name '*.go' | grep -v -E '(./vendor)') - +OS:="$(shell uname | tr '[:upper:]' '[:lower:]')" +ARCH:=amd64 + run: go run main.go all: clean check bin - bin: $(GOFILES) - mkdir -p bin/linux/ - GOOS=linux GOARCH=amd64 go build -a -tags musl -ldflags "$(LDFLAGS)" -o bin/linux/korgi main.go + mkdir -p bin/${OS}/ + GOOS=${OS} GOARCH=${ARCH} go build -a -tags musl -ldflags "$(LDFLAGS)" -o bin/${OS}/korgi main.go gofmt: gofmt -w -s pkg/ gofmt -w -s cmd/ test: - GOOS=linux GOARCH=amd64 go test ./... + GOOS=${OS} GOARCH=${ARCH} go test ./... check: @find . -name vendor -prune -o -name '*.go' -exec gofmt -s -d {} + From 2c2982a4bf1e0bbd060fda267e1dae1866214643 Mon Sep 17 00:00:00 2001 From: Anil Karatas Date: Mon, 10 Aug 2020 16:45:21 +0200 Subject: [PATCH 5/5] Reorganize command structure --- cmd/apply.go | 123 +----------------------------- cmd/applyGitOps.go | 6 +- cmd/applyGroup.go | 160 +++++++++++++++++++++++++++++++++++++++ cmd/applyNamespace.go | 20 +++-- cmd/const.go | 4 + cmd/delete.go | 24 +----- cmd/deleteGroup.go | 63 +++++++++++++++ cmd/deleteNamespace.go | 13 ++-- cmd/root.go | 2 +- pkg/utils/stringUtils.go | 2 +- 10 files changed, 247 insertions(+), 170 deletions(-) create mode 100644 cmd/applyGroup.go create mode 100644 cmd/const.go create mode 100644 cmd/deleteGroup.go diff --git a/cmd/apply.go b/cmd/apply.go index d888c20..35dd0e0 100644 --- a/cmd/apply.go +++ b/cmd/apply.go @@ -16,130 +16,9 @@ limitations under the License. package cmd import ( - "fmt" - "os" - "path/filepath" - "regexp" - - "github.com/DataReply/korgi/pkg/utils" "github.com/spf13/cobra" ) -const DefaultMatcher = "(.yaml|.yml)$" - -func templateApp(app string, inputFilePath string, appGroupDir string, lint bool) error { - - targeAppDir := utils.ConcatDirs(appGroupDir, app) - - err := os.MkdirAll(targeAppDir, os.ModePerm) - if err != nil { - return fmt.Errorf("creating app dir: %w", err) - } - if lint { - err = templateEngine.Lint(app, inputFilePath) - if err != nil { - return err - } - } - err = templateEngine.Template(app, inputFilePath, targeAppDir) - if err != nil { - return err - } - - return nil -} - -func getFinalOutputDir(outputDir string, isolated bool) string { - if isolated { - return utils.ConcatDirs(outputDir, execTime.Format("2006-01-02/15-04:05")) - } - return outputDir -} - -func applyAppGroup(group string, namespace string, outputDir string, appFilter string, lint bool, dryRun bool, match string) error { - - log.V(0).Info("applying", "group", group, "namespace", namespace, "app", appFilter, "lint", lint, "dry", dryRun) - namespaceDir := utils.GetNamespaceDir(namespace) - if _, err := os.Stat(namespaceDir); os.IsNotExist(err) { - return fmt.Errorf("%s directory does not exist", namespaceDir) - } - - appGroupDir := utils.ConcatDirs(namespaceDir, group) - - targetAppGroupDir := utils.ConcatDirs(outputDir, namespace, group) - - err := os.MkdirAll(targetAppGroupDir, os.ModePerm) - if err != nil { - return fmt.Errorf("creating group directory: %w", err) - } - - matches, err := filepath.Glob(appGroupDir + "/*") - if err != nil { - return fmt.Errorf("listing group directory: %w", err) - } - - matchers, _ := regexp.Compile(match) - - for _, matchedAppFile := range matches { - appFile := filepath.Base(matchedAppFile) - if appFile != "_app_group.yaml" { - app := utils.SanitzeAppName(appFile) - if appFilter != "" { - - if app != appFilter { - continue - } - } - - if matchers.MatchString(appFile) { - err = templateApp(app, matchedAppFile, targetAppGroupDir, lint) - if err != nil { - return fmt.Errorf("templating app: %w", err) - } - } - - } - - } - if !dryRun { - if appFilter != "" { - err = execEngine.DeployApp(group+"-"+appFilter, utils.ConcatDirs(targetAppGroupDir, appFilter), namespace) - if err != nil { - return fmt.Errorf("running kapp deploy with appFilter: %w", err) - } - return nil - } - - err = execEngine.DeployGroup(group, targetAppGroupDir, namespace) - if err != nil { - return fmt.Errorf("running kapp deploy: %w", err) - } - } - - return nil -} - -func runApplyWithMatch(cmd *cobra.Command, args []string, match string) error { - - namespace, _ := cmd.Flags().GetString("namespace") - - lint, _ := cmd.Flags().GetBool("lint") - - dryRun, _ := cmd.Flags().GetBool("dry-run") - - appFilter, _ := cmd.Flags().GetString("app") - - outputDir, _ := cmd.Flags().GetString("output-dir") - - isolated, _ := cmd.Flags().GetBool("isolate") - - err := applyAppGroup(args[0], namespace, getFinalOutputDir(outputDir, isolated), appFilter, lint, dryRun, match) - if err != nil { - return err - } - return nil -} - // applyCmd represents the apply command var applyCmd = &cobra.Command{ Use: "apply", @@ -148,7 +27,7 @@ var applyCmd = &cobra.Command{ TraverseChildren: true, RunE: func(cmd *cobra.Command, args []string) error { - err := runApplyWithMatch(cmd, args, DefaultMatcher) + err := runApplyWithMatch(cmd, args, defaultMatcher) if err != nil { return err } diff --git a/cmd/applyGitOps.go b/cmd/applyGitOps.go index bcf6026..6466fef 100644 --- a/cmd/applyGitOps.go +++ b/cmd/applyGitOps.go @@ -19,15 +19,13 @@ import ( "github.com/spf13/cobra" ) -const GitOpsMatcher = "(.yaml.gitops|.yml.gitops)$" - // applyAppDefs represents the apply command var applyAppDefs = &cobra.Command{ - Use: "gitops-defs", + Use: "gitops-manifests", Short: "Apply custom definition for generated app resources to k8s", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - err := runApplyWithMatch(cmd, args, GitOpsMatcher) + err := runApplyWithMatch(cmd, args, gitOpsMatcher) if err != nil { return err } diff --git a/cmd/applyGroup.go b/cmd/applyGroup.go new file mode 100644 index 0000000..bbbebfd --- /dev/null +++ b/cmd/applyGroup.go @@ -0,0 +1,160 @@ +/* +Copyright © 2020 Artyom Topchyan a.topchyan@reply.de + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package cmd + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + + "github.com/DataReply/korgi/pkg/utils" + "github.com/spf13/cobra" +) + +func templateApp(app string, inputFilePath string, appGroupDir string, lint bool) error { + + targetAppDir := utils.ConcatDirs(appGroupDir, app) + + err := os.MkdirAll(targetAppDir, os.ModePerm) + if err != nil { + return fmt.Errorf("creating app dir: %w", err) + } + if lint { + err = templateEngine.Lint(app, inputFilePath) + if err != nil { + return err + } + } + err = templateEngine.Template(app, inputFilePath, targetAppDir) + if err != nil { + return err + } + + return nil +} + +func getFinalOutputDir(outputDir string, isolated bool) string { + if isolated { + return utils.ConcatDirs(outputDir, execTime.Format("2006-01-02/15-04:05")) + } + return outputDir +} + +func applyAppGroup(group string, namespace string, outputDir string, appFilter string, lint bool, dryRun bool, match string) error { + + log.V(0).Info("applying", "group", group, "namespace", namespace, "app", appFilter, "lint", lint, "dry", dryRun) + namespaceDir := utils.GetNamespaceDir(namespace) + if _, err := os.Stat(namespaceDir); os.IsNotExist(err) { + return fmt.Errorf("%s directory does not exist", namespaceDir) + } + + appGroupDir := utils.ConcatDirs(namespaceDir, group) + + targetAppGroupDir := utils.ConcatDirs(outputDir, namespace, group) + + err := os.MkdirAll(targetAppGroupDir, os.ModePerm) + if err != nil { + return fmt.Errorf("creating group directory: %w", err) + } + + matches, err := filepath.Glob(appGroupDir + "/*") + if err != nil { + return fmt.Errorf("listing group directory: %w", err) + } + + matchers, _ := regexp.Compile(match) + + for _, matchedAppFile := range matches { + appFile := filepath.Base(matchedAppFile) + if appFile != "_app_group.yaml" { + app := utils.SanitizeAppName(appFile) + if appFilter != "" { + + if app != appFilter { + continue + } + } + + if matchers.MatchString(appFile) { + err = templateApp(app, matchedAppFile, targetAppGroupDir, lint) + if err != nil { + return fmt.Errorf("templating app: %w", err) + } + } + + } + + } + if !dryRun { + if appFilter != "" { + err = execEngine.DeployApp(group+"-"+appFilter, utils.ConcatDirs(targetAppGroupDir, appFilter), namespace) + if err != nil { + return fmt.Errorf("running kapp deploy with appFilter: %w", err) + } + return nil + } + + err = execEngine.DeployGroup(group, targetAppGroupDir, namespace) + if err != nil { + return fmt.Errorf("running kapp deploy: %w", err) + } + } + + return nil +} + +func runApplyWithMatch(cmd *cobra.Command, args []string, match string) error { + + namespace, _ := cmd.Flags().GetString("namespace") + + lint, _ := cmd.Flags().GetBool("lint") + + dryRun, _ := cmd.Flags().GetBool("dry-run") + + appFilter, _ := cmd.Flags().GetString("app") + + outputDir, _ := cmd.Flags().GetString("output-dir") + + isolated, _ := cmd.Flags().GetBool("isolate") + + err := applyAppGroup(args[0], namespace, getFinalOutputDir(outputDir, isolated), appFilter, lint, dryRun, match) + if err != nil { + return err + } + return nil +} + +// applyCmd represents the apply command +var applyGroupCmd = &cobra.Command{ + Use: "group", + Short: "App-group scoped apply", + Args: cobra.ExactArgs(1), + TraverseChildren: true, + RunE: func(cmd *cobra.Command, args []string) error { + + err := runApplyWithMatch(cmd, args, defaultMatcher) + if err != nil { + return err + } + return nil + }, +} + +func init() { + + applyCmd.AddCommand(applyGroupCmd) +} diff --git a/cmd/applyNamespace.go b/cmd/applyNamespace.go index 50006bb..65e674c 100644 --- a/cmd/applyNamespace.go +++ b/cmd/applyNamespace.go @@ -24,11 +24,13 @@ import ( "github.com/spf13/cobra" ) -// namespaceCmd represents the namespace command -var namespaceCmd = &cobra.Command{ - Use: "apply-namespace", - Short: "Namespace apply", - Args: cobra.ExactArgs(1), +// applyNamespaceCmd represents the namespace command +var applyNamespaceCmd = &cobra.Command{ + Use: "namespace", + Aliases: []string{"ns"}, + Short: "Namespace scoped apply", + Long: "Iterates over the entire namespace directory, running the apply command on each app-group.", + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { lint, _ := cmd.Flags().GetBool("lint") @@ -51,7 +53,7 @@ var namespaceCmd = &cobra.Command{ if info.IsDir() && path != namespaceDir { group := filepath.Base(path) - err := applyAppGroup(group, namespace, getFinalOutputDir(outputDir, isolated), appFilter, lint, dryRun, DefaultMatcher) + err := applyAppGroup(group, namespace, getFinalOutputDir(outputDir, isolated), appFilter, lint, dryRun, defaultMatcher) if err != nil { return err } @@ -68,9 +70,5 @@ var namespaceCmd = &cobra.Command{ } func init() { - rootCmd.AddCommand(namespaceCmd) - - namespaceCmd.Flags().BoolP("lint", "l", false, "Lint temlate") - namespaceCmd.Flags().BoolP("dry-run", "d", false, "Dry Run") - + applyCmd.AddCommand(applyNamespaceCmd) } diff --git a/cmd/const.go b/cmd/const.go new file mode 100644 index 0000000..4ce1063 --- /dev/null +++ b/cmd/const.go @@ -0,0 +1,4 @@ +package cmd + +const gitOpsMatcher = "(.gitops.y(a)?ml)$" +const defaultMatcher = "(.yaml|.yml)$" diff --git a/cmd/delete.go b/cmd/delete.go index cf50bb2..39a6a4d 100644 --- a/cmd/delete.go +++ b/cmd/delete.go @@ -16,31 +16,13 @@ limitations under the License. package cmd import ( - "fmt" - "github.com/spf13/cobra" ) -func deleteAppGroup(group string, namespace string, appFilter string) error { - if appFilter != "" { - err := execEngine.DeleteApp(group+"-"+appFilter, namespace) - if err != nil { - return fmt.Errorf("kapp app delete: %w", err) - } - return nil - } - - err := execEngine.DeleteGroup(group, namespace) - if err != nil { - return fmt.Errorf("kapp group delete: %w", err) - } - return nil -} - // deleteCmd represents the delete command var deleteCmd = &cobra.Command{ Use: "delete", - Short: "Delete app group or app", + Short: "Delete k8s resources", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { group := args[0] @@ -60,8 +42,4 @@ var deleteCmd = &cobra.Command{ func init() { rootCmd.AddCommand(deleteCmd) - - deleteCmd.Flags().StringP("namespace", "n", "", "Target namespace") - deleteCmd.MarkFlagRequired("namespace") - } diff --git a/cmd/deleteGroup.go b/cmd/deleteGroup.go new file mode 100644 index 0000000..30b7cb9 --- /dev/null +++ b/cmd/deleteGroup.go @@ -0,0 +1,63 @@ +/* +Copyright © 2020 Artyom Topchyan a.topchyan@reply.de + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +func deleteAppGroup(group string, namespace string, appFilter string) error { + if appFilter != "" { + err := execEngine.DeleteApp(group+"-"+appFilter, namespace) + if err != nil { + return fmt.Errorf("kapp app delete: %w", err) + } + return nil + } + + err := execEngine.DeleteGroup(group, namespace) + if err != nil { + return fmt.Errorf("kapp group delete: %w", err) + } + return nil +} + +// deleteCmd represents the delete command +var deleteGroupCmd = &cobra.Command{ + Use: "group", + Short: "Delete app group or app", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + group := args[0] + + namespace, _ := cmd.Flags().GetString("namespace") + + appFilter, _ := cmd.Flags().GetString("app") + + err := deleteAppGroup(group, namespace, appFilter) + if err != nil { + return err + } + + return nil + }, +} + +func init() { + deleteCmd.AddCommand(deleteGroupCmd) +} diff --git a/cmd/deleteNamespace.go b/cmd/deleteNamespace.go index de1c884..4e599d3 100644 --- a/cmd/deleteNamespace.go +++ b/cmd/deleteNamespace.go @@ -26,9 +26,10 @@ import ( // deleteNamespaceCmd represents the namespace command var deleteNamespaceCmd = &cobra.Command{ - Use: "delete-namespace", - Short: "Namespace apply", - Args: cobra.ExactArgs(1), + Use: "namespace", + Aliases: []string{"ns"}, + Short: "Namespace apply", + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { appFilter, _ := cmd.Flags().GetString("app") @@ -60,9 +61,5 @@ var deleteNamespaceCmd = &cobra.Command{ } func init() { - rootCmd.AddCommand(deleteNamespaceCmd) - - deleteNamespaceCmd.Flags().BoolP("lint", "l", false, "Lint temlate") - deleteNamespaceCmd.Flags().BoolP("dry-run", "d", false, "Dry Run") - + deleteCmd.AddCommand(deleteNamespaceCmd) } diff --git a/cmd/root.go b/cmd/root.go index 7d3c952..45b9819 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -96,7 +96,7 @@ func init() { rootCmd.PersistentFlags().StringP("output-dir", "o", "/tmp/kapp", "Working directory") rootCmd.PersistentFlags().BoolP("isolate", "i", true, "By default all output is isolated by appending the time in the following format 2006-01-02/15-04:05") - rootCmd.PersistentFlags().StringP("app", "a", "", "only include this app") + rootCmd.PersistentFlags().StringP("app", "a", "", "Deploy a single app which typically belongs to an app-group.") rootCmd.PersistentFlags().StringP("template-engine", "t", "helmfile", "Template engine") rootCmd.PersistentFlags().StringArray("exec-engine-args", []string{}, "Execution engine extra args(only kapp is supported)s") diff --git a/pkg/utils/stringUtils.go b/pkg/utils/stringUtils.go index a4cfa6f..601d618 100644 --- a/pkg/utils/stringUtils.go +++ b/pkg/utils/stringUtils.go @@ -22,7 +22,7 @@ import ( "strings" ) -func SanitzeAppName(input string) string { +func SanitizeAppName(input string) string { r, _ := regexp.Compile("\\.ya?ml") return r.ReplaceAllString(input, "") }