From c7866c856cb6be38fb04d3f903996bbe29ea702e Mon Sep 17 00:00:00 2001 From: Christophe VILA Date: Mon, 12 Jul 2021 23:08:45 +0200 Subject: [PATCH] Fix https://github.com/ThalesGroup/helm-spray/issues/73 --- CHANGELOG.md | 3 + go.mod | 9 ++- go.sum | 1 + pkg/helm/helm.go | 91 ++++++++++------------- pkg/helmspray/helmspray.go | 145 +++++++++++++++---------------------- pkg/kubectl/kubectl.go | 31 +++++--- plugin.yaml | 2 +- 7 files changed, 128 insertions(+), 154 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8c5bec..3fb8ff4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Release Notes +## Version 4.0.9 - 07/12/2021 +* Fixed [`#73`](https://github.com/ThalesGroup/helm-spray/issues/73) (cvila84) + ## Version 4.0.8 - 06/24/2021 * Exposed helm install/update --create-namespace flag on spray. Since 4.0.6, --create-namespace is automatically passed to helm install/update commands but because it is trying to create namespace even if it already exists, it can generate errors when user rights on cluster do not include namespace creation diff --git a/go.mod b/go.mod index d95bf94..6185117 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,9 @@ module github.com/gemalto/helm-spray/v4 go 1.15 -require github.com/spf13/cobra v1.1.3 - -require helm.sh/helm/v3 v3.6.1 +require ( + github.com/spf13/cobra v1.1.3 + helm.sh/helm/v3 v3.6.1 + k8s.io/api v0.21.0 + k8s.io/client-go v0.21.0 +) diff --git a/go.sum b/go.sum index a41222f..46af698 100644 --- a/go.sum +++ b/go.sum @@ -552,6 +552,7 @@ github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUb github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= diff --git a/pkg/helm/helm.go b/pkg/helm/helm.go index aed92e7..a218053 100644 --- a/pkg/helm/helm.go +++ b/pkg/helm/helm.go @@ -20,18 +20,21 @@ import ( "io/ioutil" "os" "os/exec" - "regexp" "runtime" "strconv" "strings" ) -// Types returned by some of the functions type Status struct { Namespace string Status string } +type UpgradedRelease struct { + Info map[string]interface{} `json:"info"` + Manifest string `json:"manifest"` +} + type Release struct { Name string `json:"name"` Revision string `json:"revision"` @@ -42,64 +45,46 @@ type Release struct { Namespace string `json:"namespace"` } -// Parse the "helm status"-like output to extract relevant information -// WARNING: this code has been developed and tested with version 'v3.2.4' of Helm -// it may need to be adapted to other versions of Helm. -func parseStatusOutput(outs []byte, helmstatus *Status) { - var outStr = string(outs) - - // Extract the namespace - var namespace = regexp.MustCompile(`(?m)^NAMESPACE: (.*)$`) - result := namespace.FindStringSubmatch(outStr) - if len(result) > 1 { - helmstatus.Namespace = result[1] - } - - // Extract the status - var status = regexp.MustCompile(`(?m)^STATUS: (.*)$`) - result = status.FindStringSubmatch(outStr) - if len(result) > 1 { - helmstatus.Status = result[1] - } -} - -// Helm functions calls -// -------------------- - // List ... -func List(namespace string) (map[string]Release, error) { - helmlist := make(map[string]Release, 0) +func List(level int, namespace string, debug bool) (map[string]Release, error) { + // Prepare parameters... + var myargs = []string{"list", "--namespace", namespace, "-o", "json"} - // Get the list of Releases of the chunk - cmd := exec.Command("helm", "list", "--namespace", namespace, "-o", "json") + // Run the list command + if debug { + log.Info(level, "running helm command : %v", myargs) + } + cmd := exec.Command("helm", myargs...) cmdOutput := &bytes.Buffer{} cmd.Stdout = cmdOutput cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { + err := cmd.Run() + output := cmdOutput.Bytes() + if debug { + log.Info(level, "helm command returned:\n%s", string(output)) + } + if err != nil { return nil, err } - // Transform the received json into structs - output := cmdOutput.Bytes() var releases []Release - err := json.Unmarshal(output, &releases) + err = json.Unmarshal(output, &releases) if err != nil { return nil, err } - // Add the Releases into a map + // Return the Releases into a map + releasesMap := make(map[string]Release, 0) for _, r := range releases { - helmlist[r.Name] = r + releasesMap[r.Name] = r } - - return helmlist, nil + return releasesMap, nil } // UpgradeWithValues ... -func UpgradeWithValues(namespace string, createNamespace bool, releaseName string, chartPath string, resetValues bool, reuseValues bool, valueFiles []string, valuesSet []string, valuesSetString []string, valuesSetFile []string, force bool, timeout int, dryRun bool, debug bool) (Status, error) { +func UpgradeWithValues(level int, namespace string, createNamespace bool, releaseName string, chartPath string, resetValues bool, reuseValues bool, valueFiles []string, valuesSet []string, valuesSetString []string, valuesSetFile []string, force bool, timeout int, dryRun bool, debug bool) (UpgradedRelease, error) { // Prepare parameters... - var myargs = []string{"upgrade", "--install", releaseName, chartPath, "--namespace", namespace, "--timeout", strconv.Itoa(timeout) + "s"} - + var myargs = []string{"upgrade", "--install", releaseName, chartPath, "--namespace", namespace, "--timeout", strconv.Itoa(timeout) + "s", "-o", "json"} for _, v := range valuesSet { myargs = append(myargs, "--set") myargs = append(myargs, v) @@ -131,31 +116,31 @@ func UpgradeWithValues(namespace string, createNamespace bool, releaseName strin if createNamespace { myargs = append(myargs, "--create-namespace") } - if debug { - myargs = append(myargs, "--debug") - log.Info(1, "running helm command for \"%s\": %v\n", releaseName, myargs) - } // Run the upgrade command + if debug { + log.Info(level, "running helm command for \"%s\": %v", releaseName, myargs) + } cmd := exec.Command("helm", myargs...) - cmdOutput := &bytes.Buffer{} cmd.Stderr = os.Stderr cmd.Stdout = cmdOutput err := cmd.Run() output := cmdOutput.Bytes() - if debug { - log.Info(1, "helm command for \"%s\" returned: \n%s\n", releaseName, string(output)) + log.Info(level, "helm command for \"%s\" returned:\n%s", releaseName, string(output)) } if err != nil { - return Status{}, err + return UpgradedRelease{}, err + } + + var upgradedRelease UpgradedRelease + err = json.Unmarshal(output, &upgradedRelease) + if err != nil { + return UpgradedRelease{}, err } - // Parse the ending helm status. - status := Status{} - parseStatusOutput(output, &status) - return status, nil + return upgradedRelease, nil } // Fetch ... diff --git a/pkg/helmspray/helmspray.go b/pkg/helmspray/helmspray.go index 481ba7f..cb60f1f 100644 --- a/pkg/helmspray/helmspray.go +++ b/pkg/helmspray/helmspray.go @@ -12,8 +12,12 @@ import ( "helm.sh/helm/v3/pkg/chart/loader" cliValues "helm.sh/helm/v3/pkg/cli/values" "io/ioutil" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + "k8s.io/client-go/kubernetes/scheme" "os" "strconv" + "strings" "text/tabwriter" "time" ) @@ -35,16 +39,16 @@ type Spray struct { DryRun bool Verbose bool Debug bool - deployments map[string]struct{} - statefulsets map[string]struct{} - jobs map[string]struct{} + deployments []string + statefulSets []string + jobs []string } // Spray ... func (s *Spray) Spray() error { if s.Debug { - log.Info(1, "starting spray with flags: %+v\n", s) + log.Info(1, "starting spray with flags: %+v", s) } startTime := time.Now() @@ -101,7 +105,7 @@ func (s *Spray) Spray() error { log.Info(1, "deploying solution chart \"%s\" in namespace \"%s\"", s.ChartName, s.Namespace) } - releases, err := helm.List(s.Namespace) + releases, err := helm.List(1, s.Namespace, s.Debug) if err != nil { return fmt.Errorf("listing releases: %w", err) } @@ -115,33 +119,6 @@ func (s *Spray) Spray() error { return fmt.Errorf("checking targets and excludes: %w", err) } - s.deployments = map[string]struct{}{} - s.statefulsets = map[string]struct{}{} - s.jobs = map[string]struct{}{} - - allDeployments, err := kubectl.GetDeployments(s.Namespace) - if err != nil { - return errors.New("cannot list deployments") - } - allStatefulSets, err := kubectl.GetStatefulSets(s.Namespace) - if err != nil { - return errors.New("cannot list statefulsets") - } - allJobs, err := kubectl.GetJobs(s.Namespace) - if err != nil { - return errors.New("cannot list jobs") - } - - for _, deployment := range allDeployments { - s.deployments[deployment] = struct{}{} - } - for _, statefulSet := range allStatefulSets { - s.statefulsets[statefulSet] = struct{}{} - } - for _, job := range allJobs { - s.jobs[job] = struct{}{} - } - // Loop on the increasing weight for i := 0; i <= maxWeight(deps); i++ { shouldWait, err := s.upgrade(releases, deps, i) @@ -172,6 +149,9 @@ func (s *Spray) upgrade(releases map[string]helm.Release, deps []dependencies.De if firstInWeight { log.Info(1, "processing sub-charts of weight %d", dependency.Weight) firstInWeight = false + s.deployments = make([]string, 0) + s.statefulSets = make([]string, 0) + s.jobs = make([]string, 0) } if release, ok := releases[dependency.CorrespondingReleaseName]; ok { @@ -198,7 +178,7 @@ func (s *Spray) upgrade(releases map[string]helm.Release, deps []dependencies.De valuesSet = append(valuesSet, depValuesSet) // Upgrade the Deployment - helmstatus, err := helm.UpgradeWithValues( + upgradedRelease, err := helm.UpgradeWithValues(3, s.Namespace, s.CreateNamespace, dependency.CorrespondingReleaseName, @@ -221,12 +201,45 @@ func (s *Spray) upgrade(releases map[string]helm.Release, deps []dependencies.De log.Info(3, "release: \"%s\" upgraded", dependency.CorrespondingReleaseName) if s.Verbose { - log.Info(3, "helm status: %s", helmstatus.Status) + log.Info(3, "helm status: %s", upgradedRelease.Info["status"]) } - - if !s.DryRun && helmstatus.Status != "deployed" { + if !s.DryRun && upgradedRelease.Info["status"] != "deployed" { return false, errors.New("status returned by helm differs from \"deployed\", spray interrupted") } + + for _, yaml := range strings.Split(upgradedRelease.Manifest, "---") { + manifest, _, err := scheme.Codecs.UniversalDeserializer().Decode([]byte(yaml), nil, nil) + if err != nil && len(yaml) > 0 { + log.Info(3, "warning: ignored part of helm upgrade output") + if s.Debug { + log.Info(3, "warning: ignored '%s'", yaml) + } + } + deployment, ok := manifest.(*appsv1.Deployment) + if ok { + s.deployments = append(s.deployments, deployment.Name) + } + statefulSet, ok := manifest.(*appsv1.StatefulSet) + if ok { + s.statefulSets = append(s.statefulSets, statefulSet.Name) + } + job, ok := manifest.(*batchv1.Job) + if ok { + s.jobs = append(s.jobs, job.Name) + } + } + + if s.Verbose { + if len(s.deployments) > 0 { + log.Info(3, "release deployments: %v", s.deployments) + } + if len(s.statefulSets) > 0 { + log.Info(3, "release statefulsets: %v", s.statefulSets) + } + if len(s.jobs) > 0 { + log.Info(3, "release jobs: %v", s.jobs) + } + } } } } @@ -236,38 +249,6 @@ func (s *Spray) upgrade(releases map[string]helm.Release, deps []dependencies.De func (s *Spray) wait() error { log.Info(2, "waiting for liveness and readiness...") - allDeployments, err := kubectl.GetDeployments(s.Namespace) - if err != nil { - return errors.New("cannot list deployments") - } - allStatefulSets, err := kubectl.GetStatefulSets(s.Namespace) - if err != nil { - return errors.New("cannot list statefulsets") - } - allJobs, err := kubectl.GetJobs(s.Namespace) - if err != nil { - return errors.New("cannot list jobs") - } - - deployments := make([]string, 0) - for _, deployment := range allDeployments { - if _, ok := s.deployments[deployment]; !ok { - deployments = append(deployments, deployment) - } - } - statefulSets := make([]string, 0) - for _, statefulset := range allStatefulSets { - if _, ok := s.statefulsets[statefulset]; !ok { - statefulSets = append(statefulSets, statefulset) - } - } - jobs := make([]string, 0) - for _, job := range allJobs { - if _, ok := s.jobs[job]; !ok { - jobs = append(jobs, job) - } - } - sleepTime := 5 doneDeployments := false doneStatefulSets := false @@ -275,27 +256,27 @@ func (s *Spray) wait() error { // Wait for completion of the Deployments/StatefulSets/Jobs for i := 0; i < s.Timeout; { - if len(deployments) > 0 && !doneDeployments { + if len(s.deployments) > 0 && !doneDeployments { if s.Verbose { - log.Info(3, "waiting for Deployments %v", deployments) + log.Info(3, "waiting for deployments %v", s.deployments) } - doneDeployments, _ = kubectl.AreDeploymentsReady(deployments, s.Namespace, s.Debug) + doneDeployments, _ = kubectl.AreDeploymentsReady(s.deployments, s.Namespace, s.Debug) } else { doneDeployments = true } - if len(statefulSets) > 0 && !doneStatefulSets { + if len(s.statefulSets) > 0 && !doneStatefulSets { if s.Verbose { - log.Info(3, "waiting for StatefulSets %v", statefulSets) + log.Info(3, "waiting for statefulsets %v", s.statefulSets) } - doneStatefulSets, _ = kubectl.AreStatefulSetsReady(statefulSets, s.Namespace, s.Debug) + doneStatefulSets, _ = kubectl.AreStatefulSetsReady(s.statefulSets, s.Namespace, s.Debug) } else { doneStatefulSets = true } - if len(jobs) > 0 && !doneJobs { + if len(s.jobs) > 0 && !doneJobs { if s.Verbose { - log.Info(3, "waiting for Jobs %v", jobs) + log.Info(3, "waiting for jobs %v", s.jobs) } - doneJobs, _ = kubectl.AreJobsReady(jobs, s.Namespace, s.Debug) + doneJobs, _ = kubectl.AreJobsReady(s.jobs, s.Namespace, s.Debug) } else { doneJobs = true } @@ -310,16 +291,6 @@ func (s *Spray) wait() error { return errors.New("timed out waiting for liveness and readiness") } - for _, deployment := range deployments { - s.deployments[deployment] = struct{}{} - } - for _, statefulSet := range statefulSets { - s.statefulsets[statefulSet] = struct{}{} - } - for _, job := range jobs { - s.jobs[job] = struct{}{} - } - return nil } diff --git a/pkg/kubectl/kubectl.go b/pkg/kubectl/kubectl.go index 26af64a..a8a629d 100644 --- a/pkg/kubectl/kubectl.go +++ b/pkg/kubectl/kubectl.go @@ -1,3 +1,5 @@ +package kubectl + /* (c) Copyright 2018, Gemalto. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); @@ -10,10 +12,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -package kubectl import ( - "fmt" + "github.com/gemalto/helm-spray/v4/internal/log" "os" "os/exec" "strings" @@ -45,9 +46,9 @@ func AreJobsReady(names []string, namespace string, debug bool) (bool, error) { } template := generateTemplate(names, "{{if .status.succeeded}}{{if lt .status.succeeded 1}}{{printf \"%s \" .metadata.name}}{{end}}{{end}}") if debug { - fmt.Printf("kubectl template: %s\n", template) + log.Info(3, "kubectl template: %s", template) } - cmd := exec.Command("kubectl", "--namespace", namespace, "get", "jobs", "-o", "go-template="+template+"") + cmd := exec.Command("kubectl", "--namespace", namespace, "get", "jobs", "-o", "go-template="+template) cmd.Stderr = os.Stderr result, err := cmd.Output() if err != nil { @@ -56,7 +57,7 @@ func AreJobsReady(names []string, namespace string, debug bool) (bool, error) { } strResult := string(result) if debug { - fmt.Printf("kubectl output: %s\n", strResult) + log.Info(3, "kubectl output: %s", strResult) } if len(strResult) > 0 { return false, nil @@ -79,20 +80,30 @@ func areWorkloadsReady(k8sObjectType string, names []string, namespace string, d if len(names) == 0 { return true, nil } - template := generateTemplate(names, "{{if .status.readyReplicas}}{{if lt .status.readyReplicas .spec.replicas}}{{printf \"%s \" .metadata.name}}{{end}}{{else}}{{printf \"%s \" .metadata.name}}{{end}}") if debug { - fmt.Printf("kubectl template: %s\n", template) + template := generateTemplate(names, "{{$ready := 0}}{{if .status.readyReplicas}}{{$ready = .status.readyReplicas}}{{end}}{{$current := .spec.replicas}}{{if .status.currentReplicas}}{{$current = .status.currentReplicas}}{{end}}{{$updated := 0}}{{if .status.updatedReplicas}}{{$updated = .status.updatedReplicas}}{{end}}{{printf \"{name: %s, ready: %d, current: %d, updated: %d}\" .metadata.name $ready $current $updated}}") + log.Info(3, "kubectl template: %s", template) + cmd := exec.Command("kubectl", "--namespace", namespace, "get", k8sObjectType, "-o", "go-template="+template) + result, _ := cmd.Output() + log.Info(3, "kubectl output: %s", string(result)) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + _ = cmd.Run() } - cmd := exec.Command("kubectl", "--namespace", namespace, "get", k8sObjectType, "-o", "go-template="+template+"") + template := generateTemplate(names, "{{$ready := 0}}{{if .status.readyReplicas}}{{$ready = .status.readyReplicas}}{{end}}{{$current := .spec.replicas}}{{if .status.currentReplicas}}{{$current = .status.currentReplicas}}{{end}}{{$updated := 0}}{{if .status.updatedReplicas}}{{$updated = .status.updatedReplicas}}{{end}}{{if or (lt $ready .spec.replicas) (lt $current .spec.replicas) (lt $updated .spec.replicas)}}{{printf \"%s \" .metadata.name}}{{end}}") + if debug { + log.Info(3, "kubectl template: %s", template) + } + cmd := exec.Command("kubectl", "--namespace", namespace, "get", k8sObjectType, "-o", "go-template="+template) cmd.Stderr = os.Stderr result, err := cmd.Output() if err != nil { - // Cannot make the difference between an error when calling kubectl and no corresponding resource found. Return "" in any case. + // Cannot make the difference between an error when calling kubectl and no corresponding resource found. Return false in any case. return false, err } strResult := string(result) if debug { - fmt.Printf("kubectl output: %s\n", strResult) + log.Info(3, "kubectl output: %s", strResult) } if len(strResult) > 0 { return false, nil diff --git a/plugin.yaml b/plugin.yaml index 06ad49c..30d2807 100644 --- a/plugin.yaml +++ b/plugin.yaml @@ -1,5 +1,5 @@ name: "spray" -version: 4.0.8 +version: 4.0.9 usage: "upgrade sub-charts from an umbrella chart with dependency orders" description: "Helm plugin for upgrading sub-charts from umbrella chart with dependency orders" command: "$HELM_PLUGIN_DIR/bin/helm-spray"