feat(manifests/helmfile): add helmfile templating engine

Dockerfile.slim

@@ -4,6 +4,7 @@ LABEL maintainer="sig-platform@spinnaker.io"
 ENV KUSTOMIZE_VERSION=3.8.6
 ENV KUSTOMIZE4_VERSION=4.5.5
 ENV PACKER_VERSION=1.8.1
+ENV HELMFILE_VERSION=0.153.1
 
 
 ARG TARGETARCH
@@ -42,6 +43,12 @@ RUN mkdir kustomize && \
  mv ./kustomize/kustomize /usr/local/bin/kustomize4 && \
  rm -rf ./kustomize
 
+RUN mkdir helmfile && \
+ curl -s -L https://github.com/helmfile/helmfile/releases/download/v${HELMFILE_VERSION}/helmfile_${HELMFILE_VERSION}_linux_${TARGETARCH}.tar.gz|\
+ tar xvz -C helmfile/ && \
+ mv ./helmfile/helmfile /usr/local/bin/helmfile && \
+ rm -rf ./helmfile
+
 RUN addgroup -S -g 10111 spinnaker
 RUN adduser -S -G spinnaker -u 10111 spinnaker
 COPY rosco-web/build/install/rosco /opt/rosco

Dockerfile.ubuntu

@@ -4,6 +4,7 @@ LABEL maintainer="sig-platform@spinnaker.io"
 ENV KUSTOMIZE_VERSION=3.8.6
 ENV KUSTOMIZE4_VERSION=4.5.5
 ENV PACKER_VERSION=1.8.1
+ENV HELMFILE_VERSION=0.153.1
 
 ARG TARGETARCH
 
@@ -41,6 +42,12 @@ RUN mkdir kustomize && \
  mv ./kustomize/kustomize /usr/local/bin/kustomize4 && \
  rm -rf ./kustomize
 
+RUN mkdir helmfile && \
+ curl -s -L https://github.com/helmfile/helmfile/releases/download/v${HELMFILE_VERSION}/helmfile_${HELMFILE_VERSION}_linux_${TARGETARCH}.tar.gz|\
+ tar xvz -C helmfile/ && \
+ mv ./helmfile/helmfile /usr/local/bin/helmfile && \
+ rm -rf ./helmfile
+
 RUN adduser --system --uid 10111 --group spinnaker
 COPY rosco-web/build/install/rosco /opt/rosco
 COPY rosco-web/config /opt/rosco

rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/BakeManifestRequest.java

@@ -17,6 +17,7 @@ public enum TemplateRenderer {
  HELM3,
  KUSTOMIZE,
  KUSTOMIZE4,
+ HELMFILE,
  CF;
 
  @JsonCreator

rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/HelmBakeTemplateUtils.java

@@ -0,0 +1,106 @@
+/*
+ * Copyright 2023 Grab Holdings, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.rosco.manifests;
+
+import com.netflix.spinnaker.kork.artifacts.model.Artifact;
+import com.netflix.spinnaker.kork.exceptions.SpinnakerException;
+import com.netflix.spinnaker.kork.retrofit.exceptions.SpinnakerHttpException;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+public abstract class HelmBakeTemplateUtils<T extends BakeManifestRequest> {
+ private static final String MANIFEST_SEPARATOR = "---\n";
+ private static final Pattern REGEX_TESTS_MANIFESTS =
+ Pattern.compile("# Source: .*/templates/tests/.*");
+
+ private final ArtifactDownloader artifactDownloader;
+
+ protected HelmBakeTemplateUtils(ArtifactDownloader artifactDownloader) {
+ this.artifactDownloader = artifactDownloader;
+ }
+
+ public ArtifactDownloader getArtifactDownloader() {
+ return artifactDownloader;
+ }
+
+ public abstract String fetchFailureMessage(String description, Exception e);
+
+ public String removeTestsDirectoryTemplates(String inputString) {
+ return Arrays.stream(inputString.split(MANIFEST_SEPARATOR))
+ .filter(manifest -> !REGEX_TESTS_MANIFESTS.matcher(manifest).find())
+ .collect(Collectors.joining(MANIFEST_SEPARATOR));
+ }
+
+ protected Path downloadArtifactToTmpFile(BakeManifestEnvironment env, Artifact artifact)
+ throws IOException {
+ String fileName = UUID.randomUUID().toString();
+ Path targetPath = env.resolvePath(fileName);
+ artifactDownloader.downloadArtifactToFile(artifact, targetPath);
+ return targetPath;
+ }
+
+ public abstract String getHelmExecutableForRequest(T request);
+
+ protected List<Path> getValuePaths(List<Artifact> artifacts, BakeManifestEnvironment env) {
+ List<Path> valuePaths = new ArrayList<>();
+
+ try {
+ // not a stream to keep exception handling cleaner
+ for (Artifact valueArtifact : artifacts.subList(1, artifacts.size())) {
+ valuePaths.add(downloadArtifactToTmpFile(env, valueArtifact));
+ }
+ } catch (SpinnakerHttpException e) {
+ throw new SpinnakerHttpException(fetchFailureMessage("values file", e), e);
+ } catch (IOException | SpinnakerException e) {
+ throw new IllegalStateException(fetchFailureMessage("values file", e), e);
+ }
+
+ return valuePaths;
+ }
+
+ protected Path getHelmTypePathFromArtifact(
+ BakeManifestEnvironment env, List<Artifact> inputArtifacts, String filePath)
+ throws IOException {
+ Path helmTypeFilePath;
+
+ Artifact helmTypeTemplateArtifact = inputArtifacts.get(0);
+ String artifactType = Optional.ofNullable(helmTypeTemplateArtifact.getType()).orElse("");
+
+ if ("git/repo".equals(artifactType)) {
+ env.downloadArtifactTarballAndExtract(getArtifactDownloader(), helmTypeTemplateArtifact);
+
+ helmTypeFilePath = env.resolvePath(Optional.ofNullable(filePath).orElse(""));
+ } else {
+ try {
+ helmTypeFilePath = downloadArtifactToTmpFile(env, helmTypeTemplateArtifact);
+ } catch (SpinnakerHttpException e) {
+ throw new SpinnakerHttpException(fetchFailureMessage("template", e), e);
+ } catch (IOException | SpinnakerException e) {
+ throw new IllegalStateException(fetchFailureMessage("template", e), e);
+ }
+ }
+
+ return helmTypeFilePath;
+ }
+}

rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/config/RoscoHelmfileConfigurationProperties.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2023 Grab Holdings, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.rosco.manifests.config;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+
+@ConfigurationProperties("helmfile")
+@Data
+public class RoscoHelmfileConfigurationProperties {
+ private String executablePath = "helmfile";
+}

rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helm/HelmTemplateUtils.java

@@ -1,88 +1,63 @@
 package com.netflix.spinnaker.rosco.manifests.helm;
 
 import com.netflix.spinnaker.kork.artifacts.model.Artifact;
-import com.netflix.spinnaker.kork.exceptions.SpinnakerException;
-import com.netflix.spinnaker.kork.retrofit.exceptions.SpinnakerHttpException;
 import com.netflix.spinnaker.rosco.jobs.BakeRecipe;
 import com.netflix.spinnaker.rosco.manifests.ArtifactDownloader;
 import com.netflix.spinnaker.rosco.manifests.BakeManifestEnvironment;
 import com.netflix.spinnaker.rosco.manifests.BakeManifestRequest;
+import com.netflix.spinnaker.rosco.manifests.HelmBakeTemplateUtils;
 import com.netflix.spinnaker.rosco.manifests.config.RoscoHelmConfigurationProperties;
 import java.io.IOException;
 import java.nio.file.Path;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
-import java.util.UUID;
-import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Component;
 
 @Component
 @Slf4j
-public class HelmTemplateUtils {
- private static final String MANIFEST_SEPARATOR = "---\n";
- private static final Pattern REGEX_TESTS_MANIFESTS =
- Pattern.compile("# Source: .*/templates/tests/.*");
-
- private final ArtifactDownloader artifactDownloader;
+public class HelmTemplateUtils extends HelmBakeTemplateUtils<HelmBakeManifestRequest> {
  private final RoscoHelmConfigurationProperties helmConfigurationProperties;
 
  public HelmTemplateUtils(
  ArtifactDownloader artifactDownloader,
  RoscoHelmConfigurationProperties helmConfigurationProperties) {
- this.artifactDownloader = artifactDownloader;
+ super(artifactDownloader);
  this.helmConfigurationProperties = helmConfigurationProperties;
  }
 
  public BakeRecipe buildBakeRecipe(BakeManifestEnvironment env, HelmBakeManifestRequest request)
  throws IOException {
- BakeRecipe result = new BakeRecipe();
- result.setName(request.getOutputName());
-
  Path templatePath;
- List<Path> valuePaths = new ArrayList<>();
+
  List<Artifact> inputArtifacts = request.getInputArtifacts();
  if (inputArtifacts == null || inputArtifacts.isEmpty()) {
  throw new IllegalArgumentException("At least one input artifact must be provided to bake");
  }
 
- Artifact helmTemplateArtifact = inputArtifacts.get(0);
- String artifactType = Optional.ofNullable(helmTemplateArtifact.getType()).orElse("");
- if ("git/repo".equals(artifactType)) {
- env.downloadArtifactTarballAndExtract(artifactDownloader, helmTemplateArtifact);
-
- log.info("helmChartFilePath: '{}'", request.getHelmChartFilePath());
-
- // If there's no helm chart path specified, assume it lives in the root of
- // the git/repo artifact.
- templatePath =
- env.resolvePath(Optional.ofNullable(request.getHelmChartFilePath()).orElse(""));
- } else {
- try {
- templatePath = downloadArtifactToTmpFile(env, helmTemplateArtifact);
- } catch (SpinnakerHttpException e) {
- throw new SpinnakerHttpException(fetchFailureMessage("template", e), e);
- } catch (IOException | SpinnakerException e) {
- throw new IllegalStateException(fetchFailureMessage("template", e), e);
- }
- }
+ templatePath = getHelmTypePathFromArtifact(env, inputArtifacts, request.getHelmChartFilePath());
 
  log.info("path to Chart.yaml: {}", templatePath);
+ return buildCommand(request, getValuePaths(inputArtifacts, env), templatePath);
+ }
 
- try {
- // not a stream to keep exception handling cleaner
- for (Artifact valueArtifact : inputArtifacts.subList(1, inputArtifacts.size())) {
- valuePaths.add(downloadArtifactToTmpFile(env, valueArtifact));
- }
- } catch (SpinnakerHttpException e) {
- throw new SpinnakerHttpException(fetchFailureMessage("values file", e), e);
- } catch (IOException | SpinnakerException e) {
- throw new IllegalStateException(fetchFailureMessage("values file", e), e);
+ public String fetchFailureMessage(String description, Exception e) {
+ return "Failed to fetch helm " + description + ": " + e.getMessage();
+ }
+
+ public String getHelmExecutableForRequest(HelmBakeManifestRequest request) {
+ if (BakeManifestRequest.TemplateRenderer.HELM2.equals(request.getTemplateRenderer())) {
+ return helmConfigurationProperties.getV2ExecutablePath();
  }
+ return helmConfigurationProperties.getV3ExecutablePath();
+ }
+
+ public BakeRecipe buildCommand(
+ HelmBakeManifestRequest request, List<Path> valuePaths, Path templatePath) {
+ BakeRecipe result = new BakeRecipe();
+ result.setName(request.getOutputName());
 
  List<String> command = new ArrayList<>();
  String executable = getHelmExecutableForRequest(request);
@@ -133,29 +108,4 @@ public BakeRecipe buildBakeRecipe(BakeManifestEnvironment env, HelmBakeManifestR
 
  return result;
  }
-
- private String fetchFailureMessage(String description, Exception e) {
- return "Failed to fetch helm " + description + ": " + e.getMessage();
- }
-
- public String removeTestsDirectoryTemplates(String inputString) {
- return Arrays.stream(inputString.split(MANIFEST_SEPARATOR))
- .filter(manifest -> !REGEX_TESTS_MANIFESTS.matcher(manifest).find())
- .collect(Collectors.joining(MANIFEST_SEPARATOR));
- }
-
- private Path downloadArtifactToTmpFile(BakeManifestEnvironment env, Artifact artifact)
- throws IOException {
- String fileName = UUID.randomUUID().toString();
- Path targetPath = env.resolvePath(fileName);
- artifactDownloader.downloadArtifactToFile(artifact, targetPath);
- return targetPath;
- }
-
- private String getHelmExecutableForRequest(HelmBakeManifestRequest request) {
- if (BakeManifestRequest.TemplateRenderer.HELM2.equals(request.getTemplateRenderer())) {
- return helmConfigurationProperties.getV2ExecutablePath();
- }
- return helmConfigurationProperties.getV3ExecutablePath();
- }
 }

rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileBakeManifestRequest.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2023 Grab Holdings, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.rosco.manifests.helmfile;
+
+import com.netflix.spinnaker.kork.artifacts.model.Artifact;
+import com.netflix.spinnaker.rosco.manifests.BakeManifestRequest;
+import java.util.List;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+
+@Data
+@EqualsAndHashCode(callSuper = true)
+public class HelmfileBakeManifestRequest extends BakeManifestRequest {
+ private String helmfileFilePath;
+
+ /**
+ * The environment name used to customize the content of the helmfile manifest. The environment
+ * name defaults to default.
+ */
+ private String environment;
+
+ /** The namespace to be released into. */
+ private String namespace;
+
+ /**
+ * The 0th element is (or contains) the helmfile template. The rest (possibly none) are values
+ * files.
+ */
+ List<Artifact> inputArtifacts;
+
+ /**
+ * Include custom resource definition manifests in the templated output. Helmfile uses Helm v3
+ * only which provides the option to include CRDs as part of the rendered output.
+ */
+ boolean includeCRDs;
+}