feat(manifests/helmfile): add helmfile templating engine

Dockerfile.slim

@@ -4,6 +4,7 @@ LABEL maintainer="sig-platform@spinnaker.io"
 ENV KUSTOMIZE_VERSION=3.8.6
 ENV KUSTOMIZE4_VERSION=4.5.5
 ENV PACKER_VERSION=1.8.1
+ENV HELMFILE_VERSION=0.153.1
 
 
 ARG TARGETARCH
@@ -42,6 +43,12 @@ RUN mkdir kustomize && \
   mv ./kustomize/kustomize /usr/local/bin/kustomize4 && \
   rm -rf ./kustomize
 
+RUN mkdir helmfile && \
+  curl -s -L https://github.com/helmfile/helmfile/releases/download/v${HELMFILE_VERSION}/helmfile_${HELMFILE_VERSION}_linux_${TARGETARCH}.tar.gz|\
+  tar xvz -C helmfile/ && \
+  mv ./helmfile/helmfile /usr/local/bin/helmfile && \
+  rm -rf ./helmfile
+
 RUN addgroup -S -g 10111 spinnaker
 RUN adduser -S -G spinnaker -u 10111 spinnaker
 COPY rosco-web/build/install/rosco /opt/rosco

Dockerfile.ubuntu

@@ -4,6 +4,7 @@ LABEL maintainer="sig-platform@spinnaker.io"
 ENV KUSTOMIZE_VERSION=3.8.6
 ENV KUSTOMIZE4_VERSION=4.5.5
 ENV PACKER_VERSION=1.8.1
+ENV HELMFILE_VERSION=0.153.1
 
 ARG TARGETARCH
 
@@ -41,6 +42,12 @@ RUN mkdir kustomize && \
   mv ./kustomize/kustomize /usr/local/bin/kustomize4 && \
   rm -rf ./kustomize
 
+RUN mkdir helmfile && \
+  curl -s -L https://github.com/helmfile/helmfile/releases/download/v${HELMFILE_VERSION}/helmfile_${HELMFILE_VERSION}_linux_${TARGETARCH}.tar.gz|\
+  tar xvz -C helmfile/ && \
+  mv ./helmfile/helmfile /usr/local/bin/helmfile && \
+  rm -rf ./helmfile
+
 RUN adduser --system --uid 10111 --group spinnaker
 COPY rosco-web/build/install/rosco /opt/rosco
 COPY rosco-web/config              /opt/rosco

rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/BakeManifestRequest.java

@@ -17,6 +17,7 @@ public enum TemplateRenderer {
     HELM3,
     KUSTOMIZE,
     KUSTOMIZE4,
+    HELMFILE,
     CF;
 
     @JsonCreator

rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/HelmBakeTemplateUtils.java

@@ -0,0 +1,106 @@
+/*
+ * Copyright 2023 Grab Holdings, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.rosco.manifests;
+
+import com.netflix.spinnaker.kork.artifacts.model.Artifact;
+import com.netflix.spinnaker.kork.exceptions.SpinnakerException;
+import com.netflix.spinnaker.kork.retrofit.exceptions.SpinnakerHttpException;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+public abstract class HelmBakeTemplateUtils<T extends BakeManifestRequest> {
+  private static final String MANIFEST_SEPARATOR = "---\n";
+  private static final Pattern REGEX_TESTS_MANIFESTS =
+      Pattern.compile("# Source: .*/templates/tests/.*");
+
+  private final ArtifactDownloader artifactDownloader;
+
+  protected HelmBakeTemplateUtils(ArtifactDownloader artifactDownloader) {
+    this.artifactDownloader = artifactDownloader;
+  }
+
+  public ArtifactDownloader getArtifactDownloader() {
+    return artifactDownloader;
+  }
+
+  public abstract String fetchFailureMessage(String description, Exception e);
+
+  public String removeTestsDirectoryTemplates(String inputString) {
+    return Arrays.stream(inputString.split(MANIFEST_SEPARATOR))
+        .filter(manifest -> !REGEX_TESTS_MANIFESTS.matcher(manifest).find())
+        .collect(Collectors.joining(MANIFEST_SEPARATOR));
+  }
+
+  protected Path downloadArtifactToTmpFile(BakeManifestEnvironment env, Artifact artifact)
+      throws IOException {
+    String fileName = UUID.randomUUID().toString();
+    Path targetPath = env.resolvePath(fileName);
+    artifactDownloader.downloadArtifactToFile(artifact, targetPath);
+    return targetPath;
+  }
+
+  public abstract String getHelmExecutableForRequest(T request);
+
+  protected List<Path> getValuePaths(List<Artifact> artifacts, BakeManifestEnvironment env) {
+    List<Path> valuePaths = new ArrayList<>();
+
+    try {
+      // not a stream to keep exception handling cleaner
+      for (Artifact valueArtifact : artifacts.subList(1, artifacts.size())) {
+        valuePaths.add(downloadArtifactToTmpFile(env, valueArtifact));
+      }
+    } catch (SpinnakerHttpException e) {
+      throw new SpinnakerHttpException(fetchFailureMessage("values file", e), e);
+    } catch (IOException | SpinnakerException e) {
+      throw new IllegalStateException(fetchFailureMessage("values file", e), e);
+    }
+
+    return valuePaths;
+  }
+
+  protected Path getHelmTypePathFromArtifact(
+      BakeManifestEnvironment env, List<Artifact> inputArtifacts, String filePath)
+      throws IOException {
+    Path helmTypeFilePath;
+
+    Artifact helmTypeTemplateArtifact = inputArtifacts.get(0);
+    String artifactType = Optional.ofNullable(helmTypeTemplateArtifact.getType()).orElse("");
+
+    if ("git/repo".equals(artifactType)) {
+      env.downloadArtifactTarballAndExtract(getArtifactDownloader(), helmTypeTemplateArtifact);
+
+      helmTypeFilePath = env.resolvePath(Optional.ofNullable(filePath).orElse(""));
+    } else {
+      try {
+        helmTypeFilePath = downloadArtifactToTmpFile(env, helmTypeTemplateArtifact);
+      } catch (SpinnakerHttpException e) {
+        throw new SpinnakerHttpException(fetchFailureMessage("template", e), e);
+      } catch (IOException | SpinnakerException e) {
+        throw new IllegalStateException(fetchFailureMessage("template", e), e);
+      }
+    }
+
+    return helmTypeFilePath;
+  }
+}

rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/config/RoscoHelmfileConfigurationProperties.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2023 Grab Holdings, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.rosco.manifests.config;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+
+@ConfigurationProperties("helmfile")
+@Data
+public class RoscoHelmfileConfigurationProperties {
+  private String executablePath = "helmfile";
+}

rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helm/HelmTemplateUtils.java

@@ -1,88 +1,63 @@
 package com.netflix.spinnaker.rosco.manifests.helm;
 
 import com.netflix.spinnaker.kork.artifacts.model.Artifact;
-import com.netflix.spinnaker.kork.exceptions.SpinnakerException;
-import com.netflix.spinnaker.kork.retrofit.exceptions.SpinnakerHttpException;
 import com.netflix.spinnaker.rosco.jobs.BakeRecipe;
 import com.netflix.spinnaker.rosco.manifests.ArtifactDownloader;
 import com.netflix.spinnaker.rosco.manifests.BakeManifestEnvironment;
 import com.netflix.spinnaker.rosco.manifests.BakeManifestRequest;
+import com.netflix.spinnaker.rosco.manifests.HelmBakeTemplateUtils;
 import com.netflix.spinnaker.rosco.manifests.config.RoscoHelmConfigurationProperties;
 import java.io.IOException;
 import java.nio.file.Path;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
-import java.util.UUID;
-import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Component;
 
 @Component
 @Slf4j
-public class HelmTemplateUtils {
-  private static final String MANIFEST_SEPARATOR = "---\n";
-  private static final Pattern REGEX_TESTS_MANIFESTS =
-      Pattern.compile("# Source: .*/templates/tests/.*");
-
-  private final ArtifactDownloader artifactDownloader;
+public class HelmTemplateUtils extends HelmBakeTemplateUtils<HelmBakeManifestRequest> {
   private final RoscoHelmConfigurationProperties helmConfigurationProperties;
 
   public HelmTemplateUtils(
       ArtifactDownloader artifactDownloader,
       RoscoHelmConfigurationProperties helmConfigurationProperties) {
-    this.artifactDownloader = artifactDownloader;
+    super(artifactDownloader);
     this.helmConfigurationProperties = helmConfigurationProperties;
   }
 
   public BakeRecipe buildBakeRecipe(BakeManifestEnvironment env, HelmBakeManifestRequest request)
       throws IOException {
-    BakeRecipe result = new BakeRecipe();
-    result.setName(request.getOutputName());
-
     Path templatePath;
-    List<Path> valuePaths = new ArrayList<>();
+
     List<Artifact> inputArtifacts = request.getInputArtifacts();
     if (inputArtifacts == null || inputArtifacts.isEmpty()) {
       throw new IllegalArgumentException("At least one input artifact must be provided to bake");
     }
 
-    Artifact helmTemplateArtifact = inputArtifacts.get(0);
-    String artifactType = Optional.ofNullable(helmTemplateArtifact.getType()).orElse("");
-    if ("git/repo".equals(artifactType)) {
-      env.downloadArtifactTarballAndExtract(artifactDownloader, helmTemplateArtifact);
-
-      log.info("helmChartFilePath: '{}'", request.getHelmChartFilePath());
-
-      // If there's no helm chart path specified, assume it lives in the root of
-      // the git/repo artifact.
-      templatePath =
-          env.resolvePath(Optional.ofNullable(request.getHelmChartFilePath()).orElse(""));
-    } else {
-      try {
-        templatePath = downloadArtifactToTmpFile(env, helmTemplateArtifact);
-      } catch (SpinnakerHttpException e) {
-        throw new SpinnakerHttpException(fetchFailureMessage("template", e), e);
-      } catch (IOException | SpinnakerException e) {
-        throw new IllegalStateException(fetchFailureMessage("template", e), e);
-      }
-    }
+    templatePath = getHelmTypePathFromArtifact(env, inputArtifacts, request.getHelmChartFilePath());
 
     log.info("path to Chart.yaml: {}", templatePath);
+    return buildCommand(request, getValuePaths(inputArtifacts, env), templatePath);
+  }
 
-    try {
-      // not a stream to keep exception handling cleaner
-      for (Artifact valueArtifact : inputArtifacts.subList(1, inputArtifacts.size())) {
-        valuePaths.add(downloadArtifactToTmpFile(env, valueArtifact));
-      }
-    } catch (SpinnakerHttpException e) {
-      throw new SpinnakerHttpException(fetchFailureMessage("values file", e), e);
-    } catch (IOException | SpinnakerException e) {
-      throw new IllegalStateException(fetchFailureMessage("values file", e), e);
+  public String fetchFailureMessage(String description, Exception e) {
+    return "Failed to fetch helm " + description + ": " + e.getMessage();
+  }
+
+  public String getHelmExecutableForRequest(HelmBakeManifestRequest request) {
+    if (BakeManifestRequest.TemplateRenderer.HELM2.equals(request.getTemplateRenderer())) {
+      return helmConfigurationProperties.getV2ExecutablePath();
     }
+    return helmConfigurationProperties.getV3ExecutablePath();
+  }
+
+  public BakeRecipe buildCommand(
+      HelmBakeManifestRequest request, List<Path> valuePaths, Path templatePath) {
+    BakeRecipe result = new BakeRecipe();
+    result.setName(request.getOutputName());
 
     List<String> command = new ArrayList<>();
     String executable = getHelmExecutableForRequest(request);
@@ -133,29 +108,4 @@ public BakeRecipe buildBakeRecipe(BakeManifestEnvironment env, HelmBakeManifestR
 
     return result;
   }
-
-  private String fetchFailureMessage(String description, Exception e) {
-    return "Failed to fetch helm " + description + ": " + e.getMessage();
-  }
-
-  public String removeTestsDirectoryTemplates(String inputString) {
-    return Arrays.stream(inputString.split(MANIFEST_SEPARATOR))
-        .filter(manifest -> !REGEX_TESTS_MANIFESTS.matcher(manifest).find())
-        .collect(Collectors.joining(MANIFEST_SEPARATOR));
-  }
-
-  private Path downloadArtifactToTmpFile(BakeManifestEnvironment env, Artifact artifact)
-      throws IOException {
-    String fileName = UUID.randomUUID().toString();
-    Path targetPath = env.resolvePath(fileName);
-    artifactDownloader.downloadArtifactToFile(artifact, targetPath);
-    return targetPath;
-  }
-
-  private String getHelmExecutableForRequest(HelmBakeManifestRequest request) {
-    if (BakeManifestRequest.TemplateRenderer.HELM2.equals(request.getTemplateRenderer())) {
-      return helmConfigurationProperties.getV2ExecutablePath();
-    }
-    return helmConfigurationProperties.getV3ExecutablePath();
-  }
 }

rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileBakeManifestRequest.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2023 Grab Holdings, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.rosco.manifests.helmfile;
+
+import com.netflix.spinnaker.kork.artifacts.model.Artifact;
+import com.netflix.spinnaker.rosco.manifests.BakeManifestRequest;
+import java.util.List;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+
+@Data
+@EqualsAndHashCode(callSuper = true)
+public class HelmfileBakeManifestRequest extends BakeManifestRequest {
+  private String helmfileFilePath;
+
+  /**
+   * The environment name used to customize the content of the helmfile manifest. The environment
+   * name defaults to default.
+   */
+  private String environment;
+
+  /** The namespace to be released into. */
+  private String namespace;
+
+  /**
+   * The 0th element is (or contains) the helmfile template. The rest (possibly none) are values
+   * files.
+   */
+  List<Artifact> inputArtifacts;
+
+  /**
+   * Include custom resource definition manifests in the templated output. Helmfile uses Helm v3
+   * only which provides the option to include CRDs as part of the rendered output.
+   */
+  boolean includeCRDs;
+}