feat(manifests/helmfile): add helmfile templating engine

Dockerfile.slim

@@ -4,6 +4,7 @@ LABEL maintainer="sig-platform@spinnaker.io"
 ENV KUSTOMIZE_VERSION=3.8.6
 ENV KUSTOMIZE4_VERSION=4.5.5
 ENV PACKER_VERSION=1.8.1
+ENV HELMFILE_VERSION=0.153.1
 
 
 ARG TARGETARCH
@@ -42,6 +43,12 @@ RUN mkdir kustomize && \
  mv ./kustomize/kustomize /usr/local/bin/kustomize4 && \
  rm -rf ./kustomize
 
+RUN mkdir helmfile && \
+ curl -s -L https://github.com/helmfile/helmfile/releases/download/v${HELMFILE_VERSION}/helmfile_${HELMFILE_VERSION}_linux_${TARGETARCH}.tar.gz|\
+ tar xvz -C helmfile/ && \
+ mv ./helmfile/helmfile /usr/local/bin/helmfile && \
+ rm -rf ./helmfile
+
 RUN addgroup -S -g 10111 spinnaker
 RUN adduser -S -G spinnaker -u 10111 spinnaker
 COPY rosco-web/build/install/rosco /opt/rosco

Dockerfile.ubuntu

@@ -4,6 +4,7 @@ LABEL maintainer="sig-platform@spinnaker.io"
 ENV KUSTOMIZE_VERSION=3.8.6
 ENV KUSTOMIZE4_VERSION=4.5.5
 ENV PACKER_VERSION=1.8.1
+ENV HELMFILE_VERSION=0.153.1
 
 ARG TARGETARCH
 
@@ -41,6 +42,12 @@ RUN mkdir kustomize && \
  mv ./kustomize/kustomize /usr/local/bin/kustomize4 && \
  rm -rf ./kustomize
 
+RUN mkdir helmfile && \
+ curl -s -L https://github.com/helmfile/helmfile/releases/download/v${HELMFILE_VERSION}/helmfile_${HELMFILE_VERSION}_linux_${TARGETARCH}.tar.gz|\
+ tar xvz -C helmfile/ && \
+ mv ./helmfile/helmfile /usr/local/bin/helmfile && \
+ rm -rf ./helmfile
+
 RUN adduser --system --uid 10111 --group spinnaker
 COPY rosco-web/build/install/rosco /opt/rosco
 COPY rosco-web/config /opt/rosco

rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/BakeManifestRequest.java

@@ -17,6 +17,7 @@ public enum TemplateRenderer {
  HELM3,
  KUSTOMIZE,
  KUSTOMIZE4,
+ HELMFILE,
  CF;
 
  @JsonCreator

rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/HelmBakeTemplateUtils.java

@@ -0,0 +1,78 @@
+/*
+ * Copyright 2023 Grab Holdings, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.rosco.manifests;
+
+import com.netflix.spinnaker.kork.artifacts.model.Artifact;
+import com.netflix.spinnaker.kork.exceptions.SpinnakerException;
+import com.netflix.spinnaker.kork.retrofit.exceptions.SpinnakerHttpException;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.*;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+public abstract class HelmBakeTemplateUtils<T extends BakeManifestRequest> {
+ private static final String MANIFEST_SEPARATOR = "---\n";
+ private static final Pattern REGEX_TESTS_MANIFESTS =
+ Pattern.compile("# Source: .*/templates/tests/.*");
+
+ private final ArtifactDownloader artifactDownloader;
+
+ protected HelmBakeTemplateUtils(ArtifactDownloader artifactDownloader) {
+ this.artifactDownloader = artifactDownloader;
+ }
+
+ public ArtifactDownloader getArtifactDownloader() {
+ return artifactDownloader;
+ }
+
+ public abstract String fetchFailureMessage(String description, Exception e);
+
+ public String removeTestsDirectoryTemplates(String inputString) {
+ return Arrays.stream(inputString.split(MANIFEST_SEPARATOR))
+ .filter(manifest -> !REGEX_TESTS_MANIFESTS.matcher(manifest).find())
+ .collect(Collectors.joining(MANIFEST_SEPARATOR));
+ }
+
+ protected Path downloadArtifactToTmpFile(BakeManifestEnvironment env, Artifact artifact)
+ throws IOException {
+ String fileName = UUID.randomUUID().toString();
+ Path targetPath = env.resolvePath(fileName);
+ artifactDownloader.downloadArtifactToFile(artifact, targetPath);
+ return targetPath;
+ }
+
+ public abstract String getHelmExecutableForRequest(T request);
+
+ protected List<Path> getValuePaths(List<Artifact> artifacts, BakeManifestEnvironment env) {
+ List<Path> valuePaths = new ArrayList<>();
+
+ try {
+ // not a stream to keep exception handling cleaner
+ for (Artifact valueArtifact : artifacts.subList(1, artifacts.size())) {
+ valuePaths.add(downloadArtifactToTmpFile(env, valueArtifact));
+ }
+ } catch (SpinnakerHttpException e) {
+ throw new SpinnakerHttpException(fetchFailureMessage("values file", e), e);
+ } catch (IOException | SpinnakerException e) {
+ throw new IllegalStateException(fetchFailureMessage("values file", e), e);
+ }
+
+ return valuePaths;
+ }
+}

rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/config/RoscoHelmfileConfigurationProperties.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2023 Grab Holdings, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.rosco.manifests.config;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+
+@ConfigurationProperties("helmfile")
+@Data
+public class RoscoHelmfileConfigurationProperties {
+ private String executablePath = "helmfile";
+}

rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helm/HelmTemplateUtils.java

@@ -7,34 +7,27 @@
 import com.netflix.spinnaker.rosco.manifests.ArtifactDownloader;
 import com.netflix.spinnaker.rosco.manifests.BakeManifestEnvironment;
 import com.netflix.spinnaker.rosco.manifests.BakeManifestRequest;
+import com.netflix.spinnaker.rosco.manifests.HelmBakeTemplateUtils;
 import com.netflix.spinnaker.rosco.manifests.config.RoscoHelmConfigurationProperties;
 import java.io.IOException;
 import java.nio.file.Path;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.UUID;
-import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Component;
 
 @Component
 @Slf4j
-public class HelmTemplateUtils {
- private static final String MANIFEST_SEPARATOR = "---\n";
- private static final Pattern REGEX_TESTS_MANIFESTS =
- Pattern.compile("# Source: .*/templates/tests/.*");
-
- private final ArtifactDownloader artifactDownloader;
+public class HelmTemplateUtils extends HelmBakeTemplateUtils<HelmBakeManifestRequest> {
  private final RoscoHelmConfigurationProperties helmConfigurationProperties;
 
  public HelmTemplateUtils(
  ArtifactDownloader artifactDownloader,
  RoscoHelmConfigurationProperties helmConfigurationProperties) {
- this.artifactDownloader = artifactDownloader;
+ super(artifactDownloader);
  this.helmConfigurationProperties = helmConfigurationProperties;
  }
 
@@ -44,7 +37,8 @@ public BakeRecipe buildBakeRecipe(BakeManifestEnvironment env, HelmBakeManifestR
  result.setName(request.getOutputName());
 
  Path templatePath;
- List<Path> valuePaths = new ArrayList<>();
+ List<Path> valuePaths;
+
  List<Artifact> inputArtifacts = request.getInputArtifacts();
  if (inputArtifacts == null || inputArtifacts.isEmpty()) {
  throw new IllegalArgumentException("At least one input artifact must be provided to bake");
@@ -53,7 +47,7 @@ public BakeRecipe buildBakeRecipe(BakeManifestEnvironment env, HelmBakeManifestR
  Artifact helmTemplateArtifact = inputArtifacts.get(0);
  String artifactType = Optional.ofNullable(helmTemplateArtifact.getType()).orElse("");
  if ("git/repo".equals(artifactType)) {
- env.downloadArtifactTarballAndExtract(artifactDownloader, helmTemplateArtifact);
+ env.downloadArtifactTarballAndExtract(super.getArtifactDownloader(), helmTemplateArtifact);
 
  log.info("helmChartFilePath: '{}'", request.getHelmChartFilePath());
 
@@ -73,16 +67,7 @@ public BakeRecipe buildBakeRecipe(BakeManifestEnvironment env, HelmBakeManifestR
 
  log.info("path to Chart.yaml: {}", templatePath);
 
- try {
- // not a stream to keep exception handling cleaner
- for (Artifact valueArtifact : inputArtifacts.subList(1, inputArtifacts.size())) {
- valuePaths.add(downloadArtifactToTmpFile(env, valueArtifact));
- }
- } catch (SpinnakerHttpException e) {
- throw new SpinnakerHttpException(fetchFailureMessage("values file", e), e);
- } catch (IOException | SpinnakerException e) {
- throw new IllegalStateException(fetchFailureMessage("values file", e), e);
- }
+ valuePaths = getValuePaths(inputArtifacts, env);
 
  List<String> command = new ArrayList<>();
  String executable = getHelmExecutableForRequest(request);
@@ -134,25 +119,11 @@ public BakeRecipe buildBakeRecipe(BakeManifestEnvironment env, HelmBakeManifestR
  return result;
  }
 
- private String fetchFailureMessage(String description, Exception e) {
+ public String fetchFailureMessage(String description, Exception e) {
  return "Failed to fetch helm " + description + ": " + e.getMessage();
  }
 
- public String removeTestsDirectoryTemplates(String inputString) {
- return Arrays.stream(inputString.split(MANIFEST_SEPARATOR))
- .filter(manifest -> !REGEX_TESTS_MANIFESTS.matcher(manifest).find())
- .collect(Collectors.joining(MANIFEST_SEPARATOR));
- }
-
- private Path downloadArtifactToTmpFile(BakeManifestEnvironment env, Artifact artifact)
- throws IOException {
- String fileName = UUID.randomUUID().toString();
- Path targetPath = env.resolvePath(fileName);
- artifactDownloader.downloadArtifactToFile(artifact, targetPath);
- return targetPath;
- }
-
- private String getHelmExecutableForRequest(HelmBakeManifestRequest request) {
+ public String getHelmExecutableForRequest(HelmBakeManifestRequest request) {
  if (BakeManifestRequest.TemplateRenderer.HELM2.equals(request.getTemplateRenderer())) {
  return helmConfigurationProperties.getV2ExecutablePath();
  }

rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileBakeManifestRequest.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright 2023 Grab Holdings, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.rosco.manifests.helmfile;
+
+import com.netflix.spinnaker.kork.artifacts.model.Artifact;
+import com.netflix.spinnaker.rosco.manifests.BakeManifestRequest;
+import java.util.List;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+
+@Data
+@EqualsAndHashCode(callSuper = true)
+public class HelmfileBakeManifestRequest extends BakeManifestRequest {
+ private String helmfileFilePath;
+
+ /**
+ * The environment name used to customize the content of the helmfile manifest.
+ * The environment name defaults to default.
+ */
+ private String environment;
+
+ /**
+ * The namespace to be released into.
+ */
+ private String namespace;
+
+ /**
+ * The 0th element is (or contains) the helmfile template. The rest (possibly none) are values
+ * files.
+ */
+ List<Artifact> inputArtifacts;
+
+ /**
+ * Include custom resource definition manifests in the templated output.
+ * Helmfile uses Helm v3 only which provides the option to include CRDs as part of the rendered output.
+ */
+ boolean includeCRDs;
+}

rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileBakeManifestService.java

@@ -0,0 +1,65 @@
+/*
+ * Copyright 2023 Grab Holdings, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.rosco.manifests.helmfile;
+
+import static com.netflix.spinnaker.rosco.manifests.BakeManifestRequest.TemplateRenderer;
+
+import com.google.common.collect.ImmutableSet;
+import com.netflix.spinnaker.kork.artifacts.model.Artifact;
+import com.netflix.spinnaker.rosco.jobs.BakeRecipe;
+import com.netflix.spinnaker.rosco.jobs.JobExecutor;
+import com.netflix.spinnaker.rosco.manifests.BakeManifestEnvironment;
+import com.netflix.spinnaker.rosco.manifests.BakeManifestService;
+import java.io.IOException;
+import java.util.Base64;
+import org.springframework.stereotype.Component;
+
+@Component
+public class HelmfileBakeManifestService extends BakeManifestService<HelmfileBakeManifestRequest> {
+ private final HelmfileTemplateUtils helmfileTemplateUtils;
+ private static final ImmutableSet<String> supportedTemplates =
+ ImmutableSet.of(TemplateRenderer.HELMFILE.toString());
+
+ public HelmfileBakeManifestService(
+ HelmfileTemplateUtils helmTemplateUtils, JobExecutor jobExecutor) {
+ super(jobExecutor);
+ this.helmfileTemplateUtils = helmTemplateUtils;
+ }
+
+ @Override
+ public Class<HelmfileBakeManifestRequest> requestType() {
+ return HelmfileBakeManifestRequest.class;
+ }
+
+ @Override
+ public boolean handles(String type) {
+ return supportedTemplates.contains(type);
+ }
+
+ public Artifact bake(HelmfileBakeManifestRequest helmfileBakeManifestRequest) throws IOException {
+ try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) {
+ BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, helmfileBakeManifestRequest);
+
+ String bakeResult = helmfileTemplateUtils.removeTestsDirectoryTemplates(doBake(recipe));
+ return Artifact.builder()
+ .type("embedded/base64")
+ .name(helmfileBakeManifestRequest.getOutputArtifactName())
+ .reference(Base64.getEncoder().encodeToString(bakeResult.getBytes()))
+ .build();
+ }
+ }
+}