From 7eab89f31d9108ff95e434ce1b10b048bc7caccf Mon Sep 17 00:00:00 2001 From: ThameezBo Date: Tue, 23 May 2023 17:04:56 +0200 Subject: [PATCH] feat(manifests/helmfile): add code feat(manifests/helmfile): add tests --- .../rosco/manifests/BakeManifestRequest.java | 1 + .../RoscoHelmfileConfigurationProperties.java | 26 + .../helmfile/HelmfileBakeManifestRequest.java | 18 + .../helmfile/HelmfileBakeManifestService.java | 49 ++ .../helmfile/HelmfileTemplateUtils.java | 157 ++++ .../helmfile/HelmfileTemplateUtilsTest.java | 728 ++++++++++++++++++ .../com/netflix/spinnaker/rosco/Main.groovy | 2 + 7 files changed, 981 insertions(+) create mode 100644 rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/config/RoscoHelmfileConfigurationProperties.java create mode 100644 rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileBakeManifestRequest.java create mode 100644 rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileBakeManifestService.java create mode 100644 rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileTemplateUtils.java create mode 100644 rosco-manifests/src/test/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileTemplateUtilsTest.java diff --git a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/BakeManifestRequest.java b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/BakeManifestRequest.java index be5c8b67e..6a1eb36c8 100644 --- a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/BakeManifestRequest.java +++ b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/BakeManifestRequest.java @@ -17,6 +17,7 @@ public enum TemplateRenderer { HELM3, KUSTOMIZE, KUSTOMIZE4, + HELMFILE, CF; @JsonCreator diff --git a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/config/RoscoHelmfileConfigurationProperties.java b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/config/RoscoHelmfileConfigurationProperties.java new file mode 100644 index 000000000..f91deadd9 --- /dev/null +++ b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/config/RoscoHelmfileConfigurationProperties.java @@ -0,0 +1,26 @@ +/* + * Copyright 2020 Grab Holdings, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.rosco.manifests.config; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@ConfigurationProperties("helmfile") +@Data +public class RoscoHelmfileConfigurationProperties { + private String ExecutablePath = "helmfile"; +} diff --git a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileBakeManifestRequest.java b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileBakeManifestRequest.java new file mode 100644 index 000000000..99a4ed4e4 --- /dev/null +++ b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileBakeManifestRequest.java @@ -0,0 +1,18 @@ +package com.netflix.spinnaker.rosco.manifests.helmfile; + +import com.netflix.spinnaker.kork.artifacts.model.Artifact; +import com.netflix.spinnaker.rosco.manifests.BakeManifestRequest; +import java.util.List; +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class HelmfileBakeManifestRequest extends BakeManifestRequest { + private String helmfileFilePath; + private String environment; + private String namespace; + + List inputArtifacts; + boolean includeCRDs; +} diff --git a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileBakeManifestService.java b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileBakeManifestService.java new file mode 100644 index 000000000..ce2fce969 --- /dev/null +++ b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileBakeManifestService.java @@ -0,0 +1,49 @@ +package com.netflix.spinnaker.rosco.manifests.helmfile; + +import static com.netflix.spinnaker.rosco.manifests.BakeManifestRequest.TemplateRenderer; + +import com.google.common.collect.ImmutableSet; +import com.netflix.spinnaker.kork.artifacts.model.Artifact; +import com.netflix.spinnaker.rosco.jobs.BakeRecipe; +import com.netflix.spinnaker.rosco.jobs.JobExecutor; +import com.netflix.spinnaker.rosco.manifests.BakeManifestEnvironment; +import com.netflix.spinnaker.rosco.manifests.BakeManifestService; +import java.io.IOException; +import java.util.Base64; +import org.springframework.stereotype.Component; + +@Component +public class HelmfileBakeManifestService extends BakeManifestService { + private final HelmfileTemplateUtils helmfileTemplateUtils; + private static final ImmutableSet supportedTemplates = + ImmutableSet.of(TemplateRenderer.HELMFILE.toString()); + + public HelmfileBakeManifestService( + HelmfileTemplateUtils helmTemplateUtils, JobExecutor jobExecutor) { + super(jobExecutor); + this.helmfileTemplateUtils = helmTemplateUtils; + } + + @Override + public Class requestType() { + return HelmfileBakeManifestRequest.class; + } + + @Override + public boolean handles(String type) { + return supportedTemplates.contains(type); + } + + public Artifact bake(HelmfileBakeManifestRequest helmfileBakeManifestRequest) throws IOException { + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, helmfileBakeManifestRequest); + + String bakeResult = helmfileTemplateUtils.removeTestsDirectoryTemplates(doBake(recipe)); + return Artifact.builder() + .type("embedded/base64") + .name(helmfileBakeManifestRequest.getOutputArtifactName()) + .reference(Base64.getEncoder().encodeToString(bakeResult.getBytes())) + .build(); + } + } +} diff --git a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileTemplateUtils.java b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileTemplateUtils.java new file mode 100644 index 000000000..b7eae94f0 --- /dev/null +++ b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileTemplateUtils.java @@ -0,0 +1,157 @@ +package com.netflix.spinnaker.rosco.manifests.helmfile; + +import com.netflix.spinnaker.kork.artifacts.model.Artifact; +import com.netflix.spinnaker.kork.exceptions.SpinnakerException; +import com.netflix.spinnaker.kork.retrofit.exceptions.SpinnakerHttpException; +import com.netflix.spinnaker.rosco.jobs.BakeRecipe; +import com.netflix.spinnaker.rosco.manifests.ArtifactDownloader; +import com.netflix.spinnaker.rosco.manifests.BakeManifestEnvironment; +import com.netflix.spinnaker.rosco.manifests.config.RoscoHelmConfigurationProperties; +import com.netflix.spinnaker.rosco.manifests.config.RoscoHelmfileConfigurationProperties; +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +@Component +@Slf4j +public class HelmfileTemplateUtils { + private static final String MANIFEST_SEPARATOR = "---\n"; + private static final Pattern REGEX_TESTS_MANIFESTS = + Pattern.compile("# Source: .*/templates/tests/.*"); + + private final ArtifactDownloader artifactDownloader; + private final RoscoHelmfileConfigurationProperties helmfileConfigurationProperties; + private final RoscoHelmConfigurationProperties helmConfigurationProperties = + new RoscoHelmConfigurationProperties(); + + public HelmfileTemplateUtils( + ArtifactDownloader artifactDownloader, + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties) { + this.artifactDownloader = artifactDownloader; + this.helmfileConfigurationProperties = helmfileConfigurationProperties; + } + + public BakeRecipe buildBakeRecipe( + BakeManifestEnvironment env, HelmfileBakeManifestRequest request) throws IOException { + BakeRecipe result = new BakeRecipe(); + result.setName(request.getOutputName()); + + Path helmfileFilePath; + + List valuePaths = new ArrayList<>(); + List inputArtifacts = request.getInputArtifacts(); + if (inputArtifacts == null || inputArtifacts.isEmpty()) { + throw new IllegalArgumentException("At least one input artifact must be provided to bake"); + } + + log.info("helmfileFilePath: '{}'", request.getHelmfileFilePath()); + Artifact helmfileTemplateArtifact = inputArtifacts.get(0); + String artifactType = Optional.ofNullable(helmfileTemplateArtifact.getType()).orElse(""); + if ("git/repo".equals(artifactType)) { + env.downloadArtifactTarballAndExtract(artifactDownloader, helmfileTemplateArtifact); + + // If there's no helmfile path specified, assume it lives in the root of + // the git/repo artifact. + helmfileFilePath = + env.resolvePath(Optional.ofNullable(request.getHelmfileFilePath()).orElse("")); + } else { + try { + helmfileFilePath = downloadArtifactToTmpFile(env, helmfileTemplateArtifact); + } catch (SpinnakerHttpException e) { + throw new SpinnakerHttpException(fetchFailureMessage("template", e), e); + } catch (IOException | SpinnakerException e) { + throw new IllegalStateException(fetchFailureMessage("template", e), e); + } + } + + log.info("path to helmfile: {}", helmfileFilePath); + + try { + // not a stream to keep exception handling cleaner + for (Artifact valueArtifact : inputArtifacts.subList(1, inputArtifacts.size())) { + valuePaths.add(downloadArtifactToTmpFile(env, valueArtifact)); + } + } catch (SpinnakerHttpException e) { + throw new SpinnakerHttpException(fetchFailureMessage("values file", e), e); + } catch (IOException | SpinnakerException e) { + throw new IllegalStateException(fetchFailureMessage("values file", e), e); + } + + List command = new ArrayList<>(); + String executable = helmfileConfigurationProperties.getExecutablePath(); + + command.add(executable); + command.add("template"); + command.add("--file"); + command.add(helmfileFilePath.toString()); + + command.add("--helm-binary"); + command.add(getHelm3ExecutablePath()); + + String environment = request.getEnvironment(); + if (environment != null && !environment.isEmpty()) { + command.add("--environment"); + command.add(environment); + } + + String namespace = request.getNamespace(); + if (namespace != null && !namespace.isEmpty()) { + command.add("--namespace"); + command.add(namespace); + } + + if (request.isIncludeCRDs()) { + command.add("--include-crds"); + } + + Map overrides = request.getOverrides(); + if (!overrides.isEmpty()) { + List overrideList = new ArrayList<>(); + for (Map.Entry entry : overrides.entrySet()) { + overrideList.add(entry.getKey() + "=" + entry.getValue().toString()); + } + command.add("--set"); + command.add(String.join(",", overrideList)); + } + + if (!valuePaths.isEmpty()) { + command.add("--values"); + command.add(valuePaths.stream().map(Path::toString).collect(Collectors.joining(","))); + } + + result.setCommand(command); + + return result; + } + + private String fetchFailureMessage(String description, Exception e) { + return "Failed to fetch helmfile " + description + ": " + e.getMessage(); + } + + public String removeTestsDirectoryTemplates(String inputString) { + return Arrays.stream(inputString.split(MANIFEST_SEPARATOR)) + .filter(manifest -> !REGEX_TESTS_MANIFESTS.matcher(manifest).find()) + .collect(Collectors.joining(MANIFEST_SEPARATOR)); + } + + private Path downloadArtifactToTmpFile(BakeManifestEnvironment env, Artifact artifact) + throws IOException { + String fileName = UUID.randomUUID().toString(); + Path targetPath = env.resolvePath(fileName); + artifactDownloader.downloadArtifactToFile(artifact, targetPath); + return targetPath; + } + + private String getHelm3ExecutablePath() { + return helmConfigurationProperties.getV3ExecutablePath(); + } +} diff --git a/rosco-manifests/src/test/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileTemplateUtilsTest.java b/rosco-manifests/src/test/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileTemplateUtilsTest.java new file mode 100644 index 000000000..6cc3324cf --- /dev/null +++ b/rosco-manifests/src/test/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileTemplateUtilsTest.java @@ -0,0 +1,728 @@ +/* + * Copyright 2019 Google, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.rosco.manifests.helmfile; + +import static com.netflix.spinnaker.rosco.manifests.ManifestTestUtils.makeSpinnakerHttpException; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.netflix.spinnaker.kork.artifacts.model.Artifact; +import com.netflix.spinnaker.kork.exceptions.SpinnakerException; +import com.netflix.spinnaker.kork.retrofit.exceptions.SpinnakerHttpException; +import com.netflix.spinnaker.rosco.jobs.BakeRecipe; +import com.netflix.spinnaker.rosco.manifests.ArtifactDownloader; +import com.netflix.spinnaker.rosco.manifests.BakeManifestEnvironment; +import com.netflix.spinnaker.rosco.manifests.BakeManifestRequest; +import com.netflix.spinnaker.rosco.manifests.config.RoscoHelmConfigurationProperties; +import com.netflix.spinnaker.rosco.manifests.config.RoscoHelmfileConfigurationProperties; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.FileVisitOption; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; +import org.apache.commons.compress.utils.IOUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.platform.runner.JUnitPlatform; +import org.junit.runner.RunWith; +import org.springframework.http.HttpStatus; + +@RunWith(JUnitPlatform.class) +final class HelmfileTemplateUtilsTest { + + private ArtifactDownloader artifactDownloader; + + private HelmfileTemplateUtils helmfileTemplateUtils; + + private HelmfileBakeManifestRequest bakeManifestRequest; + + @BeforeEach + private void init(TestInfo testInfo) { + System.out.println("--------------- Test " + testInfo.getDisplayName()); + + artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + Artifact chartArtifact = Artifact.builder().name("test-artifact").version("3").build(); + + bakeManifestRequest = new HelmfileBakeManifestRequest(); + bakeManifestRequest.setInputArtifacts(ImmutableList.of(chartArtifact)); + } + + @Test + public void nullReferenceTest() throws IOException { + bakeManifestRequest.setOverrides(ImmutableMap.of()); + + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, bakeManifestRequest); + } + } + + @Test + public void exceptionDownloading() throws IOException { + // When artifactDownloader throws an exception, make sure we wrap it and get + // a chance to include our own message, so the exception that goes up the + // chain includes something about helmfile. + SpinnakerException spinnakerException = new SpinnakerException("error from ArtifactDownloader"); + doThrow(spinnakerException) + .when(artifactDownloader) + .downloadArtifactToFile(any(Artifact.class), any(Path.class)); + + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + IllegalStateException thrown = + assertThrows( + IllegalStateException.class, + () -> helmfileTemplateUtils.buildBakeRecipe(env, bakeManifestRequest)); + + assertThat(thrown.getMessage()).contains("Failed to fetch helmfile template"); + assertThat(thrown.getCause()).isEqualTo(spinnakerException); + } + } + + @Test + public void httpExceptionDownloading() throws IOException { + // When artifactDownloader throws a SpinnakerHttpException, make sure we + // wrap it and get a chance to include our own message, so the exception + // that goes up the chain includes something about helm charts. It's + // important that HelmTemplateUtils also throws a SpinnakerHttpException so + // it's eventually handled properly...meaning the status code in the http + // response and the logging correspond to what happened. For example, if + // there's a 404 from clouddriver, rosco also responds with 404, and doesn't + // log an error. + + SpinnakerHttpException spinnakerHttpException = + makeSpinnakerHttpException(HttpStatus.NOT_FOUND.value()); + doThrow(spinnakerHttpException) + .when(artifactDownloader) + .downloadArtifactToFile(any(Artifact.class), any(Path.class)); + + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + SpinnakerHttpException thrown = + assertThrows( + SpinnakerHttpException.class, + () -> helmfileTemplateUtils.buildBakeRecipe(env, bakeManifestRequest)); + + assertThat(thrown.getMessage()).contains("Failed to fetch helmfile template"); + assertThat(thrown.getResponseCode()).isEqualTo(HttpStatus.NOT_FOUND.value()); + assertThat(thrown.getCause()).isEqualTo(spinnakerHttpException); + } + } + + @Test + public void removeTestsDirectoryTemplatesWithTests() throws IOException { + String inputManifests = + "---\n" + + "# Source: mysql/templates/pvc.yaml\n" + + "\n" + + "kind: PersistentVolumeClaim\n" + + "apiVersion: v1\n" + + "metadata:\n" + + " name: release-name-mysql\n" + + " namespace: default\n" + + "spec:\n" + + " accessModes:\n" + + " - \"ReadWriteOnce\"\n" + + " resources:\n" + + " requests:\n" + + " storage: \"8Gi\"\n" + + "---\n" + + "# Source: mysql/templates/tests/test-configmap.yaml\n" + + "apiVersion: v1\n" + + "kind: ConfigMap\n" + + "metadata:\n" + + " name: release-name-mysql-test\n" + + " namespace: default\n" + + "data:\n" + + " run.sh: |-\n"; + + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + + String output = helmfileTemplateUtils.removeTestsDirectoryTemplates(inputManifests); + + String expected = + "---\n" + + "# Source: mysql/templates/pvc.yaml\n" + + "\n" + + "kind: PersistentVolumeClaim\n" + + "apiVersion: v1\n" + + "metadata:\n" + + " name: release-name-mysql\n" + + " namespace: default\n" + + "spec:\n" + + " accessModes:\n" + + " - \"ReadWriteOnce\"\n" + + " resources:\n" + + " requests:\n" + + " storage: \"8Gi\"\n"; + + assertEquals(expected.trim(), output.trim()); + } + + @Test + public void removeTestsDirectoryTemplatesWithoutTests() throws IOException { + String inputManifests = + "---\n" + + "# Source: mysql/templates/pvc.yaml\n" + + "\n" + + "kind: PersistentVolumeClaim\n" + + "apiVersion: v1\n" + + "metadata:\n" + + " name: release-name-mysql\n" + + " namespace: default\n" + + "spec:\n" + + " accessModes:\n" + + " - \"ReadWriteOnce\"\n" + + " resources:\n" + + " requests:\n" + + " storage: \"8Gi\"\n" + + "---\n" + + "# Source: mysql/templates/configmap.yaml\n" + + "apiVersion: v1\n" + + "kind: ConfigMap\n" + + "metadata:\n" + + " name: release-name-mysql-test\n" + + " namespace: default\n" + + "data:\n" + + " run.sh: |-\n"; + + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + + String output = helmfileTemplateUtils.removeTestsDirectoryTemplates(inputManifests); + + assertEquals(inputManifests.trim(), output.trim()); + } + + @ParameterizedTest + @MethodSource("helmfileRendererArgs") + public void buildBakeRecipeSelectsHelm3ExecutableWhenNoneSet( + String command, BakeManifestRequest.TemplateRenderer templateRenderer) throws IOException { + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + RoscoHelmConfigurationProperties helmConfigurationProperties = + new RoscoHelmConfigurationProperties(); + + HelmfileBakeManifestRequest request = new HelmfileBakeManifestRequest(); + Artifact artifact = Artifact.builder().build(); + request.setInputArtifacts(Collections.singletonList(artifact)); + request.setOverrides(Collections.emptyMap()); + + request.setTemplateRenderer(templateRenderer); + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, request); + + assertEquals(helmConfigurationProperties.getV3ExecutablePath(), recipe.getCommand().get(5)); + } + } + + @ParameterizedTest + @MethodSource("helmfileRendererArgs") + public void buildBakeRecipeSelectsHelm3ExecutableWhenSet( + String command, BakeManifestRequest.TemplateRenderer templateRenderer) throws IOException { + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + RoscoHelmConfigurationProperties helmConfigurationProperties = + new RoscoHelmConfigurationProperties(); + + HelmfileBakeManifestRequest request = new HelmfileBakeManifestRequest(); + Artifact artifact = Artifact.builder().build(); + request.setInputArtifacts(Collections.singletonList(artifact)); + request.setOverrides(Collections.emptyMap()); + request.setHelmEngine(BakeManifestRequest.TemplateRenderer.HELM3); + + request.setTemplateRenderer(templateRenderer); + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, request); + + assertEquals(helmConfigurationProperties.getV3ExecutablePath(), recipe.getCommand().get(5)); + } + } + + @ParameterizedTest + @MethodSource("helmfileRendererArgs") + public void buildBakeRecipeSelectsHelm2ExecutableWhenSet( + String command, BakeManifestRequest.TemplateRenderer templateRenderer) throws IOException { + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + RoscoHelmConfigurationProperties helmConfigurationProperties = + new RoscoHelmConfigurationProperties(); + + HelmfileBakeManifestRequest request = new HelmfileBakeManifestRequest(); + Artifact artifact = Artifact.builder().build(); + request.setInputArtifacts(Collections.singletonList(artifact)); + request.setOverrides(Collections.emptyMap()); + request.setHelmEngine(BakeManifestRequest.TemplateRenderer.HELM2); + + request.setTemplateRenderer(templateRenderer); + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, request); + + assertEquals(helmConfigurationProperties.getV2ExecutablePath(), recipe.getCommand().get(5)); + } + } + + private static Stream helmfileRendererArgs() { + // The command here (e.g. helmfile) must match the defaults in + // RoscoHelmfileConfigurationProperties + return Stream.of(Arguments.of("helmfile", BakeManifestRequest.TemplateRenderer.HELMFILE)); + } + + @Test + public void buildBakeRecipeWithGitRepoArtifact(@TempDir Path tempDir) throws IOException { + // git/repo artifacts appear as a tarball, so create one that contains a helmfile file + // and helm chart. + addTestHelmfile(tempDir); + + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + + HelmfileBakeManifestRequest request = new HelmfileBakeManifestRequest(); + + Artifact artifact = + Artifact.builder().type("git/repo").reference("https://github.com/some/repo.git").build(); + + // Set up the mock artifactDownloader to supply the tarball that represents + // the git/repo artifact + when(artifactDownloader.downloadArtifact(artifact)).thenReturn(makeTarball(tempDir)); + + request.setInputArtifacts(Collections.singletonList(artifact)); + request.setOverrides(Collections.emptyMap()); + + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, request); + + // Make sure we're really testing the git/repo logic + verify(artifactDownloader).downloadArtifact(artifact); + + // Make sure the BakeManifestEnvironment has the files in our git/repo artifact. + assertTrue(env.resolvePath("helmfile.yaml").toFile().exists()); + assertTrue(env.resolvePath("Chart.yaml").toFile().exists()); + assertTrue(env.resolvePath("values.yaml").toFile().exists()); + assertTrue(env.resolvePath("templates/foo.yaml").toFile().exists()); + } + } + + @Test + public void buildBakeRecipeWithGitRepoArtifactUsingHelmfileFilePath(@TempDir Path tempDir) + throws IOException { + // Create a tarball with a helmfile in a sub directory + String subDirName = "subdir"; + Path subDir = tempDir.resolve(subDirName); + addTestHelmfile(subDir); + + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + + HelmfileBakeManifestRequest request = new HelmfileBakeManifestRequest(); + + // Note that supplying a location for a git/repo artifact doesn't change the + // path in the resulting tarball. It's here because it's likely that it's + // used together with helmChartFilePath. Removing it wouldn't change the + // test. + Artifact artifact = + Artifact.builder() + .type("git/repo") + .reference("https://github.com/some/repo.git") + .location(subDirName) + .build(); + + // Set up the mock artifactDownloader to supply the tarball that represents + // the git/repo artifact + when(artifactDownloader.downloadArtifact(artifact)).thenReturn(makeTarball(tempDir)); + + request.setInputArtifacts(Collections.singletonList(artifact)); + request.setOverrides(Collections.emptyMap()); + + // This is the key part of this test. + request.setHelmfileFilePath(subDirName); + + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, request); + + // Make sure we're really testing the git/repo logic + verify(artifactDownloader).downloadArtifact(artifact); + + // Make sure the BakeManifestEnvironment has the files in our git/repo + // artifact in the expected location. + assertTrue(env.resolvePath(Path.of(subDirName, "helmfile.yaml")).toFile().exists()); + assertTrue(env.resolvePath(Path.of(subDirName, "Chart.yaml")).toFile().exists()); + assertTrue(env.resolvePath(Path.of(subDirName, "values.yaml")).toFile().exists()); + assertTrue(env.resolvePath(Path.of(subDirName, "templates/foo.yaml")).toFile().exists()); + + // And that the helm template command includes the path to the subdirectory + // + // The expected elements in the + // command list are: + // + // 0 - the helm executable + // 1 - template + // 2 - --file flag + // 3 - the path to helmfile.yaml + assertEquals(env.resolvePath(subDirName).toString(), recipe.getCommand().get(3)); + } + } + + /** + * Add a helmfile and helm chart for testing + * + * @param path the location of the helmfile file (e.g. helmfile.yaml) + */ + void addTestHelmfile(Path path) throws IOException { + addFile( + path, + "helmfile.yaml", + "releases:\n" + + " - name: test\n" + + " namespace: namespace\n" + + " chart: Chart.yaml\n" + + " values:\n" + + " - values.yaml\n"); + + addFile( + path, + "Chart.yaml", + "apiVersion: v1\n" + + "name: example\n" + + "description: chart for testing\n" + + "version: 0.1\n" + + "engine: gotpl\n"); + + addFile(path, "values.yaml", "foo: bar\n"); + + addFile( + path, + "templates/foo.yaml", + "labels:\n" + "helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}\n"); + } + + /** + * Create a new file in the temp directory + * + * @param path the path of the file to create (relative to the temp directory's root) + * @param content the content of the file, or null for an empty file + */ + void addFile(Path tempDir, String path, String content) throws IOException { + Path pathToCreate = tempDir.resolve(path); + pathToCreate.toFile().getParentFile().mkdirs(); + Files.write(pathToCreate, content.getBytes()); + } + + /** + * Make a gzipped tarball of all files in a path + * + * @param rootPath the root path of the tarball + * @return an InputStream containing the gzipped tarball + */ + InputStream makeTarball(Path rootPath) throws IOException { + ArrayList filePathsToAdd = + Files.walk(rootPath, FileVisitOption.FOLLOW_LINKS) + .filter(path -> !path.equals(rootPath)) + .collect(Collectors.toCollection(ArrayList::new)); + + // See + // https://commons.apache.org/proper/commons-compress/examples.html#Common_Archival_Logic + // for background + try (ByteArrayOutputStream os = new ByteArrayOutputStream(); + GzipCompressorOutputStream gzo = new GzipCompressorOutputStream(os); + TarArchiveOutputStream tarArchive = new TarArchiveOutputStream(gzo)) { + for (Path path : filePathsToAdd) { + TarArchiveEntry tarEntry = + new TarArchiveEntry(path.toFile(), rootPath.relativize(path).toString()); + tarArchive.setBigNumberMode(tarArchive.BIGNUMBER_POSIX); + tarArchive.setLongFileMode(tarArchive.LONGFILE_POSIX); + tarArchive.putArchiveEntry(tarEntry); + if (path.toFile().isFile()) { + IOUtils.copy(Files.newInputStream(path), tarArchive); + } + tarArchive.closeArchiveEntry(); + } + + tarArchive.finish(); + gzo.finish(); + + return new ByteArrayInputStream(os.toByteArray()); + } + } + + @Test + public void buildBakeRecipeIncludesEnvironmentWhenSet() throws IOException { + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + + HelmfileBakeManifestRequest request = new HelmfileBakeManifestRequest(); + Artifact artifact = Artifact.builder().build(); + String envName = "testEnvironment"; + request.setEnvironment(envName); + request.setInputArtifacts(Collections.singletonList(artifact)); + request.setOverrides(Collections.emptyMap()); + + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, request); + assertTrue(recipe.getCommand().contains("--environment")); + assertTrue(recipe.getCommand().contains(envName)); + // Assert that the flag position goes after 'helmfile template' subcommand + assertTrue(recipe.getCommand().indexOf("--environment") > 1); + } + } + + @Test + public void buildBakeRecipeDoesNotIncludeEnvironmentWhenNotSet() throws IOException { + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + + HelmfileBakeManifestRequest request = new HelmfileBakeManifestRequest(); + Artifact artifact = Artifact.builder().build(); + request.setInputArtifacts(Collections.singletonList(artifact)); + request.setOverrides(Collections.emptyMap()); + + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, request); + assertFalse(recipe.getCommand().contains("--environment")); + } + } + + @Test + public void buildBakeRecipeIncludesNamespaceWhenSet() throws IOException { + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + + HelmfileBakeManifestRequest request = new HelmfileBakeManifestRequest(); + Artifact artifact = Artifact.builder().build(); + String namespaceName = "testNamespace"; + request.setNamespace(namespaceName); + request.setInputArtifacts(Collections.singletonList(artifact)); + request.setOverrides(Collections.emptyMap()); + + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, request); + assertTrue(recipe.getCommand().contains("--namespace")); + assertTrue(recipe.getCommand().contains(namespaceName)); + // Assert that the flag position goes after 'helmfile template' subcommand + assertTrue(recipe.getCommand().indexOf("--namespace") > 1); + } + } + + @Test + public void buildBakeRecipeDoesNotIncludeNamespaceWhenNotSet() throws IOException { + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + + HelmfileBakeManifestRequest request = new HelmfileBakeManifestRequest(); + Artifact artifact = Artifact.builder().build(); + request.setInputArtifacts(Collections.singletonList(artifact)); + request.setOverrides(Collections.emptyMap()); + + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, request); + assertFalse(recipe.getCommand().contains("--namespace")); + } + } + + @Test + public void buildBakeRecipeNotIncludingCRDsWithHelm2() throws IOException { + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + + HelmfileBakeManifestRequest request = new HelmfileBakeManifestRequest(); + Artifact artifact = Artifact.builder().build(); + request.setHelmEngine(BakeManifestRequest.TemplateRenderer.HELM2); + request.setIncludeCRDs(true); + request.setInputArtifacts(Collections.singletonList(artifact)); + request.setOverrides(Collections.emptyMap()); + + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, request); + assertFalse(recipe.getCommand().contains("--include-crds")); + } + } + + @Test + public void buildBakeRecipeIncludingCRDsWithHelm3() throws IOException { + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + + HelmfileBakeManifestRequest request = new HelmfileBakeManifestRequest(); + Artifact artifact = Artifact.builder().build(); + request.setHelmEngine(BakeManifestRequest.TemplateRenderer.HELM3); + request.setIncludeCRDs(true); + request.setInputArtifacts(Collections.singletonList(artifact)); + request.setOverrides(Collections.emptyMap()); + + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, request); + assertTrue(recipe.getCommand().contains("--include-crds")); + // Assert that the flag position goes after 'helmfile template' subcommand + assertTrue(recipe.getCommand().indexOf("--include-crds") > 1); + } + } + + @Test + public void buildBakeRecipeIncludesOverridesWhenSet() throws IOException { + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + + HelmfileBakeManifestRequest request = new HelmfileBakeManifestRequest(); + Artifact artifact = Artifact.builder().build(); + String overrideKey = "testOverrideKey"; + String overrideValue = "testOverrideValue"; + request.setInputArtifacts(Collections.singletonList(artifact)); + request.setOverrides(Collections.singletonMap(overrideKey, overrideValue)); + + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, request); + assertTrue(recipe.getCommand().contains("--set")); + assertTrue(recipe.getCommand().contains(overrideKey + "=" + overrideValue)); + // Assert that the flag position goes after 'helmfile template' subcommand + assertTrue(recipe.getCommand().indexOf("--set") > 1); + } + } + + @Test + public void buildBakeRecipeDoesNotIncludeOverridesWhenNotSet() throws IOException { + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + + HelmfileBakeManifestRequest request = new HelmfileBakeManifestRequest(); + Artifact artifact = Artifact.builder().build(); + request.setInputArtifacts(Collections.singletonList(artifact)); + request.setOverrides(Collections.emptyMap()); + + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, request); + assertFalse(recipe.getCommand().contains("--set")); + } + } + + @Test + public void buildBakeRecipeIncludesValuesWhenSet() throws IOException { + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + + HelmfileBakeManifestRequest request = new HelmfileBakeManifestRequest(); + + List artifacts = new ArrayList<>(); + Artifact artifact = Artifact.builder().build(); + Artifact testValue = Artifact.builder().build(); + artifacts.add(artifact); + artifacts.add(testValue); + + request.setInputArtifacts(artifacts); + request.setOverrides(Collections.emptyMap()); + + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, request); + assertTrue(recipe.getCommand().contains("--values")); + // Assert that the flag position goes after 'helmfile template' subcommand + assertTrue(recipe.getCommand().indexOf("--values") > 1); + } + } + + @Test + public void buildBakeRecipeDoesNotIncludeValuesWhenNotSet() throws IOException { + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + + HelmfileBakeManifestRequest request = new HelmfileBakeManifestRequest(); + Artifact artifact = Artifact.builder().build(); + request.setInputArtifacts(Collections.singletonList(artifact)); + request.setOverrides(Collections.emptyMap()); + + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, request); + assertFalse(recipe.getCommand().contains("--values")); + } + } +} diff --git a/rosco-web/src/main/groovy/com/netflix/spinnaker/rosco/Main.groovy b/rosco-web/src/main/groovy/com/netflix/spinnaker/rosco/Main.groovy index 1f6fa34a8..4349529e7 100644 --- a/rosco-web/src/main/groovy/com/netflix/spinnaker/rosco/Main.groovy +++ b/rosco-web/src/main/groovy/com/netflix/spinnaker/rosco/Main.groovy @@ -19,6 +19,7 @@ package com.netflix.spinnaker.rosco import com.netflix.spinnaker.rosco.config.RoscoPackerConfigurationProperties import com.netflix.spinnaker.rosco.jobs.config.LocalJobConfig import com.netflix.spinnaker.rosco.manifests.config.RoscoHelmConfigurationProperties +import com.netflix.spinnaker.rosco.manifests.config.RoscoHelmfileConfigurationProperties import com.netflix.spinnaker.rosco.manifests.config.RoscoKustomizeConfigurationProperties import com.netflix.spinnaker.rosco.providers.alicloud.config.RoscoAliCloudConfiguration import com.netflix.spinnaker.rosco.providers.aws.config.RoscoAWSConfiguration @@ -68,6 +69,7 @@ import javax.servlet.Filter RoscoTencentCloudConfiguration, RoscoPackerConfigurationProperties, RoscoHelmConfigurationProperties, + RoscoHelmfileConfigurationProperties, RoscoKustomizeConfigurationProperties, LocalJobConfig ])