diff --git a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/BakeManifestRequest.java b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/BakeManifestRequest.java index be5c8b67e..6a1eb36c8 100644 --- a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/BakeManifestRequest.java +++ b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/BakeManifestRequest.java @@ -17,6 +17,7 @@ public enum TemplateRenderer { HELM3, KUSTOMIZE, KUSTOMIZE4, + HELMFILE, CF; @JsonCreator diff --git a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/config/RoscoHelmfileConfigurationProperties.java b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/config/RoscoHelmfileConfigurationProperties.java new file mode 100644 index 000000000..f91deadd9 --- /dev/null +++ b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/config/RoscoHelmfileConfigurationProperties.java @@ -0,0 +1,26 @@ +/* + * Copyright 2020 Grab Holdings, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.rosco.manifests.config; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@ConfigurationProperties("helmfile") +@Data +public class RoscoHelmfileConfigurationProperties { + private String ExecutablePath = "helmfile"; +} diff --git a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileBakeManifestRequest.java b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileBakeManifestRequest.java new file mode 100644 index 000000000..6a31179ae --- /dev/null +++ b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileBakeManifestRequest.java @@ -0,0 +1,20 @@ +package com.netflix.spinnaker.rosco.manifests.helmfile; + +import com.netflix.spinnaker.kork.artifacts.model.Artifact; +import com.netflix.spinnaker.rosco.manifests.BakeManifestRequest; +import java.util.List; +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class HelmfileBakeManifestRequest extends BakeManifestRequest { + private String helmfileFilePath; + private String environment; + private String namespace; + + private BakeManifestRequest.TemplateRenderer helmEngine; + + List inputArtifacts; + boolean includeCRDs; +} diff --git a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileBakeManifestService.java b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileBakeManifestService.java new file mode 100644 index 000000000..ce2fce969 --- /dev/null +++ b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileBakeManifestService.java @@ -0,0 +1,49 @@ +package com.netflix.spinnaker.rosco.manifests.helmfile; + +import static com.netflix.spinnaker.rosco.manifests.BakeManifestRequest.TemplateRenderer; + +import com.google.common.collect.ImmutableSet; +import com.netflix.spinnaker.kork.artifacts.model.Artifact; +import com.netflix.spinnaker.rosco.jobs.BakeRecipe; +import com.netflix.spinnaker.rosco.jobs.JobExecutor; +import com.netflix.spinnaker.rosco.manifests.BakeManifestEnvironment; +import com.netflix.spinnaker.rosco.manifests.BakeManifestService; +import java.io.IOException; +import java.util.Base64; +import org.springframework.stereotype.Component; + +@Component +public class HelmfileBakeManifestService extends BakeManifestService { + private final HelmfileTemplateUtils helmfileTemplateUtils; + private static final ImmutableSet supportedTemplates = + ImmutableSet.of(TemplateRenderer.HELMFILE.toString()); + + public HelmfileBakeManifestService( + HelmfileTemplateUtils helmTemplateUtils, JobExecutor jobExecutor) { + super(jobExecutor); + this.helmfileTemplateUtils = helmTemplateUtils; + } + + @Override + public Class requestType() { + return HelmfileBakeManifestRequest.class; + } + + @Override + public boolean handles(String type) { + return supportedTemplates.contains(type); + } + + public Artifact bake(HelmfileBakeManifestRequest helmfileBakeManifestRequest) throws IOException { + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, helmfileBakeManifestRequest); + + String bakeResult = helmfileTemplateUtils.removeTestsDirectoryTemplates(doBake(recipe)); + return Artifact.builder() + .type("embedded/base64") + .name(helmfileBakeManifestRequest.getOutputArtifactName()) + .reference(Base64.getEncoder().encodeToString(bakeResult.getBytes())) + .build(); + } + } +} diff --git a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileTemplateUtils.java b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileTemplateUtils.java new file mode 100644 index 000000000..ff1e6b66d --- /dev/null +++ b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileTemplateUtils.java @@ -0,0 +1,166 @@ +package com.netflix.spinnaker.rosco.manifests.helmfile; + +import com.netflix.spinnaker.kork.artifacts.model.Artifact; +import com.netflix.spinnaker.kork.exceptions.SpinnakerException; +import com.netflix.spinnaker.kork.retrofit.exceptions.SpinnakerHttpException; +import com.netflix.spinnaker.rosco.jobs.BakeRecipe; +import com.netflix.spinnaker.rosco.manifests.ArtifactDownloader; +import com.netflix.spinnaker.rosco.manifests.BakeManifestEnvironment; +import com.netflix.spinnaker.rosco.manifests.BakeManifestRequest; +import com.netflix.spinnaker.rosco.manifests.config.RoscoHelmConfigurationProperties; +import com.netflix.spinnaker.rosco.manifests.config.RoscoHelmfileConfigurationProperties; +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +@Component +@Slf4j +public class HelmfileTemplateUtils { + private static final String MANIFEST_SEPARATOR = "---\n"; + private static final Pattern REGEX_TESTS_MANIFESTS = + Pattern.compile("# Source: .*/templates/tests/.*"); + + private final ArtifactDownloader artifactDownloader; + private final RoscoHelmfileConfigurationProperties helmfileConfigurationProperties; + private final RoscoHelmConfigurationProperties helmConfigurationProperties = + new RoscoHelmConfigurationProperties(); + + public HelmfileTemplateUtils( + ArtifactDownloader artifactDownloader, + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties) { + this.artifactDownloader = artifactDownloader; + this.helmfileConfigurationProperties = helmfileConfigurationProperties; + } + + public BakeRecipe buildBakeRecipe( + BakeManifestEnvironment env, HelmfileBakeManifestRequest request) throws IOException { + BakeRecipe result = new BakeRecipe(); + result.setName(request.getOutputName()); + + Path helmfileFilePath; + BakeManifestRequest.TemplateRenderer helmEngineVersion = + Optional.ofNullable(request.getHelmEngine()) + .orElse(BakeManifestRequest.TemplateRenderer.HELM3); + + List valuePaths = new ArrayList<>(); + List inputArtifacts = request.getInputArtifacts(); + if (inputArtifacts == null || inputArtifacts.isEmpty()) { + throw new IllegalArgumentException("At least one input artifact must be provided to bake"); + } + + Artifact helmfileTemplateArtifact = inputArtifacts.get(0); + String artifactType = Optional.ofNullable(helmfileTemplateArtifact.getType()).orElse(""); + if ("git/repo".equals(artifactType)) { + env.downloadArtifactTarballAndExtract(artifactDownloader, helmfileTemplateArtifact); + + log.info("helmfileFilePath: '{}'", request.getHelmfileFilePath()); + + // If there's no helmfile path specified, assume it lives in the root of + // the git/repo artifact. + helmfileFilePath = + env.resolvePath(Optional.ofNullable(request.getHelmfileFilePath()).orElse("")); + } else { + try { + helmfileFilePath = downloadArtifactToTmpFile(env, helmfileTemplateArtifact); + } catch (SpinnakerHttpException e) { + throw new SpinnakerHttpException(fetchFailureMessage("template", e), e); + } catch (IOException | SpinnakerException e) { + throw new IllegalStateException(fetchFailureMessage("template", e), e); + } + } + + log.info("path to helmfile: {}", helmfileFilePath); + + try { + // not a stream to keep exception handling cleaner + for (Artifact valueArtifact : inputArtifacts.subList(1, inputArtifacts.size())) { + valuePaths.add(downloadArtifactToTmpFile(env, valueArtifact)); + } + } catch (SpinnakerHttpException e) { + throw new SpinnakerHttpException(fetchFailureMessage("values file", e), e); + } catch (IOException | SpinnakerException e) { + throw new IllegalStateException(fetchFailureMessage("values file", e), e); + } + + List command = new ArrayList<>(); + String executable = helmfileConfigurationProperties.getExecutablePath(); + + command.add(executable); + command.add("template"); + command.add("--file"); + command.add(helmfileFilePath.toString()); + + command.add("--helm-binary"); + command.add(getHelmExecutablePath(helmEngineVersion)); + + String environment = request.getEnvironment(); + if (environment != null && !environment.isEmpty()) { + command.add("--environment"); + command.add(environment); + } + + String namespace = request.getNamespace(); + if (namespace != null && !namespace.isEmpty()) { + command.add("--namespace"); + command.add(namespace); + } + + if (request.isIncludeCRDs() + && BakeManifestRequest.TemplateRenderer.HELM3.equals(helmEngineVersion)) { + command.add("--include-crds"); + } + + Map overrides = request.getOverrides(); + if (!overrides.isEmpty()) { + List overrideList = new ArrayList<>(); + for (Map.Entry entry : overrides.entrySet()) { + overrideList.add(entry.getKey() + "=" + entry.getValue().toString()); + } + command.add("--set"); + command.add(String.join(",", overrideList)); + } + // + if (!valuePaths.isEmpty()) { + command.add("--values"); + command.add(valuePaths.stream().map(Path::toString).collect(Collectors.joining(","))); + } + + result.setCommand(command); + + return result; + } + + private String fetchFailureMessage(String description, Exception e) { + return "Failed to fetch helmfile " + description + ": " + e.getMessage(); + } + + public String removeTestsDirectoryTemplates(String inputString) { + return Arrays.stream(inputString.split(MANIFEST_SEPARATOR)) + .filter(manifest -> !REGEX_TESTS_MANIFESTS.matcher(manifest).find()) + .collect(Collectors.joining(MANIFEST_SEPARATOR)); + } + + private Path downloadArtifactToTmpFile(BakeManifestEnvironment env, Artifact artifact) + throws IOException { + String fileName = UUID.randomUUID().toString(); + Path targetPath = env.resolvePath(fileName); + artifactDownloader.downloadArtifactToFile(artifact, targetPath); + return targetPath; + } + + private String getHelmExecutablePath(BakeManifestRequest.TemplateRenderer helmEngineVersion) { + if (BakeManifestRequest.TemplateRenderer.HELM3.equals(helmEngineVersion)) { + return helmConfigurationProperties.getV3ExecutablePath(); + } + return helmConfigurationProperties.getV2ExecutablePath(); + } +} diff --git a/rosco-web/src/main/groovy/com/netflix/spinnaker/rosco/Main.groovy b/rosco-web/src/main/groovy/com/netflix/spinnaker/rosco/Main.groovy index 1f6fa34a8..4349529e7 100644 --- a/rosco-web/src/main/groovy/com/netflix/spinnaker/rosco/Main.groovy +++ b/rosco-web/src/main/groovy/com/netflix/spinnaker/rosco/Main.groovy @@ -19,6 +19,7 @@ package com.netflix.spinnaker.rosco import com.netflix.spinnaker.rosco.config.RoscoPackerConfigurationProperties import com.netflix.spinnaker.rosco.jobs.config.LocalJobConfig import com.netflix.spinnaker.rosco.manifests.config.RoscoHelmConfigurationProperties +import com.netflix.spinnaker.rosco.manifests.config.RoscoHelmfileConfigurationProperties import com.netflix.spinnaker.rosco.manifests.config.RoscoKustomizeConfigurationProperties import com.netflix.spinnaker.rosco.providers.alicloud.config.RoscoAliCloudConfiguration import com.netflix.spinnaker.rosco.providers.aws.config.RoscoAWSConfiguration @@ -68,6 +69,7 @@ import javax.servlet.Filter RoscoTencentCloudConfiguration, RoscoPackerConfigurationProperties, RoscoHelmConfigurationProperties, + RoscoHelmfileConfigurationProperties, RoscoKustomizeConfigurationProperties, LocalJobConfig ])