diff --git a/Dockerfile.slim b/Dockerfile.slim index 4c5ba1b15..759d909b0 100644 --- a/Dockerfile.slim +++ b/Dockerfile.slim @@ -4,6 +4,7 @@ LABEL maintainer="sig-platform@spinnaker.io" ENV KUSTOMIZE_VERSION=3.8.6 ENV KUSTOMIZE4_VERSION=4.5.5 ENV PACKER_VERSION=1.8.1 +ENV HELMFILE_VERSION=0.153.1 ARG TARGETARCH @@ -42,6 +43,12 @@ RUN mkdir kustomize && \ mv ./kustomize/kustomize /usr/local/bin/kustomize4 && \ rm -rf ./kustomize +RUN mkdir helmfile && \ + curl -s -L https://github.com/helmfile/helmfile/releases/download/v${HELMFILE_VERSION}/helmfile_${HELMFILE_VERSION}_linux_${TARGETARCH}.tar.gz|\ + tar xvz -C helmfile/ && \ + mv ./helmfile/helmfile /usr/local/bin/helmfile && \ + rm -rf ./helmfile + RUN addgroup -S -g 10111 spinnaker RUN adduser -S -G spinnaker -u 10111 spinnaker COPY rosco-web/build/install/rosco /opt/rosco diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 0bbd3f963..080147d92 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -4,6 +4,7 @@ LABEL maintainer="sig-platform@spinnaker.io" ENV KUSTOMIZE_VERSION=3.8.6 ENV KUSTOMIZE4_VERSION=4.5.5 ENV PACKER_VERSION=1.8.1 +ENV HELMFILE_VERSION=0.153.1 ARG TARGETARCH @@ -41,6 +42,12 @@ RUN mkdir kustomize && \ mv ./kustomize/kustomize /usr/local/bin/kustomize4 && \ rm -rf ./kustomize +RUN mkdir helmfile && \ + curl -s -L https://github.com/helmfile/helmfile/releases/download/v${HELMFILE_VERSION}/helmfile_${HELMFILE_VERSION}_linux_${TARGETARCH}.tar.gz|\ + tar xvz -C helmfile/ && \ + mv ./helmfile/helmfile /usr/local/bin/helmfile && \ + rm -rf ./helmfile + RUN adduser --system --uid 10111 --group spinnaker COPY rosco-web/build/install/rosco /opt/rosco COPY rosco-web/config /opt/rosco diff --git a/gradle.properties b/gradle.properties index 84c034302..5413edd78 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ -korkVersion=7.170.0 +korkVersion=7.182.1 org.gradle.parallel=true -spinnakerGradleVersion=8.26.0 +spinnakerGradleVersion=8.31.0 targetJava11=true # To enable a composite reference to a project, set the diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index e708b1c02..943f0cbfa 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 28ff446a2..508322917 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.8.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.1-bin.zip +networkTimeout=10000 zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index 4f906e0c8..65dcd68d6 100755 --- a/gradlew +++ b/gradlew @@ -1,7 +1,7 @@ -#!/usr/bin/env sh +#!/bin/sh # -# Copyright 2015 the original author or authors. +# Copyright © 2015-2021 the original authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,67 +17,101 @@ # ############################################################################## -## -## Gradle start up script for UN*X -## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# ############################################################################## # Attempt to set APP_HOME + # Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null -APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" +MAX_FD=maximum warn () { echo "$*" -} +} >&2 die () { echo echo "$*" echo exit 1 -} +} >&2 # OS specific support (must be 'true' or 'false'). cygwin=false msys=false darwin=false nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar @@ -87,9 +121,9 @@ CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACMD=$JAVA_HOME/jre/sh/java else - JAVACMD="$JAVA_HOME/bin/java" + JAVACMD=$JAVA_HOME/bin/java fi if [ ! -x "$JAVACMD" ] ; then die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME @@ -98,7 +132,7 @@ Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi else - JAVACMD="java" + JAVACMD=java which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the @@ -106,80 +140,105 @@ location of your Java installation." fi # Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac fi -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. # For Cygwin or MSYS, switch paths to Windows format before running java -if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) fi - i=`expr $i + 1` + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg done - case $i in - 0) set -- ;; - 1) set -- "$args0" ;; - 2) set -- "$args0" "$args1" ;; - 3) set -- "$args0" "$args1" "$args2" ;; - 4) set -- "$args0" "$args1" "$args2" "$args3" ;; - 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac fi -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=`save "$@"` +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat index ac1b06f93..6689b85be 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -14,7 +14,7 @@ @rem limitations under the License. @rem -@if "%DEBUG%" == "" @echo off +@if "%DEBUG%"=="" @echo off @rem ########################################################################## @rem @rem Gradle startup script for Windows @@ -25,7 +25,8 @@ if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @@ -40,7 +41,7 @@ if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto execute +if %ERRORLEVEL% equ 0 goto execute echo. echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. @@ -75,13 +76,15 @@ set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar :end @rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd +if %ERRORLEVEL% equ 0 goto mainEnd :fail rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% :mainEnd if "%OS%"=="Windows_NT" endlocal diff --git a/rosco-core/rosco-core.gradle b/rosco-core/rosco-core.gradle index d36d4b7ee..f73a397a0 100644 --- a/rosco-core/rosco-core.gradle +++ b/rosco-core/rosco-core.gradle @@ -11,14 +11,13 @@ dependencies { implementation "io.spinnaker.kork:kork-retrofit" implementation "io.spinnaker.kork:kork-swagger" implementation "io.spinnaker.kork:kork-web" - implementation "com.squareup.retrofit:converter-jackson" - implementation "com.squareup.retrofit:retrofit" + implementation "com.squareup.retrofit2:converter-jackson" + implementation "com.squareup.retrofit2:retrofit" implementation "io.reactivex:rxjava" implementation "org.apache.commons:commons-exec" implementation "org.codehaus.groovy:groovy" implementation "org.springframework.boot:spring-boot-starter-web" implementation "redis.clients:jedis" - implementation "com.jakewharton.retrofit:retrofit1-okhttp3-client" testImplementation "org.spockframework:spock-core" testImplementation "org.objenesis:objenesis" } diff --git a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/config/RoscoConfiguration.groovy b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/config/RoscoConfiguration.groovy index 9589ae6aa..8637a67c6 100644 --- a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/config/RoscoConfiguration.groovy +++ b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/config/RoscoConfiguration.groovy @@ -17,6 +17,7 @@ package com.netflix.spinnaker.rosco.config import com.netflix.spinnaker.config.PluginsAutoConfiguration +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStoreConfiguration; import com.netflix.spinnaker.kork.jedis.JedisClientDelegate import com.netflix.spinnaker.kork.jedis.RedisClientDelegate import com.netflix.spinnaker.rosco.executor.BakePoller @@ -39,7 +40,7 @@ import redis.clients.jedis.JedisPool @Configuration @CompileStatic @Slf4j -@Import(PluginsAutoConfiguration.class) +@Import([ PluginsAutoConfiguration.class, ArtifactStoreConfiguration.class ]) class RoscoConfiguration { @Bean @@ -88,5 +89,4 @@ class RoscoConfiguration { PackerCommandFactory localJobFriendlyPackerCommandFactory() { return new LocalJobFriendlyPackerCommandFactory() } - } diff --git a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/providers/aws/AWSBakeHandler.groovy b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/providers/aws/AWSBakeHandler.groovy index ff255d8f8..02b7e9ae1 100644 --- a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/providers/aws/AWSBakeHandler.groovy +++ b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/providers/aws/AWSBakeHandler.groovy @@ -18,6 +18,7 @@ package com.netflix.spinnaker.rosco.providers.aws import com.netflix.spinnaker.kork.artifacts.model.Artifact import com.netflix.spinnaker.kork.core.RetrySupport +import com.netflix.spinnaker.kork.retrofit.Retrofit2SyncCall import com.netflix.spinnaker.rosco.api.Bake import com.netflix.spinnaker.rosco.api.BakeOptions import com.netflix.spinnaker.rosco.api.BakeRequest @@ -26,6 +27,7 @@ import com.netflix.spinnaker.rosco.providers.CloudProviderBakeHandler import com.netflix.spinnaker.rosco.providers.aws.config.RoscoAWSConfiguration import com.netflix.spinnaker.rosco.providers.util.ImageNameFactory import com.netflix.spinnaker.rosco.services.ClouddriverService + import com.netflix.spinnaker.security.AuthenticatedRequest import groovy.util.logging.Slf4j import org.springframework.beans.factory.annotation.Autowired @@ -240,7 +242,7 @@ public class AWSBakeHandler extends CloudProviderBakeHandler { def images = AuthenticatedRequest.allowAnonymous( { retrySupport.retry({ - clouddriverService.findAmazonImageByName(name, account, region) + Retrofit2SyncCall.execute(clouddriverService.findAmazonImageByName(name, account, region)) }, 3, Duration.ofSeconds(3), false) } ) diff --git a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/services/ClouddriverService.java b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/services/ClouddriverService.java index 1a286590d..e1ccd6a76 100644 --- a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/services/ClouddriverService.java +++ b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/services/ClouddriverService.java @@ -1,19 +1,36 @@ +/* + * Copyright 2023 OpsMx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.netflix.spinnaker.rosco.services; import com.netflix.spinnaker.kork.artifacts.model.Artifact; import com.netflix.spinnaker.rosco.providers.aws.config.RoscoAWSConfiguration.AWSNamedImage; import java.util.List; -import retrofit.client.Response; -import retrofit.http.Body; -import retrofit.http.GET; -import retrofit.http.PUT; -import retrofit.http.Query; +import okhttp3.ResponseBody; +import retrofit2.Call; +import retrofit2.http.Body; +import retrofit2.http.GET; +import retrofit2.http.PUT; +import retrofit2.http.Query; public interface ClouddriverService { @PUT("/artifacts/fetch/") - Response fetchArtifact(@Body Artifact artifact); + Call fetchArtifact(@Body Artifact artifact); @GET("/aws/images/find") - List findAmazonImageByName( + Call> findAmazonImageByName( @Query("q") String name, @Query("account") String account, @Query("region") String region); } diff --git a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/services/ServiceConfig.java b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/services/ServiceConfig.java index 503926032..185173abe 100644 --- a/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/services/ServiceConfig.java +++ b/rosco-core/src/main/groovy/com/netflix/spinnaker/rosco/services/ServiceConfig.java @@ -18,30 +18,20 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import com.jakewharton.retrofit.Ok3Client; import com.netflix.spinnaker.config.OkHttp3ClientConfiguration; import com.netflix.spinnaker.kork.core.RetrySupport; -import com.netflix.spinnaker.kork.retrofit.exceptions.SpinnakerRetrofitErrorHandler; +import com.netflix.spinnaker.kork.retrofit.ErrorHandlingExecutorCallAdapterFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import retrofit.RequestInterceptor; -import retrofit.RestAdapter; -import retrofit.converter.JacksonConverter; +import retrofit2.Retrofit; +import retrofit2.converter.jackson.JacksonConverterFactory; @Configuration public class ServiceConfig { @Value("${services.clouddriver.base-url:http://localhost:7002}") String clouddriverBaseUrl; - @Value("${retrofit.log-level:BASIC}") - String retrofitLogLevel; - - @Bean - Ok3Client okClient(OkHttp3ClientConfiguration okHttpClientConfig) { - return new Ok3Client(okHttpClientConfig.create().build()); - } - @Bean RetrySupport retrySupport() { return new RetrySupport(); @@ -49,20 +39,21 @@ RetrySupport retrySupport() { // This should be service-agnostic if more integrations than clouddriver are used @Bean - ClouddriverService clouddriverService( - Ok3Client ok3Client, RequestInterceptor spinnakerRequestInterceptor) { + ClouddriverService clouddriverService(OkHttp3ClientConfiguration okHttpClientConfig) { + ObjectMapper objectMapper = new ObjectMapper() .enable(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL) .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); - return new RestAdapter.Builder() - .setEndpoint(clouddriverBaseUrl) - .setRequestInterceptor(spinnakerRequestInterceptor) - .setClient(ok3Client) - .setConverter(new JacksonConverter(objectMapper)) - .setLogLevel(RestAdapter.LogLevel.valueOf(retrofitLogLevel)) - .setErrorHandler(SpinnakerRetrofitErrorHandler.getInstance()) + /* + * ErrorHandlingExecutorCallAdapterFactory handles exceptions globally in retrofit2, similar to SpinnakerRetrofitErrorHandler with retrofit. + * */ + return new Retrofit.Builder() + .baseUrl(clouddriverBaseUrl) + .client(okHttpClientConfig.createForRetrofit2().build()) + .addCallAdapterFactory(ErrorHandlingExecutorCallAdapterFactory.getInstance()) + .addConverterFactory(JacksonConverterFactory.create(objectMapper)) .build() .create(ClouddriverService.class); } diff --git a/rosco-core/src/test/groovy/com/netflix/spinnaker/rosco/providers/aws/AWSBakeHandlerSpec.groovy b/rosco-core/src/test/groovy/com/netflix/spinnaker/rosco/providers/aws/AWSBakeHandlerSpec.groovy index c22feae09..ee6dd8f88 100644 --- a/rosco-core/src/test/groovy/com/netflix/spinnaker/rosco/providers/aws/AWSBakeHandlerSpec.groovy +++ b/rosco-core/src/test/groovy/com/netflix/spinnaker/rosco/providers/aws/AWSBakeHandlerSpec.groovy @@ -27,6 +27,9 @@ import com.netflix.spinnaker.rosco.providers.util.PackageNameConverter import com.netflix.spinnaker.rosco.providers.util.PackerCommandFactory import com.netflix.spinnaker.rosco.providers.util.TestDefaults import com.netflix.spinnaker.rosco.services.ClouddriverService + +import retrofit2.Call +import retrofit2.Response import spock.lang.Shared import spock.lang.Specification import spock.lang.Subject @@ -489,7 +492,25 @@ class AWSBakeHandlerSpec extends Specification implements TestDefaults { void 'looks up base ami id from name'() { def packerCommandFactoryMock = Mock(PackerCommandFactory) def imageNameFactoryMock = Mock(ImageNameFactory) - def clouddriverService = Mock(ClouddriverService) + def mockCall = Mock(Call.class) { + execute() >> { + return Response.success(200, [ + new RoscoAWSConfiguration.AWSNamedImage( + imageName: SOURCE_BIONIC_HVM_IMAGE_NAME, + attributes: new RoscoAWSConfiguration.AWSImageAttributes(virtualizationType: BakeRequest.VmType.hvm), + amis: [ + (REGION): [ SOURCE_BIONIC_HVM_IMAGE_ID ] + ] + ) + ]) + } + } + + def clouddriverService = Mock(ClouddriverService) { + findAmazonImageByName(_, _, _) >> { + return mockCall + } + } def bakeRequest = new BakeRequest(user: "someuser@gmail.com", package_name: PACKAGES_NAME, base_os: "bionic", @@ -511,16 +532,6 @@ class AWSBakeHandlerSpec extends Specification implements TestDefaults { def vmSettings = awsBakeHandler.findVirtualizationSettings(REGION, bakeRequest) then: - 1 * clouddriverService.findAmazonImageByName(_, _, _) >> [ - new RoscoAWSConfiguration.AWSNamedImage( - imageName: SOURCE_BIONIC_HVM_IMAGE_NAME, - attributes: new RoscoAWSConfiguration.AWSImageAttributes(virtualizationType: BakeRequest.VmType.hvm), - amis: [ - (REGION): [ SOURCE_BIONIC_HVM_IMAGE_ID ] - ] - ) - ] - vmSettings.sourceAmi == SOURCE_BIONIC_HVM_IMAGE_ID } @@ -568,9 +579,15 @@ class AWSBakeHandlerSpec extends Specification implements TestDefaults { void 'produces packer command with all required parameters for ubuntu, using ami lookup by name'() { setup: - def clouddriverService = Stub(ClouddriverService) { + def mockCall = Mock(Call.class) { + execute() >> { + return Response.success(200, searchByNameResults) + } + } + + def clouddriverService = Mock(ClouddriverService) { findAmazonImageByName(_, _, _) >> { - return searchByNameResults + return mockCall } } def packerCommandFactoryMock = Mock(PackerCommandFactory) diff --git a/rosco-manifests/rosco-manifests.gradle b/rosco-manifests/rosco-manifests.gradle index 22ad479ff..d135d0997 100644 --- a/rosco-manifests/rosco-manifests.gradle +++ b/rosco-manifests/rosco-manifests.gradle @@ -13,12 +13,13 @@ dependencies { implementation "org.apache.commons:commons-compress:1.21" implementation "org.yaml:snakeyaml" - implementation "com.squareup.retrofit:retrofit" + implementation "com.squareup.retrofit2:retrofit" testImplementation "org.assertj:assertj-core" testImplementation "org.junit.jupiter:junit-jupiter-api" testImplementation "org.junit.jupiter:junit-jupiter-params" - testImplementation "org.junit.platform:junit-platform-runner" testImplementation "org.mockito:mockito-core" testImplementation "org.spockframework:spock-core" testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine" + testImplementation "com.squareup.retrofit2:converter-jackson" + testCompileOnly "com.squareup.retrofit:retrofit" } diff --git a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/ArtifactDownloaderImpl.java b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/ArtifactDownloaderImpl.java index 133243be8..63fe3a35e 100644 --- a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/ArtifactDownloaderImpl.java +++ b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/ArtifactDownloaderImpl.java @@ -3,7 +3,7 @@ import com.netflix.spinnaker.kork.artifacts.model.Artifact; import com.netflix.spinnaker.kork.core.RetrySupport; import com.netflix.spinnaker.kork.exceptions.SpinnakerException; -import com.netflix.spinnaker.kork.retrofit.exceptions.SpinnakerHttpException; +import com.netflix.spinnaker.kork.retrofit.Retrofit2SyncCall; import com.netflix.spinnaker.rosco.services.ClouddriverService; import java.io.IOException; import java.io.InputStream; @@ -11,9 +11,9 @@ import java.nio.file.Files; import java.nio.file.Path; import lombok.extern.slf4j.Slf4j; +import okhttp3.ResponseBody; import org.apache.commons.io.IOUtils; import org.springframework.stereotype.Component; -import retrofit.client.Response; @Component @Slf4j @@ -26,12 +26,16 @@ public ArtifactDownloaderImpl(ClouddriverService clouddriverService) { } public InputStream downloadArtifact(Artifact artifact) throws IOException { - Response response = - retrySupport.retry(() -> clouddriverService.fetchArtifact(artifact), 5, 1000, true); - if (response.getBody() == null) { + ResponseBody response = + retrySupport.retry( + () -> Retrofit2SyncCall.execute(clouddriverService.fetchArtifact(artifact)), + 5, + 1000, + true); + if (response == null) { throw new IOException("Failure to fetch artifact: empty response"); } - return response.getBody().in(); + return response.byteStream(); } public void downloadArtifactToFile(Artifact artifact, Path targetFile) throws IOException { @@ -45,10 +49,8 @@ public void downloadArtifactToFile(Artifact artifact, Path targetFile) throws IO artifact, e.getMessage()), e); } - } catch (SpinnakerHttpException e) { - throw new SpinnakerHttpException(downloadFailureMessage(artifact, e), e); } catch (SpinnakerException e) { - throw new SpinnakerException(downloadFailureMessage(artifact, e), e); + throw e.newInstance(downloadFailureMessage(artifact, e)); } } diff --git a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/BakeManifestRequest.java b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/BakeManifestRequest.java index be5c8b67e..6a1eb36c8 100644 --- a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/BakeManifestRequest.java +++ b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/BakeManifestRequest.java @@ -17,6 +17,7 @@ public enum TemplateRenderer { HELM3, KUSTOMIZE, KUSTOMIZE4, + HELMFILE, CF; @JsonCreator diff --git a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/HelmBakeTemplateUtils.java b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/HelmBakeTemplateUtils.java new file mode 100644 index 000000000..5e1f99075 --- /dev/null +++ b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/HelmBakeTemplateUtils.java @@ -0,0 +1,106 @@ +/* + * Copyright 2023 Grab Holdings, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.rosco.manifests; + +import com.netflix.spinnaker.kork.artifacts.model.Artifact; +import com.netflix.spinnaker.kork.exceptions.SpinnakerException; +import com.netflix.spinnaker.kork.retrofit.exceptions.SpinnakerHttpException; +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public abstract class HelmBakeTemplateUtils { + private static final String MANIFEST_SEPARATOR = "---\n"; + private static final Pattern REGEX_TESTS_MANIFESTS = + Pattern.compile("# Source: .*/templates/tests/.*"); + + private final ArtifactDownloader artifactDownloader; + + protected HelmBakeTemplateUtils(ArtifactDownloader artifactDownloader) { + this.artifactDownloader = artifactDownloader; + } + + public ArtifactDownloader getArtifactDownloader() { + return artifactDownloader; + } + + public abstract String fetchFailureMessage(String description, Exception e); + + public String removeTestsDirectoryTemplates(String inputString) { + return Arrays.stream(inputString.split(MANIFEST_SEPARATOR)) + .filter(manifest -> !REGEX_TESTS_MANIFESTS.matcher(manifest).find()) + .collect(Collectors.joining(MANIFEST_SEPARATOR)); + } + + protected Path downloadArtifactToTmpFile(BakeManifestEnvironment env, Artifact artifact) + throws IOException { + String fileName = UUID.randomUUID().toString(); + Path targetPath = env.resolvePath(fileName); + artifactDownloader.downloadArtifactToFile(artifact, targetPath); + return targetPath; + } + + public abstract String getHelmExecutableForRequest(T request); + + protected List getValuePaths(List artifacts, BakeManifestEnvironment env) { + List valuePaths = new ArrayList<>(); + + try { + // not a stream to keep exception handling cleaner + for (Artifact valueArtifact : artifacts.subList(1, artifacts.size())) { + valuePaths.add(downloadArtifactToTmpFile(env, valueArtifact)); + } + } catch (SpinnakerHttpException e) { + throw new SpinnakerHttpException(fetchFailureMessage("values file", e), e); + } catch (IOException | SpinnakerException e) { + throw new IllegalStateException(fetchFailureMessage("values file", e), e); + } + + return valuePaths; + } + + protected Path getHelmTypePathFromArtifact( + BakeManifestEnvironment env, List inputArtifacts, String filePath) + throws IOException { + Path helmTypeFilePath; + + Artifact helmTypeTemplateArtifact = inputArtifacts.get(0); + String artifactType = Optional.ofNullable(helmTypeTemplateArtifact.getType()).orElse(""); + + if ("git/repo".equals(artifactType)) { + env.downloadArtifactTarballAndExtract(getArtifactDownloader(), helmTypeTemplateArtifact); + + helmTypeFilePath = env.resolvePath(Optional.ofNullable(filePath).orElse("")); + } else { + try { + helmTypeFilePath = downloadArtifactToTmpFile(env, helmTypeTemplateArtifact); + } catch (SpinnakerHttpException e) { + throw new SpinnakerHttpException(fetchFailureMessage("template", e), e); + } catch (IOException | SpinnakerException e) { + throw new IllegalStateException(fetchFailureMessage("template", e), e); + } + } + + return helmTypeFilePath; + } +} diff --git a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/config/RoscoHelmfileConfigurationProperties.java b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/config/RoscoHelmfileConfigurationProperties.java new file mode 100644 index 000000000..c9bd9fa93 --- /dev/null +++ b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/config/RoscoHelmfileConfigurationProperties.java @@ -0,0 +1,26 @@ +/* + * Copyright 2023 Grab Holdings, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.rosco.manifests.config; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@ConfigurationProperties("helmfile") +@Data +public class RoscoHelmfileConfigurationProperties { + private String executablePath = "helmfile"; +} diff --git a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helm/HelmTemplateUtils.java b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helm/HelmTemplateUtils.java index cd92fd3e8..f0d74e943 100644 --- a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helm/HelmTemplateUtils.java +++ b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helm/HelmTemplateUtils.java @@ -1,88 +1,63 @@ package com.netflix.spinnaker.rosco.manifests.helm; import com.netflix.spinnaker.kork.artifacts.model.Artifact; -import com.netflix.spinnaker.kork.exceptions.SpinnakerException; -import com.netflix.spinnaker.kork.retrofit.exceptions.SpinnakerHttpException; import com.netflix.spinnaker.rosco.jobs.BakeRecipe; import com.netflix.spinnaker.rosco.manifests.ArtifactDownloader; import com.netflix.spinnaker.rosco.manifests.BakeManifestEnvironment; import com.netflix.spinnaker.rosco.manifests.BakeManifestRequest; +import com.netflix.spinnaker.rosco.manifests.HelmBakeTemplateUtils; import com.netflix.spinnaker.rosco.manifests.config.RoscoHelmConfigurationProperties; import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.Optional; -import java.util.UUID; -import java.util.regex.Pattern; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; @Component @Slf4j -public class HelmTemplateUtils { - private static final String MANIFEST_SEPARATOR = "---\n"; - private static final Pattern REGEX_TESTS_MANIFESTS = - Pattern.compile("# Source: .*/templates/tests/.*"); - - private final ArtifactDownloader artifactDownloader; +public class HelmTemplateUtils extends HelmBakeTemplateUtils { private final RoscoHelmConfigurationProperties helmConfigurationProperties; public HelmTemplateUtils( ArtifactDownloader artifactDownloader, RoscoHelmConfigurationProperties helmConfigurationProperties) { - this.artifactDownloader = artifactDownloader; + super(artifactDownloader); this.helmConfigurationProperties = helmConfigurationProperties; } public BakeRecipe buildBakeRecipe(BakeManifestEnvironment env, HelmBakeManifestRequest request) throws IOException { - BakeRecipe result = new BakeRecipe(); - result.setName(request.getOutputName()); - Path templatePath; - List valuePaths = new ArrayList<>(); + List inputArtifacts = request.getInputArtifacts(); if (inputArtifacts == null || inputArtifacts.isEmpty()) { throw new IllegalArgumentException("At least one input artifact must be provided to bake"); } - Artifact helmTemplateArtifact = inputArtifacts.get(0); - String artifactType = Optional.ofNullable(helmTemplateArtifact.getType()).orElse(""); - if ("git/repo".equals(artifactType)) { - env.downloadArtifactTarballAndExtract(artifactDownloader, helmTemplateArtifact); - - log.info("helmChartFilePath: '{}'", request.getHelmChartFilePath()); - - // If there's no helm chart path specified, assume it lives in the root of - // the git/repo artifact. - templatePath = - env.resolvePath(Optional.ofNullable(request.getHelmChartFilePath()).orElse("")); - } else { - try { - templatePath = downloadArtifactToTmpFile(env, helmTemplateArtifact); - } catch (SpinnakerHttpException e) { - throw new SpinnakerHttpException(fetchFailureMessage("template", e), e); - } catch (IOException | SpinnakerException e) { - throw new IllegalStateException(fetchFailureMessage("template", e), e); - } - } + templatePath = getHelmTypePathFromArtifact(env, inputArtifacts, request.getHelmChartFilePath()); log.info("path to Chart.yaml: {}", templatePath); + return buildCommand(request, getValuePaths(inputArtifacts, env), templatePath); + } - try { - // not a stream to keep exception handling cleaner - for (Artifact valueArtifact : inputArtifacts.subList(1, inputArtifacts.size())) { - valuePaths.add(downloadArtifactToTmpFile(env, valueArtifact)); - } - } catch (SpinnakerHttpException e) { - throw new SpinnakerHttpException(fetchFailureMessage("values file", e), e); - } catch (IOException | SpinnakerException e) { - throw new IllegalStateException(fetchFailureMessage("values file", e), e); + public String fetchFailureMessage(String description, Exception e) { + return "Failed to fetch helm " + description + ": " + e.getMessage(); + } + + public String getHelmExecutableForRequest(HelmBakeManifestRequest request) { + if (BakeManifestRequest.TemplateRenderer.HELM2.equals(request.getTemplateRenderer())) { + return helmConfigurationProperties.getV2ExecutablePath(); } + return helmConfigurationProperties.getV3ExecutablePath(); + } + + public BakeRecipe buildCommand( + HelmBakeManifestRequest request, List valuePaths, Path templatePath) { + BakeRecipe result = new BakeRecipe(); + result.setName(request.getOutputName()); List command = new ArrayList<>(); String executable = getHelmExecutableForRequest(request); @@ -133,29 +108,4 @@ public BakeRecipe buildBakeRecipe(BakeManifestEnvironment env, HelmBakeManifestR return result; } - - private String fetchFailureMessage(String description, Exception e) { - return "Failed to fetch helm " + description + ": " + e.getMessage(); - } - - public String removeTestsDirectoryTemplates(String inputString) { - return Arrays.stream(inputString.split(MANIFEST_SEPARATOR)) - .filter(manifest -> !REGEX_TESTS_MANIFESTS.matcher(manifest).find()) - .collect(Collectors.joining(MANIFEST_SEPARATOR)); - } - - private Path downloadArtifactToTmpFile(BakeManifestEnvironment env, Artifact artifact) - throws IOException { - String fileName = UUID.randomUUID().toString(); - Path targetPath = env.resolvePath(fileName); - artifactDownloader.downloadArtifactToFile(artifact, targetPath); - return targetPath; - } - - private String getHelmExecutableForRequest(HelmBakeManifestRequest request) { - if (BakeManifestRequest.TemplateRenderer.HELM2.equals(request.getTemplateRenderer())) { - return helmConfigurationProperties.getV2ExecutablePath(); - } - return helmConfigurationProperties.getV3ExecutablePath(); - } } diff --git a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileBakeManifestRequest.java b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileBakeManifestRequest.java new file mode 100644 index 000000000..2367059d2 --- /dev/null +++ b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileBakeManifestRequest.java @@ -0,0 +1,50 @@ +/* + * Copyright 2023 Grab Holdings, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.rosco.manifests.helmfile; + +import com.netflix.spinnaker.kork.artifacts.model.Artifact; +import com.netflix.spinnaker.rosco.manifests.BakeManifestRequest; +import java.util.List; +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class HelmfileBakeManifestRequest extends BakeManifestRequest { + private String helmfileFilePath; + + /** + * The environment name used to customize the content of the helmfile manifest. The environment + * name defaults to default. + */ + private String environment; + + /** The namespace to be released into. */ + private String namespace; + + /** + * The 0th element is (or contains) the helmfile template. The rest (possibly none) are values + * files. + */ + List inputArtifacts; + + /** + * Include custom resource definition manifests in the templated output. Helmfile uses Helm v3 + * only which provides the option to include CRDs as part of the rendered output. + */ + boolean includeCRDs; +} diff --git a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileBakeManifestService.java b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileBakeManifestService.java new file mode 100644 index 000000000..a3061927a --- /dev/null +++ b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileBakeManifestService.java @@ -0,0 +1,65 @@ +/* + * Copyright 2023 Grab Holdings, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.rosco.manifests.helmfile; + +import static com.netflix.spinnaker.rosco.manifests.BakeManifestRequest.TemplateRenderer; + +import com.google.common.collect.ImmutableSet; +import com.netflix.spinnaker.kork.artifacts.model.Artifact; +import com.netflix.spinnaker.rosco.jobs.BakeRecipe; +import com.netflix.spinnaker.rosco.jobs.JobExecutor; +import com.netflix.spinnaker.rosco.manifests.BakeManifestEnvironment; +import com.netflix.spinnaker.rosco.manifests.BakeManifestService; +import java.io.IOException; +import java.util.Base64; +import org.springframework.stereotype.Component; + +@Component +public class HelmfileBakeManifestService extends BakeManifestService { + private final HelmfileTemplateUtils helmfileTemplateUtils; + private static final ImmutableSet supportedTemplates = + ImmutableSet.of(TemplateRenderer.HELMFILE.toString()); + + public HelmfileBakeManifestService( + HelmfileTemplateUtils helmTemplateUtils, JobExecutor jobExecutor) { + super(jobExecutor); + this.helmfileTemplateUtils = helmTemplateUtils; + } + + @Override + public Class requestType() { + return HelmfileBakeManifestRequest.class; + } + + @Override + public boolean handles(String type) { + return supportedTemplates.contains(type); + } + + public Artifact bake(HelmfileBakeManifestRequest helmfileBakeManifestRequest) throws IOException { + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, helmfileBakeManifestRequest); + + String bakeResult = helmfileTemplateUtils.removeTestsDirectoryTemplates(doBake(recipe)); + return Artifact.builder() + .type("embedded/base64") + .name(helmfileBakeManifestRequest.getOutputArtifactName()) + .reference(Base64.getEncoder().encodeToString(bakeResult.getBytes())) + .build(); + } + } +} diff --git a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileTemplateUtils.java b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileTemplateUtils.java new file mode 100644 index 000000000..3abb66c95 --- /dev/null +++ b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileTemplateUtils.java @@ -0,0 +1,125 @@ +/* + * Copyright 2023 Grab Holdings, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.rosco.manifests.helmfile; + +import com.netflix.spinnaker.kork.artifacts.model.Artifact; +import com.netflix.spinnaker.rosco.jobs.BakeRecipe; +import com.netflix.spinnaker.rosco.manifests.ArtifactDownloader; +import com.netflix.spinnaker.rosco.manifests.BakeManifestEnvironment; +import com.netflix.spinnaker.rosco.manifests.HelmBakeTemplateUtils; +import com.netflix.spinnaker.rosco.manifests.config.RoscoHelmConfigurationProperties; +import com.netflix.spinnaker.rosco.manifests.config.RoscoHelmfileConfigurationProperties; +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +@Component +@Slf4j +public class HelmfileTemplateUtils extends HelmBakeTemplateUtils { + private final RoscoHelmfileConfigurationProperties helmfileConfigurationProperties; + private final RoscoHelmConfigurationProperties helmConfigurationProperties = + new RoscoHelmConfigurationProperties(); + + public HelmfileTemplateUtils( + ArtifactDownloader artifactDownloader, + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties) { + super(artifactDownloader); + this.helmfileConfigurationProperties = helmfileConfigurationProperties; + } + + public BakeRecipe buildBakeRecipe( + BakeManifestEnvironment env, HelmfileBakeManifestRequest request) throws IOException { + Path helmfileFilePath; + + List inputArtifacts = request.getInputArtifacts(); + if (inputArtifacts == null || inputArtifacts.isEmpty()) { + throw new IllegalArgumentException("At least one input artifact must be provided to bake"); + } + + log.info("helmfileFilePath: '{}'", request.getHelmfileFilePath()); + helmfileFilePath = + getHelmTypePathFromArtifact(env, inputArtifacts, request.getHelmfileFilePath()); + + log.info("path to helmfile: {}", helmfileFilePath); + return buildCommand(request, getValuePaths(inputArtifacts, env), helmfileFilePath); + } + + public String fetchFailureMessage(String description, Exception e) { + return "Failed to fetch helmfile " + description + ": " + e.getMessage(); + } + + public String getHelmExecutableForRequest(HelmfileBakeManifestRequest request) { + return helmConfigurationProperties.getV3ExecutablePath(); + } + + public BakeRecipe buildCommand( + HelmfileBakeManifestRequest request, List valuePaths, Path helmfileFilePath) { + BakeRecipe result = new BakeRecipe(); + result.setName(request.getOutputName()); + + List command = new ArrayList<>(); + String executable = helmfileConfigurationProperties.getExecutablePath(); + + command.add(executable); + command.add("template"); + command.add("--file"); + command.add(helmfileFilePath.toString()); + + command.add("--helm-binary"); + command.add(getHelmExecutableForRequest(null)); + + String environment = request.getEnvironment(); + if (environment != null && !environment.isEmpty()) { + command.add("--environment"); + command.add(environment); + } + + String namespace = request.getNamespace(); + if (namespace != null && !namespace.isEmpty()) { + command.add("--namespace"); + command.add(namespace); + } + + if (request.isIncludeCRDs()) { + command.add("--include-crds"); + } + + Map overrides = request.getOverrides(); + if (!overrides.isEmpty()) { + List overrideList = new ArrayList<>(); + for (Map.Entry entry : overrides.entrySet()) { + overrideList.add(entry.getKey() + "=" + entry.getValue().toString()); + } + command.add("--set"); + command.add(String.join(",", overrideList)); + } + + if (!valuePaths.isEmpty()) { + command.add("--values"); + command.add(valuePaths.stream().map(Path::toString).collect(Collectors.joining(","))); + } + + result.setCommand(command); + + return result; + } +} diff --git a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/kustomize/KustomizationFileReader.java b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/kustomize/KustomizationFileReader.java index b0a1dbd42..f4d89302f 100644 --- a/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/kustomize/KustomizationFileReader.java +++ b/rosco-manifests/src/main/java/com/netflix/spinnaker/rosco/manifests/kustomize/KustomizationFileReader.java @@ -19,6 +19,7 @@ import com.google.common.collect.ImmutableList; import com.netflix.spinnaker.kork.artifacts.model.Artifact; import com.netflix.spinnaker.kork.core.RetrySupport; +import com.netflix.spinnaker.kork.retrofit.Retrofit2SyncCall; import com.netflix.spinnaker.rosco.manifests.kustomize.mapping.Kustomization; import com.netflix.spinnaker.rosco.services.ClouddriverService; import java.io.IOException; @@ -29,11 +30,11 @@ import java.util.*; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; +import okhttp3.ResponseBody; import org.springframework.stereotype.Component; import org.yaml.snakeyaml.Yaml; import org.yaml.snakeyaml.constructor.Constructor; import org.yaml.snakeyaml.representer.Representer; -import retrofit.client.Response; @Component @Slf4j @@ -95,8 +96,12 @@ private Kustomization convert(Artifact artifact) throws IOException { private InputStream downloadFile(Artifact artifact) throws IOException { log.info("downloading kustomization file {}", artifact.getReference()); - Response response = - retrySupport.retry(() -> clouddriverService.fetchArtifact(artifact), 5, 1000, true); - return response.getBody().in(); + ResponseBody response = + retrySupport.retry( + () -> Retrofit2SyncCall.execute(clouddriverService.fetchArtifact(artifact)), + 5, + 1000, + true); + return response.byteStream(); } } diff --git a/rosco-manifests/src/test/groovy/com/netflix/spinnaker/rosco/manifests/kustomize/KustomizationFileReaderSpec.groovy b/rosco-manifests/src/test/groovy/com/netflix/spinnaker/rosco/manifests/kustomize/KustomizationFileReaderSpec.groovy index 0292721c0..8513467c2 100644 --- a/rosco-manifests/src/test/groovy/com/netflix/spinnaker/rosco/manifests/kustomize/KustomizationFileReaderSpec.groovy +++ b/rosco-manifests/src/test/groovy/com/netflix/spinnaker/rosco/manifests/kustomize/KustomizationFileReaderSpec.groovy @@ -19,8 +19,10 @@ package com.netflix.spinnaker.rosco.manifests.kustomize import com.netflix.spinnaker.kork.artifacts.model.Artifact import com.netflix.spinnaker.rosco.manifests.kustomize.mapping.Kustomization import com.netflix.spinnaker.rosco.services.ClouddriverService -import retrofit.client.Response -import retrofit.mime.TypedString +import okhttp3.MediaType +import okhttp3.ResponseBody +import retrofit2.Call +import retrofit2.Response import spock.lang.Specification class KustomizationFileReaderSpec extends Specification { @@ -34,7 +36,16 @@ class KustomizationFileReaderSpec extends Specification { namePrefix: demo- """ - def clouddriverService = Mock(ClouddriverService) + def mockCall = Mock(Call.class) { + execute() >> { + return Response.success(200, ResponseBody.create(MediaType.parse("text/plain"), kustomizationYaml.stripMargin())); + } + } + def clouddriverService = Mock(ClouddriverService) { + fetchArtifact(_) >> { + return mockCall + } + } def kustomizationFileReader = new KustomizationFileReader(clouddriverService) def baseArtifact = Artifact.builder() .name("base") @@ -47,18 +58,20 @@ class KustomizationFileReaderSpec extends Specification { Kustomization k = kustomizationFileReader.getKustomization(baseArtifact,"kustomization.yml") then: - 1 * clouddriverService.fetchArtifact(_ as Artifact) >> { Artifact a -> - return new Response("test", 200, "", [], new TypedString(kustomizationYaml.stripMargin())) - } k.getResources().sort() == ["deployment.yml", "service.yml"].sort() k.getSelfReference() == "https://api.github.com/repos/org/repo/contents/base/kustomization.yml" } def "getKustomization throws an exception if it can't find a valid kustomization file"() { given: + def mockCall = Mock(Call.class) { + execute() >> { + return Response.error(500, ResponseBody.create(null, "")) + } + } def clouddriverService = Mock(ClouddriverService) { fetchArtifact(_) >> { - return new Response("test", 500, "", [], null) + return mockCall } } def kustomizationFileReader = new KustomizationFileReader(clouddriverService) diff --git a/rosco-manifests/src/test/java/com/netflix/spinnaker/rosco/manifests/ArtifactDownloaderImplTest.java b/rosco-manifests/src/test/java/com/netflix/spinnaker/rosco/manifests/ArtifactDownloaderImplTest.java index caf389f0c..041ad76b4 100644 --- a/rosco-manifests/src/test/java/com/netflix/spinnaker/rosco/manifests/ArtifactDownloaderImplTest.java +++ b/rosco-manifests/src/test/java/com/netflix/spinnaker/rosco/manifests/ArtifactDownloaderImplTest.java @@ -22,25 +22,22 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import com.google.common.collect.ImmutableList; import com.netflix.spinnaker.kork.artifacts.model.Artifact; import com.netflix.spinnaker.kork.exceptions.SpinnakerException; import com.netflix.spinnaker.kork.retrofit.exceptions.SpinnakerHttpException; +import com.netflix.spinnaker.kork.retrofit.exceptions.SpinnakerNetworkException; import com.netflix.spinnaker.rosco.services.ClouddriverService; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import okhttp3.ResponseBody; import org.junit.jupiter.api.Test; -import org.junit.platform.runner.JUnitPlatform; -import org.junit.runner.RunWith; -import org.springframework.http.HttpStatus; -import retrofit.RetrofitError; -import retrofit.client.Response; -import retrofit.mime.TypedByteArray; - -@RunWith(JUnitPlatform.class) +import retrofit2.Call; +import retrofit2.Response; + final class ArtifactDownloaderImplTest { private final ClouddriverService clouddriverService = mock(ClouddriverService.class); + private final Call mockCall = mock(Call.class); private static final Artifact testArtifact = Artifact.builder().name("test-artifact").version("3").build(); @@ -50,8 +47,8 @@ public void downloadsArtifactContent() throws IOException { String testContent = "abcdefg"; try (ArtifactDownloaderImplTest.AutoDeletingFile file = new AutoDeletingFile()) { - when(clouddriverService.fetchArtifact(testArtifact)) - .thenReturn(successfulResponse(testContent)); + when(clouddriverService.fetchArtifact(testArtifact)).thenReturn(mockCall); + when(mockCall.execute()).thenReturn(successfulResponse(testContent)); artifactDownloader.downloadArtifactToFile(testArtifact, file.path); assertThat(file.path).hasContent(testContent); @@ -62,10 +59,10 @@ public void downloadsArtifactContent() throws IOException { public void retries() throws IOException { ArtifactDownloaderImpl artifactDownloader = new ArtifactDownloaderImpl(clouddriverService); String testContent = "abcdefg"; - try (ArtifactDownloaderImplTest.AutoDeletingFile file = new AutoDeletingFile()) { - when(clouddriverService.fetchArtifact(testArtifact)) - .thenThrow(RetrofitError.networkError("", new IOException("timeout"))) + when(clouddriverService.fetchArtifact(testArtifact)).thenReturn(mockCall); + when(mockCall.execute()) + .thenThrow(new SpinnakerNetworkException(new IOException("timeout"))) .thenReturn(successfulResponse(testContent)); artifactDownloader.downloadArtifactToFile(testArtifact, file.path); @@ -78,8 +75,8 @@ public void exceptionDownloadingArtifactContent() throws IOException { ArtifactDownloaderImpl artifactDownloader = new ArtifactDownloaderImpl(clouddriverService); SpinnakerException spinnakerException = new SpinnakerException("error from clouddriver"); try (ArtifactDownloaderImplTest.AutoDeletingFile file = new AutoDeletingFile()) { - when(clouddriverService.fetchArtifact(testArtifact)).thenThrow(spinnakerException); - + when(clouddriverService.fetchArtifact(testArtifact)).thenReturn(mockCall); + when(mockCall.execute()).thenThrow(spinnakerException); SpinnakerException thrown = assertThrows( SpinnakerException.class, @@ -99,7 +96,8 @@ public void artifactNotFound() throws IOException { ArtifactDownloaderImpl artifactDownloader = new ArtifactDownloaderImpl(clouddriverService); SpinnakerHttpException spinnakerHttpException = makeSpinnakerHttpException(404); try (ArtifactDownloaderImplTest.AutoDeletingFile file = new AutoDeletingFile()) { - when(clouddriverService.fetchArtifact(testArtifact)).thenThrow(spinnakerHttpException); + when(clouddriverService.fetchArtifact(testArtifact)).thenReturn(mockCall); + when(mockCall.execute()).thenThrow(spinnakerHttpException); // Make sure that the exception from artifactDownloader is also a // SpinnakerHttpException with 404 status since that'll make rosco @@ -122,12 +120,7 @@ public void artifactNotFound() throws IOException { } private Response successfulResponse(String content) { - return new Response( - "", - HttpStatus.OK.value(), - "", - ImmutableList.of(), - new TypedByteArray(null, content.getBytes())); + return Response.success(200, ResponseBody.create(null, content.getBytes())); } private static class AutoDeletingFile implements AutoCloseable { diff --git a/rosco-manifests/src/test/java/com/netflix/spinnaker/rosco/manifests/BakeManifestEnvironmentTest.java b/rosco-manifests/src/test/java/com/netflix/spinnaker/rosco/manifests/BakeManifestEnvironmentTest.java index 205039c00..69766d435 100644 --- a/rosco-manifests/src/test/java/com/netflix/spinnaker/rosco/manifests/BakeManifestEnvironmentTest.java +++ b/rosco-manifests/src/test/java/com/netflix/spinnaker/rosco/manifests/BakeManifestEnvironmentTest.java @@ -23,10 +23,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import org.junit.jupiter.api.Test; -import org.junit.platform.runner.JUnitPlatform; -import org.junit.runner.RunWith; -@RunWith(JUnitPlatform.class) final class BakeManifestEnvironmentTest { @Test void rejectsInvalidPaths() throws IOException { diff --git a/rosco-manifests/src/test/java/com/netflix/spinnaker/rosco/manifests/ManifestTestUtils.java b/rosco-manifests/src/test/java/com/netflix/spinnaker/rosco/manifests/ManifestTestUtils.java index 7ffee330d..3946cc1f2 100644 --- a/rosco-manifests/src/test/java/com/netflix/spinnaker/rosco/manifests/ManifestTestUtils.java +++ b/rosco-manifests/src/test/java/com/netflix/spinnaker/rosco/manifests/ManifestTestUtils.java @@ -16,13 +16,12 @@ package com.netflix.spinnaker.rosco.manifests; -import com.google.gson.Gson; +import com.netflix.spinnaker.kork.retrofit.exceptions.RetrofitException; import com.netflix.spinnaker.kork.retrofit.exceptions.SpinnakerHttpException; -import java.util.List; -import retrofit.RetrofitError; -import retrofit.client.Response; -import retrofit.converter.GsonConverter; -import retrofit.mime.TypedString; +import okhttp3.MediaType; +import okhttp3.ResponseBody; +import retrofit2.Retrofit; +import retrofit2.converter.jackson.JacksonConverterFactory; public class ManifestTestUtils { @@ -35,17 +34,20 @@ public static SpinnakerHttpException makeSpinnakerHttpException(int status) { // would be sufficient, except in the chained case, where the return value // of this method is the cause of a real SpinnakerHttpException object. // There, getResponseCode needs a real underlying response, at least real - // enough for response.getStatus() to work. So, go ahead and build one. + // enough for retrofit2Response.code() to work. So, go ahead and build one. String url = "https://some-url"; - Response response = - new Response( - url, + retrofit2.Response retrofit2Response = + retrofit2.Response.error( status, - "arbitrary reason", - List.of(), - new TypedString("{ message: \"arbitrary message\" }")); + ResponseBody.create( + MediaType.parse("application/json"), "{ \"message\": \"arbitrary message\" }")); - return new SpinnakerHttpException( - RetrofitError.httpError(url, response, new GsonConverter(new Gson()), String.class)); + Retrofit retrofit = + new Retrofit.Builder() + .baseUrl(url) + .addConverterFactory(JacksonConverterFactory.create()) + .build(); + + return new SpinnakerHttpException(RetrofitException.httpError(retrofit2Response, retrofit)); } } diff --git a/rosco-manifests/src/test/java/com/netflix/spinnaker/rosco/manifests/cloudfoundry/CloudFoundryBakeManifestServiceTest.java b/rosco-manifests/src/test/java/com/netflix/spinnaker/rosco/manifests/cloudfoundry/CloudFoundryBakeManifestServiceTest.java index 5c9c89b43..0be8d2bee 100644 --- a/rosco-manifests/src/test/java/com/netflix/spinnaker/rosco/manifests/cloudfoundry/CloudFoundryBakeManifestServiceTest.java +++ b/rosco-manifests/src/test/java/com/netflix/spinnaker/rosco/manifests/cloudfoundry/CloudFoundryBakeManifestServiceTest.java @@ -31,10 +31,7 @@ import java.io.InputStream; import java.util.Base64; import org.junit.jupiter.api.Test; -import org.junit.platform.runner.JUnitPlatform; -import org.junit.runner.RunWith; -@RunWith(JUnitPlatform.class) public class CloudFoundryBakeManifestServiceTest { private ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); diff --git a/rosco-manifests/src/test/java/com/netflix/spinnaker/rosco/manifests/helm/HelmTemplateUtilsTest.java b/rosco-manifests/src/test/java/com/netflix/spinnaker/rosco/manifests/helm/HelmTemplateUtilsTest.java index f88de02fd..1a6eb41e9 100644 --- a/rosco-manifests/src/test/java/com/netflix/spinnaker/rosco/manifests/helm/HelmTemplateUtilsTest.java +++ b/rosco-manifests/src/test/java/com/netflix/spinnaker/rosco/manifests/helm/HelmTemplateUtilsTest.java @@ -18,10 +18,10 @@ import static com.netflix.spinnaker.rosco.manifests.ManifestTestUtils.makeSpinnakerHttpException; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -60,11 +60,8 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.junit.platform.runner.JUnitPlatform; -import org.junit.runner.RunWith; import org.springframework.http.HttpStatus; -@RunWith(JUnitPlatform.class) final class HelmTemplateUtilsTest { private ArtifactDownloader artifactDownloader; diff --git a/rosco-manifests/src/test/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileTemplateUtilsTest.java b/rosco-manifests/src/test/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileTemplateUtilsTest.java new file mode 100644 index 000000000..a6119e6be --- /dev/null +++ b/rosco-manifests/src/test/java/com/netflix/spinnaker/rosco/manifests/helmfile/HelmfileTemplateUtilsTest.java @@ -0,0 +1,671 @@ +/* + * Copyright 2023 Grab Holdings, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.rosco.manifests.helmfile; + +import static com.netflix.spinnaker.rosco.manifests.ManifestTestUtils.makeSpinnakerHttpException; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.netflix.spinnaker.kork.artifacts.model.Artifact; +import com.netflix.spinnaker.kork.exceptions.SpinnakerException; +import com.netflix.spinnaker.kork.retrofit.exceptions.SpinnakerHttpException; +import com.netflix.spinnaker.rosco.jobs.BakeRecipe; +import com.netflix.spinnaker.rosco.manifests.ArtifactDownloader; +import com.netflix.spinnaker.rosco.manifests.BakeManifestEnvironment; +import com.netflix.spinnaker.rosco.manifests.BakeManifestRequest; +import com.netflix.spinnaker.rosco.manifests.config.RoscoHelmConfigurationProperties; +import com.netflix.spinnaker.rosco.manifests.config.RoscoHelmfileConfigurationProperties; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.FileVisitOption; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; +import org.apache.commons.compress.utils.IOUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.springframework.http.HttpStatus; + +final class HelmfileTemplateUtilsTest { + + private ArtifactDownloader artifactDownloader; + + private HelmfileTemplateUtils helmfileTemplateUtils; + + private HelmfileBakeManifestRequest bakeManifestRequest; + + @BeforeEach + private void init(TestInfo testInfo) { + System.out.println("--------------- Test " + testInfo.getDisplayName()); + + artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + Artifact chartArtifact = Artifact.builder().name("test-artifact").version("3").build(); + + bakeManifestRequest = new HelmfileBakeManifestRequest(); + bakeManifestRequest.setInputArtifacts(ImmutableList.of(chartArtifact)); + } + + @Test + public void nullReferenceTest() throws IOException { + bakeManifestRequest.setOverrides(ImmutableMap.of()); + + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, bakeManifestRequest); + } + } + + @Test + public void exceptionDownloading() throws IOException { + // When artifactDownloader throws an exception, make sure we wrap it and get + // a chance to include our own message, so the exception that goes up the + // chain includes something about helmfile. + SpinnakerException spinnakerException = new SpinnakerException("error from ArtifactDownloader"); + doThrow(spinnakerException) + .when(artifactDownloader) + .downloadArtifactToFile(any(Artifact.class), any(Path.class)); + + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + IllegalStateException thrown = + assertThrows( + IllegalStateException.class, + () -> helmfileTemplateUtils.buildBakeRecipe(env, bakeManifestRequest)); + + assertThat(thrown.getMessage()).contains("Failed to fetch helmfile template"); + assertThat(thrown.getCause()).isEqualTo(spinnakerException); + } + } + + @Test + public void httpExceptionDownloading() throws IOException { + // When artifactDownloader throws a SpinnakerHttpException, make sure we + // wrap it and get a chance to include our own message, so the exception + // that goes up the chain includes something about helm charts. It's + // important that HelmTemplateUtils also throws a SpinnakerHttpException so + // it's eventually handled properly...meaning the status code in the http + // response and the logging correspond to what happened. For example, if + // there's a 404 from clouddriver, rosco also responds with 404, and doesn't + // log an error. + + SpinnakerHttpException spinnakerHttpException = + makeSpinnakerHttpException(HttpStatus.NOT_FOUND.value()); + doThrow(spinnakerHttpException) + .when(artifactDownloader) + .downloadArtifactToFile(any(Artifact.class), any(Path.class)); + + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + SpinnakerHttpException thrown = + assertThrows( + SpinnakerHttpException.class, + () -> helmfileTemplateUtils.buildBakeRecipe(env, bakeManifestRequest)); + + assertThat(thrown.getMessage()).contains("Failed to fetch helmfile template"); + assertThat(thrown.getResponseCode()).isEqualTo(HttpStatus.NOT_FOUND.value()); + assertThat(thrown.getCause()).isEqualTo(spinnakerHttpException); + } + } + + @Test + public void removeTestsDirectoryTemplatesWithTests() throws IOException { + String inputManifests = + "---\n" + + "# Source: mysql/templates/pvc.yaml\n" + + "\n" + + "kind: PersistentVolumeClaim\n" + + "apiVersion: v1\n" + + "metadata:\n" + + " name: release-name-mysql\n" + + " namespace: default\n" + + "spec:\n" + + " accessModes:\n" + + " - \"ReadWriteOnce\"\n" + + " resources:\n" + + " requests:\n" + + " storage: \"8Gi\"\n" + + "---\n" + + "# Source: mysql/templates/tests/test-configmap.yaml\n" + + "apiVersion: v1\n" + + "kind: ConfigMap\n" + + "metadata:\n" + + " name: release-name-mysql-test\n" + + " namespace: default\n" + + "data:\n" + + " run.sh: |-\n"; + + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + + String output = helmfileTemplateUtils.removeTestsDirectoryTemplates(inputManifests); + + String expected = + "---\n" + + "# Source: mysql/templates/pvc.yaml\n" + + "\n" + + "kind: PersistentVolumeClaim\n" + + "apiVersion: v1\n" + + "metadata:\n" + + " name: release-name-mysql\n" + + " namespace: default\n" + + "spec:\n" + + " accessModes:\n" + + " - \"ReadWriteOnce\"\n" + + " resources:\n" + + " requests:\n" + + " storage: \"8Gi\"\n"; + + assertEquals(expected.trim(), output.trim()); + } + + @Test + public void removeTestsDirectoryTemplatesWithoutTests() throws IOException { + String inputManifests = + "---\n" + + "# Source: mysql/templates/pvc.yaml\n" + + "\n" + + "kind: PersistentVolumeClaim\n" + + "apiVersion: v1\n" + + "metadata:\n" + + " name: release-name-mysql\n" + + " namespace: default\n" + + "spec:\n" + + " accessModes:\n" + + " - \"ReadWriteOnce\"\n" + + " resources:\n" + + " requests:\n" + + " storage: \"8Gi\"\n" + + "---\n" + + "# Source: mysql/templates/configmap.yaml\n" + + "apiVersion: v1\n" + + "kind: ConfigMap\n" + + "metadata:\n" + + " name: release-name-mysql-test\n" + + " namespace: default\n" + + "data:\n" + + " run.sh: |-\n"; + + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + + String output = helmfileTemplateUtils.removeTestsDirectoryTemplates(inputManifests); + + assertEquals(inputManifests.trim(), output.trim()); + } + + @ParameterizedTest + @MethodSource("helmfileRendererArgs") + public void buildBakeRecipeSelectsHelm3ExecutableWhenNoneSet( + String command, BakeManifestRequest.TemplateRenderer templateRenderer) throws IOException { + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + RoscoHelmConfigurationProperties helmConfigurationProperties = + new RoscoHelmConfigurationProperties(); + + HelmfileBakeManifestRequest request = new HelmfileBakeManifestRequest(); + Artifact artifact = Artifact.builder().build(); + request.setInputArtifacts(Collections.singletonList(artifact)); + request.setOverrides(Collections.emptyMap()); + + request.setTemplateRenderer(templateRenderer); + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, request); + + assertEquals(helmConfigurationProperties.getV3ExecutablePath(), recipe.getCommand().get(5)); + } + } + + private static Stream helmfileRendererArgs() { + // The command here (e.g. helmfile) must match the defaults in + // RoscoHelmfileConfigurationProperties + return Stream.of(Arguments.of("helmfile", BakeManifestRequest.TemplateRenderer.HELMFILE)); + } + + @Test + public void buildBakeRecipeWithGitRepoArtifact(@TempDir Path tempDir) throws IOException { + // git/repo artifacts appear as a tarball, so create one that contains a + // helmfile file + // and helm chart. + addTestHelmfile(tempDir); + + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + + HelmfileBakeManifestRequest request = new HelmfileBakeManifestRequest(); + + Artifact artifact = + Artifact.builder().type("git/repo").reference("https://github.com/some/repo.git").build(); + + // Set up the mock artifactDownloader to supply the tarball that represents + // the git/repo artifact + when(artifactDownloader.downloadArtifact(artifact)).thenReturn(makeTarball(tempDir)); + + request.setInputArtifacts(Collections.singletonList(artifact)); + request.setOverrides(Collections.emptyMap()); + + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, request); + + // Make sure we're really testing the git/repo logic + verify(artifactDownloader).downloadArtifact(artifact); + + // Make sure the BakeManifestEnvironment has the files in our git/repo artifact. + assertTrue(env.resolvePath("helmfile.yaml").toFile().exists()); + assertTrue(env.resolvePath("Chart.yaml").toFile().exists()); + assertTrue(env.resolvePath("values.yaml").toFile().exists()); + assertTrue(env.resolvePath("templates/foo.yaml").toFile().exists()); + } + } + + @Test + public void buildBakeRecipeWithGitRepoArtifactUsingHelmfileFilePath(@TempDir Path tempDir) + throws IOException { + // Create a tarball with a helmfile in a sub directory + String subDirName = "subdir"; + Path subDir = tempDir.resolve(subDirName); + addTestHelmfile(subDir); + + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + + HelmfileBakeManifestRequest request = new HelmfileBakeManifestRequest(); + + // Note that supplying a location for a git/repo artifact doesn't change the + // path in the resulting tarball. It's here because it's likely that it's + // used together with helmChartFilePath. Removing it wouldn't change the + // test. + Artifact artifact = + Artifact.builder() + .type("git/repo") + .reference("https://github.com/some/repo.git") + .location(subDirName) + .build(); + + // Set up the mock artifactDownloader to supply the tarball that represents + // the git/repo artifact + when(artifactDownloader.downloadArtifact(artifact)).thenReturn(makeTarball(tempDir)); + + request.setInputArtifacts(Collections.singletonList(artifact)); + request.setOverrides(Collections.emptyMap()); + + // This is the key part of this test. + request.setHelmfileFilePath(subDirName); + + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, request); + + // Make sure we're really testing the git/repo logic + verify(artifactDownloader).downloadArtifact(artifact); + + // Make sure the BakeManifestEnvironment has the files in our git/repo + // artifact in the expected location. + assertTrue(env.resolvePath(Path.of(subDirName, "helmfile.yaml")).toFile().exists()); + assertTrue(env.resolvePath(Path.of(subDirName, "Chart.yaml")).toFile().exists()); + assertTrue(env.resolvePath(Path.of(subDirName, "values.yaml")).toFile().exists()); + assertTrue(env.resolvePath(Path.of(subDirName, "templates/foo.yaml")).toFile().exists()); + + // And that the helm template command includes the path to the subdirectory + // + // The expected elements in the + // command list are: + // + // 0 - the helm executable + // 1 - template + // 2 - --file flag + // 3 - the path to helmfile.yaml + assertEquals(env.resolvePath(subDirName).toString(), recipe.getCommand().get(3)); + } + } + + /** + * Add a helmfile and helm chart for testing + * + * @param path the location of the helmfile file (e.g. helmfile.yaml) + */ + void addTestHelmfile(Path path) throws IOException { + addFile( + path, + "helmfile.yaml", + "releases:\n" + + " - name: test\n" + + " namespace: namespace\n" + + " chart: Chart.yaml\n" + + " values:\n" + + " - values.yaml\n"); + + addFile( + path, + "Chart.yaml", + "apiVersion: v1\n" + + "name: example\n" + + "description: chart for testing\n" + + "version: 0.1\n" + + "engine: gotpl\n"); + + addFile(path, "values.yaml", "foo: bar\n"); + + addFile( + path, + "templates/foo.yaml", + "labels:\n" + "helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}\n"); + } + + /** + * Create a new file in the temp directory + * + * @param path the path of the file to create (relative to the temp directory's root) + * @param content the content of the file, or null for an empty file + */ + void addFile(Path tempDir, String path, String content) throws IOException { + Path pathToCreate = tempDir.resolve(path); + pathToCreate.toFile().getParentFile().mkdirs(); + Files.write(pathToCreate, content.getBytes()); + } + + /** + * Make a gzipped tarball of all files in a path + * + * @param rootPath the root path of the tarball + * @return an InputStream containing the gzipped tarball + */ + InputStream makeTarball(Path rootPath) throws IOException { + ArrayList filePathsToAdd = + Files.walk(rootPath, FileVisitOption.FOLLOW_LINKS) + .filter(path -> !path.equals(rootPath)) + .collect(Collectors.toCollection(ArrayList::new)); + + // See + // https://commons.apache.org/proper/commons-compress/examples.html#Common_Archival_Logic + // for background + try (ByteArrayOutputStream os = new ByteArrayOutputStream(); + GzipCompressorOutputStream gzo = new GzipCompressorOutputStream(os); + TarArchiveOutputStream tarArchive = new TarArchiveOutputStream(gzo)) { + for (Path path : filePathsToAdd) { + TarArchiveEntry tarEntry = + new TarArchiveEntry(path.toFile(), rootPath.relativize(path).toString()); + tarArchive.setBigNumberMode(tarArchive.BIGNUMBER_POSIX); + tarArchive.setLongFileMode(tarArchive.LONGFILE_POSIX); + tarArchive.putArchiveEntry(tarEntry); + if (path.toFile().isFile()) { + IOUtils.copy(Files.newInputStream(path), tarArchive); + } + tarArchive.closeArchiveEntry(); + } + + tarArchive.finish(); + gzo.finish(); + + return new ByteArrayInputStream(os.toByteArray()); + } + } + + @Test + public void buildBakeRecipeIncludesEnvironmentWhenSet() throws IOException { + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + + HelmfileBakeManifestRequest request = new HelmfileBakeManifestRequest(); + Artifact artifact = Artifact.builder().build(); + String envName = "testEnvironment"; + request.setEnvironment(envName); + request.setInputArtifacts(Collections.singletonList(artifact)); + request.setOverrides(Collections.emptyMap()); + + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, request); + assertTrue(recipe.getCommand().contains("--environment")); + assertTrue(recipe.getCommand().contains(envName)); + // Assert that the flag position goes after 'helmfile template' subcommand + assertTrue(recipe.getCommand().indexOf("--environment") > 1); + } + } + + @Test + public void buildBakeRecipeDoesNotIncludeEnvironmentWhenNotSet() throws IOException { + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + + HelmfileBakeManifestRequest request = new HelmfileBakeManifestRequest(); + Artifact artifact = Artifact.builder().build(); + request.setInputArtifacts(Collections.singletonList(artifact)); + request.setOverrides(Collections.emptyMap()); + + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, request); + assertFalse(recipe.getCommand().contains("--environment")); + } + } + + @Test + public void buildBakeRecipeIncludesNamespaceWhenSet() throws IOException { + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + + HelmfileBakeManifestRequest request = new HelmfileBakeManifestRequest(); + Artifact artifact = Artifact.builder().build(); + String namespaceName = "testNamespace"; + request.setNamespace(namespaceName); + request.setInputArtifacts(Collections.singletonList(artifact)); + request.setOverrides(Collections.emptyMap()); + + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, request); + assertTrue(recipe.getCommand().contains("--namespace")); + assertTrue(recipe.getCommand().contains(namespaceName)); + // Assert that the flag position goes after 'helmfile template' subcommand + assertTrue(recipe.getCommand().indexOf("--namespace") > 1); + } + } + + @Test + public void buildBakeRecipeDoesNotIncludeNamespaceWhenNotSet() throws IOException { + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + + HelmfileBakeManifestRequest request = new HelmfileBakeManifestRequest(); + Artifact artifact = Artifact.builder().build(); + request.setInputArtifacts(Collections.singletonList(artifact)); + request.setOverrides(Collections.emptyMap()); + + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, request); + assertFalse(recipe.getCommand().contains("--namespace")); + } + } + + @Test + public void buildBakeRecipeIncludingCRDsWithHelm3() throws IOException { + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + + HelmfileBakeManifestRequest request = new HelmfileBakeManifestRequest(); + Artifact artifact = Artifact.builder().build(); + request.setIncludeCRDs(true); + request.setInputArtifacts(Collections.singletonList(artifact)); + request.setOverrides(Collections.emptyMap()); + + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, request); + assertTrue(recipe.getCommand().contains("--include-crds")); + // Assert that the flag position goes after 'helmfile template' subcommand + assertTrue(recipe.getCommand().indexOf("--include-crds") > 1); + } + } + + @Test + public void buildBakeRecipeNotIncludingCRDsWithHelm3() throws IOException { + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + + HelmfileBakeManifestRequest request = new HelmfileBakeManifestRequest(); + Artifact artifact = Artifact.builder().build(); + request.setInputArtifacts(Collections.singletonList(artifact)); + request.setOverrides(Collections.emptyMap()); + + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, request); + assertFalse(recipe.getCommand().contains("--include-crds")); + } + } + + @Test + public void buildBakeRecipeIncludesOverridesWhenSet() throws IOException { + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + + HelmfileBakeManifestRequest request = new HelmfileBakeManifestRequest(); + Artifact artifact = Artifact.builder().build(); + String overrideKey = "testOverrideKey"; + String overrideValue = "testOverrideValue"; + request.setInputArtifacts(Collections.singletonList(artifact)); + request.setOverrides(Collections.singletonMap(overrideKey, overrideValue)); + + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, request); + assertTrue(recipe.getCommand().contains("--set")); + assertTrue(recipe.getCommand().contains(overrideKey + "=" + overrideValue)); + // Assert that the flag position goes after 'helmfile template' subcommand + assertTrue(recipe.getCommand().indexOf("--set") > 1); + } + } + + @Test + public void buildBakeRecipeDoesNotIncludeOverridesWhenNotSet() throws IOException { + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + + HelmfileBakeManifestRequest request = new HelmfileBakeManifestRequest(); + Artifact artifact = Artifact.builder().build(); + request.setInputArtifacts(Collections.singletonList(artifact)); + request.setOverrides(Collections.emptyMap()); + + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, request); + assertFalse(recipe.getCommand().contains("--set")); + } + } + + @Test + public void buildBakeRecipeIncludesValuesWhenSet() throws IOException { + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + + HelmfileBakeManifestRequest request = new HelmfileBakeManifestRequest(); + + List artifacts = new ArrayList<>(); + Artifact artifact = Artifact.builder().build(); + Artifact testValue = Artifact.builder().build(); + artifacts.add(artifact); + artifacts.add(testValue); + + request.setInputArtifacts(artifacts); + request.setOverrides(Collections.emptyMap()); + + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, request); + assertTrue(recipe.getCommand().contains("--values")); + // Assert that the flag position goes after 'helmfile template' subcommand + assertTrue(recipe.getCommand().indexOf("--values") > 1); + } + } + + @Test + public void buildBakeRecipeDoesNotIncludeValuesWhenNotSet() throws IOException { + ArtifactDownloader artifactDownloader = mock(ArtifactDownloader.class); + RoscoHelmfileConfigurationProperties helmfileConfigurationProperties = + new RoscoHelmfileConfigurationProperties(); + HelmfileTemplateUtils helmfileTemplateUtils = + new HelmfileTemplateUtils(artifactDownloader, helmfileConfigurationProperties); + + HelmfileBakeManifestRequest request = new HelmfileBakeManifestRequest(); + Artifact artifact = Artifact.builder().build(); + request.setInputArtifacts(Collections.singletonList(artifact)); + request.setOverrides(Collections.emptyMap()); + + try (BakeManifestEnvironment env = BakeManifestEnvironment.create()) { + BakeRecipe recipe = helmfileTemplateUtils.buildBakeRecipe(env, request); + assertFalse(recipe.getCommand().contains("--values")); + } + } +} diff --git a/rosco-manifests/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/rosco-manifests/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker new file mode 100644 index 000000000..1f0955d45 --- /dev/null +++ b/rosco-manifests/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker @@ -0,0 +1 @@ +mock-maker-inline diff --git a/rosco-web/rosco-web.gradle b/rosco-web/rosco-web.gradle index 9615f1b76..c5517c8c3 100644 --- a/rosco-web/rosco-web.gradle +++ b/rosco-web/rosco-web.gradle @@ -21,11 +21,14 @@ dependencies { runtimeOnly "io.spinnaker.kork:kork-runtime" - testImplementation "com.jakewharton.retrofit:retrofit1-okhttp3-client" testImplementation "io.spinnaker.kork:kork-retrofit" testImplementation "org.spockframework:spock-core" testImplementation "org.springframework.boot:spring-boot-starter-test" testImplementation "org.objenesis:objenesis" + testImplementation "com.squareup.retrofit2:retrofit" + testImplementation "com.squareup.retrofit2:converter-jackson" + testImplementation "com.github.tomakehurst:wiremock-jre8-standalone" + testCompileOnly "com.squareup.retrofit:retrofit" } ospackage { diff --git a/rosco-web/src/main/groovy/com/netflix/spinnaker/rosco/Main.groovy b/rosco-web/src/main/groovy/com/netflix/spinnaker/rosco/Main.groovy index 1f6fa34a8..d98596f69 100644 --- a/rosco-web/src/main/groovy/com/netflix/spinnaker/rosco/Main.groovy +++ b/rosco-web/src/main/groovy/com/netflix/spinnaker/rosco/Main.groovy @@ -16,9 +16,15 @@ package com.netflix.spinnaker.rosco +import com.fasterxml.jackson.databind.ObjectMapper +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStore +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStoreConfiguration +import com.netflix.spinnaker.kork.artifacts.artifactstore.EmbeddedArtifactSerializer +import com.netflix.spinnaker.kork.artifacts.model.Artifact import com.netflix.spinnaker.rosco.config.RoscoPackerConfigurationProperties import com.netflix.spinnaker.rosco.jobs.config.LocalJobConfig import com.netflix.spinnaker.rosco.manifests.config.RoscoHelmConfigurationProperties +import com.netflix.spinnaker.rosco.manifests.config.RoscoHelmfileConfigurationProperties import com.netflix.spinnaker.rosco.manifests.config.RoscoKustomizeConfigurationProperties import com.netflix.spinnaker.rosco.providers.alicloud.config.RoscoAliCloudConfiguration import com.netflix.spinnaker.rosco.providers.aws.config.RoscoAWSConfiguration @@ -29,8 +35,11 @@ import com.netflix.spinnaker.rosco.providers.huaweicloud.config.RoscoHuaweiCloud import com.netflix.spinnaker.rosco.providers.oracle.config.RoscoOracleConfiguration import com.netflix.spinnaker.rosco.providers.tencentcloud.config.RoscoTencentCloudConfiguration import com.netflix.spinnaker.rosco.services.ServiceConfig +import org.springframework.beans.factory.annotation.Qualifier import org.springframework.boot.autoconfigure.EnableAutoConfiguration import org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression import org.springframework.boot.autoconfigure.groovy.template.GroovyTemplateAutoConfiguration import org.springframework.boot.builder.SpringApplicationBuilder import org.springframework.boot.web.servlet.support.SpringBootServletInitializer @@ -38,6 +47,7 @@ import org.springframework.context.annotation.Bean import org.springframework.context.annotation.ComponentScan import org.springframework.context.annotation.Configuration import org.springframework.context.annotation.Import +import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder import org.springframework.scheduling.annotation.EnableScheduling import org.springframework.web.filter.ShallowEtagHeaderFilter import com.netflix.spinnaker.kork.boot.DefaultPropertiesBuilder @@ -68,8 +78,10 @@ import javax.servlet.Filter RoscoTencentCloudConfiguration, RoscoPackerConfigurationProperties, RoscoHelmConfigurationProperties, + RoscoHelmfileConfigurationProperties, RoscoKustomizeConfigurationProperties, - LocalJobConfig + LocalJobConfig, + ArtifactStoreConfiguration ]) @EnableAutoConfiguration(exclude = [BatchAutoConfiguration, GroovyTemplateAutoConfiguration]) @EnableScheduling @@ -90,4 +102,18 @@ class Main extends SpringBootServletInitializer { Filter eTagFilter() { new ShallowEtagHeaderFilter() } + + @Bean + @ConditionalOnExpression('${artifact-store.enabled:false}') + EmbeddedArtifactSerializer artifactSerializer(ArtifactStore store, @Qualifier("artifactObjectMapper") ObjectMapper objectMapper) { + return new EmbeddedArtifactSerializer(objectMapper, store); + } + + @Bean + @ConditionalOnBean(EmbeddedArtifactSerializer.class) + ObjectMapper objectMapper(Jackson2ObjectMapperBuilder builder, EmbeddedArtifactSerializer serializer) { + return builder.createXmlMapper(false) + .serializerByType(Artifact.class, serializer) + .build(); + } } diff --git a/rosco-web/src/test/java/com/netflix/spinnaker/rosco/controllers/V2BakeryControllerTest.java b/rosco-web/src/test/java/com/netflix/spinnaker/rosco/controllers/V2BakeryControllerTest.java index 7527c7870..4441fbc52 100644 --- a/rosco-web/src/test/java/com/netflix/spinnaker/rosco/controllers/V2BakeryControllerTest.java +++ b/rosco-web/src/test/java/com/netflix/spinnaker/rosco/controllers/V2BakeryControllerTest.java @@ -25,15 +25,15 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; -import com.google.gson.Gson; import com.netflix.spinnaker.kork.artifacts.model.Artifact; +import com.netflix.spinnaker.kork.retrofit.exceptions.RetrofitException; import com.netflix.spinnaker.kork.retrofit.exceptions.SpinnakerHttpException; import com.netflix.spinnaker.kork.retrofit.exceptions.SpinnakerServerException; import com.netflix.spinnaker.rosco.Main; import com.netflix.spinnaker.rosco.manifests.helm.HelmBakeManifestRequest; import com.netflix.spinnaker.rosco.services.ClouddriverService; import java.nio.charset.StandardCharsets; -import java.util.List; +import okhttp3.ResponseBody; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; @@ -44,10 +44,8 @@ import org.springframework.test.context.TestPropertySource; import org.springframework.test.web.servlet.MockMvc; import org.springframework.web.context.WebApplicationContext; -import retrofit.RetrofitError; -import retrofit.client.Response; -import retrofit.converter.GsonConverter; -import retrofit.mime.TypedString; +import retrofit2.Retrofit; +import retrofit2.converter.jackson.JacksonConverterFactory; @SpringBootTest(classes = Main.class) @TestPropertySource(properties = "spring.application.name = rosco") @@ -116,15 +114,19 @@ private static SpinnakerHttpException makeSpinnakerHttpException(int status) { // There, getResponseCode needs a real underlying response, at least real // enough for response.getStatus() to work. So, go ahead and build one. String url = "https://some-url"; - Response response = - new Response( - url, + retrofit2.Response retrofit2Response = + retrofit2.Response.error( status, - "arbitrary reason", - List.of(), - new TypedString("{ message: \"arbitrary message\" }")); + ResponseBody.create( + okhttp3.MediaType.parse("application/json"), + "{ \"message\": \"arbitrary message\" }")); - return new SpinnakerHttpException( - RetrofitError.httpError(url, response, new GsonConverter(new Gson()), String.class)); + Retrofit retrofit = + new Retrofit.Builder() + .baseUrl(url) + .addConverterFactory(JacksonConverterFactory.create()) + .build(); + + return new SpinnakerHttpException(RetrofitException.httpError(retrofit2Response, retrofit)); } } diff --git a/rosco-web/src/test/java/com/netflix/spinnaker/rosco/controllers/V2BakeryControllerWithClouddriverServiceTest.java b/rosco-web/src/test/java/com/netflix/spinnaker/rosco/controllers/V2BakeryControllerWithClouddriverServiceTest.java index 76b9c35bf..d7113d05c 100644 --- a/rosco-web/src/test/java/com/netflix/spinnaker/rosco/controllers/V2BakeryControllerWithClouddriverServiceTest.java +++ b/rosco-web/src/test/java/com/netflix/spinnaker/rosco/controllers/V2BakeryControllerWithClouddriverServiceTest.java @@ -15,10 +15,14 @@ */ package com.netflix.spinnaker.rosco.controllers; +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; +import static com.github.tomakehurst.wiremock.client.WireMock.putRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.client.WireMock.urlMatching; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; import static com.netflix.spinnaker.kork.common.Header.USER; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; @@ -26,36 +30,33 @@ import static org.springframework.test.web.servlet.setup.MockMvcBuilders.webAppContextSetup; import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.junit5.WireMockExtension; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; -import com.jakewharton.retrofit.Ok3Client; import com.netflix.spinnaker.filters.AuthenticatedRequestFilter; import com.netflix.spinnaker.kork.artifacts.model.Artifact; import com.netflix.spinnaker.rosco.Main; import com.netflix.spinnaker.rosco.api.BakeStatus; +import com.netflix.spinnaker.rosco.executor.BakePoller; import com.netflix.spinnaker.rosco.jobs.JobExecutor; import com.netflix.spinnaker.rosco.jobs.JobRequest; import com.netflix.spinnaker.rosco.manifests.helm.HelmBakeManifestRequest; -import com.netflix.spinnaker.rosco.services.ClouddriverService; import java.nio.charset.StandardCharsets; -import java.util.List; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; -import org.mockito.ArgumentCaptor; +import org.junit.jupiter.api.extension.RegisterExtension; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; import org.springframework.test.context.TestPropertySource; import org.springframework.test.web.servlet.MockMvc; import org.springframework.web.context.WebApplicationContext; -import retrofit.client.Header; -import retrofit.client.Request; -import retrofit.client.Response; -import retrofit.mime.TypedByteArray; @SpringBootTest(classes = Main.class) @TestPropertySource(properties = "spring.application.name = rosco") @@ -63,12 +64,14 @@ class V2BakeryControllerWithClouddriverServiceTest { private MockMvc webAppMockMvc; - @Autowired private WebApplicationContext webApplicationContext; + /** See https://wiremock.org/docs/junit-jupiter/#advanced-usage---programmatic */ + @RegisterExtension + static WireMockExtension wm = + WireMockExtension.newInstance().options(wireMockConfig().dynamicPort()).build(); - @Autowired ClouddriverService clouddriverService; + @Autowired private WebApplicationContext webApplicationContext; - /** So it's possible to verify e.g. request headers sent to clouddriver */ - @MockBean Ok3Client ok3Client; + @MockBean BakePoller bakePoller; @Autowired ObjectMapper objectMapper; @@ -84,6 +87,13 @@ class V2BakeryControllerWithClouddriverServiceTest { private HelmBakeManifestRequest bakeManifestRequest; + @DynamicPropertySource + static void registerClouddriverBaseUrl(DynamicPropertyRegistry registry) { + // Configure wiremock's random port into rosco + System.out.println("wiremock server url: " + wm.baseUrl()); + registry.add("services.clouddriver.base-url", wm::baseUrl); + } + @BeforeEach private void init(TestInfo testInfo) { System.out.println("--------------- Test " + testInfo.getDisplayName()); @@ -122,7 +132,9 @@ void testRequestHeadersToClouddriver() throws Exception { // Simulate a successful response from clouddriver. The actual response // isn't important since we're verifying headers in the request to // clouddriver. - when(ok3Client.execute(any(Request.class))).thenReturn(successfulResponse("")); + wm.stubFor( + WireMock.put(urlMatching("/artifacts/fetch/")) + .willReturn(aResponse().withStatus(HttpStatus.OK.value()).withBody("success"))); String userValue = "some user"; webAppMockMvc @@ -136,22 +148,9 @@ void testRequestHeadersToClouddriver() throws Exception { .andDo(print()) .andExpect(status().isOk()); - // Make sure the request to clouddriver has the same headers as the request - // to rosco - ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(Request.class); - verify(ok3Client).execute(requestCaptor.capture()); - List
headers = requestCaptor.getValue().getHeaders(); - Header header = Iterables.getOnlyElement(headers); - assertEquals(USER.getHeader(), header.getName()); - assertEquals(userValue, header.getValue()); - } - - private Response successfulResponse(String content) { - return new Response( - "", - HttpStatus.OK.value(), - "", - ImmutableList.of(), - new TypedByteArray(null, content.getBytes())); + // verifing request to clouddriver contains header passing to rosco + wm.verify( + putRequestedFor(urlEqualTo("/artifacts/fetch/")) + .withHeader(USER.getHeader(), equalTo(userValue))); } } diff --git a/settings.gradle b/settings.gradle index 95c7a5d2f..53e004d6a 100644 --- a/settings.gradle +++ b/settings.gradle @@ -6,8 +6,6 @@ } } -enableFeaturePreview("VERSION_ORDERING_V2") - rootProject.name = 'rosco' include 'rosco-bom', 'rosco-core', 'rosco-web', 'rosco-manifests'