From 88d8709c6a9ae3d37c1f7a72dea44bb70f4b0c91 Mon Sep 17 00:00:00 2001 From: Lucas Persson Date: Tue, 1 Nov 2022 16:54:31 +0100 Subject: [PATCH] add support for so caled interpolation of values in a properties map --- .settings/org.eclipse.buildship.core.prefs | 2 +- README.md | 71 +++++++++++++++++-- build.gradle | 2 +- .../CreateOrUpdateStackTask.groovy | 14 ++-- .../cloudformation/ParameterResolver.groovy | 60 ++++++++++++++++ .../PrintEnviromentParametersTask.groovy | 57 +++++++++++++++ 6 files changed, 192 insertions(+), 14 deletions(-) create mode 100644 src/main/groovy/se/solrike/cloudformation/ParameterResolver.groovy create mode 100644 src/main/groovy/se/solrike/cloudformation/PrintEnviromentParametersTask.groovy diff --git a/.settings/org.eclipse.buildship.core.prefs b/.settings/org.eclipse.buildship.core.prefs index c6d5084..c7f00bd 100644 --- a/.settings/org.eclipse.buildship.core.prefs +++ b/.settings/org.eclipse.buildship.core.prefs @@ -1,2 +1,2 @@ -connection.project.dir=../spring-aws-serverless +connection.project.dir=../s3deploy eclipse.preferences.version=1 diff --git a/README.md b/README.md index ff0cbe2..54b6ad3 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Apply the plugin to your project. ```groovy plugins { - id 'se.solrike.cloudformation' version '1.0.0-beta.1' + id 'se.solrike.cloudformation' version '1.0.0-beta.2' } ``` Gradle 7.0 or later must be used. @@ -23,7 +23,7 @@ The tasks have to be created. Minimal example on how to create a task that creat ```groovy plugins { - id 'se.solrike.cloudformation' version '1.0.0-beta.1' + id 'se.solrike.cloudformation' version '1.0.0-beta.2' } task deployS3Stack(type: se.solrike.cloudformation.CreateOrUpdateStackTask) { parameters = [ s3BucketName : 's3-bucket4711'] @@ -65,6 +65,10 @@ The task will also create the following tags on the stack: * TemplateGitVersion - If the Gradle project is using GIT then the last commit info for the template is added. * CreatedBy/UpdatedBy - The local OS user's userID that executes the task. +## How the DeleteStackTask works + +## How the PrintEnviromentParametersTask works + ## More advanced example ### Example 1 Some stacks need additional permission when they are created, called "capabilities". @@ -128,9 +132,9 @@ If you have all sorts of properties in the properties file you can filter out th ```java # Java properties file for stage 25 # Properties for the S3 template -slrf.deploy.s3.s3BucketName: my-bucket-for-stage25 +slrk.deploy.s3.s3BucketName: my-bucket-for-stage25 # Some property for another template -slrf.deploy.sam.handler: se.solrike.cloud.serverless.StreamLambdaHandler::handleRequest +slrk.deploy.sam.handler: se.solrike.cloud.serverless.StreamLambdaHandler::handleRequest ``` @@ -139,12 +143,69 @@ task deployS3Stack(type: se.solrike.cloudformation.CreateOrUpdateStackTask) { group = 'AWS' description = 'Create S3 buckets using Cloudformation template. ' + 'Specify the environment to use with -Penv=.' + parameterPrefix = 'slrk.deploy.s3.' // only include properties which begins with 'slrk.deploy.s3.' parameters = project.objects.mapProperty(String, String).convention(project.provider({ Properties props = new Properties() file("environments/${env}.properties").withInputStream { props.load(it) } props })) - parameterPrefix = 'slrf.deploy.s3.' // only include properties which begins with 'slrf.deploy.s3.' + stackName = project.objects.property(String).convention(project.provider( {"s3-buckets-${env}"} )) + templateFileName = 'aws-cloudformation/aws-s3-buckets.yaml' +} +``` + +### Example 3 +In a large complex setup there are more than one template and consequently a lot more stack parameters. Sometime you need to have the same value as input to several stacks and following DRY you need to be able to have variables in the properties files where you have the stacks parameters. + +The following example has two stacks which shares an S3 bucket. One creates it and the other consumes it. So the bucket name needs to be as a parameter to both of them. Also the a DB password is needed and you don't want to have that as clear text in the environment's properties files. The example is simply base 64 encoding the password but in real life it shall be encrypted of course. + +Properties file: + +```java +# Java properties file for stage 25. +# Values are "interpolated" (evaluated as they where Groovy script) so it means those can reference +# other parameter and also be functions. + +# common props +slrk.deploy.env.name: ${env} + +# Properties for the S3 template +slrk.deploy.s3.s3BucketName: my-bucket-for-${slrk.deploy.env.name} + +# Some property for another template +slrk.deploy.sam.handler: se.solrike.cloud.serverless.StreamLambdaHandler::handleRequest +slrk.deploy.sam.s3BucketName: ${slrk.deploy.s3.s3BucketName} + +# DB user and password +slrk.deploy.db.user: application-user +slrk.deploy.db.password: ${MyUtils.decode('bXlub3Rzb3NlY3JldHBhc3N3b3JkCg==')} +``` + +Task to deploy and code in build.gradle to support the task. + +```groovy +Properties loadProperties(String env) { + Properties props = new Properties() + file("environments/${env}.properties").withInputStream { props.load(it) } + // so that the property 'env' can be referenced in the env's properties file + props.setProperty('env', env) + return props +} + +class MyUtils { + static String decode(String text) { + return new String(Base64.getDecoder().decode(text)).trim() + } +} + +task deployS3Stack(type: se.solrike.cloudformation.CreateOrUpdateStackTask) { + group = 'AWS' + description = 'Create S3 buckets using Cloudformation template. ' + + 'Specify the enviroment to use with -Penv=.' + parameterPrefix = 'slrk.deploy.s3.' // only include properties which begins with 'slrf.deploy.s3.' + parameters = project.objects.mapProperty(String, String).convention(project.provider({loadProperties(env)})) + // pass in classloader so the MyUtils class above can be referenced in the env's properties file + parentClassLoader = getClass().getClassLoader() stackName = project.objects.property(String).convention(project.provider( {"s3-buckets-${env}"} )) templateFileName = 'aws-cloudformation/aws-s3-buckets.yaml' } diff --git a/build.gradle b/build.gradle index d74ab52..84fe33a 100644 --- a/build.gradle +++ b/build.gradle @@ -26,7 +26,7 @@ dependencies { } group = 'se.solrike.cloudformation' -version = '1.0.0-beta.1' +version = '1.0.0-beta.2' sourceCompatibility = '11' targetCompatibility = '11' diff --git a/src/main/groovy/se/solrike/cloudformation/CreateOrUpdateStackTask.groovy b/src/main/groovy/se/solrike/cloudformation/CreateOrUpdateStackTask.groovy index 3ffb790..5f12678 100644 --- a/src/main/groovy/se/solrike/cloudformation/CreateOrUpdateStackTask.groovy +++ b/src/main/groovy/se/solrike/cloudformation/CreateOrUpdateStackTask.groovy @@ -48,21 +48,21 @@ abstract class CreateOrUpdateStackTask extends DefaultTask { public abstract Property getEnableTerminationProtection() /** - * The template parameters. + * Optionally specify a parameter name prefix to filter out the parameters to use. *

- * Parameter names will be converted to PascalCase-ish since that is what Cloudformation templates expects. + * The submitted parameter names will have the prefix string removed from the names. */ @Input - public abstract MapProperty getParameters() + @Optional + public abstract Property getParameterPrefix() /** - * Optionally specify a parameter name prefix to filter out the parameters to use. + * The template parameters. *

- * The submitted parameter names will have the prefix string removed from the names. + * Parameter names will be converted to PascalCase-ish since that is what Cloudformation templates expects. */ @Input - @Optional - public abstract Property getParameterPrefix() + public abstract MapProperty getParameters() /** * Stack name. diff --git a/src/main/groovy/se/solrike/cloudformation/ParameterResolver.groovy b/src/main/groovy/se/solrike/cloudformation/ParameterResolver.groovy new file mode 100644 index 0000000..1aca430 --- /dev/null +++ b/src/main/groovy/se/solrike/cloudformation/ParameterResolver.groovy @@ -0,0 +1,60 @@ +package se.solrike.cloudformation + +/** + * @author Lucas Persson + */ +public class ParameterResolver { + + public static Map resolve(Map parameters, ClassLoader parentClassLoader) { + Binding binding = new Binding() + + Properties p = new Properties() + p.putAll(parameters) + // ConfigSlurper will create the properties as beans in case a dot notation is used + ConfigObject conf = new ConfigSlurper().parse(p) + conf.each { key, value -> + binding.setVariable(key, value) + } + GroovyShell shell = new GroovyShell(parentClassLoader, binding) + + // multipass so nested references (in two hops) can be used. + resolveValues(shell, binding) + resolveValues(shell, binding) + + Map flatMap = [:] + flatternMap(binding.getVariables(), flatMap, null) + + return flatMap + } + + static void resolveValues(GroovyShell shell, Binding binding) { + visitMap(binding.getVariables(), { + it.value = shell.evaluate("\"$it.value\"") + }) + } + + static void visitMap(Map map, Closure action) { + map.each { + if (it.value instanceof Map) { + visitMap(it.value, action) + } + else { + action.call(it) + } + } + } + + // flattern the map and re-create the dot notation or the keys + static void flatternMap(Map source, Map target, String prefix) { + source.each { + String key = prefix ? prefix + '.' + it.key : it.key + if (it.value instanceof Map) { + flatternMap(it.value, target, key) + } + else { + target[key] = it.value + } + } + } + +} diff --git a/src/main/groovy/se/solrike/cloudformation/PrintEnviromentParametersTask.groovy b/src/main/groovy/se/solrike/cloudformation/PrintEnviromentParametersTask.groovy new file mode 100644 index 0000000..9095173 --- /dev/null +++ b/src/main/groovy/se/solrike/cloudformation/PrintEnviromentParametersTask.groovy @@ -0,0 +1,57 @@ +package se.solrike.cloudformation + +import org.gradle.api.DefaultTask +import org.gradle.api.provider.MapProperty +import org.gradle.api.provider.Property +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.Internal +import org.gradle.api.tasks.Optional +import org.gradle.api.tasks.TaskAction + +import software.amazon.awssdk.core.waiters.WaiterResponse +import software.amazon.awssdk.services.cloudformation.CloudFormationClient +import software.amazon.awssdk.services.cloudformation.model.Capability +import software.amazon.awssdk.services.cloudformation.model.CloudFormationException +import software.amazon.awssdk.services.cloudformation.model.CreateStackRequest +import software.amazon.awssdk.services.cloudformation.model.CreateStackResponse +import software.amazon.awssdk.services.cloudformation.model.DeleteStackResponse +import software.amazon.awssdk.services.cloudformation.model.DescribeStacksResponse +import software.amazon.awssdk.services.cloudformation.model.Parameter +import software.amazon.awssdk.services.cloudformation.model.Tag +import software.amazon.awssdk.services.cloudformation.model.UpdateStackRequest +import software.amazon.awssdk.services.cloudformation.model.UpdateStackResponse + +/** + * Task to print the resolved parameters for an environment. + *

+ * The task will first resolve parameters using Groovy's evaluation support. + * + * @author Lucas Persson + */ +abstract class PrintEnviromentParametersTask extends DefaultTask { + + /** + * The build script's class loader so that this task can find e.g. classes defined in the build.gradle. + */ + @Input + @Optional + public abstract Property getParentClassLoader() + + /** + * The parameters. + */ + @Input + public abstract MapProperty getParameters() + + + + @TaskAction + void execute() { + Map resolvedParameters = ParameterResolver.resolve(getParameters().get(), + getParentClassLoader().getOrElse(getClass().getClassLoader())) + + resolvedParameters.sort { it.key }.each {key, value -> + println "$key : $value" + } + } +}