feat(intent): Pipeline intent processing (#51)

keel-clouddriver/src/test/kotlin/com/netflix/spinnaker/keel/clouddriver/model/BaseModelParsingTest.kt

@@ -26,7 +26,7 @@ abstract class BaseModelParsingTest<out T> {
 
  private val mapper = KeelConfiguration()
  .apply { properties = KeelProperties() }
- .objectMapper(ObjectMapper())
+ .objectMapper(ObjectMapper(), listOf())
  private val client = mock<Client>()
  private val cloudDriver = RestAdapter.Builder()
  .setEndpoint(newFixedEndpoint("https://spinnaker.💩"))

keel-core/src/main/kotlin/com/netflix/spinnaker/config/KeelConfiguration.kt

@@ -56,13 +56,19 @@ open class KeelConfiguration {
  // have the subtypes registered.
  // TODO rz - Move keiko subtype configurer into kork so we can use it here instead
  @Autowired
- open fun objectMapper(objectMapper: ObjectMapper) =
+ open fun objectMapper(objectMapper: ObjectMapper, subtypeLocators: List<KeelSubTypeLocator>) =
  objectMapper.apply {
  registerSubtypes(*findAllSubtypes(log, Intent::class.java, "com.netflix.spinnaker.keel.intent"))
  registerSubtypes(*findAllSubtypes(log, IntentSpec::class.java, "com.netflix.spinnaker.keel.intent"))
  registerSubtypes(*findAllSubtypes(log, Policy::class.java, "com.netflix.spinnaker.keel.policy"))
  registerSubtypes(*findAllSubtypes(log, PolicySpec::class.java, "com.netflix.spinnaker.keel.policy"))
  registerSubtypes(*findAllSubtypes(log, Attribute::class.java, "com.netflix.spinnaker.keel.attribute"))
+
+ subtypeLocators.forEach { subtype ->
+ subtype.packages.forEach { pkg ->
+ registerSubtypes(*findAllSubtypes(log, subtype.cls, pkg))
+ }
+ }
  }
  .registerModule(KotlinModule())
  .registerModule(VersioningModule())

keel-core/src/main/kotlin/com/netflix/spinnaker/keel/model/OrchestrationRequest.kt

@@ -20,12 +20,12 @@ data class OrchestrationRequest(
  val application: String,
  val description: String,
  val job: List<Job>,
- val trigger: Trigger
+ val trigger: OrchestrationTrigger
 )
 
 class Job(type: String, m: MutableMap<String, Any?>): HashMap<String, Any?>(m.apply { put("type", type) })
 
-data class Trigger(
+data class OrchestrationTrigger(
  val correlationId: String,
  val type: String = "keel",
  val user: String = "keel"

keel-core/src/main/kotlin/com/netflix/spinnaker/keel/objectMapperUtil.kt

@@ -15,18 +15,27 @@
  */
 package com.netflix.spinnaker.keel
 
+import com.fasterxml.jackson.annotation.JsonTypeName
+import com.fasterxml.jackson.databind.jsontype.NamedType
 import org.slf4j.Logger
 import org.springframework.context.annotation.ClassPathScanningCandidateComponentProvider
 import org.springframework.core.type.filter.AssignableTypeFilter
 import org.springframework.util.ClassUtils
 
-fun findAllSubtypes(log: Logger, clazz: Class<*>, pkg: String): Array<Class<*>>
+fun findAllSubtypes(log: Logger, clazz: Class<*>, pkg: String): Array<NamedType>
  = ClassPathScanningCandidateComponentProvider(false)
  .apply { addIncludeFilter(AssignableTypeFilter(clazz)) }
  .findCandidateComponents(pkg)
  .map {
  val cls = ClassUtils.resolveClassName(it.beanClassName, ClassUtils.getDefaultClassLoader())
- log.info("Registering ${cls.simpleName}")
- return@map cls
+
+ val serializationName = cls.annotations
+ .filterIsInstance<JsonTypeName>()
+ .firstOrNull()
+ ?.value
+
+ log.info("Registering ${cls.simpleName}: $serializationName")
+
+ return@map NamedType(cls, serializationName )
  }
  .toTypedArray()

keel-core/src/test/kotlin/com/netflix/spinnaker/keel/dryrun/DryRunIntentLauncherTest.kt

@@ -29,7 +29,7 @@ import com.netflix.spinnaker.keel.IntentSpec
 import com.netflix.spinnaker.keel.IntentStatus.ACTIVE
 import com.netflix.spinnaker.keel.model.Job
 import com.netflix.spinnaker.keel.model.OrchestrationRequest
-import com.netflix.spinnaker.keel.model.Trigger
+import com.netflix.spinnaker.keel.model.OrchestrationTrigger
 import com.nhaarman.mockito_kotlin.any
 import com.nhaarman.mockito_kotlin.doAnswer
 import com.nhaarman.mockito_kotlin.doReturn
@@ -65,7 +65,7 @@ object DryRunIntentLauncherTest {
  Job("wait", mutableMapOf("waitTime" to 5)),
  Job("wait", mutableMapOf("name" to "wait for more time", "waitTime" to 5))
  ),
- Trigger("1", "keel", "keel")
+ OrchestrationTrigger("1", "keel", "keel")
  )
  ),
  changeSummary

keel-core/src/test/kotlin/com/netflix/spinnaker/keel/model/OrchestrationRequestTest.kt

@@ -25,7 +25,7 @@ object OrchestrationRequestTest {
  fun `should build job request`() {
  val req = OrchestrationRequest("wait for nothing", "keel", "my orchestration", listOf(
  Job("wait", mutableMapOf("waitTime" to 30))
- ), Trigger("1", "keel", "keel"))
+ ), OrchestrationTrigger("1", "keel", "keel"))
 
  req.name shouldMatch equalTo("wait for nothing")
  req.application shouldMatch equalTo("keel")

keel-front50/src/main/kotlin/com/netflix/spinnaker/keel/front50/Front50Service.kt

@@ -19,8 +19,14 @@ import com.netflix.spinnaker.keel.Intent
 import com.netflix.spinnaker.keel.IntentSpec
 import com.netflix.spinnaker.keel.IntentStatus
 import com.netflix.spinnaker.keel.front50.model.Application
+import com.netflix.spinnaker.keel.front50.model.PipelineConfig
 import retrofit.client.Response
-import retrofit.http.*
+import retrofit.http.Body
+import retrofit.http.DELETE
+import retrofit.http.GET
+import retrofit.http.POST
+import retrofit.http.Path
+import retrofit.http.Query
 
 interface Front50Service {
 
@@ -41,4 +47,7 @@ interface Front50Service {
 
  @GET("/v2/applications/{applicationName}")
  fun getApplication(@Path("applicationName") applicationName: String): Application
+
+ @GET("/pipelines/{applicationName}")
+ fun getPipelineConfigs(@Path("applicationName") applicationName: String): List<PipelineConfig>
 }

keel-front50/src/main/kotlin/com/netflix/spinnaker/keel/front50/model/Application.kt

@@ -28,7 +28,7 @@ data class Application(
  @Computed val createTs: String? = null,
  val platformHealthOnly: Boolean,
  val platformHealthOnlyShowOverride: Boolean,
- val owner: String
+ val owner: String?
 ) : ComputedPropertyProvider {
 
  val details: MutableMap<String, Any?> = mutableMapOf()

keel-front50/src/main/kotlin/com/netflix/spinnaker/keel/front50/model/PipelineConfig.kt

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2018 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.netflix.spinnaker.keel.front50.model
+
+import com.fasterxml.jackson.annotation.JsonAnyGetter
+import com.fasterxml.jackson.annotation.JsonAnySetter
+import com.netflix.spinnaker.keel.annotation.Computed
+
+data class PipelineConfig(
+ val application: String,
+ val name: String,
+ val parameterConfig: List<Map<String, Any?>>?,
+ val triggers: List<Map<String, Any?>>?,
+ val notifications: List<Map<String, Any?>>?,
+ val stages: List<Map<String, Any?>>,
+ val spelEvaluator: String?,
+ val executionEngine: String?,
+ val limitConcurrent: Boolean?,
+ val keepWaitingPipelines: Boolean?,
+ @Computed val id: String?,
+ @Computed val index: Int?,
+ @Computed val stageCounter: Int?,
+ @Computed val lastModifiedBy: String?,
+ @Computed val updateTs: String?
+) {
+
+ private val details: MutableMap<String, Any?> = mutableMapOf()
+
+ @JsonAnySetter
+ fun set(name: String, value: Any?) {
+ details[name] = value
+ }
+
+ @JsonAnyGetter
+ fun details() = details
+}

keel-intent/src/main/kotlin/com/netflix/spinnaker/config/IntentConfiguration.kt

@@ -16,6 +16,7 @@
 package com.netflix.spinnaker.config
 
 import com.netflix.spinnaker.keel.intent.SecurityGroupRule
+import com.netflix.spinnaker.keel.intent.Trigger
 import org.springframework.context.annotation.Bean
 import org.springframework.context.annotation.ComponentScan
 import org.springframework.context.annotation.Configuration
@@ -30,4 +31,7 @@ open class IntentConfiguration {
 
  @Bean open fun securityGroupSubTypeLocator() =
  KeelSubTypeLocator(SecurityGroupRule::class.java, listOf("com.netflix.spinnaker.keel.intent"))
+
+ @Bean open fun pipelineTriggerSubTypeLocator() =
+ KeelSubTypeLocator(Trigger::class.java, listOf("com.netflix.spinnaker.keel.intent"))
 }

keel-intent/src/main/kotlin/com/netflix/spinnaker/keel/intent/PipelineIntent.kt

@@ -35,30 +35,33 @@ class PipelineIntent
  schema = CURRENT_SCHEMA,
  spec = spec
 ) {
- @JsonIgnore override val defaultId = "$KIND:${spec.application}:${spec.slug}"
+ @JsonIgnore override val defaultId = "$KIND:${spec.application}:${spec.name}"
 }
 
 @JsonTypeName("pipeline")
 data class PipelineSpec(
  override val application: String,
- // Used for idempotency of id
- @ForcesNew val slug: String,
- val stages: List<Map<String, Any?>>,
+ // TODO rz - Support renaming without re-creation. Probably require getting all pipelines for an
+ // application and finding a match on name before writes happen?
+ @ForcesNew val name: String,
+ val stages: List<PipelineStage>,
  val triggers: List<Trigger>,
- val flags: Flags,
- val properties: Properties
+ val parameters: List<Map<String, Any?>>,
+ val notifications: List<Map<String, Any?>>,
+ val flags: PipelineFlags,
+ val properties: PipelineProperties
 ) : ApplicationAwareIntentSpec()
 
-class Flags : HashMap<String, Boolean>() {
+class PipelineFlags : HashMap<String, Boolean>() {
 
- val keepWaitingPipelines: Boolean
- get() = get("keepWaitingPipelines") ?: true
+ val keepWaitingPipelines: Boolean?
+ get() = get("keepWaitingPipelines")
 
- val limitConcurrent: Boolean
- get() = get("limitConcurrent") ?: false
+ val limitConcurrent: Boolean?
+ get() = get("limitConcurrent")
 }
 
-class Properties : HashMap<String, String>() {
+class PipelineProperties : HashMap<String, String>() {
 
  val executionEngine: String?
  get() = get("executionEngine")
@@ -67,26 +70,41 @@ class Properties : HashMap<String, String>() {
  get() = get("spelEvaluator")
 }
 
+class PipelineStage : HashMap<String, Any>() {
+
+ val kind: String
+ get() = get("kind").toString()
+
+ val refId: String
+ get() = get("refId").toString()
+
+ val dependsOn: List<String>
+ get() = if (containsKey("dependsOn")) this["dependsOn"] as List<String> else listOf()
+}
+
 @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "kind")
-abstract class Trigger : HashMap<String, Any?>()
+interface Trigger
+//abstract class Trigger : HashMap<String, Any> {
+// @JsonCreator constructor(m: Map<String, Any>) : super(m)
+//}
 
 @JsonTypeName("cron")
-class CronTrigger : Trigger()
+class CronTrigger(m: Map<String, Any>) : HashMap<String, Any>(m), Trigger
 
 @JsonTypeName("docker")
-class DockerTrigger : Trigger()
+class DockerTrigger(m: Map<String, Any>) : HashMap<String, Any>(m), Trigger
 
 @JsonTypeName("dryRun")
-class DryRunTrigger : Trigger()
+class DryRunTrigger(m: Map<String, Any>) : HashMap<String, Any>(m), Trigger
 
 @JsonTypeName("git")
-class GitTrigger : Trigger()
+class GitTrigger(m: Map<String, Any>) : HashMap<String, Any>(m), Trigger
 
 @JsonTypeName("jenkins")
-class JenkinsTrigger : Trigger()
+class JenkinsTrigger(m: Map<String, Any>) : HashMap<String, Any>(m), Trigger
 
 @JsonTypeName("manual")
-class ManualTrigger : Trigger()
+class ManualTrigger(m: Map<String, Any>) : HashMap<String, Any>(m), Trigger
 
 @JsonTypeName("pipeline")
-class PipelineTrigger : Trigger()
+class PipelineTrigger(m: Map<String, Any>) : HashMap<String, Any>(m), Trigger

keel-intent/src/main/kotlin/com/netflix/spinnaker/keel/intent/processor/ApplicationIntentProcessor.kt

@@ -30,7 +30,7 @@ import com.netflix.spinnaker.keel.intent.ApplicationIntent
 import com.netflix.spinnaker.keel.intent.BaseApplicationSpec
 import com.netflix.spinnaker.keel.model.Job
 import com.netflix.spinnaker.keel.model.OrchestrationRequest
-import com.netflix.spinnaker.keel.model.Trigger
+import com.netflix.spinnaker.keel.model.OrchestrationTrigger
 import com.netflix.spinnaker.keel.state.FieldMutator
 import com.netflix.spinnaker.keel.state.StateInspector
 import com.netflix.spinnaker.keel.tracing.Trace
@@ -82,7 +82,7 @@ class ApplicationIntentProcessor
  )
  )
  ),
- trigger = Trigger(intent.id())
+ trigger = OrchestrationTrigger(intent.id())
  )
  ),
  changeSummary

keel-intent/src/main/kotlin/com/netflix/spinnaker/keel/intent/processor/ParrotIntentProcessor.kt

@@ -23,7 +23,7 @@ import com.netflix.spinnaker.keel.dryrun.ChangeSummary
 import com.netflix.spinnaker.keel.intent.ParrotIntent
 import com.netflix.spinnaker.keel.model.Job
 import com.netflix.spinnaker.keel.model.OrchestrationRequest
-import com.netflix.spinnaker.keel.model.Trigger
+import com.netflix.spinnaker.keel.model.OrchestrationTrigger
 import com.netflix.spinnaker.keel.tracing.Trace
 import com.netflix.spinnaker.keel.tracing.TraceRepository
 import org.springframework.beans.factory.annotation.Autowired
@@ -49,7 +49,7 @@ class ParrotIntentProcessor
  job = listOf(
  Job("wait", mutableMapOf("waitTime" to intent.spec.waitTime))
  ),
- trigger = Trigger(intent.id())
+ trigger = OrchestrationTrigger(intent.id())
  )
  ), changeSummary)
  }