feat(artifacts): Introduce ArtifactSupplier plugin interface (3 of 3) (…

…#1321)

* feat(artifacts): Introduce ArtifactPublisher plugin interface (3 of 3)

* chore(pr): Improve logging

* fix(pr): Address review feedback

* chore(pr): More logging

* fix(pr): Ignore statuses for DockerArtifact

* chore(logging): Yet more logging improvements

* refactor(pr): Rename ArtifactPublisher to ArtifactSupplier

* fix(pr): Cleanup left-over file

keel-api-jackson/keel-api-jackson.gradle.kts

@@ -26,8 +26,7 @@ dependencies {
   implementation("org.springframework:spring-context")
   implementation("org.springframework.boot:spring-boot-autoconfigure")
 
-  // this is really just so we can access `configuredObjectMapper`
-  testImplementation(project(":keel-core"))
+  testImplementation(project(":keel-test"))
 
   testImplementation("io.strikt:strikt-jackson")
   testImplementation("dev.minutest:minutest")

keel-api-jackson/src/main/kotlin/com/netflix/spinnaker/keel/json/KeelApiModule.kt

@@ -23,16 +23,11 @@ import com.netflix.spinnaker.keel.api.Locatable
 import com.netflix.spinnaker.keel.api.Monikered
 import com.netflix.spinnaker.keel.api.ResourceKind
 import com.netflix.spinnaker.keel.api.SubnetAwareRegionSpec
-import com.netflix.spinnaker.keel.api.artifacts.ArtifactType
-import com.netflix.spinnaker.keel.api.artifacts.DebianSemVerVersioningStrategy
 import com.netflix.spinnaker.keel.api.artifacts.DeliveryArtifact
-import com.netflix.spinnaker.keel.api.artifacts.DockerVersioningStrategy
 import com.netflix.spinnaker.keel.api.artifacts.TagVersionStrategy
 import com.netflix.spinnaker.keel.api.artifacts.VersioningStrategy
 import com.netflix.spinnaker.keel.api.constraints.ConstraintState
 import com.netflix.spinnaker.keel.api.constraints.ConstraintStateAttributes
-import com.netflix.spinnaker.keel.artifacts.DebianArtifact
-import com.netflix.spinnaker.keel.artifacts.DockerArtifact
 import com.netflix.spinnaker.keel.json.mixins.ConstraintStateMixin
 import com.netflix.spinnaker.keel.json.mixins.DeliveryArtifactMixin
 import com.netflix.spinnaker.keel.json.mixins.LocatableMixin
@@ -55,13 +50,6 @@ object KeelApiModule : SimpleModule("Keel API") {
       setMixInAnnotations<Monikered, MonikeredMixin>()
       setMixInAnnotations<SubnetAwareRegionSpec, SubnetAwareRegionSpecMixin>()
       setMixInAnnotations<ConstraintState, ConstraintStateMixin>()
-
-      registerSubtypes(
-        NamedType<DebianArtifact>(ArtifactType.deb.name),
-        NamedType<DockerArtifact>(ArtifactType.docker.name),
-        NamedType<DockerVersioningStrategy>(ArtifactType.docker.name),
-        NamedType<DebianSemVerVersioningStrategy>(ArtifactType.deb.name)
-      )
     }
   }
 }
@@ -86,7 +74,8 @@ internal object KeelApiAnnotationIntrospector : NopAnnotationIntrospector() {
   private val types = setOf(
     Constraint::class.java,
     ConstraintStateAttributes::class.java,
-    DeliveryArtifact::class.java
+    DeliveryArtifact::class.java,
+    VersioningStrategy::class.java
   )
 
   override fun findTypeResolver(config: MapperConfig<*>, ac: AnnotatedClass, baseType: JavaType): TypeResolverBuilder<*>? =
@@ -120,12 +109,6 @@ internal object KeelApiDeserializers : Deserializers.Base() {
       TagVersionStrategy::class.java -> TagVersionStrategyDeserializer
       else -> null
     }
-
-  override fun findBeanDeserializer(type: JavaType, config: DeserializationConfig, beanDesc: BeanDescription): JsonDeserializer<*>? =
-    when (type.rawClass) {
-      VersioningStrategy::class.java -> VersioningStrategyDeserializer
-      else -> null
-    }
 }
 
 internal inline fun <reified T> NamedType(name: String) = NamedType(T::class.java, name)

keel-api-jackson/src/test/kotlin/com/netflix/spinnaker/keel/api/DeliveryArtifactTests.kt

@@ -5,7 +5,7 @@ import com.fasterxml.jackson.module.kotlin.readValue
 import com.netflix.spinnaker.keel.api.artifacts.DeliveryArtifact
 import com.netflix.spinnaker.keel.artifacts.DebianArtifact
 import com.netflix.spinnaker.keel.artifacts.DockerArtifact
-import com.netflix.spinnaker.keel.serialization.configuredObjectMapper
+import com.netflix.spinnaker.keel.test.configuredTestObjectMapper
 import dev.minutest.junit.JUnit5Minutests
 import dev.minutest.rootContext
 import strikt.api.expectCatching
@@ -82,5 +82,5 @@ internal class DeliveryArtifactTests : JUnit5Minutests {
 private data class Fixture(
   val json: String
 ) {
-  val mapper = configuredObjectMapper()
+  val mapper = configuredTestObjectMapper()
 }

keel-api/src/main/kotlin/com/netflix/spinnaker/keel/api/artifacts/DeliveryArtifact.kt

@@ -1,20 +1,28 @@
 package com.netflix.spinnaker.keel.api.artifacts
 
-enum class ArtifactType {
-  deb, docker;
-
-  override fun toString() = name
-}
+typealias ArtifactType = String
 
+/**
+ * The release status of an artifact. This may not necessarily be applicable to all
+ * [DeliveryArtifact] sub-classes.
+ */
 enum class ArtifactStatus {
   FINAL, CANDIDATE, SNAPSHOT, RELEASE, UNKNOWN
 }
 
+/**
+ * An artifact as defined in a [DeliveryConfig].
+ *
+ * Unlike other places within Spinnaker, this class does not describe a specific instance of a software artifact
+ * (i.e. the output of a build that is published to an artifact repository), but rather the high-level properties
+ * that allow keel and [ArtifactPublisher] plugins to find/process the actual artifacts.
+ */
 abstract class DeliveryArtifact {
   abstract val name: String
   abstract val type: ArtifactType
   abstract val versioningStrategy: VersioningStrategy
   abstract val reference: String // friendly reference to use within a delivery config
   abstract val deliveryConfigName: String? // the delivery config this artifact is a part of
-  override fun toString() = "$type:$name (ref: $reference)"
+  open val statuses: Set<ArtifactStatus> = emptySet()
+  override fun toString() = "${type.toUpperCase()} artifact $name (ref: $reference)"
 }

keel-api/src/main/kotlin/com/netflix/spinnaker/keel/api/artifacts/PublishedArtifact.kt

@@ -0,0 +1,21 @@
+package com.netflix.spinnaker.keel.api.artifacts
+
+/**
+ * An immutable data class that represents a published software artifact in the Spinnaker ecosystem.
+ *
+ * This class mirrors [com.netflix.spinnaker.kork.artifacts.model.Artifact], but without all the Jackson baggage.
+ * One notable difference from the kork counterpart is that this class enforces non-nullability of a few
+ * key fields without which it doesn't make sense for an artifact to exist in Managed Delivery terms.
+ */
+data class PublishedArtifact(
+  val name: String,
+  val type: String,
+  val reference: String,
+  val version: String,
+  val customKind: Boolean? = null,
+  val location: String? = null,
+  val artifactAccount: String? = null,
+  val provenance: String? = null,
+  val uuid: String? = null,
+  val metadata: Map<String, Any?> = emptyMap()
+)

keel-api/src/main/kotlin/com/netflix/spinnaker/keel/api/artifacts/VersioningStrategy.kt

@@ -3,21 +3,7 @@ package com.netflix.spinnaker.keel.api.artifacts
 /**
  * Strategy for how to sort versions of artifacts.
  */
-abstract class VersioningStrategy
-
-object DebianSemVerVersioningStrategy : VersioningStrategy() {
-  override fun toString(): String =
-    javaClass.simpleName
-
-  override fun equals(other: Any?): Boolean {
-    return other is DebianSemVerVersioningStrategy
-  }
-}
-
-data class DockerVersioningStrategy(
-  val strategy: TagVersionStrategy,
-  val captureGroupRegex: String? = null
-) : VersioningStrategy() {
-  override fun toString(): String =
-    "${javaClass.simpleName}[strategy=$strategy, captureGroupRegex=$captureGroupRegex]}"
+abstract class VersioningStrategy {
+  abstract val type: String
+  abstract val comparator: Comparator<String>
 }

keel-api/src/main/kotlin/com/netflix/spinnaker/keel/api/events/artifacts.kt

@@ -1,14 +1,14 @@
 package com.netflix.spinnaker.keel.api.events
 
 import com.netflix.spinnaker.keel.api.artifacts.DeliveryArtifact
-import com.netflix.spinnaker.keel.api.artifacts.SpinnakerArtifact
+import com.netflix.spinnaker.keel.api.artifacts.PublishedArtifact
 
 /**
- * An event that conveys information about one or more software artifacts that are
+ * An event that conveys information about one or more [PublishedArtifact] that are
  * potentially relevant to keel.
  */
-data class ArtifactEvent(
-  val artifacts: List<SpinnakerArtifact>,
+data class ArtifactPublishedEvent(
+  val artifacts: List<PublishedArtifact>,
   val details: Map<String, Any>?
 )
 

keel-api/src/main/kotlin/com/netflix/spinnaker/keel/api/plugins/ArtifactSupplier.kt

@@ -0,0 +1,92 @@
+package com.netflix.spinnaker.keel.api.plugins
+
+import com.netflix.spinnaker.keel.api.DeliveryConfig
+import com.netflix.spinnaker.keel.api.artifacts.ArtifactStatus
+import com.netflix.spinnaker.keel.api.artifacts.ArtifactType
+import com.netflix.spinnaker.keel.api.artifacts.BuildMetadata
+import com.netflix.spinnaker.keel.api.artifacts.DeliveryArtifact
+import com.netflix.spinnaker.keel.api.artifacts.GitMetadata
+import com.netflix.spinnaker.keel.api.artifacts.PublishedArtifact
+import com.netflix.spinnaker.keel.api.artifacts.VersioningStrategy
+import com.netflix.spinnaker.keel.api.events.ArtifactPublishedEvent
+import com.netflix.spinnaker.keel.api.support.EventPublisher
+import com.netflix.spinnaker.kork.plugins.api.internal.SpinnakerExtensionPoint
+
+/**
+ * Keel plugin interface to be implemented by suppliers of artifact information.
+ *
+ * The primary responsibility of an [ArtifactSupplier] is to detect new versions of artifacts, using
+ * whatever mechanism they choose (e.g. they could receive events from another system,
+ * or poll an artifact repository for artifact versions), and notify keel via the [publishArtifact]
+ * method, so that the artifact versions can be persisted and evaluated for promotion.
+ *
+ * Secondarily, [ArtifactSupplier]s are also periodically called to retrieve the latest available
+ * version of an artifact. This is so that we don't miss any versions in case of missed or failure
+ * to handle events in case of downtime, etc.
+ */
+interface ArtifactSupplier<T : DeliveryArtifact> : SpinnakerExtensionPoint {
+  val eventPublisher: EventPublisher
+  val supportedArtifact: SupportedArtifact<T>
+  val supportedVersioningStrategies: List<SupportedVersioningStrategy<*>>
+
+  /**
+   * Publishes an [ArtifactPublishedEvent] to core Keel so that the corresponding artifact version can be
+   * persisted and evaluated for promotion into deployment environments.
+   *
+   * The default implementation of [publishArtifact] simply publishes the event via the [EventPublisher],
+   * and should *not* be overridden by implementors.
+   */
+  fun publishArtifact(artifactPublishedEvent: ArtifactPublishedEvent) =
+    eventPublisher.publishEvent(artifactPublishedEvent)
+
+  /**
+   * Returns the latest available version for the given [DeliveryArtifact], represented
+   * as a [PublishedArtifact].
+   *
+   * This function may interact with external systems to retrieve artifact information as needed.
+   */
+  suspend fun getLatestArtifact(deliveryConfig: DeliveryConfig, artifact: DeliveryArtifact): PublishedArtifact?
+
+  /**
+   * Given a [PublishedArtifact] supported by this [ArtifactSupplier], return the full representation of
+   * a version string, if different from [PublishedArtifact.version].
+   */
+  fun getFullVersionString(artifact: PublishedArtifact): String = artifact.version
+
+  /**
+   * Given a [PublishedArtifact] supported by this [ArtifactSupplier], return the display name for the
+   * artifact version, if different from [PublishedArtifact.version].
+   */
+  fun getVersionDisplayName(artifact: PublishedArtifact): String = artifact.version
+
+  /**
+   * Given a [PublishedArtifact] supported by this [ArtifactSupplier], return the [ArtifactStatus] for
+   * the artifact, if applicable.
+   */
+  fun getReleaseStatus(artifact: PublishedArtifact): ArtifactStatus? = null
+
+  /**
+   * Given a [PublishedArtifact] and a [VersioningStrategy] supported by this [ArtifactSupplier],
+   * return the [BuildMetadata] for the artifact, if available.
+   *
+   * This function is currently *not* expected to make calls to other systems, but only look into
+   * the metadata available within the [PublishedArtifact] object itself.
+   */
+  fun getBuildMetadata(artifact: PublishedArtifact, versioningStrategy: VersioningStrategy): BuildMetadata? = null
+
+  /**
+   * Given a [PublishedArtifact] and a [VersioningStrategy] supported by this [ArtifactSupplier],
+   * return the [GitMetadata] for the artifact, if available.
+   *
+   * This function is currently *not* expected to make calls to other systems, but only look into
+   * the metadata available within the [PublishedArtifact] object itself.
+   */
+  fun getGitMetadata(artifact: PublishedArtifact, versioningStrategy: VersioningStrategy): GitMetadata? = null
+}
+
+/**
+ * Return the [ArtifactSupplier] supporting the specified artifact type.
+ */
+fun List<ArtifactSupplier<*>>.supporting(type: ArtifactType) =
+  find { it.supportedArtifact.name.toLowerCase() == type.toLowerCase() }
+    ?: error("Artifact type '$type' is not supported.")

keel-api/src/main/kotlin/com/netflix/spinnaker/keel/api/plugins/SupportedArtifact.kt

@@ -5,7 +5,7 @@ import com.netflix.spinnaker.keel.api.artifacts.DeliveryArtifact
 /**
  * Tuple of [DeliveryArtifact] type name (e.g. "deb", "docker", etc.) and the
  * corresponding [DeliveryArtifact] sub-class, used to facilitate registration
- * and discovery of supported artifact types from [ArtifactPublisher] plugins.
+ * and discovery of supported artifact types from [ArtifactSupplier] plugins.
  */
 data class SupportedArtifact<T : DeliveryArtifact>(
   val name: String,

keel-api/src/main/kotlin/com/netflix/spinnaker/keel/api/plugins/SupportedVersioningStrategy.kt

@@ -5,7 +5,7 @@ import com.netflix.spinnaker.keel.api.artifacts.VersioningStrategy
 /**
  * Tuple of [VersioningStrategy] type name (e.g. "deb", "docker", etc.) and the
  * corresponding [VersioningStrategy] sub-class, used to facilitate registration
- * and discovery of supported versioning strategies from [ArtifactPublisher] plugins.
+ * and discovery of supported versioning strategies from [ArtifactSupplier] plugins.
  */
 data class SupportedVersioningStrategy<T : VersioningStrategy>(
   val name: String,