From 0bab1d4c94f983b9c78d4be26417ef41cd19e92d Mon Sep 17 00:00:00 2001 From: Matthew Tovbin Date: Wed, 10 Apr 2019 14:25:35 -0700 Subject: [PATCH] Proper UID for Raw Feature Filter (#276) --- .../op/filters/RawFeatureFilter.scala | 3 +++ .../op/filters/RawFeatureFilterResults.scala | 23 ++++++++----------- .../com/salesforce/op/ModelInsightsTest.scala | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/core/src/main/scala/com/salesforce/op/filters/RawFeatureFilter.scala b/core/src/main/scala/com/salesforce/op/filters/RawFeatureFilter.scala index afb32fade7..4d0db28229 100644 --- a/core/src/main/scala/com/salesforce/op/filters/RawFeatureFilter.scala +++ b/core/src/main/scala/com/salesforce/op/filters/RawFeatureFilter.scala @@ -592,6 +592,9 @@ object RawFeatureFilter { // scoring sets since they will not be reliable. Currently, this is set to the same as the minimum training size. val minScoringRowsDefault = 500 + val stageName = classOf[RawFeatureFilter[_]].getSimpleName + + val uid = s"${stageName}_100000000000" } /** diff --git a/core/src/main/scala/com/salesforce/op/filters/RawFeatureFilterResults.scala b/core/src/main/scala/com/salesforce/op/filters/RawFeatureFilterResults.scala index 917f53a219..458f4077cd 100644 --- a/core/src/main/scala/com/salesforce/op/filters/RawFeatureFilterResults.scala +++ b/core/src/main/scala/com/salesforce/op/filters/RawFeatureFilterResults.scala @@ -39,13 +39,6 @@ import org.json4s.{DefaultFormats, Formats} import scala.util.Try -trait RawFeatureFilterFormats { - implicit val jsonFormats: Formats = DefaultFormats + - new SpecialDoubleSerializer + - EnumEntrySerializer.json4s[CorrelationType](CorrelationType) + - EnumEntrySerializer.json4s[FeatureDistributionType](FeatureDistributionType) -} - /** * Contains configuration and results from RawFeatureFilter * @@ -62,7 +55,15 @@ case class RawFeatureFilterResults exclusionReasons: Seq[ExclusionReasons] = Seq.empty ) +trait RawFeatureFilterFormats { + implicit val jsonFormats: Formats = DefaultFormats + + new SpecialDoubleSerializer + + EnumEntrySerializer.json4s[CorrelationType](CorrelationType) + + EnumEntrySerializer.json4s[FeatureDistributionType](FeatureDistributionType) +} + object RawFeatureFilterResults extends RawFeatureFilterFormats { + /** * RawFeatureFilterResults to json * @@ -77,8 +78,7 @@ object RawFeatureFilterResults extends RawFeatureFilterFormats { * @param json json * @return raw feature filter results */ - def fromJson(json: String): Try[RawFeatureFilterResults] = - Try { Serialization.read[RawFeatureFilterResults](json) } + def fromJson(json: String): Try[RawFeatureFilterResults] = Try { Serialization.read[RawFeatureFilterResults](json) } } @@ -116,10 +116,7 @@ object RawFeatureFilterConfig extends RawFeatureFilterFormats { * @return Map[String, Map[String, Any] ] */ def toStageInfo(config: RawFeatureFilterConfig): Map[String, Map[String, Any]] = { - val stageName = "rawFeatureFilter" - val uid = "rawFeatureFilter" - Map(stageName -> Map("uid" -> uid, "params" -> toStringMap(config)) - ) + Map(RawFeatureFilter.stageName -> Map("uid" -> RawFeatureFilter.uid, "params" -> toStringMap(config))) } } diff --git a/core/src/test/scala/com/salesforce/op/ModelInsightsTest.scala b/core/src/test/scala/com/salesforce/op/ModelInsightsTest.scala index fcb722b081..3402c0829a 100644 --- a/core/src/test/scala/com/salesforce/op/ModelInsightsTest.scala +++ b/core/src/test/scala/com/salesforce/op/ModelInsightsTest.scala @@ -403,7 +403,7 @@ class ModelInsightsTest extends FlatSpec with PassengerSparkFixtureTest with Dou // check that raw feature filter config is correctly serialized and deserialized def getRawFeatureFilterConfig(modelInsights: ModelInsights): Map[String, String] = { - modelInsights.stageInfo("rawFeatureFilter") match { + modelInsights.stageInfo(RawFeatureFilter.stageName) match { case configInfo: Map[String, Map[String, String]] => configInfo.getOrElse("params", Map.empty[String, String]) case _ => Map.empty[String, String]