Enable Html stripping (#478)

core/src/main/scala/com/salesforce/op/dsl/RichMapFeature.scala

@@ -277,6 +277,7 @@ trait RichMapFeature {
  * options are from the full entry or from the tokens
  * @param minLengthStdDev minimum standard deviation of the lengths of tokens in a text field for it to
  * be hashed instead of ignored
+ * @param stripHtml indicates whether to strip HTML tags from the text or not before analyzing
  * @param others additional text features
  * @return result feature of type Vector
  */
@@ -304,6 +305,7 @@ trait RichMapFeature {
  hashAlgorithm: HashAlgorithm = TransmogrifierDefaults.HashAlgorithm,
  textLengthType: TextLengthType = SmartTextVectorizer.LengthType,
  minLengthStdDev: Double = SmartTextVectorizer.MinTextLengthStdDev,
+ stripHtml: Boolean = TextTokenizer.StripHtml,
  others: Array[FeatureLike[TextMap]] = Array.empty
  ): FeatureLike[OPVector] = {
  // scalastyle:on parameter.number
@@ -318,6 +320,7 @@ trait RichMapFeature {
  .setAutoDetectThreshold(autoDetectThreshold)
  .setDefaultLanguage(defaultLanguage)
  .setMinTokenLength(minTokenLength)
+ .setStripHtml(stripHtml)
  .setToLowercase(toLowercase)
  .setTopK(topK)
  .setMinSupport(minSupport)
@@ -426,10 +429,9 @@ trait RichMapFeature {
  * @param defaultLanguage default language to assume in case autoDetectLanguage is disabled or
  * failed to make a good enough prediction.
  * @param hashAlgorithm hash algorithm to use
- * @param tokenizeForLengths If true, then the length counts will be lengths of the tokens in the entries.
- * If false, then the length counts will be the lengths of the entire entries
  * @param minLengthStdDev minimum standard deviation of the lengths of tokens in a text field for it to
  * be hashed instead of ignored
+ * @param stripHtml indicates whether to strip HTML tags from the text or not before analyzing
  * @param others additional text features
  * @return result feature of type Vector
  */
@@ -457,6 +459,7 @@ trait RichMapFeature {
  hashAlgorithm: HashAlgorithm = TransmogrifierDefaults.HashAlgorithm,
  textLengthType: TextLengthType = SmartTextVectorizer.LengthType,
  minLengthStdDev: Double = SmartTextVectorizer.MinTextLengthStdDev,
+ stripHtml: Boolean = TextTokenizer.StripHtml,
  others: Array[FeatureLike[TextAreaMap]] = Array.empty
  ): FeatureLike[OPVector] = {
  // scalastyle:on parameter.number
@@ -471,6 +474,7 @@ trait RichMapFeature {
  .setAutoDetectThreshold(autoDetectThreshold)
  .setDefaultLanguage(defaultLanguage)
  .setMinTokenLength(minTokenLength)
+ .setStripHtml(stripHtml)
  .setToLowercase(toLowercase)
  .setTopK(topK)
  .setMinSupport(minSupport)

core/src/main/scala/com/salesforce/op/dsl/RichTextFeature.scala

@@ -115,6 +115,7 @@ trait RichTextFeature {
  * confidence greater than the threshold then defaultLanguage is used.
  * @param hashSpaceStrategy strategy to determine whether to use shared hash space for all included features
  * @param minTokenLength minimum token length, >= 1.
+ * @param stripHtml indicates whether to strip HTML tags from the text or not before analyzing
  * @param trackNulls indicates whether or not to track null values in a separate column.
  * Since features may be combined into a shared hash space here, the null value
  * should be tracked separately
@@ -137,6 +138,7 @@ trait RichTextFeature {
  autoDetectLanguage: Boolean,
  minTokenLength: Int,
  toLowercase: Boolean,
+ stripHtml: Boolean = TextTokenizer.StripHtml,
  trackNulls: Boolean = TransmogrifierDefaults.TrackNulls,
  trackTextLen: Boolean = TransmogrifierDefaults.TrackTextLen,
  hashWithIndex: Boolean = TransmogrifierDefaults.HashWithIndex,
@@ -153,7 +155,7 @@ trait RichTextFeature {
  // scalastyle:on parameter.number
  val tokenized = (f +: others).map(_.tokenize(
  languageDetector = languageDetector,
- analyzer = analyzer,
+ analyzer = if (stripHtml) TextTokenizer.AnalyzerHtmlStrip else analyzer,
  autoDetectLanguage = autoDetectLanguage,
  autoDetectThreshold = autoDetectThreshold,
  defaultLanguage = defaultLanguage,
@@ -241,6 +243,7 @@ trait RichTextFeature {
  hashAlgorithm: HashAlgorithm = TransmogrifierDefaults.HashAlgorithm,
  textLengthType: TextLengthType = SmartTextVectorizer.LengthType,
  minLengthStdDev: Double = SmartTextVectorizer.MinTextLengthStdDev,
+ stripHtml: Boolean = TextTokenizer.StripHtml,
  others: Array[FeatureLike[T]] = Array.empty
  ): FeatureLike[OPVector] = {
  // scalastyle:on parameter.number
@@ -254,6 +257,7 @@ trait RichTextFeature {
  .setAutoDetectThreshold(autoDetectThreshold)
  .setDefaultLanguage(defaultLanguage)
  .setMinTokenLength(minTokenLength)
+ .setStripHtml(stripHtml)
  .setToLowercase(toLowercase)
  .setTopK(topK)
  .setMinSupport(minSupport)
@@ -375,7 +379,7 @@ trait RichTextFeature {
  minTokenLength: Int = TextTokenizer.MinTokenLength,
  toLowercase: Boolean = TextTokenizer.ToLowercase
  ): FeatureLike[TextList] = {
-
+ // html stripping won't work here due since LuceneRegexTextAnalyzer
  tokenize(
  languageDetector = TextTokenizer.LanguageDetector,
  analyzer = new LuceneRegexTextAnalyzer(pattern, group),

core/src/main/scala/com/salesforce/op/stages/impl/feature/SmartTextMapVectorizer.scala

@@ -221,6 +221,7 @@ class SmartTextMapVectorizer[T <: OPMap[String]]
  .setMinTokenLength(getMinTokenLength)
  .setToLowercase(getToLowercase)
  .setTrackTextLen($(trackTextLen))
+ .setStripHtml(getStripHtml)
  }
 }
 

core/src/main/scala/com/salesforce/op/stages/impl/feature/SmartTextVectorizer.scala

@@ -148,6 +148,7 @@ class SmartTextVectorizer[T <: Text](uid: String = UID[SmartTextVectorizer[T]])(
  .setMinTokenLength(getMinTokenLength)
  .setToLowercase(getToLowercase)
  .setTrackTextLen($(trackTextLen))
+ .setStripHtml(getStripHtml)
  }
 
  private def makeVectorMetadata(smartTextParams: SmartTextVectorizerModelArgs): OpVectorMetadata = {

core/src/main/scala/com/salesforce/op/stages/impl/feature/TextTokenizer.scala

@@ -83,18 +83,24 @@ trait TextTokenizerParams extends LanguageDetectionParams with TextMatchingParam
  def setMinTokenLength(value: Int): this.type = set(minTokenLength, value)
  def getMinTokenLength: Int = $(minTokenLength)
 
+ final val stripHtml =
+ new BooleanParam(this, "stripHtml", "enable html stripping")
+ def setStripHtml(value: Boolean): this.type = set(stripHtml, value)
+ def getStripHtml: Boolean = $(stripHtml)
+
  setDefault(
  minTokenLength -> TextTokenizer.MinTokenLength,
  toLowercase -> TextTokenizer.ToLowercase,
  autoDetectLanguage -> TextTokenizer.AutoDetectLanguage,
  autoDetectThreshold -> TextTokenizer.AutoDetectThreshold,
- defaultLanguage -> TextTokenizer.DefaultLanguage.entryName
+ defaultLanguage -> TextTokenizer.DefaultLanguage.entryName,
+ stripHtml -> TextTokenizer.StripHtml
  )
 
  def tokenize(
  text: Text,
  languageDetector: LanguageDetector = TextTokenizer.LanguageDetector,
- analyzer: TextAnalyzer = TextTokenizer.Analyzer
+ analyzer: TextAnalyzer = if (getStripHtml) TextTokenizer.AnalyzerHtmlStrip else TextTokenizer.Analyzer
  ): TextTokenizerResult = TextTokenizer.tokenize(
  text = text,
  languageDetector = languageDetector,

core/src/main/scala/com/salesforce/op/stages/impl/feature/Transmogrifier.scala

@@ -53,7 +53,7 @@ private[op] trait TransmogrifierDefaults {
  val NullString: String = OpVectorColumnMetadata.NullString
  val OtherString: String = OpVectorColumnMetadata.OtherString
  val DefaultNumOfFeatures: Int = 512
- val MaxNumOfFeatures: Int = 16384
+ val MaxNumOfFeatures: Int = 1 << 17 // 2^17
  val DateListDefault: DateListPivot = DateListPivot.SinceLast
  val ReferenceDate: org.joda.time.DateTime = DateTimeUtils.now()
  val TopK: Int = 20

core/src/test/scala/com/salesforce/op/stages/impl/feature/SmartTextMapVectorizerTest.scala

@@ -772,6 +772,16 @@ class SmartTextMapVectorizerTest
  checkDerivedQuantities(res, "f2", Seq(4, 5, 5, 5, 3).map(_.toLong))
  }
 
+ it should "turn on stripHTML flag is equivalent to passing in a custom AnalyzerHtmlStrip" +
+ "inside SmartTextMapVectorizer" in {
+ val exampleHTML = "<body>Big ones, small <h1>ones</h1>, some as big as your head</body>".toText
+ val tokensWithFlag = new SmartTextMapVectorizer()
+ .setStripHtml(true).setInput(m1).tokenize(exampleHTML).tokens.value
+ val tokensWithAnalyzer = new SmartTextMapVectorizer().setInput(m1)
+ .tokenize(exampleHTML, analyzer = TextTokenizer.AnalyzerHtmlStrip).tokens.value
+ tokensWithFlag should contain theSameElementsInOrderAs tokensWithAnalyzer
+ }
+
  private[op] def assertVectorLength(df: DataFrame, output: FeatureLike[OPVector],
  expectedLength: Int, textVectorizationMethod: TextVectorizationMethod): Unit = {
  val result = df.collect(output)

core/src/test/scala/com/salesforce/op/stages/impl/feature/SmartTextVectorizerTest.scala

@@ -712,4 +712,14 @@ class SmartTextVectorizerTest
  ts.lengthStdDev.isNaN shouldBe true
  }
 
+ it should "turn on stripHTML flag is equivalent to passing in a custom AnalyzerHtmlStrip" +
+ "inside SmartTextVectorizer" in {
+ val exampleHTML = "<body>Big ones, small <h1>ones</h1>, some as big as your head</body>".toText
+ val tokensWithFlag = new SmartTextVectorizer()
+ .setStripHtml(true).setInput(f1).tokenize(exampleHTML).tokens.value
+ val tokensWithAnalyzer = new SmartTextVectorizer().setInput(f1)
+ .tokenize(exampleHTML, analyzer = TextTokenizer.AnalyzerHtmlStrip).tokens.value
+ tokensWithFlag should contain theSameElementsInOrderAs tokensWithAnalyzer
+ }
+
 }