optional name detection by annotation (close #143)

pkg/ent/token/features.go

@@ -25,6 +25,11 @@ type Features struct {
  // Abbr feature: token ends with a period.
  Abbr bool
 
+ // HasAnnotation is true if there is an apparent nomenclatural annotation
+ // attached to a name candidate. This feature can only be set for the
+ // first token of a candidate.
+ HasAnnotation bool
+
  // PotentialBinomialGenus feature: the token might be a genus of name.
  PotentialBinomialGenus bool
 

pkg/ent/token/token.go

@@ -13,8 +13,8 @@ import (
  "github.com/gnames/gnfinder/pkg/io/dict"
 )
 
-// tokenSN represents a word separated by spaces in a text. Words that are
-// split by new lines are concatenated.
+// tokenSN (a token for a 'Scientific Name') represents a word separated by
+// spaces in a text. Words that are split by new lines are concatenated.
 type tokenSN struct {
  gner.TokenNER
 
@@ -218,10 +218,13 @@ func checkRank(t TokenSN, d *dict.Dictionary) bool {
 
 // UpperIndex takes an index of a token and length of the tokens slice and
 // returns an upper index of what could be a slice of a name. We expect that
-// that most of the names will fit into 5 words. Other cases would require
-// more thorough algorithims that we can run later as plugins.
+// that most of the names will fit into 5 words. We also expect that annotation
+// will be located not further away than 10 words away.
+// Other cases would require more thorough algorithims that we can run
+// later as plugins.
 func UpperIndex(i int, l int) int {
- upperIndex := i + 5
+ // 10 here is 5 tokens for name, 5 tokens for annotation
+ upperIndex := i + 10
  if l < upperIndex {
  upperIndex = l
  }