Add lowercase tagger and rename indexing method to be more accurate

chatterbot/chatterbot.py

@@ -108,10 +108,10 @@ def get_response(self, statement=None, **kwargs):
  # Make sure the input statement has its search text saved
 
  if not input_statement.search_text:
- input_statement.search_text = self.storage.tagger.get_bigram_pair_string(input_statement.text)
+ input_statement.search_text = self.storage.tagger.get_text_index_string(input_statement.text)
 
  if not input_statement.search_in_response_to and input_statement.in_response_to:
- input_statement.search_in_response_to = self.storage.tagger.get_bigram_pair_string(input_statement.in_response_to)
+ input_statement.search_in_response_to = self.storage.tagger.get_text_index_string(input_statement.in_response_to)
 
  response = self.generate_response(input_statement, additional_response_selection_parameters)
 

chatterbot/comparisons.py

@@ -108,6 +108,7 @@ class JaccardSimilarity(Comparator):
 
  def __init__(self, language):
  super().__init__(language)
+ import spacy
 
  self.nlp = spacy.load(self.language.ISO_639_1)
 

chatterbot/logic/best_match.py

@@ -57,7 +57,7 @@ def process(self, input_statement, additional_response_selection_parameters=None
  }
 
  alternate_response_selection_parameters = {
- 'search_in_response_to': self.chatbot.storage.tagger.get_bigram_pair_string(
+ 'search_in_response_to': self.chatbot.storage.tagger.get_text_index_string(
  input_statement.text
  ),
  'exclude_text': recent_repeated_responses,

chatterbot/search.py

@@ -50,7 +50,7 @@ def search(self, input_statement, **additional_parameters):
  'No value for search_text was available on the provided input'
  )
 
- input_search_text = self.chatbot.storage.tagger.get_bigram_pair_string(
+ input_search_text = self.chatbot.storage.tagger.get_text_index_string(
  input_statement.text
  )
 

chatterbot/storage/django_storage.py

@@ -100,11 +100,11 @@ def create(self, **kwargs):
  tags = kwargs.pop('tags', [])
 
  if 'search_text' not in kwargs:
- kwargs['search_text'] = self.tagger.get_bigram_pair_string(kwargs['text'])
+ kwargs['search_text'] = self.tagger.get_text_index_string(kwargs['text'])
 
  if 'search_in_response_to' not in kwargs:
  if kwargs.get('in_response_to'):
- kwargs['search_in_response_to'] = self.tagger.get_bigram_pair_string(kwargs['in_response_to'])
+ kwargs['search_in_response_to'] = self.tagger.get_text_index_string(kwargs['in_response_to'])
 
  statement = Statement(**kwargs)
 
@@ -137,10 +137,10 @@ def create_many(self, statements):
  statement_model_object = Statement(**statement_data)
 
  if not statement.search_text:
- statement_model_object.search_text = self.tagger.get_bigram_pair_string(statement.text)
+ statement_model_object.search_text = self.tagger.get_text_index_string(statement.text)
 
  if not statement.search_in_response_to and statement.in_response_to:
- statement_model_object.search_in_response_to = self.tagger.get_bigram_pair_string(statement.in_response_to)
+ statement_model_object.search_in_response_to = self.tagger.get_text_index_string(statement.in_response_to)
 
  statement_model_object.save()
 
@@ -168,10 +168,10 @@ def update(self, statement):
  else:
  statement = Statement.objects.create(
  text=statement.text,
- search_text=self.tagger.get_bigram_pair_string(statement.text),
+ search_text=self.tagger.get_text_index_string(statement.text),
  conversation=statement.conversation,
  in_response_to=statement.in_response_to,
- search_in_response_to=self.tagger.get_bigram_pair_string(statement.in_response_to),
+ search_in_response_to=self.tagger.get_text_index_string(statement.in_response_to),
  created_at=statement.created_at
  )
 

chatterbot/storage/mongodb.py

@@ -159,11 +159,11 @@ def create(self, **kwargs):
  kwargs['tags'] = list(set(kwargs['tags']))
 
  if 'search_text' not in kwargs:
- kwargs['search_text'] = self.tagger.get_bigram_pair_string(kwargs['text'])
+ kwargs['search_text'] = self.tagger.get_text_index_string(kwargs['text'])
 
  if 'search_in_response_to' not in kwargs:
  if kwargs.get('in_response_to'):
- kwargs['search_in_response_to'] = self.tagger.get_bigram_pair_string(kwargs['in_response_to'])
+ kwargs['search_in_response_to'] = self.tagger.get_text_index_string(kwargs['in_response_to'])
 
  inserted = self.statements.insert_one(kwargs)
 
@@ -183,10 +183,10 @@ def create_many(self, statements):
  statement_data['tags'] = tag_data
 
  if not statement.search_text:
- statement_data['search_text'] = self.tagger.get_bigram_pair_string(statement.text)
+ statement_data['search_text'] = self.tagger.get_text_index_string(statement.text)
 
  if not statement.search_in_response_to and statement.in_response_to:
- statement_data['search_in_response_to'] = self.tagger.get_bigram_pair_string(statement.in_response_to)
+ statement_data['search_in_response_to'] = self.tagger.get_text_index_string(statement.in_response_to)
 
  create_statements.append(statement_data)
 
@@ -197,10 +197,10 @@ def update(self, statement):
  data.pop('id', None)
  data.pop('tags', None)
 
- data['search_text'] = self.tagger.get_bigram_pair_string(data['text'])
+ data['search_text'] = self.tagger.get_text_index_string(data['text'])
 
  if data.get('in_response_to'):
- data['search_in_response_to'] = self.tagger.get_bigram_pair_string(data['in_response_to'])
+ data['search_in_response_to'] = self.tagger.get_text_index_string(data['in_response_to'])
 
  update_data = {
  '$set': data

chatterbot/storage/sql_storage.py

@@ -186,12 +186,12 @@ def create(self, **kwargs):
  tags = set(kwargs.pop('tags', []))
 
  if 'search_text' not in kwargs:
- kwargs['search_text'] = self.tagger.get_bigram_pair_string(kwargs['text'])
+ kwargs['search_text'] = self.tagger.get_text_index_string(kwargs['text'])
 
  if 'search_in_response_to' not in kwargs:
  in_response_to = kwargs.get('in_response_to')
  if in_response_to:
- kwargs['search_in_response_to'] = self.tagger.get_bigram_pair_string(in_response_to)
+ kwargs['search_in_response_to'] = self.tagger.get_text_index_string(in_response_to)
 
  statement = Statement(**kwargs)
 
@@ -236,10 +236,10 @@ def create_many(self, statements):
  statement_model_object = Statement(**statement_data)
 
  if not statement.search_text:
- statement_model_object.search_text = self.tagger.get_bigram_pair_string(statement.text)
+ statement_model_object.search_text = self.tagger.get_text_index_string(statement.text)
 
  if not statement.search_in_response_to and statement.in_response_to:
- statement_model_object.search_in_response_to = self.tagger.get_bigram_pair_string(statement.in_response_to)
+ statement_model_object.search_in_response_to = self.tagger.get_text_index_string(statement.in_response_to)
 
  new_tags = set(tag_data) - set(create_tags.keys())
 
@@ -299,10 +299,10 @@ def update(self, statement):
 
  record.created_at = statement.created_at
 
- record.search_text = self.tagger.get_bigram_pair_string(statement.text)
+ record.search_text = self.tagger.get_text_index_string(statement.text)
 
  if statement.in_response_to:
- record.search_in_response_to = self.tagger.get_bigram_pair_string(statement.in_response_to)
+ record.search_in_response_to = self.tagger.get_text_index_string(statement.in_response_to)
 
  for tag_name in statement.get_tags():
  tag = session.query(Tag).filter_by(name=tag_name).first()

chatterbot/storage/storage_adapter.py

@@ -17,7 +17,9 @@ def __init__(self, *args, **kwargs):
  """
  self.logger = kwargs.get('logger', logging.getLogger(__name__))
 
- self.tagger = PosLemmaTagger(language=kwargs.get(
+ Tagger = kwargs.get('tagger', PosLemmaTagger)
+
+ self.tagger = Tagger(language=kwargs.get(
  'tagger_language', languages.ENG
  ))
 

chatterbot/tagging.py

@@ -1,18 +1,31 @@
 import string
 from chatterbot import languages
-import spacy
+
+
+class LowercaseTagger(object):
+ """
+ Returns the text in lowercase.
+ """
+
+ def __init__(self, language=None):
+ pass
+
+ def get_text_index_string(self, text):
+ return text.lower()
 
 
 class PosLemmaTagger(object):
 
  def __init__(self, language=None):
+ import spacy
+
  self.language = language or languages.ENG
 
  self.punctuation_table = str.maketrans(dict.fromkeys(string.punctuation))
 
  self.nlp = spacy.load(self.language.ISO_639_1.lower())
 
- def get_bigram_pair_string(self, text):
+ def get_text_index_string(self, text):
  """
  Return a string of text containing part-of-speech, lemma pairs.
  """

chatterbot/trainers.py

@@ -97,7 +97,7 @@ def train(self, conversation):
  conversation_count + 1, len(conversation)
  )
 
- statement_search_text = self.chatbot.storage.tagger.get_bigram_pair_string(text)
+ statement_search_text = self.chatbot.storage.tagger.get_text_index_string(text)
 
  statement = self.get_preprocessed_statement(
  Statement(
@@ -151,7 +151,7 @@ def train(self, *corpus_paths):
 
  for text in conversation:
 
- statement_search_text = self.chatbot.storage.tagger.get_bigram_pair_string(text)
+ statement_search_text = self.chatbot.storage.tagger.get_text_index_string(text)
 
  statement = Statement(
  text=text,
@@ -336,7 +336,7 @@ def chunks(items, items_per_chunk):
  for preprocessor in self.chatbot.preprocessors:
  statement = preprocessor(statement)
 
- statement.search_text = tagger.get_bigram_pair_string(statement.text)
+ statement.search_text = tagger.get_text_index_string(statement.text)
  statement.search_in_response_to = previous_statement_search_text
 
  previous_statement_text = statement.text

tests/test_chatbot.py

@@ -308,7 +308,7 @@ def test_search_text_results_after_training(self):
  ])
 
  results = list(self.chatbot.storage.filter(
- search_text=self.chatbot.storage.tagger.get_bigram_pair_string(
+ search_text=self.chatbot.storage.tagger.get_text_index_string(
  'Example A for search.'
  )
  ))

tests/test_tagging.py

@@ -9,14 +9,14 @@ def setUp(self):
  self.tagger = tagging.PosLemmaTagger()
 
  def test_empty_string(self):
- tagged_text = self.tagger.get_bigram_pair_string(
+ tagged_text = self.tagger.get_text_index_string(
  ''
  )
 
  self.assertEqual(tagged_text, '')
 
  def test_tagging(self):
- tagged_text = self.tagger.get_bigram_pair_string(
+ tagged_text = self.tagger.get_text_index_string(
  'Hello, how are you doing on this awesome day?'
  )
 
@@ -27,7 +27,7 @@ def test_tagging_english(self):
  language=languages.ENG
  )
 
- tagged_text = self.tagger.get_bigram_pair_string(
+ tagged_text = self.tagger.get_text_index_string(
  'Hello, how are you doing on this awesome day?'
  )
 
@@ -38,99 +38,99 @@ def test_tagging_german(self):
  language=languages.GER
  )
 
- tagged_text = self.tagger.get_bigram_pair_string(
+ tagged_text = self.tagger.get_text_index_string(
  'Ich spreche nicht viel Deutsch.'
  )
 
  self.assertEqual(tagged_text, 'VERB:deutsch')
 
  def test_string_becomes_lowercase(self):
- tagged_text = self.tagger.get_bigram_pair_string('THIS IS HOW IT BEGINS!')
+ tagged_text = self.tagger.get_text_index_string('THIS IS HOW IT BEGINS!')
 
  self.assertEqual(tagged_text, 'DET:be VERB:how ADV:it NOUN:begin')
 
  def test_tagging_medium_sized_words(self):
- tagged_text = self.tagger.get_bigram_pair_string('Hello, my name is Gunther.')
+ tagged_text = self.tagger.get_text_index_string('Hello, my name is Gunther.')
 
  self.assertEqual(tagged_text, 'INTJ:gunther')
 
  def test_tagging_long_words(self):
- tagged_text = self.tagger.get_bigram_pair_string('I play several orchestra instruments for pleasure.')
+ tagged_text = self.tagger.get_text_index_string('I play several orchestra instruments for pleasure.')
 
  self.assertEqual(tagged_text, 'VERB:orchestra ADJ:instrument NOUN:pleasure')
 
- def test_get_bigram_pair_string_punctuation_only(self):
- bigram_string = self.tagger.get_bigram_pair_string(
+ def test_get_text_index_string_punctuation_only(self):
+ bigram_string = self.tagger.get_text_index_string(
  '?'
  )
 
  self.assertEqual(bigram_string, '?')
 
- def test_get_bigram_pair_string_single_character(self):
- bigram_string = self.tagger.get_bigram_pair_string(
+ def test_get_text_index_string_single_character(self):
+ bigram_string = self.tagger.get_text_index_string(
  '🙂'
  )
 
  self.assertEqual(bigram_string, '🙂')
 
- def test_get_bigram_pair_string_single_character_punctuated(self):
- bigram_string = self.tagger.get_bigram_pair_string(
+ def test_get_text_index_string_single_character_punctuated(self):
+ bigram_string = self.tagger.get_text_index_string(
  '🤷?'
  )
 
  self.assertEqual(bigram_string, '🤷')
 
- def test_get_bigram_pair_string_two_characters(self):
- bigram_string = self.tagger.get_bigram_pair_string(
+ def test_get_text_index_string_two_characters(self):
+ bigram_string = self.tagger.get_text_index_string(
  'AB'
  )
 
  self.assertEqual(bigram_string, 'ab')
 
- def test_get_bigram_pair_string_three_characters(self):
- bigram_string = self.tagger.get_bigram_pair_string(
+ def test_get_text_index_string_three_characters(self):
+ bigram_string = self.tagger.get_text_index_string(
  'ABC'
  )
 
  self.assertEqual(bigram_string, 'abc')
 
- def test_get_bigram_pair_string_four_characters(self):
- bigram_string = self.tagger.get_bigram_pair_string(
+ def test_get_text_index_string_four_characters(self):
+ bigram_string = self.tagger.get_text_index_string(
  'ABCD'
  )
 
  self.assertEqual(bigram_string, 'abcd')
 
- def test_get_bigram_pair_string_five_characters(self):
- bigram_string = self.tagger.get_bigram_pair_string(
+ def test_get_text_index_string_five_characters(self):
+ bigram_string = self.tagger.get_text_index_string(
  'ABCDE'
  )
 
  self.assertEqual(bigram_string, 'abcde')
 
- def test_get_bigram_pair_string_single_word(self):
- bigram_string = self.tagger.get_bigram_pair_string(
+ def test_get_text_index_string_single_word(self):
+ bigram_string = self.tagger.get_text_index_string(
  'Hello'
  )
 
  self.assertEqual(bigram_string, 'hello')
 
- def test_get_bigram_pair_string_multiple_words(self):
- bigram_string = self.tagger.get_bigram_pair_string(
+ def test_get_text_index_string_multiple_words(self):
+ bigram_string = self.tagger.get_text_index_string(
  'Hello Dr. Salazar. How are you today?'
  )
 
  self.assertEqual(bigram_string, 'INTJ:salazar PROPN:today')
 
- def test_get_bigram_pair_string_single_character_words(self):
- bigram_string = self.tagger.get_bigram_pair_string(
+ def test_get_text_index_string_single_character_words(self):
+ bigram_string = self.tagger.get_text_index_string(
  'a e i o u'
  )
 
  self.assertEqual(bigram_string, 'NOUN:o NOUN:u')
 
- def test_get_bigram_pair_string_two_character_words(self):
- bigram_string = self.tagger.get_bigram_pair_string(
+ def test_get_text_index_string_two_character_words(self):
+ bigram_string = self.tagger.get_text_index_string(
  'Lo my mu it is of us'
  )