Merge branch '2.4.x' into additional_events

RasaHQ · Mar 24, 2021 · 792ebdd · 792ebdd
2 parents cc6b633 + 35651b9
commit 792ebdd
Show file tree

Hide file tree

Showing 5 changed files with 187 additions and 1 deletion.
diff --git a/changelog/7835.bugfix.md b/changelog/7835.bugfix.md
@@ -0,0 +1 @@
+Fix `UnicodeException` in `is_key_in_yaml`.
diff --git a/data/test_nlu_no_responses/nlu_with_unicode.yml b/data/test_nlu_no_responses/nlu_with_unicode.yml
@@ -0,0 +1,163 @@
+version: "2.0"
+
+nlu:
+- intent: greet
+  examples: |
+    - hey
+    - hello
+    - hi
+    - hello there
+    - good morning
+    - good evening
+    - moin
+    - hey there
+    - let's go
+    - hey dude
+    - goodmorning
+    - goodevening
+    - good afternoon
+    
+
+- intent: goodbye
+  examples: |
+    - good afternoon
+    - cu
+    - good by
+    - cee you later
+    - good night
+    - bye
+    - goodbye
+    - have a nice day
+    - see you around
+    - bye bye
+    - see you later
+
+- intent: affirm
+  examples: |
+    - yes
+    - y
+    - indeed
+    - of course
+    - that sounds good
+    - correct
+
+- intent: deny
+  examples: |
+    - no
+    - n
+    - never
+    - I don't think so
+    - don't like that
+    - no way
+    - not really
+
+- intent: mood_great
+  examples: |
+    - perfect
+    - great
+    - amazing
+    - feeling like a king
+    - wonderful
+    - I am feeling very good
+    - I am great
+    - I am amazing
+    - I am going to save the world
+    - super stoked
+    - extremely good
+    - so so perfect
+    - so good
+    - so perfect
+
+- intent: mood_unhappy
+  examples: |
+    - my day was horrible
+    - I am sad
+    - I don't feel very well
+    - I am disappointed
+    - super sad
+    - I'm so sad
+    - sad
+    - very sad
+    - unhappy
+    - not good
+    - not very good
+    - extremly sad
+    - so saad
+    - so sad
+
+- intent: bot_challenge
+  examples: |
+    - are you a bot?
+    - are you a human?
+    - am I talking to a bot?
+    - am I talking to a human?
+
+- intent: internet_problem
+  examples: |
+    - my internet is slow
+    - i have very slow wifi
+    - my internet is very bad
+    - internet is down
+    - router not working
+    - modem not giving internet
+    - modem problem
+    - i cant use the internet
+    - i cant go to google
+    - google is not working
+    - ma connection est lente
+    - j'ai un wifi très lent
+    - mon internet est très mauvais
+    - Internet est en panne
+    - le routeur ne marche pas
+    - le modem ne donne pas Internet
+    - problème de modem
+    - je ne peux pas utiliser Internet
+    - je ne peux pas aller sur google
+    - google ne fonctionne pas
+    - mafi internet
+    - ma 3ande wifi
+    - mech meche el internet
+    - ma fie fout 3al internet
+    - ma fie esta3mil el wifi
+    - el connextion manna mechye
+    - internet khara
+    - internet 3atil
+    - el router mano meche
+    - mech edir fout 3ala el facebook
+    - الانترنت لدي بطيء
+    - لدي wifi بطيء جدًا
+    - الإنترنت الخاص بي سيء للغاية
+    - الإنترنت معطل
+    - جهاز التوجيه لا يعمل
+    - مودم لا يعطي الإنترنت
+    - مشكلة المودم
+    - لا أستطيع استخدام الإنترنت
+    - لا أستطيع الذهاب إلى جوجل
+    - جوجل لا يعمل
+    - իմ ինտերնետը դանդաղ է
+    - ես շատ դանդաղ wifi ունեմ
+    - իմ ինտերնետը շատ վատն է
+    - ինտերնետն անջատված է
+    - երթուղիչը չի աշխատում
+    - մոդեմ, որը ինտերնետ չի տալիս
+    - մոդեմի խնդիր
+    - Ես չեմ կարող օգտվել ինտերնետից
+    - չեմ կարող մտնել google
+    - google- ը չի աշխատում
+    - my connection is down
+    - my internet is bad
+    - my wifi is slow
+    - my interent is not working
+    - i have internet problems
+    - i have bad wifi
+    - i have called ogero on 1515 and they have resolved the noise issue
+    - my connection is still down
+    - i reached to ogero on 1515 and they solved the noise
+    - check my connection
+    - I contacted ogero
+    - i don't have internet
+    - wifi khara
+    - internet us slow
+    - My internet is slow
+    - BAD INTERNET
+    - my internt is rlly shitty
diff --git a/rasa/shared/core/training_data/story_reader/yaml_story_reader.py b/rasa/shared/core/training_data/story_reader/yaml_story_reader.py
@@ -179,7 +179,9 @@ def is_key_in_yaml(cls, file_path: Union[Text, Path], *keys: Text) -> bool:
             FileNotFoundException: if the file cannot be found.
         """
         try:
-            with open(file_path) as file:
+            with open(
+                file_path, encoding=rasa.shared.utils.io.DEFAULT_ENCODING
+            ) as file:
                 return any(
                     any(line.lstrip().startswith(f"{key}:") for key in keys)
                     for line in file

diff --git a/tests/shared/core/training_data/story_reader/test_yaml_story_reader.py b/tests/shared/core/training_data/story_reader/test_yaml_story_reader.py
@@ -224,6 +224,13 @@ async def test_is_key_in_yaml(file: Text, keys: List[Text], expected_result: boo
     assert YAMLStoryReader.is_key_in_yaml(file, *keys) == expected_result
 
 
+async def test_is_key_in_yaml_with_unicode_files():
+    # This shouldn't raise
+    assert YAMLStoryReader.is_key_in_yaml(
+        "./data/test_nlu_no_responses/nlu_with_unicode.yml", "nlu"
+    )
+
+
 async def test_yaml_intent_with_leading_slash_warning(default_domain: Domain):
     yaml_file = "data/test_wrong_yaml_stories/intent_with_leading_slash.yml"
 

diff --git a/tests/shared/importers/test_importer.py b/tests/shared/importers/test_importer.py
@@ -360,3 +360,16 @@ def test_importer_with_invalid_model_config(tmp_path: Path):
 
     with pytest.raises(YamlValidationException):
         TrainingDataImporter.load_from_config(str(config_file))
+
+
+async def test_importer_with_unicode_files():
+    importer = TrainingDataImporter.load_from_dict(
+        training_data_paths=["./data/test_nlu_no_responses/nlu_with_unicode.yml"]
+    )
+
+    # None of these should raise
+    nlu_data = await importer.get_nlu_data()
+    assert not nlu_data.is_empty()
+
+    await importer.get_stories()
+    await importer.get_domain()