From 96d05cbf993e2316d0143d91f102f9eddb6778f6 Mon Sep 17 00:00:00 2001
From: Tobias Wochinger <t.wochinger@rasa.com>
Date: Wed, 24 Mar 2021 12:06:41 +0100
Subject: [PATCH] add test

---
 .../nlu_with_unicode.yml                      | 163 ++++++++++++++++++
 tests/shared/importers/test_importer.py       |  13 ++
 2 files changed, 176 insertions(+)
 create mode 100644 data/test_nlu_no_responses/nlu_with_unicode.yml

diff --git a/data/test_nlu_no_responses/nlu_with_unicode.yml b/data/test_nlu_no_responses/nlu_with_unicode.yml
new file mode 100644
index 000000000000..f3dae3c3f89a
--- /dev/null
+++ b/data/test_nlu_no_responses/nlu_with_unicode.yml
@@ -0,0 +1,163 @@
+version: "2.0"
+
+nlu:
+- intent: greet
+  examples: |
+    - hey
+    - hello
+    - hi
+    - hello there
+    - good morning
+    - good evening
+    - moin
+    - hey there
+    - let's go
+    - hey dude
+    - goodmorning
+    - goodevening
+    - good afternoon
+    
+
+- intent: goodbye
+  examples: |
+    - good afternoon
+    - cu
+    - good by
+    - cee you later
+    - good night
+    - bye
+    - goodbye
+    - have a nice day
+    - see you around
+    - bye bye
+    - see you later
+
+- intent: affirm
+  examples: |
+    - yes
+    - y
+    - indeed
+    - of course
+    - that sounds good
+    - correct
+
+- intent: deny
+  examples: |
+    - no
+    - n
+    - never
+    - I don't think so
+    - don't like that
+    - no way
+    - not really
+
+- intent: mood_great
+  examples: |
+    - perfect
+    - great
+    - amazing
+    - feeling like a king
+    - wonderful
+    - I am feeling very good
+    - I am great
+    - I am amazing
+    - I am going to save the world
+    - super stoked
+    - extremely good
+    - so so perfect
+    - so good
+    - so perfect
+
+- intent: mood_unhappy
+  examples: |
+    - my day was horrible
+    - I am sad
+    - I don't feel very well
+    - I am disappointed
+    - super sad
+    - I'm so sad
+    - sad
+    - very sad
+    - unhappy
+    - not good
+    - not very good
+    - extremly sad
+    - so saad
+    - so sad
+
+- intent: bot_challenge
+  examples: |
+    - are you a bot?
+    - are you a human?
+    - am I talking to a bot?
+    - am I talking to a human?
+
+- intent: internet_problem
+  examples: |
+    - my internet is slow
+    - i have very slow wifi
+    - my internet is very bad
+    - internet is down
+    - router not working
+    - modem not giving internet
+    - modem problem
+    - i cant use the internet
+    - i cant go to google
+    - google is not working
+    - ma connection est lente
+    - j'ai un wifi très lent
+    - mon internet est très mauvais
+    - Internet est en panne
+    - le routeur ne marche pas
+    - le modem ne donne pas Internet
+    - problème de modem
+    - je ne peux pas utiliser Internet
+    - je ne peux pas aller sur google
+    - google ne fonctionne pas
+    - mafi internet
+    - ma 3ande wifi
+    - mech meche el internet
+    - ma fie fout 3al internet
+    - ma fie esta3mil el wifi
+    - el connextion manna mechye
+    - internet khara
+    - internet 3atil
+    - el router mano meche
+    - mech edir fout 3ala el facebook
+    - الانترنت لدي بطيء
+    - لدي wifi بطيء جدًا
+    - الإنترنت الخاص بي سيء للغاية
+    - الإنترنت معطل
+    - جهاز التوجيه لا يعمل
+    - مودم لا يعطي الإنترنت
+    - مشكلة المودم
+    - لا أستطيع استخدام الإنترنت
+    - لا أستطيع الذهاب إلى جوجل
+    - جوجل لا يعمل
+    - իմ ինտերնետը դանդաղ է
+    - ես շատ դանդաղ wifi ունեմ
+    - իմ ինտերնետը շատ վատն է
+    - ինտերնետն անջատված է
+    - երթուղիչը չի աշխատում
+    - մոդեմ, որը ինտերնետ չի տալիս
+    - մոդեմի խնդիր
+    - Ես չեմ կարող օգտվել ինտերնետից
+    - չեմ կարող մտնել google
+    - google- ը չի աշխատում
+    - my connection is down
+    - my internet is bad
+    - my wifi is slow
+    - my interent is not working
+    - i have internet problems
+    - i have bad wifi
+    - i have called ogero on 1515 and they have resolved the noise issue
+    - my connection is still down
+    - i reached to ogero on 1515 and they solved the noise
+    - check my connection
+    - I contacted ogero
+    - i don't have internet
+    - wifi khara
+    - internet us slow
+    - My internet is slow
+    - BAD INTERNET
+    - my internt is rlly shitty
diff --git a/tests/shared/importers/test_importer.py b/tests/shared/importers/test_importer.py
index 6e1534847e75..e2a7d8ea4bf1 100644
--- a/tests/shared/importers/test_importer.py
+++ b/tests/shared/importers/test_importer.py
@@ -360,3 +360,16 @@ def test_importer_with_invalid_model_config(tmp_path: Path):
 
     with pytest.raises(YamlValidationException):
         TrainingDataImporter.load_from_config(str(config_file))
+
+
+async def test_importer_with_unicode_files():
+    importer = TrainingDataImporter.load_from_dict(
+        training_data_paths=["./data/test_nlu_no_responses/nlu_with_unicode.yml"]
+    )
+
+    # None of these should raise
+    nlu_data = await importer.get_nlu_data()
+    assert not nlu_data.is_empty()
+
+    await importer.get_stories()
+    await importer.get_domain()