Force unicode conversion.

chatterbot/ext/sqlalchemy_app/models.py

@@ -1,7 +1,8 @@
-from sqlalchemy import Table, Column, Integer, String, DateTime, ForeignKey, PickleType
+from sqlalchemy import Table, Column, Integer, DateTime, ForeignKey, PickleType
 from sqlalchemy.orm import relationship
 from sqlalchemy.sql import func
 from sqlalchemy.ext.declarative import declared_attr, declarative_base
+from chatterbot.ext.sqlalchemy_app.types import UnicodeString
 from chatterbot.conversation.statement import StatementMixin
 
 
@@ -40,15 +41,15 @@ class Tag(Base):
     A tag that describes a statement.
     """
 
-    name = Column(String)
+    name = Column(UnicodeString)
 
 
 class Statement(Base, StatementMixin):
     """
     A Statement represents a sentence or phrase.
     """
 
-    text = Column(String, unique=True)
+    text = Column(UnicodeString, unique=True)
 
     tags = relationship(
         'Tag',
@@ -90,7 +91,7 @@ class Response(Base):
     Response, contains responses related to a given statement.
     """
 
-    text = Column(String)
+    text = Column(UnicodeString)
 
     created_at = Column(
         DateTime(timezone=True),
@@ -99,7 +100,7 @@ class Response(Base):
 
     occurrence = Column(Integer, default=1)
 
-    statement_text = Column(String, ForeignKey('statement.text'))
+    statement_text = Column(UnicodeString, ForeignKey('statement.text'))
 
     statement_table = relationship(
         'Statement',

chatterbot/ext/sqlalchemy_app/types.py

@@ -0,0 +1,21 @@
+from sqlalchemy.types import TypeDecorator, Unicode
+
+
+class UnicodeString(TypeDecorator):
+    """
+    Type for unicode strings.
+    """
+
+    impl = Unicode
+
+    def process_bind_param(self, value, dialect):
+        """
+        Coerce Python bytestrings to unicode before
+        saving them to the database.
+        """
+        import sys
+
+        if sys.version_info[0] < 3:
+            if isinstance(value, str):
+                value = value.decode('utf-8')
+        return value