PY3 compat

docs/conf.py

@@ -12,6 +12,7 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
+from __future__ import unicode_literals
 import sys
 import os
 import shlex

setup.py

@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 from setuptools import setup, find_packages
 
 

sippers/__init__.py

@@ -1,12 +1,13 @@
 from __future__ import absolute_import
+from __future__ import unicode_literals
 import os
 
 from osconf import config_from_environment
 
 try:
     VERSION = __import__('pkg_resources') \
         .get_distribution(__name__).version
-except Exception, e:
+except Exception as e:
     VERSION = 'unknown'
 
 

sippers/adapters/__init__.py

@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 from marshmallow import Schema, post_load
 from marshmallow.decorators import tag_processor
 from sippers.models import Document

sippers/adapters/cnmc.py

@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 from sippers.adapters import SipsAdapter, MeasuresAdapter
 from sippers.models import SipsSchema, MeasuresSchema
 from marshmallow import pre_load, fields
@@ -21,7 +22,7 @@ def add_distri_description(self, data):
 
     @pre_load
     def fix_dates(self, data):
-        for attr, field in self.fields.iteritems():
+        for attr, field in self.fields.items():
             if isinstance(field, fields.DateTime):
                 # si ve alguna data, assumim que ve correcta
                 if data[attr] == u'':   data[attr] = None
@@ -103,7 +104,7 @@ class CnmcMeasuresAdapter(MeasuresAdapter, MeasuresSchema):
 
     @pre_load
     def fix_dates(self, data):
-        for attr, field in self.fields.iteritems():
+        for attr, field in self.fields.items():
             if isinstance(field, fields.DateTime):
                 # si ve alguna dada, assumim que ve correcta
                 if data[attr] == u'':   data[attr] = None

sippers/adapters/cnmc_gas.py

@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 from sippers.adapters import SipsAdapter, MeasuresAdapter
 from sippers.models.cnmc_gas import CnmcGasSipsSchema, CnmcGasMeasuresSchema
 from marshmallow import pre_load, fields
@@ -27,7 +28,7 @@ def parse_propiedad_equipo_medida(self, data):
 
     @pre_load
     def fix_dates(self, data):
-        for attr, field in self.fields.iteritems():
+        for attr, field in self.fields.items():
             if isinstance(field, fields.DateTime):
                 # si ve alguna data, assumim que ve correcta
                 if data[attr] == u'':
@@ -51,7 +52,7 @@ def parse_selections(self, data):
 
     @pre_load
     def fix_dates(self, data):
-        for attr, field in self.fields.iteritems():
+        for attr, field in self.fields.items():
             if isinstance(field, fields.DateTime):
                 # si ve alguna data, assumim que ve correcta
                 if data[attr] == u'':

sippers/adapters/cnmc_v2.py

@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 from sippers.adapters import SipsAdapter, MeasuresAdapter
 from sippers.models.cnmc_v2 import CnmcV2SipsSchema, CnmcV2MeasuresSchema
 from marshmallow import pre_load, fields
@@ -36,7 +37,7 @@ def parse_selections(self, data):
 
     @pre_load
     def fix_dates(self, data):
-        for attr, field in self.fields.iteritems():
+        for attr, field in self.fields.items():
             if isinstance(field, fields.DateTime):
                 # si ve alguna data, assumim que ve correcta
                 if data[attr] == u'':
@@ -61,7 +62,7 @@ def parse_selections(self, data):
 
     @pre_load
     def fix_dates(self, data):
-        for attr, field in self.fields.iteritems():
+        for attr, field in self.fields.items():
             if isinstance(field, fields.DateTime):
                 # si ve alguna data, assumim que ve correcta
                 if data[attr] == u'':

sippers/adapters/endesa.py

@@ -1,8 +1,9 @@
+from __future__ import unicode_literals
 from sippers import logger
 from sippers.adapters import SipsAdapter, MeasuresAdapter
 from sippers.models import SipsSchema, MeasuresSchema, TARIFFS as BASE_TARIFFS
 from marshmallow import Schema, fields, pre_load
-
+from six import string_types
 
 TARIFFS = {
     '20A': '2.0A',
@@ -36,7 +37,7 @@ def fix_dates(self, data):
         With this all ``fields.DateTime`` fields are caugth and parsed to a
         correct format ``YYYY-MM-DDT00:00:00``.
         """
-        for attr, field in self.fields.iteritems():
+        for attr, field in self.fields.items():
             if isinstance(field, fields.DateTime):
                 orig = data.get(attr)
                 if orig and orig not in ('0', '00000000'):
@@ -49,7 +50,7 @@ def fix_dates(self, data):
 
     @pre_load
     def fix_numbers(self, data):
-        for attr, field in self.fields.iteritems():
+        for attr, field in self.fields.items():
             if isinstance(field, fields.Integer):
                 if not data.get(attr):
                     data[attr] = 0
@@ -61,11 +62,11 @@ def fix_floats(self, data):
 
         Replace ``,`` to ``.``
         """
-        for attr, field in self.fields.iteritems():
+        for attr, field in self.fields.items():
             if isinstance(field, fields.Float):
                 if not data.get(attr):
                     data[attr] = 0
-                if isinstance(data[attr], basestring):
+                if isinstance(data[attr], string_types):
                     data[attr] = data[attr].replace(',', '.')
         return data
 

sippers/adapters/hidrocantabrico.py

@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 import json
 
 from sippers import get_data
@@ -27,8 +28,8 @@
 
 class HidrocantabricoSipsAdapter(SipsAdapter, SipsSchema):
 
-    with open(get_data('hc_poblacions.json'), 'r') as f:
-        poblacions = json.load(f, encoding="utf-8")
+    with open(get_data('hc_poblacions.json'), 'rb') as f:
+        poblacions = json.load(f)
 
     @pre_load
     def adapt_tarifa(self, data):
@@ -40,7 +41,7 @@ def adapt_tarifa(self, data):
 
     @pre_load
     def fix_floats(self, data):
-        for attr, field in self.fields.iteritems():
+        for attr, field in self.fields.items():
             if isinstance(field, fields.Float):
                 if not data.get(attr):
                     data[attr] = '0'
@@ -61,7 +62,7 @@ def fix_primera_vivenda(self, data):
 
     @pre_load
     def fix_dates(self, data):
-        for attr, field in self.fields.iteritems():
+        for attr, field in self.fields.items():
             if isinstance(field, fields.DateTime):
                 orig = data.get(attr)
                 if orig not in ('', '0', '00000000'):
@@ -107,7 +108,7 @@ class HidrocantabricoMeasuresAdapter(MeasuresAdapter, MeasuresSchema):
 
     @pre_load
     def fix_numbers(self, data):
-        for attr, field in self.fields.iteritems():
+        for attr, field in self.fields.items():
             if isinstance(field, (fields.Integer, fields.Float)):
                 if not data.get(attr):
                     data[attr] = 0
@@ -117,7 +118,7 @@ def fix_numbers(self, data):
 
     @pre_load
     def fix_dates(self, data):
-        for attr, field in self.fields.iteritems():
+        for attr, field in self.fields.items():
             if isinstance(field, fields.DateTime):
                 orig = data.get(attr)
                 if orig not in ('', '0', '00000000'):

sippers/adapters/iberdrola.py

@@ -1,14 +1,15 @@
+from __future__ import unicode_literals
 from sippers.adapters import SipsAdapter, MeasuresAdapter
 from sippers.models import SipsSchema, MeasuresSchema
 from sippers.models.iberdrola import TARIFFS_OCSUM
 from marshmallow import pre_load, fields
-
+from six import string_types
 
 class IberdrolaSipsAdapter(SipsAdapter, SipsSchema):
 
     @pre_load
     def fix_dates(self, data):
-        for attr, field in self.fields.iteritems():
+        for attr, field in self.fields.items():
             if isinstance(field, fields.DateTime):
                 data[attr] += 'T00:00:00'
         return data
@@ -21,11 +22,11 @@ def adapt_distribuidora(self, data):
 
     @pre_load
     def fix_floats(self, data):
-        for attr, field in self.fields.iteritems():
+        for attr, field in self.fields.items():
             if isinstance(field, fields.Float):
                 if not data.get(attr):
                     data[attr] = 0
-                if isinstance(data[attr], basestring):
+                if isinstance(data[attr], string_types):
                     data[attr] = data[attr].replace(',', '.')
         return data
 
@@ -72,7 +73,7 @@ class IberdrolaMeasuresAdapter(MeasuresAdapter, MeasuresSchema):
 
     @pre_load
     def fix_dates(self, data):
-        for attr, field in self.fields.iteritems():
+        for attr, field in self.fields.items():
             if isinstance(field, fields.DateTime):
                 data[attr] += 'T00:00:00'
         return data

sippers/backends/__init__.py

@@ -1,6 +1,11 @@
 from __future__ import absolute_import
+from __future__ import unicode_literals
 
-from urlparse import urlparse as std_urlparse
+import six
+if six.PY2:
+    from urlparse import urlparse as std_urlparse
+else:
+    from urllib.parse import urlparse as std_urlparse
 from sippers.backends.base import BaseBackend
 
 _AVAILABLE_BACKENDS = {}

sippers/backends/base.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+from __future__ import unicode_literals
 
 
 class BaseBackend(object):

sippers/backends/mongodb.py

@@ -1,4 +1,5 @@
 from __future__ import (absolute_import)
+from __future__ import unicode_literals
 from sippers.backends import BaseBackend, register, urlparse
 from sippers.parsers.cnmc_v2 import CnmcV2, CnmcV2Cons
 from sippers.parsers.cnmc_gas import CnmcGas, CnmcGasCons

sippers/cli/__init__.py

@@ -1,4 +1,6 @@
 from __future__ import absolute_import
+from __future__ import print_function
+from __future__ import unicode_literals
 import zipfile
 
 import click
@@ -23,21 +25,21 @@ def import_file(file, backend):
             with PackedSipsFile(file) as psf:
                 pstats = psf.stats
                 for sips_file in psf:
-                    print sips_file.path
+                    print(sips_file.path)
                     stats = sips_file.stats
                     for line in sips_file:
                         if not line:
                             continue
                         bnd.insert(line)
-                        print pstats.progress, stats.progress, stats.elapsed_time, stats.speed
+                        print(pstats.progress, stats.progress, stats.elapsed_time, stats.speed)
         else:
             with SipsFile(file) as sips_file:
                 stats = sips_file.stats
                 for line in sips_file:
                     if not line:
                         continue
                     bnd.insert(line)
-                    print stats.progress, stats.elapsed_time, stats.speed
+                    print(stats.progress, stats.elapsed_time, stats.speed)
 
 
 

sippers/exceptions.py

@@ -1,2 +1,3 @@
+from __future__ import unicode_literals
 class ParserNotFoundException(Exception):
     pass

sippers/file.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+from __future__ import unicode_literals
 from datetime import datetime
 import os
 import zipfile
@@ -116,6 +117,9 @@ def __init__(self, path, strict=False):
     def __iter__(self):
         return self
 
+    def __next__(self):
+        return self.next()
+
     def next(self):
         for filename in self.files:
             self.stats.idx_file += 1
@@ -167,7 +171,7 @@ def __init__(self, path, fd=None, parser=None, strict=False, resume=None):
         else:
             self.parser = parser
         if fd is None:
-            self.fd = open(path, 'r')
+            self.fd = open(path, 'rb')
             self.stats = SipsFileStats(os.fstat(self.fd.fileno()).st_size)
         else:
             self.fd = fd
@@ -208,6 +212,9 @@ def next(self):
     def __enter__(self):
         return self
 
+    def __next__(self):
+        return self.next()
+
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.close()
 

sippers/logging.py

@@ -11,12 +11,14 @@
         logger.info('Info message')
 """
 from __future__ import absolute_import
+from __future__ import unicode_literals
 
 import logging
 
 from raven import Client as SentryClient
 from raven.handlers.logging import SentryHandler
 from sippers import VERSION
+from six import string_types
 
 LOG_FORMAT = '[%(asctime)s] %(levelname)s in %(module)s: %(message)s'
 
@@ -35,8 +37,9 @@ def setup_logging(level=None, logfile=None):
     stream.setFormatter(logging.Formatter(LOG_FORMAT))
 
     logger = logging.getLogger('sippers')
-    del logger.handlers[:]
-    logger.addHandler(stream)
+    for _h in logger.handlers[:]:
+        _h.close()
+        logger.removeHandler(_h)
 
     if logfile:
         hdlr = logging.FileHandler(logfile)
@@ -49,7 +52,7 @@ def setup_logging(level=None, logfile=None):
     sentry_handler = SentryHandler(sentry, level=logging.ERROR)
     logger.addHandler(sentry_handler)
 
-    if isinstance(level, basestring):
+    if isinstance(level, string_types):
         level = getattr(logging, level.upper(), None)
 
     if level is None:

sippers/models/__init__.py

@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 from marshmallow import Schema, fields
 from marshmallow.validate import OneOf
 

sippers/models/cnmc.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+from __future__ import unicode_literals
 from marshmallow import Schema, fields
 
 class CnmcMeasuresSchema(Schema):

sippers/models/cnmc_gas.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+from __future__ import unicode_literals
 from marshmallow import Schema, fields
 from marshmallow.validate import OneOf
 

sippers/models/cnmc_v2.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+from __future__ import unicode_literals
 from marshmallow import Schema, fields
 from sippers.utils import TABLA_6, TABLA_9, TABLA_17, TABLA_30, TABLA_32, TABLA_35, TABLA_62, TABLA_64, TABLA_108, TABLA_111
 from marshmallow.validate import OneOf

sippers/models/endesa.py

@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 from marshmallow import Schema, fields