diff --git a/Jenkinsfile b/Jenkinsfile index 7805396..0cc8bae 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -107,35 +107,31 @@ pipeline { "KGS": { node(label: 'docker') { - sh '''docker run -i --rm --name="$BUILD_TAG-kgs" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/kgs-devel /debug.sh bin/test --test-path /plone/instance/src/$GIT_NAME -v -vv -s $GIT_NAME''' + sh '''docker pull eeacms/kgs-devel;docker run -i --rm --name="$BUILD_TAG-kgs" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/kgs-devel /debug.sh bin/test --test-path /plone/instance/src/$GIT_NAME -v -vv -s $GIT_NAME''' } }, "Plone4": { node(label: 'docker') { - sh '''docker run -i --rm --name="$BUILD_TAG-plone4" -e GIT_BRANCH="$BRANCH_NAME" -e ADDONS="$GIT_NAME[test]" -e DEVELOP="src/$GIT_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/plone-test:4 -v -vv -s $GIT_NAME''' + sh '''docker pull eeacms/plone-test:4;docker run -i --rm --name="$BUILD_TAG-plone4" -e GIT_BRANCH="$BRANCH_NAME" -e ADDONS="$GIT_NAME[test]" -e DEVELOP="src/$GIT_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/plone-test:4 -v -vv -s $GIT_NAME''' } }, "Plone5 & Python2": { node(label: 'docker') { - sh '''docker run -i --rm --name="$BUILD_TAG-plone5py2" -e GIT_BRANCH="$BRANCH_NAME" -e ADDONS="$GIT_NAME[test]" -e DEVELOP="src/$GIT_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/plone-test:5 -v -vv -s $GIT_NAME''' + sh '''docker pull eeacms/plone-test:5;docker run -i --rm --name="$BUILD_TAG-plone5py2" -e GIT_BRANCH="$BRANCH_NAME" -e ADDONS="$GIT_NAME[test]" -e DEVELOP="src/$GIT_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/plone-test:5 -v -vv -s $GIT_NAME''' } }, "Plone5 & Python3": { node(label: 'docker') { - catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { - sh '''docker run -i --rm --name="$BUILD_TAG-plone5py3" -e GIT_BRANCH="$BRANCH_NAME" -e ADDONS="$GIT_NAME[test]" -e DEVELOP="src/$GIT_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/plone-test:5-python3 -v -vv -s $GIT_NAME''' - } + sh '''docker pull eeacms/plone-test:5-python3;docker run -i --rm --name="$BUILD_TAG-plone5py3" -e GIT_BRANCH="$BRANCH_NAME" -e ADDONS="$GIT_NAME[test]" -e DEVELOP="src/$GIT_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/plone-test:5-python3 -v -vv -s $GIT_NAME''' } }, "PloneSaaS": { node(label: 'docker') { - catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { - sh '''docker run -i --rm --name="$BUILD_TAG-plonesaas" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/plonesaas-devel /debug.sh bin/test --test-path /plone/instance/src/$GIT_NAME -v -vv -s $GIT_NAME''' - } + sh '''docker pull eeacms/plonesaas-devel;docker run -i --rm --name="$BUILD_TAG-plonesaas" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/plonesaas-devel /debug.sh bin/test --test-path /plone/instance/src/$GIT_NAME -v -vv -s $GIT_NAME''' } } diff --git a/README.rst b/README.rst index 8208b7e..0828d44 100644 --- a/README.rst +++ b/README.rst @@ -102,6 +102,7 @@ Literals. These can take a data type or a language code. Blank node. Similar to IRI but lacks a stable identifier. Query utilities +=============== ``class sparql.Service(endpoint, qs_encoding='utf-8')`` diff --git a/__init__.py b/__init__.py index 8b13789..e69de29 100644 --- a/__init__.py +++ b/__init__.py @@ -1 +0,0 @@ - diff --git a/docs/HISTORY.txt b/docs/HISTORY.txt index d897ea8..9523f26 100644 --- a/docs/HISTORY.txt +++ b/docs/HISTORY.txt @@ -1,9 +1,16 @@ Changelog ========= +3.8 - (2020-06-17) +--------------------------- +* Bug fix: fixed Python3 query execution + [alecghica refs #111217] +* Feature: PEP8 and linting + [alecghica refs #111217] + 3.7 - (2020-06-17) --------------------------- -* Feature: added the documentation originally found under eionet.euripa.eu +* Feature: added the documentation originally found under eionet.europa.eu [alecghica refs #111217] 3.6 - (2020-03-03) diff --git a/docs/conf.py b/docs/conf.py index f7b2c95..1532813 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from sparql import __version__ as version extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx'] @@ -9,7 +10,6 @@ project = u'sparql-client' copyright = u'2011, European Environment Agency' -from sparql import __version__ as version release = version exclude_patterns = ['_build'] diff --git a/setup.py b/setup.py index 6542dd6..9df4e3b 100644 --- a/setup.py +++ b/setup.py @@ -11,12 +11,14 @@ setup(name=NAME, version=VERSION, description='Python API to query a SPARQL endpoint', - long_description = open('README.rst').read() + "\n\n" + - open(os.path.join("docs", "HISTORY.txt")).read(), + long_description=open('README.rst').read() + "\n\n" + \ + open(os.path.join("docs", "HISTORY.txt")).read(), classifiers=[ 'Environment :: Console', 'Intended Audience :: Developers', "Programming Language :: Python", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 2.7", 'Operating System :: OS Independent', 'Topic :: Software Development :: Libraries :: Python Modules', ], @@ -26,9 +28,11 @@ long_description_content_type='text/x-rst', url='https://github.com/eea/sparql-client', license="MPL", - py_modules =['sparql'], + py_modules=['sparql'], install_requires=[ - 'eventlet', 'six' + 'eventlet', + 'six', + 'dnspython < 2.0.0', ], extras_require={ 'test': [ diff --git a/sparql-client/tests/genquery.py b/sparql-client/tests/genquery.py index b55df0a..e29d9da 100644 --- a/sparql-client/tests/genquery.py +++ b/sparql-client/tests/genquery.py @@ -1,18 +1,27 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- -import six.moves.urllib.request, six.moves.urllib.parse, six.moves.urllib.error,six.moves.urllib.request,six.moves.urllib.error,six.moves.urllib.parse +import six.moves.urllib.request +import six.moves.urllib.parse +import six.moves.urllib.error +import six.moves.urllib.request +import six.moves.urllib.error +import six.moves.urllib.parse -statement = open("code.rq").read() -query = { 'query': statement, - 'format':'xml' } +statement = open('code.rq').read() +query = {'query': statement, 'format': 'xml'} qs = six.moves.urllib.parse.urlencode(query) print(qs) -url = "http://dbpedia.org/sparql?" + six.moves.urllib.parse.urlencode(query) +url = 'http://dbpedia.org/sparql?' \ + + six.moves.urllib.parse.urlencode(query) -opener = six.moves.urllib.request.build_opener(six.moves.urllib.request.HTTPHandler) +opener = \ + six.moves.urllib.request.build_opener(six.moves.urllib.request.HTTPHandler) six.moves.urllib.request.install_opener(opener) req = six.moves.urllib.request.Request(url) -#req.add_header("Accept", "application/xml") + +# req.add_header("Accept", "application/xml") try: conn = six.moves.urllib.request.urlopen(req, timeout=10) @@ -20,7 +29,7 @@ conn = None if not conn: - raise IOError("Failure in open") + raise IOError('Failure in open') data = conn.read() conn.close() diff --git a/sparql-client/tests/testconversion.py b/sparql-client/tests/testconversion.py index 94e681e..e8189d7 100644 --- a/sparql-client/tests/testconversion.py +++ b/sparql-client/tests/testconversion.py @@ -3,8 +3,8 @@ import unittest from datetime import datetime, date, time -import sparql import os.path +import sparql from mock import Mock, patch from six.moves import map diff --git a/sparql-client/tests/testdatatypes.py b/sparql-client/tests/testdatatypes.py index 9d18785..1501116 100644 --- a/sparql-client/tests/testdatatypes.py +++ b/sparql-client/tests/testdatatypes.py @@ -3,38 +3,43 @@ import unittest import sparql +import six class TestLiterals(unittest.TestCase): def test_literal_same(self): """ Two literals with same language must be the same """ - l1 = sparql.Literal(u"Hello world",lang=u"en") - l2 = sparql.Literal(u"Hello world",lang=u"en") + + l1 = sparql.Literal(u'Hello world', lang=u'en') + l2 = sparql.Literal(u'Hello world', lang=u'en') self.assertEqual(l1, l2) def test_literal_notsame1(self): """ Two literals different language must be different """ - l1 = sparql.Literal(u"Hello world",lang=u"en") - l2 = sparql.Literal(u"Hello world",lang=u"en-US") + + l1 = sparql.Literal(u'Hello world', lang=u'en') + l2 = sparql.Literal(u'Hello world', lang=u'en-US') self.assertNotEqual(l1, l2) def test_literal_notsame2(self): """ Difference on both value and language """ - l1 = sparql.Literal(u"Hello world",lang=u"en") - l2 = sparql.Literal(u"Hallo Welt",lang=u"de") + + l1 = sparql.Literal(u'Hello world', lang=u'en') + l2 = sparql.Literal(u'Hallo Welt', lang=u'de') self.assertNotEqual(l1, l2) def test_literal_notsame3(self): """ Two literals with same language must be the same """ - l1 = sparql.Literal(u"Hello world",lang=u"en") - self.assertNotEqual(u"Hello world", l1) - self.assertNotEqual(l1, u"Hello world") + + l1 = sparql.Literal(u'Hello world', lang=u'en') + self.assertNotEqual(u'Hello world', l1) + self.assertNotEqual(l1, u'Hello world') def test_compare_with_non_literal(self): - l1 = sparql.Literal("hi") - self.assertFalse(l1 == "hi") - self.assertFalse(l1 == u"hi") + l1 = sparql.Literal('hi') + self.assertFalse(l1 == 'hi') + self.assertFalse(l1 == u'hi') self.assertFalse(l1 == None) self.assertFalse(l1 == 13) self.assertFalse(l1 == 13.0) @@ -43,63 +48,97 @@ def test_compare_with_non_literal(self): def test_convert_to_unicode(self): """ Literals should convert values to unicode when saving them """ + class SomeType(object): + def __unicode__(self): - return u"hello world" - l = sparql.Literal(SomeType()) - self.assertEqual(str(l), "hello world") + return u'hello world' + + if six.PY2: + l = sparql.Literal(SomeType()) + else: + l = sparql.Literal(SomeType().__unicode__()) + self.assertEqual(str(l), 'hello world') def test_repr(self): """ repr should return the literal in N3 syntax """ - l = sparql.Literal(u"Hello world") + + l = sparql.Literal(u'Hello world') self.assertEqual(u'', repr(l)) + class TestTypedLiterals(unittest.TestCase): def test_isinstance(self): """ Type literals are instances of RDFTerm """ - l = sparql.Literal(u"Hello world",u"http://www.w3.org/2001/XMLSchema#string") + + l = sparql.Literal(u'Hello world', + u'http://www.w3.org/2001/XMLSchema#string') assert isinstance(l, sparql.RDFTerm) def test_repr(self): """ repr should return the literal in N3 syntax """ - l = sparql.Literal(u"Hello world",u"http://www.w3.org/2001/XMLSchema#string") - self.assertEqual(u'>', repr(l)) + + l = sparql.Literal(u'Hello world', + u'http://www.w3.org/2001/XMLSchema#string') + self.assertEqual(u'>' + , repr(l)) def test_str(self): """ str should return the literal without type """ - l = sparql.Literal(u"Hello world",u"http://www.w3.org/2001/XMLSchema#string") - assert str(l) == u"Hello world" + + l = sparql.Literal(u'Hello world', + u'http://www.w3.org/2001/XMLSchema#string') + assert str(l) == u'Hello world' def test_literal_same(self): """ Two literals with same language must be the same """ - l1 = sparql.Literal(u"Hello world",u"http://aims.fao.org/aos/geopolitical.owl#MillionUSD") - l2 = sparql.Literal(u"Hello world",u"http://aims.fao.org/aos/geopolitical.owl#MillionUSD") + + l1 = sparql.Literal(u'Hello world', + u'http://aims.fao.org/aos/geopolitical.owl#MillionUSD' + ) + l2 = sparql.Literal(u'Hello world', + u'http://aims.fao.org/aos/geopolitical.owl#MillionUSD' + ) self.assertEqual(l1, l2) def test_literal_notsame1(self): """ Two literals different language must be different """ - l1 = sparql.Literal(u"Hello world",u"http://aims.fao.org/aos/geopolitical.owl#MillionUSD") - l2 = sparql.Literal(u"Hello world",u"http://www.w3.org/2001/XMLSchema#string") + + l1 = sparql.Literal(u'Hello world', + u'http://aims.fao.org/aos/geopolitical.owl#MillionUSD' + ) + l2 = sparql.Literal(u'Hello world', + u'http://www.w3.org/2001/XMLSchema#string') self.assertNotEqual(l1, l2) def test_literal_notsame2(self): """ Difference on both value and language """ - l1 = sparql.Literal(u"Hello world",u"http://aims.fao.org/aos/geopolitical.owl#MillionUSD") - l2 = sparql.Literal(u"Hallo Welt",u"http://aims.fao.org/aos/geopolitical.owl#MillionUSD") + + l1 = sparql.Literal(u'Hello world', + u'http://aims.fao.org/aos/geopolitical.owl#MillionUSD' + ) + l2 = sparql.Literal(u'Hallo Welt', + u'http://aims.fao.org/aos/geopolitical.owl#MillionUSD' + ) self.assertNotEqual(l1, l2) def test_literal_notsame3(self): """ Two literals with same language must be the same """ - l1 = sparql.Literal(u"Hello world",u"http://aims.fao.org/aos/geopolitical.owl#MillionUSD") - self.assertNotEqual(u"Hello world", l1) - self.assertNotEqual(l1, u"Hello world") - assert l1 != u"Hello world" + + l1 = sparql.Literal(u'Hello world', + u'http://aims.fao.org/aos/geopolitical.owl#MillionUSD' + ) + self.assertNotEqual(u'Hello world', l1) + self.assertNotEqual(l1, u'Hello world') + assert l1 != u'Hello world' def test_compare_with_non_literal(self): - l1 = sparql.Literal("hi",u"http://aims.fao.org/aos/geopolitical.owl#MillionUSD") - self.assertFalse(l1 == "hi") - self.assertFalse(l1 == u"hi") + l1 = sparql.Literal('hi', + u'http://aims.fao.org/aos/geopolitical.owl#MillionUSD' + ) + self.assertFalse(l1 == 'hi') + self.assertFalse(l1 == u'hi') self.assertFalse(l1 == None) self.assertFalse(l1 == 13) self.assertFalse(l1 == 13.0) @@ -108,29 +147,43 @@ def test_compare_with_non_literal(self): def test_convert_to_unicode(self): """ Literals should convert values to unicode when saving them """ + class SomeType(object): + def __unicode__(self): - return u"hello world" - l = sparql.Literal(SomeType(),u"http://aims.fao.org/aos/geopolitical.owl#MillionUSD") - self.assertEqual(str(l), "hello world") + return u'hello world' + + if six.PY2: + lt = sparql.Literal(SomeType(), + u'http://aims.fao.org/aos/geopolitical.owl#MillionUSD' + ) + else: + lt = sparql.Literal(SomeType().__unicode__(), + u'http://aims.fao.org/aos/geopolitical.owl#MillionUSD' + ) + self.assertEqual(str(lt), 'hello world') + class TestIRIs(unittest.TestCase): def test_repr(self): """ repr should return the literal in N3 syntax """ - i = sparql.IRI("http://example.com/asdf") - self.assertEqual(repr(i), ">") + + i = sparql.IRI('http://example.com/asdf') + self.assertEqual(repr(i), '>') def test_compare_with_non_iri(self): - i1 = sparql.IRI("http://example.com/asdf") - self.assertFalse(i1 == "http://example.com/asdf") - self.assertFalse(i1 == u"http://example.com/asdf") + i1 = sparql.IRI('http://example.com/asdf') + self.assertFalse(i1 == 'http://example.com/asdf') + self.assertFalse(i1 == u'http://example.com/asdf') self.assertFalse(i1 == None) self.assertFalse(i1 == 13) self.assertFalse(i1 == 13.0) self.assertFalse(i1 == ['http://example.com/asdf']) - self.assertFalse(i1 == {'http://example.com/asdf': - 'http://example.com/asdf'}) + self.assertFalse(i1 + == {'http://example.com/asdf': 'http://example.com/asdf' + }) + _literal_data = [ ('' , '""'), @@ -142,24 +195,33 @@ def test_compare_with_non_iri(self): ("new\nlines" , '"new\\nlines"'), ("ta\tbs" , '"ta\\tbs"'), (u"ascii-unicode" , '"ascii-unicode"'), - (u"̈Ünɨcøðé" , '"\\u0308\\u00dcn\\u0268c\\u00f8\\u00f0\\u00e9"'), - (u"\u6f22\u5b57(kanji)" , '"\u6f22\u5b57(kanji)"'), + (u"̈Ünɨcøðé" , '"\\u0308\\u00dcn\\u0268c\\u00f8\\u00f0\\u00e9"') ] +if six.PY2: + _literal_data.append((u"\u6f22\u5b57(kanji)" , '"\u6f22\u5b57(kanji)"')) +else: + _literal_data.append((u"\u6f22\u5b57(kanji)" , '"\\u6f22\\u5b57(kanji)"')) + + class TestNotation3(unittest.TestCase): def test_literal(self): """ Notation3 representation of a literal """ - for value, expected in _literal_data: + + for (value, expected) in _literal_data: self.assertEqual(sparql.Literal(value).n3(), expected) - self.assertEqual(sparql.Literal(value, lang='en').n3(), expected+'@en') + self.assertEqual(sparql.Literal(value, lang='en').n3(), + expected + '@en') def test_typed_literal(self): """ N3 notation of a typed literal """ - datatype = u"http://www.w3.org/2001/XMLSchema#string" - for value, expected in _literal_data: + + datatype = u'http://www.w3.org/2001/XMLSchema#string' + for (value, expected) in _literal_data: tl = sparql.Literal(value, datatype) self.assertEqual(tl.n3(), '%s^^<%s>' % (expected, datatype)) + if __name__ == '__main__': unittest.main() diff --git a/sparql-client/tests/testhttp.py b/sparql-client/tests/testhttp.py index b410522..5402fd2 100644 --- a/sparql-client/tests/testhttp.py +++ b/sparql-client/tests/testhttp.py @@ -1,8 +1,12 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + import unittest import sparql +import six QUERIES = { -"SELECT * WHERE {?s ?p ?o} LIMIT 2": """\ + "SELECT * WHERE {?s ?p ?o} LIMIT 2": """\ @@ -46,7 +50,15 @@ def getcode(self): class MockQuery(sparql._Query): def _get_response(self, opener, request, buf, timeout): - self.querystring = request.get_data() + if six.PY2: + self.querystring = request.get_data() + else: + if not request.data: + self.querystring = request.selector.split('?')[1] + else: + self.querystring = request.data + if isinstance(self.querystring, six.binary_type): + self.querystring = self.querystring.decode("utf-8") return MockResponse() def _read_response(self, response, buf, timeout): @@ -55,7 +67,11 @@ def _read_response(self, response, buf, timeout): except ImportError: from cgi import parse_qs query = parse_qs(self.querystring).get('query', [''])[0] - buf.write(QUERIES[query]) + if not six.PY2: + value = QUERIES[query].encode() + else: + value = QUERIES[query] + buf.write(value) class TestSparqlEndpoint(unittest.TestCase): diff --git a/sparql-client/tests/testn3parse.py b/sparql-client/tests/testn3parse.py index d7d2971..a626729 100644 --- a/sparql-client/tests/testn3parse.py +++ b/sparql-client/tests/testn3parse.py @@ -1,5 +1,11 @@ import unittest import sparql +import six + +try: + from testdatatypes import _literal_data +except ImportError: + from .testdatatypes import _literal_data _string_literals = [ ('""', ''), # empty string @@ -14,7 +20,6 @@ ("'''some\ntext\n with spaces'''", 'some\ntext\n with spaces'), ] -from testdatatypes import _literal_data for _value, _n3 in _literal_data: _string_literals.append((_n3, _value)) @@ -23,10 +28,17 @@ class N3ParsingTest(unittest.TestCase): def test_unicode(self): value = 'http://example.com/some_iri' + class Tricky(object): + def __unicode__(self): return '<%s>' % value - self.assertEqual(sparql.parse_n3_term(Tricky()), sparql.IRI(value)) + + if six.PY2: + parsed = sparql.parse_n3_term(Tricky()) + else: + parsed = sparql.parse_n3_term(Tricky().__unicode__()) + self.assertEqual(parsed, sparql.IRI(value)) def test_parse_IRI(self): value = 'http://example.com/some_iri' @@ -45,7 +57,7 @@ def test_IRI_error(self): self.assertRaises(ValueError, parse, 'ri>') def test_literal(self): - for n3_value, value in _string_literals: + for (n3_value, value) in _string_literals: result = sparql.parse_n3_term(n3_value) self.assertTrue(type(result) is sparql.Literal) self.assertEqual(result.lang, None) @@ -55,7 +67,7 @@ def test_literal(self): self.assertEqual(sparql.parse_n3_term(l.n3()), l) def test_literal_with_lang(self): - for n3_value, value in _string_literals: + for (n3_value, value) in _string_literals: n3_value_with_lang = n3_value + '@en' result = sparql.parse_n3_term(n3_value_with_lang) self.assertTrue(type(result) is sparql.Literal) @@ -66,8 +78,8 @@ def test_literal_with_lang(self): self.assertEqual(sparql.parse_n3_term(l.n3()), l) def test_typed_literals(self): - million_uri = u"http://aims.fao.org/aos/geopolitical.owl#MillionUSD" - for n3_value, value in _string_literals: + million_uri = u'http://aims.fao.org/aos/geopolitical.owl#MillionUSD' + for (n3_value, value) in _string_literals: n3_value_with_type = n3_value + '^^<' + million_uri + '>' result = sparql.parse_n3_term(n3_value_with_type) self.assertTrue(type(result) is sparql.Literal) @@ -81,4 +93,4 @@ def test_evil_literals(self): parse = sparql.parse_n3_term self.assertRaises(ValueError, parse, '"hello" + " world"') self.assertRaises(ValueError, parse, '"hello"\nx = " world"') - self.assertRaises(ValueError, parse, 'hello') + self.assertRaises(ValueError, parse, 'hello') \ No newline at end of file diff --git a/sparql-client/tests/testparser.py b/sparql-client/tests/testparser.py index 376a5ab..34abcc8 100644 --- a/sparql-client/tests/testparser.py +++ b/sparql-client/tests/testparser.py @@ -9,62 +9,104 @@ from six.moves import map import six + def _open_datafile(name): return open(os.path.join(os.path.dirname(__file__), name)) -XSD_FAO_MILLION = "http://aims.fao.org/aos/geopolitical.owl#MillionUSD" + +XSD_FAO_MILLION = 'http://aims.fao.org/aos/geopolitical.owl#MillionUSD' + class TestParser(unittest.TestCase): def test_simple(self): """ Simple query with unbound variables """ - resultfp = _open_datafile("countries.srx") + + resultfp = _open_datafile('countries.srx') result = sparql._ResultsParser(resultfp) - self.assertEqual([u'eeaURI', u'gdpTotal', u'eeacode', u'nutscode', u'faocode', u'gdp', u'name'], result.variables) + self.assertEqual([ + u'eeaURI', + u'gdpTotal', + u'eeacode', + u'nutscode', + u'faocode', + u'gdp', + u'name', + ], result.variables) rows = result.fetchall() row0 = rows[0] - self.assertEqual(sparql.IRI(u"http://rdfdata.eionet.europa.eu/eea/countries/BE"), row0[0]) - self.assertEqual(sparql.Literal("471161.0", XSD_FAO_MILLION), row0[1]) - self.assertEqual(sparql.Literal("44.252934", sparql.XSD_FLOAT), row0[5]) + self.assertEqual(sparql.IRI(u'http://rdfdata.eionet.europa.eu/eea/countries/BE' + ), row0[0]) + self.assertEqual(sparql.Literal('471161.0', XSD_FAO_MILLION), + row0[1]) + self.assertEqual(sparql.Literal('44.252934', sparql.XSD_FLOAT), + row0[5]) def test_unpack(self): - resultfp = _open_datafile("countries.srx") + resultfp = _open_datafile('countries.srx') result = sparql._ResultsParser(resultfp) - self.assertEqual([u'eeaURI', u'gdpTotal', u'eeacode', u'nutscode', u'faocode', u'gdp', u'name'], result.variables) + self.assertEqual([ + u'eeaURI', + u'gdpTotal', + u'eeacode', + u'nutscode', + u'faocode', + u'gdp', + u'name', + ], result.variables) rows = list(map(sparql.unpack_row, result.fetchall())) row0 = rows[0] - self.assertEqual(u"http://rdfdata.eionet.europa.eu/eea/countries/BE", row0[0]) + self.assertEqual(u'http://rdfdata.eionet.europa.eu/eea/countries/BE' + , row0[0]) + # XSD_FAO_MILLION unpacked as string - self.assertEqual("471161.0", row0[1]) + + self.assertEqual('471161.0', row0[1]) + # XSD_FLOAT unpacked as float - self.assertNotEqual("44.252934", row0[5]) + + self.assertNotEqual('44.252934', row0[5]) self.assertEqual(44.252934, row0[5]) def test_fetchmany(self): """ Simple query with unbound variables """ - resultfp = _open_datafile("countries.srx") + + resultfp = _open_datafile('countries.srx') result = sparql._ResultsParser(resultfp) - self.assertEqual([u'eeaURI', u'gdpTotal', u'eeacode', u'nutscode', u'faocode', u'gdp', u'name'], result.variables) + self.assertEqual([ + u'eeaURI', + u'gdpTotal', + u'eeacode', + u'nutscode', + u'faocode', + u'gdp', + u'name', + ], result.variables) rows = result.fetchmany(2) self.assertEqual(2, len(rows)) row0 = rows[0] - self.assertEqual("http://rdfdata.eionet.europa.eu/eea/countries/BE", str(row0[0])) + if six.PY2: + self.assertEqual('http://rdfdata.eionet.europa.eu/eea/countries/BE' + , str(row0[0])) + else: + self.assertEqual('http://rdfdata.eionet.europa.eu/eea/countries/BE' + , row0[0].value) rows = result.fetchmany(2) self.assertEqual(1, len(rows)) row0 = rows[0] - assert str(row0[6]) == "Japan" + assert str(row0[6]) == 'Japan' def test_ask_query(self): """ Check that http://www.w3.org/TR/rdf-sparql-XMLres/output2.srx works """ - resultfp = _open_datafile("w3-output2.srx") + + resultfp = _open_datafile('w3-output2.srx') result = sparql._ResultsParser(resultfp) rows = result.fetchall() assert len(rows) == 0 - # for row in result.fetchone(): # print row # row1 = result.fetchone() @@ -72,54 +114,83 @@ def test_ask_query(self): def test_w3_example(self): """ Check that http://www.w3.org/TR/rdf-sparql-XMLres/output.srx works """ - resultfp = _open_datafile("w3-output.srx") + + resultfp = _open_datafile('w3-output.srx') result = sparql._ResultsParser(resultfp) - self.assertEqual([u'x', u'hpage', u'name', u'mbox', u'age', u'blurb', u'friend'], result.variables) + self.assertEqual([ + u'x', + u'hpage', + u'name', + u'mbox', + u'age', + u'blurb', + u'friend', + ], result.variables) rows = result.fetchall() row0 = rows[0] - self.assertEqual("http://work.example.org/alice/", str(row0[1])) + if six.PY2: + self.assertEqual('http://work.example.org/alice/', + str(row0[1])) + else: + self.assertEqual('http://work.example.org/alice/', + row0[1].value) def test_hasresult(self): """ Check that http://www.w3.org/TR/rdf-sparql-XMLres/output2.srx works """ - resultfp = _open_datafile("w3-output2.srx") + + resultfp = _open_datafile('w3-output2.srx') result = sparql._ResultsParser(resultfp) assert result.hasresult() == True def test_national(self): """ Simple query with UTF-8 """ - resultfp = _open_datafile("national.srx") + + resultfp = _open_datafile('national.srx') result = sparql._ResultsParser(resultfp) - self.assertEqual([u'subj', u'nameen', u'nameru'], result.variables) + self.assertEqual([u'subj', u'nameen', u'nameru'], + result.variables) rows = result.fetchall() row0 = rows[0] - self.assertEqual("http://aims.fao.org/aos/geopolitical.owl#Germany", str(row0[0])) - self.assertEqual(sparql.IRI(u"http://aims.fao.org/aos/geopolitical.owl#Germany"), row0[0]) + if six.PY2: + self.assertEqual('http://aims.fao.org/aos/geopolitical.owl#Germany' + , str(row0[0])) + else: + self.assertEqual('http://aims.fao.org/aos/geopolitical.owl#Germany' + , row0[0].value) + + self.assertEqual(sparql.IRI(u'http://aims.fao.org/aos/geopolitical.owl#Germany' + ), row0[0]) self.assertEqual(u"Германия", six.text_type(row0[2])) def test_big_text(self): + # `xml.dom.pulldom` may return several text nodes within a single # binding. This seems to be triggered especially by entities, e.g. # "<". - resultfp = _open_datafile("big_text.srx") + + resultfp = _open_datafile('big_text.srx') result = sparql._ResultsParser(resultfp) row0 = result.fetchall()[0] - self.assertEqual("multiple
paragraphs
here", row0[0].value) - self.assertEqual("http://example.com/", row0[1].value) - self.assertEqual("bnode.id", row0[2].value) + self.assertEqual('multiple
paragraphs
here', + row0[0].value) + self.assertEqual('http://example.com/', row0[1].value) + self.assertEqual('bnode.id', row0[2].value) def side_effect_fetchhead(): - fp = _open_datafile("invalid-result.srx") + fp = _open_datafile('invalid-result.srx') return pulldom.parse(fp) - @patch('sparql._ResultsParser._fetchhead', side_effect=side_effect_fetchhead) + @patch('sparql._ResultsParser._fetchhead', + side_effect=side_effect_fetchhead) def test_invalid_fetchone(self, mocked_element): """ Simple query with invalid characters """ - resultfp = _open_datafile("invalid-result.srx") + + resultfp = _open_datafile('invalid-result.srx') result = sparql._ResultsParser(resultfp) setattr(result, 'events', result._fetchhead()) for row in result.fetchone(): - print(row) + print(row) if __name__ == '__main__': diff --git a/sparql.py b/sparql.py index 083bdcc..1ec2807 100755 --- a/sparql.py +++ b/sparql.py @@ -30,7 +30,7 @@ >>> result = sparql.query(endpoint, query) >>> for row in result: - >>> print row + >>> print(row) Command-line use @@ -48,6 +48,7 @@ """ from base64 import encodestring +from six.moves import input, map from six.moves.urllib.parse import urlencode from xml.dom import pulldom from xml.sax import SAXParseException @@ -62,30 +63,28 @@ import compiler else: from eventlet.green.urllib import request as ev_request -from six.moves import map -import six -from six.moves import input + import ast as astcompiler try: __version__ = open('version.txt').read().strip() except Exception: __version__ = "2.6" -USER_AGENT = "sparql-client/%s +https://www.eionet.europa.eu/software/sparql-client/" % __version__ +USER_AGENT = "sparql-client/%s +https://www.eionet.europa.eu/software/sparql-client/" % __version__ CONTENT_TYPE = { - 'turtle' : "application/turtle" , - 'n3' :"application/n3", - 'rdfxml' : "application/rdf+xml" , - 'ntriples' : "application/n-triples" , - 'xml' : "application/xml" + 'turtle': "application/turtle", + 'n3': "application/n3", + 'rdfxml': "application/rdf+xml", + 'ntriples': "application/n-triples", + 'xml': "application/xml" } RESULTS_TYPES = { - 'xml' : "application/sparql-results+xml" , - 'xmlschema' : "application/x-ms-access-export+xml", - 'json' : "application/sparql-results+json" + 'xml': "application/sparql-results+xml", + 'xmlschema': "application/x-ms-access-export+xml", + 'json': "application/sparql-results+json" } # The purpose of this construction is to use shared strings when @@ -105,22 +104,26 @@ datatype_dict = { '': '', - XSD_STRING : XSD_STRING, - XSD_INT : XSD_INT, - XSD_LONG : XSD_LONG, - XSD_DOUBLE : XSD_DOUBLE, - XSD_FLOAT : XSD_FLOAT, - XSD_INTEGER : XSD_INTEGER, - XSD_DECIMAL : XSD_DECIMAL, - XSD_DATETIME : XSD_DATETIME, - XSD_DATE : XSD_DATE, - XSD_TIME : XSD_TIME, - XSD_BOOLEAN : XSD_BOOLEAN + XSD_STRING: XSD_STRING, + XSD_INT: XSD_INT, + XSD_LONG: XSD_LONG, + XSD_DOUBLE: XSD_DOUBLE, + XSD_FLOAT: XSD_FLOAT, + XSD_INTEGER: XSD_INTEGER, + XSD_DECIMAL: XSD_DECIMAL, + XSD_DATETIME: XSD_DATETIME, + XSD_DATE: XSD_DATE, + XSD_TIME: XSD_TIME, + XSD_BOOLEAN: XSD_BOOLEAN } # allow import from RestrictedPython -__allow_access_to_unprotected_subobjects__ = {'Datatype': 1, 'unpack_row': 1, - 'RDFTerm': 1, 'IRI': 1, 'Literal': 1, 'BlankNode': 1} +__allow_access_to_unprotected_subobjects__ = { + 'Datatype': 1, 'unpack_row': 1, + 'RDFTerm': 1, 'IRI': 1, + 'Literal': 1, 'BlankNode': 1 +} + def Datatype(value): """ @@ -128,7 +131,7 @@ def Datatype(value): intern() only works for plain strings - not unicode. We make it look like a class, because it conceptually could be. """ - if value==None: + if value is None: r = None elif value in datatype_dict: r = datatype_dict[value] @@ -136,6 +139,7 @@ def Datatype(value): r = datatype_dict[value] = value return r + class RDFTerm(object): """ Super class containing methods to override. :class:`IRI`, @@ -159,6 +163,7 @@ def n3(self): def __repr__(self): return '<%s %s>' % (type(self).__name__, self.n3()) + class IRI(RDFTerm): """ An RDF resource. """ @@ -171,7 +176,8 @@ def __str__(self): def __eq__(self, other): if type(self) != type(other): return False - if self.value == other.value: return True + if self.value == other.value: + return True return False def n3(self): @@ -184,15 +190,17 @@ def n3(self): '\t': '\\t', '\\': '\\\\', } + + def _n3_quote(string): def escape(m): ch = m.group() if ch in _n3_quote_map: return _n3_quote_map[ch] - else: - return "\\u%04x" % ord(ch) + return "\\u%04x" % ord(ch) return '"' + _n3_quote_char.sub(escape, string) + '"' + class Literal(RDFTerm): """ Literals. These can take a data type or a language code. @@ -203,16 +211,15 @@ def __init__(self, value, datatype=None, lang=None): self.datatype = datatype def __eq__(self, other): - if type(self) != type(other): - return False + if type(self) != type(other): + return False - elif (self.value == other.value and - self.lang == other.lang and - self.datatype == other.datatype): - return True + elif (self.value == other.value and + self.lang == other.lang and + self.datatype == other.datatype): + return True - else: - return False + return False def n3(self): n3_value = _n3_quote(self.value) @@ -225,17 +232,18 @@ def n3(self): return n3_value + class BlankNode(RDFTerm): """ Blank node. Similar to `IRI` but lacks a stable identifier. """ def __init__(self, value): self.value = value def __eq__(self, other): - if type(self) != type(other): - return False - if self.value == other.value: - return True - return False + if type(self) != type(other): + return False + if self.value == other.value: + return True + return False def n3(self): return '_:%s' % self.value @@ -243,6 +251,7 @@ def n3(self): _n3parser_lang = re.compile(r'@(?P\w+)$') _n3parser_datatype = re.compile(r'\^\^<(?P[^\^"\'>]+)>$') + def parse_n3_term(src): """ Parse a Notation3 value into a RDFTerm object (IRI or Literal). @@ -286,26 +295,39 @@ def parse_n3_term(src): ast = compiler.parse("value = u" + src) else: # should be fixed due to #111217 - ast = compile("value = u" + src) + ast = astcompiler.parse("value = " + src) except: raise ValueError - # Don't allow any extra tokens in the AST - if len(ast.node.getChildNodes()) != 1: - raise ValueError - assign_node = ast.node.getChildNodes()[0] - if len(assign_node.getChildNodes()) != 2: - raise ValueError - value_node = assign_node.getChildNodes()[1] - if value_node.getChildNodes(): - raise ValueError if six.PY2: + # Don't allow any extra tokens in the AST + if len(ast.node.getChildNodes()) != 1: + raise ValueError + assign_node = ast.node.getChildNodes()[0] + if len(assign_node.getChildNodes()) != 2: + raise ValueError + value_node = assign_node.getChildNodes()[1] + if value_node.getChildNodes(): + raise ValueError if value_node.__class__ != compiler.ast.Const: raise ValueError + value = value_node.value else: - if value_node.__class__ != ast.Constant(): + # Don't allow any extra tokens in the AST + if len(ast.body) != 1: + raise ValueError + assign_node = ast.body[0] + + if len(assign_node._fields) != 2: raise ValueError - value = value_node.value + + value_node = assign_node.value + if len(value_node._fields) != 1: + raise ValueError + + # if value_node.__class__ != ast.Constant(): + # raise ValueError + value = getattr(value_node, value_node._fields[0]) if type(value) is not six.text_type: raise ValueError @@ -317,6 +339,8 @@ def parse_n3_term(src): # _ServiceMixin # ######################################### + + class _ServiceMixin(object): def __init__(self, endpoint, method="POST", accept=RESULTS_TYPES['xml']): self._method = method @@ -332,7 +356,8 @@ def __init__(self, endpoint, method="POST", accept=RESULTS_TYPES['xml']): def _setMethod(self, method): if method in ("GET", "POST"): self._method = method - else: raise ValueError("Only GET or POST is allowed") + else: + raise ValueError("Only GET or POST is allowed") def _getMethod(self): return self._method @@ -366,14 +391,15 @@ def headers(self): # ######################################### + class Service(_ServiceMixin): """ This is the main entry to the library. The user creates a :class:`Service`, then sends a query to it. If we want to have persistent connections, then open them here. """ - def __init__(self, endpoint, qs_encoding = "utf-8", method = "POST", - accept = "application/sparql-results+xml"): + def __init__(self, endpoint, qs_encoding="utf-8", method="POST", + accept="application/sparql-results+xml"): _ServiceMixin.__init__(self, endpoint, method, accept) self.qs_encoding = qs_encoding @@ -385,7 +411,7 @@ def createQuery(self): q._prefix_map = copy.deepcopy(self._prefix_map) return q - def query(self, query, timeout = 0, raw=False): + def query(self, query, timeout=0, raw=False): q = self.createQuery() return q.query(query, timeout, raw=raw) @@ -395,11 +421,9 @@ def authenticate(self, username, password): head = "Basic %s" % encodestring("%s:%s" % (username, password)).replace("\012", "") self._headers_map['Authorization'] = head + def _parseBoolean(val): - if val.lower() in ('true', '1'): - return True - else: - return False + return val.lower() in ('true', '1') # XMLSchema types and cast functions @@ -408,8 +432,8 @@ def _parseBoolean(val): XSD_LONG: int, XSD_DOUBLE: float, XSD_FLOAT: float, - XSD_INTEGER: int, # INTEGER is a DECIMAL, but Python `int` has no size - # limit, so it's safe to use + XSD_INTEGER: int, # INTEGER is a DECIMAL, but Python `int` has no size + # limit, so it's safe to use XSD_DECIMAL: decimal.Decimal, XSD_BOOLEAN: _parseBoolean, } @@ -422,6 +446,7 @@ def _parseBoolean(val): except ImportError: pass + def unpack_row(row, convert=None, convert_type={}): """ Convert values in the given row from :class:`RDFTerm` objects to plain @@ -464,6 +489,8 @@ def unpack_row(row, convert=None, convert_type={}): # _Query # ######################################### + + class _Query(_ServiceMixin): def __init__(self, service): @@ -510,8 +537,10 @@ def _read_response(self, response, buf, timeout): def _build_response(self, query, opener, buf, timeout): request = self._build_request(query) + if type(query) is not bytes and not six.PY2: + query = query.encode() return self._get_response(opener, request, buf, - timeout if timeout > 0 else None) + timeout if timeout > 0 else None) def _request(self, statement, timeout=0): """ @@ -524,6 +553,8 @@ def _request(self, statement, timeout=0): opener = ev_request.build_opener(RedirectHandler) opener.addheaders = list(self.headers().items()) try: + if type(query) is not bytes and not six.PY2: + query = query.encode() response = self._build_response(query, opener, buf, timeout) except SparqlException as error: self.endpoint = error.message @@ -549,13 +580,15 @@ def _queryString(self, statement): """ args = [] # refs #72876 removing the replace of newline to allow the comments in sparql queries - #statement = statement.replace("\n", " ").encode('utf-8') + # statement = statement.replace("\n", " ").encode('utf-8') # not needed py3 # statement = statement.encode('utf-8') pref = ' '.join(["PREFIX %s: <%s> " % (p, self._prefix_map[p]) for p in self._prefix_map]) - - statement = pref + statement + if six.PY2: + statement = pref + statement + else: + statement = pref.encode() + statement.encode() args.append(('query', statement)) @@ -564,7 +597,9 @@ def _queryString(self, statement): for uri in self.namedGraphs(): args.append(('named-graph-uri', uri)) - return urlencode(args).encode('utf-8') + if six.PY2: + return urlencode(args).encode('utf-8') + return urlencode(args) class RedirectHandler(ev_request.HTTPRedirectHandler): @@ -583,8 +618,13 @@ class _ResultsParser(object): Parse the XML result. """ - __allow_access_to_unprotected_subobjects__ = {'fetchone': 1, - 'fetchmany': 1, 'fetchall': 1, 'hasresult': 1, 'variables': 1} + __allow_access_to_unprotected_subobjects__ = { + 'fetchone': 1, + 'fetchmany': 1, + 'fetchall': 1, + 'hasresult': 1, + 'variables': 1 + } def __init__(self, fp): self.__fp = fp @@ -611,7 +651,7 @@ def _fetchhead(self): self.events.expandNode(node) self._hasResult = (node.firstChild.data == 'true') elif node.tagName == 'result': - return # We should not arrive here + return # We should not arrive here elif event == pulldom.END_ELEMENT: if node.tagName == 'head' and self.variables: return @@ -647,7 +687,7 @@ def fetchone(self): for (event, node) in self.events: if event == pulldom.START_ELEMENT: if node.tagName == 'result': - self._vals = [None] * len(self.variables) + self._vals = [None] * len(self.variables) elif node.tagName == 'binding': idx = self.variables.index(node.attributes['name'].value) elif node.tagName == 'uri': @@ -666,10 +706,14 @@ def fetchone(self): self._vals[idx] = BlankNode(data) elif event == pulldom.END_ELEMENT: if node.tagName == 'result': - #print "rtn:", len(self._vals), self._vals + # print "rtn:", len(self._vals), self._vals yield tuple(self._vals) except SAXParseException as e: - faultString = 'The data is ' + e.message + if six.PY2: + message = e.message + else: + message = e.getMessage() + faultString = 'The data is ' + message print(faultString) yield tuple() @@ -687,9 +731,11 @@ def fetchmany(self, num): for row in self.fetchone(): result.append(row) num -= 1 - if num <= 0: return result + if num <= 0: + return result return result + def query(endpoint, query, timeout=0, qs_encoding="utf-8", method="POST", accept="application/sparql-results+xml", raw=False): """ @@ -700,6 +746,7 @@ def query(endpoint, query, timeout=0, qs_encoding="utf-8", method="POST", s = Service(endpoint, qs_encoding, method, accept) return s.query(query, timeout, raw=raw) + def _interactive(endpoint): while True: try: @@ -737,15 +784,14 @@ def __init__(self, code, message): try: c = codecs.getwriter(sys.stdout.encoding) - except: + except Exception: c = codecs.getwriter('ascii') sys.stdout = c(sys.stdout, 'replace') - parser = OptionParser(usage="%prog [-i] endpoint", - version="%prog " + str(__version__)) + version="%prog " + str(__version__)) parser.add_option("-i", dest="interactive", action="store_true", - help="Enables interactive mode") + help="Enables interactive mode") (options, args) = parser.parse_args() @@ -763,5 +809,9 @@ def __init__(self, code, message): for row in result.fetchone(): print("\t".join(map(six.text_type, row))) except SparqlException as e: - faultString = e.message + if six.PY2: + message = e.message + else: + message = e.getMessage() + faultString = message print(faultString) diff --git a/version.txt b/version.txt index 475ba51..cc1923a 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -3.7 +3.8