Merge pull request #28 from ayoub-belarbi/connector_meta_data

Return cursor meta_data with the result set

python-shaptools.changes

@@ -1,3 +1,9 @@
+-------------------------------------------------------------------
+Wed May 29 12:26:08 UTC 2019 - Ayoub Belarbi (abelarbi@suse.com)
+
+- Update hdb connector to return metadata besides the query
+  records. 
+
 -------------------------------------------------------------------
 Thu May 16 09:35:41 UTC 2019 - Xabier Arbulu Insausti <xarbulu@suse.com>
 

shaptools/hdb_connector/connectors/base_connector.py

@@ -34,6 +34,33 @@ class QueryError(BaseError):
     Error during query
     """
 
+class QueryResult(object):
+    """
+    Class to manage query results
+
+    Args:
+        records (list of tuples): rows of a query result
+        metadata (tuple): Sequence of 7-item sequences that describe one result column
+    """
+
+    def __init__(self, records, metadata):
+        self._logger = logging.getLogger(__name__)
+        self.records = records
+        self.metadata = metadata
+
+    @classmethod
+    def load_cursor(cls, cursor):
+        """
+        load cursor and extract records and metadata
+
+        Args:
+            cursor (obj): Cursor object created by the connector (dbapi or pydhb)
+        """
+        records = cursor.fetchall() # TODO: catch any exceptions raised by fetchall()
+        metadata = cursor.description
+        instance = cls(records, metadata)
+        instance._logger.info('query records: %s', instance.records)
+        return instance
 
 class BaseConnector(object):
     """
@@ -69,7 +96,7 @@ def query(self, sql_statement):
 
     def disconnect(self):
         """
-        Disconnecto from SAP HANA database
+        Disconnect from SAP HANA database
         """
         raise NotImplementedError(
             'method must be implemented in inherited connectors')

shaptools/hdb_connector/connectors/dbapi_connector.py

@@ -38,7 +38,7 @@ def connect(self, host, port=30015, **kwargs):
             user (str): Existing username in the database
             password (str): User password
         """
-        self._logger.info('connecting to SAP HANA database at %s:%s' % (host, port))
+        self._logger.info('connecting to SAP HANA database at %s:%s', host, port)
         try:
             self._connection = dbapi.connect(
                 address=host,
@@ -48,26 +48,25 @@ def connect(self, host, port=30015, **kwargs):
             )
         except dbapi.Error as err:
             raise base_connector.ConnectionError('connection failed: {}'.format(err))
-        self._logger.info('connected succesfully')
+        self._logger.info('connected successfully')
 
     def query(self, sql_statement):
         """
-        Query a sql statement and return response
+        Query a sql query result and return a result object
         """
-        self._logger.info('executing sql query: %s' % sql_statement)
+        self._logger.info('executing sql query: %s', sql_statement)
         try:
             with self._connection.cursor() as cursor:
                 cursor.execute(sql_statement)
-                result = cursor.fetchall()
+                result = base_connector.QueryResult.load_cursor(cursor)
         except dbapi.Error as err:
             raise base_connector.QueryError('query failed: {}'.format(err))
-        self._logger.info('query result: %s' % result)
         return result
 
     def disconnect(self):
         """
-        Disconnecto from SAP HANA database
+        Disconnect from SAP HANA database
         """
         self._logger.info('disconnecting from SAP HANA database')
         self._connection.close()
-        self._logger.info('disconnected succesfully')
+        self._logger.info('disconnected successfully')

shaptools/hdb_connector/connectors/pyhdb_connector.py

@@ -39,7 +39,7 @@ def connect(self, host, port=30015, **kwargs):
             user (str): Existing username in the database
             password (str): User password
         """
-        self._logger.info('connecting to SAP HANA database at %s:%s' % (host, port))
+        self._logger.info('connecting to SAP HANA database at %s:%s', host, port)
         try:
             self._connection = pyhdb.connect(
                 host=host,
@@ -49,27 +49,29 @@ def connect(self, host, port=30015, **kwargs):
             )
         except socket.error as err:
             raise base_connector.ConnectionError('connection failed: {}'.format(err))
-        self._logger.info('connected succesfully')
+        self._logger.info('connected successfully')
 
     def query(self, sql_statement):
         """
-        Query a sql statement and return response
+        Query a sql query result and return a result object
         """
-        self._logger.info('executing sql query: %s' % sql_statement)
+        self._logger.info('executing sql query: %s', sql_statement)
         try:
+            cursor = None
             cursor = self._connection.cursor()
             cursor.execute(sql_statement)
-            result = cursor.fetchall()
-            cursor.close()
+            result = base_connector.QueryResult.load_cursor(cursor)
         except pyhdb.exceptions.DatabaseError as err:
             raise base_connector.QueryError('query failed: {}'.format(err))
-        self._logger.info('query result: %s' % result)
+        finally:
+            if cursor:
+                cursor.close()
         return result
 
     def disconnect(self):
         """
-        Disconnecto from SAP HANA database
+        Disconnect from SAP HANA database
         """
         self._logger.info('disconnecting from SAP HANA database')
         self._connection.close()
-        self._logger.info('disconnected succesfully')
+        self._logger.info('disconnected successfully')

tests/hdb_connector/base_connect_test.py

@@ -23,9 +23,51 @@
     import mock
 
 
+class TestQueryResul(unittest.TestCase):
+    """
+    Unitary tests for base_connector.py QueryResult class
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        """
+        Global setUp.
+        """
+
+        logging.basicConfig(level=logging.INFO)
+        from shaptools.hdb_connector.connectors import base_connector
+        cls._base_connector = base_connector
+
+    def setUp(self):
+        """
+        Test setUp.
+        """
+
+    def tearDown(self):
+        """
+        Test tearDown.
+        """
+
+    @classmethod
+    def tearDownClass(cls):
+        """
+        Global tearDown.
+        """
+
+    @mock.patch('logging.Logger.info')
+    def test_load_cursor(self, logger):
+        mock_cursor = mock.Mock()
+        mock_cursor.description = 'metadata'
+        mock_cursor.fetchall.return_value = ['data1', 'data2']
+        result = self._base_connector.QueryResult.load_cursor(mock_cursor)
+        logger.assert_called_once_with('query records: %s', ['data1', 'data2'])
+        self.assertEqual(result.records, ['data1', 'data2'])
+        self.assertEqual(result.metadata, 'metadata')
+
+
 class TestHana(unittest.TestCase):
     """
-    Unitary tests for hana.py.
+    Unitary tests for base_connector.py BaseConnector class
     """
 
     @classmethod

tests/hdb_connector/dbapi_connector_test.py

@@ -70,8 +70,8 @@ def test_connect(self, mock_logger, mock_dbapi):
         mock_dbapi.connect.assert_called_once_with(
             address='host', port=1234, user='user', password='pass')
         mock_logger.assert_has_calls([
-            mock.call('connecting to SAP HANA database at %s:%s' % ('host', 1234)),
-            mock.call('connected succesfully')
+            mock.call('connecting to SAP HANA database at %s:%s', 'host', 1234),
+            mock.call('connected successfully')
         ])
 
     @mock.patch('shaptools.hdb_connector.connectors.dbapi_connector.dbapi')
@@ -87,30 +87,32 @@ def test_connect_error(self, mock_logger, mock_dbapi):
             address='host', port=1234, user='user', password='pass')
 
         mock_logger.assert_called_once_with(
-            'connecting to SAP HANA database at %s:%s' % ('host', 1234))
+            'connecting to SAP HANA database at %s:%s', 'host', 1234)
 
+    @mock.patch('shaptools.hdb_connector.connectors.base_connector.QueryResult')
     @mock.patch('logging.Logger.info')
-    def test_query(self, mock_logger):
+    def test_query(self, mock_logger, mock_result):
         cursor_mock_instance = mock.Mock()
         cursor_mock = mock.Mock(return_value=cursor_mock_instance)
-        cursor_mock_instance.fetchall.return_value = 'result'
+        mock_result_inst = mock.Mock()
+        mock_result_inst.records = ['data1', 'data2']
+        mock_result_inst.metadata = 'metadata'
+        mock_result.load_cursor.return_value = mock_result_inst
         context_manager_mock = mock.Mock(
             __enter__ = cursor_mock,
             __exit__ = mock.Mock()
         )
         self._conn._connection = mock.Mock()
         self._conn._connection.cursor.return_value = context_manager_mock
 
-        response = self._conn.query('query')
+        result = self._conn.query('query')
 
         cursor_mock_instance.execute.assert_called_once_with('query')
-        cursor_mock_instance.fetchall.assert_called_once_with()
+        mock_result.load_cursor.assert_called_once_with(cursor_mock_instance)
 
-        self.assertEqual(response, 'result')
-        mock_logger.assert_has_calls([
-            mock.call('executing sql query: %s' % 'query'),
-            mock.call('query result: %s' % 'result')
-        ])
+        self.assertEqual(result.records, ['data1', 'data2'])
+        self.assertEqual(result.metadata, 'metadata')
+        mock_logger.assert_called_once_with('executing sql query: %s', 'query')
 
     @mock.patch('shaptools.hdb_connector.connectors.dbapi_connector.dbapi')
     @mock.patch('logging.Logger.info')
@@ -123,7 +125,7 @@ def test_query_error(self, mock_logger, mock_dbapi):
 
         self.assertTrue('query failed: {}'.format('error') in str(err.exception))
         self._conn._connection.cursor.assert_called_once_with()
-        mock_logger.assert_called_once_with('executing sql query: %s' % 'query')
+        mock_logger.assert_called_once_with('executing sql query: %s', 'query')
 
     @mock.patch('logging.Logger.info')
     def test_disconnect(self, mock_logger):
@@ -132,5 +134,5 @@ def test_disconnect(self, mock_logger):
         self._conn._connection.close.assert_called_once_with()
         mock_logger.assert_has_calls([
             mock.call('disconnecting from SAP HANA database'),
-            mock.call('disconnected succesfully')
+            mock.call('disconnected successfully')
         ])

tests/hdb_connector/pyhdb_connector_test.py

@@ -70,8 +70,8 @@ def test_connect(self, mock_logger, mock_pyhdb):
         mock_pyhdb.connect.assert_called_once_with(
             host='host', port=1234, user='user', password='pass')
         mock_logger.assert_has_calls([
-            mock.call('connecting to SAP HANA database at %s:%s' % ('host', 1234)),
-            mock.call('connected succesfully')
+            mock.call('connecting to SAP HANA database at %s:%s', 'host', 1234),
+            mock.call('connected successfully')
         ])
 
     @mock.patch('shaptools.hdb_connector.connectors.pyhdb_connector.socket')
@@ -88,39 +88,60 @@ def test_connect_error(self, mock_logger, mock_pyhdb, mock_socket):
             host='host', port=1234, user='user', password='pass')
 
         mock_logger.assert_called_once_with(
-            'connecting to SAP HANA database at %s:%s' % ('host', 1234))
+            'connecting to SAP HANA database at %s:%s', 'host', 1234)
 
+    @mock.patch('shaptools.hdb_connector.connectors.base_connector.QueryResult')
     @mock.patch('logging.Logger.info')
-    def test_query(self, mock_logger):
+    def test_query(self, mock_logger, mock_result):
 
         mock_cursor = mock.Mock()
-        mock_cursor.fetchall.return_value = 'result'
         self._conn._connection = mock.Mock()
         self._conn._connection.cursor.return_value = mock_cursor
 
-        response = self._conn.query('query')
+        mock_result_inst = mock.Mock()
+        mock_result_inst.records = ['data1', 'data2']
+        mock_result_inst.metadata = 'metadata'
+        mock_result.load_cursor.return_value = mock_result_inst
+
+        result = self._conn.query('query')
 
         mock_cursor.execute.assert_called_once_with('query')
-        mock_cursor.fetchall.assert_called_once_with()
+        mock_result.load_cursor.assert_called_once_with(mock_cursor)
 
-        self.assertEqual(response, 'result')
-        mock_logger.assert_has_calls([
-            mock.call('executing sql query: %s' % 'query'),
-            mock.call('query result: %s' % 'result')
-        ])
+        self.assertEqual(result.records, ['data1', 'data2'])
+        self.assertEqual(result.metadata, 'metadata')
+        mock_logger.assert_called_once_with('executing sql query: %s', 'query')
+        mock_cursor.close.assert_called_once_with()
 
     @mock.patch('shaptools.hdb_connector.connectors.pyhdb_connector.pyhdb')
     @mock.patch('logging.Logger.info')
     def test_query_error(self, mock_logger, mock_pyhdb):
         mock_pyhdb.exceptions.DatabaseError = PyhdbException
         self._conn._connection = mock.Mock()
-        self._conn._connection.cursor.side_effect = mock_pyhdb.exceptions.DatabaseError('error')
+        self._conn._connection.cursor.side_effect = PyhdbException('error')
+        with self.assertRaises(self._pyhdb_connector.base_connector.QueryError) as err:
+            self._conn.query('query')
+
+        self.assertTrue('query failed: {}'.format('error') in str(err.exception))
+        self._conn._connection.cursor.assert_called_once_with()
+        mock_logger.assert_called_once_with('executing sql query: %s', 'query')
+
+    @mock.patch('shaptools.hdb_connector.connectors.pyhdb_connector.pyhdb')
+    @mock.patch('logging.Logger.info')
+    def test_query_error_execute(self, mock_logger, mock_pyhdb):
+        mock_pyhdb.exceptions.DatabaseError = PyhdbException
+        self._conn._connection = mock.Mock()
+        cursor_mock = mock.Mock()
+        self._conn._connection.cursor.return_value = cursor_mock
+        cursor_mock.execute = mock.Mock()
+        cursor_mock.execute.side_effect = PyhdbException('error')
         with self.assertRaises(self._pyhdb_connector.base_connector.QueryError) as err:
             self._conn.query('query')
 
         self.assertTrue('query failed: {}'.format('error') in str(err.exception))
         self._conn._connection.cursor.assert_called_once_with()
-        mock_logger.assert_called_once_with('executing sql query: %s' % 'query')
+        mock_logger.assert_called_once_with('executing sql query: %s', 'query')
+        cursor_mock.close.assert_called_once_with()
 
     @mock.patch('logging.Logger.info')
     def test_disconnect(self, mock_logger):
@@ -129,5 +150,5 @@ def test_disconnect(self, mock_logger):
         self._conn._connection.close.assert_called_once_with()
         mock_logger.assert_has_calls([
             mock.call('disconnecting from SAP HANA database'),
-            mock.call('disconnected succesfully')
+            mock.call('disconnected successfully')
         ])