From 961a1e3cca72e8f378dd3b307cdd9b3c8eab785b Mon Sep 17 00:00:00 2001 From: Emin Mastizada Date: Mon, 3 Apr 2017 00:52:07 +0400 Subject: [PATCH] Version 2.0 (#18) * Sum Function (#13) * Changed columns to fields in increment function * PostgreSQL support. (#17) * Missing default settings variable added for connection with json file * Version 2.0 Changelog and version update for files. --- .gitignore | 2 + CHANGELOG.md | 6 ++ README.rst | 29 +++++-- credentials.json | 3 +- credentials.postgres.json | 8 ++ dbConnect/__init__.py | 8 +- dbConnect/dbConnect.py | 142 +++++++++++++++++++++++-------- docs/_templates/sidebarlogo.html | 2 +- docs/conf.py | 2 +- docs/index.rst | 8 +- docs/user/install.rst | 5 +- docs/user/intro.rst | 2 +- docs/user/quickstart.rst | 29 ++++++- setup.py | 7 +- tests.py | 36 ++++---- 15 files changed, 215 insertions(+), 74 deletions(-) create mode 100644 credentials.postgres.json diff --git a/.gitignore b/.gitignore index 1b16bfd..b5ede90 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ __pycache__/ +venv/ *.pyc .idea/* build/ @@ -7,3 +8,4 @@ dist/ docs/_build/ NOTES.md test_credentials.json +test_pg_credentials.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 06e3525..fc2ed8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,3 +50,9 @@ * Added example for execution of custom queries * Added Link for available connection parameters * Flake8 improvements (PEP8) + +** v 2.0 **: + +* Added SUM function +* Added support for Postgresql +* Minor fixes diff --git a/README.rst b/README.rst index 4c9ddc4..f89c562 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -dbConnect: MySQL for Humans +dbConnect: Database for Humans =========================== .. image:: https://readthedocs.org/projects/dbconnect/badge/?version=latest :target: http://dbconnect.readthedocs.org/?badge=latest @@ -9,11 +9,22 @@ dbConnect: MySQL for Humans :target: https://landscape.io/github/mastizada/dbConnect/master :alt: Code Health + +WHY? +==== + +dbConnect was made as a little module to be used in small projects +that need to do some interactions with MySQL or PostgreSQL databases. + +It's just a big time saver for developers specially for making data analyzing and data scraping +and it helps to keep your code clean and readable by using python-like structure. + + Installation ============= requirements: ^^^^^^^^^^^^^ -dbConnect uses mysql.connector module, install it using: +dbConnect uses mysql.connector module for mysql, install it using: .. code-block:: bash @@ -22,6 +33,12 @@ dbConnect uses mysql.connector module, install it using: Or using offical site: `https://dev.mysql.com/downloads/connector/python/` +For PostgreSQL install psycopg2 module: + +.. code-block:: bash + + $ pip install psycopg2 + using pip: ^^^^^^^^^^ @@ -36,7 +53,7 @@ from source code: $ git clone git@github.com:mastizada/dbConnect.git $ cd dbConnect - $ sudo pip install -r requirements.txt --allow-external mysql-connector-python + $ # install required module for database $ sudo python setup.py install Usage @@ -53,7 +70,7 @@ Documentation ============= - Docs: http://dbconnect.readthedocs.org/ -- Another Docs: https://pythonhosted.org/dbConnect/ +- Alternate Docs: https://pythonhosted.org/dbConnect/ - Check generated documentation using: .. code-block:: bash @@ -64,9 +81,9 @@ Documentation .. code-block:: bash - $ pydoc3 -p 1994 + $ pydoc3 -p 8000 - and open localhost:1994/ in browser + and open http://localhost:8000/ in browser Enjoy ===== diff --git a/credentials.json b/credentials.json index 4b6f58f..96d0964 100644 --- a/credentials.json +++ b/credentials.json @@ -3,5 +3,6 @@ "user": "", "password": "", "database": "", - "port": 3306 + "port": 3306, + "engine": "mysql" } diff --git a/credentials.postgres.json b/credentials.postgres.json new file mode 100644 index 0000000..afda5e3 --- /dev/null +++ b/credentials.postgres.json @@ -0,0 +1,8 @@ +{ + "host": "", + "user": "", + "password": "", + "dbname": "", + "port": 5432, + "engine": "postgres" +} diff --git a/dbConnect/__init__.py b/dbConnect/__init__.py index dab0371..b33b573 100644 --- a/dbConnect/__init__.py +++ b/dbConnect/__init__.py @@ -1,12 +1,12 @@ """ -MySQL for Humans +Database for Humans """ from .dbConnect import DBConnect -__description__ = 'MySQL for Humans' +__description__ = 'Database for Humans' __author__ = "Emin Mastizada " -__version__ = '1.6.0' +__version__ = '2.0' __license__ = "MPL 2.0" -__help__ = """MySQL for Humans""" +__help__ = """Database for Humans""" __all__ = ['DBConnect', ] diff --git a/dbConnect/dbConnect.py b/dbConnect/dbConnect.py index e008952..1adac1c 100644 --- a/dbConnect/dbConnect.py +++ b/dbConnect/dbConnect.py @@ -2,30 +2,27 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals import json -try: - import mysql.connector # MySQL Connector - from mysql.connector import errorcode -except: - raise ValueError( - 'Please, install mysql-connector module before using plugin.' - ) class DBConnect: - """ - Light database connection object - """ + """Light database connection object.""" + settings = {} def _check_settings(self): """ Check configuration file :return: True if all settings are correct """ - keys = ['host', 'user', 'password', 'database'] + keys = ['host', 'user', 'password'] if not all(key in self.settings.keys() for key in keys): raise ValueError( 'Please check credentials file for correct keys: host, user, ' 'password, database' ) + if self.engine == "mysql" and 'database' not in self.settings.keys(): + raise ValueError( + 'database parameter is missing in credentials' + ) + # @NOTE PostgreSQL uses dbname and is automatically set to username def connect(self): """ @@ -33,21 +30,41 @@ def connect(self): Connection to database can be loosed, if that happens you can use this function to reconnect to database """ - try: - self.connection = mysql.connector.connect(**self.settings) - except mysql.connector.Error as err: - if err.errno == errorcode.ER_ACCESS_DENIED_ERROR: - raise ValueError("Wrong credentials, ACCESS DENIED") - elif err.errno == errorcode.ER_BAD_DB_ERROR: + if self.engine == "mysql": + try: + import mysql.connector # MySQL Connector + from mysql.connector import errorcode + self.connection = mysql.connector.connect(**self.settings) + except ImportError: raise ValueError( - "Database %s does not exists" % (self.settings['database']) + 'Please, install mysql-connector module before using plugin.' ) - else: - raise ValueError(err) + except mysql.connector.Error as err: + if err.errno == errorcode.ER_ACCESS_DENIED_ERROR: + raise ValueError("Wrong credentials, ACCESS DENIED") + elif err.errno == errorcode.ER_BAD_DB_ERROR: + raise ValueError( + "Database %s does not exists" % (self.settings['database']) + ) + else: + raise ValueError(err) + elif self.engine == "postgres": + try: + import psycopg2 + except ImportError: + raise ValueError( + 'Please, install psycopg2 module before using plugin.' + ) + self.connection = psycopg2.connect(**self.settings) + else: + raise NotImplementedError( + "Database engine %s not implemented!" % self.engine + ) + self.cursor = self.connection.cursor() def __init__(self, credentials_file=None, charset='utf8', - port=3306, **kwargs): + port=3306, engine="mysql", **kwargs): """ Initialise object with credentials file provided You can choose between providing file or connection details @@ -63,6 +80,10 @@ def __init__(self, credentials_file=None, charset='utf8', self.settings['charset'] = charset # Merge with kwargs self.settings.update(**kwargs) + self.engine = self.settings.pop('engine', engine) + # @NOTE Charset parameter not supported in PostgreSQL + if self.engine == 'postgres': + self.settings.pop('charset', None) self._check_settings() self.connection = None self.cursor = None @@ -197,14 +218,30 @@ def insert(self, data, table, commit=True, update=None): query = query_insert + query_value if update and bool(update): # bool(dict) checks if dict is not empty - query += ' ON DUPLICATE KEY UPDATE ' - for key in update: - query += key + ' = ' - if isinstance(update[key], int): - query += update[key] + ', ' - else: - query += '"' + update[key] + '", ' - query = query.rstrip(', ') + if self.engine == "mysql": + query += ' ON DUPLICATE KEY UPDATE ' + for key in update: + query += key + ' = ' + if isinstance(update[key], int): + query += update[key] + ', ' + else: + query += '"' + update[key] + '", ' + query = query.rstrip(', ') + elif self.engine == "postgres": + query += ' ON CONFLICT ON CONSTRAINT ' + query += table + '_pkey' + query += ' DO UPDATE SET ' + for key in update: + query = key + ' = ' + if isinstance(update[key], int): + query += update[key] + ', ' + else: + query += '"' + update[key] + '", ' + query = query.rstrip(', ') + else: + raise NotImplementedError( + "Update on insert not implemented for choosen engine" + ) # Format, execute and send to database: self.cursor.execute(query, data) if commit: @@ -283,22 +320,24 @@ def delete(self, table, filters=None, case='AND', commit=True): if commit: self.commit() - def increment(self, table, columns, steps=1, filters=None, + def increment(self, table, fields, steps=1, filters=None, case="AND", commit=True): """ Increment column in table :param table: str table name - :param columns: list column names to increment + :param fields: list column names to increment :param steps: int steps to increment, default is 1 :param filters: dict filters for rows to use :param case: Search case, Should be 'AND' or 'OR' :param commit: Commit at the end or add to pool :note: If you use safe update mode, filters should be provided """ - if not columns: - raise ValueError("You must provide which columns to update") + if not fields: + raise ValueError( + "You must provide which columns (fields) to update" + ) query = "UPDATE %s SET " % str(table) - for column in columns: + for column in fields: query += "{column} = {column} + {steps}, ".format( column=column, steps=steps) query = query.rstrip(', ') @@ -317,6 +356,41 @@ def increment(self, table, columns, steps=1, filters=None, self.commit() return {'status': True, 'message': "Columns incremented"} + def value_sum(self, table, fields, filters=None, case='AND'): + """ + Get total sum of a numeric column(s) + :param table: name of the table + :type table: str + :param fields: fields to get sum of + :type fields: list + :param filters: filters to get custom results (where) + :type filters: dict + :param case: [AND, OR] for filter type + :type case: str + :return: dict with column name and value as Decimal + """ + query = 'SELECT ' + for field in fields: + query += 'SUM(' + field + '), ' + query = query.rstrip(', ') + ' FROM ' + str(table) + data = None + if filters: + data = {} + query += ' WHERE ' + update_query, where_data = self._where_builder(filters, case) + query += update_query + for key in where_data: + data['where_' + key] = where_data[key] + if data: + self.cursor.execute(query, data) + else: + self.cursor.execute(query) + row = self.cursor.fetchone() + result = {} + for i in range(len(row)): + result[fields[i]] = row[i] + return result + def commit(self): """ Commit collected data for making changes to database diff --git a/docs/_templates/sidebarlogo.html b/docs/_templates/sidebarlogo.html index 8178128..d069af4 100644 --- a/docs/_templates/sidebarlogo.html +++ b/docs/_templates/sidebarlogo.html @@ -8,7 +8,7 @@

- dbConnect is Light MySQL Database Module. If ORM is too big for your project and MySQL module looks ugly and difficult then dbConnect is for you. + dbConnect is Light MySQL and PostgreSQL Database Module. If ORM is too big for your project and MySQL module looks ugly and difficult then dbConnect is for you.

diff --git a/docs/conf.py b/docs/conf.py index 76e91c5..8cbd6a1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,7 +15,7 @@ import sys import os import shlex -__version__ = '1.6.0' +__version__ = '2.0' sys.path.insert(0, os.path.abspath('../dbConnect')) diff --git a/docs/index.rst b/docs/index.rst index 4459839..1e08e47 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,7 +3,7 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -dbConnect: MySQL for Humans +dbConnect: Database for Humans ====================================== Release v\ |version|. (:ref:`Installation `) @@ -11,8 +11,8 @@ Release v\ |version|. (:ref:`Installation `) .. image:: https://readthedocs.org/projects/dbconnect/badge/?version=latest dbConnect is an :ref:`MPLv2 Licensed` Module for **little projects** -using *mysql* database. It generates mysql queries automatically, -you just send data in pythonic style and it does the rest. +using *mysql* or *postgresql* databases. It generates mysql and postgresql +queries automatically, you just send data in pythonic style and it does the rest. >>> from dbConnect import DBConnect >>> database = DBConnect('credentials.json') @@ -31,6 +31,8 @@ Feature Support - **insert** to table - **update** row - **delete** row +- **increment** column in table +- **sum** of a numeric column(s) - **custom sql query** diff --git a/docs/user/install.rst b/docs/user/install.rst index 64dfda7..afa1c70 100644 --- a/docs/user/install.rst +++ b/docs/user/install.rst @@ -9,11 +9,14 @@ This part of the documentation covers the installation of dbConnect. Requirements ------------ -dbConnect uses mysql.connector, install it using:: +dbConnect uses mysql.connector for mysql, install it using:: $ apt-get install python3-mysql.connector $ apt-get install python-mysql.connector +For PostgreSQL install psycopg2 module:: + + $ pip install psycopg2 Distribute & Pip ---------------- diff --git a/docs/user/intro.rst b/docs/user/intro.rst index bf5001c..a2f4fe7 100644 --- a/docs/user/intro.rst +++ b/docs/user/intro.rst @@ -9,7 +9,7 @@ If you have big project then building models (entities) and using ORM will help you a lot and will be a lot safer. dbConnect was made as little module to be used in small projects -that need to do some interactions with MySQL database. +that need to do some interactions with MySQL or PostgreSQL databases. It's just a big time saver for developers and helps to keep your code clean and readable. diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 81cb580..2a5519d 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -28,11 +28,16 @@ Or provide database details: >>> database = DBConnect(host='127.0.0.1', user='root', password='', database='test') -You can provide any other parameters that are available in `mysql.connector `_ +You can provide any other parameters that are available in `mysql.connector `_ or `PostgreSQL Documentation `_ * After successfull connection there will be **database.connection** and -**database.cursor** variables that can be used as in official MySQL -documentation. +**database.cursor** variables that can be used as in official MySQL or PostgreSQL documentation. + + +Engines +------- + +dbConnect supports `mysql` and `postgres` as `engine` options. Fetch Data @@ -126,7 +131,7 @@ Increment provided columns. Fields: - table: ``str`` : name of table, must be provided - - columns: ``array`` : column names to increment, must be provided + - fields: ``array`` : list of column names to increment, required - steps: ``int`` : Steps to increment, must be provided - filters: ``dict`` : filters to find row(s) - case: ``str`` : search case for filter [AND, OR], default ``'AND'`` @@ -137,6 +142,22 @@ Example: >>> database.increment('user', ['views'], steps=2, filters={'id': 1}) + SUM + ----------------- + + Total sum of a numeric column(s). + + Fields: + - table: ``str`` : name of table, must be provided + - fields: ``array`` : list of numeric column names, required + - filters: ``dict`` : filters to find row(s) + - case: ``str`` : search case for filter [AND, OR], default ``'AND'`` + + Example: + + >>> database.value_sum('user', fields=['views']) + + Custom SQL Query ---------------- diff --git a/setup.py b/setup.py index df12228..f94df0d 100644 --- a/setup.py +++ b/setup.py @@ -5,10 +5,10 @@ setup( name='dbConnect', - version='1.6.0', - description='MySQL for Humans', + version='2.0', + description='Database for Humans', long_description=readme, - keywords='dbConnect, mysql, simple, easy, light, connection module', + keywords='dbConnect, mysql, postgresql, postgres, simple, easy, light, module', url='https://github.com/mastizada/dbConnect', author='Emin Mastizada', author_email='emin@linux.com', @@ -30,6 +30,7 @@ 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Database :: Front-Ends', diff --git a/tests.py b/tests.py index c86cb9d..92d0143 100644 --- a/tests.py +++ b/tests.py @@ -12,15 +12,11 @@ class dbTest(TestCase): def setUp(self): - """ - Prepare for Test - """ + """Prepare for Test.""" self.database = DBConnect('travis_credentials.json') def tearDown(self): - """ - Finish Testing - """ + """Finish Testing.""" # Delete all created rows self.database.cursor.execute("truncate test") self.database.commit() @@ -28,9 +24,7 @@ def tearDown(self): self.database.disconnect() def test_insert(self): - """ - Test inserting information into database - """ + """Test inserting information into database.""" new_user = { 'name': 'Emin Mastizada', 'email': 'emin@linux.com', @@ -41,9 +35,7 @@ def test_insert(self): "Insert Failed with message %s" % result["message"]) def test_commit(self): - """ - Test committing all users at once - """ + """Test committing all users at once.""" for user in USERS: result = self.database.insert(user, 'test', commit=False) self.assertTrue(result["status"], @@ -64,11 +56,25 @@ def test_commit(self): "Number of new users in table should be 3") def test_fetch(self): - """ - Test fetching information from database - """ + """Test fetching information from database.""" pass + def test_sum(self): + """Test value_sum functionality.""" + counter=1 + for user in USERS[:3]: + user['views'] = counter + self.database.insert(user, 'test') + counter+=1 + sum_result = self.database.value_sum( + 'test', + fields=['views'] + ) + # views = 1 + 2 + 3 = 6 + self.assertEqual( + sum_result['views'], 6, "Sum of 3 new users should be 6" + ) + if __name__ == '__main__': main()