From 19eda095f3340b52c61317e8d94f28bf85290704 Mon Sep 17 00:00:00 2001 From: Joe Mancuso Date: Mon, 21 Oct 2024 15:01:49 -0400 Subject: [PATCH 01/26] Refactor MySQLConnection to use connection pooling --- .../connections/MySQLConnection.py | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/src/masoniteorm/connections/MySQLConnection.py b/src/masoniteorm/connections/MySQLConnection.py index 5d0c2175..7ff44d9b 100644 --- a/src/masoniteorm/connections/MySQLConnection.py +++ b/src/masoniteorm/connections/MySQLConnection.py @@ -68,16 +68,30 @@ def make_connection(self): if self.has_global_connection(): return self.get_global_connection() - self._connection = pymysql.connect( - cursorclass=pymysql.cursors.DictCursor, - autocommit=True, - host=self.host, - user=self.user, - password=self.password, - port=self.port, - db=self.database, - **self.options - ) + # Check if there is an available connection in the pool + if CONNECTION_POOL: + self._connection = CONNECTION_POOL.pop() + else: + if len(CONNECTION_POOL) < self.options.get("pool_size", 5): # Default pool size is 5 + self._connection = pymysql.connect( + cursorclass=pymysql.cursors.DictCursor, + autocommit=True, + host=self.host, + user=self.user, + password=self.password, + port=self.port, + database=self.database, + **self.options + ) + else: + raise ConnectionError("Connection pool limit reached") + + # Add the connection back to the pool when it's closed + def close_connection(): + CONNECTION_POOL.append(self._connection) + self._connection = None + + self._connection.close = close_connection self.enable_disable_foreign_keys() From fd7d20088244ac512df093abc9496dd0b384687b Mon Sep 17 00:00:00 2001 From: Joe Mancuso Date: Mon, 21 Oct 2024 20:44:43 -0400 Subject: [PATCH 02/26] Refactor PostgresConnection to use connection pooling --- .../connections/PostgresConnection.py | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/masoniteorm/connections/PostgresConnection.py b/src/masoniteorm/connections/PostgresConnection.py index 19919d95..5304365a 100644 --- a/src/masoniteorm/connections/PostgresConnection.py +++ b/src/masoniteorm/connections/PostgresConnection.py @@ -4,6 +4,7 @@ from ..schema.platforms import PostgresPlatform from ..query.processors import PostgresPostProcessor from ..exceptions import QueryException +from psycopg2 import pool CONNECTION_POOL = [] @@ -58,14 +59,22 @@ def make_connection(self): schema = self.schema or self.full_details.get("schema") - self._connection = psycopg2.connect( - database=self.database, - user=self.user, - password=self.password, - host=self.host, - port=self.port, - options=f"-c search_path={schema}" if schema else "", - ) + # if connection pool is empty, create a new connection pool + if not CONNECTION_POOL: + CONNECTION_POOL.append( + pool.SimpleConnectionPool( + 1, 20, # minconn, maxconn + database=self.database, + user=self.user, + password=self.password, + host=self.host, + port=self.port, + options=f"-c search_path={schema}" if schema else "", + ) + ) + + # get a connection from the pool + self._connection = CONNECTION_POOL[0].getconn() self._connection.autocommit = True @@ -122,6 +131,7 @@ def get_transaction_level(self): def set_cursor(self): from psycopg2.extras import RealDictCursor + self._cursor = self._connection.cursor(cursor_factory=RealDictCursor) return self._cursor From 3ff0a839a35b173aad5151c4ffa3bc8ffeec55fb Mon Sep 17 00:00:00 2001 From: Joe Mancuso Date: Sat, 26 Oct 2024 11:32:18 -0400 Subject: [PATCH 03/26] Refactor MySQLConnection to use connection pooling --- .../connections/MySQLConnection.py | 78 ++++++++++++------- tests/integrations/config/database.py | 6 +- 2 files changed, 54 insertions(+), 30 deletions(-) diff --git a/src/masoniteorm/connections/MySQLConnection.py b/src/masoniteorm/connections/MySQLConnection.py index 7ff44d9b..578cac4c 100644 --- a/src/masoniteorm/connections/MySQLConnection.py +++ b/src/masoniteorm/connections/MySQLConnection.py @@ -55,27 +55,45 @@ def make_connection(self): "You must have the 'pymysql' package installed to make a connection to MySQL. Please install it using 'pip install pymysql'" ) - try: - import pendulum - import pymysql.converters - - pymysql.converters.conversions[ - pendulum.DateTime - ] = pymysql.converters.escape_datetime - except ImportError: - pass if self.has_global_connection(): return self.get_global_connection() # Check if there is an available connection in the pool - if CONNECTION_POOL: - self._connection = CONNECTION_POOL.pop() + self._connection = self.create_connection() + + self._connection.close = self.close_connection + self.enable_disable_foreign_keys() + + # self._connection._open = 1 + self.open = 1 + + return self + + # Add the connection back to the pool when it's closed + def close_connection(self): + if self.full_details.get("connection_pooling_enabled") and len(CONNECTION_POOL) < self.full_details.get("connection_pooling_size", 10): + print("connection closing. pool append", self._connection, CONNECTION_POOL, len(CONNECTION_POOL)) + CONNECTION_POOL.append(self._connection) + self._connection = None + + def create_connection(self, autocommit=True): + import pymysql + import pendulum + import pymysql.converters + pymysql.converters.conversions[ + pendulum.DateTime + ] = pymysql.converters.escape_datetime + + print("STARTING POOL", CONNECTION_POOL, len(CONNECTION_POOL)) + + if self.full_details.get("connection_pooling_enabled") and CONNECTION_POOL: + connection = CONNECTION_POOL.pop() + print("pool popped", connection, "remaining:", CONNECTION_POOL, len(CONNECTION_POOL)) else: - if len(CONNECTION_POOL) < self.options.get("pool_size", 5): # Default pool size is 5 - self._connection = pymysql.connect( + connection = pymysql.connect( cursorclass=pymysql.cursors.DictCursor, - autocommit=True, + autocommit=autocommit, host=self.host, user=self.user, password=self.password, @@ -83,21 +101,18 @@ def make_connection(self): database=self.database, **self.options ) - else: - raise ConnectionError("Connection pool limit reached") + + # Add the connection to the pool if pooling is enabled and the pool size is not exceeded + if self.full_details.get("connection_pooling_enabled"): + connection_pooling_size = self.full_details.get("connection_pooling_size", 10) + if len(CONNECTION_POOL) < connection_pooling_size: + CONNECTION_POOL.append(connection) + + return connection + + + return - # Add the connection back to the pool when it's closed - def close_connection(): - CONNECTION_POOL.append(self._connection) - self._connection = None - - self._connection.close = close_connection - - self.enable_disable_foreign_keys() - - self.open = 1 - - return self def reconnect(self): self._connection.connect() @@ -170,10 +185,15 @@ def query(self, query, bindings=(), results="*"): if self._dry: return {} - if not self._connection.open: + if not self.open: + if self._connection is None: + self._connection = self.create_connection() + self._connection.connect() self._cursor = self._connection.cursor() + + print("pool", self._connection, CONNECTION_POOL) try: with self._cursor as cursor: diff --git a/tests/integrations/config/database.py b/tests/integrations/config/database.py index fe4473fa..452dda0e 100644 --- a/tests/integrations/config/database.py +++ b/tests/integrations/config/database.py @@ -31,12 +31,14 @@ "host": os.getenv("MYSQL_DATABASE_HOST"), "user": os.getenv("MYSQL_DATABASE_USER"), "password": os.getenv("MYSQL_DATABASE_PASSWORD"), - "database": os.getenv("MYSQL_DATABASE_DATABASE"), + "database": "rothco_ll_preproduction", "port": os.getenv("MYSQL_DATABASE_PORT"), "prefix": "", "options": {"charset": "utf8mb4"}, "log_queries": True, "propagate": False, + "connection_pooling_enabled": True, + "connection_pooling_size": 2, }, "t": {"driver": "sqlite", "database": "orm.sqlite3", "log_queries": True, "foreign_keys": True}, "devprod": { @@ -101,6 +103,8 @@ "authentication": "ActiveDirectoryPassword", "driver": "ODBC Driver 17 for SQL Server", "connection_timeout": 15, + "connection_pooling": False, + "connection_pooling_size": 100, }, }, } From bc08c93823494b667c9cafc9591a1f6d5bcf6a3b Mon Sep 17 00:00:00 2001 From: Joe Mancuso Date: Sun, 27 Oct 2024 11:47:13 -0400 Subject: [PATCH 04/26] Refactor MySQLConnection to use connection pooling and add connection_pool_size parameter --- .../connections/MySQLConnection.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/src/masoniteorm/connections/MySQLConnection.py b/src/masoniteorm/connections/MySQLConnection.py index 578cac4c..613e6e69 100644 --- a/src/masoniteorm/connections/MySQLConnection.py +++ b/src/masoniteorm/connections/MySQLConnection.py @@ -31,10 +31,12 @@ def __init__( if str(port).isdigit(): self.port = int(self.port) self.database = database + self.user = user self.password = password self.prefix = prefix self.full_details = full_details or {} + self.connection_pool_size = full_details.get("connection_pooling_size", 100) self.options = options or {} self._cursor = None self.open = 0 @@ -72,9 +74,10 @@ def make_connection(self): # Add the connection back to the pool when it's closed def close_connection(self): - if self.full_details.get("connection_pooling_enabled") and len(CONNECTION_POOL) < self.full_details.get("connection_pooling_size", 10): - print("connection closing. pool append", self._connection, CONNECTION_POOL, len(CONNECTION_POOL)) + if self.full_details.get("connection_pooling_enabled") and len(CONNECTION_POOL) < self.connection_pool_size: CONNECTION_POOL.append(self._connection) + print("connection closing. pool append", self._connection, CONNECTION_POOL, len(CONNECTION_POOL)) + self._connection = None def create_connection(self, autocommit=True): @@ -87,7 +90,7 @@ def create_connection(self, autocommit=True): print("STARTING POOL", CONNECTION_POOL, len(CONNECTION_POOL)) - if self.full_details.get("connection_pooling_enabled") and CONNECTION_POOL: + if self.full_details.get("connection_pooling_enabled") and CONNECTION_POOL and len(CONNECTION_POOL) > 0: connection = CONNECTION_POOL.pop() print("pool popped", connection, "remaining:", CONNECTION_POOL, len(CONNECTION_POOL)) else: @@ -102,17 +105,7 @@ def create_connection(self, autocommit=True): **self.options ) - # Add the connection to the pool if pooling is enabled and the pool size is not exceeded - if self.full_details.get("connection_pooling_enabled"): - connection_pooling_size = self.full_details.get("connection_pooling_size", 10) - if len(CONNECTION_POOL) < connection_pooling_size: - CONNECTION_POOL.append(connection) - return connection - - - return - def reconnect(self): self._connection.connect() From 28418eb792bc8c32ba4515201b18276d019d03d8 Mon Sep 17 00:00:00 2001 From: Joe Mancuso Date: Sun, 27 Oct 2024 11:54:26 -0400 Subject: [PATCH 05/26] Refactor MySQLConnection to use connection pooling and initialize connection pool with specified size --- src/masoniteorm/connections/MySQLConnection.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/masoniteorm/connections/MySQLConnection.py b/src/masoniteorm/connections/MySQLConnection.py index 613e6e69..122a80df 100644 --- a/src/masoniteorm/connections/MySQLConnection.py +++ b/src/masoniteorm/connections/MySQLConnection.py @@ -90,6 +90,23 @@ def create_connection(self, autocommit=True): print("STARTING POOL", CONNECTION_POOL, len(CONNECTION_POOL)) + # Initialize the connection pool if the option is set + initialize_size = self.full_details.get("connection_pool_initialize_size") + if initialize_size and len(CONNECTION_POOL) < initialize_size: + for _ in range(initialize_size - len(CONNECTION_POOL)): + connection = pymysql.connect( + cursorclass=pymysql.cursors.DictCursor, + autocommit=autocommit, + host=self.host, + user=self.user, + password=self.password, + port=self.port, + database=self.database, + **self.options + ) + CONNECTION_POOL.append(connection) + print(f"Initialized connection pool with {initialize_size} connections") + if self.full_details.get("connection_pooling_enabled") and CONNECTION_POOL and len(CONNECTION_POOL) > 0: connection = CONNECTION_POOL.pop() print("pool popped", connection, "remaining:", CONNECTION_POOL, len(CONNECTION_POOL)) From 341db08a2a3580aae42255bd86b498e35062e96a Mon Sep 17 00:00:00 2001 From: Joe Mancuso Date: Sun, 27 Oct 2024 11:55:23 -0400 Subject: [PATCH 06/26] Refactor MySQLConnection to use connection pooling and remove debug print statements --- src/masoniteorm/connections/MySQLConnection.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/masoniteorm/connections/MySQLConnection.py b/src/masoniteorm/connections/MySQLConnection.py index 122a80df..938813c1 100644 --- a/src/masoniteorm/connections/MySQLConnection.py +++ b/src/masoniteorm/connections/MySQLConnection.py @@ -76,7 +76,6 @@ def make_connection(self): def close_connection(self): if self.full_details.get("connection_pooling_enabled") and len(CONNECTION_POOL) < self.connection_pool_size: CONNECTION_POOL.append(self._connection) - print("connection closing. pool append", self._connection, CONNECTION_POOL, len(CONNECTION_POOL)) self._connection = None @@ -88,7 +87,6 @@ def create_connection(self, autocommit=True): pendulum.DateTime ] = pymysql.converters.escape_datetime - print("STARTING POOL", CONNECTION_POOL, len(CONNECTION_POOL)) # Initialize the connection pool if the option is set initialize_size = self.full_details.get("connection_pool_initialize_size") @@ -105,11 +103,9 @@ def create_connection(self, autocommit=True): **self.options ) CONNECTION_POOL.append(connection) - print(f"Initialized connection pool with {initialize_size} connections") if self.full_details.get("connection_pooling_enabled") and CONNECTION_POOL and len(CONNECTION_POOL) > 0: connection = CONNECTION_POOL.pop() - print("pool popped", connection, "remaining:", CONNECTION_POOL, len(CONNECTION_POOL)) else: connection = pymysql.connect( cursorclass=pymysql.cursors.DictCursor, @@ -203,7 +199,6 @@ def query(self, query, bindings=(), results="*"): self._cursor = self._connection.cursor() - print("pool", self._connection, CONNECTION_POOL) try: with self._cursor as cursor: From 1ad5ad74e6423bd4abe59596231aad7a500f61d8 Mon Sep 17 00:00:00 2001 From: Joe Mancuso Date: Sun, 27 Oct 2024 11:59:06 -0400 Subject: [PATCH 07/26] Refactor MySQLConnection to use connection pooling and update connection pool size parameter --- src/masoniteorm/connections/MySQLConnection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/masoniteorm/connections/MySQLConnection.py b/src/masoniteorm/connections/MySQLConnection.py index 938813c1..9eb13bea 100644 --- a/src/masoniteorm/connections/MySQLConnection.py +++ b/src/masoniteorm/connections/MySQLConnection.py @@ -36,7 +36,7 @@ def __init__( self.password = password self.prefix = prefix self.full_details = full_details or {} - self.connection_pool_size = full_details.get("connection_pooling_size", 100) + self.connection_pool_size = full_details.get("connection_pooling_max_size", 100) self.options = options or {} self._cursor = None self.open = 0 @@ -89,7 +89,7 @@ def create_connection(self, autocommit=True): # Initialize the connection pool if the option is set - initialize_size = self.full_details.get("connection_pool_initialize_size") + initialize_size = self.full_details.get("connection_pool_min_size") if initialize_size and len(CONNECTION_POOL) < initialize_size: for _ in range(initialize_size - len(CONNECTION_POOL)): connection = pymysql.connect( From ba7372f2dbd1ea3a36a5b688c73a6df229c65673 Mon Sep 17 00:00:00 2001 From: Joe Mancuso Date: Sun, 27 Oct 2024 12:04:00 -0400 Subject: [PATCH 08/26] Refactor MySQLConnection to use connection pooling and update connection pool size parameter --- tests/integrations/config/database.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integrations/config/database.py b/tests/integrations/config/database.py index 452dda0e..4cf7b3a7 100644 --- a/tests/integrations/config/database.py +++ b/tests/integrations/config/database.py @@ -24,6 +24,7 @@ They can be named whatever you want. """ + DATABASES = { "default": "mysql", "mysql": { @@ -31,14 +32,15 @@ "host": os.getenv("MYSQL_DATABASE_HOST"), "user": os.getenv("MYSQL_DATABASE_USER"), "password": os.getenv("MYSQL_DATABASE_PASSWORD"), - "database": "rothco_ll_preproduction", + "database": os.getenv("MYSQL_DATABASE_DATABASE"), "port": os.getenv("MYSQL_DATABASE_PORT"), "prefix": "", "options": {"charset": "utf8mb4"}, "log_queries": True, "propagate": False, "connection_pooling_enabled": True, - "connection_pooling_size": 2, + "connection_pooling_max_size": 10, + "connection_pool_min_size": None, }, "t": {"driver": "sqlite", "database": "orm.sqlite3", "log_queries": True, "foreign_keys": True}, "devprod": { From cf9bc847966e200656cca8d9b816d4bc79c7fb3c Mon Sep 17 00:00:00 2001 From: Joe Mancuso Date: Sun, 27 Oct 2024 12:04:37 -0400 Subject: [PATCH 09/26] Refactor MySQLConnection to use connection pooling and update connection pool size parameter --- src/masoniteorm/connections/MySQLConnection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/masoniteorm/connections/MySQLConnection.py b/src/masoniteorm/connections/MySQLConnection.py index 9eb13bea..5045f59c 100644 --- a/src/masoniteorm/connections/MySQLConnection.py +++ b/src/masoniteorm/connections/MySQLConnection.py @@ -89,7 +89,7 @@ def create_connection(self, autocommit=True): # Initialize the connection pool if the option is set - initialize_size = self.full_details.get("connection_pool_min_size") + initialize_size = self.full_details.get("connection_pooling_min_size") if initialize_size and len(CONNECTION_POOL) < initialize_size: for _ in range(initialize_size - len(CONNECTION_POOL)): connection = pymysql.connect( From 3010894ff7e710d6c9aee74577dc8ae2c0ef756a Mon Sep 17 00:00:00 2001 From: Joe Mancuso Date: Sun, 27 Oct 2024 12:09:44 -0400 Subject: [PATCH 10/26] Refactor MySQLConnection to use connection pooling and update connection pool size parameter --- .../connections/MySQLConnection.py | 34 +++++++++++-------- tests/integrations/config/database.py | 2 +- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/src/masoniteorm/connections/MySQLConnection.py b/src/masoniteorm/connections/MySQLConnection.py index 5045f59c..1875414d 100644 --- a/src/masoniteorm/connections/MySQLConnection.py +++ b/src/masoniteorm/connections/MySQLConnection.py @@ -31,7 +31,7 @@ def __init__( if str(port).isdigit(): self.port = int(self.port) self.database = database - + self.user = user self.password = password self.prefix = prefix @@ -57,13 +57,12 @@ def make_connection(self): "You must have the 'pymysql' package installed to make a connection to MySQL. Please install it using 'pip install pymysql'" ) - if self.has_global_connection(): return self.get_global_connection() # Check if there is an available connection in the pool self._connection = self.create_connection() - + self._connection.close = self.close_connection self.enable_disable_foreign_keys() @@ -73,21 +72,25 @@ def make_connection(self): return self # Add the connection back to the pool when it's closed + def close_connection(self): - if self.full_details.get("connection_pooling_enabled") and len(CONNECTION_POOL) < self.connection_pool_size: + if ( + self.full_details.get("connection_pooling_enabled") + and len(CONNECTION_POOL) < self.connection_pool_size + ): CONNECTION_POOL.append(self._connection) self._connection = None - + def create_connection(self, autocommit=True): import pymysql import pendulum import pymysql.converters - pymysql.converters.conversions[ - pendulum.DateTime - ] = pymysql.converters.escape_datetime - - + + pymysql.converters.conversions[pendulum.DateTime] = ( + pymysql.converters.escape_datetime + ) + # Initialize the connection pool if the option is set initialize_size = self.full_details.get("connection_pooling_min_size") if initialize_size and len(CONNECTION_POOL) < initialize_size: @@ -104,7 +107,11 @@ def create_connection(self, autocommit=True): ) CONNECTION_POOL.append(connection) - if self.full_details.get("connection_pooling_enabled") and CONNECTION_POOL and len(CONNECTION_POOL) > 0: + if ( + self.full_details.get("connection_pooling_enabled") + and CONNECTION_POOL + and len(CONNECTION_POOL) > 0 + ): connection = CONNECTION_POOL.pop() else: connection = pymysql.connect( @@ -117,7 +124,7 @@ def create_connection(self, autocommit=True): database=self.database, **self.options ) - + return connection def reconnect(self): @@ -194,11 +201,10 @@ def query(self, query, bindings=(), results="*"): if not self.open: if self._connection is None: self._connection = self.create_connection() - + self._connection.connect() self._cursor = self._connection.cursor() - try: with self._cursor as cursor: diff --git a/tests/integrations/config/database.py b/tests/integrations/config/database.py index 4cf7b3a7..48edb0b1 100644 --- a/tests/integrations/config/database.py +++ b/tests/integrations/config/database.py @@ -40,7 +40,7 @@ "propagate": False, "connection_pooling_enabled": True, "connection_pooling_max_size": 10, - "connection_pool_min_size": None, + "connection_pooling_min_size": None, }, "t": {"driver": "sqlite", "database": "orm.sqlite3", "log_queries": True, "foreign_keys": True}, "devprod": { From db112cade634e072321cd3fc4e7509d025b85985 Mon Sep 17 00:00:00 2001 From: Joe Mancuso Date: Sun, 27 Oct 2024 12:10:55 -0400 Subject: [PATCH 11/26] format --- src/masoniteorm/connections/PostgresConnection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/masoniteorm/connections/PostgresConnection.py b/src/masoniteorm/connections/PostgresConnection.py index 5304365a..8864411e 100644 --- a/src/masoniteorm/connections/PostgresConnection.py +++ b/src/masoniteorm/connections/PostgresConnection.py @@ -63,7 +63,8 @@ def make_connection(self): if not CONNECTION_POOL: CONNECTION_POOL.append( pool.SimpleConnectionPool( - 1, 20, # minconn, maxconn + 1, + 20, # minconn, maxconn database=self.database, user=self.user, password=self.password, @@ -131,7 +132,6 @@ def get_transaction_level(self): def set_cursor(self): from psycopg2.extras import RealDictCursor - self._cursor = self._connection.cursor(cursor_factory=RealDictCursor) return self._cursor From 8291bc708884803a91c37d9ccbb9772545a6e061 Mon Sep 17 00:00:00 2001 From: Joe Mancuso Date: Sun, 27 Oct 2024 12:36:32 -0400 Subject: [PATCH 12/26] Refactor PostgresConnection to use connection pooling and update connection pool size parameter --- .../connections/PostgresConnection.py | 71 ++++++++++++++----- 1 file changed, 52 insertions(+), 19 deletions(-) diff --git a/src/masoniteorm/connections/PostgresConnection.py b/src/masoniteorm/connections/PostgresConnection.py index 8864411e..abaf8d85 100644 --- a/src/masoniteorm/connections/PostgresConnection.py +++ b/src/masoniteorm/connections/PostgresConnection.py @@ -4,7 +4,6 @@ from ..schema.platforms import PostgresPlatform from ..query.processors import PostgresPostProcessor from ..exceptions import QueryException -from psycopg2 import pool CONNECTION_POOL = [] @@ -35,8 +34,10 @@ def __init__( self.database = database self.user = user self.password = password + self.prefix = prefix self.full_details = full_details or {} + self.connection_pool_size = full_details.get("connection_pooling_max_size", 100) self.options = options or {} self._cursor = None self.transaction_level = 0 @@ -59,23 +60,7 @@ def make_connection(self): schema = self.schema or self.full_details.get("schema") - # if connection pool is empty, create a new connection pool - if not CONNECTION_POOL: - CONNECTION_POOL.append( - pool.SimpleConnectionPool( - 1, - 20, # minconn, maxconn - database=self.database, - user=self.user, - password=self.password, - host=self.host, - port=self.port, - options=f"-c search_path={schema}" if schema else "", - ) - ) - - # get a connection from the pool - self._connection = CONNECTION_POOL[0].getconn() + self._connection = self.create_connection() self._connection.autocommit = True @@ -84,7 +69,43 @@ def make_connection(self): self.open = 1 return self + + def create_connection(self): + import psycopg2 + + # Initialize the connection pool if the option is set + initialize_size = self.full_details.get("connection_pooling_min_size") + if self.full_details.get("connection_pooling_enabled") and initialize_size and len(CONNECTION_POOL) < initialize_size: + for _ in range(initialize_size - len(CONNECTION_POOL)): + connection = psycopg2.connect( + database=self.database, + user=self.user, + password=self.password, + host=self.host, + port=self.port, + options=f"-c search_path={self.schema or self.full_details.get('schema')}" if self.schema or self.full_details.get('schema') else "", + ) + CONNECTION_POOL.append(connection) + + + if ( + self.full_details.get("connection_pooling_enabled") + and CONNECTION_POOL + and len(CONNECTION_POOL) > 0 + ): + connection = CONNECTION_POOL.pop() + else: + connection = psycopg2.connect( + database=self.database, + user=self.user, + password=self.password, + host=self.host, + port=self.port, + options=f"-c search_path={self.schema or self.full_details.get('schema')}" if self.schema or self.full_details.get('schema') else "", + ) + return connection + def get_database_name(self): return self.database @@ -102,6 +123,17 @@ def get_default_post_processor(cls): def reconnect(self): pass + + def close_connection(self): + if ( + self.full_details.get("connection_pooling_enabled") + and len(CONNECTION_POOL) < self.connection_pool_size + ): + CONNECTION_POOL.append(self._connection) + else: + self._connection.close() + + self._connection = None def commit(self): """Transaction""" @@ -174,4 +206,5 @@ def query(self, query, bindings=(), results="*"): finally: if self.get_transaction_level() <= 0: self.open = 0 - self._connection.close() + self.close_connection() + # self._connection.close() From 4cbd0e40b3bc186c223eba2b1ac70ab84326552d Mon Sep 17 00:00:00 2001 From: Joe Mancuso Date: Sun, 27 Oct 2024 12:38:00 -0400 Subject: [PATCH 13/26] Refactor PostgresConnection to remove unused variable --- src/masoniteorm/connections/PostgresConnection.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/masoniteorm/connections/PostgresConnection.py b/src/masoniteorm/connections/PostgresConnection.py index abaf8d85..deb3075c 100644 --- a/src/masoniteorm/connections/PostgresConnection.py +++ b/src/masoniteorm/connections/PostgresConnection.py @@ -58,8 +58,6 @@ def make_connection(self): if self.has_global_connection(): return self.get_global_connection() - schema = self.schema or self.full_details.get("schema") - self._connection = self.create_connection() self._connection.autocommit = True From a4656b90928fa15c907dbb285c32430814e286e4 Mon Sep 17 00:00:00 2001 From: Joe Mancuso Date: Sun, 27 Oct 2024 12:40:33 -0400 Subject: [PATCH 14/26] Refactor PostgresConnection to enable connection pooling with a maximum size of 10 --- tests/integrations/config/database.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integrations/config/database.py b/tests/integrations/config/database.py index 48edb0b1..d7a64acb 100644 --- a/tests/integrations/config/database.py +++ b/tests/integrations/config/database.py @@ -73,6 +73,9 @@ "password": os.getenv("POSTGRES_DATABASE_PASSWORD"), "database": os.getenv("POSTGRES_DATABASE_DATABASE"), "port": os.getenv("POSTGRES_DATABASE_PORT"), + "connection_pooling_enabled": True, + "connection_pooling_max_size": 10, + "connection_pooling_min_size": None, "prefix": "", "log_queries": True, "propagate": False, From 6a237fa58128c3644f269eea62c1d6a143c894df Mon Sep 17 00:00:00 2001 From: Joe Mancuso Date: Sun, 27 Oct 2024 12:45:18 -0400 Subject: [PATCH 15/26] Refactor database configuration to update connection pool minimum size --- tests/integrations/config/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integrations/config/database.py b/tests/integrations/config/database.py index d7a64acb..ed1fd02e 100644 --- a/tests/integrations/config/database.py +++ b/tests/integrations/config/database.py @@ -75,7 +75,7 @@ "port": os.getenv("POSTGRES_DATABASE_PORT"), "connection_pooling_enabled": True, "connection_pooling_max_size": 10, - "connection_pooling_min_size": None, + "connection_pooling_min_size": 2, "prefix": "", "log_queries": True, "propagate": False, From 6c4ff02617c8ff0e164e1e5748682d42da8793ac Mon Sep 17 00:00:00 2001 From: Joe Mancuso Date: Sun, 27 Oct 2024 13:03:54 -0400 Subject: [PATCH 16/26] Refactor SQLiteConnection to use a separate method for creating the connection --- src/masoniteorm/connections/SQLiteConnection.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/masoniteorm/connections/SQLiteConnection.py b/src/masoniteorm/connections/SQLiteConnection.py index 24bd7f0d..0bbd2d87 100644 --- a/src/masoniteorm/connections/SQLiteConnection.py +++ b/src/masoniteorm/connections/SQLiteConnection.py @@ -59,16 +59,20 @@ def make_connection(self): if self.has_global_connection(): return self.get_global_connection() - self._connection = sqlite3.connect(self.database, isolation_level=None) - self._connection.create_function("REGEXP", 2, regexp) - - self._connection.row_factory = sqlite3.Row + self._connection = self.create_connection() self.enable_disable_foreign_keys() self.open = 1 return self + + def create_connection(self): + import sqlite3 + connection = sqlite3.connect(self.database, isolation_level=None) + connection.create_function("REGEXP", 2, regexp) + connection.row_factory = sqlite3.Row + return connection @classmethod def get_default_query_grammar(cls): From d14c826ad1ffbc1f0208a52915cc86c453446e28 Mon Sep 17 00:00:00 2001 From: Joe Mancuso Date: Sun, 27 Oct 2024 14:44:17 -0400 Subject: [PATCH 17/26] Refactor SQLiteConnection to use a separate method for creating the connection --- src/masoniteorm/connections/SQLiteConnection.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/masoniteorm/connections/SQLiteConnection.py b/src/masoniteorm/connections/SQLiteConnection.py index 0bbd2d87..24bd7f0d 100644 --- a/src/masoniteorm/connections/SQLiteConnection.py +++ b/src/masoniteorm/connections/SQLiteConnection.py @@ -59,20 +59,16 @@ def make_connection(self): if self.has_global_connection(): return self.get_global_connection() - self._connection = self.create_connection() + self._connection = sqlite3.connect(self.database, isolation_level=None) + self._connection.create_function("REGEXP", 2, regexp) + + self._connection.row_factory = sqlite3.Row self.enable_disable_foreign_keys() self.open = 1 return self - - def create_connection(self): - import sqlite3 - connection = sqlite3.connect(self.database, isolation_level=None) - connection.create_function("REGEXP", 2, regexp) - connection.row_factory = sqlite3.Row - return connection @classmethod def get_default_query_grammar(cls): From 76590f84423e40c90ef9108d95142f9fd1d4ded4 Mon Sep 17 00:00:00 2001 From: Joe Mancuso Date: Sun, 27 Oct 2024 14:45:29 -0400 Subject: [PATCH 18/26] linted --- .../connections/PostgresConnection.py | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/masoniteorm/connections/PostgresConnection.py b/src/masoniteorm/connections/PostgresConnection.py index deb3075c..20a96b4b 100644 --- a/src/masoniteorm/connections/PostgresConnection.py +++ b/src/masoniteorm/connections/PostgresConnection.py @@ -34,7 +34,7 @@ def __init__( self.database = database self.user = user self.password = password - + self.prefix = prefix self.full_details = full_details or {} self.connection_pool_size = full_details.get("connection_pooling_max_size", 100) @@ -67,13 +67,17 @@ def make_connection(self): self.open = 1 return self - + def create_connection(self): import psycopg2 # Initialize the connection pool if the option is set initialize_size = self.full_details.get("connection_pooling_min_size") - if self.full_details.get("connection_pooling_enabled") and initialize_size and len(CONNECTION_POOL) < initialize_size: + if ( + self.full_details.get("connection_pooling_enabled") + and initialize_size + and len(CONNECTION_POOL) < initialize_size + ): for _ in range(initialize_size - len(CONNECTION_POOL)): connection = psycopg2.connect( database=self.database, @@ -81,10 +85,13 @@ def create_connection(self): password=self.password, host=self.host, port=self.port, - options=f"-c search_path={self.schema or self.full_details.get('schema')}" if self.schema or self.full_details.get('schema') else "", + options=( + f"-c search_path={self.schema or self.full_details.get('schema')}" + if self.schema or self.full_details.get("schema") + else "" + ), ) CONNECTION_POOL.append(connection) - if ( self.full_details.get("connection_pooling_enabled") @@ -99,11 +106,15 @@ def create_connection(self): password=self.password, host=self.host, port=self.port, - options=f"-c search_path={self.schema or self.full_details.get('schema')}" if self.schema or self.full_details.get('schema') else "", + options=( + f"-c search_path={self.schema or self.full_details.get('schema')}" + if self.schema or self.full_details.get("schema") + else "" + ), ) return connection - + def get_database_name(self): return self.database @@ -121,7 +132,7 @@ def get_default_post_processor(cls): def reconnect(self): pass - + def close_connection(self): if ( self.full_details.get("connection_pooling_enabled") From d28435b6ee5bededa3b16748bb13a14e0a2752b5 Mon Sep 17 00:00:00 2001 From: Joe Mancuso Date: Sun, 27 Oct 2024 14:50:08 -0400 Subject: [PATCH 19/26] Refactor PostgresConnection to handle closed connections during query execution --- src/masoniteorm/connections/PostgresConnection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/masoniteorm/connections/PostgresConnection.py b/src/masoniteorm/connections/PostgresConnection.py index 20a96b4b..b5c6253e 100644 --- a/src/masoniteorm/connections/PostgresConnection.py +++ b/src/masoniteorm/connections/PostgresConnection.py @@ -191,7 +191,7 @@ def query(self, query, bindings=(), results="*"): dict|None -- Returns a dictionary of results or None """ try: - if self._connection.closed: + if not hasattr(self, '_connection') or self._connection.closed: self.make_connection() self.set_cursor() From 9475b9eee179a5b70aba81098355d98eec3963d7 Mon Sep 17 00:00:00 2001 From: Joe Mancuso Date: Sun, 27 Oct 2024 14:50:37 -0400 Subject: [PATCH 20/26] Refactor PostgresConnection to handle closed connections during query execution --- src/masoniteorm/connections/PostgresConnection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/masoniteorm/connections/PostgresConnection.py b/src/masoniteorm/connections/PostgresConnection.py index b5c6253e..0bbfe172 100644 --- a/src/masoniteorm/connections/PostgresConnection.py +++ b/src/masoniteorm/connections/PostgresConnection.py @@ -191,7 +191,7 @@ def query(self, query, bindings=(), results="*"): dict|None -- Returns a dictionary of results or None """ try: - if not hasattr(self, '_connection') or self._connection.closed: + if not self._connection or self._connection.closed: self.make_connection() self.set_cursor() From 09d082b1c4e2f489c92155e9c97e78bf8032a810 Mon Sep 17 00:00:00 2001 From: Joe Mancuso Date: Sun, 27 Oct 2024 15:01:58 -0400 Subject: [PATCH 21/26] Refactor MySQLConnection to handle closed connections during query execution --- .../connections/MySQLConnection.py | 34 +++++++------------ 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/src/masoniteorm/connections/MySQLConnection.py b/src/masoniteorm/connections/MySQLConnection.py index 1875414d..fa3c2548 100644 --- a/src/masoniteorm/connections/MySQLConnection.py +++ b/src/masoniteorm/connections/MySQLConnection.py @@ -31,7 +31,7 @@ def __init__( if str(port).isdigit(): self.port = int(self.port) self.database = database - + self.user = user self.password = password self.prefix = prefix @@ -57,40 +57,35 @@ def make_connection(self): "You must have the 'pymysql' package installed to make a connection to MySQL. Please install it using 'pip install pymysql'" ) + if self.has_global_connection(): return self.get_global_connection() # Check if there is an available connection in the pool self._connection = self.create_connection() - self._connection.close = self.close_connection self.enable_disable_foreign_keys() - # self._connection._open = 1 self.open = 1 return self # Add the connection back to the pool when it's closed - def close_connection(self): - if ( - self.full_details.get("connection_pooling_enabled") - and len(CONNECTION_POOL) < self.connection_pool_size - ): + if self.full_details.get("connection_pooling_enabled") and len(CONNECTION_POOL) < self.connection_pool_size: CONNECTION_POOL.append(self._connection) self._connection = None - + def create_connection(self, autocommit=True): import pymysql import pendulum import pymysql.converters - - pymysql.converters.conversions[pendulum.DateTime] = ( - pymysql.converters.escape_datetime - ) - + pymysql.converters.conversions[ + pendulum.DateTime + ] = pymysql.converters.escape_datetime + + # Initialize the connection pool if the option is set initialize_size = self.full_details.get("connection_pooling_min_size") if initialize_size and len(CONNECTION_POOL) < initialize_size: @@ -107,11 +102,7 @@ def create_connection(self, autocommit=True): ) CONNECTION_POOL.append(connection) - if ( - self.full_details.get("connection_pooling_enabled") - and CONNECTION_POOL - and len(CONNECTION_POOL) > 0 - ): + if self.full_details.get("connection_pooling_enabled") and CONNECTION_POOL and len(CONNECTION_POOL) > 0: connection = CONNECTION_POOL.pop() else: connection = pymysql.connect( @@ -124,7 +115,7 @@ def create_connection(self, autocommit=True): database=self.database, **self.options ) - + return connection def reconnect(self): @@ -201,10 +192,11 @@ def query(self, query, bindings=(), results="*"): if not self.open: if self._connection is None: self._connection = self.create_connection() - + self._connection.connect() self._cursor = self._connection.cursor() + try: with self._cursor as cursor: From e4662828aa8819577a3011362c04b96611fd672c Mon Sep 17 00:00:00 2001 From: Joe Mancuso Date: Sun, 27 Oct 2024 15:04:57 -0400 Subject: [PATCH 22/26] Refactor MySQLConnection to handle closed connections during query execution --- src/masoniteorm/connections/MySQLConnection.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/masoniteorm/connections/MySQLConnection.py b/src/masoniteorm/connections/MySQLConnection.py index fa3c2548..f972238a 100644 --- a/src/masoniteorm/connections/MySQLConnection.py +++ b/src/masoniteorm/connections/MySQLConnection.py @@ -63,7 +63,6 @@ def make_connection(self): # Check if there is an available connection in the pool self._connection = self.create_connection() - self._connection.close = self.close_connection self.enable_disable_foreign_keys() self.open = 1 @@ -115,6 +114,8 @@ def create_connection(self, autocommit=True): database=self.database, **self.options ) + + connection.close = self.close_connection return connection From 94b1347981a1cc67e0343b0aa8e3bad182238794 Mon Sep 17 00:00:00 2001 From: Joe Mancuso Date: Sun, 27 Oct 2024 15:07:45 -0400 Subject: [PATCH 23/26] Refactor MySQLConnection to handle closed connections during query execution --- .../connections/MySQLConnection.py | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/src/masoniteorm/connections/MySQLConnection.py b/src/masoniteorm/connections/MySQLConnection.py index f972238a..fba0777a 100644 --- a/src/masoniteorm/connections/MySQLConnection.py +++ b/src/masoniteorm/connections/MySQLConnection.py @@ -31,7 +31,7 @@ def __init__( if str(port).isdigit(): self.port = int(self.port) self.database = database - + self.user = user self.password = password self.prefix = prefix @@ -57,7 +57,6 @@ def make_connection(self): "You must have the 'pymysql' package installed to make a connection to MySQL. Please install it using 'pip install pymysql'" ) - if self.has_global_connection(): return self.get_global_connection() @@ -70,21 +69,25 @@ def make_connection(self): return self # Add the connection back to the pool when it's closed + def close_connection(self): - if self.full_details.get("connection_pooling_enabled") and len(CONNECTION_POOL) < self.connection_pool_size: + if ( + self.full_details.get("connection_pooling_enabled") + and len(CONNECTION_POOL) < self.connection_pool_size + ): CONNECTION_POOL.append(self._connection) self._connection = None - + def create_connection(self, autocommit=True): import pymysql import pendulum import pymysql.converters - pymysql.converters.conversions[ - pendulum.DateTime - ] = pymysql.converters.escape_datetime - - + + pymysql.converters.conversions[pendulum.DateTime] = ( + pymysql.converters.escape_datetime + ) + # Initialize the connection pool if the option is set initialize_size = self.full_details.get("connection_pooling_min_size") if initialize_size and len(CONNECTION_POOL) < initialize_size: @@ -101,7 +104,11 @@ def create_connection(self, autocommit=True): ) CONNECTION_POOL.append(connection) - if self.full_details.get("connection_pooling_enabled") and CONNECTION_POOL and len(CONNECTION_POOL) > 0: + if ( + self.full_details.get("connection_pooling_enabled") + and CONNECTION_POOL + and len(CONNECTION_POOL) > 0 + ): connection = CONNECTION_POOL.pop() else: connection = pymysql.connect( @@ -114,9 +121,9 @@ def create_connection(self, autocommit=True): database=self.database, **self.options ) - + connection.close = self.close_connection - + return connection def reconnect(self): @@ -193,11 +200,10 @@ def query(self, query, bindings=(), results="*"): if not self.open: if self._connection is None: self._connection = self.create_connection() - + self._connection.connect() self._cursor = self._connection.cursor() - try: with self._cursor as cursor: From fd6785fd36091c9db4483552af29a0b0fe62f8bb Mon Sep 17 00:00:00 2001 From: Joe Mancuso Date: Sun, 27 Oct 2024 15:43:02 -0400 Subject: [PATCH 24/26] linted --- .../connections/MySQLConnection.py | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/src/masoniteorm/connections/MySQLConnection.py b/src/masoniteorm/connections/MySQLConnection.py index fba0777a..c96ac8e9 100644 --- a/src/masoniteorm/connections/MySQLConnection.py +++ b/src/masoniteorm/connections/MySQLConnection.py @@ -36,7 +36,11 @@ def __init__( self.password = password self.prefix = prefix self.full_details = full_details or {} - self.connection_pool_size = full_details.get("connection_pooling_max_size", 100) + self.connection_pool_size = ( + full_details.get( + "connection_pooling_max_size", 100 + ) + ) self.options = options or {} self._cursor = None self.open = 0 @@ -50,13 +54,6 @@ def make_connection(self): if self._dry: return - try: - import pymysql - except ModuleNotFoundError: - raise DriverNotFound( - "You must have the 'pymysql' package installed to make a connection to MySQL. Please install it using 'pip install pymysql'" - ) - if self.has_global_connection(): return self.get_global_connection() @@ -80,7 +77,15 @@ def close_connection(self): self._connection = None def create_connection(self, autocommit=True): - import pymysql + + try: + import pymysql + except ModuleNotFoundError: + raise DriverNotFound( + "You must have the 'pymysql' package " + "installed to make a connection to MySQL. " + "Please install it using 'pip install pymysql'" + ) import pendulum import pymysql.converters @@ -180,15 +185,19 @@ def get_cursor(self): return self._cursor def query(self, query, bindings=(), results="*"): - """Make the actual query that will reach the database and come back with a result. + """Make the actual query that + will reach the database and come back with a result. Arguments: - query {string} -- A string query. This could be a qmarked string or a regular query. + query {string} -- A string query. + This could be a qmarked string or a regular query. bindings {tuple} -- A tuple of bindings Keyword Arguments: - results {str|1} -- If the results is equal to an asterisks it will call 'fetchAll' - else it will return 'fetchOne' and return a single record. (default: {"*"}) + results {str|1} -- If the results is equal to an + asterisks it will call 'fetchAll' + else it will return 'fetchOne' and + return a single record. (default: {"*"}) Returns: dict|None -- Returns a dictionary of results or None From 0468c45a44fb47ccd6a1b5e29b1838ef97d20e81 Mon Sep 17 00:00:00 2001 From: Joe Mancuso Date: Sun, 27 Oct 2024 18:22:26 -0400 Subject: [PATCH 25/26] Refactor MySQLConnection to handle closed connections during query execution --- src/masoniteorm/connections/MySQLConnection.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/masoniteorm/connections/MySQLConnection.py b/src/masoniteorm/connections/MySQLConnection.py index c96ac8e9..27d1a204 100644 --- a/src/masoniteorm/connections/MySQLConnection.py +++ b/src/masoniteorm/connections/MySQLConnection.py @@ -61,7 +61,7 @@ def make_connection(self): self._connection = self.create_connection() self.enable_disable_foreign_keys() - self.open = 1 + return self @@ -73,7 +73,7 @@ def close_connection(self): and len(CONNECTION_POOL) < self.connection_pool_size ): CONNECTION_POOL.append(self._connection) - + self.open = 0 self._connection = None def create_connection(self, autocommit=True): @@ -128,6 +128,8 @@ def create_connection(self, autocommit=True): ) connection.close = self.close_connection + + self.open = 1 return connection From 7a6e7a8b19650d14cfabe879ccfc21d85ac612ee Mon Sep 17 00:00:00 2001 From: Joe Mancuso Date: Mon, 28 Oct 2024 15:07:15 -0400 Subject: [PATCH 26/26] linted --- src/masoniteorm/connections/MySQLConnection.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/masoniteorm/connections/MySQLConnection.py b/src/masoniteorm/connections/MySQLConnection.py index 27d1a204..355f2da8 100644 --- a/src/masoniteorm/connections/MySQLConnection.py +++ b/src/masoniteorm/connections/MySQLConnection.py @@ -61,12 +61,8 @@ def make_connection(self): self._connection = self.create_connection() self.enable_disable_foreign_keys() - - return self - # Add the connection back to the pool when it's closed - def close_connection(self): if ( self.full_details.get("connection_pooling_enabled") @@ -128,7 +124,7 @@ def create_connection(self, autocommit=True): ) connection.close = self.close_connection - + self.open = 1 return connection