diff --git a/bandit-baseline.json b/bandit-baseline.json index aee9ade..32c65c2 100644 --- a/bandit-baseline.json +++ b/bandit-baseline.json @@ -1,17 +1,17 @@ { "errors": [], - "generated_at": "2024-08-06T12:35:09Z", + "generated_at": "2024-08-14T14:00:34Z", "metrics": { "_totals": { - "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.HIGH": 9.0, "CONFIDENCE.LOW": 3.0, - "CONFIDENCE.MEDIUM": 1.0, + "CONFIDENCE.MEDIUM": 9.0, "CONFIDENCE.UNDEFINED": 0.0, "SEVERITY.HIGH": 0.0, - "SEVERITY.LOW": 1.0, - "SEVERITY.MEDIUM": 3.0, + "SEVERITY.LOW": 10.0, + "SEVERITY.MEDIUM": 11.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 3497, + "loc": 4127, "nosec": 0 }, "src/setup.py": { @@ -35,7 +35,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 3, + "loc": 20, "nosec": 0 }, "src/sqlitecloud/client.py": { @@ -59,7 +59,7 @@ "SEVERITY.LOW": 1.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 177, + "loc": 178, "nosec": 0 }, "src/sqlitecloud/dbapi2.py": { @@ -71,7 +71,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 376, + "loc": 510, "nosec": 0 }, "src/sqlitecloud/download.py": { @@ -95,7 +95,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 787, + "loc": 794, "nosec": 0 }, "src/sqlitecloud/pubsub.py": { @@ -119,7 +119,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 99, + "loc": 89, "nosec": 0 }, "src/sqlitecloud/upload.py": { @@ -155,7 +155,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 35, + "loc": 46, "nosec": 0 }, "src/tests/integration/__init__.py": { @@ -171,15 +171,15 @@ "nosec": 0 }, "src/tests/integration/test_client.py": { - "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.HIGH": 1.0, "CONFIDENCE.LOW": 1.0, "CONFIDENCE.MEDIUM": 0.0, "CONFIDENCE.UNDEFINED": 0.0, "SEVERITY.HIGH": 0.0, - "SEVERITY.LOW": 0.0, + "SEVERITY.LOW": 1.0, "SEVERITY.MEDIUM": 1.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 543, + "loc": 557, "nosec": 0 }, "src/tests/integration/test_dbapi2.py": { @@ -215,7 +215,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 18, + "loc": 31, "nosec": 0 }, "src/tests/integration/test_pandas.py": { @@ -243,15 +243,15 @@ "nosec": 0 }, "src/tests/integration/test_sqlite3_parity.py": { - "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.HIGH": 8.0, "CONFIDENCE.LOW": 0.0, - "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.MEDIUM": 8.0, "CONFIDENCE.UNDEFINED": 0.0, "SEVERITY.HIGH": 0.0, - "SEVERITY.LOW": 0.0, - "SEVERITY.MEDIUM": 0.0, + "SEVERITY.LOW": 8.0, + "SEVERITY.MEDIUM": 8.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 176, + "loc": 600, "nosec": 0 }, "src/tests/integration/test_upload.py": { @@ -287,7 +287,7 @@ "SEVERITY.LOW": 0.0, "SEVERITY.MEDIUM": 0.0, "SEVERITY.UNDEFINED": 0.0, - "loc": 241, + "loc": 260, "nosec": 0 }, "src/tests/unit/test_driver.py": { @@ -329,18 +329,14 @@ }, "results": [ { - "code": "87 class SQLiteCloudAccount:\n88 def __init__(\n89 self,\n90 username: Optional[str] = \"\",\n91 password: Optional[str] = \"\",\n92 hostname: str = \"\",\n93 dbname: Optional[str] = \"\",\n94 port: int = SQLITECLOUD_DEFAULT.PORT.value,\n95 apikey: Optional[str] = \"\",\n96 ) -> None:\n97 # User name is required unless connectionstring is provided\n98 self.username = username\n99 # Password is required unless connection string is provided\n100 self.password = password\n101 # Password is hashed\n102 self.password_hashed = False\n103 # API key instead of username and password\n104 self.apikey = apikey\n105 # Name of database to open\n106 self.dbname = dbname\n107 # Like mynode.sqlitecloud.io\n108 self.hostname = hostname\n109 self.port = port\n110 \n", + "code": "91 class SQLiteCloudAccount:\n92 def __init__(\n93 self,\n94 username: Optional[str] = \"\",\n95 password: Optional[str] = \"\",\n96 hostname: str = \"\",\n97 dbname: Optional[str] = \"\",\n98 port: int = SQLITECLOUD_DEFAULT.PORT.value,\n99 apikey: Optional[str] = \"\",\n100 ) -> None:\n101 # User name is required unless connectionstring is provided\n102 self.username = username\n103 # Password is required unless connection string is provided\n104 self.password = password\n105 # Password is hashed\n106 self.password_hashed = False\n107 # API key instead of username and password\n108 self.apikey = apikey\n109 # Name of database to open\n110 self.dbname = dbname\n111 # Like mynode.sqlitecloud.io\n112 self.hostname = hostname\n113 self.port = port\n114 \n", "col_offset": 4, "filename": "src/sqlitecloud/datatypes.py", "issue_confidence": "MEDIUM", "issue_severity": "LOW", "issue_text": "Possible hardcoded password: ''", - "line_number": 88, + "line_number": 92, "line_range": [ - 88, - 89, - 90, - 91, 92, 93, 94, @@ -358,22 +354,41 @@ 106, 107, 108, - 109 + 109, + 110, + 111, + 112, + 113 ], "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b107_hardcoded_password_default.html", "test_id": "B107", "test_name": "hardcoded_password_default" }, { - "code": "639 for i in range(nRows):\n640 sql += f\"INSERT INTO TestCompress (name) VALUES ('Test {i}'); \"\n641 \n", - "col_offset": 23, + "code": "644 \n645 table_name = \"TestCompress\" + str(random.randint(0, 99999))\n646 try:\n", + "col_offset": 42, + "filename": "src/tests/integration/test_client.py", + "issue_confidence": "HIGH", + "issue_severity": "LOW", + "issue_text": "Standard pseudo-random generators are not suitable for security/cryptographic purposes.", + "line_number": 645, + "line_range": [ + 645 + ], + "more_info": "https://bandit.readthedocs.io/en/latest/blacklists/blacklist_calls.html#b311-random", + "test_id": "B311", + "test_name": "blacklist" + }, + { + "code": "660 rowset = client.exec_query(\n661 f\"SELECT * from {table_name}\",\n662 connection,\n", + "col_offset": 16, "filename": "src/tests/integration/test_client.py", "issue_confidence": "LOW", "issue_severity": "MEDIUM", "issue_text": "Possible SQL injection vector through string-based query construction.", - "line_number": 640, + "line_number": 661, "line_range": [ - 640 + 661 ], "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b608_hardcoded_sql_expressions.html", "test_id": "B608", @@ -394,6 +409,246 @@ "test_id": "B608", "test_name": "hardcoded_sql_expressions" }, + { + "code": "385 \n386 tableName = \"TestTextFactory\" + str(random.randint(0, 99999))\n387 try:\n", + "col_offset": 44, + "filename": "src/tests/integration/test_sqlite3_parity.py", + "issue_confidence": "HIGH", + "issue_severity": "LOW", + "issue_text": "Standard pseudo-random generators are not suitable for security/cryptographic purposes.", + "line_number": 386, + "line_range": [ + 386 + ], + "more_info": "https://bandit.readthedocs.io/en/latest/blacklists/blacklist_calls.html#b311-random", + "test_id": "B311", + "test_name": "blacklist" + }, + { + "code": "390 cursor.execute(f\"INSERT INTO {tableName}(p) VALUES (?)\", (15,))\n391 cursor.execute(f\"SELECT p FROM {tableName}\")\n392 \n", + "col_offset": 27, + "filename": "src/tests/integration/test_sqlite3_parity.py", + "issue_confidence": "MEDIUM", + "issue_severity": "MEDIUM", + "issue_text": "Possible SQL injection vector through string-based query construction.", + "line_number": 391, + "line_range": [ + 391 + ], + "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b608_hardcoded_sql_expressions.html", + "test_id": "B608", + "test_name": "hardcoded_sql_expressions" + }, + { + "code": "414 \n415 tableName = \"TestTextFactory\" + str(random.randint(0, 99999))\n416 try:\n", + "col_offset": 44, + "filename": "src/tests/integration/test_sqlite3_parity.py", + "issue_confidence": "HIGH", + "issue_severity": "LOW", + "issue_text": "Standard pseudo-random generators are not suitable for security/cryptographic purposes.", + "line_number": 415, + "line_range": [ + 415 + ], + "more_info": "https://bandit.readthedocs.io/en/latest/blacklists/blacklist_calls.html#b311-random", + "test_id": "B311", + "test_name": "blacklist" + }, + { + "code": "419 cursor.execute(f\"INSERT INTO {tableName}(p) VALUES (?)\", (\"15\",))\n420 cursor.execute(f\"SELECT p FROM {tableName}\")\n421 \n", + "col_offset": 27, + "filename": "src/tests/integration/test_sqlite3_parity.py", + "issue_confidence": "MEDIUM", + "issue_severity": "MEDIUM", + "issue_text": "Possible SQL injection vector through string-based query construction.", + "line_number": 420, + "line_range": [ + 420 + ], + "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b608_hardcoded_sql_expressions.html", + "test_id": "B608", + "test_name": "hardcoded_sql_expressions" + }, + { + "code": "476 \n477 tableName = \"TestParseDeclTypes\" + str(random.randint(0, 99999))\n478 try:\n", + "col_offset": 47, + "filename": "src/tests/integration/test_sqlite3_parity.py", + "issue_confidence": "HIGH", + "issue_severity": "LOW", + "issue_text": "Standard pseudo-random generators are not suitable for security/cryptographic purposes.", + "line_number": 477, + "line_range": [ + 477 + ], + "more_info": "https://bandit.readthedocs.io/en/latest/blacklists/blacklist_calls.html#b311-random", + "test_id": "B311", + "test_name": "blacklist" + }, + { + "code": "481 cursor.executemany(f\"INSERT INTO {tableName}(p) VALUES (?)\", [(p1,), (p2,)])\n482 cursor.execute(f\"SELECT p FROM {tableName}\")\n483 \n", + "col_offset": 27, + "filename": "src/tests/integration/test_sqlite3_parity.py", + "issue_confidence": "MEDIUM", + "issue_severity": "MEDIUM", + "issue_text": "Possible SQL injection vector through string-based query construction.", + "line_number": 482, + "line_range": [ + 482 + ], + "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b608_hardcoded_sql_expressions.html", + "test_id": "B608", + "test_name": "hardcoded_sql_expressions" + }, + { + "code": "663 \n664 tableName = \"TestParseDeclTypes\" + str(random.randint(0, 99999))\n665 try:\n", + "col_offset": 47, + "filename": "src/tests/integration/test_sqlite3_parity.py", + "issue_confidence": "HIGH", + "issue_severity": "LOW", + "issue_text": "Standard pseudo-random generators are not suitable for security/cryptographic purposes.", + "line_number": 664, + "line_range": [ + 664 + ], + "more_info": "https://bandit.readthedocs.io/en/latest/blacklists/blacklist_calls.html#b311-random", + "test_id": "B311", + "test_name": "blacklist" + }, + { + "code": "668 cursor.execute(f\"INSERT INTO {tableName}(p) VALUES (?)\", (str(p),))\n669 cursor.execute(f\"SELECT p FROM {tableName}\")\n670 \n", + "col_offset": 27, + "filename": "src/tests/integration/test_sqlite3_parity.py", + "issue_confidence": "MEDIUM", + "issue_severity": "MEDIUM", + "issue_text": "Possible SQL injection vector through string-based query construction.", + "line_number": 669, + "line_range": [ + 669 + ], + "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b608_hardcoded_sql_expressions.html", + "test_id": "B608", + "test_name": "hardcoded_sql_expressions" + }, + { + "code": "693 \n694 tableName = \"TestParseDeclTypes\" + str(random.randint(0, 99999))\n695 try:\n", + "col_offset": 47, + "filename": "src/tests/integration/test_sqlite3_parity.py", + "issue_confidence": "HIGH", + "issue_severity": "LOW", + "issue_text": "Standard pseudo-random generators are not suitable for security/cryptographic purposes.", + "line_number": 694, + "line_range": [ + 694 + ], + "more_info": "https://bandit.readthedocs.io/en/latest/blacklists/blacklist_calls.html#b311-random", + "test_id": "B311", + "test_name": "blacklist" + }, + { + "code": "698 cursor.execute(f\"INSERT INTO {tableName}(p) VALUES (?)\", (mynumber,))\n699 cursor.execute(f\"SELECT p FROM {tableName}\")\n700 \n", + "col_offset": 27, + "filename": "src/tests/integration/test_sqlite3_parity.py", + "issue_confidence": "MEDIUM", + "issue_severity": "MEDIUM", + "issue_text": "Possible SQL injection vector through string-based query construction.", + "line_number": 699, + "line_range": [ + 699 + ], + "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b608_hardcoded_sql_expressions.html", + "test_id": "B608", + "test_name": "hardcoded_sql_expressions" + }, + { + "code": "726 \n727 tableName = \"TestParseDeclTypes\" + str(random.randint(0, 99999))\n728 try:\n", + "col_offset": 47, + "filename": "src/tests/integration/test_sqlite3_parity.py", + "issue_confidence": "HIGH", + "issue_severity": "LOW", + "issue_text": "Standard pseudo-random generators are not suitable for security/cryptographic purposes.", + "line_number": 727, + "line_range": [ + 727 + ], + "more_info": "https://bandit.readthedocs.io/en/latest/blacklists/blacklist_calls.html#b311-random", + "test_id": "B311", + "test_name": "blacklist" + }, + { + "code": "731 cursor.execute(f\"INSERT INTO {tableName}(p) VALUES (?)\", (pippo,))\n732 cursor.execute(f\"SELECT p FROM {tableName}\")\n733 \n", + "col_offset": 27, + "filename": "src/tests/integration/test_sqlite3_parity.py", + "issue_confidence": "MEDIUM", + "issue_severity": "MEDIUM", + "issue_text": "Possible SQL injection vector through string-based query construction.", + "line_number": 732, + "line_range": [ + 732 + ], + "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b608_hardcoded_sql_expressions.html", + "test_id": "B608", + "test_name": "hardcoded_sql_expressions" + }, + { + "code": "754 \n755 tableName = \"TestParseDeclTypes\" + str(random.randint(0, 99999))\n756 try:\n", + "col_offset": 47, + "filename": "src/tests/integration/test_sqlite3_parity.py", + "issue_confidence": "HIGH", + "issue_severity": "LOW", + "issue_text": "Standard pseudo-random generators are not suitable for security/cryptographic purposes.", + "line_number": 755, + "line_range": [ + 755 + ], + "more_info": "https://bandit.readthedocs.io/en/latest/blacklists/blacklist_calls.html#b311-random", + "test_id": "B311", + "test_name": "blacklist" + }, + { + "code": "759 cursor.execute(f\"INSERT INTO {tableName}(p) VALUES (?)\", (mynumber,))\n760 cursor.execute(f\"SELECT p FROM {tableName}\")\n761 \n", + "col_offset": 27, + "filename": "src/tests/integration/test_sqlite3_parity.py", + "issue_confidence": "MEDIUM", + "issue_severity": "MEDIUM", + "issue_text": "Possible SQL injection vector through string-based query construction.", + "line_number": 760, + "line_range": [ + 760 + ], + "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b608_hardcoded_sql_expressions.html", + "test_id": "B608", + "test_name": "hardcoded_sql_expressions" + }, + { + "code": "778 \n779 tableName = \"TestParseDeclTypes\" + str(random.randint(0, 99999))\n780 try:\n", + "col_offset": 47, + "filename": "src/tests/integration/test_sqlite3_parity.py", + "issue_confidence": "HIGH", + "issue_severity": "LOW", + "issue_text": "Standard pseudo-random generators are not suitable for security/cryptographic purposes.", + "line_number": 779, + "line_range": [ + 779 + ], + "more_info": "https://bandit.readthedocs.io/en/latest/blacklists/blacklist_calls.html#b311-random", + "test_id": "B311", + "test_name": "blacklist" + }, + { + "code": "791 )\n792 cursor.execute(f\"SELECT d, t FROM {tableName}\")\n793 \n", + "col_offset": 27, + "filename": "src/tests/integration/test_sqlite3_parity.py", + "issue_confidence": "MEDIUM", + "issue_severity": "MEDIUM", + "issue_text": "Possible SQL injection vector through string-based query construction.", + "line_number": 792, + "line_range": [ + 792 + ], + "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b608_hardcoded_sql_expressions.html", + "test_id": "B608", + "test_name": "hardcoded_sql_expressions" + }, { "code": "18 rowset = client.exec_query(\n19 f\"USE DATABASE {dbname}; SELECT * FROM contacts\", connection\n20 )\n", "col_offset": 16, diff --git a/src/sqlitecloud/__init__.py b/src/sqlitecloud/__init__.py index 387635f..06a78db 100644 --- a/src/sqlitecloud/__init__.py +++ b/src/sqlitecloud/__init__.py @@ -2,8 +2,25 @@ # the classes and functions from the dbapi2 module. # eg: sqlite3.connect() -> sqlitecloud.connect() # -from .dbapi2 import Connection, Cursor, connect, register_adapter +from .dbapi2 import ( + PARSE_COLNAMES, + PARSE_DECLTYPES, + Connection, + Cursor, + connect, + register_adapter, + register_converter, +) -__all__ = ["VERSION", "Connection", "Cursor", "connect", "register_adapter"] +__all__ = [ + "VERSION", + "Connection", + "Cursor", + "connect", + "register_adapter", + "register_converter", + "PARSE_DECLTYPES", + "PARSE_COLNAMES", +] VERSION = "0.0.79" diff --git a/src/sqlitecloud/dbapi2.py b/src/sqlitecloud/dbapi2.py index bda003a..12b7c63 100644 --- a/src/sqlitecloud/dbapi2.py +++ b/src/sqlitecloud/dbapi2.py @@ -50,8 +50,10 @@ PARSE_DECLTYPES = 1 PARSE_COLNAMES = 2 -# Adapter registry to convert Python types to SQLite types -adapters = {} +# Adapters registry to convert Python types to SQLite types +_adapters = {} +# Converters registry to convert SQLite types to Python types +_converters = {} @overload @@ -106,6 +108,11 @@ def connect( It can be either a connection string or a `SqliteCloudAccount` object. config (Optional[SQLiteCloudConfig]): The configuration options for the connection. Defaults to None. + detect_types (int): Default (0), disabled. How data types not natively supported + by SQLite are looked up to be converted to Python types, using the converters + registered with register_converter(). + Accepts any combination (using |, bitwise or) of PARSE_DECLTYPES and PARSE_COLNAMES. + Column names takes precedence over declared types if both flags are set. Returns: Connection: A DB-API 2.0 connection object representing the connection to the database. @@ -122,13 +129,16 @@ def connect( else: config = SQLiteCloudConfig(connection_info) - return Connection( - driver.connect(config.account.hostname, config.account.port, config) + connection = Connection( + driver.connect(config.account.hostname, config.account.port, config), + detect_types=detect_types, ) + return connection + def register_adapter( - pytype: Type, adapter_callable: Callable[[object], SQLiteTypes] + pytype: Type, adapter_callable: Callable[[Any], SQLiteTypes] ) -> None: """ Registers a callable to convert the type into one of the supported SQLite types. @@ -138,8 +148,21 @@ def register_adapter( callable (Callable): The callable that converts the type into a supported SQLite supported type. """ - global adapters - adapters[pytype] = adapter_callable + global _adapters + _adapters[pytype] = adapter_callable + + +def register_converter(type_name: str, converter: Callable[[bytes], Any]) -> None: + """ + Registers a callable to convert a bytestring from the database into a custom Python type. + + Args: + type_name (str): The name of the type to convert. + The match with the name of the type in the query is case-insensitive. + converter (Callable): The callable that converts the bytestring into the custom Python type. + """ + global _converters + _converters[type_name.lower()] = converter class Connection: @@ -154,16 +177,16 @@ class Connection: SQLiteCloud_connection (SQLiteCloudConnect): The SQLite Cloud connection object. """ - def __init__(self, sqlitecloud_connection: SQLiteCloudConnect) -> None: + def __init__( + self, sqlitecloud_connection: SQLiteCloudConnect, detect_types: int = 0 + ) -> None: self._driver = Driver() self.sqlitecloud_connection = sqlitecloud_connection self.row_factory: Optional[Callable[["Cursor", Tuple], object]] = None - self.text_factory: Union[ - Type[Union[str, bytes]], Callable[[bytes], object] - ] = str + self.text_factory: Union[Type[Union[str, bytes]], Callable[[bytes], Any]] = str - self.detect_types = 0 + self.detect_types = detect_types @property def sqlcloud_connection(self) -> SQLiteCloudConnect: @@ -273,19 +296,19 @@ def cursor(self): cursor.row_factory = self.row_factory return cursor - def _apply_adapter(self, value: object) -> SQLiteTypes: + def _apply_adapter(self, value: Any) -> SQLiteTypes: """ Applies the registered adapter to convert the Python type into a SQLite supported type. In the case there is no registered adapter, it calls the __conform__() method when the value object implements it. Args: - value (object): The Python type to convert. + value (Any): The Python type to convert. Returns: SQLiteTypes: The SQLite supported type or the given value when no adapter is found. """ - if type(value) in adapters: - return adapters[type(value)](value) + if type(value) in _adapters: + return _adapters[type(value)](value) if hasattr(value, "__conform__"): # we don't support sqlite3.PrepareProtocol @@ -445,6 +468,8 @@ def executemany( commands = "" for parameters in seq_of_parameters: + parameters = self._adapt_parameters(parameters) + prepared_statement = self._driver.prepare_statement(sql, parameters) commands += prepared_statement + ";" @@ -547,24 +572,51 @@ def _adapt_parameters(self, parameters: Union[Dict, Tuple]) -> Union[Dict, Tuple return tuple(self._connection._apply_adapter(p) for p in parameters) + def _convert_value(self, value: Any, decltype: Optional[str]) -> Any: + # todo: parse columns first + + if (self.connection.detect_types & PARSE_DECLTYPES) == PARSE_DECLTYPES: + return self._parse_decltypes(value, decltype) + + if decltype == SQLITECLOUD_VALUE_TYPE.TEXT.value or ( + decltype is None and isinstance(value, str) + ): + return self._apply_text_factory(value) + + return value + + def _parse_decltypes(self, value: Any, decltype: str) -> Any: + decltype = decltype.lower() + if decltype in _converters: + # sqlite3 always passes value as bytes + value = ( + str(value).encode("utf-8") if not isinstance(value, bytes) else value + ) + return _converters[decltype](value) + + return value + + def _apply_text_factory(self, value: Any) -> Any: + """Use Connection.text_factory to convert value with TEXT column or + string value with undleclared column type.""" + + if self._connection.text_factory is bytes: + return value.encode("utf-8") + if self._connection.text_factory is not str and callable( + self._connection.text_factory + ): + return self._connection.text_factory(value.encode("utf-8")) + + return value + def _get_value(self, row: int, col: int) -> Optional[Any]: if not self._is_result_rowset(): return None - # Convert TEXT type with text_factory + value = self._resultset.get_value(row, col) decltype = self._resultset.get_decltype(col) - if decltype is None or decltype == SQLITECLOUD_VALUE_TYPE.TEXT.value: - value = self._resultset.get_value(row, col, False) - - if self._connection.text_factory is bytes: - return value.encode("utf-8") - if self._connection.text_factory is not str and callable( - self._connection.text_factory - ): - return self._connection.text_factory(value.encode("utf-8")) - return value - return self._resultset.get_value(row, col) + return self._convert_value(value, decltype) def __iter__(self) -> "Cursor": return self @@ -602,7 +654,7 @@ def adapt_datetime(val): return val.isoformat(" ") def convert_date(val): - return datetime.date(*map(int, val.split(b"-"))) + return date(*map(int, val.split(b"-"))) def convert_timestamp(val): datepart, timepart = val.split(b" ") @@ -614,13 +666,13 @@ def convert_timestamp(val): else: microseconds = 0 - val = datetime.datetime(year, month, day, hours, minutes, seconds, microseconds) + val = datetime(year, month, day, hours, minutes, seconds, microseconds) return val register_adapter(date, adapt_date) register_adapter(datetime, adapt_datetime) - # register_converter("date", convert_date) - # register_converter("timestamp", convert_timestamp) + register_converter("date", convert_date) + register_converter("timestamp", convert_timestamp) register_adapters_and_converters() diff --git a/src/sqlitecloud/resultset.py b/src/sqlitecloud/resultset.py index 034a6b0..d0b6d7d 100644 --- a/src/sqlitecloud/resultset.py +++ b/src/sqlitecloud/resultset.py @@ -60,13 +60,12 @@ def _compute_index(self, row: int, col: int) -> int: return -1 return row * self.ncols + col - def get_value(self, row: int, col: int, convert: bool = True) -> Optional[any]: + def get_value(self, row: int, col: int) -> Optional[any]: index = self._compute_index(row, col) if index < 0 or not self.data or index >= len(self.data): return None - value = self.data[index] - return self._convert(value, col) if convert else value + return self.data[index] def get_name(self, col: int) -> Optional[str]: if col < 0 or col >= self.ncols: @@ -79,23 +78,6 @@ def get_decltype(self, col: int) -> Optional[str]: return self.decltype[col] - def _convert(self, value: str, col: int) -> any: - if col < 0 or col >= len(self.decltype): - return value - - decltype = self.decltype[col] - if decltype == SQLITECLOUD_VALUE_TYPE.INTEGER.value: - return int(value) - if decltype == SQLITECLOUD_VALUE_TYPE.FLOAT.value: - return float(value) - if decltype == SQLITECLOUD_VALUE_TYPE.BLOB.value: - # values are received as bytes before being strings - return bytes(value) - if decltype == SQLITECLOUD_VALUE_TYPE.NULL.value: - return None - - return value - class SQLiteCloudResultSet: def __init__(self, result: SQLiteCloudResult) -> None: diff --git a/src/tests/conftest.py b/src/tests/conftest.py index 891ce1e..78d03ea 100644 --- a/src/tests/conftest.py +++ b/src/tests/conftest.py @@ -45,7 +45,7 @@ def sqlitecloud_dbapi2_connection(): yield next(get_sqlitecloud_dbapi2_connection()) -def get_sqlitecloud_dbapi2_connection(): +def get_sqlitecloud_dbapi2_connection(detect_types: int = 0): account = SQLiteCloudAccount() account.username = os.getenv("SQLITE_USER") account.password = os.getenv("SQLITE_PASSWORD") @@ -53,7 +53,7 @@ def get_sqlitecloud_dbapi2_connection(): account.hostname = os.getenv("SQLITE_HOST") account.port = int(os.getenv("SQLITE_PORT")) - connection = sqlitecloud.connect(account) + connection = sqlitecloud.connect(account, detect_types=detect_types) assert isinstance(connection, sqlitecloud.Connection) @@ -62,12 +62,13 @@ def get_sqlitecloud_dbapi2_connection(): connection.close() -def get_sqlite3_connection(): +def get_sqlite3_connection(detect_types: int = 0): # set isolation_level=None to enable autocommit # and to be aligned with the behavior of SQLite Cloud connection = sqlite3.connect( os.path.join(os.path.dirname(__file__), "./assets/chinook.sqlite"), isolation_level=None, + detect_types=detect_types, ) yield connection connection.close() diff --git a/src/tests/integration/test_client.py b/src/tests/integration/test_client.py index 67d8fe4..ec31a7f 100644 --- a/src/tests/integration/test_client.py +++ b/src/tests/integration/test_client.py @@ -1,4 +1,5 @@ import os +import random import time import pytest @@ -641,10 +642,10 @@ def test_big_rowset(self): connection = client.open_connection() - table_name = "TestCompress" + str(int(time.time())) + table_name = "TestCompress" + str(random.randint(0, 99999)) try: client.exec_query( - f"CREATE TABLE IF NOT EXISTS {table_name} (id INTEGER PRIMARY KEY, name TEXT)", + f"CREATE TABLE {table_name} (id INTEGER PRIMARY KEY, name TEXT)", connection, ) @@ -663,7 +664,7 @@ def test_big_rowset(self): assert rowset.nrows == nRows finally: - client.exec_query(f"DROP TABLE {table_name}", connection) + client.exec_query(f"DROP TABLE IF EXISTS {table_name}", connection) client.disconnect(connection) def test_compression_single_column(self): diff --git a/src/tests/integration/test_sqlite3_parity.py b/src/tests/integration/test_sqlite3_parity.py index abb395c..40fae1b 100644 --- a/src/tests/integration/test_sqlite3_parity.py +++ b/src/tests/integration/test_sqlite3_parity.py @@ -1,3 +1,4 @@ +import random import sqlite3 import time from datetime import date, datetime @@ -367,6 +368,63 @@ def test_text_factory_with_callable( assert result[0] == "barFoo" + @pytest.mark.parametrize( + "connection", + [ + "sqlitecloud_dbapi2_connection", + "sqlite3_connection", + ], + ) + def test_apply_text_factory_to_int_value_with_text_decltype( + self, connection, request + ): + """Expect the text_factory to be applied when the inserted + value is an integer but the declared type for the column is TEXT.""" + connection = request.getfixturevalue(connection) + connection.text_factory = bytes + + tableName = "TestTextFactory" + str(random.randint(0, 99999)) + try: + cursor = connection.execute(f"CREATE TABLE {tableName}(p TEXT)") + + cursor.execute(f"INSERT INTO {tableName}(p) VALUES (?)", (15,)) + cursor.execute(f"SELECT p FROM {tableName}") + + result = cursor.fetchone() + + assert result[0] == b"15" + finally: + connection.execute(f"DROP TABLE IF EXISTS {tableName}") + + @pytest.mark.parametrize( + "connection", + [ + "sqlitecloud_dbapi2_connection", + "sqlite3_connection", + ], + ) + def test_not_apply_text_factory_to_string_value_without_text_decltype( + self, connection, request + ): + """Expect the text_factory to be not applied when the inserted + value is a string but the declared type for the column is not TEXT.""" + + connection = request.getfixturevalue(connection) + connection.text_factory = bytes + + tableName = "TestTextFactory" + str(random.randint(0, 99999)) + try: + cursor = connection.execute(f"CREATE TABLE {tableName}(p INTEGER)") + + cursor.execute(f"INSERT INTO {tableName}(p) VALUES (?)", ("15",)) + cursor.execute(f"SELECT p FROM {tableName}") + + result = cursor.fetchone() + + assert result[0] == 15 + finally: + connection.execute(f"DROP TABLE IF EXISTS {tableName}") + @pytest.mark.parametrize( "connection, module", [ @@ -394,6 +452,42 @@ def adapt_point(point): assert result[0] == "4.0, -3.2" + @pytest.mark.parametrize( + "connection, module", + [ + ("sqlitecloud_dbapi2_connection", sqlitecloud), + ("sqlite3_connection", sqlite3), + ], + ) + def test_register_adapter_and_executemany(self, connection, module, request): + connection = request.getfixturevalue(connection) + + class Point: + def __init__(self, x, y): + self.x, self.y = x, y + + def adapt_point(point): + return f"{point.x}, {point.y}" + + module.register_adapter(Point, adapt_point) + + p1 = Point(4.0, -3.2) + p2 = Point(2.1, 1.9) + + tableName = "TestParseDeclTypes" + str(random.randint(0, 99999)) + try: + cursor = connection.execute(f"CREATE TABLE {tableName}(p)") + + cursor.executemany(f"INSERT INTO {tableName}(p) VALUES (?)", [(p1,), (p2,)]) + cursor.execute(f"SELECT p FROM {tableName}") + + result = cursor.fetchall() + + assert result[0][0] == "4.0, -3.2" + assert result[1][0] == "2.1, 1.9" + finally: + connection.execute(f"DROP TABLE IF EXISTS {tableName}") + @pytest.mark.parametrize( "connection, module", [ @@ -541,51 +635,213 @@ def adapt_point(point): assert result[0] == "4.0, -3.2" - # def test_datatypes(self, sqlite3_connection): - # class Point: - # def __init__(self, x, y): - # self.x, self.y = x, y - - # def __repr__(self): - # return "(%f;%f)" % (self.x, self.y) - - # def adapt_point(point): - # return ("%f;%f" % (point.x, point.y)).encode('ascii') - - # def convert_point(s): - # x, y = list(map(float, s.split(b";"))) - # return Point(x, y) - - # # Register the adapter - # sqlite3.register_adapter(Point, adapt_point) - - # # Register the converter - # sqlite3.register_converter("point", convert_point) - - # p = Point(4.0, -3.2) - - # ######################### - # # 1) Using declared types - # con = sqlite3.connect(":memory:", detect_types=sqlite3.PARSE_DECLTYPES) - # cur = con.cursor() - # cur.execute("create table test(p point)") - - # cur.execute("insert into test(p) values (?)", (p,)) - # cur.execute("select p from test") - # r = cur.fetchone() - # print("with declared types:", r[0]) - # cur.close() - # con.close() - - # ####################### - # # 1) Using column names - # con = sqlite3.connect(":memory:", detect_types=sqlite3.PARSE_COLNAMES) - # cur = con.cursor() - # cur.execute("create table test(p)") - - # cur.execute("insert into test(p) values (?)", (p,)) - # cur.execute('select p as "p [point]" from test') - # r = cur.fetchone() - # print("with column names:", r[0]) - # cur.close() - # con.close() + @pytest.mark.parametrize( + "connection, module", + [ + ( + next(get_sqlitecloud_dbapi2_connection(sqlitecloud.PARSE_DECLTYPES)), + sqlitecloud, + ), + (next(get_sqlite3_connection(sqlite3.PARSE_DECLTYPES)), sqlite3), + ], + ) + def test_parse_decltype(self, connection, module): + class Point: + def __init__(self, x, y): + self.x, self.y = x, y + + def __repr__(self): + return f"{self.x};{self.y}" + + def convert_point(s): + x, y = list(map(float, s.split(b";"))) + return Point(x, y) + + module.register_converter("point", convert_point) + + p = Point(4.0, -3.2) + + tableName = "TestParseDeclTypes" + str(random.randint(0, 99999)) + try: + cursor = connection.execute(f"CREATE TABLE {tableName}(p point)") + + cursor.execute(f"INSERT INTO {tableName}(p) VALUES (?)", (str(p),)) + cursor.execute(f"SELECT p FROM {tableName}") + + result = cursor.fetchone() + + assert isinstance(result[0], Point) + assert result[0].x == p.x + assert result[0].y == p.y + finally: + connection.execute(f"DROP TABLE IF EXISTS {tableName}") + + @pytest.mark.parametrize( + "connection, module", + [ + ( + next(get_sqlitecloud_dbapi2_connection(sqlitecloud.PARSE_DECLTYPES)), + sqlitecloud, + ), + (next(get_sqlite3_connection(sqlite3.PARSE_DECLTYPES)), sqlite3), + ], + ) + def test_register_converter_case_insensitive(self, connection, module): + module.register_converter("integer", lambda x: int(x.decode("utf-8")) + 7) + + mynumber = 10 + + tableName = "TestParseDeclTypes" + str(random.randint(0, 99999)) + try: + cursor = connection.execute(f"CREATE TABLE {tableName}(p INTEGER)") + + cursor.execute(f"INSERT INTO {tableName}(p) VALUES (?)", (mynumber,)) + cursor.execute(f"SELECT p FROM {tableName}") + + result = cursor.fetchone() + + assert result[0] == 17 + finally: + connection.execute(f"DROP TABLE IF EXISTS {tableName}") + + @pytest.mark.parametrize( + "connection, module", + [ + ( + next(get_sqlitecloud_dbapi2_connection(sqlitecloud.PARSE_DECLTYPES)), + sqlitecloud, + ), + (next(get_sqlite3_connection(sqlite3.PARSE_DECLTYPES)), sqlite3), + ], + ) + def test_registered_converter_on_text_decltype_replaces_text_factory( + self, connection, module + ): + """Expect the registered converter to the TEXT decltype to be used in place of the text_factory.""" + + module.register_converter("TEXT", lambda x: x.decode("utf-8") + "Foo") + connection.text_factory = bytes + + pippo = "Pippo" + + tableName = "TestParseDeclTypes" + str(random.randint(0, 99999)) + try: + cursor = connection.execute(f"CREATE TABLE {tableName}(p TEXT)") + + cursor.execute(f"INSERT INTO {tableName}(p) VALUES (?)", (pippo,)) + cursor.execute(f"SELECT p FROM {tableName}") + + result = cursor.fetchone() + + assert result[0] == pippo + "Foo" + finally: + connection.execute(f"DROP TABLE IF EXISTS {tableName}") + + @pytest.mark.parametrize( + "connection, module", + [ + ( + next(get_sqlitecloud_dbapi2_connection(sqlitecloud.PARSE_DECLTYPES)), + sqlitecloud, + ), + (next(get_sqlite3_connection(sqlite3.PARSE_DECLTYPES)), sqlite3), + ], + ) + def test_parse_native_decltype(self, connection, module): + module.register_converter("INTEGER", lambda x: int(x.decode("utf-8")) + 10) + + mynumber = 10 + + tableName = "TestParseDeclTypes" + str(random.randint(0, 99999)) + try: + cursor = connection.execute(f"CREATE TABLE {tableName}(p INTEGER)") + + cursor.execute(f"INSERT INTO {tableName}(p) VALUES (?)", (mynumber,)) + cursor.execute(f"SELECT p FROM {tableName}") + + result = cursor.fetchone() + + assert result[0] == 20 + finally: + connection.execute(f"DROP TABLE IF EXISTS {tableName}") + + @pytest.mark.parametrize( + "connection", + [ + next(get_sqlitecloud_dbapi2_connection(sqlitecloud.PARSE_DECLTYPES)), + next(get_sqlite3_connection(sqlite3.PARSE_DECLTYPES)), + ], + ) + def test_register_adapters_and_converters_for_date_and_datetime_by_default( + self, connection + ): + + tableName = "TestParseDeclTypes" + str(random.randint(0, 99999)) + try: + today = date.today() + now = datetime.now() + + cursor = connection.execute( + f"CREATE TABLE {tableName}(d DATE, t timestamp)" + ) + + cursor.execute( + f"INSERT INTO {tableName}(d, t) VALUES (:date, :timestamp)", + {"date": today, "timestamp": now}, + ) + cursor.execute(f"SELECT d, t FROM {tableName}") + + result = cursor.fetchone() + + assert isinstance(result[0], date) + assert isinstance(result[1], datetime) + assert result[0] == today + assert result[1] == now + finally: + connection.execute(f"DROP TABLE IF EXISTS {tableName}") + + @pytest.mark.parametrize( + "connection, module", + [ + ( + next(get_sqlitecloud_dbapi2_connection(sqlitecloud.PARSE_DECLTYPES)), + sqlitecloud, + ), + (next(get_sqlite3_connection(sqlite3.PARSE_DECLTYPES)), sqlite3), + ], + ) + def test_adapt_and_convert_custom_decltype(self, connection, module): + class Point: + def __init__(self, x, y): + self.x, self.y = x, y + + def __repr__(self): + return f"({self.x};{self.y})" + + def adapt_point(point): + return f"{point.x};{point.y}".encode("ascii") + + def convert_point(s): + x, y = list(map(float, s.split(b";"))) + return Point(x, y) + + module.register_adapter(Point, adapt_point) + module.register_converter("point", convert_point) + + p = Point(4.0, -3.2) + + tableName = "TestParseDeclTypes" + str(random.randint(0, 99999)) + try: + cursor = connection.cursor() + cursor.execute(f"CREATE TABLE {tableName}(p point)") + + cursor.execute(f"INSERT INTO {tableName}(p) VALUES (?)", (p,)) + cursor.execute(f"SELECT p FROM {tableName}") + + result = cursor.fetchone() + + assert isinstance(result[0], Point) + assert result[0].x == p.x + assert result[0].y == p.y + finally: + connection.execute(f"DROP TABLE IF EXISTS {tableName}") diff --git a/src/tests/unit/test_dbapi2.py b/src/tests/unit/test_dbapi2.py index 722ae1b..fd85da4 100644 --- a/src/tests/unit/test_dbapi2.py +++ b/src/tests/unit/test_dbapi2.py @@ -105,16 +105,15 @@ def test_rowcount_with_no_resultset(self, mocker): assert cursor.rowcount == -1 def test_execute_escaped(self, mocker: MockerFixture): - connection = mocker.patch("sqlitecloud.Connection") - apply_adapter_mock = mocker.patch.object(connection, "_apply_adapter") - apply_adapter_mock.return_value = "John's" + parameters = ("John's",) execute_mock = mocker.patch.object(Driver, "execute") - cursor = Cursor(connection) + cursor = Cursor(mocker.patch("sqlitecloud.Connection")) + apply_adapter_mock = mocker.patch.object(cursor, "_adapt_parameters") + apply_adapter_mock.return_value = parameters sql = "SELECT * FROM users WHERE name = ?" - parameters = ("John's",) cursor.execute(sql, parameters) @@ -123,11 +122,14 @@ def test_execute_escaped(self, mocker: MockerFixture): ) def test_executemany(self, mocker): + seq_of_parameters = [("John", 25), ("Jane", 30), ("Bob", 40)] + cursor = Cursor(mocker.patch("sqlitecloud.Connection")) execute_mock = mocker.patch.object(cursor, "execute") + apply_adapter_mock = mocker.patch.object(cursor, "_adapt_parameters") + apply_adapter_mock.side_effect = seq_of_parameters sql = "INSERT INTO users (name, age) VALUES (?, ?)" - seq_of_parameters = [("John", 25), ("Jane", 30), ("Bob", 40)] cursor.executemany(sql, seq_of_parameters) @@ -136,11 +138,14 @@ def test_executemany(self, mocker): ) def test_executemany_escaped(self, mocker): + seq_of_parameters = [("O'Conner", 25)] + cursor = Cursor(mocker.patch("sqlitecloud.Connection")) execute_mock = mocker.patch.object(cursor, "execute") + apply_adapter_mock = mocker.patch.object(cursor, "_adapt_parameters") + apply_adapter_mock.side_effect = seq_of_parameters sql = "INSERT INTO users (name, age) VALUES (?, ?)" - seq_of_parameters = [("O'Conner", 25)] cursor.executemany(sql, seq_of_parameters) diff --git a/src/tests/unit/test_resultset.py b/src/tests/unit/test_resultset.py index 18a43f2..8090120 100644 --- a/src/tests/unit/test_resultset.py +++ b/src/tests/unit/test_resultset.py @@ -2,7 +2,6 @@ from sqlitecloud.resultset import ( SQLITECLOUD_RESULT_TYPE, - SQLITECLOUD_VALUE_TYPE, SQLiteCloudResult, SQLiteCloudResultSet, ) @@ -83,30 +82,8 @@ def test_get_value_with_convert_false(self): result.data = ["John", "42"] result.decltype = ["TEXT", "INTEGER"] - assert "John" == result.get_value(0, 0, convert=False) - assert "42" == result.get_value(0, 1, convert=False) - - @pytest.mark.parametrize( - "value_type, value, expected_value", - [ - (SQLITECLOUD_VALUE_TYPE.INTEGER.value, "24", 24), - (SQLITECLOUD_VALUE_TYPE.FLOAT.value, "3.14", 3.14), - (SQLITECLOUD_VALUE_TYPE.TEXT.value, "John", "John"), - (SQLITECLOUD_VALUE_TYPE.BLOB.value, b"hello", b"hello"), - (SQLITECLOUD_VALUE_TYPE.NULL.value, "NULL", None), - ], - ) - def test_get_value_to_convert_text(self, value_type, value, expected_value): - result = SQLiteCloudResult(SQLITECLOUD_RESULT_TYPE.RESULT_ROWSET) - result.nrows = 1 - result.ncols = 1 - result.colname = ["mycol"] - result.data = [value] - result.decltype = [value_type] - - result_set = SQLiteCloudResultSet(result) - - assert expected_value == result_set.get_value(0, 0) + assert "John" == result.get_value(0, 0) + assert "42" == result.get_value(0, 1) class TestSqliteCloudResultSet: