diff --git a/DESCRIPTION.md b/DESCRIPTION.md index 0de5c43ca..aec46651c 100644 --- a/DESCRIPTION.md +++ b/DESCRIPTION.md @@ -13,7 +13,6 @@ Source code is also available at: https://github.com/snowflakedb/snowflake-conne - Added for non-Windows platforms command suggestions (chown/chmod) for insufficient file permissions of config files. - Fixed issue with connection diagnostics failing to complete certificate checks. - Fixed issue that arrow iterator causes `ImportError` when the c extensions are not compiled. - - Removed dependencies on Cryptodome and oscrypto and removed the `use_openssl_only` parameter. All connections now go through OpenSSL via the cryptography library, which was already a dependency. - v3.3.0(October 10,2023) diff --git a/ci/test_fips_docker.sh b/ci/test_fips_docker.sh index 4150296de..e7c8c8dea 100755 --- a/ci/test_fips_docker.sh +++ b/ci/test_fips_docker.sh @@ -21,6 +21,7 @@ user_id=$(id -u $USER) docker run --network=host \ -e LANG=en_US.UTF-8 \ -e TERM=vt102 \ + -e SF_USE_OPENSSL_ONLY=True \ -e PIP_DISABLE_PIP_VERSION_CHECK=1 \ -e LOCAL_USER_ID=$user_id \ -e CRYPTOGRAPHY_ALLOW_OPENSSL_102=1 \ diff --git a/setup.cfg b/setup.cfg index 2836f2087..3b49dc264 100644 --- a/setup.cfg +++ b/setup.cfg @@ -45,7 +45,9 @@ install_requires = asn1crypto>0.24.0,<2.0.0 cffi>=1.9,<2.0.0 cryptography>=3.1.0,<42.0.0 + oscrypto<2.0.0 pyOpenSSL>=16.2.0,<24.0.0 + pycryptodomex!=3.5.0,>=3.2,<4.0.0 pyjwt<3.0.0 pytz requests<3.0.0 diff --git a/src/snowflake/connector/connection.py b/src/snowflake/connector/connection.py index 5fd4c1cda..9f2d9844b 100644 --- a/src/snowflake/connector/connection.py +++ b/src/snowflake/connector/connection.py @@ -189,9 +189,9 @@ def DefaultConverterClass() -> type: "client_store_temporary_credential": (False, bool), "client_request_mfa_token": (False, bool), "use_openssl_only": ( - True, + False, bool, - ), # ignored - python only crypto modules are no longer used + ), # only use openssl instead of python only crypto modules # whether to convert Arrow number values to decimal instead of doubles "arrow_number_to_decimal": (False, bool), "enable_stage_s3_privatelink_for_us_east_1": ( @@ -287,6 +287,7 @@ class SnowflakeConnection: validate_default_parameters: Validate database, schema, role and warehouse used on Snowflake. is_pyformat: Whether the current argument binding is pyformat or format. consent_cache_id_token: Consented cache ID token. + use_openssl_only: Use OpenSSL instead of pure Python libraries for signature verification and encryption. enable_stage_s3_privatelink_for_us_east_1: when true, clients use regional s3 url to upload files. enable_connection_diag: when true, clients will generate a connectivity diagnostic report. connection_diag_log_path: path to location to create diag report with enable_connection_diag. @@ -573,8 +574,7 @@ def disable_request_pooling(self, value) -> None: @property def use_openssl_only(self) -> bool: - # Deprecated, kept for backwards compatibility - return True + return self._use_openssl_only @property def arrow_number_to_decimal(self): @@ -1117,6 +1117,19 @@ def __config(self, **kwargs): "CHECKED." ) + if "SF_USE_OPENSSL_ONLY" not in os.environ: + logger.info("Setting use_openssl_only mode to %s", self.use_openssl_only) + os.environ["SF_USE_OPENSSL_ONLY"] = str(self.use_openssl_only) + elif ( + os.environ.get("SF_USE_OPENSSL_ONLY", "False") == "True" + ) != self.use_openssl_only: + logger.warning( + "Mode use_openssl_only is already set to: %s, ignoring set request to: %s", + os.environ["SF_USE_OPENSSL_ONLY"], + self.use_openssl_only, + ) + self._use_openssl_only = os.environ["SF_USE_OPENSSL_ONLY"] == "True" + def cmd_query( self, sql: str, diff --git a/src/snowflake/connector/encryption_util.py b/src/snowflake/connector/encryption_util.py index c1c34079e..d8f767ecb 100644 --- a/src/snowflake/connector/encryption_util.py +++ b/src/snowflake/connector/encryption_util.py @@ -12,6 +12,7 @@ from logging import getLogger from typing import IO, TYPE_CHECKING +from Cryptodome.Cipher import AES from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes @@ -68,6 +69,7 @@ def encrypt_stream( The encryption metadata. """ logger = getLogger(__name__) + use_openssl_only = os.getenv("SF_USE_OPENSSL_ONLY", "False") == "True" decoded_key = base64.standard_b64decode( encryption_material.query_stage_master_key ) @@ -77,9 +79,14 @@ def encrypt_stream( # Generate key for data encryption iv_data = SnowflakeEncryptionUtil.get_secure_random(block_size) file_key = SnowflakeEncryptionUtil.get_secure_random(key_size) - backend = default_backend() - cipher = Cipher(algorithms.AES(file_key), modes.CBC(iv_data), backend=backend) - encryptor = cipher.encryptor() + if not use_openssl_only: + data_cipher = AES.new(key=file_key, mode=AES.MODE_CBC, IV=iv_data) + else: + backend = default_backend() + cipher = Cipher( + algorithms.AES(file_key), modes.CBC(iv_data), backend=backend + ) + encryptor = cipher.encryptor() padded = False while True: @@ -89,17 +96,30 @@ def encrypt_stream( elif len(chunk) % block_size != 0: chunk = PKCS5_PAD(chunk, block_size) padded = True - out.write(encryptor.update(chunk)) + if not use_openssl_only: + out.write(data_cipher.encrypt(chunk)) + else: + out.write(encryptor.update(chunk)) if not padded: - out.write(encryptor.update(block_size * chr(block_size).encode(UTF8))) - out.write(encryptor.finalize()) + if not use_openssl_only: + out.write( + data_cipher.encrypt(block_size * chr(block_size).encode(UTF8)) + ) + else: + out.write(encryptor.update(block_size * chr(block_size).encode(UTF8))) + if use_openssl_only: + out.write(encryptor.finalize()) # encrypt key with QRMK - cipher = Cipher(algorithms.AES(decoded_key), modes.ECB(), backend=backend) - encryptor = cipher.encryptor() - enc_kek = ( - encryptor.update(PKCS5_PAD(file_key, block_size)) + encryptor.finalize() - ) + if not use_openssl_only: + key_cipher = AES.new(key=decoded_key, mode=AES.MODE_ECB) + enc_kek = key_cipher.encrypt(PKCS5_PAD(file_key, block_size)) + else: + cipher = Cipher(algorithms.AES(decoded_key), modes.ECB(), backend=backend) + encryptor = cipher.encryptor() + enc_kek = ( + encryptor.update(PKCS5_PAD(file_key, block_size)) + encryptor.finalize() + ) mat_desc = MaterialDescriptor( smk_id=encryption_material.smk_id, @@ -158,6 +178,7 @@ def decrypt_stream( ) -> None: """To read from `src` stream then decrypt to `out` stream.""" + use_openssl_only = os.getenv("SF_USE_OPENSSL_ONLY", "False") == "True" key_base64 = metadata.key iv_base64 = metadata.iv decoded_key = base64.standard_b64decode( @@ -166,26 +187,37 @@ def decrypt_stream( key_bytes = base64.standard_b64decode(key_base64) iv_bytes = base64.standard_b64decode(iv_base64) - backend = default_backend() - cipher = Cipher(algorithms.AES(decoded_key), modes.ECB(), backend=backend) - decryptor = cipher.decryptor() - file_key = PKCS5_UNPAD(decryptor.update(key_bytes) + decryptor.finalize()) - cipher = Cipher(algorithms.AES(file_key), modes.CBC(iv_bytes), backend=backend) - decryptor = cipher.decryptor() + if not use_openssl_only: + key_cipher = AES.new(key=decoded_key, mode=AES.MODE_ECB) + file_key = PKCS5_UNPAD(key_cipher.decrypt(key_bytes)) + data_cipher = AES.new(key=file_key, mode=AES.MODE_CBC, IV=iv_bytes) + else: + backend = default_backend() + cipher = Cipher(algorithms.AES(decoded_key), modes.ECB(), backend=backend) + decryptor = cipher.decryptor() + file_key = PKCS5_UNPAD(decryptor.update(key_bytes) + decryptor.finalize()) + cipher = Cipher( + algorithms.AES(file_key), modes.CBC(iv_bytes), backend=backend + ) + decryptor = cipher.decryptor() last_decrypted_chunk = None chunk = src.read(chunk_size) while len(chunk) != 0: if last_decrypted_chunk is not None: out.write(last_decrypted_chunk) - d = decryptor.update(chunk) + if not use_openssl_only: + d = data_cipher.decrypt(chunk) + else: + d = decryptor.update(chunk) last_decrypted_chunk = d chunk = src.read(chunk_size) if last_decrypted_chunk is not None: offset = PKCS5_OFFSET(last_decrypted_chunk) out.write(last_decrypted_chunk[:-offset]) - out.write(decryptor.finalize()) + if use_openssl_only: + out.write(decryptor.finalize()) @staticmethod def decrypt_file( diff --git a/src/snowflake/connector/file_util.py b/src/snowflake/connector/file_util.py index d89e72185..ac4bca4a8 100644 --- a/src/snowflake/connector/file_util.py +++ b/src/snowflake/connector/file_util.py @@ -13,6 +13,7 @@ from logging import getLogger from typing import IO +from Cryptodome.Hash import SHA256 from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes @@ -32,17 +33,27 @@ def get_digest_and_size(src: IO[bytes]) -> tuple[str, int]: Returns: Tuple of src's digest and src's size in bytes. """ + use_openssl_only = os.getenv("SF_USE_OPENSSL_ONLY", "False") == "True" CHUNK_SIZE = 64 * kilobyte - backend = default_backend() - chosen_hash = hashes.SHA256() - hasher = hashes.Hash(chosen_hash, backend) + if not use_openssl_only: + m = SHA256.new() + else: + backend = default_backend() + chosen_hash = hashes.SHA256() + hasher = hashes.Hash(chosen_hash, backend) while True: chunk = src.read(CHUNK_SIZE) if chunk == b"": break - hasher.update(chunk) - - digest = base64.standard_b64encode(hasher.finalize()).decode(UTF8) + if not use_openssl_only: + m.update(chunk) + else: + hasher.update(chunk) + + if not use_openssl_only: + digest = base64.standard_b64encode(m.digest()).decode(UTF8) + else: + digest = base64.standard_b64encode(hasher.finalize()).decode(UTF8) size = src.tell() src.seek(0) diff --git a/src/snowflake/connector/ocsp_asn1crypto.py b/src/snowflake/connector/ocsp_asn1crypto.py index 50efb3fc1..41fb73cb5 100644 --- a/src/snowflake/connector/ocsp_asn1crypto.py +++ b/src/snowflake/connector/ocsp_asn1crypto.py @@ -5,6 +5,10 @@ from __future__ import annotations +import os +import platform +import sys +import warnings from base64 import b64decode, b64encode from collections import OrderedDict from datetime import datetime, timezone @@ -24,6 +28,9 @@ Version, ) from asn1crypto.x509 import Certificate +from Cryptodome.Hash import SHA1, SHA256, SHA384, SHA512 +from Cryptodome.PublicKey import RSA +from Cryptodome.Signature import PKCS1_v1_5 from cryptography.exceptions import InvalidSignature from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes, serialization @@ -41,6 +48,20 @@ from snowflake.connector.errors import RevocationCheckError from snowflake.connector.ocsp_snowflake import SnowflakeOCSP, generate_cache_key +with warnings.catch_warnings(): + warnings.simplefilter("ignore") + # force versioned dylibs onto oscrypto ssl on catalina + if sys.platform == "darwin" and platform.mac_ver()[0].startswith("10.15"): + from oscrypto import _module_values, use_openssl + + if _module_values["backend"] is None: + use_openssl( + libcrypto_path="/usr/lib/libcrypto.35.dylib", + libssl_path="/usr/lib/libssl.35.dylib", + ) + from oscrypto import asymmetric + + logger = getLogger(__name__) @@ -49,6 +70,12 @@ class SnowflakeOCSPAsn1Crypto(SnowflakeOCSP): # map signature algorithm name to digest class SIGNATURE_ALGORITHM_TO_DIGEST_CLASS = { + "sha256": SHA256, + "sha384": SHA384, + "sha512": SHA512, + } + + SIGNATURE_ALGORITHM_TO_DIGEST_CLASS_OPENSSL = { "sha256": hashes.SHA256, "sha384": hashes.SHA3_384, "sha512": hashes.SHA3_512, @@ -351,29 +378,51 @@ def process_ocsp_response(self, issuer, cert_id, ocsp_response): raise RevocationCheckError(msg=debug_msg, errno=op_er.errno) def verify_signature(self, signature_algorithm, signature, cert, data): - backend = default_backend() - public_key = serialization.load_der_public_key( - cert.public_key.dump(), backend=default_backend() - ) - if ( - signature_algorithm - in SnowflakeOCSPAsn1Crypto.SIGNATURE_ALGORITHM_TO_DIGEST_CLASS - ): - chosen_hash = SnowflakeOCSPAsn1Crypto.SIGNATURE_ALGORITHM_TO_DIGEST_CLASS[ + use_openssl_only = os.getenv("SF_USE_OPENSSL_ONLY", "False") == "True" + if not use_openssl_only: + pubkey = asymmetric.load_public_key(cert.public_key).unwrap().dump() + rsakey = RSA.importKey(pubkey) + signer = PKCS1_v1_5.new(rsakey) + if ( signature_algorithm - ]() + in SnowflakeOCSPAsn1Crypto.SIGNATURE_ALGORITHM_TO_DIGEST_CLASS + ): + digest = SnowflakeOCSPAsn1Crypto.SIGNATURE_ALGORITHM_TO_DIGEST_CLASS[ + signature_algorithm + ].new() + else: + # the last resort. should not happen. + digest = SHA1.new() + digest.update(data.dump()) + if not signer.verify(digest, signature): + raise RevocationCheckError(msg="Failed to verify the signature") + else: - # the last resort. should not happen. - chosen_hash = hashes.SHA1() - hasher = hashes.Hash(chosen_hash, backend) - hasher.update(data.dump()) - digest = hasher.finalize() - try: - public_key.verify( - signature, digest, padding.PKCS1v15(), utils.Prehashed(chosen_hash) + backend = default_backend() + public_key = serialization.load_der_public_key( + cert.public_key.dump(), backend=default_backend() ) - except InvalidSignature: - raise RevocationCheckError(msg="Failed to verify the signature") + if ( + signature_algorithm + in SnowflakeOCSPAsn1Crypto.SIGNATURE_ALGORITHM_TO_DIGEST_CLASS + ): + chosen_hash = ( + SnowflakeOCSPAsn1Crypto.SIGNATURE_ALGORITHM_TO_DIGEST_CLASS_OPENSSL[ + signature_algorithm + ]() + ) + else: + # the last resort. should not happen. + chosen_hash = hashes.SHA1() + hasher = hashes.Hash(chosen_hash, backend) + hasher.update(data.dump()) + digest = hasher.finalize() + try: + public_key.verify( + signature, digest, padding.PKCS1v15(), utils.Prehashed(chosen_hash) + ) + except InvalidSignature: + raise RevocationCheckError(msg="Failed to verify the signature") def extract_certificate_chain( self, connection: Connection diff --git a/test/integ/test_connection.py b/test/integ/test_connection.py index 5dd20ece1..f5518813d 100644 --- a/test/integ/test_connection.py +++ b/test/integ/test_connection.py @@ -681,6 +681,35 @@ def mock_auth(self, auth_instance): assert cnx +@pytest.mark.skipolddriver +def test_use_openssl_only(db_parameters): + cnx = snowflake.connector.connect( + user=db_parameters["user"], + password=db_parameters["password"], + host=db_parameters["host"], + port=db_parameters["port"], + account=db_parameters["account"], + protocol=db_parameters["protocol"], + use_openssl_only=True, + ) + assert cnx + assert "SF_USE_OPENSSL_ONLY" in os.environ + # Note during testing conftest will default this value to False, so if testing this we need to manually clear it + # Let's test it again, after clearing it + del os.environ["SF_USE_OPENSSL_ONLY"] + cnx = snowflake.connector.connect( + user=db_parameters["user"], + password=db_parameters["password"], + host=db_parameters["host"], + port=db_parameters["port"], + account=db_parameters["account"], + protocol=db_parameters["protocol"], + use_openssl_only=True, + ) + assert cnx + assert os.environ["SF_USE_OPENSSL_ONLY"] == "True" + + def test_dashed_url(db_parameters): """Test whether dashed URLs get created correctly.""" with mock.patch(