Prep 2.2.3 (#294)

* updated Python Connector reqs * SNOW-102876 secure sso python copy * SNOW-141822 bumped pandas to newest versions * SNOW-141822 bumped pandas to newest versions * SNOW-141932 build manylinux1 wheels * SNOW-118103 fix unclosed file issue * SNOW-144663 added missing test directories to tox commans * SNOW-145906 update python docs * SNOW-143923 tox housekeeping * SNOW-146266 updated Python test * SNOW-146266 fix import ordering * SNOW-145814 wrongly default keyring package * SNOW-146213 Add google storage api url to whitelist for ocsp validation * SNOW-67159 update column size python connector * SNOW-145814 fix mac sso unit test with mock * SNOW-83085 use_openssl_only mode for Python connector * SNOW-147687 in band telemetry update python * SNOW-144043: Add new optional config client_store_temporary_credential into SnowSQL and made it the same in python connector * SNOW-144043 fix lint error * SNOW-148015 Added type checking workaround for Python 3.5.1 * SNOW-121925 Adding a test to verify that the Python connector supports dashed URLs * Revert SNOW-121925 Adding a test to verify that the Python connector supports dashed URLs * python connector version bump to 2.2.3 * skip new sso tests on Travis * reenabled pandas tests
snowflakedb · Mar 30, 2020 · a574cad · a574cad
1 parent fbd29c0
commit a574cad
Show file tree

Hide file tree

Showing 35 changed files with 550 additions and 879 deletions.
diff --git a/DESCRIPTION.rst b/DESCRIPTION.rst
@@ -9,6 +9,15 @@ Source code is also available at: https://github.com/snowflakedb/snowflake-conne
 Release Notes
 -------------------------------------------------------------------------------
 
+- v2.2.3(March 30,2020)
+
+    - Secure SSO ID Token
+    - Add use_openssl_only connection parameter, which disables the usage of pure Python cryptographic libraries for FIPS
+    - Add manylinux1 as well as manylinux2010
+    - Fix a bug where a certificate file was opened and never closed in snowflake-connector-python.
+    - Fix python connector skips validating GCP URLs
+    - Adds additional client driver config information to in band telemetry.
+
 - v2.2.2(March 9,2020)
 
     - Fix retry with chunck_downloader.py for stability.

diff --git a/auth.py b/auth.py
@@ -8,7 +8,6 @@
 import copy
 import json
 import logging
-import platform
 import tempfile
 import time
 import uuid
@@ -18,18 +17,24 @@
 from threading import Lock, Thread
 
 from .auth_keypair import AuthByKeyPair
-from .compat import IS_LINUX, TO_UNICODE, urlencode
+from .compat import IS_LINUX, IS_WINDOWS, IS_MACOS, TO_UNICODE, urlencode
 from .constants import (
     HTTP_HEADER_ACCEPT,
     HTTP_HEADER_CONTENT_TYPE,
     HTTP_HEADER_SERVICE_NAME,
     HTTP_HEADER_USER_AGENT,
     PARAMETER_CLIENT_STORE_TEMPORARY_CREDENTIAL,
-    PARAMETER_CLIENT_USE_SECURE_STORAGE_FOR_TEMPORARY_CREDENTIAL,
 )
 from .description import COMPILER, IMPLEMENTATION, OPERATING_SYSTEM, PLATFORM, PYTHON_VERSION
 from .errorcode import ER_FAILED_TO_CONNECT_TO_DB
-from .errors import BadGatewayError, DatabaseError, Error, ForbiddenError, ServiceUnavailableError
+from .errors import (
+    BadGatewayError,
+    DatabaseError,
+    Error,
+    ForbiddenError,
+    ServiceUnavailableError,
+    MissingDependencyError,
+)
 from .network import (
     ACCEPT_TYPE_APPLICATION_SNOWFLAKE,
     CONTENT_TYPE_APPLICATION_JSON,
@@ -41,13 +46,19 @@
 
 logger = logging.getLogger(__name__)
 
+try:
+    import keyring
+except ImportError as ie:
+    keyring = None
+    logger.debug('Failed to import keyring module. err=[%s]', ie)
+
 # Cache directory
 CACHE_ROOT_DIR = getenv('SF_TEMPORARY_CREDENTIAL_CACHE_DIR') or \
                  expanduser("~") or tempfile.gettempdir()
-if platform.system() == 'Windows':
+if IS_WINDOWS:
     CACHE_DIR = path.join(CACHE_ROOT_DIR, 'AppData', 'Local', 'Snowflake',
                           'Caches')
-elif platform.system() == 'Darwin':
+elif IS_MACOS:
     CACHE_DIR = path.join(CACHE_ROOT_DIR, 'Library', 'Caches', 'Snowflake')
 else:
     CACHE_DIR = path.join(CACHE_ROOT_DIR, '.cache', 'snowflake')
@@ -77,6 +88,7 @@
 # keyring
 KEYRING_SERVICE_NAME = "net.snowflake.temporary_token"
 KEYRING_USER = "temp_token"
+KEYRING_DRIVER_NAME = "SNOWFLAKE-PYTHON-DRIVER"
 
 
 class Auth(object):
@@ -91,22 +103,28 @@ def __init__(self, rest):
     def base_auth_data(user, account, application,
                        internal_application_name,
                        internal_application_version,
-                       ocsp_mode):
+                       ocsp_mode, login_timeout,
+                       network_timeout=None,
+                       store_temp_cred=None):
         return {
-            u'data': {
-                u"CLIENT_APP_ID": internal_application_name,
-                u"CLIENT_APP_VERSION": internal_application_version,
-                u"SVN_REVISION": VERSION[3],
-                u"ACCOUNT_NAME": account,
-                u"LOGIN_NAME": user,
-                u"CLIENT_ENVIRONMENT": {
-                    u"APPLICATION": application,
-                    u"OS": OPERATING_SYSTEM,
-                    u"OS_VERSION": PLATFORM,
-                    u"PYTHON_VERSION": PYTHON_VERSION,
-                    u"PYTHON_RUNTIME": IMPLEMENTATION,
-                    u"PYTHON_COMPILER": COMPILER,
-                    u"OCSP_MODE": ocsp_mode.name,
+            'data': {
+                "CLIENT_APP_ID": internal_application_name,
+                "CLIENT_APP_VERSION": internal_application_version,
+                "SVN_REVISION": VERSION[3],
+                "ACCOUNT_NAME": account,
+                "LOGIN_NAME": user,
+                "CLIENT_ENVIRONMENT": {
+                    "APPLICATION": application,
+                    "OS": OPERATING_SYSTEM,
+                    "OS_VERSION": PLATFORM,
+                    "PYTHON_VERSION": PYTHON_VERSION,
+                    "PYTHON_RUNTIME": IMPLEMENTATION,
+                    "PYTHON_COMPILER": COMPILER,
+                    "OCSP_MODE": ocsp_mode.name,
+                    "TRACING": logger.getEffectiveLevel(),
+                    "LOGIN_TIMEOUT": login_timeout,
+                    "NETWORK_TIMEOUT": network_timeout,
+                    "CLIENT_STORE_TEMPORARY_CREDENTIAL": store_temp_cred,
                 }
             },
         }
@@ -132,11 +150,22 @@ def authenticate(self, auth_instance, account, user,
             headers[HTTP_HEADER_SERVICE_NAME] = \
                 session_parameters[HTTP_HEADER_SERVICE_NAME]
         url = u"/session/v1/login-request"
+        if session_parameters is not None \
+                and PARAMETER_CLIENT_STORE_TEMPORARY_CREDENTIAL in session_parameters:
+            store_temp_cred = session_parameters[
+                PARAMETER_CLIENT_STORE_TEMPORARY_CREDENTIAL]
+        else:
+            store_temp_cred = None
+
         body_template = Auth.base_auth_data(
             user, account, self._rest._connection.application,
             self._rest._connection._internal_application_name,
             self._rest._connection._internal_application_version,
-            self._rest._connection._ocsp_mode())
+            self._rest._connection._ocsp_mode(),
+            self._rest._connection._login_timeout,
+            self._rest._connection._network_timeout,
+            store_temp_cred,
+        )
 
         body = copy.deepcopy(body_template)
         # updating request body
@@ -317,10 +346,10 @@ def post_request_wrapper(self, url, headers, body):
                 id_token=ret[u'data'].get(u'idToken')
             )
             if self._rest._connection.consent_cache_id_token:
-                write_temporary_credential_file(
-                    account, user, self._rest.id_token,
+                write_temporary_credential(
+                    self._rest._host, account, user, self._rest.id_token,
                     session_parameters.get(
-                        PARAMETER_CLIENT_USE_SECURE_STORAGE_FOR_TEMPORARY_CREDENTIAL))
+                        PARAMETER_CLIENT_STORE_TEMPORARY_CREDENTIAL))
             if u'sessionId' in ret[u'data']:
                 self._rest._connection._session_id = ret[u'data'][u'sessionId']
             if u'sessionInfo' in ret[u'data']:
@@ -333,17 +362,26 @@ def post_request_wrapper(self, url, headers, body):
 
         return session_parameters
 
-    def read_temporary_credential(self, account, user, session_parameters):
-        if session_parameters.get(PARAMETER_CLIENT_STORE_TEMPORARY_CREDENTIAL):
-            read_temporary_credential_file(
-                session_parameters.get(
-                    PARAMETER_CLIENT_USE_SECURE_STORAGE_FOR_TEMPORARY_CREDENTIAL)
-            )
-            id_token = TEMPORARY_CREDENTIAL.get(
-                account.upper(), {}).get(user.upper())
+    def read_temporary_credential(self, host, account, user, session_parameters):
+        if session_parameters.get(PARAMETER_CLIENT_STORE_TEMPORARY_CREDENTIAL, False):
+            id_token = None
+            if IS_MACOS or IS_WINDOWS:
+                if not keyring:
+                    # we will leave the exception for write_temporary_credential function to raise
+                    return False
+                new_target = convert_target(host, user)
+                try:
+                    id_token = keyring.get_password(new_target, user.upper())
+                except keyring.errors.KeyringError as ke:
+                    logger.debug("Could not retrieve id_token from secure storage : {}".format(str(ke)))
+            elif IS_LINUX:
+                read_temporary_credential_file()
+                id_token = TEMPORARY_CREDENTIAL.get(
+                    account.upper(), {}).get(user.upper())
+            else:
+                logger.debug("connection parameter enable_sso_temporary_credential not set or OS not support")
             if id_token:
                 self._rest.id_token = id_token
-            if self._rest.id_token:
                 try:
                     self._rest._id_token_session()
                     return True
@@ -354,11 +392,31 @@ def read_temporary_credential(self, account, user, session_parameters):
         return False
 
 
-def write_temporary_credential_file(
-        account, user, id_token,
-        use_secure_storage_for_temporary_credential=False):
-    if not CACHE_DIR or not id_token:
-        # no cache is enabled or no id_token is given
+def write_temporary_credential(host, account, user, id_token, store_temporary_credential=False):
+    if not id_token:
+        logger.debug("no ID token is given when try to store temporary credential")
+        return
+    if IS_MACOS or IS_WINDOWS:
+        if not keyring:
+            raise MissingDependencyError("Please install keyring module to enable SSO token cache feature.")
+
+        new_target = convert_target(host, user)
+        try:
+            keyring.set_password(new_target, user.upper(), id_token)
+        except keyring.errors.KeyringError as ke:
+            logger.debug("Could not store id_token to keyring, %s", str(ke))
+    elif IS_LINUX and store_temporary_credential:
+        write_temporary_credential_file(host, account, user, id_token)
+    else:
+        logger.debug("connection parameter client_store_temporary_credential not set or OS not support")
+
+
+def write_temporary_credential_file(host, account, user, id_token):
+    """
+    Write temporary credential file when OS is Linux
+    """
+    if not CACHE_DIR:
+        # no cache is enabled
         return
     global TEMPORARY_CREDENTIAL
     global TEMPORARY_CREDENTIAL_LOCK
@@ -377,27 +435,19 @@ def write_temporary_credential_file(
                            "write the temporary credential file: %s",
                            TEMPORARY_CREDENTIAL_FILE)
         try:
-            if IS_LINUX or not use_secure_storage_for_temporary_credential:
-                with codecs.open(TEMPORARY_CREDENTIAL_FILE, 'w',
-                                 encoding='utf-8', errors='ignore') as f:
-                    json.dump(TEMPORARY_CREDENTIAL, f)
-            else:
-                import keyring
-                keyring.set_password(
-                    KEYRING_SERVICE_NAME, KEYRING_USER,
-                    json.dumps(TEMPORARY_CREDENTIAL))
-
+            with codecs.open(TEMPORARY_CREDENTIAL_FILE, 'w',
+                             encoding='utf-8', errors='ignore') as f:
+                json.dump(TEMPORARY_CREDENTIAL, f)
         except Exception as ex:
             logger.debug("Failed to write a credential file: "
                          "file=[%s], err=[%s]", TEMPORARY_CREDENTIAL_FILE, ex)
         finally:
             unlock_temporary_credential_file()
 
 
-def read_temporary_credential_file(
-        use_secure_storage_for_temporary_credential=False):
+def read_temporary_credential_file():
     """
-    Read temporary credential file
+    Read temporary credential file when OS is Linux
     """
     if not CACHE_DIR:
         # no cache is enabled
@@ -416,15 +466,9 @@ def read_temporary_credential_file(
                            "write the temporary credential file: %s",
                            TEMPORARY_CREDENTIAL_FILE)
         try:
-            if IS_LINUX or not use_secure_storage_for_temporary_credential:
-                with codecs.open(TEMPORARY_CREDENTIAL_FILE, 'r',
-                                 encoding='utf-8', errors='ignore') as f:
-                    TEMPORARY_CREDENTIAL = json.load(f)
-            else:
-                import keyring
-                f = keyring.get_password(
-                    KEYRING_SERVICE_NAME, KEYRING_USER) or "{}"
-                TEMPORARY_CREDENTIAL = json.loads(f)
+            with codecs.open(TEMPORARY_CREDENTIAL_FILE, 'r',
+                             encoding='utf-8', errors='ignore') as f:
+                TEMPORARY_CREDENTIAL = json.load(f)
             return TEMPORARY_CREDENTIAL
         except Exception as ex:
             logger.debug("Failed to read a credential file. The file may not"
@@ -456,26 +500,34 @@ def unlock_temporary_credential_file():
         return False
 
 
-def delete_temporary_credential_file(
-        use_secure_storage_for_temporary_credential=False):
-    """
-    Delete temporary credential file and its lock file
-    """
-    global TEMPORARY_CREDENTIAL_FILE
-    if IS_LINUX or not use_secure_storage_for_temporary_credential:
+def delete_temporary_credential(host, user, store_temporary_credential=False):
+    if (IS_MACOS or IS_WINDOWS) and keyring:
+        new_target = convert_target(host, user)
         try:
-            remove(TEMPORARY_CREDENTIAL_FILE)
-        except Exception as ex:
-            logger.debug("Failed to delete a credential file: "
-                         "file=[%s], err=[%s]", TEMPORARY_CREDENTIAL_FILE, ex)
-    else:
-        try:
-            import keyring
-            keyring.delete_password(KEYRING_SERVICE_NAME, KEYRING_USER)
+            keyring.delete_password(new_target, user.upper())
         except Exception as ex:
             logger.debug("Failed to delete credential in the keyring: err=[%s]",
                          ex)
+    elif IS_LINUX and store_temporary_credential:
+        delete_temporary_credential_file()
+
+
+def delete_temporary_credential_file():
+    """
+    Delete temporary credential file and its lock file
+    """
+    global TEMPORARY_CREDENTIAL_FILE
+    try:
+        remove(TEMPORARY_CREDENTIAL_FILE)
+    except Exception as ex:
+        logger.debug("Failed to delete a credential file: "
+                     "file=[%s], err=[%s]", TEMPORARY_CREDENTIAL_FILE, ex)
     try:
         removedirs(TEMPORARY_CREDENTIAL_FILE_LOCK)
     except Exception as ex:
         logger.debug("Failed to delete credential lock file: err=[%s]", ex)
+
+
+def convert_target(host, user):
+    return "{host}:{user}:{driver}".format(
+            host=host.upper(), user=user.upper(), driver=KEYRING_DRIVER_NAME)
diff --git a/auth_okta.py b/auth_okta.py
@@ -9,7 +9,8 @@
 from .auth import Auth
 from .auth_by_plugin import AuthByPlugin
 from .compat import unescape, urlencode, urlsplit
-from .constants import HTTP_HEADER_ACCEPT, HTTP_HEADER_CONTENT_TYPE, HTTP_HEADER_SERVICE_NAME, HTTP_HEADER_USER_AGENT
+from .constants import HTTP_HEADER_ACCEPT, HTTP_HEADER_CONTENT_TYPE, \
+    HTTP_HEADER_SERVICE_NAME, HTTP_HEADER_USER_AGENT
 from .errorcode import ER_IDP_CONNECTION_ERROR, ER_INCORRECT_DESTINATION
 from .errors import DatabaseError, Error
 from .network import CONTENT_TYPE_APPLICATION_JSON, PYTHON_CONNECTOR_USER_AGENT
@@ -121,7 +122,10 @@ def _step1(self, authenticator, service_name, account, user):
             self._rest._connection.application,
             self._rest._connection._internal_application_name,
             self._rest._connection._internal_application_version,
-            self._rest._connection._ocsp_mode())
+            self._rest._connection._ocsp_mode(),
+            self._rest._connection._login_timeout,
+            self._rest._connection._network_timeout,
+        )
 
         body[u"data"][u"AUTHENTICATOR"] = authenticator
         logger.debug(

diff --git a/auth_webbrowser.py b/auth_webbrowser.py
@@ -12,7 +12,8 @@
 from .auth import Auth
 from .auth_by_plugin import AuthByPlugin
 from .compat import parse_qs, urlparse, urlsplit
-from .constants import HTTP_HEADER_ACCEPT, HTTP_HEADER_CONTENT_TYPE, HTTP_HEADER_SERVICE_NAME, HTTP_HEADER_USER_AGENT
+from .constants import HTTP_HEADER_ACCEPT, HTTP_HEADER_CONTENT_TYPE, HTTP_HEADER_SERVICE_NAME, \
+    HTTP_HEADER_USER_AGENT
 from .errorcode import ER_IDP_CONNECTION_ERROR, ER_NO_HOSTNAME_FOUND, ER_UNABLE_TO_OPEN_BROWSER
 from .errors import OperationalError
 from .network import CONTENT_TYPE_APPLICATION_JSON, EXTERNAL_BROWSER_AUTHENTICATOR, PYTHON_CONNECTOR_USER_AGENT
@@ -291,7 +292,10 @@ def _get_sso_url(
             self._rest._connection.application,
             self._rest._connection._internal_application_name,
             self._rest._connection._internal_application_version,
-            self._rest._connection._ocsp_mode())
+            self._rest._connection._ocsp_mode(),
+            self._rest._connection._login_timeout,
+            self._rest._connection._network_timeout,
+        )
 
         body[u'data'][u'AUTHENTICATOR'] = authenticator
         body[u'data'][u"BROWSER_MODE_REDIRECT_PORT"] = str(callback_port)

diff --git a/compat.py b/compat.py
@@ -17,6 +17,7 @@
 
 IS_LINUX = platform.system() == 'Linux'
 IS_WINDOWS = platform.system() == 'Windows'
+IS_MACOS = platform.system() == 'Darwin'
 
 NUM_DATA_TYPES = []
 try: