From a3d18eb26147236da7f8217569ad8684121326fb Mon Sep 17 00:00:00 2001 From: Zane Date: Fri, 18 Oct 2024 15:41:46 -0700 Subject: [PATCH 01/34] feat: add connection file path and connection name cli arguments --- schemachange/config/parse_cli_args.py | 12 ++++++++++++ tests/config/test_parse_cli_args.py | 6 ++++++ 2 files changed, 18 insertions(+) diff --git a/schemachange/config/parse_cli_args.py b/schemachange/config/parse_cli_args.py index f287cd5..e4db894 100644 --- a/schemachange/config/parse_cli_args.py +++ b/schemachange/config/parse_cli_args.py @@ -134,6 +134,18 @@ def parse_cli_args(args) -> dict: help="The name of the default schema to use. Can be overridden in the change scripts.", required=False, ) + parser_deploy.add_argument( + "--connections-file-path", + type=str, + help="Override the default connections file path at snowflake.connector.constants.CONNECTIONS_FILE (OS specific)", + required=False, + ) + parser_deploy.add_argument( + "--connection-name", + type=str, + help="Override the default connection name. Other connection-related values will override these connection values.", + required=False, + ) parser_deploy.add_argument( "-c", "--change-history-table", diff --git a/tests/config/test_parse_cli_args.py b/tests/config/test_parse_cli_args.py index 31e5fa7..71de14d 100644 --- a/tests/config/test_parse_cli_args.py +++ b/tests/config/test_parse_cli_args.py @@ -43,6 +43,12 @@ def test_parse_args_deploy_names(): ), ("--snowflake-database", "some_snowflake_database", "some_snowflake_database"), ("--snowflake-schema", "some_snowflake_schema", "some_snowflake_schema"), + ( + "--connections-file-path", + "some_connections_file_path", + "some_connections_file_path", + ), + ("--connection-name", "some_connection_name", "some_connection_name"), ("--change-history-table", "some_history_table", "some_history_table"), ("--query-tag", "some_query_tag", "some_query_tag"), ("--oauth-config", json.dumps({"some": "values"}), {"some": "values"}), From be0e37a2d6e0784453584e3caf078ecc01ba776f Mon Sep 17 00:00:00 2001 From: Zane Date: Sat, 19 Oct 2024 11:15:00 -0700 Subject: [PATCH 02/34] feat: add connections.toml options to DeployConfig and parse_cli_args.py --- schemachange/config/DeployConfig.py | 61 +++++--- schemachange/config/parse_cli_args.py | 21 +++ schemachange/config/utils.py | 33 ++++- tests/config/connections.toml | 14 ++ tests/config/test_Config.py | 206 +++++++++++++++++++++++++- tests/config/test_parse_cli_args.py | 33 +++++ 6 files changed, 349 insertions(+), 19 deletions(-) create mode 100644 tests/config/connections.toml diff --git a/schemachange/config/DeployConfig.py b/schemachange/config/DeployConfig.py index c0bcc2e..bb4aec6 100644 --- a/schemachange/config/DeployConfig.py +++ b/schemachange/config/DeployConfig.py @@ -6,7 +6,12 @@ from schemachange.config.BaseConfig import BaseConfig from schemachange.config.ChangeHistoryTable import ChangeHistoryTable -from schemachange.config.utils import get_snowflake_identifier_string +from schemachange.config.utils import ( + get_snowflake_identifier_string, + validate_file_path, + set_connections_toml_path, + get_connection_kwargs, +) @dataclasses.dataclass(frozen=True, kw_only=True) @@ -18,6 +23,12 @@ class DeployConfig(BaseConfig): snowflake_warehouse: str | None = None snowflake_database: str | None = None snowflake_schema: str | None = None + snowflake_authenticator: str | None = None + snowflake_password: str | None = None + snowflake_private_key_path: Path | None = None + snowflake_token_path: Path | None = None + connections_file_path: Path | None = None + connection_name: str | None = None # TODO: Turn change_history_table into three arguments. There's no need to parse it from a string change_history_table: ChangeHistoryTable | None = dataclasses.field( default_factory=ChangeHistoryTable @@ -32,16 +43,42 @@ class DeployConfig(BaseConfig): def factory( cls, config_file_path: Path, - snowflake_role: str | None = None, - snowflake_warehouse: str | None = None, - snowflake_database: str | None = None, - snowflake_schema: str | None = None, change_history_table: str | None = None, + connections_file_path: str | None = None, + connection_name: str | None = None, **kwargs, ): if "subcommand" in kwargs: kwargs.pop("subcommand") + if connections_file_path is not None: + connections_file_path = validate_file_path(file_path=connections_file_path) + set_connections_toml_path(connections_file_path=connections_file_path) + + if connection_name is not None: + connection_kwargs = get_connection_kwargs(connection_name=connection_name) + kwargs = {**connection_kwargs, **kwargs} + + for sf_input in [ + "snowflake_role", + "snowflake_warehouse", + "snowflake_database", + "snowflake_schema", + ]: + if sf_input in kwargs: + kwargs[sf_input] = get_snowflake_identifier_string( + kwargs[sf_input], sf_input + ) + + for sf_path_input in [ + "snowflake_private_key_path", + "snowflake_token_path", + ]: + if sf_path_input in kwargs: + kwargs[sf_path_input] = validate_file_path( + file_path=kwargs[sf_path_input] + ) + change_history_table = ChangeHistoryTable.from_str( table_str=change_history_table ) @@ -49,19 +86,9 @@ def factory( return super().factory( subcommand="deploy", config_file_path=config_file_path, - snowflake_role=get_snowflake_identifier_string( - snowflake_role, "snowflake_role" - ), - snowflake_warehouse=get_snowflake_identifier_string( - snowflake_warehouse, "snowflake_warehouse" - ), - snowflake_database=get_snowflake_identifier_string( - snowflake_database, "snowflake_database" - ), - snowflake_schema=get_snowflake_identifier_string( - snowflake_schema, "snowflake_schema" - ), change_history_table=change_history_table, + connections_file_path=connections_file_path, + connection_name=connection_name, **kwargs, ) diff --git a/schemachange/config/parse_cli_args.py b/schemachange/config/parse_cli_args.py index e4db894..9be9cd7 100644 --- a/schemachange/config/parse_cli_args.py +++ b/schemachange/config/parse_cli_args.py @@ -134,6 +134,27 @@ def parse_cli_args(args) -> dict: help="The name of the default schema to use. Can be overridden in the change scripts.", required=False, ) + parser_deploy.add_argument( + "-A", + "--snowflake-authenticator", + type=str, + help="The Snowflake Authenticator to use. One of snowflake, oauth, externalbrowser, or https://.okta.com", + required=False, + ) + parser_deploy.add_argument( + "-k", + "--snowflake-private-key-path", + type=str, + help="Path to file containing private key.", + required=False, + ) + parser_deploy.add_argument( + "-t", + "--snowflake-token-path", + type=str, + help="Path to the file containing the OAuth token to be used when authenticating with Snowflake.", + required=False, + ) parser_deploy.add_argument( "--connections-file-path", type=str, diff --git a/schemachange/config/utils.py b/schemachange/config/utils.py index cd98495..b2a8257 100644 --- a/schemachange/config/utils.py +++ b/schemachange/config/utils.py @@ -8,7 +8,7 @@ import jinja2.ext import structlog import yaml - +from snowflake.connector.config_manager import CONFIG_MANAGER from schemachange.JinjaEnvVar import JinjaEnvVar logger = structlog.getLogger(__name__) @@ -130,3 +130,34 @@ def load_yaml_config(config_file_path: Path | None) -> dict[str, Any]: config = yaml.load(config_template.render(), Loader=yaml.FullLoader) logger.info("Using config file", config_file_path=str(config_file_path)) return config + + +def set_connections_toml_path(connections_file_path: Path) -> None: + # Change config file path and force update cache + for i, s in enumerate(CONFIG_MANAGER._slices): + if s.section == "connections": + CONFIG_MANAGER._slices[i] = s._replace(path=connections_file_path) + CONFIG_MANAGER.read_config() + break + + +def get_connection_kwargs(connection_name: str) -> dict: + connections = CONFIG_MANAGER["connections"] + connection = connections.get(connection_name) + if connection is None: + raise Exception( + f"Invalid connection_name '{connection_name}'," + f" known ones are {list(connections.keys())}" + ) + return { + "snowflake_account": connection.get("account"), + "snowflake_user": connection.get("user"), + "snowflake_role": connection.get("role"), + "snowflake_warehouse": connection.get("warehouse"), + "snowflake_database": connection.get("database"), + "snowflake_schema": connection.get("schema"), + "snowflake_authenticator": connection.get("authenticator"), + "snowflake_password": connection.get("password"), + "snowflake_private_key_path": connection.get("private-key"), + "snowflake_token_path": connection.get("token-file-path"), + } diff --git a/tests/config/connections.toml b/tests/config/connections.toml new file mode 100644 index 0000000..ca5c602 --- /dev/null +++ b/tests/config/connections.toml @@ -0,0 +1,14 @@ +[myconnection] +account = "connections.toml-account" +user = "connections.toml-user" +role = "connections.toml-role" +warehouse = "connections.toml-warehouse" +database = "connections.toml-database" +schema = "connections.toml-schema" +authenticator = "connections.toml-authenticator" +password = "connections.toml-password" +host = "connections.toml-host" +port = "connections.toml-port" +region = "connections.toml-region" +private-key = "connections.toml-private-key" +token-file-path = "connections.toml-token-file-path" diff --git a/tests/config/test_Config.py b/tests/config/test_Config.py index eca2e26..6d0d44c 100644 --- a/tests/config/test_Config.py +++ b/tests/config/test_Config.py @@ -1,5 +1,6 @@ from __future__ import annotations +import tomllib from pathlib import Path from unittest import mock @@ -9,7 +10,10 @@ from schemachange.config.ChangeHistoryTable import ChangeHistoryTable from schemachange.config.DeployConfig import DeployConfig from schemachange.config.RenderConfig import RenderConfig -from schemachange.config.utils import get_config_secrets +from schemachange.config.utils import ( + get_config_secrets, + get_snowflake_identifier_string, +) @pytest.fixture @@ -229,6 +233,206 @@ def test_invalid_modules_folder(self, _): e_info_value = str(e_info.value) assert "Path is not valid directory: some_modules_folder_name" in e_info_value + @mock.patch("pathlib.Path.is_dir", side_effect=[True, True]) + @mock.patch("pathlib.Path.is_file", side_effect=[False]) + def test_invalid_snowflake_private_key_path(self, _, __): + with pytest.raises(Exception) as e_info: + DeployConfig.factory( + config_file_path=Path("some_config_file_name"), + root_folder="some_root_folder_name", + modules_folder="some_modules_folder_name", + config_vars={"some": "config_vars"}, + snowflake_account="some_snowflake_account", + snowflake_user="some_snowflake_user", + snowflake_role="some_snowflake_role", + snowflake_warehouse="some_snowflake_warehouse", + snowflake_database="some_snowflake_database", + snowflake_schema="some_snowflake_schema", + snowflake_private_key_path="invalid_snowflake_private_key_path", + snowflake_token_path="invalid_snowflake_token_path", + connections_file_path="invalid_connections_file_path", + connection_name="invalid_connection_name", + change_history_table="some_history_table", + query_tag="some_query_tag", + oauth_config={"some": "values"}, + ) + e_info_value = str(e_info.value) + assert "invalid file path: invalid_snowflake_private_key_path" in e_info_value + + @mock.patch("pathlib.Path.is_dir", side_effect=[True, True]) + @mock.patch("pathlib.Path.is_file", side_effect=[True, False]) + def test_invalid_snowflake_token_path(self, _, __): + with pytest.raises(Exception) as e_info: + DeployConfig.factory( + config_file_path=Path("some_config_file_name"), + root_folder="some_root_folder_name", + modules_folder="some_modules_folder_name", + config_vars={"some": "config_vars"}, + snowflake_account="some_snowflake_account", + snowflake_user="some_snowflake_user", + snowflake_role="some_snowflake_role", + snowflake_warehouse="some_snowflake_warehouse", + snowflake_database="some_snowflake_database", + snowflake_schema="some_snowflake_schema", + snowflake_private_key_path="valid_snowflake_private_key_path", + snowflake_token_path="invalid_snowflake_token_path", + connections_file_path="invalid_connections_file_path", + connection_name="invalid_connection_name", + change_history_table="some_history_table", + query_tag="some_query_tag", + oauth_config={"some": "values"}, + ) + e_info_value = str(e_info.value) + assert "invalid file path: invalid_snowflake_token_path" in e_info_value + + @mock.patch("pathlib.Path.is_dir", side_effect=[True, True]) + @mock.patch("pathlib.Path.is_file", side_effect=[True, True, False]) + def test_invalid_connections_file_path(self, _, __): + with pytest.raises(Exception) as e_info: + DeployConfig.factory( + config_file_path=Path("some_config_file_name"), + root_folder="some_root_folder_name", + modules_folder="some_modules_folder_name", + config_vars={"some": "config_vars"}, + snowflake_account="some_snowflake_account", + snowflake_user="some_snowflake_user", + snowflake_role="some_snowflake_role", + snowflake_warehouse="some_snowflake_warehouse", + snowflake_database="some_snowflake_database", + snowflake_schema="some_snowflake_schema", + snowflake_private_key_path="valid_snowflake_private_key_path", + snowflake_token_path="valid_snowflake_token_path", + connections_file_path="invalid_connections_file_path", + connection_name="invalid_connection_name", + change_history_table="some_history_table", + query_tag="some_query_tag", + oauth_config={"some": "values"}, + ) + e_info_value = str(e_info.value) + assert "invalid file path: invalid_connections_file_path" in e_info_value + + @mock.patch("pathlib.Path.is_dir", side_effect=[True, True]) + def test_invalid_connection_name(self, _): + with pytest.raises(Exception) as e_info: + DeployConfig.factory( + config_file_path=Path("some_config_file_name"), + root_folder="some_root_folder_name", + modules_folder="some_modules_folder_name", + config_vars={"some": "config_vars"}, + snowflake_account="some_snowflake_account", + snowflake_user="some_snowflake_user", + snowflake_role="some_snowflake_role", + snowflake_warehouse="some_snowflake_warehouse", + snowflake_database="some_snowflake_database", + snowflake_schema="some_snowflake_schema", + connections_file_path=str(Path(__file__).parent / "connections.toml"), + connection_name="invalid_connection_name", + change_history_table="some_history_table", + query_tag="some_query_tag", + oauth_config={"some": "values"}, + ) + e_info_value = str(e_info.value) + assert "Invalid connection_name 'invalid_connection_name'" in e_info_value + + @mock.patch("pathlib.Path.is_dir", side_effect=[True, True]) + @mock.patch("pathlib.Path.is_file", side_effect=[True, True, True]) + def test_connection_happy_path(self, _, __): + connections_file_path = Path(__file__).parent / "connections.toml" + connection_name = "myconnection" + with connections_file_path.open("rb") as f: + connection_data = tomllib.load(f) + + config = DeployConfig.factory( + config_file_path=Path("some_config_file_name"), + root_folder="some_root_folder_name", + modules_folder="some_modules_folder_name", + config_vars={"some": "config_vars"}, + connections_file_path=str(connections_file_path), + connection_name=connection_name, + change_history_table="some_history_table", + query_tag="some_query_tag", + oauth_config={"some": "values"}, + ) + assert connection_data is not None + assert config.connection_name == connection_name + assert config.connections_file_path == connections_file_path + assert config.snowflake_account == connection_data[connection_name]["account"] + assert config.snowflake_user == connection_data[connection_name]["user"] + assert config.snowflake_role == get_snowflake_identifier_string( + connection_data[connection_name]["role"], "placeholder" + ) + assert config.snowflake_warehouse == get_snowflake_identifier_string( + connection_data[connection_name]["warehouse"], "placeholder" + ) + assert config.snowflake_database == get_snowflake_identifier_string( + connection_data[connection_name]["database"], "placeholder" + ) + assert config.snowflake_schema == get_snowflake_identifier_string( + connection_data[connection_name]["schema"], "placeholder" + ) + assert ( + config.snowflake_authenticator + == connection_data[connection_name]["authenticator"] + ) + assert config.snowflake_password == connection_data[connection_name]["password"] + assert config.snowflake_private_key_path == Path( + connection_data[connection_name]["private-key"] + ) + assert config.snowflake_token_path == Path( + connection_data[connection_name]["token-file-path"] + ) + + @mock.patch("pathlib.Path.is_dir", side_effect=[True, True]) + @mock.patch("pathlib.Path.is_file", side_effect=[True, True, True]) + def test_connection_overrides(self, _, __): + connections_file_path = Path(__file__).parent / "connections.toml" + connection_name = "myconnection" + snowflake_account = "some_snowflake_account" + snowflake_user = "some_snowflake_user" + snowflake_role = "some_snowflake_role" + snowflake_warehouse = "some_snowflake_warehouse" + snowflake_database = "some_snowflake_database" + snowflake_schema = "some_snowflake_schema" + snowflake_authenticator = "some_snowflake_authenticator" + snowflake_password = "some_snowflake_password" + snowflake_private_key_path = "some_snowflake_private_key_path" + snowflake_token_path = "some_snowflake_token_path" + + config = DeployConfig.factory( + config_file_path=Path("some_config_file_name"), + root_folder="some_root_folder_name", + modules_folder="some_modules_folder_name", + config_vars={"some": "config_vars"}, + snowflake_account=snowflake_account, + snowflake_user=snowflake_user, + snowflake_role=snowflake_role, + snowflake_warehouse=snowflake_warehouse, + snowflake_database=snowflake_database, + snowflake_schema=snowflake_schema, + snowflake_authenticator=snowflake_authenticator, + snowflake_password=snowflake_password, + snowflake_private_key_path=snowflake_private_key_path, + snowflake_token_path=snowflake_token_path, + connections_file_path=str(connections_file_path), + connection_name=connection_name, + change_history_table="some_history_table", + query_tag="some_query_tag", + oauth_config={"some": "values"}, + ) + + assert config.connection_name == connection_name + assert config.connections_file_path == connections_file_path + assert config.snowflake_account == snowflake_account + assert config.snowflake_user == snowflake_user + assert config.snowflake_role == snowflake_role + assert config.snowflake_warehouse == snowflake_warehouse + assert config.snowflake_database == snowflake_database + assert config.snowflake_schema == snowflake_schema + assert config.snowflake_authenticator == snowflake_authenticator + assert config.snowflake_password == snowflake_password + assert config.snowflake_private_key_path == Path(snowflake_private_key_path) + assert config.snowflake_token_path == Path(snowflake_token_path) + def test_config_vars_not_a_dict(self): with pytest.raises(Exception) as e_info: BaseConfig.factory( diff --git a/tests/config/test_parse_cli_args.py b/tests/config/test_parse_cli_args.py index 71de14d..2536558 100644 --- a/tests/config/test_parse_cli_args.py +++ b/tests/config/test_parse_cli_args.py @@ -43,6 +43,21 @@ def test_parse_args_deploy_names(): ), ("--snowflake-database", "some_snowflake_database", "some_snowflake_database"), ("--snowflake-schema", "some_snowflake_schema", "some_snowflake_schema"), + ( + "--snowflake-authenticator", + "some_snowflake_authenticator", + "some_snowflake_authenticator", + ), + ( + "--snowflake-private-key-path", + "some_snowflake_private_key_path", + "some_snowflake_private_key_path", + ), + ( + "--snowflake-token-path", + "some_snowflake_token_path", + "some_snowflake_token_path", + ), ( "--connections-file-path", "some_connections_file_path", @@ -107,6 +122,24 @@ def test_parse_args_deploy_flags(): "some_snowflake_database", ), ("-s", "snowflake_schema", "some_snowflake_schema", "some_snowflake_schema"), + ( + "-A", + "snowflake_authenticator", + "some_snowflake_authenticator", + "some_snowflake_authenticator", + ), + ( + "-k", + "snowflake_private_key_path", + "some_snowflake_private_key_path", + "some_snowflake_private_key_path", + ), + ( + "-t", + "snowflake_token_path", + "some_snowflake_token_path", + "some_snowflake_token_path", + ), ("-c", "change_history_table", "some_history_table", "some_history_table"), ] From 41a4995c15ac98ecde25e49e82bd6ad1755de2a4 Mon Sep 17 00:00:00 2001 From: Zane Date: Sat, 19 Oct 2024 11:40:07 -0700 Subject: [PATCH 03/34] feat: fetch snowflake_password in config object --- schemachange/config/DeployConfig.py | 8 ++++++- schemachange/config/utils.py | 33 +++++++++++++++++++++++++++++ schemachange/session/utils.py | 25 ---------------------- tests/config/test_Config.py | 23 +++++++++++++------- 4 files changed, 55 insertions(+), 34 deletions(-) diff --git a/schemachange/config/DeployConfig.py b/schemachange/config/DeployConfig.py index bb4aec6..1d3a8b2 100644 --- a/schemachange/config/DeployConfig.py +++ b/schemachange/config/DeployConfig.py @@ -11,6 +11,7 @@ validate_file_path, set_connections_toml_path, get_connection_kwargs, + get_snowflake_password, ) @@ -51,13 +52,18 @@ def factory( if "subcommand" in kwargs: kwargs.pop("subcommand") + kwargs["snowflake_password"] = get_snowflake_password() + if connections_file_path is not None: connections_file_path = validate_file_path(file_path=connections_file_path) set_connections_toml_path(connections_file_path=connections_file_path) if connection_name is not None: connection_kwargs = get_connection_kwargs(connection_name=connection_name) - kwargs = {**connection_kwargs, **kwargs} + kwargs = { + **connection_kwargs, + **{k: v for k, v in kwargs.items() if v is not None}, + } for sf_input in [ "snowflake_role", diff --git a/schemachange/config/utils.py b/schemachange/config/utils.py index b2a8257..2cde45e 100644 --- a/schemachange/config/utils.py +++ b/schemachange/config/utils.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os import re from pathlib import Path from typing import Any @@ -10,6 +11,7 @@ import yaml from snowflake.connector.config_manager import CONFIG_MANAGER from schemachange.JinjaEnvVar import JinjaEnvVar +import warnings logger = structlog.getLogger(__name__) @@ -161,3 +163,34 @@ def get_connection_kwargs(connection_name: str) -> dict: "snowflake_private_key_path": connection.get("private-key"), "snowflake_token_path": connection.get("token-file-path"), } + + +def get_snowsql_pwd() -> str | None: + snowsql_pwd = os.getenv("SNOWSQL_PWD") + if snowsql_pwd is not None and snowsql_pwd: + warnings.warn( + "The SNOWSQL_PWD environment variable is deprecated and " + "will be removed in a later version of schemachange. " + "Please use SNOWFLAKE_PASSWORD instead.", + DeprecationWarning, + ) + return snowsql_pwd + + +def get_snowflake_password() -> str | None: + snowflake_password = os.getenv("SNOWFLAKE_PASSWORD") + snowsql_pwd = get_snowsql_pwd() + + if snowflake_password is not None and snowflake_password: + # Check legacy/deprecated env variable + if snowsql_pwd is not None and snowsql_pwd: + warnings.warn( + "Environment variables SNOWFLAKE_PASSWORD and SNOWSQL_PWD " + "are both present, using SNOWFLAKE_PASSWORD", + DeprecationWarning, + ) + return snowflake_password + elif snowsql_pwd is not None and snowsql_pwd: + return snowsql_pwd + else: + return None diff --git a/schemachange/session/utils.py b/schemachange/session/utils.py index 9cc58f3..d8d9df2 100644 --- a/schemachange/session/utils.py +++ b/schemachange/session/utils.py @@ -2,7 +2,6 @@ import json import os -import warnings import requests import structlog @@ -12,30 +11,6 @@ logger = structlog.getLogger(__name__) -def get_snowflake_password() -> str | None: - snowflake_password = None - if os.getenv("SNOWFLAKE_PASSWORD") is not None and os.getenv("SNOWFLAKE_PASSWORD"): - snowflake_password = os.getenv("SNOWFLAKE_PASSWORD") - - # Check legacy/deprecated env variable - if os.getenv("SNOWSQL_PWD") is not None and os.getenv("SNOWSQL_PWD"): - if snowflake_password: - warnings.warn( - "Environment variables SNOWFLAKE_PASSWORD and SNOWSQL_PWD " - "are both present, using SNOWFLAKE_PASSWORD", - DeprecationWarning, - ) - else: - warnings.warn( - "The SNOWSQL_PWD environment variable is deprecated and " - "will be removed in a later version of schemachange. " - "Please use SNOWFLAKE_PASSWORD instead.", - DeprecationWarning, - ) - snowflake_password = os.getenv("SNOWSQL_PWD") - return snowflake_password - - def get_private_key_password() -> bytes | None: private_key_password = os.getenv("SNOWFLAKE_PRIVATE_KEY_PASSPHRASE", "") diff --git a/tests/config/test_Config.py b/tests/config/test_Config.py index 6d0d44c..9ed5ea6 100644 --- a/tests/config/test_Config.py +++ b/tests/config/test_Config.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os import tomllib from pathlib import Path from unittest import mock @@ -234,8 +235,11 @@ def test_invalid_modules_folder(self, _): assert "Path is not valid directory: some_modules_folder_name" in e_info_value @mock.patch("pathlib.Path.is_dir", side_effect=[True, True]) - @mock.patch("pathlib.Path.is_file", side_effect=[False]) + @mock.patch("pathlib.Path.is_file", side_effect=[True, False]) def test_invalid_snowflake_private_key_path(self, _, __): + connections_file_path = Path(__file__).parent / "connections.toml" + connection_name = "myconnection" + with pytest.raises(Exception) as e_info: DeployConfig.factory( config_file_path=Path("some_config_file_name"), @@ -250,8 +254,8 @@ def test_invalid_snowflake_private_key_path(self, _, __): snowflake_schema="some_snowflake_schema", snowflake_private_key_path="invalid_snowflake_private_key_path", snowflake_token_path="invalid_snowflake_token_path", - connections_file_path="invalid_connections_file_path", - connection_name="invalid_connection_name", + connections_file_path=str(connections_file_path), + connection_name=connection_name, change_history_table="some_history_table", query_tag="some_query_tag", oauth_config={"some": "values"}, @@ -260,8 +264,11 @@ def test_invalid_snowflake_private_key_path(self, _, __): assert "invalid file path: invalid_snowflake_private_key_path" in e_info_value @mock.patch("pathlib.Path.is_dir", side_effect=[True, True]) - @mock.patch("pathlib.Path.is_file", side_effect=[True, False]) + @mock.patch("pathlib.Path.is_file", side_effect=[True, True, False]) def test_invalid_snowflake_token_path(self, _, __): + connections_file_path = Path(__file__).parent / "connections.toml" + connection_name = "myconnection" + with pytest.raises(Exception) as e_info: DeployConfig.factory( config_file_path=Path("some_config_file_name"), @@ -276,8 +283,8 @@ def test_invalid_snowflake_token_path(self, _, __): snowflake_schema="some_snowflake_schema", snowflake_private_key_path="valid_snowflake_private_key_path", snowflake_token_path="invalid_snowflake_token_path", - connections_file_path="invalid_connections_file_path", - connection_name="invalid_connection_name", + connections_file_path=str(connections_file_path), + connection_name=connection_name, change_history_table="some_history_table", query_tag="some_query_tag", oauth_config={"some": "values"}, @@ -286,7 +293,7 @@ def test_invalid_snowflake_token_path(self, _, __): assert "invalid file path: invalid_snowflake_token_path" in e_info_value @mock.patch("pathlib.Path.is_dir", side_effect=[True, True]) - @mock.patch("pathlib.Path.is_file", side_effect=[True, True, False]) + @mock.patch("pathlib.Path.is_file", side_effect=[False]) def test_invalid_connections_file_path(self, _, __): with pytest.raises(Exception) as e_info: DeployConfig.factory( @@ -397,6 +404,7 @@ def test_connection_overrides(self, _, __): snowflake_password = "some_snowflake_password" snowflake_private_key_path = "some_snowflake_private_key_path" snowflake_token_path = "some_snowflake_token_path" + os.environ["SNOWFLAKE_PASSWORD"] = snowflake_password config = DeployConfig.factory( config_file_path=Path("some_config_file_name"), @@ -410,7 +418,6 @@ def test_connection_overrides(self, _, __): snowflake_database=snowflake_database, snowflake_schema=snowflake_schema, snowflake_authenticator=snowflake_authenticator, - snowflake_password=snowflake_password, snowflake_private_key_path=snowflake_private_key_path, snowflake_token_path=snowflake_token_path, connections_file_path=str(connections_file_path), From 660bb0a717fae01e22b6a69f1f269c39a42e04ad Mon Sep 17 00:00:00 2001 From: Zane Date: Sat, 19 Oct 2024 11:53:49 -0700 Subject: [PATCH 04/34] feat: prioritize SNOWFLAKE_PRIVATE_KEY_PATH env var in DeployConfig --- schemachange/config/DeployConfig.py | 7 ++++++- schemachange/session/utils.py | 5 +++-- tests/config/test_Config.py | 17 +++++++++++------ 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/schemachange/config/DeployConfig.py b/schemachange/config/DeployConfig.py index 1d3a8b2..ed870b7 100644 --- a/schemachange/config/DeployConfig.py +++ b/schemachange/config/DeployConfig.py @@ -1,6 +1,7 @@ from __future__ import annotations import dataclasses +import os from pathlib import Path from typing import Literal @@ -53,6 +54,10 @@ def factory( kwargs.pop("subcommand") kwargs["snowflake_password"] = get_snowflake_password() + if os.getenv("SNOWFLAKE_PRIVATE_KEY_PATH") is not None: + kwargs["snowflake_private_key_path"] = os.getenv( + "SNOWFLAKE_PRIVATE_KEY_PATH" + ) if connections_file_path is not None: connections_file_path = validate_file_path(file_path=connections_file_path) @@ -80,7 +85,7 @@ def factory( "snowflake_private_key_path", "snowflake_token_path", ]: - if sf_path_input in kwargs: + if sf_path_input in kwargs and kwargs[sf_path_input] is not None: kwargs[sf_path_input] = validate_file_path( file_path=kwargs[sf_path_input] ) diff --git a/schemachange/session/utils.py b/schemachange/session/utils.py index d8d9df2..d4ed5f6 100644 --- a/schemachange/session/utils.py +++ b/schemachange/session/utils.py @@ -2,6 +2,7 @@ import json import os +from pathlib import Path import requests import structlog @@ -24,9 +25,9 @@ def get_private_key_password() -> bytes | None: return None -def get_private_key_bytes() -> bytes: +def get_private_key_bytes(snowflake_private_key_path: Path) -> bytes: private_key_password = get_private_key_password() - with open(os.environ["SNOWFLAKE_PRIVATE_KEY_PATH"], "rb") as key: + with snowflake_private_key_path.open("rb") as key: p_key = serialization.load_pem_private_key( key.read(), password=private_key_password, diff --git a/tests/config/test_Config.py b/tests/config/test_Config.py index 9ed5ea6..be61875 100644 --- a/tests/config/test_Config.py +++ b/tests/config/test_Config.py @@ -391,6 +391,13 @@ def test_connection_happy_path(self, _, __): @mock.patch("pathlib.Path.is_dir", side_effect=[True, True]) @mock.patch("pathlib.Path.is_file", side_effect=[True, True, True]) + @mock.patch.dict( + os.environ, + { + "SNOWFLAKE_PASSWORD": "some_snowflake_password", + "SNOWFLAKE_PRIVATE_KEY_PATH": "some_snowflake_private_key_path", + }, + ) def test_connection_overrides(self, _, __): connections_file_path = Path(__file__).parent / "connections.toml" connection_name = "myconnection" @@ -401,10 +408,7 @@ def test_connection_overrides(self, _, __): snowflake_database = "some_snowflake_database" snowflake_schema = "some_snowflake_schema" snowflake_authenticator = "some_snowflake_authenticator" - snowflake_password = "some_snowflake_password" - snowflake_private_key_path = "some_snowflake_private_key_path" snowflake_token_path = "some_snowflake_token_path" - os.environ["SNOWFLAKE_PASSWORD"] = snowflake_password config = DeployConfig.factory( config_file_path=Path("some_config_file_name"), @@ -418,7 +422,6 @@ def test_connection_overrides(self, _, __): snowflake_database=snowflake_database, snowflake_schema=snowflake_schema, snowflake_authenticator=snowflake_authenticator, - snowflake_private_key_path=snowflake_private_key_path, snowflake_token_path=snowflake_token_path, connections_file_path=str(connections_file_path), connection_name=connection_name, @@ -436,8 +439,10 @@ def test_connection_overrides(self, _, __): assert config.snowflake_database == snowflake_database assert config.snowflake_schema == snowflake_schema assert config.snowflake_authenticator == snowflake_authenticator - assert config.snowflake_password == snowflake_password - assert config.snowflake_private_key_path == Path(snowflake_private_key_path) + assert config.snowflake_password == "some_snowflake_password" + assert config.snowflake_private_key_path == Path( + "some_snowflake_private_key_path" + ) assert config.snowflake_token_path == Path(snowflake_token_path) def test_config_vars_not_a_dict(self): From aadf7a1d93d38c706575f855e2b28c621e17290f Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Wed, 23 Oct 2024 15:48:29 -0700 Subject: [PATCH 05/34] feat: introduce env var and connections.toml layering into get_merged_config.py --- schemachange/config/DeployConfig.py | 77 +- schemachange/config/get_merged_config.py | 39 +- schemachange/config/parse_cli_args.py | 22 +- schemachange/config/utils.py | 65 +- schemachange/session/Credential.py | 112 -- schemachange/session/utils.py | 61 - tests/config/alt-connections.toml | 14 + ...schemachange-config-full-no-connection.yml | 33 + tests/config/schemachange-config-full.yml | 35 + ...achange-config-partial-with-connection.yml | 26 + tests/config/schemachange-config.yml | 29 - tests/config/test_ChangeHistoryTable.py | 81 ++ tests/config/test_Config.py | 496 -------- tests/config/test_DeployConfig.py | 261 +++++ tests/config/test_RenderConfig.py | 14 + tests/config/test_get_config_secrets.py | 79 ++ tests/config/test_get_merged_config.py | 1041 ++++++++++++++--- tests/config/test_get_yaml_config.py | 8 +- tests/config/test_parse_cli_args.py | 10 +- tests/config/test_utils.py | 116 ++ tests/session/test_Credential.py | 109 -- tests/session/test_utils.py | 43 - 22 files changed, 1708 insertions(+), 1063 deletions(-) delete mode 100644 schemachange/session/Credential.py delete mode 100644 schemachange/session/utils.py create mode 100644 tests/config/alt-connections.toml create mode 100644 tests/config/schemachange-config-full-no-connection.yml create mode 100644 tests/config/schemachange-config-full.yml create mode 100644 tests/config/schemachange-config-partial-with-connection.yml create mode 100644 tests/config/test_ChangeHistoryTable.py delete mode 100644 tests/config/test_Config.py create mode 100644 tests/config/test_DeployConfig.py create mode 100644 tests/config/test_RenderConfig.py create mode 100644 tests/config/test_get_config_secrets.py create mode 100644 tests/config/test_utils.py delete mode 100644 tests/session/test_Credential.py delete mode 100644 tests/session/test_utils.py diff --git a/schemachange/config/DeployConfig.py b/schemachange/config/DeployConfig.py index ed870b7..59e44d8 100644 --- a/schemachange/config/DeployConfig.py +++ b/schemachange/config/DeployConfig.py @@ -1,7 +1,6 @@ from __future__ import annotations import dataclasses -import os from pathlib import Path from typing import Literal @@ -10,9 +9,7 @@ from schemachange.config.utils import ( get_snowflake_identifier_string, validate_file_path, - set_connections_toml_path, - get_connection_kwargs, - get_snowflake_password, + get_oauth_token, ) @@ -25,10 +22,10 @@ class DeployConfig(BaseConfig): snowflake_warehouse: str | None = None snowflake_database: str | None = None snowflake_schema: str | None = None - snowflake_authenticator: str | None = None + snowflake_authenticator: str | None = "snowflake" snowflake_password: str | None = None + snowflake_oauth_token: str | None = None snowflake_private_key_path: Path | None = None - snowflake_token_path: Path | None = None connections_file_path: Path | None = None connection_name: str | None = None # TODO: Turn change_history_table into three arguments. There's no need to parse it from a string @@ -39,44 +36,24 @@ class DeployConfig(BaseConfig): autocommit: bool = False dry_run: bool = False query_tag: str | None = None - oauth_config: dict | None = None @classmethod def factory( cls, config_file_path: Path, change_history_table: str | None = None, - connections_file_path: str | None = None, - connection_name: str | None = None, **kwargs, ): if "subcommand" in kwargs: kwargs.pop("subcommand") - kwargs["snowflake_password"] = get_snowflake_password() - if os.getenv("SNOWFLAKE_PRIVATE_KEY_PATH") is not None: - kwargs["snowflake_private_key_path"] = os.getenv( - "SNOWFLAKE_PRIVATE_KEY_PATH" - ) - - if connections_file_path is not None: - connections_file_path = validate_file_path(file_path=connections_file_path) - set_connections_toml_path(connections_file_path=connections_file_path) - - if connection_name is not None: - connection_kwargs = get_connection_kwargs(connection_name=connection_name) - kwargs = { - **connection_kwargs, - **{k: v for k, v in kwargs.items() if v is not None}, - } - for sf_input in [ "snowflake_role", "snowflake_warehouse", "snowflake_database", "snowflake_schema", ]: - if sf_input in kwargs: + if sf_input in kwargs and kwargs[sf_input] is not None: kwargs[sf_input] = get_snowflake_identifier_string( kwargs[sf_input], sf_input ) @@ -90,6 +67,24 @@ def factory( file_path=kwargs[sf_path_input] ) + # If set by an environment variable, pop snowflake_token_path from kwargs + if "snowflake_oauth_token" in kwargs: + kwargs.pop("snowflake_token_path", None) + kwargs.pop("oauthconfig", None) + # Load it from a file, if provided + elif "snowflake_token_path" in kwargs: + kwargs.pop("oauthconfig", None) + oauth_token_path = kwargs.pop("snowflake_token_path") + with open(oauth_token_path) as f: + kwargs["snowflake_oauth_token"] = f.read() + # Make the oauth call if authenticator == "oauth" + + elif "oauth_config" in kwargs: + oauth_config = kwargs.pop("oauth_config") + authenticator = kwargs.get("snowflake_authenticator") + if authenticator is not None and authenticator.lower() == "oauth": + kwargs["snowflake_oauth_token"] = get_oauth_token(oauth_config) + change_history_table = ChangeHistoryTable.from_str( table_str=change_history_table ) @@ -98,8 +93,6 @@ def factory( subcommand="deploy", config_file_path=config_file_path, change_history_table=change_history_table, - connections_file_path=connections_file_path, - connection_name=connection_name, **kwargs, ) @@ -112,6 +105,32 @@ def check_for_deploy_args(self) -> None: "snowflake_role": self.snowflake_role, "snowflake_warehouse": self.snowflake_warehouse, } + + # OAuth based authentication + if self.snowflake_authenticator.lower() == "oauth": + # TODO: defer to an existing token or fetch one here? + req_args["snowflake_oauth_token"] = self.snowflake_oauth_token + + # External Browser based SSO + elif self.snowflake_authenticator.lower() == "externalbrowser": + pass + + # IDP based Authentication, limited to Okta + elif self.snowflake_authenticator.lower()[:8] == "https://": + req_args["snowflake_password"] = self.snowflake_password + + elif self.snowflake_authenticator.lower() == "snowflake_jwt": + req_args["snowflake_private_key_path"] = self.snowflake_private_key_path + + elif self.snowflake_authenticator.lower() == "snowflake": + req_args["snowflake_password"] = self.snowflake_password + + else: + raise ValueError( + f"{self.snowflake_authenticator} is not supported authenticator option. " + "Choose from snowflake, snowflake_jwt, externalbrowser, oauth, https://.okta.com." + ) + missing_args = [key for key, value in req_args.items() if value is None] if len(missing_args) == 0: diff --git a/schemachange/config/get_merged_config.py b/schemachange/config/get_merged_config.py index 607c09e..8f1c6c7 100644 --- a/schemachange/config/get_merged_config.py +++ b/schemachange/config/get_merged_config.py @@ -6,7 +6,12 @@ from schemachange.config.DeployConfig import DeployConfig from schemachange.config.RenderConfig import RenderConfig from schemachange.config.parse_cli_args import parse_cli_args -from schemachange.config.utils import load_yaml_config, validate_directory +from schemachange.config.utils import ( + load_yaml_config, + validate_directory, + get_env_kwargs, + get_connection_kwargs, +) def get_yaml_config_kwargs(config_file_path: Optional[Path]) -> dict: @@ -26,22 +31,21 @@ def get_yaml_config_kwargs(config_file_path: Optional[Path]) -> dict: if "vars" in kwargs: kwargs["config_vars"] = kwargs.pop("vars") - return kwargs + return {k: v for k, v in kwargs.items() if v is not None} def get_merged_config() -> Union[DeployConfig, RenderConfig]: - cli_kwargs = parse_cli_args(sys.argv[1:]) + env_kwargs: dict[str, str] = get_env_kwargs() - if "verbose" in cli_kwargs and cli_kwargs["verbose"]: - cli_kwargs["log_level"] = logging.DEBUG - cli_kwargs.pop("verbose") + cli_kwargs = parse_cli_args(sys.argv[1:]) - cli_config_vars = cli_kwargs.pop("config_vars", None) - if cli_config_vars is None: - cli_config_vars = {} + cli_config_vars = cli_kwargs.pop("config_vars") + connections_file_path = cli_kwargs.get("connections_file_path") + connection_name = cli_kwargs.get("connection_name") config_folder = validate_directory(path=cli_kwargs.pop("config_folder", ".")) - config_file_path = Path(config_folder) / "schemachange-config.yml" + config_file_name = cli_kwargs.pop("config_file_name") + config_file_path = Path(config_folder) / config_file_name yaml_kwargs = get_yaml_config_kwargs( config_file_path=config_file_path, @@ -50,6 +54,19 @@ def get_merged_config() -> Union[DeployConfig, RenderConfig]: if yaml_config_vars is None: yaml_config_vars = {} + if connections_file_path is None: + connections_file_path = yaml_kwargs.get("connections_file_path") + if config_folder is not None and connections_file_path is not None: + connections_file_path = config_folder / connections_file_path + + if connection_name is None: + connection_name = yaml_kwargs.get("connection_name") + + connection_kwargs: dict[str, str] = get_connection_kwargs( + connections_file_path=connections_file_path, + connection_name=connection_name, + ) + config_vars = { **yaml_config_vars, **cli_config_vars, @@ -59,8 +76,10 @@ def get_merged_config() -> Union[DeployConfig, RenderConfig]: kwargs = { "config_file_path": config_file_path, "config_vars": config_vars, + **{k: v for k, v in connection_kwargs.items() if v is not None}, **{k: v for k, v in yaml_kwargs.items() if v is not None}, **{k: v for k, v in cli_kwargs.items() if v is not None}, + **{k: v for k, v in env_kwargs.items() if v is not None}, } if cli_kwargs["subcommand"] == "deploy": diff --git a/schemachange/config/parse_cli_args.py b/schemachange/config/parse_cli_args.py index 9be9cd7..3e1ea4e 100644 --- a/schemachange/config/parse_cli_args.py +++ b/schemachange/config/parse_cli_args.py @@ -2,6 +2,7 @@ import argparse import json +import logging from enum import Enum import structlog @@ -58,6 +59,14 @@ def parse_cli_args(args) -> dict: "(the default is the current working directory)", required=False, ) + parent_parser.add_argument( + "--config-file-name", + type=str, + default="schemachange-config.yml", + help="The schemachange config YAML file name. Must be in the directory supplied as the config-folder " + "(Default: schemachange-config.yml)", + required=False, + ) parent_parser.add_argument( "-f", "--root-folder", @@ -237,7 +246,16 @@ def parse_cli_args(args) -> dict: if "log_level" in parsed_kwargs and isinstance(parsed_kwargs["log_level"], Enum): parsed_kwargs["log_level"] = parsed_kwargs["log_level"].value + parsed_kwargs["config_vars"] = {} if "vars" in parsed_kwargs: - parsed_kwargs["config_vars"] = parsed_kwargs.pop("vars") + config_vars = parsed_kwargs.pop("vars") + if config_vars is not None: + parsed_kwargs["config_vars"] = config_vars + + if "verbose" in parsed_kwargs: + parsed_kwargs["log_level"] = ( + logging.DEBUG if parsed_kwargs["verbose"] else logging.INFO + ) + parsed_kwargs.pop("verbose") - return parsed_kwargs + return {k: v for k, v in parsed_kwargs.items() if v is not None} diff --git a/schemachange/config/utils.py b/schemachange/config/utils.py index 2cde45e..23e280c 100644 --- a/schemachange/config/utils.py +++ b/schemachange/config/utils.py @@ -5,6 +5,9 @@ from pathlib import Path from typing import Any +import json + +import requests import jinja2 import jinja2.ext import structlog @@ -18,16 +21,14 @@ snowflake_identifier_pattern = re.compile(r"^[\w]+$") -def get_snowflake_identifier_string(input_value: str, input_type: str) -> str: +def get_snowflake_identifier_string(input_value: str, input_type: str) -> str | None: # Words with alphanumeric characters and underscores only. - result = "" - if input_value is None: - result = None + return None elif snowflake_identifier_pattern.match(input_value): - result = input_value + return input_value elif input_value.startswith('"') and input_value.endswith('"'): - result = input_value + return input_value elif input_value.startswith('"') and not input_value.endswith('"'): raise ValueError( f"Invalid {input_type}: {input_value}. Missing ending double quote" @@ -37,9 +38,7 @@ def get_snowflake_identifier_string(input_value: str, input_type: str) -> str: f"Invalid {input_type}: {input_value}. Missing beginning double quote" ) else: - result = f'"{input_value}"' - - return result + return f'"{input_value}"' def get_config_secrets(config_vars: dict[str, dict | str] | None) -> set[str]: @@ -136,14 +135,25 @@ def load_yaml_config(config_file_path: Path | None) -> dict[str, Any]: def set_connections_toml_path(connections_file_path: Path) -> None: # Change config file path and force update cache + # noinspection PyProtectedMember for i, s in enumerate(CONFIG_MANAGER._slices): if s.section == "connections": + # noinspection PyProtectedMember CONFIG_MANAGER._slices[i] = s._replace(path=connections_file_path) CONFIG_MANAGER.read_config() break -def get_connection_kwargs(connection_name: str) -> dict: +def get_connection_kwargs( + connections_file_path: Path | None = None, connection_name: str | None = None +) -> dict: + if connections_file_path is not None: + connections_file_path = validate_file_path(file_path=connections_file_path) + set_connections_toml_path(connections_file_path=connections_file_path) + + if connection_name is None: + return {} + connections = CONFIG_MANAGER["connections"] connection = connections.get(connection_name) if connection is None: @@ -151,7 +161,8 @@ def get_connection_kwargs(connection_name: str) -> dict: f"Invalid connection_name '{connection_name}'," f" known ones are {list(connections.keys())}" ) - return { + + connection_kwargs = { "snowflake_account": connection.get("account"), "snowflake_user": connection.get("user"), "snowflake_role": connection.get("role"), @@ -164,6 +175,8 @@ def get_connection_kwargs(connection_name: str) -> dict: "snowflake_token_path": connection.get("token-file-path"), } + return {k: v for k, v in connection_kwargs.items() if v is not None} + def get_snowsql_pwd() -> str | None: snowsql_pwd = os.getenv("SNOWSQL_PWD") @@ -194,3 +207,33 @@ def get_snowflake_password() -> str | None: return snowsql_pwd else: return None + + +def get_env_kwargs() -> dict[str, str]: + env_kwargs = { + "snowflake_password": get_snowflake_password(), + "snowflake_private_key_path": os.getenv("SNOWFLAKE_PRIVATE_KEY_PATH"), + "snowflake_authenticator": os.getenv("SNOWFLAKE_AUTHENTICATOR"), + "snowflake_oauth_token": os.getenv("SNOWFLAKE_TOKEN"), + } + return {k: v for k, v in env_kwargs.items() if v is not None} + + +def get_oauth_token(oauth_config: dict): + req_info = { + "url": oauth_config["token-provider-url"], + "headers": oauth_config["token-request-headers"], + "data": oauth_config["token-request-payload"], + } + token_name = oauth_config["token-response-name"] + response = requests.post(**req_info) + response_dict = json.loads(response.text) + try: + return response_dict[token_name] + except KeyError: + keys = ", ".join(response_dict.keys()) + errormessage = f"Response Json contains keys: {keys} \n but not {token_name}" + # if there is an error passed with the response include that + if "error_description" in response_dict.keys(): + errormessage = f"{errormessage}\n error description: {response_dict['error_description']}" + raise KeyError(errormessage) diff --git a/schemachange/session/Credential.py b/schemachange/session/Credential.py deleted file mode 100644 index b39e180..0000000 --- a/schemachange/session/Credential.py +++ /dev/null @@ -1,112 +0,0 @@ -from __future__ import annotations - -import dataclasses -import os -from abc import ABC -from typing import Literal, Union - -import structlog - -from schemachange.session.utils import ( - get_snowflake_password, - get_private_key_bytes, - get_oauth_token, -) - - -@dataclasses.dataclass(kw_only=True, frozen=True) -class Credential(ABC): - authenticator: str - - -@dataclasses.dataclass(kw_only=True, frozen=True) -class OauthCredential(Credential): - authenticator: Literal["oauth"] = "oauth" - token: str - - -@dataclasses.dataclass(kw_only=True, frozen=True) -class PasswordCredential(Credential): - authenticator: Literal["snowflake"] = "snowflake" - password: str - - -@dataclasses.dataclass(kw_only=True, frozen=True) -class PrivateKeyCredential(Credential): - authenticator: Literal["snowflake"] = "snowflake" - private_key: bytes - - -@dataclasses.dataclass(kw_only=True, frozen=True) -class ExternalBrowserCredential(Credential): - authenticator: Literal["externalbrowser"] = "externalbrowser" - password: str | None = None - - -@dataclasses.dataclass(kw_only=True, frozen=True) -class OktaCredential(Credential): - authenticator: str - password: str - - -SomeCredential = Union[ - OauthCredential, - PasswordCredential, - ExternalBrowserCredential, - OktaCredential, - PrivateKeyCredential, -] - - -def credential_factory( - logger: structlog.BoundLogger, - oauth_config: dict | None = None, -) -> SomeCredential: - snowflake_authenticator = os.getenv("SNOWFLAKE_AUTHENTICATOR", default="snowflake") - - # OAuth based authentication - if snowflake_authenticator.lower() == "oauth": - logger.debug("Proceeding with Oauth Access Token authentication") - return OauthCredential(token=get_oauth_token(oauth_config)) - - # External Browser based SSO - if snowflake_authenticator.lower() == "externalbrowser": - logger.debug("Proceeding with External Browser authentication") - return ExternalBrowserCredential() - - snowflake_password = get_snowflake_password() - - # IDP based Authentication, limited to Okta - if snowflake_authenticator.lower()[:8] == "https://": - logger.debug( - "Proceeding with Okta authentication", okta_endpoint=snowflake_authenticator - ) - return OktaCredential( - authenticator=snowflake_authenticator, password=snowflake_password - ) - - if snowflake_authenticator.lower() != "snowflake": - logger.debug( - "Supplied authenticator is not supported authenticator option. Choose from snowflake, " - "externalbrowser, oauth, https://.okta.com. " - "Using default value = 'snowflake'", - snowflake_authenticator=snowflake_authenticator, - ) - - if snowflake_password: - logger.debug("Proceeding with password authentication") - - return PasswordCredential(password=snowflake_password) - - if os.getenv("SNOWFLAKE_PRIVATE_KEY_PATH", ""): - logger.debug("Proceeding with private key authentication") - - return PrivateKeyCredential(private_key=get_private_key_bytes()) - - raise NameError( - "Missing environment variable(s). \n" - "SNOWFLAKE_PASSWORD must be defined for password authentication. \n" - "SNOWFLAKE_PRIVATE_KEY_PATH and (optional) SNOWFLAKE_PRIVATE_KEY_PASSPHRASE " - "must be defined for private key authentication. \n" - "SNOWFLAKE_AUTHENTICATOR must be defined is using Oauth, OKTA or external Browser Authentication." - ) diff --git a/schemachange/session/utils.py b/schemachange/session/utils.py deleted file mode 100644 index d4ed5f6..0000000 --- a/schemachange/session/utils.py +++ /dev/null @@ -1,61 +0,0 @@ -from __future__ import annotations - -import json -import os -from pathlib import Path - -import requests -import structlog -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import serialization - -logger = structlog.getLogger(__name__) - - -def get_private_key_password() -> bytes | None: - private_key_password = os.getenv("SNOWFLAKE_PRIVATE_KEY_PASSPHRASE", "") - - if private_key_password: - return private_key_password.encode() - - logger.debug( - "No private key passphrase provided. Assuming the key is not encrypted." - ) - - return None - - -def get_private_key_bytes(snowflake_private_key_path: Path) -> bytes: - private_key_password = get_private_key_password() - with snowflake_private_key_path.open("rb") as key: - p_key = serialization.load_pem_private_key( - key.read(), - password=private_key_password, - backend=default_backend(), - ) - - return p_key.private_bytes( - encoding=serialization.Encoding.DER, - format=serialization.PrivateFormat.PKCS8, - encryption_algorithm=serialization.NoEncryption(), - ) - - -def get_oauth_token(oauth_config: dict): - req_info = { - "url": oauth_config["token-provider-url"], - "headers": oauth_config["token-request-headers"], - "data": oauth_config["token-request-payload"], - } - token_name = oauth_config["token-response-name"] - response = requests.post(**req_info) - response_dict = json.loads(response.text) - try: - return response_dict[token_name] - except KeyError: - keys = ", ".join(response_dict.keys()) - errormessage = f"Response Json contains keys: {keys} \n but not {token_name}" - # if there is an error passed with the response include that - if "error_description" in response_dict.keys(): - errormessage = f"{errormessage}\n error description: {response_dict['error_description']}" - raise KeyError(errormessage) diff --git a/tests/config/alt-connections.toml b/tests/config/alt-connections.toml new file mode 100644 index 0000000..7231d94 --- /dev/null +++ b/tests/config/alt-connections.toml @@ -0,0 +1,14 @@ +[myaltconnection] +account = "alt-connections.toml-account" +user = "alt-connections.toml-user" +role = "alt-connections.toml-role" +warehouse = "alt-connections.toml-warehouse" +database = "alt-connections.toml-database" +schema = "alt-connections.toml-schema" +authenticator = "alt-connections.toml-authenticator" +password = "alt-connections.toml-password" +host = "alt-connections.toml-host" +port = "alt-connections.toml-port" +region = "alt-connections.toml-region" +private-key = "alt-connections.toml-private-key" +token-file-path = "alt-connections.toml-token-file-path" diff --git a/tests/config/schemachange-config-full-no-connection.yml b/tests/config/schemachange-config-full-no-connection.yml new file mode 100644 index 0000000..7997fef --- /dev/null +++ b/tests/config/schemachange-config-full-no-connection.yml @@ -0,0 +1,33 @@ +config-version: 1 +root-folder: 'root-folder-from-yaml' +modules-folder: 'modules-folder-from-yaml' +snowflake-account: 'snowflake-account-from-yaml' +snowflake-user: 'snowflake-user-from-yaml' +snowflake-role: 'snowflake-role-from-yaml' +snowflake-warehouse: 'snowflake-warehouse-from-yaml' +snowflake-database: 'snowflake-database-from-yaml' +snowflake-schema: 'snowflake-schema-from-yaml' +snowflake-authenticator: 'snowflake-authenticator-from-yaml' +snowflake-private-key-path: 'snowflake-private-key-path-from-yaml' +snowflake-token-path: 'snowflake-token-path-from-yaml' +change-history-table: 'change-history-table-from-yaml' +vars: + var1: 'from_yaml' + var2: 'also_from_yaml' +create-change-history-table: false +autocommit: false +verbose: false +dry-run: false +query-tag: 'query-tag-from-yaml' +oauthconfig: + token-provider-url: 'token-provider-url-from-yaml' + token-response-name: 'token-response-name-from-yaml' + token-request-headers: + Content-Type: 'Content-Type-from-yaml' + User-Agent: 'User-Agent-from-yaml' + token-request-payload: + client_id: 'id-from-yaml' + username: 'username-from-yaml' + password: 'password-from-yaml' + grant_type: 'type-from-yaml' + scope: 'scope-from-yaml' diff --git a/tests/config/schemachange-config-full.yml b/tests/config/schemachange-config-full.yml new file mode 100644 index 0000000..a3a2c15 --- /dev/null +++ b/tests/config/schemachange-config-full.yml @@ -0,0 +1,35 @@ +config-version: 1 +root-folder: 'root-folder-from-yaml' +modules-folder: 'modules-folder-from-yaml' +snowflake-account: 'snowflake-account-from-yaml' +snowflake-user: 'snowflake-user-from-yaml' +snowflake-role: 'snowflake-role-from-yaml' +snowflake-warehouse: 'snowflake-warehouse-from-yaml' +snowflake-database: 'snowflake-database-from-yaml' +snowflake-schema: 'snowflake-schema-from-yaml' +snowflake-authenticator: 'snowflake-authenticator-from-yaml' +snowflake-private-key-path: 'snowflake-private-key-path-from-yaml' +snowflake-token-path: 'snowflake-token-path-from-yaml' +connections-file-path: 'connections.toml' +connection-name: 'myconnection' +change-history-table: 'change-history-table-from-yaml' +vars: + var1: 'from_yaml' + var2: 'also_from_yaml' +create-change-history-table: false +autocommit: false +verbose: false +dry-run: false +query-tag: 'query-tag-from-yaml' +oauthconfig: + token-provider-url: 'token-provider-url-from-yaml' + token-response-name: 'token-response-name-from-yaml' + token-request-headers: + Content-Type: 'Content-Type-from-yaml' + User-Agent: 'User-Agent-from-yaml' + token-request-payload: + client_id: 'id-from-yaml' + username: 'username-from-yaml' + password: 'password-from-yaml' + grant_type: 'type-from-yaml' + scope: 'scope-from-yaml' diff --git a/tests/config/schemachange-config-partial-with-connection.yml b/tests/config/schemachange-config-partial-with-connection.yml new file mode 100644 index 0000000..bc74981 --- /dev/null +++ b/tests/config/schemachange-config-partial-with-connection.yml @@ -0,0 +1,26 @@ +config-version: 1 +root-folder: 'root-folder-from-yaml' +modules-folder: 'modules-folder-from-yaml' +connections-file-path: 'connections.toml' +connection-name: 'myconnection' +change-history-table: 'change-history-table-from-yaml' +vars: + var1: 'from_yaml' + var2: 'also_from_yaml' +create-change-history-table: false +autocommit: false +verbose: false +dry-run: false +query-tag: 'query-tag-from-yaml' +oauthconfig: + token-provider-url: 'token-provider-url-from-yaml' + token-response-name: 'token-response-name-from-yaml' + token-request-headers: + Content-Type: 'Content-Type-from-yaml' + User-Agent: 'User-Agent-from-yaml' + token-request-payload: + client_id: 'id-from-yaml' + username: 'username-from-yaml' + password: 'password-from-yaml' + grant_type: 'type-from-yaml' + scope: 'scope-from-yaml' diff --git a/tests/config/schemachange-config.yml b/tests/config/schemachange-config.yml index 73a5e56..3dae2c5 100644 --- a/tests/config/schemachange-config.yml +++ b/tests/config/schemachange-config.yml @@ -1,30 +1 @@ config-version: 1 -root-folder: 'root-folder-from-yaml' -modules-folder: 'modules-folder-from-yaml' -snowflake-account: 'snowflake-account-from-yaml' -snowflake-user: 'snowflake-user-from-yaml' -snowflake-role: 'snowflake-role-from-yaml' -snowflake-warehouse: 'snowflake-warehouse-from-yaml' -snowflake-database: 'snowflake-database-from-yaml' -snowflake-schema: 'snowflake-schema-from-yaml' -change-history-table: 'change-history-table-from-yaml' -vars: - var1: 'from_yaml' - var2: 'also_from_yaml' -create-change-history-table: false -autocommit: false -verbose: false -dry-run: false -query-tag: 'query-tag-from-yaml' -oauthconfig: - token-provider-url: 'token-provider-url-from-yaml' - token-response-name: 'token-response-name-from-yaml' - token-request-headers: - Content-Type: 'Content-Type-from-yaml' - User-Agent: 'User-Agent-from-yaml' - token-request-payload: - client_id: 'id-from-yaml' - username: 'username-from-yaml' - password: 'password-from-yaml' - grant_type: 'type-from-yaml' - scope: 'scope-from-yaml' diff --git a/tests/config/test_ChangeHistoryTable.py b/tests/config/test_ChangeHistoryTable.py new file mode 100644 index 0000000..fa88b45 --- /dev/null +++ b/tests/config/test_ChangeHistoryTable.py @@ -0,0 +1,81 @@ +from __future__ import annotations + + +import pytest + +from schemachange.config.ChangeHistoryTable import ChangeHistoryTable + + +@pytest.mark.parametrize( + "table_str, expected", + [ + ( + "DATABASE_NAME.SCHEMA_NAME.TABLE_NAME", + ChangeHistoryTable( + table_name="TABLE_NAME", + schema_name="SCHEMA_NAME", + database_name="DATABASE_NAME", + ), + ), + ( + "SCHEMA_NAME.TABLE_NAME", + ChangeHistoryTable( + table_name="TABLE_NAME", + schema_name="SCHEMA_NAME", + database_name="METADATA", + ), + ), + ( + "TABLE_NAME", + ChangeHistoryTable( + table_name="TABLE_NAME", + schema_name="SCHEMACHANGE", + database_name="METADATA", + ), + ), + ], +) +def test_from_str_happy_path(table_str: str, expected: ChangeHistoryTable): + result = ChangeHistoryTable.from_str(table_str) + assert result == expected + + +def test_from_str_exception(): + with pytest.raises(ValueError) as e: + ChangeHistoryTable.from_str("FOUR.THREE.TWO.ONE") + + assert "Invalid change history table name:" in str(e.value) + + +@pytest.mark.parametrize( + "table, expected", + [ + ( + ChangeHistoryTable( + table_name="TABLE_NAME", + schema_name="SCHEMA_NAME", + database_name="DATABASE_NAME", + ), + "DATABASE_NAME.SCHEMA_NAME.TABLE_NAME", + ), + ( + ChangeHistoryTable( + table_name="TABLE_NAME", + schema_name="SCHEMA_NAME", + database_name="METADATA", + ), + "METADATA.SCHEMA_NAME.TABLE_NAME", + ), + ( + ChangeHistoryTable( + table_name="TABLE_NAME", + schema_name="SCHEMACHANGE", + database_name="METADATA", + ), + "METADATA.SCHEMACHANGE.TABLE_NAME", + ), + ], +) +def test_fully_qualified(table: ChangeHistoryTable, expected: str): + result = table.fully_qualified + assert result == expected diff --git a/tests/config/test_Config.py b/tests/config/test_Config.py deleted file mode 100644 index be61875..0000000 --- a/tests/config/test_Config.py +++ /dev/null @@ -1,496 +0,0 @@ -from __future__ import annotations - -import os -import tomllib -from pathlib import Path -from unittest import mock - -import pytest - -from schemachange.config.BaseConfig import BaseConfig -from schemachange.config.ChangeHistoryTable import ChangeHistoryTable -from schemachange.config.DeployConfig import DeployConfig -from schemachange.config.RenderConfig import RenderConfig -from schemachange.config.utils import ( - get_config_secrets, - get_snowflake_identifier_string, -) - - -@pytest.fixture -@mock.patch("pathlib.Path.is_dir", return_value=True) -def yaml_config(_) -> DeployConfig: - return DeployConfig.factory( - config_file_path=Path(__file__).parent.parent.parent - / "demo" - / "basics_demo" - / "schemachange-config.yml", - root_folder=Path(__file__).parent.parent.parent / "demo" / "basics_demo", - modules_folder=Path(__file__).parent.parent.parent / "demo" / "basics_demo", - config_vars={"var1": "yaml_vars"}, - snowflake_account="yaml_snowflake_account", - snowflake_user="yaml_snowflake_user", - snowflake_role="yaml_snowflake_role", - snowflake_warehouse="yaml_snowflake_warehouse", - snowflake_database="yaml_snowflake_database", - snowflake_schema="yaml_snowflake_schema", - change_history_table="yaml_change_history_table", - create_change_history_table=True, - autocommit=True, - dry_run=True, - query_tag="yaml_query_tag", - oauth_config={"oauth": "yaml_oauth"}, - ) - - -class TestGetConfigSecrets: - def test_given_empty_config_should_not_error(self): - get_config_secrets(config_vars={}) - - def test_given_none_should_not_error(self): - get_config_secrets(None) - - @pytest.mark.parametrize( - "config_vars, secret", - [ - ({"secret": "secret_val1"}, "secret_val1"), - ({"SECret": "secret_val2"}, "secret_val2"), - ({"secret_key": "secret_val3"}, "secret_val3"), - ({"s3_bucket_secret": "secret_val4"}, "secret_val4"), - ({"s3SecretKey": "secret_val5"}, "secret_val5"), - ({"nested": {"s3_bucket_secret": "secret_val6"}}, "secret_val6"), - ], - ) - def test_given__vars_with_keys_should_extract_secret(self, config_vars, secret): - results = get_config_secrets(config_vars) - assert secret in results - - def test_given_vars_with_secrets_key_then_all_children_should_be_treated_as_secrets( - self, - ): - config_vars = { - "secrets": { - "database_name": "database_name_val", - "schema_name": "schema_name_val", - "nested_secrets": {"SEC_ONE": "SEC_ONE_VAL"}, - } - } - results = get_config_secrets(config_vars=config_vars) - - assert len(results) == 3 - assert "database_name_val" in results - assert "schema_name_val" in results - assert "SEC_ONE_VAL" in results - - def test_given_vars_with_nested_secrets_key_then_all_children_should_be_treated_as_secrets( - self, - ): - config_vars = { - "nested": { - "secrets": { - "database_name": "database_name_val", - "schema_name": "schema_name_val", - "nested": {"SEC_ONE": "SEC_ONE_VAL"}, - } - } - } - - results = get_config_secrets(config_vars) - - assert len(results) == 3 - assert "database_name_val" in results - assert "schema_name_val" in results - assert "SEC_ONE_VAL" in results - - def test_given_vars_with_same_secret_twice_then_only_extracted_once(self): - config_vars = { - "secrets": { - "database_name": "SECRET_VALUE", - "schema_name": "SECRET_VALUE", - "nested_secrets": {"SEC_ONE": "SECRET_VALUE"}, - } - } - - results = get_config_secrets(config_vars) - - assert len(results) == 1 - assert "SECRET_VALUE" in results - - -class TestTable: - @pytest.mark.parametrize( - "table_str, expected", - [ - ( - "DATABASE_NAME.SCHEMA_NAME.TABLE_NAME", - ChangeHistoryTable( - table_name="TABLE_NAME", - schema_name="SCHEMA_NAME", - database_name="DATABASE_NAME", - ), - ), - ( - "SCHEMA_NAME.TABLE_NAME", - ChangeHistoryTable( - table_name="TABLE_NAME", - schema_name="SCHEMA_NAME", - database_name="METADATA", - ), - ), - ( - "TABLE_NAME", - ChangeHistoryTable( - table_name="TABLE_NAME", - schema_name="SCHEMACHANGE", - database_name="METADATA", - ), - ), - ], - ) - def test_from_str_happy_path(self, table_str: str, expected: ChangeHistoryTable): - result = ChangeHistoryTable.from_str(table_str) - assert result == expected - - def test_from_str_exception(self): - with pytest.raises(ValueError) as e: - ChangeHistoryTable.from_str("FOUR.THREE.TWO.ONE") - - assert "Invalid change history table name:" in str(e.value) - - @pytest.mark.parametrize( - "table, expected", - [ - ( - ChangeHistoryTable( - table_name="TABLE_NAME", - schema_name="SCHEMA_NAME", - database_name="DATABASE_NAME", - ), - "DATABASE_NAME.SCHEMA_NAME.TABLE_NAME", - ), - ( - ChangeHistoryTable( - table_name="TABLE_NAME", - schema_name="SCHEMA_NAME", - database_name="METADATA", - ), - "METADATA.SCHEMA_NAME.TABLE_NAME", - ), - ( - ChangeHistoryTable( - table_name="TABLE_NAME", - schema_name="SCHEMACHANGE", - database_name="METADATA", - ), - "METADATA.SCHEMACHANGE.TABLE_NAME", - ), - ], - ) - def test_fully_qualified(self, table: ChangeHistoryTable, expected: str): - result = table.fully_qualified - assert result == expected - - -class TestConfig: - @mock.patch("pathlib.Path.is_dir", side_effect=[False]) - def test_invalid_root_folder(self, _): - with pytest.raises(Exception) as e_info: - DeployConfig.factory( - config_file_path=Path("some_config_file_name"), - root_folder="some_root_folder_name", - modules_folder="some_modules_folder_name", - config_vars={"some": "config_vars"}, - snowflake_account="some_snowflake_account", - snowflake_user="some_snowflake_user", - snowflake_role="some_snowflake_role", - snowflake_warehouse="some_snowflake_warehouse", - snowflake_database="some_snowflake_database", - snowflake_schema="some_snowflake_schema", - change_history_table="some_history_table", - query_tag="some_query_tag", - oauth_config={"some": "values"}, - ) - e_info_value = str(e_info.value) - assert "Path is not valid directory: some_root_folder_name" in e_info_value - - @mock.patch("pathlib.Path.is_dir", side_effect=[True, False]) - def test_invalid_modules_folder(self, _): - with pytest.raises(Exception) as e_info: - DeployConfig.factory( - config_file_path=Path("some_config_file_name"), - root_folder="some_root_folder_name", - modules_folder="some_modules_folder_name", - config_vars={"some": "config_vars"}, - snowflake_account="some_snowflake_account", - snowflake_user="some_snowflake_user", - snowflake_role="some_snowflake_role", - snowflake_warehouse="some_snowflake_warehouse", - snowflake_database="some_snowflake_database", - snowflake_schema="some_snowflake_schema", - change_history_table="some_history_table", - query_tag="some_query_tag", - oauth_config={"some": "values"}, - ) - e_info_value = str(e_info.value) - assert "Path is not valid directory: some_modules_folder_name" in e_info_value - - @mock.patch("pathlib.Path.is_dir", side_effect=[True, True]) - @mock.patch("pathlib.Path.is_file", side_effect=[True, False]) - def test_invalid_snowflake_private_key_path(self, _, __): - connections_file_path = Path(__file__).parent / "connections.toml" - connection_name = "myconnection" - - with pytest.raises(Exception) as e_info: - DeployConfig.factory( - config_file_path=Path("some_config_file_name"), - root_folder="some_root_folder_name", - modules_folder="some_modules_folder_name", - config_vars={"some": "config_vars"}, - snowflake_account="some_snowflake_account", - snowflake_user="some_snowflake_user", - snowflake_role="some_snowflake_role", - snowflake_warehouse="some_snowflake_warehouse", - snowflake_database="some_snowflake_database", - snowflake_schema="some_snowflake_schema", - snowflake_private_key_path="invalid_snowflake_private_key_path", - snowflake_token_path="invalid_snowflake_token_path", - connections_file_path=str(connections_file_path), - connection_name=connection_name, - change_history_table="some_history_table", - query_tag="some_query_tag", - oauth_config={"some": "values"}, - ) - e_info_value = str(e_info.value) - assert "invalid file path: invalid_snowflake_private_key_path" in e_info_value - - @mock.patch("pathlib.Path.is_dir", side_effect=[True, True]) - @mock.patch("pathlib.Path.is_file", side_effect=[True, True, False]) - def test_invalid_snowflake_token_path(self, _, __): - connections_file_path = Path(__file__).parent / "connections.toml" - connection_name = "myconnection" - - with pytest.raises(Exception) as e_info: - DeployConfig.factory( - config_file_path=Path("some_config_file_name"), - root_folder="some_root_folder_name", - modules_folder="some_modules_folder_name", - config_vars={"some": "config_vars"}, - snowflake_account="some_snowflake_account", - snowflake_user="some_snowflake_user", - snowflake_role="some_snowflake_role", - snowflake_warehouse="some_snowflake_warehouse", - snowflake_database="some_snowflake_database", - snowflake_schema="some_snowflake_schema", - snowflake_private_key_path="valid_snowflake_private_key_path", - snowflake_token_path="invalid_snowflake_token_path", - connections_file_path=str(connections_file_path), - connection_name=connection_name, - change_history_table="some_history_table", - query_tag="some_query_tag", - oauth_config={"some": "values"}, - ) - e_info_value = str(e_info.value) - assert "invalid file path: invalid_snowflake_token_path" in e_info_value - - @mock.patch("pathlib.Path.is_dir", side_effect=[True, True]) - @mock.patch("pathlib.Path.is_file", side_effect=[False]) - def test_invalid_connections_file_path(self, _, __): - with pytest.raises(Exception) as e_info: - DeployConfig.factory( - config_file_path=Path("some_config_file_name"), - root_folder="some_root_folder_name", - modules_folder="some_modules_folder_name", - config_vars={"some": "config_vars"}, - snowflake_account="some_snowflake_account", - snowflake_user="some_snowflake_user", - snowflake_role="some_snowflake_role", - snowflake_warehouse="some_snowflake_warehouse", - snowflake_database="some_snowflake_database", - snowflake_schema="some_snowflake_schema", - snowflake_private_key_path="valid_snowflake_private_key_path", - snowflake_token_path="valid_snowflake_token_path", - connections_file_path="invalid_connections_file_path", - connection_name="invalid_connection_name", - change_history_table="some_history_table", - query_tag="some_query_tag", - oauth_config={"some": "values"}, - ) - e_info_value = str(e_info.value) - assert "invalid file path: invalid_connections_file_path" in e_info_value - - @mock.patch("pathlib.Path.is_dir", side_effect=[True, True]) - def test_invalid_connection_name(self, _): - with pytest.raises(Exception) as e_info: - DeployConfig.factory( - config_file_path=Path("some_config_file_name"), - root_folder="some_root_folder_name", - modules_folder="some_modules_folder_name", - config_vars={"some": "config_vars"}, - snowflake_account="some_snowflake_account", - snowflake_user="some_snowflake_user", - snowflake_role="some_snowflake_role", - snowflake_warehouse="some_snowflake_warehouse", - snowflake_database="some_snowflake_database", - snowflake_schema="some_snowflake_schema", - connections_file_path=str(Path(__file__).parent / "connections.toml"), - connection_name="invalid_connection_name", - change_history_table="some_history_table", - query_tag="some_query_tag", - oauth_config={"some": "values"}, - ) - e_info_value = str(e_info.value) - assert "Invalid connection_name 'invalid_connection_name'" in e_info_value - - @mock.patch("pathlib.Path.is_dir", side_effect=[True, True]) - @mock.patch("pathlib.Path.is_file", side_effect=[True, True, True]) - def test_connection_happy_path(self, _, __): - connections_file_path = Path(__file__).parent / "connections.toml" - connection_name = "myconnection" - with connections_file_path.open("rb") as f: - connection_data = tomllib.load(f) - - config = DeployConfig.factory( - config_file_path=Path("some_config_file_name"), - root_folder="some_root_folder_name", - modules_folder="some_modules_folder_name", - config_vars={"some": "config_vars"}, - connections_file_path=str(connections_file_path), - connection_name=connection_name, - change_history_table="some_history_table", - query_tag="some_query_tag", - oauth_config={"some": "values"}, - ) - assert connection_data is not None - assert config.connection_name == connection_name - assert config.connections_file_path == connections_file_path - assert config.snowflake_account == connection_data[connection_name]["account"] - assert config.snowflake_user == connection_data[connection_name]["user"] - assert config.snowflake_role == get_snowflake_identifier_string( - connection_data[connection_name]["role"], "placeholder" - ) - assert config.snowflake_warehouse == get_snowflake_identifier_string( - connection_data[connection_name]["warehouse"], "placeholder" - ) - assert config.snowflake_database == get_snowflake_identifier_string( - connection_data[connection_name]["database"], "placeholder" - ) - assert config.snowflake_schema == get_snowflake_identifier_string( - connection_data[connection_name]["schema"], "placeholder" - ) - assert ( - config.snowflake_authenticator - == connection_data[connection_name]["authenticator"] - ) - assert config.snowflake_password == connection_data[connection_name]["password"] - assert config.snowflake_private_key_path == Path( - connection_data[connection_name]["private-key"] - ) - assert config.snowflake_token_path == Path( - connection_data[connection_name]["token-file-path"] - ) - - @mock.patch("pathlib.Path.is_dir", side_effect=[True, True]) - @mock.patch("pathlib.Path.is_file", side_effect=[True, True, True]) - @mock.patch.dict( - os.environ, - { - "SNOWFLAKE_PASSWORD": "some_snowflake_password", - "SNOWFLAKE_PRIVATE_KEY_PATH": "some_snowflake_private_key_path", - }, - ) - def test_connection_overrides(self, _, __): - connections_file_path = Path(__file__).parent / "connections.toml" - connection_name = "myconnection" - snowflake_account = "some_snowflake_account" - snowflake_user = "some_snowflake_user" - snowflake_role = "some_snowflake_role" - snowflake_warehouse = "some_snowflake_warehouse" - snowflake_database = "some_snowflake_database" - snowflake_schema = "some_snowflake_schema" - snowflake_authenticator = "some_snowflake_authenticator" - snowflake_token_path = "some_snowflake_token_path" - - config = DeployConfig.factory( - config_file_path=Path("some_config_file_name"), - root_folder="some_root_folder_name", - modules_folder="some_modules_folder_name", - config_vars={"some": "config_vars"}, - snowflake_account=snowflake_account, - snowflake_user=snowflake_user, - snowflake_role=snowflake_role, - snowflake_warehouse=snowflake_warehouse, - snowflake_database=snowflake_database, - snowflake_schema=snowflake_schema, - snowflake_authenticator=snowflake_authenticator, - snowflake_token_path=snowflake_token_path, - connections_file_path=str(connections_file_path), - connection_name=connection_name, - change_history_table="some_history_table", - query_tag="some_query_tag", - oauth_config={"some": "values"}, - ) - - assert config.connection_name == connection_name - assert config.connections_file_path == connections_file_path - assert config.snowflake_account == snowflake_account - assert config.snowflake_user == snowflake_user - assert config.snowflake_role == snowflake_role - assert config.snowflake_warehouse == snowflake_warehouse - assert config.snowflake_database == snowflake_database - assert config.snowflake_schema == snowflake_schema - assert config.snowflake_authenticator == snowflake_authenticator - assert config.snowflake_password == "some_snowflake_password" - assert config.snowflake_private_key_path == Path( - "some_snowflake_private_key_path" - ) - assert config.snowflake_token_path == Path(snowflake_token_path) - - def test_config_vars_not_a_dict(self): - with pytest.raises(Exception) as e_info: - BaseConfig.factory( - subcommand="deploy", - config_vars="a string", - config_file_path=Path("."), - ) - assert ( - "config_vars did not parse correctly, please check its configuration" - in str(e_info.value) - ) - - def test_config_vars_reserved_word(self): - with pytest.raises(Exception) as e_info: - BaseConfig.factory( - subcommand="deploy", - config_vars={"schemachange": "not allowed"}, - config_file_path=Path("."), - ) - assert ( - "The variable 'schemachange' has been reserved for use by schemachange, please use a different name" - in str(e_info.value) - ) - - def test_check_for_deploy_args_happy_path(self): - config = DeployConfig.factory( - snowflake_account="account", - snowflake_user="user", - snowflake_role="role", - snowflake_warehouse="warehouse", - config_file_path=Path("."), - ) - config.check_for_deploy_args() - - def test_check_for_deploy_args_exception(self): - config = DeployConfig.factory(config_file_path=Path(".")) - with pytest.raises(ValueError) as e: - config.check_for_deploy_args() - - assert "Missing config values. The following config values are required" in str( - e.value - ) - - -@mock.patch("pathlib.Path.is_file", return_value=False) -def test_render_config_invalid_path(_): - with pytest.raises(Exception) as e_info: - RenderConfig.factory(script_path="invalid path") - assert "invalid file path" in str(e_info) diff --git a/tests/config/test_DeployConfig.py b/tests/config/test_DeployConfig.py new file mode 100644 index 0000000..41bd785 --- /dev/null +++ b/tests/config/test_DeployConfig.py @@ -0,0 +1,261 @@ +from __future__ import annotations + +from pathlib import Path +from unittest import mock +from unittest.mock import mock_open + +import pytest + +from schemachange.config.BaseConfig import BaseConfig +from schemachange.config.DeployConfig import DeployConfig + + +@mock.patch("pathlib.Path.is_dir", side_effect=[False]) +def test_invalid_root_folder(_): + with pytest.raises(Exception) as e_info: + DeployConfig.factory( + config_file_path=Path("some_config_file_name"), + root_folder="some_root_folder_name", + modules_folder="some_modules_folder_name", + config_vars={"some": "config_vars"}, + snowflake_account="some_snowflake_account", + snowflake_user="some_snowflake_user", + snowflake_role="some_snowflake_role", + snowflake_warehouse="some_snowflake_warehouse", + snowflake_database="some_snowflake_database", + snowflake_schema="some_snowflake_schema", + change_history_table="some_history_table", + query_tag="some_query_tag", + oauth_config={"some": "values"}, + ) + e_info_value = str(e_info.value) + assert "Path is not valid directory: some_root_folder_name" in e_info_value + + +@mock.patch("pathlib.Path.is_dir", side_effect=[True, False]) +def test_invalid_modules_folder(_): + with pytest.raises(Exception) as e_info: + DeployConfig.factory( + config_file_path=Path("some_config_file_name"), + root_folder="some_root_folder_name", + modules_folder="some_modules_folder_name", + config_vars={"some": "config_vars"}, + snowflake_account="some_snowflake_account", + snowflake_user="some_snowflake_user", + snowflake_role="some_snowflake_role", + snowflake_warehouse="some_snowflake_warehouse", + snowflake_database="some_snowflake_database", + snowflake_schema="some_snowflake_schema", + change_history_table="some_history_table", + query_tag="some_query_tag", + oauth_config={"some": "values"}, + ) + e_info_value = str(e_info.value) + assert "Path is not valid directory: some_modules_folder_name" in e_info_value + + +@mock.patch("pathlib.Path.is_dir", side_effect=[True, True]) +@mock.patch("pathlib.Path.is_file", side_effect=[False]) +def test_invalid_snowflake_private_key_path(_, __): + connections_file_path = Path(__file__).parent / "connections.toml" + connection_name = "myconnection" + + with pytest.raises(Exception) as e_info: + DeployConfig.factory( + config_file_path=Path("some_config_file_name"), + root_folder="some_root_folder_name", + modules_folder="some_modules_folder_name", + config_vars={"some": "config_vars"}, + snowflake_account="some_snowflake_account", + snowflake_user="some_snowflake_user", + snowflake_role="some_snowflake_role", + snowflake_warehouse="some_snowflake_warehouse", + snowflake_database="some_snowflake_database", + snowflake_schema="some_snowflake_schema", + snowflake_private_key_path="invalid_snowflake_private_key_path", + snowflake_token_path="invalid_snowflake_token_path", + connections_file_path=str(connections_file_path), + connection_name=connection_name, + change_history_table="some_history_table", + query_tag="some_query_tag", + oauth_config={"some": "values"}, + ) + e_info_value = str(e_info.value) + assert "invalid file path: invalid_snowflake_private_key_path" in e_info_value + + +@mock.patch("pathlib.Path.is_dir", side_effect=[True, True]) +@mock.patch("pathlib.Path.is_file", side_effect=[True, False]) +def test_invalid_snowflake_token_path(_, __): + connections_file_path = Path(__file__).parent / "connections.toml" + connection_name = "myconnection" + + with pytest.raises(Exception) as e_info: + DeployConfig.factory( + config_file_path=Path("some_config_file_name"), + root_folder="some_root_folder_name", + modules_folder="some_modules_folder_name", + config_vars={"some": "config_vars"}, + snowflake_account="some_snowflake_account", + snowflake_user="some_snowflake_user", + snowflake_role="some_snowflake_role", + snowflake_warehouse="some_snowflake_warehouse", + snowflake_database="some_snowflake_database", + snowflake_schema="some_snowflake_schema", + snowflake_private_key_path="valid_snowflake_private_key_path", + snowflake_token_path="invalid_snowflake_token_path", + connections_file_path=str(connections_file_path), + connection_name=connection_name, + change_history_table="some_history_table", + query_tag="some_query_tag", + oauth_config={"some": "values"}, + ) + e_info_value = str(e_info.value) + assert "invalid file path: invalid_snowflake_token_path" in e_info_value + + +def test_config_vars_not_a_dict(): + with pytest.raises(Exception) as e_info: + BaseConfig.factory( + subcommand="deploy", + config_vars="a string", + config_file_path=Path("."), + ) + assert "config_vars did not parse correctly, please check its configuration" in str( + e_info.value + ) + + +def test_config_vars_reserved_word(): + with pytest.raises(Exception) as e_info: + BaseConfig.factory( + subcommand="deploy", + config_vars={"schemachange": "not allowed"}, + config_file_path=Path("."), + ) + assert ( + "The variable 'schemachange' has been reserved for use by schemachange, please use a different name" + in str(e_info.value) + ) + + +def test_check_for_deploy_args_oauth_with_token_happy_path(): + config = DeployConfig.factory( + snowflake_account="account", + snowflake_user="user", + snowflake_role="role", + snowflake_warehouse="warehouse", + snowflake_authenticator="oauth", + snowflake_oauth_token="my-oauth-token", + config_file_path=Path("."), + ) + config.check_for_deploy_args() + + +@mock.patch("pathlib.Path.is_file", return_value=True) +def test_check_for_deploy_args_oauth_with_file_happy_path(_): + with mock.patch("builtins.open", mock_open(read_data="my-oauth-token-from-a-file")): + config = DeployConfig.factory( + snowflake_account="account", + snowflake_user="user", + snowflake_role="role", + snowflake_warehouse="warehouse", + snowflake_authenticator="oauth", + snowflake_token_path="token_path", + config_file_path=Path("."), + ) + config.check_for_deploy_args() + assert config.snowflake_oauth_token == "my-oauth-token-from-a-file" + + +@mock.patch("schemachange.config.DeployConfig.get_oauth_token") +def test_check_for_deploy_args_oauth_with_request_happy_path(mock_get_oauth_token): + oauth_token = "my-oauth-token-from-a-request" + mock_get_oauth_token.return_value = oauth_token + oauth_config = {"my_oauth_config": "values"} + config = DeployConfig.factory( + snowflake_account="account", + snowflake_user="user", + snowflake_role="role", + snowflake_warehouse="warehouse", + snowflake_authenticator="oauth", + oauth_config=oauth_config, + config_file_path=Path("."), + ) + config.check_for_deploy_args() + assert config.snowflake_oauth_token == oauth_token + mock_get_oauth_token.call_args.args[0] == oauth_config + + +def test_check_for_deploy_args_externalbrowser_happy_path(): + config = DeployConfig.factory( + snowflake_account="account", + snowflake_user="user", + snowflake_role="role", + snowflake_warehouse="warehouse", + snowflake_authenticator="externalbrowser", + config_file_path=Path("."), + ) + config.check_for_deploy_args() + + +def test_check_for_deploy_args_okta_happy_path(): + config = DeployConfig.factory( + snowflake_account="account", + snowflake_user="user", + snowflake_role="role", + snowflake_warehouse="warehouse", + snowflake_authenticator="https://okta...", + snowflake_password="password", + config_file_path=Path("."), + ) + config.check_for_deploy_args() + + +@mock.patch("pathlib.Path.is_file", return_value=True) +def test_check_for_deploy_args_snowflake_jwt_happy_path(_): + config = DeployConfig.factory( + snowflake_account="account", + snowflake_user="user", + snowflake_role="role", + snowflake_warehouse="warehouse", + snowflake_authenticator="snowflake_jwt", + snowflake_private_key_path="private_key_path", + config_file_path=Path("."), + ) + config.check_for_deploy_args() + + +def test_check_for_deploy_args_snowflake_happy_path(): + config = DeployConfig.factory( + snowflake_account="account", + snowflake_user="user", + snowflake_role="role", + snowflake_warehouse="warehouse", + snowflake_authenticator="snowflake", + snowflake_password="password", + config_file_path=Path("."), + ) + config.check_for_deploy_args() + + +def test_check_for_deploy_args_default_happy_path(): + config = DeployConfig.factory( + snowflake_account="account", + snowflake_user="user", + snowflake_role="role", + snowflake_warehouse="warehouse", + snowflake_password="password", + config_file_path=Path("."), + ) + config.check_for_deploy_args() + + +def test_check_for_deploy_args_exception(): + config = DeployConfig.factory(config_file_path=Path(".")) + with pytest.raises(ValueError) as e: + config.check_for_deploy_args() + + assert "Missing config values. The following config values are required" in str( + e.value + ) diff --git a/tests/config/test_RenderConfig.py b/tests/config/test_RenderConfig.py new file mode 100644 index 0000000..af435e2 --- /dev/null +++ b/tests/config/test_RenderConfig.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +from unittest import mock + +import pytest + +from schemachange.config.RenderConfig import RenderConfig + + +@mock.patch("pathlib.Path.is_file", return_value=False) +def test_render_config_invalid_path(_): + with pytest.raises(Exception) as e_info: + RenderConfig.factory(script_path="invalid path") + assert "invalid file path" in str(e_info) diff --git a/tests/config/test_get_config_secrets.py b/tests/config/test_get_config_secrets.py new file mode 100644 index 0000000..cb4f03f --- /dev/null +++ b/tests/config/test_get_config_secrets.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import pytest + +from schemachange.config.utils import get_config_secrets + + +def test_given_empty_config_should_not_error(): + get_config_secrets(config_vars={}) + + +def test_given_none_should_not_error(): + get_config_secrets(None) + + +@pytest.mark.parametrize( + "config_vars, secret", + [ + ({"secret": "secret_val1"}, "secret_val1"), + ({"SECret": "secret_val2"}, "secret_val2"), + ({"secret_key": "secret_val3"}, "secret_val3"), + ({"s3_bucket_secret": "secret_val4"}, "secret_val4"), + ({"s3SecretKey": "secret_val5"}, "secret_val5"), + ({"nested": {"s3_bucket_secret": "secret_val6"}}, "secret_val6"), + ], +) +def test_given__vars_with_keys_should_extract_secret(config_vars, secret): + results = get_config_secrets(config_vars) + assert secret in results + + +def test_given_vars_with_secrets_key_then_all_children_should_be_treated_as_secrets(): + config_vars = { + "secrets": { + "database_name": "database_name_val", + "schema_name": "schema_name_val", + "nested_secrets": {"SEC_ONE": "SEC_ONE_VAL"}, + } + } + results = get_config_secrets(config_vars=config_vars) + + assert len(results) == 3 + assert "database_name_val" in results + assert "schema_name_val" in results + assert "SEC_ONE_VAL" in results + + +def test_given_vars_with_nested_secrets_key_then_all_children_should_be_treated_as_secrets(): + config_vars = { + "nested": { + "secrets": { + "database_name": "database_name_val", + "schema_name": "schema_name_val", + "nested": {"SEC_ONE": "SEC_ONE_VAL"}, + } + } + } + + results = get_config_secrets(config_vars) + + assert len(results) == 3 + assert "database_name_val" in results + assert "schema_name_val" in results + assert "SEC_ONE_VAL" in results + + +def test_given_vars_with_same_secret_twice_then_only_extracted_once(): + config_vars = { + "secrets": { + "database_name": "SECRET_VALUE", + "schema_name": "SECRET_VALUE", + "nested_secrets": {"SEC_ONE": "SECRET_VALUE"}, + } + } + + results = get_config_secrets(config_vars) + + assert len(results) == 1 + assert "SECRET_VALUE" in results diff --git a/tests/config/test_get_merged_config.py b/tests/config/test_get_merged_config.py index f2676a9..b981874 100644 --- a/tests/config/test_get_merged_config.py +++ b/tests/config/test_get_merged_config.py @@ -1,160 +1,895 @@ +import json +import logging +import os +import tomllib from pathlib import Path from unittest import mock import pytest -from schemachange.config.ChangeHistoryTable import ChangeHistoryTable -from schemachange.config.get_merged_config import get_merged_config - -required_args = [ - "--snowflake-account", - "account", - "--snowflake-user", - "user", - "--snowflake-warehouse", - "warehouse", - "--snowflake-role", - "role", -] - - -class TestGetMergedConfig: - @mock.patch("pathlib.Path.is_dir", return_value=True) - def test_default_config_folder(self, _): - with mock.patch("sys.argv", ["schemachange", *required_args]): - config = get_merged_config() - assert ( - config.config_file_path == Path(".") / config.default_config_file_name - ) - - @mock.patch("pathlib.Path.is_dir", return_value=True) - def test_config_folder(self, _): - with mock.patch( - "sys.argv", ["schemachange", "--config-folder", "DUMMY", *required_args] - ): - config = get_merged_config() - assert ( - config.config_file_path - == Path("DUMMY") / config.default_config_file_name - ) - - @mock.patch("pathlib.Path.is_dir", return_value=False) - def test_invalid_config_folder(self, _): - with pytest.raises(Exception) as e_info: - with mock.patch( - "sys.argv", ["schemachange", "--config-folder", "DUMMY", *required_args] - ): - config = get_merged_config() - assert ( - config.config_file_path - == Path("DUMMY") / config.default_config_file_name - ) - e_info_value = str(e_info.value) - assert "Path is not valid directory: DUMMY" in e_info_value - - @mock.patch("pathlib.Path.is_dir", return_value=True) - def test_no_cli_args(self, _): - with mock.patch( - "sys.argv", ["schemachange", "--config-folder", str(Path(__file__).parent)] - ): - config = get_merged_config() - - assert config.snowflake_account == "snowflake-account-from-yaml" - assert config.snowflake_user == "snowflake-user-from-yaml" - assert config.snowflake_warehouse == '"snowflake-warehouse-from-yaml"' - assert config.snowflake_role == '"snowflake-role-from-yaml"' - assert str(config.root_folder) == "root-folder-from-yaml" - assert str(config.modules_folder) == "modules-folder-from-yaml" - assert config.snowflake_database == '"snowflake-database-from-yaml"' - assert config.snowflake_schema == '"snowflake-schema-from-yaml"' - assert config.change_history_table == ChangeHistoryTable( - table_name='"change-history-table-from-yaml"', - schema_name="SCHEMACHANGE", - database_name="METADATA", - ) - assert config.config_vars == {"var1": "from_yaml", "var2": "also_from_yaml"} - assert config.create_change_history_table is False - assert config.autocommit is False - assert config.dry_run is False - assert config.query_tag == "query-tag-from-yaml" - assert config.oauth_config == { - "token-provider-url": "token-provider-url-from-yaml", - "token-response-name": "token-response-name-from-yaml", - "token-request-headers": { - "Content-Type": "Content-Type-from-yaml", - "User-Agent": "User-Agent-from-yaml", +from schemachange.config.get_merged_config import ( + get_merged_config, + get_yaml_config_kwargs, +) + +default_cli_kwargs = { + "subcommand": "deploy", + "config_file_name": "schemachange-config.yml", + "config_vars": {}, +} + +assets_path = Path(__file__).parent + + +def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: + with file_path.open("rb") as f: + connections = tomllib.load(f) + return connections[connection_name] + + +my_connection = get_connection_from_toml( + file_path=assets_path / "connections.toml", connection_name="myconnection" +) + +alt_connection = get_connection_from_toml( + file_path=assets_path / "alt-connections.toml", + connection_name="myaltconnection", +) + +schemachange_config = get_yaml_config_kwargs(assets_path / "schemachange-config.yml") +schemachange_config_full = get_yaml_config_kwargs( + assets_path / "schemachange-config-full.yml" +) +schemachange_config_full_no_connection = get_yaml_config_kwargs( + assets_path / "schemachange-config-full-no-connection.yml" +) +schemachange_config_partial_with_connection = get_yaml_config_kwargs( + assets_path / "schemachange-config-partial-with-connection.yml" +) + + +@pytest.mark.parametrize( + "env_kwargs, cli_kwargs, yaml_kwargs, connection_kwargs, expected", + [ + pytest.param( + { # env_kwargs + "snowflake_password": None, + "snowflake_private_key_path": None, + "snowflake_authenticator": None, + }, + {**default_cli_kwargs}, # cli_kwargs + {}, # yaml_kwargs + {}, # connection_kwargs + { # expected + "config_file_path": Path("schemachange-config.yml"), + "config_vars": {}, + "subcommand": "deploy", + }, + id="Deploy: Only required arguments", + ), + pytest.param( + { # env_kwargs + "snowflake_password": None, + "snowflake_private_key_path": None, + "snowflake_authenticator": None, + }, + {**default_cli_kwargs}, # cli_kwargs + {}, # yaml_kwargs + { # connection_kwargs + "snowflake_account": "connection_snowflake_account", + "snowflake_user": "connection_snowflake_user", + "snowflake_role": "connection_snowflake_role", + "snowflake_warehouse": "connection_snowflake_warehouse", + "snowflake_database": "connection_snowflake_database", + "snowflake_schema": "connection_snowflake_schema", + "snowflake_authenticator": "connection_snowflake_authenticator", + "snowflake_password": "connection_snowflake_password", + "snowflake_private_key_path": "connection_snowflake_private_key_path", + "snowflake_token_path": "connection_snowflake_token_path", + }, + { # expected + "log_level": logging.INFO, + "config_file_path": Path("schemachange-config.yml"), + "config_vars": {}, + "subcommand": "deploy", + "snowflake_account": "connection_snowflake_account", + "snowflake_user": "connection_snowflake_user", + "snowflake_role": "connection_snowflake_role", + "snowflake_warehouse": "connection_snowflake_warehouse", + "snowflake_database": "connection_snowflake_database", + "snowflake_schema": "connection_snowflake_schema", + "snowflake_authenticator": "connection_snowflake_authenticator", + "snowflake_password": "connection_snowflake_password", + "snowflake_private_key_path": "connection_snowflake_private_key_path", + "snowflake_token_path": "connection_snowflake_token_path", + }, + id="Deploy: all connection_kwargs", + ), + pytest.param( + { # env_kwargs + "snowflake_password": None, + "snowflake_private_key_path": None, + "snowflake_authenticator": None, + }, + {**default_cli_kwargs}, # cli_kwargs + { # yaml_kwargs + "root_folder": "yaml_root_folder", + "modules_folder": "yaml_modules_folder", + "config_vars": { + "variable_1": "yaml_variable_1", + "variable_2": "yaml_variable_2", + "variable_3": "yaml_variable_3", + }, + "log_level": logging.DEBUG, + "snowflake_account": "yaml_snowflake_account", + "snowflake_user": "yaml_snowflake_user", + "snowflake_role": "yaml_snowflake_role", + "snowflake_warehouse": "yaml_snowflake_warehouse", + "snowflake_database": "yaml_snowflake_database", + "snowflake_schema": "yaml_snowflake_schema", + "snowflake_authenticator": "yaml_snowflake_authenticator", + "snowflake_private_key_path": "yaml_snowflake_private_key_path", + "snowflake_token_path": "yaml_snowflake_token_path", + "connections_file_path": "yaml_connections_file_path", + "connection_name": "yaml_connection_name", + "change_history_table": "yaml_change_history_table", + "create_change_history_table": True, + "autocommit": True, + "dry_run": True, + "query_tag": "yaml_query_tag", + "oauth_config": {"oauth_config_variable": "yaml_oauth_config_value"}, + }, + { # connection_kwargs + "snowflake_account": "connection_snowflake_account", + "snowflake_user": "connection_snowflake_user", + "snowflake_role": "connection_snowflake_role", + "snowflake_warehouse": "connection_snowflake_warehouse", + "snowflake_database": "connection_snowflake_database", + "snowflake_schema": "connection_snowflake_schema", + "snowflake_authenticator": "connection_snowflake_authenticator", + "snowflake_password": "connection_snowflake_password", + "snowflake_private_key_path": "connection_snowflake_private_key_path", + "snowflake_token_path": "connection_snowflake_token_path", + }, + { # expected + "log_level": logging.DEBUG, + "config_file_path": Path("schemachange-config.yml"), + "config_vars": { + "variable_1": "yaml_variable_1", + "variable_2": "yaml_variable_2", + "variable_3": "yaml_variable_3", + }, + "subcommand": "deploy", + "root_folder": "yaml_root_folder", + "modules_folder": "yaml_modules_folder", + "snowflake_account": "yaml_snowflake_account", + "snowflake_user": "yaml_snowflake_user", + "snowflake_role": "yaml_snowflake_role", + "snowflake_warehouse": "yaml_snowflake_warehouse", + "snowflake_database": "yaml_snowflake_database", + "snowflake_schema": "yaml_snowflake_schema", + "snowflake_authenticator": "yaml_snowflake_authenticator", + "snowflake_password": "connection_snowflake_password", + "snowflake_private_key_path": "yaml_snowflake_private_key_path", + "snowflake_token_path": "yaml_snowflake_token_path", + "connections_file_path": "yaml_connections_file_path", + "connection_name": "yaml_connection_name", + "change_history_table": "yaml_change_history_table", + "create_change_history_table": True, + "autocommit": True, + "dry_run": True, + "query_tag": "yaml_query_tag", + "oauth_config": {"oauth_config_variable": "yaml_oauth_config_value"}, + }, + id="Deploy: all yaml, all connection_kwargs", + ), + pytest.param( + { # env_kwargs + "snowflake_password": None, + "snowflake_private_key_path": None, + "snowflake_authenticator": None, + }, + { # cli_kwargs + **default_cli_kwargs, + "config_folder": "cli_config_folder", + "root_folder": "cli_root_folder", + "modules_folder": "cli_modules_folder", + "config_vars": { + "variable_1": "cli_variable_1", + "variable_2": "cli_variable_2", + }, + "log_level": logging.INFO, + "snowflake_account": "cli_snowflake_account", + "snowflake_user": "cli_snowflake_user", + "snowflake_role": "cli_snowflake_role", + "snowflake_warehouse": "cli_snowflake_warehouse", + "snowflake_database": "cli_snowflake_database", + "snowflake_schema": "cli_snowflake_schema", + "snowflake_authenticator": "cli_snowflake_authenticator", + "snowflake_private_key_path": "cli_snowflake_private_key_path", + "snowflake_token_path": "cli_snowflake_token_path", + "connections_file_path": "cli_connections_file_path", + "connection_name": "cli_connection_name", + "change_history_table": "cli_change_history_table", + "create_change_history_table": False, + "autocommit": False, + "dry_run": False, + "query_tag": "cli_query_tag", + "oauth_config": {"oauth_config_variable": "cli_oauth_config_value"}, + }, + { # yaml_kwargs + "root_folder": "yaml_root_folder", + "modules_folder": "yaml_modules_folder", + "config_vars": { + "variable_1": "yaml_variable_1", + "variable_2": "yaml_variable_2", + "variable_3": "yaml_variable_3", + }, + "log_level": logging.DEBUG, + "snowflake_account": "yaml_snowflake_account", + "snowflake_user": "yaml_snowflake_user", + "snowflake_role": "yaml_snowflake_role", + "snowflake_warehouse": "yaml_snowflake_warehouse", + "snowflake_database": "yaml_snowflake_database", + "snowflake_schema": "yaml_snowflake_schema", + "snowflake_authenticator": "yaml_snowflake_authenticator", + "snowflake_private_key_path": "yaml_snowflake_private_key_path", + "snowflake_token_path": "yaml_snowflake_token_path", + "connections_file_path": "yaml_connections_file_path", + "connection_name": "yaml_connection_name", + "change_history_table": "yaml_change_history_table", + "create_change_history_table": True, + "autocommit": True, + "dry_run": True, + "query_tag": "yaml_query_tag", + "oauth_config": {"oauth_config_variable": "yaml_oauth_config_value"}, + }, + { # connection_kwargs + "snowflake_account": "connection_snowflake_account", + "snowflake_user": "connection_snowflake_user", + "snowflake_role": "connection_snowflake_role", + "snowflake_warehouse": "connection_snowflake_warehouse", + "snowflake_database": "connection_snowflake_database", + "snowflake_schema": "connection_snowflake_schema", + "snowflake_authenticator": "connection_snowflake_authenticator", + "snowflake_password": "connection_snowflake_password", + "snowflake_private_key_path": "connection_snowflake_private_key_path", + "snowflake_token_path": "connection_snowflake_token_path", + }, + { # expected + "log_level": logging.INFO, + "config_file_path": Path("cli_config_folder/schemachange-config.yml"), + "config_vars": { + "variable_1": "cli_variable_1", + "variable_2": "cli_variable_2", + "variable_3": "yaml_variable_3", + }, + "subcommand": "deploy", + "root_folder": "cli_root_folder", + "modules_folder": "cli_modules_folder", + "snowflake_account": "cli_snowflake_account", + "snowflake_user": "cli_snowflake_user", + "snowflake_role": "cli_snowflake_role", + "snowflake_warehouse": "cli_snowflake_warehouse", + "snowflake_database": "cli_snowflake_database", + "snowflake_schema": "cli_snowflake_schema", + "snowflake_authenticator": "cli_snowflake_authenticator", + "snowflake_password": "connection_snowflake_password", + "snowflake_private_key_path": "cli_snowflake_private_key_path", + "snowflake_token_path": "cli_snowflake_token_path", + "connections_file_path": "cli_connections_file_path", + "connection_name": "cli_connection_name", + "change_history_table": "cli_change_history_table", + "create_change_history_table": False, + "autocommit": False, + "dry_run": False, + "query_tag": "cli_query_tag", + "oauth_config": {"oauth_config_variable": "cli_oauth_config_value"}, + }, + id="Deploy: all cli, all yaml, all connection_kwargs", + ), + pytest.param( + { # env_kwargs + "snowflake_password": "env_snowflake_password", + "snowflake_private_key_path": "env_snowflake_private_key_path", + "snowflake_authenticator": "env_snowflake_authenticator", + }, + { # cli_kwargs + **default_cli_kwargs, + "config_folder": "cli_config_folder", + "root_folder": "cli_root_folder", + "modules_folder": "cli_modules_folder", + "config_vars": { + "variable_1": "cli_variable_1", + "variable_2": "cli_variable_2", }, - "token-request-payload": { - "client_id": "id-from-yaml", - "username": "username-from-yaml", - "password": "password-from-yaml", - "grant_type": "type-from-yaml", - "scope": "scope-from-yaml", + "log_level": logging.INFO, + "snowflake_account": "cli_snowflake_account", + "snowflake_user": "cli_snowflake_user", + "snowflake_role": "cli_snowflake_role", + "snowflake_warehouse": "cli_snowflake_warehouse", + "snowflake_database": "cli_snowflake_database", + "snowflake_schema": "cli_snowflake_schema", + "snowflake_authenticator": "cli_snowflake_authenticator", + "snowflake_private_key_path": "cli_snowflake_private_key_path", + "snowflake_token_path": "cli_snowflake_token_path", + "connections_file_path": "cli_connections_file_path", + "connection_name": "cli_connection_name", + "change_history_table": "cli_change_history_table", + "create_change_history_table": False, + "autocommit": False, + "dry_run": False, + "query_tag": "cli_query_tag", + "oauth_config": {"oauth_config_variable": "cli_oauth_config_value"}, + }, + { # yaml_kwargs + "root_folder": "yaml_root_folder", + "modules_folder": "yaml_modules_folder", + "config_vars": { + "variable_1": "yaml_variable_1", + "variable_2": "yaml_variable_2", + "variable_3": "yaml_variable_3", }, - } - - @mock.patch("pathlib.Path.is_dir", return_value=True) - def test_all_cli_args(self, _): - with mock.patch( - "sys.argv", - [ - "schemachange", - "--config-folder", - str(Path(__file__).parent), - "--root-folder", - "root-folder-from-cli", - "--modules-folder", - "modules-folder-from-cli", - "--vars", - '{"var1": "from_cli", "var3": "also_from_cli"}', - "--snowflake-account", - "snowflake-account-from-cli", - "--snowflake-user", - "snowflake-user-from-cli", - "--snowflake-role", - "snowflake-role-from-cli", - "--snowflake-warehouse", - "snowflake-warehouse-from-cli", - "--snowflake-database", - "snowflake-database-from-cli", - "--snowflake-schema", - "snowflake-schema-from-cli", - "--change-history-table", - "change-history-table-from-cli", - "--create-change-history-table", - "--autocommit", - "--dry-run", - "--query-tag", - "query-tag-from-cli", - "--oauth-config", - '{"token-provider-url": "https//...", "token-request-payload": {"client_id": "GUID_xyz"} }', - ], - ): - config = get_merged_config() - - assert config.snowflake_account == "snowflake-account-from-cli" - assert config.snowflake_user == "snowflake-user-from-cli" - assert config.snowflake_warehouse == '"snowflake-warehouse-from-cli"' - assert config.snowflake_role == '"snowflake-role-from-cli"' - assert str(config.root_folder) == "root-folder-from-cli" - assert str(config.modules_folder) == "modules-folder-from-cli" - assert config.snowflake_database == '"snowflake-database-from-cli"' - assert config.snowflake_schema == '"snowflake-schema-from-cli"' - assert config.change_history_table == ChangeHistoryTable( - table_name='"change-history-table-from-cli"', - schema_name="SCHEMACHANGE", - database_name="METADATA", - ) - assert config.config_vars == { - "var1": "from_cli", - "var2": "also_from_yaml", - "var3": "also_from_cli", - } - assert config.create_change_history_table is True - assert config.autocommit is True - assert config.dry_run is True - assert config.query_tag == "query-tag-from-cli" - assert config.oauth_config == { - "token-provider-url": "https//...", - "token-request-payload": {"client_id": "GUID_xyz"}, - } + "log_level": logging.DEBUG, + "snowflake_account": "yaml_snowflake_account", + "snowflake_user": "yaml_snowflake_user", + "snowflake_role": "yaml_snowflake_role", + "snowflake_warehouse": "yaml_snowflake_warehouse", + "snowflake_database": "yaml_snowflake_database", + "snowflake_schema": "yaml_snowflake_schema", + "snowflake_authenticator": "yaml_snowflake_authenticator", + "snowflake_private_key_path": "yaml_snowflake_private_key_path", + "snowflake_token_path": "yaml_snowflake_token_path", + "connections_file_path": "yaml_connections_file_path", + "connection_name": "yaml_connection_name", + "change_history_table": "yaml_change_history_table", + "create_change_history_table": True, + "autocommit": True, + "dry_run": True, + "query_tag": "yaml_query_tag", + "oauth_config": {"oauth_config_variable": "yaml_oauth_config_value"}, + }, + { # connection_kwargs + "snowflake_account": "connection_snowflake_account", + "snowflake_user": "connection_snowflake_user", + "snowflake_role": "connection_snowflake_role", + "snowflake_warehouse": "connection_snowflake_warehouse", + "snowflake_database": "connection_snowflake_database", + "snowflake_schema": "connection_snowflake_schema", + "snowflake_authenticator": "connection_snowflake_authenticator", + "snowflake_password": "connection_snowflake_password", + "snowflake_private_key_path": "connection_snowflake_private_key_path", + "snowflake_token_path": "connection_snowflake_token_path", + }, + { # expected + "log_level": logging.INFO, + "config_file_path": Path("cli_config_folder/schemachange-config.yml"), + "config_vars": { + "variable_1": "cli_variable_1", + "variable_2": "cli_variable_2", + "variable_3": "yaml_variable_3", + }, + "subcommand": "deploy", + "root_folder": "cli_root_folder", + "modules_folder": "cli_modules_folder", + "snowflake_account": "cli_snowflake_account", + "snowflake_user": "cli_snowflake_user", + "snowflake_role": "cli_snowflake_role", + "snowflake_warehouse": "cli_snowflake_warehouse", + "snowflake_database": "cli_snowflake_database", + "snowflake_schema": "cli_snowflake_schema", + "snowflake_authenticator": "env_snowflake_authenticator", + "snowflake_password": "env_snowflake_password", + "snowflake_private_key_path": "env_snowflake_private_key_path", + "snowflake_token_path": "cli_snowflake_token_path", + "connections_file_path": "cli_connections_file_path", + "connection_name": "cli_connection_name", + "change_history_table": "cli_change_history_table", + "create_change_history_table": False, + "autocommit": False, + "dry_run": False, + "query_tag": "cli_query_tag", + "oauth_config": {"oauth_config_variable": "cli_oauth_config_value"}, + }, + id="Deploy: all env, all cli, all yaml, all connection_kwargs", + ), + ], +) +@mock.patch("pathlib.Path.is_dir", return_value=True) +@mock.patch("schemachange.config.get_merged_config.get_env_kwargs") +@mock.patch("schemachange.config.get_merged_config.parse_cli_args") +@mock.patch("schemachange.config.get_merged_config.get_yaml_config_kwargs") +@mock.patch("schemachange.config.get_merged_config.get_connection_kwargs") +@mock.patch("schemachange.config.get_merged_config.DeployConfig.factory") +def test_get_merged_config_inheritance( + mock_deploy_config_factory, + mock_get_connection_kwargs, + mock_get_yaml_config_kwargs, + mock_parse_cli_args, + mock_get_env_kwargs, + _, + env_kwargs, + cli_kwargs, + yaml_kwargs, + connection_kwargs, + expected, +): + mock_get_env_kwargs.return_value = {**env_kwargs} + mock_parse_cli_args.return_value = {**cli_kwargs} + mock_get_yaml_config_kwargs.return_value = {**yaml_kwargs} + mock_get_connection_kwargs.return_value = {**connection_kwargs} + get_merged_config() + factory_kwargs = mock_deploy_config_factory.call_args.kwargs + for actual_key, actual_value in factory_kwargs.items(): + assert expected[actual_key] == actual_value + + +@mock.patch("pathlib.Path.is_dir", return_value=False) +@mock.patch("schemachange.config.get_merged_config.parse_cli_args") +def test_invalid_config_folder(mock_parse_cli_args, _): + cli_kwargs = { + "config_folder": "cli_config_folder", + **default_cli_kwargs, + } + mock_parse_cli_args.return_value = {**cli_kwargs} + with pytest.raises(Exception) as e_info: + get_merged_config() + assert f"Path is not valid directory: {cli_kwargs['config_folder']}" in str( + e_info.value + ) + + +param_only_required_cli_arguments = pytest.param( + {}, # env_kwargs + [ # cli_args + "schemachange", + ], + { # expected + "subcommand": "deploy", + "config_file_path": Path("schemachange-config.yml"), + "config_version": 1, + "config_vars": {}, + "log_level": logging.INFO, + }, + id="Deploy: Only required cli arguments", +) + +param_full_cli_and_connection = pytest.param( + {}, # env_kwargs + [ # cli_args + "schemachange", + "--root-folder", + "root-folder-from-cli", + "--modules-folder", + "modules-folder-from-cli", + "--vars", + '{"var1": "from_cli", "var3": "also_from_cli"}', + "--snowflake-account", + "snowflake-account-from-cli", + "--snowflake-user", + "snowflake-user-from-cli", + "--snowflake-role", + "snowflake-role-from-cli", + "--snowflake-warehouse", + "snowflake-warehouse-from-cli", + "--snowflake-database", + "snowflake-database-from-cli", + "--snowflake-schema", + "snowflake-schema-from-cli", + "--snowflake-authenticator", + "snowflake-authenticator-from-cli", + "--snowflake-private-key-path", + "snowflake-private-key-path-from-cli", + "--snowflake-token-path", + "snowflake-token-path-from-cli", + "--connections-file-path", + str(assets_path / "alt-connections.toml"), + "--connection-name", + "myaltconnection", + "--change-history-table", + "change-history-table-from-cli", + "--create-change-history-table", + "--autocommit", + "--dry-run", + "--query-tag", + "query-tag-from-cli", + "--oauth-config", + json.dumps({"oauth_config_variable": "cli_oauth_config_value"}), + ], + { # expected + "subcommand": "deploy", + "config_file_path": Path("schemachange-config.yml"), + "config_version": 1, + "root_folder": "root-folder-from-cli", + "modules_folder": "modules-folder-from-cli", + "snowflake_account": "snowflake-account-from-cli", + "snowflake_user": "snowflake-user-from-cli", + "snowflake_role": "snowflake-role-from-cli", + "snowflake_warehouse": "snowflake-warehouse-from-cli", + "snowflake_database": "snowflake-database-from-cli", + "snowflake_schema": "snowflake-schema-from-cli", + "snowflake_authenticator": "snowflake-authenticator-from-cli", + "snowflake_private_key_path": "snowflake-private-key-path-from-cli", + "snowflake_token_path": "snowflake-token-path-from-cli", + "change_history_table": "change-history-table-from-cli", + "config_vars": { + "var1": "from_cli", + "var3": "also_from_cli", + }, + "create_change_history_table": True, + "autocommit": True, + "log_level": logging.INFO, + "dry_run": True, + "query_tag": "query-tag-from-cli", + "oauth_config": {"oauth_config_variable": "cli_oauth_config_value"}, + "connection_name": "myaltconnection", + "connections_file_path": str(assets_path / "alt-connections.toml"), + "snowflake_password": alt_connection["password"], + }, + id="Deploy: full cli and connections.toml", +) + +param_full_yaml_no_connection = pytest.param( + {}, # env_kwargs + [ # cli_args + "schemachange", + "--config-folder", + str(assets_path), + "--config-file-name", + "schemachange-config-full-no-connection.yml", + ], + { # expected + "subcommand": "deploy", + "config_file_path": assets_path / "schemachange-config-full-no-connection.yml", + "log_level": logging.INFO, + **{ + k: v + for k, v in schemachange_config_full_no_connection.items() + if k + in [ + "config_version", + "root_folder", + "modules_folder", + "snowflake_account", + "snowflake_user", + "snowflake_role", + "snowflake_warehouse", + "snowflake_database", + "snowflake_schema", + "snowflake_authenticator", + "snowflake_private_key_path", + "snowflake_token_path", + "change_history_table", + "config_vars", + "create_change_history_table", + "autocommit", + "dry_run", + "query_tag", + "oauth_config", + ] + }, + }, + id="Deploy: yaml, no connections.toml", +) + +param_full_yaml_and_connection = pytest.param( + {}, # env_kwargs + [ # cli_args + "schemachange", + "--config-folder", + str(assets_path), + "--config-file-name", + "schemachange-config-full.yml", + ], + { # expected + "subcommand": "deploy", + "config_file_path": assets_path / "schemachange-config-full.yml", + "snowflake_password": my_connection["password"], + "log_level": logging.INFO, + **{ + k: v + for k, v in schemachange_config_full.items() + if k + in [ + "config_version", + "root_folder", + "modules_folder", + "snowflake_account", + "snowflake_user", + "snowflake_role", + "snowflake_warehouse", + "snowflake_database", + "snowflake_schema", + "snowflake_authenticator", + "snowflake_private_key_path", + "snowflake_token_path", + "change_history_table", + "snowflake_private_key_path", + "config_vars", + "create_change_history_table", + "autocommit", + "dry_run", + "query_tag", + "oauth_config", + "connection_name", + "connections_file_path", + ] + }, + }, + id="Deploy: full yaml and connections.toml", +) + +param_full_yaml_and_connection_and_cli = pytest.param( + {}, # env_kwargs + [ # cli_args + "schemachange", + "--config-folder", + str(assets_path), + "--config-file-name", + "schemachange-config-full.yml", + "--root-folder", + "root-folder-from-cli", + "--modules-folder", + "modules-folder-from-cli", + "--vars", + '{"var1": "from_cli", "var3": "also_from_cli"}', + "--snowflake-account", + "snowflake-account-from-cli", + "--snowflake-user", + "snowflake-user-from-cli", + "--snowflake-role", + "snowflake-role-from-cli", + "--snowflake-warehouse", + "snowflake-warehouse-from-cli", + "--snowflake-database", + "snowflake-database-from-cli", + "--snowflake-schema", + "snowflake-schema-from-cli", + "--snowflake-authenticator", + "snowflake-authenticator-from-cli", + "--snowflake-private-key-path", + "snowflake-private-key-path-from-cli", + "--snowflake-token-path", + "snowflake-token-path-from-cli", + "--connections-file-path", + str(assets_path / "alt-connections.toml"), + "--connection-name", + "myaltconnection", + "--change-history-table", + "change-history-table-from-cli", + "--create-change-history-table", + "--autocommit", + "--dry-run", + "--query-tag", + "query-tag-from-cli", + "--oauth-config", + json.dumps({"oauth_config_variable": "cli_oauth_config_value"}), + ], + { # expected + "subcommand": "deploy", + "config_file_path": assets_path / "schemachange-config-full.yml", + "config_version": 1, + "root_folder": "root-folder-from-cli", + "modules_folder": "modules-folder-from-cli", + "snowflake_account": "snowflake-account-from-cli", + "snowflake_user": "snowflake-user-from-cli", + "snowflake_role": "snowflake-role-from-cli", + "snowflake_warehouse": "snowflake-warehouse-from-cli", + "snowflake_database": "snowflake-database-from-cli", + "snowflake_schema": "snowflake-schema-from-cli", + "snowflake_authenticator": "snowflake-authenticator-from-cli", + "snowflake_private_key_path": "snowflake-private-key-path-from-cli", + "snowflake_token_path": "snowflake-token-path-from-cli", + "change_history_table": "change-history-table-from-cli", + "config_vars": { + "var1": "from_cli", + "var2": "also_from_yaml", + "var3": "also_from_cli", + }, + "create_change_history_table": True, + "autocommit": True, + "log_level": logging.INFO, + "dry_run": True, + "query_tag": "query-tag-from-cli", + "oauth_config": {"oauth_config_variable": "cli_oauth_config_value"}, + "connection_name": "myaltconnection", + "connections_file_path": str(assets_path / "alt-connections.toml"), + "snowflake_password": alt_connection["password"], + }, + id="Deploy: full yaml, connections.toml, and cli", +) + +param_full_yaml_and_connection_and_cli_and_env = pytest.param( + { + "SNOWFLAKE_PASSWORD": "env_snowflake_password", + "SNOWFLAKE_PRIVATE_KEY_PATH": "env_snowflake_private_key_path", + "SNOWFLAKE_AUTHENTICATOR": "env_snowflake_authenticator", + }, # env_kwargs + [ # cli_args + "schemachange", + "--config-folder", + str(assets_path), + "--config-file-name", + "schemachange-config-full.yml", + "--root-folder", + "root-folder-from-cli", + "--modules-folder", + "modules-folder-from-cli", + "--vars", + '{"var1": "from_cli", "var3": "also_from_cli"}', + "--snowflake-account", + "snowflake-account-from-cli", + "--snowflake-user", + "snowflake-user-from-cli", + "--snowflake-role", + "snowflake-role-from-cli", + "--snowflake-warehouse", + "snowflake-warehouse-from-cli", + "--snowflake-database", + "snowflake-database-from-cli", + "--snowflake-schema", + "snowflake-schema-from-cli", + "--snowflake-authenticator", + "snowflake-authenticator-from-cli", + "--snowflake-private-key-path", + "snowflake-private-key-path-from-cli", + "--snowflake-token-path", + "snowflake-token-path-from-cli", + "--connections-file-path", + str(assets_path / "alt-connections.toml"), + "--connection-name", + "myaltconnection", + "--change-history-table", + "change-history-table-from-cli", + "--create-change-history-table", + "--autocommit", + "--dry-run", + "--query-tag", + "query-tag-from-cli", + "--oauth-config", + json.dumps({"oauth_config_variable": "cli_oauth_config_value"}), + ], + { # expected + "subcommand": "deploy", + "config_file_path": assets_path / "schemachange-config-full.yml", + "config_version": 1, + "root_folder": "root-folder-from-cli", + "modules_folder": "modules-folder-from-cli", + "snowflake_account": "snowflake-account-from-cli", + "snowflake_user": "snowflake-user-from-cli", + "snowflake_role": "snowflake-role-from-cli", + "snowflake_warehouse": "snowflake-warehouse-from-cli", + "snowflake_database": "snowflake-database-from-cli", + "snowflake_schema": "snowflake-schema-from-cli", + "snowflake_authenticator": "env_snowflake_authenticator", + "snowflake_private_key_path": "env_snowflake_private_key_path", + "snowflake_token_path": "snowflake-token-path-from-cli", + "change_history_table": "change-history-table-from-cli", + "config_vars": { + "var1": "from_cli", + "var2": "also_from_yaml", + "var3": "also_from_cli", + }, + "create_change_history_table": True, + "autocommit": True, + "log_level": logging.INFO, + "dry_run": True, + "query_tag": "query-tag-from-cli", + "oauth_config": {"oauth_config_variable": "cli_oauth_config_value"}, + "connection_name": "myaltconnection", + "connections_file_path": str(assets_path / "alt-connections.toml"), + "snowflake_password": "env_snowflake_password", + }, + id="Deploy: full yaml, connections.toml, cli, and env", +) + + +param_connection_no_yaml = pytest.param( + {}, # env_kwargs + [ # cli_args + "schemachange", + "--config-folder", + str(assets_path), + "--connections-file-path", + str(assets_path / "connections.toml"), + "--connection-name", + "myconnection", + ], + { # expected + "subcommand": "deploy", + "connections_file_path": str(assets_path / "connections.toml"), + "connection_name": "myconnection", + "config_file_path": assets_path / "schemachange-config.yml", + "config_version": 1, + "snowflake_account": my_connection["account"], + "snowflake_user": my_connection["user"], + "snowflake_role": my_connection["role"], + "snowflake_warehouse": my_connection["warehouse"], + "snowflake_database": my_connection["database"], + "snowflake_schema": my_connection["schema"], + "snowflake_authenticator": my_connection["authenticator"], + "snowflake_password": my_connection["password"], + "snowflake_private_key_path": my_connection["private-key"], + "snowflake_token_path": my_connection["token-file-path"], + "config_vars": {}, + "log_level": logging.INFO, + }, + id="Deploy: connections.toml, no yaml", +) + +param_partial_yaml_and_connection = pytest.param( + {}, # env_kwargs + [ # cli_arg + "schemachange", + "--config-folder", + str(assets_path), + "--config-file-name", + "schemachange-config-partial-with-connection.yml", + ], + { # expected + "subcommand": "deploy", + "config_file_path": assets_path + / "schemachange-config-partial-with-connection.yml", + "snowflake_account": my_connection["account"], + "snowflake_user": my_connection["user"], + "snowflake_role": my_connection["role"], + "snowflake_warehouse": my_connection["warehouse"], + "snowflake_database": my_connection["database"], + "snowflake_schema": my_connection["schema"], + "snowflake_authenticator": my_connection["authenticator"], + "snowflake_private_key_path": my_connection["private-key"], + "snowflake_token_path": my_connection["token-file-path"], + "log_level": logging.INFO, + "snowflake_password": my_connection["password"], + **{ + k: v + for k, v in schemachange_config_partial_with_connection.items() + if k + in [ + "config_version", + "root_folder", + "modules_folder", + "change_history_table", + "config_vars", + "create_change_history_table", + "autocommit", + "dry_run", + "query_tag", + "oauth_config", + "connection_name", + "connections_file_path", + ] + }, + }, + id="Deploy: partial yaml and connections.toml", +) + + +@pytest.mark.parametrize( + "env_vars, cli_args, expected", + [ + param_only_required_cli_arguments, + param_full_cli_and_connection, + param_full_yaml_no_connection, + param_full_yaml_and_connection, + param_full_yaml_and_connection_and_cli, + param_full_yaml_and_connection_and_cli_and_env, + param_connection_no_yaml, + param_partial_yaml_and_connection, + ], +) +@mock.patch("pathlib.Path.is_dir", return_value=True) +@mock.patch("schemachange.config.get_merged_config.DeployConfig.factory") +def test_integration_get_merged_config_inheritance( + mock_deploy_config_factory, + _, + env_vars, + cli_args, + expected, +): + with mock.patch.dict(os.environ, env_vars): + with mock.patch("sys.argv", cli_args): + get_merged_config() + factory_kwargs = mock_deploy_config_factory.call_args.kwargs + for actual_key, actual_value in factory_kwargs.items(): + assert expected[actual_key] == actual_value diff --git a/tests/config/test_get_yaml_config.py b/tests/config/test_get_yaml_config.py index 9748fff..9e8d4ac 100644 --- a/tests/config/test_get_yaml_config.py +++ b/tests/config/test_get_yaml_config.py @@ -22,7 +22,7 @@ def test_load_yaml_config__simple_config_file(tmp_path: Path): vars: database_name: SCHEMACHANGE_DEMO_JINJA """ - config_file = tmp_path / "schemachange-config.yml" + config_file = tmp_path / "schemachange-config-full.yml" config_file.write_text(config_contents) # noinspection PyTypeChecker @@ -45,7 +45,7 @@ def test_load_yaml_config__with_env_var_should_populate_value( vars: database_name: SCHEMACHANGE_DEMO_JINJA """ - config_file = tmp_path / "schemachange-config.yml" + config_file = tmp_path / "schemachange-config-full.yml" config_file.write_text(config_contents) config = load_yaml_config(config_file) @@ -63,7 +63,7 @@ def test_load_yaml_config__requiring_env_var_but_env_var_not_set_should_raise_ex vars: database_name: SCHEMACHANGE_DEMO_JINJA """ - config_file = tmp_path / "schemachange-config.yml" + config_file = tmp_path / "schemachange-config-full.yml" config_file.write_text(config_contents) with pytest.raises(ValueError) as e: @@ -76,7 +76,7 @@ def test_load_yaml_config__requiring_env_var_but_env_var_not_set_should_raise_ex @mock.patch("pathlib.Path.is_dir", return_value=True) def test_get_yaml_config(_): - config_file_path = Path(__file__).parent / "schemachange-config.yml" + config_file_path = Path(__file__).parent / "schemachange-config-full.yml" yaml_config = get_yaml_config_kwargs(config_file_path=config_file_path) assert str(yaml_config["root_folder"]) == "root-folder-from-yaml" assert str(yaml_config["modules_folder"]) == "modules-folder-from-yaml" diff --git a/tests/config/test_parse_cli_args.py b/tests/config/test_parse_cli_args.py index 2536558..545002b 100644 --- a/tests/config/test_parse_cli_args.py +++ b/tests/config/test_parse_cli_args.py @@ -7,7 +7,11 @@ def test_parse_args_defaults(): args: list[str] = [] - test_args = [("--config-folder", None, ".")] + test_args = [ + ("--config-folder", None, "."), + ("--config-file-name", None, "schemachange-config.yml"), + ("--config-vars", None, {}), + ] expected: dict[str, str | int | None] = {} for arg, value, expected_value in test_args: if value: @@ -18,9 +22,6 @@ def test_parse_args_defaults(): parsed_args = parse_cli_args(args) for expected_arg, expected_value in expected.items(): assert parsed_args[expected_arg] == expected_value - assert parsed_args["create_change_history_table"] is None - assert parsed_args["autocommit"] is None - assert parsed_args["dry_run"] is None assert parsed_args["subcommand"] == "deploy" @@ -30,6 +31,7 @@ def test_parse_args_deploy_names(): valued_test_args: list[tuple[str, str, str]] = [ ("--config-folder", "some_config_folder_name", "some_config_folder_name"), + ("--config-file-name", "some_config_file_name", "some_config_file_name"), ("--root-folder", "some_root_folder_name", "some_root_folder_name"), ("--modules-folder", "some_modules_folder_name", "some_modules_folder_name"), ("--vars", json.dumps({"some": "vars"}), {"some": "vars"}), diff --git a/tests/config/test_utils.py b/tests/config/test_utils.py new file mode 100644 index 0000000..cc81135 --- /dev/null +++ b/tests/config/test_utils.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import os +from pathlib import Path +from unittest import mock + +import pytest + +from schemachange.config.utils import ( + get_snowflake_password, + get_env_kwargs, + get_connection_kwargs, +) + +assets_path = Path(__file__).parent + + +@pytest.mark.parametrize( + "env_vars, expected", + [ + ({"SNOWFLAKE_PASSWORD": "my-password"}, "my-password"), + ({"SNOWFLAKE_PASSWORD": ""}, None), + ({}, None), + ({"SNOWSQL_PWD": "my-password"}, "my-password"), + ( + {"SNOWSQL_PWD": "my-password", "SNOWFLAKE_PASSWORD": "my-password"}, + "my-password", + ), + ], +) +def test_get_snowflake_password(env_vars: dict, expected: str): + with mock.patch.dict(os.environ, env_vars, clear=True): + result = get_snowflake_password() + assert result == expected + + +@pytest.mark.parametrize( + "env_vars, expected", + [ + ( + {"SNOWSQL_PWD": "ignored", "SNOWFLAKE_PASSWORD": "my_snowflake_password"}, + {"snowflake_password": "my_snowflake_password"}, + ), + ( + {"SNOWSQL_PWD": "my_snowflake_password"}, + {"snowflake_password": "my_snowflake_password"}, + ), + ( + {"SNOWFLAKE_PASSWORD": "my_snowflake_password"}, + {"snowflake_password": "my_snowflake_password"}, + ), + ( + {"SNOWFLAKE_PRIVATE_KEY_PATH": "my_snowflake_private_key_path"}, + {"snowflake_private_key_path": "my_snowflake_private_key_path"}, + ), + ( + {"SNOWFLAKE_AUTHENTICATOR": "my_snowflake_authenticator"}, + {"snowflake_authenticator": "my_snowflake_authenticator"}, + ), + ], +) +def test_get_env_kwargs(env_vars: dict, expected: str): + with mock.patch.dict(os.environ, env_vars, clear=True): + result = get_env_kwargs() + assert result == expected + + +class TestGetConnectionKwargs: + @mock.patch("pathlib.Path.is_dir", side_effect=[True, True]) + @mock.patch("pathlib.Path.is_file", return_value=False) + def test_get_connection_kwargs_invalid_connections_file_path(self, _, __): + with pytest.raises(Exception) as e_info: + get_connection_kwargs( + connections_file_path=Path("invalid_connections_file_path"), + connection_name="invalid_connection_name", + ) + + e_info_value = str(e_info.value) + assert "invalid file path: invalid_connections_file_path" in e_info_value + + @mock.patch("pathlib.Path.is_dir", return_value=True) + def test_get_connection_kwargs_no_connection_name(self, _): + connection_kwargs = get_connection_kwargs( + connections_file_path=assets_path / "connections.toml", + connection_name=None, + ) + assert connection_kwargs == {} + + @mock.patch("pathlib.Path.is_dir", side_effect=[True, True]) + def test_get_connection_kwargs_invalid_connection_name(self, _): + with pytest.raises(Exception) as e_info: + get_connection_kwargs( + connections_file_path=assets_path / "connections.toml", + connection_name="invalid_connection_name", + ) + e_info_value = str(e_info.value) + assert "Invalid connection_name 'invalid_connection_name'" in e_info_value + + @mock.patch("pathlib.Path.is_dir", return_value=True) + def test_get_connection_kwargs_happy_path(self, _): + connection_kwargs = get_connection_kwargs( + connections_file_path=assets_path / "connections.toml", + connection_name="myconnection", + ) + assert connection_kwargs == { + "snowflake_account": "connections.toml-account", + "snowflake_authenticator": "connections.toml-authenticator", + "snowflake_database": "connections.toml-database", + "snowflake_token_path": "connections.toml-token-file-path", + "snowflake_password": "connections.toml-password", + "snowflake_private_key_path": "connections.toml-private-key", + "snowflake_role": "connections.toml-role", + "snowflake_schema": "connections.toml-schema", + "snowflake_user": "connections.toml-user", + "snowflake_warehouse": "connections.toml-warehouse", + } diff --git a/tests/session/test_Credential.py b/tests/session/test_Credential.py deleted file mode 100644 index 6239645..0000000 --- a/tests/session/test_Credential.py +++ /dev/null @@ -1,109 +0,0 @@ -from __future__ import annotations - -import json -import os -from unittest import mock -from unittest.mock import MagicMock - -import pytest -import structlog - -from schemachange.session.Credential import ( - credential_factory, - PasswordCredential, - ExternalBrowserCredential, - OktaCredential, - PrivateKeyCredential, - OauthCredential, -) - - -# noinspection PyTypeChecker -@pytest.mark.parametrize( - "env_vars, oauth_config, expected", - [ - ( - {"SNOWFLAKE_PASSWORD": "my-password"}, - None, - PasswordCredential(password="my-password"), - ), - ( - { - "SNOWFLAKE_PASSWORD": "my-password", - "SNOWFLAKE_AUTHENTICATOR": "snowflake", - }, - None, - PasswordCredential(password="my-password"), - ), - ( - { - "SNOWFLAKE_AUTHENTICATOR": "oauth", - }, - { - "token-provider-url": "token-provider-url-from-yaml", - "token-response-name": "token-response-name-from-yaml", - "token-request-headers": { - "Content-Type": "Content-Type-from-yaml", - "User-Agent": "User-Agent-from-yaml", - }, - "token-request-payload": { - "client_id": "id-from-yaml", - "username": "username-from-yaml", - "password": "password-from-yaml", - "grant_type": "type-from-yaml", - "scope": "scope-from-yaml", - }, - }, - OauthCredential(token="my-token"), - ), - ( - { - "SNOWFLAKE_AUTHENTICATOR": "externalbrowser", - }, - None, - ExternalBrowserCredential(), - ), - ( - { - "SNOWFLAKE_AUTHENTICATOR": "https://someurl.com", - "SNOWFLAKE_PASSWORD": "my-password", - }, - None, - OktaCredential(authenticator="https://someurl.com", password="my-password"), - ), - ( - { - "SNOWFLAKE_PRIVATE_KEY_PATH": "some_path", - "SNOWFLAKE_AUTHENTICATOR": "snowflake", - }, - None, - PrivateKeyCredential(private_key="some_path"), - ), - ], -) -@mock.patch( - "schemachange.session.Credential.get_private_key_bytes", - return_value="some_path", -) -@mock.patch("requests.post") -def test_credential_factory( - mock_post, _, env_vars: dict, oauth_config: dict | None, expected: str -): - mock_response = MagicMock() - mock_response.text = json.dumps({"token-response-name-from-yaml": "my-token"}) - mock_post.return_value = mock_response - logger = structlog.testing.CapturingLogger() - - with mock.patch.dict(os.environ, env_vars, clear=True): - # noinspection PyTypeChecker - result = credential_factory(oauth_config=oauth_config, logger=logger) - assert result == expected - - -@pytest.mark.parametrize("env_vars", [{}]) -def test_credential_factory_unhandled(env_vars): - logger = structlog.testing.CapturingLogger() - with pytest.raises(NameError): - with mock.patch.dict(os.environ, env_vars, clear=True): - # noinspection PyTypeChecker - credential_factory(logger=logger) diff --git a/tests/session/test_utils.py b/tests/session/test_utils.py deleted file mode 100644 index a3077b5..0000000 --- a/tests/session/test_utils.py +++ /dev/null @@ -1,43 +0,0 @@ -from __future__ import annotations - -import os -from unittest import mock - -import pytest - -from schemachange.session.utils import ( - get_snowflake_password, - get_private_key_password, -) - - -@pytest.mark.parametrize( - "env_vars, expected", - [ - ({"SNOWFLAKE_PASSWORD": "my-password"}, "my-password"), - ({"SNOWFLAKE_PASSWORD": ""}, None), - ({}, None), - ({"SNOWSQL_PWD": "my-password"}, "my-password"), - ( - {"SNOWSQL_PWD": "my-password", "SNOWFLAKE_PASSWORD": "my-password"}, - "my-password", - ), - ], -) -def test_get_snowflake_password(env_vars: dict, expected: str): - with mock.patch.dict(os.environ, env_vars, clear=True): - result = get_snowflake_password() - assert result == expected - - -@pytest.mark.parametrize( - "env_vars, expected", - [ - ({"SNOWFLAKE_PRIVATE_KEY_PASSPHRASE": "my-passphrase"}, b"my-passphrase"), - ({}, None), - ], -) -def test_get_private_key_password(env_vars: dict, expected: str): - with mock.patch.dict(os.environ, env_vars, clear=True): - result = get_private_key_password() - assert result == expected From f2635a2713fe7d4e65936760a2f780a59c374251 Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Thu, 24 Oct 2024 09:41:42 -0600 Subject: [PATCH 06/34] feat: remove get_session_from_config, opt for config.get_session_kwargs --- schemachange/cli.py | 8 +- schemachange/config/DeployConfig.py | 24 +- schemachange/config/get_merged_config.py | 17 +- schemachange/config/utils.py | 6 +- schemachange/session/SnowflakeSession.py | 86 ++-- tests/config/__init__.py | 0 tests/config/alt_private_key.txt | 1 + tests/config/oauth_token_path.txt | 1 + tests/config/private_key.txt | 1 + tests/config/test_get_merged_config.py | 22 +- tests/session/test_SnowflakeSession.py | 11 +- tests/test_main.py | 611 +++++++++++++++++------ 12 files changed, 540 insertions(+), 248 deletions(-) create mode 100644 tests/config/__init__.py create mode 100644 tests/config/alt_private_key.txt create mode 100644 tests/config/oauth_token_path.txt create mode 100644 tests/config/private_key.txt diff --git a/schemachange/cli.py b/schemachange/cli.py index d6fe54f..895fc81 100644 --- a/schemachange/cli.py +++ b/schemachange/cli.py @@ -9,7 +9,7 @@ from schemachange.config.get_merged_config import get_merged_config from schemachange.deploy import deploy from schemachange.redact_config_secrets import redact_config_secrets -from schemachange.session.SnowflakeSession import get_session_from_config +from schemachange.session.SnowflakeSession import SnowflakeSession # region Global Variables # metadata @@ -63,11 +63,11 @@ def main(): ) else: config.check_for_deploy_args() - session = get_session_from_config( - config=config, + session = SnowflakeSession( schemachange_version=SCHEMACHANGE_VERSION, - snowflake_application_name=SNOWFLAKE_APPLICATION_NAME, + application=SNOWFLAKE_APPLICATION_NAME, logger=logger, + **config.get_session_kwargs(), ) deploy(config=config, session=session) diff --git a/schemachange/config/DeployConfig.py b/schemachange/config/DeployConfig.py index 59e44d8..055ad32 100644 --- a/schemachange/config/DeployConfig.py +++ b/schemachange/config/DeployConfig.py @@ -70,10 +70,10 @@ def factory( # If set by an environment variable, pop snowflake_token_path from kwargs if "snowflake_oauth_token" in kwargs: kwargs.pop("snowflake_token_path", None) - kwargs.pop("oauthconfig", None) + kwargs.pop("oauth_config", None) # Load it from a file, if provided elif "snowflake_token_path" in kwargs: - kwargs.pop("oauthconfig", None) + kwargs.pop("oauth_config", None) oauth_token_path = kwargs.pop("snowflake_token_path") with open(oauth_token_path) as f: kwargs["snowflake_oauth_token"] = f.read() @@ -140,3 +140,23 @@ def check_for_deploy_args(self) -> None: raise ValueError( f"Missing config values. The following config values are required: {missing_args}" ) + + def get_session_kwargs(self) -> dict: + session_kwargs = { + "account": self.snowflake_account, + "user": self.snowflake_user, + "role": self.snowflake_role, + "warehouse": self.snowflake_warehouse, + "database": self.snowflake_database, + "schema": self.snowflake_schema, + "authenticator": self.snowflake_authenticator, + "password": self.snowflake_password, + "oauth_token": self.snowflake_oauth_token, + "private_key_path": self.snowflake_private_key_path, + "connections_file_path": self.connections_file_path, + "connection_name": self.connection_name, + "change_history_table": self.change_history_table, + "autocommit": self.autocommit, + "query_tag": self.query_tag, + } + return {k: v for k, v in session_kwargs.items() if v is not None} diff --git a/schemachange/config/get_merged_config.py b/schemachange/config/get_merged_config.py index 8f1c6c7..a75a9e4 100644 --- a/schemachange/config/get_merged_config.py +++ b/schemachange/config/get_merged_config.py @@ -11,6 +11,7 @@ validate_directory, get_env_kwargs, get_connection_kwargs, + validate_file_path, ) @@ -41,8 +42,10 @@ def get_merged_config() -> Union[DeployConfig, RenderConfig]: cli_config_vars = cli_kwargs.pop("config_vars") - connections_file_path = cli_kwargs.get("connections_file_path") - connection_name = cli_kwargs.get("connection_name") + connections_file_path = validate_file_path( + file_path=cli_kwargs.pop("connections_file_path", None) + ) + connection_name = cli_kwargs.pop("connection_name", None) config_folder = validate_directory(path=cli_kwargs.pop("config_folder", ".")) config_file_name = cli_kwargs.pop("config_file_name") config_file_path = Path(config_folder) / config_file_name @@ -55,12 +58,14 @@ def get_merged_config() -> Union[DeployConfig, RenderConfig]: yaml_config_vars = {} if connections_file_path is None: - connections_file_path = yaml_kwargs.get("connections_file_path") + connections_file_path = yaml_kwargs.pop("connections_file_path", None) if config_folder is not None and connections_file_path is not None: connections_file_path = config_folder / connections_file_path + connections_file_path = validate_file_path(file_path=connections_file_path) + if connection_name is None: - connection_name = yaml_kwargs.get("connection_name") + connection_name = yaml_kwargs.pop("connection_name", None) connection_kwargs: dict[str, str] = get_connection_kwargs( connections_file_path=connections_file_path, @@ -81,6 +86,10 @@ def get_merged_config() -> Union[DeployConfig, RenderConfig]: **{k: v for k, v in cli_kwargs.items() if v is not None}, **{k: v for k, v in env_kwargs.items() if v is not None}, } + if connections_file_path is not None: + kwargs["connections_file_path"] = connections_file_path + if connection_name is not None: + kwargs["connection_name"] = connection_name if cli_kwargs["subcommand"] == "deploy": return DeployConfig.factory(**kwargs) diff --git a/schemachange/config/utils.py b/schemachange/config/utils.py index 23e280c..d7bc42c 100644 --- a/schemachange/config/utils.py +++ b/schemachange/config/utils.py @@ -74,7 +74,9 @@ def inner_extract_dictionary_secrets( return inner_extract_dictionary_secrets(config_vars) -def validate_file_path(file_path: Path | str) -> Path: +def validate_file_path(file_path: Path | str | None) -> Path | None: + if file_path is None: + return None if isinstance(file_path, str): file_path = Path(file_path) if not file_path.is_file(): @@ -84,7 +86,7 @@ def validate_file_path(file_path: Path | str) -> Path: def validate_directory(path: Path | str | None) -> Path | None: if path is None: - return path + return None if isinstance(path, str): path = Path(path) if not path.is_dir(): diff --git a/schemachange/session/SnowflakeSession.py b/schemachange/session/SnowflakeSession.py index f6c88be..08353bb 100644 --- a/schemachange/session/SnowflakeSession.py +++ b/schemachange/session/SnowflakeSession.py @@ -3,26 +3,21 @@ import hashlib import time from collections import defaultdict -from dataclasses import asdict from textwrap import dedent, indent import snowflake.connector import structlog from schemachange.config.ChangeHistoryTable import ChangeHistoryTable -from schemachange.config.DeployConfig import DeployConfig -from schemachange.session.Credential import SomeCredential, credential_factory from schemachange.session.Script import VersionedScript, RepeatableScript, AlwaysScript class SnowflakeSession: - user: str - account: str - role: str - warehouse: str + user: str | None + role: str | None + warehouse: str | None database: str | None schema: str | None - query_tag: str | None autocommit: bool change_history_table: ChangeHistoryTable logger: structlog.BoundLogger @@ -35,28 +30,26 @@ class SnowflakeSession: def __init__( self, - snowflake_user: str, - snowflake_account: str, - snowflake_role: str, - snowflake_warehouse: str, schemachange_version: str, application: str, - credential: SomeCredential, change_history_table: ChangeHistoryTable, logger: structlog.BoundLogger, - autocommit: bool = False, - snowflake_database: str | None = None, - snowflake_schema: str | None = None, + user: str | None = None, + role: str | None = None, + warehouse: str | None = None, + database: str | None = None, + schema: str | None = None, query_tag: str | None = None, + autocommit: bool = False, + **kwargs, ): - self.user = snowflake_user - self.account = snowflake_account - self.role = snowflake_role - self.warehouse = snowflake_warehouse - self.database = snowflake_database - self.schema = snowflake_schema - self.autocommit = autocommit + self.user = user + self.role = role + self.warehouse = warehouse + self.database = database + self.schema = schema self.change_history_table = change_history_table + self.autocommit = autocommit self.logger = logger self.session_parameters = {"QUERY_TAG": f"schemachange {schemachange_version}"} @@ -64,15 +57,22 @@ def __init__( self.session_parameters["QUERY_TAG"] += f";{query_tag}" self.con = snowflake.connector.connect( + account=kwargs["account"], user=self.user, - account=self.account, + database=kwargs.get("database"), + schema=kwargs.get("schema"), role=self.role, warehouse=self.warehouse, - database=self.database, - schema=self.schema, + private_key=kwargs.get("private_key"), + private_key_file=kwargs.get("private_key_path"), + private_key_file_pwd=kwargs.get("private_key_path_password"), + token=kwargs.get("oauth_token"), + password=kwargs.get("password"), + authenticator=kwargs.get("authenticator"), + connection_name=kwargs.get("connection_name"), + connections_file_path=kwargs.get("connections_file_path"), application=application, session_parameters=self.session_parameters, - **asdict(credential), ) print(f"Current session ID: {self.con.session_id}") @@ -210,9 +210,9 @@ def get_script_metadata( self.logger.info( "Max applied change script version %(max_published_version)s" % { - "max_published_version": max_published_version - if max_published_version != "" - else "None" + "max_published_version": ( + max_published_version if max_published_version != "" else "None" + ) } ) return change_history, r_scripts_checksum, max_published_version @@ -261,6 +261,7 @@ def fetch_versioned_scripts( "checksum": checksum, } + # noinspection PyTypeChecker return versioned_scripts, versions[0] if versions else None def reset_session(self, logger: structlog.BoundLogger): @@ -298,6 +299,7 @@ def apply_change_script( return logger.info("Applying change script") # Define a few other change related variables + # noinspection PyTypeChecker checksum = hashlib.sha224(script_content.encode("utf-8")).hexdigest() execution_time = 0 status = "Success" @@ -341,27 +343,3 @@ def apply_change_script( ); """ self.execute_snowflake_query(dedent(query), logger=logger) - - -def get_session_from_config( - config: DeployConfig, - logger: structlog.BoundLogger, - schemachange_version: str, - snowflake_application_name: str, -) -> SnowflakeSession: - credential = credential_factory(logger=logger, oauth_config=config.oauth_config) - return SnowflakeSession( - snowflake_user=config.snowflake_user, - snowflake_account=config.snowflake_account, - snowflake_role=config.snowflake_role, - snowflake_warehouse=config.snowflake_warehouse, - schemachange_version=schemachange_version, - application=snowflake_application_name, - credential=credential, - change_history_table=config.change_history_table, - logger=logger, - autocommit=config.autocommit, - snowflake_database=config.snowflake_database, - snowflake_schema=config.snowflake_schema, - query_tag=config.query_tag, - ) diff --git a/tests/config/__init__.py b/tests/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/config/alt_private_key.txt b/tests/config/alt_private_key.txt new file mode 100644 index 0000000..c856730 --- /dev/null +++ b/tests/config/alt_private_key.txt @@ -0,0 +1 @@ +my-alt-private-key diff --git a/tests/config/oauth_token_path.txt b/tests/config/oauth_token_path.txt new file mode 100644 index 0000000..49dbda6 --- /dev/null +++ b/tests/config/oauth_token_path.txt @@ -0,0 +1 @@ +my-oauth-token diff --git a/tests/config/private_key.txt b/tests/config/private_key.txt new file mode 100644 index 0000000..4764fdd --- /dev/null +++ b/tests/config/private_key.txt @@ -0,0 +1 @@ +my-private-key diff --git a/tests/config/test_get_merged_config.py b/tests/config/test_get_merged_config.py index b981874..33b1cb4 100644 --- a/tests/config/test_get_merged_config.py +++ b/tests/config/test_get_merged_config.py @@ -172,7 +172,7 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "snowflake_password": "connection_snowflake_password", "snowflake_private_key_path": "yaml_snowflake_private_key_path", "snowflake_token_path": "yaml_snowflake_token_path", - "connections_file_path": "yaml_connections_file_path", + "connections_file_path": Path("yaml_connections_file_path"), "connection_name": "yaml_connection_name", "change_history_table": "yaml_change_history_table", "create_change_history_table": True, @@ -277,7 +277,7 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "snowflake_password": "connection_snowflake_password", "snowflake_private_key_path": "cli_snowflake_private_key_path", "snowflake_token_path": "cli_snowflake_token_path", - "connections_file_path": "cli_connections_file_path", + "connections_file_path": Path("cli_connections_file_path"), "connection_name": "cli_connection_name", "change_history_table": "cli_change_history_table", "create_change_history_table": False, @@ -382,7 +382,7 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "snowflake_password": "env_snowflake_password", "snowflake_private_key_path": "env_snowflake_private_key_path", "snowflake_token_path": "cli_snowflake_token_path", - "connections_file_path": "cli_connections_file_path", + "connections_file_path": Path("cli_connections_file_path"), "connection_name": "cli_connection_name", "change_history_table": "cli_change_history_table", "create_change_history_table": False, @@ -396,6 +396,7 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: ], ) @mock.patch("pathlib.Path.is_dir", return_value=True) +@mock.patch("pathlib.Path.is_file", return_value=True) @mock.patch("schemachange.config.get_merged_config.get_env_kwargs") @mock.patch("schemachange.config.get_merged_config.parse_cli_args") @mock.patch("schemachange.config.get_merged_config.get_yaml_config_kwargs") @@ -408,6 +409,7 @@ def test_get_merged_config_inheritance( mock_parse_cli_args, mock_get_env_kwargs, _, + __, env_kwargs, cli_kwargs, yaml_kwargs, @@ -523,7 +525,7 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "query_tag": "query-tag-from-cli", "oauth_config": {"oauth_config_variable": "cli_oauth_config_value"}, "connection_name": "myaltconnection", - "connections_file_path": str(assets_path / "alt-connections.toml"), + "connections_file_path": assets_path / "alt-connections.toml", "snowflake_password": alt_connection["password"], }, id="Deploy: full cli and connections.toml", @@ -586,6 +588,7 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "config_file_path": assets_path / "schemachange-config-full.yml", "snowflake_password": my_connection["password"], "log_level": logging.INFO, + "connections_file_path": assets_path / "connections.toml", **{ k: v for k, v in schemachange_config_full.items() @@ -612,7 +615,6 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "query_tag", "oauth_config", "connection_name", - "connections_file_path", ] }, }, @@ -693,7 +695,7 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "query_tag": "query-tag-from-cli", "oauth_config": {"oauth_config_variable": "cli_oauth_config_value"}, "connection_name": "myaltconnection", - "connections_file_path": str(assets_path / "alt-connections.toml"), + "connections_file_path": assets_path / "alt-connections.toml", "snowflake_password": alt_connection["password"], }, id="Deploy: full yaml, connections.toml, and cli", @@ -704,6 +706,7 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "SNOWFLAKE_PASSWORD": "env_snowflake_password", "SNOWFLAKE_PRIVATE_KEY_PATH": "env_snowflake_private_key_path", "SNOWFLAKE_AUTHENTICATOR": "env_snowflake_authenticator", + "SNOWFLAKE_TOKEN": "env_snowflake_token", }, # env_kwargs [ # cli_args "schemachange", @@ -775,9 +778,10 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "log_level": logging.INFO, "dry_run": True, "query_tag": "query-tag-from-cli", + "snowflake_oauth_token": "env_snowflake_token", "oauth_config": {"oauth_config_variable": "cli_oauth_config_value"}, "connection_name": "myaltconnection", - "connections_file_path": str(assets_path / "alt-connections.toml"), + "connections_file_path": assets_path / "alt-connections.toml", "snowflake_password": "env_snowflake_password", }, id="Deploy: full yaml, connections.toml, cli, and env", @@ -797,7 +801,7 @@ def test_invalid_config_folder(mock_parse_cli_args, _): ], { # expected "subcommand": "deploy", - "connections_file_path": str(assets_path / "connections.toml"), + "connections_file_path": assets_path / "connections.toml", "connection_name": "myconnection", "config_file_path": assets_path / "schemachange-config.yml", "config_version": 1, @@ -841,6 +845,7 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "snowflake_token_path": my_connection["token-file-path"], "log_level": logging.INFO, "snowflake_password": my_connection["password"], + "connections_file_path": assets_path / "connections.toml", **{ k: v for k, v in schemachange_config_partial_with_connection.items() @@ -857,7 +862,6 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "query_tag", "oauth_config", "connection_name", - "connections_file_path", ] }, }, diff --git a/tests/session/test_SnowflakeSession.py b/tests/session/test_SnowflakeSession.py index 68f308a..96b39ac 100644 --- a/tests/session/test_SnowflakeSession.py +++ b/tests/session/test_SnowflakeSession.py @@ -6,26 +6,23 @@ import structlog from schemachange.config.ChangeHistoryTable import ChangeHistoryTable -from schemachange.session.Credential import ExternalBrowserCredential from schemachange.session.SnowflakeSession import SnowflakeSession @pytest.fixture def session() -> SnowflakeSession: - credential = ExternalBrowserCredential(password="password") change_history_table = ChangeHistoryTable() logger = structlog.testing.CapturingLogger() with mock.patch("snowflake.connector.connect"): # noinspection PyTypeChecker return SnowflakeSession( - snowflake_user="user", - snowflake_account="account", - snowflake_role="role", - snowflake_warehouse="warehouse", + user="user", + account="account", + role="role", + warehouse="warehouse", schemachange_version="3.6.1.dev", application="schemachange", - credential=credential, change_history_table=change_history_table, logger=logger, ) diff --git a/tests/test_main.py b/tests/test_main.py index 384c404..7393c1d 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,7 +1,9 @@ from __future__ import annotations +import json import logging import os +import tomllib import tempfile import unittest.mock as mock from dataclasses import asdict @@ -12,7 +14,21 @@ from schemachange.config.ChangeHistoryTable import ChangeHistoryTable import schemachange.cli as cli +from schemachange.config.utils import get_snowflake_identifier_string +assets_path = Path(__file__).parent / "config" + + +def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: + with file_path.open("rb") as f: + connections = tomllib.load(f) + return connections[connection_name] + + +alt_connection = get_connection_from_toml( + file_path=assets_path / "alt-connections.toml", + connection_name="myaltconnection", +) default_base_config = { # Shared configuration options "config_file_path": Path(".") / "schemachange-config.yml", @@ -30,6 +46,12 @@ "snowflake_warehouse": None, "snowflake_database": None, "snowflake_schema": None, + "snowflake_authenticator": "snowflake", + "snowflake_password": None, + "snowflake_oauth_token": None, + "snowflake_private_key_path": None, + "connections_file_path": None, + "connection_name": None, "change_history_table": ChangeHistoryTable( table_name="CHANGE_HISTORY", schema_name="SCHEMACHANGE", @@ -39,7 +61,6 @@ "autocommit": False, "dry_run": False, "query_tag": None, - "oauth_config": None, } required_args = [ @@ -58,201 +79,458 @@ "snowflake_user": "user", "snowflake_warehouse": "warehouse", "snowflake_role": "role", + "snowflake_password": "password", } script_path = Path(__file__).parent.parent / "demo" / "basics_demo" / "A__basic001.sql" +no_command = pytest.param( + "schemachange.cli.deploy", + {"SNOWFLAKE_PASSWORD": "password"}, + ["schemachange", *required_args], + {**default_deploy_config, **required_config}, + None, + id="no command", +) -@pytest.mark.parametrize( - "to_mock, cli_args, expected_config, expected_script_path", +deploy_only_required = pytest.param( + "schemachange.cli.deploy", + {"SNOWFLAKE_PASSWORD": "password"}, + ["schemachange", "deploy", *required_args], + {**default_deploy_config, **required_config}, + None, + id="deploy: only required", +) + +deploy_all_cli_arg_names = pytest.param( + "schemachange.cli.deploy", + {}, [ - ( - "schemachange.cli.deploy", - ["schemachange", *required_args], - {**default_deploy_config, **required_config}, - None, - ), - ( - "schemachange.cli.deploy", - ["schemachange", "deploy", *required_args], - {**default_deploy_config, **required_config}, - None, - ), - ( - "schemachange.cli.deploy", - ["schemachange", "deploy", "-f", ".", *required_args], - {**default_deploy_config, **required_config, "root_folder": Path(".")}, - None, - ), - ( - "schemachange.cli.deploy", - [ - "schemachange", - "deploy", - *required_args, - "--snowflake-database", - "database", - ], - { - **default_deploy_config, - **required_config, - "snowflake_database": "database", - }, - None, + "schemachange", + "deploy", + "--config-folder", + str(assets_path), + "--config-file-name", + "schemachange-config.yml", + "--root-folder", + str(assets_path), + "--modules-folder", + str(assets_path), + "--vars", + '{"var1": "from_cli", "var3": "also_from_cli"}', + "--verbose", + "--snowflake-account", + "snowflake-account-from-cli", + "--snowflake-user", + "snowflake-user-from-cli", + "--snowflake-role", + "snowflake-role-from-cli", + "--snowflake-warehouse", + "snowflake-warehouse-from-cli", + "--snowflake-database", + "snowflake-database-from-cli", + "--snowflake-schema", + "snowflake-schema-from-cli", + "--snowflake-authenticator", + "externalbrowser", + "--snowflake-private-key-path", + str(assets_path / "private_key.txt"), + "--snowflake-token-path", + str(assets_path / "oauth_token_path.txt"), + "--connections-file-path", + str(assets_path / "alt-connections.toml"), + "--connection-name", + "myaltconnection", + "--change-history-table", + "db.schema.table_from_cli", + "--create-change-history-table", + "--autocommit", + "--dry-run", + "--query-tag", + "query-tag-from-cli", + "--oauth-config", + json.dumps({"oauth_config_variable": "cli_oauth_config_value"}), + ], + { # expected + "subcommand": "deploy", + "config_file_path": assets_path / "schemachange-config.yml", + "config_version": 1, + "root_folder": assets_path, + "modules_folder": assets_path, + "snowflake_account": "snowflake-account-from-cli", + "snowflake_user": "snowflake-user-from-cli", + "snowflake_role": get_snowflake_identifier_string( + "snowflake-role-from-cli", "placeholder" ), - ( - "schemachange.cli.deploy", - ["schemachange", "deploy", *required_args, "--snowflake-schema", "schema"], - {**default_deploy_config, **required_config, "snowflake_schema": "schema"}, - None, + "snowflake_warehouse": get_snowflake_identifier_string( + "snowflake-warehouse-from-cli", "placeholder" ), - ( - "schemachange.cli.deploy", - [ - "schemachange", - "deploy", - *required_args, - "--change-history-table", - "db.schema.table", - ], - { - **default_deploy_config, - **required_config, - "change_history_table": ChangeHistoryTable( - database_name="db", schema_name="schema", table_name="table" - ), - }, - None, + "snowflake_database": get_snowflake_identifier_string( + "snowflake-database-from-cli", "placeholder" ), - ( - "schemachange.cli.deploy", - ["schemachange", "deploy", *required_args, "--vars", '{"var1": "val"}'], - { - **default_deploy_config, - **required_config, - "config_vars": {"var1": "val"}, - }, - None, + "snowflake_schema": get_snowflake_identifier_string( + "snowflake-schema-from-cli", "placeholder" ), - ( - "schemachange.cli.deploy", - ["schemachange", "deploy", *required_args, "--create-change-history-table"], - { - **default_deploy_config, - **required_config, - "create_change_history_table": True, - }, - None, + "snowflake_authenticator": "externalbrowser", + "snowflake_private_key_path": assets_path / "private_key.txt", + "change_history_table": ChangeHistoryTable( + database_name="db", + schema_name="schema", + table_name="table_from_cli", ), - ( - "schemachange.cli.deploy", - ["schemachange", "deploy", *required_args, "--autocommit"], - {**default_deploy_config, **required_config, "autocommit": True}, - None, + "config_vars": { + "var1": "from_cli", + "var3": "also_from_cli", + }, + "create_change_history_table": True, + "autocommit": True, + "log_level": logging.DEBUG, + "dry_run": True, + "query_tag": "query-tag-from-cli", + "connection_name": "myaltconnection", + "connections_file_path": assets_path / "alt-connections.toml", + "snowflake_password": alt_connection["password"], + }, + None, + id="deploy: all cli argument names", +) + +deploy_all_cli_arg_flags = pytest.param( + "schemachange.cli.deploy", + {}, + [ + "schemachange", + "deploy", + "--config-folder", + str(assets_path), + "--config-file-name", + "schemachange-config.yml", + "-f", + str(assets_path), + "-m", + str(assets_path), + "--vars", + '{"var1": "from_cli", "var3": "also_from_cli"}', + "-v", + "-a", + "snowflake-account-from-cli", + "-u", + "snowflake-user-from-cli", + "-r", + "snowflake-role-from-cli", + "-w", + "snowflake-warehouse-from-cli", + "-d", + "snowflake-database-from-cli", + "-s", + "snowflake-schema-from-cli", + "-A", + "externalbrowser", + "-k", + str(assets_path / "private_key.txt"), + "-t", + str(assets_path / "oauth_token_path.txt"), + "--connections-file-path", + str(assets_path / "alt-connections.toml"), + "--connection-name", + "myaltconnection", + "-c", + "db.schema.table_from_cli", + "--create-change-history-table", + "-ac", + "--dry-run", + "--query-tag", + "query-tag-from-cli", + "--oauth-config", + json.dumps({"oauth_config_variable": "cli_oauth_config_value"}), + ], + { # expected + "subcommand": "deploy", + "config_file_path": assets_path / "schemachange-config.yml", + "config_version": 1, + "root_folder": assets_path, + "modules_folder": assets_path, + "snowflake_account": "snowflake-account-from-cli", + "snowflake_user": "snowflake-user-from-cli", + "snowflake_role": get_snowflake_identifier_string( + "snowflake-role-from-cli", "placeholder" ), - ( - "schemachange.cli.deploy", - ["schemachange", "deploy", *required_args, "--verbose"], - {**default_deploy_config, **required_config, "log_level": logging.DEBUG}, - None, + "snowflake_warehouse": get_snowflake_identifier_string( + "snowflake-warehouse-from-cli", "placeholder" ), - ( - "schemachange.cli.deploy", - ["schemachange", "deploy", *required_args, "--dry-run"], - {**default_deploy_config, **required_config, "dry_run": True}, - None, + "snowflake_database": get_snowflake_identifier_string( + "snowflake-database-from-cli", "placeholder" ), - ( - "schemachange.cli.deploy", - ["schemachange", "deploy", *required_args, "--query-tag", "querytag"], - {**default_deploy_config, **required_config, "query_tag": "querytag"}, - None, + "snowflake_schema": get_snowflake_identifier_string( + "snowflake-schema-from-cli", "placeholder" ), - ( - "schemachange.cli.deploy", - [ - "schemachange", - "deploy", - *required_args, - "--oauth-config", - '{"token-provider-url": "https//..."}', - ], - { - **default_deploy_config, - **required_config, - "oauth_config": {"token-provider-url": "https//..."}, - }, - None, + "snowflake_authenticator": "externalbrowser", + "snowflake_private_key_path": assets_path / "private_key.txt", + "change_history_table": ChangeHistoryTable( + database_name="db", + schema_name="schema", + table_name="table_from_cli", ), - ( - "schemachange.cli.deploy", - [ - "schemachange", - "deploy", - *required_args, - ], - { - **default_deploy_config, - **required_config, - "log_level": 20, - }, - None, + "config_vars": { + "var1": "from_cli", + "var3": "also_from_cli", + }, + "create_change_history_table": True, + "autocommit": True, + "log_level": logging.DEBUG, + "dry_run": True, + "query_tag": "query-tag-from-cli", + "connection_name": "myaltconnection", + "connections_file_path": assets_path / "alt-connections.toml", + "snowflake_password": alt_connection["password"], + }, + None, + id="deploy: all cli argument flags", +) + +deploy_all_env_all_cli = pytest.param( + "schemachange.cli.deploy", + { + "SNOWFLAKE_PASSWORD": "env_snowflake_password", + "SNOWFLAKE_PRIVATE_KEY_PATH": str(assets_path / "alt_private_key.txt"), + "SNOWFLAKE_AUTHENTICATOR": "snowflake_jwt", + "SNOWFLAKE_TOKEN": "env_snowflake_oauth_token", + }, + [ + "schemachange", + "deploy", + "--config-folder", + str(assets_path), + "--config-file-name", + "schemachange-config.yml", + "--root-folder", + str(assets_path), + "--modules-folder", + str(assets_path), + "--vars", + '{"var1": "from_cli", "var3": "also_from_cli"}', + "--verbose", + "--snowflake-account", + "snowflake-account-from-cli", + "--snowflake-user", + "snowflake-user-from-cli", + "--snowflake-role", + "snowflake-role-from-cli", + "--snowflake-warehouse", + "snowflake-warehouse-from-cli", + "--snowflake-database", + "snowflake-database-from-cli", + "--snowflake-schema", + "snowflake-schema-from-cli", + "--snowflake-authenticator", + "externalbrowser", + "--snowflake-private-key-path", + str(assets_path / "private_key.txt"), + "--snowflake-token-path", + str(assets_path / "oauth_token_path.txt"), + "--connections-file-path", + str(assets_path / "alt-connections.toml"), + "--connection-name", + "myaltconnection", + "--change-history-table", + "db.schema.table_from_cli", + "--create-change-history-table", + "--autocommit", + "--dry-run", + "--query-tag", + "query-tag-from-cli", + "--oauth-config", + json.dumps({"oauth_config_variable": "cli_oauth_config_value"}), + ], + { # expected + "subcommand": "deploy", + "config_file_path": assets_path / "schemachange-config.yml", + "config_version": 1, + "root_folder": assets_path, + "modules_folder": assets_path, + "snowflake_account": "snowflake-account-from-cli", + "snowflake_user": "snowflake-user-from-cli", + "snowflake_role": get_snowflake_identifier_string( + "snowflake-role-from-cli", "placeholder" ), - ( - "schemachange.cli.render", - [ - "schemachange", - "render", - str(script_path), - ], - {**default_base_config}, - script_path, + "snowflake_warehouse": get_snowflake_identifier_string( + "snowflake-warehouse-from-cli", "placeholder" ), - ( - "schemachange.cli.render", - [ - "schemachange", - "render", - "--root-folder", - ".", - str(script_path), - ], - {**default_base_config, "root_folder": Path(".")}, - script_path, + "snowflake_database": get_snowflake_identifier_string( + "snowflake-database-from-cli", "placeholder" ), - ( - "schemachange.cli.render", - [ - "schemachange", - "render", - "--vars", - '{"var1": "val"}', - str(script_path), - ], - {**default_base_config, "config_vars": {"var1": "val"}}, - script_path, + "snowflake_schema": get_snowflake_identifier_string( + "snowflake-schema-from-cli", "placeholder" ), - ( - "schemachange.cli.render", - [ - "schemachange", - "render", - "--verbose", - str(script_path), - ], - {**default_base_config, "log_level": logging.DEBUG}, - script_path, + "snowflake_authenticator": "snowflake_jwt", + "snowflake_private_key_path": assets_path / "alt_private_key.txt", + "change_history_table": ChangeHistoryTable( + database_name="db", + schema_name="schema", + table_name="table_from_cli", ), + "config_vars": { + "var1": "from_cli", + "var3": "also_from_cli", + }, + "create_change_history_table": True, + "autocommit": True, + "log_level": logging.DEBUG, + "dry_run": True, + "query_tag": "query-tag-from-cli", + "connection_name": "myaltconnection", + "connections_file_path": assets_path / "alt-connections.toml", + "snowflake_password": "env_snowflake_password", + }, + None, + id="deploy: all env_vars and all cli argument names", +) + +deploy_snowflake_oauth_env_var = pytest.param( + "schemachange.cli.deploy", + {"SNOWFLAKE_TOKEN": "env_snowflake_oauth_token"}, + [ + "schemachange", + "deploy", + *required_args, + "--snowflake-authenticator", + "oauth", + "--snowflake-token-path", + str(assets_path / "oauth_token_path.txt"), + "--oauth-config", + json.dumps({"oauth_config_variable": "cli_oauth_config_value"}), ], + { + **default_deploy_config, + "snowflake_account": "account", + "snowflake_user": "user", + "snowflake_warehouse": "warehouse", + "snowflake_role": "role", + "snowflake_authenticator": "oauth", + "snowflake_oauth_token": "env_snowflake_oauth_token", + }, + None, + id="deploy: oauth env var", +) + +deploy_snowflake_oauth_file = pytest.param( + "schemachange.cli.deploy", + {}, + [ + "schemachange", + "deploy", + *required_args, + "--snowflake-authenticator", + "oauth", + "--snowflake-token-path", + str(assets_path / "oauth_token_path.txt"), + "--oauth-config", + json.dumps({"oauth_config_variable": "cli_oauth_config_value"}), + ], + { + **default_deploy_config, + "snowflake_account": "account", + "snowflake_user": "user", + "snowflake_warehouse": "warehouse", + "snowflake_role": "role", + "snowflake_authenticator": "oauth", + "snowflake_oauth_token": "my-oauth-token", + }, + None, + id="deploy: oauth file", +) + +deploy_snowflake_oauth_request = pytest.param( + "schemachange.cli.deploy", + {}, + [ + "schemachange", + "deploy", + *required_args, + "--snowflake-authenticator", + "oauth", + "--oauth-config", + json.dumps({"oauth_config_variable": "cli_oauth_config_value"}), + ], + { + **default_deploy_config, + "snowflake_account": "account", + "snowflake_user": "user", + "snowflake_warehouse": "warehouse", + "snowflake_role": "role", + "snowflake_authenticator": "oauth", + "snowflake_oauth_token": "requested_oauth_token", + }, + None, + id="deploy: oauth request", +) + +render_only_required = pytest.param( + "schemachange.cli.render", + {}, + [ + "schemachange", + "render", + str(script_path), + ], + {**default_base_config}, + script_path, + id="render: only required", +) + +render_all_cli_arg_names = pytest.param( + "schemachange.cli.render", + {}, + [ + "schemachange", + "render", + "--root-folder", + ".", + "--vars", + '{"var1": "val"}', + "--verbose", + str(script_path), + ], + { + **default_base_config, + "root_folder": Path("."), + "config_vars": {"var1": "val"}, + "log_level": logging.DEBUG, + }, + script_path, + id="render: all cli argument names", +) + + +@pytest.mark.parametrize( + "to_mock, env_vars, cli_args, expected_config, expected_script_path", + [ + no_command, + deploy_only_required, + deploy_all_cli_arg_names, + deploy_all_cli_arg_flags, + deploy_all_env_all_cli, + deploy_snowflake_oauth_env_var, + deploy_snowflake_oauth_file, + deploy_snowflake_oauth_request, + render_only_required, + render_all_cli_arg_names, + ], +) +@mock.patch( + "schemachange.config.DeployConfig.get_oauth_token", + return_value="requested_oauth_token", ) @mock.patch("schemachange.session.SnowflakeSession.snowflake.connector.connect") def test_main_deploy_subcommand_given_arguments_make_sure_arguments_set_on_call( _, + __, to_mock: str, + env_vars: dict[str, str], cli_args: list[str], expected_config: dict, expected_script_path: Path | None, ): - with mock.patch.dict(os.environ, {"SNOWFLAKE_PASSWORD": "password"}, clear=True): + with mock.patch.dict(os.environ, env_vars, clear=True): with mock.patch("sys.argv", cli_args): with mock.patch(to_mock) as mock_command: cli.main() @@ -285,6 +563,7 @@ def test_main_deploy_subcommand_given_arguments_make_sure_arguments_set_on_call( "snowflake_warehouse": "warehouse", "snowflake_role": "role", "snowflake_account": "account", + "snowflake_password": "password", }, None, ), From 7dda5ded6edfb0ccac81eeb3c65bab0559e7402b Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Thu, 24 Oct 2024 10:40:53 -0600 Subject: [PATCH 07/34] fix: fix verbose handling in get_yaml_config_kwargs --- schemachange/config/DeployConfig.py | 1 - schemachange/config/get_merged_config.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/schemachange/config/DeployConfig.py b/schemachange/config/DeployConfig.py index 055ad32..e0bfbba 100644 --- a/schemachange/config/DeployConfig.py +++ b/schemachange/config/DeployConfig.py @@ -108,7 +108,6 @@ def check_for_deploy_args(self) -> None: # OAuth based authentication if self.snowflake_authenticator.lower() == "oauth": - # TODO: defer to an existing token or fetch one here? req_args["snowflake_oauth_token"] = self.snowflake_oauth_token # External Browser based SSO diff --git a/schemachange/config/get_merged_config.py b/schemachange/config/get_merged_config.py index a75a9e4..8181465 100644 --- a/schemachange/config/get_merged_config.py +++ b/schemachange/config/get_merged_config.py @@ -26,7 +26,8 @@ def get_yaml_config_kwargs(config_file_path: Optional[Path]) -> dict: } if "verbose" in kwargs: - kwargs["log_level"] = logging.DEBUG + if kwargs["verbose"]: + kwargs["log_level"] = logging.DEBUG kwargs.pop("verbose") if "vars" in kwargs: From fdb7554189dca5323c4b85bdd0a18dcd3e170be8 Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Thu, 24 Oct 2024 10:55:32 -0600 Subject: [PATCH 08/34] fix: return account as session attribute --- schemachange/session/SnowflakeSession.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/schemachange/session/SnowflakeSession.py b/schemachange/session/SnowflakeSession.py index 08353bb..574fd0b 100644 --- a/schemachange/session/SnowflakeSession.py +++ b/schemachange/session/SnowflakeSession.py @@ -13,6 +13,7 @@ class SnowflakeSession: + account: str user: str | None role: str | None warehouse: str | None @@ -34,6 +35,7 @@ def __init__( application: str, change_history_table: ChangeHistoryTable, logger: structlog.BoundLogger, + account: str | None = None, user: str | None = None, role: str | None = None, warehouse: str | None = None, @@ -43,6 +45,7 @@ def __init__( autocommit: bool = False, **kwargs, ): + self.account = account self.user = user self.role = role self.warehouse = warehouse @@ -57,7 +60,7 @@ def __init__( self.session_parameters["QUERY_TAG"] += f";{query_tag}" self.con = snowflake.connector.connect( - account=kwargs["account"], + account=self.account, user=self.user, database=kwargs.get("database"), schema=kwargs.get("schema"), From 8046f5ace31a0b5ad889ee94280a67496bcc7e7d Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Thu, 24 Oct 2024 10:56:43 -0600 Subject: [PATCH 09/34] fix: account for token path newline --- tests/test_main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_main.py b/tests/test_main.py index 7393c1d..ae78874 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -433,7 +433,7 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "snowflake_warehouse": "warehouse", "snowflake_role": "role", "snowflake_authenticator": "oauth", - "snowflake_oauth_token": "my-oauth-token", + "snowflake_oauth_token": "my-oauth-token\n", }, None, id="deploy: oauth file", From cec87a54f61bb504e84e7b2cd909aa3d672741bc Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Fri, 25 Oct 2024 13:01:50 -0600 Subject: [PATCH 10/34] fix: connections.toml requires token_file_path instead of token-file-path --- schemachange/config/utils.py | 2 +- tests/config/alt-connections.toml | 2 +- tests/config/connections.toml | 2 +- tests/config/test_get_merged_config.py | 4 ++-- tests/config/test_utils.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/schemachange/config/utils.py b/schemachange/config/utils.py index d7bc42c..7dfe572 100644 --- a/schemachange/config/utils.py +++ b/schemachange/config/utils.py @@ -174,7 +174,7 @@ def get_connection_kwargs( "snowflake_authenticator": connection.get("authenticator"), "snowflake_password": connection.get("password"), "snowflake_private_key_path": connection.get("private-key"), - "snowflake_token_path": connection.get("token-file-path"), + "snowflake_token_path": connection.get("token_file_path"), } return {k: v for k, v in connection_kwargs.items() if v is not None} diff --git a/tests/config/alt-connections.toml b/tests/config/alt-connections.toml index 7231d94..2d43d54 100644 --- a/tests/config/alt-connections.toml +++ b/tests/config/alt-connections.toml @@ -11,4 +11,4 @@ host = "alt-connections.toml-host" port = "alt-connections.toml-port" region = "alt-connections.toml-region" private-key = "alt-connections.toml-private-key" -token-file-path = "alt-connections.toml-token-file-path" +token_file_path = "alt-connections.toml-token_file_path" diff --git a/tests/config/connections.toml b/tests/config/connections.toml index ca5c602..729c7f6 100644 --- a/tests/config/connections.toml +++ b/tests/config/connections.toml @@ -11,4 +11,4 @@ host = "connections.toml-host" port = "connections.toml-port" region = "connections.toml-region" private-key = "connections.toml-private-key" -token-file-path = "connections.toml-token-file-path" +token_file_path = "connections.toml-token_file_path" diff --git a/tests/config/test_get_merged_config.py b/tests/config/test_get_merged_config.py index 33b1cb4..8725b9b 100644 --- a/tests/config/test_get_merged_config.py +++ b/tests/config/test_get_merged_config.py @@ -814,7 +814,7 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "snowflake_authenticator": my_connection["authenticator"], "snowflake_password": my_connection["password"], "snowflake_private_key_path": my_connection["private-key"], - "snowflake_token_path": my_connection["token-file-path"], + "snowflake_token_path": my_connection["token_file_path"], "config_vars": {}, "log_level": logging.INFO, }, @@ -842,7 +842,7 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "snowflake_schema": my_connection["schema"], "snowflake_authenticator": my_connection["authenticator"], "snowflake_private_key_path": my_connection["private-key"], - "snowflake_token_path": my_connection["token-file-path"], + "snowflake_token_path": my_connection["token_file_path"], "log_level": logging.INFO, "snowflake_password": my_connection["password"], "connections_file_path": assets_path / "connections.toml", diff --git a/tests/config/test_utils.py b/tests/config/test_utils.py index cc81135..13331da 100644 --- a/tests/config/test_utils.py +++ b/tests/config/test_utils.py @@ -106,7 +106,7 @@ def test_get_connection_kwargs_happy_path(self, _): "snowflake_account": "connections.toml-account", "snowflake_authenticator": "connections.toml-authenticator", "snowflake_database": "connections.toml-database", - "snowflake_token_path": "connections.toml-token-file-path", + "snowflake_token_path": "connections.toml-token_file_path", "snowflake_password": "connections.toml-password", "snowflake_private_key_path": "connections.toml-private-key", "snowflake_role": "connections.toml-role", From f9e9917a8c6def12f0ac1627ed47b275d51e8d13 Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Thu, 31 Oct 2024 07:48:46 -0700 Subject: [PATCH 11/34] feat: swap tomllib with tomlkit --- tests/config/test_get_merged_config.py | 4 ++-- tests/test_main.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/config/test_get_merged_config.py b/tests/config/test_get_merged_config.py index 8725b9b..9ada158 100644 --- a/tests/config/test_get_merged_config.py +++ b/tests/config/test_get_merged_config.py @@ -1,7 +1,7 @@ import json import logging import os -import tomllib +import tomlkit from pathlib import Path from unittest import mock @@ -23,7 +23,7 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: with file_path.open("rb") as f: - connections = tomllib.load(f) + connections = tomlkit.load(f) return connections[connection_name] diff --git a/tests/test_main.py b/tests/test_main.py index ae78874..178f2aa 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -3,7 +3,7 @@ import json import logging import os -import tomllib +import tomlkit import tempfile import unittest.mock as mock from dataclasses import asdict @@ -21,7 +21,7 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: with file_path.open("rb") as f: - connections = tomllib.load(f) + connections = tomlkit.load(f) return connections[connection_name] @@ -603,6 +603,7 @@ def test_main_deploy_config_folder( ) ) + # noinspection PyTypeChecker args[args.index("DUMMY")] = d expected_config["config_file_path"] = Path(d) / "schemachange-config.yml" @@ -654,6 +655,7 @@ def test_main_deploy_modules_folder( ): with mock.patch.dict(os.environ, {"SNOWFLAKE_PASSWORD": "password"}, clear=True): with tempfile.TemporaryDirectory() as d: + # noinspection PyTypeChecker args[args.index("DUMMY")] = d expected_config["modules_folder"] = Path(d) From 95fbe72c153f3a8cd66a3d9bbc7bb9f83927973a Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Thu, 31 Oct 2024 08:20:42 -0700 Subject: [PATCH 12/34] feat: avoid DeployConfig argument repetition --- tests/config/test_DeployConfig.py | 110 ++++++++---------------------- 1 file changed, 30 insertions(+), 80 deletions(-) diff --git a/tests/config/test_DeployConfig.py b/tests/config/test_DeployConfig.py index 41bd785..419782f 100644 --- a/tests/config/test_DeployConfig.py +++ b/tests/config/test_DeployConfig.py @@ -9,25 +9,31 @@ from schemachange.config.BaseConfig import BaseConfig from schemachange.config.DeployConfig import DeployConfig +minimal_deploy_config_kwargs: dict = { + "snowflake_account": "some_snowflake_account", + "snowflake_user": "some_snowflake_user", + "snowflake_role": "some_snowflake_role", + "snowflake_warehouse": "some_snowflake_warehouse", +} + +complete_deploy_config_kwargs: dict = { + **minimal_deploy_config_kwargs, + "config_file_path": Path("some_config_file_name"), + "root_folder": "some_root_folder_name", + "modules_folder": "some_modules_folder_name", + "config_vars": {"some": "config_vars"}, + "snowflake_database": "some_snowflake_database", + "snowflake_schema": "some_snowflake_schema", + "change_history_table": "some_history_table", + "query_tag": "some_query_tag", + "oauth_config": {"some": "values"}, +} + @mock.patch("pathlib.Path.is_dir", side_effect=[False]) def test_invalid_root_folder(_): with pytest.raises(Exception) as e_info: - DeployConfig.factory( - config_file_path=Path("some_config_file_name"), - root_folder="some_root_folder_name", - modules_folder="some_modules_folder_name", - config_vars={"some": "config_vars"}, - snowflake_account="some_snowflake_account", - snowflake_user="some_snowflake_user", - snowflake_role="some_snowflake_role", - snowflake_warehouse="some_snowflake_warehouse", - snowflake_database="some_snowflake_database", - snowflake_schema="some_snowflake_schema", - change_history_table="some_history_table", - query_tag="some_query_tag", - oauth_config={"some": "values"}, - ) + DeployConfig.factory(**complete_deploy_config_kwargs) e_info_value = str(e_info.value) assert "Path is not valid directory: some_root_folder_name" in e_info_value @@ -35,21 +41,7 @@ def test_invalid_root_folder(_): @mock.patch("pathlib.Path.is_dir", side_effect=[True, False]) def test_invalid_modules_folder(_): with pytest.raises(Exception) as e_info: - DeployConfig.factory( - config_file_path=Path("some_config_file_name"), - root_folder="some_root_folder_name", - modules_folder="some_modules_folder_name", - config_vars={"some": "config_vars"}, - snowflake_account="some_snowflake_account", - snowflake_user="some_snowflake_user", - snowflake_role="some_snowflake_role", - snowflake_warehouse="some_snowflake_warehouse", - snowflake_database="some_snowflake_database", - snowflake_schema="some_snowflake_schema", - change_history_table="some_history_table", - query_tag="some_query_tag", - oauth_config={"some": "values"}, - ) + DeployConfig.factory(**complete_deploy_config_kwargs) e_info_value = str(e_info.value) assert "Path is not valid directory: some_modules_folder_name" in e_info_value @@ -62,23 +54,11 @@ def test_invalid_snowflake_private_key_path(_, __): with pytest.raises(Exception) as e_info: DeployConfig.factory( - config_file_path=Path("some_config_file_name"), - root_folder="some_root_folder_name", - modules_folder="some_modules_folder_name", - config_vars={"some": "config_vars"}, - snowflake_account="some_snowflake_account", - snowflake_user="some_snowflake_user", - snowflake_role="some_snowflake_role", - snowflake_warehouse="some_snowflake_warehouse", - snowflake_database="some_snowflake_database", - snowflake_schema="some_snowflake_schema", + **complete_deploy_config_kwargs, snowflake_private_key_path="invalid_snowflake_private_key_path", snowflake_token_path="invalid_snowflake_token_path", connections_file_path=str(connections_file_path), connection_name=connection_name, - change_history_table="some_history_table", - query_tag="some_query_tag", - oauth_config={"some": "values"}, ) e_info_value = str(e_info.value) assert "invalid file path: invalid_snowflake_private_key_path" in e_info_value @@ -92,23 +72,11 @@ def test_invalid_snowflake_token_path(_, __): with pytest.raises(Exception) as e_info: DeployConfig.factory( - config_file_path=Path("some_config_file_name"), - root_folder="some_root_folder_name", - modules_folder="some_modules_folder_name", - config_vars={"some": "config_vars"}, - snowflake_account="some_snowflake_account", - snowflake_user="some_snowflake_user", - snowflake_role="some_snowflake_role", - snowflake_warehouse="some_snowflake_warehouse", - snowflake_database="some_snowflake_database", - snowflake_schema="some_snowflake_schema", + **complete_deploy_config_kwargs, snowflake_private_key_path="valid_snowflake_private_key_path", snowflake_token_path="invalid_snowflake_token_path", connections_file_path=str(connections_file_path), connection_name=connection_name, - change_history_table="some_history_table", - query_tag="some_query_tag", - oauth_config={"some": "values"}, ) e_info_value = str(e_info.value) assert "invalid file path: invalid_snowflake_token_path" in e_info_value @@ -174,10 +142,7 @@ def test_check_for_deploy_args_oauth_with_request_happy_path(mock_get_oauth_toke mock_get_oauth_token.return_value = oauth_token oauth_config = {"my_oauth_config": "values"} config = DeployConfig.factory( - snowflake_account="account", - snowflake_user="user", - snowflake_role="role", - snowflake_warehouse="warehouse", + **minimal_deploy_config_kwargs, snowflake_authenticator="oauth", oauth_config=oauth_config, config_file_path=Path("."), @@ -189,10 +154,7 @@ def test_check_for_deploy_args_oauth_with_request_happy_path(mock_get_oauth_toke def test_check_for_deploy_args_externalbrowser_happy_path(): config = DeployConfig.factory( - snowflake_account="account", - snowflake_user="user", - snowflake_role="role", - snowflake_warehouse="warehouse", + **minimal_deploy_config_kwargs, snowflake_authenticator="externalbrowser", config_file_path=Path("."), ) @@ -201,10 +163,7 @@ def test_check_for_deploy_args_externalbrowser_happy_path(): def test_check_for_deploy_args_okta_happy_path(): config = DeployConfig.factory( - snowflake_account="account", - snowflake_user="user", - snowflake_role="role", - snowflake_warehouse="warehouse", + **minimal_deploy_config_kwargs, snowflake_authenticator="https://okta...", snowflake_password="password", config_file_path=Path("."), @@ -215,10 +174,7 @@ def test_check_for_deploy_args_okta_happy_path(): @mock.patch("pathlib.Path.is_file", return_value=True) def test_check_for_deploy_args_snowflake_jwt_happy_path(_): config = DeployConfig.factory( - snowflake_account="account", - snowflake_user="user", - snowflake_role="role", - snowflake_warehouse="warehouse", + **minimal_deploy_config_kwargs, snowflake_authenticator="snowflake_jwt", snowflake_private_key_path="private_key_path", config_file_path=Path("."), @@ -228,10 +184,7 @@ def test_check_for_deploy_args_snowflake_jwt_happy_path(_): def test_check_for_deploy_args_snowflake_happy_path(): config = DeployConfig.factory( - snowflake_account="account", - snowflake_user="user", - snowflake_role="role", - snowflake_warehouse="warehouse", + **minimal_deploy_config_kwargs, snowflake_authenticator="snowflake", snowflake_password="password", config_file_path=Path("."), @@ -241,10 +194,7 @@ def test_check_for_deploy_args_snowflake_happy_path(): def test_check_for_deploy_args_default_happy_path(): config = DeployConfig.factory( - snowflake_account="account", - snowflake_user="user", - snowflake_role="role", - snowflake_warehouse="warehouse", + **minimal_deploy_config_kwargs, snowflake_password="password", config_file_path=Path("."), ) From c724850628ea389138ccb89fe3549554de5ee560 Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Thu, 31 Oct 2024 08:47:36 -0700 Subject: [PATCH 13/34] fix: clear environment variables to allow GitHub Actions tests to pass --- tests/config/test_get_merged_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/config/test_get_merged_config.py b/tests/config/test_get_merged_config.py index 9ada158..46a0d46 100644 --- a/tests/config/test_get_merged_config.py +++ b/tests/config/test_get_merged_config.py @@ -891,7 +891,7 @@ def test_integration_get_merged_config_inheritance( cli_args, expected, ): - with mock.patch.dict(os.environ, env_vars): + with mock.patch.dict(os.environ, env_vars, clear=True): with mock.patch("sys.argv", cli_args): get_merged_config() factory_kwargs = mock_deploy_config_factory.call_args.kwargs From addf64225837fddd3e1a8817e4a3fca38229dfe3 Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Thu, 31 Oct 2024 08:57:23 -0700 Subject: [PATCH 14/34] fix: remove unused default_config_file_name class attribute --- schemachange/config/BaseConfig.py | 2 -- tests/test_cli_misc.py | 1 - 2 files changed, 3 deletions(-) diff --git a/schemachange/config/BaseConfig.py b/schemachange/config/BaseConfig.py index a77c8da..0051d40 100644 --- a/schemachange/config/BaseConfig.py +++ b/schemachange/config/BaseConfig.py @@ -20,8 +20,6 @@ @dataclasses.dataclass(frozen=True) class BaseConfig(ABC): - default_config_file_name: ClassVar[str] = "schemachange-config.yml" - subcommand: Literal["deploy", "render"] config_version: int | None = None config_file_path: Path | None = None diff --git a/tests/test_cli_misc.py b/tests/test_cli_misc.py index 97bbc68..e2b9d7e 100644 --- a/tests/test_cli_misc.py +++ b/tests/test_cli_misc.py @@ -14,7 +14,6 @@ def test_cli_given__schemachange_version_change_updated_in_setup_config_file(): def test_cli_given__constants_exist(): - assert BaseConfig.default_config_file_name == "schemachange-config.yml" assert ChangeHistoryTable._default_database_name == "METADATA" assert ChangeHistoryTable._default_schema_name == "SCHEMACHANGE" assert ChangeHistoryTable._default_table_name == "CHANGE_HISTORY" From 10b867c1c3af828d3b4fb8c7f018aaac9aa34258 Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Thu, 31 Oct 2024 09:01:36 -0700 Subject: [PATCH 15/34] fix: make ownership grant idempotent --- demo/setup/basics_demo/A__setup_basics_demo.sql | 2 +- demo/setup/citibike_demo/A__setup_citibike_demo.sql | 2 +- demo/setup/citibike_demo_jinja/A__setup_citibike_demo_jinja.sql | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/demo/setup/basics_demo/A__setup_basics_demo.sql b/demo/setup/basics_demo/A__setup_basics_demo.sql index 2d7edf7..d6af411 100644 --- a/demo/setup/basics_demo/A__setup_basics_demo.sql +++ b/demo/setup/basics_demo/A__setup_basics_demo.sql @@ -31,7 +31,7 @@ GRANT DATABASE ROLE IDENTIFIER($SC_W) TO DATABASE ROLE IDENTIFIER($SC_C); CREATE SCHEMA IF NOT EXISTS IDENTIFIER($TARGET_SCHEMA_NAME) WITH MANAGED ACCESS; -- USE SCHEMA INFORMATION_SCHEMA; -- DROP SCHEMA IF EXISTS PUBLIC; -GRANT OWNERSHIP ON SCHEMA IDENTIFIER($TARGET_SCHEMA_NAME) TO ROLE IDENTIFIER($DEPLOY_ROLE); +GRANT OWNERSHIP ON SCHEMA IDENTIFIER($TARGET_SCHEMA_NAME) TO ROLE IDENTIFIER($DEPLOY_ROLE) REVOKE CURRENT GRANTS; USE SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE); -- SCHEMA diff --git a/demo/setup/citibike_demo/A__setup_citibike_demo.sql b/demo/setup/citibike_demo/A__setup_citibike_demo.sql index 2d7edf7..d6af411 100644 --- a/demo/setup/citibike_demo/A__setup_citibike_demo.sql +++ b/demo/setup/citibike_demo/A__setup_citibike_demo.sql @@ -31,7 +31,7 @@ GRANT DATABASE ROLE IDENTIFIER($SC_W) TO DATABASE ROLE IDENTIFIER($SC_C); CREATE SCHEMA IF NOT EXISTS IDENTIFIER($TARGET_SCHEMA_NAME) WITH MANAGED ACCESS; -- USE SCHEMA INFORMATION_SCHEMA; -- DROP SCHEMA IF EXISTS PUBLIC; -GRANT OWNERSHIP ON SCHEMA IDENTIFIER($TARGET_SCHEMA_NAME) TO ROLE IDENTIFIER($DEPLOY_ROLE); +GRANT OWNERSHIP ON SCHEMA IDENTIFIER($TARGET_SCHEMA_NAME) TO ROLE IDENTIFIER($DEPLOY_ROLE) REVOKE CURRENT GRANTS; USE SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE); -- SCHEMA diff --git a/demo/setup/citibike_demo_jinja/A__setup_citibike_demo_jinja.sql b/demo/setup/citibike_demo_jinja/A__setup_citibike_demo_jinja.sql index 2d7edf7..d6af411 100644 --- a/demo/setup/citibike_demo_jinja/A__setup_citibike_demo_jinja.sql +++ b/demo/setup/citibike_demo_jinja/A__setup_citibike_demo_jinja.sql @@ -31,7 +31,7 @@ GRANT DATABASE ROLE IDENTIFIER($SC_W) TO DATABASE ROLE IDENTIFIER($SC_C); CREATE SCHEMA IF NOT EXISTS IDENTIFIER($TARGET_SCHEMA_NAME) WITH MANAGED ACCESS; -- USE SCHEMA INFORMATION_SCHEMA; -- DROP SCHEMA IF EXISTS PUBLIC; -GRANT OWNERSHIP ON SCHEMA IDENTIFIER($TARGET_SCHEMA_NAME) TO ROLE IDENTIFIER($DEPLOY_ROLE); +GRANT OWNERSHIP ON SCHEMA IDENTIFIER($TARGET_SCHEMA_NAME) TO ROLE IDENTIFIER($DEPLOY_ROLE) REVOKE CURRENT GRANTS; USE SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE); -- SCHEMA From 6ee659fb605c0bb6307637ecfd5b95fb1e9a9afa Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Thu, 31 Oct 2024 09:10:38 -0700 Subject: [PATCH 16/34] feat: make tests runnable on Snowflake standard edition --- demo/provision/setup_schemachange_schema.sql | 1 - demo/setup/basics_demo/A__setup_basics_demo.sql | 1 - demo/setup/citibike_demo/A__setup_citibike_demo.sql | 1 - demo/setup/citibike_demo_jinja/A__setup_citibike_demo_jinja.sql | 1 - 4 files changed, 4 deletions(-) diff --git a/demo/provision/setup_schemachange_schema.sql b/demo/provision/setup_schemachange_schema.sql index e1eef07..9e1c1fd 100644 --- a/demo/provision/setup_schemachange_schema.sql +++ b/demo/provision/setup_schemachange_schema.sql @@ -53,7 +53,6 @@ GRANT MONITOR ON SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDE -- SC_W -- None -- SC_C -GRANT MODIFY, APPLYBUDGET, ADD SEARCH OPTIMIZATION ON SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_C); -- TABLES -- SC_M diff --git a/demo/setup/basics_demo/A__setup_basics_demo.sql b/demo/setup/basics_demo/A__setup_basics_demo.sql index d6af411..911048b 100644 --- a/demo/setup/basics_demo/A__setup_basics_demo.sql +++ b/demo/setup/basics_demo/A__setup_basics_demo.sql @@ -44,7 +44,6 @@ GRANT MONITOR ON SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDE -- SC_W -- None -- SC_C -GRANT MODIFY, APPLYBUDGET, ADD SEARCH OPTIMIZATION ON SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_C); -- TABLES -- SC_M diff --git a/demo/setup/citibike_demo/A__setup_citibike_demo.sql b/demo/setup/citibike_demo/A__setup_citibike_demo.sql index d6af411..911048b 100644 --- a/demo/setup/citibike_demo/A__setup_citibike_demo.sql +++ b/demo/setup/citibike_demo/A__setup_citibike_demo.sql @@ -44,7 +44,6 @@ GRANT MONITOR ON SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDE -- SC_W -- None -- SC_C -GRANT MODIFY, APPLYBUDGET, ADD SEARCH OPTIMIZATION ON SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_C); -- TABLES -- SC_M diff --git a/demo/setup/citibike_demo_jinja/A__setup_citibike_demo_jinja.sql b/demo/setup/citibike_demo_jinja/A__setup_citibike_demo_jinja.sql index d6af411..911048b 100644 --- a/demo/setup/citibike_demo_jinja/A__setup_citibike_demo_jinja.sql +++ b/demo/setup/citibike_demo_jinja/A__setup_citibike_demo_jinja.sql @@ -44,7 +44,6 @@ GRANT MONITOR ON SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDE -- SC_W -- None -- SC_C -GRANT MODIFY, APPLYBUDGET, ADD SEARCH OPTIMIZATION ON SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE) TO DATABASE ROLE IDENTIFIER($SC_C); -- TABLES -- SC_M From 6eaeaa9ef04bbadff13f3e34604b6d432fa7b440 Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Thu, 31 Oct 2024 09:14:54 -0700 Subject: [PATCH 17/34] feat: respect SNOWFLAKE_DEFAULT_CONNECTION_NAME environment variable --- schemachange/config/get_merged_config.py | 7 +++++- schemachange/config/utils.py | 1 + tests/config/alt-connections.toml | 14 +++++++++++ tests/config/test_get_merged_config.py | 30 +++++++----------------- tests/config/test_utils.py | 4 ++++ 5 files changed, 33 insertions(+), 23 deletions(-) diff --git a/schemachange/config/get_merged_config.py b/schemachange/config/get_merged_config.py index 8181465..c1a868d 100644 --- a/schemachange/config/get_merged_config.py +++ b/schemachange/config/get_merged_config.py @@ -38,6 +38,7 @@ def get_yaml_config_kwargs(config_file_path: Optional[Path]) -> dict: def get_merged_config() -> Union[DeployConfig, RenderConfig]: env_kwargs: dict[str, str] = get_env_kwargs() + connection_name = env_kwargs.pop("connection_name", None) cli_kwargs = parse_cli_args(sys.argv[1:]) @@ -46,7 +47,10 @@ def get_merged_config() -> Union[DeployConfig, RenderConfig]: connections_file_path = validate_file_path( file_path=cli_kwargs.pop("connections_file_path", None) ) - connection_name = cli_kwargs.pop("connection_name", None) + + if connection_name is None: + connection_name = cli_kwargs.pop("connection_name", None) + config_folder = validate_directory(path=cli_kwargs.pop("config_folder", ".")) config_file_name = cli_kwargs.pop("config_file_name") config_file_path = Path(config_folder) / config_file_name @@ -61,6 +65,7 @@ def get_merged_config() -> Union[DeployConfig, RenderConfig]: if connections_file_path is None: connections_file_path = yaml_kwargs.pop("connections_file_path", None) if config_folder is not None and connections_file_path is not None: + # noinspection PyTypeChecker connections_file_path = config_folder / connections_file_path connections_file_path = validate_file_path(file_path=connections_file_path) diff --git a/schemachange/config/utils.py b/schemachange/config/utils.py index 7dfe572..b90af1c 100644 --- a/schemachange/config/utils.py +++ b/schemachange/config/utils.py @@ -217,6 +217,7 @@ def get_env_kwargs() -> dict[str, str]: "snowflake_private_key_path": os.getenv("SNOWFLAKE_PRIVATE_KEY_PATH"), "snowflake_authenticator": os.getenv("SNOWFLAKE_AUTHENTICATOR"), "snowflake_oauth_token": os.getenv("SNOWFLAKE_TOKEN"), + "connection_name": os.getenv("SNOWFLAKE_DEFAULT_CONNECTION_NAME"), } return {k: v for k, v in env_kwargs.items() if v is not None} diff --git a/tests/config/alt-connections.toml b/tests/config/alt-connections.toml index 2d43d54..4b44631 100644 --- a/tests/config/alt-connections.toml +++ b/tests/config/alt-connections.toml @@ -12,3 +12,17 @@ port = "alt-connections.toml-port" region = "alt-connections.toml-region" private-key = "alt-connections.toml-private-key" token_file_path = "alt-connections.toml-token_file_path" +[anotherconnection] +account = "another-connections.toml-account" +user = "another-connections.toml-user" +role = "another-connections.toml-role" +warehouse = "another-connections.toml-warehouse" +database = "another-connections.toml-database" +schema = "another-connections.toml-schema" +authenticator = "another-connections.toml-authenticator" +password = "another-connections.toml-password" +host = "another-connections.toml-host" +port = "another-connections.toml-port" +region = "another-connections.toml-region" +private-key = "another-connections.toml-private-key" +token_file_path = "another-connections.toml-token_file_path" diff --git a/tests/config/test_get_merged_config.py b/tests/config/test_get_merged_config.py index 46a0d46..7ca2989 100644 --- a/tests/config/test_get_merged_config.py +++ b/tests/config/test_get_merged_config.py @@ -52,11 +52,7 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "env_kwargs, cli_kwargs, yaml_kwargs, connection_kwargs, expected", [ pytest.param( - { # env_kwargs - "snowflake_password": None, - "snowflake_private_key_path": None, - "snowflake_authenticator": None, - }, + {}, # env_kwargs {**default_cli_kwargs}, # cli_kwargs {}, # yaml_kwargs {}, # connection_kwargs @@ -68,11 +64,7 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: id="Deploy: Only required arguments", ), pytest.param( - { # env_kwargs - "snowflake_password": None, - "snowflake_private_key_path": None, - "snowflake_authenticator": None, - }, + {}, # env_kwargs {**default_cli_kwargs}, # cli_kwargs {}, # yaml_kwargs { # connection_kwargs @@ -106,11 +98,7 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: id="Deploy: all connection_kwargs", ), pytest.param( - { # env_kwargs - "snowflake_password": None, - "snowflake_private_key_path": None, - "snowflake_authenticator": None, - }, + {}, # env_kwargs {**default_cli_kwargs}, # cli_kwargs { # yaml_kwargs "root_folder": "yaml_root_folder", @@ -184,11 +172,7 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: id="Deploy: all yaml, all connection_kwargs", ), pytest.param( - { # env_kwargs - "snowflake_password": None, - "snowflake_private_key_path": None, - "snowflake_authenticator": None, - }, + {}, # env_kwargs { # cli_kwargs **default_cli_kwargs, "config_folder": "cli_config_folder", @@ -293,6 +277,7 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "snowflake_password": "env_snowflake_password", "snowflake_private_key_path": "env_snowflake_private_key_path", "snowflake_authenticator": "env_snowflake_authenticator", + "connection_name": "env_connection_name", }, { # cli_kwargs **default_cli_kwargs, @@ -383,7 +368,7 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "snowflake_private_key_path": "env_snowflake_private_key_path", "snowflake_token_path": "cli_snowflake_token_path", "connections_file_path": Path("cli_connections_file_path"), - "connection_name": "cli_connection_name", + "connection_name": "env_connection_name", "change_history_table": "cli_change_history_table", "create_change_history_table": False, "autocommit": False, @@ -707,6 +692,7 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "SNOWFLAKE_PRIVATE_KEY_PATH": "env_snowflake_private_key_path", "SNOWFLAKE_AUTHENTICATOR": "env_snowflake_authenticator", "SNOWFLAKE_TOKEN": "env_snowflake_token", + "SNOWFLAKE_DEFAULT_CONNECTION_NAME": "anotherconnection", }, # env_kwargs [ # cli_args "schemachange", @@ -780,7 +766,7 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "query_tag": "query-tag-from-cli", "snowflake_oauth_token": "env_snowflake_token", "oauth_config": {"oauth_config_variable": "cli_oauth_config_value"}, - "connection_name": "myaltconnection", + "connection_name": "anotherconnection", "connections_file_path": assets_path / "alt-connections.toml", "snowflake_password": "env_snowflake_password", }, diff --git a/tests/config/test_utils.py b/tests/config/test_utils.py index 13331da..67b3585 100644 --- a/tests/config/test_utils.py +++ b/tests/config/test_utils.py @@ -57,6 +57,10 @@ def test_get_snowflake_password(env_vars: dict, expected: str): {"SNOWFLAKE_AUTHENTICATOR": "my_snowflake_authenticator"}, {"snowflake_authenticator": "my_snowflake_authenticator"}, ), + ( + {"SNOWFLAKE_DEFAULT_CONNECTION_NAME": "my_connection_name"}, + {"connection_name": "my_connection_name"}, + ), ], ) def test_get_env_kwargs(env_vars: dict, expected: str): From 6e53452b66a56f9c7998f15d016315465d1203e3 Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Thu, 31 Oct 2024 09:41:24 -0700 Subject: [PATCH 18/34] feat: log snowflake.connector.connect arguments --- schemachange/config/BaseConfig.py | 5 +++- schemachange/config/DeployConfig.py | 10 +++++++ schemachange/session/SnowflakeSession.py | 36 ++++++++++++------------ tests/session/test_SnowflakeSession.py | 4 +-- 4 files changed, 34 insertions(+), 21 deletions(-) diff --git a/schemachange/config/BaseConfig.py b/schemachange/config/BaseConfig.py index 0051d40..5f9b192 100644 --- a/schemachange/config/BaseConfig.py +++ b/schemachange/config/BaseConfig.py @@ -4,7 +4,7 @@ import logging from abc import ABC from pathlib import Path -from typing import Literal, ClassVar, TypeVar +from typing import Literal, TypeVar import structlog @@ -38,10 +38,13 @@ def factory( modules_folder: Path | str | None = None, config_vars: str | dict | None = None, log_level: int = logging.INFO, + connection_secrets: set[str] | None = None, **kwargs, ): try: secrets = get_config_secrets(config_vars) + if connection_secrets is not None: + secrets.update(connection_secrets) except Exception as e: raise Exception( "config_vars did not parse correctly, please check its configuration" diff --git a/schemachange/config/DeployConfig.py b/schemachange/config/DeployConfig.py index 1558b94..3cffeaf 100644 --- a/schemachange/config/DeployConfig.py +++ b/schemachange/config/DeployConfig.py @@ -89,10 +89,20 @@ def factory( table_str=change_history_table ) + connection_secrets = { + secret + for secret in [ + kwargs.get("snowflake_password"), + kwargs.get("snowflake_oauth_token"), + ] + if secret is not None + } + return super().factory( subcommand="deploy", config_file_path=config_file_path, change_history_table=change_history_table, + connection_secrets=connection_secrets, **kwargs, ) diff --git a/schemachange/session/SnowflakeSession.py b/schemachange/session/SnowflakeSession.py index 6078f9f..1c3bd96 100644 --- a/schemachange/session/SnowflakeSession.py +++ b/schemachange/session/SnowflakeSession.py @@ -59,24 +59,24 @@ def __init__( if query_tag: self.session_parameters["QUERY_TAG"] += f";{query_tag}" - self.con = snowflake.connector.connect( - account=self.account, - user=self.user, - database=kwargs.get("database"), - schema=kwargs.get("schema"), - role=self.role, - warehouse=self.warehouse, - private_key=kwargs.get("private_key"), - private_key_file=kwargs.get("private_key_path"), - private_key_file_pwd=kwargs.get("private_key_path_password"), - token=kwargs.get("oauth_token"), - password=kwargs.get("password"), - authenticator=kwargs.get("authenticator"), - connection_name=kwargs.get("connection_name"), - connections_file_path=kwargs.get("connections_file_path"), - application=application, - session_parameters=self.session_parameters, - ) + connect_kwargs = { + "account": self.account, + "user": self.user, + "database": kwargs.get("database"), + "schema": kwargs.get("schema"), + "role": self.role, + "warehouse": self.warehouse, + "private_key_file": kwargs.get("private_key_path"), + "token": kwargs.get("oauth_token"), + "password": kwargs.get("password"), + "authenticator": kwargs.get("authenticator"), + "connection_name": kwargs.get("connection_name"), + "connections_file_path": kwargs.get("connections_file_path"), + "application": application, + "session_parameters": self.session_parameters, + } + self.logger.info("snowflake.connector.connect kwargs", **connect_kwargs) + self.con = snowflake.connector.connect(**connect_kwargs) print(f"Current session ID: {self.con.session_id}") if not self.autocommit: diff --git a/tests/session/test_SnowflakeSession.py b/tests/session/test_SnowflakeSession.py index 96b39ac..32b9d4b 100644 --- a/tests/session/test_SnowflakeSession.py +++ b/tests/session/test_SnowflakeSession.py @@ -34,7 +34,7 @@ def test_fetch_change_history_metadata_exists(self, session: SnowflakeSession): result = session.fetch_change_history_metadata() assert result == {"created": "created", "last_altered": "last_altered"} assert session.con.execute_string.call_count == 1 - assert session.logger.calls[0][1][0] == "Executing query" + assert session.logger.calls[1][1][0] == "Executing query" def test_fetch_change_history_metadata_does_not_exist( self, session: SnowflakeSession @@ -43,4 +43,4 @@ def test_fetch_change_history_metadata_does_not_exist( result = session.fetch_change_history_metadata() assert result == {} assert session.con.execute_string.call_count == 1 - assert session.logger.calls[0][1][0] == "Executing query" + assert session.logger.calls[1][1][0] == "Executing query" From 713ae2db0a3ba5f1c4507a9e4d4831641b0c6311 Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Thu, 31 Oct 2024 13:26:48 -0700 Subject: [PATCH 19/34] feat: log get_merged_config steps --- schemachange/cli.py | 2 +- schemachange/config/get_merged_config.py | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/schemachange/cli.py b/schemachange/cli.py index d8c3325..f2b5dc3 100644 --- a/schemachange/cli.py +++ b/schemachange/cli.py @@ -42,7 +42,7 @@ def main(): % {"schemachange_version": SCHEMACHANGE_VERSION} ) - config = get_merged_config() + config = get_merged_config(logger=module_logger) redact_config_secrets(config_secrets=config.secrets) structlog.configure( diff --git a/schemachange/config/get_merged_config.py b/schemachange/config/get_merged_config.py index c1a868d..17f0831 100644 --- a/schemachange/config/get_merged_config.py +++ b/schemachange/config/get_merged_config.py @@ -3,6 +3,8 @@ from pathlib import Path from typing import Union, Optional +import structlog + from schemachange.config.DeployConfig import DeployConfig from schemachange.config.RenderConfig import RenderConfig from schemachange.config.parse_cli_args import parse_cli_args @@ -36,11 +38,16 @@ def get_yaml_config_kwargs(config_file_path: Optional[Path]) -> dict: return {k: v for k, v in kwargs.items() if v is not None} -def get_merged_config() -> Union[DeployConfig, RenderConfig]: +def get_merged_config( + logger: structlog.BoundLogger, +) -> Union[DeployConfig, RenderConfig]: env_kwargs: dict[str, str] = get_env_kwargs() + logger.info("env_kwargs", **env_kwargs) + connection_name = env_kwargs.pop("connection_name", None) cli_kwargs = parse_cli_args(sys.argv[1:]) + logger.info("cli_kwargs", **cli_kwargs) cli_config_vars = cli_kwargs.pop("config_vars") @@ -58,6 +65,8 @@ def get_merged_config() -> Union[DeployConfig, RenderConfig]: yaml_kwargs = get_yaml_config_kwargs( config_file_path=config_file_path, ) + logger.info("yaml_kwargs", **yaml_kwargs) + yaml_config_vars = yaml_kwargs.pop("config_vars", None) if yaml_config_vars is None: yaml_config_vars = {} @@ -77,6 +86,7 @@ def get_merged_config() -> Union[DeployConfig, RenderConfig]: connections_file_path=connections_file_path, connection_name=connection_name, ) + logger.info("connection_kwargs", **connection_kwargs) config_vars = { **yaml_config_vars, @@ -97,6 +107,8 @@ def get_merged_config() -> Union[DeployConfig, RenderConfig]: if connection_name is not None: kwargs["connection_name"] = connection_name + logger.info("final kwargs", **kwargs) + if cli_kwargs["subcommand"] == "deploy": return DeployConfig.factory(**kwargs) elif cli_kwargs["subcommand"] == "render": From 9fec7589aa07aec0cc3c6c219578126653b1233f Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Thu, 31 Oct 2024 13:31:45 -0700 Subject: [PATCH 20/34] fix: fix get_merged_config tests --- tests/config/test_get_merged_config.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/config/test_get_merged_config.py b/tests/config/test_get_merged_config.py index 7ca2989..b86a5d1 100644 --- a/tests/config/test_get_merged_config.py +++ b/tests/config/test_get_merged_config.py @@ -1,6 +1,8 @@ import json import logging import os + +import structlog import tomlkit from pathlib import Path from unittest import mock @@ -405,7 +407,9 @@ def test_get_merged_config_inheritance( mock_parse_cli_args.return_value = {**cli_kwargs} mock_get_yaml_config_kwargs.return_value = {**yaml_kwargs} mock_get_connection_kwargs.return_value = {**connection_kwargs} - get_merged_config() + logger = structlog.testing.CapturingLogger() + # noinspection PyTypeChecker + get_merged_config(logger=logger) factory_kwargs = mock_deploy_config_factory.call_args.kwargs for actual_key, actual_value in factory_kwargs.items(): assert expected[actual_key] == actual_value @@ -419,8 +423,10 @@ def test_invalid_config_folder(mock_parse_cli_args, _): **default_cli_kwargs, } mock_parse_cli_args.return_value = {**cli_kwargs} + logger = structlog.testing.CapturingLogger() with pytest.raises(Exception) as e_info: - get_merged_config() + # noinspection PyTypeChecker + get_merged_config(logger=logger) assert f"Path is not valid directory: {cli_kwargs['config_folder']}" in str( e_info.value ) @@ -877,9 +883,11 @@ def test_integration_get_merged_config_inheritance( cli_args, expected, ): + logger = structlog.testing.CapturingLogger() with mock.patch.dict(os.environ, env_vars, clear=True): with mock.patch("sys.argv", cli_args): - get_merged_config() + # noinspection PyTypeChecker + get_merged_config(logger=logger) factory_kwargs = mock_deploy_config_factory.call_args.kwargs for actual_key, actual_value in factory_kwargs.items(): assert expected[actual_key] == actual_value From 19319b6ee2440c5f55d32c6fb3e9901b5a805172 Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Thu, 31 Oct 2024 13:45:02 -0700 Subject: [PATCH 21/34] fix: fix get_merged_config tests --- schemachange/config/DeployConfig.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/schemachange/config/DeployConfig.py b/schemachange/config/DeployConfig.py index 3cffeaf..ba5d900 100644 --- a/schemachange/config/DeployConfig.py +++ b/schemachange/config/DeployConfig.py @@ -168,4 +168,6 @@ def get_session_kwargs(self) -> dict: "autocommit": self.autocommit, "query_tag": self.query_tag, } + print("get_session_kwargs") + print(session_kwargs) return {k: v for k, v in session_kwargs.items() if v is not None} From e82b3b94083d3ce969d79017c2af70bb2260f505 Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Thu, 31 Oct 2024 13:53:44 -0700 Subject: [PATCH 22/34] fix: fix missing database reference --- schemachange/config/DeployConfig.py | 2 -- schemachange/config/get_merged_config.py | 10 +++++----- schemachange/session/SnowflakeSession.py | 6 +++--- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/schemachange/config/DeployConfig.py b/schemachange/config/DeployConfig.py index ba5d900..3cffeaf 100644 --- a/schemachange/config/DeployConfig.py +++ b/schemachange/config/DeployConfig.py @@ -168,6 +168,4 @@ def get_session_kwargs(self) -> dict: "autocommit": self.autocommit, "query_tag": self.query_tag, } - print("get_session_kwargs") - print(session_kwargs) return {k: v for k, v in session_kwargs.items() if v is not None} diff --git a/schemachange/config/get_merged_config.py b/schemachange/config/get_merged_config.py index 17f0831..dbd8367 100644 --- a/schemachange/config/get_merged_config.py +++ b/schemachange/config/get_merged_config.py @@ -42,12 +42,12 @@ def get_merged_config( logger: structlog.BoundLogger, ) -> Union[DeployConfig, RenderConfig]: env_kwargs: dict[str, str] = get_env_kwargs() - logger.info("env_kwargs", **env_kwargs) + logger.debug("env_kwargs", **env_kwargs) connection_name = env_kwargs.pop("connection_name", None) cli_kwargs = parse_cli_args(sys.argv[1:]) - logger.info("cli_kwargs", **cli_kwargs) + logger.debug("cli_kwargs", **cli_kwargs) cli_config_vars = cli_kwargs.pop("config_vars") @@ -65,7 +65,7 @@ def get_merged_config( yaml_kwargs = get_yaml_config_kwargs( config_file_path=config_file_path, ) - logger.info("yaml_kwargs", **yaml_kwargs) + logger.debug("yaml_kwargs", **yaml_kwargs) yaml_config_vars = yaml_kwargs.pop("config_vars", None) if yaml_config_vars is None: @@ -86,7 +86,7 @@ def get_merged_config( connections_file_path=connections_file_path, connection_name=connection_name, ) - logger.info("connection_kwargs", **connection_kwargs) + logger.debug("connection_kwargs", **connection_kwargs) config_vars = { **yaml_config_vars, @@ -107,7 +107,7 @@ def get_merged_config( if connection_name is not None: kwargs["connection_name"] = connection_name - logger.info("final kwargs", **kwargs) + logger.debug("final kwargs", **kwargs) if cli_kwargs["subcommand"] == "deploy": return DeployConfig.factory(**kwargs) diff --git a/schemachange/session/SnowflakeSession.py b/schemachange/session/SnowflakeSession.py index 1c3bd96..06e8e2c 100644 --- a/schemachange/session/SnowflakeSession.py +++ b/schemachange/session/SnowflakeSession.py @@ -62,8 +62,8 @@ def __init__( connect_kwargs = { "account": self.account, "user": self.user, - "database": kwargs.get("database"), - "schema": kwargs.get("schema"), + "database": self.database, + "schema": self.schema, "role": self.role, "warehouse": self.warehouse, "private_key_file": kwargs.get("private_key_path"), @@ -75,7 +75,7 @@ def __init__( "application": application, "session_parameters": self.session_parameters, } - self.logger.info("snowflake.connector.connect kwargs", **connect_kwargs) + self.logger.debug("snowflake.connector.connect kwargs", **connect_kwargs) self.con = snowflake.connector.connect(**connect_kwargs) print(f"Current session ID: {self.con.session_id}") From d5bf05a062bb30a9122d9359e011c5f5547214bb Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Thu, 31 Oct 2024 14:19:28 -0700 Subject: [PATCH 23/34] docs: reference new cli arguments --- README.md | 46 +++++++++++++++------------ schemachange/config/parse_cli_args.py | 4 +-- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 93dcd74..f78be09 100644 --- a/README.md +++ b/README.md @@ -496,26 +496,32 @@ This is the main command that runs the deployment process. usage: schemachange deploy [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-m MODULES_FOLDER] [-a SNOWFLAKE_ACCOUNT] [-u SNOWFLAKE_USER] [-r SNOWFLAKE_ROLE] [-w SNOWFLAKE_WAREHOUSE] [-d SNOWFLAKE_DATABASE] [-s SNOWFLAKE_SCHEMA] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] ``` -| Parameter | Description | -|----------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| -h, --help | Show the help message and exit | -| --config-folder CONFIG_FOLDER | The folder to look in for the schemachange-config.yml file (the default is the current working directory) | -| -f ROOT_FOLDER, --root-folder ROOT_FOLDER | The root folder for the database change scripts. The default is the current directory. | -| -m MODULES_FOLDER, --modules-folder MODULES_FOLDER | The modules folder for jinja macros and templates to be used across mutliple scripts | -| -a SNOWFLAKE_ACCOUNT, --snowflake-account SNOWFLAKE_ACCOUNT | The name of the snowflake account (e.g. xy12345.east-us-2.azure). | -| -u SNOWFLAKE_USER, --snowflake-user SNOWFLAKE_USER | The name of the snowflake user | -| -r SNOWFLAKE_ROLE, --snowflake-role SNOWFLAKE_ROLE | The name of the role to use | -| -w SNOWFLAKE_WAREHOUSE, --snowflake-warehouse SNOWFLAKE_WAREHOUSE | The name of the default warehouse to use. Can be overridden in the change scripts. | -| -d SNOWFLAKE_DATABASE, --snowflake-database SNOWFLAKE_DATABASE | The name of the default database to use. Can be overridden in the change scripts. | -| -s SNOWFLAKE_SCHEMA, --snowflake-schema SNOWFLAKE_SCHEMA | The name of the default schema to use. Can be overridden in the change scripts. | -| -c CHANGE_HISTORY_TABLE, --change-history-table CHANGE_HISTORY_TABLE | Used to override the default name of the change history table (which is METADATA.SCHEMACHANGE.CHANGE_HISTORY) | -| --vars VARS | Define values for the variables to replaced in change scripts, given in JSON format (e.g. '{"variable1": "value1", "variable2": "value2"}') | -| --create-change-history-table | Create the change history table if it does not exist. The default is 'False'. | -| -ac, --autocommit | Enable autocommit feature for DML commands. The default is 'False'. | -| -v, --verbose | Display verbose debugging details during execution. The default is 'False'. | -| --dry-run | Run schemachange in dry run mode. The default is 'False'. | -| --query-tag | A string to include in the QUERY_TAG that is attached to every SQL statement executed. | -| --oauth-config | Define values for the variables to Make Oauth Token requests (e.g. {"token-provider-url": "https//...", "token-request-payload": {"client_id": "GUID_xyz",...},... })' | +| Parameter | Description | +|----------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| -h, --help | Show the help message and exit | +| --config-folder CONFIG_FOLDER | The folder to look in for the schemachange config file (the default is the current working directory) | +| --config-file=name CONFIG_FILE_NAME | The file name of the schemachange config file. (the default is schemachange-config.yml) | +| -f ROOT_FOLDER, --root-folder ROOT_FOLDER | The root folder for the database change scripts. The default is the current directory. | +| -m MODULES_FOLDER, --modules-folder MODULES_FOLDER | The modules folder for jinja macros and templates to be used across mutliple scripts | +| -a SNOWFLAKE_ACCOUNT, --snowflake-account SNOWFLAKE_ACCOUNT | The name of the snowflake account (e.g. xy12345.east-us-2.azure). | +| -u SNOWFLAKE_USER, --snowflake-user SNOWFLAKE_USER | The name of the snowflake user | +| -r SNOWFLAKE_ROLE, --snowflake-role SNOWFLAKE_ROLE | The name of the role to use | +| -w SNOWFLAKE_WAREHOUSE, --snowflake-warehouse SNOWFLAKE_WAREHOUSE | The name of the default warehouse to use. Can be overridden in the change scripts. | +| -d SNOWFLAKE_DATABASE, --snowflake-database SNOWFLAKE_DATABASE | The name of the default database to use. Can be overridden in the change scripts. | +| -s SNOWFLAKE_SCHEMA, --snowflake-schema SNOWFLAKE_SCHEMA | The name of the default schema to use. Can be overridden in the change scripts. | +| -A SNOWFLAKE_AUTHENTICATOR, --snowflake-authenticator SNOWFLAKE_AUTHENTICATOR | The Snowflake Authenticator to use. One of snowflake, oauth, externalbrowser, or https://.okta.com | +| -k SNOWFLAKE_PRIVATE_KEY_PATH, --snowflake-private-key-path SNOWFLAKE_PRIVATE_KEY_PATH | Path to file containing private key. | +| -t SNOWFLAKE_TOKEN_PATH, --snowflake-token-path SNOWFLAKE_TOKEN_PATH | Path to the file containing the OAuth token to be used when authenticating with Snowflake. | +| --connections-file-path CONNECTIONS_FILE_PATH | Override the default [connections.toml](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-connect#connecting-using-the-connections-toml-file) file path at snowflake.connector.constants.CONNECTIONS_FILE (OS specific) | +| --connection-name CONNECTION_NAME | Override the default [connections.toml](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-connect#connecting-using-the-connections-toml-file) connection name. Other connection-related values will override these connection values. | +| -c CHANGE_HISTORY_TABLE, --change-history-table CHANGE_HISTORY_TABLE | Used to override the default name of the change history table (which is METADATA.SCHEMACHANGE.CHANGE_HISTORY) | +| --vars VARS | Define values for the variables to replaced in change scripts, given in JSON format (e.g. '{"variable1": "value1", "variable2": "value2"}') | +| --create-change-history-table | Create the change history table if it does not exist. The default is 'False'. | +| -ac, --autocommit | Enable autocommit feature for DML commands. The default is 'False'. | +| -v, --verbose | Display verbose debugging details during execution. The default is 'False'. | +| --dry-run | Run schemachange in dry run mode. The default is 'False'. | +| --query-tag | A string to include in the QUERY_TAG that is attached to every SQL statement executed. | +| --oauth-config | Define values for the variables to Make Oauth Token requests (e.g. {"token-provider-url": "https//...", "token-request-payload": {"client_id": "GUID_xyz",...},... })' | #### render diff --git a/schemachange/config/parse_cli_args.py b/schemachange/config/parse_cli_args.py index 3e1ea4e..f6a31c1 100644 --- a/schemachange/config/parse_cli_args.py +++ b/schemachange/config/parse_cli_args.py @@ -167,13 +167,13 @@ def parse_cli_args(args) -> dict: parser_deploy.add_argument( "--connections-file-path", type=str, - help="Override the default connections file path at snowflake.connector.constants.CONNECTIONS_FILE (OS specific)", + help="Override the default connections.toml file path at snowflake.connector.constants.CONNECTIONS_FILE (OS specific)", required=False, ) parser_deploy.add_argument( "--connection-name", type=str, - help="Override the default connection name. Other connection-related values will override these connection values.", + help="Override the default connections.toml connection name. Other connection-related values will override these connection values.", required=False, ) parser_deploy.add_argument( From d218e5215fd48a34255cc924750deb581457c6c0 Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Thu, 31 Oct 2024 16:11:46 -0700 Subject: [PATCH 24/34] docs: reference new cli arguments --- README.md | 281 +++++++++++++++++++++++++++++++++++------------------- 1 file changed, 185 insertions(+), 96 deletions(-) diff --git a/README.md b/README.md index f78be09..7787e67 100644 --- a/README.md +++ b/README.md @@ -45,17 +45,21 @@ support or warranty. 1. [Change History Table](#change-history-table) 1. [Authentication](#authentication) 1. [Password Authentication](#password-authentication) - 1. [Private Key Authentication](#private-key-authentication) - 1. [Oauth Authentication](#oauth-authentication) + 1. [External OAuth Authentication](#external-oauth-authentication) 1. [External Browser Authentication](#external-browser-authentication) 1. [Okta Authentication](#okta-authentication) + 1. [Private Key Authentication](#private-key-authentication) 1. [Configuration](#configuration) + 1. [Environment Variables](#environment-variables) 1. [YAML Config File](#yaml-config-file) 1. [Yaml Jinja support](#yaml-jinja-support) - 1. [Command Line Arguments](#command-line-arguments) + 1. [connections.toml File](#connectionstoml-file) +1. [Commands](#commands) + 1. [deploy](#deploy) + 1. [render](#render) 1. [Running schemachange](#running-schemachange) 1. [Prerequisites](#prerequisites) - 1. [Running The Script](#running-the-script) + 1. [Running the Script](#running-the-script) 1. [Integrating With DevOps](#integrating-with-devops) 1. [Sample DevOps Process Flow](#sample-devops-process-flow) 1. [Using in a CI/CD Pipeline](#using-in-a-cicd-pipeline) @@ -237,16 +241,16 @@ Within change scripts: schemachange records all applied changes scripts to the change history table. By default, schemachange will attempt to log all activities to the `METADATA.SCHEMACHANGE.CHANGE_HISTORY` table. The name and location of the change history -table can be overriden by using the `-c` (or `--change-history-table`) parameter. The value passed to the parameter can -have a one, two, or three part name (e.g. "TABLE_NAME", or "SCHEMA_NAME.TABLE_NAME", or " -DATABASE_NAME.SCHEMA_NAME.TABLE_NAME"). This can be used to support multiple environments (dev, test, prod) or multiple -subject areas within the same Snowflake account. By default, schemachange will not try to create the change history -table, and will fail if the table does not exist. +table can be overriden via a command line argument (`-c` or `--change-history-table`) or the `schemachange-config.yml` +file ( `change-history-table`). The value passed to the parameter can have a one, two, or three part name (e.g. " +TABLE_NAME", or "SCHEMA_NAME.TABLE_NAME", or " DATABASE_NAME.SCHEMA_NAME.TABLE_NAME"). This can be used to support +multiple environments (dev, test, prod) or multiple subject areas within the same Snowflake account. -Additionally, if the `--create-change-history-table` parameter is given, then schemachange will attempt to create the -schema and table associated with the change history table. schemachange will not attempt to create the database for the -change history table, so that must be created ahead of time, even when using the `--create-change-history-table` -parameter. +By default, schemachange will not try to create the change history table, and it will fail if the table does not exist. +This behavior can be altered by passing in the `--create-change-history-table` argument or adding +`create-change-history-table: true` to the `schemachange-config.yml` file. Even with the `--create-change-history-table` +parameter, schemachange will not attempt to create the database for the change history table. That must be created +before running schemachange. The structure of the `CHANGE_HISTORY` table is as follows: @@ -272,119 +276,155 @@ script), in case you choose to create it manually and not use the `--create-chan ```sql CREATE TABLE IF NOT EXISTS SCHEMACHANGE.CHANGE_HISTORY ( - VERSION VARCHAR - ,DESCRIPTION VARCHAR - ,SCRIPT VARCHAR - ,SCRIPT_TYPE VARCHAR - ,CHECKSUM VARCHAR - ,EXECUTION_TIME NUMBER - ,STATUS VARCHAR - ,INSTALLED_BY VARCHAR - ,INSTALLED_ON TIMESTAMP_LTZ + VERSION VARCHAR, + DESCRIPTION VARCHAR, + SCRIPT VARCHAR, + SCRIPT_TYPE VARCHAR, + CHECKSUM VARCHAR, + EXECUTION_TIME NUMBER, + STATUS VARCHAR, + INSTALLED_BY VARCHAR, + INSTALLED_ON TIMESTAMP_LTZ ) ``` ## Authentication -Schemachange supports snowflake's default authenticator, External Oauth, Browswer based SSO and Programmatic SSO options -supported by -the [Snowflake Python Connector](https://docs.snowflake.com/en/user-guide/python-connector-example.html#connecting-to-snowflake). -Set the environment variable `SNOWFLAKE_AUTHENTICATOR` to one of the following -Authentication Option | Expected Value ---- | --- -Default [Password](https://docs.snowflake.com/en/user-guide/python-connector-example.html#connecting-using-the-default-authenticator) -Authenticator | `snowflake` -[Key Pair](https://docs.snowflake.com/en/user-guide/python-connector-example.html#using-key-pair-authentication) -Authenticator| `snowflake` -[External Oauth](https://docs.snowflake.com/en/user-guide/oauth-external.html) | `oauth` -[Browser based SSO](https://docs.snowflake.com/en/user-guide/admin-security-fed-auth-use.html#setting-up-browser-based-sso) | `externalbrowser` -[Programmatic SSO](https://docs.snowflake.com/en/user-guide/admin-security-fed-auth-use.html#native-sso-okta-only) (Okta -Only) | Okta URL endpoint for your Okta account typically in the form `https://.okta.com` -OR `https://.oktapreview.com` - -If an authenticator is unsupported, then schemachange will default to `snowflake`. If the authenticator is `snowflake`, -and both password and key pair values are provided then schemachange will use the password over the key pair values. +Schemachange supports the many of the authentication methods supported by +the [Snowflake Python Connector](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-connect). +In order of priority, the authenticator can be set with: + +1. The `SNOWFLAKE_AUTHENTICATOR` [environment variable](#environment-variables) +2. The `--snowflake-authenticator` [command-line argument](#commands) +3. Setting a `snowflake-authenticator` in the [schemachange-config.yml](#yaml-config-file) file +4. Setting an `authenticator` in the [connections.toml](#connectionstoml-file) file + +The following authenticators are supported: + +- `snowflake`: [Password](#password-authentication) +- `oauth`: [External OAuth](#external-oauth-authentication) +- `externalbrowser`: [Browser-based SSO](#external-browser-authentication) +- `https://.okta.com`: [Okta SSO](#okta-authentication) +- `snowflake_jwt`: [Private Key](#private-key-authentication) + +If an authenticator is unsupported, an exception will be raised. ### Password Authentication -The Snowflake user password for `SNOWFLAKE_USER` is required to be set in the environment variable `SNOWFLAKE_PASSWORD` -prior to calling the script. schemachange will fail if the `SNOWFLAKE_PASSWORD` environment variable is not set. The -environment variable `SNOWFLAKE_AUTHENTICATOR` will be set to `snowflake` if it not explicitly set. +Password authentication is the authenticator. Supplying `snowflake` as your authenticator will set it explicitly. A +password must be supplied in one of the following ways (in order of priority): -_**DEPRECATION NOTICE**: The `SNOWSQL_PWD` environment variable is deprecated but currently still supported. Support for -it will be removed in a later version of schemachange. Please use `SNOWFLAKE_PASSWORD` instead._ +1. The `SNOWFLAKE_PASSWORD` [environment variable](#environment-variables) +2. The `SNOWSQL_PWD` [environment variable](#environment-variables). _**DEPRECATION NOTICE**: The `SNOWSQL_PWD` + environment variable is deprecated but currently still supported. Support for + it will be removed in a later version of schemachange. Please use `SNOWFLAKE_PASSWORD` instead._ +3. Setting a `password` in the [connections.toml](#connectionstoml-file) file -### Private Key Authentication +### External OAuth Authentication -The Snowflake user encrypted private key for `SNOWFLAKE_USER` is required to be in a file with the file path set in the -environment variable `SNOWFLAKE_PRIVATE_KEY_PATH`. Additionally, the password for the encrypted private key file is -required to be set in the environment variable `SNOWFLAKE_PRIVATE_KEY_PASSPHRASE`. If the variable is not set, -schemachange will assume the private key is not encrypted. These two environment variables must be set prior to calling -the script. Schemachange will fail if the `SNOWFLAKE_PRIVATE_KEY_PATH` is not set. +External OAuth authentication can be selected by supplying `oauth` as your authenticator. In order of preference, a +token will need to be supplied or acquired in one of the following ways: -### Oauth Authentication +1. Supply a token via the `SNOWFLAKE_TOKEN` [environment variable](#environment-variables) +2. Supply a token path in one of the following ways (in order of priority). The token path will be passed directly to + the Snowflake connector. + 1. The `--snowflake-token-path` [command-line argument](#commands) + 2. Setting a `snowflake-token-path` in the [schemachange-config.yml](#yaml-config-file) file + 3. Setting a `token_file_path` in the [connections.toml](#connectionstoml-file) file +3. Supply an "OAuth config" in one of the following ways (in order of priority): -An Oauth Configuration can be made in the [YAML Config File](#yaml-config-file) or passing an equivalent json dictionary -to the switch `--oauth-config`. Invoke this method by setting the environment variable `SNOWFLAKE_AUTHENTICATOR` to the -value `oauth` prior to calling schemachange. Since different Oauth providers may require different information the Oauth -configuration uses four named variables that are fed into a POST request to obtain a token. Azure is shown in the -example YAML but other providers should use a similar pattern and request payload contents. + 1. The `--oauth-config` [command-line argument](#commands) + 2. Setting an `oauthconfig` in the [schemachange-config.yml](#yaml-config-file) file -* token-provider-url - The URL of the authenticator resource that will receive the POST request. -* token-response-name - The Expected name of the JSON element containing the Token in the return response from the authenticator resource. -* token-request-payload - The Set of variables passed as a dictionary to the `data` element of the request. -* token-request-headers - The Set of variables passed as a dictionary to the `headers` element of the request. + Since different Oauth providers may require different information the Oauth + configuration uses four named variables that are fed into a POST request to obtain a token. Azure is shown in the + example YAML but other providers should use a similar pattern and request payload contents. -It is recomended to use the YAML file and pass oauth secrets into the configuration using the templating engine instead -of the command line option. + * token-provider-url + The URL of the authenticator resource that will receive the POST request. + * token-response-name + The Expected name of the JSON element containing the Token in the return response from the authenticator resource. + * token-request-payload + The Set of variables passed as a dictionary to the `data` element of the request. + * token-request-headers + The Set of variables passed as a dictionary to the `headers` element of the request. + + It is recommended to use the YAML file and pass oauth secrets into the configuration using the templating engine + instead of the command line option. + + The OAuth POST call will only be made if a token or token filepath isn't discovered. ### External Browser Authentication -External browser authentication can be used for local development by setting the environment -variable `SNOWFLAKE_AUTHENTICATOR` to the value `externalbrowser` prior to calling schemachange. -The client will be prompted to authenticate in a browser that pops up. Refer to +External browser authentication can be selected by supplying `externalbrowser` as your authenticator. The client will be +prompted to authenticate in a browser that pops up. Refer to the [documentation](https://docs.snowflake.com/en/user-guide/admin-security-fed-auth-use.html#setting-up-browser-based-sso) to cache the token to minimize the number of times the browser pops up to authenticate the user. ### Okta Authentication -For clients that do not have a browser, can use the popular SaaS Idp option to connect via Okta. This will require the -Okta URL that you utilize for SSO. -Okta authentication can be used setting the environment variable `SNOWFLAKE_AUTHENTICATOR` to the value of your okta -endpoint as a fully formed URL ( E.g. `https://.okta.com`) prior to calling schemachange. +External browser authentication can be selected by supplying your Okta endpoint as your authenticator (e.g. +`https://.okta.com`). For clients that do not have a browser, can use the popular SaaS Idp option to connect +via Okta. A password must be supplied in one of the following ways (in order of priority): + +1. The `SNOWFLAKE_PASSWORD` [environment variable](#environment-variables) +2. The `SNOWSQL_PWD` [environment variable](#environment-variables). _**DEPRECATION NOTICE**: The `SNOWSQL_PWD` + environment variable is deprecated but currently still supported. Support for + it will be removed in a later version of schemachange. Please use `SNOWFLAKE_PASSWORD` instead._ +3. Setting a `password` in the [connections.toml](#connectionstoml-file) file _** NOTE**: Please disable Okta MFA for the user who uses Native SSO authentication with client drivers. Please consult your Okta administrator for more information._ -## Configuration +### Private Key Authentication + +External browser authentication can be selected by supplying `snowflake_jwt` as your authenticator. The filepath to a +Snowflake user-encrypted private key must be supplied in one of the following ways (in order of priority): -Parameters to schemachange can be supplied in two different ways: +1. The `SNOWFLAKE_PRIVATE_KEY_PATH` [environment variable](#environment-variables) +2. The `--snowflake-private-key-path` [command-line argument](#commands) +3. Setting a `snowflake-private-key-path` in the [schemachange-config.yml](#yaml-config-file) file +4. Setting an `private-key` in the [connections.toml](#connectionstoml-file) file -1. Through a YAML config file -2. Via command line arguments +Additionally, the password for the encrypted private key file is required to be set in the environment variable +`SNOWFLAKE_PRIVATE_KEY_PASSPHRASE`. If the variable is not set, schemachange will assume the private key is not +encrypted. -If supplied by both the command line and the YAML file, The command line overides the YAML values. +## Configuration + +Parameters to schemachange can be supplied in four different ways (in order of priority): -Additionally, regardless of the approach taken, the following paramaters are required to run schemachange: +1. Environment Variables +2. Command Line Arguments +3. YAML config file +4. connections.toml file -* snowflake-account -* snowflake-user -* snowflake-role -* snowflake-warehouse +**Note:** `vars` provided via command-line argument will be merged with vars provided via YAML config. -Plese +Not all parameters can be supplied via every method. + +Please see [Usage Notes for the account Parameter (for the connect Method)](https://docs.snowflake.com/en/user-guide/python-connector-api.html#label-account-format-info) for more details on how to structure the account name. +### Environment Variables + +The Snowflake Python connector subscribes to many variables. Schemachange won't alter environment variables, but it will +consider the following variables to detect an incomplete configuration: + +- SNOWFLAKE_PASSWORD +- _Deprecated_ SNOWSQL_PWD +- SNOWFLAKE_PRIVATE_KEY_PATH +- SNOWFLAKE_AUTHENTICATOR +- SNOWFLAKE_TOKEN +- SNOWFLAKE_DEFAULT_CONNECTION_NAME + ### YAML Config File -schemachange expects the YAML config file to be named `schemachange-config.yml` and looks for it by default in the -current folder. The folder can be overridden by using the `--config-folder` command line argument ( -see [Command Line Arguments](#command-line-arguments) below for more details). +By default, Schemachange expects the YAML config file to be named `schemachange-config.yml`, located in the current +working directory. The YAML file name can be overridden with the +`--config-file-name` [command-line argument](#commands). The folder can be overridden by using the +`--config-folder` [command-line argument](#commands) Here is the list of available configurations in the `schemachange-config.yml` file: @@ -417,10 +457,25 @@ snowflake-database: null # The name of the default schema to use. Can be overridden in the change scripts. snowflake-schema: null +# The Snowflake Authenticator to use. One of snowflake, oauth, externalbrowser, or https://.okta.com +snowflake-authenticator: null + +# Path to file containing private key. +snowflake-private-key-path: null + +# Path to the file containing the OAuth token to be used when authenticating with Snowflake. +snowflake-token-path: null + +# Override the default connections.toml file path at snowflake.connector.constants.CONNECTIONS_FILE (OS specific) +connections-file-path: null + +# Override the default connections.toml connection name. Other connection-related values will override these connection values. +connection-name: null + # Used to override the default name of the change history table (the default is METADATA.SCHEMACHANGE.CHANGE_HISTORY) change-history-table: null -# Define values for the variables to replaced in change scripts +# Define values for the variables to replaced in change scripts. vars supplied via the command line will be merged into YAML-supplied vars vars: var1: 'value1' var2: 'value2' @@ -443,7 +498,7 @@ dry-run: false query-tag: 'QUERY_TAG' # Information for Oauth token requests -oauthconfig: +oauth-config: # url Where token request are posted to token-provider-url: 'https://login.microsoftonline.com/{{ env_var('AZURE_ORG_GUID', 'default') }}/oauth2/v2.0/token' # name of Json entity returned by request @@ -483,24 +538,58 @@ Return the value of the environmental variable if it exists, otherwise raise an {{ env_var('') }} ``` -### Command Line Arguments +### connections.toml File + +A `[connections.toml](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-connect#connecting-using-the-connections-toml-file) +filepath can be supplied in the following ways (in order of priority): + +1. The `--connections-file-path` [command-line argument](#commands) +2. The `connections-file-path` [YAML value](#yaml-config-file) + +A connection name can be supplied in the following ways (in order of priority): + +1. The `SNOWFLAKE_DEFAULT_CONNECTION_NAME` [environment variable](#environment-variables) +2. The `--connection-name` [command-line argument](#commands) +3. The `connection-name` [YAML value](#yaml-config-file) + +Schemachange will consider these connection parameters after environment variables, command line arguments, and the YAML +configuration. + +```txt +[example connection name] +account = "example account" +user = "example user" +role = "example role" +warehouse = "example warehouse" +database = "example database" +schema = "example schema" +authenticator = "example authenticator" +password = "example password" +host = "example host" +port = "example port" +region = "example region" +private-key = "example private-key" +token_file_path = "example token_file_path" +``` + +## Commands Schemachange supports a few subcommands. If the subcommand is not provided it defaults to deploy. This behaviour keeps compatibility with versions prior to 3.2. -#### deploy +### deploy This is the main command that runs the deployment process. ```bash -usage: schemachange deploy [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-m MODULES_FOLDER] [-a SNOWFLAKE_ACCOUNT] [-u SNOWFLAKE_USER] [-r SNOWFLAKE_ROLE] [-w SNOWFLAKE_WAREHOUSE] [-d SNOWFLAKE_DATABASE] [-s SNOWFLAKE_SCHEMA] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] +usage: schemachange deploy [-h] [--config-folder CONFIG_FOLDER] [--config-file-name CONFIG_FILE_NAME] [-f ROOT_FOLDER] [-m MODULES_FOLDER] [-a SNOWFLAKE_ACCOUNT] [-u SNOWFLAKE_USER] [-r SNOWFLAKE_ROLE] [-w SNOWFLAKE_WAREHOUSE] [-d SNOWFLAKE_DATABASE] [-s SNOWFLAKE_SCHEMA] [--snowflake-authenticator SNOWFLAKE_AUTHENTICATOR] [--snowflake-private-key-path SNOWFLAKE_PRIVATE_KEY_PATH] [--snowflake-token-path SNOWFLAKE_TOKEN_PATH] [--connections-file-path CONNECTIONS_FILE_PATH] [--connection-name CONNECTION_NAME] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] ``` | Parameter | Description | |----------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | -h, --help | Show the help message and exit | | --config-folder CONFIG_FOLDER | The folder to look in for the schemachange config file (the default is the current working directory) | -| --config-file=name CONFIG_FILE_NAME | The file name of the schemachange config file. (the default is schemachange-config.yml) | +| --config-file-name CONFIG_FILE_NAME | The file name of the schemachange config file. (the default is schemachange-config.yml) | | -f ROOT_FOLDER, --root-folder ROOT_FOLDER | The root folder for the database change scripts. The default is the current directory. | | -m MODULES_FOLDER, --modules-folder MODULES_FOLDER | The modules folder for jinja macros and templates to be used across mutliple scripts | | -a SNOWFLAKE_ACCOUNT, --snowflake-account SNOWFLAKE_ACCOUNT | The name of the snowflake account (e.g. xy12345.east-us-2.azure). | @@ -515,7 +604,7 @@ usage: schemachange deploy [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] | --connections-file-path CONNECTIONS_FILE_PATH | Override the default [connections.toml](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-connect#connecting-using-the-connections-toml-file) file path at snowflake.connector.constants.CONNECTIONS_FILE (OS specific) | | --connection-name CONNECTION_NAME | Override the default [connections.toml](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-connect#connecting-using-the-connections-toml-file) connection name. Other connection-related values will override these connection values. | | -c CHANGE_HISTORY_TABLE, --change-history-table CHANGE_HISTORY_TABLE | Used to override the default name of the change history table (which is METADATA.SCHEMACHANGE.CHANGE_HISTORY) | -| --vars VARS | Define values for the variables to replaced in change scripts, given in JSON format (e.g. '{"variable1": "value1", "variable2": "value2"}') | +| --vars VARS | Define values for the variables to replaced in change scripts, given in JSON format. Vars supplied via the command line will be merged with YAML-supplied vars (e.g. '{"variable1": "value1", "variable2": "value2"}') | | --create-change-history-table | Create the change history table if it does not exist. The default is 'False'. | | -ac, --autocommit | Enable autocommit feature for DML commands. The default is 'False'. | | -v, --verbose | Display verbose debugging details during execution. The default is 'False'. | @@ -523,7 +612,7 @@ usage: schemachange deploy [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] | --query-tag | A string to include in the QUERY_TAG that is attached to every SQL statement executed. | | --oauth-config | Define values for the variables to Make Oauth Token requests (e.g. {"token-provider-url": "https//...", "token-request-payload": {"client_id": "GUID_xyz",...},... })' | -#### render +### render This subcommand is used to render a single script to the console. It is intended to support the development and troubleshooting of script that use features from the jinja template engine. From 81aaeaea1e98d314c179c670d457dd451eb7f96b Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Thu, 31 Oct 2024 16:15:39 -0700 Subject: [PATCH 25/34] docs: adding to the change log --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d1c933..e99cafd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,12 +7,14 @@ All notable changes to this project will be documented in this file. ### Added - Use of `structlog~=24.1.0` for standard log outputs - Verified Schemachange against Python 3.12 +- Support for connections.toml configurations +- Support for supplying the authenticator, private key path, token path, connections file path, and connection name via the YAML and command-line configurations. ### Changed - Refactored the main cli.py into multiple modules - config, session. - Updated contributing guidelines and demo readme content to help contributors setup local snowflake account to run the github actions in their fork before pushing the PR to upstream repository. - Removed tests against Python 3.8 [End of Life on 2024-10-07](https://devguide.python.org/versions/#supported-versions) - +- Command-line vars are now merged into YAML vars instead of overwriting them entirely ## [3.7.0] - 2024-07-22 ### Added From f659acb973126c00e6b55d506779c1590d6a9fc2 Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Fri, 8 Nov 2024 14:06:49 -0800 Subject: [PATCH 26/34] feat: prioritize cli arguments over environment variables --- schemachange/config/get_merged_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemachange/config/get_merged_config.py b/schemachange/config/get_merged_config.py index dbd8367..b056ee2 100644 --- a/schemachange/config/get_merged_config.py +++ b/schemachange/config/get_merged_config.py @@ -99,8 +99,8 @@ def get_merged_config( "config_vars": config_vars, **{k: v for k, v in connection_kwargs.items() if v is not None}, **{k: v for k, v in yaml_kwargs.items() if v is not None}, - **{k: v for k, v in cli_kwargs.items() if v is not None}, **{k: v for k, v in env_kwargs.items() if v is not None}, + **{k: v for k, v in cli_kwargs.items() if v is not None}, } if connections_file_path is not None: kwargs["connections_file_path"] = connections_file_path From db86db46fb1f97a4bfe1955401ed4af484a0d3f9 Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Fri, 8 Nov 2024 14:17:36 -0800 Subject: [PATCH 27/34] feat: prioritize cli arguments over environment variables --- schemachange/config/get_merged_config.py | 5 ++--- tests/config/test_get_merged_config.py | 12 ++++++------ tests/test_main.py | 4 ++-- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/schemachange/config/get_merged_config.py b/schemachange/config/get_merged_config.py index b056ee2..0d4789d 100644 --- a/schemachange/config/get_merged_config.py +++ b/schemachange/config/get_merged_config.py @@ -44,8 +44,6 @@ def get_merged_config( env_kwargs: dict[str, str] = get_env_kwargs() logger.debug("env_kwargs", **env_kwargs) - connection_name = env_kwargs.pop("connection_name", None) - cli_kwargs = parse_cli_args(sys.argv[1:]) logger.debug("cli_kwargs", **cli_kwargs) @@ -55,8 +53,9 @@ def get_merged_config( file_path=cli_kwargs.pop("connections_file_path", None) ) + connection_name = cli_kwargs.pop("connection_name", None) if connection_name is None: - connection_name = cli_kwargs.pop("connection_name", None) + connection_name = env_kwargs.pop("connection_name", None) config_folder = validate_directory(path=cli_kwargs.pop("config_folder", ".")) config_file_name = cli_kwargs.pop("config_file_name") diff --git a/tests/config/test_get_merged_config.py b/tests/config/test_get_merged_config.py index b86a5d1..8338ab5 100644 --- a/tests/config/test_get_merged_config.py +++ b/tests/config/test_get_merged_config.py @@ -365,12 +365,12 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "snowflake_warehouse": "cli_snowflake_warehouse", "snowflake_database": "cli_snowflake_database", "snowflake_schema": "cli_snowflake_schema", - "snowflake_authenticator": "env_snowflake_authenticator", + "snowflake_authenticator": "cli_snowflake_authenticator", "snowflake_password": "env_snowflake_password", - "snowflake_private_key_path": "env_snowflake_private_key_path", + "snowflake_private_key_path": "cli_snowflake_private_key_path", "snowflake_token_path": "cli_snowflake_token_path", "connections_file_path": Path("cli_connections_file_path"), - "connection_name": "env_connection_name", + "connection_name": "cli_connection_name", "change_history_table": "cli_change_history_table", "create_change_history_table": False, "autocommit": False, @@ -756,8 +756,8 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "snowflake_warehouse": "snowflake-warehouse-from-cli", "snowflake_database": "snowflake-database-from-cli", "snowflake_schema": "snowflake-schema-from-cli", - "snowflake_authenticator": "env_snowflake_authenticator", - "snowflake_private_key_path": "env_snowflake_private_key_path", + "snowflake_authenticator": "snowflake-authenticator-from-cli", + "snowflake_private_key_path": "snowflake-private-key-path-from-cli", "snowflake_token_path": "snowflake-token-path-from-cli", "change_history_table": "change-history-table-from-cli", "config_vars": { @@ -772,7 +772,7 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "query_tag": "query-tag-from-cli", "snowflake_oauth_token": "env_snowflake_token", "oauth_config": {"oauth_config_variable": "cli_oauth_config_value"}, - "connection_name": "anotherconnection", + "connection_name": "myaltconnection", "connections_file_path": assets_path / "alt-connections.toml", "snowflake_password": "env_snowflake_password", }, diff --git a/tests/test_main.py b/tests/test_main.py index 178f2aa..84180f8 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -361,8 +361,8 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "snowflake_schema": get_snowflake_identifier_string( "snowflake-schema-from-cli", "placeholder" ), - "snowflake_authenticator": "snowflake_jwt", - "snowflake_private_key_path": assets_path / "alt_private_key.txt", + "snowflake_authenticator": "externalbrowser", + "snowflake_private_key_path": assets_path / "private_key.txt", "change_history_table": ChangeHistoryTable( database_name="db", schema_name="schema", From 1beb3a53c9fa85b4d1ad7137c6a765cfde06b8aa Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Fri, 8 Nov 2024 14:52:43 -0800 Subject: [PATCH 28/34] feat: remove oauth-config support --- README.md | 55 ++++--------------- schemachange/config/DeployConfig.py | 10 ---- schemachange/config/get_merged_config.py | 6 +- schemachange/config/parse_cli_args.py | 8 --- schemachange/config/utils.py | 22 -------- ...schemachange-config-full-no-connection.yml | 12 ---- tests/config/schemachange-config-full.yml | 12 ---- ...achange-config-partial-with-connection.yml | 12 ---- tests/config/test_DeployConfig.py | 17 ------ tests/config/test_get_merged_config.py | 21 ------- tests/config/test_get_yaml_config.py | 15 ----- tests/config/test_parse_cli_args.py | 1 - tests/test_cli_misc.py | 1 - tests/test_main.py | 42 -------------- 14 files changed, 11 insertions(+), 223 deletions(-) diff --git a/README.md b/README.md index 7787e67..a131ee9 100644 --- a/README.md +++ b/README.md @@ -331,28 +331,12 @@ token will need to be supplied or acquired in one of the following ways: 1. The `--snowflake-token-path` [command-line argument](#commands) 2. Setting a `snowflake-token-path` in the [schemachange-config.yml](#yaml-config-file) file 3. Setting a `token_file_path` in the [connections.toml](#connectionstoml-file) file -3. Supply an "OAuth config" in one of the following ways (in order of priority): - 1. The `--oauth-config` [command-line argument](#commands) - 2. Setting an `oauthconfig` in the [schemachange-config.yml](#yaml-config-file) file - - Since different Oauth providers may require different information the Oauth - configuration uses four named variables that are fed into a POST request to obtain a token. Azure is shown in the - example YAML but other providers should use a similar pattern and request payload contents. - - * token-provider-url - The URL of the authenticator resource that will receive the POST request. - * token-response-name - The Expected name of the JSON element containing the Token in the return response from the authenticator resource. - * token-request-payload - The Set of variables passed as a dictionary to the `data` element of the request. - * token-request-headers - The Set of variables passed as a dictionary to the `headers` element of the request. - - It is recommended to use the YAML file and pass oauth secrets into the configuration using the templating engine - instead of the command line option. - - The OAuth POST call will only be made if a token or token filepath isn't discovered. +**Schemachange no longer supports the `--oauth-config` option.** Prior to the 4.0 release, this library supported +supplying an `--oauth-config` that would be used to fetch an OAuth token via the `requests` library. This required +Schemachange to keep track of connection arguments that could otherwise be passed directly to the Snowflake Python +connector. Maintaining this logic in Schemachange added unnecessary complication to the repo and prevented access to +recent connector parameterization features offered by the Snowflake connector. ### External Browser Authentication @@ -460,13 +444,13 @@ snowflake-schema: null # The Snowflake Authenticator to use. One of snowflake, oauth, externalbrowser, or https://.okta.com snowflake-authenticator: null -# Path to file containing private key. +# Path to file containing private key. snowflake-private-key-path: null # Path to the file containing the OAuth token to be used when authenticating with Snowflake. snowflake-token-path: null -# Override the default connections.toml file path at snowflake.connector.constants.CONNECTIONS_FILE (OS specific) +# Override the default connections.toml file path at snowflake.connector.constants.CONNECTIONS_FILE (OS specific) connections-file-path: null # Override the default connections.toml connection name. Other connection-related values will override these connection values. @@ -496,24 +480,6 @@ dry-run: false # A string to include in the QUERY_TAG that is attached to every SQL statement executed query-tag: 'QUERY_TAG' - -# Information for Oauth token requests -oauth-config: - # url Where token request are posted to - token-provider-url: 'https://login.microsoftonline.com/{{ env_var('AZURE_ORG_GUID', 'default') }}/oauth2/v2.0/token' - # name of Json entity returned by request - token-response-name: 'access_token' - # Headers needed for successful post or other security markings ( multiple labeled items permitted - token-request-headers: - Content-Type: "application/x-www-form-urlencoded" - User-Agent: "python/schemachange" - # Request Payload for Token (it is recommended pass - token-request-payload: - client_id: '{{ env_var('CLIENT_ID', 'default') }}' - username: '{{ env_var('USER_ID', 'default') }}' - password: '{{ env_var('USER_PASSWORD', 'default') }}' - grant_type: 'password' - scope: '{{ env_var('SESSION_SCOPE', 'default') }}' ``` #### Yaml Jinja support @@ -610,7 +576,6 @@ usage: schemachange deploy [-h] [--config-folder CONFIG_FOLDER] [--config-file-n | -v, --verbose | Display verbose debugging details during execution. The default is 'False'. | | --dry-run | Run schemachange in dry run mode. The default is 'False'. | | --query-tag | A string to include in the QUERY_TAG that is attached to every SQL statement executed. | -| --oauth-config | Define values for the variables to Make Oauth Token requests (e.g. {"token-provider-url": "https//...", "token-request-payload": {"client_id": "GUID_xyz",...},... })' | ### render @@ -657,13 +622,13 @@ schemachange is a single python script located at [schemachange/cli.py](schemach follows: ``` -python schemachange/cli.py [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-a SNOWFLAKE_ACCOUNT] [-u SNOWFLAKE_USER] [-r SNOWFLAKE_ROLE] [-w SNOWFLAKE_WAREHOUSE] [-d SNOWFLAKE_DATABASE] [-s SNOWFLAKE_SCHEMA] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] [--oauth-config OUATH_CONFIG] +python schemachange/cli.py [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-a SNOWFLAKE_ACCOUNT] [-u SNOWFLAKE_USER] [-r SNOWFLAKE_ROLE] [-w SNOWFLAKE_WAREHOUSE] [-d SNOWFLAKE_DATABASE] [-s SNOWFLAKE_SCHEMA] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] ``` Or if installed via `pip`, it can be executed as follows: ``` -schemachange [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-a SNOWFLAKE_ACCOUNT] [-u SNOWFLAKE_USER] [-r SNOWFLAKE_ROLE] [-w SNOWFLAKE_WAREHOUSE] [-d SNOWFLAKE_DATABASE] [-s SNOWFLAKE_SCHEMA] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] [--oauth-config OUATH_CONFIG] +schemachange [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-a SNOWFLAKE_ACCOUNT] [-u SNOWFLAKE_USER] [-r SNOWFLAKE_ROLE] [-w SNOWFLAKE_WAREHOUSE] [-d SNOWFLAKE_DATABASE] [-s SNOWFLAKE_SCHEMA] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] ``` The [demo](demo) folder in this project repository contains three schemachange demo projects for you to try out. These @@ -702,7 +667,7 @@ If your build agent has a recent version of python 3 installed, the script can b ```bash pip install schemachange --upgrade -schemachange [-h] [-f ROOT_FOLDER] -a SNOWFLAKE_ACCOUNT -u SNOWFLAKE_USER -r SNOWFLAKE_ROLE -w SNOWFLAKE_WAREHOUSE [-d SNOWFLAKE_DATABASE] [-s SNOWFLAKE_SCHEMA] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] [--oauth-config OUATH_CONFIG] +schemachange [-h] [-f ROOT_FOLDER] -a SNOWFLAKE_ACCOUNT -u SNOWFLAKE_USER -r SNOWFLAKE_ROLE -w SNOWFLAKE_WAREHOUSE [-d SNOWFLAKE_DATABASE] [-s SNOWFLAKE_SCHEMA] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] ``` Or if you prefer docker, set the environment variables and run like so: diff --git a/schemachange/config/DeployConfig.py b/schemachange/config/DeployConfig.py index 3cffeaf..3446d4c 100644 --- a/schemachange/config/DeployConfig.py +++ b/schemachange/config/DeployConfig.py @@ -9,7 +9,6 @@ from schemachange.config.utils import ( get_snowflake_identifier_string, validate_file_path, - get_oauth_token, ) @@ -70,20 +69,11 @@ def factory( # If set by an environment variable, pop snowflake_token_path from kwargs if "snowflake_oauth_token" in kwargs: kwargs.pop("snowflake_token_path", None) - kwargs.pop("oauth_config", None) # Load it from a file, if provided elif "snowflake_token_path" in kwargs: - kwargs.pop("oauth_config", None) oauth_token_path = kwargs.pop("snowflake_token_path") with open(oauth_token_path) as f: kwargs["snowflake_oauth_token"] = f.read() - # Make the oauth call if authenticator == "oauth" - - elif "oauth_config" in kwargs: - oauth_config = kwargs.pop("oauth_config") - authenticator = kwargs.get("snowflake_authenticator") - if authenticator is not None and authenticator.lower() == "oauth": - kwargs["snowflake_oauth_token"] = get_oauth_token(oauth_config) change_history_table = ChangeHistoryTable.from_str( table_str=change_history_table diff --git a/schemachange/config/get_merged_config.py b/schemachange/config/get_merged_config.py index 0d4789d..88bfe0d 100644 --- a/schemachange/config/get_merged_config.py +++ b/schemachange/config/get_merged_config.py @@ -18,13 +18,9 @@ def get_yaml_config_kwargs(config_file_path: Optional[Path]) -> dict: - # TODO: I think the configuration key for oauthconfig should be oauth-config. - # This looks like a bug in the current state of the repo to me - # load YAML inputs and convert kebabs to snakes kwargs = { - k.replace("-", "_").replace("oauthconfig", "oauth_config"): v - for (k, v) in load_yaml_config(config_file_path).items() + k.replace("-", "_"): v for (k, v) in load_yaml_config(config_file_path).items() } if "verbose" in kwargs: diff --git a/schemachange/config/parse_cli_args.py b/schemachange/config/parse_cli_args.py index f6a31c1..c3376ae 100644 --- a/schemachange/config/parse_cli_args.py +++ b/schemachange/config/parse_cli_args.py @@ -215,14 +215,6 @@ def parse_cli_args(args) -> dict: help="The string to add to the Snowflake QUERY_TAG session value for each query executed", required=False, ) - parser_deploy.add_argument( - "--oauth-config", - type=json.loads, - help='Define values for the variables to Make Oauth Token requests (e.g. {"token-provider-url": ' - '"https//...", "token-request-payload": {"client_id": "GUID_xyz",...},... })', - required=False, - ) - parser_render = subcommands.add_parser( "render", description="Renders a script to the console, used to check and verify jinja output from scripts.", diff --git a/schemachange/config/utils.py b/schemachange/config/utils.py index b90af1c..f6eaff3 100644 --- a/schemachange/config/utils.py +++ b/schemachange/config/utils.py @@ -5,9 +5,7 @@ from pathlib import Path from typing import Any -import json -import requests import jinja2 import jinja2.ext import structlog @@ -220,23 +218,3 @@ def get_env_kwargs() -> dict[str, str]: "connection_name": os.getenv("SNOWFLAKE_DEFAULT_CONNECTION_NAME"), } return {k: v for k, v in env_kwargs.items() if v is not None} - - -def get_oauth_token(oauth_config: dict): - req_info = { - "url": oauth_config["token-provider-url"], - "headers": oauth_config["token-request-headers"], - "data": oauth_config["token-request-payload"], - } - token_name = oauth_config["token-response-name"] - response = requests.post(**req_info) - response_dict = json.loads(response.text) - try: - return response_dict[token_name] - except KeyError: - keys = ", ".join(response_dict.keys()) - errormessage = f"Response Json contains keys: {keys} \n but not {token_name}" - # if there is an error passed with the response include that - if "error_description" in response_dict.keys(): - errormessage = f"{errormessage}\n error description: {response_dict['error_description']}" - raise KeyError(errormessage) diff --git a/tests/config/schemachange-config-full-no-connection.yml b/tests/config/schemachange-config-full-no-connection.yml index 7997fef..ad85113 100644 --- a/tests/config/schemachange-config-full-no-connection.yml +++ b/tests/config/schemachange-config-full-no-connection.yml @@ -19,15 +19,3 @@ autocommit: false verbose: false dry-run: false query-tag: 'query-tag-from-yaml' -oauthconfig: - token-provider-url: 'token-provider-url-from-yaml' - token-response-name: 'token-response-name-from-yaml' - token-request-headers: - Content-Type: 'Content-Type-from-yaml' - User-Agent: 'User-Agent-from-yaml' - token-request-payload: - client_id: 'id-from-yaml' - username: 'username-from-yaml' - password: 'password-from-yaml' - grant_type: 'type-from-yaml' - scope: 'scope-from-yaml' diff --git a/tests/config/schemachange-config-full.yml b/tests/config/schemachange-config-full.yml index a3a2c15..49f3a57 100644 --- a/tests/config/schemachange-config-full.yml +++ b/tests/config/schemachange-config-full.yml @@ -21,15 +21,3 @@ autocommit: false verbose: false dry-run: false query-tag: 'query-tag-from-yaml' -oauthconfig: - token-provider-url: 'token-provider-url-from-yaml' - token-response-name: 'token-response-name-from-yaml' - token-request-headers: - Content-Type: 'Content-Type-from-yaml' - User-Agent: 'User-Agent-from-yaml' - token-request-payload: - client_id: 'id-from-yaml' - username: 'username-from-yaml' - password: 'password-from-yaml' - grant_type: 'type-from-yaml' - scope: 'scope-from-yaml' diff --git a/tests/config/schemachange-config-partial-with-connection.yml b/tests/config/schemachange-config-partial-with-connection.yml index bc74981..863925b 100644 --- a/tests/config/schemachange-config-partial-with-connection.yml +++ b/tests/config/schemachange-config-partial-with-connection.yml @@ -12,15 +12,3 @@ autocommit: false verbose: false dry-run: false query-tag: 'query-tag-from-yaml' -oauthconfig: - token-provider-url: 'token-provider-url-from-yaml' - token-response-name: 'token-response-name-from-yaml' - token-request-headers: - Content-Type: 'Content-Type-from-yaml' - User-Agent: 'User-Agent-from-yaml' - token-request-payload: - client_id: 'id-from-yaml' - username: 'username-from-yaml' - password: 'password-from-yaml' - grant_type: 'type-from-yaml' - scope: 'scope-from-yaml' diff --git a/tests/config/test_DeployConfig.py b/tests/config/test_DeployConfig.py index 419782f..cbc7441 100644 --- a/tests/config/test_DeployConfig.py +++ b/tests/config/test_DeployConfig.py @@ -26,7 +26,6 @@ "snowflake_schema": "some_snowflake_schema", "change_history_table": "some_history_table", "query_tag": "some_query_tag", - "oauth_config": {"some": "values"}, } @@ -136,22 +135,6 @@ def test_check_for_deploy_args_oauth_with_file_happy_path(_): assert config.snowflake_oauth_token == "my-oauth-token-from-a-file" -@mock.patch("schemachange.config.DeployConfig.get_oauth_token") -def test_check_for_deploy_args_oauth_with_request_happy_path(mock_get_oauth_token): - oauth_token = "my-oauth-token-from-a-request" - mock_get_oauth_token.return_value = oauth_token - oauth_config = {"my_oauth_config": "values"} - config = DeployConfig.factory( - **minimal_deploy_config_kwargs, - snowflake_authenticator="oauth", - oauth_config=oauth_config, - config_file_path=Path("."), - ) - config.check_for_deploy_args() - assert config.snowflake_oauth_token == oauth_token - mock_get_oauth_token.call_args.args[0] == oauth_config - - def test_check_for_deploy_args_externalbrowser_happy_path(): config = DeployConfig.factory( **minimal_deploy_config_kwargs, diff --git a/tests/config/test_get_merged_config.py b/tests/config/test_get_merged_config.py index 8338ab5..236f98c 100644 --- a/tests/config/test_get_merged_config.py +++ b/tests/config/test_get_merged_config.py @@ -1,4 +1,3 @@ -import json import logging import os @@ -127,7 +126,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "autocommit": True, "dry_run": True, "query_tag": "yaml_query_tag", - "oauth_config": {"oauth_config_variable": "yaml_oauth_config_value"}, }, { # connection_kwargs "snowflake_account": "connection_snowflake_account", @@ -169,7 +167,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "autocommit": True, "dry_run": True, "query_tag": "yaml_query_tag", - "oauth_config": {"oauth_config_variable": "yaml_oauth_config_value"}, }, id="Deploy: all yaml, all connection_kwargs", ), @@ -201,7 +198,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "autocommit": False, "dry_run": False, "query_tag": "cli_query_tag", - "oauth_config": {"oauth_config_variable": "cli_oauth_config_value"}, }, { # yaml_kwargs "root_folder": "yaml_root_folder", @@ -228,7 +224,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "autocommit": True, "dry_run": True, "query_tag": "yaml_query_tag", - "oauth_config": {"oauth_config_variable": "yaml_oauth_config_value"}, }, { # connection_kwargs "snowflake_account": "connection_snowflake_account", @@ -270,7 +265,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "autocommit": False, "dry_run": False, "query_tag": "cli_query_tag", - "oauth_config": {"oauth_config_variable": "cli_oauth_config_value"}, }, id="Deploy: all cli, all yaml, all connection_kwargs", ), @@ -307,7 +301,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "autocommit": False, "dry_run": False, "query_tag": "cli_query_tag", - "oauth_config": {"oauth_config_variable": "cli_oauth_config_value"}, }, { # yaml_kwargs "root_folder": "yaml_root_folder", @@ -334,7 +327,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "autocommit": True, "dry_run": True, "query_tag": "yaml_query_tag", - "oauth_config": {"oauth_config_variable": "yaml_oauth_config_value"}, }, { # connection_kwargs "snowflake_account": "connection_snowflake_account", @@ -376,7 +368,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "autocommit": False, "dry_run": False, "query_tag": "cli_query_tag", - "oauth_config": {"oauth_config_variable": "cli_oauth_config_value"}, }, id="Deploy: all env, all cli, all yaml, all connection_kwargs", ), @@ -486,8 +477,6 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "--dry-run", "--query-tag", "query-tag-from-cli", - "--oauth-config", - json.dumps({"oauth_config_variable": "cli_oauth_config_value"}), ], { # expected "subcommand": "deploy", @@ -514,7 +503,6 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "log_level": logging.INFO, "dry_run": True, "query_tag": "query-tag-from-cli", - "oauth_config": {"oauth_config_variable": "cli_oauth_config_value"}, "connection_name": "myaltconnection", "connections_file_path": assets_path / "alt-connections.toml", "snowflake_password": alt_connection["password"], @@ -558,7 +546,6 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "autocommit", "dry_run", "query_tag", - "oauth_config", ] }, }, @@ -604,7 +591,6 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "autocommit", "dry_run", "query_tag", - "oauth_config", "connection_name", ] }, @@ -655,8 +641,6 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "--dry-run", "--query-tag", "query-tag-from-cli", - "--oauth-config", - json.dumps({"oauth_config_variable": "cli_oauth_config_value"}), ], { # expected "subcommand": "deploy", @@ -684,7 +668,6 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "log_level": logging.INFO, "dry_run": True, "query_tag": "query-tag-from-cli", - "oauth_config": {"oauth_config_variable": "cli_oauth_config_value"}, "connection_name": "myaltconnection", "connections_file_path": assets_path / "alt-connections.toml", "snowflake_password": alt_connection["password"], @@ -741,8 +724,6 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "--dry-run", "--query-tag", "query-tag-from-cli", - "--oauth-config", - json.dumps({"oauth_config_variable": "cli_oauth_config_value"}), ], { # expected "subcommand": "deploy", @@ -771,7 +752,6 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "dry_run": True, "query_tag": "query-tag-from-cli", "snowflake_oauth_token": "env_snowflake_token", - "oauth_config": {"oauth_config_variable": "cli_oauth_config_value"}, "connection_name": "myaltconnection", "connections_file_path": assets_path / "alt-connections.toml", "snowflake_password": "env_snowflake_password", @@ -852,7 +832,6 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "autocommit", "dry_run", "query_tag", - "oauth_config", "connection_name", ] }, diff --git a/tests/config/test_get_yaml_config.py b/tests/config/test_get_yaml_config.py index 9e8d4ac..a389163 100644 --- a/tests/config/test_get_yaml_config.py +++ b/tests/config/test_get_yaml_config.py @@ -94,18 +94,3 @@ def test_get_yaml_config(_): assert yaml_config["dry_run"] is False assert yaml_config["config_vars"] == {"var1": "from_yaml", "var2": "also_from_yaml"} - assert yaml_config["oauth_config"] == { - "token-provider-url": "token-provider-url-from-yaml", - "token-request-headers": { - "Content-Type": "Content-Type-from-yaml", - "User-Agent": "User-Agent-from-yaml", - }, - "token-request-payload": { - "client_id": "id-from-yaml", - "grant_type": "type-from-yaml", - "password": "password-from-yaml", - "scope": "scope-from-yaml", - "username": "username-from-yaml", - }, - "token-response-name": "token-response-name-from-yaml", - } diff --git a/tests/config/test_parse_cli_args.py b/tests/config/test_parse_cli_args.py index 545002b..d181707 100644 --- a/tests/config/test_parse_cli_args.py +++ b/tests/config/test_parse_cli_args.py @@ -68,7 +68,6 @@ def test_parse_args_deploy_names(): ("--connection-name", "some_connection_name", "some_connection_name"), ("--change-history-table", "some_history_table", "some_history_table"), ("--query-tag", "some_query_tag", "some_query_tag"), - ("--oauth-config", json.dumps({"some": "values"}), {"some": "values"}), ] for arg, value, expected_value in valued_test_args: diff --git a/tests/test_cli_misc.py b/tests/test_cli_misc.py index e2b9d7e..08e9066 100644 --- a/tests/test_cli_misc.py +++ b/tests/test_cli_misc.py @@ -3,7 +3,6 @@ import pytest from schemachange.cli import SCHEMACHANGE_VERSION, SNOWFLAKE_APPLICATION_NAME -from schemachange.config.BaseConfig import BaseConfig from schemachange.config.ChangeHistoryTable import ChangeHistoryTable from schemachange.config.utils import get_snowflake_identifier_string from schemachange.deploy import alphanum_convert, get_alphanum_key, sorted_alphanumeric diff --git a/tests/test_main.py b/tests/test_main.py index 84180f8..bfaf266 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,6 +1,5 @@ from __future__ import annotations -import json import logging import os import tomlkit @@ -147,8 +146,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "--dry-run", "--query-tag", "query-tag-from-cli", - "--oauth-config", - json.dumps({"oauth_config_variable": "cli_oauth_config_value"}), ], { # expected "subcommand": "deploy", @@ -240,8 +237,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "--dry-run", "--query-tag", "query-tag-from-cli", - "--oauth-config", - json.dumps({"oauth_config_variable": "cli_oauth_config_value"}), ], { # expected "subcommand": "deploy", @@ -338,8 +333,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "--dry-run", "--query-tag", "query-tag-from-cli", - "--oauth-config", - json.dumps({"oauth_config_variable": "cli_oauth_config_value"}), ], { # expected "subcommand": "deploy", @@ -396,8 +389,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "oauth", "--snowflake-token-path", str(assets_path / "oauth_token_path.txt"), - "--oauth-config", - json.dumps({"oauth_config_variable": "cli_oauth_config_value"}), ], { **default_deploy_config, @@ -423,8 +414,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "oauth", "--snowflake-token-path", str(assets_path / "oauth_token_path.txt"), - "--oauth-config", - json.dumps({"oauth_config_variable": "cli_oauth_config_value"}), ], { **default_deploy_config, @@ -439,31 +428,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: id="deploy: oauth file", ) -deploy_snowflake_oauth_request = pytest.param( - "schemachange.cli.deploy", - {}, - [ - "schemachange", - "deploy", - *required_args, - "--snowflake-authenticator", - "oauth", - "--oauth-config", - json.dumps({"oauth_config_variable": "cli_oauth_config_value"}), - ], - { - **default_deploy_config, - "snowflake_account": "account", - "snowflake_user": "user", - "snowflake_warehouse": "warehouse", - "snowflake_role": "role", - "snowflake_authenticator": "oauth", - "snowflake_oauth_token": "requested_oauth_token", - }, - None, - id="deploy: oauth request", -) - render_only_required = pytest.param( "schemachange.cli.render", {}, @@ -511,19 +475,13 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: deploy_all_env_all_cli, deploy_snowflake_oauth_env_var, deploy_snowflake_oauth_file, - deploy_snowflake_oauth_request, render_only_required, render_all_cli_arg_names, ], ) -@mock.patch( - "schemachange.config.DeployConfig.get_oauth_token", - return_value="requested_oauth_token", -) @mock.patch("schemachange.session.SnowflakeSession.snowflake.connector.connect") def test_main_deploy_subcommand_given_arguments_make_sure_arguments_set_on_call( _, - __, to_mock: str, env_vars: dict[str, str], cli_args: list[str], From eb8882ba558f9f12df0ecfba4f0f7046391c0422 Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Thu, 14 Nov 2024 09:58:48 -0800 Subject: [PATCH 29/34] feat: deprecate cli connection arguments --- schemachange/config/parse_cli_args.py | 30 +++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/schemachange/config/parse_cli_args.py b/schemachange/config/parse_cli_args.py index c3376ae..60a1a6d 100644 --- a/schemachange/config/parse_cli_args.py +++ b/schemachange/config/parse_cli_args.py @@ -3,6 +3,7 @@ import argparse import json import logging +import sys from enum import Enum import structlog @@ -10,6 +11,25 @@ logger = structlog.getLogger(__name__) +class DeprecateConnectionArgAction(argparse.Action): + def __init__(self, *args, **kwargs): + self.call_count = 0 + if "help" in kwargs: + kwargs["help"] = ( + f'[DEPRECATED - Set in connections.toml instead.] {kwargs["help"]}' + ) + super().__init__(*args, **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + if self.call_count == 0: + sys.stderr.write( + f"{', '.join(self.option_strings)} is deprecated. It will be ignored in future versions.\n" + ) + sys.stderr.write(self.help + "\n") + self.call_count += 1 + setattr(namespace, self.dest, values) + + class EnumAction(argparse.Action): """ Argparse action for handling Enums @@ -101,12 +121,14 @@ def parse_cli_args(args) -> dict: subcommands = parser.add_subparsers(dest="subcommand") parser_deploy = subcommands.add_parser("deploy", parents=[parent_parser]) + parser_deploy.register("action", "deprecate", DeprecateConnectionArgAction) parser_deploy.add_argument( "-a", "--snowflake-account", type=str, help="The name of the snowflake account (e.g. xy12345.east-us-2.azure)", required=False, + action="deprecate", ) parser_deploy.add_argument( "-u", @@ -114,6 +136,7 @@ def parse_cli_args(args) -> dict: type=str, help="The name of the snowflake user", required=False, + action="deprecate", ) parser_deploy.add_argument( "-r", @@ -121,6 +144,7 @@ def parse_cli_args(args) -> dict: type=str, help="The name of the default role to use", required=False, + action="deprecate", ) parser_deploy.add_argument( "-w", @@ -128,6 +152,7 @@ def parse_cli_args(args) -> dict: type=str, help="The name of the default warehouse to use. Can be overridden in the change scripts.", required=False, + action="deprecate", ) parser_deploy.add_argument( "-d", @@ -135,6 +160,7 @@ def parse_cli_args(args) -> dict: type=str, help="The name of the default database to use. Can be overridden in the change scripts.", required=False, + action="deprecate", ) parser_deploy.add_argument( "-s", @@ -142,6 +168,7 @@ def parse_cli_args(args) -> dict: type=str, help="The name of the default schema to use. Can be overridden in the change scripts.", required=False, + action="deprecate", ) parser_deploy.add_argument( "-A", @@ -149,6 +176,7 @@ def parse_cli_args(args) -> dict: type=str, help="The Snowflake Authenticator to use. One of snowflake, oauth, externalbrowser, or https://.okta.com", required=False, + action="deprecate", ) parser_deploy.add_argument( "-k", @@ -156,6 +184,7 @@ def parse_cli_args(args) -> dict: type=str, help="Path to file containing private key.", required=False, + action="deprecate", ) parser_deploy.add_argument( "-t", @@ -163,6 +192,7 @@ def parse_cli_args(args) -> dict: type=str, help="Path to the file containing the OAuth token to be used when authenticating with Snowflake.", required=False, + action="deprecate", ) parser_deploy.add_argument( "--connections-file-path", From f29925afe004b704da9e94087bc64237fd439a44 Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Thu, 14 Nov 2024 11:04:51 -0800 Subject: [PATCH 30/34] feat: drive SnowflakeSession attributes from connection instead of arguments, comment later removals --- schemachange/session/SnowflakeSession.py | 69 ++++++++++++++---------- 1 file changed, 40 insertions(+), 29 deletions(-) diff --git a/schemachange/session/SnowflakeSession.py b/schemachange/session/SnowflakeSession.py index 06e8e2c..5694456 100644 --- a/schemachange/session/SnowflakeSession.py +++ b/schemachange/session/SnowflakeSession.py @@ -14,10 +14,10 @@ class SnowflakeSession: account: str - user: str | None - role: str | None - warehouse: str | None - database: str | None + user: str | None # TODO: user: str when connections.toml is enforced + role: str | None # TODO: role: str when connections.toml is enforced + warehouse: str | None # TODO: warehouse: str when connections.toml is enforced + database: str | None # TODO: database: str when connections.toml is enforced schema: str | None autocommit: bool change_history_table: ChangeHistoryTable @@ -35,22 +35,19 @@ def __init__( application: str, change_history_table: ChangeHistoryTable, logger: structlog.BoundLogger, - account: str | None = None, - user: str | None = None, - role: str | None = None, - warehouse: str | None = None, - database: str | None = None, - schema: str | None = None, + connection_name: str | None = None, + connections_file_path: str | None = None, + account: str | None = None, # TODO: Remove when connections.toml is enforced + user: str | None = None, # TODO: Remove when connections.toml is enforced + role: str | None = None, # TODO: Remove when connections.toml is enforced + warehouse: str | None = None, # TODO: Remove when connections.toml is enforced + database: str | None = None, # TODO: Remove when connections.toml is enforced + schema: str | None = None, # TODO: Remove when connections.toml is enforced query_tag: str | None = None, autocommit: bool = False, - **kwargs, + **kwargs, # TODO: Remove when connections.toml is enforced ): - self.account = account - self.user = user - self.role = role - self.warehouse = warehouse - self.database = database - self.schema = schema + self.change_history_table = change_history_table self.autocommit = autocommit self.logger = logger @@ -60,24 +57,38 @@ def __init__( self.session_parameters["QUERY_TAG"] += f";{query_tag}" connect_kwargs = { - "account": self.account, - "user": self.user, - "database": self.database, - "schema": self.schema, - "role": self.role, - "warehouse": self.warehouse, - "private_key_file": kwargs.get("private_key_path"), - "token": kwargs.get("oauth_token"), - "password": kwargs.get("password"), - "authenticator": kwargs.get("authenticator"), - "connection_name": kwargs.get("connection_name"), - "connections_file_path": kwargs.get("connections_file_path"), + "account": account, # TODO: Remove when connections.toml is enforced + "user": user, # TODO: Remove when connections.toml is enforced + "database": database, # TODO: Remove when connections.toml is enforced + "schema": schema, # TODO: Remove when connections.toml is enforced + "role": role, # TODO: Remove when connections.toml is enforced + "warehouse": warehouse, # TODO: Remove when connections.toml is enforced + "private_key_file": kwargs.get( + "private_key_path" + ), # TODO: Remove when connections.toml is enforced + "token": kwargs.get( + "oauth_token" + ), # TODO: Remove when connections.toml is enforced + "password": kwargs.get( + "password" + ), # TODO: Remove when connections.toml is enforced + "authenticator": kwargs.get( + "authenticator" + ), # TODO: Remove when connections.toml is enforced + "connection_name": connection_name, + "connections_file_path": connections_file_path, "application": application, "session_parameters": self.session_parameters, } self.logger.debug("snowflake.connector.connect kwargs", **connect_kwargs) self.con = snowflake.connector.connect(**connect_kwargs) print(f"Current session ID: {self.con.session_id}") + self.account = self.con.account + self.user = self.con.user + self.role = self.con.role + self.warehouse = self.con.warehouse + self.database = self.con.database + self.schema = self.con.schema if not self.autocommit: self.con.autocommit(False) From f35f1a648345e8dfa0674eb19a1c4015df6375e6 Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Thu, 14 Nov 2024 15:28:17 -0800 Subject: [PATCH 31/34] feat: defer to snowflake python connector for connection arguments --- README.md | 217 +++++------------ schemachange/cli.py | 1 - schemachange/config/BaseConfig.py | 3 - schemachange/config/DeployConfig.py | 118 ++------- schemachange/config/get_merged_config.py | 28 +-- schemachange/config/parse_cli_args.py | 24 -- schemachange/config/utils.py | 57 ----- schemachange/session/SnowflakeSession.py | 1 - tests/config/alt-connections.toml | 28 --- tests/config/alt_private_key.txt | 1 - tests/config/connections.toml | 14 -- tests/config/oauth_token_path.txt | 1 - tests/config/private_key.txt | 1 - ...schemachange-config-full-no-connection.yml | 3 - tests/config/schemachange-config-full.yml | 3 - tests/config/test_DeployConfig.py | 125 ---------- tests/config/test_get_merged_config.py | 227 ++---------------- tests/config/test_parse_cli_args.py | 33 --- tests/config/test_utils.py | 92 +------ tests/test_main.py | 204 ++++++---------- 20 files changed, 183 insertions(+), 998 deletions(-) delete mode 100644 tests/config/alt-connections.toml delete mode 100644 tests/config/alt_private_key.txt delete mode 100644 tests/config/connections.toml delete mode 100644 tests/config/oauth_token_path.txt delete mode 100644 tests/config/private_key.txt diff --git a/README.md b/README.md index a131ee9..cd19fe5 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,6 @@ support or warranty. 1. [Okta Authentication](#okta-authentication) 1. [Private Key Authentication](#private-key-authentication) 1. [Configuration](#configuration) - 1. [Environment Variables](#environment-variables) 1. [YAML Config File](#yaml-config-file) 1. [Yaml Jinja support](#yaml-jinja-support) 1. [connections.toml File](#connectionstoml-file) @@ -123,7 +122,7 @@ numbers separated by periods. Here are a few valid version strings: Every script within a database folder must have a unique version number. schemachange will check for duplicate version numbers and throw an error if it finds any. This helps to ensure that developers who are working in parallel don't -accidently (re-)use the same version number. +accidentally (re-)use the same version number. ### Repeatable Script Naming @@ -168,7 +167,8 @@ schemachange is designed to be very lightweight and not impose too many limitati number of SQL statements within it and must supply the necessary context, like database and schema names. The context can be supplied by using an explicit `USE ` command or by naming all objects with a three-part name (`..`). schemachange will simply run the contents of each script against -the target Snowflake account, in the correct order. +the target Snowflake account, in the correct order. After each script, Schemachange will execute "reset" the context ( +role, warehouse, database, schema) to the values used to configure the connector. ### Using Variables in Scripts @@ -226,7 +226,7 @@ These files can be stored in the root-folder but schemachange also provides a se folder `--modules-folder`. This allows common logic to be stored outside of the main changes scripts. The [demo/citibike_demo_jinja](demo/citibike_demo_jinja) has a simple example that demonstrates this. -The Jinja autoescaping feature is disabled in schemachange, this feature in Jinja is currently designed for where the +The Jinja auto-escaping feature is disabled in schemachange, this feature in Jinja is currently designed for where the output language is HTML/XML. So if you are using schemachange with untrusted inputs you will need to handle this within your change scripts. @@ -292,12 +292,7 @@ CREATE TABLE IF NOT EXISTS SCHEMACHANGE.CHANGE_HISTORY Schemachange supports the many of the authentication methods supported by the [Snowflake Python Connector](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-connect). -In order of priority, the authenticator can be set with: - -1. The `SNOWFLAKE_AUTHENTICATOR` [environment variable](#environment-variables) -2. The `--snowflake-authenticator` [command-line argument](#commands) -3. Setting a `snowflake-authenticator` in the [schemachange-config.yml](#yaml-config-file) file -4. Setting an `authenticator` in the [connections.toml](#connectionstoml-file) file +The authenticator can be set by setting an `authenticator` in the [connections.toml](#connectionstoml-file) file The following authenticators are supported: @@ -311,26 +306,13 @@ If an authenticator is unsupported, an exception will be raised. ### Password Authentication -Password authentication is the authenticator. Supplying `snowflake` as your authenticator will set it explicitly. A -password must be supplied in one of the following ways (in order of priority): - -1. The `SNOWFLAKE_PASSWORD` [environment variable](#environment-variables) -2. The `SNOWSQL_PWD` [environment variable](#environment-variables). _**DEPRECATION NOTICE**: The `SNOWSQL_PWD` - environment variable is deprecated but currently still supported. Support for - it will be removed in a later version of schemachange. Please use `SNOWFLAKE_PASSWORD` instead._ -3. Setting a `password` in the [connections.toml](#connectionstoml-file) file +Password authentication is the default authenticator. Supplying `snowflake` as your authenticator will set it +explicitly. A `password` must be supplied in the [connections.toml](#connectionstoml-file) file ### External OAuth Authentication -External OAuth authentication can be selected by supplying `oauth` as your authenticator. In order of preference, a -token will need to be supplied or acquired in one of the following ways: - -1. Supply a token via the `SNOWFLAKE_TOKEN` [environment variable](#environment-variables) -2. Supply a token path in one of the following ways (in order of priority). The token path will be passed directly to - the Snowflake connector. - 1. The `--snowflake-token-path` [command-line argument](#commands) - 2. Setting a `snowflake-token-path` in the [schemachange-config.yml](#yaml-config-file) file - 3. Setting a `token_file_path` in the [connections.toml](#connectionstoml-file) file +External OAuth authentication can be selected by supplying `oauth` as your authenticator. A `token_file_path` must be +supplied in the [connections.toml](#connectionstoml-file) file **Schemachange no longer supports the `--oauth-config` option.** Prior to the 4.0 release, this library supported supplying an `--oauth-config` that would be used to fetch an OAuth token via the `requests` library. This required @@ -349,13 +331,7 @@ to cache the token to minimize the number of times the browser pops up to authen External browser authentication can be selected by supplying your Okta endpoint as your authenticator (e.g. `https://.okta.com`). For clients that do not have a browser, can use the popular SaaS Idp option to connect -via Okta. A password must be supplied in one of the following ways (in order of priority): - -1. The `SNOWFLAKE_PASSWORD` [environment variable](#environment-variables) -2. The `SNOWSQL_PWD` [environment variable](#environment-variables). _**DEPRECATION NOTICE**: The `SNOWSQL_PWD` - environment variable is deprecated but currently still supported. Support for - it will be removed in a later version of schemachange. Please use `SNOWFLAKE_PASSWORD` instead._ -3. Setting a `password` in the [connections.toml](#connectionstoml-file) file +via Okta. A `password` must be supplied in the [connections.toml](#connectionstoml-file) file _** NOTE**: Please disable Okta MFA for the user who uses Native SSO authentication with client drivers. Please consult your Okta administrator for more information._ @@ -363,45 +339,42 @@ your Okta administrator for more information._ ### Private Key Authentication External browser authentication can be selected by supplying `snowflake_jwt` as your authenticator. The filepath to a -Snowflake user-encrypted private key must be supplied in one of the following ways (in order of priority): - -1. The `SNOWFLAKE_PRIVATE_KEY_PATH` [environment variable](#environment-variables) -2. The `--snowflake-private-key-path` [command-line argument](#commands) -3. Setting a `snowflake-private-key-path` in the [schemachange-config.yml](#yaml-config-file) file -4. Setting an `private-key` in the [connections.toml](#connectionstoml-file) file - -Additionally, the password for the encrypted private key file is required to be set in the environment variable -`SNOWFLAKE_PRIVATE_KEY_PASSPHRASE`. If the variable is not set, schemachange will assume the private key is not -encrypted. +Snowflake user-encrypted private key must be supplied as `private-key` in the [connections.toml](#connectionstoml-file) +file. If the private key file is password protected, supply the password as `private_key_file_pwd` in +the [connections.toml](#connectionstoml-file) file. If the variable is not set, the Snowflake Python connector will +assume the private key is not encrypted. ## Configuration -Parameters to schemachange can be supplied in four different ways (in order of priority): +As of version 4.0, Snowflake connection parameters must be supplied via +a [connections.toml file](#connectionstoml-file). Command-line and yaml arguments will still be supported with a +deprecation warning until support is completely dropped. -1. Environment Variables -2. Command Line Arguments -3. YAML config file -4. connections.toml file +Schemachange-specific parameters can be supplied in two different ways (in order of priority): -**Note:** `vars` provided via command-line argument will be merged with vars provided via YAML config. +1. Command Line Arguments +2. YAML config file -Not all parameters can be supplied via every method. +**Note:** As of 4.0, `vars` provided via command-line argument will be merged with vars provided via YAML config. +Previously, one overwrote the other completely Please see [Usage Notes for the account Parameter (for the connect Method)](https://docs.snowflake.com/en/user-guide/python-connector-api.html#label-account-format-info) for more details on how to structure the account name. -### Environment Variables +### connections.toml File -The Snowflake Python connector subscribes to many variables. Schemachange won't alter environment variables, but it will -consider the following variables to detect an incomplete configuration: +A `[connections.toml](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-connect#connecting-using-the-connections-toml-file) +filepath can be supplied in the following ways (in order of priority): -- SNOWFLAKE_PASSWORD -- _Deprecated_ SNOWSQL_PWD -- SNOWFLAKE_PRIVATE_KEY_PATH -- SNOWFLAKE_AUTHENTICATOR -- SNOWFLAKE_TOKEN -- SNOWFLAKE_DEFAULT_CONNECTION_NAME +1. The `--connections-file-path` [command-line argument](#commands) +2. The `connections-file-path` [YAML value](#yaml-config-file) + +A connection name can be supplied in the following ways (in order of priority): + +1. The `SNOWFLAKE_DEFAULT_CONNECTION_NAME` [environment variable](#environment-variables) +2. The `--connection-name` [command-line argument](#commands) +3. The `connection-name` [YAML value](#yaml-config-file) ### YAML Config File @@ -421,35 +394,6 @@ root-folder: '/path/to/folder' # The modules folder for jinja macros and templates to be used across multiple scripts. modules-folder: null -# The name of the snowflake account (e.g. xy12345.east-us-2.azure). -# You can also use the regionless format (e.g. myorgname-accountname) -# for privatelink accounts, suffix the account value with privatelink (e.g. .privatelink) -snowflake-account: 'xy12345.east-us-2.azure' - -# The name of the snowflake user -snowflake-user: 'user' - -# The name of the default role to use. Can be overridden in the change scripts. -snowflake-role: 'role' - -# The name of the default warehouse to use. Can be overridden in the change scripts. -snowflake-warehouse: 'warehouse' - -# The name of the default database to use. Can be overridden in the change scripts. -snowflake-database: null - -# The name of the default schema to use. Can be overridden in the change scripts. -snowflake-schema: null - -# The Snowflake Authenticator to use. One of snowflake, oauth, externalbrowser, or https://.okta.com -snowflake-authenticator: null - -# Path to file containing private key. -snowflake-private-key-path: null - -# Path to the file containing the OAuth token to be used when authenticating with Snowflake. -snowflake-token-path: null - # Override the default connections.toml file path at snowflake.connector.constants.CONNECTIONS_FILE (OS specific) connections-file-path: null @@ -504,40 +448,6 @@ Return the value of the environmental variable if it exists, otherwise raise an {{ env_var('') }} ``` -### connections.toml File - -A `[connections.toml](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-connect#connecting-using-the-connections-toml-file) -filepath can be supplied in the following ways (in order of priority): - -1. The `--connections-file-path` [command-line argument](#commands) -2. The `connections-file-path` [YAML value](#yaml-config-file) - -A connection name can be supplied in the following ways (in order of priority): - -1. The `SNOWFLAKE_DEFAULT_CONNECTION_NAME` [environment variable](#environment-variables) -2. The `--connection-name` [command-line argument](#commands) -3. The `connection-name` [YAML value](#yaml-config-file) - -Schemachange will consider these connection parameters after environment variables, command line arguments, and the YAML -configuration. - -```txt -[example connection name] -account = "example account" -user = "example user" -role = "example role" -warehouse = "example warehouse" -database = "example database" -schema = "example schema" -authenticator = "example authenticator" -password = "example password" -host = "example host" -port = "example port" -region = "example region" -private-key = "example private-key" -token_file_path = "example token_file_path" -``` - ## Commands Schemachange supports a few subcommands. If the subcommand is not provided it defaults to deploy. This behaviour keeps @@ -548,34 +458,25 @@ compatibility with versions prior to 3.2. This is the main command that runs the deployment process. ```bash -usage: schemachange deploy [-h] [--config-folder CONFIG_FOLDER] [--config-file-name CONFIG_FILE_NAME] [-f ROOT_FOLDER] [-m MODULES_FOLDER] [-a SNOWFLAKE_ACCOUNT] [-u SNOWFLAKE_USER] [-r SNOWFLAKE_ROLE] [-w SNOWFLAKE_WAREHOUSE] [-d SNOWFLAKE_DATABASE] [-s SNOWFLAKE_SCHEMA] [--snowflake-authenticator SNOWFLAKE_AUTHENTICATOR] [--snowflake-private-key-path SNOWFLAKE_PRIVATE_KEY_PATH] [--snowflake-token-path SNOWFLAKE_TOKEN_PATH] [--connections-file-path CONNECTIONS_FILE_PATH] [--connection-name CONNECTION_NAME] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] +usage: schemachange deploy [-h] [--config-folder CONFIG_FOLDER] [--config-file-name CONFIG_FILE_NAME] [-f ROOT_FOLDER] [-m MODULES_FOLDER] [--connections-file-path CONNECTIONS_FILE_PATH] [--connection-name CONNECTION_NAME] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] ``` -| Parameter | Description | -|----------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| -h, --help | Show the help message and exit | -| --config-folder CONFIG_FOLDER | The folder to look in for the schemachange config file (the default is the current working directory) | -| --config-file-name CONFIG_FILE_NAME | The file name of the schemachange config file. (the default is schemachange-config.yml) | -| -f ROOT_FOLDER, --root-folder ROOT_FOLDER | The root folder for the database change scripts. The default is the current directory. | -| -m MODULES_FOLDER, --modules-folder MODULES_FOLDER | The modules folder for jinja macros and templates to be used across mutliple scripts | -| -a SNOWFLAKE_ACCOUNT, --snowflake-account SNOWFLAKE_ACCOUNT | The name of the snowflake account (e.g. xy12345.east-us-2.azure). | -| -u SNOWFLAKE_USER, --snowflake-user SNOWFLAKE_USER | The name of the snowflake user | -| -r SNOWFLAKE_ROLE, --snowflake-role SNOWFLAKE_ROLE | The name of the role to use | -| -w SNOWFLAKE_WAREHOUSE, --snowflake-warehouse SNOWFLAKE_WAREHOUSE | The name of the default warehouse to use. Can be overridden in the change scripts. | -| -d SNOWFLAKE_DATABASE, --snowflake-database SNOWFLAKE_DATABASE | The name of the default database to use. Can be overridden in the change scripts. | -| -s SNOWFLAKE_SCHEMA, --snowflake-schema SNOWFLAKE_SCHEMA | The name of the default schema to use. Can be overridden in the change scripts. | -| -A SNOWFLAKE_AUTHENTICATOR, --snowflake-authenticator SNOWFLAKE_AUTHENTICATOR | The Snowflake Authenticator to use. One of snowflake, oauth, externalbrowser, or https://.okta.com | -| -k SNOWFLAKE_PRIVATE_KEY_PATH, --snowflake-private-key-path SNOWFLAKE_PRIVATE_KEY_PATH | Path to file containing private key. | -| -t SNOWFLAKE_TOKEN_PATH, --snowflake-token-path SNOWFLAKE_TOKEN_PATH | Path to the file containing the OAuth token to be used when authenticating with Snowflake. | -| --connections-file-path CONNECTIONS_FILE_PATH | Override the default [connections.toml](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-connect#connecting-using-the-connections-toml-file) file path at snowflake.connector.constants.CONNECTIONS_FILE (OS specific) | -| --connection-name CONNECTION_NAME | Override the default [connections.toml](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-connect#connecting-using-the-connections-toml-file) connection name. Other connection-related values will override these connection values. | -| -c CHANGE_HISTORY_TABLE, --change-history-table CHANGE_HISTORY_TABLE | Used to override the default name of the change history table (which is METADATA.SCHEMACHANGE.CHANGE_HISTORY) | -| --vars VARS | Define values for the variables to replaced in change scripts, given in JSON format. Vars supplied via the command line will be merged with YAML-supplied vars (e.g. '{"variable1": "value1", "variable2": "value2"}') | -| --create-change-history-table | Create the change history table if it does not exist. The default is 'False'. | -| -ac, --autocommit | Enable autocommit feature for DML commands. The default is 'False'. | -| -v, --verbose | Display verbose debugging details during execution. The default is 'False'. | -| --dry-run | Run schemachange in dry run mode. The default is 'False'. | -| --query-tag | A string to include in the QUERY_TAG that is attached to every SQL statement executed. | +| Parameter | Description | +|----------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| -h, --help | Show the help message and exit | +| --config-folder CONFIG_FOLDER | The folder to look in for the schemachange config file (the default is the current working directory) | +| --config-file-name CONFIG_FILE_NAME | The file name of the schemachange config file. (the default is schemachange-config.yml) | +| -f ROOT_FOLDER, --root-folder ROOT_FOLDER | The root folder for the database change scripts. The default is the current directory. | +| -m MODULES_FOLDER, --modules-folder MODULES_FOLDER | The modules folder for jinja macros and templates to be used across mutliple scripts | +| --connections-file-path CONNECTIONS_FILE_PATH | Override the default [connections.toml](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-connect#connecting-using-the-connections-toml-file) file path at snowflake.connector.constants.CONNECTIONS_FILE (OS specific) | +| --connection-name CONNECTION_NAME | Override the default [connections.toml](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-connect#connecting-using-the-connections-toml-file) connection name. Other connection-related values will override these connection values. | +| -c CHANGE_HISTORY_TABLE, --change-history-table CHANGE_HISTORY_TABLE | Used to override the default name of the change history table (which is METADATA.SCHEMACHANGE.CHANGE_HISTORY) | +| --vars VARS | Define values for the variables to replaced in change scripts, given in JSON format. Vars supplied via the command line will be merged with YAML-supplied vars (e.g. '{"variable1": "value1", "variable2": "value2"}') | +| --create-change-history-table | Create the change history table if it does not exist. The default is 'False'. | +| -ac, --autocommit | Enable autocommit feature for DML commands. The default is 'False'. | +| -v, --verbose | Display verbose debugging details during execution. The default is 'False'. | +| --dry-run | Run schemachange in dry run mode. The default is 'False'. | +| --query-tag | A string to include in the QUERY_TAG that is attached to every SQL statement executed. | ### render @@ -622,13 +523,13 @@ schemachange is a single python script located at [schemachange/cli.py](schemach follows: ``` -python schemachange/cli.py [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-a SNOWFLAKE_ACCOUNT] [-u SNOWFLAKE_USER] [-r SNOWFLAKE_ROLE] [-w SNOWFLAKE_WAREHOUSE] [-d SNOWFLAKE_DATABASE] [-s SNOWFLAKE_SCHEMA] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] +python schemachange/cli.py [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] [--connections-file-path] [--connection-name] ``` Or if installed via `pip`, it can be executed as follows: ``` -schemachange [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-a SNOWFLAKE_ACCOUNT] [-u SNOWFLAKE_USER] [-r SNOWFLAKE_ROLE] [-w SNOWFLAKE_WAREHOUSE] [-d SNOWFLAKE_DATABASE] [-s SNOWFLAKE_SCHEMA] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] +schemachange [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] [--connections-file-path] [--connection-name] ``` The [demo](demo) folder in this project repository contains three schemachange demo projects for you to try out. These @@ -667,10 +568,10 @@ If your build agent has a recent version of python 3 installed, the script can b ```bash pip install schemachange --upgrade -schemachange [-h] [-f ROOT_FOLDER] -a SNOWFLAKE_ACCOUNT -u SNOWFLAKE_USER -r SNOWFLAKE_ROLE -w SNOWFLAKE_WAREHOUSE [-d SNOWFLAKE_DATABASE] [-s SNOWFLAKE_SCHEMA] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] +schemachange [-h] [-f ROOT_FOLDER] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG] [--connections-file-path] [--connection-name] ``` -Or if you prefer docker, set the environment variables and run like so: +Or if you prefer docker, run like so: ```bash docker run -it --rm \ @@ -678,15 +579,11 @@ docker run -it --rm \ -v "$PWD":/usr/src/schemachange \ -w /usr/src/schemachange \ -e ROOT_FOLDER \ - -e SNOWFLAKE_ACCOUNT \ - -e SNOWFLAKE_USER \ - -e SNOWFLAKE_ROLE \ - -e SNOWFLAKE_WAREHOUSE \ - -e SNOWFLAKE_PASSWORD \ - python:3 /bin/bash -c "pip install schemachange --upgrade && schemachange -f $ROOT_FOLDER -a $SNOWFLAKE_ACCOUNT -u $SNOWFLAKE_USER -r $SNOWFLAKE_ROLE -w $SNOWFLAKE_WAREHOUSE" + -e $CONNECTION_NAME \ + python:3 /bin/bash -c "pip install schemachange --upgrade && schemachange -f $ROOT_FOLDER --connections-file-path connections.toml --connection-name $CONNECTION_NAME" ``` -Either way, don't forget to set the `SNOWFLAKE_PASSWORD` environment variable if using password authentication! +Either way, don't forget to configure a [connections.toml file](#connectionstoml-file) for connection parameters ## Maintainers diff --git a/schemachange/cli.py b/schemachange/cli.py index f2b5dc3..fc04e3d 100644 --- a/schemachange/cli.py +++ b/schemachange/cli.py @@ -62,7 +62,6 @@ def main(): logger=logger, ) else: - config.check_for_deploy_args() session = SnowflakeSession( schemachange_version=SCHEMACHANGE_VERSION, application=SNOWFLAKE_APPLICATION_NAME, diff --git a/schemachange/config/BaseConfig.py b/schemachange/config/BaseConfig.py index 5f9b192..e2ecccf 100644 --- a/schemachange/config/BaseConfig.py +++ b/schemachange/config/BaseConfig.py @@ -38,13 +38,10 @@ def factory( modules_folder: Path | str | None = None, config_vars: str | dict | None = None, log_level: int = logging.INFO, - connection_secrets: set[str] | None = None, **kwargs, ): try: secrets = get_config_secrets(config_vars) - if connection_secrets is not None: - secrets.update(connection_secrets) except Exception as e: raise Exception( "config_vars did not parse correctly, please check its configuration" diff --git a/schemachange/config/DeployConfig.py b/schemachange/config/DeployConfig.py index 3446d4c..e183470 100644 --- a/schemachange/config/DeployConfig.py +++ b/schemachange/config/DeployConfig.py @@ -6,25 +6,26 @@ from schemachange.config.BaseConfig import BaseConfig from schemachange.config.ChangeHistoryTable import ChangeHistoryTable -from schemachange.config.utils import ( - get_snowflake_identifier_string, - validate_file_path, -) +from schemachange.config.utils import get_snowflake_identifier_string @dataclasses.dataclass(frozen=True) class DeployConfig(BaseConfig): subcommand: Literal["deploy"] = "deploy" - snowflake_account: str | None = None - snowflake_user: str | None = None - snowflake_role: str | None = None - snowflake_warehouse: str | None = None - snowflake_database: str | None = None - snowflake_schema: str | None = None - snowflake_authenticator: str | None = "snowflake" - snowflake_password: str | None = None - snowflake_oauth_token: str | None = None - snowflake_private_key_path: Path | None = None + snowflake_account: str | None = ( + None # TODO: Remove when connections.toml is enforced + ) + snowflake_user: str | None = None # TODO: Remove when connections.toml is enforced + snowflake_role: str | None = None # TODO: Remove when connections.toml is enforced + snowflake_warehouse: str | None = ( + None # TODO: Remove when connections.toml is enforced + ) + snowflake_database: str | None = ( + None # TODO: Remove when connections.toml is enforced + ) + snowflake_schema: str | None = ( + None # TODO: Remove when connections.toml is enforced + ) connections_file_path: Path | None = None connection_name: str | None = None # TODO: Turn change_history_table into three arguments. There's no need to parse it from a string @@ -46,6 +47,7 @@ def factory( if "subcommand" in kwargs: kwargs.pop("subcommand") + # TODO: Remove when connections.toml is enforced for sf_input in [ "snowflake_role", "snowflake_warehouse", @@ -57,101 +59,25 @@ def factory( kwargs[sf_input], sf_input ) - for sf_path_input in [ - "snowflake_private_key_path", - "snowflake_token_path", - ]: - if sf_path_input in kwargs and kwargs[sf_path_input] is not None: - kwargs[sf_path_input] = validate_file_path( - file_path=kwargs[sf_path_input] - ) - - # If set by an environment variable, pop snowflake_token_path from kwargs - if "snowflake_oauth_token" in kwargs: - kwargs.pop("snowflake_token_path", None) - # Load it from a file, if provided - elif "snowflake_token_path" in kwargs: - oauth_token_path = kwargs.pop("snowflake_token_path") - with open(oauth_token_path) as f: - kwargs["snowflake_oauth_token"] = f.read() - change_history_table = ChangeHistoryTable.from_str( table_str=change_history_table ) - connection_secrets = { - secret - for secret in [ - kwargs.get("snowflake_password"), - kwargs.get("snowflake_oauth_token"), - ] - if secret is not None - } - return super().factory( subcommand="deploy", config_file_path=config_file_path, change_history_table=change_history_table, - connection_secrets=connection_secrets, **kwargs, ) - def check_for_deploy_args(self) -> None: - """Make sure we have the required connection info""" - - req_args = { - "snowflake_account": self.snowflake_account, - "snowflake_user": self.snowflake_user, - "snowflake_role": self.snowflake_role, - "snowflake_warehouse": self.snowflake_warehouse, - } - - # OAuth based authentication - if self.snowflake_authenticator.lower() == "oauth": - req_args["snowflake_oauth_token"] = self.snowflake_oauth_token - - # External Browser based SSO - elif self.snowflake_authenticator.lower() == "externalbrowser": - pass - - # IDP based Authentication, limited to Okta - elif self.snowflake_authenticator.lower()[:8] == "https://": - req_args["snowflake_password"] = self.snowflake_password - - elif self.snowflake_authenticator.lower() == "snowflake_jwt": - req_args["snowflake_private_key_path"] = self.snowflake_private_key_path - - elif self.snowflake_authenticator.lower() == "snowflake": - req_args["snowflake_password"] = self.snowflake_password - - else: - raise ValueError( - f"{self.snowflake_authenticator} is not supported authenticator option. " - "Choose from snowflake, snowflake_jwt, externalbrowser, oauth, https://.okta.com." - ) - - missing_args = [key for key, value in req_args.items() if value is None] - - if len(missing_args) == 0: - return - - missing_args = ", ".join({arg.replace("_", " ") for arg in missing_args}) - raise ValueError( - f"Missing config values. The following config values are required: {missing_args}" - ) - def get_session_kwargs(self) -> dict: session_kwargs = { - "account": self.snowflake_account, - "user": self.snowflake_user, - "role": self.snowflake_role, - "warehouse": self.snowflake_warehouse, - "database": self.snowflake_database, - "schema": self.snowflake_schema, - "authenticator": self.snowflake_authenticator, - "password": self.snowflake_password, - "oauth_token": self.snowflake_oauth_token, - "private_key_path": self.snowflake_private_key_path, + "account": self.snowflake_account, # TODO: Remove when connections.toml is enforced + "user": self.snowflake_user, # TODO: Remove when connections.toml is enforced + "role": self.snowflake_role, # TODO: Remove when connections.toml is enforced + "warehouse": self.snowflake_warehouse, # TODO: Remove when connections.toml is enforced + "database": self.snowflake_database, # TODO: Remove when connections.toml is enforced + "schema": self.snowflake_schema, # TODO: Remove when connections.toml is enforced "connections_file_path": self.connections_file_path, "connection_name": self.connection_name, "change_history_table": self.change_history_table, diff --git a/schemachange/config/get_merged_config.py b/schemachange/config/get_merged_config.py index 88bfe0d..f1a2ad5 100644 --- a/schemachange/config/get_merged_config.py +++ b/schemachange/config/get_merged_config.py @@ -11,8 +11,6 @@ from schemachange.config.utils import ( load_yaml_config, validate_directory, - get_env_kwargs, - get_connection_kwargs, validate_file_path, ) @@ -31,15 +29,25 @@ def get_yaml_config_kwargs(config_file_path: Optional[Path]) -> dict: if "vars" in kwargs: kwargs["config_vars"] = kwargs.pop("vars") + for deprecated_arg in [ + "snowflake_account", + "snowflake_user", + "snowflake_role", + "snowflake_warehouse", + "snowflake_database", + "snowflake_schema", + ]: + if deprecated_arg in kwargs: + sys.stderr.write( + f"DEPRECATED - Set in connections.toml instead: {deprecated_arg}\n" + ) + return {k: v for k, v in kwargs.items() if v is not None} def get_merged_config( logger: structlog.BoundLogger, ) -> Union[DeployConfig, RenderConfig]: - env_kwargs: dict[str, str] = get_env_kwargs() - logger.debug("env_kwargs", **env_kwargs) - cli_kwargs = parse_cli_args(sys.argv[1:]) logger.debug("cli_kwargs", **cli_kwargs) @@ -50,8 +58,6 @@ def get_merged_config( ) connection_name = cli_kwargs.pop("connection_name", None) - if connection_name is None: - connection_name = env_kwargs.pop("connection_name", None) config_folder = validate_directory(path=cli_kwargs.pop("config_folder", ".")) config_file_name = cli_kwargs.pop("config_file_name") @@ -77,12 +83,6 @@ def get_merged_config( if connection_name is None: connection_name = yaml_kwargs.pop("connection_name", None) - connection_kwargs: dict[str, str] = get_connection_kwargs( - connections_file_path=connections_file_path, - connection_name=connection_name, - ) - logger.debug("connection_kwargs", **connection_kwargs) - config_vars = { **yaml_config_vars, **cli_config_vars, @@ -92,9 +92,7 @@ def get_merged_config( kwargs = { "config_file_path": config_file_path, "config_vars": config_vars, - **{k: v for k, v in connection_kwargs.items() if v is not None}, **{k: v for k, v in yaml_kwargs.items() if v is not None}, - **{k: v for k, v in env_kwargs.items() if v is not None}, **{k: v for k, v in cli_kwargs.items() if v is not None}, } if connections_file_path is not None: diff --git a/schemachange/config/parse_cli_args.py b/schemachange/config/parse_cli_args.py index 60a1a6d..8b4cd01 100644 --- a/schemachange/config/parse_cli_args.py +++ b/schemachange/config/parse_cli_args.py @@ -170,30 +170,6 @@ def parse_cli_args(args) -> dict: required=False, action="deprecate", ) - parser_deploy.add_argument( - "-A", - "--snowflake-authenticator", - type=str, - help="The Snowflake Authenticator to use. One of snowflake, oauth, externalbrowser, or https://.okta.com", - required=False, - action="deprecate", - ) - parser_deploy.add_argument( - "-k", - "--snowflake-private-key-path", - type=str, - help="Path to file containing private key.", - required=False, - action="deprecate", - ) - parser_deploy.add_argument( - "-t", - "--snowflake-token-path", - type=str, - help="Path to the file containing the OAuth token to be used when authenticating with Snowflake.", - required=False, - action="deprecate", - ) parser_deploy.add_argument( "--connections-file-path", type=str, diff --git a/schemachange/config/utils.py b/schemachange/config/utils.py index f6eaff3..af3f284 100644 --- a/schemachange/config/utils.py +++ b/schemachange/config/utils.py @@ -10,7 +10,6 @@ import jinja2.ext import structlog import yaml -from snowflake.connector.config_manager import CONFIG_MANAGER from schemachange.JinjaEnvVar import JinjaEnvVar import warnings @@ -133,51 +132,6 @@ def load_yaml_config(config_file_path: Path | None) -> dict[str, Any]: return config -def set_connections_toml_path(connections_file_path: Path) -> None: - # Change config file path and force update cache - # noinspection PyProtectedMember - for i, s in enumerate(CONFIG_MANAGER._slices): - if s.section == "connections": - # noinspection PyProtectedMember - CONFIG_MANAGER._slices[i] = s._replace(path=connections_file_path) - CONFIG_MANAGER.read_config() - break - - -def get_connection_kwargs( - connections_file_path: Path | None = None, connection_name: str | None = None -) -> dict: - if connections_file_path is not None: - connections_file_path = validate_file_path(file_path=connections_file_path) - set_connections_toml_path(connections_file_path=connections_file_path) - - if connection_name is None: - return {} - - connections = CONFIG_MANAGER["connections"] - connection = connections.get(connection_name) - if connection is None: - raise Exception( - f"Invalid connection_name '{connection_name}'," - f" known ones are {list(connections.keys())}" - ) - - connection_kwargs = { - "snowflake_account": connection.get("account"), - "snowflake_user": connection.get("user"), - "snowflake_role": connection.get("role"), - "snowflake_warehouse": connection.get("warehouse"), - "snowflake_database": connection.get("database"), - "snowflake_schema": connection.get("schema"), - "snowflake_authenticator": connection.get("authenticator"), - "snowflake_password": connection.get("password"), - "snowflake_private_key_path": connection.get("private-key"), - "snowflake_token_path": connection.get("token_file_path"), - } - - return {k: v for k, v in connection_kwargs.items() if v is not None} - - def get_snowsql_pwd() -> str | None: snowsql_pwd = os.getenv("SNOWSQL_PWD") if snowsql_pwd is not None and snowsql_pwd: @@ -207,14 +161,3 @@ def get_snowflake_password() -> str | None: return snowsql_pwd else: return None - - -def get_env_kwargs() -> dict[str, str]: - env_kwargs = { - "snowflake_password": get_snowflake_password(), - "snowflake_private_key_path": os.getenv("SNOWFLAKE_PRIVATE_KEY_PATH"), - "snowflake_authenticator": os.getenv("SNOWFLAKE_AUTHENTICATOR"), - "snowflake_oauth_token": os.getenv("SNOWFLAKE_TOKEN"), - "connection_name": os.getenv("SNOWFLAKE_DEFAULT_CONNECTION_NAME"), - } - return {k: v for k, v in env_kwargs.items() if v is not None} diff --git a/schemachange/session/SnowflakeSession.py b/schemachange/session/SnowflakeSession.py index 5694456..4531db1 100644 --- a/schemachange/session/SnowflakeSession.py +++ b/schemachange/session/SnowflakeSession.py @@ -47,7 +47,6 @@ def __init__( autocommit: bool = False, **kwargs, # TODO: Remove when connections.toml is enforced ): - self.change_history_table = change_history_table self.autocommit = autocommit self.logger = logger diff --git a/tests/config/alt-connections.toml b/tests/config/alt-connections.toml deleted file mode 100644 index 4b44631..0000000 --- a/tests/config/alt-connections.toml +++ /dev/null @@ -1,28 +0,0 @@ -[myaltconnection] -account = "alt-connections.toml-account" -user = "alt-connections.toml-user" -role = "alt-connections.toml-role" -warehouse = "alt-connections.toml-warehouse" -database = "alt-connections.toml-database" -schema = "alt-connections.toml-schema" -authenticator = "alt-connections.toml-authenticator" -password = "alt-connections.toml-password" -host = "alt-connections.toml-host" -port = "alt-connections.toml-port" -region = "alt-connections.toml-region" -private-key = "alt-connections.toml-private-key" -token_file_path = "alt-connections.toml-token_file_path" -[anotherconnection] -account = "another-connections.toml-account" -user = "another-connections.toml-user" -role = "another-connections.toml-role" -warehouse = "another-connections.toml-warehouse" -database = "another-connections.toml-database" -schema = "another-connections.toml-schema" -authenticator = "another-connections.toml-authenticator" -password = "another-connections.toml-password" -host = "another-connections.toml-host" -port = "another-connections.toml-port" -region = "another-connections.toml-region" -private-key = "another-connections.toml-private-key" -token_file_path = "another-connections.toml-token_file_path" diff --git a/tests/config/alt_private_key.txt b/tests/config/alt_private_key.txt deleted file mode 100644 index c856730..0000000 --- a/tests/config/alt_private_key.txt +++ /dev/null @@ -1 +0,0 @@ -my-alt-private-key diff --git a/tests/config/connections.toml b/tests/config/connections.toml deleted file mode 100644 index 729c7f6..0000000 --- a/tests/config/connections.toml +++ /dev/null @@ -1,14 +0,0 @@ -[myconnection] -account = "connections.toml-account" -user = "connections.toml-user" -role = "connections.toml-role" -warehouse = "connections.toml-warehouse" -database = "connections.toml-database" -schema = "connections.toml-schema" -authenticator = "connections.toml-authenticator" -password = "connections.toml-password" -host = "connections.toml-host" -port = "connections.toml-port" -region = "connections.toml-region" -private-key = "connections.toml-private-key" -token_file_path = "connections.toml-token_file_path" diff --git a/tests/config/oauth_token_path.txt b/tests/config/oauth_token_path.txt deleted file mode 100644 index 49dbda6..0000000 --- a/tests/config/oauth_token_path.txt +++ /dev/null @@ -1 +0,0 @@ -my-oauth-token diff --git a/tests/config/private_key.txt b/tests/config/private_key.txt deleted file mode 100644 index 4764fdd..0000000 --- a/tests/config/private_key.txt +++ /dev/null @@ -1 +0,0 @@ -my-private-key diff --git a/tests/config/schemachange-config-full-no-connection.yml b/tests/config/schemachange-config-full-no-connection.yml index ad85113..189c65f 100644 --- a/tests/config/schemachange-config-full-no-connection.yml +++ b/tests/config/schemachange-config-full-no-connection.yml @@ -7,9 +7,6 @@ snowflake-role: 'snowflake-role-from-yaml' snowflake-warehouse: 'snowflake-warehouse-from-yaml' snowflake-database: 'snowflake-database-from-yaml' snowflake-schema: 'snowflake-schema-from-yaml' -snowflake-authenticator: 'snowflake-authenticator-from-yaml' -snowflake-private-key-path: 'snowflake-private-key-path-from-yaml' -snowflake-token-path: 'snowflake-token-path-from-yaml' change-history-table: 'change-history-table-from-yaml' vars: var1: 'from_yaml' diff --git a/tests/config/schemachange-config-full.yml b/tests/config/schemachange-config-full.yml index 49f3a57..d16ed79 100644 --- a/tests/config/schemachange-config-full.yml +++ b/tests/config/schemachange-config-full.yml @@ -7,9 +7,6 @@ snowflake-role: 'snowflake-role-from-yaml' snowflake-warehouse: 'snowflake-warehouse-from-yaml' snowflake-database: 'snowflake-database-from-yaml' snowflake-schema: 'snowflake-schema-from-yaml' -snowflake-authenticator: 'snowflake-authenticator-from-yaml' -snowflake-private-key-path: 'snowflake-private-key-path-from-yaml' -snowflake-token-path: 'snowflake-token-path-from-yaml' connections-file-path: 'connections.toml' connection-name: 'myconnection' change-history-table: 'change-history-table-from-yaml' diff --git a/tests/config/test_DeployConfig.py b/tests/config/test_DeployConfig.py index cbc7441..7c44c48 100644 --- a/tests/config/test_DeployConfig.py +++ b/tests/config/test_DeployConfig.py @@ -2,7 +2,6 @@ from pathlib import Path from unittest import mock -from unittest.mock import mock_open import pytest @@ -45,42 +44,6 @@ def test_invalid_modules_folder(_): assert "Path is not valid directory: some_modules_folder_name" in e_info_value -@mock.patch("pathlib.Path.is_dir", side_effect=[True, True]) -@mock.patch("pathlib.Path.is_file", side_effect=[False]) -def test_invalid_snowflake_private_key_path(_, __): - connections_file_path = Path(__file__).parent / "connections.toml" - connection_name = "myconnection" - - with pytest.raises(Exception) as e_info: - DeployConfig.factory( - **complete_deploy_config_kwargs, - snowflake_private_key_path="invalid_snowflake_private_key_path", - snowflake_token_path="invalid_snowflake_token_path", - connections_file_path=str(connections_file_path), - connection_name=connection_name, - ) - e_info_value = str(e_info.value) - assert "invalid file path: invalid_snowflake_private_key_path" in e_info_value - - -@mock.patch("pathlib.Path.is_dir", side_effect=[True, True]) -@mock.patch("pathlib.Path.is_file", side_effect=[True, False]) -def test_invalid_snowflake_token_path(_, __): - connections_file_path = Path(__file__).parent / "connections.toml" - connection_name = "myconnection" - - with pytest.raises(Exception) as e_info: - DeployConfig.factory( - **complete_deploy_config_kwargs, - snowflake_private_key_path="valid_snowflake_private_key_path", - snowflake_token_path="invalid_snowflake_token_path", - connections_file_path=str(connections_file_path), - connection_name=connection_name, - ) - e_info_value = str(e_info.value) - assert "invalid file path: invalid_snowflake_token_path" in e_info_value - - def test_config_vars_not_a_dict(): with pytest.raises(Exception) as e_info: BaseConfig.factory( @@ -104,91 +67,3 @@ def test_config_vars_reserved_word(): "The variable 'schemachange' has been reserved for use by schemachange, please use a different name" in str(e_info.value) ) - - -def test_check_for_deploy_args_oauth_with_token_happy_path(): - config = DeployConfig.factory( - snowflake_account="account", - snowflake_user="user", - snowflake_role="role", - snowflake_warehouse="warehouse", - snowflake_authenticator="oauth", - snowflake_oauth_token="my-oauth-token", - config_file_path=Path("."), - ) - config.check_for_deploy_args() - - -@mock.patch("pathlib.Path.is_file", return_value=True) -def test_check_for_deploy_args_oauth_with_file_happy_path(_): - with mock.patch("builtins.open", mock_open(read_data="my-oauth-token-from-a-file")): - config = DeployConfig.factory( - snowflake_account="account", - snowflake_user="user", - snowflake_role="role", - snowflake_warehouse="warehouse", - snowflake_authenticator="oauth", - snowflake_token_path="token_path", - config_file_path=Path("."), - ) - config.check_for_deploy_args() - assert config.snowflake_oauth_token == "my-oauth-token-from-a-file" - - -def test_check_for_deploy_args_externalbrowser_happy_path(): - config = DeployConfig.factory( - **minimal_deploy_config_kwargs, - snowflake_authenticator="externalbrowser", - config_file_path=Path("."), - ) - config.check_for_deploy_args() - - -def test_check_for_deploy_args_okta_happy_path(): - config = DeployConfig.factory( - **minimal_deploy_config_kwargs, - snowflake_authenticator="https://okta...", - snowflake_password="password", - config_file_path=Path("."), - ) - config.check_for_deploy_args() - - -@mock.patch("pathlib.Path.is_file", return_value=True) -def test_check_for_deploy_args_snowflake_jwt_happy_path(_): - config = DeployConfig.factory( - **minimal_deploy_config_kwargs, - snowflake_authenticator="snowflake_jwt", - snowflake_private_key_path="private_key_path", - config_file_path=Path("."), - ) - config.check_for_deploy_args() - - -def test_check_for_deploy_args_snowflake_happy_path(): - config = DeployConfig.factory( - **minimal_deploy_config_kwargs, - snowflake_authenticator="snowflake", - snowflake_password="password", - config_file_path=Path("."), - ) - config.check_for_deploy_args() - - -def test_check_for_deploy_args_default_happy_path(): - config = DeployConfig.factory( - **minimal_deploy_config_kwargs, - snowflake_password="password", - config_file_path=Path("."), - ) - config.check_for_deploy_args() - - -def test_check_for_deploy_args_exception(): - config = DeployConfig.factory(config_file_path=Path(".")) - with pytest.raises(ValueError) as e: - config.check_for_deploy_args() - - assert "Missing config values. The following config values are required" in str( - e.value - ) diff --git a/tests/config/test_get_merged_config.py b/tests/config/test_get_merged_config.py index 236f98c..fa7c9b4 100644 --- a/tests/config/test_get_merged_config.py +++ b/tests/config/test_get_merged_config.py @@ -1,8 +1,6 @@ import logging -import os import structlog -import tomlkit from pathlib import Path from unittest import mock @@ -21,22 +19,6 @@ assets_path = Path(__file__).parent - -def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: - with file_path.open("rb") as f: - connections = tomlkit.load(f) - return connections[connection_name] - - -my_connection = get_connection_from_toml( - file_path=assets_path / "connections.toml", connection_name="myconnection" -) - -alt_connection = get_connection_from_toml( - file_path=assets_path / "alt-connections.toml", - connection_name="myaltconnection", -) - schemachange_config = get_yaml_config_kwargs(assets_path / "schemachange-config.yml") schemachange_config_full = get_yaml_config_kwargs( assets_path / "schemachange-config-full.yml" @@ -50,13 +32,11 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: @pytest.mark.parametrize( - "env_kwargs, cli_kwargs, yaml_kwargs, connection_kwargs, expected", + "cli_kwargs, yaml_kwargs, expected", [ pytest.param( - {}, # env_kwargs {**default_cli_kwargs}, # cli_kwargs {}, # yaml_kwargs - {}, # connection_kwargs { # expected "config_file_path": Path("schemachange-config.yml"), "config_vars": {}, @@ -65,41 +45,16 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: id="Deploy: Only required arguments", ), pytest.param( - {}, # env_kwargs {**default_cli_kwargs}, # cli_kwargs {}, # yaml_kwargs - { # connection_kwargs - "snowflake_account": "connection_snowflake_account", - "snowflake_user": "connection_snowflake_user", - "snowflake_role": "connection_snowflake_role", - "snowflake_warehouse": "connection_snowflake_warehouse", - "snowflake_database": "connection_snowflake_database", - "snowflake_schema": "connection_snowflake_schema", - "snowflake_authenticator": "connection_snowflake_authenticator", - "snowflake_password": "connection_snowflake_password", - "snowflake_private_key_path": "connection_snowflake_private_key_path", - "snowflake_token_path": "connection_snowflake_token_path", - }, { # expected - "log_level": logging.INFO, "config_file_path": Path("schemachange-config.yml"), "config_vars": {}, "subcommand": "deploy", - "snowflake_account": "connection_snowflake_account", - "snowflake_user": "connection_snowflake_user", - "snowflake_role": "connection_snowflake_role", - "snowflake_warehouse": "connection_snowflake_warehouse", - "snowflake_database": "connection_snowflake_database", - "snowflake_schema": "connection_snowflake_schema", - "snowflake_authenticator": "connection_snowflake_authenticator", - "snowflake_password": "connection_snowflake_password", - "snowflake_private_key_path": "connection_snowflake_private_key_path", - "snowflake_token_path": "connection_snowflake_token_path", }, id="Deploy: all connection_kwargs", ), pytest.param( - {}, # env_kwargs {**default_cli_kwargs}, # cli_kwargs { # yaml_kwargs "root_folder": "yaml_root_folder", @@ -116,9 +71,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "snowflake_warehouse": "yaml_snowflake_warehouse", "snowflake_database": "yaml_snowflake_database", "snowflake_schema": "yaml_snowflake_schema", - "snowflake_authenticator": "yaml_snowflake_authenticator", - "snowflake_private_key_path": "yaml_snowflake_private_key_path", - "snowflake_token_path": "yaml_snowflake_token_path", "connections_file_path": "yaml_connections_file_path", "connection_name": "yaml_connection_name", "change_history_table": "yaml_change_history_table", @@ -127,18 +79,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "dry_run": True, "query_tag": "yaml_query_tag", }, - { # connection_kwargs - "snowflake_account": "connection_snowflake_account", - "snowflake_user": "connection_snowflake_user", - "snowflake_role": "connection_snowflake_role", - "snowflake_warehouse": "connection_snowflake_warehouse", - "snowflake_database": "connection_snowflake_database", - "snowflake_schema": "connection_snowflake_schema", - "snowflake_authenticator": "connection_snowflake_authenticator", - "snowflake_password": "connection_snowflake_password", - "snowflake_private_key_path": "connection_snowflake_private_key_path", - "snowflake_token_path": "connection_snowflake_token_path", - }, { # expected "log_level": logging.DEBUG, "config_file_path": Path("schemachange-config.yml"), @@ -156,10 +96,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "snowflake_warehouse": "yaml_snowflake_warehouse", "snowflake_database": "yaml_snowflake_database", "snowflake_schema": "yaml_snowflake_schema", - "snowflake_authenticator": "yaml_snowflake_authenticator", - "snowflake_password": "connection_snowflake_password", - "snowflake_private_key_path": "yaml_snowflake_private_key_path", - "snowflake_token_path": "yaml_snowflake_token_path", "connections_file_path": Path("yaml_connections_file_path"), "connection_name": "yaml_connection_name", "change_history_table": "yaml_change_history_table", @@ -171,7 +107,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: id="Deploy: all yaml, all connection_kwargs", ), pytest.param( - {}, # env_kwargs { # cli_kwargs **default_cli_kwargs, "config_folder": "cli_config_folder", @@ -188,9 +123,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "snowflake_warehouse": "cli_snowflake_warehouse", "snowflake_database": "cli_snowflake_database", "snowflake_schema": "cli_snowflake_schema", - "snowflake_authenticator": "cli_snowflake_authenticator", - "snowflake_private_key_path": "cli_snowflake_private_key_path", - "snowflake_token_path": "cli_snowflake_token_path", "connections_file_path": "cli_connections_file_path", "connection_name": "cli_connection_name", "change_history_table": "cli_change_history_table", @@ -214,9 +146,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "snowflake_warehouse": "yaml_snowflake_warehouse", "snowflake_database": "yaml_snowflake_database", "snowflake_schema": "yaml_snowflake_schema", - "snowflake_authenticator": "yaml_snowflake_authenticator", - "snowflake_private_key_path": "yaml_snowflake_private_key_path", - "snowflake_token_path": "yaml_snowflake_token_path", "connections_file_path": "yaml_connections_file_path", "connection_name": "yaml_connection_name", "change_history_table": "yaml_change_history_table", @@ -225,18 +154,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "dry_run": True, "query_tag": "yaml_query_tag", }, - { # connection_kwargs - "snowflake_account": "connection_snowflake_account", - "snowflake_user": "connection_snowflake_user", - "snowflake_role": "connection_snowflake_role", - "snowflake_warehouse": "connection_snowflake_warehouse", - "snowflake_database": "connection_snowflake_database", - "snowflake_schema": "connection_snowflake_schema", - "snowflake_authenticator": "connection_snowflake_authenticator", - "snowflake_password": "connection_snowflake_password", - "snowflake_private_key_path": "connection_snowflake_private_key_path", - "snowflake_token_path": "connection_snowflake_token_path", - }, { # expected "log_level": logging.INFO, "config_file_path": Path("cli_config_folder/schemachange-config.yml"), @@ -254,10 +171,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "snowflake_warehouse": "cli_snowflake_warehouse", "snowflake_database": "cli_snowflake_database", "snowflake_schema": "cli_snowflake_schema", - "snowflake_authenticator": "cli_snowflake_authenticator", - "snowflake_password": "connection_snowflake_password", - "snowflake_private_key_path": "cli_snowflake_private_key_path", - "snowflake_token_path": "cli_snowflake_token_path", "connections_file_path": Path("cli_connections_file_path"), "connection_name": "cli_connection_name", "change_history_table": "cli_change_history_table", @@ -269,12 +182,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: id="Deploy: all cli, all yaml, all connection_kwargs", ), pytest.param( - { # env_kwargs - "snowflake_password": "env_snowflake_password", - "snowflake_private_key_path": "env_snowflake_private_key_path", - "snowflake_authenticator": "env_snowflake_authenticator", - "connection_name": "env_connection_name", - }, { # cli_kwargs **default_cli_kwargs, "config_folder": "cli_config_folder", @@ -291,9 +198,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "snowflake_warehouse": "cli_snowflake_warehouse", "snowflake_database": "cli_snowflake_database", "snowflake_schema": "cli_snowflake_schema", - "snowflake_authenticator": "cli_snowflake_authenticator", - "snowflake_private_key_path": "cli_snowflake_private_key_path", - "snowflake_token_path": "cli_snowflake_token_path", "connections_file_path": "cli_connections_file_path", "connection_name": "cli_connection_name", "change_history_table": "cli_change_history_table", @@ -317,9 +221,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "snowflake_warehouse": "yaml_snowflake_warehouse", "snowflake_database": "yaml_snowflake_database", "snowflake_schema": "yaml_snowflake_schema", - "snowflake_authenticator": "yaml_snowflake_authenticator", - "snowflake_private_key_path": "yaml_snowflake_private_key_path", - "snowflake_token_path": "yaml_snowflake_token_path", "connections_file_path": "yaml_connections_file_path", "connection_name": "yaml_connection_name", "change_history_table": "yaml_change_history_table", @@ -328,18 +229,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "dry_run": True, "query_tag": "yaml_query_tag", }, - { # connection_kwargs - "snowflake_account": "connection_snowflake_account", - "snowflake_user": "connection_snowflake_user", - "snowflake_role": "connection_snowflake_role", - "snowflake_warehouse": "connection_snowflake_warehouse", - "snowflake_database": "connection_snowflake_database", - "snowflake_schema": "connection_snowflake_schema", - "snowflake_authenticator": "connection_snowflake_authenticator", - "snowflake_password": "connection_snowflake_password", - "snowflake_private_key_path": "connection_snowflake_private_key_path", - "snowflake_token_path": "connection_snowflake_token_path", - }, { # expected "log_level": logging.INFO, "config_file_path": Path("cli_config_folder/schemachange-config.yml"), @@ -357,10 +246,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "snowflake_warehouse": "cli_snowflake_warehouse", "snowflake_database": "cli_snowflake_database", "snowflake_schema": "cli_snowflake_schema", - "snowflake_authenticator": "cli_snowflake_authenticator", - "snowflake_password": "env_snowflake_password", - "snowflake_private_key_path": "cli_snowflake_private_key_path", - "snowflake_token_path": "cli_snowflake_token_path", "connections_file_path": Path("cli_connections_file_path"), "connection_name": "cli_connection_name", "change_history_table": "cli_change_history_table", @@ -375,35 +260,29 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: ) @mock.patch("pathlib.Path.is_dir", return_value=True) @mock.patch("pathlib.Path.is_file", return_value=True) -@mock.patch("schemachange.config.get_merged_config.get_env_kwargs") @mock.patch("schemachange.config.get_merged_config.parse_cli_args") @mock.patch("schemachange.config.get_merged_config.get_yaml_config_kwargs") -@mock.patch("schemachange.config.get_merged_config.get_connection_kwargs") @mock.patch("schemachange.config.get_merged_config.DeployConfig.factory") def test_get_merged_config_inheritance( mock_deploy_config_factory, - mock_get_connection_kwargs, mock_get_yaml_config_kwargs, mock_parse_cli_args, - mock_get_env_kwargs, _, __, - env_kwargs, cli_kwargs, yaml_kwargs, - connection_kwargs, expected, ): - mock_get_env_kwargs.return_value = {**env_kwargs} mock_parse_cli_args.return_value = {**cli_kwargs} mock_get_yaml_config_kwargs.return_value = {**yaml_kwargs} - mock_get_connection_kwargs.return_value = {**connection_kwargs} logger = structlog.testing.CapturingLogger() # noinspection PyTypeChecker get_merged_config(logger=logger) factory_kwargs = mock_deploy_config_factory.call_args.kwargs for actual_key, actual_value in factory_kwargs.items(): assert expected[actual_key] == actual_value + del expected[actual_key] + assert len(expected.keys()) == 0 @mock.patch("pathlib.Path.is_dir", return_value=False) @@ -424,13 +303,14 @@ def test_invalid_config_folder(mock_parse_cli_args, _): param_only_required_cli_arguments = pytest.param( - {}, # env_kwargs [ # cli_args "schemachange", + "--config-folder", + str(assets_path), ], { # expected "subcommand": "deploy", - "config_file_path": Path("schemachange-config.yml"), + "config_file_path": assets_path / "schemachange-config.yml", "config_version": 1, "config_vars": {}, "log_level": logging.INFO, @@ -439,9 +319,10 @@ def test_invalid_config_folder(mock_parse_cli_args, _): ) param_full_cli_and_connection = pytest.param( - {}, # env_kwargs [ # cli_args "schemachange", + "--config-folder", + str(assets_path), "--root-folder", "root-folder-from-cli", "--modules-folder", @@ -460,12 +341,6 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "snowflake-database-from-cli", "--snowflake-schema", "snowflake-schema-from-cli", - "--snowflake-authenticator", - "snowflake-authenticator-from-cli", - "--snowflake-private-key-path", - "snowflake-private-key-path-from-cli", - "--snowflake-token-path", - "snowflake-token-path-from-cli", "--connections-file-path", str(assets_path / "alt-connections.toml"), "--connection-name", @@ -480,7 +355,7 @@ def test_invalid_config_folder(mock_parse_cli_args, _): ], { # expected "subcommand": "deploy", - "config_file_path": Path("schemachange-config.yml"), + "config_file_path": assets_path / "schemachange-config.yml", "config_version": 1, "root_folder": "root-folder-from-cli", "modules_folder": "modules-folder-from-cli", @@ -490,9 +365,6 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "snowflake_warehouse": "snowflake-warehouse-from-cli", "snowflake_database": "snowflake-database-from-cli", "snowflake_schema": "snowflake-schema-from-cli", - "snowflake_authenticator": "snowflake-authenticator-from-cli", - "snowflake_private_key_path": "snowflake-private-key-path-from-cli", - "snowflake_token_path": "snowflake-token-path-from-cli", "change_history_table": "change-history-table-from-cli", "config_vars": { "var1": "from_cli", @@ -505,13 +377,11 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "query_tag": "query-tag-from-cli", "connection_name": "myaltconnection", "connections_file_path": assets_path / "alt-connections.toml", - "snowflake_password": alt_connection["password"], }, id="Deploy: full cli and connections.toml", ) param_full_yaml_no_connection = pytest.param( - {}, # env_kwargs [ # cli_args "schemachange", "--config-folder", @@ -537,9 +407,6 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "snowflake_warehouse", "snowflake_database", "snowflake_schema", - "snowflake_authenticator", - "snowflake_private_key_path", - "snowflake_token_path", "change_history_table", "config_vars", "create_change_history_table", @@ -553,7 +420,6 @@ def test_invalid_config_folder(mock_parse_cli_args, _): ) param_full_yaml_and_connection = pytest.param( - {}, # env_kwargs [ # cli_args "schemachange", "--config-folder", @@ -564,7 +430,6 @@ def test_invalid_config_folder(mock_parse_cli_args, _): { # expected "subcommand": "deploy", "config_file_path": assets_path / "schemachange-config-full.yml", - "snowflake_password": my_connection["password"], "log_level": logging.INFO, "connections_file_path": assets_path / "connections.toml", **{ @@ -581,9 +446,6 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "snowflake_warehouse", "snowflake_database", "snowflake_schema", - "snowflake_authenticator", - "snowflake_private_key_path", - "snowflake_token_path", "change_history_table", "snowflake_private_key_path", "config_vars", @@ -599,7 +461,6 @@ def test_invalid_config_folder(mock_parse_cli_args, _): ) param_full_yaml_and_connection_and_cli = pytest.param( - {}, # env_kwargs [ # cli_args "schemachange", "--config-folder", @@ -624,12 +485,6 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "snowflake-database-from-cli", "--snowflake-schema", "snowflake-schema-from-cli", - "--snowflake-authenticator", - "snowflake-authenticator-from-cli", - "--snowflake-private-key-path", - "snowflake-private-key-path-from-cli", - "--snowflake-token-path", - "snowflake-token-path-from-cli", "--connections-file-path", str(assets_path / "alt-connections.toml"), "--connection-name", @@ -654,9 +509,6 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "snowflake_warehouse": "snowflake-warehouse-from-cli", "snowflake_database": "snowflake-database-from-cli", "snowflake_schema": "snowflake-schema-from-cli", - "snowflake_authenticator": "snowflake-authenticator-from-cli", - "snowflake_private_key_path": "snowflake-private-key-path-from-cli", - "snowflake_token_path": "snowflake-token-path-from-cli", "change_history_table": "change-history-table-from-cli", "config_vars": { "var1": "from_cli", @@ -670,19 +522,11 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "query_tag": "query-tag-from-cli", "connection_name": "myaltconnection", "connections_file_path": assets_path / "alt-connections.toml", - "snowflake_password": alt_connection["password"], }, id="Deploy: full yaml, connections.toml, and cli", ) param_full_yaml_and_connection_and_cli_and_env = pytest.param( - { - "SNOWFLAKE_PASSWORD": "env_snowflake_password", - "SNOWFLAKE_PRIVATE_KEY_PATH": "env_snowflake_private_key_path", - "SNOWFLAKE_AUTHENTICATOR": "env_snowflake_authenticator", - "SNOWFLAKE_TOKEN": "env_snowflake_token", - "SNOWFLAKE_DEFAULT_CONNECTION_NAME": "anotherconnection", - }, # env_kwargs [ # cli_args "schemachange", "--config-folder", @@ -707,12 +551,6 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "snowflake-database-from-cli", "--snowflake-schema", "snowflake-schema-from-cli", - "--snowflake-authenticator", - "snowflake-authenticator-from-cli", - "--snowflake-private-key-path", - "snowflake-private-key-path-from-cli", - "--snowflake-token-path", - "snowflake-token-path-from-cli", "--connections-file-path", str(assets_path / "alt-connections.toml"), "--connection-name", @@ -737,9 +575,6 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "snowflake_warehouse": "snowflake-warehouse-from-cli", "snowflake_database": "snowflake-database-from-cli", "snowflake_schema": "snowflake-schema-from-cli", - "snowflake_authenticator": "snowflake-authenticator-from-cli", - "snowflake_private_key_path": "snowflake-private-key-path-from-cli", - "snowflake_token_path": "snowflake-token-path-from-cli", "change_history_table": "change-history-table-from-cli", "config_vars": { "var1": "from_cli", @@ -751,17 +586,14 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "log_level": logging.INFO, "dry_run": True, "query_tag": "query-tag-from-cli", - "snowflake_oauth_token": "env_snowflake_token", "connection_name": "myaltconnection", "connections_file_path": assets_path / "alt-connections.toml", - "snowflake_password": "env_snowflake_password", }, id="Deploy: full yaml, connections.toml, cli, and env", ) param_connection_no_yaml = pytest.param( - {}, # env_kwargs [ # cli_args "schemachange", "--config-folder", @@ -777,16 +609,6 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "connection_name": "myconnection", "config_file_path": assets_path / "schemachange-config.yml", "config_version": 1, - "snowflake_account": my_connection["account"], - "snowflake_user": my_connection["user"], - "snowflake_role": my_connection["role"], - "snowflake_warehouse": my_connection["warehouse"], - "snowflake_database": my_connection["database"], - "snowflake_schema": my_connection["schema"], - "snowflake_authenticator": my_connection["authenticator"], - "snowflake_password": my_connection["password"], - "snowflake_private_key_path": my_connection["private-key"], - "snowflake_token_path": my_connection["token_file_path"], "config_vars": {}, "log_level": logging.INFO, }, @@ -794,7 +616,6 @@ def test_invalid_config_folder(mock_parse_cli_args, _): ) param_partial_yaml_and_connection = pytest.param( - {}, # env_kwargs [ # cli_arg "schemachange", "--config-folder", @@ -806,17 +627,7 @@ def test_invalid_config_folder(mock_parse_cli_args, _): "subcommand": "deploy", "config_file_path": assets_path / "schemachange-config-partial-with-connection.yml", - "snowflake_account": my_connection["account"], - "snowflake_user": my_connection["user"], - "snowflake_role": my_connection["role"], - "snowflake_warehouse": my_connection["warehouse"], - "snowflake_database": my_connection["database"], - "snowflake_schema": my_connection["schema"], - "snowflake_authenticator": my_connection["authenticator"], - "snowflake_private_key_path": my_connection["private-key"], - "snowflake_token_path": my_connection["token_file_path"], "log_level": logging.INFO, - "snowflake_password": my_connection["password"], "connections_file_path": assets_path / "connections.toml", **{ k: v @@ -841,7 +652,7 @@ def test_invalid_config_folder(mock_parse_cli_args, _): @pytest.mark.parametrize( - "env_vars, cli_args, expected", + "cli_args, expected", [ param_only_required_cli_arguments, param_full_cli_and_connection, @@ -854,19 +665,21 @@ def test_invalid_config_folder(mock_parse_cli_args, _): ], ) @mock.patch("pathlib.Path.is_dir", return_value=True) +@mock.patch("pathlib.Path.is_file", return_value=True) @mock.patch("schemachange.config.get_merged_config.DeployConfig.factory") def test_integration_get_merged_config_inheritance( mock_deploy_config_factory, _, - env_vars, + __, cli_args, expected, ): logger = structlog.testing.CapturingLogger() - with mock.patch.dict(os.environ, env_vars, clear=True): - with mock.patch("sys.argv", cli_args): - # noinspection PyTypeChecker - get_merged_config(logger=logger) - factory_kwargs = mock_deploy_config_factory.call_args.kwargs - for actual_key, actual_value in factory_kwargs.items(): - assert expected[actual_key] == actual_value + with mock.patch("sys.argv", cli_args): + # noinspection PyTypeChecker + get_merged_config(logger=logger) + factory_kwargs = mock_deploy_config_factory.call_args.kwargs + for actual_key, actual_value in factory_kwargs.items(): + assert expected[actual_key] == actual_value + del expected[actual_key] + assert len(expected.keys()) == 0 diff --git a/tests/config/test_parse_cli_args.py b/tests/config/test_parse_cli_args.py index d181707..11f5c55 100644 --- a/tests/config/test_parse_cli_args.py +++ b/tests/config/test_parse_cli_args.py @@ -45,21 +45,6 @@ def test_parse_args_deploy_names(): ), ("--snowflake-database", "some_snowflake_database", "some_snowflake_database"), ("--snowflake-schema", "some_snowflake_schema", "some_snowflake_schema"), - ( - "--snowflake-authenticator", - "some_snowflake_authenticator", - "some_snowflake_authenticator", - ), - ( - "--snowflake-private-key-path", - "some_snowflake_private_key_path", - "some_snowflake_private_key_path", - ), - ( - "--snowflake-token-path", - "some_snowflake_token_path", - "some_snowflake_token_path", - ), ( "--connections-file-path", "some_connections_file_path", @@ -123,24 +108,6 @@ def test_parse_args_deploy_flags(): "some_snowflake_database", ), ("-s", "snowflake_schema", "some_snowflake_schema", "some_snowflake_schema"), - ( - "-A", - "snowflake_authenticator", - "some_snowflake_authenticator", - "some_snowflake_authenticator", - ), - ( - "-k", - "snowflake_private_key_path", - "some_snowflake_private_key_path", - "some_snowflake_private_key_path", - ), - ( - "-t", - "snowflake_token_path", - "some_snowflake_token_path", - "some_snowflake_token_path", - ), ("-c", "change_history_table", "some_history_table", "some_history_table"), ] diff --git a/tests/config/test_utils.py b/tests/config/test_utils.py index 67b3585..048814e 100644 --- a/tests/config/test_utils.py +++ b/tests/config/test_utils.py @@ -6,11 +6,7 @@ import pytest -from schemachange.config.utils import ( - get_snowflake_password, - get_env_kwargs, - get_connection_kwargs, -) +from schemachange.config.utils import get_snowflake_password assets_path = Path(__file__).parent @@ -32,89 +28,3 @@ def test_get_snowflake_password(env_vars: dict, expected: str): with mock.patch.dict(os.environ, env_vars, clear=True): result = get_snowflake_password() assert result == expected - - -@pytest.mark.parametrize( - "env_vars, expected", - [ - ( - {"SNOWSQL_PWD": "ignored", "SNOWFLAKE_PASSWORD": "my_snowflake_password"}, - {"snowflake_password": "my_snowflake_password"}, - ), - ( - {"SNOWSQL_PWD": "my_snowflake_password"}, - {"snowflake_password": "my_snowflake_password"}, - ), - ( - {"SNOWFLAKE_PASSWORD": "my_snowflake_password"}, - {"snowflake_password": "my_snowflake_password"}, - ), - ( - {"SNOWFLAKE_PRIVATE_KEY_PATH": "my_snowflake_private_key_path"}, - {"snowflake_private_key_path": "my_snowflake_private_key_path"}, - ), - ( - {"SNOWFLAKE_AUTHENTICATOR": "my_snowflake_authenticator"}, - {"snowflake_authenticator": "my_snowflake_authenticator"}, - ), - ( - {"SNOWFLAKE_DEFAULT_CONNECTION_NAME": "my_connection_name"}, - {"connection_name": "my_connection_name"}, - ), - ], -) -def test_get_env_kwargs(env_vars: dict, expected: str): - with mock.patch.dict(os.environ, env_vars, clear=True): - result = get_env_kwargs() - assert result == expected - - -class TestGetConnectionKwargs: - @mock.patch("pathlib.Path.is_dir", side_effect=[True, True]) - @mock.patch("pathlib.Path.is_file", return_value=False) - def test_get_connection_kwargs_invalid_connections_file_path(self, _, __): - with pytest.raises(Exception) as e_info: - get_connection_kwargs( - connections_file_path=Path("invalid_connections_file_path"), - connection_name="invalid_connection_name", - ) - - e_info_value = str(e_info.value) - assert "invalid file path: invalid_connections_file_path" in e_info_value - - @mock.patch("pathlib.Path.is_dir", return_value=True) - def test_get_connection_kwargs_no_connection_name(self, _): - connection_kwargs = get_connection_kwargs( - connections_file_path=assets_path / "connections.toml", - connection_name=None, - ) - assert connection_kwargs == {} - - @mock.patch("pathlib.Path.is_dir", side_effect=[True, True]) - def test_get_connection_kwargs_invalid_connection_name(self, _): - with pytest.raises(Exception) as e_info: - get_connection_kwargs( - connections_file_path=assets_path / "connections.toml", - connection_name="invalid_connection_name", - ) - e_info_value = str(e_info.value) - assert "Invalid connection_name 'invalid_connection_name'" in e_info_value - - @mock.patch("pathlib.Path.is_dir", return_value=True) - def test_get_connection_kwargs_happy_path(self, _): - connection_kwargs = get_connection_kwargs( - connections_file_path=assets_path / "connections.toml", - connection_name="myconnection", - ) - assert connection_kwargs == { - "snowflake_account": "connections.toml-account", - "snowflake_authenticator": "connections.toml-authenticator", - "snowflake_database": "connections.toml-database", - "snowflake_token_path": "connections.toml-token_file_path", - "snowflake_password": "connections.toml-password", - "snowflake_private_key_path": "connections.toml-private-key", - "snowflake_role": "connections.toml-role", - "snowflake_schema": "connections.toml-schema", - "snowflake_user": "connections.toml-user", - "snowflake_warehouse": "connections.toml-warehouse", - } diff --git a/tests/test_main.py b/tests/test_main.py index bfaf266..17d96e1 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -2,7 +2,6 @@ import logging import os -import tomlkit import tempfile import unittest.mock as mock from dataclasses import asdict @@ -17,20 +16,9 @@ assets_path = Path(__file__).parent / "config" - -def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: - with file_path.open("rb") as f: - connections = tomlkit.load(f) - return connections[connection_name] - - -alt_connection = get_connection_from_toml( - file_path=assets_path / "alt-connections.toml", - connection_name="myaltconnection", -) default_base_config = { # Shared configuration options - "config_file_path": Path(".") / "schemachange-config.yml", + "config_file_path": assets_path / "schemachange-config.yml", "root_folder": Path("."), "modules_folder": None, "config_vars": {}, @@ -45,10 +33,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "snowflake_warehouse": None, "snowflake_database": None, "snowflake_schema": None, - "snowflake_authenticator": "snowflake", - "snowflake_password": None, - "snowflake_oauth_token": None, - "snowflake_private_key_path": None, "connections_file_path": None, "connection_name": None, "change_history_table": ChangeHistoryTable( @@ -63,6 +47,8 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: } required_args = [ + "--config-folder", + str(assets_path), "--snowflake-account", "account", "--snowflake-user", @@ -74,17 +60,16 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: ] required_config = { + "config_file_path": assets_path / "schemachange-config.yml", "snowflake_account": "account", "snowflake_user": "user", "snowflake_warehouse": "warehouse", "snowflake_role": "role", - "snowflake_password": "password", } script_path = Path(__file__).parent.parent / "demo" / "basics_demo" / "A__basic001.sql" no_command = pytest.param( "schemachange.cli.deploy", - {"SNOWFLAKE_PASSWORD": "password"}, ["schemachange", *required_args], {**default_deploy_config, **required_config}, None, @@ -93,7 +78,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: deploy_only_required = pytest.param( "schemachange.cli.deploy", - {"SNOWFLAKE_PASSWORD": "password"}, ["schemachange", "deploy", *required_args], {**default_deploy_config, **required_config}, None, @@ -102,7 +86,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: deploy_all_cli_arg_names = pytest.param( "schemachange.cli.deploy", - {}, [ "schemachange", "deploy", @@ -129,12 +112,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "snowflake-database-from-cli", "--snowflake-schema", "snowflake-schema-from-cli", - "--snowflake-authenticator", - "externalbrowser", - "--snowflake-private-key-path", - str(assets_path / "private_key.txt"), - "--snowflake-token-path", - str(assets_path / "oauth_token_path.txt"), "--connections-file-path", str(assets_path / "alt-connections.toml"), "--connection-name", @@ -167,8 +144,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "snowflake_schema": get_snowflake_identifier_string( "snowflake-schema-from-cli", "placeholder" ), - "snowflake_authenticator": "externalbrowser", - "snowflake_private_key_path": assets_path / "private_key.txt", "change_history_table": ChangeHistoryTable( database_name="db", schema_name="schema", @@ -185,7 +160,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "query_tag": "query-tag-from-cli", "connection_name": "myaltconnection", "connections_file_path": assets_path / "alt-connections.toml", - "snowflake_password": alt_connection["password"], }, None, id="deploy: all cli argument names", @@ -193,7 +167,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: deploy_all_cli_arg_flags = pytest.param( "schemachange.cli.deploy", - {}, [ "schemachange", "deploy", @@ -220,12 +193,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "snowflake-database-from-cli", "-s", "snowflake-schema-from-cli", - "-A", - "externalbrowser", - "-k", - str(assets_path / "private_key.txt"), - "-t", - str(assets_path / "oauth_token_path.txt"), "--connections-file-path", str(assets_path / "alt-connections.toml"), "--connection-name", @@ -258,8 +225,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "snowflake_schema": get_snowflake_identifier_string( "snowflake-schema-from-cli", "placeholder" ), - "snowflake_authenticator": "externalbrowser", - "snowflake_private_key_path": assets_path / "private_key.txt", "change_history_table": ChangeHistoryTable( database_name="db", schema_name="schema", @@ -276,7 +241,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "query_tag": "query-tag-from-cli", "connection_name": "myaltconnection", "connections_file_path": assets_path / "alt-connections.toml", - "snowflake_password": alt_connection["password"], }, None, id="deploy: all cli argument flags", @@ -284,12 +248,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: deploy_all_env_all_cli = pytest.param( "schemachange.cli.deploy", - { - "SNOWFLAKE_PASSWORD": "env_snowflake_password", - "SNOWFLAKE_PRIVATE_KEY_PATH": str(assets_path / "alt_private_key.txt"), - "SNOWFLAKE_AUTHENTICATOR": "snowflake_jwt", - "SNOWFLAKE_TOKEN": "env_snowflake_oauth_token", - }, [ "schemachange", "deploy", @@ -316,12 +274,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "snowflake-database-from-cli", "--snowflake-schema", "snowflake-schema-from-cli", - "--snowflake-authenticator", - "externalbrowser", - "--snowflake-private-key-path", - str(assets_path / "private_key.txt"), - "--snowflake-token-path", - str(assets_path / "oauth_token_path.txt"), "--connections-file-path", str(assets_path / "alt-connections.toml"), "--connection-name", @@ -354,8 +306,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "snowflake_schema": get_snowflake_identifier_string( "snowflake-schema-from-cli", "placeholder" ), - "snowflake_authenticator": "externalbrowser", - "snowflake_private_key_path": assets_path / "private_key.txt", "change_history_table": ChangeHistoryTable( database_name="db", schema_name="schema", @@ -372,7 +322,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "query_tag": "query-tag-from-cli", "connection_name": "myaltconnection", "connections_file_path": assets_path / "alt-connections.toml", - "snowflake_password": "env_snowflake_password", }, None, id="deploy: all env_vars and all cli argument names", @@ -380,15 +329,10 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: deploy_snowflake_oauth_env_var = pytest.param( "schemachange.cli.deploy", - {"SNOWFLAKE_TOKEN": "env_snowflake_oauth_token"}, [ "schemachange", "deploy", *required_args, - "--snowflake-authenticator", - "oauth", - "--snowflake-token-path", - str(assets_path / "oauth_token_path.txt"), ], { **default_deploy_config, @@ -396,8 +340,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "snowflake_user": "user", "snowflake_warehouse": "warehouse", "snowflake_role": "role", - "snowflake_authenticator": "oauth", - "snowflake_oauth_token": "env_snowflake_oauth_token", }, None, id="deploy: oauth env var", @@ -405,15 +347,10 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: deploy_snowflake_oauth_file = pytest.param( "schemachange.cli.deploy", - {}, [ "schemachange", "deploy", *required_args, - "--snowflake-authenticator", - "oauth", - "--snowflake-token-path", - str(assets_path / "oauth_token_path.txt"), ], { **default_deploy_config, @@ -421,8 +358,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: "snowflake_user": "user", "snowflake_warehouse": "warehouse", "snowflake_role": "role", - "snowflake_authenticator": "oauth", - "snowflake_oauth_token": "my-oauth-token\n", }, None, id="deploy: oauth file", @@ -430,11 +365,12 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: render_only_required = pytest.param( "schemachange.cli.render", - {}, [ "schemachange", "render", str(script_path), + "--config-folder", + str(assets_path), ], {**default_base_config}, script_path, @@ -443,7 +379,6 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: render_all_cli_arg_names = pytest.param( "schemachange.cli.render", - {}, [ "schemachange", "render", @@ -453,6 +388,8 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: '{"var1": "val"}', "--verbose", str(script_path), + "--config-folder", + str(assets_path), ], { **default_base_config, @@ -466,7 +403,7 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: @pytest.mark.parametrize( - "to_mock, env_vars, cli_args, expected_config, expected_script_path", + "to_mock, cli_args, expected_config, expected_script_path", [ no_command, deploy_only_required, @@ -479,29 +416,29 @@ def get_connection_from_toml(file_path: Path, connection_name: str) -> dict: render_all_cli_arg_names, ], ) +@mock.patch("pathlib.Path.is_file", return_value=True) @mock.patch("schemachange.session.SnowflakeSession.snowflake.connector.connect") def test_main_deploy_subcommand_given_arguments_make_sure_arguments_set_on_call( _, + __, to_mock: str, - env_vars: dict[str, str], cli_args: list[str], expected_config: dict, expected_script_path: Path | None, ): - with mock.patch.dict(os.environ, env_vars, clear=True): - with mock.patch("sys.argv", cli_args): - with mock.patch(to_mock) as mock_command: - cli.main() - mock_command.assert_called_once() - _, call_kwargs = mock_command.call_args - for expected_arg, expected_value in expected_config.items(): - actual_value = getattr(call_kwargs["config"], expected_arg) - if hasattr(actual_value, "table_name"): - assert asdict(actual_value) == asdict(expected_value) - else: - assert actual_value == expected_value - if expected_script_path is not None: - assert call_kwargs["script_path"] == expected_script_path + with mock.patch("sys.argv", cli_args): + with mock.patch(to_mock) as mock_command: + cli.main() + mock_command.assert_called_once() + _, call_kwargs = mock_command.call_args + for expected_arg, expected_value in expected_config.items(): + actual_value = getattr(call_kwargs["config"], expected_arg) + if hasattr(actual_value, "table_name"): + assert asdict(actual_value) == asdict(expected_value) + else: + assert actual_value == expected_value + if expected_script_path is not None: + assert call_kwargs["script_path"] == expected_script_path @pytest.mark.parametrize( @@ -521,7 +458,6 @@ def test_main_deploy_subcommand_given_arguments_make_sure_arguments_set_on_call( "snowflake_warehouse": "warehouse", "snowflake_role": "role", "snowflake_account": "account", - "snowflake_password": "password", }, None, ), @@ -547,37 +483,36 @@ def test_main_deploy_config_folder( expected_config: dict, expected_script_path: Path | None, ): - with mock.patch.dict(os.environ, {"SNOWFLAKE_PASSWORD": "password"}, clear=True): - with tempfile.TemporaryDirectory() as d: - with open(os.path.join(d, "schemachange-config.yml"), "w") as f: - f.write( - dedent( - """ - snowflake_account: account - snowflake_user: user - snowflake_warehouse: warehouse - snowflake_role: role - """ - ) + with tempfile.TemporaryDirectory() as d: + with open(os.path.join(d, "schemachange-config.yml"), "w") as f: + f.write( + dedent( + """ + snowflake_account: account + snowflake_user: user + snowflake_warehouse: warehouse + snowflake_role: role + """ ) + ) - # noinspection PyTypeChecker - args[args.index("DUMMY")] = d - expected_config["config_file_path"] = Path(d) / "schemachange-config.yml" + # noinspection PyTypeChecker + args[args.index("DUMMY")] = d + expected_config["config_file_path"] = Path(d) / "schemachange-config.yml" - with mock.patch(to_mock) as mock_command: - with mock.patch("sys.argv", args): - cli.main() - mock_command.assert_called_once() - _, call_kwargs = mock_command.call_args - for expected_arg, expected_value in expected_config.items(): - actual_value = getattr(call_kwargs["config"], expected_arg) - if hasattr(actual_value, "table_name"): - assert asdict(actual_value) == asdict(expected_value) - else: - assert actual_value == expected_value - if expected_script_path is not None: - assert call_kwargs["script_path"] == expected_script_path + with mock.patch(to_mock) as mock_command: + with mock.patch("sys.argv", args): + cli.main() + mock_command.assert_called_once() + _, call_kwargs = mock_command.call_args + for expected_arg, expected_value in expected_config.items(): + actual_value = getattr(call_kwargs["config"], expected_arg) + if hasattr(actual_value, "table_name"): + assert asdict(actual_value) == asdict(expected_value) + else: + assert actual_value == expected_value + if expected_script_path is not None: + assert call_kwargs["script_path"] == expected_script_path @pytest.mark.parametrize( @@ -597,6 +532,8 @@ def test_main_deploy_config_folder( str(script_path), "--modules-folder", "DUMMY", + "--config-folder", + str(assets_path), ], {**default_base_config, "modules_folder": "DUMMY"}, script_path, @@ -611,22 +548,21 @@ def test_main_deploy_modules_folder( expected_config: dict, expected_script_path: Path | None, ): - with mock.patch.dict(os.environ, {"SNOWFLAKE_PASSWORD": "password"}, clear=True): - with tempfile.TemporaryDirectory() as d: - # noinspection PyTypeChecker - args[args.index("DUMMY")] = d - expected_config["modules_folder"] = Path(d) + with tempfile.TemporaryDirectory() as d: + # noinspection PyTypeChecker + args[args.index("DUMMY")] = d + expected_config["modules_folder"] = Path(d) - with mock.patch(to_mock) as mock_command: - with mock.patch("sys.argv", args): - cli.main() - mock_command.assert_called_once() - _, call_kwargs = mock_command.call_args - for expected_arg, expected_value in expected_config.items(): - actual_value = getattr(call_kwargs["config"], expected_arg) - if hasattr(actual_value, "table_name"): - assert asdict(actual_value) == asdict(expected_value) - else: - assert actual_value == expected_value - if expected_script_path is not None: - assert call_kwargs["script_path"] == expected_script_path + with mock.patch(to_mock) as mock_command: + with mock.patch("sys.argv", args): + cli.main() + mock_command.assert_called_once() + _, call_kwargs = mock_command.call_args + for expected_arg, expected_value in expected_config.items(): + actual_value = getattr(call_kwargs["config"], expected_arg) + if hasattr(actual_value, "table_name"): + assert asdict(actual_value) == asdict(expected_value) + else: + assert actual_value == expected_value + if expected_script_path is not None: + assert call_kwargs["script_path"] == expected_script_path From fb2f2a4000b36dc8ff243234fbf527a1f7bf17a3 Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Thu, 14 Nov 2024 18:54:58 -0800 Subject: [PATCH 32/34] feat: restructure integration test --- .github/workflows/master-pytest.yml | 57 ++++++++++++++++--- .../1_setup/A__setup.sql} | 0 demo/basics_demo/{ => 2_test}/A__basic001.sql | 0 demo/basics_demo/{ => 2_test}/A__render.sql | 0 demo/basics_demo/{ => 2_test}/R__basic001.sql | 0 demo/basics_demo/{ => 2_test}/R__render.sql | 0 .../{ => 2_test}/V1.0.0__render.sql | 0 .../{ => 2_test}/V1.0.1__EOF_FIle.sql | 0 .../{ => 2_test}/V1.0.2__StoredProc.sql | 0 .../3_teardown/A__teardown.sql} | 0 demo/basics_demo/schemachange-config.yml | 6 -- .../1_setup/A__setup.sql} | 0 demo/citibike_demo/{ => 2_test}/A__checks.sql | 0 demo/citibike_demo/{ => 2_test}/A__render.sql | 0 demo/citibike_demo/{ => 2_test}/R__checks.sql | 0 demo/citibike_demo/{ => 2_test}/R__render.sql | 0 .../{ => 2_test}/V1.0.0__render.sql | 0 .../V1.1.0__initial_database_objects.sql | 0 .../V1.2.0__load_tables_from_s3.sql | 0 .../3_teardown/A__teardown.sql} | 0 demo/citibike_demo/schemachange-config.yml | 6 -- .../1_setup/A__setup.sql} | 0 .../{ => 2_test}/A__render.sql | 0 .../{ => 2_test}/R__render.sql | 0 .../{ => 2_test}/V1.0.0__render.sql | 0 .../V1.1.0__initial_database_objects.sql | 0 .../V1.2.0__load_tables_from_s3.sql | 0 .../3_teardown/A__teardown.sql} | 0 .../schemachange-config.yml | 6 -- ...nfig.yml => schemachange-config-setup.yml} | 5 -- ...g.yml => schemachange-config-teardown.yml} | 5 -- .../citibike_demo/schemachange-config.yml | 16 ------ .../schemachange-config.yml | 16 ------ .../basics_demo/schemachange-config.yml | 16 ------ .../schemachange-config.yml | 16 ------ 35 files changed, 50 insertions(+), 99 deletions(-) rename demo/{setup/basics_demo/A__setup_basics_demo.sql => basics_demo/1_setup/A__setup.sql} (100%) rename demo/basics_demo/{ => 2_test}/A__basic001.sql (100%) rename demo/basics_demo/{ => 2_test}/A__render.sql (100%) rename demo/basics_demo/{ => 2_test}/R__basic001.sql (100%) rename demo/basics_demo/{ => 2_test}/R__render.sql (100%) rename demo/basics_demo/{ => 2_test}/V1.0.0__render.sql (100%) rename demo/basics_demo/{ => 2_test}/V1.0.1__EOF_FIle.sql (100%) rename demo/basics_demo/{ => 2_test}/V1.0.2__StoredProc.sql (100%) rename demo/{teardown/basics_demo/A__teardown_basics_demo.sql => basics_demo/3_teardown/A__teardown.sql} (100%) rename demo/{setup/citibike_demo/A__setup_citibike_demo.sql => citibike_demo/1_setup/A__setup.sql} (100%) rename demo/citibike_demo/{ => 2_test}/A__checks.sql (100%) rename demo/citibike_demo/{ => 2_test}/A__render.sql (100%) rename demo/citibike_demo/{ => 2_test}/R__checks.sql (100%) rename demo/citibike_demo/{ => 2_test}/R__render.sql (100%) rename demo/citibike_demo/{ => 2_test}/V1.0.0__render.sql (100%) rename demo/citibike_demo/{ => 2_test}/V1.1.0__initial_database_objects.sql (100%) rename demo/citibike_demo/{ => 2_test}/V1.2.0__load_tables_from_s3.sql (100%) rename demo/{teardown/citibike_demo/A__teardown_citibike_demo.sql => citibike_demo/3_teardown/A__teardown.sql} (100%) rename demo/{setup/citibike_demo_jinja/A__setup_citibike_demo_jinja.sql => citibike_demo_jinja/1_setup/A__setup.sql} (100%) rename demo/citibike_demo_jinja/{ => 2_test}/A__render.sql (100%) rename demo/citibike_demo_jinja/{ => 2_test}/R__render.sql (100%) rename demo/citibike_demo_jinja/{ => 2_test}/V1.0.0__render.sql (100%) rename demo/citibike_demo_jinja/{ => 2_test}/V1.1.0__initial_database_objects.sql (100%) rename demo/citibike_demo_jinja/{ => 2_test}/V1.2.0__load_tables_from_s3.sql (100%) rename demo/{teardown/citibike_demo_jinja/A__teardown_citibike_demo_jinja.sql => citibike_demo_jinja/3_teardown/A__teardown.sql} (100%) rename demo/{setup/basics_demo/schemachange-config.yml => schemachange-config-setup.yml} (63%) rename demo/{teardown/citibike_demo/schemachange-config.yml => schemachange-config-teardown.yml} (63%) delete mode 100644 demo/setup/citibike_demo/schemachange-config.yml delete mode 100644 demo/setup/citibike_demo_jinja/schemachange-config.yml delete mode 100644 demo/teardown/basics_demo/schemachange-config.yml delete mode 100644 demo/teardown/citibike_demo_jinja/schemachange-config.yml diff --git a/.github/workflows/master-pytest.yml b/.github/workflows/master-pytest.yml index 5caa505..1cbe21b 100644 --- a/.github/workflows/master-pytest.yml +++ b/.github/workflows/master-pytest.yml @@ -58,6 +58,19 @@ jobs: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Create and populate connections.toml + run: | + touch connections.toml + echo [default] >> connections.toml + echo account = "$SNOWFLAKE_ACCOUNT" >> connections.toml + echo user = "$SNOWFLAKE_USER" >> connections.toml + echo role = "$SNOWFLAKE_ROLE" >> connections.toml + echo warehouse = "$SNOWFLAKE_WAREHOUSE" >> connections.toml + echo database = "$SNOWFLAKE_DATABASE" >> connections.toml + echo schema = "$MY_TARGET_SCHEMA" >> connections.toml + echo password = "$SCHEMACHANGE_SNOWFLAKE_PASSWORD" >> connections.toml + echo "cat connections.toml" + cat connections.toml - name: Test with pytest id: pytest run: | @@ -66,16 +79,39 @@ jobs: - name: Test Schemachange on ${{ matrix.os }} targeting ${{ env.SNOWFLAKE_DATABASE }}.${{ env.MY_TARGET_SCHEMA }} schema run: | echo "::group::Setting up ${MY_TARGET_SCHEMA}" - schemachange deploy --config-folder ./demo/setup/${SCENARIO_NAME} - echo "::endgroup::" + schemachange deploy \ + --config-folder ./demo \ + --config-file-name schemachange-config-setup.yml \ + --root-folder ./demo/${SCENARIO_NAME}/1_setup \ + --connection-name default \ + --connections-file-path connections.toml + echo "::endgroup::"' + echo "::group::Testing Rendering to ${MY_TARGET_SCHEMA}" - schemachange render --config-folder ./demo/${SCENARIO_NAME} ./demo/${SCENARIO_NAME}/A__render.sql - schemachange render --config-folder ./demo/${SCENARIO_NAME} ./demo/${SCENARIO_NAME}/R__render.sql - schemachange render --config-folder ./demo/${SCENARIO_NAME} ./demo/${SCENARIO_NAME}/V1.0.0__render.sql + schemachange render \ + --config-folder ./demo/${SCENARIO_NAME} \ + --connection-name default \ + --connections-file-path connections.toml \ + ./demo/${SCENARIO_NAME}/2_test/A__render.sql + schemachange render \ + --config-folder ./demo/${SCENARIO_NAME} \ + --connection-name default \ + --connections-file-path connections.toml \ + ./demo/${SCENARIO_NAME}/2_test/R__render.sql + schemachange render \ + --config-folder ./demo/${SCENARIO_NAME} \ + --connection-name default \ + --connections-file-path connections.toml + ./demo/${SCENARIO_NAME}/2_test/V1.0.0__render.sql echo "::endgroup::" + echo "::group::Testing Deployment using ${MY_TARGET_SCHEMA}" set +e - schemachange deploy --config-folder ./demo/${SCENARIO_NAME} + schemachange deploy \ + --config-folder ./demo/${SCENARIO_NAME} \ + --connection-name default \ + --connections-file-path connections.toml \ + --root-folder ./demo/${SCENARIO_NAME}/2_test RESULT=$? if [ $RESULT -eq 0 ]; then echo "Deployment Completed!" @@ -84,9 +120,16 @@ jobs: fi echo "::endgroup::" set -e + echo "::group::Tearing down up ${MY_TARGET_SCHEMA}" - schemachange deploy --config-folder ./demo/teardown/${SCENARIO_NAME} + schemachange deploy \ + --config-folder ./demo \ + --config-file-name schemachange-config-teardown.yml \ + --connection-name default \ + --connections-file-path connections.toml \ + --root-folder ./demo/${SCENARIO_NAME}/3_teardown \ echo "::endgroup::" + if [ $RESULT -ne 0 ]; then exit 1 fi diff --git a/demo/setup/basics_demo/A__setup_basics_demo.sql b/demo/basics_demo/1_setup/A__setup.sql similarity index 100% rename from demo/setup/basics_demo/A__setup_basics_demo.sql rename to demo/basics_demo/1_setup/A__setup.sql diff --git a/demo/basics_demo/A__basic001.sql b/demo/basics_demo/2_test/A__basic001.sql similarity index 100% rename from demo/basics_demo/A__basic001.sql rename to demo/basics_demo/2_test/A__basic001.sql diff --git a/demo/basics_demo/A__render.sql b/demo/basics_demo/2_test/A__render.sql similarity index 100% rename from demo/basics_demo/A__render.sql rename to demo/basics_demo/2_test/A__render.sql diff --git a/demo/basics_demo/R__basic001.sql b/demo/basics_demo/2_test/R__basic001.sql similarity index 100% rename from demo/basics_demo/R__basic001.sql rename to demo/basics_demo/2_test/R__basic001.sql diff --git a/demo/basics_demo/R__render.sql b/demo/basics_demo/2_test/R__render.sql similarity index 100% rename from demo/basics_demo/R__render.sql rename to demo/basics_demo/2_test/R__render.sql diff --git a/demo/basics_demo/V1.0.0__render.sql b/demo/basics_demo/2_test/V1.0.0__render.sql similarity index 100% rename from demo/basics_demo/V1.0.0__render.sql rename to demo/basics_demo/2_test/V1.0.0__render.sql diff --git a/demo/basics_demo/V1.0.1__EOF_FIle.sql b/demo/basics_demo/2_test/V1.0.1__EOF_FIle.sql similarity index 100% rename from demo/basics_demo/V1.0.1__EOF_FIle.sql rename to demo/basics_demo/2_test/V1.0.1__EOF_FIle.sql diff --git a/demo/basics_demo/V1.0.2__StoredProc.sql b/demo/basics_demo/2_test/V1.0.2__StoredProc.sql similarity index 100% rename from demo/basics_demo/V1.0.2__StoredProc.sql rename to demo/basics_demo/2_test/V1.0.2__StoredProc.sql diff --git a/demo/teardown/basics_demo/A__teardown_basics_demo.sql b/demo/basics_demo/3_teardown/A__teardown.sql similarity index 100% rename from demo/teardown/basics_demo/A__teardown_basics_demo.sql rename to demo/basics_demo/3_teardown/A__teardown.sql diff --git a/demo/basics_demo/schemachange-config.yml b/demo/basics_demo/schemachange-config.yml index 18680db..7252e33 100644 --- a/demo/basics_demo/schemachange-config.yml +++ b/demo/basics_demo/schemachange-config.yml @@ -2,12 +2,6 @@ config-version: 1 root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}" -snowflake-user: "{{ env_var('SNOWFLAKE_USER')}}" -snowflake-account: "{{ env_var('SNOWFLAKE_ACCOUNT')}}" -snowflake-role: "{{ env_var('SNOWFLAKE_ROLE')}}" -snowflake-warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE')}}" -snowflake-database: "{{ env_var('SNOWFLAKE_DATABASE')}}" -snowflake-schema: "{{ env_var('MY_TARGET_SCHEMA')}}" change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.{{ env_var('MY_TARGET_SCHEMA')}}.CHANGE_HISTORY" create-change-history-table: true diff --git a/demo/setup/citibike_demo/A__setup_citibike_demo.sql b/demo/citibike_demo/1_setup/A__setup.sql similarity index 100% rename from demo/setup/citibike_demo/A__setup_citibike_demo.sql rename to demo/citibike_demo/1_setup/A__setup.sql diff --git a/demo/citibike_demo/A__checks.sql b/demo/citibike_demo/2_test/A__checks.sql similarity index 100% rename from demo/citibike_demo/A__checks.sql rename to demo/citibike_demo/2_test/A__checks.sql diff --git a/demo/citibike_demo/A__render.sql b/demo/citibike_demo/2_test/A__render.sql similarity index 100% rename from demo/citibike_demo/A__render.sql rename to demo/citibike_demo/2_test/A__render.sql diff --git a/demo/citibike_demo/R__checks.sql b/demo/citibike_demo/2_test/R__checks.sql similarity index 100% rename from demo/citibike_demo/R__checks.sql rename to demo/citibike_demo/2_test/R__checks.sql diff --git a/demo/citibike_demo/R__render.sql b/demo/citibike_demo/2_test/R__render.sql similarity index 100% rename from demo/citibike_demo/R__render.sql rename to demo/citibike_demo/2_test/R__render.sql diff --git a/demo/citibike_demo/V1.0.0__render.sql b/demo/citibike_demo/2_test/V1.0.0__render.sql similarity index 100% rename from demo/citibike_demo/V1.0.0__render.sql rename to demo/citibike_demo/2_test/V1.0.0__render.sql diff --git a/demo/citibike_demo/V1.1.0__initial_database_objects.sql b/demo/citibike_demo/2_test/V1.1.0__initial_database_objects.sql similarity index 100% rename from demo/citibike_demo/V1.1.0__initial_database_objects.sql rename to demo/citibike_demo/2_test/V1.1.0__initial_database_objects.sql diff --git a/demo/citibike_demo/V1.2.0__load_tables_from_s3.sql b/demo/citibike_demo/2_test/V1.2.0__load_tables_from_s3.sql similarity index 100% rename from demo/citibike_demo/V1.2.0__load_tables_from_s3.sql rename to demo/citibike_demo/2_test/V1.2.0__load_tables_from_s3.sql diff --git a/demo/teardown/citibike_demo/A__teardown_citibike_demo.sql b/demo/citibike_demo/3_teardown/A__teardown.sql similarity index 100% rename from demo/teardown/citibike_demo/A__teardown_citibike_demo.sql rename to demo/citibike_demo/3_teardown/A__teardown.sql diff --git a/demo/citibike_demo/schemachange-config.yml b/demo/citibike_demo/schemachange-config.yml index 18680db..7252e33 100644 --- a/demo/citibike_demo/schemachange-config.yml +++ b/demo/citibike_demo/schemachange-config.yml @@ -2,12 +2,6 @@ config-version: 1 root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}" -snowflake-user: "{{ env_var('SNOWFLAKE_USER')}}" -snowflake-account: "{{ env_var('SNOWFLAKE_ACCOUNT')}}" -snowflake-role: "{{ env_var('SNOWFLAKE_ROLE')}}" -snowflake-warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE')}}" -snowflake-database: "{{ env_var('SNOWFLAKE_DATABASE')}}" -snowflake-schema: "{{ env_var('MY_TARGET_SCHEMA')}}" change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.{{ env_var('MY_TARGET_SCHEMA')}}.CHANGE_HISTORY" create-change-history-table: true diff --git a/demo/setup/citibike_demo_jinja/A__setup_citibike_demo_jinja.sql b/demo/citibike_demo_jinja/1_setup/A__setup.sql similarity index 100% rename from demo/setup/citibike_demo_jinja/A__setup_citibike_demo_jinja.sql rename to demo/citibike_demo_jinja/1_setup/A__setup.sql diff --git a/demo/citibike_demo_jinja/A__render.sql b/demo/citibike_demo_jinja/2_test/A__render.sql similarity index 100% rename from demo/citibike_demo_jinja/A__render.sql rename to demo/citibike_demo_jinja/2_test/A__render.sql diff --git a/demo/citibike_demo_jinja/R__render.sql b/demo/citibike_demo_jinja/2_test/R__render.sql similarity index 100% rename from demo/citibike_demo_jinja/R__render.sql rename to demo/citibike_demo_jinja/2_test/R__render.sql diff --git a/demo/citibike_demo_jinja/V1.0.0__render.sql b/demo/citibike_demo_jinja/2_test/V1.0.0__render.sql similarity index 100% rename from demo/citibike_demo_jinja/V1.0.0__render.sql rename to demo/citibike_demo_jinja/2_test/V1.0.0__render.sql diff --git a/demo/citibike_demo_jinja/V1.1.0__initial_database_objects.sql b/demo/citibike_demo_jinja/2_test/V1.1.0__initial_database_objects.sql similarity index 100% rename from demo/citibike_demo_jinja/V1.1.0__initial_database_objects.sql rename to demo/citibike_demo_jinja/2_test/V1.1.0__initial_database_objects.sql diff --git a/demo/citibike_demo_jinja/V1.2.0__load_tables_from_s3.sql b/demo/citibike_demo_jinja/2_test/V1.2.0__load_tables_from_s3.sql similarity index 100% rename from demo/citibike_demo_jinja/V1.2.0__load_tables_from_s3.sql rename to demo/citibike_demo_jinja/2_test/V1.2.0__load_tables_from_s3.sql diff --git a/demo/teardown/citibike_demo_jinja/A__teardown_citibike_demo_jinja.sql b/demo/citibike_demo_jinja/3_teardown/A__teardown.sql similarity index 100% rename from demo/teardown/citibike_demo_jinja/A__teardown_citibike_demo_jinja.sql rename to demo/citibike_demo_jinja/3_teardown/A__teardown.sql diff --git a/demo/citibike_demo_jinja/schemachange-config.yml b/demo/citibike_demo_jinja/schemachange-config.yml index 0cd98e4..88cac3e 100644 --- a/demo/citibike_demo_jinja/schemachange-config.yml +++ b/demo/citibike_demo_jinja/schemachange-config.yml @@ -3,12 +3,6 @@ config-version: 1 root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}" modules-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/modules" -snowflake-user: "{{ env_var('SNOWFLAKE_USER')}}" -snowflake-account: "{{ env_var('SNOWFLAKE_ACCOUNT')}}" -snowflake-role: "{{ env_var('SNOWFLAKE_ROLE')}}" -snowflake-warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE')}}" -snowflake-database: "{{ env_var('SNOWFLAKE_DATABASE')}}" -snowflake-schema: "{{ env_var('MY_TARGET_SCHEMA')}}" change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.{{ env_var('MY_TARGET_SCHEMA')}}.CHANGE_HISTORY" create-change-history-table: true diff --git a/demo/setup/basics_demo/schemachange-config.yml b/demo/schemachange-config-setup.yml similarity index 63% rename from demo/setup/basics_demo/schemachange-config.yml rename to demo/schemachange-config-setup.yml index 16a4c46..e68543c 100644 --- a/demo/setup/basics_demo/schemachange-config.yml +++ b/demo/schemachange-config-setup.yml @@ -2,11 +2,6 @@ config-version: 1 root-folder: "./demo/setup/{{ env_var('SCENARIO_NAME')}}" -snowflake-user: "{{ env_var('SNOWFLAKE_USER')}}" -snowflake-account: "{{ env_var('SNOWFLAKE_ACCOUNT')}}" -snowflake-role: "{{ env_var('SNOWFLAKE_ROLE')}}" -snowflake-warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE')}}" -snowflake-database: "{{ env_var('SNOWFLAKE_DATABASE')}}" # tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" create-change-history-table: true diff --git a/demo/teardown/citibike_demo/schemachange-config.yml b/demo/schemachange-config-teardown.yml similarity index 63% rename from demo/teardown/citibike_demo/schemachange-config.yml rename to demo/schemachange-config-teardown.yml index d4f8fc0..800415a 100644 --- a/demo/teardown/citibike_demo/schemachange-config.yml +++ b/demo/schemachange-config-teardown.yml @@ -2,11 +2,6 @@ config-version: 1 root-folder: "./demo/teardown/{{ env_var('SCENARIO_NAME')}}" -snowflake-user: "{{ env_var('SNOWFLAKE_USER')}}" -snowflake-account: "{{ env_var('SNOWFLAKE_ACCOUNT')}}" -snowflake-role: "{{ env_var('SNOWFLAKE_ROLE')}}" -snowflake-warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE')}}" -snowflake-database: "{{ env_var('SNOWFLAKE_DATABASE')}}" # tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" create-change-history-table: true diff --git a/demo/setup/citibike_demo/schemachange-config.yml b/demo/setup/citibike_demo/schemachange-config.yml deleted file mode 100644 index 16a4c46..0000000 --- a/demo/setup/citibike_demo/schemachange-config.yml +++ /dev/null @@ -1,16 +0,0 @@ -config-version: 1 - -root-folder: "./demo/setup/{{ env_var('SCENARIO_NAME')}}" - -snowflake-user: "{{ env_var('SNOWFLAKE_USER')}}" -snowflake-account: "{{ env_var('SNOWFLAKE_ACCOUNT')}}" -snowflake-role: "{{ env_var('SNOWFLAKE_ROLE')}}" -snowflake-warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE')}}" -snowflake-database: "{{ env_var('SNOWFLAKE_DATABASE')}}" -# tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. -change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" -create-change-history-table: true - -vars: - database_name: "{{env_var('SNOWFLAKE_DATABASE')}}" - schema_name: "{{env_var('MY_TARGET_SCHEMA')}}" diff --git a/demo/setup/citibike_demo_jinja/schemachange-config.yml b/demo/setup/citibike_demo_jinja/schemachange-config.yml deleted file mode 100644 index 16a4c46..0000000 --- a/demo/setup/citibike_demo_jinja/schemachange-config.yml +++ /dev/null @@ -1,16 +0,0 @@ -config-version: 1 - -root-folder: "./demo/setup/{{ env_var('SCENARIO_NAME')}}" - -snowflake-user: "{{ env_var('SNOWFLAKE_USER')}}" -snowflake-account: "{{ env_var('SNOWFLAKE_ACCOUNT')}}" -snowflake-role: "{{ env_var('SNOWFLAKE_ROLE')}}" -snowflake-warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE')}}" -snowflake-database: "{{ env_var('SNOWFLAKE_DATABASE')}}" -# tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. -change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" -create-change-history-table: true - -vars: - database_name: "{{env_var('SNOWFLAKE_DATABASE')}}" - schema_name: "{{env_var('MY_TARGET_SCHEMA')}}" diff --git a/demo/teardown/basics_demo/schemachange-config.yml b/demo/teardown/basics_demo/schemachange-config.yml deleted file mode 100644 index d4f8fc0..0000000 --- a/demo/teardown/basics_demo/schemachange-config.yml +++ /dev/null @@ -1,16 +0,0 @@ -config-version: 1 - -root-folder: "./demo/teardown/{{ env_var('SCENARIO_NAME')}}" - -snowflake-user: "{{ env_var('SNOWFLAKE_USER')}}" -snowflake-account: "{{ env_var('SNOWFLAKE_ACCOUNT')}}" -snowflake-role: "{{ env_var('SNOWFLAKE_ROLE')}}" -snowflake-warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE')}}" -snowflake-database: "{{ env_var('SNOWFLAKE_DATABASE')}}" -# tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. -change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" -create-change-history-table: true - -vars: - database_name: "{{env_var('SNOWFLAKE_DATABASE')}}" - schema_name: "{{env_var('MY_TARGET_SCHEMA')}}" diff --git a/demo/teardown/citibike_demo_jinja/schemachange-config.yml b/demo/teardown/citibike_demo_jinja/schemachange-config.yml deleted file mode 100644 index d4f8fc0..0000000 --- a/demo/teardown/citibike_demo_jinja/schemachange-config.yml +++ /dev/null @@ -1,16 +0,0 @@ -config-version: 1 - -root-folder: "./demo/teardown/{{ env_var('SCENARIO_NAME')}}" - -snowflake-user: "{{ env_var('SNOWFLAKE_USER')}}" -snowflake-account: "{{ env_var('SNOWFLAKE_ACCOUNT')}}" -snowflake-role: "{{ env_var('SNOWFLAKE_ROLE')}}" -snowflake-warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE')}}" -snowflake-database: "{{ env_var('SNOWFLAKE_DATABASE')}}" -# tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment. -change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY" -create-change-history-table: true - -vars: - database_name: "{{env_var('SNOWFLAKE_DATABASE')}}" - schema_name: "{{env_var('MY_TARGET_SCHEMA')}}" From c07f97cfd3ade48ec155f94ba81d2e4df3c833c4 Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Thu, 14 Nov 2024 19:10:33 -0800 Subject: [PATCH 33/34] fix: fix integration tests --- .github/workflows/master-pytest.yml | 47 +++++++++++------------- schemachange/session/SnowflakeSession.py | 14 ++++--- tests/session/test_SnowflakeSession.py | 25 +++++++------ tests/test_main.py | 10 ++++- 4 files changed, 54 insertions(+), 42 deletions(-) diff --git a/.github/workflows/master-pytest.yml b/.github/workflows/master-pytest.yml index 1cbe21b..385403e 100644 --- a/.github/workflows/master-pytest.yml +++ b/.github/workflows/master-pytest.yml @@ -29,13 +29,13 @@ jobs: runs-on: ${{ matrix.os }} if: ${{ github.event.label.name == 'ci-run-tests' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} env: - SNOWFLAKE_PASSWORD: ${{ secrets.SCHEMACHANGE_SNOWFLAKE_PASSWORD }} - SNOWFLAKE_USER: ${{ secrets.SCHEMACHANGE_SNOWFLAKE_USER }} SNOWFLAKE_ACCOUNT: ${{ secrets.SCHEMACHANGE_SNOWFLAKE_ACCOUNT }} - SNOWFLAKE_DATABASE: SCHEMACHANGE_DEMO - SNOWFLAKE_WAREHOUSE: SCHEMACHANGE_DEMO_WH + SNOWFLAKE_USER: ${{ secrets.SCHEMACHANGE_SNOWFLAKE_USER }} SNOWFLAKE_ROLE: SCHEMACHANGE_DEMO-DEPLOY + SNOWFLAKE_WAREHOUSE: SCHEMACHANGE_DEMO_WH + SNOWFLAKE_DATABASE: SCHEMACHANGE_DEMO MY_TARGET_SCHEMA: ${{ matrix.scenario-name }}_${{ github.run_number }}_${{ strategy.job-index }} + SNOWFLAKE_PASSWORD: ${{ secrets.SCHEMACHANGE_SNOWFLAKE_PASSWORD }} SCENARIO_NAME: ${{ matrix.scenario-name }} steps: - uses: actions/checkout@v4 @@ -60,17 +60,16 @@ jobs: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Create and populate connections.toml run: | - touch connections.toml - echo [default] >> connections.toml - echo account = "$SNOWFLAKE_ACCOUNT" >> connections.toml - echo user = "$SNOWFLAKE_USER" >> connections.toml - echo role = "$SNOWFLAKE_ROLE" >> connections.toml - echo warehouse = "$SNOWFLAKE_WAREHOUSE" >> connections.toml - echo database = "$SNOWFLAKE_DATABASE" >> connections.toml - echo schema = "$MY_TARGET_SCHEMA" >> connections.toml - echo password = "$SCHEMACHANGE_SNOWFLAKE_PASSWORD" >> connections.toml + touch ./connections.toml + echo [default] >> ./connections.toml + echo account = \"${SNOWFLAKE_ACCOUNT}\" >> ./connections.toml + echo user = \"${SNOWFLAKE_USER}\" >> ./connections.toml + echo role = \"${SNOWFLAKE_ROLE}\" >> ./connections.toml + echo warehouse = \"${SNOWFLAKE_WAREHOUSE}\" >> ./connections.toml + echo database = \"${SNOWFLAKE_DATABASE}\" >> ./connections.toml + echo password = \"${SNOWFLAKE_PASSWORD}\" >> ./connections.toml echo "cat connections.toml" - cat connections.toml + cat ./connections.toml - name: Test with pytest id: pytest run: | @@ -84,24 +83,20 @@ jobs: --config-file-name schemachange-config-setup.yml \ --root-folder ./demo/${SCENARIO_NAME}/1_setup \ --connection-name default \ - --connections-file-path connections.toml - echo "::endgroup::"' + --connections-file-path ./connections.toml \ + --verbose + echo "::endgroup::" echo "::group::Testing Rendering to ${MY_TARGET_SCHEMA}" + schemachange render \ --config-folder ./demo/${SCENARIO_NAME} \ - --connection-name default \ - --connections-file-path connections.toml \ ./demo/${SCENARIO_NAME}/2_test/A__render.sql schemachange render \ --config-folder ./demo/${SCENARIO_NAME} \ - --connection-name default \ - --connections-file-path connections.toml \ ./demo/${SCENARIO_NAME}/2_test/R__render.sql schemachange render \ --config-folder ./demo/${SCENARIO_NAME} \ - --connection-name default \ - --connections-file-path connections.toml ./demo/${SCENARIO_NAME}/2_test/V1.0.0__render.sql echo "::endgroup::" @@ -110,8 +105,9 @@ jobs: schemachange deploy \ --config-folder ./demo/${SCENARIO_NAME} \ --connection-name default \ - --connections-file-path connections.toml \ - --root-folder ./demo/${SCENARIO_NAME}/2_test + --connections-file-path ./connections.toml \ + --root-folder ./demo/${SCENARIO_NAME}/2_test \ + --verbose RESULT=$? if [ $RESULT -eq 0 ]; then echo "Deployment Completed!" @@ -126,8 +122,9 @@ jobs: --config-folder ./demo \ --config-file-name schemachange-config-teardown.yml \ --connection-name default \ - --connections-file-path connections.toml \ + --connections-file-path ./connections.toml \ --root-folder ./demo/${SCENARIO_NAME}/3_teardown \ + --verbose echo "::endgroup::" if [ $RESULT -ne 0 ]; then diff --git a/schemachange/session/SnowflakeSession.py b/schemachange/session/SnowflakeSession.py index 4531db1..ba987f4 100644 --- a/schemachange/session/SnowflakeSession.py +++ b/schemachange/session/SnowflakeSession.py @@ -9,6 +9,7 @@ import structlog from schemachange.config.ChangeHistoryTable import ChangeHistoryTable +from schemachange.config.utils import get_snowflake_identifier_string from schemachange.session.Script import VersionedScript, RepeatableScript, AlwaysScript @@ -79,15 +80,18 @@ def __init__( "application": application, "session_parameters": self.session_parameters, } + connect_kwargs = {k: v for k, v in connect_kwargs.items() if v is not None} self.logger.debug("snowflake.connector.connect kwargs", **connect_kwargs) self.con = snowflake.connector.connect(**connect_kwargs) print(f"Current session ID: {self.con.session_id}") self.account = self.con.account - self.user = self.con.user - self.role = self.con.role - self.warehouse = self.con.warehouse - self.database = self.con.database - self.schema = self.con.schema + self.user = get_snowflake_identifier_string(self.con.user, "user") + self.role = get_snowflake_identifier_string(self.con.role, "role") + self.warehouse = get_snowflake_identifier_string( + self.con.warehouse, "warehouse" + ) + self.database = get_snowflake_identifier_string(self.con.database, "database") + self.schema = get_snowflake_identifier_string(self.con.schema, "schema") if not self.autocommit: self.con.autocommit(False) diff --git a/tests/session/test_SnowflakeSession.py b/tests/session/test_SnowflakeSession.py index 32b9d4b..647852d 100644 --- a/tests/session/test_SnowflakeSession.py +++ b/tests/session/test_SnowflakeSession.py @@ -15,17 +15,20 @@ def session() -> SnowflakeSession: logger = structlog.testing.CapturingLogger() with mock.patch("snowflake.connector.connect"): - # noinspection PyTypeChecker - return SnowflakeSession( - user="user", - account="account", - role="role", - warehouse="warehouse", - schemachange_version="3.6.1.dev", - application="schemachange", - change_history_table=change_history_table, - logger=logger, - ) + with mock.patch( + "schemachange.session.SnowflakeSession.get_snowflake_identifier_string" + ): + # noinspection PyTypeChecker + return SnowflakeSession( + user="user", + account="account", + role="role", + warehouse="warehouse", + schemachange_version="3.6.1.dev", + application="schemachange", + change_history_table=change_history_table, + logger=logger, + ) class TestSnowflakeSession: diff --git a/tests/test_main.py b/tests/test_main.py index 17d96e1..cdbc661 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -66,7 +66,9 @@ "snowflake_warehouse": "warehouse", "snowflake_role": "role", } -script_path = Path(__file__).parent.parent / "demo" / "basics_demo" / "A__basic001.sql" +script_path = ( + Path(__file__).parent.parent / "demo" / "basics_demo" / "2_test" / "A__basic001.sql" +) no_command = pytest.param( "schemachange.cli.deploy", @@ -418,9 +420,11 @@ ) @mock.patch("pathlib.Path.is_file", return_value=True) @mock.patch("schemachange.session.SnowflakeSession.snowflake.connector.connect") +@mock.patch("schemachange.session.SnowflakeSession.get_snowflake_identifier_string") def test_main_deploy_subcommand_given_arguments_make_sure_arguments_set_on_call( _, __, + ___, to_mock: str, cli_args: list[str], expected_config: dict, @@ -476,8 +480,10 @@ def test_main_deploy_subcommand_given_arguments_make_sure_arguments_set_on_call( ], ) @mock.patch("schemachange.session.SnowflakeSession.snowflake.connector.connect") +@mock.patch("schemachange.session.SnowflakeSession.get_snowflake_identifier_string") def test_main_deploy_config_folder( _, + __, to_mock: str, args: list[str], expected_config: dict, @@ -541,8 +547,10 @@ def test_main_deploy_config_folder( ], ) @mock.patch("schemachange.session.SnowflakeSession.snowflake.connector.connect") +@mock.patch("schemachange.session.SnowflakeSession.get_snowflake_identifier_string") def test_main_deploy_modules_folder( _, + __, to_mock: str, args: list[str], expected_config: dict, From 53efca1fb11ff073f6017eb358a5b3c6ad8e6264 Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Mon, 18 Nov 2024 14:03:00 -0800 Subject: [PATCH 34/34] docs: clarify integration test seup --- .github/CONTRIBUTING.md | 27 +++++++++++++++++---------- .github/workflows/master-pytest.yml | 2 +- demo/README.MD | 21 +++++++++++++-------- 3 files changed, 31 insertions(+), 19 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index a7bd6ad..d4d275b 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -7,7 +7,8 @@ operating environment, schemachange version and python version. Whenever possibl also include a brief, self-contained code example that demonstrates the problem. We have -included [issue templates](https://github.com/Snowflake-Labs/schemachange/issues/new/choose) for reporting bugs, requesting features and seeking clarifications. Choose the appropriate issue template to contribute to the repository. +included [issue templates](https://github.com/Snowflake-Labs/schemachange/issues/new/choose) for reporting bugs, +requesting features and seeking clarifications. Choose the appropriate issue template to contribute to the repository. ## Contributing code @@ -22,8 +23,6 @@ Thank you for your interest in contributing code to schemachange! ### Guide to contributing to schemachange -> **IMPORTANT** : You will need to follow the [provisioning and schemachange setup instructions](../demo/README.MD) to ensure you can run GitHub actions against your Snowflake account before placing a PR with main schemachange repository so that your PR can be merged into schemachange master branch. - 1. If you are a first-time contributor + Go to [Snowflake-Labs/Schemachange](https://github.com/Snowflake-Labs/schemachange) and click the "fork" button to create your own copy of the project. @@ -53,8 +52,8 @@ Thank you for your interest in contributing code to schemachange! + [Pull](https://github.com/git-guides/git-pull) the latest changes from upstream, including tags: ```shell - git checkout main - git pull upstream main --tags + git checkout master + git pull upstream master --tags ``` 2. Create and Activate a Virtual Environment @@ -98,24 +97,32 @@ Thank you for your interest in contributing code to schemachange! + Commit locally as you progress ( [git add](https://github.com/git-guides/git-add) and [git commit](https://github.com/git-guides/git-commit) ). Use a properly formatted commit message. Be sure to - document any changed behavior in the [CHANGELOG.md](../CHANGELOG.md) file to help us collate the changes for a specific release. + document any changed behavior in the [CHANGELOG.md](../CHANGELOG.md) file to help us collate the changes for a + specific release. 4. Test your contribution locally ```bash python -m pytest ``` - PS: Please add test cases to the features you are developing so that over time, we can capture any lapse in functionality changes. + PS: Please add test cases to the features you are developing so that over time, we can capture any lapse in + functionality changes. + +5. Perform integration tests on your branch from your fork + - Follow the [provisioning and schemachange setup instructions](../demo/README.MD) to configure your Snowflake + account for testing. + - Follow [these](https://docs.github.com/en/actions/managing-workflow-runs-and-deployments/managing-workflow-runs/manually-running-a-workflow) + instructions to manually run the `master-pytest` workflow on your fork of the repo, targeting your feature branch. -5. Push your contribution to GitHub +6. Push your contribution to GitHub - [Push](https://github.com/git-guides/git-push) your changes back to your fork on GitHub + [Push](https://github.com/git-guides/git-push) your changes back to your fork on GitHub ```shell git push origin update-build-library-dependencies ``` -6. Raise a Pull Request to merge your contribution into the a Schemachange Release +7. Raise a Pull Request to merge your contribution into the a Schemachange Release + Go to GitHub. The new branch will show up with a green [Pull Request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests#initiating-the-pull-request) button. Make sure the title and message are clear, concise and self-explanatory. Then click the button to submit diff --git a/.github/workflows/master-pytest.yml b/.github/workflows/master-pytest.yml index 385403e..027a09e 100644 --- a/.github/workflows/master-pytest.yml +++ b/.github/workflows/master-pytest.yml @@ -88,7 +88,7 @@ jobs: echo "::endgroup::" echo "::group::Testing Rendering to ${MY_TARGET_SCHEMA}" - + schemachange render \ --config-folder ./demo/${SCENARIO_NAME} \ ./demo/${SCENARIO_NAME}/2_test/A__render.sql diff --git a/demo/README.MD b/demo/README.MD index a36dadf..f67b2f3 100644 --- a/demo/README.MD +++ b/demo/README.MD @@ -5,7 +5,7 @@ to see how schemachange works with the main feature set. For the contributor, wh codebase, this will serve as a basis to test the PR against your own snowflake account to ensure your code change does not break any existing functionality. -## Prerequisite +## Prerequisites - You will need your own snowflake Account to test the Demo - Both as a contributor and consumer. - You will need to review and run statements in the provision folder or set up your own database and schema. @@ -43,14 +43,19 @@ the demo DDL scripts. ### Contributors -As a contributor, you will have to set up schemachange demo database and schemachange schema (See Initialize and Setup -scripts below). Along with that you will also set up the following Secrets in your forked repository so that the GitHub -actions can set up, test and teardown the temporary schema it creates to test the changes to your code in the master and -dev branches respectively. +1. Execute the [initialize.sql](provision/initialize.sql) + and [setup_schemachange_schema.sql](provision/setup_schemachange_schema.sql) scripts to create up a + `SCHEMACHANGE_DEMO` + database and `SCHEMACHANGE` schema (See [Prerequisites](#prerequisites)). -- SCHEMACHANGE_SNOWFLAKE_PASSWORD -- SCHEMACHANGE_SNOWFLAKE_USER -- SCHEMACHANGE_SNOWFLAKE_ACCOUNT +2. Create the + following [GitHub Action Secrets](https://docs.github.com/en/actions/security-for-github-actions/security-guides/using-secrets-in-github-actions) + in your forked repository. GitHub will use these actions to set up and teardown the temporary schema(s) it creates to + test your code. + + - `SCHEMACHANGE_SNOWFLAKE_PASSWORD` + - `SCHEMACHANGE_SNOWFLAKE_USER` + - `SCHEMACHANGE_SNOWFLAKE_ACCOUNT` # Setup