From cce63748a0c74e93e361c97990923492b90ec900 Mon Sep 17 00:00:00 2001 From: Vida Maleki Date: Thu, 19 Sep 2024 13:00:46 -0700 Subject: [PATCH 01/10] Added execute command and tests for Streamlit app --- .../cli/_plugins/streamlit/commands.py | 12 +++ .../cli/_plugins/streamlit/manager.py | 4 + tests/__snapshots__/test_help_messages.ambr | 78 +++++++++++++++++++ tests/stage/__snapshots__/test_stage.ambr | 22 ------ tests/streamlit/test_commands.py | 11 +++ tests/streamlit/test_streamlit_manager.py | 13 ++++ 6 files changed, 118 insertions(+), 22 deletions(-) diff --git a/src/snowflake/cli/_plugins/streamlit/commands.py b/src/snowflake/cli/_plugins/streamlit/commands.py index 1d7ef2d571..9449cbc66a 100644 --- a/src/snowflake/cli/_plugins/streamlit/commands.py +++ b/src/snowflake/cli/_plugins/streamlit/commands.py @@ -81,6 +81,18 @@ ) +@app.command(requires_connection=True) +def execute( + name: FQN = StreamlitNameArgument, + **options, +): + """ + Executes a streamlit in a headless mode. + """ + _ = StreamlitManager().execute(app_name=name) + return MessageResult(f"Streamlit {name} executed.") + + @app.command("share", requires_connection=True) def streamlit_share( name: FQN = StreamlitNameArgument, diff --git a/src/snowflake/cli/_plugins/streamlit/manager.py b/src/snowflake/cli/_plugins/streamlit/manager.py index 62df6329d3..3ce2893dcf 100644 --- a/src/snowflake/cli/_plugins/streamlit/manager.py +++ b/src/snowflake/cli/_plugins/streamlit/manager.py @@ -43,6 +43,10 @@ class StreamlitManager(SqlExecutionMixin): + def execute(self, app_name: FQN): + query = f"EXECUTE STREAMLIT {app_name.sql_identifier}()" + return self._execute_query(query=query) + def share(self, streamlit_name: FQN, to_role: str) -> SnowflakeCursor: return self._execute_query( f"grant usage on streamlit {streamlit_name.sql_identifier} to role {to_role}" diff --git a/tests/__snapshots__/test_help_messages.ambr b/tests/__snapshots__/test_help_messages.ambr index 915a1dd223..76e13cc657 100644 --- a/tests/__snapshots__/test_help_messages.ambr +++ b/tests/__snapshots__/test_help_messages.ambr @@ -7797,6 +7797,82 @@ +------------------------------------------------------------------------------+ + ''' +# --- +# name: test_help_messages[streamlit.execute] + ''' + + Usage: default streamlit execute [OPTIONS] NAME + + Executes a streamlit in a headless mode. + + +- Arguments ------------------------------------------------------------------+ + | * name TEXT Identifier of the Streamlit app. For example: | + | my_streamlit | + | [required] | + +------------------------------------------------------------------------------+ + +- Options --------------------------------------------------------------------+ + | --help -h Show this message and exit. | + +------------------------------------------------------------------------------+ + +- Connection configuration ---------------------------------------------------+ + | --connection,--environment -c TEXT Name of the connection, as | + | defined in your config.toml. | + | Default: default. | + | --account,--accountname TEXT Name assigned to your | + | Snowflake account. Overrides | + | the value specified for the | + | connection. | + | --user,--username TEXT Username to connect to | + | Snowflake. Overrides the value | + | specified for the connection. | + | --password TEXT Snowflake password. Overrides | + | the value specified for the | + | connection. | + | --authenticator TEXT Snowflake authenticator. | + | Overrides the value specified | + | for the connection. | + | --private-key-file,--private-… TEXT Snowflake private key file | + | path. Overrides the value | + | specified for the connection. | + | --token-file-path TEXT Path to file with an OAuth | + | token that should be used when | + | connecting to Snowflake | + | --database,--dbname TEXT Database to use. Overrides the | + | value specified for the | + | connection. | + | --schema,--schemaname TEXT Database schema to use. | + | Overrides the value specified | + | for the connection. | + | --role,--rolename TEXT Role to use. Overrides the | + | value specified for the | + | connection. | + | --warehouse TEXT Warehouse to use. Overrides | + | the value specified for the | + | connection. | + | --temporary-connection -x Uses connection defined with | + | command line parameters, | + | instead of one defined in | + | config | + | --mfa-passcode TEXT Token to use for multi-factor | + | authentication (MFA) | + | --enable-diag Run python connector | + | diagnostic test | + | --diag-log-path TEXT Diagnostic report path | + | --diag-allowlist-path TEXT Diagnostic report path to | + | optional allowlist | + +------------------------------------------------------------------------------+ + +- Global configuration -------------------------------------------------------+ + | --format [TABLE|JSON] Specifies the output format. | + | [default: TABLE] | + | --verbose -v Displays log entries for log levels info | + | and higher. | + | --debug Displays log entries for log levels debug | + | and higher; debug logs contains additional | + | information. | + | --silent Turns off intermediate output to console. | + +------------------------------------------------------------------------------+ + + ''' # --- # name: test_help_messages[streamlit.get-url] @@ -8055,6 +8131,7 @@ | command will raise an error. | | describe Provides description of streamlit. | | drop Drops streamlit with given name. | + | execute Executes a streamlit in a headless mode. | | get-url Returns a URL to the specified Streamlit app | | list Lists all available streamlits. | | share Shares a Streamlit app with another role. | @@ -8476,6 +8553,7 @@ | command will raise an error. | | describe Provides description of streamlit. | | drop Drops streamlit with given name. | + | execute Executes a streamlit in a headless mode. | | get-url Returns a URL to the specified Streamlit app | | list Lists all available streamlits. | | share Shares a Streamlit app with another role. | diff --git a/tests/stage/__snapshots__/test_stage.ambr b/tests/stage/__snapshots__/test_stage.ambr index af8887291c..f3b8576c06 100644 --- a/tests/stage/__snapshots__/test_stage.ambr +++ b/tests/stage/__snapshots__/test_stage.ambr @@ -34,17 +34,6 @@ ''' # --- -# name: test_execute[@DB.SCHEMA.EXE/s1.sql-@db.schema.exe-expected_files19] - ''' - SUCCESS - @db.schema.exe/s1.sql - +-----------------------------------------+ - | File | Status | Error | - |-----------------------+---------+-------| - | @db.schema.exe/s1.sql | SUCCESS | None | - +-----------------------------------------+ - - ''' -# --- # name: test_execute[@DB.schema.EXE/a/S3.sql-@DB.schema.EXE-expected_files20] ''' SUCCESS - @DB.schema.EXE/a/S3.sql @@ -56,17 +45,6 @@ ''' # --- -# name: test_execute[@DB.schema.EXE/a/S3.sql-@db.schema.exe-expected_files20] - ''' - SUCCESS - @db.schema.exe/a/S3.sql - +-------------------------------------------+ - | File | Status | Error | - |-------------------------+---------+-------| - | @db.schema.exe/a/S3.sql | SUCCESS | None | - +-------------------------------------------+ - - ''' -# --- # name: test_execute[@db.schema.exe-@db.schema.exe-expected_files15] ''' SUCCESS - @db.schema.exe/s1.sql diff --git a/tests/streamlit/test_commands.py b/tests/streamlit/test_commands.py index 02720eea84..8d6d0cf5ab 100644 --- a/tests/streamlit/test_commands.py +++ b/tests/streamlit/test_commands.py @@ -18,6 +18,8 @@ import pytest from snowflake.cli._plugins.connection.util import REGIONLESS_QUERY +from snowflake.cli._plugins.streamlit.manager import StreamlitManager +from snowflake.cli.api.identifiers import FQN STREAMLIT_NAME = "test_streamlit" TEST_WAREHOUSE = "test_warehouse" @@ -911,3 +913,12 @@ def test_multiple_streamlit_raise_error_if_multiple_entities( assert result.exit_code == 2, result.output assert result.output == os_agnostic_snapshot + + +@mock.patch.object(StreamlitManager, "execute") +def test_execute_streamlit(mock_execute, runner): + result = runner.invoke(["streamlit", "execute", STREAMLIT_NAME]) + + assert result.exit_code == 0, result.output + assert result.output == f"Streamlit {STREAMLIT_NAME} executed.\n" + mock_execute.assert_called_once_with(app_name=FQN.from_string(STREAMLIT_NAME)) diff --git a/tests/streamlit/test_streamlit_manager.py b/tests/streamlit/test_streamlit_manager.py index 2d6f82e9d0..bf0763f25b 100644 --- a/tests/streamlit/test_streamlit_manager.py +++ b/tests/streamlit/test_streamlit_manager.py @@ -7,6 +7,7 @@ from snowflake.cli._plugins.streamlit.streamlit_entity_model import ( StreamlitEntityModel, ) +from snowflake.cli.api.identifiers import FQN mock_streamlit_exists = mock.patch( "snowflake.cli._plugins.streamlit.manager.ObjectManager.object_exists", @@ -90,3 +91,15 @@ def test_deploy_streamlit_with_api_integrations( secrets=('my_secret'=SecretOfTheSecrets, 'other'=other_secret)""" ) ) + + +@mock.patch("snowflake.cli._plugins.streamlit.manager.StreamlitManager._execute_query") +@mock_streamlit_exists +def test_execute_streamlit(mock_execute_query): + app_name = FQN(database="DB", schema="SH", name="my_streamlit_app") + + StreamlitManager(MagicMock()).execute(app_name=app_name) + + mock_execute_query.assert_called_once_with( + query="EXECUTE STREAMLIT IDENTIFIER('DB.SH.my_streamlit_app')()" + ) From 58bad5b164cbc896b6db0e89746fb03dad356693 Mon Sep 17 00:00:00 2001 From: Vida Maleki Date: Thu, 19 Sep 2024 13:23:49 -0700 Subject: [PATCH 02/10] Added note for execute command in release note --- RELEASE-NOTES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 5427d27b29..6887119701 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -45,6 +45,7 @@ * Added `SNOWFLAKE_..._PRIVATE_KEY_RAW` environment variable to pass private key as a raw string. * Added periodic check for newest version of Snowflake CLI. When new version is available, user will be notified. * Added support for `imports` in Streamlit definition. +* Added `snow streamlit execute app-name` command to run Streamlit apps in a Snowflake environment in headless mode. ## Fixes and improvements * Fixed problem with whitespaces in `snow connection add` command. From 31c739b7930da4a9af4498037a2c35f50f574960 Mon Sep 17 00:00:00 2001 From: Adam Stus Date: Fri, 20 Sep 2024 12:11:04 +0200 Subject: [PATCH 03/10] Added class to hide variables in tracebacks (#1599) --- src/snowflake/cli/_app/secret.py | 9 ++++ src/snowflake/cli/_app/snow_connector.py | 66 ++++++++++++++---------- tests/test_connection.py | 3 +- tests/test_snow_connector.py | 10 ++-- tests_integration/snowflake_connector.py | 1 + 5 files changed, 57 insertions(+), 32 deletions(-) create mode 100644 src/snowflake/cli/_app/secret.py diff --git a/src/snowflake/cli/_app/secret.py b/src/snowflake/cli/_app/secret.py new file mode 100644 index 0000000000..d833988aee --- /dev/null +++ b/src/snowflake/cli/_app/secret.py @@ -0,0 +1,9 @@ +class SecretType: + def __init__(self, value): + self.value = value + + def __repr__(self): + return "SecretType(***)" + + def __str___(self): + return "***" diff --git a/src/snowflake/cli/_app/snow_connector.py b/src/snowflake/cli/_app/snow_connector.py index d0f3a729c4..ccd48c75c4 100644 --- a/src/snowflake/cli/_app/snow_connector.py +++ b/src/snowflake/cli/_app/snow_connector.py @@ -24,6 +24,7 @@ from snowflake.cli._app.constants import ( PARAM_APPLICATION_NAME, ) +from snowflake.cli._app.secret import SecretType from snowflake.cli._app.telemetry import command_info from snowflake.cli.api.config import ( get_connection_dict, @@ -205,7 +206,7 @@ def _load_private_key(connection_parameters: Dict, private_key_var_name: str) -> connection_parameters[private_key_var_name] ) private_key = _load_pem_to_der(private_key_pem) - connection_parameters["private_key"] = private_key + connection_parameters["private_key"] = private_key.value del connection_parameters[private_key_var_name] else: raise ClickException( @@ -217,10 +218,11 @@ def _load_private_key_from_parameters( connection_parameters: Dict, private_key_var_name: str ) -> None: if connection_parameters.get("authenticator") == "SNOWFLAKE_JWT": - private_key_pem = connection_parameters[private_key_var_name] - private_key_pem = private_key_pem.encode("utf-8") + private_key_pem = _load_pem_from_parameters( + connection_parameters[private_key_var_name] + ) private_key = _load_pem_to_der(private_key_pem) - connection_parameters["private_key"] = private_key + connection_parameters["private_key"] = private_key.value del connection_parameters[private_key_var_name] else: raise ClickException( @@ -236,43 +238,49 @@ def _update_connection_application_name(connection_parameters: Dict): connection_parameters.update(connection_application_params) -def _load_pem_from_file(private_key_file: str) -> bytes: +def _load_pem_from_file(private_key_file: str) -> SecretType: with SecurePath(private_key_file).open( "rb", read_file_limit_mb=DEFAULT_SIZE_LIMIT_MB ) as f: - private_key_pem = f.read() + private_key_pem = SecretType(f.read()) return private_key_pem -def _load_pem_to_der(private_key_pem: bytes) -> bytes: +def _load_pem_from_parameters(private_key_raw: str) -> SecretType: + return SecretType(private_key_raw.encode("utf-8")) + + +def _load_pem_to_der(private_key_pem: SecretType) -> SecretType: """ Given a private key file path (in PEM format), decode key data into DER format """ - private_key_passphrase = os.getenv("PRIVATE_KEY_PASSPHRASE", None) + private_key_passphrase = SecretType(os.getenv("PRIVATE_KEY_PASSPHRASE", None)) if ( - private_key_pem.startswith(ENCRYPTED_PKCS8_PK_HEADER) - and private_key_passphrase is None + private_key_pem.value.startswith(ENCRYPTED_PKCS8_PK_HEADER) + and private_key_passphrase.value is None ): raise ClickException( "Encrypted private key, you must provide the" "passphrase in the environment variable PRIVATE_KEY_PASSPHRASE" ) - if not private_key_pem.startswith( + if not private_key_pem.value.startswith( ENCRYPTED_PKCS8_PK_HEADER - ) and not private_key_pem.startswith(UNENCRYPTED_PKCS8_PK_HEADER): + ) and not private_key_pem.value.startswith(UNENCRYPTED_PKCS8_PK_HEADER): raise ClickException( "Private key provided is not in PKCS#8 format. Please use correct format." ) - if private_key_pem.startswith(UNENCRYPTED_PKCS8_PK_HEADER): - private_key_passphrase = None + if private_key_pem.value.startswith(UNENCRYPTED_PKCS8_PK_HEADER): + private_key_passphrase = SecretType(None) return prepare_private_key(private_key_pem, private_key_passphrase) -def prepare_private_key(private_key_pem, private_key_passphrase=None): +def prepare_private_key( + private_key_pem: SecretType, private_key_passphrase: SecretType = SecretType(None) +): from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.serialization import ( Encoding, @@ -281,17 +289,21 @@ def prepare_private_key(private_key_pem, private_key_passphrase=None): load_pem_private_key, ) - private_key = load_pem_private_key( - private_key_pem, - ( - str.encode(private_key_passphrase) - if private_key_passphrase is not None - else private_key_passphrase - ), - default_backend(), + private_key = SecretType( + load_pem_private_key( + private_key_pem.value, + ( + str.encode(private_key_passphrase.value) + if private_key_passphrase.value is not None + else private_key_passphrase.value + ), + default_backend(), + ) ) - return private_key.private_bytes( - encoding=Encoding.DER, - format=PrivateFormat.PKCS8, - encryption_algorithm=NoEncryption(), + return SecretType( + private_key.value.private_bytes( + encoding=Encoding.DER, + format=PrivateFormat.PKCS8, + encryption_algorithm=NoEncryption(), + ) ) diff --git a/tests/test_connection.py b/tests/test_connection.py index 6e7bb52a29..f3a9a966fa 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -21,6 +21,7 @@ import pytest import tomlkit +from snowflake.cli._app.secret import SecretType from snowflake.cli.api.constants import ObjectType from tests_common import IS_WINDOWS @@ -705,7 +706,7 @@ def test_key_pair_authentication_from_config( ): ctx = mock_ctx() mock_connector.return_value = ctx - mock_convert.return_value = "secret value" + mock_convert.return_value = SecretType("secret value") with NamedTemporaryFile("w+", suffix="toml") as tmp_file: tmp_file.write( diff --git a/tests/test_snow_connector.py b/tests/test_snow_connector.py index 81b755e64d..302bf684bd 100644 --- a/tests/test_snow_connector.py +++ b/tests/test_snow_connector.py @@ -16,6 +16,7 @@ from unittest import mock import pytest +from snowflake.cli._app.secret import SecretType # Used as a solution to syrupy having some problems with comparing multilines string @@ -118,9 +119,10 @@ def test_private_key_loading_and_aliases( else: overrides[user_input] = override_value + key = SecretType(b"bytes") mock_command_info.return_value = "SNOWCLI.SQL" - mock_load_pem_from_file.return_value = b"bytes" - mock_load_pem_to_der.return_value = b"bytes" + mock_load_pem_from_file.return_value = key + mock_load_pem_to_der.return_value = key conn_dict = get_connection_dict(connection_name) default_value = conn_dict.get("private_key_file", None) or conn_dict.get( @@ -135,7 +137,7 @@ def test_private_key_loading_and_aliases( expected_private_key_args = ( {} if expected_private_key_file_value is None - else dict(private_key=mock_load_pem_to_der.return_value) + else dict(private_key=b"bytes") ) mock_connect.assert_called_once_with( application=mock_command_info.return_value, @@ -145,7 +147,7 @@ def test_private_key_loading_and_aliases( ) if expected_private_key_file_value is not None: mock_load_pem_from_file.assert_called_with(expected_private_key_file_value) - mock_load_pem_to_der.assert_called_with(b"bytes") + mock_load_pem_to_der.assert_called_with(key) @mock.patch.dict(os.environ, {}, clear=True) diff --git a/tests_integration/snowflake_connector.py b/tests_integration/snowflake_connector.py index 8dc7bcbcf5..ee1a6d6f0a 100644 --- a/tests_integration/snowflake_connector.py +++ b/tests_integration/snowflake_connector.py @@ -22,6 +22,7 @@ import pytest from snowflake import connector +from snowflake.cli._app.secret import SecretType from snowflake.cli.api.exceptions import EnvironmentVariableNotFoundError from snowflake.cli._app.snow_connector import update_connection_details_with_private_key From 412c0cf8b9a492a35021dfadae676efd22b51934 Mon Sep 17 00:00:00 2001 From: Tomasz Urbaszek Date: Fri, 20 Sep 2024 12:50:14 +0200 Subject: [PATCH 04/10] Rename ws migrate to migrate-definition (#1598) --- RELEASE-NOTES.md | 2 +- .../commands_registration/builtin_plugins.py | 2 + .../cli/_plugins/helpers/__init__.py | 13 + .../cli/_plugins/helpers/commands.py | 57 +++ .../cli/_plugins/helpers/plugin_spec.py | 30 ++ .../cli/_plugins/workspace/commands.py | 34 +- tests/__snapshots__/test_help_messages.ambr | 4 +- tests/helpers/__init__.py | 0 .../helpers/__snapshots__/test_v1_to_v2.ambr | 324 ++++++++++++++++++ tests/helpers/test_v1_to_v2.py | 143 ++++++++ tests/test_help_messages.py | 4 +- tests/workspace/test_manager.py | 142 +------- .../__snapshots__/test_installation.ambr | 2 +- 13 files changed, 577 insertions(+), 180 deletions(-) create mode 100644 src/snowflake/cli/_plugins/helpers/__init__.py create mode 100644 src/snowflake/cli/_plugins/helpers/commands.py create mode 100644 src/snowflake/cli/_plugins/helpers/plugin_spec.py create mode 100644 tests/helpers/__init__.py create mode 100644 tests/helpers/__snapshots__/test_v1_to_v2.ambr create mode 100644 tests/helpers/test_v1_to_v2.py diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 6887119701..afb992325f 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -39,7 +39,7 @@ * Added support for external access (api integrations and secrets) in Streamlit. * Added support for `<% ... %>` syntax in SQL templating. * Support multiple Streamlit application in single snowflake.yml project definition file. -* Added `snow ws migrate` command to migrate `snowflake.yml` file from V1 to V2. +* Added `snow helpers v1-to-v2` command to migrate `snowflake.yml` file from V1 to V2. * Added `--package-entity-id` and `--app-entity-id` options to `snow app` commands to allow targeting specific entities when the `definition_version` in `snowflake.yml` is `2` or higher and it contains multiple `application package` or `application` entities. * Added templates expansion of arbitrary files for Native Apps through `templates` processor. * Added `SNOWFLAKE_..._PRIVATE_KEY_RAW` environment variable to pass private key as a raw string. diff --git a/src/snowflake/cli/_app/commands_registration/builtin_plugins.py b/src/snowflake/cli/_app/commands_registration/builtin_plugins.py index 018f41a943..072955ae43 100644 --- a/src/snowflake/cli/_app/commands_registration/builtin_plugins.py +++ b/src/snowflake/cli/_app/commands_registration/builtin_plugins.py @@ -15,6 +15,7 @@ from snowflake.cli._plugins.connection import plugin_spec as connection_plugin_spec from snowflake.cli._plugins.cortex import plugin_spec as cortex_plugin_spec from snowflake.cli._plugins.git import plugin_spec as git_plugin_spec +from snowflake.cli._plugins.helpers import plugin_spec as migrate_plugin_spec from snowflake.cli._plugins.init import plugin_spec as init_plugin_spec from snowflake.cli._plugins.nativeapp import plugin_spec as nativeapp_plugin_spec from snowflake.cli._plugins.notebook import plugin_spec as notebook_plugin_spec @@ -31,6 +32,7 @@ def get_builtin_plugin_name_to_plugin_spec(): plugin_specs = { "connection": connection_plugin_spec, + "helpers": migrate_plugin_spec, "spcs": spcs_plugin_spec, "nativeapp": nativeapp_plugin_spec, "object": object_plugin_spec, diff --git a/src/snowflake/cli/_plugins/helpers/__init__.py b/src/snowflake/cli/_plugins/helpers/__init__.py new file mode 100644 index 0000000000..ada0a4e13d --- /dev/null +++ b/src/snowflake/cli/_plugins/helpers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024 Snowflake Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/snowflake/cli/_plugins/helpers/commands.py b/src/snowflake/cli/_plugins/helpers/commands.py new file mode 100644 index 0000000000..4799f29361 --- /dev/null +++ b/src/snowflake/cli/_plugins/helpers/commands.py @@ -0,0 +1,57 @@ +# Copyright (c) 2024 Snowflake Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import typer +import yaml +from snowflake.cli.api.commands.snow_typer import SnowTyperFactory +from snowflake.cli.api.output.types import MessageResult +from snowflake.cli.api.project.definition_conversion import ( + convert_project_definition_to_v2, +) +from snowflake.cli.api.project.definition_manager import DefinitionManager +from snowflake.cli.api.secure_path import SecurePath + +app = SnowTyperFactory( + name="helpers", + help="Helper commands.", +) + + +@app.command() +def v1_to_v2( + accept_templates: bool = typer.Option( + False, "-t", "--accept-templates", help="Allows the migration of templates." + ), + **options, +): + """Migrates the Snowpark, Streamlit, and Native App project definition files from V1 to V2.""" + manager = DefinitionManager() + pd = manager.unrendered_project_definition + + if pd.meets_version_requirement("2"): + return MessageResult("Project definition is already at version 2.") + + pd_v2 = convert_project_definition_to_v2(manager.project_root, pd, accept_templates) + + SecurePath("snowflake.yml").rename("snowflake_V1.yml") + with open("snowflake.yml", "w") as file: + yaml.dump( + pd_v2.model_dump( + exclude_unset=True, exclude_none=True, mode="json", by_alias=True + ), + file, + ) + return MessageResult("Project definition migrated to version 2.") diff --git a/src/snowflake/cli/_plugins/helpers/plugin_spec.py b/src/snowflake/cli/_plugins/helpers/plugin_spec.py new file mode 100644 index 0000000000..492cf65c91 --- /dev/null +++ b/src/snowflake/cli/_plugins/helpers/plugin_spec.py @@ -0,0 +1,30 @@ +# Copyright (c) 2024 Snowflake Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from snowflake.cli._plugins.helpers import commands +from snowflake.cli.api.plugins.command import ( + SNOWCLI_ROOT_COMMAND_PATH, + CommandSpec, + CommandType, + plugin_hook_impl, +) + + +@plugin_hook_impl +def command_spec(): + return CommandSpec( + parent_command_path=SNOWCLI_ROOT_COMMAND_PATH, + command_type=CommandType.COMMAND_GROUP, + typer_instance=commands.app.create_instance(), + ) diff --git a/src/snowflake/cli/_plugins/workspace/commands.py b/src/snowflake/cli/_plugins/workspace/commands.py index 3d45f0cc68..99c602e377 100644 --- a/src/snowflake/cli/_plugins/workspace/commands.py +++ b/src/snowflake/cli/_plugins/workspace/commands.py @@ -20,7 +20,6 @@ from typing import List, Optional import typer -import yaml from snowflake.cli._plugins.nativeapp.artifacts import BundleMap from snowflake.cli._plugins.nativeapp.common_flags import ( ForceOption, @@ -34,46 +33,15 @@ from snowflake.cli.api.entities.common import EntityActions from snowflake.cli.api.exceptions import IncompatibleParametersError from snowflake.cli.api.output.types import MessageResult, QueryResult -from snowflake.cli.api.project.definition_conversion import ( - convert_project_definition_to_v2, -) -from snowflake.cli.api.project.definition_manager import DefinitionManager -from snowflake.cli.api.secure_path import SecurePath ws = SnowTyperFactory( name="ws", help="Deploy and interact with snowflake.yml-based entities.", + is_hidden=lambda: True, ) log = logging.getLogger(__name__) -@ws.command() -def migrate( - accept_templates: bool = typer.Option( - False, "-t", "--accept-templates", help="Allows the migration of templates." - ), - **options, -): - """Migrates the Snowpark, Streamlit, and Native App project definition files from V1 to V2.""" - manager = DefinitionManager() - pd = manager.unrendered_project_definition - - if pd.meets_version_requirement("2"): - return MessageResult("Project definition is already at version 2.") - - pd_v2 = convert_project_definition_to_v2(manager.project_root, pd, accept_templates) - - SecurePath("snowflake.yml").rename("snowflake_V1.yml") - with open("snowflake.yml", "w") as file: - yaml.dump( - pd_v2.model_dump( - exclude_unset=True, exclude_none=True, mode="json", by_alias=True - ), - file, - ) - return MessageResult("Project definition migrated to version 2.") - - @ws.command(requires_connection=True, hidden=True) @with_project_definition() def bundle( diff --git a/tests/__snapshots__/test_help_messages.ambr b/tests/__snapshots__/test_help_messages.ambr index 2c0042a7d1..fa32bdcdc1 100644 --- a/tests/__snapshots__/test_help_messages.ambr +++ b/tests/__snapshots__/test_help_messages.ambr @@ -25,6 +25,7 @@ | connection Manages connections to Snowflake. | | cortex Provides access to Snowflake Cortex. | | git Manages git repositories in Snowflake. | + | helpers Helper commands. | | init Creates project directory from template. | | notebook Manages notebooks in Snowflake. | | object Manages Snowflake objects like warehouses and stages | @@ -34,7 +35,6 @@ | sql Executes Snowflake query. | | stage Manages stages. | | streamlit Manages a Streamlit app in Snowflake. | - | ws Deploy and interact with snowflake.yml-based entities. | +------------------------------------------------------------------------------+ @@ -8161,6 +8161,7 @@ | connection Manages connections to Snowflake. | | cortex Provides access to Snowflake Cortex. | | git Manages git repositories in Snowflake. | + | helpers Helper commands. | | init Creates project directory from template. | | notebook Manages notebooks in Snowflake. | | object Manages Snowflake objects like warehouses and stages | @@ -8170,7 +8171,6 @@ | sql Executes Snowflake query. | | stage Manages stages. | | streamlit Manages a Streamlit app in Snowflake. | - | ws Deploy and interact with snowflake.yml-based entities. | +------------------------------------------------------------------------------+ diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/helpers/__snapshots__/test_v1_to_v2.ambr b/tests/helpers/__snapshots__/test_v1_to_v2.ambr new file mode 100644 index 0000000000..913229f454 --- /dev/null +++ b/tests/helpers/__snapshots__/test_v1_to_v2.ambr @@ -0,0 +1,324 @@ +# serializer version: 1 +# name: test_if_template_is_not_rendered_during_migration_with_option_checked[snowpark_templated_v1] + ''' + definition_version: '2' + entities: + hello_function: + artifacts: + - dest: + src: <% ctx.env.project_source %> + external_access_integrations: [] + handler: functions.hello_function + identifier: + name: hello_function + imports: [] + meta: + use_mixins: + - snowpark_shared + returns: string + secrets: {} + signature: + - name: name + type: string + stage: + type: function + hello_procedure: + artifacts: + - dest: + src: <% ctx.env.project_source %> + execute_as_caller: false + external_access_integrations: [] + handler: procedures.hello_procedure + identifier: + name: hello_procedure + imports: [] + meta: + use_mixins: + - snowpark_shared + returns: string + secrets: {} + signature: + - name: name + type: string + stage: + type: procedure + test_procedure: + artifacts: + - dest: + src: <% ctx.env.project_source %> + execute_as_caller: false + external_access_integrations: [] + handler: procedures.test_procedure + identifier: + name: test_procedure + imports: [] + meta: + use_mixins: + - snowpark_shared + returns: string + secrets: {} + signature: '' + stage: + type: procedure + env: + project_source: app/ + mixins: + snowpark_shared: + artifacts: + - dest: + src: <% ctx.env.project_source %> + stage: + + ''' +# --- +# name: test_if_template_is_not_rendered_during_migration_with_option_checked[snowpark_templated_v1].1 + ''' + definition_version: "1.1" + snowpark: + project_name: "" + stage_name: "" + src: <% ctx.env.project_source %> + functions: + - name: hello_function + handler: "functions.hello_function" + signature: + - name: "name" + type: "string" + returns: string + procedures: + - name: hello_procedure + handler: "procedures.hello_procedure" + signature: + - name: "name" + type: "string" + returns: string + - name: test_procedure + handler: "procedures.test_procedure" + signature: "" + returns: string + env: + project_source: "app/" + + ''' +# --- +# name: test_if_template_is_not_rendered_during_migration_with_option_checked[streamlit_templated_v1] + ''' + definition_version: '2' + entities: + streamlit_entity_1: + artifacts: + - streamlit_app.py + - environment.yml + - pages + - common/hello.py + identifier: + name: + main_file: streamlit_app.py + pages_dir: pages + query_warehouse: + stage: + title: <% ctx.env.streamlit_title %> + type: streamlit + env: + streamlit_title: My Fancy Streamlit + + ''' +# --- +# name: test_if_template_is_not_rendered_during_migration_with_option_checked[streamlit_templated_v1].1 + ''' + definition_version: "1.1" + streamlit: + name: + stage: + query_warehouse: + main_file: streamlit_app.py + env_file: environment.yml + pages_dir: pages/ + title: <% ctx.env.streamlit_title %> + additional_source_files: + - common/hello.py + env: + streamlit_title: "My Fancy Streamlit" + + ''' +# --- +# name: test_migrating_a_file_with_duplicated_keys_raises_an_error[\n - name: test\n handler: "test"\n signature: ""\n returns: string\n handler: test\n runtime: "3.10"\n ] + ''' + +- Error ----------------------------------------------------------------------+ + | While loading the project definition file, duplicate key was found: handler | + +------------------------------------------------------------------------------+ + + ''' +# --- +# name: test_migrating_a_file_with_duplicated_keys_raises_an_error[\n - name: test\n handler: "test"\n signature: ""\n returns: string\n runtime: "3.10"\n ] + ''' + +- Error ----------------------------------------------------------------------+ + | Entity with name test seems to be duplicated. Please rename it and try | + | again. | + +------------------------------------------------------------------------------+ + + ''' +# --- +# name: test_migrating_a_file_with_duplicated_keys_raises_an_error[\nstreamlit:\n name: test\n stage: streamlit\n query_warehouse: test_warehouse\n main_file: "streamlit_app.py"\n title: "My Fancy Streamlit"\n ] + ''' + +- Error ----------------------------------------------------------------------+ + | In your project, streamlit and snowpark entities share the same name. Please | + | rename them and try again. | + +------------------------------------------------------------------------------+ + + ''' +# --- +# name: test_migrations_with_multiple_entities + ''' + definition_version: '2' + entities: + app: + from: + target: pkg + identifier: myapp_app + meta: + warehouse: app_wh + type: application + func1: + artifacts: + - dest: my_snowpark_project + src: app + external_access_integrations: [] + handler: app.func1_handler + identifier: + name: func1 + imports: [] + meta: + use_mixins: + - snowpark_shared + returns: string + runtime: '3.10' + secrets: {} + signature: + - default: default value + name: a + type: string + - name: b + type: variant + stage: dev_deployment + type: function + pkg: + artifacts: + - dest: ./ + src: app/* + - dest: ./ + processors: + - name: native app setup + - name: templates + properties: + foo: bar + src: to_process/* + bundle_root: output/bundle/ + deploy_root: output/deploy/ + distribution: external + generated_root: __generated/ + identifier: <% fn.concat_ids('myapp', '_pkg_', fn.sanitize_id(fn.get_username('unknown_user')) + | lower) %> + manifest: app/manifest.yml + meta: + role: pkg_role + scratch_stage: app_src.scratch + stage: app_src.stage + type: application package + procedureName: + artifacts: + - dest: my_snowpark_project + src: app + execute_as_caller: false + external_access_integrations: [] + handler: hello + identifier: + name: procedureName + imports: [] + meta: + use_mixins: + - snowpark_shared + returns: string + secrets: {} + signature: + - name: name + type: string + stage: dev_deployment + type: procedure + test_streamlit: + artifacts: + - streamlit_app.py + - environment.yml + - pages + identifier: + name: test_streamlit + main_file: streamlit_app.py + pages_dir: None + query_warehouse: test_warehouse + stage: streamlit + title: My Fancy Streamlit + type: streamlit + mixins: + snowpark_shared: + artifacts: + - dest: my_snowpark_project + src: app/ + stage: dev_deployment + + ''' +# --- +# name: test_migrations_with_multiple_entities.1 + ''' + definition_version: 1 + streamlit: + name: test_streamlit + stage: streamlit + query_warehouse: test_warehouse + main_file: "streamlit_app.py" + title: "My Fancy Streamlit" + snowpark: + project_name: "my_snowpark_project" + stage_name: "dev_deployment" + src: "app/" + functions: + - name: func1 + handler: "app.func1_handler" + signature: + - name: "a" + type: "string" + default: "default value" + - name: "b" + type: "variant" + returns: string + runtime: 3.10 + procedures: + - name: procedureName + handler: "hello" + signature: + - name: "name" + type: "string" + returns: string + native_app: + name: myapp + source_stage: app_src.stage + scratch_stage: app_src.scratch + artifacts: + - src: app/* + dest: ./ + - src: to_process/* + dest: ./ + processors: + - native app setup + - name: templates + properties: + foo: bar + package: + role: pkg_role + distribution: external + application: + name: myapp_app + warehouse: app_wh + debug: true + + ''' +# --- diff --git a/tests/helpers/test_v1_to_v2.py b/tests/helpers/test_v1_to_v2.py new file mode 100644 index 0000000000..6a230dac9b --- /dev/null +++ b/tests/helpers/test_v1_to_v2.py @@ -0,0 +1,143 @@ +import logging +from pathlib import Path +from textwrap import dedent + +import pytest +import yaml + + +def test_migration_already_v2( + runner, + project_directory, +): + with project_directory("migration_already_v2"): + result = runner.invoke(["helpers", "v1-to-v2"]) + + assert result.exit_code == 0 + assert "Project definition is already at version 2." in result.output + + +def test_migrations_with_multiple_entities( + runner, project_directory, os_agnostic_snapshot +): + with project_directory("migration_multiple_entities"): + result = runner.invoke(["helpers", "v1-to-v2"]) + assert result.exit_code == 0 + assert Path("snowflake.yml").read_text() == os_agnostic_snapshot + assert Path("snowflake_V1.yml").read_text() == os_agnostic_snapshot + + +def test_migration_native_app_missing_manifest(runner, project_directory): + with project_directory("migration_multiple_entities") as project_dir: + (project_dir / "app" / "manifest.yml").unlink() + result = runner.invoke(["helpers", "v1-to-v2"]) + assert result.exit_code == 1 + assert "manifest.yml file not found" in result.output + + +def test_migration_native_app_no_artifacts(runner, project_directory): + with project_directory("migration_multiple_entities") as project_dir: + with (project_dir / "snowflake.yml").open("r+") as snowflake_yml: + pdf = yaml.safe_load(snowflake_yml) + pdf["native_app"]["artifacts"] = [] + snowflake_yml.seek(0) + yaml.safe_dump(pdf, snowflake_yml) + snowflake_yml.truncate() + result = runner.invoke(["helpers", "v1-to-v2"]) + assert result.exit_code == 1 + assert "No artifacts mapping found in project definition" in result.output + assert "Could not bundle Native App artifacts" in result.output + + +def test_migration_native_app_package_scripts(runner, project_directory): + with project_directory("migration_package_scripts") as project_dir: + result = runner.invoke(["helpers", "v1-to-v2"]) + assert result.exit_code == 0 + package_scripts_dir = project_dir / "package_scripts" + for file in package_scripts_dir.iterdir(): + assert file.read_text() == dedent( + """\ + -- Just a demo package script, won't actually be executed in tests + select * from <% ctx.entities.pkg.identifier %>.my_schema.my_table + """ + ) + + +@pytest.mark.parametrize( + "project_directory_name", ["snowpark_templated_v1", "streamlit_templated_v1"] +) +def test_if_template_is_not_rendered_during_migration_with_option_checked( + runner, project_directory, project_directory_name, os_agnostic_snapshot, caplog +): + with project_directory(project_directory_name): + with caplog.at_level(logging.WARNING): + result = runner.invoke(["helpers", "v1-to-v2", "--accept-templates"]) + + assert result.exit_code == 0 + assert Path("snowflake.yml").read_text() == os_agnostic_snapshot + assert Path("snowflake_V1.yml").read_text() == os_agnostic_snapshot + assert ( + "Your V1 definition contains templates. We cannot guarantee the correctness of the migration." + in caplog.text + ) + + +@pytest.mark.parametrize( + "project_directory_name", ["snowpark_templated_v1", "streamlit_templated_v1"] +) +def test_if_template_raises_error_during_migrations( + runner, project_directory, project_directory_name, os_agnostic_snapshot +): + with project_directory(project_directory_name): + result = runner.invoke(["helpers", "v1-to-v2"]) + assert result.exit_code == 1, result.output + assert "Project definition contains templates" in result.output + + +def test_migration_with_only_envs(project_directory, runner): + with project_directory("sql_templating"): + result = runner.invoke(["helpers", "v1-to-v2"]) + + assert result.exit_code == 0 + + +@pytest.mark.parametrize( + "duplicated_entity", + [ + """ + - name: test + handler: "test" + signature: "" + returns: string + runtime: "3.10" + """, + """ +streamlit: + name: test + stage: streamlit + query_warehouse: test_warehouse + main_file: "streamlit_app.py" + title: "My Fancy Streamlit" + """, + """ + - name: test + handler: "test" + signature: "" + returns: string + handler: test + runtime: "3.10" + """, + ], +) +def test_migrating_a_file_with_duplicated_keys_raises_an_error( + runner, project_directory, os_agnostic_snapshot, duplicated_entity +): + with project_directory("snowpark_procedures") as pd: + definition_path = pd / "snowflake.yml" + + with open(definition_path, "a") as definition_file: + definition_file.write(duplicated_entity) + + result = runner.invoke(["helpers", "v1-to-v2"]) + assert result.exit_code == 1, result.output + assert result.output == os_agnostic_snapshot diff --git a/tests/test_help_messages.py b/tests/test_help_messages.py index 4370d043c4..f7f1ed2074 100644 --- a/tests/test_help_messages.py +++ b/tests/test_help_messages.py @@ -34,7 +34,7 @@ def iter_through_all_commands(command_groups_only: bool = False): Generator iterating through all commands. Paths are yielded as List[str] """ - ignore_plugins = ["render", "cortex", "workspace"] + ignore_plugins = ["helpers", "cortex", "workspace"] no_command: List[str] = [] yield no_command @@ -67,7 +67,7 @@ def test_help_messages(runner, snapshot, command): Check help messages against the snapshot """ result = runner.invoke(command + ["--help"]) - assert result.exit_code == 0 + assert result.exit_code == 0, result.output assert result.output == snapshot diff --git a/tests/workspace/test_manager.py b/tests/workspace/test_manager.py index a95425f30d..f4ecfbf586 100644 --- a/tests/workspace/test_manager.py +++ b/tests/workspace/test_manager.py @@ -13,10 +13,7 @@ # limitations under the License. from __future__ import annotations -import logging import os -from pathlib import Path -from textwrap import dedent from unittest import mock import pytest @@ -86,143 +83,6 @@ def test_bundle_of_invalid_entity_type(temp_dir): ws_manager.perform_action("app", EntityActions.BUNDLE) -def test_migration_already_v2( - runner, - project_directory, -): - with project_directory("migration_already_v2"): - result = runner.invoke(["ws", "migrate"]) - - assert result.exit_code == 0 - assert "Project definition is already at version 2." in result.output - - -def test_migrations_with_multiple_entities( - runner, project_directory, os_agnostic_snapshot -): - with project_directory("migration_multiple_entities"): - result = runner.invoke(["ws", "migrate"]) - assert result.exit_code == 0 - assert Path("snowflake.yml").read_text() == os_agnostic_snapshot - assert Path("snowflake_V1.yml").read_text() == os_agnostic_snapshot - - -def test_migration_native_app_missing_manifest(runner, project_directory): - with project_directory("migration_multiple_entities") as project_dir: - (project_dir / "app" / "manifest.yml").unlink() - result = runner.invoke(["ws", "migrate"]) - assert result.exit_code == 1 - assert "manifest.yml file not found" in result.output - - -def test_migration_native_app_no_artifacts(runner, project_directory): - with project_directory("migration_multiple_entities") as project_dir: - with (project_dir / "snowflake.yml").open("r+") as snowflake_yml: - pdf = yaml.safe_load(snowflake_yml) - pdf["native_app"]["artifacts"] = [] - snowflake_yml.seek(0) - yaml.safe_dump(pdf, snowflake_yml) - snowflake_yml.truncate() - result = runner.invoke(["ws", "migrate"]) - assert result.exit_code == 1 - assert "No artifacts mapping found in project definition" in result.output - assert "Could not bundle Native App artifacts" in result.output - - -def test_migration_native_app_package_scripts(runner, project_directory): - with project_directory("migration_package_scripts") as project_dir: - result = runner.invoke(["ws", "migrate"]) - assert result.exit_code == 0 - package_scripts_dir = project_dir / "package_scripts" - for file in package_scripts_dir.iterdir(): - assert file.read_text() == dedent( - """\ - -- Just a demo package script, won't actually be executed in tests - select * from <% ctx.entities.pkg.identifier %>.my_schema.my_table - """ - ) - - -@pytest.mark.parametrize( - "project_directory_name", ["snowpark_templated_v1", "streamlit_templated_v1"] -) -def test_if_template_is_not_rendered_during_migration_with_option_checked( - runner, project_directory, project_directory_name, os_agnostic_snapshot, caplog -): - with project_directory(project_directory_name): - with caplog.at_level(logging.WARNING): - result = runner.invoke(["ws", "migrate", "--accept-templates"]) - - assert result.exit_code == 0 - assert Path("snowflake.yml").read_text() == os_agnostic_snapshot - assert Path("snowflake_V1.yml").read_text() == os_agnostic_snapshot - assert ( - "Your V1 definition contains templates. We cannot guarantee the correctness of the migration." - in caplog.text - ) - - -@pytest.mark.parametrize( - "project_directory_name", ["snowpark_templated_v1", "streamlit_templated_v1"] -) -def test_if_template_raises_error_during_migrations( - runner, project_directory, project_directory_name, os_agnostic_snapshot -): - with project_directory(project_directory_name): - result = runner.invoke(["ws", "migrate"]) - assert result.exit_code == 1 - assert "Project definition contains templates" in result.output - - -def test_migration_with_only_envs(project_directory, runner): - with project_directory("sql_templating"): - result = runner.invoke(["ws", "migrate"]) - - assert result.exit_code == 0 - - -@pytest.mark.parametrize( - "duplicated_entity", - [ - """ - - name: test - handler: "test" - signature: "" - returns: string - runtime: "3.10" - """, - """ -streamlit: - name: test - stage: streamlit - query_warehouse: test_warehouse - main_file: "streamlit_app.py" - title: "My Fancy Streamlit" - """, - """ - - name: test - handler: "test" - signature: "" - returns: string - handler: test - runtime: "3.10" - """, - ], -) -def test_migrating_a_file_with_duplicated_keys_raises_an_error( - runner, project_directory, os_agnostic_snapshot, duplicated_entity -): - with project_directory("snowpark_procedures") as pd: - definition_path = pd / "snowflake.yml" - - with open(definition_path, "a") as definition_file: - definition_file.write(duplicated_entity) - - result = runner.invoke(["ws", "migrate"]) - assert result.exit_code == 1 - assert result.output == os_agnostic_snapshot - - @pytest.mark.parametrize("definition_version", [1, "1.1"]) def test_migrate_nativeapp_fields_with_username( runner, project_directory, definition_version @@ -236,7 +96,7 @@ def test_migrate_nativeapp_fields_with_username( yaml.safe_dump(old_definition, f) f.truncate() - result = runner.invoke(["ws", "migrate", "--accept-templates"]) + result = runner.invoke(["helpers", "v1-to-v2", "--accept-templates"]) assert result.exit_code == 0, result.output with definition_path.open("r") as f: diff --git a/tests_e2e/__snapshots__/test_installation.ambr b/tests_e2e/__snapshots__/test_installation.ambr index a99e2982cb..98a5128707 100644 --- a/tests_e2e/__snapshots__/test_installation.ambr +++ b/tests_e2e/__snapshots__/test_installation.ambr @@ -35,6 +35,7 @@ | connection Manages connections to Snowflake. | | cortex Provides access to Snowflake Cortex. | | git Manages git repositories in Snowflake. | + | helpers Helper commands. | | init Creates project directory from template. | | notebook Manages notebooks in Snowflake. | | object Manages Snowflake objects like warehouses and stages | @@ -44,7 +45,6 @@ | sql Executes Snowflake query. | | stage Manages stages. | | streamlit Manages a Streamlit app in Snowflake. | - | ws Deploy and interact with snowflake.yml-based entities. | +------------------------------------------------------------------------------+ From 1cb58d550fee33fb4c9e53b502364fa5f470373d Mon Sep 17 00:00:00 2001 From: Tomasz Urbaszek Date: Fri, 20 Sep 2024 14:18:16 +0200 Subject: [PATCH 05/10] Bring back glob support in streamlit (#1605) * Bring back glob support in streamlit * fixup! Bring back glob support in streamlit --- .../streamlit/streamlit_entity_model.py | 13 ++---------- .../__snapshots__/test_commands.ambr | 15 ------------- tests/streamlit/test_commands.py | 17 --------------- tests_integration/test_streamlit.py | 21 +++++++++++++++++++ 4 files changed, 23 insertions(+), 43 deletions(-) diff --git a/src/snowflake/cli/_plugins/streamlit/streamlit_entity_model.py b/src/snowflake/cli/_plugins/streamlit/streamlit_entity_model.py index 98fe54c69f..55068adb5a 100644 --- a/src/snowflake/cli/_plugins/streamlit/streamlit_entity_model.py +++ b/src/snowflake/cli/_plugins/streamlit/streamlit_entity_model.py @@ -50,23 +50,14 @@ class StreamlitEntityModel(EntityModelBase, ExternalAccessBaseModel, ImportsBase default=None, ) - @model_validator(mode="after") - def main_file_must_be_in_artifacts(self): - if not self.artifacts: - return self - - if Path(self.main_file) not in self.artifacts: - raise ValueError( - f"Specified main file {self.main_file} is not included in artifacts." - ) - return self - @model_validator(mode="after") def artifacts_must_exists(self): if not self.artifacts: return self for artifact in self.artifacts: + if "*" in artifact.name: + continue if not artifact.exists(): raise ValueError( f"Specified artifact {artifact} does not exist locally." diff --git a/tests/streamlit/__snapshots__/test_commands.ambr b/tests/streamlit/__snapshots__/test_commands.ambr index 2e525c5430..a849210505 100644 --- a/tests/streamlit/__snapshots__/test_commands.ambr +++ b/tests/streamlit/__snapshots__/test_commands.ambr @@ -14,21 +14,6 @@ ''' # --- -# name: test_main_file_must_be_in_artifacts - ''' - +- Error ----------------------------------------------------------------------+ - | During evaluation of DefinitionV20 in project definition following errors | - | were encountered: | - | For field entities.my_streamlit.streamlit you provided '{'artifacts': | - | ['streamlit_app.py', 'utils/utils.py', 'pages/', 'environment.yml'], | - | 'identifier': 'test_streamlit_deploy_snowcli', 'main_file': 'foo_bar.py', | - | 'query_warehouse': 'xsmall', 'stage': 'streamlit', 'title': 'My Fancy | - | Streamlit', 'type': 'streamlit'}'. This caused: Value error, Specified main | - | file foo_bar.py is not included in artifacts. | - +------------------------------------------------------------------------------+ - - ''' -# --- # name: test_multiple_streamlit_raise_error_if_multiple_entities ''' Usage: default streamlit deploy [OPTIONS] [ENTITY_ID] diff --git a/tests/streamlit/test_commands.py b/tests/streamlit/test_commands.py index 2d3ab5db2f..32251dd73a 100644 --- a/tests/streamlit/test_commands.py +++ b/tests/streamlit/test_commands.py @@ -268,23 +268,6 @@ def test_deploy_only_streamlit_file_replace( mock_typer.launch.assert_not_called() -def test_main_file_must_be_in_artifacts( - runner, mock_ctx, project_directory, alter_snowflake_yml, snapshot -): - with project_directory("example_streamlit_v2") as pdir: - alter_snowflake_yml( - pdir / "snowflake.yml", - parameter_path="entities.my_streamlit.main_file", - value="foo_bar.py", - ) - - result = runner.invoke( - ["streamlit", "deploy"], - ) - assert result.exit_code == 1 - assert result.output == snapshot - - def test_artifacts_must_exists( runner, mock_ctx, project_directory, alter_snowflake_yml, snapshot ): diff --git a/tests_integration/test_streamlit.py b/tests_integration/test_streamlit.py index 64943a3acf..0da14a626d 100644 --- a/tests_integration/test_streamlit.py +++ b/tests_integration/test_streamlit.py @@ -135,6 +135,27 @@ def test_streamlit_deploy_with_imports( assert result.json[0]["import_urls"] == '["@stage/foo.py","@stage/bar.py"]' +@pytest.mark.integration +@pytest.mark.parametrize("pattern", ["*.py", "*"]) +def test_streamlit_deploy_with_glob_patterns( + pattern, + runner, + snowflake_session, + test_database, + _new_streamlit_role, + project_directory, + alter_snowflake_yml, +): + with project_directory(f"streamlit_v2"): + alter_snowflake_yml( + "snowflake.yml", "entities.my_streamlit.artifacts", [pattern] + ) + result = runner.invoke_with_connection_json( + ["streamlit", "deploy", "--replace"] + ) + assert result.exit_code == 0 + + @pytest.mark.integration @pytest.mark.skip( reason="only works in accounts with experimental checkout behavior enabled" From ce6a9a49d4517956ae61c981189f55873e5f5649 Mon Sep 17 00:00:00 2001 From: Parya Jafari Date: Fri, 20 Sep 2024 10:37:01 -0400 Subject: [PATCH 06/10] Catch error on deploy for package with special comment (#1600) --- RELEASE-NOTES.md | 1 + .../_plugins/nativeapp/application_entity.py | 2 + .../nativeapp/application_package_entity.py | 43 ++++- .../cli/_plugins/nativeapp/commands.py | 14 +- .../cli/_plugins/nativeapp/manager.py | 4 + .../cli/_plugins/nativeapp/policy.py | 3 + .../cli/_plugins/nativeapp/run_processor.py | 6 +- .../cli/_plugins/workspace/commands.py | 4 + tests/__snapshots__/test_help_messages.ambr | 105 +++++++----- tests/nativeapp/test_manager.py | 37 +++++ .../test_application_package_entity.py | 8 +- tests_integration/nativeapp/test_deploy.py | 152 ++++++++++++++++++ 12 files changed, 327 insertions(+), 52 deletions(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index afb992325f..c0f59103cc 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -58,6 +58,7 @@ * `snow streamlit deploy` will check for existing streamlit instance before deploying anything. * Fixed `snow git execute` with `/` in name of the branch. * `snow app` commands don't enforce ownership of the objects they manage, and rely on RBAC instead. +* `snow app deploy` for package entity now allows operating on application packages created outside the CLI # v2.8.1 ## Backward incompatibility diff --git a/src/snowflake/cli/_plugins/nativeapp/application_entity.py b/src/snowflake/cli/_plugins/nativeapp/application_entity.py index 5864dfb585..14c9793e92 100644 --- a/src/snowflake/cli/_plugins/nativeapp/application_entity.py +++ b/src/snowflake/cli/_plugins/nativeapp/application_entity.py @@ -146,6 +146,8 @@ def deploy_package(): paths=[], validate=validate, stage_fqn=stage_fqn, + interactive=interactive, + force=force, ) self.deploy( diff --git a/src/snowflake/cli/_plugins/nativeapp/application_package_entity.py b/src/snowflake/cli/_plugins/nativeapp/application_package_entity.py index a5fb58ee62..5476c5d911 100644 --- a/src/snowflake/cli/_plugins/nativeapp/application_package_entity.py +++ b/src/snowflake/cli/_plugins/nativeapp/application_package_entity.py @@ -27,12 +27,19 @@ CouldNotDropApplicationPackageWithVersions, SetupScriptFailedValidation, ) +from snowflake.cli._plugins.nativeapp.policy import ( + AllowAlwaysPolicy, + AskAlwaysPolicy, + DenyAlwaysPolicy, + PolicyBase, +) from snowflake.cli._plugins.nativeapp.utils import ( needs_confirmation, ) from snowflake.cli._plugins.stage.diff import DiffResult from snowflake.cli._plugins.stage.manager import StageManager from snowflake.cli._plugins.workspace.action_context import ActionContext +from snowflake.cli.api.console import cli_console as cc from snowflake.cli.api.console.abc import AbstractConsole from snowflake.cli.api.entities.common import EntityBase, get_sql_executor from snowflake.cli.api.entities.utils import ( @@ -80,12 +87,22 @@ def action_deploy( recursive: bool, paths: List[Path], validate: bool, + interactive: bool, + force: bool, stage_fqn: Optional[str] = None, *args, **kwargs, ): model = self._entity_model package_name = model.fqn.identifier + + if force: + policy = AllowAlwaysPolicy() + elif interactive: + policy = AskAlwaysPolicy() + else: + policy = DenyAlwaysPolicy() + return self.deploy( console=ctx.console, project_root=ctx.project_root, @@ -107,6 +124,7 @@ def action_deploy( ), post_deploy_hooks=model.meta and model.meta.post_deploy, package_scripts=[], # Package scripts are not supported in PDFv2 + policy=policy, ) def action_drop(self, ctx: ActionContext, force_drop: bool, *args, **kwargs): @@ -124,7 +142,9 @@ def action_drop(self, ctx: ActionContext, force_drop: bool, *args, **kwargs): force_drop=force_drop, ) - def action_validate(self, ctx: ActionContext, *args, **kwargs): + def action_validate( + self, ctx: ActionContext, interactive: bool, force: bool, *args, **kwargs + ): model = self._entity_model package_name = model.fqn.identifier stage_fqn = f"{package_name}.{model.stage}" @@ -141,6 +161,8 @@ def deploy_to_scratch_stage_fn(): paths=[], validate=False, stage_fqn=model.scratch_stage, + interactive=interactive, + force=force, ) self.validate_setup_script( @@ -206,6 +228,7 @@ def deploy( stage_fqn: str, post_deploy_hooks: list[PostDeployHook] | None, package_scripts: List[str], + policy: PolicyBase, ) -> DiffResult: # 1. Create a bundle bundle_map = cls.bundle( @@ -218,13 +241,17 @@ def deploy( ) # 2. Create an empty application package, if none exists - cls.create_app_package( - console=console, - package_name=package_name, - package_role=package_role, - package_distribution=package_distribution, - ) - + try: + cls.create_app_package( + console=console, + package_name=package_name, + package_role=package_role, + package_distribution=package_distribution, + ) + except ApplicationPackageAlreadyExistsError as e: + cc.warning(e.message) + if not policy.should_proceed("Proceed with using this package?"): + raise typer.Abort() from e with get_sql_executor().use_role(package_role): if package_scripts: cls.apply_package_scripts( diff --git a/src/snowflake/cli/_plugins/nativeapp/commands.py b/src/snowflake/cli/_plugins/nativeapp/commands.py index ba7a888678..f08d0bd14b 100644 --- a/src/snowflake/cli/_plugins/nativeapp/commands.py +++ b/src/snowflake/cli/_plugins/nativeapp/commands.py @@ -406,6 +406,8 @@ def app_deploy( unspecified, the command syncs all local changes to the stage.""" ).strip(), ), + interactive: bool = InteractiveOption, + force: Optional[bool] = ForceOption, validate: bool = ValidateOption, **options, ) -> CommandResult: @@ -416,6 +418,13 @@ def app_deploy( assert_project_type("native_app") + if force: + policy = AllowAlwaysPolicy() + elif interactive: + policy = AskAlwaysPolicy() + else: + policy = DenyAlwaysPolicy() + has_paths = paths is not None and len(paths) > 0 if prune is None and recursive is None and not has_paths: prune = True @@ -442,6 +451,7 @@ def app_deploy( recursive=recursive, local_paths_to_sync=paths, validate=validate, + policy=policy, ) return MessageResult( @@ -452,7 +462,9 @@ def app_deploy( @app.command("validate", requires_connection=True) @with_project_definition() @nativeapp_definition_v2_to_v1() -def app_validate(**options): +def app_validate( + **options, +): """ Validates a deployed Snowflake Native App's setup script. """ diff --git a/src/snowflake/cli/_plugins/nativeapp/manager.py b/src/snowflake/cli/_plugins/nativeapp/manager.py index 0946350b0f..eb5bde56f2 100644 --- a/src/snowflake/cli/_plugins/nativeapp/manager.py +++ b/src/snowflake/cli/_plugins/nativeapp/manager.py @@ -36,6 +36,7 @@ from snowflake.cli._plugins.nativeapp.exceptions import ( NoEventTableForAccount, ) +from snowflake.cli._plugins.nativeapp.policy import AllowAlwaysPolicy, PolicyBase from snowflake.cli._plugins.nativeapp.project_model import ( NativeAppProjectModel, ) @@ -306,6 +307,7 @@ def deploy( bundle_map: BundleMap, prune: bool, recursive: bool, + policy: PolicyBase, stage_fqn: Optional[str] = None, local_paths_to_sync: List[Path] | None = None, validate: bool = True, @@ -330,6 +332,7 @@ def deploy( package_warehouse=self.package_warehouse, post_deploy_hooks=self.package_post_deploy_hooks, package_scripts=self.package_scripts, + policy=policy, ) def deploy_to_scratch_stage_fn(self): @@ -341,6 +344,7 @@ def deploy_to_scratch_stage_fn(self): stage_fqn=self.scratch_stage_fqn, validate=False, print_diff=False, + policy=AllowAlwaysPolicy(), ) def validate(self, use_scratch_stage: bool = False): diff --git a/src/snowflake/cli/_plugins/nativeapp/policy.py b/src/snowflake/cli/_plugins/nativeapp/policy.py index 5dc5a38696..71fa91584f 100644 --- a/src/snowflake/cli/_plugins/nativeapp/policy.py +++ b/src/snowflake/cli/_plugins/nativeapp/policy.py @@ -27,6 +27,9 @@ class PolicyBase(ABC): def should_proceed(self, user_prompt: Optional[str]) -> bool: pass + def __eq__(self, value: object) -> bool: + return self.__class__ == value.__class__ + class AllowAlwaysPolicy(PolicyBase): """Always allow a Snowflake CLI command to continue execution.""" diff --git a/src/snowflake/cli/_plugins/nativeapp/run_processor.py b/src/snowflake/cli/_plugins/nativeapp/run_processor.py index dd9b2eea4c..7f93ee6767 100644 --- a/src/snowflake/cli/_plugins/nativeapp/run_processor.py +++ b/src/snowflake/cli/_plugins/nativeapp/run_processor.py @@ -151,7 +151,11 @@ def process( ): def deploy_package(): self.deploy( - bundle_map=bundle_map, prune=True, recursive=True, validate=validate + bundle_map=bundle_map, + prune=True, + recursive=True, + validate=validate, + policy=policy, ) def drop_app(): diff --git a/src/snowflake/cli/_plugins/workspace/commands.py b/src/snowflake/cli/_plugins/workspace/commands.py index 99c602e377..0e45c3423b 100644 --- a/src/snowflake/cli/_plugins/workspace/commands.py +++ b/src/snowflake/cli/_plugins/workspace/commands.py @@ -179,6 +179,8 @@ def validate( entity_id: str = typer.Option( help=f"""The ID of the entity you want to validate.""", ), + interactive: bool = InteractiveOption, + force: Optional[bool] = ForceOption, **options, ): """Validates the specified entity.""" @@ -191,6 +193,8 @@ def validate( ws.perform_action( entity_id, EntityActions.VALIDATE, + interactive=interactive, + force=force, ) diff --git a/tests/__snapshots__/test_help_messages.ambr b/tests/__snapshots__/test_help_messages.ambr index fa32bdcdc1..2234377af6 100644 --- a/tests/__snapshots__/test_help_messages.ambr +++ b/tests/__snapshots__/test_help_messages.ambr @@ -96,47 +96,70 @@ | syncs all local changes to the stage. | +------------------------------------------------------------------------------+ +- Options --------------------------------------------------------------------+ - | --prune --no-prune Whether to delete specified | - | files from the stage if | - | they don't exist locally. | - | If set, the command deletes | - | files that exist in the | - | stage, but not in the local | - | filesystem. This option | - | cannot be used when paths | - | are specified. | - | [default: no-prune] | - | --recursive -r --no-recursive Whether to traverse and | - | deploy files from | - | subdirectories. If set, the | - | command deploys all files | - | and subdirectories; | - | otherwise, only files in | - | the current directory are | - | deployed. | - | [default: no-recursive] | - | --validate --no-validate When enabled, this option | - | triggers validation of a | - | deployed Snowflake Native | - | App's setup script SQL | - | [default: validate] | - | --package-entity-id TEXT The ID of the package | - | entity on which to operate | - | when definition_version is | - | 2 or higher. | - | --app-entity-id TEXT The ID of the application | - | entity on which to operate | - | when definition_version is | - | 2 or higher. | - | --project -p TEXT Path where Snowflake | - | project resides. Defaults | - | to current working | - | directory. | - | --env TEXT String in format of | - | key=value. Overrides | - | variables from env section | - | used for templates. | - | --help -h Show this message and exit. | + | --prune --no-prune Whether to delete | + | specified files from the | + | stage if they don't exist | + | locally. If set, the | + | command deletes files | + | that exist in the stage, | + | but not in the local | + | filesystem. This option | + | cannot be used when paths | + | are specified. | + | [default: no-prune] | + | --recursive -r --no-recursive Whether to traverse and | + | deploy files from | + | subdirectories. If set, | + | the command deploys all | + | files and subdirectories; | + | otherwise, only files in | + | the current directory are | + | deployed. | + | [default: no-recursive] | + | --interactive --no-interactive When enabled, this option | + | displays prompts even if | + | the standard input and | + | output are not terminal | + | devices. Defaults to True | + | in an interactive shell | + | environment, and False | + | otherwise. | + | --force When enabled, this option | + | causes the command to | + | implicitly approve any | + | prompts that arise. You | + | should enable this option | + | if interactive mode is | + | not specified and if you | + | want perform potentially | + | destructive actions. | + | Defaults to unset. | + | --validate --no-validate When enabled, this option | + | triggers validation of a | + | deployed Snowflake Native | + | App's setup script SQL | + | [default: validate] | + | --package-entity-id TEXT The ID of the package | + | entity on which to | + | operate when | + | definition_version is 2 | + | or higher. | + | --app-entity-id TEXT The ID of the application | + | entity on which to | + | operate when | + | definition_version is 2 | + | or higher. | + | --project -p TEXT Path where Snowflake | + | project resides. Defaults | + | to current working | + | directory. | + | --env TEXT String in format of | + | key=value. Overrides | + | variables from env | + | section used for | + | templates. | + | --help -h Show this message and | + | exit. | +------------------------------------------------------------------------------+ +- Connection configuration ---------------------------------------------------+ | --connection,--environment -c TEXT Name of the connection, as | diff --git a/tests/nativeapp/test_manager.py b/tests/nativeapp/test_manager.py index c423b93b90..52c15119b2 100644 --- a/tests/nativeapp/test_manager.py +++ b/tests/nativeapp/test_manager.py @@ -40,6 +40,7 @@ from snowflake.cli._plugins.nativeapp.manager import ( NativeAppManager, ) +from snowflake.cli._plugins.nativeapp.policy import AllowAlwaysPolicy from snowflake.cli._plugins.stage.diff import ( DiffResult, StagePath, @@ -927,6 +928,40 @@ def test_create_app_pkg_internal_distribution_no_special_comment( ) +# Test create_app_package() with existing package without special comment +@mock.patch(APP_PACKAGE_ENTITY_IS_DISTRIBUTION_SAME) +@mock_get_app_pkg_distribution_in_sf() +@mock.patch(APP_PACKAGE_ENTITY_GET_EXISTING_APP_PKG_INFO) +@mock.patch(SQL_EXECUTOR_EXECUTE) +def test_existing_app_pkg_without_special_comment( + mock_execute, + mock_get_existing_app_pkg_info, + mock_get_distribution, + mock_is_distribution_same, + temp_dir, + mock_cursor, +): + mock_get_existing_app_pkg_info.return_value = { + "name": "APP_PKG", + "comment": "NOT_SPECIAL_COMMENT", + "version": LOOSE_FILES_MAGIC_VERSION, + "owner": "package_role", + } + mock_get_distribution.return_value = "internal" + mock_is_distribution_same.return_value = True + + current_working_directory = os.getcwd() + create_named_file( + file_name="snowflake.yml", + dir_name=current_working_directory, + contents=[mock_snowflake_yml_file], + ) + + native_app_manager = _get_na_manager() + with pytest.raises(ApplicationPackageAlreadyExistsError): + native_app_manager.create_app_package() + + @pytest.mark.parametrize( "paths_to_sync,expected_result", [ @@ -1187,6 +1222,7 @@ def test_validate_use_scratch_stage( stage_fqn=native_app_manager.scratch_stage_fqn, validate=False, print_diff=False, + policy=AllowAlwaysPolicy(), ) assert mock_execute.mock_calls == expected @@ -1253,6 +1289,7 @@ def test_validate_failing_drops_scratch_stage( stage_fqn=native_app_manager.scratch_stage_fqn, validate=False, print_diff=False, + policy=AllowAlwaysPolicy(), ) assert mock_execute.mock_calls == expected diff --git a/tests/workspace/test_application_package_entity.py b/tests/workspace/test_application_package_entity.py index 54980190bb..34aff9d890 100644 --- a/tests/workspace/test_application_package_entity.py +++ b/tests/workspace/test_application_package_entity.py @@ -135,7 +135,13 @@ def test_deploy( app_pkg, bundle_ctx, mock_console = _get_app_pkg_entity(project_directory) app_pkg.action_deploy( - bundle_ctx, prune=False, recursive=False, paths=["a/b", "c"], validate=True + bundle_ctx, + prune=False, + recursive=False, + paths=["a/b", "c"], + validate=True, + interactive=False, + force=False, ) mock_sync.assert_called_once_with( diff --git a/tests_integration/nativeapp/test_deploy.py b/tests_integration/nativeapp/test_deploy.py index 3f4925964b..8312fcd3cc 100644 --- a/tests_integration/nativeapp/test_deploy.py +++ b/tests_integration/nativeapp/test_deploy.py @@ -577,3 +577,155 @@ def test_nativeapp_deploy_validate_failing( assert result.exit_code == 1, result.output assert "Snowflake Native App setup script failed validation." in result.output assert "syntax error" in result.output + + +@pytest.mark.integration +@pytest.mark.parametrize( + "test_project", + [ + "napp_init_v1", + "napp_init_v2", + ], +) +def test_nativeapp_deploy_package_no_magic_comment( + runner, + snowflake_session, + default_username, + resource_suffix, + nativeapp_teardown, + snapshot, + nativeapp_project_directory, + test_project, +): + project_name = "myapp" + with nativeapp_project_directory(test_project): + result_create_abort = runner.invoke_with_connection_json(["app", "deploy"]) + assert result_create_abort.exit_code == 0 + + # package exists + package_name = f"{project_name}_pkg_{default_username}{resource_suffix}".upper() + assert contains_row_with( + row_from_snowflake_session( + snowflake_session.execute_string( + f"show application packages like '{package_name}'", + ) + ), + dict(name=package_name), + ) + + assert contains_row_with( + row_from_snowflake_session( + snowflake_session.execute_string( + f"alter application package {package_name} set comment = 'not the magic comment'" + ) + ), + dict(status="Statement executed successfully."), + ) + + # app command - say no + result_create_abort = runner.invoke_with_connection( + ["app", "deploy", "--interactive"], + input="n\n", + ) + assert result_create_abort.exit_code == 1 + assert ( + f"An Application Package {package_name} already exists in account " + "that may have been created without Snowflake CLI.".upper() + in result_create_abort.output.upper() + ) + assert "Aborted." in result_create_abort.output + + # app command - say yes + result_create_yes = runner.invoke_with_connection( + ["app", "deploy", "--interactive"], + input="y\n", + ) + assert result_create_yes.exit_code == 0 + assert ( + f"An Application Package {package_name} already exists in account " + "that may have been created without Snowflake CLI.".upper() + in result_create_yes.output.upper() + ) + + # app command - force + result_create_force = runner.invoke_with_connection( + ["app", "deploy", "--force"] + ) + assert result_create_force.exit_code == 0 + assert ( + f"An Application Package {package_name} already exists in account " + "that may have been created without Snowflake CLI.".upper() + in result_create_force.output.upper() + ) + + +@pytest.mark.integration +def test_ws_deploy_package_no_magic_comment( + runner, + snowflake_session, + default_username, + resource_suffix, + nativeapp_teardown, + snapshot, + nativeapp_project_directory, +): + project_name = "myapp" + with nativeapp_project_directory("napp_init_v2"): + result_create_abort = runner.invoke_with_connection_json(["app", "deploy"]) + assert result_create_abort.exit_code == 0 + + # package exists + package_name = f"{project_name}_pkg_{default_username}{resource_suffix}".upper() + assert contains_row_with( + row_from_snowflake_session( + snowflake_session.execute_string( + f"show application packages like '{package_name}'", + ) + ), + dict(name=package_name), + ) + + assert contains_row_with( + row_from_snowflake_session( + snowflake_session.execute_string( + f"alter application package {package_name} set comment = 'not the magic comment'" + ) + ), + dict(status="Statement executed successfully."), + ) + + # ws command - say no + result_create_abort = runner.invoke_with_connection( + ["ws", "deploy", "--entity-id=pkg", "--interactive"], + input="n\n", + ) + assert result_create_abort.exit_code == 1 + assert ( + f"An Application Package {package_name} already exists in account " + "that may have been created without Snowflake CLI.".upper() + in result_create_abort.output.upper() + ) + assert "Aborted." in result_create_abort.output + + # ws command - say yes + result_create_yes = runner.invoke_with_connection( + ["ws", "deploy", "--entity-id=pkg", "--interactive"], + input="y\n", + ) + assert result_create_yes.exit_code == 0 + assert ( + f"An Application Package {package_name} already exists in account " + "that may have been created without Snowflake CLI.".upper() + in result_create_yes.output.upper() + ) + + # ws command - force + result_create_force = runner.invoke_with_connection( + ["ws", "deploy", "--entity-id=pkg", "--force"] + ) + assert result_create_force.exit_code == 0 + assert ( + f"An Application Package {package_name} already exists in account " + "that may have been created without Snowflake CLI.".upper() + in result_create_force.output.upper() + ) From 657e3fc99653dea134637115e1d6cdad50a23d49 Mon Sep 17 00:00:00 2001 From: Vida Maleki Date: Fri, 20 Sep 2024 09:15:30 -0700 Subject: [PATCH 07/10] Added integration test for execute streamlit command --- tests_integration/test_streamlit.py | 56 +++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tests_integration/test_streamlit.py b/tests_integration/test_streamlit.py index 0da14a626d..41def53deb 100644 --- a/tests_integration/test_streamlit.py +++ b/tests_integration/test_streamlit.py @@ -23,6 +23,7 @@ rows_from_snowflake_session, ) from tests_integration.testing_utils import assert_that_result_is_successful +from snowflake.cli._plugins.streamlit.manager import StreamlitManager @pytest.mark.integration @@ -367,3 +368,58 @@ def _new_streamlit_role(snowflake_session, test_database): row_from_snowflake_session(result), {"status": f"{role_name.upper()} successfully dropped."}, ) + + +def _execute_streamlit(runner, streamlit_name): + result = runner.invoke_with_connection_json( + ["streamlit", "execute", streamlit_name, "--format", "json"] + ) + assert result.exit_code == 0 + assert result.json == {"message": f"Streamlit {streamlit_name} executed."} + + +def _execute_streamlit_failure(runner, streamlit_name): + result = runner.invoke_with_connection(["streamlit", "execute", streamlit_name]) + assert result.exit_code == 1 + assert f"NameError: name {streamlit_name} is not defined" in result.output + + +@pytest.mark.integration +def test_execute_streamlit(runner, snowflake_session, test_database): + stage_name = "streamlit_stage" + snowflake_session.execute_string(f"create stage {stage_name};") + + # Create a Streamlit app and upload it to the stage + streamlit_app = Path("test_streamlit_app.py") + + _create_streamlit( + streamlit=streamlit_app, + runner=runner, + snowflake_session=snowflake_session, + stage_name=stage_name, + ) + + # Test successful and failure executions + _execute_streamlit(runner, streamlit_app.stem) + _execute_streamlit_failure(runner, "nonexistent_app") + + +def _create_streamlit(local_streamlit_file, runner, snowflake_session, stage_name): + streamlit_name = local_streamlit_file.stem + stage_path = f"@{stage_name}/{local_streamlit_file.name}" + snowflake_session.execute_string( + f"put file://{local_streamlit_file.absolute()} @{stage_name} AUTO_COMPRESS=FALSE;" + ) + command = ( + "streamlit", + "create", + streamlit_name, + "--streamlit-file", + stage_path, + "--format", + "json", + ) + result = runner.invoke_with_connection_json(command) + assert result.exit_code == 0 + message: str = result.json.get("message", "") + assert message.endswith(streamlit_name.upper()) From f6484dc72f48e8aa047dcbae3b40bd6ce8606243 Mon Sep 17 00:00:00 2001 From: Vida Maleki Date: Fri, 20 Sep 2024 10:25:32 -0700 Subject: [PATCH 08/10] Fixed integration test about unexpected keyword argument --- tests_integration/test_streamlit.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests_integration/test_streamlit.py b/tests_integration/test_streamlit.py index 41def53deb..0a1480e4e5 100644 --- a/tests_integration/test_streamlit.py +++ b/tests_integration/test_streamlit.py @@ -390,10 +390,10 @@ def test_execute_streamlit(runner, snowflake_session, test_database): snowflake_session.execute_string(f"create stage {stage_name};") # Create a Streamlit app and upload it to the stage - streamlit_app = Path("test_streamlit_app.py") + streamlit_app = Path("test_data/streamlit_v2/streamlit_app.py") _create_streamlit( - streamlit=streamlit_app, + local_streamlit_file=streamlit_app, runner=runner, snowflake_session=snowflake_session, stage_name=stage_name, From b0f2cbe66f24b65b236a41e0852870eaf190b8d8 Mon Sep 17 00:00:00 2001 From: Vida Maleki Date: Fri, 20 Sep 2024 12:26:31 -0700 Subject: [PATCH 09/10] Fixed path for integration test environment --- tests_integration/test_streamlit.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests_integration/test_streamlit.py b/tests_integration/test_streamlit.py index 0a1480e4e5..246b38bb77 100644 --- a/tests_integration/test_streamlit.py +++ b/tests_integration/test_streamlit.py @@ -390,7 +390,9 @@ def test_execute_streamlit(runner, snowflake_session, test_database): snowflake_session.execute_string(f"create stage {stage_name};") # Create a Streamlit app and upload it to the stage - streamlit_app = Path("test_data/streamlit_v2/streamlit_app.py") + streamlit_app = Path( + "tests_integration/test_data/projects/streamlit_v2/streamlit_app.py" + ) _create_streamlit( local_streamlit_file=streamlit_app, From 6dabd1c0bf58c7fa7dab09085a1b5d1048397cac Mon Sep 17 00:00:00 2001 From: Vida Maleki Date: Fri, 20 Sep 2024 14:34:55 -0700 Subject: [PATCH 10/10] Created new test enviroments for execute command --- .../execute_streamlit/environment.yml | 6 ++ .../projects/execute_streamlit/snowflake.yml | 10 ++++ .../execute_streamlit/streamlit_app.py | 0 tests_integration/test_streamlit.py | 58 ++++++++++++++----- 4 files changed, 59 insertions(+), 15 deletions(-) create mode 100644 tests_integration/test_data/projects/execute_streamlit/environment.yml create mode 100644 tests_integration/test_data/projects/execute_streamlit/snowflake.yml create mode 100644 tests_integration/test_data/projects/execute_streamlit/streamlit_app.py diff --git a/tests_integration/test_data/projects/execute_streamlit/environment.yml b/tests_integration/test_data/projects/execute_streamlit/environment.yml new file mode 100644 index 0000000000..fafbe1709c --- /dev/null +++ b/tests_integration/test_data/projects/execute_streamlit/environment.yml @@ -0,0 +1,6 @@ +name: sf_env +channels: + - snowflake +dependencies: + - streamlit + - snowflake-snowpark-python diff --git a/tests_integration/test_data/projects/execute_streamlit/snowflake.yml b/tests_integration/test_data/projects/execute_streamlit/snowflake.yml new file mode 100644 index 0000000000..a2605cffc0 --- /dev/null +++ b/tests_integration/test_data/projects/execute_streamlit/snowflake.yml @@ -0,0 +1,10 @@ +definition_version: "1.1" +streamlit: + name: "vmaleki_cli_app" + title: "Vida Uploaded File" + stage: "ST_DB.SIS.MY_STREAMLIT_STAGE" + database: "ST_DB" + schema: "SIS" + query_warehouse: "regress" + main_file: "streamlit_app.py" + env_file: "environment.yml" diff --git a/tests_integration/test_data/projects/execute_streamlit/streamlit_app.py b/tests_integration/test_data/projects/execute_streamlit/streamlit_app.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests_integration/test_streamlit.py b/tests_integration/test_streamlit.py index 246b38bb77..d616961292 100644 --- a/tests_integration/test_streamlit.py +++ b/tests_integration/test_streamlit.py @@ -370,57 +370,85 @@ def _new_streamlit_role(snowflake_session, test_database): ) -def _execute_streamlit(runner, streamlit_name): +def _execute_streamlit(runner, streamlit_name, database, schema): result = runner.invoke_with_connection_json( - ["streamlit", "execute", streamlit_name, "--format", "json"] + [ + "streamlit", + "execute", + streamlit_name, + "--database", + database, + "--schema", + schema, + "--format", + "json", + ] ) assert result.exit_code == 0 assert result.json == {"message": f"Streamlit {streamlit_name} executed."} -def _execute_streamlit_failure(runner, streamlit_name): - result = runner.invoke_with_connection(["streamlit", "execute", streamlit_name]) +def _execute_streamlit_failure(runner, streamlit_name, database, schema): + result = runner.invoke_with_connection( + [ + "streamlit", + "execute", + streamlit_name, + "--database", + database, + "--schema", + schema, + ] + ) assert result.exit_code == 1 assert f"NameError: name {streamlit_name} is not defined" in result.output @pytest.mark.integration def test_execute_streamlit(runner, snowflake_session, test_database): - stage_name = "streamlit_stage" + stage_name = "ST_DB.SIS.MY_STREAMLIT_STAGE" + database = "ST_DB" + schema = "SIS" # Update schema to match your snowflake.yml + snowflake_session.execute_string(f"create stage {stage_name};") - # Create a Streamlit app and upload it to the stage + # Create and deploy a Streamlit app streamlit_app = Path( - "tests_integration/test_data/projects/streamlit_v2/streamlit_app.py" + "tests_integration/test_data/projects/execute_streamlit/streamlit_app.py" ) - _create_streamlit( + _deploy_streamlit( local_streamlit_file=streamlit_app, runner=runner, snowflake_session=snowflake_session, stage_name=stage_name, + replace=True, ) # Test successful and failure executions - _execute_streamlit(runner, streamlit_app.stem) - _execute_streamlit_failure(runner, "nonexistent_app") + _execute_streamlit(runner, streamlit_app.stem, database, schema) + _execute_streamlit_failure(runner, "nonexistent_app", database, schema) -def _create_streamlit(local_streamlit_file, runner, snowflake_session, stage_name): +def _deploy_streamlit( + local_streamlit_file, runner, snowflake_session, stage_name, replace=True +): streamlit_name = local_streamlit_file.stem stage_path = f"@{stage_name}/{local_streamlit_file.name}" snowflake_session.execute_string( f"put file://{local_streamlit_file.absolute()} @{stage_name} AUTO_COMPRESS=FALSE;" ) - command = ( + command = [ "streamlit", - "create", - streamlit_name, + "deploy", "--streamlit-file", stage_path, "--format", "json", - ) + ] + if replace: + command.append("--replace") + result = runner.invoke_with_connection_json(command) assert result.exit_code == 0 message: str = result.json.get("message", "")