Merge branch 'main' into mchok-metrics-counter-feature-tracking

RELEASE-NOTES.md

@@ -39,7 +39,7 @@
 * Added support for external access (api integrations and secrets) in Streamlit.
 * Added support for `<% ... %>` syntax in SQL templating.
 * Support multiple Streamlit application in single snowflake.yml project definition file.
-* Added `snow ws migrate` command to migrate `snowflake.yml` file from V1 to V2.
+* Added `snow helpers v1-to-v2` command to migrate `snowflake.yml` file from V1 to V2.
 * Added `--package-entity-id` and `--app-entity-id` options to `snow app` commands to allow targeting specific entities when the `definition_version` in `snowflake.yml` is `2` or higher and it contains multiple `application package` or `application` entities.
 * Added templates expansion of arbitrary files for Native Apps through `templates` processor.
 * Added `SNOWFLAKE_..._PRIVATE_KEY_RAW` environment variable to pass private key as a raw string.

src/snowflake/cli/_app/commands_registration/builtin_plugins.py

@@ -15,6 +15,7 @@
 from snowflake.cli._plugins.connection import plugin_spec as connection_plugin_spec
 from snowflake.cli._plugins.cortex import plugin_spec as cortex_plugin_spec
 from snowflake.cli._plugins.git import plugin_spec as git_plugin_spec
+from snowflake.cli._plugins.helpers import plugin_spec as migrate_plugin_spec
 from snowflake.cli._plugins.init import plugin_spec as init_plugin_spec
 from snowflake.cli._plugins.nativeapp import plugin_spec as nativeapp_plugin_spec
 from snowflake.cli._plugins.notebook import plugin_spec as notebook_plugin_spec
@@ -31,6 +32,7 @@
 def get_builtin_plugin_name_to_plugin_spec():
     plugin_specs = {
         "connection": connection_plugin_spec,
+        "helpers": migrate_plugin_spec,
         "spcs": spcs_plugin_spec,
         "nativeapp": nativeapp_plugin_spec,
         "object": object_plugin_spec,

src/snowflake/cli/_app/secret.py

@@ -0,0 +1,9 @@
+class SecretType:
+    def __init__(self, value):
+        self.value = value
+
+    def __repr__(self):
+        return "SecretType(***)"
+
+    def __str___(self):
+        return "***"

src/snowflake/cli/_app/snow_connector.py

@@ -24,6 +24,7 @@
 from snowflake.cli._app.constants import (
     PARAM_APPLICATION_NAME,
 )
+from snowflake.cli._app.secret import SecretType
 from snowflake.cli._app.telemetry import command_info
 from snowflake.cli.api.config import (
     get_connection_dict,
@@ -205,7 +206,7 @@ def _load_private_key(connection_parameters: Dict, private_key_var_name: str) ->
             connection_parameters[private_key_var_name]
         )
         private_key = _load_pem_to_der(private_key_pem)
-        connection_parameters["private_key"] = private_key
+        connection_parameters["private_key"] = private_key.value
         del connection_parameters[private_key_var_name]
     else:
         raise ClickException(
@@ -217,10 +218,11 @@ def _load_private_key_from_parameters(
     connection_parameters: Dict, private_key_var_name: str
 ) -> None:
     if connection_parameters.get("authenticator") == "SNOWFLAKE_JWT":
-        private_key_pem = connection_parameters[private_key_var_name]
-        private_key_pem = private_key_pem.encode("utf-8")
+        private_key_pem = _load_pem_from_parameters(
+            connection_parameters[private_key_var_name]
+        )
         private_key = _load_pem_to_der(private_key_pem)
-        connection_parameters["private_key"] = private_key
+        connection_parameters["private_key"] = private_key.value
         del connection_parameters[private_key_var_name]
     else:
         raise ClickException(
@@ -236,43 +238,49 @@ def _update_connection_application_name(connection_parameters: Dict):
     connection_parameters.update(connection_application_params)
 
 
-def _load_pem_from_file(private_key_file: str) -> bytes:
+def _load_pem_from_file(private_key_file: str) -> SecretType:
     with SecurePath(private_key_file).open(
         "rb", read_file_limit_mb=DEFAULT_SIZE_LIMIT_MB
     ) as f:
-        private_key_pem = f.read()
+        private_key_pem = SecretType(f.read())
     return private_key_pem
 
 
-def _load_pem_to_der(private_key_pem: bytes) -> bytes:
+def _load_pem_from_parameters(private_key_raw: str) -> SecretType:
+    return SecretType(private_key_raw.encode("utf-8"))
+
+
+def _load_pem_to_der(private_key_pem: SecretType) -> SecretType:
     """
     Given a private key file path (in PEM format), decode key data into DER
     format
     """
-    private_key_passphrase = os.getenv("PRIVATE_KEY_PASSPHRASE", None)
+    private_key_passphrase = SecretType(os.getenv("PRIVATE_KEY_PASSPHRASE", None))
     if (
-        private_key_pem.startswith(ENCRYPTED_PKCS8_PK_HEADER)
-        and private_key_passphrase is None
+        private_key_pem.value.startswith(ENCRYPTED_PKCS8_PK_HEADER)
+        and private_key_passphrase.value is None
     ):
         raise ClickException(
             "Encrypted private key, you must provide the"
             "passphrase in the environment variable PRIVATE_KEY_PASSPHRASE"
         )
 
-    if not private_key_pem.startswith(
+    if not private_key_pem.value.startswith(
         ENCRYPTED_PKCS8_PK_HEADER
-    ) and not private_key_pem.startswith(UNENCRYPTED_PKCS8_PK_HEADER):
+    ) and not private_key_pem.value.startswith(UNENCRYPTED_PKCS8_PK_HEADER):
         raise ClickException(
             "Private key provided is not in PKCS#8 format. Please use correct format."
         )
 
-    if private_key_pem.startswith(UNENCRYPTED_PKCS8_PK_HEADER):
-        private_key_passphrase = None
+    if private_key_pem.value.startswith(UNENCRYPTED_PKCS8_PK_HEADER):
+        private_key_passphrase = SecretType(None)
 
     return prepare_private_key(private_key_pem, private_key_passphrase)
 
 
-def prepare_private_key(private_key_pem, private_key_passphrase=None):
+def prepare_private_key(
+    private_key_pem: SecretType, private_key_passphrase: SecretType = SecretType(None)
+):
     from cryptography.hazmat.backends import default_backend
     from cryptography.hazmat.primitives.serialization import (
         Encoding,
@@ -281,17 +289,21 @@ def prepare_private_key(private_key_pem, private_key_passphrase=None):
         load_pem_private_key,
     )
 
-    private_key = load_pem_private_key(
-        private_key_pem,
-        (
-            str.encode(private_key_passphrase)
-            if private_key_passphrase is not None
-            else private_key_passphrase
-        ),
-        default_backend(),
+    private_key = SecretType(
+        load_pem_private_key(
+            private_key_pem.value,
+            (
+                str.encode(private_key_passphrase.value)
+                if private_key_passphrase.value is not None
+                else private_key_passphrase.value
+            ),
+            default_backend(),
+        )
     )
-    return private_key.private_bytes(
-        encoding=Encoding.DER,
-        format=PrivateFormat.PKCS8,
-        encryption_algorithm=NoEncryption(),
+    return SecretType(
+        private_key.value.private_bytes(
+            encoding=Encoding.DER,
+            format=PrivateFormat.PKCS8,
+            encryption_algorithm=NoEncryption(),
+        )
     )

src/snowflake/cli/_plugins/helpers/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

src/snowflake/cli/_plugins/helpers/commands.py

@@ -0,0 +1,57 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import typer
+import yaml
+from snowflake.cli.api.commands.snow_typer import SnowTyperFactory
+from snowflake.cli.api.output.types import MessageResult
+from snowflake.cli.api.project.definition_conversion import (
+    convert_project_definition_to_v2,
+)
+from snowflake.cli.api.project.definition_manager import DefinitionManager
+from snowflake.cli.api.secure_path import SecurePath
+
+app = SnowTyperFactory(
+    name="helpers",
+    help="Helper commands.",
+)
+
+
+@app.command()
+def v1_to_v2(
+    accept_templates: bool = typer.Option(
+        False, "-t", "--accept-templates", help="Allows the migration of templates."
+    ),
+    **options,
+):
+    """Migrates the Snowpark, Streamlit, and Native App project definition files from V1 to V2."""
+    manager = DefinitionManager()
+    pd = manager.unrendered_project_definition
+
+    if pd.meets_version_requirement("2"):
+        return MessageResult("Project definition is already at version 2.")
+
+    pd_v2 = convert_project_definition_to_v2(manager.project_root, pd, accept_templates)
+
+    SecurePath("snowflake.yml").rename("snowflake_V1.yml")
+    with open("snowflake.yml", "w") as file:
+        yaml.dump(
+            pd_v2.model_dump(
+                exclude_unset=True, exclude_none=True, mode="json", by_alias=True
+            ),
+            file,
+        )
+    return MessageResult("Project definition migrated to version 2.")

src/snowflake/cli/_plugins/helpers/plugin_spec.py

@@ -0,0 +1,30 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from snowflake.cli._plugins.helpers import commands
+from snowflake.cli.api.plugins.command import (
+    SNOWCLI_ROOT_COMMAND_PATH,
+    CommandSpec,
+    CommandType,
+    plugin_hook_impl,
+)
+
+
+@plugin_hook_impl
+def command_spec():
+    return CommandSpec(
+        parent_command_path=SNOWCLI_ROOT_COMMAND_PATH,
+        command_type=CommandType.COMMAND_GROUP,
+        typer_instance=commands.app.create_instance(),
+    )

src/snowflake/cli/_plugins/nativeapp/artifacts.py

@@ -23,6 +23,7 @@
 from click.exceptions import ClickException
 from snowflake.cli.api.constants import DEFAULT_SIZE_LIMIT_MB
 from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import PathMapping
+from snowflake.cli.api.project.util import to_identifier
 from snowflake.cli.api.secure_path import SecurePath
 from yaml import safe_load
 
@@ -734,23 +735,23 @@ def find_setup_script_file(deploy_root: Path) -> Path:
 
 def find_version_info_in_manifest_file(
     deploy_root: Path,
-) -> Tuple[Optional[str], Optional[str]]:
+) -> Tuple[Optional[str], Optional[int]]:
     """
     Find version and patch, if available, in the manifest.yml file.
     """
-    version_field = "version"
     name_field = "name"
     patch_field = "patch"
 
     manifest_content = find_and_read_manifest_file(deploy_root=deploy_root)
 
     version_name: Optional[str] = None
-    patch_name: Optional[str] = None
+    patch_number: Optional[int] = None
 
-    if version_field in manifest_content:
-        version_info = manifest_content[version_field]
-        version_name = version_info[name_field]
+    version_info = manifest_content.get("version", None)
+    if version_info:
+        if name_field in version_info:
+            version_name = to_identifier(str(version_info[name_field]))
         if patch_field in version_info:
-            patch_name = version_info[patch_field]
+            patch_number = int(version_info[patch_field])
 
-    return version_name, patch_name
+    return version_name, patch_number