diff --git a/src/snowflake/cli/_app/telemetry.py b/src/snowflake/cli/_app/telemetry.py index 326e453ee8..106ae969c4 100644 --- a/src/snowflake/cli/_app/telemetry.py +++ b/src/snowflake/cli/_app/telemetry.py @@ -54,6 +54,8 @@ class CLITelemetryField(Enum): COMMAND_EXECUTION_TIME = "command_execution_time" # Configuration CONFIG_FEATURE_FLAGS = "config_feature_flags" + # Metrics + COUNTERS = "counters" # Information EVENT = "event" ERROR_MSG = "error_msg" @@ -72,6 +74,16 @@ class TelemetryEvent(Enum): TelemetryDict = Dict[Union[CLITelemetryField, TelemetryField], Any] +def _get_command_metrics() -> TelemetryDict: + cli_context = get_cli_context() + + return { + CLITelemetryField.COUNTERS: { + **cli_context.metrics.counters, + } + } + + def _find_command_info() -> TelemetryDict: ctx = click.get_current_context() command_path = ctx.command_path.split(" ")[1:] @@ -168,6 +180,7 @@ def log_command_result(execution: ExecutionMetadata): CLITelemetryField.COMMAND_EXECUTION_ID: execution.execution_id, CLITelemetryField.COMMAND_RESULT_STATUS: execution.status.value, CLITelemetryField.COMMAND_EXECUTION_TIME: execution.get_duration(), + **_get_command_metrics(), } ) @@ -183,6 +196,7 @@ def log_command_execution_error(exception: Exception, execution: ExecutionMetada CLITelemetryField.ERROR_TYPE: exception_type, CLITelemetryField.IS_CLI_EXCEPTION: is_cli_exception, CLITelemetryField.COMMAND_EXECUTION_TIME: execution.get_duration(), + **_get_command_metrics(), } ) diff --git a/src/snowflake/cli/_plugins/nativeapp/codegen/compiler.py b/src/snowflake/cli/_plugins/nativeapp/codegen/compiler.py index 8b254887a4..84e2060bdd 100644 --- a/src/snowflake/cli/_plugins/nativeapp/codegen/compiler.py +++ b/src/snowflake/cli/_plugins/nativeapp/codegen/compiler.py @@ -34,7 +34,9 @@ TemplatesProcessor, ) from snowflake.cli._plugins.nativeapp.feature_flags import FeatureFlag +from snowflake.cli.api.cli_global_context import get_cli_context from snowflake.cli.api.console import cli_console as cc +from snowflake.cli.api.metrics import CLICounterField from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import ( ProcessorMapping, ) @@ -76,6 +78,10 @@ def compile_artifacts(self): if not self._should_invoke_processors(): return + metrics = get_cli_context().metrics + metrics.set_counter(CLICounterField.TEMPLATES_PROCESSOR, 0) + metrics.set_counter(CLICounterField.SNOWPARK_PROCESSOR, 0) + with cc.phase("Invoking artifact processors"): if self._bundle_ctx.generated_root.exists(): raise ClickException( diff --git a/src/snowflake/cli/_plugins/nativeapp/codegen/snowpark/python_processor.py b/src/snowflake/cli/_plugins/nativeapp/codegen/snowpark/python_processor.py index f0bdf66a09..98f9ca1eae 100644 --- a/src/snowflake/cli/_plugins/nativeapp/codegen/snowpark/python_processor.py +++ b/src/snowflake/cli/_plugins/nativeapp/codegen/snowpark/python_processor.py @@ -48,7 +48,9 @@ NativeAppExtensionFunction, ) from snowflake.cli._plugins.stage.diff import to_stage_path +from snowflake.cli.api.cli_global_context import get_cli_context from snowflake.cli.api.console import cli_console as cc +from snowflake.cli.api.metrics import CLICounterField from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import ( PathMapping, ProcessorMapping, @@ -176,6 +178,8 @@ def process( setup script with generated SQL that registers these functions. """ + get_cli_context().metrics.set_counter(CLICounterField.SNOWPARK_PROCESSOR, 1) + bundle_map = BundleMap( project_root=self._bundle_ctx.project_root, deploy_root=self._bundle_ctx.deploy_root, diff --git a/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py b/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py index 779da8717e..4c04443653 100644 --- a/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py +++ b/src/snowflake/cli/_plugins/nativeapp/codegen/templates/templates_processor.py @@ -25,6 +25,7 @@ from snowflake.cli._plugins.nativeapp.exceptions import InvalidTemplateInFileError from snowflake.cli.api.cli_global_context import get_cli_context from snowflake.cli.api.console import cli_console as cc +from snowflake.cli.api.metrics import CLICounterField from snowflake.cli.api.project.schemas.v1.native_app.path_mapping import ( PathMapping, ProcessorMapping, @@ -98,6 +99,8 @@ def process( Process the artifact by executing the template expansion logic on it. """ + get_cli_context().metrics.set_counter(CLICounterField.TEMPLATES_PROCESSOR, 1) + bundle_map = BundleMap( project_root=self._bundle_ctx.project_root, deploy_root=self._bundle_ctx.deploy_root, diff --git a/src/snowflake/cli/api/cli_global_context.py b/src/snowflake/cli/api/cli_global_context.py index e810da9d62..0b53495166 100644 --- a/src/snowflake/cli/api/cli_global_context.py +++ b/src/snowflake/cli/api/cli_global_context.py @@ -22,6 +22,7 @@ from snowflake.cli.api.connections import ConnectionContext, OpenConnectionCache from snowflake.cli.api.exceptions import MissingConfiguration +from snowflake.cli.api.metrics import CLIMetrics from snowflake.cli.api.output.formats import OutputFormat from snowflake.cli.api.rendering.jinja import CONTEXT_KEY from snowflake.connector import SnowflakeConnection @@ -46,6 +47,8 @@ class _CliGlobalContextManager: experimental: bool = False enable_tracebacks: bool = True + metrics: CLIMetrics = field(default_factory=CLIMetrics) + project_path_arg: str | None = None project_is_optional: bool = True project_env_overrides_args: dict[str, str] = field(default_factory=dict) @@ -152,6 +155,10 @@ def connection_context(self) -> ConnectionContext: def enable_tracebacks(self) -> bool: return self._manager.enable_tracebacks + @property + def metrics(self): + return self._manager.metrics + @property def output_format(self) -> OutputFormat: return self._manager.output_format diff --git a/src/snowflake/cli/api/entities/utils.py b/src/snowflake/cli/api/entities/utils.py index 9b514a9ba7..a588b063be 100644 --- a/src/snowflake/cli/api/entities/utils.py +++ b/src/snowflake/cli/api/entities/utils.py @@ -31,6 +31,7 @@ NO_WAREHOUSE_SELECTED_IN_SESSION, ) from snowflake.cli.api.exceptions import SnowflakeSQLExecutionError +from snowflake.cli.api.metrics import CLICounterField from snowflake.cli.api.project.schemas.entities.common import PostDeployHook from snowflake.cli.api.rendering.sql_templates import ( choose_sql_jinja_env_based_on_template_syntax, @@ -246,9 +247,14 @@ def execute_post_deploy_hooks( While executing SQL post deploy hooks, it first switches to the database provided in the input. All post deploy scripts templates will first be expanded using the global template context. """ + metrics = get_cli_context().metrics + metrics.set_counter(CLICounterField.POST_DEPLOY_SCRIPTS, 0) + if not post_deploy_hooks: return + metrics.set_counter(CLICounterField.POST_DEPLOY_SCRIPTS, 1) + with console.phase(f"Executing {deployed_object_type} post-deploy actions"): sql_scripts_paths = [] for hook in post_deploy_hooks: diff --git a/src/snowflake/cli/api/metrics.py b/src/snowflake/cli/api/metrics.py new file mode 100644 index 0000000000..7736f87e5d --- /dev/null +++ b/src/snowflake/cli/api/metrics.py @@ -0,0 +1,57 @@ +# Copyright (c) 2024 Snowflake Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, Optional + +_FEATURES_PREFIX = "features" +_APP_PREFIX = "app" + + +class CLICounterField: + TEMPLATES_PROCESSOR = f"{_FEATURES_PREFIX}.templates_processor" + SQL_TEMPLATES = f"{_FEATURES_PREFIX}.sql_templates" + PDF_TEMPLATES = f"{_FEATURES_PREFIX}.pdf_templates" + SNOWPARK_PROCESSOR = f"{_FEATURES_PREFIX}.{_APP_PREFIX}.snowpark_processor" + POST_DEPLOY_SCRIPTS = f"{_FEATURES_PREFIX}.{_APP_PREFIX}.post_deploy_scripts" + + +class CLIMetrics: + """ + Class to track various metrics across the execution of a command + """ + + def __init__(self): + self._counters: Dict[str, int] = {} + + def __eq__(self, other): + if isinstance(other, CLIMetrics): + return self._counters == other._counters + return False + + def get_counter(self, name: str) -> Optional[int]: + return self._counters.get(name) + + def set_counter(self, name: str, value: int) -> None: + self._counters[name] = value + + def increment_counter(self, name: str, value: int = 1) -> None: + if name not in self._counters: + self.set_counter(name, value) + else: + self._counters[name] += value + + @property + def counters(self) -> Dict[str, int]: + # return a copy of the original dict to avoid mutating the original + return self._counters.copy() diff --git a/src/snowflake/cli/api/rendering/sql_templates.py b/src/snowflake/cli/api/rendering/sql_templates.py index 5834d1cd44..1c400b07f8 100644 --- a/src/snowflake/cli/api/rendering/sql_templates.py +++ b/src/snowflake/cli/api/rendering/sql_templates.py @@ -21,6 +21,7 @@ from snowflake.cli.api.cli_global_context import get_cli_context from snowflake.cli.api.console.console import cli_console from snowflake.cli.api.exceptions import InvalidTemplate +from snowflake.cli.api.metrics import CLICounterField from snowflake.cli.api.rendering.jinja import ( CONTEXT_KEY, FUNCTION_KEY, @@ -96,4 +97,9 @@ def snowflake_sql_jinja_render(content: str, data: Dict | None = None) -> str: context_data = get_cli_context().template_context context_data.update(data) env = choose_sql_jinja_env_based_on_template_syntax(content) + + get_cli_context().metrics.set_counter( + CLICounterField.SQL_TEMPLATES, int(has_sql_templates(content)) + ) + return env.from_string(content).render(context_data) diff --git a/src/snowflake/cli/api/utils/definition_rendering.py b/src/snowflake/cli/api/utils/definition_rendering.py index 1755b31609..48c41d0051 100644 --- a/src/snowflake/cli/api/utils/definition_rendering.py +++ b/src/snowflake/cli/api/utils/definition_rendering.py @@ -19,8 +19,10 @@ from jinja2 import Environment, TemplateSyntaxError, nodes from packaging.version import Version +from snowflake.cli.api.cli_global_context import get_cli_context from snowflake.cli.api.console import cli_console as cc from snowflake.cli.api.exceptions import CycleDetectedError, InvalidTemplate +from snowflake.cli.api.metrics import CLICounterField from snowflake.cli.api.project.schemas.project_definition import ( ProjectProperties, build_project_definition, @@ -266,6 +268,12 @@ def find_any_template_vars(element): return referenced_vars +def _has_referenced_vars_in_definition( + template_env: TemplatedEnvironment, definition: Definition +) -> bool: + return len(_get_referenced_vars_in_definition(template_env, definition)) > 0 + + def _template_version_warning(): cc.warning( "Ignoring template pattern in project definition file. " @@ -291,6 +299,17 @@ def _add_defaults_to_definition(original_definition: Definition) -> Definition: return definition_with_defaults +def _update_metrics(template_env: TemplatedEnvironment, definition: Definition): + metrics = get_cli_context().metrics + + # render_definition_template is invoked multiple times both by the user + # and by us so we should make sure we don't overwrite a 1 with a 0 here + metrics.increment_counter(CLICounterField.PDF_TEMPLATES, 0) + + if _has_referenced_vars_in_definition(template_env, definition): + metrics.set_counter(CLICounterField.PDF_TEMPLATES, 1) + + def render_definition_template( original_definition: Optional[Definition], context_overrides: Context ) -> ProjectProperties: @@ -326,10 +345,7 @@ def render_definition_template( definition["definition_version"] ) < Version("1.1"): try: - referenced_vars = _get_referenced_vars_in_definition( - template_env, definition - ) - if referenced_vars: + if _has_referenced_vars_in_definition(template_env, definition): _template_version_warning() except Exception: # also warn on Exception, as it means the user is incorrectly attempting to use templating @@ -340,6 +356,10 @@ def render_definition_template( project_context[CONTEXT_KEY]["env"] = environment_overrides return ProjectProperties(project_definition, project_context) + # need to have the metrics added here since we add defaults to the + # definition that the user might not have added themselves later + _update_metrics(template_env, definition) + definition = _add_defaults_to_definition(definition) project_context = {CONTEXT_KEY: definition} diff --git a/tests/api/test_metrics.py b/tests/api/test_metrics.py new file mode 100644 index 0000000000..220bc3de5f --- /dev/null +++ b/tests/api/test_metrics.py @@ -0,0 +1,78 @@ +# Copyright (c) 2024 Snowflake Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from snowflake.cli.api.metrics import CLIMetrics + + +def test_metrics_no_counters(): + # given + metrics = CLIMetrics() + + # when + + # then + assert metrics.counters == {} + assert metrics.get_counter("counter1") is None + + +def test_metrics_set_one_counter(): + # given + metrics = CLIMetrics() + + # when + metrics.set_counter("counter1", 1) + + # then + assert metrics.counters == {"counter1": 1} + assert metrics.get_counter("counter1") == 1 + + +def test_metrics_increment_new_counter(): + # given + metrics = CLIMetrics() + + # when + metrics.increment_counter("counter1") + + # then + assert metrics.counters == {"counter1": 1} + assert metrics.get_counter("counter1") == 1 + + +def test_metrics_increment_existing_counter(): + # given + metrics = CLIMetrics() + + # when + metrics.set_counter("counter1", 2) + metrics.increment_counter(name="counter1", value=2) + + # then + assert metrics.counters == {"counter1": 4} + assert metrics.get_counter("counter1") == 4 + + +def test_metrics_set_multiple_counters(): + # given + metrics = CLIMetrics() + + # when + metrics.set_counter("counter1", 1) + metrics.set_counter("counter2", 0) + metrics.set_counter(name="counter2", value=2) + + # then + assert metrics.counters == {"counter1": 1, "counter2": 2} + assert metrics.get_counter("counter1") == 1 + assert metrics.get_counter("counter2") == 2 diff --git a/tests_common/__init__.py b/tests_common/__init__.py index 9648e4f95d..62fe867531 100644 --- a/tests_common/__init__.py +++ b/tests_common/__init__.py @@ -14,5 +14,6 @@ import platform from tests_common.path_utils import * +from tests_common.telemetry_utils import * IS_WINDOWS = platform.system() == "Windows" diff --git a/tests_common/telemetry_utils.py b/tests_common/telemetry_utils.py new file mode 100644 index 0000000000..c8a885d54a --- /dev/null +++ b/tests_common/telemetry_utils.py @@ -0,0 +1,35 @@ +import unittest.mock as mock +from typing import Dict, Any + +import pytest + + +class _MockTelemetryUtils: + """ + collection of shorthand utilities for mocked telemetry object + """ + + def __init__(self, mocked_telemetry: mock.MagicMock): + self.mocked_telemetry = mocked_telemetry + + def extract_first_result_executing_command_telemetry_message( + self, + ) -> Dict[str, Any]: + return next( + args.args[0].to_dict()["message"] + for args in self.mocked_telemetry.call_args_list + if args.args[0].to_dict().get("message").get("type") + == "result_executing_command" + ) + + +@pytest.fixture +def mock_telemetry(): + """ + fixture for mocking telemetry calls, providing + useful utility functions to validate calls + """ + with mock.patch( + "snowflake.connector.telemetry.TelemetryClient.try_add_log_to_batch" + ) as mocked_telemetry: + yield _MockTelemetryUtils(mocked_telemetry) diff --git a/tests_e2e/test_nativeapp.py b/tests_e2e/test_nativeapp.py index 712378ae84..1bc654a940 100644 --- a/tests_e2e/test_nativeapp.py +++ b/tests_e2e/test_nativeapp.py @@ -18,6 +18,7 @@ from textwrap import dedent import pytest +from snowflake.cli.api.metrics import CLICounterField from tests_e2e.conftest import subprocess_check_output, subprocess_run @@ -69,7 +70,7 @@ def assert_snapshot_match_with_query_result(output: str, snapshot) -> bool: @pytest.mark.e2e def test_full_lifecycle_with_codegen( - snowcli, test_root_path, project_directory, snapshot + snowcli, test_root_path, project_directory, snapshot, mock_telemetry ): config_path = test_root_path / "config" / "config.toml" # FYI: when testing locally and you want to quickly get this running without all the setup, @@ -128,6 +129,15 @@ def test_full_lifecycle_with_codegen( assert result.returncode == 0 + telemetry_message = ( + mock_telemetry.extract_first_result_executing_command_telemetry_message() + ) + + assert telemetry_message["counters"] == { + CLICounterField.SNOWPARK_PROCESSOR: 1, + CLICounterField.TEMPLATES_PROCESSOR: 0, + } + app_name_and_schema = f"{app_name}.ext_code_schema" # Disable debug mode to call functions and procedures. diff --git a/tests_integration/nativeapp/test_feature_metrics.py b/tests_integration/nativeapp/test_feature_metrics.py new file mode 100644 index 0000000000..401e9bcaa1 --- /dev/null +++ b/tests_integration/nativeapp/test_feature_metrics.py @@ -0,0 +1,97 @@ +# Copyright (c) 2024 Snowflake Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from shlex import split +from typing import Dict + +from snowflake.cli.api.metrics import CLICounterField +from tests.project.fixtures import * + + +@pytest.mark.integration +@pytest.mark.parametrize( + "command,expected_counter", + [ + ( + [ + "sql", + "-q", + "select '<% ctx.env.test %>'", + "--env", + "test=value_from_cli", + ], + 1, + ), + (["sql", "-q", "select 'string'"], 0), + ], +) +def test_sql_templating_emits_counter_one( + command: List[str], + expected_counter, + runner, + mock_telemetry, +): + result = runner.invoke_with_connection_json(command) + + assert result.exit_code == 0 + + message = mock_telemetry.extract_first_result_executing_command_telemetry_message() + + assert message["counters"][CLICounterField.SQL_TEMPLATES] == expected_counter + + +@pytest.mark.integration +@pytest.mark.parametrize( + "command," "test_project," "expected_counters", + [ + ( + "app deploy", + "napp_application_post_deploy_v1", + {CLICounterField.PDF_TEMPLATES: 0, CLICounterField.POST_DEPLOY_SCRIPTS: 1}, + ), + ( + "ws bundle --entity-id=pkg", + "napp_templates_processors_v2", + { + CLICounterField.SNOWPARK_PROCESSOR: 0, + CLICounterField.TEMPLATES_PROCESSOR: 1, + CLICounterField.PDF_TEMPLATES: 1, + }, + ), + ], +) +def test_nativeapp_feature_counter_has_expected_value( + mock_telemetry, + runner, + nativeapp_teardown, + nativeapp_project_directory, + command: str, + test_project: str, + expected_counters: Dict[str, int], +): + local_test_env = { + "APP_DIR": "app", + "schema_name": "test_schema", + "table_name": "test_table", + "value": "test_value", + } + + with nativeapp_project_directory(test_project): + runner.invoke_with_connection(split(command), env=local_test_env) + + # The method is called with a TelemetryData type, so we cast it to dict for simpler comparison + message = ( + mock_telemetry.extract_first_result_executing_command_telemetry_message() + ) + + assert message["counters"] == expected_counters