feat: --initialize command to onboard a new account (#80)

Closes #21

This is kind of a new idea that we talked about in office hours a while
back for leveraging the target's capabilities to configure the database.
Related to meltano/hub#1407.

Adds a CLI flag for initializing a new account. It interactively prompts
for all the information it needs and will execute it if you want it to,
otherwise it will just print the sql and you can run it yourself. Based
on https://fivetran.com/docs/destinations/snowflake/setup-guide.

I created a personal trial snowflake account and was able to initialize
using this script then immediately run a tap using the new
user/role/warehouse/database.

---------

Co-authored-by: Edgar R. M <edgar@meltano.com>

README.md

@@ -32,6 +32,37 @@ Built with the [Meltano Singer SDK](https://sdk.meltano.com).
 
 A full list of supported settings and capabilities is available by running: `target-snowflake --about`
 
+### Initializing a Snowflake Account
+
+This target has an interactive feature that will help you get a Snowflake account initialized with everything needed to get started loading data.
+
+- User
+- Role
+- Warehouse
+- Database
+- Proper grants
+
+The CLI will ask you to provide information about the new user/role/etc. you want to create but it will also need SYSADMIN credentials to execute the queries.
+You should prepare the following inputs:
+
+- Account
+- User that has SYSADMIN and SECURITYADMIN access. These comes default with the user that created the Snowflake account.
+- The password for your SYSADMIN user.
+
+Run the following command to get started with the interactive CLI.
+Note - the CLI will print the SQL queries it is planning to run and confirm with you before it makes any changes.
+
+```bash
+poetry run target-snowflake --initialize
+
+# Alternatively using Meltano CLI
+meltano invoke target-snowflake --initialize
+```
+
+The CLI also has a "dry run" mode that will print the queries without executing them.
+
+Check out the demo of this [on YouTube](https://youtu.be/9vEFxw-0nxI).
+
 ### Configure using environment variables
 
 This Singer target will automatically import any environment variables within the working directory's

target_snowflake/connector.py

@@ -461,3 +461,52 @@ def remove_staged_files(self, sync_id: str) -> None:
  )
  self.logger.debug(f"Removing staged files with SQL: {remove_statement!s}")
  conn.execute(remove_statement, **kwargs)
+
+ @staticmethod
+ def get_initialize_script(role, user, password, warehouse, database):
+ # https://fivetran.com/docs/destinations/snowflake/setup-guide
+ return f"""
+ begin;
+
+ -- change role to securityadmin for user / role steps
+ use role securityadmin;
+
+ -- create role
+ create role if not exists {role};
+ grant role {role} to role SYSADMIN;
+
+ -- create a user
+ create user if not exists {user}
+ password = '{password}'
+ default_role = {role}
+ default_warehouse = {warehouse};
+
+ grant role {role} to user {user};
+
+ -- change role to sysadmin for warehouse / database steps
+ use role sysadmin;
+
+ -- create a warehouse
+ create warehouse if not exists {warehouse}
+ warehouse_size = xsmall
+ warehouse_type = standard
+ auto_suspend = 60
+ auto_resume = true
+ initially_suspended = true;
+
+ -- create database
+ create database if not exists {database};
+
+ -- grant role access to warehouse
+ grant USAGE
+ on warehouse {warehouse}
+ to role {role};
+
+ -- grant access to database
+ grant CREATE SCHEMA, MONITOR, USAGE
+ on database {database}
+ to role {role};
+ 
+ commit;
+
+ """

target_snowflake/initializer.py

@@ -0,0 +1,61 @@
+import click
+from target_snowflake.connector import SnowflakeConnector
+import sys
+from sqlalchemy import text
+
+
+def initializer():
+ click.echo("")
+ click.echo("")
+ click.echo("✨Initializing Snowflake account.✨")
+ click.echo("Note: You will always be asked to confirm before anything is executed.")
+ click.echo("")
+ click.echo("Additionally you can run in `dry_run` mode which will print the SQL without running it.")
+ dry_run = click.prompt("Would you like to run in `dry_run` mode?", default=False, type=bool)
+ click.echo("")
+ click.echo("We will now interactively create (or the print queries) for all the following objects in your Snowflake account:")
+ click.echo(" - Role")
+ click.echo(" - User")
+ click.echo(" - Warehouse")
+ click.echo(" - Database")
+ click.echo("")
+ role = click.prompt("Meltano Role Name:", type=str, default="MELTANO_ROLE")
+ user = click.prompt("Meltano User Name:", type=str, default="MELTANO_USER")
+ password = click.prompt("Meltano Password", type=str, confirmation_prompt=True)
+ warehouse = click.prompt("Meltano Warehouse Name", type=str, default="MELTANO_WAREHOUSE")
+ database = click.prompt("Meltano Database Name", type=str, default="MELTANO_DATABASE")
+ script = SnowflakeConnector.get_initialize_script(role, user, password, warehouse, database)
+ if dry_run:
+ click.echo(script)
+ sys.exit(0)
+ else:
+ account = click.prompt("Account (i.e. lqnwlrc-onb17812)", type=str)
+ admin_user = click.prompt("User w/SYSADMIN access", type=str)
+ admin_pass = click.prompt("User Password", type=str)
+ connector = SnowflakeConnector(
+ {
+ "account": account,
+ "database": "SNOWFLAKE",
+ "password": admin_pass,
+ "role": "SYSADMIN",
+ "user": admin_user,
+ }
+ )
+ connector
+ try:
+ click.echo("Initialization Started")
+ with connector._connect() as conn:
+ click.echo(f"Executing:")
+ click.echo(f"{script}")
+ click.prompt("Confirm?", default=True, type=bool)
+ click.echo("Initialization Started...")
+ for statement in script.split(';'):
+ if len(statement.strip()) > 0:
+ conn.execute(
+ text(statement)
+ )
+ click.echo("Success!")
+ click.echo("Initialization Complete")
+ except Exception as e:
+ click.echo(f"Initialization Failed: {e}")
+ sys.exit(1)

target_snowflake/target.py

@@ -2,11 +2,15 @@
 
 from __future__ import annotations
 
+import sys
+
+import click
 from singer_sdk import typing as th
-from singer_sdk.target_base import SQLTarget
+from singer_sdk.target_base import SQLTarget, Target
 
+from target_snowflake.initializer import initializer
 from target_snowflake.sinks import SnowflakeSink
-from singer_sdk.sinks import Sink
+
 
 class TargetSnowflake(SQLTarget):
  """Target for Snowflake."""
@@ -69,5 +73,39 @@ class TargetSnowflake(SQLTarget):
 
  default_sink_class = SnowflakeSink
 
+ @classmethod
+ def cb_inititalize(
+ cls: type[TargetSnowflake],
+ ctx: click.Context,
+ param: click.Option, # noqa: ARG003
+ value: bool, # noqa: FBT001
+ ) -> None:
+ if value:
+ initializer()
+ ctx.exit()
+
+ @classmethod
+ def get_singer_command(cls: type[TargetSnowflake]) -> click.Command:
+ """Execute standard CLI handler for targets.
+
+ Returns:
+ A click.Command object.
+ """
+ command = super().get_singer_command()
+ command.params.extend(
+ [
+ click.Option(
+ ["--initialize"],
+ is_flag=True,
+ help="Interactive Snowflake account initialization.",
+ callback=cls.cb_inititalize,
+ expose_value=False,
+ ),
+ ],
+ )
+
+ return command
+
+
 if __name__ == "__main__":
  TargetSnowflake.cli()