diff --git a/hub_utils/main.py b/hub_utils/main.py index 2fe78c0..ccdef05 100644 --- a/hub_utils/main.py +++ b/hub_utils/main.py @@ -3,7 +3,7 @@ import json import os from copy import copy -from datetime import datetime +from datetime import datetime, timezone from enum import Enum import requests @@ -260,6 +260,7 @@ def get_variant_names( def extract_sdk_metadata_to_s3( variant_path_list: str, output_dir: str, + python: str | None = None, ): """ NOTE: USED FOR @@ -279,6 +280,7 @@ def extract_sdk_metadata_to_s3( data.get("namespace"), data.get("executable", p_name), True, + python=python, ) hash_id = hashlib.md5( json.dumps(sdk_def, sort_keys=True, indent=2).encode("utf-8") @@ -287,7 +289,7 @@ def extract_sdk_metadata_to_s3( file_name = file_path + ".json" local_file_path = f"{output_dir}/{p_type}/{p_name}/{hash_id}--{file_name}" util._write_dict(local_file_path, sdk_def) - date_now = datetime.utcnow().strftime("%Y-%m-%d") + date_now = datetime.now(tz=timezone.utc).strftime("%Y-%m-%d") s3_file_path = f"{p_type}/{p_name}/{file_path}/{hash_id}--{date_now}.json" s3_bucket = os.environ.get("AWS_S3_BUCKET") if not S3().hash_exists(s3_bucket, s3_file_path): @@ -317,7 +319,7 @@ def upload_airbyte( json.dumps(spec_data, sort_keys=True, indent=2).encode("utf-8") ).hexdigest() file_path = os.path.basename(yaml_file).replace(".yml", "") - date_now = datetime.utcnow().strftime("%Y-%m-%d") + date_now = datetime.now(tz=timezone.utc).strftime("%Y-%m-%d") s3_file_path = f"{p_type}/{p_name}/{file_path}/{hash_id}--{date_now}.json" s3_bucket = os.environ.get("AWS_S3_BUCKET") if not S3().hash_exists(s3_bucket, s3_file_path): diff --git a/hub_utils/meltano_util.py b/hub_utils/meltano_util.py index d4e1b55..9a6e373 100644 --- a/hub_utils/meltano_util.py +++ b/hub_utils/meltano_util.py @@ -1,5 +1,7 @@ import json import pathlib +import shlex +import shutil import subprocess import tempfile @@ -15,17 +17,22 @@ def get_cwd(): return pathlib.Path(__file__).parent.resolve() @staticmethod - def add(plugin_name, namespace, executable, pip_url, plugin_type): - python_version = subprocess.run( - "which python".split(" "), stdout=subprocess.PIPE, text=True - ).stdout.replace("\n", "") + def add( + plugin_name, + namespace, + executable, + pip_url, + plugin_type, + python: str | None = None, + ): + python_version = python or shutil.which("python") subprocess.run( - f"pipx uninstall {plugin_name}".split(" "), + shlex.split(f"pipx uninstall {plugin_name}"), stdout=subprocess.PIPE, text=True, ) subprocess.run( - f"pipx install {pip_url} --python {python_version}".split(" "), + shlex.split(f"pipx install {pip_url} --python {python_version}"), stdout=subprocess.PIPE, text=True, check=True, @@ -38,14 +45,14 @@ def help_test(plugin_name, config=None): json.dump(config, tmp) tmp.flush() subprocess.run( - f"{plugin_name} --help --config {tmp.name}".split(" "), + shlex.split(f"{plugin_name} --help --config {tmp.name}"), stdout=subprocess.PIPE, text=True, check=True, ) else: subprocess.run( - f"{plugin_name} --help".split(" "), + shlex.split(f"{plugin_name} --help"), stdout=subprocess.PIPE, text=True, check=True, @@ -69,7 +76,7 @@ def sdk_about(plugin_name, config=None): return json.loads(about_json_str) else: about_content = subprocess.run( - f"{plugin_name} --about --format=json".split(" "), + shlex.split(f"{plugin_name} --about --format=json"), stdout=subprocess.PIPE, text=True, check=True, diff --git a/hub_utils/utilities.py b/hub_utils/utilities.py index 20dbaa5..49bd8fc 100644 --- a/hub_utils/utilities.py +++ b/hub_utils/utilities.py @@ -384,8 +384,22 @@ def _reformat_all(self, plugin_type, plugin_name, variant): self._reformat(f"{self.hub_root}/{file_path}") @staticmethod - def _install_test(plugin_name, plugin_type, pip_url, namespace, executable): - MeltanoUtil.add(plugin_name, namespace, executable, pip_url, plugin_type) + def _install_test( + plugin_name, + plugin_type, + pip_url, + namespace, + executable, + python: str | None = None, + ): + MeltanoUtil.add( + plugin_name, + namespace, + executable, + pip_url, + plugin_type, + python=python, + ) MeltanoUtil.help_test(executable) def add(self, repo_url: str | None = None, definition_seed: dict | None = None): @@ -587,10 +601,24 @@ def _merge_definitions(self, existing_def, settings, keywords, m_status, caps, s return new_def def _test_exception( - self, plugin_name, plugin_type, pip_url, namespace, executable, is_meltano_sdk + self, + plugin_name, + plugin_type, + pip_url, + namespace, + executable, + is_meltano_sdk, + python: str | None = None, ): if self._prompt("Run install test?", True, type=bool): - self._install_test(plugin_name, plugin_type, pip_url, namespace, executable) + self._install_test( + plugin_name, + plugin_type, + pip_url, + namespace, + executable, + python=python, + ) if is_meltano_sdk: if self._prompt("Scrape SDK --about settings?", True, type=bool): try: @@ -600,11 +628,24 @@ def _test_exception( return json.loads(self._prompt("Provide --about output")) def _test( - self, plugin_name, plugin_type, pip_url, namespace, executable, is_meltano_sdk + self, + plugin_name, + plugin_type, + pip_url, + namespace, + executable, + is_meltano_sdk, + python: str | None = None, ): try: return self._test_exception( - plugin_name, plugin_type, pip_url, namespace, executable, is_meltano_sdk + plugin_name, + plugin_type, + pip_url, + namespace, + executable, + is_meltano_sdk, + python=python, ) except Exception as e: print(e)