feat: Allow passing a custom Python exectuable to `extract-sdk-metada…

…ta-to-s3`

hub_utils/main.py

@@ -3,7 +3,7 @@
 import json
 import os
 from copy import copy
-from datetime import datetime
+from datetime import datetime, timezone
 from enum import Enum
 
 import requests
@@ -260,6 +260,7 @@ def get_variant_names(
 def extract_sdk_metadata_to_s3(
     variant_path_list: str,
     output_dir: str,
+    python: str | None = None,
 ):
     """
     NOTE: USED FOR
@@ -272,13 +273,19 @@ def extract_sdk_metadata_to_s3(
         data = util._read_yaml(yaml_file)
         p_type = util.get_plugin_type(data.get("repo"))
         p_name = data.get("name")
+        print(f"Extracting: {p_type}/{p_name}")
+
+        if python:
+            print(f"Using Python: {python}")
+
         sdk_def = util._test_exception(
             p_name,
             p_type,
             data.get("pip_url"),
             data.get("namespace"),
             data.get("executable", p_name),
-            True,
+            is_meltano_sdk=True,
+            python=python,
         )
         hash_id = hashlib.md5(
             json.dumps(sdk_def, sort_keys=True, indent=2).encode("utf-8")
@@ -287,7 +294,7 @@ def extract_sdk_metadata_to_s3(
         file_name = file_path + ".json"
         local_file_path = f"{output_dir}/{p_type}/{p_name}/{hash_id}--{file_name}"
         util._write_dict(local_file_path, sdk_def)
-        date_now = datetime.utcnow().strftime("%Y-%m-%d")
+        date_now = datetime.now(tz=timezone.utc).strftime("%Y-%m-%d")
         s3_file_path = f"{p_type}/{p_name}/{file_path}/{hash_id}--{date_now}.json"
         s3_bucket = os.environ.get("AWS_S3_BUCKET")
         if not S3().hash_exists(s3_bucket, s3_file_path):
@@ -317,7 +324,7 @@ def upload_airbyte(
             json.dumps(spec_data, sort_keys=True, indent=2).encode("utf-8")
         ).hexdigest()
         file_path = os.path.basename(yaml_file).replace(".yml", "")
-        date_now = datetime.utcnow().strftime("%Y-%m-%d")
+        date_now = datetime.now(tz=timezone.utc).strftime("%Y-%m-%d")
         s3_file_path = f"{p_type}/{p_name}/{file_path}/{hash_id}--{date_now}.json"
         s3_bucket = os.environ.get("AWS_S3_BUCKET")
         if not S3().hash_exists(s3_bucket, s3_file_path):

hub_utils/meltano_util.py

@@ -1,5 +1,7 @@
 import json
 import pathlib
+import shlex
+import shutil
 import subprocess
 import tempfile
 
@@ -15,17 +17,22 @@ def get_cwd():
         return pathlib.Path(__file__).parent.resolve()
 
     @staticmethod
-    def add(plugin_name, namespace, executable, pip_url, plugin_type):
-        python_version = subprocess.run(
-            "which python".split(" "), stdout=subprocess.PIPE, text=True
-        ).stdout.replace("\n", "")
+    def add(
+        plugin_name,
+        namespace,
+        executable,
+        pip_url,
+        plugin_type,
+        python: str | None = None,
+    ):
+        python_version = python or shutil.which("python")
         subprocess.run(
-            f"pipx uninstall {plugin_name}".split(" "),
+            shlex.split(f"pipx uninstall {plugin_name}"),
             stdout=subprocess.PIPE,
             text=True,
         )
         subprocess.run(
-            f"pipx install {pip_url} --python {python_version}".split(" "),
+            shlex.split(f"pipx install {pip_url} --python {python_version}"),
             stdout=subprocess.PIPE,
             text=True,
             check=True,
@@ -38,14 +45,14 @@ def help_test(plugin_name, config=None):
                 json.dump(config, tmp)
                 tmp.flush()
                 subprocess.run(
-                    f"{plugin_name} --help --config {tmp.name}".split(" "),
+                    shlex.split(f"{plugin_name} --help --config {tmp.name}"),
                     stdout=subprocess.PIPE,
                     text=True,
                     check=True,
                 )
         else:
             subprocess.run(
-                f"{plugin_name} --help".split(" "),
+                shlex.split(f"{plugin_name} --help"),
                 stdout=subprocess.PIPE,
                 text=True,
                 check=True,
@@ -69,7 +76,7 @@ def sdk_about(plugin_name, config=None):
                 return json.loads(about_json_str)
         else:
             about_content = subprocess.run(
-                f"{plugin_name} --about --format=json".split(" "),
+                shlex.split(f"{plugin_name} --about --format=json"),
                 stdout=subprocess.PIPE,
                 text=True,
                 check=True,

hub_utils/utilities.py

@@ -144,6 +144,7 @@ def _get_plugin_variant(repo_url: str):
 
     @staticmethod
     def get_plugin_type(plugin_name: str):
+        plugin_name = plugin_name.lower()
         if "tap-" in plugin_name and "target-" in plugin_name:
             raise Exception(f"Type Unknown: {plugin_name}")
         if "tap-" in plugin_name:
@@ -384,8 +385,22 @@ def _reformat_all(self, plugin_type, plugin_name, variant):
             self._reformat(f"{self.hub_root}/{file_path}")
 
     @staticmethod
-    def _install_test(plugin_name, plugin_type, pip_url, namespace, executable):
-        MeltanoUtil.add(plugin_name, namespace, executable, pip_url, plugin_type)
+    def _install_test(
+        plugin_name,
+        plugin_type,
+        pip_url,
+        namespace,
+        executable,
+        python: str | None = None,
+    ):
+        MeltanoUtil.add(
+            plugin_name,
+            namespace,
+            executable,
+            pip_url,
+            plugin_type,
+            python=python,
+        )
         MeltanoUtil.help_test(executable)
 
     def add(self, repo_url: str | None = None, definition_seed: dict | None = None):
@@ -587,10 +602,25 @@ def _merge_definitions(self, existing_def, settings, keywords, m_status, caps, s
         return new_def
 
     def _test_exception(
-        self, plugin_name, plugin_type, pip_url, namespace, executable, is_meltano_sdk
+        self,
+        plugin_name,
+        plugin_type,
+        pip_url,
+        namespace,
+        executable,
+        *,
+        is_meltano_sdk: bool,
+        python: str | None = None,
     ):
         if self._prompt("Run install test?", True, type=bool):
-            self._install_test(plugin_name, plugin_type, pip_url, namespace, executable)
+            self._install_test(
+                plugin_name,
+                plugin_type,
+                pip_url,
+                namespace,
+                executable,
+                python=python,
+            )
         if is_meltano_sdk:
             if self._prompt("Scrape SDK --about settings?", True, type=bool):
                 try:
@@ -600,11 +630,24 @@ def _test_exception(
                         return json.loads(self._prompt("Provide --about output"))
 
     def _test(
-        self, plugin_name, plugin_type, pip_url, namespace, executable, is_meltano_sdk
+        self,
+        plugin_name,
+        plugin_type,
+        pip_url,
+        namespace,
+        executable,
+        is_meltano_sdk,
+        python: str | None = None,
     ):
         try:
             return self._test_exception(
-                plugin_name, plugin_type, pip_url, namespace, executable, is_meltano_sdk
+                plugin_name,
+                plugin_type,
+                pip_url,
+                namespace,
+                executable,
+                is_meltano_sdk=is_meltano_sdk,
+                python=python,
             )
         except Exception as e:
             print(e)