optimize commands and operations, add --ini support for project disco…

…very

utilmeta/bin/base.py

@@ -151,6 +151,10 @@ def __init__(self, *argv: str, cwd: str):
         self.cwd = cwd
         if argv:
             self.arg_name, *self.args = argv
+            # like meta --ini ...
+            if self.arg_name.startswith("--"):
+                self.arg_name = ""
+                self.args = argv
         else:
             self.arg_name = ""
             self.args = []

utilmeta/bin/commands/base.py

@@ -16,16 +16,17 @@ def __init__(self, exe: str = None, *args: str, cwd: str = os.getcwd()):
         self.exe = exe  # absolute path of meta command tool
         self.sys_args = list(args)
         if exe:
-            os.environ.setdefault("META_ABSOLUTE_PATH", exe)
+            os.environ.setdefault("UTILMETA_EXECUTABLE_PATH", exe)
 
         if not os.path.isabs(cwd):
             cwd = path_join(os.getcwd(), cwd)
 
         self.cwd = cwd.replace("\\", "/")
-        self.ini_path = search_file("utilmeta.ini", path=cwd) or search_file(
-            META_INI, path=cwd
-        )
+        self.ini_path = self.get_ini_file(*args)
+
         self.base_path = os.path.dirname(self.ini_path) if self.ini_path else self.cwd
+        os.environ.setdefault("UTILMETA_PROJECT_DIR", self.base_path)
+
         self.service_config = {}
         self._service = None
         self._application = None
@@ -38,6 +39,41 @@ def __init__(self, exe: str = None, *args: str, cwd: str = os.getcwd()):
 
         super().__init__(*self.sys_args, cwd=self.cwd)
 
+    def get_ini_file(self, *args: str):
+        project_dir = str(os.getenv("UTILMETA_PROJECT_DIR") or self.cwd)
+        file = search_file("utilmeta.ini", path=project_dir) or search_file(
+            META_INI, path=project_dir
+        )
+        if file:
+            # if inside a project, use the found ini file
+            return file
+        # check
+        ini_file = None
+        exclude_params = []
+        for i, arg in enumerate(args):
+            if arg.startswith('--ini'):
+                if '=' in arg:
+                    ini_file = arg.split('=')[1]
+                    exclude_params.append(i)
+                else:
+                    try:
+                        ini_file = args[i + 1]
+                        exclude_params.extend([i, i + 1])
+                    except IndexError:
+                        ini_file = None
+                break
+        if exclude_params:
+            sys.argv = [self.exe] + [arg for i, arg in enumerate(sys.argv[1:]) if i not in exclude_params]
+            self.sys_args = [arg for i, arg in enumerate(args) if i not in exclude_params]
+        if ini_file:
+            path = path_join(self.cwd, ini_file)
+            if os.path.isdir(path):
+                return search_file("utilmeta.ini", path=path) or search_file(
+                    META_INI, path=path
+                )
+            return path
+        return None
+
     def command_not_found(self):
         print(RED % f'{self.script_name or "meta"}: command not found: {self.arg_name}')
         if not self.ini_path:

utilmeta/bin/meta.py

@@ -232,19 +232,23 @@ def run(
         print(
             f"UtilMeta service {BLUE % self.service.name} running at {self.main_file}"
         )
+        if os.name == 'nt':
+            env = f'set PYTHONPATH=%PYTHONPATH%;{self.service.project_dir} &&'
+        else:
+            env = f'PYTHONPATH=$PYTHONPATH:{self.service.project_dir}'
         cmd = f"{sys.executable} {self.main_file}"
         if daemon:
             if os.name == "posix":
                 print(f"running service with nohup in background, writing log to {log}")
-                cmd = f"nohup {cmd} > {log} 2>&1 &"
+                cmd = f"{env} nohup {cmd} > {log} 2>&1 &"
             else:
                 print(YELLOW % "ignoring daemon mode since only posix system support")
         if connect:
             from utilmeta.ops.cmd import try_to_connect
 
             try_to_connect()
 
-        run(cmd)
+        run(f'{env} {cmd}')
 
     @command
     def down(self):
@@ -293,6 +297,7 @@ def restart(
             print(RED % f"meta restart: load process: {pid} failed with error: {e}")
             exit(1)
         proc.kill()
+        print(f'current service [{self.service.name}](pid={pid}) stopped')
         return self.run(daemon=True, connect=connect, log=log)
 
 

utilmeta/conf/env.py

@@ -54,11 +54,21 @@ def _load_from_sys_env(self) -> Mapping:
                 data[key[len(self._sys_env_prefix) :]] = value
         return data
 
+    @classmethod
+    def _get_base_dir(cls):
+        try:
+            from utilmeta import service
+            if service.project_dir:
+                return service.project_dir
+        except ImportError:
+            pass
+        return os.getenv("UTILMETA_PROJECT_DIR") or os.getcwd()
+
     def _load_from_file(self) -> Mapping:
         if not self._file:
             return {}
         if not os.path.exists(self._file):
-            rel_file = os.path.join(os.getcwd(), self._file)
+            rel_file = os.path.join(self._get_base_dir(), self._file)
             if not os.path.exists(rel_file):
                 raise FileNotFoundError(
                     f"{self.__class__}: file: {repr(self._file)} not exists"

utilmeta/core/server/backends/django/cmd.py

@@ -181,13 +181,19 @@ def mergemigrations(self, app_name: str):
             #     cfg: AppConfig
             #     self.mergemigrations(cfg.label)
             for key, cfg in apps.app_configs.items():
-                if not cfg.path.startswith(self.service.project_dir):
+                if not cfg.path.startswith(str(self.service.project_dir)):
                     # eg. django content types / utilmeta.ops
                     continue
                 cfg: AppConfig
                 # if cfg.label == app_name:
                 migrations_path = os.path.join(cfg.path, "migrations")
-                files = next(os.walk(migrations_path))[2]
+                try:
+                    files = next(os.walk(migrations_path))[2]
+                except (StopIteration, IndexError):
+                    # migrations does not exists
+                    continue
+                if not files:
+                    continue
                 for file in files:
                     if file.startswith(SEG):
                         continue
@@ -206,7 +212,13 @@ def mergemigrations(self, app_name: str):
                 cfg: AppConfig
                 # if cfg.label == app_name:
                 migrations_path = os.path.join(cfg.path, "migrations")
-                files = next(os.walk(migrations_path))[2]
+                try:
+                    files = next(os.walk(migrations_path))[2]
+                except (StopIteration, IndexError):
+                    # migrations does not exists
+                    continue
+                if not files:
+                    continue
                 for file in files:
                     if file.startswith(SEG):
                         continue

utilmeta/core/server/backends/django/settings.py

@@ -592,7 +592,7 @@ def setup(self, service: UtilMeta):
                     service,
                     "default",
                     database=Database(
-                        name=os.path.join(service.project_dir, "__default_db"),
+                        name=str(os.path.join(service.project_dir, "__default_db")),
                         engine="sqlite3",
                     ),
                 )

utilmeta/core/server/service.py

@@ -69,7 +69,7 @@ def __init__(
         # 2. os.path.dirname(self.module.__file__)
         # 3. sys.path[0] / os.getcwd()
         self.meta_path = None
-        self.project_dir = Path(os.getcwd())
+        self.project_dir = Path(os.getenv("UTILMETA_PROJECT_DIR") or os.getcwd())
         self.meta_config = {}
         self.root_url = str(route or "").strip("/")
 
@@ -189,7 +189,11 @@ def root_api(self, api):
             )
 
     def load_meta(self):
-        self.meta_path = search_file("utilmeta.ini") or search_file("meta.ini")
+        self.meta_path = search_file(
+            "utilmeta.ini", path=self.project_dir
+        ) or search_file(
+            "meta.ini", path=self.project_dir
+        )
 
         if self.meta_path:
             self.project_dir = Path(os.path.dirname(self.meta_path))
@@ -647,7 +651,7 @@ def get_origin(self, no_localhost: bool = False, force_ip: bool = False):
     def origin(self):
         if self._origin:
             return self._origin
-        return self.get_origin(no_localhost=self.production)
+        return self.get_origin()
 
     # def get_base_url(self, no_localhost: bool = False):
     #     origin = self.get_origin(no_localhost=no_localhost)

utilmeta/ops/client.py

@@ -5,7 +5,7 @@
 from utilmeta.core import request
 from utype.types import *
 
-from .key import encrypt_data
+from .key import encrypt_data, decode_key
 from .schema import (
     NodeMetadata,
     SupervisorBasic,
@@ -214,14 +214,15 @@ def __init__(
 
         headers = {}
         if access_key:
+            access_key = decode_key(access_key)
             # only required in ADD_NODE operation
             headers.update(
                 {
                     "X-Access-Key": access_key,
                 }
             )
-
         if cluster_key:
+            cluster_key = decode_key(cluster_key)
             headers.update({"X-Cluster-Key": cluster_key})
         if cluster_id:
             headers.update({"X-Cluster-Id": cluster_id})
@@ -246,6 +247,7 @@ def __init__(
                 self._base_url = supervisor.base_url
 
         if node_key:
+            node_key = decode_key(node_key)
             headers.update({"X-Node-Key": node_key})
         if service_id:
             headers.update({"X-Service-ID": service_id})

utilmeta/ops/cmd.py

@@ -144,20 +144,35 @@ def connect(
                 exit(0)
 
         if not self.config.is_local:
-            if not self.config.proxy and self.config.proxy_required:
-                print(
-                    YELLOW
-                    % f"meta connect: it seems that you are using a private base_url: {self.config.base_url} "
-                    f"without setting "
-                    "a proxy in Operations, this service will be unable to access in the platform"
-                )
-            if not self.config.is_secure:
-                print(
-                    YELLOW
-                    % f"meta connect: you are trying to connect an insecure node:"
-                    f" {self.config.ops_api} (with HTTP protocol), "
-                    "we strongly recommend using HTTPS protocol instead"
-                )
+            if not self.config.proxy:
+                if self.config.proxy_required:
+                    print(
+                        YELLOW
+                        % f"meta connect: it seems that you are using a private base_url: {self.config.base_url} "
+                        f"without setting "
+                        "a proxy in Operations, this service will be unable to access in the platform"
+                    )
+
+                if not self.config.is_secure:
+                    print(
+                        YELLOW
+                        % f"meta connect: you are trying to connect an insecure node:"
+                        f" {self.config.ops_api} (with HTTP protocol), "
+                        "we strongly recommend using HTTPS protocol instead"
+                    )
+
+        if self.config.proxy:
+            print(f'Connect to supervisor using proxy: {self.config.proxy.base_url}')
+            manager = self.config.resources_manager_cls(service=self.service)
+            node_id = manager.init_service_resources(force=force)
+            if node_id:
+                from .models import Supervisor
+                supervisor: Supervisor = Supervisor.filter(node_id=node_id).first()
+                if supervisor:
+                    print(f'UtilMeta supervisor[{node_id}] connected')
+                    if supervisor.url:
+                        print(f"please visit {supervisor.url} to view and manage your APIs")
+            return
 
         if not key:
             webbrowser.open_new_tab(__website__)
@@ -175,7 +190,6 @@ def connect(
             exit(1)
 
         from .connect import connect_supervisor
-
         connect_supervisor(key=key, base_url=to, service_id=service)
 
     @command

utilmeta/ops/config.py

@@ -589,6 +589,8 @@ def migrate(self, with_default: bool = False):
 
     @property
     def ops_api(self):
+        if self.proxy:
+            return self.proxy_ops_api
         parsed = urlsplit(self.route)
         if parsed.scheme:
             # is url

utilmeta/ops/connect.py

@@ -265,7 +265,7 @@ def connect_supervisor(
     if not supervisor_obj.local:
         resources.sync_resources(supervisor_obj)
 
-    print("supervisor connected successfully!")
+    print("UtilMeta supervisor connected successfully!")
     if url:
         print(f"please visit {url} to view and manage your APIs")
 

utilmeta/ops/key.py

@@ -14,13 +14,18 @@ def generate_key_pair(identifier: str):
     return public_key, private_key
 
 
+def decode_key(key: str):
+    if not key.startswith("{") or not key.endswith("}"):
+        # BASE64
+        key = base64.decodebytes(key.encode()).decode()
+    return key
+
+
 def encrypt_data(payload, public_key: Union[str, dict]) -> str:
     if not isinstance(public_key, dict):
         if isinstance(public_key, str):
-            if not public_key.startswith("{") or not public_key.endswith("}"):
-                # BASE64
-                public_key = base64.decodebytes(public_key.encode()).decode()
-
+            # BASE64
+            public_key = decode_key(public_key)
         public_key = json_decode(public_key)
     pubkey_obj = jwk.JWK(**public_key)
     protected_header = {

utilmeta/ops/resources.py

@@ -511,16 +511,25 @@ def sync_resources(self, supervisor: Supervisor = None, force: bool = False):
                         f"you can visit {resp.result.url} to view the updated resources"
                     )
 
+    def get_instance(self):
+        from .log import setup_locals
+        from .models import Resource
+        setup_locals(self.ops_config)
+        from .log import _instance
+        return _instance or Resource.get_current_instance()
+
     def init_service_resources(
         self,
         supervisor: Supervisor = None,
         instance: Resource = None,
         force: bool = False,
     ):
         if self.ops_config.proxy:
-            self.register_service(supervisor=supervisor, instance=instance)
+            if not instance:
+                instance = self.get_instance()
+            return self.register_service(supervisor=supervisor, instance=instance)
         else:
-            self.sync_resources(supervisor=supervisor, force=force)
+            return self.sync_resources(supervisor=supervisor, force=force)
 
     def register_service(
         self, supervisor: Supervisor = None, instance: Resource = None
@@ -563,8 +572,8 @@ def register_service(
             if isinstance(resp, RegistryResponse):
                 if resp.result.node_id:
                     from utilmeta.bin.utils import update_meta_ini_file
-
                     update_meta_ini_file(node=resp.result.node_id)
+                    return resp.result.node_id
             else:
                 warnings.warn(
                     f"register service: [{self.service.name}] to proxy: "