From 3653a443fea1795a0473eaac1741c02c298ad471 Mon Sep 17 00:00:00 2001 From: Ashish Mahendra Date: Wed, 9 Oct 2024 12:49:48 +0000 Subject: [PATCH] adding setup.py --- jac-splice-orc/.gitignore | 3 + jac-splice-orc/managers/proxy_manager.py | 36 +++++------ jac-splice-orc/plugin/__init__.py | 0 jac-splice-orc/plugin/splice_plugin.py | 81 ++++++++++++++++++++++++ jac-splice-orc/setup.py | 24 +++++++ jac-splice-orc/test.jac | 1 + jac/jaclang/settings.py | 3 + 7 files changed, 130 insertions(+), 18 deletions(-) create mode 100644 jac-splice-orc/.gitignore create mode 100644 jac-splice-orc/plugin/__init__.py create mode 100644 jac-splice-orc/plugin/splice_plugin.py create mode 100644 jac-splice-orc/test.jac diff --git a/jac-splice-orc/.gitignore b/jac-splice-orc/.gitignore new file mode 100644 index 0000000000..726c7f2249 --- /dev/null +++ b/jac-splice-orc/.gitignore @@ -0,0 +1,3 @@ +*.egg-info +__jac_gen__ +build \ No newline at end of file diff --git a/jac-splice-orc/managers/proxy_manager.py b/jac-splice-orc/managers/proxy_manager.py index f3a87e07ea..527732b272 100644 --- a/jac-splice-orc/managers/proxy_manager.py +++ b/jac-splice-orc/managers/proxy_manager.py @@ -60,21 +60,21 @@ def method(*args, **kwargs): return ModuleRemoteProxy(module_name, self.pod_manager) -module_config = { - "lib_mem_size_req": "150MB", - "lib_cpu_req": "600m", - "dependency": ["numpy", "mkl"], - "load_type": "remote", -} - -# Example usage -if __name__ == "__main__": - pod_manager_url = ( - "http://smartimport.apps.bcstechnology.com.au" # Pod Manager service URL - ) - proxy = ModuleProxy(pod_manager_url) - numpy_proxy = proxy.get_module_proxy("numpy", module_config) - - # Call methods of the numpy module remotely - result = numpy_proxy.array(1, 2, 3) - print(f"Result: {result}") +# module_config = { +# "lib_mem_size_req": "150MB", +# "lib_cpu_req": "600m", +# "dependency": ["numpy", "mkl"], +# "load_type": "remote", +# } + +# # Example usage +# if __name__ == "__main__": +# pod_manager_url = ( +# "http://smartimport.apps.bcstechnology.com.au" # Pod Manager service URL +# ) +# proxy = ModuleProxy(pod_manager_url) +# numpy_proxy = proxy.get_module_proxy("numpy", module_config) + +# # Call methods of the numpy module remotely +# result = numpy_proxy.array(1, 2, 3) +# print(f"Result: {result}") diff --git a/jac-splice-orc/plugin/__init__.py b/jac-splice-orc/plugin/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/jac-splice-orc/plugin/splice_plugin.py b/jac-splice-orc/plugin/splice_plugin.py new file mode 100644 index 0000000000..75c32d1cde --- /dev/null +++ b/jac-splice-orc/plugin/splice_plugin.py @@ -0,0 +1,81 @@ +import types +from typing import Optional, Union + +from jaclang.plugin.default import hookimpl +from jaclang.runtimelib.importer import ImportPathSpec, JacImporter, PythonImporter +from jaclang.runtimelib.machine import JacMachine, JacProgram +from jaclang.settings import settings +from managers.proxy_manager import ModuleProxy +import pluggy + +hookimpl = pluggy.HookimplMarker("jac") + + +class SpliceOrcPlugin: + @staticmethod + @hookimpl + def jac_import( + target: str, + base_path: str, + absorb: bool, + cachable: bool, + mdl_alias: Optional[str], + override_name: Optional[str], + lng: Optional[str], + items: Optional[dict[str, Union[str, Optional[str]]]], + reload_module: Optional[bool], + ) -> tuple[types.ModuleType, ...]: + """Core Import Process with Kubernetes Pod Integration.""" + print("Importing %s" % target) + # Check if the target module is configured for remote execution + if ( + target in settings.module_config + and settings.module_config[target]["load_type"] == "remote" + ): + # Handle remote import using ProxyManager + print(f"Loading module '{target}' remotely via Kubernetes.") + + # Initialize the ProxyManager for Kubernetes pod management + proxy = ModuleProxy(settings.pod_manager_url) + + # This will handle pod creation, module loading, and execution + remote_module_proxy = proxy.get_module_proxy( + module_name=target, module_config=settings.module_config[target] + ) + + # Return the proxy for remote execution of methods + return (remote_module_proxy,) + + # Default import process if not a remote module + spec = ImportPathSpec( + target, + base_path, + absorb, + cachable, + mdl_alias, + override_name, + lng, + items, + ) + + jac_machine = JacMachine.get(base_path) + if not jac_machine.jac_program: + jac_machine.attach_program(JacProgram(mod_bundle=None, bytecode=None)) + + if lng == "py": + # Import using PythonImporter if it's a Python module + import_result = PythonImporter(JacMachine.get()).run_import(spec) + else: + # Otherwise, import using JacImporter + import_result = JacImporter(JacMachine.get()).run_import( + spec, reload_module + ) + + return ( + (import_result.ret_mod,) + if absorb or not items + else tuple(import_result.ret_items) + ) + + +plugin = SpliceOrcPlugin() diff --git a/jac-splice-orc/setup.py b/jac-splice-orc/setup.py index e69de29bb2..44d5a96e6d 100644 --- a/jac-splice-orc/setup.py +++ b/jac-splice-orc/setup.py @@ -0,0 +1,24 @@ +from setuptools import setup, find_packages + +setup( + name="jac-splice-orc", + version="0.1.0", + description="JAC Splice-Orchestrator: Kubernetes-based dynamic remote module management for JacLang", + author="Jason Mars", + author_email="jason@jaseci.org", + packages=find_packages(), + install_requires=[ + "fastapi", + "uvicorn", + "grpcio", + "grpcio-tools", + "kubernetes", + "pydantic", + ], + entry_points={ + "jaclang.plugins": [ + "splice_orc = plugin.splice_plugin", + ], + }, + python_requires=">=3.11", +) diff --git a/jac-splice-orc/test.jac b/jac-splice-orc/test.jac new file mode 100644 index 0000000000..60e3eb8b33 --- /dev/null +++ b/jac-splice-orc/test.jac @@ -0,0 +1 @@ +import:py numpy; \ No newline at end of file diff --git a/jac/jaclang/settings.py b/jac/jaclang/settings.py index b2360b5670..99628813ca 100644 --- a/jac/jaclang/settings.py +++ b/jac/jaclang/settings.py @@ -30,6 +30,9 @@ class Settings: # module import configuration remote_module_handling: bool = True # Enable/Disable remote module handling modules_to_remote: dict = None # Dictionary specifying module configurations + pod_manager_url: str = ( + "http://smartimport.apps.bcstechnology.com.au" # "localhost:8080" # URL for pod manager + ) # Example module specification: module_config: dict = field(