From 966adef27ab7adb1e7ee2fce3177c311aa56b1a4 Mon Sep 17 00:00:00 2001 From: "Alexie (Boyong) Madolid" Date: Fri, 21 Jul 2023 03:17:12 +0800 Subject: [PATCH] [HOTFIX]: Remote action support *args and *kwargs --- jaseci_core/jaseci/jsorc/live_actions.py | 36 +- jaseci_core/jaseci/jsorc/remote_actions.py | 63 ++- .../jaseci/jsorc/tests/test_actions.py | 459 ++++++++++++++++++ 3 files changed, 545 insertions(+), 13 deletions(-) diff --git a/jaseci_core/jaseci/jsorc/live_actions.py b/jaseci_core/jaseci/jsorc/live_actions.py index 7d25b1191a..7e2fa4bd16 100644 --- a/jaseci_core/jaseci/jsorc/live_actions.py +++ b/jaseci_core/jaseci/jsorc/live_actions.py @@ -11,6 +11,10 @@ import inspect import importlib import gc +import re + +var_args = re.compile(r"^\*[^\*]") +var_kwargs = re.compile(r"^\*\*[^\*]") live_actions = {} # {"act.func": func_obj, ...} live_action_modules = {} # {__module__: ["act.func1", "act.func2", ...], ...} @@ -305,14 +309,32 @@ def gen_remote_func_hook(url, act_name, param_names): def func(*args, **kwargs): params = {} - for i in range(len(param_names)): - if i < len(args): - params[param_names[i]] = args[i] + _args = list(args) + _param_names: list = param_names.copy() + + args_len = len(args) + for idx, name in enumerate(param_names): + if idx >= args_len or not _args: + break + _param_names.remove(name) + if var_args.match(name): + if _args: + params[name] = _args + break else: - params[param_names[i]] = None - for i in kwargs.keys(): - if i in param_names: - params[i] = kwargs[i] + params[name] = _args.pop(0) + + for name in _param_names: + if var_args.match(name): + if _args: + params[name] = _args + elif var_kwargs.match(name): + if kwargs: + params[name] = kwargs + break + elif name in kwargs: + params[name] = kwargs.pop(name) + # Remove any None-valued parameters to use the default value of the action def params = dict([(k, v) for k, v in params.items() if v is not None]) act_url = f"{url.rstrip('/')}/{act_name.split('.')[-1]}" diff --git a/jaseci_core/jaseci/jsorc/remote_actions.py b/jaseci_core/jaseci/jsorc/remote_actions.py index ac095b4eac..6986cd95b0 100644 --- a/jaseci_core/jaseci/jsorc/remote_actions.py +++ b/jaseci_core/jaseci/jsorc/remote_actions.py @@ -9,7 +9,12 @@ import inspect import uvicorn import os +import re +var_args = re.compile(r"^\*[^\*]") +var_kwargs = re.compile(r"^\*\*[^\*]") + +args_kwargs = (2, 4) remote_actions = {} registered_apis = [] registered_endpoints = [] @@ -51,12 +56,15 @@ def action_list(): def gen_api_service(app, func, act_group, aliases, caller_globals): """Helper for jaseci_action decorator""" # Construct list of action apis available - varnames = list(inspect.signature(func).parameters.keys()) act_group = ( [os.path.basename(inspect.getfile(func)).split(".")[0]] if act_group is None else act_group ) + varnames = [] + for param in inspect.signature(func).parameters.values(): + varnames.append(str(param) if param.kind in args_kwargs else param.name) + remote_actions[f"{'.'.join(act_group+[func.__name__])}"] = varnames # Need to get pydatic model for func signature for fastAPI post @@ -64,19 +72,62 @@ def gen_api_service(app, func, act_group, aliases, caller_globals): # Keep only fields present in param list in base model keep_fields = {} - for i in model.__fields__.keys(): - if i in varnames: - keep_fields[i] = model.__fields__[i] + for name in varnames: + if var_args.match(name): + keep_fields[name] = model.__fields__[name[1:]] + keep_fields[name].name = name + keep_fields[name].alias = name + elif var_kwargs.match(name): + keep_fields[name] = model.__fields__[name[2:]] + keep_fields[name].name = name + keep_fields[name].alias = name + else: + field = model.__fields__.get(name) + if field: + keep_fields[name] = field model.__fields__ = keep_fields # Create duplicate funtion for api endpoint and inject in call site globals @app.post(f"/{func.__name__}/") def new_func(params: model = model.construct()): - pl_peek = str(dict(params.__dict__))[:128] + params: dict = params.__dict__ + pl_peek = str(params)[:128] logger.info(str(f"Incoming call to {func.__name__} with {pl_peek}")) start_time = time() - ret = validate_arguments(func)(**(params.__dict__)) + args = [] + kwargs = {} + fp1 = inspect.signature(func).parameters.values() + fp2 = list(fp1) + + # try to process args + for param in fp1: + _param = str(param) if param.kind in args_kwargs else param.name + if _param in params: + fp2.remove(param) + if var_args.match(_param): + for arg in params.get(_param) or []: + args.append(arg) + break + elif var_kwargs.match(_param): + kwargs.update(params.get(_param) or {}) + break + args.append(params.get(_param)) + else: + break + + # try to process kwargs + for param in fp2: + param = str(param) if param.kind in args_kwargs else param.name + if param in params: + if var_kwargs.match(param): + kwargs.update(params.get(param) or {}) + break + kwargs[param] = params.get(param) + else: + break + + ret = validate_arguments(func)(*args, **kwargs) tot_time = time() - start_time logger.info( str( diff --git a/jaseci_core/jaseci/jsorc/tests/test_actions.py b/jaseci_core/jaseci/jsorc/tests/test_actions.py index 722f491061..9c35eab541 100644 --- a/jaseci_core/jaseci/jsorc/tests/test_actions.py +++ b/jaseci_core/jaseci/jsorc/tests/test_actions.py @@ -4,6 +4,26 @@ import jaseci.jsorc.live_actions as jla import jaseci.jsorc.remote_actions as jra from jaseci.jsorc.live_actions import gen_remote_func_hook +from pydantic import BaseModel +from typing import Any, List, Mapping, Optional + + +class RequestBody(BaseModel): + a: str + b: str + c: str + d: Optional[List[Any]] = None + e: str = None + f: str = "" + g: Optional[Mapping[Any, Any]] = None + + +def proc_req_body(route, body: RequestBody): + if "d" in body.__dict__: + body.__dict__["*d"] = body.__dict__.pop("d") + if "g" in body.__dict__: + body.__dict__["**g"] = body.__dict__.pop("g") + return route(body) class JacActionsTests(TestCaseHelper, TestCase): @@ -71,6 +91,445 @@ def test_remote_action_kwargs(self, mock_post): assert kwargs["json"] == payload + @patch("requests.post") + def test_remote_action_with_args_only(self, mock_post): + @jla.jaseci_action(act_group=["test"], allow_remote=True) + def args(a, b, c, *d, e=None, f=""): + return {"a": a, "b": b, "c": c, "*d": d, "e": e, "f": f} + + app = jra.serv_actions() + self.assertEqual( + jra.remote_actions, + {"test.args": ["a", "b", "c", "*d", "e", "f"]}, + ) + + remote_action = gen_remote_func_hook( + url="https://example.com/api/v1/endpoint", + act_name="test.args", + param_names=["a", "b", "c", "*d", "e", "f"], + ) + + ################################################# + # --------------- complete call --------------- # + ################################################# + + remote_action( + "1", # a + "2", # b + "3", # c + "4", # inside "*d": [4] + "5", # inside "*d": [4, 5] + "6", # inside "*d": [4, 5, 6] + e="1", # e + f="2", # f + ) + + self.assertEqual( + { + "a": "1", + "b": "2", + "c": "3", + "*d": ["4", "5", "6"], + "e": "1", + "f": "2", + }, + mock_post.call_args[1]["json"], + ) + + ################################################# + # ---------------- no args call --------------- # + ################################################# + + remote_action( + "1", # a + "2", # b + "3", # c + e="1", # e + f="2", # f + ) + + self.assertEqual( + { + "a": "1", + "b": "2", + "c": "3", + "e": "1", + "f": "2", + }, + mock_post.call_args[1]["json"], + ) + + # --------------------------------------------- action call --------------------------------------------- # + + route = app.routes[-1].endpoint + + ######################################################## + # --------------- complete action call --------------- # + ######################################################## + + self.assertEqual( + { + "a": "1", + "b": "2", + "c": "3", + "*d": ("4", "5", "6"), + "e": "1", + "f": "2", + }, + proc_req_body( + route, + RequestBody( + a="1", + b="2", + c="3", + d=["4", "5", "6"], + e="1", + f="2", + ), + ), + ) + + ######################################################## + # ---------------- no args action call --------------- # + ######################################################## + + self.assertEqual( + { + "a": "1", + "b": "2", + "c": "3", + "*d": (), + "e": "1", + "f": "2", + }, + proc_req_body( + route, + RequestBody( + a="1", + b="2", + c="3", + e="1", + f="2", + ), + ), + ) + + @patch("requests.post") + def test_remote_action_with_kwargs_only(self, mock_post): + @jla.jaseci_action(act_group=["test"], allow_remote=True) + def kwargs(a, b, c, e=None, f="", **g): + return {"a": a, "b": b, "c": c, "e": e, "f": f, "**g": g} + + app = jra.serv_actions() + self.assertEqual( + jra.remote_actions, + {"test.kwargs": ["a", "b", "c", "e", "f", "**g"]}, + ) + + remote_action = gen_remote_func_hook( + url="https://example.com/api/v1/endpoint", + act_name="test.kwargs", + param_names=["a", "b", "c", "e", "f", "**g"], + ) + + ################################################# + # --------------- complete call --------------- # + ################################################# + + remote_action( + "1", # a + "2", # b + "3", # c + e="1", # e + f="2", # f + g="3", # inside "**g": {"g": 3} + h="4", # inside "**g": {"g": 3, "h": 4} + i="5", # inside "**g": {"g": 3, "h": 4, "i": 5} + ) + + self.assertEqual( + { + "a": "1", + "b": "2", + "c": "3", + "e": "1", + "f": "2", + "**g": {"g": "3", "h": "4", "i": "5"}, + }, + mock_post.call_args[1]["json"], + ) + + ################################################# + # --------------- no kwargs call -------------- # + ################################################# + + remote_action( + "1", # a + "2", # b + "3", # c + e="1", # e + f="2", # f + ) + + self.assertEqual( + {"a": "1", "b": "2", "c": "3", "e": "1", "f": "2"}, + mock_post.call_args[1]["json"], + ) + + # --------------------------------------------- action call --------------------------------------------- # + + route = app.routes[-1].endpoint + + ######################################################## + # --------------- complete action call --------------- # + ######################################################## + + self.assertEqual( + { + "a": "1", + "b": "2", + "c": "3", + "e": "1", + "f": "2", + "**g": {"g": "3", "h": "4", "i": "5"}, + }, + proc_req_body( + route, + RequestBody( + a="1", + b="2", + c="3", + e="1", + f="2", + g={"g": "3", "h": "4", "i": "5"}, + ), + ), + ) + + ######################################################## + # --------------- no kwargs action call -------------- # + ######################################################## + + self.assertEqual( + { + "a": "1", + "b": "2", + "c": "3", + "e": "1", + "f": "2", + "**g": {}, + }, + proc_req_body(route, RequestBody(a="1", b="2", c="3", e="1", f="2")), + ) + + @patch("requests.post") + def test_remote_action_with_args_kwargs(self, mock_post): + @jla.jaseci_action(act_group=["test"], allow_remote=True) + def args_kwargs(a, b, c, *d, e=None, f="", **g): + return {"a": a, "b": b, "c": c, "*d": d, "e": e, "f": f, "**g": g} + + app = jra.serv_actions() + self.assertEqual( + jra.remote_actions, + {"test.args_kwargs": ["a", "b", "c", "*d", "e", "f", "**g"]}, + ) + + remote_action = gen_remote_func_hook( + url="https://example.com/api/v1/endpoint", + act_name="test.args_kwargs", + param_names=["a", "b", "c", "*d", "e", "f", "**g"], + ) + + ################################################# + # --------------- complete call --------------- # + ################################################# + + remote_action( + "1", # a + "2", # b + "3", # c + "4", # inside "*d": [4] + "5", # inside "*d": [4, 5] + "6", # inside "*d": [4, 5, 6] + e="1", # e + f="2", # f + g="3", # inside "**g": {"g": 3} + h="4", # inside "**g": {"g": 3, "h": 4} + i="5", # inside "**g": {"g": 3, "h": 4, "i": 5} + ) + + self.assertEqual( + { + "a": "1", + "b": "2", + "c": "3", + "*d": ["4", "5", "6"], + "e": "1", + "f": "2", + "**g": {"g": "3", "h": "4", "i": "5"}, + }, + mock_post.call_args[1]["json"], + ) + + ################################################# + # --------------- no kwargs call -------------- # + ################################################# + + remote_action( + "1", # a + "2", # b + "3", # c + "4", # inside "*d": [4] + "5", # inside "*d": [4, 5] + "6", # inside "*d": [4, 5, 6] + e="1", # e + f="2", # f + ) + + self.assertEqual( + {"a": "1", "b": "2", "c": "3", "*d": ["4", "5", "6"], "e": "1", "f": "2"}, + mock_post.call_args[1]["json"], + ) + + ################################################# + # ---------------- no args call --------------- # + ################################################# + + remote_action( + "1", # a + "2", # b + "3", # c + e="1", # e + f="2", # f + g="3", # inside "**g": {"g": 3} + h="4", # inside "**g": {"g": 3, "h": 4} + i="5", # inside "**g": {"g": 3, "h": 4, "i": 5} + ) + + self.assertEqual( + { + "a": "1", + "b": "2", + "c": "3", + "e": "1", + "f": "2", + "**g": {"g": "3", "h": "4", "i": "5"}, + }, + mock_post.call_args[1]["json"], + ) + + ################################################# + # ---------- no args and kwargs call ---------- # + ################################################# + + remote_action( + "1", # a + "2", # b + "3", # c + e="1", # e + f="2", # f + ) + + self.assertEqual( + {"a": "1", "b": "2", "c": "3", "e": "1", "f": "2"}, + mock_post.call_args[1]["json"], + ) + + # --------------------------------------------- action call --------------------------------------------- # + + route = app.routes[-1].endpoint + + ######################################################## + # --------------- complete action call --------------- # + ######################################################## + + self.assertEqual( + { + "a": "1", + "b": "2", + "c": "3", + "*d": ("4", "5", "6"), + "e": "1", + "f": "2", + "**g": {"g": "3", "h": "4", "i": "5"}, + }, + proc_req_body( + route, + RequestBody( + a="1", + b="2", + c="3", + d=["4", "5", "6"], + e="1", + f="2", + g={"g": "3", "h": "4", "i": "5"}, + ), + ), + ) + + ######################################################## + # --------------- no kwargs action call -------------- # + ######################################################## + + self.assertEqual( + { + "a": "1", + "b": "2", + "c": "3", + "*d": ("4", "5", "6"), + "e": "1", + "f": "2", + "**g": {}, + }, + proc_req_body( + route, RequestBody(a="1", b="2", c="3", d=["4", "5", "6"], e="1", f="2") + ), + ) + + ######################################################## + # ---------------- no args action call --------------- # + ######################################################## + + self.assertEqual( + { + "a": "1", + "b": "2", + "c": "3", + "*d": (), + "e": "1", + "f": "2", + "**g": {"g": "3", "h": "4", "i": "5"}, + }, + proc_req_body( + route, + RequestBody( + a="1", + b="2", + c="3", + e="1", + f="2", + g={"g": "3", "h": "4", "i": "5"}, + ), + ), + ) + + ######################################################## + # ---------- no args and kwargs action call ---------- # + ######################################################## + + self.assertEqual( + { + "a": "1", + "b": "2", + "c": "3", + "*d": (), + "e": "1", + "f": "2", + "**g": {}, + }, + proc_req_body(route, RequestBody(a="1", b="2", c="3", e="1", f="2")), + ) + def test_setup_decorated_as_startup(self): @jla.jaseci_action(act_group=["ex"], allow_remote=True) def setup(param: str = ""):