From 288d4cc2511e49bbf1c05373a8b01e1d73c154c1 Mon Sep 17 00:00:00 2001 From: David Erb Date: Thu, 1 Jun 2023 13:38:03 +0100 Subject: [PATCH 1/8] new docs structure from template --- docs/explanations/conventions.rst | 4 ++++ docs/explanations/docs_structure.rst | 4 ++++ docs/explanations/index.rst | 12 ++++++++++++ docs/explanations/todo.rst | 7 +++++++ docs/how-to/developing.rst | 4 ++++ docs/how-to/devops.rst | 4 ++++ docs/how-to/documenting.rst | 4 ++++ docs/how-to/index.rst | 16 ++++++++++++++++ docs/how-to/installing.rst | 6 ++++++ docs/how-to/testing.rst | 4 ++++ docs/index.rst | 13 ++++++++++--- docs/{api => reference}/classes.rst | 0 docs/{api => reference}/command_line.rst | 0 docs/{api => reference}/index.rst | 0 docs/{api => reference}/modules.rst | 0 docs/tutorials/index.rst | 11 +++++++++++ docs/tutorials/tbd.rst | 4 ++++ 17 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 docs/explanations/conventions.rst create mode 100644 docs/explanations/docs_structure.rst create mode 100644 docs/explanations/index.rst create mode 100644 docs/explanations/todo.rst create mode 100644 docs/how-to/developing.rst create mode 100644 docs/how-to/devops.rst create mode 100644 docs/how-to/documenting.rst create mode 100644 docs/how-to/index.rst create mode 100644 docs/how-to/installing.rst create mode 100644 docs/how-to/testing.rst rename docs/{api => reference}/classes.rst (100%) rename docs/{api => reference}/command_line.rst (100%) rename docs/{api => reference}/index.rst (100%) rename docs/{api => reference}/modules.rst (100%) create mode 100644 docs/tutorials/index.rst create mode 100644 docs/tutorials/tbd.rst diff --git a/docs/explanations/conventions.rst b/docs/explanations/conventions.rst new file mode 100644 index 0000000..c1b5c81 --- /dev/null +++ b/docs/explanations/conventions.rst @@ -0,0 +1,4 @@ +.. + Use the file automatically generated by dae-devops. + +.. include:: ../../.dae-devops/docs/conventions.rst \ No newline at end of file diff --git a/docs/explanations/docs_structure.rst b/docs/explanations/docs_structure.rst new file mode 100644 index 0000000..18c92dc --- /dev/null +++ b/docs/explanations/docs_structure.rst @@ -0,0 +1,4 @@ +.. + Use the file automatically generated by dae-devops. + +.. include:: ../../.dae-devops/docs/docs_structure.rst \ No newline at end of file diff --git a/docs/explanations/index.rst b/docs/explanations/index.rst new file mode 100644 index 0000000..0e5e221 --- /dev/null +++ b/docs/explanations/index.rst @@ -0,0 +1,12 @@ +:orphan: + +Explanations +============ + +Explanation, or discussions, clarify and illuminate a particular topic. They broaden the documentation's coverage of a topic. + +.. toctree:: + + docs_structure + conventions + todo \ No newline at end of file diff --git a/docs/explanations/todo.rst b/docs/explanations/todo.rst new file mode 100644 index 0000000..15ef74f --- /dev/null +++ b/docs/explanations/todo.rst @@ -0,0 +1,7 @@ +TODO +======================================================================= + +- Remove matplotlib from imports done by mib2hdfConvert package to decrease load time and debug volume. +- Make sure we can collect and process mib files which are not in a material subdirectory. +- Add unit tests for the mib_convert task. + \ No newline at end of file diff --git a/docs/how-to/developing.rst b/docs/how-to/developing.rst new file mode 100644 index 0000000..5964239 --- /dev/null +++ b/docs/how-to/developing.rst @@ -0,0 +1,4 @@ +.. + Use the file automatically generated by dae-devops. + +.. include:: ../../.dae-devops/docs/developing.rst \ No newline at end of file diff --git a/docs/how-to/devops.rst b/docs/how-to/devops.rst new file mode 100644 index 0000000..334cc58 --- /dev/null +++ b/docs/how-to/devops.rst @@ -0,0 +1,4 @@ +.. + Use the file automatically generated by dae-devops. + +.. include:: ../../.dae-devops/docs/devops.rst \ No newline at end of file diff --git a/docs/how-to/documenting.rst b/docs/how-to/documenting.rst new file mode 100644 index 0000000..cdb5964 --- /dev/null +++ b/docs/how-to/documenting.rst @@ -0,0 +1,4 @@ +.. + Use the file automatically generated by dae-devops. + +.. include:: ../../.dae-devops/docs/documenting.rst \ No newline at end of file diff --git a/docs/how-to/index.rst b/docs/how-to/index.rst new file mode 100644 index 0000000..21b92fc --- /dev/null +++ b/docs/how-to/index.rst @@ -0,0 +1,16 @@ +:orphan: + +How-to Guides +============= + +How-to guides take the reader through the steps required to solve a real-world problem. +Practical step-by-step guides for the more experienced user. + +.. toctree:: + + installing + developing + testing + documenting + devops + \ No newline at end of file diff --git a/docs/how-to/installing.rst b/docs/how-to/installing.rst new file mode 100644 index 0000000..3a0f3fc --- /dev/null +++ b/docs/how-to/installing.rst @@ -0,0 +1,6 @@ +.. + Use the file automatically generated by dae-devops. + +.. include:: ../../.dae-devops/docs/installing.rst + +Since this package is just a library, the command line doesn't do anything else besides print the version. diff --git a/docs/how-to/testing.rst b/docs/how-to/testing.rst new file mode 100644 index 0000000..2cd284b --- /dev/null +++ b/docs/how-to/testing.rst @@ -0,0 +1,4 @@ +.. + Use the file automatically generated by dae-devops. + +.. include:: ../../.dae-devops/docs/testing.rst \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index e328032..be06b53 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,7 +1,14 @@ +dls-servbase +======================================================================= +Simple HTTP service for database operations. + +Agnostic of type of underlying sql client, provides single-writer access to native database and has a predefined database schema for cookies. .. toctree:: - :caption: API-caption - :hidden: + :titlesonly: - api/index + tutorials/index + how-to/index + explanations/index + reference/index diff --git a/docs/api/classes.rst b/docs/reference/classes.rst similarity index 100% rename from docs/api/classes.rst rename to docs/reference/classes.rst diff --git a/docs/api/command_line.rst b/docs/reference/command_line.rst similarity index 100% rename from docs/api/command_line.rst rename to docs/reference/command_line.rst diff --git a/docs/api/index.rst b/docs/reference/index.rst similarity index 100% rename from docs/api/index.rst rename to docs/reference/index.rst diff --git a/docs/api/modules.rst b/docs/reference/modules.rst similarity index 100% rename from docs/api/modules.rst rename to docs/reference/modules.rst diff --git a/docs/tutorials/index.rst b/docs/tutorials/index.rst new file mode 100644 index 0000000..2af6711 --- /dev/null +++ b/docs/tutorials/index.rst @@ -0,0 +1,11 @@ +:orphan: + +Tutorials +========= + +Tutorials are lessons that take the reader by the hand through a series of steps to complete a project of some kind. +They are what your project needs in order to show a beginner that they can achieve something with it. + +.. toctree:: + + tbd diff --git a/docs/tutorials/tbd.rst b/docs/tutorials/tbd.rst new file mode 100644 index 0000000..14489c4 --- /dev/null +++ b/docs/tutorials/tbd.rst @@ -0,0 +1,4 @@ +TBD Tutorial +======================================================================= + +TBD \ No newline at end of file From 8b767676948057d3c43000e45186205440a000b8 Mon Sep 17 00:00:00 2001 From: David Erb Date: Fri, 2 Jun 2023 07:27:15 +0100 Subject: [PATCH 2/8] removing contexts --- good_config1.yaml | 0 src/dls_servbase_api/context_base.py | 35 +++ .../guis}/__init__.py | 0 src/dls_servbase_api/guis/aiohttp.py | 62 ++++ src/dls_servbase_api/guis/constants.py | 11 + src/dls_servbase_api/guis/context.py | 43 +++ src/dls_servbase_api/guis/guis.py | 68 +++++ src/dls_servbase_lib/contexts/base.py | 52 ---- src/dls_servbase_lib/contexts/classic.py | 154 ---------- src/dls_servbase_lib/contexts/contexts.py | 57 ---- src/dls_servbase_lib/datafaces/context.py | 10 +- src/dls_servbase_lib/guis/context.py | 10 +- src/dls_servbase_lib/guis/guis.py | 5 +- tests/configurations/mysql.yaml | 2 +- tests/configurations/sqlite.yaml | 2 +- tests/test_gui.py | 285 ++++++++++-------- 16 files changed, 388 insertions(+), 408 deletions(-) delete mode 100644 good_config1.yaml create mode 100644 src/dls_servbase_api/context_base.py rename src/{dls_servbase_lib/contexts => dls_servbase_api/guis}/__init__.py (100%) create mode 100644 src/dls_servbase_api/guis/aiohttp.py create mode 100644 src/dls_servbase_api/guis/constants.py create mode 100644 src/dls_servbase_api/guis/context.py create mode 100644 src/dls_servbase_api/guis/guis.py delete mode 100644 src/dls_servbase_lib/contexts/base.py delete mode 100644 src/dls_servbase_lib/contexts/classic.py delete mode 100644 src/dls_servbase_lib/contexts/contexts.py diff --git a/good_config1.yaml b/good_config1.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/src/dls_servbase_api/context_base.py b/src/dls_servbase_api/context_base.py new file mode 100644 index 0000000..b536781 --- /dev/null +++ b/src/dls_servbase_api/context_base.py @@ -0,0 +1,35 @@ +import logging + +logger = logging.getLogger(__name__) + + +class ContextBase: + """ """ + + # ---------------------------------------------------------------------------------------- + def __init__(self, specification): + self.__specification = specification + self.__interface = None + + # ---------------------------------------------------------------------------------------- + def get_interface(self): + return self.__interface + + def set_interface(self, interface): + self.__interface = interface + + interface = property(get_interface, set_interface) + + # ---------------------------------------------------------------------------------------- + async def __aenter__(self): + """ """ + + await self.aenter() + + return self.interface + + # ---------------------------------------------------------------------------------------- + async def __aexit__(self, type, value, traceback): + """ """ + + await self.aexit() diff --git a/src/dls_servbase_lib/contexts/__init__.py b/src/dls_servbase_api/guis/__init__.py similarity index 100% rename from src/dls_servbase_lib/contexts/__init__.py rename to src/dls_servbase_api/guis/__init__.py diff --git a/src/dls_servbase_api/guis/aiohttp.py b/src/dls_servbase_api/guis/aiohttp.py new file mode 100644 index 0000000..413f331 --- /dev/null +++ b/src/dls_servbase_api/guis/aiohttp.py @@ -0,0 +1,62 @@ +import logging + +# Class for an aiohttp client. +from dls_servbase_api.aiohttp_client import AiohttpClient + +# Dataface protocolj things. +from dls_servbase_api.guis.constants import Commands, Keywords + +logger = logging.getLogger(__name__) + + +# ------------------------------------------------------------------------------------------ +class Aiohttp: + """ + Object implementing client side API for talking to the dls_servbase_gui server. + Please see doctopic [A01]. + """ + + # ---------------------------------------------------------------------------------------- + def __init__(self, specification=None): + self.__specification = specification + + self.__aiohttp_client = AiohttpClient( + specification["type_specific_tbd"]["aiohttp_specification"], + ) + + # ---------------------------------------------------------------------------------------- + def specification(self): + return self.__specification + + # ---------------------------------------------------------------------------------------- + async def report_health(self): + """""" + return await self.__send_protocolj("report_health") + + # ---------------------------------------------------------------------------------------- + async def __send_protocolj(self, function, *args, **kwargs): + """""" + + return await self.__aiohttp_client.client_protocolj( + { + Keywords.COMMAND: Commands.EXECUTE, + Keywords.PAYLOAD: { + "function": function, + "args": args, + "kwargs": kwargs, + }, + }, + ) + + # ---------------------------------------------------------------------------------------- + async def close_client_session(self): + """""" + + if self.__aiohttp_client is not None: + await self.__aiohttp_client.close_client_session() + + # ---------------------------------------------------------------------------------------- + async def client_report_health(self): + """""" + + return await self.__aiohttp_client.client_report_health() diff --git a/src/dls_servbase_api/guis/constants.py b/src/dls_servbase_api/guis/constants.py new file mode 100644 index 0000000..d99e207 --- /dev/null +++ b/src/dls_servbase_api/guis/constants.py @@ -0,0 +1,11 @@ +class Keywords: + COMMAND = "dls_servbase_guis::keywords::command" + PAYLOAD = "dls_servbase_guis::keywords::payload" + + +class Commands: + EXECUTE = "dls_servbase_guis::commands::execute" + + +class Types: + AIOHTTP = "dls_servbase_lib.guis.aiohttp" diff --git a/src/dls_servbase_api/guis/context.py b/src/dls_servbase_api/guis/context.py new file mode 100644 index 0000000..12d97fb --- /dev/null +++ b/src/dls_servbase_api/guis/context.py @@ -0,0 +1,43 @@ +import logging + +# Base class. +from dls_servbase_api.context_base import ContextBase + +# Things created in the context. +from dls_servbase_api.guis.guis import Guis, dls_servbase_guis_set_default + +logger = logging.getLogger(__name__) + + +class Context(ContextBase): + """ + Client context for a dls_servbase_gui object. + On entering, it creates the object according to the specification (a dict). + On exiting, it closes client connection. + + The aenter and aexit methods are exposed for use by an enclosing context. + """ + + # ---------------------------------------------------------------------------------------- + def __init__(self, specification): + self.__specification = specification + + # ---------------------------------------------------------------------------------------- + async def aenter(self): + """ """ + + # Build the object according to the specification. + self.interface = Guis().build_object(self.__specification) + + # If there is more than one gui, the last one defined will be the default. + dls_servbase_guis_set_default(self.interface) + + # ---------------------------------------------------------------------------------------- + async def aexit(self): + """ """ + + if self.interface is not None: + await self.interface.close_client_session() + + # Clear the global variable. Important between pytests. + dls_servbase_guis_set_default(None) diff --git a/src/dls_servbase_api/guis/guis.py b/src/dls_servbase_api/guis/guis.py new file mode 100644 index 0000000..003a84a --- /dev/null +++ b/src/dls_servbase_api/guis/guis.py @@ -0,0 +1,68 @@ +# Use standard logging in this module. +import logging + +# Exceptions. +from dls_utilpack.exceptions import NotFound + +# Class managing list of things. +from dls_utilpack.things import Things + +# Types. +from dls_servbase_api.guis.constants import Types + +logger = logging.getLogger(__name__) + +# ----------------------------------------------------------------------------------------- +__default_dls_servbase_gui = None + + +def dls_servbase_guis_set_default(dls_servbase_gui): + global __default_dls_servbase_gui + __default_dls_servbase_gui = dls_servbase_gui + + +def dls_servbase_guis_get_default(): + global __default_dls_servbase_gui + if __default_dls_servbase_gui is None: + raise RuntimeError("dls_servbase_guis_get_default instance is None") + return __default_dls_servbase_gui + + +# ----------------------------------------------------------------------------------------- + + +class Guis(Things): + """ + List of available dls_servbase_guis. + """ + + # ---------------------------------------------------------------------------------------- + def __init__(self, name=None): + Things.__init__(self, name) + + # ---------------------------------------------------------------------------------------- + def build_object(self, specification): + """""" + + dls_servbase_gui_class = self.lookup_class(specification["type"]) + + try: + dls_servbase_gui_object = dls_servbase_gui_class(specification) + except Exception as exception: + raise RuntimeError( + "unable to build dls_servbase gui object for type %s" + % (dls_servbase_gui_class) + ) from exception + + return dls_servbase_gui_object + + # ---------------------------------------------------------------------------------------- + def lookup_class(self, class_type): + """""" + + if class_type == Types.AIOHTTP: + from dls_servbase_api.guis.aiohttp import Aiohttp + + return Aiohttp + + raise NotFound(f"unable to get dls_servbase gui class for type {class_type}") diff --git a/src/dls_servbase_lib/contexts/base.py b/src/dls_servbase_lib/contexts/base.py deleted file mode 100644 index 025b544..0000000 --- a/src/dls_servbase_lib/contexts/base.py +++ /dev/null @@ -1,52 +0,0 @@ -import logging - -# Utilities. -from dls_utilpack.callsign import callsign - -# Base class for a Thing which has a name and traits. -from dls_utilpack.thing import Thing - -logger = logging.getLogger(__name__) - - -class Base(Thing): - """ """ - - # ---------------------------------------------------------------------------------------- - def __init__(self, thing_type, specification=None, predefined_uuid=None): - Thing.__init__(self, thing_type, specification, predefined_uuid=predefined_uuid) - - # Reference to object which is a server, such as BaseAiohttp. - self.server = None - - self.context_specification = self.specification().get("context", {}) - - # ---------------------------------------------------------------------------------------- - async def is_process_started(self): - """""" - - if self.server is None: - raise RuntimeError(f"{callsign(self)} a process has not been defined") - - return await self.server.is_process_started() - - # ---------------------------------------------------------------------------------------- - async def is_process_alive(self): - """""" - - if self.server is None: - raise RuntimeError(f"{callsign(self)} a process has not been defined") - - return await self.server.is_process_alive() - - # ---------------------------------------------------------------------------------------- - async def __aenter__(self): - """ """ - - await self.aenter() - - # ---------------------------------------------------------------------------------------- - async def __aexit__(self, type, value, traceback): - """ """ - - await self.aexit() diff --git a/src/dls_servbase_lib/contexts/classic.py b/src/dls_servbase_lib/contexts/classic.py deleted file mode 100644 index 09da225..0000000 --- a/src/dls_servbase_lib/contexts/classic.py +++ /dev/null @@ -1,154 +0,0 @@ -import logging - -from dls_utilpack.callsign import callsign - -# Utilities. -from dls_utilpack.explain import explain - -# Base class which maps flask requests to methods. -from dls_servbase_lib.contexts.base import Base - -# Contexts. -from dls_servbase_lib.datafaces.context import Context as DatafaceContext -from dls_servbase_lib.guis.context import Context as GuiContext - -logger = logging.getLogger(__name__) - - -thing_type = "dls_servbase_lib.dls_servbase_contexts.classic" - - -class Classic(Base): - """ - Object representing an event dls_servbase_dataface connection. - """ - - # ---------------------------------------------------------------------------------------- - def __init__(self, specification): - Base.__init__(self, thing_type, specification) - - self.__dataface = None - self.__collector = None - self.__gui = None - - # ---------------------------------------------------------------------------------------- - async def __dead_or_alive(self, server, dead, alive): - - if server is not None: - # A server was defined for this context? - if await server.is_process_started(): - if await server.is_process_alive(): - alive.append(server) - else: - dead.append(server) - - # ---------------------------------------------------------------------------------------- - async def __dead_or_alive_all(self): - """ - Return two lists, one for dead and one for alive processes. - TODO: Parallelize context process alive/dead checking. - """ - - dead = [] - alive = [] - - await self.__dead_or_alive(self.__dataface, dead, alive) - await self.__dead_or_alive(self.__collector, dead, alive) - await self.__dead_or_alive(self.__gui, dead, alive) - - return dead, alive - - # ---------------------------------------------------------------------------------------- - async def is_any_process_alive(self): - """ - Check all configured processes, return if any alive. - """ - dead, alive = await self.__dead_or_alive_all() - - logger.debug(f"{len(dead)} processes are dead, {len(alive)} are alive") - - return len(alive) > 0 - - # ---------------------------------------------------------------------------------------- - async def is_any_process_dead(self): - """ - Check all configured processes, return if any alive. - """ - dead, alive = await self.__dead_or_alive_all() - - return len(dead) > 0 - - # ---------------------------------------------------------------------------------------- - async def __aenter__(self): - """ """ - logger.debug(f"entering {callsign(self)} context") - - try: - - try: - specification = self.specification().get( - "dls_servbase_dataface_specification" - ) - if specification is not None: - logger.debug(f"at entering position {callsign(self)} DATAFACE") - self.__dataface = DatafaceContext(specification) - await self.__dataface.aenter() - except Exception as exception: - raise RuntimeError( - explain(exception, f"creating {callsign(self)} dataface context") - ) - - try: - specification = self.specification().get( - "dls_servbase_gui_specification" - ) - if specification is not None: - logger.debug(f"at entering position {callsign(self)} GUI") - self.__gui = GuiContext(specification) - await self.__gui.aenter() - except Exception as exception: - raise RuntimeError( - explain(exception, f"creating {callsign(self)} gui context") - ) - - except Exception as exception: - await self.aexit() - raise RuntimeError(explain(exception, f"entering {callsign(self)} context")) - - logger.debug(f"entered {callsign(self)} context") - - # ---------------------------------------------------------------------------------------- - async def __aexit__(self, type, value, traceback): - """ """ - - await self.aexit() - - # ---------------------------------------------------------------------------------------- - async def aexit(self): - """ """ - - logger.debug(f"exiting {callsign(self)} context") - - if self.__gui is not None: - logger.debug(f"at exiting position {callsign(self)} GUI") - try: - await self.__gui.aexit() - except Exception as exception: - logger.error( - explain(exception, f"exiting {callsign(self.__gui)} context"), - exc_info=exception, - ) - self.__gui = None - - if self.__dataface is not None: - logger.debug(f"at exiting position {callsign(self)} DATAFACE") - try: - await self.__dataface.aexit() - except Exception as exception: - logger.error( - explain(exception, f"exiting {callsign(self.__dataface)} context"), - exc_info=exception, - ) - self.__dataface = None - - logger.debug(f"exited {callsign(self)} context") diff --git a/src/dls_servbase_lib/contexts/contexts.py b/src/dls_servbase_lib/contexts/contexts.py deleted file mode 100644 index 52b6220..0000000 --- a/src/dls_servbase_lib/contexts/contexts.py +++ /dev/null @@ -1,57 +0,0 @@ -# Use standard logging in this module. -import logging - -import yaml - -# Class managing list of things. -from dls_utilpack.things import Things - -# Exceptions. -from dls_servbase_api.exceptions import NotFound - -logger = logging.getLogger(__name__) - -# ----------------------------------------------------------------------------------------- - - -class Contexts(Things): - """ - Context loader. - """ - - # ---------------------------------------------------------------------------------------- - def __init__(self, name=None): - Things.__init__(self, name) - - # ---------------------------------------------------------------------------------------- - def build_object(self, specification): - """""" - - if not isinstance(specification, dict): - with open(specification, "r") as yaml_stream: - specification = yaml.safe_load(yaml_stream) - - dls_servbase_context_class = self.lookup_class(specification["type"]) - - try: - dls_servbase_context_object = dls_servbase_context_class(specification) - except Exception as exception: - raise RuntimeError( - "unable to build dls_servbase_context object for type %s" - % (dls_servbase_context_class) - ) from exception - - return dls_servbase_context_object - - # ---------------------------------------------------------------------------------------- - def lookup_class(self, class_type): - """""" - - if class_type == "dls_servbase_lib.dls_servbase_contexts.classic": - from dls_servbase_lib.contexts.classic import Classic - - return Classic - - raise NotFound( - "unable to get dls_servbase_context class for type %s" % (class_type) - ) diff --git a/src/dls_servbase_lib/datafaces/context.py b/src/dls_servbase_lib/datafaces/context.py index e606353..1f020da 100644 --- a/src/dls_servbase_lib/datafaces/context.py +++ b/src/dls_servbase_lib/datafaces/context.py @@ -1,8 +1,8 @@ import logging from typing import Dict -# Base class for an asyncio context -from dls_servbase_lib.contexts.base import Base as ContextBase +# Base class for an asyncio server context. +from dls_utilpack.server_context_base import ServerContextBase # Things created in the context. from dls_servbase_lib.datafaces.datafaces import Datafaces @@ -13,7 +13,7 @@ thing_type = "dls_servbase_lib.datafaces.context" -class Context(ContextBase): +class Context(ServerContextBase): """ Asyncio context for a servbase object. On entering, it creates the object according to the specification (a dict). @@ -32,7 +32,7 @@ def __init__(self, specification: Dict): The only key in the specification that relates to the context is "start_as", which can be "coro", "thread", "process" or None. All other keys in the specification relate to creating the servbase object. """ - ContextBase.__init__(self, thing_type, specification) + ServerContextBase.__init__(self, thing_type, specification) # ---------------------------------------------------------------------------------------- async def aenter(self) -> None: @@ -62,7 +62,7 @@ async def aenter(self) -> None: await self.server.activate() # ---------------------------------------------------------------------------------------- - async def aexit(self) -> None: + async def aexit(self, type, value, traceback) -> None: """ Asyncio context exit. diff --git a/src/dls_servbase_lib/guis/context.py b/src/dls_servbase_lib/guis/context.py index ccffb5c..c5a1f50 100644 --- a/src/dls_servbase_lib/guis/context.py +++ b/src/dls_servbase_lib/guis/context.py @@ -1,7 +1,7 @@ import logging -# Base class which maps flask requests to methods. -from dls_servbase_lib.contexts.base import Base as ContextBase +# Base class for an asyncio server context. +from dls_utilpack.server_context_base import ServerContextBase # Things created in the context. from dls_servbase_lib.guis.guis import Guis, dls_servbase_guis_set_default @@ -12,14 +12,14 @@ thing_type = "dls_servbase_lib.dls_servbase_guis.context" -class Context(ContextBase): +class Context(ServerContextBase): """ Object representing an event dls_servbase_dataface connection. """ # ---------------------------------------------------------------------------------------- def __init__(self, specification): - ContextBase.__init__(self, thing_type, specification) + ServerContextBase.__init__(self, thing_type, specification) # ---------------------------------------------------------------------------------------- async def aenter(self): @@ -40,7 +40,7 @@ async def aenter(self): await self.server.start_process() # ---------------------------------------------------------------------------------------- - async def aexit(self): + async def aexit(self, type, value, traceback): """ """ if self.server is not None: diff --git a/src/dls_servbase_lib/guis/guis.py b/src/dls_servbase_lib/guis/guis.py index 78af02c..ff27236 100644 --- a/src/dls_servbase_lib/guis/guis.py +++ b/src/dls_servbase_lib/guis/guis.py @@ -7,6 +7,9 @@ # Exceptions. from dls_servbase_api.exceptions import NotFound +# Types. +from dls_servbase_api.guis.constants import Types + logger = logging.getLogger(__name__) # ----------------------------------------------------------------------------------------- @@ -62,7 +65,7 @@ def build_object(self, specification): def lookup_class(self, class_type): """""" - if class_type == "dls_servbase_lib.dls_servbase_guis.aiohttp": + if class_type == Types.AIOHTTP: from dls_servbase_lib.guis.aiohttp import Aiohttp return Aiohttp diff --git a/tests/configurations/mysql.yaml b/tests/configurations/mysql.yaml index 1a1a9b4..99cfce1 100644 --- a/tests/configurations/mysql.yaml +++ b/tests/configurations/mysql.yaml @@ -53,7 +53,7 @@ dls_servbase_dataface_specification: &DLS_SERVBASE_DATAFACE_SPECIFICATION # The dls_servbase_gui specification. dls_servbase_gui_specification: - type: "dls_servbase_lib.dls_servbase_guis.aiohttp" + type: "dls_servbase_lib.guis.aiohttp" type_specific_tbd: # The remote dls_servbase_gui server access. aiohttp_specification: diff --git a/tests/configurations/sqlite.yaml b/tests/configurations/sqlite.yaml index b60bbd5..f197135 100644 --- a/tests/configurations/sqlite.yaml +++ b/tests/configurations/sqlite.yaml @@ -47,7 +47,7 @@ dls_servbase_dataface_specification: &DLS_SERVBASE_DATAFACE_SPECIFICATION # The dls_servbase_gui specification. dls_servbase_gui_specification: - type: "dls_servbase_lib.dls_servbase_guis.aiohttp" + type: "dls_servbase_lib.guis.aiohttp" type_specific_tbd: # The remote dls_servbase_gui server access. aiohttp_specification: diff --git a/tests/test_gui.py b/tests/test_gui.py index c360f2a..00ce6c2 100644 --- a/tests/test_gui.py +++ b/tests/test_gui.py @@ -6,12 +6,18 @@ # API constants. from dls_servbase_api.constants import Keywords as ProtocoljKeywords -# Context creator. -from dls_servbase_lib.contexts.contexts import Contexts +# Client context. +from dls_servbase_api.guis.context import Context as GuiClientContext +from dls_servbase_api.guis.guis import dls_servbase_guis_get_default + +# Dataface server context. +from dls_servbase_lib.datafaces.context import Context as DlsServbaseDatafaceContext + +# GUI constants. from dls_servbase_lib.guis.constants import Commands, Cookies, Keywords -# Object managing gui -from dls_servbase_lib.guis.guis import dls_servbase_guis_get_default +# Server context. +from dls_servbase_lib.guis.context import Context as GuiServerContext # Base class for the tester. from tests.base_context_tester import BaseContextTester @@ -46,135 +52,150 @@ async def _main_coroutine(self, constants, output_directory): # Make a multiconf and load the context configuration file. multiconf = self.get_multiconf() - context_configuration = await multiconf.load() + multiconf_dict = await multiconf.load() - dls_servbase_gui_specification = context_configuration[ - "dls_servbase_gui_specification" - ] - type_specific_tbd = dls_servbase_gui_specification["type_specific_tbd"] + dataface_specification = multiconf_dict["dls_servbase_dataface_specification"] + dataface_context = DlsServbaseDatafaceContext(dataface_specification) + + gui_specification = multiconf_dict["dls_servbase_gui_specification"] + type_specific_tbd = gui_specification["type_specific_tbd"] aiohttp_specification = type_specific_tbd["aiohttp_specification"] aiohttp_specification["search_paths"] = [output_directory] - # Make a context instance which can start a service and stop it at the end. - dls_servbase_context = Contexts().build_object(context_configuration) - - async with dls_servbase_context: - - # -------------------------------------------------------------------- - # Use protocolj to fetch a request from the dataface. - # Simulates what javascript would do by ajax. - - # Load tabs, of which there are none at the start. - # This establishes the cookie though. - load_tabs_request = { - Keywords.COMMAND: Commands.LOAD_TABS, - ProtocoljKeywords.ENABLE_COOKIES: [Cookies.TABS_MANAGER], - } - - logger.debug("---------------------- making request 1 --------------------") - response = await dls_servbase_guis_get_default().client_protocolj( - load_tabs_request, cookies={} - ) - - # The response is json, with the last saved tab_id, which is None at first. - assert Keywords.TAB_ID in response - assert response[Keywords.TAB_ID] is None - - # We should also have cookies back. - assert "__cookies" in response - cookies = response["__cookies"] - assert Cookies.TABS_MANAGER in cookies - - # Use the tabs manager cookie value in the next requests. - cookie_uuid = cookies[Cookies.TABS_MANAGER].value - - # -------------------------------------------------------------------- - # Select a tab. - # The response is json, but nothing really to see in it. - - select_tab_request = { - Keywords.COMMAND: Commands.SELECT_TAB, - ProtocoljKeywords.ENABLE_COOKIES: [Cookies.TABS_MANAGER], - Keywords.TAB_ID: "123", - } - - logger.debug("---------------------- making request 2 --------------------") - response = await dls_servbase_guis_get_default().client_protocolj( - select_tab_request, cookies={Cookies.TABS_MANAGER: cookie_uuid} - ) - - # -------------------------------------------------------------------- - # Load tabs again, this time we should get the saved tab_id. - - logger.debug("---------------------- making request 3 --------------------") - # Put a deliberately funky cookie string into the header. - raw_cookie_header = ( - 'BadCookie={"something"}; ' + f"{Cookies.TABS_MANAGER} = {cookie_uuid};" - ) - response = await dls_servbase_guis_get_default().client_protocolj( - load_tabs_request, - headers={"Cookie": raw_cookie_header}, - ) - - logger.debug(describe("second load_tabs response", response)) - assert Keywords.TAB_ID in response - assert response[Keywords.TAB_ID] == "123" - - # -------------------------------------------------------------------- - # Update a tab. - # The response is json, but nothing really to see in it. - - select_tab_request = { - Keywords.COMMAND: Commands.SELECT_TAB, - ProtocoljKeywords.ENABLE_COOKIES: [Cookies.TABS_MANAGER], - Keywords.TAB_ID: "456", - } - - logger.debug("---------------------- making request 4 --------------------") - response = await dls_servbase_guis_get_default().client_protocolj( - select_tab_request, - cookies={Cookies.TABS_MANAGER: cookie_uuid}, - ) - - # -------------------------------------------------------------------- - # Load tabs again, this time we should get the updated tab_id. - - logger.debug("---------------------- making request 5 --------------------") - response = await dls_servbase_guis_get_default().client_protocolj( - load_tabs_request, - cookies={Cookies.TABS_MANAGER: cookie_uuid}, - ) - - logger.debug(describe("third load_tabs response", response)) - assert Keywords.TAB_ID in response - assert response[Keywords.TAB_ID] == "456" - - # -------------------------------------------------------------------- - # Write a text file and fetch it through the http server. - filename = "test.html" - contents = "some test html" - with open(f"{output_directory}/{filename}", "wt") as file: - file.write(contents) - logger.debug( - "---------------------- making request 6 (html file) --------------------" - ) - text = await dls_servbase_guis_get_default().client_get_file(filename) - # assert text == contents - - # Write a binary file and fetch it through the http server. - filename = "test.exe" - contents = "some test binary" - with open(f"{output_directory}/{filename}", "wt") as file: - file.write(contents) - binary = await dls_servbase_guis_get_default().client_get_file(filename) - # Binary comes back as bytes due to suffix of url filename. - assert binary == contents.encode() - - # -------------------------------------------------------------------- - # Get an html file automatically configured in guis/html. - filename = "javascript/version.js" - # TODO: Put proper version into javascript somehow during packaging. - contents = "dls_servbase___CURRENT_VERSION" - text = await dls_servbase_guis_get_default().client_get_file(filename) - logger.debug(f"javascript version is {text.strip()}") - assert contents in text + # Make the server context. + gui_server_context = GuiServerContext(gui_specification) + + # Make the client context. + gui_client_context = GuiClientContext(gui_specification) + + # Start the dataface the gui uses for cookies. + async with dataface_context: + # Start the gui client context. + async with gui_client_context: + # And the gui server context which starts the coro. + async with gui_server_context: + await self.__run_the_test(constants, output_directory) + + # ---------------------------------------------------------------------------------------- + + async def __run_the_test(self, constants, output_directory): + """ """ + # Reference the xchembku object which the context has set up as the default. + gui = dls_servbase_guis_get_default() + + # -------------------------------------------------------------------- + # Use protocolj to fetch a request from the dataface. + # Simulates what javascript would do by ajax. + + # Load tabs, of which there are none at the start. + # This establishes the cookie though. + load_tabs_request = { + Keywords.COMMAND: Commands.LOAD_TABS, + ProtocoljKeywords.ENABLE_COOKIES: [Cookies.TABS_MANAGER], + } + + logger.debug("---------------------- making request 1 --------------------") + response = await gui.client_protocolj(load_tabs_request, cookies={}) + + # The response is json, with the last saved tab_id, which is None at first. + assert Keywords.TAB_ID in response + assert response[Keywords.TAB_ID] is None + + # We should also have cookies back. + assert "__cookies" in response + cookies = response["__cookies"] + assert Cookies.TABS_MANAGER in cookies + + # Use the tabs manager cookie value in the next requests. + cookie_uuid = cookies[Cookies.TABS_MANAGER].value + + # -------------------------------------------------------------------- + # Select a tab. + # The response is json, but nothing really to see in it. + + select_tab_request = { + Keywords.COMMAND: Commands.SELECT_TAB, + ProtocoljKeywords.ENABLE_COOKIES: [Cookies.TABS_MANAGER], + Keywords.TAB_ID: "123", + } + + logger.debug("---------------------- making request 2 --------------------") + response = await gui.client_protocolj( + select_tab_request, cookies={Cookies.TABS_MANAGER: cookie_uuid} + ) + + # -------------------------------------------------------------------- + # Load tabs again, this time we should get the saved tab_id. + + logger.debug("---------------------- making request 3 --------------------") + # Put a deliberately funky cookie string into the header. + raw_cookie_header = ( + 'BadCookie={"something"}; ' + f"{Cookies.TABS_MANAGER} = {cookie_uuid};" + ) + response = await gui.client_protocolj( + load_tabs_request, + headers={"Cookie": raw_cookie_header}, + ) + + logger.debug(describe("second load_tabs response", response)) + assert Keywords.TAB_ID in response + assert response[Keywords.TAB_ID] == "123" + + # -------------------------------------------------------------------- + # Update a tab. + # The response is json, but nothing really to see in it. + + select_tab_request = { + Keywords.COMMAND: Commands.SELECT_TAB, + ProtocoljKeywords.ENABLE_COOKIES: [Cookies.TABS_MANAGER], + Keywords.TAB_ID: "456", + } + + logger.debug("---------------------- making request 4 --------------------") + response = await gui.client_protocolj( + select_tab_request, + cookies={Cookies.TABS_MANAGER: cookie_uuid}, + ) + + # -------------------------------------------------------------------- + # Load tabs again, this time we should get the updated tab_id. + + logger.debug("---------------------- making request 5 --------------------") + response = await gui.client_protocolj( + load_tabs_request, + cookies={Cookies.TABS_MANAGER: cookie_uuid}, + ) + + logger.debug(describe("third load_tabs response", response)) + assert Keywords.TAB_ID in response + assert response[Keywords.TAB_ID] == "456" + + # -------------------------------------------------------------------- + # Write a text file and fetch it through the http server. + filename = "test.html" + contents = "some test html" + with open(f"{output_directory}/{filename}", "wt") as file: + file.write(contents) + logger.debug( + "---------------------- making request 6 (html file) --------------------" + ) + text = await gui.client_get_file(filename) + # assert text == contents + + # Write a binary file and fetch it through the http server. + filename = "test.exe" + contents = "some test binary" + with open(f"{output_directory}/{filename}", "wt") as file: + file.write(contents) + binary = await gui.client_get_file(filename) + # Binary comes back as bytes due to suffix of url filename. + assert binary == contents.encode() + + # -------------------------------------------------------------------- + # Get an html file automatically configured in guis/html. + filename = "javascript/version.js" + # TODO: Put proper version into javascript somehow during packaging. + contents = "dls_servbase___CURRENT_VERSION" + text = await gui.client_get_file(filename) + logger.debug(f"javascript version is {text.strip()}") + assert contents in text From 70641e224d7e2ee16c592e35cb2687e8e291f14f Mon Sep 17 00:00:00 2001 From: David Erb Date: Fri, 2 Jun 2023 09:36:24 +0100 Subject: [PATCH 3/8] api and lib changed --- src/dls_servbase_api/guis/aiohttp.py | 49 +++------------------------- src/dls_servbase_lib/guis/context.py | 7 +--- src/dls_servbase_lib/guis/guis.py | 20 ------------ tests/test_gui.py | 1 + 4 files changed, 7 insertions(+), 70 deletions(-) diff --git a/src/dls_servbase_api/guis/aiohttp.py b/src/dls_servbase_api/guis/aiohttp.py index 413f331..0d9b79f 100644 --- a/src/dls_servbase_api/guis/aiohttp.py +++ b/src/dls_servbase_api/guis/aiohttp.py @@ -3,60 +3,21 @@ # Class for an aiohttp client. from dls_servbase_api.aiohttp_client import AiohttpClient -# Dataface protocolj things. -from dls_servbase_api.guis.constants import Commands, Keywords - logger = logging.getLogger(__name__) # ------------------------------------------------------------------------------------------ -class Aiohttp: +class Aiohttp(AiohttpClient): """ Object implementing client side API for talking to the dls_servbase_gui server. Please see doctopic [A01]. """ # ---------------------------------------------------------------------------------------- - def __init__(self, specification=None): - self.__specification = specification + def __init__(self, specification): - self.__aiohttp_client = AiohttpClient( + # We will get an umbrella specification which must contain an aiohttp_specification within it. + AiohttpClient.__init__( + self, specification["type_specific_tbd"]["aiohttp_specification"], ) - - # ---------------------------------------------------------------------------------------- - def specification(self): - return self.__specification - - # ---------------------------------------------------------------------------------------- - async def report_health(self): - """""" - return await self.__send_protocolj("report_health") - - # ---------------------------------------------------------------------------------------- - async def __send_protocolj(self, function, *args, **kwargs): - """""" - - return await self.__aiohttp_client.client_protocolj( - { - Keywords.COMMAND: Commands.EXECUTE, - Keywords.PAYLOAD: { - "function": function, - "args": args, - "kwargs": kwargs, - }, - }, - ) - - # ---------------------------------------------------------------------------------------- - async def close_client_session(self): - """""" - - if self.__aiohttp_client is not None: - await self.__aiohttp_client.close_client_session() - - # ---------------------------------------------------------------------------------------- - async def client_report_health(self): - """""" - - return await self.__aiohttp_client.client_report_health() diff --git a/src/dls_servbase_lib/guis/context.py b/src/dls_servbase_lib/guis/context.py index c5a1f50..b0a5753 100644 --- a/src/dls_servbase_lib/guis/context.py +++ b/src/dls_servbase_lib/guis/context.py @@ -4,7 +4,7 @@ from dls_utilpack.server_context_base import ServerContextBase # Things created in the context. -from dls_servbase_lib.guis.guis import Guis, dls_servbase_guis_set_default +from dls_servbase_lib.guis.guis import Guis logger = logging.getLogger(__name__) @@ -27,9 +27,6 @@ async def aenter(self): self.server = Guis().build_object(self.specification()) - # If there is more than one gui, the last one defined will be the default. - dls_servbase_guis_set_default(self.server) - if self.context_specification.get("start_as") == "coro": await self.server.activate_coro() @@ -49,5 +46,3 @@ async def aexit(self, type, value, traceback): # Release a client connection if we had one. await self.server.close_client_session() - - dls_servbase_guis_set_default(None) diff --git a/src/dls_servbase_lib/guis/guis.py b/src/dls_servbase_lib/guis/guis.py index ff27236..c0b5403 100644 --- a/src/dls_servbase_lib/guis/guis.py +++ b/src/dls_servbase_lib/guis/guis.py @@ -12,26 +12,6 @@ logger = logging.getLogger(__name__) -# ----------------------------------------------------------------------------------------- -__default_dls_servbase_gui = None - - -def dls_servbase_guis_set_default(dls_servbase_gui): - global __default_dls_servbase_gui - __default_dls_servbase_gui = dls_servbase_gui - - -def dls_servbase_guis_get_default(): - global __default_dls_servbase_gui - if __default_dls_servbase_gui is None: - raise RuntimeError("dls_servbase_guis_get_default instance is None") - return __default_dls_servbase_gui - - -def dls_servbase_guis_has_default(): - global __default_dls_servbase_gui - return __default_dls_servbase_gui is not None - # ----------------------------------------------------------------------------------------- diff --git a/tests/test_gui.py b/tests/test_gui.py index 00ce6c2..0a344da 100644 --- a/tests/test_gui.py +++ b/tests/test_gui.py @@ -95,6 +95,7 @@ async def __run_the_test(self, constants, output_directory): } logger.debug("---------------------- making request 1 --------------------") + logger.debug(describe("gui", gui)) response = await gui.client_protocolj(load_tabs_request, cookies={}) # The response is json, with the last saved tab_id, which is None at first. From 91bb2cbc6470f9703ea7143f29ff8842a3f54c78 Mon Sep 17 00:00:00 2001 From: David Erb Date: Fri, 2 Jun 2023 12:28:13 +0100 Subject: [PATCH 4/8] improving client, adds cli test --- src/dls_servbase_api/datafaces/aiohttp.py | 30 +--- src/dls_servbase_cli/__init__.py | 7 +- src/dls_servbase_cli/main.py | 32 +++- src/dls_servbase_cli/subcommands/base.py | 74 +++++++- src/dls_servbase_cli/subcommands/service.py | 81 +++++++++ .../subcommands/start_services.py | 101 ----------- src/dls_servbase_cli/version.py | 35 +--- src/dls_servbase_lib/envvar.py | 7 +- tests/test_cli.py | 159 ++++++++++++++++++ 9 files changed, 353 insertions(+), 173 deletions(-) create mode 100644 src/dls_servbase_cli/subcommands/service.py delete mode 100644 src/dls_servbase_cli/subcommands/start_services.py create mode 100644 tests/test_cli.py diff --git a/src/dls_servbase_api/datafaces/aiohttp.py b/src/dls_servbase_api/datafaces/aiohttp.py index 5d7e855..2fe2b7e 100644 --- a/src/dls_servbase_api/datafaces/aiohttp.py +++ b/src/dls_servbase_api/datafaces/aiohttp.py @@ -9,18 +9,17 @@ logger = logging.getLogger(__name__) -# ------------------------------------------------------------------------------------------ -class Aiohttp: +class Aiohttp(AiohttpClient): """ Object implementing client side API for talking to the dls_servbase_dataface server. - Please see doctopic [A01]. """ # ---------------------------------------------------------------------------------------- - def __init__(self, specification=None): - self.__specification = specification + def __init__(self, specification): - self.__aiohttp_client = AiohttpClient( + # We will get an umbrella specification which must contain an aiohttp_specification within it. + AiohttpClient.__init__( + self, specification["type_specific_tbd"]["aiohttp_specification"], ) @@ -50,11 +49,6 @@ async def update(self, table_name, record, where, subs=None, why=None): "update", table_name, record, where, subs=subs, why=why ) - # ---------------------------------------------------------------------------------------- - async def report_health(self): - """""" - return await self.__send_protocolj("report_health") - # ---------------------------------------------------------------------------------------- async def set_cookie(self, cookie_dict): """ """ @@ -77,7 +71,7 @@ async def update_cookie(self, row): async def __send_protocolj(self, function, *args, **kwargs): """""" - return await self.__aiohttp_client.client_protocolj( + return await self.client_protocolj( { Keywords.COMMAND: Commands.EXECUTE, Keywords.PAYLOAD: { @@ -89,14 +83,6 @@ async def __send_protocolj(self, function, *args, **kwargs): ) # ---------------------------------------------------------------------------------------- - async def close_client_session(self): - """""" - - if self.__aiohttp_client is not None: - await self.__aiohttp_client.close_client_session() - - # ---------------------------------------------------------------------------------------- - async def client_report_health(self): + async def report_health(self): """""" - - return await self.__aiohttp_client.client_report_health() + return await self.__send_protocolj("report_health") diff --git a/src/dls_servbase_cli/__init__.py b/src/dls_servbase_cli/__init__.py index 01806e8..5f86539 100644 --- a/src/dls_servbase_cli/__init__.py +++ b/src/dls_servbase_cli/__init__.py @@ -1 +1,6 @@ -__version__ = "1.3.1" +from importlib.metadata import version + +__version__ = version("dls_servbase") +del version + +__all__ = ["__version__"] diff --git a/src/dls_servbase_cli/main.py b/src/dls_servbase_cli/main.py index 4a54ac4..1bac152 100644 --- a/src/dls_servbase_cli/main.py +++ b/src/dls_servbase_cli/main.py @@ -1,16 +1,14 @@ #!/usr/bin/env python import argparse - -# Use standard python logging import logging import multiprocessing -# Base class with methods supporting command-line programs. +# Base class with methods supporting MaxIV command-line programs. from dls_mainiac_lib.mainiac import Mainiac # The subcommands. -from dls_servbase_cli.subcommands.start_services import StartServices +from dls_servbase_cli.subcommands.service import Service # The package version. from dls_servbase_cli.version import meta as version_meta @@ -28,8 +26,8 @@ def __init__(self, app_name): def run(self): """""" - if self._args.subcommand == "start_services": - StartServices(self._args, self).run() + if self._args.subcommand == "service": + Service(self._args, self).run() else: raise RuntimeError("unhandled subcommand %s" % (self._args.subcommand)) @@ -63,8 +61,8 @@ def build_parser(self, arglist=None): subparsers.required = True # -------------------------------------------------------------------- - subparser = subparsers.add_parser("start_services", help="Start service(s).") - StartServices.add_arguments(subparser) + subparser = subparsers.add_parser("service", help="Start service (blocking).") + Service.add_arguments(subparser) return parser @@ -92,7 +90,8 @@ def configure_logging(self, settings=None): # Let the base class do most of the work. Mainiac.configure_logging(self, settings) - # See bisstis-maxiv-daqcluster for advanced logging filters. + # Don't show specific asyncio debug. + logging.getLogger("asyncio").addFilter(_asyncio_logging_filter()) # Don't show matplotlib font debug. logging.getLogger("matplotlib.font_manager").setLevel("INFO") @@ -135,6 +134,21 @@ def filter(self, record): return 1 +# -------------------------------------------------------------------------------- +class _asyncio_logging_filter: + """ + Python logging filter to remove annoying asyncio messages. + These are not super useful to see all the time at the DEBUG level. + """ + + def filter(self, record): + + if "Using selector" in record.msg: + return 0 + + return 1 + + # # -------------------------------------------------------------------------------- # class _matplotlib_logging_filter: # """ diff --git a/src/dls_servbase_cli/subcommands/base.py b/src/dls_servbase_cli/subcommands/base.py index 73ebeb9..6fc3166 100644 --- a/src/dls_servbase_cli/subcommands/base.py +++ b/src/dls_servbase_cli/subcommands/base.py @@ -1,15 +1,25 @@ import logging import os import tempfile +from typing import Optional +# Configurator. +from dls_multiconf_lib.constants import ThingTypes as MulticonfThingTypes from dls_multiconf_lib.multiconfs import Multiconfs, multiconfs_set_default # Utilities. from dls_utilpack.visit import get_visit_year +# Environment variables with some extra functionality. +from dls_servbase_lib.envvar import Envvar + logger = logging.getLogger(__name__) +class ArgKeywords: + CONFIGURATION = "configuration" + + class Base: """ Base class for femtocheck subcommands. Handles details like configuration. @@ -21,9 +31,9 @@ def __init__(self, args): self.__temporary_directory = None # ---------------------------------------------------------------------------------------- - def get_multiconf(self): + def get_multiconf(self, args_dict: dict): - dls_servbase_multiconf = Multiconfs().build_object_from_environment() + dls_servbase_multiconf = self.build_object_from_environment(args_dict=args_dict) # For convenience, make a temporary directory for this test. self.__temporary_directory = tempfile.TemporaryDirectory() @@ -34,11 +44,10 @@ def get_multiconf(self): ) substitutions = { - "APPS": "/dls_sw/apps", "CWD": os.getcwd(), "HOME": os.environ.get("HOME", "HOME"), - # Provide the PYTHONPATH at the time of service start - # to the (potentially remote) process where the dls_servbase_task.run is called. + "USER": os.environ.get("USER", "USER"), + "PATH": os.environ.get("PATH", "PATH"), "PYTHONPATH": os.environ.get("PYTHONPATH", "PYTHONPATH"), } @@ -57,3 +66,58 @@ def get_multiconf(self): multiconfs_set_default(dls_servbase_multiconf) return dls_servbase_multiconf + + # ---------------------------------------------------------------------------------------- + def build_object_from_environment( + self, + environ: Optional[dict] = None, + args_dict: Optional[dict] = None, + ): + + configuration_keyword = "configuration" + + multiconf_filename = None + + if args_dict is not None: + multiconf_filename = args_dict.get(configuration_keyword) + + if multiconf_filename is not None: + # Make sure the path exists. + if not os.path.exists(multiconf_filename): + raise RuntimeError( + f"unable to find --{configuration_keyword} file {multiconf_filename}" + ) + else: + # Get the explicit name of the config file. + multiconf_filename = Envvar( + Envvar.DLS_SERVBASE_CONFIGFILE, + environ=environ, + ) + + # Config file is explicitly named? + if multiconf_filename.is_set: + # Make sure the path exists. + multiconf_filename = multiconf_filename.value + if not os.path.exists(multiconf_filename): + raise RuntimeError( + f"unable to find {Envvar.DLS_SERVBASE_CONFIGFILE} {multiconf_filename}" + ) + # Config file is not explicitly named? + else: + raise RuntimeError( + f"command line --{configuration_keyword} not given" + f" and environment variable {Envvar.DLS_SERVBASE_CONFIGFILE} is not set" + ) + + configurator = Multiconfs().build_object( + { + "type": MulticonfThingTypes.YAML, + "type_specific_tbd": {"filename": multiconf_filename}, + } + ) + + configurator.substitute( + {"configurator_directory": os.path.dirname(multiconf_filename)} + ) + + return configurator diff --git a/src/dls_servbase_cli/subcommands/service.py b/src/dls_servbase_cli/subcommands/service.py new file mode 100644 index 0000000..3948fa1 --- /dev/null +++ b/src/dls_servbase_cli/subcommands/service.py @@ -0,0 +1,81 @@ +import asyncio + +# Use standard logging in this module. +import logging + +from dls_utilpack.require import require + +# Things created in the context. +from dls_servbase_api.guis.guis import dls_servbase_guis_get_default + +# Base class for cli subcommands. +from dls_servbase_cli.subcommands.base import ArgKeywords, Base + +# Servbase context creator. +from dls_servbase_lib.datafaces.context import Context as DlsServbaseDatafaceContext + +# Servbase context creator. +from dls_servbase_lib.guis.context import Context as GuiContext + +logger = logging.getLogger() + + +# -------------------------------------------------------------- +class Service(Base): + """ + Start single service and keep running until ^C or remotely requested shutdown. + """ + + def __init__(self, args, mainiac): + super().__init__(args) + + # ---------------------------------------------------------------------------------------- + def run(self): + """ """ + + # Run in asyncio event loop. + asyncio.run(self.__run_coro()) + + # ---------------------------------------------------------- + async def __run_coro(self): + """""" + + # Load the configuration. + multiconf = self.get_multiconf(vars(self._args)) + configuration = await multiconf.load() + + # Get the specfication we want out of the configuration. + specification = require( + "configuration", + configuration, + "dls_servbase_dataface_specification", + ) + + # Context always starts as coro. + if "context" not in specification: + specification["context"] = {} + + specification["context"]["start_as"] = "coro" + + # Make the servbase service context from the specification in the configuration. + servbase_context = DlsServbaseDatafaceContext(specification) + + # Open the servbase context which starts the service process. + async with servbase_context: + # Wait for the coro to finish. + await servbase_context.server.wait_for_shutdown() + + # ---------------------------------------------------------- + def add_arguments(parser): + + parser.add_argument( + "--configuration", + "-c", + help="Configuration file.", + type=str, + metavar="yaml filename", + default=None, + dest=ArgKeywords.CONFIGURATION, + ) + + return parser diff --git a/src/dls_servbase_cli/subcommands/start_services.py b/src/dls_servbase_cli/subcommands/start_services.py deleted file mode 100644 index 7127888..0000000 --- a/src/dls_servbase_cli/subcommands/start_services.py +++ /dev/null @@ -1,101 +0,0 @@ -import asyncio - -# Use standard logging in this module. -import logging - -# Utilities. -from dls_utilpack.callsign import callsign - -# Base class for cli subcommands. -from dls_servbase_cli.subcommands.base import Base - -# Context creator. -from dls_servbase_lib.contexts.contexts import Contexts - -# Special reference to the gui so we give the url to the user on the info. -from dls_servbase_lib.guis.guis import dls_servbase_guis_get_default - -logger = logging.getLogger() - -# Specifications of services we can start, and their short names for parse args. -services = { - "dls_servbase_dataface_specification": "dataface", - "dls_servbase_collector_specification": "collector", - "dls_servbase_gui_specification": "gui", -} - - -# -------------------------------------------------------------- -class StartServices(Base): - """ - Start one or more services and keep them running until ^C. - """ - - def __init__(self, args, mainiac): - super().__init__(args) - - self.__mainiac = mainiac - - # ---------------------------------------------------------------------------------------- - def run(self): - """ """ - - # Run in asyncio event loop. - asyncio.run(self.__run_coro()) - - # ---------------------------------------------------------- - async def __run_coro(self): - """""" - - # Load the configuration. - dls_servbase_multiconf = self.get_multiconf() - context_configuration = await dls_servbase_multiconf.load() - - if "all" in self._args.service_names: - selected_service_names = [] - for _, service_name in services.items(): - selected_service_names.append(service_name) - else: - selected_service_names = self._args.service_names - - # Change all start_as to None, except the one we are starting. - for keyword, specification in context_configuration.items(): - if keyword in services: - service_name = services[keyword] - if service_name in selected_service_names: - specification["context"] = {"start_as": "process"} - - # Make a context from the configuration. - dls_servbase_context = Contexts().build_object(context_configuration) - - # Open the context (servers and clients). - async with dls_servbase_context: - if "gui" in selected_service_names: - logger.info(f"starting gui {callsign(dls_servbase_guis_get_default())}") - - try: - # Stay up until all processes are dead. - # TODO: Use asyncio wait or sentinel for all started processes to be dead. - while True: - await asyncio.sleep(1.0) - if not await dls_servbase_context.is_any_process_alive(): - logger.info("all processes have shutdown") - break - - except KeyboardInterrupt: - pass - - # ---------------------------------------------------------- - def add_arguments(parser): - - services_list = list(services.values()) - - parser.add_argument( - help='"all" or any combination of {%s}' % (" ".join(services_list)), - nargs="+", - type=str, - metavar="service name(s)", - dest="service_names", - ) - - return parser diff --git a/src/dls_servbase_cli/version.py b/src/dls_servbase_cli/version.py index b916475..214fe41 100644 --- a/src/dls_servbase_cli/version.py +++ b/src/dls_servbase_cli/version.py @@ -1,11 +1,13 @@ -import argparse -import json +import logging import dls_mainiac_lib.version -import dls_servbase_cli import dls_servbase_lib.version +from . import __version__ + +logger = logging.getLogger(__name__) + # ---------------------------------------------------------- def version(): @@ -13,7 +15,7 @@ def version(): Current version. """ - return dls_servbase_cli.__version__ + return __version__ # ---------------------------------------------------------- @@ -32,28 +34,3 @@ def meta(given_meta=None): else: given_meta = s return given_meta - - -# ---------------------------------------------------------- -def main(): - - parser = argparse.ArgumentParser() - - parser.add_argument( - "--json", - action="store_true", - help="Print version stack in json.", - ) - - # ------------------------------------------------------------------------- - given_args, remaining_args = parser.parse_known_args() - - if given_args.json: - print(json.dumps(meta(), indent=4)) - else: - print(version()) - - -# ---------------------------------------------------------- -if __name__ == "__main__": - main() diff --git a/src/dls_servbase_lib/envvar.py b/src/dls_servbase_lib/envvar.py index 7a0b9e5..c8a3152 100644 --- a/src/dls_servbase_lib/envvar.py +++ b/src/dls_servbase_lib/envvar.py @@ -7,12 +7,7 @@ class Envvar: """Class which covers environment variables, with default values.""" - DLS_BILLY_CONFIGFILE = "DLS_BILLY_CONFIGFILE" - XCHEM_BEFLOW_DATA = "XCHEM_BEFLOW_DATA" - XCHEM_BEFLOW_DLS_ROOT = "XCHEM_BEFLOW_DLS_ROOT" - BEAMLINE = "BEAMLINE" - VISIT_YEAR = "VISIT_YEAR" - VISIT = "VISIT" + DLS_SERVBASE_CONFIGFILE = "DLS_SERVBASE_CONFIGFILE" def __init__(self, name, **kwargs): diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..d21dafc --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,159 @@ +import asyncio +import copy +import logging +import subprocess +import time + +from ruamel.yaml import YAML + +from dls_servbase_api.databases.constants import CookieFieldnames, Tablenames +from dls_servbase_api.datafaces.context import Context as ClientContext +from dls_servbase_api.datafaces.datafaces import dls_servbase_datafaces_get_default +from dls_servbase_lib.datafaces.context import Context as ServerContext + +# Base class for the tester. +from tests.base_context_tester import BaseContextTester + +logger = logging.getLogger(__name__) + + +# ---------------------------------------------------------------------------------------- +class TestCliSqlite: + """ + Test that we can do a basic database operation through the service. + """ + + def test(self, constants, logging_setup, output_directory): + """ """ + + configuration_file = "tests/configurations/sqlite.yaml" + CliTester(configuration_file).main( + constants, configuration_file, output_directory + ) + + +# ---------------------------------------------------------------------------------------- +class TestCliMysql: + """ + Test that we can do a basic database operation through the service. + """ + + def test(self, constants, logging_setup, output_directory): + """ """ + + configuration_file = "tests/configurations/mysql.yaml" + CliTester(configuration_file).main( + constants, configuration_file, output_directory + ) + + +# ---------------------------------------------------------------------------------------- +class CliTester(BaseContextTester): + """ + Class to test the dataface. + """ + + def __init__(self, configuration_file): + BaseContextTester.__init__(self) + + self.__configuration_file = configuration_file + + async def _main_coroutine(self, constants, output_directory): + """ """ + + # Command to run the service. + dls_servbase_server_cli = [ + "python", + "-m", + "dls_servbase_cli.main", + "service", + "--verbose", + "--configuration", + self.__configuration_file, + ] + + # Launch the service as a process. + logger.debug(f"launching {' '.join(dls_servbase_server_cli)}") + process = subprocess.Popen( + dls_servbase_server_cli, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + try: + # Read the configuration. + multiconf_object = self.get_multiconf() + multiconf_dict = await multiconf_object.load() + + # Get a client context to the server in the process we just started. + servbase_specification = multiconf_dict[ + "dls_servbase_dataface_specification" + ] + dls_servbase_client_context = ClientContext(servbase_specification) + async with dls_servbase_client_context: + dataface = dls_servbase_datafaces_get_default() + + # Some easy sql query. + all_sql = f"SELECT * FROM {Tablenames.COOKIES}" + + # Wait for process to be up. + start_time = time.time() + max_seconds = 5.0 + while True: + try: + records = await dataface.query(all_sql) + break + except Exception as exception: + logger.debug(f"retrying after failure {exception}") + + if process.poll() is not None: + raise RuntimeError( + "server apprently died before first command worked" + ) + + if time.time() - start_time > max_seconds: + raise RuntimeError( + f"server not answering within {max_seconds} seconds" + ) + + await asyncio.sleep(1.0) + + # Interact with it. + await dataface.insert( + Tablenames.COOKIES, + [ + { + CookieFieldnames.UUID: "f0", + CookieFieldnames.CONTENTS: "{'a': 'f000'}", + } + ], + ) + + records = await dataface.query(all_sql) + + assert len(records) == 1 + assert records[0][CookieFieldnames.UUID] == "f0" + assert records[0][CookieFieldnames.CONTENTS] == "{'a': 'f000'}" + + await dataface.client_shutdown() + finally: + try: + # Wait for the process to finish and get the output + stdout_bytes, stderr_bytes = process.communicate(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + stdout_bytes, stderr_bytes = process.communicate() + + # Get the return code of the process + return_code = process.returncode + logger.debug(f"server return_code is {return_code}") + + logger.debug( + f"================================== server stderr is:\n{stderr_bytes.decode()}" + ) + logger.debug( + f"================================== server stdout is:\n{stdout_bytes.decode()}" + ) + logger.debug(f"==================================") + + assert return_code == 0 From 4fd3bfbc66ae03b34e7e1c8777f707a1a80ac231 Mon Sep 17 00:00:00 2001 From: David Erb Date: Fri, 2 Jun 2023 14:09:26 +0100 Subject: [PATCH 5/8] test_cli working --- .../dls_servbase_dataface.sqlite | Bin 0 -> 32768 bytes src/dls_servbase_cli/subcommands/base.py | 18 +----------- src/dls_servbase_cli/subcommands/service.py | 1 + src/dls_servbase_lib/base_aiohttp.py | 3 ++ src/dls_servbase_lib/datafaces/aiohttp.py | 8 +++--- src/dls_servbase_lib/datafaces/context.py | 17 +++++++----- tests/test_cli.py | 26 +++++++++--------- 7 files changed, 32 insertions(+), 41 deletions(-) create mode 100644 ${output_directory}/dls_servbase_dataface.sqlite diff --git a/${output_directory}/dls_servbase_dataface.sqlite b/${output_directory}/dls_servbase_dataface.sqlite new file mode 100644 index 0000000000000000000000000000000000000000..0481da417f686447deb937c11a55b2097d92da0b GIT binary patch literal 32768 zcmeI((N5Y>7{KvUDpXvsdc_4{>CJ*9!xT`rakXx17NgURaiQ4~idqsfX)`E`$+9Ip z!d_u-vpXMQ*<%Gv)pCoJo&Hy^~4u%>3k6 z5x3KZW94y5)$#>7>rX-F=rpc3ARG*Zrrl%T3tTT4MiJ%8LMCq&6cNG=zn;qVaJw@K{IK7S<*SoMTHc=F zs<@ld^7W!J&W3RV_gin+^S$ None: # Build the object according to the specification. self.server = Datafaces().build_object(self.specification()) - if self.context_specification.get("start_as") == "coro": + start_as = self.context_specification.get("start_as") + if start_as == "coro": await self.server.activate_coro() - elif self.context_specification.get("start_as") == "thread": + elif start_as == "thread": await self.server.start_thread() - elif self.context_specification.get("start_as") == "process": + elif start_as == "process": await self.server.start_process() # Not running as a service? - elif self.context_specification.get("start_as") == "direct": + elif start_as == "direct": # We need to activate the tick() task. await self.server.activate() @@ -70,7 +71,9 @@ async def aexit(self, type, value, traceback) -> None: """ if self.server is not None: - if self.context_specification.get("start_as") == "process": + start_as = self.context_specification.get("start_as") + + if start_as == "process": logger.info( "[DISSHU] in context exit, sending shutdown to client process" ) @@ -78,8 +81,8 @@ async def aexit(self, type, value, traceback) -> None: await self.server.client_shutdown() logger.info("[DISSHU] in context exit, sent shutdown to client process") - if self.context_specification.get("start_as") == "coro": + if start_as == "coro": await self.server.direct_shutdown() - if self.context_specification.get("start_as") == "direct": + if start_as == "direct": await self.server.deactivate() diff --git a/tests/test_cli.py b/tests/test_cli.py index d21dafc..25bc5fc 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,6 @@ import asyncio -import copy import logging +import os import subprocess import time @@ -9,7 +9,6 @@ from dls_servbase_api.databases.constants import CookieFieldnames, Tablenames from dls_servbase_api.datafaces.context import Context as ClientContext from dls_servbase_api.datafaces.datafaces import dls_servbase_datafaces_get_default -from dls_servbase_lib.datafaces.context import Context as ServerContext # Base class for the tester. from tests.base_context_tester import BaseContextTester @@ -73,6 +72,7 @@ async def _main_coroutine(self, constants, output_directory): ] # Launch the service as a process. + os.environ["output_directory"] = output_directory logger.debug(f"launching {' '.join(dls_servbase_server_cli)}") process = subprocess.Popen( dls_servbase_server_cli, @@ -144,16 +144,16 @@ async def _main_coroutine(self, constants, output_directory): process.kill() stdout_bytes, stderr_bytes = process.communicate() - # Get the return code of the process - return_code = process.returncode - logger.debug(f"server return_code is {return_code}") - - logger.debug( - f"================================== server stderr is:\n{stderr_bytes.decode()}" - ) - logger.debug( - f"================================== server stdout is:\n{stdout_bytes.decode()}" - ) - logger.debug(f"==================================") + # Get the return code of the process + return_code = process.returncode + logger.debug(f"server return_code is {return_code}") + + logger.debug( + f"================================== server stderr is:\n{stderr_bytes.decode()}" + ) + logger.debug( + f"================================== server stdout is:\n{stdout_bytes.decode()}" + ) + logger.debug(f"==================================") assert return_code == 0 From 153c732cc62fbecbbe35298d84c9322709802fc4 Mon Sep 17 00:00:00 2001 From: David Erb Date: Fri, 2 Jun 2023 14:40:11 +0100 Subject: [PATCH 6/8] finishes command line doc --- docs/reference/command_line.rst | 7 +++ src/dls_servbase_cli/main.py | 57 +++++++-------------- src/dls_servbase_cli/subcommands/base.py | 6 +-- src/dls_servbase_cli/subcommands/service.py | 46 +++++++++-------- src/dls_servbase_lib/base_aiohttp.py | 3 -- src/dls_servbase_lib/datafaces/aiohttp.py | 2 - tests/test_cli.py | 17 +++--- 7 files changed, 60 insertions(+), 78 deletions(-) diff --git a/docs/reference/command_line.rst b/docs/reference/command_line.rst index 8e4f1e9..8e63583 100644 --- a/docs/reference/command_line.rst +++ b/docs/reference/command_line.rst @@ -7,3 +7,10 @@ dls-servbase :module: dls_servbase_lib.__main__ :func: get_parser :prog: dls-servbase + +dls_servbase_cli.main +----------------------------------------------------------------------- +.. argparse:: + :module: dls_servbase_cli.main + :func: get_parser + :prog: python -m dls_servbase_cli.main diff --git a/src/dls_servbase_cli/main.py b/src/dls_servbase_cli/main.py index 1bac152..af6873a 100644 --- a/src/dls_servbase_cli/main.py +++ b/src/dls_servbase_cli/main.py @@ -41,7 +41,7 @@ def build_parser(self, arglist=None): # Make a parser. parser = argparse.ArgumentParser( - description="Command line app for checking quality of femtoscan file in progress.", + description="Command line interface to dls-servbase.", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) @@ -61,7 +61,10 @@ def build_parser(self, arglist=None): subparsers.required = True # -------------------------------------------------------------------- - subparser = subparsers.add_parser("service", help="Start service (blocking).") + subparser = subparsers.add_parser( + "service", + help="Start single service and block until ^C or remotely requested shutdown.", + ) Service.add_arguments(subparser) return parser @@ -93,12 +96,6 @@ def configure_logging(self, settings=None): # Don't show specific asyncio debug. logging.getLogger("asyncio").addFilter(_asyncio_logging_filter()) - # Don't show matplotlib font debug. - logging.getLogger("matplotlib.font_manager").setLevel("INFO") - - # Set filter on the ispyb logger to ignore the annoying NOTICE. - logging.getLogger("ispyb").addFilter(_ispyb_logging_filter()) - # ---------------------------------------------------------- def version(self): """ @@ -117,23 +114,6 @@ def about(self): return {"versions": version_meta()} -# -------------------------------------------------------------------------------- -class _ispyb_logging_filter: - """ - Python logging filter to remove annoying traitlets messages. - These are not super useful to see all the time at the DEBUG level. - """ - - def filter(self, record): - - if record.msg.startswith( - "NOTICE: This code uses __future__ functionality in the ISPyB API." - ): - return 0 - - return 1 - - # -------------------------------------------------------------------------------- class _asyncio_logging_filter: """ @@ -149,20 +129,6 @@ def filter(self, record): return 1 -# # -------------------------------------------------------------------------------- -# class _matplotlib_logging_filter: -# """ -# Python logging filter to remove annoying matplotlib messages. -# These are not super useful to see all the time at the INIT level. -# """ - -# def filter(self, record): -# if "loaded modules" in record.msg: -# return 0 - -# return 1 - - # --------------------------------------------------------------- def main(): @@ -176,6 +142,19 @@ def main(): main.try_run_catch() +# --------------------------------------------------------------- +def get_parser(): + """ + Called from sphinx automodule. + """ + + # Instantiate the app. + main = Main("dls_servbase_cli") + + # Configure the app from command line arguments. + return main.build_parser() + + # --------------------------------------------------------------- # From command line, invoke the main method. if __name__ == "__main__": diff --git a/src/dls_servbase_cli/subcommands/base.py b/src/dls_servbase_cli/subcommands/base.py index d4bc5b2..72629a3 100644 --- a/src/dls_servbase_cli/subcommands/base.py +++ b/src/dls_servbase_cli/subcommands/base.py @@ -1,15 +1,11 @@ import logging import os -import tempfile from typing import Optional # Configurator. from dls_multiconf_lib.constants import ThingTypes as MulticonfThingTypes from dls_multiconf_lib.multiconfs import Multiconfs, multiconfs_set_default -# Utilities. -from dls_utilpack.visit import get_visit_year - # Environment variables with some extra functionality. from dls_servbase_lib.envvar import Envvar @@ -22,7 +18,7 @@ class ArgKeywords: class Base: """ - Base class for femtocheck subcommands. Handles details like configuration. + Base class for subcommands. Handles details like configuration. """ def __init__(self, args): diff --git a/src/dls_servbase_cli/subcommands/service.py b/src/dls_servbase_cli/subcommands/service.py index 2724189..22e4f1b 100644 --- a/src/dls_servbase_cli/subcommands/service.py +++ b/src/dls_servbase_cli/subcommands/service.py @@ -1,3 +1,4 @@ +import argparse import asyncio # Use standard logging in this module. @@ -5,25 +6,19 @@ from dls_utilpack.require import require -# Things created in the context. -from dls_servbase_api.guis.guis import dls_servbase_guis_get_default - # Base class for cli subcommands. from dls_servbase_cli.subcommands.base import ArgKeywords, Base # Servbase context creator. from dls_servbase_lib.datafaces.context import Context as DlsServbaseDatafaceContext -# Servbase context creator. -from dls_servbase_lib.guis.context import Context as GuiContext - logger = logging.getLogger() # -------------------------------------------------------------- class Service(Base): """ - Start single service and keep running until ^C or remotely requested shutdown. + Start single service and block until ^C or remotely requested shutdown. """ def __init__(self, args, mainiac): @@ -38,43 +33,54 @@ def run(self): # ---------------------------------------------------------- async def __run_coro(self): - """""" + """ + Run the service as an asyncio coro. + """ # Load the configuration. - multiconf = self.get_multiconf(vars(self._args)) - configuration = await multiconf.load() + multiconf_object = self.get_multiconf(vars(self._args)) + # Resolve the symbols and give configuration as a dict. + multiconf_dict = await multiconf_object.load() - # Get the specfication we want out of the configuration. + # Get the specfication we want by keyword in the full configuration. specification = require( "configuration", - configuration, + multiconf_dict, "dls_servbase_dataface_specification", ) - # Context always starts as coro. + # We need the context to always start the service as a coro. if "context" not in specification: specification["context"] = {} - specification["context"]["start_as"] = "coro" # Make the servbase service context from the specification in the configuration. - servbase_context = DlsServbaseDatafaceContext(specification) + context = DlsServbaseDatafaceContext(specification) # Open the servbase context which starts the service process. - logger.debug("[CLIOPS] starting servbase coro context") - async with servbase_context: + async with context: # Wait for the coro to finish. - await servbase_context.server.wait_for_shutdown() + await context.server.wait_for_shutdown() # ---------------------------------------------------------- - def add_arguments(parser): + @staticmethod + def add_arguments(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: + """ + Add arguments for this subcommand. + + This is a static method called from the main program. + + Args: + parser (argparse.ArgumentParser): Parser object which has been created already. + + """ parser.add_argument( "--configuration", "-c", help="Configuration file.", type=str, - metavar="yaml filename", + metavar="filename.yaml", default=None, dest=ArgKeywords.CONFIGURATION, ) diff --git a/src/dls_servbase_lib/base_aiohttp.py b/src/dls_servbase_lib/base_aiohttp.py index 8f0fa16..e570bcb 100644 --- a/src/dls_servbase_lib/base_aiohttp.py +++ b/src/dls_servbase_lib/base_aiohttp.py @@ -280,8 +280,6 @@ async def activate_coro_base(self, route_tuples=[]): This is called from each server's activate_coro() method. """ try: - logger.debug(f"[CLIOPS] shutting down other instances of {callsign(self)}") - # Shut down any existing servers. result = await self.client_shutdown() @@ -333,7 +331,6 @@ async def activate_coro_base(self, route_tuples=[]): site = aiohttp.web.UnixSite(self.__app_runner, parts.path) await site.start() - logger.debug(f"[CLIOPS] started web site of {callsign(self)}") except Exception as exception: raise RuntimeError( explain(exception, f"exception starting {callsign(self)} site") diff --git a/src/dls_servbase_lib/datafaces/aiohttp.py b/src/dls_servbase_lib/datafaces/aiohttp.py index 672cb7e..6988ae1 100644 --- a/src/dls_servbase_lib/datafaces/aiohttp.py +++ b/src/dls_servbase_lib/datafaces/aiohttp.py @@ -94,8 +94,6 @@ async def activate_coro(self): # Get the local implementation started. await self.__actual_dls_servbase_dataface.start() - logger.debug(f"[CLIOPS] activating coro base of {callsign(self)}") - await self.activate_coro_base(route_tuples) except Exception: diff --git a/tests/test_cli.py b/tests/test_cli.py index 25bc5fc..8600dff 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -4,8 +4,6 @@ import subprocess import time -from ruamel.yaml import YAML - from dls_servbase_api.databases.constants import CookieFieldnames, Tablenames from dls_servbase_api.datafaces.context import Context as ClientContext from dls_servbase_api.datafaces.datafaces import dls_servbase_datafaces_get_default @@ -19,11 +17,10 @@ # ---------------------------------------------------------------------------------------- class TestCliSqlite: """ - Test that we can do a basic database operation through the service. + Test that we can do a basic database operation through the service on sqlite databases. """ def test(self, constants, logging_setup, output_directory): - """ """ configuration_file = "tests/configurations/sqlite.yaml" CliTester(configuration_file).main( @@ -34,11 +31,10 @@ def test(self, constants, logging_setup, output_directory): # ---------------------------------------------------------------------------------------- class TestCliMysql: """ - Test that we can do a basic database operation through the service. + Test that we can do a basic database operation through the service on mysql databases. """ def test(self, constants, logging_setup, output_directory): - """ """ configuration_file = "tests/configurations/mysql.yaml" CliTester(configuration_file).main( @@ -71,8 +67,10 @@ async def _main_coroutine(self, constants, output_directory): self.__configuration_file, ] - # Launch the service as a process. + # Let the output_directory symbol be replaced in the multiconf. os.environ["output_directory"] = output_directory + + # Launch the service as a process. logger.debug(f"launching {' '.join(dls_servbase_server_cli)}") process = subprocess.Popen( dls_servbase_server_cli, @@ -138,9 +136,10 @@ async def _main_coroutine(self, constants, output_directory): await dataface.client_shutdown() finally: try: - # Wait for the process to finish and get the output + # Wait for the process to finish and get the output. stdout_bytes, stderr_bytes = process.communicate(timeout=5) except subprocess.TimeoutExpired: + # Timeout happens when client dies but server hasn't been told to shutdown. process.kill() stdout_bytes, stderr_bytes = process.communicate() @@ -154,6 +153,6 @@ async def _main_coroutine(self, constants, output_directory): logger.debug( f"================================== server stdout is:\n{stdout_bytes.decode()}" ) - logger.debug(f"==================================") + logger.debug("==================================") assert return_code == 0 From 05ee57552535ff5617fa0f29db8f8dd8bf6bd035 Mon Sep 17 00:00:00 2001 From: David Erb Date: Fri, 2 Jun 2023 15:05:32 +0100 Subject: [PATCH 7/8] tweaks docs and cleans up --- .../dls_servbase_dataface.sqlite | Bin 32768 -> 0 bytes src/dls_servbase_api/constants.py | 6 +++ src/dls_servbase_lib/datafaces/aiohttp.py | 5 ++- src/dls_servbase_lib/datafaces/datafaces.py | 41 +++++++++++++++--- src/dls_servbase_lib/datafaces/normsql.py | 4 +- tests/base.py | 34 --------------- tests/base_specification_tester.py | 37 ---------------- tests/base_tester.py | 28 ------------ tests/test_database.py | 4 +- 9 files changed, 49 insertions(+), 110 deletions(-) delete mode 100644 ${output_directory}/dls_servbase_dataface.sqlite delete mode 100644 tests/base.py delete mode 100644 tests/base_specification_tester.py diff --git a/${output_directory}/dls_servbase_dataface.sqlite b/${output_directory}/dls_servbase_dataface.sqlite deleted file mode 100644 index 0481da417f686447deb937c11a55b2097d92da0b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 32768 zcmeI((N5Y>7{KvUDpXvsdc_4{>CJ*9!xT`rakXx17NgURaiQ4~idqsfX)`E`$+9Ip z!d_u-vpXMQ*<%Gv)pCoJo&Hy^~4u%>3k6 z5x3KZW94y5)$#>7>rX-F=rpc3ARG*Zrrl%T3tTT4MiJ%8LMCq&6cNG=zn;qVaJw@K{IK7S<*SoMTHc=F zs<@ld^7W!J&W3RV_gin+^S$ Any: + """ + Build an object whose type is contained in the specification as a string. + + Args: + specification (Dict): Specification, must contain at least the keyword "type". + + Returns: + The dls_servbase_dataface instance. + """ dls_servbase_dataface_class = self.lookup_class(specification["type"]) @@ -39,15 +61,20 @@ def build_object(self, specification): return dls_servbase_dataface_object # ---------------------------------------------------------------------------------------- - def lookup_class(self, class_type): - """""" + def lookup_class(self, class_type: str) -> Type: + """ + From the given class type string, return a class corresponding to the type. + + Returns: + A class which can be instantiated. + """ - if class_type == "dls_servbase_lib.datafaces.aiohttp": + if class_type == ClassTypes.AIOHTTP: from dls_servbase_lib.datafaces.aiohttp import Aiohttp return Aiohttp - elif class_type == "dls_servbase_lib.datafaces.normsql": + elif class_type == ClassTypes.NORMSQL: from dls_servbase_lib.datafaces.normsql import Normsql return Normsql diff --git a/src/dls_servbase_lib/datafaces/normsql.py b/src/dls_servbase_lib/datafaces/normsql.py index ec5b794..6336218 100644 --- a/src/dls_servbase_lib/datafaces/normsql.py +++ b/src/dls_servbase_lib/datafaces/normsql.py @@ -6,12 +6,14 @@ # Base class for generic things. from dls_utilpack.thing import Thing +# Class types. +from dls_servbase_api.constants import ClassTypes from dls_servbase_api.databases.constants import Tablenames from dls_servbase_api.databases.database_definition import DatabaseDefinition logger = logging.getLogger(__name__) -thing_type = "dls_servbase_lib.datafaces.normsql" +thing_type = ClassTypes.NORMSQL class Normsql(Thing): diff --git a/tests/base.py b/tests/base.py deleted file mode 100644 index c59587a..0000000 --- a/tests/base.py +++ /dev/null @@ -1,34 +0,0 @@ -import asyncio -import logging -import multiprocessing - -import pytest - -logger = logging.getLogger(__name__) - - -class Base: - - # ---------------------------------------------------------------------------------------- - def main(self, constants, infrastrcuture_context, output_directory): - """ """ - - multiprocessing.current_process().name = "main" - - failure_message = None - try: - # Run main test in asyncio event loop. - asyncio.run( - self._main_coroutine( - constants, infrastrcuture_context, output_directory - ) - ) - - except Exception as exception: - logger.exception( - "unexpected exception in the test method", exc_info=exception - ) - failure_message = str(exception) - - if failure_message is not None: - pytest.fail(failure_message) diff --git a/tests/base_specification_tester.py b/tests/base_specification_tester.py deleted file mode 100644 index 929e975..0000000 --- a/tests/base_specification_tester.py +++ /dev/null @@ -1,37 +0,0 @@ -import asyncio -import logging -import multiprocessing - -import pytest - -logger = logging.getLogger(__name__) - - -# ---------------------------------------------------------------------------------------- -class BaseSpecificationTester: - """ - This is a base class for tests which take a specification. - """ - - def main(self, constants, specification, output_directory): - """ - This is the main program which calls the test using asyncio. - """ - - multiprocessing.current_process().name = "main" - - failure_message = None - try: - # Run main test in asyncio event loop. - asyncio.run( - self._main_coroutine(constants, specification, output_directory) - ) - - except Exception as exception: - logger.exception( - "unexpected exception in the test method", exc_info=exception - ) - failure_message = str(exception) - - if failure_message is not None: - pytest.fail(failure_message) diff --git a/tests/base_tester.py b/tests/base_tester.py index 8184704..07c3fd9 100644 --- a/tests/base_tester.py +++ b/tests/base_tester.py @@ -9,34 +9,6 @@ # ---------------------------------------------------------------------------------------- class BaseTester: - """ - This is a base class for simplest tests. - """ - - def main(self, constants, output_directory): - """ - This is the main program which calls the test using asyncio. - """ - - multiprocessing.current_process().name = "main" - - failure_message = None - try: - # Run main test in asyncio event loop. - asyncio.run(self._main_coroutine(constants, output_directory)) - - except Exception as exception: - logger.exception( - "unexpected exception in the test method", exc_info=exception - ) - failure_message = str(exception) - - if failure_message is not None: - pytest.fail(failure_message) - - -# ---------------------------------------------------------------------------------------- -class BaseTester2: """ Provide asyncio loop and error checking over *Tester classes. """ diff --git a/tests/test_database.py b/tests/test_database.py index 09b8797..1002833 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -5,7 +5,7 @@ from dls_servbase_api.databases.constants import CookieFieldnames, Tablenames from dls_servbase_api.databases.database_definition import DatabaseDefinition -from tests.base_tester import BaseTester2 +from tests.base_tester import BaseTester logger = logging.getLogger(__name__) @@ -61,7 +61,7 @@ def test(self, constants, logging_setup, output_directory): # ---------------------------------------------------------------------------------------- -class DatabaseTester(BaseTester2): +class DatabaseTester(BaseTester): """ Test direct SQL access to the database. """ From 3c6d920f84f3ff7d24e9433cfe9acd00f71e10d8 Mon Sep 17 00:00:00 2001 From: David Erb Date: Sat, 3 Jun 2023 06:09:40 +0100 Subject: [PATCH 8/8] get client context base from utilpack --- src/dls_servbase_api/context_base.py | 35 ----------------------- src/dls_servbase_api/datafaces/context.py | 34 +++++++++------------- src/dls_servbase_api/guis/context.py | 10 +++---- 3 files changed, 19 insertions(+), 60 deletions(-) delete mode 100644 src/dls_servbase_api/context_base.py diff --git a/src/dls_servbase_api/context_base.py b/src/dls_servbase_api/context_base.py deleted file mode 100644 index b536781..0000000 --- a/src/dls_servbase_api/context_base.py +++ /dev/null @@ -1,35 +0,0 @@ -import logging - -logger = logging.getLogger(__name__) - - -class ContextBase: - """ """ - - # ---------------------------------------------------------------------------------------- - def __init__(self, specification): - self.__specification = specification - self.__interface = None - - # ---------------------------------------------------------------------------------------- - def get_interface(self): - return self.__interface - - def set_interface(self, interface): - self.__interface = interface - - interface = property(get_interface, set_interface) - - # ---------------------------------------------------------------------------------------- - async def __aenter__(self): - """ """ - - await self.aenter() - - return self.interface - - # ---------------------------------------------------------------------------------------- - async def __aexit__(self, type, value, traceback): - """ """ - - await self.aexit() diff --git a/src/dls_servbase_api/datafaces/context.py b/src/dls_servbase_api/datafaces/context.py index 2f11209..f76fb86 100644 --- a/src/dls_servbase_api/datafaces/context.py +++ b/src/dls_servbase_api/datafaces/context.py @@ -1,5 +1,8 @@ import logging +# Base class. +from dls_utilpack.client_context_base import ClientContextBase + # Things created in the context. from dls_servbase_api.datafaces.datafaces import ( Datafaces, @@ -9,48 +12,39 @@ logger = logging.getLogger(__name__) -class Context: +class Context(ClientContextBase): """ Client context for a dls_servbase_dataface object. On entering, it creates the object according to the specification (a dict). On exiting, it closes client connection. - The aenter and aexit methods are exposed for use by an enclosing context. + The aenter and aexit methods are exposed for use by an enclosing context and the base class. """ # ---------------------------------------------------------------------------------------- def __init__(self, specification): - self.__specification = specification - self.__dls_servbase_dataface = None - - # ---------------------------------------------------------------------------------------- - async def __aenter__(self): - """ """ - - await self.aenter() - - # ---------------------------------------------------------------------------------------- - async def __aexit__(self, type, value, traceback): - """ """ - - await self.aexit() + ClientContextBase.__init__(self, specification) # ---------------------------------------------------------------------------------------- async def aenter(self): """ """ # Build the object according to the specification. - self.__dls_servbase_dataface = Datafaces().build_object(self.__specification) + self.interface = Datafaces().build_object(self.specification) # If there is more than one dataface, the last one defined will be the default. - dls_servbase_datafaces_set_default(self.__dls_servbase_dataface) + dls_servbase_datafaces_set_default(self.interface) + + # Open client session to the service or direct connection. + await self.interface.open_client_session() # ---------------------------------------------------------------------------------------- async def aexit(self): """ """ - if self.__dls_servbase_dataface is not None: - await self.__dls_servbase_dataface.close_client_session() + if self.interface is not None: + # Close client session to the service or direct connection. + await self.interface.close_client_session() # Clear the global variable. Important between pytests. dls_servbase_datafaces_set_default(None) diff --git a/src/dls_servbase_api/guis/context.py b/src/dls_servbase_api/guis/context.py index 12d97fb..58a1b19 100644 --- a/src/dls_servbase_api/guis/context.py +++ b/src/dls_servbase_api/guis/context.py @@ -1,7 +1,7 @@ import logging # Base class. -from dls_servbase_api.context_base import ContextBase +from dls_utilpack.client_context_base import ClientContextBase # Things created in the context. from dls_servbase_api.guis.guis import Guis, dls_servbase_guis_set_default @@ -9,25 +9,25 @@ logger = logging.getLogger(__name__) -class Context(ContextBase): +class Context(ClientContextBase): """ Client context for a dls_servbase_gui object. On entering, it creates the object according to the specification (a dict). On exiting, it closes client connection. - The aenter and aexit methods are exposed for use by an enclosing context. + The aenter and aexit methods are exposed for use by an enclosing context and the base class. """ # ---------------------------------------------------------------------------------------- def __init__(self, specification): - self.__specification = specification + ClientContextBase.__init__(self, specification) # ---------------------------------------------------------------------------------------- async def aenter(self): """ """ # Build the object according to the specification. - self.interface = Guis().build_object(self.__specification) + self.interface = Guis().build_object(self.specification) # If there is more than one gui, the last one defined will be the default. dls_servbase_guis_set_default(self.interface)