From c02587b01071ef8543b436be3cb671ddfef56fa4 Mon Sep 17 00:00:00 2001 From: Joe Date: Wed, 4 Sep 2024 16:21:00 -0500 Subject: [PATCH] feat: Add watch mode to `plugin_builder.py` (#777) Fixes #632 Adds a `--watch` mode to `plugin_builder.py` This goes beyond just editable installs with the goal of automatically running any command that is reasonable to rerun (not config, for example). So this can also be used to rebuild `--docs` automatically. Some examples are in the readme --- .gitignore | 4 + .pre-commit-config.yaml | 3 +- README.md | 47 +++++- tools/plugin_builder.py | 348 ++++++++++++++++++++++++++++++++++++---- 4 files changed, 365 insertions(+), 37 deletions(-) diff --git a/.gitignore b/.gitignore index 9b472a161..88ed2dab5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +# When adding new files here, consider adding them to `tools/plugin_builder.py` as well. +# particularly for large build directories with lots of files or directories that may cause infinite loops +# because they are built by the plugin builder. + node_modules .vscode/ tsconfig.tsbuildinfo diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4a675c916..005a5854a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,7 +25,8 @@ repos: matplotlib, deephaven-plugin-utilities, sphinx, - click + click, + watchdog, ] - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.2.2 diff --git a/README.md b/README.md index 6d6feea28..2d2eb0252 100644 --- a/README.md +++ b/README.md @@ -208,12 +208,12 @@ services: ### Using plugin_builder.py The `tools/plugin_builder.py` script is a utility script that makes common plugin development cases easier. -The tool uses `click` for command line argument parsing, so install it if you haven't already: +The tool uses `click` for command line argument parsing and `watchdog` for file watching. Skip the venv setup if you already have one ```shell python -m venv .venv source .venv/bin/activate -pip install click +pip install click watchdog ``` The script can then be used to help set up your venv. @@ -238,8 +238,8 @@ python tools/plugin_builder.py plotly-express ui ``` This targeting works for all commands that target the plugins directly, such as `--docs` or `--install`. -To build docs, pass the `--docs` flag. -First install the necessary dependencies (if setup with `--configure=full` this is already done) +To build docs, pass the `--docs` flag. +First install the necessary dependencies (if setup with `--configure=full` this is already done) ```shell pip install -r sphinx_ext/sphinx-requirements.txt ``` @@ -249,24 +249,57 @@ This example builds the docs for the `ui` plugin: python tools/plugin_builder.py --docs ui ``` -To run the server, pass the `--server` flag. -First install `deephaven-server` if it is not already installed (if setup with `--configure=full` this is already done): +It is necessary to install the latest version of the plugin you're building docs for before building the docs themselves. +Run with `--install` or `--reinstall` to install the plugin (depending on if you're installing a new version or not) +before building the docs. +```shell +python tools/plugin_builder.py --docs --install ui +``` +After the first time install, you can drop the `--install` flag and just run the script with `--docs` unless you have plugin changes. + + +To run the server, pass the `--server` flag. +First install `deephaven-server` if it is not already installed (if setup with `--configure=full` this is already done): ```shell pip install deephaven-server ``` -This example reinstalls the `plotly-express` plugin, then starts the server: +This example reinstalls the `plotly-express` plugin, then starts the server: ```shell python tools/plugin_builder.py --reinstall --server plotly-express ``` Reinstall will force reinstall the plugins (but only the plugins, not the dependencies), which is useful if there are changes to the plugins but without a bumped version number. +To run the server with specific args, pass the `--server-arg` flag. +By default, the server is passed the `--no-browser` flag, which will prevent the server from opening a browser window. +This example will override that default and open the browser: +```shell +python tools/plugin_builder.py --server-arg --browser +``` +Similar to other arguments, this argument can be shortened to `-sa`. +This example changes the port and psk and reinstalls the `ui` plugin before starting the server: +```shell +python tools/plugin_builder.py -r -sa --port=9999 -sa --jvm-args="-Dauthentication.psk=mypsk" ui +``` + The js plugins can be built with the `--js` flag. This will build all js plugins or target specific ones if specified. This example reinstalls the `ui` plugin with js, and starts the server with shorthand flags. ```shell python tools/plugin_builder.py --js -r -s ui ``` +Enable `watch` mode with the `--watch` flag. This will watch the project for changes and rerun the script with the same arguments. +Note that when using `--watch`, the script will not exit until stopped manually. +For example, to watch the `plotly-express` plugin for changes and rebuild the docs when changes are made: +```shell +python tools/plugin_builder.py --docs --watch plotly-express +``` + +This example reinstalls the `ui` plugin with js, starts the server, and watches for changes. +```shell +python tools/plugin_builder.py -jrsw ui +``` + ## Release Management In order to manage changelogs, version bumps and github releases, we use [cocogitto](https://github.com/cocogitto/cocogitto), or `cog` for short. Follow the [Installation instructions](https://github.com/cocogitto/cocogitto?tab=readme-ov-file#installation) to install `cog`. For Linux and Windows, we recommend using [cargo](https://doc.rust-lang.org/cargo/getting-started/installation.html) to install. For MacOS, we recommend using [brew](https://brew.sh/). diff --git a/tools/plugin_builder.py b/tools/plugin_builder.py index 20d127342..1a51ea16e 100644 --- a/tools/plugin_builder.py +++ b/tools/plugin_builder.py @@ -3,12 +3,137 @@ import click import os import sys -from typing import Generator +from typing import Generator, Callable +import time +import subprocess +from watchdog.events import FileSystemEvent, RegexMatchingEventHandler +from watchdog.observers import Observer +import threading # get the directory of the current file current_dir = os.path.dirname(os.path.abspath(__file__)) # navigate out one directory to get to the plugins directory plugins_dir = os.path.join(current_dir, "../plugins") +# navigate up one directory to get to the project directory +project_path = os.path.split(current_dir)[0] + +# these are the patterns to watch for changes in the plugins directory +# if in editable mode, the builder will rerun when these files change +REBUILD_REGEXES = [ + ".*\.py$", + ".*\.js$", + ".*\.md$", + ".*\.svg$", + ".*\.ts$", + ".*\.tsx$", + ".*\.scss$", +] + +# ignore these patterns in particular +# prevents infinite loops when the builder is rerun +IGNORE_REGEXES = [ + ".*/dist/.*", + ".*/build/.*", + ".*/node_modules/.*", + ".*/_js/.*", + # ignore hidden files and directories + ".*/\..*/.*", +] + + +class PluginsChangedHandler(RegexMatchingEventHandler): + """ + A handler that watches for changes reruns the function when changes are detected + + Args: + func: The function to run when changes are detected + stop_event: The event to signal the function to stop + + Attributes: + func: The function to run when changes are detected + stop_event: The event to signal the function to stop + rerun_lock: A lock to prevent multiple reruns from occurring at the same time + """ + + def __init__(self, func: Callable, stop_event: threading.Event) -> None: + super().__init__(regexes=REBUILD_REGEXES, ignore_regexes=IGNORE_REGEXES) + + self.func = func + + # A flag to indicate whether the function should continue running + # Also prevents unnecessary reruns + self.stop_event = stop_event + + # A lock to prevent multiple reruns from occurring at the same time + self.rerun_lock = threading.Lock() + + # always have an initial run + threading.Thread(target=self.attempt_rerun).start() + + def attempt_rerun(self) -> None: + """ + Attempt to rerun the function. + If the stop event is set, do not rerun because a rerun has already been scheduled. + """ + self.stop_event.set() + with self.rerun_lock: + self.stop_event.clear() + self.func() + + def event_handler(self, event: FileSystemEvent) -> None: + """ + Handle any file system event + + Args: + event: The event that occurred + """ + if self.stop_event.is_set(): + # a rerun has already been scheduled on another thread + print( + f"File {event.src_path} {event.event_type}, rerun has already been scheduled" + ) + return + print(f"File {event.src_path} {event.event_type}, new rerun scheduled") + threading.Thread(target=self.attempt_rerun).start() + + def on_created(self, event: FileSystemEvent) -> None: + """ + Handle a file creation event + + Args: + event: The event that occurred + """ + self.event_handler(event) + + def on_deleted(self, event: FileSystemEvent) -> None: + """ + Handle a file deletion event + + Args: + event: The event that occurred + """ + self.event_handler(event) + + def on_modified(self, event: FileSystemEvent) -> None: + """ + Handle a file modification event + + Args: + event: The event that occurred + """ + self.event_handler(event) + + def on_moved(self, event: FileSystemEvent) -> None: + """ + Handle a file move event + + Args: + event: The event that occurred + + Returns: + + """ + self.event_handler(event) def clean_build_dist(plugin: str) -> None: @@ -51,6 +176,23 @@ def plugin_names( def run_command(command: str) -> None: """ Run a command and exit if it fails. + This should only be used in a non-main thread. + + Args: + command: The command to run. + + Returns: + None + """ + code = os.system(command) + if code != 0: + os._exit(1) + + +def run_main_command(command: str) -> None: + """ + Run a command and exit if it fails. + This should only be used in the main thread. Args: command: The command to run. @@ -86,7 +228,7 @@ def run_build( run_command(f"python -m build --wheel {plugins_dir}/{plugin}") elif error_on_missing: click.echo(f"Error: setup.cfg not found in {plugin}") - sys.exit(1) + os._exit(1) def run_install( @@ -138,7 +280,7 @@ def run_docs( run_command(f"python {plugins_dir}/{plugin}/make_docs.py") elif error_on_missing: click.echo(f"Error: make_docs.py not found in {plugin}") - sys.exit(1) + os._exit(1) def run_build_js(plugins: tuple[str]) -> None: @@ -177,12 +319,117 @@ def run_configure( None """ if configure in ["min", "full"]: - run_command("pip install -r requirements.txt") - run_command("pre-commit install") - run_command("npm install") + run_main_command("pip install -r requirements.txt") + run_main_command("pre-commit install") + run_main_command("npm install") if configure == "full": # currently deephaven-server is installed as part of the sphinx_ext requirements - run_command("pip install -r sphinx_ext/sphinx-requirements.txt") + run_main_command("pip install -r sphinx_ext/sphinx-requirements.txt") + + +def build_server_args(server_arg: tuple[str]) -> list[str]: + """ + Build the server arguments to pass to the deephaven server + By default, the --no-browser flag is added to the server arguments unless the --browser flag is present + + Args: + server_arg: The arguments to pass to the server + """ + server_args = ["--no-browser"] + if server_arg: + if "--no-browser" in server_arg or "--browser" in server_arg: + server_args = list(server_arg) + else: + server_args = server_args + list(server_arg) + return server_args + + +def handle_args( + build: bool, + install: bool, + reinstall: bool, + docs: bool, + server: bool, + server_arg: tuple[str], + js: bool, + configure: str | None, + plugins: tuple[str], + stop_event: threading.Event, +) -> None: + """ + Handle all arguments for the builder command + + Args: + build: True to build the plugins + install: True to install the plugins + reinstall: True to reinstall the plugins + docs: True to generate the docs + server: True to run the deephaven server after building and installing the plugins + server_arg: The arguments to pass to the server + js: True to build the JS files for the plugins + configure: The configuration to use. 'min' will install the minimum requirements for development. + 'full' will install some optional packages for development, such as sphinx and deephaven-server. + plugins: Plugins to build and install + stop_event: The event to signal the function to stop + """ + # it is possible that the stop event is set before this function is called + if stop_event.is_set(): + return + + # default is to install, but don't if just configuring + if not any([build, install, reinstall, docs, js, configure]): + js = True + install = True + + # if this thread is signaled to stop, return after the current command + # instead of in the middle of a command, which could leave the environment in a bad state + if stop_event.is_set(): + return + + if js: + run_build_js(plugins) + + if stop_event.is_set(): + return + + if build or install or reinstall: + run_build(plugins, len(plugins) > 0) + + if stop_event.is_set(): + return + + if install or reinstall: + run_install(plugins, reinstall) + + if stop_event.is_set(): + return + + if docs: + run_docs(plugins, len(plugins) > 0) + + if stop_event.is_set(): + return + + if server or server_arg: + server_args = build_server_args(server_arg) + + click.echo(f"Running deephaven server with args: {server_args}") + process = subprocess.Popen(["deephaven", "server"] + server_args) + + # waiting on either the process to finish or the stop event to be set + while not stop_event.wait(1): + poll = process.poll() + if poll is not None: + # process threw an error or was killed, so exit + os._exit(process.returncode) + + # stop event is set, so kill the process + process.terminate() + try: + process.wait(timeout=1) + except subprocess.TimeoutExpired: + process.kill() + process.wait() @click.command( @@ -211,7 +458,9 @@ def run_configure( "--docs", "-d", is_flag=True, - help="Generate docs for all plugins that have a make_docs.py.", + help="Generate docs for all plugins that have a make_docs.py. " + "There must be an installed version of the plugin to generate the docs." + "Consider using the --reinstall or --install flags to update the plugin before generating the docs.", ) @click.option( "--server", @@ -219,6 +468,13 @@ def run_configure( is_flag=True, help="Run the deephaven server after building and installing the plugins.", ) +@click.option( + "--server-arg", + "-sa", + default=tuple(), + multiple=True, + help="Run the deephaven server after building and installing the plugins with the provided argument.", +) @click.option( "--js", "-j", @@ -229,9 +485,17 @@ def run_configure( "--configure", "-c", default=None, - help="Configure your venv for plugin development. 'min' will install the minimum requirements for development." + help="Configure your venv for plugin development. 'min' will install the minimum requirements for development. " "'full' will install some optional packages for development, such as sphinx and deephaven-server.", ) +@click.option( + "--watch", + "-w", + is_flag=True, + help="Run the other provided commands in an editable-like mode, watching for changes " + "This will rerun all other commands (except configure) when files are changed. " + "The top level directory of this project is watched.", +) @click.argument("plugins", nargs=-1) def builder( build: bool, @@ -239,8 +503,10 @@ def builder( reinstall: bool, docs: bool, server: bool, + server_arg: tuple[str], js: bool, configure: str | None, + watch: bool, plugins: tuple[str], ) -> None: """ @@ -252,33 +518,57 @@ def builder( reinstall: True to reinstall the plugins docs: True to generate the docs server: True to run the deephaven server after building and installing the plugins + server_arg: The arguments to pass to the server js: True to build the JS files for the plugins configure: The configuration to use. 'min' will install the minimum requirements for development. 'full' will install some optional packages for development, such as sphinx and deephaven-server. + watch: True to rerun the other commands when files are changed plugins: Plugins to build and install """ + # no matter what, only run the configure command once run_configure(configure) - # default is to install, but don't if just configuring - if not any([build, install, reinstall, docs, js, configure]): - js = True - install = True - - if js: - run_build_js(plugins) - - if build or install or reinstall: - run_build(plugins, len(plugins) > 0) - - if install or reinstall: - run_install(plugins, reinstall) - - if docs: - run_docs(plugins, len(plugins) > 0) - - if server: - click.echo("Running deephaven server") - os.system("deephaven server") + stop_event = threading.Event() + + def run_handle_args() -> None: + """ + Run the handle_args function with the provided arguments + """ + handle_args( + build, + install, + reinstall, + docs, + server, + server_arg, + js, + configure, + plugins, + stop_event, + ) + + if not watch: + # since editable is not specified, only run the handler once + # call it from a thread to allow the usage of os._exit to exit the process + # rather than sys.exit because sys.exit will not exit the process when called from a thread + # and os._exit should be called from a thread + thread = threading.Thread(target=run_handle_args) + thread.start() + thread.join() + return + + # editable is specified, so run the handler in a loop that watches for changes and + # reruns the handler when changes are detected + event_handler = PluginsChangedHandler(run_handle_args, stop_event) + observer = Observer() + observer.schedule(event_handler, project_path, recursive=True) + observer.start() + try: + while True: + input() + finally: + observer.stop() + observer.join() if __name__ == "__main__":